google-cloud-bigquery 1.9.0 → 1.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 783828eac7c8335eced73772ee8d7f154daa10adfb2976c0961bca8132d449f5
-  data.tar.gz: baf70806a64f209d6fd7d5567b11df9287879ea941478054ca4806dafb8980ec
+  metadata.gz: 475fe669391639d85bc48bdabb141ab8efa86dd9cdcfa97544d92b77099a1f63
+  data.tar.gz: ad906eefc520fff184d81d1496e7e65914be90d631d4d31aead94fa41f4fff0f
 SHA512:
-  metadata.gz: dd35d5d59e70c1d6aa417bed8e48a5d94454353953b8a674f28f94a03ee748d8c559028ef5551a9e2c6d3ec98a67b036e48c8bbc261cf44671eb3f4d1ccd6086
-  data.tar.gz: 2bffb6a6249419fdc9745e8f829a04f53daae9bdbc67fd72c9e58e8a70741e9261df1f189139bd73b5aacfe0584acacbd438720a6d314c5410cd40c30c675063
+  metadata.gz: 9639819d2a90cf202dcc9f2d4ef14fb1ddfab512ddb965af8fd394d7228a6ac5d061ba4e7444a98ac5c5a09a2bb189b0f6c81146375cd5d5bbbe78576f249d05
+  data.tar.gz: d3c854d3eb4ed7a053262842ab48e8bdcde7b929f1059695c8e67e36447c52fdaa1225b78334590d54ee0e445da8dcefd0402d04d5202c4d66155e9d27edb3b2

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Release History
 
+### 1.10.0 / 2018-12-06
+
+* Add dryrun param to Project#query_job and Dataset#query_job
+* Add copy and extract methods to Project
+  * Add Project#extract and Project#extract_job
+  * Add Project#copy and Project#copy_job
+  * Deprecate dryrun param in Table#copy_job, Table#extract_job and 
+    Table#load_job
+* Fix memoization in Dataset#exists? and Table#exists?
+  * Add force param to Dataset#exists? and Table#exists?
+
 ### 1.9.0 / 2018-10-25
 
 * Add clustering fields to LoadJob, QueryJob and Table

data/lib/google/cloud/bigquery/data.rb

@@ -150,6 +150,7 @@ module Google
         #   field.required? #=> true
         #
         def schema
+          return nil unless @table_gapi
           Schema.from_gapi(@table_gapi.schema).freeze
         end
 

data/lib/google/cloud/bigquery/dataset.rb

@@ -418,6 +418,8 @@ module Google
         def delete force: nil
           ensure_service!
           service.delete_dataset dataset_id, force
+          # Set flag for #exists?
+          @exists = false
           true
         end
 
@@ -748,6 +750,10 @@ module Google
         #   * `append` - BigQuery appends the data to the table.
         #   * `empty` - A 'duplicate' error is returned in the job result if the
         #     table exists and contains data.
+        # @param [Boolean] dryrun If set to true, BigQuery doesn't run the job.
+        #   Instead, if the query is valid, BigQuery returns statistics about
+        #   the job such as how many bytes would be processed. If the query is
+        #   invalid, an error returns. The default value is false.
         # @param [Boolean] standard_sql Specifies whether to use BigQuery's
         #   [standard
         #   SQL](https://cloud.google.com/bigquery/docs/reference/standard-sql/)
@@ -795,7 +801,8 @@ module Google
         #   contain lowercase letters, numeric characters, underscores and
         #   dashes. International characters are allowed. Label values are
         #   optional. Label keys must start with a letter and each label in the
-        #   list must have a different key.
+        #   list must have a different key. See [Requirements for
+        #   labels](https://cloud.google.com/bigquery/docs/creating-managing-labels#requirements).
         # @param [Array<String>, String] udfs User-defined function resources
         #   used in the query. May be either a code resource to load from a
         #   Google Cloud Storage URI (`gs://bucket/path`), or an inline resource
@@ -928,13 +935,13 @@ module Google
         #
         def query_job query, params: nil, external: nil,
                       priority: "INTERACTIVE", cache: true, table: nil,
-                      create: nil, write: nil, standard_sql: nil,
+                      create: nil, write: nil, dryrun: nil, standard_sql: nil,
                       legacy_sql: nil, large_results: nil, flatten: nil,
                       maximum_billing_tier: nil, maximum_bytes_billed: nil,
                       job_id: nil, prefix: nil, labels: nil, udfs: nil
           ensure_service!
           options = { priority: priority, cache: cache, table: table,
-                      create: create, write: write,
+                      create: create, write: write, dryrun: dryrun,
                       large_results: large_results, flatten: flatten,
                       legacy_sql: legacy_sql, standard_sql: standard_sql,
                       maximum_billing_tier: maximum_billing_tier,
@@ -1341,14 +1348,17 @@ module Google
         #   contain lowercase letters, numeric characters, underscores and
         #   dashes. International characters are allowed. Label values are
         #   optional. Label keys must start with a letter and each label in the
-        #   list must have a different key.
-        #
+        #   list must have a different key. See [Requirements for
+        #   labels](https://cloud.google.com/bigquery/docs/creating-managing-labels#requirements).
         # @yield [updater] A block for setting the schema and other
         #   options for the destination table. The schema can be omitted if the
         #   destination table already exists, or if you're loading data from a
         #   Google Cloud Datastore backup.
         # @yieldparam [Google::Cloud::Bigquery::LoadJob::Updater] updater An
         #   updater to modify the load job and its schema.
+        # @param [Boolean] dryrun  If set, don't actually run this job. Behavior
+        #   is undefined however for non-query jobs and may result in an error.
+        #   Deprecated.
         #
         # @return [Google::Cloud::Bigquery::LoadJob] A new load job object.
         #
@@ -1437,8 +1447,8 @@ module Google
                      projection_fields: nil, jagged_rows: nil,
                      quoted_newlines: nil, encoding: nil, delimiter: nil,
                      ignore_unknown: nil, max_bad_records: nil, quote: nil,
-                     skip_leading: nil, dryrun: nil, schema: nil, job_id: nil,
-                     prefix: nil, labels: nil, autodetect: nil, null_marker: nil
+                     skip_leading: nil, schema: nil, job_id: nil, prefix: nil,
+                     labels: nil, autodetect: nil, null_marker: nil, dryrun: nil
           ensure_service!
 
           updater = load_job_updater table_id,
@@ -1709,16 +1719,21 @@ module Google
         #
         def reload!
           ensure_service!
-          reloaded_gapi = service.get_dataset dataset_id
+          @gapi = service.get_dataset dataset_id
           @reference = nil
-          @gapi = reloaded_gapi
+          @exists = nil
           self
         end
         alias refresh! reload!
 
         ##
         # Determines whether the dataset exists in the BigQuery service. The
-        # result is cached locally.
+        # result is cached locally. To refresh state, set `force` to `true`.
+        #
+        # @param [Boolean] force Force the latest resource representation to be
+        #   retrieved from the BigQuery service when `true`. Otherwise the
+        #   return value of this method will be memoized to reduce the number of
+        #   API calls made to the BigQuery service. The default is `false`.
         #
         # @return [Boolean] `true` when the dataset exists in the BigQuery
         #   service, `false` otherwise.
@@ -1731,15 +1746,13 @@ module Google
         #   dataset = bigquery.dataset "my_dataset", skip_lookup: true
         #   dataset.exists? # true
         #
-        def exists?
-          # Always true if we have a gapi object
-          return true unless reference?
-          # If we have a value, return it
+        def exists? force: nil
+          return gapi_exists? if force
+          # If we have a memoized value, return it
           return @exists unless @exists.nil?
-          ensure_gapi!
-          @exists = true
-        rescue Google::Cloud::NotFoundError
-          @exists = false
+          # Always true if we have a gapi object
+          return true if resource?
+          gapi_exists?
         end
 
         ##
@@ -2056,6 +2069,15 @@ module Google
           reload!
         end
 
+        ##
+        # Fetch gapi and memoize whether resource exists.
+        def gapi_exists?
+          reload!
+          @exists = true
+        rescue Google::Cloud::NotFoundError
+          @exists = false
+        end
+
         def patch_gapi! *attributes
           return if attributes.empty?
           ensure_service!

data/lib/google/cloud/bigquery/dataset/access.rb

@@ -191,10 +191,12 @@ module Google
           ##
           # Add reader access to a view.
           #
-          # @param [Google::Cloud::Bigquery::Table, String] view A table object
-          #   or a string identifier as specified by the [Query
-          #   Reference](https://cloud.google.com/bigquery/query-reference#from):
-          #   `project_name:datasetId.tableId`.
+          # @param [Google::Cloud::Bigquery::Table, String] view A table object,
+          #   or a string identifier as specified by the [Standard SQL Query
+          #   Reference](https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#from-clause)
+          #   (`project-name.dataset_id.table_id`) or the [Legacy SQL Query
+          #   Reference](https://cloud.google.com/bigquery/query-reference#from)
+          #   (`project-name:dataset_id.table_id`).
           #
           # @example
           #   require "google/cloud/bigquery"
@@ -450,10 +452,12 @@ module Google
           ##
           # Remove reader access from a view.
           #
-          # @param [Google::Cloud::Bigquery::Table, String] view A table object
-          #   or a string identifier as specified by the [Query
-          #   Reference](https://cloud.google.com/bigquery/query-reference#from):
-          #   `project_name:datasetId.tableId`.
+          # @param [Google::Cloud::Bigquery::Table, String] view A table object,
+          #   or a string identifier as specified by the [Standard SQL Query
+          #   Reference](https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#from-clause)
+          #   (`project-name.dataset_id.table_id`) or the [Legacy SQL Query
+          #   Reference](https://cloud.google.com/bigquery/query-reference#from)
+          #   (`project-name:dataset_id.table_id`).
           #
           # @example
           #   require "google/cloud/bigquery"
@@ -705,10 +709,12 @@ module Google
           ##
           # Checks reader access for a view.
           #
-          # @param [Google::Cloud::Bigquery::Table, String] view A table object
-          #   or a string identifier as specified by the [Query
-          #   Reference](https://cloud.google.com/bigquery/query-reference#from):
-          #   `project_name:datasetId.tableId`.
+          # @param [Google::Cloud::Bigquery::Table, String] view A table object,
+          #   or a string identifier as specified by the [Standard SQL Query
+          #   Reference](https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#from-clause)
+          #   (`project-name.dataset_id.table_id`) or the [Legacy SQL Query
+          #   Reference](https://cloud.google.com/bigquery/query-reference#from)
+          #   (`project-name:dataset_id.table_id`).
           #
           # @example
           #   require "google/cloud/bigquery"
@@ -923,7 +929,7 @@ module Google
             if view.respond_to? :table_ref
               view.table_ref
             else
-              Service.table_ref_from_s view, @dataset_reference
+              Service.table_ref_from_s view, default_ref: @dataset_reference
             end
           end
 

data/lib/google/cloud/bigquery/project.rb

@@ -95,6 +95,184 @@ module Google
             service.project_service_account.email
         end
 
+        ##
+        # Copies the data from the source table to the destination table using
+        # an asynchronous method. In this method, a {CopyJob} is immediately
+        # returned. The caller may poll the service by repeatedly calling
+        # {Job#reload!} and {Job#done?} to detect when the job is done, or
+        # simply block until the job is done by calling #{Job#wait_until_done!}.
+        # See {#copy} for the synchronous version. Use this method instead of
+        # {Table#copy_job} to copy from source tables in other projects.
+        #
+        # The geographic location for the job ("US", "EU", etc.) can be set via
+        # {CopyJob::Updater#location=} in a block passed to this method.
+        #
+        # @param [String, Table] source_table The source table for the
+        #   copied data. This can be a table object; or a string ID as specified
+        #   by the [Standard SQL Query
+        #   Reference](https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#from-clause)
+        #   (`project-name.dataset_id.table_id`) or the [Legacy SQL Query
+        #   Reference](https://cloud.google.com/bigquery/query-reference#from)
+        #   (`project-name:dataset_id.table_id`).
+        # @param [String, Table] destination_table The destination table for the
+        #   copied data. This can be a table object; or a string ID as specified
+        #   by the [Standard SQL Query
+        #   Reference](https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#from-clause)
+        #   (`project-name.dataset_id.table_id`) or the [Legacy SQL Query
+        #   Reference](https://cloud.google.com/bigquery/query-reference#from)
+        #   (`project-name:dataset_id.table_id`).
+        # @param [String] create Specifies whether the job is allowed to create
+        #   new tables. The default value is `needed`.
+        #
+        #   The following values are supported:
+        #
+        #   * `needed` - Create the table if it does not exist.
+        #   * `never` - The table must already exist. A 'notFound' error is
+        #     raised if the table does not exist.
+        # @param [String] write Specifies how to handle data already present in
+        #   the destination table. The default value is `empty`.
+        #
+        #   The following values are supported:
+        #
+        #   * `truncate` - BigQuery overwrites the table data.
+        #   * `append` - BigQuery appends the data to the table.
+        #   * `empty` - An error will be returned if the destination table
+        #     already contains data.
+        # @param [String] job_id A user-defined ID for the copy job. The ID
+        #   must contain only letters (a-z, A-Z), numbers (0-9), underscores
+        #   (_), or dashes (-). The maximum length is 1,024 characters. If
+        #   `job_id` is provided, then `prefix` will not be used.
+        #
+        #   See [Generating a job
+        #   ID](https://cloud.google.com/bigquery/docs/managing-jobs#generate-jobid).
+        # @param [String] prefix A string, usually human-readable, that will be
+        #   prepended to a generated value to produce a unique job ID. For
+        #   example, the prefix `daily_import_job_` can be given to generate a
+        #   job ID such as `daily_import_job_12vEDtMQ0mbp1Mo5Z7mzAFQJZazh`. The
+        #   prefix must contain only letters (a-z, A-Z), numbers (0-9),
+        #   underscores (_), or dashes (-). The maximum length of the entire ID
+        #   is 1,024 characters. If `job_id` is provided, then `prefix` will not
+        #   be used.
+        # @param [Hash] labels A hash of user-provided labels associated with
+        #   the job. You can use these to organize and group your jobs. Label
+        #   keys and values can be no longer than 63 characters, can only
+        #   contain lowercase letters, numeric characters, underscores and
+        #   dashes. International characters are allowed. Label values are
+        #   optional. Label keys must start with a letter and each label in the
+        #   list must have a different key. See [Requirements for
+        #   labels](https://cloud.google.com/bigquery/docs/creating-managing-labels#requirements).
+        # @yield [job] a job configuration object
+        # @yieldparam [Google::Cloud::Bigquery::CopyJob::Updater] job a job
+        #   configuration object for setting additional options.
+        #
+        # @return [Google::Cloud::Bigquery::CopyJob]
+        #
+        # @example
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #   dataset = bigquery.dataset "my_dataset"
+        #   source_table_id = "bigquery-public-data.samples.shakespeare"
+        #   destination_table = dataset.table "my_destination_table"
+        #
+        #   copy_job = bigquery.copy_job source_table_id, destination_table
+        #
+        #   copy_job.wait_until_done!
+        #   copy_job.done? #=> true
+        #
+        # @!group Data
+        #
+        def copy_job source_table, destination_table, create: nil, write: nil,
+                     job_id: nil, prefix: nil, labels: nil
+          ensure_service!
+          options = { create: create, write: write, labels: labels,
+                      job_id: job_id, prefix: prefix }
+
+          updater = CopyJob::Updater.from_options(
+            service,
+            Service.get_table_ref(source_table, default_ref: project_ref),
+            Service.get_table_ref(destination_table, default_ref: project_ref),
+            options
+          )
+
+          yield updater if block_given?
+
+          job_gapi = updater.to_gapi
+          gapi = service.copy_table job_gapi
+          Job.from_gapi gapi, service
+        end
+
+        ##
+        # Copies the data from the source table to the destination table using a
+        # synchronous method that blocks for a response. Timeouts and transient
+        # errors are generally handled as needed to complete the job. See
+        # {#copy_job} for the asynchronous version. Use this method instead of
+        # {Table#copy} to copy from source tables in other projects.
+        #
+        # The geographic location for the job ("US", "EU", etc.) can be set via
+        # {CopyJob::Updater#location=} in a block passed to this method.
+        #
+        # @param [String, Table] source_table The source table for the
+        #   copied data. This can be a table object; or a string ID as specified
+        #   by the [Standard SQL Query
+        #   Reference](https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#from-clause)
+        #   (`project-name.dataset_id.table_id`) or the [Legacy SQL Query
+        #   Reference](https://cloud.google.com/bigquery/query-reference#from)
+        #   (`project-name:dataset_id.table_id`).
+        # @param [String, Table] destination_table The destination table for the
+        #   copied data. This can be a table object; or a string ID as specified
+        #   by the [Standard SQL Query
+        #   Reference](https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#from-clause)
+        #   (`project-name.dataset_id.table_id`) or the [Legacy SQL Query
+        #   Reference](https://cloud.google.com/bigquery/query-reference#from)
+        #   (`project-name:dataset_id.table_id`).
+        # @param [String] create Specifies whether the job is allowed to create
+        #   new tables. The default value is `needed`.
+        #
+        #   The following values are supported:
+        #
+        #   * `needed` - Create the table if it does not exist.
+        #   * `never` - The table must already exist. A 'notFound' error is
+        #     raised if the table does not exist.
+        # @param [String] write Specifies how to handle data already present in
+        #   the destination table. The default value is `empty`.
+        #
+        #   The following values are supported:
+        #
+        #   * `truncate` - BigQuery overwrites the table data.
+        #   * `append` - BigQuery appends the data to the table.
+        #   * `empty` - An error will be returned if the destination table
+        #     already contains data.
+        # @yield [job] a job configuration object
+        # @yieldparam [Google::Cloud::Bigquery::CopyJob::Updater] job a job
+        #   configuration object for setting additional options.
+        #
+        # @return [Boolean] Returns `true` if the copy operation succeeded.
+        #
+        # @example
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #   dataset = bigquery.dataset "my_dataset"
+        #   destination_table = dataset.table "my_destination_table"
+        #
+        #   bigquery.copy "bigquery-public-data.samples.shakespeare",
+        #                 destination_table
+        #
+        # @!group Data
+        #
+        def copy source_table, destination_table, create: nil, write: nil,
+                 &block
+          job = copy_job source_table,
+                         destination_table,
+                         create: create,
+                         write: write,
+                         &block
+          job.wait_until_done!
+          ensure_job_succeeded! job
+          true
+        end
+
         ##
         # Queries data by creating a [query
         # job](https://cloud.google.com/bigquery/docs/query-overview#query_jobs).
@@ -166,6 +344,10 @@ module Google
         #   * `append` - BigQuery appends the data to the table.
         #   * `empty` - A 'duplicate' error is returned in the job result if the
         #     table exists and contains data.
+        # @param [Boolean] dryrun If set to true, BigQuery doesn't run the job.
+        #   Instead, if the query is valid, BigQuery returns statistics about
+        #   the job such as how many bytes would be processed. If the query is
+        #   invalid, an error returns. The default value is false.
         # @param [Dataset, String] dataset The default dataset to use for
         #   unqualified table names in the query. Optional.
         # @param [String] project Specifies the default projectId to assume for
@@ -221,7 +403,8 @@ module Google
         #   contain lowercase letters, numeric characters, underscores and
         #   dashes. International characters are allowed. Label values are
         #   optional. Label keys must start with a letter and each label in the
-        #   list must have a different key.
+        #   list must have a different key. See [Requirements for
+        #   labels](https://cloud.google.com/bigquery/docs/creating-managing-labels#requirements).
         # @param [Array<String>, String] udfs User-defined function resources
         #   used in the query. May be either a code resource to load from a
         #   Google Cloud Storage URI (`gs://bucket/path`), or an inline resource
@@ -357,14 +540,14 @@ module Google
         #
         def query_job query, params: nil, external: nil,
                       priority: "INTERACTIVE", cache: true, table: nil,
-                      create: nil, write: nil, dataset: nil, project: nil,
-                      standard_sql: nil, legacy_sql: nil, large_results: nil,
-                      flatten: nil, maximum_billing_tier: nil,
-                      maximum_bytes_billed: nil, job_id: nil, prefix: nil,
-                      labels: nil, udfs: nil
+                      create: nil, write: nil, dryrun: nil, dataset: nil,
+                      project: nil, standard_sql: nil, legacy_sql: nil,
+                      large_results: nil, flatten: nil,
+                      maximum_billing_tier: nil, maximum_bytes_billed: nil,
+                      job_id: nil, prefix: nil, labels: nil, udfs: nil
           ensure_service!
           options = { priority: priority, cache: cache, table: table,
-                      create: create, write: write,
+                      create: create, write: write, dryrun: dryrun,
                       large_results: large_results, flatten: flatten,
                       dataset: dataset, project: (project || self.project),
                       legacy_sql: legacy_sql, standard_sql: standard_sql,
@@ -1096,6 +1279,177 @@ module Google
           encrypt_config
         end
 
+        ##
+        # Extracts the data from the provided table to a Google Cloud Storage
+        # file using an asynchronous method. In this method, an {ExtractJob} is
+        # immediately returned. The caller may poll the service by repeatedly
+        # calling {Job#reload!} and {Job#done?} to detect when the job is done,
+        # or simply block until the job is done by calling
+        # #{Job#wait_until_done!}. See {#extract} for the synchronous version.
+        # Use this method instead of {Table#extract_job} to extract data from
+        # source tables in other projects.
+        #
+        # The geographic location for the job ("US", "EU", etc.) can be set via
+        # {ExtractJob::Updater#location=} in a block passed to this method.
+        #
+        # @see https://cloud.google.com/bigquery/exporting-data-from-bigquery
+        #   Exporting Data From BigQuery
+        #
+        # @param [String, Table] table The source table from which to extract
+        #   data. This can be a table object; or a string ID as specified by the
+        #   [Standard SQL Query
+        #   Reference](https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#from-clause)
+        #   (`project-name.dataset_id.table_id`) or the [Legacy SQL Query
+        #   Reference](https://cloud.google.com/bigquery/query-reference#from)
+        #   (`project-name:dataset_id.table_id`).
+        # @param [Google::Cloud::Storage::File, String, Array<String>]
+        #   extract_url The Google Storage file or file URI pattern(s) to which
+        #   BigQuery should extract the table data.
+        # @param [String] format The exported file format. The default value is
+        #   `csv`.
+        #
+        #   The following values are supported:
+        #
+        #   * `csv` - CSV
+        #   * `json` - [Newline-delimited JSON](http://jsonlines.org/)
+        #   * `avro` - [Avro](http://avro.apache.org/)
+        # @param [String] compression The compression type to use for exported
+        #   files. Possible values include `GZIP` and `NONE`. The default value
+        #   is `NONE`.
+        # @param [String] delimiter Delimiter to use between fields in the
+        #   exported data. Default is <code>,</code>.
+        # @param [Boolean] header Whether to print out a header row in the
+        #   results. Default is `true`.
+        # @param [String] job_id A user-defined ID for the extract job. The ID
+        #   must contain only letters (a-z, A-Z), numbers (0-9), underscores
+        #   (_), or dashes (-). The maximum length is 1,024 characters. If
+        #   `job_id` is provided, then `prefix` will not be used.
+        #
+        #   See [Generating a job
+        #   ID](https://cloud.google.com/bigquery/docs/managing-jobs#generate-jobid).
+        # @param [String] prefix A string, usually human-readable, that will be
+        #   prepended to a generated value to produce a unique job ID. For
+        #   example, the prefix `daily_import_job_` can be given to generate a
+        #   job ID such as `daily_import_job_12vEDtMQ0mbp1Mo5Z7mzAFQJZazh`. The
+        #   prefix must contain only letters (a-z, A-Z), numbers (0-9),
+        #   underscores (_), or dashes (-). The maximum length of the entire ID
+        #   is 1,024 characters. If `job_id` is provided, then `prefix` will not
+        #   be used.
+        # @param [Hash] labels A hash of user-provided labels associated with
+        #   the job. You can use these to organize and group your jobs. Label
+        #   keys and values can be no longer than 63 characters, can only
+        #   contain lowercase letters, numeric characters, underscores and
+        #   dashes. International characters are allowed. Label values are
+        #   optional. Label keys must start with a letter and each label in the
+        #   list must have a different key. See [Requirements for
+        #   labels](https://cloud.google.com/bigquery/docs/creating-managing-labels#requirements).
+        # @yield [job] a job configuration object
+        # @yieldparam [Google::Cloud::Bigquery::ExtractJob::Updater] job a job
+        #   configuration object for setting additional options.
+        #
+        # @return [Google::Cloud::Bigquery::ExtractJob]
+        #
+        # @example
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #
+        #   table_id = "bigquery-public-data.samples.shakespeare"
+        #   extract_job = bigquery.extract_job table_id,
+        #                                      "gs://my-bucket/shakespeare.csv"
+        #   extract_job.wait_until_done!
+        #   extract_job.done? #=> true
+        #
+        # @!group Data
+        #
+        def extract_job table, extract_url, format: nil, compression: nil,
+                        delimiter: nil, header: nil, job_id: nil, prefix: nil,
+                        labels: nil
+          ensure_service!
+          options = { format: format, compression: compression,
+                      delimiter: delimiter, header: header, job_id: job_id,
+                      prefix: prefix, labels: labels }
+
+          table_ref = Service.get_table_ref table, default_ref: project_ref
+          updater = ExtractJob::Updater.from_options service, table_ref,
+                                                     extract_url, options
+
+          yield updater if block_given?
+
+          job_gapi = updater.to_gapi
+          gapi = service.extract_table job_gapi
+          Job.from_gapi gapi, service
+        end
+
+        ##
+        # Extracts the data from the provided table to a Google Cloud Storage
+        # file using a synchronous method that blocks for a response. Timeouts
+        # and transient errors are generally handled as needed to complete the
+        # job. See {#extract_job} for the asynchronous version. Use this method
+        # instead of {Table#extract} to extract data from source tables in other
+        # projects.
+        #
+        # The geographic location for the job ("US", "EU", etc.) can be set via
+        # {ExtractJob::Updater#location=} in a block passed to this method.
+        #
+        # @see https://cloud.google.com/bigquery/exporting-data-from-bigquery
+        #   Exporting Data From BigQuery
+        #
+        # @param [String, Table] table The source table from which to extract
+        #   data. This can be a table object; or a string ID as specified by the
+        #   [Standard SQL Query
+        #   Reference](https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#from-clause)
+        #   (`project-name.dataset_id.table_id`) or the [Legacy SQL Query
+        #   Reference](https://cloud.google.com/bigquery/query-reference#from)
+        #   (`project-name:dataset_id.table_id`).
+        # @param [Google::Cloud::Storage::File, String, Array<String>]
+        #   extract_url The Google Storage file or file URI pattern(s) to which
+        #   BigQuery should extract the table data.
+        # @param [String] format The exported file format. The default value is
+        #   `csv`.
+        #
+        #   The following values are supported:
+        #
+        #   * `csv` - CSV
+        #   * `json` - [Newline-delimited JSON](http://jsonlines.org/)
+        #   * `avro` - [Avro](http://avro.apache.org/)
+        # @param [String] compression The compression type to use for exported
+        #   files. Possible values include `GZIP` and `NONE`. The default value
+        #   is `NONE`.
+        # @param [String] delimiter Delimiter to use between fields in the
+        #   exported data. Default is <code>,</code>.
+        # @param [Boolean] header Whether to print out a header row in the
+        #   results. Default is `true`.
+        # @yield [job] a job configuration object
+        # @yieldparam [Google::Cloud::Bigquery::ExtractJob::Updater] job a job
+        #   configuration object for setting additional options.
+        #
+        # @return [Boolean] Returns `true` if the extract operation succeeded.
+        #
+        # @example
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #
+        #   bigquery.extract "bigquery-public-data.samples.shakespeare",
+        #                    "gs://my-bucket/shakespeare.csv"
+        #
+        # @!group Data
+        #
+        def extract table, extract_url, format: nil, compression: nil,
+                    delimiter: nil, header: nil, &block
+          job = extract_job table,
+                            extract_url,
+                            format: format,
+                            compression: compression,
+                            delimiter: delimiter,
+                            header: header,
+                            &block
+          job.wait_until_done!
+          ensure_job_succeeded! job
+          true
+        end
+
         ##
         # @private New Project from a Google API Client object, using the
         # same Credentials as this project.
@@ -1122,6 +1476,21 @@ module Google
         def ensure_service!
           raise "Must have active connection" unless service
         end
+
+        def ensure_job_succeeded! job
+          return unless job.failed?
+          begin
+            # raise to activate ruby exception cause handling
+            raise job.gapi_error
+          rescue StandardError => e
+            # wrap Google::Apis::Error with Google::Cloud::Error
+            raise Google::Cloud::Error.from_error(e)
+          end
+        end
+
+        def project_ref
+          Google::Apis::BigqueryV2::ProjectReference.new project_id: project_id
+        end
       end
     end
   end

data/lib/google/cloud/bigquery/query_job.rb

@@ -99,6 +99,22 @@ module Google
           val
         end
 
+        ##
+        # If set, don't actually run this job. A valid query will return a
+        # mostly empty response with some processing statistics, while an
+        # invalid query will return the same error it would if it wasn't a dry
+        # run.
+        #
+        # @return [Boolean] `true` when the dry run flag is set for the query
+        #   job, `false` otherwise.
+        #
+        def dryrun?
+          @gapi.configuration.dry_run
+        end
+        alias dryrun dryrun?
+        alias dry_run dryrun?
+        alias dry_run? dryrun?
+
         ##
         # Checks if the query job flattens nested and repeated fields in the
         # query results. The default is `true`. If the value is `false`,
@@ -557,6 +573,9 @@ module Google
         #
         def data token: nil, max: nil, start: nil
           return nil unless done?
+          if dryrun?
+            return Data.from_gapi_json({ rows: [] }, nil, @gapi, service)
+          end
           if ddl? || dml?
             data_hash = { totalRows: nil, rows: [] }
             return Data.from_gapi_json data_hash, nil, @gapi, service
@@ -609,6 +628,7 @@ module Google
             updater.create = options[:create]
             updater.write = options[:write]
             updater.table = options[:table]
+            updater.dryrun = options[:dryrun]
             updater.maximum_bytes_billed = options[:maximum_bytes_billed]
             updater.labels = options[:labels] if options[:labels]
             updater.legacy_sql = Convert.resolve_legacy_sql(
@@ -786,6 +806,20 @@ module Google
               Convert.write_disposition value
           end
 
+          ##
+          # Sets the dry run flag for the query job.
+          #
+          # @param [Boolean] value If set, don't actually run this job. A valid
+          #   query will return a mostly empty response with some processing
+          #   statistics, while an invalid query will return the same error it
+          #   would if it wasn't a dry run..
+          #
+          # @!group Attributes
+          def dryrun= value
+            @gapi.configuration.dry_run = value
+          end
+          alias dry_run= dryrun=
+
           ##
           # Sets the destination for the query results table.
           #

data/lib/google/cloud/bigquery/service.rb

@@ -324,14 +324,26 @@ module Google
           end
         end
 
+        def self.get_table_ref table, default_ref: nil
+          if table.respond_to? :table_ref
+            table.table_ref
+          else
+            table_ref_from_s table, default_ref: default_ref
+          end
+        end
+
         ##
         # Extracts at least `tbl` group, and possibly `dts` and `prj` groups,
         # from strings in the formats: "my_table", "my_dataset.my_table", or
         # "my-project:my_dataset.my_table". Then merges project_id and
-        # dataset_id from the default table if they are missing.
-        def self.table_ref_from_s str, default_table_ref
+        # dataset_id from the default table ref if they are missing.
+        #
+        # The regex matches both Standard SQL
+        # ("bigquery-public-data.samples.shakespeare") and Legacy SQL
+        # ("bigquery-public-data:samples.shakespeare").
+        def self.table_ref_from_s str, default_ref: {}
           str = str.to_s
-          m = /\A(((?<prj>\S*):)?(?<dts>\S*)\.)?(?<tbl>\S*)\z/.match str
+          m = /\A(((?<prj>\S*)(:|\.))?(?<dts>\S*)\.)?(?<tbl>\S*)\z/.match str
           unless m
             raise ArgumentError, "unable to identify table from #{str.inspect}"
           end
@@ -340,8 +352,18 @@ module Google
             dataset_id: m["dts"],
             table_id:   m["tbl"]
           }.delete_if { |_, v| v.nil? }
-          new_table_ref_hash = default_table_ref.to_h.merge str_table_ref_hash
-          Google::Apis::BigqueryV2::TableReference.new new_table_ref_hash
+          str_table_ref_hash = default_ref.to_h.merge str_table_ref_hash
+          ref = Google::Apis::BigqueryV2::TableReference.new str_table_ref_hash
+          validate_table_ref ref
+          ref
+        end
+
+        def self.validate_table_ref table_ref
+          %i[project_id dataset_id table_id].each do |f|
+            if table_ref.send(f).nil?
+              raise ArgumentError, "TableReference is missing #{f}"
+            end
+          end
         end
 
         ##

data/lib/google/cloud/bigquery/table.rb

@@ -371,9 +371,10 @@ module Google
         ##
         # The combined Project ID, Dataset ID, and Table ID for this table, in
         # the format specified by the [Legacy SQL Query
-        # Reference](https://cloud.google.com/bigquery/query-reference#from):
-        # `project_name:datasetId.tableId`. To use this value in queries see
-        # {#query_id}.
+        # Reference](https://cloud.google.com/bigquery/query-reference#from)
+        # (`project-name:dataset_id.table_id`). This is useful for referencing
+        # tables in other projects and datasets. To use this value in queries
+        # see {#query_id}.
         #
         # @return [String, nil] The combined ID, or `nil` if the object is a
         #   reference (see {#reference?}).
@@ -386,10 +387,9 @@ module Google
         end
 
         ##
-        # The value returned by {#id}, wrapped in square brackets if the Project
-        # ID contains dashes, as specified by the [Query
-        # Reference](https://cloud.google.com/bigquery/query-reference#from).
-        # Useful in queries.
+        # The value returned by {#id}, wrapped in backticks (Standard SQL) or s
+        # quare brackets (Legacy SQL) to accommodate project IDs
+        # containing dashes. Useful in queries.
         #
         # @param [Boolean] standard_sql Specifies whether to use BigQuery's
         #   [standard
@@ -1191,9 +1191,11 @@ module Google
         #
         # @param [Table, String] destination_table The destination for the
         #   copied data. This can also be a string identifier as specified by
-        #   the [Query
-        #   Reference](https://cloud.google.com/bigquery/query-reference#from):
-        #   `project_name:datasetId.tableId`. This is useful for referencing
+        #   the [Standard SQL Query
+        #   Reference](https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#from-clause)
+        #   (`project-name.dataset_id.table_id`) or the [Legacy SQL Query
+        #   Reference](https://cloud.google.com/bigquery/query-reference#from)
+        #   (`project-name:dataset_id.table_id`). This is useful for referencing
         #   tables in other projects and datasets.
         # @param [String] create Specifies whether the job is allowed to create
         #   new tables. The default value is `needed`.
@@ -1233,7 +1235,12 @@ module Google
         #   contain lowercase letters, numeric characters, underscores and
         #   dashes. International characters are allowed. Label values are
         #   optional. Label keys must start with a letter and each label in the
-        #   list must have a different key.
+        #   list must have a different key. See [Requirements for
+        #   labels](https://cloud.google.com/bigquery/docs/creating-managing-labels#requirements).
+        # @param [Boolean] dryrun  If set, don't actually run this job. Behavior
+        #   is undefined however for non-query jobs and may result in an error.
+        #   Deprecated.
+        #
         # @yield [job] a job configuration object
         # @yieldparam [Google::Cloud::Bigquery::CopyJob::Updater] job a job
         #   configuration object for setting additional options.
@@ -1259,17 +1266,20 @@ module Google
         #
         #   copy_job = table.copy_job "other-project:other_dataset.other_table"
         #
+        #   copy_job.wait_until_done!
+        #   copy_job.done? #=> true
+        #
         # @!group Data
         #
-        def copy_job destination_table, create: nil, write: nil, dryrun: nil,
-                     job_id: nil, prefix: nil, labels: nil
+        def copy_job destination_table, create: nil, write: nil, job_id: nil,
+                     prefix: nil, labels: nil, dryrun: nil
           ensure_service!
           options = { create: create, write: write, dryrun: dryrun,
                       labels: labels, job_id: job_id, prefix: prefix }
           updater = CopyJob::Updater.from_options(
             service,
             table_ref,
-            get_table_ref(destination_table),
+            Service.get_table_ref(destination_table, default_ref: table_ref),
             options
           )
           updater.location = location if location # may be table reference
@@ -1295,9 +1305,11 @@ module Google
         #
         # @param [Table, String] destination_table The destination for the
         #   copied data. This can also be a string identifier as specified by
-        #   the [Query
-        #   Reference](https://cloud.google.com/bigquery/query-reference#from):
-        #   `project_name:datasetId.tableId`. This is useful for referencing
+        #   the [Standard SQL Query
+        #   Reference](https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#from-clause)
+        #   (`project-name.dataset_id.table_id`) or the [Legacy SQL Query
+        #   Reference](https://cloud.google.com/bigquery/query-reference#from)
+        #   (`project-name:dataset_id.table_id`). This is useful for referencing
         #   tables in other projects and datasets.
         # @param [String] create Specifies whether the job is allowed to create
         #   new tables. The default value is `needed`.
@@ -1406,7 +1418,12 @@ module Google
         #   contain lowercase letters, numeric characters, underscores and
         #   dashes. International characters are allowed. Label values are
         #   optional. Label keys must start with a letter and each label in the
-        #   list must have a different key.
+        #   list must have a different key. See [Requirements for
+        #   labels](https://cloud.google.com/bigquery/docs/creating-managing-labels#requirements).
+        # @param [Boolean] dryrun  If set, don't actually run this job. Behavior
+        #   is undefined however for non-query jobs and may result in an error.
+        #   Deprecated.
+        #
         # @yield [job] a job configuration object
         # @yieldparam [Google::Cloud::Bigquery::ExtractJob::Updater] job a job
         #   configuration object for setting additional options.
@@ -1421,13 +1438,15 @@ module Google
         #   table = dataset.table "my_table"
         #
         #   extract_job = table.extract_job "gs://my-bucket/file-name.json",
-        #                               format: "json"
+        #                                   format: "json"
+        #   extract_job.wait_until_done!
+        #   extract_job.done? #=> true
         #
         # @!group Data
         #
         def extract_job extract_url, format: nil, compression: nil,
-                        delimiter: nil, header: nil, dryrun: nil, job_id: nil,
-                        prefix: nil, labels: nil
+                        delimiter: nil, header: nil, job_id: nil, prefix: nil,
+                        labels: nil, dryrun: nil
           ensure_service!
           options = { format: format, compression: compression,
                       delimiter: delimiter, header: header, dryrun: dryrun,
@@ -1643,7 +1662,12 @@ module Google
         #   contain lowercase letters, numeric characters, underscores and
         #   dashes. International characters are allowed. Label values are
         #   optional. Label keys must start with a letter and each label in the
-        #   list must have a different key.
+        #   list must have a different key. See [Requirements for
+        #   labels](https://cloud.google.com/bigquery/docs/creating-managing-labels#requirements).
+        # @param [Boolean] dryrun  If set, don't actually run this job. Behavior
+        #   is undefined however for non-query jobs and may result in an error.
+        #   Deprecated.
+        #
         # @yield [load_job] a block for setting the load job
         # @yieldparam [LoadJob] load_job the load job object to be updated
         #
@@ -1700,8 +1724,8 @@ module Google
                      projection_fields: nil, jagged_rows: nil,
                      quoted_newlines: nil, encoding: nil, delimiter: nil,
                      ignore_unknown: nil, max_bad_records: nil, quote: nil,
-                     skip_leading: nil, dryrun: nil, job_id: nil, prefix: nil,
-                     labels: nil, autodetect: nil, null_marker: nil
+                     skip_leading: nil, job_id: nil, prefix: nil, labels: nil,
+                     autodetect: nil, null_marker: nil, dryrun: nil
           ensure_service!
 
           updater = load_job_updater format: format, create: create,
@@ -2058,6 +2082,8 @@ module Google
         def delete
           ensure_service!
           service.delete_table dataset_id, table_id
+          # Set flag for #exists?
+          @exists = false
           true
         end
 
@@ -2081,14 +2107,21 @@ module Google
         #
         def reload!
           ensure_service!
-          gapi = service.get_table dataset_id, table_id
-          @gapi = gapi
+          @gapi = service.get_table dataset_id, table_id
+          @reference = nil
+          @exists = nil
+          self
         end
         alias refresh! reload!
 
         ##
         # Determines whether the table exists in the BigQuery service. The
-        # result is cached locally.
+        # result is cached locally. To refresh state, set `force` to `true`.
+        #
+        # @param [Boolean] force Force the latest resource representation to be
+        #   retrieved from the BigQuery service when `true`. Otherwise the
+        #   return value of this method will be memoized to reduce the number of
+        #   API calls made to the BigQuery service. The default is `false`.
         #
         # @return [Boolean] `true` when the table exists in the BigQuery
         #   service, `false` otherwise.
@@ -2102,15 +2135,13 @@ module Google
         #   table = dataset.table "my_table", skip_lookup: true
         #   table.exists? # true
         #
-        def exists?
-          # Always true if we have a gapi object
-          return true unless reference?
+        def exists? force: nil
+          return gapi_exists? if force
           # If we have a value, return it
           return @exists unless @exists.nil?
-          ensure_gapi!
-          @exists = true
-        rescue Google::Cloud::NotFoundError
-          @exists = false
+          # Always true if we have a gapi object
+          return true if resource?
+          gapi_exists?
         end
 
         ##
@@ -2259,6 +2290,15 @@ module Google
           reload!
         end
 
+        ##
+        # Fetch gapi and memoize whether resource exists.
+        def gapi_exists?
+          reload!
+          @exists = true
+        rescue Google::Cloud::NotFoundError
+          @exists = false
+        end
+
         def patch_gapi! *attributes
           return if attributes.empty?
           ensure_service!
@@ -2459,16 +2499,6 @@ module Google
           end
         end
 
-        private
-
-        def get_table_ref table
-          if table.respond_to? :table_ref
-            table.table_ref
-          else
-            Service.table_ref_from_s table, table_ref
-          end
-        end
-
         ##
         # Yielded to a block to accumulate changes for a patch request.
         class Updater < Table

data/lib/google/cloud/bigquery/version.rb

@@ -16,7 +16,7 @@
 module Google
   module Cloud
     module Bigquery
-      VERSION = "1.9.0".freeze
+      VERSION = "1.10.0".freeze
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: google-cloud-bigquery
 version: !ruby/object:Gem::Version
-  version: 1.9.0
+  version: 1.10.0
 platform: ruby
 authors:
 - Mike Moore
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-25 00:00:00.000000000 Z
+date: 2018-12-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-cloud-core