google-cloud-bigquery 0.28.0 → 0.29.0

data/lib/google/cloud/bigquery/extract_job.rb

@@ -21,13 +21,25 @@ module Google
       #
       # A {Job} subclass representing an export operation that may be performed
       # on a {Table}. A ExtractJob instance is created when you call
-      # {Table#extract}.
+      # {Table#extract_job}.
       #
-      # @see https://cloud.google.com/bigquery/exporting-data-from-bigquery
+      # @see https://cloud.google.com/bigquery/docs/exporting-data
       #   Exporting Data From BigQuery
       # @see https://cloud.google.com/bigquery/docs/reference/v2/jobs Jobs API
       #   reference
       #
+      # @example
+      #   require "google/cloud/bigquery"
+      #
+      #   bigquery = Google::Cloud::Bigquery.new
+      #   dataset = bigquery.dataset "my_dataset"
+      #   table = dataset.table "my_table"
+      #
+      #   extract_job = table.extract_job "gs://my-bucket/file-name.json",
+      #                                   format: "json"
+      #   extract_job.wait_until_done!
+      #   extract_job.done? #=> true
+      #
       class ExtractJob < Job
         ##
         # The URI or URIs representing the Google Cloud Storage files to which
@@ -38,7 +50,10 @@ module Google
 
         ##
         # The table from which the data is exported. This is the table upon
-        # which {Table#extract} was called. Returns a {Table} instance.
+        # which {Table#extract_job} was called.
+        #
+        # @return [Table] A table instance.
+        #
         def source
           table = @gapi.configuration.extract.source_table
           return nil unless table
@@ -50,6 +65,9 @@ module Google
         ##
         # Checks if the export operation compresses the data using gzip. The
         # default is `false`.
+        #
+        # @return [Boolean] `true` when `GZIP`, `false` otherwise.
+        #
         def compression?
           val = @gapi.configuration.extract.compression
           val == "GZIP"
@@ -58,6 +76,10 @@ module Google
         ##
         # Checks if the destination format for the data is [newline-delimited
         # JSON](http://jsonlines.org/). The default is `false`.
+        #
+        # @return [Boolean] `true` when `NEWLINE_DELIMITED_JSON`, `false`
+        #   otherwise.
+        #
         def json?
           val = @gapi.configuration.extract.destination_format
           val == "NEWLINE_DELIMITED_JSON"
@@ -67,6 +89,9 @@ module Google
         # Checks if the destination format for the data is CSV. Tables with
         # nested or repeated fields cannot be exported as CSV. The default is
         # `true`.
+        #
+        # @return [Boolean] `true` when `CSV`, `false` otherwise.
+        #
         def csv?
           val = @gapi.configuration.extract.destination_format
           return true if val.nil?
@@ -76,14 +101,20 @@ module Google
         ##
         # Checks if the destination format for the data is
         # [Avro](http://avro.apache.org/). The default is `false`.
+        #
+        # @return [Boolean] `true` when `AVRO`, `false` otherwise.
+        #
         def avro?
           val = @gapi.configuration.extract.destination_format
           val == "AVRO"
         end
 
         ##
-        # The symbol the operation uses to delimit fields in the exported data.
-        # The default is a comma (,).
+        # The character or symbol the operation uses to delimit fields in the
+        # exported data. The default is a comma (,).
+        #
+        # @return [String] A string containing the character, such as `","`.
+        #
         def delimiter
           val = @gapi.configuration.extract.field_delimiter
           val = "," if val.nil?
@@ -93,6 +124,10 @@ module Google
         ##
         # Checks if the exported data contains a header row. The default is
         # `true`.
+        #
+        # @return [Boolean] `true` when the print header configuration is
+        #   present or `nil`, `false` otherwise.
+        #
         def print_header?
           val = @gapi.configuration.extract.print_header
           val = true if val.nil?
@@ -100,17 +135,23 @@ module Google
         end
 
         ##
-        # The count of files per destination URI or URI pattern specified in
-        # {#destinations}. Returns an Array of values in the same order as the
-        # URI patterns.
+        # The number of files per destination URI or URI pattern specified in
+        # {#destinations}.
+        #
+        # @return [Array<Integer>] An array of values in the same order as the
+        #   URI patterns.
+        #
         def destinations_file_counts
           Array @gapi.statistics.extract.destination_uri_file_counts
         end
 
         ##
-        # The count of files per destination URI or URI pattern specified in
-        # {#destinations}. Returns a Hash with the URI patterns as keys and the
-        # counts as values.
+        # A hash containing the URI or URI pattern specified in
+        # {#destinations} mapped to the counts of files per destination.
+        #
+        # @return [Hash<String, Integer>] A Hash with the URI patterns as keys
+        #   and the counts as values.
+        #
         def destinations_counts
           Hash[destinations.zip destinations_file_counts]
         end

data/lib/google/cloud/bigquery/insert_response.rb

@@ -20,6 +20,28 @@ module Google
     module Bigquery
       ##
       # InsertResponse
+      #
+      # Represents the response from BigQuery when data is inserted into a table
+      # for near-immediate querying, without the need to complete a load
+      # operation before the data can appear in query results. See
+      # {Dataset#insert} and {Table#insert}.
+      #
+      # @see https://cloud.google.com/bigquery/streaming-data-into-bigquery
+      #   Streaming Data Into BigQuery
+      #
+      # @example
+      #   require "google/cloud/bigquery"
+      #
+      #   bigquery = Google::Cloud::Bigquery.new
+      #   dataset = bigquery.dataset "my_dataset"
+      #
+      #   rows = [
+      #     { "first_name" => "Alice", "age" => 21 },
+      #     { "first_name" => "Bob", "age" => 22 }
+      #   ]
+      #
+      #   insert_response = dataset.insert "my_table", rows
+      #
       class InsertResponse
         # @private
         def initialize rows, gapi
@@ -27,18 +49,43 @@ module Google
           @gapi = gapi
         end
 
+        ##
+        # Checks if the error count is zero, meaning that all of the rows were
+        # inserted. Use {#insert_errors} to access the errors.
+        #
+        # @return [Boolean] `true` when the error count is zero, `false`
+        #   otherwise.
+        #
         def success?
           error_count.zero?
         end
 
+
+        ##
+        # The count of rows in the response, minus the count of errors for rows
+        # that were not inserted.
+        #
+        # @return [Integer] The number of rows inserted.
+        #
         def insert_count
           @rows.count - error_count
         end
 
+
+        ##
+        # The count of errors for rows that were not inserted.
+        #
+        # @return [Integer] The number of errors.
+        #
         def error_count
           Array(@gapi.insert_errors).count
         end
 
+        ##
+        # The error objects for rows that were not inserted.
+        #
+        # @return [Array<InsertError>] An array containing error objects.
+        #
         def insert_errors
           Array(@gapi.insert_errors).map do |ie|
             row = @rows[ie.index]
@@ -47,23 +94,54 @@ module Google
           end
         end
 
+        ##
+        # The rows that were not inserted.
+        #
+        # @return [Array<Hash>] An array of hash objects containing the row
+        #   data.
+        #
         def error_rows
           Array(@gapi.insert_errors).map do |ie|
             @rows[ie.index]
           end
         end
 
+        ##
+        # Returns the error object for a row that was not inserted.
+        #
+        # @param [Hash] row A hash containing the data for a row.
+        #
+        # @return [InsertError, nil] An error object, or `nil` if no error is
+        #   found in the response for the row.
+        #
         def insert_error_for row
-          json_row = Convert.to_json_row(row)
-          insert_errors.detect { |e| e.row == json_row }
+          insert_errors.detect { |e| e.row == row }
         end
 
+        ##
+        # Returns the error hashes for a row that was not inserted. Each error
+        # hash contains the following keys: `reason`, `location`, `debugInfo`,
+        # and `message`.
+        #
+        # @param [Hash] row A hash containing the data for a row.
+        #
+        # @return [Array<Hash>, nil] An array of error hashes, or `nil` if no
+        #   errors are found in the response for the row.
+        #
         def errors_for row
           ie = insert_error_for row
           return ie.errors if ie
           []
         end
 
+        ##
+        # Returns the index for a row that was not inserted.
+        #
+        # @param [Hash] row A hash containing the data for a row.
+        #
+        # @return [Integer, nil] An error object, or `nil` if no error is
+        #   found in the response for the row.
+        #
         def index_for row
           ie = insert_error_for row
           return ie.index if ie
@@ -78,6 +156,16 @@ module Google
 
         ##
         # InsertError
+        #
+        # Represents the errors for a row that was not inserted.
+        #
+        # @attr_reader [Integer] index The index of the row that error applies
+        #   to.
+        # @attr_reader [Hash] row The row that error applies to.
+        # @attr_reader [Hash] errors Error information for the row indicated by
+        #   the index property, with the following keys: `reason`, `location`,
+        #   `debugInfo`, and `message`.
+        #
         class InsertError
           attr_reader :index
           attr_reader :row

data/lib/google/cloud/bigquery/job.rb

@@ -15,7 +15,6 @@
 
 require "google/cloud/errors"
 require "google/cloud/bigquery/service"
-require "google/cloud/bigquery/query_data"
 require "google/cloud/bigquery/job/list"
 require "json"
 
@@ -31,11 +30,11 @@ module Google
       # {CopyJob}, {ExtractJob}, {LoadJob}, and {QueryJob}.
       #
       # A job instance is created when you call {Project#query_job},
-      # {Dataset#query_job}, {Table#copy}, {Table#extract}, {Table#load}, or
-      # {View#data}.
+      # {Dataset#query_job}, {Table#copy_job}, {Table#extract_job},
+      # {Table#load_job}, or {View#data}.
       #
-      # @see https://cloud.google.com/bigquery/docs/managing_jobs_datasets_projects
-      #   Managing Jobs, Datasets, and Projects
+      # @see https://cloud.google.com/bigquery/docs/managing-jobs Running and
+      #   Managing Jobs
       # @see https://cloud.google.com/bigquery/docs/reference/v2/jobs Jobs API
       #   reference
       #
@@ -52,7 +51,7 @@ module Google
       #   if job.failed?
       #     puts job.error
       #   else
-      #     puts job.query_results.first
+      #     puts job.data.first
       #   end
       #
       class Job
@@ -73,21 +72,41 @@ module Google
 
         ##
         # The ID of the job.
+        #
+        # @return [String] The ID must contain only letters (a-z, A-Z), numbers
+        #   (0-9), underscores (_), or dashes (-). The maximum length is 1,024
+        #   characters.
+        #
         def job_id
           @gapi.job_reference.job_id
         end
 
         ##
         # The ID of the project containing the job.
+        #
+        # @return [String] The project ID.
+        #
         def project_id
           @gapi.job_reference.project_id
         end
 
         ##
-        # The current state of the job. The possible values are `PENDING`,
-        # `RUNNING`, and `DONE`. A `DONE` state does not mean that the job
-        # completed successfully. Use {#failed?} to discover if an error
+        # The email address of the user who ran the job.
+        #
+        # @return [String] The email address.
+        #
+        def user_email
+          @gapi.user_email
+        end
+
+        ##
+        # The current state of the job. A `DONE` state does not mean that the
+        # job completed successfully. Use {#failed?} to discover if an error
         # occurred or if the job was successful.
+        #
+        # @return [String] The state code. The possible values are `PENDING`,
+        #   `RUNNING`, and `DONE`.
+        #
         def state
           return nil if @gapi.status.nil?
           @gapi.status.state
@@ -95,6 +114,9 @@ module Google
 
         ##
         # Checks if the job's state is `RUNNING`.
+        #
+        # @return [Boolean] `true` when `RUNNING`, `false` otherwise.
+        #
         def running?
           return false if state.nil?
           "running".casecmp(state).zero?
@@ -102,6 +124,9 @@ module Google
 
         ##
         # Checks if the job's state is `PENDING`.
+        #
+        # @return [Boolean] `true` when `PENDING`, `false` otherwise.
+        #
         def pending?
           return false if state.nil?
           "pending".casecmp(state).zero?
@@ -112,19 +137,29 @@ module Google
         # running. However, a `DONE` state does not mean that the job completed
         # successfully.  Use {#failed?} to detect if an error occurred or if the
         # job was successful.
+        #
+        # @return [Boolean] `true` when `DONE`, `false` otherwise.
+        #
         def done?
           return false if state.nil?
           "done".casecmp(state).zero?
         end
 
         ##
-        # Checks if an error is present.
+        # Checks if an error is present. Use {#error} to access the error
+        # object.
+        #
+        # @return [Boolean] `true` when there is an error, `false` otherwise.
+        #
         def failed?
           !error.nil?
         end
 
         ##
         # The time when the job was created.
+        #
+        # @return [Time, nil] The creation time from the job statistics.
+        #
         def created_at
           ::Time.at(Integer(@gapi.statistics.creation_time) / 1000.0)
         rescue
@@ -135,6 +170,9 @@ module Google
         # The time when the job was started.
         # This field is present after the job's state changes from `PENDING`
         # to either `RUNNING` or `DONE`.
+        #
+        # @return [Time, nil] The start time from the job statistics.
+        #
         def started_at
           ::Time.at(Integer(@gapi.statistics.start_time) / 1000.0)
         rescue
@@ -144,6 +182,9 @@ module Google
         ##
         # The time when the job ended.
         # This field is present when the job's state is `DONE`.
+        #
+        # @return [Time, nil] The end time from the job statistics.
+        #
         def ended_at
           ::Time.at(Integer(@gapi.statistics.end_time) / 1000.0)
         rescue
@@ -165,6 +206,9 @@ module Google
         #
         # @see https://cloud.google.com/bigquery/docs/reference/v2/jobs Jobs API
         #   reference
+        #
+        # @return [Hash] The job statistics.
+        #
         def statistics
           JSON.parse @gapi.statistics.to_json
         end
@@ -173,6 +217,9 @@ module Google
         ##
         # The job's status. Returns a hash. The values contained in the hash are
         # also exposed by {#state}, {#error}, and {#errors}.
+        #
+        # @return [Hash] The job status.
+        #
         def status
           JSON.parse @gapi.status.to_json
         end
@@ -184,7 +231,8 @@ module Google
         # @see https://cloud.google.com/bigquery/docs/reference/v2/jobs Jobs API
         #   reference
         #
-        # @return [Hash] Returns a hash containing `reason` and `message` keys:
+        # @return [Hash, nil] Returns a hash containing `reason` and `message`
+        #   keys:
         #
         #   {
         #     "reason"=>"notFound",
@@ -192,21 +240,55 @@ module Google
         #   }
         #
         def error
-          return nil if @gapi.status.nil?
-          return nil if @gapi.status.error_result.nil?
-          JSON.parse @gapi.status.error_result.to_json
+          status["errorResult"]
         end
 
         ##
         # The errors for the job, if any errors have occurred. Returns an array
         # of hash objects. See {#error}.
+        #
+        # @return [Array<Hash>, nil] Returns an array of hashes containing
+        #   `reason` and `message` keys:
+        #
+        #   {
+        #     "reason"=>"notFound",
+        #     "message"=>"Not found: Table publicdata:samples.BAD_ID"
+        #   }
+        #
         def errors
-          return [] if @gapi.status.nil?
-          Array(@gapi.status.errors).map { |e| JSON.parse e.to_json }
+          Array status["errors"]
+        end
+
+        ##
+        # A hash of user-provided labels associated with this job. Labels can be
+        # provided when the job is created, and used to organize and group jobs.
+        #
+        # The returned hash is frozen and changes are not allowed. Use
+        # {#labels=} to replace the entire hash.
+        #
+        # @return [Hash] The job labels.
+        #
+        # @!group Attributes
+        #
+        def labels
+          m = @gapi.configuration.labels
+          m = m.to_h if m.respond_to? :to_h
+          m.dup.freeze
         end
 
         ##
         # Cancels the job.
+        #
+        # @example
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #
+        #   job = bigquery.query_job "SELECT COUNT(word) as count FROM " \
+        #                            "publicdata.samples.shakespeare"
+        #
+        #   job.cancel
+        #
         def cancel
           ensure_service!
           resp = service.cancel_job job_id
@@ -216,6 +298,18 @@ module Google
 
         ##
         # Created a new job with the current configuration.
+        #
+        # @example
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #
+        #   job = bigquery.query_job "SELECT COUNT(word) as count FROM " \
+        #                            "publicdata.samples.shakespeare"
+        #
+        #   job.wait_until_done!
+        #   job.rerun!
+        #
         def rerun!
           ensure_service!
           gapi = service.insert_job @gapi.configuration
@@ -224,6 +318,19 @@ module Google
 
         ##
         # Reloads the job with current data from the BigQuery service.
+        #
+        # @example
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #
+        #   job = bigquery.query_job "SELECT COUNT(word) as count FROM " \
+        #                            "publicdata.samples.shakespeare"
+        #
+        #   job.done?
+        #   job.reload!
+        #   job.done? #=> true
+        #
         def reload!
           ensure_service!
           gapi = service.get_job job_id
@@ -232,8 +339,9 @@ module Google
         alias_method :refresh!, :reload!
 
         ##
-        # Refreshes the job until the job is `DONE`.
-        # The delay between refreshes will incrementally increase.
+        # Refreshes the job until the job is `DONE`. The delay between refreshes
+        # starts at 5 seconds and increases exponentially to a maximum of 60
+        # seconds.
         #
         # @example
         #   require "google/cloud/bigquery"
@@ -242,12 +350,16 @@ module Google
         #   dataset = bigquery.dataset "my_dataset"
         #   table = dataset.table "my_table"
         #
-        #   extract_job = table.extract "gs://my-bucket/file-name.json",
-        #                               format: "json"
+        #   extract_job = table.extract_job "gs://my-bucket/file-name.json",
+        #                                   format: "json"
         #   extract_job.wait_until_done!
         #   extract_job.done? #=> true
+        #
         def wait_until_done!
-          backoff = ->(retries) { sleep 2 * retries + 5 }
+          backoff = lambda do |retries|
+            delay = [retries ** 2 + 5, 60].min # Maximum delay is 60
+            sleep delay
+          end
           retries = 0
           until done?
             backoff.call retries
@@ -266,6 +378,20 @@ module Google
           end
         end
 
+        ##
+        # @private New Google::Apis::Error with job failure details
+        def gapi_error
+          return nil unless failed?
+
+          error_status_code = status_code_for_reason error["reason"]
+          error_body = error
+          error_body["errors"] = errors
+
+          Google::Apis::Error.new error["message"],
+                                  status_code: error_status_code,
+                                  body: error_body
+        end
+
         protected
 
         ##
@@ -296,6 +422,19 @@ module Google
         rescue Google::Cloud::NotFoundError
           nil
         end
+
+        def status_code_for_reason reason
+          codes = { "accessDenied" => 403, "backendError" => 500,
+                    "billingNotEnabled" => 403,
+                    "billingTierLimitExceeded" => 400, "blocked" => 403,
+                    "duplicate" => 409, "internalError" =>500, "invalid" => 400,
+                    "invalidQuery" => 400, "notFound" =>404,
+                    "notImplemented" => 501, "quotaExceeded" => 403,
+                    "rateLimitExceeded" => 403, "resourceInUse" => 400,
+                    "resourcesExceeded" => 400, "responseTooLarge" => 403,
+                    "tableUnavailable" => 400 }
+          codes[reason] || 0
+        end
       end
     end
   end