google-cloud-bigquery 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 41d3da96cf5cfe992d89be7d76eae719bee105acab2b2621ae9d96637963c4fa
-  data.tar.gz: 2da003b89f6ab554f97941a56aa54bc35b0a9bb239b1bf19638a1ed148df8592
+  metadata.gz: d79d312ec01c34460988a536e01fe9b32d937cd357baa16275921fa327f5474d
+  data.tar.gz: 2b24200431e4a883dbb914db75e3b0507f1b3c29d3c0401929772c1c7f4e54a4
 SHA512:
-  metadata.gz: a049fc85d22b3ea866a5beff61c46e987411acf3d946837adf465ebfabfade7d4fc633e487b29eb3b3b0be36c0f08e3b8740cc636a8c5c362b40b13fc47ad127
-  data.tar.gz: 7f8c25afa22dfb9259e73ac5b3ae6917629b84bfcde9cf52055d0a88bb680450ef1ed173d8c64edfae06afbb535b3f02e75f024a7032b14907908525d6c4d582
+  metadata.gz: ebf35594fb06f2aadd3a60de6dd44218c5e87e91ef4c735d76f640c36315df1380aa9f69d02f3ca39c89b13a1cbd2808e87c8c9fab33cb9cda588843797ae692
+  data.tar.gz: c2a19edf84a43c4c6f2cbc6e99145acf8db20a8385ce4a8e5553203811270f49dcd940ae234b192073800b2da20fb1930fe93474578fe36fd5325fb32335afb4

data/lib/google/cloud/bigquery/copy_job.rb

@@ -203,6 +203,11 @@ module Google
           # @!group Attributes
           def location= value
             @gapi.job_reference.location = value
+            return unless value.nil?
+
+            # Treat assigning value of nil the same as unsetting the value.
+            unset = @gapi.job_reference.instance_variables.include? :@location
+            @gapi.job_reference.remove_instance_variable :@location if unset
           end
 
           ##

data/lib/google/cloud/bigquery/dataset.rb

@@ -433,7 +433,9 @@ module Google
         # @param [String] name A descriptive name for the table.
         # @param [String] description A user-friendly description of the table.
         # @yield [table] a block for setting the table
-        # @yieldparam [Table] table the table object to be updated
+        # @yieldparam [Google::Cloud::Bigquery::Table::Updater] table An updater
+        #   to set additional properties on the table in the API request to
+        #   create it.
         #
         # @return [Google::Cloud::Bigquery::Table] A new table object.
         #
@@ -1086,19 +1088,10 @@ module Google
         # @!group Data
         #
         def query query, params: nil, external: nil, max: nil, cache: true,
-                  standard_sql: nil, legacy_sql: nil
-          ensure_service!
-          options = { priority: "INTERACTIVE", external: external, cache: cache,
-                      legacy_sql: legacy_sql, standard_sql: standard_sql,
-                      params: params }
-          options[:dataset] ||= self
-          updater = QueryJob::Updater.from_options service, query, options
-          updater.location = location if location # may be dataset reference
-
-          yield updater if block_given?
-
-          gapi = service.query_job updater.to_gapi
-          job = Job.from_gapi gapi, service
+                  standard_sql: nil, legacy_sql: nil, &block
+          job = query_job query, params: params, external: external,
+                                 cache: cache, standard_sql: standard_sql,
+                                 legacy_sql: legacy_sql, &block
           job.wait_until_done!
           ensure_job_succeeded! job
 
@@ -1629,27 +1622,19 @@ module Google
                  projection_fields: nil, jagged_rows: nil, quoted_newlines: nil,
                  encoding: nil, delimiter: nil, ignore_unknown: nil,
                  max_bad_records: nil, quote: nil, skip_leading: nil,
-                 schema: nil, autodetect: nil, null_marker: nil
-          ensure_service!
-
-          updater = load_job_updater table_id,
-                                     format: format, create: create,
-                                     write: write,
-                                     projection_fields: projection_fields,
-                                     jagged_rows: jagged_rows,
-                                     quoted_newlines: quoted_newlines,
-                                     encoding: encoding,
-                                     delimiter: delimiter,
-                                     ignore_unknown: ignore_unknown,
-                                     max_bad_records: max_bad_records,
-                                     quote: quote, skip_leading: skip_leading,
-                                     schema: schema,
-                                     autodetect: autodetect,
-                                     null_marker: null_marker
-
-          yield updater if block_given?
+                 schema: nil, autodetect: nil, null_marker: nil, &block
+          job = load_job table_id, files,
+                         format: format, create: create, write: write,
+                         projection_fields: projection_fields,
+                         jagged_rows: jagged_rows,
+                         quoted_newlines: quoted_newlines,
+                         encoding: encoding, delimiter: delimiter,
+                         ignore_unknown: ignore_unknown,
+                         max_bad_records: max_bad_records,
+                         quote: quote, skip_leading: skip_leading,
+                         schema: schema, autodetect: autodetect,
+                         null_marker: null_marker, &block
 
-          job = load_local_or_uri files, updater
           job.wait_until_done!
           ensure_job_succeeded! job
           true
@@ -1829,6 +1814,12 @@ module Google
         # @param [String] table_id The ID of the destination table.
         # @param [Hash, Array<Hash>] rows A hash object or array of hash objects
         #   containing the data. Required.
+        # @param [Array<String>] insert_ids A unique ID for each row. BigQuery
+        #   uses this property to detect duplicate insertion requests on a
+        #   best-effort basis. For more information, see [data
+        #   consistency](https://cloud.google.com/bigquery/streaming-data-into-bigquery#dataconsistency).
+        #   Optional. If not provided, the client library will assign a UUID to
+        #   each row before the request is sent.
         # @param [Boolean] skip_invalid Insert all valid rows of a request, even
         #   if invalid rows exist. The default value is `false`, which causes
         #   the entire request to fail if any invalid rows exist.
@@ -1884,12 +1875,19 @@ module Google
         #
         # @!group Data
         #
-        def insert table_id, rows, skip_invalid: nil, ignore_unknown: nil,
-                   autocreate: nil
+        def insert table_id, rows, insert_ids: nil, skip_invalid: nil,
+                   ignore_unknown: nil, autocreate: nil
+          rows = [rows] if rows.is_a? Hash
+          insert_ids = Array insert_ids
+          if insert_ids.count > 0 && insert_ids.count != rows.count
+            raise ArgumentError, "insert_ids must be the same size as rows"
+          end
+
           if autocreate
             begin
               insert_data table_id, rows, skip_invalid: skip_invalid,
-                                          ignore_unknown: ignore_unknown
+                                          ignore_unknown: ignore_unknown,
+                                          insert_ids: insert_ids
             rescue Google::Cloud::NotFoundError
               sleep rand(1..60)
               begin
@@ -1904,11 +1902,13 @@ module Google
               sleep 60
               insert table_id, rows, skip_invalid: skip_invalid,
                                      ignore_unknown: ignore_unknown,
-                                     autocreate: true
+                                     autocreate: true,
+                                     insert_ids: insert_ids
             end
           else
             insert_data table_id, rows, skip_invalid: skip_invalid,
-                                        ignore_unknown: ignore_unknown
+                                        ignore_unknown: ignore_unknown,
+                                        insert_ids: insert_ids
           end
         end
 
@@ -1976,12 +1976,14 @@ module Google
 
         protected
 
-        def insert_data table_id, rows, skip_invalid: nil, ignore_unknown: nil
+        def insert_data table_id, rows, skip_invalid: nil, ignore_unknown: nil,
+                        insert_ids: nil
           rows = [rows] if rows.is_a? Hash
           raise ArgumentError, "No rows provided" if rows.empty?
           ensure_service!
           options = { skip_invalid: skip_invalid,
-                      ignore_unknown: ignore_unknown }
+                      ignore_unknown: ignore_unknown,
+                      insert_ids: insert_ids }
           gapi = service.insert_tabledata dataset_id, table_id, rows, options
           InsertResponse.from_gapi rows, gapi
         end

data/lib/google/cloud/bigquery/extract_job.rb

@@ -224,6 +224,11 @@ module Google
           # @!group Attributes
           def location= value
             @gapi.job_reference.location = value
+            return unless value.nil?
+
+            # Treat assigning value of nil the same as unsetting the value.
+            unset = @gapi.job_reference.instance_variables.include? :@location
+            @gapi.job_reference.remove_instance_variable :@location if unset
           end
 
           ##

data/lib/google/cloud/bigquery/load_job.rb

@@ -745,6 +745,11 @@ module Google
           # @!group Attributes
           def location= value
             @gapi.job_reference.location = value
+            return unless value.nil?
+
+            # Treat assigning value of nil the same as unsetting the value.
+            unset = @gapi.job_reference.instance_variables.include? :@location
+            @gapi.job_reference.remove_instance_variable :@location if unset
           end
 
           ##

data/lib/google/cloud/bigquery/project.rb

@@ -84,6 +84,17 @@ module Google
         end
         alias project project_id
 
+        ##
+        # The email address of the service account for the project used to
+        # connect to BigQuery. (See also {#project_id}.)
+        #
+        # @return [String] The service account email address.
+        #
+        def service_account_email
+          @service_account_email ||= \
+            service.project_service_account.email
+        end
+
         ##
         # Queries data by creating a [query
         # job](https://cloud.google.com/bigquery/docs/query-overview#query_jobs).
@@ -514,18 +525,12 @@ module Google
         #   end
         #
         def query query, params: nil, external: nil, max: nil, cache: true,
-                  dataset: nil, project: nil, standard_sql: nil, legacy_sql: nil
-          ensure_service!
-          options = { priority: "INTERACTIVE", cache: cache, dataset: dataset,
-                      project: project || self.project,
-                      legacy_sql: legacy_sql, standard_sql: standard_sql,
-                      params: params, external: external }
-          updater = QueryJob::Updater.from_options service, query, options
-
-          yield updater if block_given?
-
-          gapi = service.query_job updater.to_gapi
-          job = Job.from_gapi gapi, service
+                  dataset: nil, project: nil, standard_sql: nil,
+                  legacy_sql: nil, &block
+          job = query_job query, params: params, external: external,
+                                 cache: cache, dataset: dataset,
+                                 project: project, standard_sql: standard_sql,
+                                 legacy_sql: legacy_sql, &block
           job.wait_until_done!
 
           if job.failed?

data/lib/google/cloud/bigquery/query_job.rb

@@ -402,6 +402,11 @@ module Google
           # @!group Attributes
           def location= value
             @gapi.job_reference.location = value
+            return unless value.nil?
+
+            # Treat assigning value of nil the same as unsetting the value.
+            unset = @gapi.job_reference.instance_variables.include? :@location
+            @gapi.job_reference.remove_instance_variable :@location if unset
           end
 
           ##

data/lib/google/cloud/bigquery/service.rb

@@ -70,6 +70,10 @@ module Google
         end
         attr_accessor :mocked_service
 
+        def project_service_account
+          service.get_project_service_account project
+        end
+
         ##
         # Lists all datasets in the specified project to which you have
         # been granted the READER dataset role.
@@ -202,15 +206,17 @@ module Google
 
         def insert_tabledata dataset_id, table_id, rows, options = {}
           json_rows = Array(rows).map { |row| Convert.to_json_row row }
-
           insert_tabledata_json_rows dataset_id, table_id, json_rows, options
         end
 
         def insert_tabledata_json_rows dataset_id, table_id, json_rows,
                                        options = {}
-          insert_rows = Array(json_rows).map do |json_row|
+
+          rows_and_ids = Array(json_rows).zip Array(options[:insert_ids])
+          insert_rows = rows_and_ids.map do |json_row, insert_id|
+            insert_id ||= SecureRandom.uuid
             {
-              insertId: SecureRandom.uuid,
+              insertId: insert_id,
               json: json_row
             }
           end

data/lib/google/cloud/bigquery/table.rb

@@ -1836,26 +1836,18 @@ module Google
                  projection_fields: nil, jagged_rows: nil, quoted_newlines: nil,
                  encoding: nil, delimiter: nil, ignore_unknown: nil,
                  max_bad_records: nil, quote: nil, skip_leading: nil,
-                 autodetect: nil, null_marker: nil
-          ensure_service!
+                 autodetect: nil, null_marker: nil, &block
+          job = load_job files, format: format, create: create, write: write,
+                                projection_fields: projection_fields,
+                                jagged_rows: jagged_rows,
+                                quoted_newlines: quoted_newlines,
+                                encoding: encoding, delimiter: delimiter,
+                                ignore_unknown: ignore_unknown,
+                                max_bad_records: max_bad_records,
+                                quote: quote, skip_leading: skip_leading,
+                                autodetect: autodetect,
+                                null_marker: null_marker, &block
 
-          updater = load_job_updater format: format, create: create,
-                                     write: write,
-                                     projection_fields: projection_fields,
-                                     jagged_rows: jagged_rows,
-                                     quoted_newlines: quoted_newlines,
-                                     encoding: encoding,
-                                     delimiter: delimiter,
-                                     ignore_unknown: ignore_unknown,
-                                     max_bad_records: max_bad_records,
-                                     quote: quote, skip_leading: skip_leading,
-                                     schema: schema,
-                                     autodetect: autodetect,
-                                     null_marker: null_marker
-
-          yield updater if block_given?
-
-          job = load_local_or_uri files, updater
           job.wait_until_done!
           ensure_job_succeeded! job
           true
@@ -1871,6 +1863,12 @@ module Google
         #
         # @param [Hash, Array<Hash>] rows A hash object or array of hash objects
         #   containing the data. Required.
+        # @param [Array<String>] insert_ids A unique ID for each row. BigQuery
+        #   uses this property to detect duplicate insertion requests on a
+        #   best-effort basis. For more information, see [data
+        #   consistency](https://cloud.google.com/bigquery/streaming-data-into-bigquery#dataconsistency).
+        #   Optional. If not provided, the client library will assign a UUID to
+        #   each row before the request is sent.
         # @param [Boolean] skip_invalid Insert all valid rows of a request, even
         #   if invalid rows exist. The default value is `false`, which causes
         #   the entire request to fail if any invalid rows exist.
@@ -1908,12 +1906,18 @@ module Google
         #
         # @!group Data
         #
-        def insert rows, skip_invalid: nil, ignore_unknown: nil
+        def insert rows, insert_ids: nil, skip_invalid: nil, ignore_unknown: nil
+          rows = [rows] if rows.is_a? Hash
+          insert_ids = Array insert_ids
+          if insert_ids.count > 0 && insert_ids.count != rows.count
+            raise ArgumentError, "insert_ids must be the same size as rows"
+          end
           rows = [rows] if rows.is_a? Hash
           raise ArgumentError, "No rows provided" if rows.empty?
           ensure_service!
           options = { skip_invalid: skip_invalid,
-                      ignore_unknown: ignore_unknown }
+                      ignore_unknown: ignore_unknown,
+                      insert_ids: insert_ids }
           gapi = service.insert_tabledata dataset_id, table_id, rows, options
           InsertResponse.from_gapi rows, gapi
         end

data/lib/google/cloud/bigquery/table/async_inserter.rb

@@ -16,6 +16,7 @@
 require "google/cloud/bigquery/convert"
 require "monitor"
 require "concurrent"
+require "securerandom"
 
 module Google
   module Cloud
@@ -100,24 +101,30 @@ module Google
           #
           # @param [Hash, Array<Hash>] rows A hash object or array of hash
           #   objects containing the data.
+          # @param [Array<String>] insert_ids A unique ID for each row. BigQuery
+          #   uses this property to detect duplicate insertion requests on a
+          #   best-effort basis. For more information, see [data
+          #   consistency](https://cloud.google.com/bigquery/streaming-data-into-bigquery#dataconsistency).
+          #   Optional. If not provided, the client library will assign a UUID
+          #   to each row before the request is sent.
           #
-          def insert rows
+          def insert rows, insert_ids: nil
             return nil if rows.nil?
             return nil if rows.is_a?(Array) && rows.empty?
-            rows = [rows] if rows.is_a? Hash
+            rows, insert_ids = validate_insert_args rows, insert_ids
 
             synchronize do
-              rows.each do |row|
+              rows.zip(Array(insert_ids)).each do |row, insert_id|
                 if @batch.nil?
                   @batch = Batch.new max_bytes: @max_bytes, max_rows: @max_rows
-                  @batch.insert row
+                  @batch.insert row, insert_id
                 else
-                  unless @batch.try_insert row
+                  unless @batch.try_insert row, insert_id
                     push_batch_request!
 
                     @batch = Batch.new max_bytes: @max_bytes,
                                        max_rows: @max_rows
-                    @batch.insert row
+                    @batch.insert row, insert_id
                   end
                 end
 
@@ -204,6 +211,15 @@ module Google
 
           protected
 
+          def validate_insert_args rows, insert_ids
+            rows = [rows] if rows.is_a? Hash
+            insert_ids = Array insert_ids
+            if insert_ids.count > 0 && insert_ids.count != rows.count
+              raise ArgumentError, "insert_ids must be the same size as rows"
+            end
+            [rows, insert_ids]
+          end
+
           def run_background
             synchronize do
               until @stopped
@@ -230,11 +246,13 @@ module Google
 
             orig_rows = @batch.rows
             json_rows = @batch.json_rows
+            insert_ids = @batch.insert_ids
             Concurrent::Future.new(executor: @thread_pool) do
               begin
                 raise ArgumentError, "No rows provided" if json_rows.empty?
                 options = { skip_invalid: @skip_invalid,
-                            ignore_unknown: @ignore_unknown }
+                            ignore_unknown: @ignore_unknown,
+                            insert_ids: insert_ids }
                 insert_resp = @table.service.insert_tabledata_json_rows(
                   @table.dataset_id, @table.table_id, json_rows, options
                 )
@@ -255,30 +273,37 @@ module Google
           ##
           # @private
           class Batch
-            attr_reader :max_bytes, :max_rows, :rows, :json_rows
+            attr_reader :max_bytes, :max_rows, :rows, :json_rows, :insert_ids
 
             def initialize max_bytes: 10000000, max_rows: 500
               @max_bytes = max_bytes
               @max_rows = max_rows
               @rows = []
               @json_rows = []
-              @current_bytes = 0
+              @insert_ids = []
+              # The default request byte size overhead is 63.
+              # "{\"rows\":[],\"ignoreUnknownValues\":false,
+              # \"skipInvalidRows\":false}".bytesize #=> 63
+              @current_bytes = 63
             end
 
-            def insert row
+            def insert row, insert_id
+              insert_id ||= SecureRandom.uuid
               json_row = to_json_row row
 
-              insert_rows_bytes row, json_row, addl_bytes_for_json_row(json_row)
+              insert_rows_bytes \
+                row, json_row, insert_id, addl_bytes_for(json_row, insert_id)
             end
 
-            def try_insert row
+            def try_insert row, insert_id
+              insert_id ||= SecureRandom.uuid
               json_row = to_json_row row
-              addl_bytes = addl_bytes_for_json_row json_row
+              addl_bytes = addl_bytes_for json_row, insert_id
 
               return false if @current_bytes + addl_bytes >= @max_bytes
               return false if @rows.count + 1 >= @max_rows
 
-              insert_rows_bytes row, json_row, addl_bytes
+              insert_rows_bytes row, json_row, insert_id, addl_bytes
               true
             end
 
@@ -288,9 +313,10 @@ module Google
 
             private
 
-            def insert_rows_bytes row, json_row, addl_bytes
+            def insert_rows_bytes row, json_row, insert_id, addl_bytes
               @rows << row
               @json_rows << json_row
+              @insert_ids << insert_id if insert_id
               @current_bytes += addl_bytes
             end
 
@@ -298,12 +324,9 @@ module Google
               Convert.to_json_row row
             end
 
-            def addl_bytes_for row
-              addl_bytes_for_json_row Convert.to_json_row(row)
-            end
-
-            def addl_bytes_for_json_row json_row
-              json_row.to_json.bytesize + 1
+            def addl_bytes_for json_row, insert_id
+              # "{\"insertId\":\"\",\"json\":},".bytesize #=> 24
+              24 + json_row.to_json.bytesize + insert_id.bytesize
             end
           end
 

data/lib/google/cloud/bigquery/version.rb

@@ -16,7 +16,7 @@
 module Google
   module Cloud
     module Bigquery
-      VERSION = "1.2.0".freeze
+      VERSION = "1.3.0".freeze
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: google-cloud-bigquery
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Mike Moore
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-31 00:00:00.000000000 Z
+date: 2018-04-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-cloud-core