google-cloud-bigquery 1.12.0 → 1.38.1

data/lib/google/cloud/bigquery/service.rb

@@ -19,7 +19,7 @@ require "google/cloud/errors"
 require "google/apis/bigquery_v2"
 require "pathname"
 require "securerandom"
-require "mime/types"
+require "mini_mime"
 require "date"
 
 module Google
@@ -39,15 +39,17 @@ module Google
         attr_accessor :credentials
 
         # @private
-        attr_reader :retries, :timeout
+        attr_reader :retries, :timeout, :host
 
         ##
         # Creates a new Service instance.
-        def initialize project, credentials, retries: nil, timeout: nil
+        def initialize project, credentials, retries: nil, timeout: nil, host: nil, quota_project: nil
           @project = project
           @credentials = credentials
           @retries = retries
           @timeout = timeout
+          @host = host
+          @quota_project = quota_project
         end
 
         def service
@@ -55,8 +57,7 @@ module Google
           @service ||= begin
             service = API::BigqueryService.new
             service.client_options.application_name    = "gcloud-ruby"
-            service.client_options.application_version = \
-              Google::Cloud::Bigquery::VERSION
+            service.client_options.application_version = Google::Cloud::Bigquery::VERSION
             service.client_options.open_timeout_sec = timeout
             service.client_options.read_timeout_sec = timeout
             service.client_options.send_timeout_sec = timeout
@@ -64,7 +65,11 @@ module Google
             service.request_options.header ||= {}
             service.request_options.header["x-goog-api-client"] = \
               "gl-ruby/#{RUBY_VERSION} gccl/#{Google::Cloud::Bigquery::VERSION}"
+            service.request_options.query ||= {}
+            service.request_options.query["prettyPrint"] = false
+            service.request_options.quota_project = @quota_project if @quota_project
             service.authorization = @credentials.client
+            service.root_url = host if host
             service
           end
         end
@@ -77,12 +82,10 @@ module Google
         ##
         # Lists all datasets in the specified project to which you have
         # been granted the READER dataset role.
-        def list_datasets options = {}
+        def list_datasets all: nil, filter: nil, max: nil, token: nil
           # The list operation is considered idempotent
           execute backoff: true do
-            service.list_datasets \
-              @project, all: options[:all], filter: options[:filter],
-                        max_results: options[:max], page_token: options[:token]
+            service.list_datasets @project, all: all, filter: filter, max_results: max, page_token: token
           end
         end
 
@@ -113,8 +116,7 @@ module Google
             patch_with_backoff = true
           end
           execute backoff: patch_with_backoff do
-            service.patch_dataset @project, dataset_id, patched_dataset_gapi,
-                                  options: options
+            service.patch_dataset @project, dataset_id, patched_dataset_gapi, options: options
           end
         end
 
@@ -133,15 +135,15 @@ module Google
         ##
         # Lists all tables in the specified dataset.
         # Requires the READER dataset role.
-        def list_tables dataset_id, options = {}
+        def list_tables dataset_id, max: nil, token: nil
           # The list operation is considered idempotent
           execute backoff: true do
-            service.list_tables @project, dataset_id,
-                                max_results: options[:max],
-                                page_token:  options[:token]
+            service.list_tables @project, dataset_id, max_results: max, page_token: token
           end
         end
 
+        ##
+        # Gets the specified table resource by full table reference.
         def get_project_table project_id, dataset_id, table_id
           # The get operation is considered idempotent
           execute backoff: true do
@@ -155,10 +157,7 @@ module Google
         # it only returns the table resource,
         # which describes the structure of this table.
         def get_table dataset_id, table_id
-          # The get operation is considered idempotent
-          execute backoff: true do
-            get_project_table @project, dataset_id, table_id
-          end
+          get_project_table @project, dataset_id, table_id
         end
 
         ##
@@ -179,8 +178,35 @@ module Google
             patch_with_backoff = true
           end
           execute backoff: patch_with_backoff do
-            service.patch_table @project, dataset_id, table_id,
-                                patched_table_gapi, options: options
+            service.patch_table @project, dataset_id, table_id, patched_table_gapi, options: options
+          end
+        end
+
+        ##
+        # Returns Google::Apis::BigqueryV2::Policy
+        def get_table_policy dataset_id, table_id
+          policy_options = API::GetPolicyOptions.new requested_policy_version: 1
+          execute do
+            service.get_table_iam_policy table_path(dataset_id, table_id),
+                                         API::GetIamPolicyRequest.new(options: policy_options)
+          end
+        end
+
+        ##
+        # @param [Google::Apis::BigqueryV2::Policy] new_policy
+        def set_table_policy dataset_id, table_id, new_policy
+          execute do
+            service.set_table_iam_policy table_path(dataset_id, table_id),
+                                         API::SetIamPolicyRequest.new(policy: new_policy)
+          end
+        end
+
+        ##
+        # Returns Google::Apis::BigqueryV2::TestIamPermissionsResponse
+        def test_table_permissions dataset_id, table_id, permissions
+          execute do
+            service.test_table_iam_permissions table_path(dataset_id, table_id),
+                                               API::TestIamPermissionsRequest.new(permissions: permissions)
           end
         end
 
@@ -193,40 +219,45 @@ module Google
 
         ##
         # Retrieves data from the table.
-        def list_tabledata dataset_id, table_id, options = {}
+        def list_tabledata dataset_id, table_id, max: nil, token: nil, start: nil
           # The list operation is considered idempotent
           execute backoff: true do
             json_txt = service.list_table_data \
               @project, dataset_id, table_id,
-              max_results: options.delete(:max),
-              page_token:  options.delete(:token),
-              start_index: options.delete(:start),
+              max_results: max,
+              page_token:  token,
+              start_index: start,
               options:     { skip_deserialization: true }
             JSON.parse json_txt, symbolize_names: true
           end
         end
 
-        def insert_tabledata dataset_id, table_id, rows, options = {}
+        def insert_tabledata dataset_id, table_id, rows, insert_ids: nil, ignore_unknown: nil, skip_invalid: nil
           json_rows = Array(rows).map { |row| Convert.to_json_row row }
-          insert_tabledata_json_rows dataset_id, table_id, json_rows, options
+          insert_tabledata_json_rows dataset_id, table_id, json_rows, insert_ids:     insert_ids,
+                                                                      ignore_unknown: ignore_unknown,
+                                                                      skip_invalid:   skip_invalid
         end
 
-        def insert_tabledata_json_rows dataset_id, table_id, json_rows,
-                                       options = {}
-
-          rows_and_ids = Array(json_rows).zip Array(options[:insert_ids])
+        def insert_tabledata_json_rows dataset_id, table_id, json_rows, insert_ids: nil, ignore_unknown: nil,
+                                       skip_invalid: nil
+          rows_and_ids = Array(json_rows).zip Array(insert_ids)
           insert_rows = rows_and_ids.map do |json_row, insert_id|
-            insert_id ||= SecureRandom.uuid
-            {
-              insertId: insert_id,
-              json:     json_row
-            }
+            if insert_id == :skip
+              { json: json_row }
+            else
+              insert_id ||= SecureRandom.uuid
+              {
+                insertId: insert_id,
+                json:     json_row
+              }
+            end
           end
 
           insert_req = {
             rows:                insert_rows,
-            ignoreUnknownValues: options[:ignore_unknown],
-            skipInvalidRows:     options[:skip_invalid]
+            ignoreUnknownValues: ignore_unknown,
+            skipInvalidRows:     skip_invalid
           }.to_json
 
           # The insertAll with insertId operation is considered idempotent
@@ -245,27 +276,26 @@ module Google
           options = { skip_deserialization: true }
           # The list operation is considered idempotent
           execute backoff: true do
-            json_txt = service.list_models @project, dataset_id,
-                                           max_results: max,
-                                           page_token:  token,
-                                           options:     options
+            json_txt = service.list_models @project, dataset_id, max_results: max, page_token: token, options: options
             JSON.parse json_txt, symbolize_names: true
           end
         end
 
-        # Gets the specified model resource by model ID.
-        # This method does not return the data in the model,
-        # it only returns the model resource,
-        # which describes the structure of this model.
-        def get_model dataset_id, model_id
+        # Gets the specified model resource by full model reference.
+        def get_project_model project_id, dataset_id, model_id
           # The get operation is considered idempotent
           execute backoff: true do
-            json_txt = service.get_model @project, dataset_id, model_id,
-                                         options: { skip_deserialization: true }
+            json_txt = service.get_model project_id, dataset_id, model_id, options: { skip_deserialization: true }
             JSON.parse json_txt, symbolize_names: true
           end
         end
 
+        # Gets the specified model resource by model ID. This method does not return the data in the model, it only
+        # returns the model resource, which describes the structure of this model.
+        def get_model dataset_id, model_id
+          get_project_model @project, dataset_id, model_id
+        end
+
         ##
         # Updates information in an existing model, replacing fields that
         # are provided in the submitted model resource.
@@ -278,9 +308,7 @@ module Google
             patch_with_backoff = true
           end
           execute backoff: patch_with_backoff do
-            json_txt = service.patch_model @project, dataset_id, model_id,
-                                           patched_model_gapi,
-                                           options: options
+            json_txt = service.patch_model @project, dataset_id, model_id, patched_model_gapi, options: options
             JSON.parse json_txt, symbolize_names: true
           end
         end
@@ -292,16 +320,69 @@ module Google
           execute { service.delete_model @project, dataset_id, model_id }
         end
 
+        ##
+        # Creates a new routine in the dataset.
+        def insert_routine dataset_id, new_routine_gapi
+          execute { service.insert_routine @project, dataset_id, new_routine_gapi }
+        end
+
+        ##
+        # Lists all routines in the specified dataset.
+        # Requires the READER dataset role.
+        # Unless readMask is set in the request, only the following fields are populated:
+        #   etag, projectId, datasetId, routineId, routineType, creationTime, lastModifiedTime, and language.
+        def list_routines dataset_id, max: nil, token: nil, filter: nil
+          # The list operation is considered idempotent
+          execute backoff: true do
+            service.list_routines @project, dataset_id, max_results: max,
+                                                        page_token:  token,
+                                                        filter:      filter
+          end
+        end
+
+        ##
+        # Gets the specified routine resource by routine ID.
+        def get_routine dataset_id, routine_id
+          # The get operation is considered idempotent
+          execute backoff: true do
+            service.get_routine @project, dataset_id, routine_id
+          end
+        end
+
+        ##
+        # Updates information in an existing routine, replacing the entire routine resource.
+        def update_routine dataset_id, routine_id, new_routine_gapi
+          update_with_backoff = false
+          options = {}
+          if new_routine_gapi.etag
+            options[:header] = { "If-Match" => new_routine_gapi.etag }
+            # The update with etag operation is considered idempotent
+            update_with_backoff = true
+          end
+          execute backoff: update_with_backoff do
+            service.update_routine @project, dataset_id, routine_id, new_routine_gapi, options: options
+          end
+        end
+
+        ##
+        # Deletes the routine specified by routine_id from the dataset.
+        def delete_routine dataset_id, routine_id
+          execute { service.delete_routine @project, dataset_id, routine_id }
+        end
+
         ##
         # Lists all jobs in the specified project to which you have
         # been granted the READER job role.
-        def list_jobs options = {}
+        def list_jobs all: nil, token: nil, max: nil, filter: nil, min_created_at: nil, max_created_at: nil,
+                      parent_job_id: nil
           # The list operation is considered idempotent
+          min_creation_time = Convert.time_to_millis min_created_at
+          max_creation_time = Convert.time_to_millis max_created_at
           execute backoff: true do
-            service.list_jobs \
-              @project, all_users: options[:all], max_results: options[:max],
-                        page_token: options[:token], projection: "full",
-                        state_filter: options[:filter]
+            service.list_jobs @project, all_users: all, max_results: max,
+                                        page_token: token, projection: "full", state_filter: filter,
+                                        min_creation_time: min_creation_time, max_creation_time: max_creation_time,
+                                        parent_job_id: parent_job_id
           end
         end
 
@@ -324,10 +405,7 @@ module Google
         end
 
         def insert_job config, location: nil
-          job_object = API::Job.new(
-            job_reference: job_ref_from(nil, nil, location: location),
-            configuration: config
-          )
+          job_object = API::Job.new job_reference: job_ref_from(nil, nil, location: location), configuration: config
           # Jobs have generated id, so this operation is considered idempotent
           execute backoff: true do
             service.insert_job @project, job_object
@@ -340,18 +418,25 @@ module Google
           end
         end
 
+        ##
+        # Deletes the job specified by jobId and location (required).
+        def delete_job job_id, location: nil
+          execute do
+            service.delete_job @project, job_id, location: location
+          end
+        end
+
         ##
         # Returns the query data for the job
-        def job_query_results job_id, options = {}
+        def job_query_results job_id, location: nil, max: nil, token: nil, start: nil, timeout: nil
           # The get operation is considered idempotent
           execute backoff: true do
-            service.get_job_query_results \
-              @project, job_id,
-              location:    options.delete(:location),
-              max_results: options.delete(:max),
-              page_token:  options.delete(:token),
-              start_index: options.delete(:start),
-              timeout_ms:  options.delete(:timeout)
+            service.get_job_query_results @project, job_id,
+                                          location:    location,
+                                          max_results: max,
+                                          page_token:  token,
+                                          start_index: start,
+                                          timeout_ms:  timeout
           end
         end
 
@@ -375,10 +460,7 @@ module Google
 
         def load_table_file file, load_job_gapi
           execute backoff: true do
-            service.insert_job \
-              @project,
-              load_job_gapi,
-              upload_source: file, content_type: mime_type_for(file)
+            service.insert_job @project, load_job_gapi, upload_source: file, content_type: mime_type_for(file)
           end
         end
 
@@ -402,34 +484,29 @@ module Google
         def self.table_ref_from_s str, default_ref: {}
           str = str.to_s
           m = /\A(((?<prj>\S*)(:|\.))?(?<dts>\S*)\.)?(?<tbl>\S*)\z/.match str
-          unless m
-            raise ArgumentError, "unable to identify table from #{str.inspect}"
-          end
+          raise ArgumentError, "unable to identify table from #{str.inspect}" unless m
           str_table_ref_hash = {
             project_id: m["prj"],
             dataset_id: m["dts"],
             table_id:   m["tbl"]
           }.delete_if { |_, v| v.nil? }
           str_table_ref_hash = default_ref.to_h.merge str_table_ref_hash
-          ref = Google::Apis::BigqueryV2::TableReference.new str_table_ref_hash
+          ref = Google::Apis::BigqueryV2::TableReference.new(**str_table_ref_hash)
           validate_table_ref ref
           ref
         end
 
         def self.validate_table_ref table_ref
-          %i[project_id dataset_id table_id].each do |f|
-            if table_ref.send(f).nil?
-              raise ArgumentError, "TableReference is missing #{f}"
-            end
+          [:project_id, :dataset_id, :table_id].each do |f|
+            raise ArgumentError, "TableReference is missing #{f}" if table_ref.send(f).nil?
           end
         end
 
         ##
         # Lists all projects to which you have been granted any project role.
-        def list_projects options = {}
+        def list_projects max: nil, token: nil
           execute backoff: true do
-            service.list_projects max_results: options[:max],
-                                  page_token:  options[:token]
+            service.list_projects max_results: max, page_token: token
           end
         end
 
@@ -439,10 +516,7 @@ module Google
         def job_ref_from job_id, prefix, location: nil
           prefix ||= "job_"
           job_id ||= "#{prefix}#{generate_id}"
-          job_ref = API::JobReference.new(
-            project_id: @project,
-            job_id:     job_id
-          )
+          job_ref = API::JobReference.new project_id: @project, job_id: job_id
           # BigQuery does not allow nil location, but missing is ok.
           job_ref.location = location if location
           job_ref
@@ -470,22 +544,27 @@ module Google
 
         protected
 
+        # Creates a formatted table path.
+        def table_path dataset_id, table_id
+          "projects/#{@project}/datasets/#{dataset_id}/tables/#{table_id}"
+        end
+
         # Generate a random string similar to the BigQuery service job IDs.
         def generate_id
           SecureRandom.urlsafe_base64 21
         end
 
         def mime_type_for file
-          mime_type = MIME::Types.of(Pathname(file).to_path).first.to_s
-          return nil if mime_type.empty?
-          mime_type
+          mime_type = MiniMime.lookup_by_filename Pathname(file).to_path
+          return nil if mime_type.nil?
+          mime_type.content_type
         rescue StandardError
           nil
         end
 
-        def execute backoff: nil
+        def execute backoff: nil, &block
           if backoff
-            Backoff.new(retries: retries).execute { yield }
+            Backoff.new(retries: retries).execute(&block)
           else
             yield
           end
@@ -500,7 +579,7 @@ module Google
             attr_accessor :backoff
           end
           self.retries = 5
-          self.reasons = %w[rateLimitExceeded backendError]
+          self.reasons = ["rateLimitExceeded", "backendError"]
           self.backoff = lambda do |retries|
             # Max delay is 32 seconds
             # See "Back-off Requirements" here:
@@ -510,31 +589,29 @@ module Google
             sleep delay
           end
 
-          def initialize options = {}
-            @retries = (options[:retries] || Backoff.retries).to_i
-            @reasons = (options[:reasons] || Backoff.reasons).to_a
-            @backoff =  options[:backoff] || Backoff.backoff
+          def initialize retries: nil, reasons: nil, backoff: nil
+            @retries = (retries || Backoff.retries).to_i
+            @reasons = (reasons || Backoff.reasons).to_a
+            @backoff = backoff || Backoff.backoff
           end
 
           def execute
             current_retries = 0
             loop do
-              begin
-                return yield
-              rescue Google::Apis::Error => e
-                raise e unless retry? e.body, current_retries
-
-                @backoff.call current_retries
-                current_retries += 1
-              end
+              return yield
+            rescue Google::Apis::Error => e
+              raise e unless retry? e.body, current_retries
+
+              @backoff.call current_retries
+              current_retries += 1
             end
           end
 
           protected
 
-          def retry? result, current_retries #:nodoc:
-            if current_retries < @retries
-              return true if retry_error_reason? result
+          def retry? result, current_retries
+            if current_retries < @retries && retry_error_reason?(result)
+              return true
             end
             false
           end