google-cloud-bigquery 1.12.0 → 1.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4fb1722765075dded454afdeea337efe2c088c118f6f966c55a84ff9bb065df1
-  data.tar.gz: 2884b2a12b6a5137234dd4a3fd1fc2420ed2c05eaae223ebeae285caa1ba44b8
+  metadata.gz: 95ecbf392c1c918219849034d540067effe661d635f7d435c1f73c0bbdcbefb3
+  data.tar.gz: 6452936f3e68a2fb9249fbdd23e54374ce67a86945b936adc6d06571b0266d5d
 SHA512:
-  metadata.gz: 851560e73e7e1f48b58e08f99ada6f2f822f33f9dfe246e19b6d3abec4b0bfb2ef4802f402e2951ca0e40cb966be2981930fb62608f81d5a57edfa6ae5626d1f
-  data.tar.gz: b1a0dc302400edfe2a499d0299cc913cf85e4f34e29b460b8f5b3564a33b0e9a13c99d37539cc65afee07b4cbcfd7103bb07b4898866ea68dcab2187105ead1a
+  metadata.gz: 3c28948fc38d7fccd1b3775f209926eacbe671ddd36432660f336457d0630d8ce921b1600170cb7d56b093c2f805dd39d84230ddc04668f9e3a3d1238d0038dc
+  data.tar.gz: 7adba9ab644ef05f683a5a1b5f647e9106b77d997796c0d867d569caeac10d778d2948823f383f3eb7cfe1b5946b5c22a46d1403352f8358c7df1500ffdf291c

data/CHANGELOG.md

@@ -1,5 +1,58 @@
 # Release History
 
+### 1.16.0 / 2019-10-03
+
+#### Features
+
+* Add Dataset default_encryption
+  * Add Dataset#default_encryption
+  * Add Dataset#default_encryption=
+
+### 1.15.0 / 2019-09-30
+
+#### Features
+
+* Add Model encryption
+  * Add Model#encryption
+  * Add Model#encryption=
+* Add range support for Google Sheets
+  * Add External::SheetsSource#range
+  * Add External::SheetsSource#range=
+* Support use_avro_logical_types on extract jobs
+  * Add ExtractJob#use_avro_logical_types?
+  * Add ExtractJob::Updater#use_avro_logical_types=
+
+### 1.14.1 / 2019-09-04
+
+#### Documentation
+
+* Add note about streaming insert issues
+  * Acknowledge tradeoffs when inserting rows soon after
+    table metadata has been changed.
+  * Add link to BigQuery Troubleshooting guide.
+
+### 1.14.0 / 2019-08-23
+
+#### Features
+
+* Support overriding of service endpoint
+
+#### Performance Improvements
+
+* Use MiniMime to detect content types
+
+#### Documentation
+
+* Update documentation
+
+### 1.13.0 / 2019-07-31
+
+* Add Table#require_partition_filter
+* List jobs using min and max created_at
+* Reduce thread usage at startup
+  * Allocate threads in pool as needed, not all up front
+* Update documentation links
+
 ### 1.12.0 / 2019-07-10
 
 * Add BigQuery Model API
@@ -34,7 +87,7 @@
 * Add copy and extract methods to Project
   * Add Project#extract and Project#extract_job
   * Add Project#copy and Project#copy_job
-  * Deprecate dryrun param in Table#copy_job, Table#extract_job and 
+  * Deprecate dryrun param in Table#copy_job, Table#extract_job and
     Table#load_job
 * Fix memoization in Dataset#exists? and Table#exists?
   * Add force param to Dataset#exists? and Table#exists?

data/LOGGING.md

@@ -6,7 +6,7 @@ Client](https://github.com/google/google-api-ruby-client/blob/master/README.md#l
 library. The logger that you set may be a Ruby stdlib
 [`Logger`](https://ruby-doc.org/stdlib-2.4.0/libdoc/logger/rdoc/Logger.html) as
 shown below, or a
-[`Google::Cloud::Logging::Logger`](https://googleapis.github.io/google-cloud-ruby/docs/google-cloud-logging/latest/Google/Cloud/Logging/Logger)
+[`Google::Cloud::Logging::Logger`](https://googleapis.dev/ruby/google-cloud-logging/latest)
 that will write logs to [Stackdriver
 Logging](https://cloud.google.com/logging/).
 

data/OVERVIEW.md

@@ -87,7 +87,7 @@ advantages over legacy SQL, including:
 * Complex `JOIN` predicates, including arbitrary expressions
 
 For examples that demonstrate some of these features, see [Standard SQL
-ghlights](https://cloud.google.com/bigquery/docs/reference/standard-sql/migrating-from-legacy-l#standard_sql_highlights).
+ghlights](https://cloud.google.com/bigquery/docs/reference/standard-sql/migrating-from-legacy-sql#standard_sql_highlights).
 
 As shown in this example, standard SQL is the library default:
 

data/lib/google-cloud-bigquery.rb

@@ -135,4 +135,5 @@ Google::Cloud.configure.add_config! :bigquery do |config|
   config.add_field! :scope, nil, match: [String, Array]
   config.add_field! :retries, nil, match: Integer
   config.add_field! :timeout, nil, match: Integer
+  config.add_field! :endpoint, nil, match: String
 end

data/lib/google/cloud/bigquery.rb

@@ -51,6 +51,8 @@ module Google
       # @param [Integer] retries Number of times to retry requests on server
       #   error. The default value is `5`. Optional.
       # @param [Integer] timeout Default timeout to use in requests. Optional.
+      # @param [String] endpoint Override of the endpoint host name. Optional.
+      #   If the param is nil, uses the default endpoint.
       # @param [String] project Alias for the `project_id` argument. Deprecated.
       # @param [String] keyfile Alias for the `credentials` argument.
       #   Deprecated.
@@ -65,26 +67,24 @@ module Google
       #   table = dataset.table "my_table"
       #
       def self.new project_id: nil, credentials: nil, scope: nil, retries: nil,
-                   timeout: nil, project: nil, keyfile: nil
-        project_id  ||= (project || default_project_id)
+                   timeout: nil, endpoint: nil, project: nil, keyfile: nil
         scope       ||= configure.scope
         retries     ||= configure.retries
         timeout     ||= configure.timeout
+        endpoint    ||= configure.endpoint
         credentials ||= (keyfile || default_credentials(scope: scope))
 
         unless credentials.is_a? Google::Auth::Credentials
           credentials = Bigquery::Credentials.new credentials, scope: scope
         end
 
-        if credentials.respond_to? :project_id
-          project_id ||= credentials.project_id
-        end
-        project_id = project_id.to_s # Always cast to a string
+        project_id = resolve_project_id(project_id || project, credentials)
         raise ArgumentError, "project_id is missing" if project_id.empty?
 
         Bigquery::Project.new(
           Bigquery::Service.new(
-            project_id, credentials, retries: retries, timeout: timeout
+            project_id, credentials,
+            retries: retries, timeout: timeout, host: endpoint
           )
         )
       end
@@ -100,6 +100,8 @@ module Google
       #   the keyfile as a String, the contents of the keyfile as a Hash, or a
       #   Google::Auth::Credentials object. (See {Bigquery::Credentials}) (The
       #   parameter `keyfile` is considered deprecated, but may also be used.)
+      # * `endpoint` - (String) Override of the endpoint host name, or `nil`
+      #   to use the default endpoint.
       # * `scope` - (String, Array<String>) The OAuth 2.0 scopes controlling
       #   the set of resources and operations that the connection can access.
       # * `retries` - (Integer) Number of times to retry requests on server
@@ -115,6 +117,16 @@ module Google
         Google::Cloud.configure.bigquery
       end
 
+      ##
+      # @private Resolve project.
+      def self.resolve_project_id given_project, credentials
+        project_id = given_project || default_project_id
+        if credentials.respond_to? :project_id
+          project_id ||= credentials.project_id
+        end
+        project_id.to_s # Always cast to a string
+      end
+
       ##
       # @private Default project.
       def self.default_project_id

data/lib/google/cloud/bigquery/dataset.rb

@@ -335,6 +335,77 @@ module Google
           patch_gapi! :labels
         end
 
+        ##
+        # The {EncryptionConfiguration} object that represents the default
+        # encryption method for all tables and models in the dataset. Once this
+        # property is set, all newly-created partitioned tables and models in
+        # the dataset will have their encryption set to this value, unless table
+        # creation request (or query) overrides it.
+        #
+        # Present only if this dataset is using custom default encryption.
+        #
+        # @see https://cloud.google.com/bigquery/docs/customer-managed-encryption
+        #   Protecting Data with Cloud KMS Keys
+        #
+        # @return [EncryptionConfiguration, nil] The default encryption
+        #   configuration.
+        #
+        #   @!group Attributes
+        #
+        # @example
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #   dataset = bigquery.dataset "my_dataset"
+        #
+        #   encrypt_config = dataset.default_encryption
+        #
+        # @!group Attributes
+        #
+        def default_encryption
+          return nil if reference?
+          ensure_full_data!
+          return nil if @gapi.default_encryption_configuration.nil?
+          EncryptionConfiguration.from_gapi(
+            @gapi.default_encryption_configuration
+          ).freeze
+        end
+
+        ##
+        # Set the {EncryptionConfiguration} object that represents the default
+        # encryption method for all tables and models in the dataset. Once this
+        # property is set, all newly-created partitioned tables and models in
+        # the dataset will have their encryption set to this value, unless table
+        # creation request (or query) overrides it.
+        #
+        # If the dataset is not a full resource representation (see
+        # {#resource_full?}), the full representation will be retrieved before
+        # the update to comply with ETag-based optimistic concurrency control.
+        #
+        # @see https://cloud.google.com/bigquery/docs/customer-managed-encryption
+        #   Protecting Data with Cloud KMS Keys
+        #
+        # @param [EncryptionConfiguration] value The new encryption config.
+        #
+        # @example
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #   dataset = bigquery.dataset "my_dataset"
+        #
+        #   key_name = "projects/a/locations/b/keyRings/c/cryptoKeys/d"
+        #   encrypt_config = bigquery.encryption kms_key: key_name
+        #
+        #   dataset.default_encryption = encrypt_config
+        #
+        # @!group Attributes
+        #
+        def default_encryption= value
+          ensure_full_data!
+          @gapi.default_encryption_configuration = value.to_gapi
+          patch_gapi! :default_encryption_configuration
+        end
+
         ##
         # Retrieves the access rules for a Dataset. The rules can be updated
         # when passing a block, see {Dataset::Access} for all the methods
@@ -1953,9 +2024,18 @@ module Google
         # the need to complete a load operation before the data can appear in
         # query results.
         #
+        # Because BigQuery's streaming API is designed for high insertion rates,
+        # modifications to the underlying table metadata are eventually
+        # consistent when interacting with the streaming system. In most cases
+        # metadata changes are propagated within minutes, but during this period
+        # API responses may reflect the inconsistent state of the table.
+        #
         # @see https://cloud.google.com/bigquery/streaming-data-into-bigquery
         #   Streaming Data Into BigQuery
         #
+        # @see https://cloud.google.com/bigquery/troubleshooting-errors#metadata-errors-for-streaming-inserts
+        #   BigQuery Troubleshooting: Metadata errors for streaming inserts
+        #
         # @param [String] table_id The ID of the destination table.
         # @param [Hash, Array<Hash>] rows A hash object or array of hash objects
         #   containing the data. Required.
@@ -2172,7 +2252,7 @@ module Google
         # Load the complete representation of the dataset if it has been
         # only partially loaded by a request to the API list method.
         def ensure_full_data!
-          reload! if resource_partial?
+          reload! unless resource_full?
         end
 
         def ensure_job_succeeded! job

data/lib/google/cloud/bigquery/external.rb

@@ -1224,6 +1224,52 @@ module Google
             frozen_check!
             @gapi.google_sheets_options.skip_leading_rows = row_count
           end
+
+          ##
+          # Range of a sheet to query from. Only used when non-empty. Typical
+          # format: `{sheet_name}!{top_left_cell_id}:{bottom_right_cell_id}`.
+          #
+          # @return [String] Range of a sheet to query from.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   sheets_url = "https://docs.google.com/spreadsheets/d/1234567980"
+          #   sheets_table = bigquery.external sheets_url do |sheets|
+          #     sheets.range = "sheet1!A1:B20"
+          #   end
+          #
+          #   sheets_table.range #=> "sheet1!A1:B20"
+          #
+          def range
+            @gapi.google_sheets_options.range
+          end
+
+          ##
+          # Set the range of a sheet to query from. Only used when non-empty.
+          # Typical format:
+          # `{sheet_name}!{top_left_cell_id}:{bottom_right_cell_id}`.
+          #
+          # @param [String] new_range New range of a sheet to query from.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   sheets_url = "https://docs.google.com/spreadsheets/d/1234567980"
+          #   sheets_table = bigquery.external sheets_url do |sheets|
+          #     sheets.range = "sheet1!A1:B20"
+          #   end
+          #
+          #   sheets_table.range #=> "sheet1!A1:B20"
+          #
+          def range= new_range
+            frozen_check!
+            @gapi.google_sheets_options.range = new_range
+          end
         end
 
         ##

data/lib/google/cloud/bigquery/extract_job.rb

@@ -156,6 +156,20 @@ module Google
           Hash[destinations.zip destinations_file_counts]
         end
 
+        ##
+        # If `#avro?` (`#format` is set to `"AVRO"`), this flag indicates
+        # whether to enable extracting applicable column types (such as
+        # `TIMESTAMP`) to their corresponding AVRO logical types
+        # (`timestamp-micros`), instead of only using their raw types
+        # (`avro-long`).
+        #
+        # @return [Boolean] `true` when applicable column types will use their
+        #   corresponding AVRO logical types, `false` otherwise.
+        #
+        def use_avro_logical_types?
+          @gapi.configuration.extract.use_avro_logical_types
+        end
+
         ##
         # Yielded to a block to accumulate changes for an API request.
         class Updater < ExtractJob
@@ -175,11 +189,8 @@ module Google
             storage_urls = Array(storage_files).map do |url|
               url.respond_to?(:to_gs_url) ? url.to_gs_url : url
             end
-            dest_format = options[:format]
-            if dest_format.nil?
-              dest_format = Convert.derive_source_format storage_urls.first
-            end
-            req = Google::Apis::BigqueryV2::Job.new(
+            options[:format] ||= Convert.derive_source_format storage_urls.first
+            job = Google::Apis::BigqueryV2::Job.new(
               job_reference: job_ref,
               configuration: Google::Apis::BigqueryV2::JobConfiguration.new(
                 extract: Google::Apis::BigqueryV2::JobConfigurationExtract.new(
@@ -190,12 +201,24 @@ module Google
               )
             )
 
-            updater = ExtractJob::Updater.new req
+            from_job_and_options job, options
+          end
+
+          ##
+          # @private Create an Updater from a Job and options hash.
+          #
+          # @return [Google::Cloud::Bigquery::ExtractJob::Updater] A job
+          #   configuration object for setting query options.
+          def self.from_job_and_options request, options = {}
+            updater = ExtractJob::Updater.new request
             updater.compression = options[:compression]
             updater.delimiter = options[:delimiter]
-            updater.format = dest_format
+            updater.format = options[:format]
             updater.header = options[:header]
             updater.labels = options[:labels] if options[:labels]
+            unless options[:use_avro_logical_types].nil?
+              updater.use_avro_logical_types = options[:use_avro_logical_types]
+            end
             updater
           end
 
@@ -300,6 +323,22 @@ module Google
             @gapi.configuration.update! labels: value
           end
 
+          ##
+          # Indicate whether to enable extracting applicable column types (such
+          # as `TIMESTAMP`) to their corresponding AVRO logical types
+          # (`timestamp-micros`), instead of only using their raw types
+          # (`avro-long`).
+          #
+          # Only used when `#format` is set to `"AVRO"` (`#avro?`).
+          #
+          # @param [Boolean] value Whether applicable column types will use
+          #   their corresponding AVRO logical types.
+          #
+          # @!group Attributes
+          def use_avro_logical_types= value
+            @gapi.configuration.extract.use_avro_logical_types = value
+          end
+
           ##
           # @private Returns the Google API client library version of this job.
           #

data/lib/google/cloud/bigquery/job/list.rb

@@ -71,9 +71,9 @@ module Google
           def next
             return nil unless next?
             ensure_service!
-            options = { all: @hidden, token: token, max: @max, filter: @filter }
-            gapi = @service.list_jobs options
-            self.class.from_gapi gapi, @service, @hidden, @max, @filter
+            next_options = @options.merge token: token
+            next_gapi = @service.list_jobs next_options
+            self.class.from_gapi next_gapi, @service, next_options
           end
 
           ##
@@ -141,17 +141,14 @@ module Google
           ##
           # @private New Job::List from a Google API Client
           # Google::Apis::BigqueryV2::JobList object.
-          def self.from_gapi gapi_list, service, hidden = nil, max = nil,
-                             filter = nil
+          def self.from_gapi gapi_list, service, options = {}
             jobs = List.new(Array(gapi_list.jobs).map do |gapi_object|
               Job.from_gapi gapi_object, service
             end)
-            jobs.instance_variable_set :@token,   gapi_list.next_page_token
-            jobs.instance_variable_set :@etag,    gapi_list.etag
-            jobs.instance_variable_set :@service, service
-            jobs.instance_variable_set :@hidden,  hidden
-            jobs.instance_variable_set :@max,     max
-            jobs.instance_variable_set :@filter,  filter
+            jobs.instance_variable_set :@token,    gapi_list.next_page_token
+            jobs.instance_variable_set :@etag,     gapi_list.etag
+            jobs.instance_variable_set :@service,  service
+            jobs.instance_variable_set :@options,  options
             jobs
           end
 

data/lib/google/cloud/bigquery/model.rb

@@ -366,6 +366,82 @@ module Google
           patch_gapi! labels: new_labels
         end
 
+        ##
+        # The {EncryptionConfiguration} object that represents the custom
+        # encryption method used to protect this model. If not set,
+        # {Dataset#default_encryption} is used.
+        #
+        # Present only if this model is using custom encryption.
+        #
+        # @see https://cloud.google.com/bigquery/docs/customer-managed-encryption
+        #   Protecting Data with Cloud KMS Keys
+        #
+        # @return [EncryptionConfiguration, nil] The encryption configuration.
+        #
+        #   @!group Attributes
+        #
+        # @example
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #   dataset = bigquery.dataset "my_dataset"
+        #   model = dataset.model "my_model"
+        #
+        #   encrypt_config = model.encryption
+        #
+        # @!group Attributes
+        #
+        def encryption
+          return nil if reference?
+          return nil if @gapi_json[:encryptionConfiguration].nil?
+          # We have to create a gapic object from the hash because that is what
+          # EncryptionConfiguration is expecing.
+          json_cmek = @gapi_json[:encryptionConfiguration].to_json
+          gapi_cmek = \
+            Google::Apis::BigqueryV2::EncryptionConfiguration.from_json(
+              json_cmek
+            )
+          EncryptionConfiguration.from_gapi(gapi_cmek).freeze
+        end
+
+        ##
+        # Set the {EncryptionConfiguration} object that represents the custom
+        # encryption method used to protect this model. If not set,
+        # {Dataset#default_encryption} is used.
+        #
+        # Present only if this model is using custom encryption.
+        #
+        # If the model is not a full resource representation (see
+        # {#resource_full?}), the full representation will be retrieved before
+        # the update to comply with ETag-based optimistic concurrency control.
+        #
+        # @see https://cloud.google.com/bigquery/docs/customer-managed-encryption
+        #   Protecting Data with Cloud KMS Keys
+        #
+        # @param [EncryptionConfiguration] value The new encryption config.
+        #
+        # @example
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #   dataset = bigquery.dataset "my_dataset"
+        #   model = dataset.model "my_model"
+        #
+        #   key_name = "projects/a/locations/b/keyRings/c/cryptoKeys/d"
+        #   encrypt_config = bigquery.encryption kms_key: key_name
+        #
+        #   model.encryption = encrypt_config
+        #
+        # @!group Attributes
+        #
+        def encryption= value
+          ensure_full_data!
+          # We have to create a hash from the gapic object's JSON because that
+          # is what Model is expecing.
+          json_cmek = JSON.parse value.to_gapi.to_json, symbolize_names: true
+          patch_gapi! encryptionConfiguration: json_cmek
+        end
+
         ##
         # The input feature columns that were used to train this model.
         #

data/lib/google/cloud/bigquery/project.rb

@@ -1024,11 +1024,17 @@ module Google
         # Retrieves the list of jobs belonging to the project.
         #
         # @param [Boolean] all Whether to display jobs owned by all users in the
-        #   project. The default is `false`.
+        #   project. The default is `false`. Optional.
         # @param [String] token A previously-returned page token representing
-        #   part of the larger set of results to view.
-        # @param [Integer] max Maximum number of jobs to return.
-        # @param [String] filter A filter for job state.
+        #   part of the larger set of results to view. Optional.
+        # @param [Integer] max Maximum number of jobs to return. Optional.
+        # @param [String] filter A filter for job state. Optional.
+        # @param [Time] min_created_at Min value for {Job#created_at}. When
+        #   provided, only jobs created after or at this time are returned.
+        #   Optional.
+        # @param [Time] max_created_at Max value for {Job#created_at}. When
+        #   provided, only jobs created before or at this time are returned.
+        #   Optional.
         #
         #   Acceptable values are:
         #
@@ -1059,6 +1065,20 @@ module Google
         #     # process job
         #   end
         #
+        # @example Retrieve only jobs created within provided times:
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #
+        #   two_days_ago = Time.now - 60*60*24*2
+        #   three_days_ago = Time.now - 60*60*24*3
+        #
+        #   jobs = bigquery.jobs min_created_at: three_days_ago,
+        #                        max_created_at: two_days_ago
+        #   jobs.each do |job|
+        #     # process job
+        #   end
+        #
         # @example Retrieve all jobs: (See {Job::List#all})
         #   require "google/cloud/bigquery"
         #
@@ -1069,11 +1089,15 @@ module Google
         #     # process job
         #   end
         #
-        def jobs all: nil, token: nil, max: nil, filter: nil
+        def jobs all: nil, token: nil, max: nil, filter: nil,
+                 min_created_at: nil, max_created_at: nil
           ensure_service!
-          options = { all: all, token: token, max: max, filter: filter }
+          options = {
+            all: all, token: token, max: max, filter: filter,
+            min_created_at: min_created_at, max_created_at: max_created_at
+          }
           gapi = service.list_jobs options
-          Job::List.from_gapi gapi, service, all, max, filter
+          Job::List.from_gapi gapi, service, options
         end
 
         ##

data/lib/google/cloud/bigquery/service.rb

@@ -19,7 +19,7 @@ require "google/cloud/errors"
 require "google/apis/bigquery_v2"
 require "pathname"
 require "securerandom"
-require "mime/types"
+require "mini_mime"
 require "date"
 
 module Google
@@ -39,15 +39,17 @@ module Google
         attr_accessor :credentials
 
         # @private
-        attr_reader :retries, :timeout
+        attr_reader :retries, :timeout, :host
 
         ##
         # Creates a new Service instance.
-        def initialize project, credentials, retries: nil, timeout: nil
+        def initialize project, credentials,
+                       retries: nil, timeout: nil, host: nil
           @project = project
           @credentials = credentials
           @retries = retries
           @timeout = timeout
+          @host = host
         end
 
         def service
@@ -65,6 +67,7 @@ module Google
             service.request_options.header["x-goog-api-client"] = \
               "gl-ruby/#{RUBY_VERSION} gccl/#{Google::Cloud::Bigquery::VERSION}"
             service.authorization = @credentials.client
+            service.root_url = host if host
             service
           end
         end
@@ -297,11 +300,15 @@ module Google
         # been granted the READER job role.
         def list_jobs options = {}
           # The list operation is considered idempotent
+          min_creation_time = Convert.time_to_millis options[:min_created_at]
+          max_creation_time = Convert.time_to_millis options[:max_created_at]
           execute backoff: true do
             service.list_jobs \
               @project, all_users: options[:all], max_results: options[:max],
                         page_token: options[:token], projection: "full",
-                        state_filter: options[:filter]
+                        state_filter: options[:filter],
+                        min_creation_time: min_creation_time,
+                        max_creation_time: max_creation_time
           end
         end
 
@@ -476,9 +483,9 @@ module Google
         end
 
         def mime_type_for file
-          mime_type = MIME::Types.of(Pathname(file).to_path).first.to_s
-          return nil if mime_type.empty?
-          mime_type
+          mime_type = MiniMime.lookup_by_filename Pathname(file).to_path
+          return nil if mime_type.nil?
+          mime_type.content_type
         rescue StandardError
           nil
         end

data/lib/google/cloud/bigquery/table.rb

@@ -325,6 +325,52 @@ module Google
           patch_gapi! :time_partitioning
         end
 
+        ###
+        # Whether queries over this table require a partition filter that can be
+        # used for partition elimination to be specified. See [Partitioned
+        # Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
+        #
+        # @return [Boolean, nil] `true` when a partition filter will be
+        #   required, `false` otherwise, or `nil` if the object is a reference
+        #   (see {#reference?}).
+        #
+        # @!group Attributes
+        #
+        def require_partition_filter
+          return nil if reference?
+          ensure_full_data!
+          @gapi.require_partition_filter
+        end
+
+        ##
+        # Sets whether queries over this table require a partition filter. See
+        # [Partitioned
+        # Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
+        #
+        # If the table is not a full resource representation (see
+        # {#resource_full?}), the full representation will be retrieved before
+        # the update to comply with ETag-based optimistic concurrency control.
+        #
+        # @param [Boolean] new_require Whether queries over this table require a
+        #   partition filter.
+        #
+        # @example
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #   dataset = bigquery.dataset "my_dataset"
+        #   table = dataset.create_table "my_table" do |table|
+        #     table.require_partition_filter = true
+        #   end
+        #
+        # @!group Attributes
+        #
+        def require_partition_filter= new_require
+          reload! unless resource_full?
+          @gapi.require_partition_filter = new_require
+          patch_gapi! :require_partition_filter
+        end
+
         ###
         # Checks if the table is clustered.
         #
@@ -829,8 +875,8 @@ module Google
 
         ##
         # The {EncryptionConfiguration} object that represents the custom
-        # encryption method used to protect the table. If not set, default
-        # encryption is used.
+        # encryption method used to protect the table. If not set,
+        # {Dataset#default_encryption} is used.
         #
         # Present only if the table is using custom encryption.
         #
@@ -851,8 +897,8 @@ module Google
 
         ##
         # Set the {EncryptionConfiguration} object that represents the custom
-        # encryption method used to protect the table. If not set, default
-        # encryption is used.
+        # encryption method used to protect the table. If not set,
+        # {Dataset#default_encryption} is used.
         #
         # Present only if the table is using custom encryption.
         #
@@ -860,7 +906,6 @@ module Google
         # {#resource_full?}), the full representation will be retrieved before
         # the update to comply with ETag-based optimistic concurrency control.
         #
-        #
         # @see https://cloud.google.com/bigquery/docs/customer-managed-encryption
         #   Protecting Data with Cloud KMS Keys
         #
@@ -1926,9 +1971,18 @@ module Google
         # need to complete a load operation before the data can appear in query
         # results.
         #
+        # Because BigQuery's streaming API is designed for high insertion rates,
+        # modifications to the underlying table metadata are eventually
+        # consistent when interacting with the streaming system. In most cases
+        # metadata changes are propagated within minutes, but during this period
+        # API responses may reflect the inconsistent state of the table.
+        #
         # @see https://cloud.google.com/bigquery/streaming-data-into-bigquery
         #   Streaming Data Into BigQuery
         #
+        # @see https://cloud.google.com/bigquery/troubleshooting-errors#metadata-errors-for-streaming-inserts
+        #   BigQuery Troubleshooting: Metadata errors for streaming inserts
+        #
         # @param [Hash, Array<Hash>] rows A hash object or array of hash objects
         #   containing the data. Required.
         # @param [Array<String>] insert_ids A unique ID for each row. BigQuery

data/lib/google/cloud/bigquery/table/async_inserter.rb

@@ -86,7 +86,8 @@ module Google
 
             @batch = nil
 
-            @thread_pool = Concurrent::FixedThreadPool.new @threads
+            @thread_pool = Concurrent::ThreadPoolExecutor.new \
+              max_threads: @threads
 
             @cond = new_cond
 
@@ -99,6 +100,19 @@ module Google
           # collected in batches and inserted together.
           # See {Google::Cloud::Bigquery::Table#insert_async}.
           #
+          # Because BigQuery's streaming API is designed for high insertion
+          # rates, modifications to the underlying table metadata are eventually
+          # consistent when interacting with the streaming system. In most cases
+          # metadata changes are propagated within minutes, but during this
+          # period API responses may reflect the inconsistent state of the
+          # table.
+          #
+          # @see https://cloud.google.com/bigquery/streaming-data-into-bigquery
+          #   Streaming Data Into BigQuery
+          #
+          # @see https://cloud.google.com/bigquery/troubleshooting-errors#metadata-errors-for-streaming-inserts
+          #   BigQuery Troubleshooting: Metadata errors for streaming inserts
+          #
           # @param [Hash, Array<Hash>] rows A hash object or array of hash
           #   objects containing the data.
           # @param [Array<String>] insert_ids A unique ID for each row. BigQuery

data/lib/google/cloud/bigquery/version.rb

@@ -16,7 +16,7 @@
 module Google
   module Cloud
     module Bigquery
-      VERSION = "1.12.0".freeze
+      VERSION = "1.16.0".freeze
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: google-cloud-bigquery
 version: !ruby/object:Gem::Version
-  version: 1.12.0
+  version: 1.16.0
 platform: ruby
 authors:
 - Mike Moore
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-10 00:00:00.000000000 Z
+date: 2019-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-cloud-core
@@ -31,14 +31,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.23'
+        version: '0.31'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.23'
+        version: '0.31'
 - !ruby/object:Gem::Dependency
   name: googleauth
   requirement: !ruby/object:Gem::Requirement
@@ -74,19 +74,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: mime-types
+  name: mini_mime
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -293,7 +293,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.4
 signing_key: 
 specification_version: 4
 summary: API Client library for Google BigQuery