google-cloud-bigquery 1.19.0 → 1.22.0

data/lib/google/cloud/bigquery/load_job.rb

@@ -348,7 +348,71 @@ module Google
         end
 
         ###
-        # Checks if the destination table will be time-partitioned. See
+        # Checks if the destination table will be range partitioned. See [Creating and using integer range partitioned
+        # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
+        #
+        # @return [Boolean] `true` when the table is range partitioned, or `false` otherwise.
+        #
+        # @!group Attributes
+        #
+        def range_partitioning?
+          !@gapi.configuration.load.range_partitioning.nil?
+        end
+
+        ###
+        # The field on which the destination table will be range partitioned, if any. The field must be a
+        # top-level `NULLABLE/REQUIRED` field. The only supported type is `INTEGER/INT64`. See
+        # [Creating and using integer range partitioned
+        # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
+        #
+        # @return [String, nil] The partition field, if a field was configured, or `nil` if not range partitioned.
+        #
+        # @!group Attributes
+        #
+        def range_partitioning_field
+          @gapi.configuration.load.range_partitioning.field if range_partitioning?
+        end
+
+        ###
+        # The start of range partitioning, inclusive. See [Creating and using integer range partitioned
+        # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
+        #
+        # @return [Integer, nil] The start of range partitioning, inclusive, or `nil` if not range partitioned.
+        #
+        # @!group Attributes
+        #
+        def range_partitioning_start
+          @gapi.configuration.load.range_partitioning.range.start if range_partitioning?
+        end
+
+        ###
+        # The width of each interval. See [Creating and using integer range partitioned
+        # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
+        #
+        # @return [Integer, nil] The width of each interval, for data in range partitions, or `nil` if not range
+        #   partitioned.
+        #
+        # @!group Attributes
+        #
+        def range_partitioning_interval
+          return nil unless range_partitioning?
+          @gapi.configuration.load.range_partitioning.range.interval
+        end
+
+        ###
+        # The end of range partitioning, exclusive. See [Creating and using integer range partitioned
+        # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
+        #
+        # @return [Integer, nil] The end of range partitioning, exclusive, or `nil` if not range partitioned.
+        #
+        # @!group Attributes
+        #
+        def range_partitioning_end
+          @gapi.configuration.load.range_partitioning.range.end if range_partitioning?
+        end
+
+        ###
+        # Checks if the destination table will be time partitioned. See
         # [Partitioned Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
         #
         # @return [Boolean, nil] `true` when the table will be time-partitioned,
@@ -361,10 +425,10 @@ module Google
         end
 
         ###
-        # The period for which the destination table will be partitioned, if
+        # The period for which the destination table will be time partitioned, if
         # any. See [Partitioned Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
         #
-        # @return [String, nil] The partition type. Currently the only supported
+        # @return [String, nil] The time partition type. Currently the only supported
         #   value is "DAY", or `nil` if not present.
         #
         # @!group Attributes
@@ -374,13 +438,13 @@ module Google
         end
 
         ###
-        # The field on which the destination table will be partitioned, if any.
-        # If not set, the destination table will be partitioned by pseudo column
-        # `_PARTITIONTIME`; if set, the table will be partitioned by this field.
+        # The field on which the destination table will be time partitioned, if any.
+        # If not set, the destination table will be time partitioned by pseudo column
+        # `_PARTITIONTIME`; if set, the table will be time partitioned by this field.
         # See [Partitioned Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
         #
-        # @return [String, nil] The partition field, if a field was configured.
-        #   `nil` if not partitioned or not set (partitioned by pseudo column
+        # @return [String, nil] The time partition field, if a field was configured.
+        #   `nil` if not time partitioned or not set (partitioned by pseudo column
         #   '_PARTITIONTIME').
         #
         # @!group Attributes
@@ -390,12 +454,12 @@ module Google
         end
 
         ###
-        # The expiration for the destination table partitions, if any, in
+        # The expiration for the destination table time partitions, if any, in
         # seconds. See [Partitioned
         # Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
         #
         # @return [Integer, nil] The expiration time, in seconds, for data in
-        #   partitions, or `nil` if not present.
+        #   time partitions, or `nil` if not present.
         #
         # @!group Attributes
         #
@@ -408,11 +472,11 @@ module Google
 
         ###
         # If set to true, queries over the destination table will require a
-        # partition filter that can be used for partition elimination to be
+        # time partition filter that can be used for partition elimination to be
         # specified. See [Partitioned
         # Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
         #
-        # @return [Boolean] `true` when a partition filter will be required,
+        # @return [Boolean] `true` when a time partition filter will be required,
         #   or `false` otherwise.
         #
         # @!group Attributes
@@ -1239,12 +1303,21 @@ module Google
           # Sets the labels to use for the load job.
           #
           # @param [Hash] val A hash of user-provided labels associated with
-          #   the job. You can use these to organize and group your jobs. Label
-          #   keys and values can be no longer than 63 characters, can only
-          #   contain lowercase letters, numeric characters, underscores and
-          #   dashes. International characters are allowed. Label values are
-          #   optional. Label keys must start with a letter and each label in
-          #   the list must have a different key.
+          #   the job. You can use these to organize and group your jobs.
+          #
+          #   The labels applied to a resource must meet the following requirements:
+          #
+          #   * Each resource can have multiple labels, up to a maximum of 64.
+          #   * Each label must be a key-value pair.
+          #   * Keys have a minimum length of 1 character and a maximum length of
+          #     63 characters, and cannot be empty. Values can be empty, and have
+          #     a maximum length of 63 characters.
+          #   * Keys and values can contain only lowercase letters, numeric characters,
+          #     underscores, and dashes. All characters must use UTF-8 encoding, and
+          #     international characters are allowed.
+          #   * The key portion of a label must be unique. However, you can use the
+          #     same key with multiple resources.
+          #   * Keys must start with a lowercase letter or international character.
           #
           # @!group Attributes
           #
@@ -1253,14 +1326,180 @@ module Google
           end
 
           ##
-          # Sets the partitioning for the destination table. See [Partitioned
+          # Sets the field on which to range partition the table. See [Creating and using integer range partitioned
+          # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
+          #
+          # See {#range_partitioning_start=}, {#range_partitioning_interval=} and {#range_partitioning_end=}.
+          #
+          # You can only set range partitioning when creating a table. BigQuery does not allow you to change
+          # partitioning on an existing table.
+          #
+          # @param [String] field The range partition field. the destination table is partitioned by this
+          #   field. The field must be a top-level `NULLABLE/REQUIRED` field. The only supported
+          #   type is `INTEGER/INT64`.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #
+          #   gs_url = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #     job.schema do |schema|
+          #       schema.integer "my_table_id", mode: :required
+          #       schema.string "my_table_data", mode: :required
+          #     end
+          #     job.range_partitioning_field = "my_table_id"
+          #     job.range_partitioning_start = 0
+          #     job.range_partitioning_interval = 10
+          #     job.range_partitioning_end = 100
+          #   end
+          #
+          #   load_job.wait_until_done!
+          #   load_job.done? #=> true
+          #
+          # @!group Attributes
+          #
+          def range_partitioning_field= field
+            @gapi.configuration.load.range_partitioning ||= Google::Apis::BigqueryV2::RangePartitioning.new(
+              range: Google::Apis::BigqueryV2::RangePartitioning::Range.new
+            )
+            @gapi.configuration.load.range_partitioning.field = field
+          end
+
+          ##
+          # Sets the start of range partitioning, inclusive, for the destination table. See [Creating and using integer
+          # range partitioned tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
+          #
+          # You can only set range partitioning when creating a table. BigQuery does not allow you to change
+          # partitioning on an existing table.
+          #
+          # See {#range_partitioning_field=}, {#range_partitioning_interval=} and {#range_partitioning_end=}.
+          #
+          # @param [Integer] range_start The start of range partitioning, inclusive.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #
+          #   gs_url = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #     job.schema do |schema|
+          #       schema.integer "my_table_id", mode: :required
+          #       schema.string "my_table_data", mode: :required
+          #     end
+          #     job.range_partitioning_field = "my_table_id"
+          #     job.range_partitioning_start = 0
+          #     job.range_partitioning_interval = 10
+          #     job.range_partitioning_end = 100
+          #   end
+          #
+          #   load_job.wait_until_done!
+          #   load_job.done? #=> true
+          #
+          # @!group Attributes
+          #
+          def range_partitioning_start= range_start
+            @gapi.configuration.load.range_partitioning ||= Google::Apis::BigqueryV2::RangePartitioning.new(
+              range: Google::Apis::BigqueryV2::RangePartitioning::Range.new
+            )
+            @gapi.configuration.load.range_partitioning.range.start = range_start
+          end
+
+          ##
+          # Sets width of each interval for data in range partitions. See [Creating and using integer range partitioned
+          # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
+          #
+          # You can only set range partitioning when creating a table. BigQuery does not allow you to change
+          # partitioning on an existing table.
+          #
+          # See {#range_partitioning_field=}, {#range_partitioning_start=} and {#range_partitioning_end=}.
+          #
+          # @param [Integer] range_interval The width of each interval, for data in partitions.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #
+          #   gs_url = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #     job.schema do |schema|
+          #       schema.integer "my_table_id", mode: :required
+          #       schema.string "my_table_data", mode: :required
+          #     end
+          #     job.range_partitioning_field = "my_table_id"
+          #     job.range_partitioning_start = 0
+          #     job.range_partitioning_interval = 10
+          #     job.range_partitioning_end = 100
+          #   end
+          #
+          #   load_job.wait_until_done!
+          #   load_job.done? #=> true
+          #
+          # @!group Attributes
+          #
+          def range_partitioning_interval= range_interval
+            @gapi.configuration.load.range_partitioning ||= Google::Apis::BigqueryV2::RangePartitioning.new(
+              range: Google::Apis::BigqueryV2::RangePartitioning::Range.new
+            )
+            @gapi.configuration.load.range_partitioning.range.interval = range_interval
+          end
+
+          ##
+          # Sets the end of range partitioning, exclusive, for the destination table. See [Creating and using integer
+          # range partitioned tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
+          #
+          # You can only set range partitioning when creating a table. BigQuery does not allow you to change
+          # partitioning on an existing table.
+          #
+          # See {#range_partitioning_start=}, {#range_partitioning_interval=} and {#range_partitioning_field=}.
+          #
+          # @param [Integer] range_end The end of range partitioning, exclusive.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #
+          #   gs_url = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #     job.schema do |schema|
+          #       schema.integer "my_table_id", mode: :required
+          #       schema.string "my_table_data", mode: :required
+          #     end
+          #     job.range_partitioning_field = "my_table_id"
+          #     job.range_partitioning_start = 0
+          #     job.range_partitioning_interval = 10
+          #     job.range_partitioning_end = 100
+          #   end
+          #
+          #   load_job.wait_until_done!
+          #   load_job.done? #=> true
+          #
+          # @!group Attributes
+          #
+          def range_partitioning_end= range_end
+            @gapi.configuration.load.range_partitioning ||= Google::Apis::BigqueryV2::RangePartitioning.new(
+              range: Google::Apis::BigqueryV2::RangePartitioning::Range.new
+            )
+            @gapi.configuration.load.range_partitioning.range.end = range_end
+          end
+
+          ##
+          # Sets the time partitioning for the destination table. See [Partitioned
           # Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
           #
-          # You can only set the partitioning field while creating a table.
+          # You can only set the time partitioning field while creating a table.
           # BigQuery does not allow you to change partitioning on an existing
           # table.
           #
-          # @param [String] type The partition type. Currently the only
+          # @param [String] type The time partition type. Currently the only
           #   supported value is "DAY".
           #
           # @example
@@ -1285,20 +1524,20 @@ module Google
           end
 
           ##
-          # Sets the field on which to partition the destination table. If not
-          # set, the destination table is partitioned by pseudo column
-          # `_PARTITIONTIME`; if set, the table is partitioned by this field.
+          # Sets the field on which to time partition the destination table. If not
+          # set, the destination table is time partitioned by pseudo column
+          # `_PARTITIONTIME`; if set, the table is time partitioned by this field.
           # See [Partitioned
           # Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
           #
-          # The destination table must also be partitioned. See
+          # The destination table must also be time partitioned. See
           # {#time_partitioning_type=}.
           #
-          # You can only set the partitioning field while creating a table.
+          # You can only set the time partitioning field while creating a table.
           # BigQuery does not allow you to change partitioning on an existing
           # table.
           #
-          # @param [String] field The partition field. The field must be a
+          # @param [String] field The time partition field. The field must be a
           #   top-level TIMESTAMP or DATE field. Its mode must be NULLABLE or
           #   REQUIRED.
           #
@@ -1328,15 +1567,15 @@ module Google
           end
 
           ##
-          # Sets the partition expiration for the destination table. See
+          # Sets the time partition expiration for the destination table. See
           # [Partitioned
           # Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
           #
-          # The destination table must also be partitioned. See
+          # The destination table must also be time partitioned. See
           # {#time_partitioning_type=}.
           #
           # @param [Integer] expiration An expiration time, in seconds,
-          #   for data in partitions.
+          #   for data in time partitions.
           #
           # @example
           #   require "google/cloud/bigquery"
@@ -1362,12 +1601,12 @@ module Google
 
           ##
           # If set to true, queries over the destination table will require a
-          # partition filter that can be used for partition elimination to be
+          # time partition filter that can be used for time partition elimination to be
           # specified. See [Partitioned
           # Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
           #
           # @param [Boolean] val Indicates if queries over the destination table
-          #   will require a partition filter. The default value is `false`.
+          #   will require a time partition filter. The default value is `false`.
           #
           # @!group Attributes
           #

data/lib/google/cloud/bigquery/model.rb

@@ -341,14 +341,19 @@ module Google
         # the update to comply with ETag-based optimistic concurrency control.
         #
         # @param [Hash<String, String>] new_labels A hash containing key/value
-        #   pairs.
-        #
-        #   * Label keys and values can be no longer than 63 characters.
-        #   * Label keys and values can contain only lowercase letters, numbers,
-        #     underscores, hyphens, and international characters.
-        #   * Label keys and values cannot exceed 128 bytes in size.
-        #   * Label keys must begin with a letter.
-        #   * Label keys must be unique within a model.
+        #   pairs. The labels applied to a resource must meet the following requirements:
+        #
+        #   * Each resource can have multiple labels, up to a maximum of 64.
+        #   * Each label must be a key-value pair.
+        #   * Keys have a minimum length of 1 character and a maximum length of
+        #     63 characters, and cannot be empty. Values can be empty, and have
+        #     a maximum length of 63 characters.
+        #   * Keys and values can contain only lowercase letters, numeric characters,
+        #     underscores, and dashes. All characters must use UTF-8 encoding, and
+        #     international characters are allowed.
+        #   * The key portion of a label must be unique. However, you can use the
+        #     same key with multiple resources.
+        #   * Keys must start with a lowercase letter or international character.
         #
         # @example
         #   require "google/cloud/bigquery"
@@ -482,6 +487,146 @@ module Google
           Array @gapi_json[:trainingRuns]
         end
 
+        ##
+        # Exports the model to Google Cloud Storage asynchronously, immediately
+        # returning an {ExtractJob} that can be used to track the progress of the
+        # export job. The caller may poll the service by repeatedly calling
+        # {Job#reload!} and {Job#done?} to detect when the job is done, or
+        # simply block until the job is done by calling #{Job#wait_until_done!}.
+        # See also {#extract}.
+        #
+        # The geographic location for the job ("US", "EU", etc.) can be set via
+        # {ExtractJob::Updater#location=} in a block passed to this method. If
+        # the model is a full resource representation (see {#resource_full?}),
+        # the location of the job will automatically be set to the location of
+        # the model.
+        #
+        # @see https://cloud.google.com/bigquery-ml/docs/exporting-models
+        #   Exporting models
+        #
+        # @param [String] extract_url The Google Storage URI to which BigQuery
+        #   should extract the model. This value should be end in an object name
+        #   prefix, since multiple objects will be exported.
+        # @param [String] format The exported file format. The default value is
+        #   `ml_tf_saved_model`.
+        #
+        #   The following values are supported:
+        #
+        #   * `ml_tf_saved_model` - TensorFlow SavedModel
+        #   * `ml_xgboost_booster` - XGBoost Booster
+        # @param [String] job_id A user-defined ID for the extract job. The ID
+        #   must contain only letters (a-z, A-Z), numbers (0-9), underscores
+        #   (_), or dashes (-). The maximum length is 1,024 characters. If
+        #   `job_id` is provided, then `prefix` will not be used.
+        #
+        #   See [Generating a job
+        #   ID](https://cloud.google.com/bigquery/docs/managing-jobs#generate-jobid).
+        # @param [String] prefix A string, usually human-readable, that will be
+        #   prepended to a generated value to produce a unique job ID. For
+        #   example, the prefix `daily_import_job_` can be given to generate a
+        #   job ID such as `daily_import_job_12vEDtMQ0mbp1Mo5Z7mzAFQJZazh`. The
+        #   prefix must contain only letters (a-z, A-Z), numbers (0-9),
+        #   underscores (_), or dashes (-). The maximum length of the entire ID
+        #   is 1,024 characters. If `job_id` is provided, then `prefix` will not
+        #   be used.
+        # @param [Hash] labels A hash of user-provided labels associated with
+        #   the job. You can use these to organize and group your jobs.
+        #
+        #   The labels applied to a resource must meet the following requirements:
+        #
+        #   * Each resource can have multiple labels, up to a maximum of 64.
+        #   * Each label must be a key-value pair.
+        #   * Keys have a minimum length of 1 character and a maximum length of
+        #     63 characters, and cannot be empty. Values can be empty, and have
+        #     a maximum length of 63 characters.
+        #   * Keys and values can contain only lowercase letters, numeric characters,
+        #     underscores, and dashes. All characters must use UTF-8 encoding, and
+        #     international characters are allowed.
+        #   * The key portion of a label must be unique. However, you can use the
+        #     same key with multiple resources.
+        #   * Keys must start with a lowercase letter or international character.
+        #
+        # @yield [job] a job configuration object
+        # @yieldparam [Google::Cloud::Bigquery::ExtractJob::Updater] job a job
+        #   configuration object for setting additional options.
+        #
+        # @return [Google::Cloud::Bigquery::ExtractJob]
+        #
+        # @example
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #   dataset = bigquery.dataset "my_dataset"
+        #   model = dataset.model "my_model"
+        #
+        #   extract_job = model.extract_job "gs://my-bucket/#{model.model_id}"
+        #
+        #   extract_job.wait_until_done!
+        #   extract_job.done? #=> true
+        #
+        # @!group Data
+        #
+        def extract_job extract_url, format: nil, job_id: nil, prefix: nil, labels: nil
+          ensure_service!
+          options = { format: format, job_id: job_id, prefix: prefix, labels: labels }
+          updater = ExtractJob::Updater.from_options service, model_ref, extract_url, options
+          updater.location = location if location # may be model reference
+
+          yield updater if block_given?
+
+          job_gapi = updater.to_gapi
+          gapi = service.extract_table job_gapi
+          Job.from_gapi gapi, service
+        end
+
+        ##
+        # Exports the model to Google Cloud Storage using a synchronous method
+        # that blocks for a response. Timeouts and transient errors are generally
+        # handled as needed to complete the job. See also {#extract_job}.
+        #
+        # The geographic location for the job ("US", "EU", etc.) can be set via
+        # {ExtractJob::Updater#location=} in a block passed to this method. If
+        # the model is a full resource representation (see {#resource_full?}),
+        # the location of the job will automatically be set to the location of
+        # the model.
+        #
+        # @see https://cloud.google.com/bigquery-ml/docs/exporting-models
+        #   Exporting models
+        #
+        # @param [String] extract_url The Google Storage URI to which BigQuery
+        #   should extract the model. This value should be end in an object name
+        #   prefix, since multiple objects will be exported.
+        # @param [String] format The exported file format. The default value is
+        #   `ml_tf_saved_model`.
+        #
+        #   The following values are supported:
+        #
+        #   * `ml_tf_saved_model` - TensorFlow SavedModel
+        #   * `ml_xgboost_booster` - XGBoost Booster
+        # @yield [job] a job configuration object
+        # @yieldparam [Google::Cloud::Bigquery::ExtractJob::Updater] job a job
+        #   configuration object for setting additional options.
+        #
+        # @return [Boolean] Returns `true` if the extract operation succeeded.
+        #
+        # @example
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #   dataset = bigquery.dataset "my_dataset"
+        #   model = dataset.model "my_model"
+        #
+        #   model.extract "gs://my-bucket/#{model.model_id}"
+        #
+        # @!group Data
+        #
+        def extract extract_url, format: nil, &block
+          job = extract_job extract_url, format: format, &block
+          job.wait_until_done!
+          ensure_job_succeeded! job
+          true
+        end
+
         ##
         # Permanently deletes the model.
         #
@@ -734,6 +879,17 @@ module Google
         def ensure_full_data!
           reload! unless resource_full?
         end
+
+        def ensure_job_succeeded! job
+          return unless job.failed?
+          begin
+            # raise to activate ruby exception cause handling
+            raise job.gapi_error
+          rescue StandardError => e
+            # wrap Google::Apis::Error with Google::Cloud::Error
+            raise Google::Cloud::Error.from_error(e)
+          end
+        end
       end
     end
   end