google-cloud-bigquery 1.21.2 → 1.26.0

data/lib/google/cloud/bigquery/load_job.rb

@@ -37,8 +37,8 @@ module Google
       #   bigquery = Google::Cloud::Bigquery.new
       #   dataset = bigquery.dataset "my_dataset"
       #
-      #   gs_url = "gs://my-bucket/file-name.csv"
-      #   load_job = dataset.load_job "my_new_table", gs_url do |schema|
+      #   gcs_uri = "gs://my-bucket/file-name.csv"
+      #   load_job = dataset.load_job "my_new_table", gcs_uri do |schema|
       #     schema.string "first_name", mode: :required
       #     schema.record "cities_lived", mode: :repeated do |nested_schema|
       #       nested_schema.string "place", mode: :required
@@ -112,8 +112,7 @@ module Google
         #   `false` otherwise.
         #
         def iso8859_1?
-          val = @gapi.configuration.load.encoding
-          val == "ISO-8859-1"
+          @gapi.configuration.load.encoding == "ISO-8859-1"
         end
 
         ##
@@ -195,8 +194,7 @@ module Google
         #   `NEWLINE_DELIMITED_JSON`, `false` otherwise.
         #
         def json?
-          val = @gapi.configuration.load.source_format
-          val == "NEWLINE_DELIMITED_JSON"
+          @gapi.configuration.load.source_format == "NEWLINE_DELIMITED_JSON"
         end
 
         ##
@@ -218,8 +216,27 @@ module Google
         #   `false` otherwise.
         #
         def backup?
-          val = @gapi.configuration.load.source_format
-          val == "DATASTORE_BACKUP"
+          @gapi.configuration.load.source_format == "DATASTORE_BACKUP"
+        end
+
+        ##
+        # Checks if the source format is ORC.
+        #
+        # @return [Boolean] `true` when the source format is `ORC`,
+        #   `false` otherwise.
+        #
+        def orc?
+          @gapi.configuration.load.source_format == "ORC"
+        end
+
+        ##
+        # Checks if the source format is Parquet.
+        #
+        # @return [Boolean] `true` when the source format is `PARQUET`,
+        #   `false` otherwise.
+        #
+        def parquet?
+          @gapi.configuration.load.source_format == "PARQUET"
         end
 
         ##
@@ -347,6 +364,58 @@ module Google
           nil
         end
 
+        ###
+        # Checks if hive partitioning options are set.
+        #
+        # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
+        #
+        # @return [Boolean] `true` when hive partitioning options are set, or `false` otherwise.
+        #
+        # @!group Attributes
+        #
+        def hive_partitioning?
+          !@gapi.configuration.load.hive_partitioning_options.nil?
+        end
+
+        ###
+        # The mode of hive partitioning to use when reading data. The following modes are supported:
+        #
+        #   1. `AUTO`: automatically infer partition key name(s) and type(s).
+        #   2. `STRINGS`: automatically infer partition key name(s). All types are interpreted as strings.
+        #   3. `CUSTOM`: partition key schema is encoded in the source URI prefix.
+        #
+        # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
+        #
+        # @return [String, nil] The mode of hive partitioning, or `nil` if not set.
+        #
+        # @!group Attributes
+        #
+        def hive_partitioning_mode
+          @gapi.configuration.load.hive_partitioning_options.mode if hive_partitioning?
+        end
+
+        ###
+        # The common prefix for all source uris when hive partition detection is requested. The prefix must end
+        # immediately before the partition key encoding begins. For example, consider files following this data layout:
+        #
+        # ```
+        # gs://bucket/path_to_table/dt=2019-01-01/country=BR/id=7/file.avro
+        # gs://bucket/path_to_table/dt=2018-12-31/country=CA/id=3/file.avro
+        # ```
+        #
+        # When hive partitioning is requested with either `AUTO` or `STRINGS` mode, the common prefix can be either of
+        # `gs://bucket/path_to_table` or `gs://bucket/path_to_table/` (trailing slash does not matter).
+        #
+        # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
+        #
+        # @return [String, nil] The common prefix for all source uris, or `nil` if not set.
+        #
+        # @!group Attributes
+        #
+        def hive_partitioning_source_uri_prefix
+          @gapi.configuration.load.hive_partitioning_options.source_uri_prefix if hive_partitioning?
+        end
+
         ###
         # Checks if the destination table will be range partitioned. See [Creating and using integer range partitioned
         # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
@@ -428,8 +497,9 @@ module Google
         # The period for which the destination table will be time partitioned, if
         # any. See [Partitioned Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
         #
-        # @return [String, nil] The time partition type. Currently the only supported
-        #   value is "DAY", or `nil` if not present.
+        # @return [String, nil] The time partition type. The supported types are `DAY`,
+        #   `HOUR`, `MONTH`, and `YEAR`, which will generate one partition per day,
+        #   hour, month, and year, respectively; or `nil` if not present.
         #
         # @!group Attributes
         #
@@ -1303,12 +1373,21 @@ module Google
           # Sets the labels to use for the load job.
           #
           # @param [Hash] val A hash of user-provided labels associated with
-          #   the job. You can use these to organize and group your jobs. Label
-          #   keys and values can be no longer than 63 characters, can only
-          #   contain lowercase letters, numeric characters, underscores and
-          #   dashes. International characters are allowed. Label values are
-          #   optional. Label keys must start with a letter and each label in
-          #   the list must have a different key.
+          #   the job. You can use these to organize and group your jobs.
+          #
+          #   The labels applied to a resource must meet the following requirements:
+          #
+          #   * Each resource can have multiple labels, up to a maximum of 64.
+          #   * Each label must be a key-value pair.
+          #   * Keys have a minimum length of 1 character and a maximum length of
+          #     63 characters, and cannot be empty. Values can be empty, and have
+          #     a maximum length of 63 characters.
+          #   * Keys and values can contain only lowercase letters, numeric characters,
+          #     underscores, and dashes. All characters must use UTF-8 encoding, and
+          #     international characters are allowed.
+          #   * The key portion of a label must be unique. However, you can use the
+          #     same key with multiple resources.
+          #   * Keys must start with a lowercase letter or international character.
           #
           # @!group Attributes
           #
@@ -1316,6 +1395,89 @@ module Google
             @gapi.configuration.update! labels: val
           end
 
+          ##
+          # Sets the mode of hive partitioning to use when reading data. The following modes are supported:
+          #
+          #   1. `auto`: automatically infer partition key name(s) and type(s).
+          #   2. `strings`: automatically infer partition key name(s). All types are interpreted as strings.
+          #   3. `custom`: partition key schema is encoded in the source URI prefix.
+          #
+          # Not all storage formats support hive partitioning. Requesting hive partitioning on an unsupported format
+          # will lead to an error. Currently supported types include: `avro`, `csv`, `json`, `orc` and `parquet`.
+          #
+          # See {#format=} and {#hive_partitioning_source_uri_prefix=}.
+          #
+          # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
+          #
+          # @param [String, Symbol] mode The mode of hive partitioning to use when reading data.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
+          #     job.format = :parquet
+          #     job.hive_partitioning_mode = :auto
+          #     job.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #
+          #   load_job.wait_until_done!
+          #   load_job.done? #=> true
+          #
+          # @!group Attributes
+          #
+          def hive_partitioning_mode= mode
+            @gapi.configuration.load.hive_partitioning_options ||= Google::Apis::BigqueryV2::HivePartitioningOptions.new
+            @gapi.configuration.load.hive_partitioning_options.mode = mode.to_s.upcase
+          end
+
+          ##
+          # Sets the common prefix for all source uris when hive partition detection is requested. The prefix must end
+          # immediately before the partition key encoding begins. For example, consider files following this data
+          # layout:
+          #
+          # ```
+          # gs://bucket/path_to_table/dt=2019-01-01/country=BR/id=7/file.avro
+          # gs://bucket/path_to_table/dt=2018-12-31/country=CA/id=3/file.avro
+          # ```
+          #
+          # When hive partitioning is requested with either `AUTO` or `STRINGS` mode, the common prefix can be either of
+          # `gs://bucket/path_to_table` or `gs://bucket/path_to_table/` (trailing slash does not matter).
+          #
+          # See {#hive_partitioning_mode=}.
+          #
+          # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
+          #
+          # @param [String] source_uri_prefix The common prefix for all source uris.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
+          #     job.format = :parquet
+          #     job.hive_partitioning_mode = :auto
+          #     job.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #
+          #   load_job.wait_until_done!
+          #   load_job.done? #=> true
+          #
+          # @!group Attributes
+          #
+          def hive_partitioning_source_uri_prefix= source_uri_prefix
+            @gapi.configuration.load.hive_partitioning_options ||= Google::Apis::BigqueryV2::HivePartitioningOptions.new
+            @gapi.configuration.load.hive_partitioning_options.source_uri_prefix = source_uri_prefix
+          end
+
           ##
           # Sets the field on which to range partition the table. See [Creating and using integer range partitioned
           # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
@@ -1335,8 +1497,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.schema do |schema|
           #       schema.integer "my_table_id", mode: :required
           #       schema.string "my_table_data", mode: :required
@@ -1376,8 +1538,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.schema do |schema|
           #       schema.integer "my_table_id", mode: :required
           #       schema.string "my_table_data", mode: :required
@@ -1417,8 +1579,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.schema do |schema|
           #       schema.integer "my_table_id", mode: :required
           #       schema.string "my_table_data", mode: :required
@@ -1458,8 +1620,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.schema do |schema|
           #       schema.integer "my_table_id", mode: :required
           #       schema.string "my_table_data", mode: :required
@@ -1490,8 +1652,9 @@ module Google
           # BigQuery does not allow you to change partitioning on an existing
           # table.
           #
-          # @param [String] type The time partition type. Currently the only
-          #   supported value is "DAY".
+          # @param [String] type The time partition type. The supported types are `DAY`,
+          #   `HOUR`, `MONTH`, and `YEAR`, which will generate one partition per day,
+          #   hour, month, and year, respectively.
           #
           # @example
           #   require "google/cloud/bigquery"
@@ -1499,8 +1662,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.time_partitioning_type = "DAY"
           #   end
           #
@@ -1538,8 +1701,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.time_partitioning_type  = "DAY"
           #     job.time_partitioning_field = "dob"
           #     job.schema do |schema|
@@ -1574,8 +1737,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.time_partitioning_type = "DAY"
           #     job.time_partitioning_expiration = 86_400
           #   end
@@ -1634,8 +1797,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.time_partitioning_type  = "DAY"
           #     job.time_partitioning_field = "dob"
           #     job.schema do |schema|

data/lib/google/cloud/bigquery/model.rb

@@ -341,14 +341,19 @@ module Google
         # the update to comply with ETag-based optimistic concurrency control.
         #
         # @param [Hash<String, String>] new_labels A hash containing key/value
-        #   pairs.
-        #
-        #   * Label keys and values can be no longer than 63 characters.
-        #   * Label keys and values can contain only lowercase letters, numbers,
-        #     underscores, hyphens, and international characters.
-        #   * Label keys and values cannot exceed 128 bytes in size.
-        #   * Label keys must begin with a letter.
-        #   * Label keys must be unique within a model.
+        #   pairs. The labels applied to a resource must meet the following requirements:
+        #
+        #   * Each resource can have multiple labels, up to a maximum of 64.
+        #   * Each label must be a key-value pair.
+        #   * Keys have a minimum length of 1 character and a maximum length of
+        #     63 characters, and cannot be empty. Values can be empty, and have
+        #     a maximum length of 63 characters.
+        #   * Keys and values can contain only lowercase letters, numeric characters,
+        #     underscores, and dashes. All characters must use UTF-8 encoding, and
+        #     international characters are allowed.
+        #   * The key portion of a label must be unique. However, you can use the
+        #     same key with multiple resources.
+        #   * Keys must start with a lowercase letter or international character.
         #
         # @example
         #   require "google/cloud/bigquery"
@@ -482,6 +487,146 @@ module Google
           Array @gapi_json[:trainingRuns]
         end
 
+        ##
+        # Exports the model to Google Cloud Storage asynchronously, immediately
+        # returning an {ExtractJob} that can be used to track the progress of the
+        # export job. The caller may poll the service by repeatedly calling
+        # {Job#reload!} and {Job#done?} to detect when the job is done, or
+        # simply block until the job is done by calling #{Job#wait_until_done!}.
+        # See also {#extract}.
+        #
+        # The geographic location for the job ("US", "EU", etc.) can be set via
+        # {ExtractJob::Updater#location=} in a block passed to this method. If
+        # the model is a full resource representation (see {#resource_full?}),
+        # the location of the job will automatically be set to the location of
+        # the model.
+        #
+        # @see https://cloud.google.com/bigquery-ml/docs/exporting-models
+        #   Exporting models
+        #
+        # @param [String] extract_url The Google Storage URI to which BigQuery
+        #   should extract the model. This value should be end in an object name
+        #   prefix, since multiple objects will be exported.
+        # @param [String] format The exported file format. The default value is
+        #   `ml_tf_saved_model`.
+        #
+        #   The following values are supported:
+        #
+        #   * `ml_tf_saved_model` - TensorFlow SavedModel
+        #   * `ml_xgboost_booster` - XGBoost Booster
+        # @param [String] job_id A user-defined ID for the extract job. The ID
+        #   must contain only letters (a-z, A-Z), numbers (0-9), underscores
+        #   (_), or dashes (-). The maximum length is 1,024 characters. If
+        #   `job_id` is provided, then `prefix` will not be used.
+        #
+        #   See [Generating a job
+        #   ID](https://cloud.google.com/bigquery/docs/managing-jobs#generate-jobid).
+        # @param [String] prefix A string, usually human-readable, that will be
+        #   prepended to a generated value to produce a unique job ID. For
+        #   example, the prefix `daily_import_job_` can be given to generate a
+        #   job ID such as `daily_import_job_12vEDtMQ0mbp1Mo5Z7mzAFQJZazh`. The
+        #   prefix must contain only letters (a-z, A-Z), numbers (0-9),
+        #   underscores (_), or dashes (-). The maximum length of the entire ID
+        #   is 1,024 characters. If `job_id` is provided, then `prefix` will not
+        #   be used.
+        # @param [Hash] labels A hash of user-provided labels associated with
+        #   the job. You can use these to organize and group your jobs.
+        #
+        #   The labels applied to a resource must meet the following requirements:
+        #
+        #   * Each resource can have multiple labels, up to a maximum of 64.
+        #   * Each label must be a key-value pair.
+        #   * Keys have a minimum length of 1 character and a maximum length of
+        #     63 characters, and cannot be empty. Values can be empty, and have
+        #     a maximum length of 63 characters.
+        #   * Keys and values can contain only lowercase letters, numeric characters,
+        #     underscores, and dashes. All characters must use UTF-8 encoding, and
+        #     international characters are allowed.
+        #   * The key portion of a label must be unique. However, you can use the
+        #     same key with multiple resources.
+        #   * Keys must start with a lowercase letter or international character.
+        #
+        # @yield [job] a job configuration object
+        # @yieldparam [Google::Cloud::Bigquery::ExtractJob::Updater] job a job
+        #   configuration object for setting additional options.
+        #
+        # @return [Google::Cloud::Bigquery::ExtractJob]
+        #
+        # @example
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #   dataset = bigquery.dataset "my_dataset"
+        #   model = dataset.model "my_model"
+        #
+        #   extract_job = model.extract_job "gs://my-bucket/#{model.model_id}"
+        #
+        #   extract_job.wait_until_done!
+        #   extract_job.done? #=> true
+        #
+        # @!group Data
+        #
+        def extract_job extract_url, format: nil, job_id: nil, prefix: nil, labels: nil
+          ensure_service!
+          options = { format: format, job_id: job_id, prefix: prefix, labels: labels }
+          updater = ExtractJob::Updater.from_options service, model_ref, extract_url, options
+          updater.location = location if location # may be model reference
+
+          yield updater if block_given?
+
+          job_gapi = updater.to_gapi
+          gapi = service.extract_table job_gapi
+          Job.from_gapi gapi, service
+        end
+
+        ##
+        # Exports the model to Google Cloud Storage using a synchronous method
+        # that blocks for a response. Timeouts and transient errors are generally
+        # handled as needed to complete the job. See also {#extract_job}.
+        #
+        # The geographic location for the job ("US", "EU", etc.) can be set via
+        # {ExtractJob::Updater#location=} in a block passed to this method. If
+        # the model is a full resource representation (see {#resource_full?}),
+        # the location of the job will automatically be set to the location of
+        # the model.
+        #
+        # @see https://cloud.google.com/bigquery-ml/docs/exporting-models
+        #   Exporting models
+        #
+        # @param [String] extract_url The Google Storage URI to which BigQuery
+        #   should extract the model. This value should be end in an object name
+        #   prefix, since multiple objects will be exported.
+        # @param [String] format The exported file format. The default value is
+        #   `ml_tf_saved_model`.
+        #
+        #   The following values are supported:
+        #
+        #   * `ml_tf_saved_model` - TensorFlow SavedModel
+        #   * `ml_xgboost_booster` - XGBoost Booster
+        # @yield [job] a job configuration object
+        # @yieldparam [Google::Cloud::Bigquery::ExtractJob::Updater] job a job
+        #   configuration object for setting additional options.
+        #
+        # @return [Boolean] Returns `true` if the extract operation succeeded.
+        #
+        # @example
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #   dataset = bigquery.dataset "my_dataset"
+        #   model = dataset.model "my_model"
+        #
+        #   model.extract "gs://my-bucket/#{model.model_id}"
+        #
+        # @!group Data
+        #
+        def extract extract_url, format: nil, &block
+          job = extract_job extract_url, format: format, &block
+          job.wait_until_done!
+          ensure_job_succeeded! job
+          true
+        end
+
         ##
         # Permanently deletes the model.
         #
@@ -734,6 +879,17 @@ module Google
         def ensure_full_data!
           reload! unless resource_full?
         end
+
+        def ensure_job_succeeded! job
+          return unless job.failed?
+          begin
+            # raise to activate ruby exception cause handling
+            raise job.gapi_error
+          rescue StandardError => e
+            # wrap Google::Apis::Error with Google::Cloud::Error
+            raise Google::Cloud::Error.from_error(e)
+          end
+        end
       end
     end
   end