google-cloud-bigquery 1.25.0 → 1.26.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ae4b6e4d7c37a945f027fe56425698b19b430f5e75451ce82f2156f2dae96719
-  data.tar.gz: d50284a47bf96d5221b574590a60b3af0f77b3a8943d201fd473d78f41a46670
+  metadata.gz: 8c2f8d8d7a564df2c916cced8adf4c0b621a1e11250a3a04decfce08995a4a24
+  data.tar.gz: 57bbda27d4c54e0522b564b8cdc62fb5e9dce8b26c91d3f0604c624ff113f057
 SHA512:
-  metadata.gz: 56c0b2d3214b385efb3866e502faa0b0c28c2fffc0c3d2ca3c5e1ad05f4d69e0638e68d59d8310c68d01c26c3db55b42225944379bfb8f6ad7a57eec558ced3a
-  data.tar.gz: 0562b98e65e880a40d634f541285241823354a966f4d05a559463a23eacca54addf8843f89b32731d313a56d91dfe2984f6098630c4126d7dbc97033d3dc6a81
+  metadata.gz: f445a86a5435cafc236d82faf91df46a06c6cee8612d8e6c4011450c93b73b61a883510909c6bb25e72516ae1a57c8a605736d82e0b63f0b10ad25cab90a1280
+  data.tar.gz: 43423f2bea5cea82c6c19bcf639bfb690f4718b0450140d299616ece59b147258ef877cafbc17b99c68274c310a0cd0123ae6b6db0b37bba2fb08be0395837f0

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Release History
 
+### 1.26.0 / 2021-01-13
+
+#### Features
+
+* Add support for Hive Partitioning
+  * Add hive partitioning options to External::DataSource
+  * Add hive partitioning options to LoadJob and LoadJob::Updater
+* Replace google-api-client with google-apis-bigquery_v2
+
 ### 1.25.0 / 2020-11-16
 
 #### Features

data/CONTRIBUTING.md

@@ -45,7 +45,7 @@ there is a small amount of setup:
 
    ```sh
    $ cd google-cloud-bigquery/
-   $ bundle exec rake bundleupdate
+   $ bundle install
    ```
 
 ## Console

data/lib/google/cloud/bigquery/external.rb

@@ -52,6 +52,24 @@ module Google
       #   # Retrieve the next page of results
       #   data = data.next if data.next?
       #
+      # @example Hive partitioning options:
+      #   require "google/cloud/bigquery"
+      #
+      #   bigquery = Google::Cloud::Bigquery.new
+      #
+      #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+      #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+      #   external_data = bigquery.external gcs_uri, format: :parquet do |ext|
+      #     ext.hive_partitioning_mode = :auto
+      #     ext.hive_partitioning_require_partition_filter = true
+      #     ext.hive_partitioning_source_uri_prefix = source_uri_prefix
+      #   end
+      #
+      #   external_data.hive_partitioning? #=> true
+      #   external_data.hive_partitioning_mode #=> "AUTO"
+      #   external_data.hive_partitioning_require_partition_filter? #=> true
+      #   external_data.hive_partitioning_source_uri_prefix #=> source_uri_prefix
+      #
       module External
         ##
         # @private New External from URLs and format
@@ -79,7 +97,8 @@ module Google
         # @private Determine source_format from inputs
         def self.source_format_for urls, format
           val = {
-            "csv" => "CSV",          "avro" => "AVRO",
+            "csv"                    => "CSV",
+            "avro"                   => "AVRO",
             "json"                   => "NEWLINE_DELIMITED_JSON",
             "newline_delimited_json" => "NEWLINE_DELIMITED_JSON",
             "sheets"                 => "GOOGLE_SHEETS",
@@ -87,7 +106,9 @@ module Google
             "datastore"              => "DATASTORE_BACKUP",
             "backup"                 => "DATASTORE_BACKUP",
             "datastore_backup"       => "DATASTORE_BACKUP",
-            "bigtable"               => "BIGTABLE"
+            "bigtable"               => "BIGTABLE",
+            "orc"                    => "ORC",
+            "parquet"                => "PARQUET"
           }[format.to_s.downcase]
           return val unless val.nil?
           Array(urls).each do |url|
@@ -110,7 +131,7 @@ module Google
           when "GOOGLE_SHEETS"          then External::SheetsSource
           when "BIGTABLE"               then External::BigtableSource
           else
-            # AVRO and DATASTORE_BACKUP
+            # AVRO, DATASTORE_BACKUP, PARQUET
             External::DataSource
           end
         end
@@ -148,6 +169,24 @@ module Google
         #   # Retrieve the next page of results
         #   data = data.next if data.next?
         #
+        # @example Hive partitioning options:
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #
+        #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+        #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+        #   external_data = bigquery.external gcs_uri, format: :parquet do |ext|
+        #     ext.hive_partitioning_mode = :auto
+        #     ext.hive_partitioning_require_partition_filter = true
+        #     ext.hive_partitioning_source_uri_prefix = source_uri_prefix
+        #   end
+        #
+        #   external_data.hive_partitioning? #=> true
+        #   external_data.hive_partitioning_mode #=> "AUTO"
+        #   external_data.hive_partitioning_require_partition_filter? #=> true
+        #   external_data.hive_partitioning_source_uri_prefix #=> source_uri_prefix
+        #
         class DataSource
           ##
           # @private The Google API Client object.
@@ -302,6 +341,52 @@ module Google
             @gapi.source_format == "BIGTABLE"
           end
 
+          ##
+          # Whether the data format is "ORC".
+          #
+          # @return [Boolean]
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   external_data = bigquery.external gcs_uri, format: :orc do |ext|
+          #     ext.hive_partitioning_mode = :auto
+          #     ext.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #   external_data.format #=> "ORC"
+          #   external_data.orc? #=> true
+          #
+          def orc?
+            @gapi.source_format == "ORC"
+          end
+
+          ##
+          # Whether the data format is "PARQUET".
+          #
+          # @return [Boolean]
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   external_data = bigquery.external gcs_uri, format: :parquet do |ext|
+          #     ext.hive_partitioning_mode = :auto
+          #     ext.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #   external_data.format #=> "PARQUET"
+          #   external_data.parquet? #=> true
+          #
+          def parquet?
+            @gapi.source_format == "PARQUET"
+          end
+
           ##
           # The fully-qualified URIs that point to your data in Google Cloud.
           # For Google Cloud Storage URIs: Each URI can contain one '*' wildcard
@@ -536,6 +621,246 @@ module Google
             @gapi.max_bad_records = new_max_bad_records
           end
 
+          ###
+          # Checks if hive partitioning options are set.
+          #
+          # Not all storage formats support hive partitioning. Requesting hive partitioning on an unsupported format
+          # will lead to an error. Currently supported types include: `avro`, `csv`, `json`, `orc` and `parquet`.
+          # If your data is stored in ORC or Parquet on Cloud Storage, see [Querying columnar formats on Cloud
+          # Storage](https://cloud.google.com/bigquery/pricing#columnar_formats_pricing).
+          #
+          # @return [Boolean] `true` when hive partitioning options are set, or `false` otherwise.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   external_data = bigquery.external gcs_uri, format: :parquet do |ext|
+          #     ext.hive_partitioning_mode = :auto
+          #     ext.hive_partitioning_require_partition_filter = true
+          #     ext.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #
+          #   external_data.hive_partitioning? #=> true
+          #   external_data.hive_partitioning_mode #=> "AUTO"
+          #   external_data.hive_partitioning_require_partition_filter? #=> true
+          #   external_data.hive_partitioning_source_uri_prefix #=> source_uri_prefix
+          #
+          def hive_partitioning?
+            !@gapi.hive_partitioning_options.nil?
+          end
+
+          ###
+          # The mode of hive partitioning to use when reading data. The following modes are supported:
+          #
+          #   1. `AUTO`: automatically infer partition key name(s) and type(s).
+          #   2. `STRINGS`: automatically infer partition key name(s). All types are interpreted as strings.
+          #   3. `CUSTOM`: partition key schema is encoded in the source URI prefix.
+          #
+          # @return [String, nil] The mode of hive partitioning, or `nil` if not set.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   external_data = bigquery.external gcs_uri, format: :parquet do |ext|
+          #     ext.hive_partitioning_mode = :auto
+          #     ext.hive_partitioning_require_partition_filter = true
+          #     ext.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #
+          #   external_data.hive_partitioning? #=> true
+          #   external_data.hive_partitioning_mode #=> "AUTO"
+          #   external_data.hive_partitioning_require_partition_filter? #=> true
+          #   external_data.hive_partitioning_source_uri_prefix #=> source_uri_prefix
+          #
+          def hive_partitioning_mode
+            @gapi.hive_partitioning_options.mode if hive_partitioning?
+          end
+
+          ##
+          # Sets the mode of hive partitioning to use when reading data. The following modes are supported:
+          #
+          #   1. `auto`: automatically infer partition key name(s) and type(s).
+          #   2. `strings`: automatically infer partition key name(s). All types are interpreted as strings.
+          #   3. `custom`: partition key schema is encoded in the source URI prefix.
+          #
+          # Not all storage formats support hive partitioning. Requesting hive partitioning on an unsupported format
+          # will lead to an error. Currently supported types include: `avro`, `csv`, `json`, `orc` and `parquet`.
+          # If your data is stored in ORC or Parquet on Cloud Storage, see [Querying columnar formats on Cloud
+          # Storage](https://cloud.google.com/bigquery/pricing#columnar_formats_pricing).
+          #
+          # See {#format}, {#hive_partitioning_require_partition_filter=} and {#hive_partitioning_source_uri_prefix=}.
+          #
+          # @param [String, Symbol] mode The mode of hive partitioning to use when reading data.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   external_data = bigquery.external gcs_uri, format: :parquet do |ext|
+          #     ext.hive_partitioning_mode = :auto
+          #     ext.hive_partitioning_require_partition_filter = true
+          #     ext.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #
+          #   external_data.hive_partitioning? #=> true
+          #   external_data.hive_partitioning_mode #=> "AUTO"
+          #   external_data.hive_partitioning_require_partition_filter? #=> true
+          #   external_data.hive_partitioning_source_uri_prefix #=> source_uri_prefix
+          #
+          def hive_partitioning_mode= mode
+            @gapi.hive_partitioning_options ||= Google::Apis::BigqueryV2::HivePartitioningOptions.new
+            @gapi.hive_partitioning_options.mode = mode.to_s.upcase
+          end
+
+          ###
+          # Whether queries over the table using this external data source require a partition filter that can be used
+          # for partition elimination to be specified. Note that this field should only be true when creating a
+          # permanent external table or querying a temporary external table.
+          #
+          # @return [Boolean] `true` when queries over this table require a partition filter, or `false` otherwise.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   external_data = bigquery.external gcs_uri, format: :parquet do |ext|
+          #     ext.hive_partitioning_mode = :auto
+          #     ext.hive_partitioning_require_partition_filter = true
+          #     ext.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #
+          #   external_data.hive_partitioning? #=> true
+          #   external_data.hive_partitioning_mode #=> "AUTO"
+          #   external_data.hive_partitioning_require_partition_filter? #=> true
+          #   external_data.hive_partitioning_source_uri_prefix #=> source_uri_prefix
+          #
+          def hive_partitioning_require_partition_filter?
+            return false unless hive_partitioning?
+            !@gapi.hive_partitioning_options.require_partition_filter.nil?
+          end
+
+          ##
+          # Sets whether queries over the table using this external data source require a partition filter
+          # that can be used for partition elimination to be specified.
+          #
+          # See {#format}, {#hive_partitioning_mode=} and {#hive_partitioning_source_uri_prefix=}.
+          #
+          # @param [Boolean] require_partition_filter `true` if a partition filter must be specified, `false` otherwise.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   external_data = bigquery.external gcs_uri, format: :parquet do |ext|
+          #     ext.hive_partitioning_mode = :auto
+          #     ext.hive_partitioning_require_partition_filter = true
+          #     ext.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #
+          #   external_data.hive_partitioning? #=> true
+          #   external_data.hive_partitioning_mode #=> "AUTO"
+          #   external_data.hive_partitioning_require_partition_filter? #=> true
+          #   external_data.hive_partitioning_source_uri_prefix #=> source_uri_prefix
+          #
+          def hive_partitioning_require_partition_filter= require_partition_filter
+            @gapi.hive_partitioning_options ||= Google::Apis::BigqueryV2::HivePartitioningOptions.new
+            @gapi.hive_partitioning_options.require_partition_filter = require_partition_filter
+          end
+
+          ###
+          # The common prefix for all source uris when hive partition detection is requested. The prefix must end
+          # immediately before the partition key encoding begins. For example, consider files following this data
+          # layout:
+          #
+          # ```
+          # gs://bucket/path_to_table/dt=2019-01-01/country=BR/id=7/file.avro
+          # gs://bucket/path_to_table/dt=2018-12-31/country=CA/id=3/file.avro
+          # ```
+          #
+          # When hive partitioning is requested with either `AUTO` or `STRINGS` mode, the common prefix can be either of
+          # `gs://bucket/path_to_table` or `gs://bucket/path_to_table/` (trailing slash does not matter).
+          #
+          # @return [String, nil] The common prefix for all source uris, or `nil` if not set.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   external_data = bigquery.external gcs_uri, format: :parquet do |ext|
+          #     ext.hive_partitioning_mode = :auto
+          #     ext.hive_partitioning_require_partition_filter = true
+          #     ext.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #
+          #   external_data.hive_partitioning? #=> true
+          #   external_data.hive_partitioning_mode #=> "AUTO"
+          #   external_data.hive_partitioning_require_partition_filter? #=> true
+          #   external_data.hive_partitioning_source_uri_prefix #=> source_uri_prefix
+          #
+          def hive_partitioning_source_uri_prefix
+            @gapi.hive_partitioning_options.source_uri_prefix if hive_partitioning?
+          end
+
+          ##
+          # Sets the common prefix for all source uris when hive partition detection is requested. The prefix must end
+          # immediately before the partition key encoding begins. For example, consider files following this data
+          # layout:
+          #
+          # ```
+          # gs://bucket/path_to_table/dt=2019-01-01/country=BR/id=7/file.avro
+          # gs://bucket/path_to_table/dt=2018-12-31/country=CA/id=3/file.avro
+          # ```
+          #
+          # When hive partitioning is requested with either `AUTO` or `STRINGS` mode, the common prefix can be either of
+          # `gs://bucket/path_to_table` or `gs://bucket/path_to_table/` (trailing slash does not matter).
+          #
+          # See {#format}, {#hive_partitioning_mode=} and {#hive_partitioning_require_partition_filter=}.
+          #
+          # @param [String] source_uri_prefix The common prefix for all source uris.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   external_data = bigquery.external gcs_uri, format: :parquet do |ext|
+          #     ext.hive_partitioning_mode = :auto
+          #     ext.hive_partitioning_require_partition_filter = true
+          #     ext.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #
+          #   external_data.hive_partitioning? #=> true
+          #   external_data.hive_partitioning_mode #=> "AUTO"
+          #   external_data.hive_partitioning_require_partition_filter? #=> true
+          #   external_data.hive_partitioning_source_uri_prefix #=> source_uri_prefix
+          #
+          def hive_partitioning_source_uri_prefix= source_uri_prefix
+            @gapi.hive_partitioning_options ||= Google::Apis::BigqueryV2::HivePartitioningOptions.new
+            @gapi.hive_partitioning_options.source_uri_prefix = source_uri_prefix
+          end
+
           ##
           # @private Google API Client object.
           def to_gapi

data/lib/google/cloud/bigquery/extract_job.rb

@@ -103,8 +103,7 @@ module Google
         #   table extraction.
         def compression?
           return false unless table?
-          val = @gapi.configuration.extract.compression
-          val == "GZIP"
+          @gapi.configuration.extract.compression == "GZIP"
         end
 
         ##
@@ -117,8 +116,7 @@ module Google
         #
         def json?
           return false unless table?
-          val = @gapi.configuration.extract.destination_format
-          val == "NEWLINE_DELIMITED_JSON"
+          @gapi.configuration.extract.destination_format == "NEWLINE_DELIMITED_JSON"
         end
 
         ##
@@ -146,8 +144,7 @@ module Google
         #
         def avro?
           return false unless table?
-          val = @gapi.configuration.extract.destination_format
-          val == "AVRO"
+          @gapi.configuration.extract.destination_format == "AVRO"
         end
 
         ##
@@ -173,8 +170,7 @@ module Google
         #
         def ml_xgboost_booster?
           return false unless model?
-          val = @gapi.configuration.extract.destination_format
-          val == "ML_XGBOOST_BOOSTER"
+          @gapi.configuration.extract.destination_format == "ML_XGBOOST_BOOSTER"
         end
 
         ##

data/lib/google/cloud/bigquery/load_job.rb

@@ -37,8 +37,8 @@ module Google
       #   bigquery = Google::Cloud::Bigquery.new
       #   dataset = bigquery.dataset "my_dataset"
       #
-      #   gs_url = "gs://my-bucket/file-name.csv"
-      #   load_job = dataset.load_job "my_new_table", gs_url do |schema|
+      #   gcs_uri = "gs://my-bucket/file-name.csv"
+      #   load_job = dataset.load_job "my_new_table", gcs_uri do |schema|
       #     schema.string "first_name", mode: :required
       #     schema.record "cities_lived", mode: :repeated do |nested_schema|
       #       nested_schema.string "place", mode: :required
@@ -112,8 +112,7 @@ module Google
         #   `false` otherwise.
         #
         def iso8859_1?
-          val = @gapi.configuration.load.encoding
-          val == "ISO-8859-1"
+          @gapi.configuration.load.encoding == "ISO-8859-1"
         end
 
         ##
@@ -195,8 +194,7 @@ module Google
         #   `NEWLINE_DELIMITED_JSON`, `false` otherwise.
         #
         def json?
-          val = @gapi.configuration.load.source_format
-          val == "NEWLINE_DELIMITED_JSON"
+          @gapi.configuration.load.source_format == "NEWLINE_DELIMITED_JSON"
         end
 
         ##
@@ -218,8 +216,27 @@ module Google
         #   `false` otherwise.
         #
         def backup?
-          val = @gapi.configuration.load.source_format
-          val == "DATASTORE_BACKUP"
+          @gapi.configuration.load.source_format == "DATASTORE_BACKUP"
+        end
+
+        ##
+        # Checks if the source format is ORC.
+        #
+        # @return [Boolean] `true` when the source format is `ORC`,
+        #   `false` otherwise.
+        #
+        def orc?
+          @gapi.configuration.load.source_format == "ORC"
+        end
+
+        ##
+        # Checks if the source format is Parquet.
+        #
+        # @return [Boolean] `true` when the source format is `PARQUET`,
+        #   `false` otherwise.
+        #
+        def parquet?
+          @gapi.configuration.load.source_format == "PARQUET"
         end
 
         ##
@@ -347,6 +364,58 @@ module Google
           nil
         end
 
+        ###
+        # Checks if hive partitioning options are set.
+        #
+        # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
+        #
+        # @return [Boolean] `true` when hive partitioning options are set, or `false` otherwise.
+        #
+        # @!group Attributes
+        #
+        def hive_partitioning?
+          !@gapi.configuration.load.hive_partitioning_options.nil?
+        end
+
+        ###
+        # The mode of hive partitioning to use when reading data. The following modes are supported:
+        #
+        #   1. `AUTO`: automatically infer partition key name(s) and type(s).
+        #   2. `STRINGS`: automatically infer partition key name(s). All types are interpreted as strings.
+        #   3. `CUSTOM`: partition key schema is encoded in the source URI prefix.
+        #
+        # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
+        #
+        # @return [String, nil] The mode of hive partitioning, or `nil` if not set.
+        #
+        # @!group Attributes
+        #
+        def hive_partitioning_mode
+          @gapi.configuration.load.hive_partitioning_options.mode if hive_partitioning?
+        end
+
+        ###
+        # The common prefix for all source uris when hive partition detection is requested. The prefix must end
+        # immediately before the partition key encoding begins. For example, consider files following this data layout:
+        #
+        # ```
+        # gs://bucket/path_to_table/dt=2019-01-01/country=BR/id=7/file.avro
+        # gs://bucket/path_to_table/dt=2018-12-31/country=CA/id=3/file.avro
+        # ```
+        #
+        # When hive partitioning is requested with either `AUTO` or `STRINGS` mode, the common prefix can be either of
+        # `gs://bucket/path_to_table` or `gs://bucket/path_to_table/` (trailing slash does not matter).
+        #
+        # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
+        #
+        # @return [String, nil] The common prefix for all source uris, or `nil` if not set.
+        #
+        # @!group Attributes
+        #
+        def hive_partitioning_source_uri_prefix
+          @gapi.configuration.load.hive_partitioning_options.source_uri_prefix if hive_partitioning?
+        end
+
         ###
         # Checks if the destination table will be range partitioned. See [Creating and using integer range partitioned
         # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
@@ -1326,6 +1395,89 @@ module Google
             @gapi.configuration.update! labels: val
           end
 
+          ##
+          # Sets the mode of hive partitioning to use when reading data. The following modes are supported:
+          #
+          #   1. `auto`: automatically infer partition key name(s) and type(s).
+          #   2. `strings`: automatically infer partition key name(s). All types are interpreted as strings.
+          #   3. `custom`: partition key schema is encoded in the source URI prefix.
+          #
+          # Not all storage formats support hive partitioning. Requesting hive partitioning on an unsupported format
+          # will lead to an error. Currently supported types include: `avro`, `csv`, `json`, `orc` and `parquet`.
+          #
+          # See {#format=} and {#hive_partitioning_source_uri_prefix=}.
+          #
+          # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
+          #
+          # @param [String, Symbol] mode The mode of hive partitioning to use when reading data.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
+          #     job.format = :parquet
+          #     job.hive_partitioning_mode = :auto
+          #     job.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #
+          #   load_job.wait_until_done!
+          #   load_job.done? #=> true
+          #
+          # @!group Attributes
+          #
+          def hive_partitioning_mode= mode
+            @gapi.configuration.load.hive_partitioning_options ||= Google::Apis::BigqueryV2::HivePartitioningOptions.new
+            @gapi.configuration.load.hive_partitioning_options.mode = mode.to_s.upcase
+          end
+
+          ##
+          # Sets the common prefix for all source uris when hive partition detection is requested. The prefix must end
+          # immediately before the partition key encoding begins. For example, consider files following this data
+          # layout:
+          #
+          # ```
+          # gs://bucket/path_to_table/dt=2019-01-01/country=BR/id=7/file.avro
+          # gs://bucket/path_to_table/dt=2018-12-31/country=CA/id=3/file.avro
+          # ```
+          #
+          # When hive partitioning is requested with either `AUTO` or `STRINGS` mode, the common prefix can be either of
+          # `gs://bucket/path_to_table` or `gs://bucket/path_to_table/` (trailing slash does not matter).
+          #
+          # See {#hive_partitioning_mode=}.
+          #
+          # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
+          #
+          # @param [String] source_uri_prefix The common prefix for all source uris.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
+          #     job.format = :parquet
+          #     job.hive_partitioning_mode = :auto
+          #     job.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #
+          #   load_job.wait_until_done!
+          #   load_job.done? #=> true
+          #
+          # @!group Attributes
+          #
+          def hive_partitioning_source_uri_prefix= source_uri_prefix
+            @gapi.configuration.load.hive_partitioning_options ||= Google::Apis::BigqueryV2::HivePartitioningOptions.new
+            @gapi.configuration.load.hive_partitioning_options.source_uri_prefix = source_uri_prefix
+          end
+
           ##
           # Sets the field on which to range partition the table. See [Creating and using integer range partitioned
           # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
@@ -1345,8 +1497,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.schema do |schema|
           #       schema.integer "my_table_id", mode: :required
           #       schema.string "my_table_data", mode: :required
@@ -1386,8 +1538,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.schema do |schema|
           #       schema.integer "my_table_id", mode: :required
           #       schema.string "my_table_data", mode: :required
@@ -1427,8 +1579,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.schema do |schema|
           #       schema.integer "my_table_id", mode: :required
           #       schema.string "my_table_data", mode: :required
@@ -1468,8 +1620,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.schema do |schema|
           #       schema.integer "my_table_id", mode: :required
           #       schema.string "my_table_data", mode: :required
@@ -1510,8 +1662,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.time_partitioning_type = "DAY"
           #   end
           #
@@ -1549,8 +1701,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.time_partitioning_type  = "DAY"
           #     job.time_partitioning_field = "dob"
           #     job.schema do |schema|
@@ -1585,8 +1737,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.time_partitioning_type = "DAY"
           #     job.time_partitioning_expiration = 86_400
           #   end
@@ -1645,8 +1797,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.time_partitioning_type  = "DAY"
           #     job.time_partitioning_field = "dob"
           #     job.schema do |schema|

data/lib/google/cloud/bigquery/query_job.rb

@@ -94,8 +94,7 @@ module Google
         #   otherwise.
         #
         def batch?
-          val = @gapi.configuration.query.priority
-          val == "BATCH"
+          @gapi.configuration.query.priority == "BATCH"
         end
 
         ##

data/lib/google/cloud/bigquery/version.rb

@@ -16,7 +16,7 @@
 module Google
   module Cloud
     module Bigquery
-      VERSION = "1.25.0".freeze
+      VERSION = "1.26.0".freeze
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: google-cloud-bigquery
 version: !ruby/object:Gem::Version
-  version: 1.25.0
+  version: 1.26.0
 platform: ruby
 authors:
 - Mike Moore
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-16 00:00:00.000000000 Z
+date: 2021-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -26,19 +26,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: google-api-client
+  name: google-apis-bigquery_v2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.47'
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.47'
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: google-cloud-core
   requirement: !ruby/object:Gem::Requirement