google-cloud-bigquery 1.21.1 → 1.27.0

data/lib/google/cloud/bigquery/external.rb

@@ -45,9 +45,30 @@ module Google
       #   data = bigquery.query "SELECT * FROM my_ext_table",
       #                         external: { my_ext_table: csv_table }
       #
+      #   # Iterate over the first page of results
       #   data.each do |row|
       #     puts row[:name]
       #   end
+      #   # Retrieve the next page of results
+      #   data = data.next if data.next?
+      #
+      # @example Hive partitioning options:
+      #   require "google/cloud/bigquery"
+      #
+      #   bigquery = Google::Cloud::Bigquery.new
+      #
+      #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+      #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+      #   external_data = bigquery.external gcs_uri, format: :parquet do |ext|
+      #     ext.hive_partitioning_mode = :auto
+      #     ext.hive_partitioning_require_partition_filter = true
+      #     ext.hive_partitioning_source_uri_prefix = source_uri_prefix
+      #   end
+      #
+      #   external_data.hive_partitioning? #=> true
+      #   external_data.hive_partitioning_mode #=> "AUTO"
+      #   external_data.hive_partitioning_require_partition_filter? #=> true
+      #   external_data.hive_partitioning_source_uri_prefix #=> source_uri_prefix
       #
       module External
         ##
@@ -76,7 +97,8 @@ module Google
         # @private Determine source_format from inputs
         def self.source_format_for urls, format
           val = {
-            "csv" => "CSV",          "avro" => "AVRO",
+            "csv"                    => "CSV",
+            "avro"                   => "AVRO",
             "json"                   => "NEWLINE_DELIMITED_JSON",
             "newline_delimited_json" => "NEWLINE_DELIMITED_JSON",
             "sheets"                 => "GOOGLE_SHEETS",
@@ -84,7 +106,9 @@ module Google
             "datastore"              => "DATASTORE_BACKUP",
             "backup"                 => "DATASTORE_BACKUP",
             "datastore_backup"       => "DATASTORE_BACKUP",
-            "bigtable"               => "BIGTABLE"
+            "bigtable"               => "BIGTABLE",
+            "orc"                    => "ORC",
+            "parquet"                => "PARQUET"
           }[format.to_s.downcase]
           return val unless val.nil?
           Array(urls).each do |url|
@@ -107,7 +131,7 @@ module Google
           when "GOOGLE_SHEETS"          then External::SheetsSource
           when "BIGTABLE"               then External::BigtableSource
           else
-            # AVRO and DATASTORE_BACKUP
+            # AVRO, DATASTORE_BACKUP, PARQUET
             External::DataSource
           end
         end
@@ -138,9 +162,30 @@ module Google
         #   data = bigquery.query "SELECT * FROM my_ext_table",
         #                         external: { my_ext_table: avro_table }
         #
+        #   # Iterate over the first page of results
         #   data.each do |row|
         #     puts row[:name]
         #   end
+        #   # Retrieve the next page of results
+        #   data = data.next if data.next?
+        #
+        # @example Hive partitioning options:
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #
+        #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+        #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+        #   external_data = bigquery.external gcs_uri, format: :parquet do |ext|
+        #     ext.hive_partitioning_mode = :auto
+        #     ext.hive_partitioning_require_partition_filter = true
+        #     ext.hive_partitioning_source_uri_prefix = source_uri_prefix
+        #   end
+        #
+        #   external_data.hive_partitioning? #=> true
+        #   external_data.hive_partitioning_mode #=> "AUTO"
+        #   external_data.hive_partitioning_require_partition_filter? #=> true
+        #   external_data.hive_partitioning_source_uri_prefix #=> source_uri_prefix
         #
         class DataSource
           ##
@@ -296,6 +341,52 @@ module Google
             @gapi.source_format == "BIGTABLE"
           end
 
+          ##
+          # Whether the data format is "ORC".
+          #
+          # @return [Boolean]
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   external_data = bigquery.external gcs_uri, format: :orc do |ext|
+          #     ext.hive_partitioning_mode = :auto
+          #     ext.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #   external_data.format #=> "ORC"
+          #   external_data.orc? #=> true
+          #
+          def orc?
+            @gapi.source_format == "ORC"
+          end
+
+          ##
+          # Whether the data format is "PARQUET".
+          #
+          # @return [Boolean]
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   external_data = bigquery.external gcs_uri, format: :parquet do |ext|
+          #     ext.hive_partitioning_mode = :auto
+          #     ext.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #   external_data.format #=> "PARQUET"
+          #   external_data.parquet? #=> true
+          #
+          def parquet?
+            @gapi.source_format == "PARQUET"
+          end
+
           ##
           # The fully-qualified URIs that point to your data in Google Cloud.
           # For Google Cloud Storage URIs: Each URI can contain one '*' wildcard
@@ -530,6 +621,246 @@ module Google
             @gapi.max_bad_records = new_max_bad_records
           end
 
+          ###
+          # Checks if hive partitioning options are set.
+          #
+          # Not all storage formats support hive partitioning. Requesting hive partitioning on an unsupported format
+          # will lead to an error. Currently supported types include: `avro`, `csv`, `json`, `orc` and `parquet`.
+          # If your data is stored in ORC or Parquet on Cloud Storage, see [Querying columnar formats on Cloud
+          # Storage](https://cloud.google.com/bigquery/pricing#columnar_formats_pricing).
+          #
+          # @return [Boolean] `true` when hive partitioning options are set, or `false` otherwise.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   external_data = bigquery.external gcs_uri, format: :parquet do |ext|
+          #     ext.hive_partitioning_mode = :auto
+          #     ext.hive_partitioning_require_partition_filter = true
+          #     ext.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #
+          #   external_data.hive_partitioning? #=> true
+          #   external_data.hive_partitioning_mode #=> "AUTO"
+          #   external_data.hive_partitioning_require_partition_filter? #=> true
+          #   external_data.hive_partitioning_source_uri_prefix #=> source_uri_prefix
+          #
+          def hive_partitioning?
+            !@gapi.hive_partitioning_options.nil?
+          end
+
+          ###
+          # The mode of hive partitioning to use when reading data. The following modes are supported:
+          #
+          #   1. `AUTO`: automatically infer partition key name(s) and type(s).
+          #   2. `STRINGS`: automatically infer partition key name(s). All types are interpreted as strings.
+          #   3. `CUSTOM`: partition key schema is encoded in the source URI prefix.
+          #
+          # @return [String, nil] The mode of hive partitioning, or `nil` if not set.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   external_data = bigquery.external gcs_uri, format: :parquet do |ext|
+          #     ext.hive_partitioning_mode = :auto
+          #     ext.hive_partitioning_require_partition_filter = true
+          #     ext.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #
+          #   external_data.hive_partitioning? #=> true
+          #   external_data.hive_partitioning_mode #=> "AUTO"
+          #   external_data.hive_partitioning_require_partition_filter? #=> true
+          #   external_data.hive_partitioning_source_uri_prefix #=> source_uri_prefix
+          #
+          def hive_partitioning_mode
+            @gapi.hive_partitioning_options.mode if hive_partitioning?
+          end
+
+          ##
+          # Sets the mode of hive partitioning to use when reading data. The following modes are supported:
+          #
+          #   1. `auto`: automatically infer partition key name(s) and type(s).
+          #   2. `strings`: automatically infer partition key name(s). All types are interpreted as strings.
+          #   3. `custom`: partition key schema is encoded in the source URI prefix.
+          #
+          # Not all storage formats support hive partitioning. Requesting hive partitioning on an unsupported format
+          # will lead to an error. Currently supported types include: `avro`, `csv`, `json`, `orc` and `parquet`.
+          # If your data is stored in ORC or Parquet on Cloud Storage, see [Querying columnar formats on Cloud
+          # Storage](https://cloud.google.com/bigquery/pricing#columnar_formats_pricing).
+          #
+          # See {#format}, {#hive_partitioning_require_partition_filter=} and {#hive_partitioning_source_uri_prefix=}.
+          #
+          # @param [String, Symbol] mode The mode of hive partitioning to use when reading data.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   external_data = bigquery.external gcs_uri, format: :parquet do |ext|
+          #     ext.hive_partitioning_mode = :auto
+          #     ext.hive_partitioning_require_partition_filter = true
+          #     ext.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #
+          #   external_data.hive_partitioning? #=> true
+          #   external_data.hive_partitioning_mode #=> "AUTO"
+          #   external_data.hive_partitioning_require_partition_filter? #=> true
+          #   external_data.hive_partitioning_source_uri_prefix #=> source_uri_prefix
+          #
+          def hive_partitioning_mode= mode
+            @gapi.hive_partitioning_options ||= Google::Apis::BigqueryV2::HivePartitioningOptions.new
+            @gapi.hive_partitioning_options.mode = mode.to_s.upcase
+          end
+
+          ###
+          # Whether queries over the table using this external data source require a partition filter that can be used
+          # for partition elimination to be specified. Note that this field should only be true when creating a
+          # permanent external table or querying a temporary external table.
+          #
+          # @return [Boolean] `true` when queries over this table require a partition filter, or `false` otherwise.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   external_data = bigquery.external gcs_uri, format: :parquet do |ext|
+          #     ext.hive_partitioning_mode = :auto
+          #     ext.hive_partitioning_require_partition_filter = true
+          #     ext.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #
+          #   external_data.hive_partitioning? #=> true
+          #   external_data.hive_partitioning_mode #=> "AUTO"
+          #   external_data.hive_partitioning_require_partition_filter? #=> true
+          #   external_data.hive_partitioning_source_uri_prefix #=> source_uri_prefix
+          #
+          def hive_partitioning_require_partition_filter?
+            return false unless hive_partitioning?
+            !@gapi.hive_partitioning_options.require_partition_filter.nil?
+          end
+
+          ##
+          # Sets whether queries over the table using this external data source require a partition filter
+          # that can be used for partition elimination to be specified.
+          #
+          # See {#format}, {#hive_partitioning_mode=} and {#hive_partitioning_source_uri_prefix=}.
+          #
+          # @param [Boolean] require_partition_filter `true` if a partition filter must be specified, `false` otherwise.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   external_data = bigquery.external gcs_uri, format: :parquet do |ext|
+          #     ext.hive_partitioning_mode = :auto
+          #     ext.hive_partitioning_require_partition_filter = true
+          #     ext.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #
+          #   external_data.hive_partitioning? #=> true
+          #   external_data.hive_partitioning_mode #=> "AUTO"
+          #   external_data.hive_partitioning_require_partition_filter? #=> true
+          #   external_data.hive_partitioning_source_uri_prefix #=> source_uri_prefix
+          #
+          def hive_partitioning_require_partition_filter= require_partition_filter
+            @gapi.hive_partitioning_options ||= Google::Apis::BigqueryV2::HivePartitioningOptions.new
+            @gapi.hive_partitioning_options.require_partition_filter = require_partition_filter
+          end
+
+          ###
+          # The common prefix for all source uris when hive partition detection is requested. The prefix must end
+          # immediately before the partition key encoding begins. For example, consider files following this data
+          # layout:
+          #
+          # ```
+          # gs://bucket/path_to_table/dt=2019-01-01/country=BR/id=7/file.avro
+          # gs://bucket/path_to_table/dt=2018-12-31/country=CA/id=3/file.avro
+          # ```
+          #
+          # When hive partitioning is requested with either `AUTO` or `STRINGS` mode, the common prefix can be either of
+          # `gs://bucket/path_to_table` or `gs://bucket/path_to_table/` (trailing slash does not matter).
+          #
+          # @return [String, nil] The common prefix for all source uris, or `nil` if not set.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   external_data = bigquery.external gcs_uri, format: :parquet do |ext|
+          #     ext.hive_partitioning_mode = :auto
+          #     ext.hive_partitioning_require_partition_filter = true
+          #     ext.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #
+          #   external_data.hive_partitioning? #=> true
+          #   external_data.hive_partitioning_mode #=> "AUTO"
+          #   external_data.hive_partitioning_require_partition_filter? #=> true
+          #   external_data.hive_partitioning_source_uri_prefix #=> source_uri_prefix
+          #
+          def hive_partitioning_source_uri_prefix
+            @gapi.hive_partitioning_options.source_uri_prefix if hive_partitioning?
+          end
+
+          ##
+          # Sets the common prefix for all source uris when hive partition detection is requested. The prefix must end
+          # immediately before the partition key encoding begins. For example, consider files following this data
+          # layout:
+          #
+          # ```
+          # gs://bucket/path_to_table/dt=2019-01-01/country=BR/id=7/file.avro
+          # gs://bucket/path_to_table/dt=2018-12-31/country=CA/id=3/file.avro
+          # ```
+          #
+          # When hive partitioning is requested with either `AUTO` or `STRINGS` mode, the common prefix can be either of
+          # `gs://bucket/path_to_table` or `gs://bucket/path_to_table/` (trailing slash does not matter).
+          #
+          # See {#format}, {#hive_partitioning_mode=} and {#hive_partitioning_require_partition_filter=}.
+          #
+          # @param [String] source_uri_prefix The common prefix for all source uris.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   external_data = bigquery.external gcs_uri, format: :parquet do |ext|
+          #     ext.hive_partitioning_mode = :auto
+          #     ext.hive_partitioning_require_partition_filter = true
+          #     ext.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #
+          #   external_data.hive_partitioning? #=> true
+          #   external_data.hive_partitioning_mode #=> "AUTO"
+          #   external_data.hive_partitioning_require_partition_filter? #=> true
+          #   external_data.hive_partitioning_source_uri_prefix #=> source_uri_prefix
+          #
+          def hive_partitioning_source_uri_prefix= source_uri_prefix
+            @gapi.hive_partitioning_options ||= Google::Apis::BigqueryV2::HivePartitioningOptions.new
+            @gapi.hive_partitioning_options.source_uri_prefix = source_uri_prefix
+          end
+
           ##
           # @private Google API Client object.
           def to_gapi
@@ -575,9 +906,12 @@ module Google
         #   data = bigquery.query "SELECT * FROM my_ext_table",
         #                         external: { my_ext_table: csv_table }
         #
+        #   # Iterate over the first page of results
         #   data.each do |row|
         #     puts row[:name]
         #   end
+        #   # Retrieve the next page of results
+        #   data = data.next if data.next?
         #
         class CsvSource < External::DataSource
           ##
@@ -1037,9 +1371,12 @@ module Google
         #   data = bigquery.query "SELECT * FROM my_ext_table",
         #                         external: { my_ext_table: json_table }
         #
+        #   # Iterate over the first page of results
         #   data.each do |row|
         #     puts row[:name]
         #   end
+        #   # Retrieve the next page of results
+        #   data = data.next if data.next?
         #
         class JsonSource < External::DataSource
           ##
@@ -1173,9 +1510,12 @@ module Google
         #   data = bigquery.query "SELECT * FROM my_ext_table",
         #                         external: { my_ext_table: sheets_table }
         #
+        #   # Iterate over the first page of results
         #   data.each do |row|
         #     puts row[:name]
         #   end
+        #   # Retrieve the next page of results
+        #   data = data.next if data.next?
         #
         class SheetsSource < External::DataSource
           ##
@@ -1318,9 +1658,12 @@ module Google
         #   data = bigquery.query "SELECT * FROM my_ext_table",
         #                         external: { my_ext_table: bigtable_table }
         #
+        #   # Iterate over the first page of results
         #   data.each do |row|
         #     puts row[:name]
         #   end
+        #   # Retrieve the next page of results
+        #   data = data.next if data.next?
         #
         class BigtableSource < External::DataSource
           ##
@@ -1516,9 +1859,12 @@ module Google
           #   data = bigquery.query "SELECT * FROM my_ext_table",
           #                         external: { my_ext_table: bigtable_table }
           #
+          #   # Iterate over the first page of results
           #   data.each do |row|
           #     puts row[:name]
           #   end
+          #   # Retrieve the next page of results
+          #   data = data.next if data.next?
           #
           class ColumnFamily
             ##
@@ -2053,9 +2399,12 @@ module Google
           #   data = bigquery.query "SELECT * FROM my_ext_table",
           #                         external: { my_ext_table: bigtable_table }
           #
+          #   # Iterate over the first page of results
           #   data.each do |row|
           #     puts row[:name]
           #   end
+          #   # Retrieve the next page of results
+          #   data = data.next if data.next?
           #
           class Column
             ##