google-cloud-bigquery 1.36.0 → 1.37.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ebd20bc9ec4776397f3530418bdfa18c09a9b154e10eaf3c411c6a3b1927065b
-  data.tar.gz: beab575327b893be124b4ff183f92c2c7ba1e4f4fb5c7986fe002d5cf1b873bb
+  metadata.gz: d23511050f9602220787610566c2f1f62ed37ad3c014e9708e16a05bf4a079a1
+  data.tar.gz: 5a61cffca84ffd8a1561f28d6d2d545d7451389f4a363279bea40f907c1ec568
 SHA512:
-  metadata.gz: b20b644003e2d74204d0639672c82138460c8403de3f0ebc6be9ff94ed7bef0c9ee1556c1fef8d17aa4c64260737d09b4cbd4ead5efdbca3facb2ec6c3fac872
-  data.tar.gz: ab3989b89ab63c5d54d176b35eac6509dd9cf3e2e88089489383ceebe96f08fb6d275d6c55605ecfcc1321b38dcee3c6deab99bc4d62c6d7c7869616b9510f39
+  metadata.gz: 4bbdca09750d11aabed55d8868b9b70501def5a8a12b35f4e0b92744c7d2a53e9ba503384771ccad85eb32ddc133b840d30a8434efc82260f4ff2fab2fbec6ac
+  data.tar.gz: 8990512c43041ad7b3b2e4d87c64d22b080a6a0bbc8d648fe09942504b2bc607a910809688b4f13410fb5c8fe2222f4b12ea8a613a9ac52864835319a1b56c66

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Release History
 
+### 1.37.0 / 2021-10-21
+
+#### Features
+
+* Add support for Avro options to external data sources
+  * Add External::AvroSource
+
 ### 1.36.0 / 2021-09-22
 
 #### Features

data/lib/google/cloud/bigquery/external/avro_source.rb

@@ -0,0 +1,107 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "google/apis/bigquery_v2"
+
+module Google
+  module Cloud
+    module Bigquery
+      module External
+        ##
+        # # AvroSource
+        #
+        # {External::AvroSource} is a subclass of {External::DataSource} and
+        # represents a Avro external data source that can be queried
+        # from directly, even though the data is not stored in BigQuery. Instead
+        # of loading or streaming the data, this object references the external
+        # data source.
+        #
+        # @example
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #
+        #   avro_url = "gs://bucket/path/to/*.avro"
+        #   avro_table = bigquery.external avro_url do |avro|
+        #     avro.use_avro_logical_types = 1
+        #   end
+        #
+        #   data = bigquery.query "SELECT * FROM my_ext_table",
+        #                         external: { my_ext_table: avro_table }
+        #
+        #   # Iterate over the first page of results
+        #   data.each do |row|
+        #     puts row[:name]
+        #   end
+        #   # Retrieve the next page of results
+        #   data = data.next if data.next?
+        #
+        class AvroSource < External::DataSource
+          ##
+          # @private Create an empty AvroSource object.
+          def initialize
+            super
+            @gapi.avro_options = Google::Apis::BigqueryV2::AvroOptions.new
+          end
+
+          ##
+          # Indicates whether to interpret logical types as the corresponding BigQuery data type (for example,
+          # `TIMESTAMP`), instead of using the raw type (for example, `INTEGER`).
+          #
+          # @return [Boolean]
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   avro_url = "gs://bucket/path/to/*.avro"
+          #   avro_table = bigquery.external avro_url do |avro|
+          #     avro.use_avro_logical_types = true
+          #   end
+          #
+          #   avro_table.use_avro_logical_types #=> true
+          #
+          def use_avro_logical_types
+            @gapi.avro_options.use_avro_logical_types
+          end
+
+          ##
+          # Sets whether to interpret logical types as the corresponding BigQuery data type (for example, `TIMESTAMP`),
+          # instead of using the raw type (for example, `INTEGER`).
+          #
+          # @param [Boolean] new_use_avro_logical_types The new `use_avro_logical_types` value.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   avro_url = "gs://bucket/path/to/*.avro"
+          #   avro_table = bigquery.external avro_url do |avro|
+          #     avro.use_avro_logical_types = true
+          #   end
+          #
+          #   avro_table.use_avro_logical_types #=> true
+          #
+          def use_avro_logical_types= new_use_avro_logical_types
+            frozen_check!
+            @gapi.avro_options.use_avro_logical_types = new_use_avro_logical_types
+          end
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/bigquery/external/data_source.rb

@@ -37,7 +37,7 @@ module Google
         #
         #   bigquery = Google::Cloud::Bigquery.new
         #
-        #   avro_url = "gs://bucket/path/to/data.avro"
+        #   avro_url = "gs://bucket/path/to/*.avro"
         #   avro_table = bigquery.external avro_url do |avro|
         #     avro.autodetect = true
         #   end
@@ -174,7 +174,7 @@ module Google
           #
           #   bigquery = Google::Cloud::Bigquery.new
           #
-          #   avro_url = "gs://bucket/path/to/data.avro"
+          #   avro_url = "gs://bucket/path/to/*.avro"
           #   avro_table = bigquery.external avro_url
           #
           #   avro_table.format #=> "AVRO"

data/lib/google/cloud/bigquery/external.rb

@@ -14,6 +14,7 @@
 
 
 require "google/cloud/bigquery/external/data_source"
+require "google/cloud/bigquery/external/avro_source"
 require "google/cloud/bigquery/external/bigtable_source"
 require "google/cloud/bigquery/external/csv_source"
 require "google/cloud/bigquery/external/json_source"
@@ -101,28 +102,28 @@ module Google
         # @private Determine source_format from inputs
         def self.source_format_for urls, format
           val = {
-            "csv"                    => "CSV",
             "avro"                   => "AVRO",
-            "json"                   => "NEWLINE_DELIMITED_JSON",
-            "newline_delimited_json" => "NEWLINE_DELIMITED_JSON",
-            "sheets"                 => "GOOGLE_SHEETS",
-            "google_sheets"          => "GOOGLE_SHEETS",
-            "datastore"              => "DATASTORE_BACKUP",
+            "bigtable"               => "BIGTABLE",
+            "csv"                    => "CSV",
             "backup"                 => "DATASTORE_BACKUP",
+            "datastore"              => "DATASTORE_BACKUP",
             "datastore_backup"       => "DATASTORE_BACKUP",
-            "bigtable"               => "BIGTABLE",
+            "sheets"                 => "GOOGLE_SHEETS",
+            "google_sheets"          => "GOOGLE_SHEETS",
+            "json"                   => "NEWLINE_DELIMITED_JSON",
+            "newline_delimited_json" => "NEWLINE_DELIMITED_JSON",
             "orc"                    => "ORC",
             "parquet"                => "PARQUET"
           }[format.to_s.downcase]
           return val unless val.nil?
           Array(urls).each do |url|
-            return "CSV" if url.end_with? ".csv"
-            return "NEWLINE_DELIMITED_JSON" if url.end_with? ".json"
-            return "PARQUET" if url.end_with? ".parquet"
             return "AVRO" if url.end_with? ".avro"
+            return "BIGTABLE" if url.start_with? "https://googleapis.com/bigtable/projects/"
+            return "CSV" if url.end_with? ".csv"
             return "DATASTORE_BACKUP" if url.end_with? ".backup_info"
             return "GOOGLE_SHEETS" if url.start_with? "https://docs.google.com/spreadsheets/"
-            return "BIGTABLE" if url.start_with? "https://googleapis.com/bigtable/projects/"
+            return "NEWLINE_DELIMITED_JSON" if url.end_with? ".json"
+            return "PARQUET" if url.end_with? ".parquet"
           end
           nil
         end
@@ -131,13 +132,14 @@ module Google
         # @private Determine table class from source_format
         def self.table_class_for format
           case format
+          when "AVRO"                   then External::AvroSource
+          when "BIGTABLE"               then External::BigtableSource
           when "CSV"                    then External::CsvSource
+          when "GOOGLE_SHEETS"          then External::SheetsSource
           when "NEWLINE_DELIMITED_JSON" then External::JsonSource
           when "PARQUET"                then External::ParquetSource
-          when "GOOGLE_SHEETS"          then External::SheetsSource
-          when "BIGTABLE"               then External::BigtableSource
           else
-            # AVRO, DATASTORE_BACKUP
+            # DATASTORE_BACKUP, ORC
             External::DataSource
           end
         end

data/lib/google/cloud/bigquery/version.rb

@@ -16,7 +16,7 @@
 module Google
   module Cloud
     module Bigquery
-      VERSION = "1.36.0".freeze
+      VERSION = "1.37.0".freeze
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: google-cloud-bigquery
 version: !ruby/object:Gem::Version
-  version: 1.36.0
+  version: 1.37.0
 platform: ruby
 authors:
 - Mike Moore
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-22 00:00:00.000000000 Z
+date: 2021-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -270,6 +270,7 @@ files:
 - lib/google/cloud/bigquery/dataset/list.rb
 - lib/google/cloud/bigquery/encryption_configuration.rb
 - lib/google/cloud/bigquery/external.rb
+- lib/google/cloud/bigquery/external/avro_source.rb
 - lib/google/cloud/bigquery/external/bigtable_source.rb
 - lib/google/cloud/bigquery/external/bigtable_source/column.rb
 - lib/google/cloud/bigquery/external/bigtable_source/column_family.rb