google-cloud-bigquery 1.14.0 → 1.42.0

data/lib/google/cloud/bigquery/external/json_source.rb

@@ -0,0 +1,170 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+module Google
+  module Cloud
+    module Bigquery
+      module External
+        ##
+        # # JsonSource
+        #
+        # {External::JsonSource} is a subclass of {External::DataSource} and
+        # represents a JSON external data source that can be queried from
+        # directly, such as Google Cloud Storage or Google Drive, even though
+        # the data is not stored in BigQuery. Instead of loading or streaming
+        # the data, this object references the external data source.
+        #
+        # @example
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #
+        #   json_url = "gs://bucket/path/to/data.json"
+        #   json_table = bigquery.external json_url do |json|
+        #     json.schema do |schema|
+        #       schema.string "name", mode: :required
+        #       schema.string "email", mode: :required
+        #       schema.integer "age", mode: :required
+        #       schema.boolean "active", mode: :required
+        #     end
+        #   end
+        #
+        #   data = bigquery.query "SELECT * FROM my_ext_table",
+        #                         external: { my_ext_table: json_table }
+        #
+        #   # Iterate over the first page of results
+        #   data.each do |row|
+        #     puts row[:name]
+        #   end
+        #   # Retrieve the next page of results
+        #   data = data.next if data.next?
+        #
+        class JsonSource < External::DataSource
+          ##
+          # The schema for the data.
+          #
+          # @param [Boolean] replace Whether to replace the existing schema with
+          #   the new schema. If `true`, the fields will replace the existing
+          #   schema. If `false`, the fields will be added to the existing
+          #   schema. The default value is `false`.
+          # @yield [schema] a block for setting the schema
+          # @yieldparam [Schema] schema the object accepting the schema
+          #
+          # @return [Google::Cloud::Bigquery::Schema]
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   json_url = "gs://bucket/path/to/data.json"
+          #   json_table = bigquery.external json_url do |json|
+          #     json.schema do |schema|
+          #       schema.string "name", mode: :required
+          #       schema.string "email", mode: :required
+          #       schema.integer "age", mode: :required
+          #       schema.boolean "active", mode: :required
+          #     end
+          #   end
+          #
+          def schema replace: false
+            @schema ||= Schema.from_gapi @gapi.schema
+            if replace
+              frozen_check!
+              @schema = Schema.from_gapi
+            end
+            @schema.freeze if frozen?
+            yield @schema if block_given?
+            @schema
+          end
+
+          ##
+          # Set the schema for the data.
+          #
+          # @param [Schema] new_schema The schema object.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   json_shema = bigquery.schema do |schema|
+          #     schema.string "name", mode: :required
+          #     schema.string "email", mode: :required
+          #     schema.integer "age", mode: :required
+          #     schema.boolean "active", mode: :required
+          #   end
+          #
+          #   json_url = "gs://bucket/path/to/data.json"
+          #   json_table = bigquery.external json_url
+          #   json_table.schema = json_shema
+          #
+          def schema= new_schema
+            frozen_check!
+            @schema = new_schema
+          end
+
+          ##
+          # The fields of the schema.
+          #
+          # @return [Array<Schema::Field>] An array of field objects.
+          #
+          def fields
+            schema.fields
+          end
+
+          ##
+          # The names of the columns in the schema.
+          #
+          # @return [Array<Symbol>] An array of column names.
+          #
+          def headers
+            schema.headers
+          end
+
+          ##
+          # The types of the fields in the data in the schema, using the same
+          # format as the optional query parameter types.
+          #
+          # @return [Hash] A hash with field names as keys, and types as values.
+          #
+          def param_types
+            schema.param_types
+          end
+
+          ##
+          # @private Google API Client object.
+          def to_gapi
+            @gapi.schema = @schema.to_gapi if @schema
+            @gapi
+          end
+
+          ##
+          # @private Google API Client object.
+          def self.from_gapi gapi
+            new_table = super
+            schema = Schema.from_gapi gapi.schema
+            new_table.instance_variable_set :@schema, schema
+            new_table
+          end
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/bigquery/external/parquet_source.rb

@@ -0,0 +1,148 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "google/apis/bigquery_v2"
+
+module Google
+  module Cloud
+    module Bigquery
+      module External
+        ##
+        # # ParquetSource
+        #
+        # {External::ParquetSource} is a subclass of {External::DataSource} and
+        # represents a Parquet external data source that can be queried
+        # from directly, even though the data is not stored in BigQuery. Instead
+        # of loading or streaming the data, this object references the external
+        # data source.
+        #
+        # @example
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #
+        #   parquet_url = "gs://bucket/path/to/data.parquet"
+        #   parquet_table = bigquery.external parquet_url do |parquet|
+        #     parquet.enable_list_inference = 1
+        #   end
+        #
+        #   data = bigquery.query "SELECT * FROM my_ext_table",
+        #                         external: { my_ext_table: parquet_table }
+        #
+        #   # Iterate over the first page of results
+        #   data.each do |row|
+        #     puts row[:name]
+        #   end
+        #   # Retrieve the next page of results
+        #   data = data.next if data.next?
+        #
+        class ParquetSource < External::DataSource
+          ##
+          # @private Create an empty ParquetSource object.
+          def initialize
+            super
+            @gapi.parquet_options = Google::Apis::BigqueryV2::ParquetOptions.new
+          end
+
+          ##
+          # Indicates whether to use schema inference specifically for Parquet `LIST` logical type.
+          #
+          # @return [Boolean]
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   parquet_url = "gs://bucket/path/to/data.parquet"
+          #   parquet_table = bigquery.external parquet_url do |parquet|
+          #     parquet.enable_list_inference = true
+          #   end
+          #
+          #   parquet_table.enable_list_inference #=> true
+          #
+          def enable_list_inference
+            @gapi.parquet_options.enable_list_inference
+          end
+
+          ##
+          # Sets whether to use schema inference specifically for Parquet `LIST` logical type.
+          #
+          # @param [Boolean] new_enable_list_inference The new `enable_list_inference` value.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   parquet_url = "gs://bucket/path/to/data.parquet"
+          #   parquet_table = bigquery.external parquet_url do |parquet|
+          #     parquet.enable_list_inference = true
+          #   end
+          #
+          #   parquet_table.enable_list_inference #=> true
+          #
+          def enable_list_inference= new_enable_list_inference
+            frozen_check!
+            @gapi.parquet_options.enable_list_inference = new_enable_list_inference
+          end
+
+          ##
+          # Indicates whether to infer Parquet `ENUM` logical type as `STRING` instead of `BYTES` by default.
+          #
+          # @return [Boolean]
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   parquet_url = "gs://bucket/path/to/data.parquet"
+          #   parquet_table = bigquery.external parquet_url do |parquet|
+          #     parquet.enum_as_string = true
+          #   end
+          #
+          #   parquet_table.enum_as_string #=> true
+          #
+          def enum_as_string
+            @gapi.parquet_options.enum_as_string
+          end
+
+          ##
+          # Sets whether to infer Parquet `ENUM` logical type as `STRING` instead of `BYTES` by default.
+          #
+          # @param [Boolean] new_enum_as_string The new `enum_as_string` value.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   parquet_url = "gs://bucket/path/to/data.parquet"
+          #   parquet_table = bigquery.external parquet_url do |parquet|
+          #     parquet.enum_as_string = true
+          #   end
+          #
+          #   parquet_table.enum_as_string #=> true
+          #
+          def enum_as_string= new_enum_as_string
+            frozen_check!
+            @gapi.parquet_options.enum_as_string = new_enum_as_string
+          end
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/bigquery/external/sheets_source.rb

@@ -0,0 +1,166 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "google/apis/bigquery_v2"
+
+module Google
+  module Cloud
+    module Bigquery
+      module External
+        ##
+        # # SheetsSource
+        #
+        # {External::SheetsSource} is a subclass of {External::DataSource} and
+        # represents a Google Sheets external data source that can be queried
+        # from directly, even though the data is not stored in BigQuery. Instead
+        # of loading or streaming the data, this object references the external
+        # data source.
+        #
+        # @example
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #
+        #   sheets_url = "https://docs.google.com/spreadsheets/d/1234567980"
+        #   sheets_table = bigquery.external sheets_url do |sheets|
+        #     sheets.skip_leading_rows = 1
+        #   end
+        #
+        #   data = bigquery.query "SELECT * FROM my_ext_table",
+        #                         external: { my_ext_table: sheets_table }
+        #
+        #   # Iterate over the first page of results
+        #   data.each do |row|
+        #     puts row[:name]
+        #   end
+        #   # Retrieve the next page of results
+        #   data = data.next if data.next?
+        #
+        class SheetsSource < External::DataSource
+          ##
+          # @private Create an empty SheetsSource object.
+          def initialize
+            super
+            @gapi.google_sheets_options = Google::Apis::BigqueryV2::GoogleSheetsOptions.new
+          end
+
+          ##
+          # The number of rows at the top of a sheet that BigQuery will skip
+          # when reading the data. The default value is `0`.
+          #
+          # This property is useful if you have header rows that should be
+          # skipped. When `autodetect` is on, behavior is the following:
+          #
+          # * `nil` - Autodetect tries to detect headers in the first row. If
+          #   they are not detected, the row is read as data. Otherwise data is
+          #   read starting from the second row.
+          # * `0` - Instructs autodetect that there are no headers and data
+          #   should be read starting from the first row.
+          # * `N > 0` - Autodetect skips `N-1` rows and tries to detect headers
+          #   in row `N`. If headers are not detected, row `N` is just skipped.
+          #   Otherwise row `N` is used to extract column names for the detected
+          #   schema.
+          #
+          # @return [Integer]
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   sheets_url = "https://docs.google.com/spreadsheets/d/1234567980"
+          #   sheets_table = bigquery.external sheets_url do |sheets|
+          #     sheets.skip_leading_rows = 1
+          #   end
+          #
+          #   sheets_table.skip_leading_rows #=> 1
+          #
+          def skip_leading_rows
+            @gapi.google_sheets_options.skip_leading_rows
+          end
+
+          ##
+          # Set the number of rows at the top of a sheet that BigQuery will skip
+          # when reading the data.
+          #
+          # @param [Integer] row_count New skip_leading_rows value
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   sheets_url = "https://docs.google.com/spreadsheets/d/1234567980"
+          #   sheets_table = bigquery.external sheets_url do |sheets|
+          #     sheets.skip_leading_rows = 1
+          #   end
+          #
+          #   sheets_table.skip_leading_rows #=> 1
+          #
+          def skip_leading_rows= row_count
+            frozen_check!
+            @gapi.google_sheets_options.skip_leading_rows = row_count
+          end
+
+          ##
+          # Range of a sheet to query from. Only used when non-empty. Typical
+          # format: `{sheet_name}!{top_left_cell_id}:{bottom_right_cell_id}`.
+          #
+          # @return [String] Range of a sheet to query from.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   sheets_url = "https://docs.google.com/spreadsheets/d/1234567980"
+          #   sheets_table = bigquery.external sheets_url do |sheets|
+          #     sheets.range = "sheet1!A1:B20"
+          #   end
+          #
+          #   sheets_table.range #=> "sheet1!A1:B20"
+          #
+          def range
+            @gapi.google_sheets_options.range
+          end
+
+          ##
+          # Set the range of a sheet to query from. Only used when non-empty.
+          # Typical format:
+          # `{sheet_name}!{top_left_cell_id}:{bottom_right_cell_id}`.
+          #
+          # @param [String] new_range New range of a sheet to query from.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   sheets_url = "https://docs.google.com/spreadsheets/d/1234567980"
+          #   sheets_table = bigquery.external sheets_url do |sheets|
+          #     sheets.range = "sheet1!A1:B20"
+          #   end
+          #
+          #   sheets_table.range #=> "sheet1!A1:B20"
+          #
+          def range= new_range
+            frozen_check!
+            @gapi.google_sheets_options.range = new_range
+          end
+        end
+      end
+    end
+  end
+end