google-cloud-bigquery 1.38.1 → 1.42.0

data/lib/google/cloud/bigquery/dataset.rb

@@ -22,6 +22,7 @@ require "google/cloud/bigquery/routine"
 require "google/cloud/bigquery/external"
 require "google/cloud/bigquery/dataset/list"
 require "google/cloud/bigquery/dataset/access"
+require "google/cloud/bigquery/dataset/tag"
 require "google/cloud/bigquery/convert"
 require "google/apis/bigquery_v2"
 
@@ -466,6 +467,21 @@ module Google
           access_builder.freeze
         end
 
+        ##
+        # Retrieves the tags associated with this dataset. Tag keys are
+        # globally unique, and managed via the resource manager API.
+        #
+        # @see https://cloud.google.com/resource-manager/docs/tags/tags-overview
+        # for more information.
+        #
+        # @return [Google::Cloud::Bigquery::Dataset::Tag] The list of tags.
+        #
+        def tags
+          ensure_full_data!
+          return nil if @gapi.tags.nil?
+          @gapi.tags.map { |gapi| Tag.from_gapi(gapi) }
+        end
+
         ##
         # Permanently deletes the dataset. The dataset must be empty before it
         # can be deleted unless the `force` option is set to `true`.
@@ -793,6 +809,11 @@ module Google
         #   object without verifying that the resource exists on the BigQuery
         #   service. Calls made on this object will raise errors if the resource
         #   does not exist. Default is `false`. Optional.
+        # @param [String] view Specifies the view that determines which table information is returned.
+        #   By default, basic table information and storage statistics (STORAGE_STATS) are returned.
+        #   Accepted values include `:unspecified`, `:basic`, `:storage`, and
+        #   `:full`. For more information, see [BigQuery Classes](@todo: Update the link).
+        #   The default value is the `:unspecified` view type.
         #
         # @return [Google::Cloud::Bigquery::Table, nil] Returns `nil` if the
         #   table does not exist.
@@ -815,13 +836,22 @@ module Google
         #
         #   table = dataset.table "my_table", skip_lookup: true
         #
+        # @example Avoid retrieving transient stats of the table with `view`:
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #
+        #   dataset = bigquery.dataset "my_dataset"
+        #
+        #   table = dataset.table "my_table", view: "basic"
+        #
         # @!group Table
         #
-        def table table_id, skip_lookup: nil
+        def table table_id, skip_lookup: nil, view: nil
           ensure_service!
           return Table.new_reference project_id, dataset_id, table_id, service if skip_lookup
-          gapi = service.get_table dataset_id, table_id
-          Table.from_gapi gapi, service
+          gapi = service.get_table dataset_id, table_id, metadata_view: view
+          Table.from_gapi gapi, service, metadata_view: view
         rescue Google::Cloud::NotFoundError
           nil
         end
@@ -1816,7 +1846,7 @@ module Google
         #   The following values are supported:
         #
         #   * `csv` - CSV
-        #   * `json` - [Newline-delimited JSON](http://jsonlines.org/)
+        #   * `json` - [Newline-delimited JSON](https://jsonlines.org/)
         #   * `avro` - [Avro](http://avro.apache.org/)
         #   * `sheets` - Google Sheets
         #   * `datastore_backup` - Cloud Datastore backup
@@ -1879,7 +1909,7 @@ module Google
         #   The following values are supported:
         #
         #   * `csv` - CSV
-        #   * `json` - [Newline-delimited JSON](http://jsonlines.org/)
+        #   * `json` - [Newline-delimited JSON](https://jsonlines.org/)
         #   * `avro` - [Avro](http://avro.apache.org/)
         #   * `orc` - [ORC](https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-orc)
         #   * `parquet` - [Parquet](https://parquet.apache.org/)
@@ -2141,7 +2171,7 @@ module Google
         #   The following values are supported:
         #
         #   * `csv` - CSV
-        #   * `json` - [Newline-delimited JSON](http://jsonlines.org/)
+        #   * `json` - [Newline-delimited JSON](https://jsonlines.org/)
         #   * `avro` - [Avro](http://avro.apache.org/)
         #   * `orc` - [ORC](https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-orc)
         #   * `parquet` - [Parquet](https://parquet.apache.org/)
@@ -2658,6 +2688,11 @@ module Google
         #   messages before the batch is published. Default is 10.
         # @attr_reader [Numeric] threads The number of threads used to insert
         #   batches of rows. Default is 4.
+        # @param [String] view Specifies the view that determines which table information is returned.
+        #   By default, basic table information and storage statistics (STORAGE_STATS) are returned.
+        #   Accepted values include `:unspecified`, `:basic`, `:storage`, and
+        #   `:full`. For more information, see [BigQuery Classes](@todo: Update the link).
+        #   The default value is the `:unspecified` view type.
         # @yield [response] the callback for when a batch of rows is inserted
         # @yieldparam [Table::AsyncInserter::Result] result the result of the
         #   asynchronous insert
@@ -2686,13 +2721,35 @@ module Google
         #
         #   inserter.stop.wait!
         #
+        # @example Avoid retrieving transient stats of the table with while inserting :
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #   dataset = bigquery.dataset "my_dataset"
+        #   inserter = dataset.insert_async("my_table", view: "basic") do |result|
+        #     if result.error?
+        #       log_error result.error
+        #     else
+        #       log_insert "inserted #{result.insert_count} rows " \
+        #         "with #{result.error_count} errors"
+        #     end
+        #   end
+        #
+        #   rows = [
+        #     { "first_name" => "Alice", "age" => 21 },
+        #     { "first_name" => "Bob", "age" => 22 }
+        #   ]
+        #   inserter.insert rows
+        #
+        #   inserter.stop.wait!
+        #
         def insert_async table_id, skip_invalid: nil, ignore_unknown: nil, max_bytes: 10_000_000, max_rows: 500,
-                         interval: 10, threads: 4, &block
+                         interval: 10, threads: 4, view: nil, &block
           ensure_service!
 
           # Get table, don't use Dataset#table which handles NotFoundError
-          gapi = service.get_table dataset_id, table_id
-          table = Table.from_gapi gapi, service
+          gapi = service.get_table dataset_id, table_id, metadata_view: view
+          table = Table.from_gapi gapi, service, metadata_view: view
           # Get the AsyncInserter from the table
           table.insert_async skip_invalid: skip_invalid,
                              ignore_unknown: ignore_unknown,
@@ -2700,6 +2757,29 @@ module Google
                              interval: interval, threads: threads, &block
         end
 
+        ##
+        # Build an object of type Google::Apis::BigqueryV2::DatasetAccessEntry from
+        # the self.
+        #
+        # @param [Array<String>] target_types The list of target types within the dataset.
+        #
+        # @return [Google::Apis::BigqueryV2::DatasetAccessEntry] Returns a DatasetAccessEntry object.
+        #
+        # @example
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #   dataset = bigquery.dataset "my_dataset"
+        #   dataset_access_entry = dataset.access_entry target_types: ["VIEWS"]
+        #
+        def build_access_entry target_types: nil
+          params = {
+            dataset: dataset_ref,
+            target_types: target_types
+          }.delete_if { |_, v| v.nil? }
+          Google::Apis::BigqueryV2::DatasetAccessEntry.new(**params)
+        end
+
         protected
 
         def insert_data_with_autocreate table_id, rows, skip_invalid: nil, ignore_unknown: nil, insert_ids: nil
@@ -2754,7 +2834,7 @@ module Google
         def patch_gapi! *attributes
           return if attributes.empty?
           ensure_service!
-          patch_args = Hash[attributes.map { |attr| [attr, @gapi.send(attr)] }]
+          patch_args = attributes.to_h { |attr| [attr, @gapi.send(attr)] }
           patch_gapi = Google::Apis::BigqueryV2::Dataset.new(**patch_args)
           patch_gapi.etag = etag if etag
           @gapi = service.patch_dataset dataset_id, patch_gapi
@@ -2944,8 +3024,6 @@ module Google
             @access
           end
 
-          # rubocop:disable Style/MethodDefParentheses
-
           ##
           # @raise [RuntimeError] not implemented
           def delete(*)
@@ -3049,8 +3127,6 @@ module Google
           end
           alias refresh! reload!
 
-          # rubocop:enable Style/MethodDefParentheses
-
           ##
           # @private Make sure any access changes are saved
           def check_for_mutated_access!

data/lib/google/cloud/bigquery/extract_job.rb

@@ -65,11 +65,17 @@ module Google
         ##
         # The table or model which is exported.
         #
+        # @param [String] view Specifies the view that determines which table information is returned.
+        #   By default, basic table information and storage statistics (STORAGE_STATS) are returned.
+        #   Accepted values include `:unspecified`, `:basic`, `:storage`, and
+        #   `:full`. For more information, see [BigQuery Classes](@todo: Update the link).
+        #   The default value is the `:unspecified` view type.
+        #
         # @return [Table, Model, nil] A table or model instance, or `nil`.
         #
-        def source
+        def source view: nil
           if (table = @gapi.configuration.extract.source_table)
-            retrieve_table table.project_id, table.dataset_id, table.table_id
+            retrieve_table table.project_id, table.dataset_id, table.table_id, metadata_view: view
           elsif (model = @gapi.configuration.extract.source_model)
             retrieve_model model.project_id, model.dataset_id, model.model_id
           end
@@ -108,7 +114,7 @@ module Google
 
         ##
         # Checks if the destination format for the table data is [newline-delimited
-        # JSON](http://jsonlines.org/). The default is `false`. Not applicable when
+        # JSON](https://jsonlines.org/). The default is `false`. Not applicable when
         # extracting models.
         #
         # @return [Boolean] `true` when `NEWLINE_DELIMITED_JSON`, `false` if not
@@ -221,7 +227,7 @@ module Google
         #   and the counts as values.
         #
         def destinations_counts
-          Hash[destinations.zip destinations_file_counts]
+          destinations.zip(destinations_file_counts).to_h
         end
 
         ##
@@ -362,7 +368,7 @@ module Google
           # Supported values for tables:
           #
           # * `csv` - CSV
-          # * `json` - [Newline-delimited JSON](http://jsonlines.org/)
+          # * `json` - [Newline-delimited JSON](https://jsonlines.org/)
           # * `avro` - [Avro](http://avro.apache.org/)
           #
           # Supported values for models:

data/lib/google/cloud/bigquery/job.rb

@@ -489,7 +489,7 @@ module Google
         #
         def wait_until_done!
           backoff = lambda do |retries|
-            delay = [retries**2 + 5, 60].min # Maximum delay is 60
+            delay = [(retries**2) + 5, 60].min # Maximum delay is 60
             sleep delay
           end
           retries = 0
@@ -710,9 +710,9 @@ module Google
           raise "Must have active connection" unless service
         end
 
-        def retrieve_table project_id, dataset_id, table_id
+        def retrieve_table project_id, dataset_id, table_id, metadata_view: nil
           ensure_service!
-          gapi = service.get_project_table project_id, dataset_id, table_id
+          gapi = service.get_project_table project_id, dataset_id, table_id, metadata_view: metadata_view
           Table.from_gapi gapi, service
         rescue Google::Cloud::NotFoundError
           nil

data/lib/google/cloud/bigquery/load_job.rb

@@ -62,12 +62,18 @@ module Google
         # The table into which the operation loads data. This is the table on
         # which {Table#load_job} was invoked.
         #
+        # @param [String] view Specifies the view that determines which table information is returned.
+        #   By default, basic table information and storage statistics (STORAGE_STATS) are returned.
+        #   Accepted values include `:unspecified`, `:basic`, `:storage`, and
+        #   `:full`. For more information, see [BigQuery Classes](@todo: Update the link).
+        #   The default value is the `:unspecified` view type.
+        #
         # @return [Table] A table instance.
         #
-        def destination
+        def destination view: nil
           table = @gapi.configuration.load.destination_table
           return nil unless table
-          retrieve_table table.project_id, table.dataset_id, table.table_id
+          retrieve_table table.project_id, table.dataset_id, table.table_id, metadata_view: view
         end
 
         ##
@@ -188,7 +194,7 @@ module Google
 
         ##
         # Checks if the format of the source data is [newline-delimited
-        # JSON](http://jsonlines.org/). The default is `false`.
+        # JSON](https://jsonlines.org/). The default is `false`.
         #
         # @return [Boolean] `true` when the source format is
         #   `NEWLINE_DELIMITED_JSON`, `false` otherwise.
@@ -1269,7 +1275,7 @@ module Google
           # The following values are supported:
           #
           # * `csv` - CSV
-          # * `json` - [Newline-delimited JSON](http://jsonlines.org/)
+          # * `json` - [Newline-delimited JSON](https://jsonlines.org/)
           # * `avro` - [Avro](http://avro.apache.org/)
           # * `orc` - [ORC](https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-orc)
           # * `parquet` - [Parquet](https://parquet.apache.org/)

data/lib/google/cloud/bigquery/project.rb

@@ -961,7 +961,7 @@ module Google
         #   The following values are supported:
         #
         #   * `csv` - CSV
-        #   * `json` - [Newline-delimited JSON](http://jsonlines.org/)
+        #   * `json` - [Newline-delimited JSON](https://jsonlines.org/)
         #   * `avro` - [Avro](http://avro.apache.org/)
         #   * `sheets` - Google Sheets
         #   * `datastore_backup` - Cloud Datastore backup
@@ -1554,7 +1554,7 @@ module Google
         #   Supported values for tables:
         #
         #   * `csv` - CSV
-        #   * `json` - [Newline-delimited JSON](http://jsonlines.org/)
+        #   * `json` - [Newline-delimited JSON](https://jsonlines.org/)
         #   * `avro` - [Avro](http://avro.apache.org/)
         #
         #   Supported values for models:
@@ -1683,7 +1683,7 @@ module Google
         #   Supported values for tables:
         #
         #   * `csv` - CSV
-        #   * `json` - [Newline-delimited JSON](http://jsonlines.org/)
+        #   * `json` - [Newline-delimited JSON](https://jsonlines.org/)
         #   * `avro` - [Avro](http://avro.apache.org/)
         #
         #   Supported values for models:

data/lib/google/cloud/bigquery/query_job.rb

@@ -437,14 +437,21 @@ module Google
         ##
         # The table in which the query results are stored.
         #
+        # @param [String] view Specifies the view that determines which table information is returned.
+        #   By default, basic table information and storage statistics (STORAGE_STATS) are returned.
+        #   Accepted values include `:unspecified`, `:basic`, `:storage`, and
+        #   `:full`. For more information, see [BigQuery Classes](@todo: Update the link).
+        #   The default value is the `:unspecified` view type.
+        #
         # @return [Table] A table instance.
         #
-        def destination
+        def destination view: nil
           table = @gapi.configuration.query.destination_table
           return nil unless table
           retrieve_table table.project_id,
                          table.dataset_id,
-                         table.table_id
+                         table.table_id,
+                         metadata_view: view
         end
 
         ##
@@ -1193,7 +1200,7 @@ module Google
           #
           def external= value
             external_table_pairs = value.map { |name, obj| [String(name), obj.to_gapi] }
-            external_table_hash = Hash[external_table_pairs]
+            external_table_hash = external_table_pairs.to_h
             @gapi.configuration.query.table_definitions = external_table_hash
           end
 

data/lib/google/cloud/bigquery/schema/field.rb

@@ -471,7 +471,7 @@ module Google
           #
           def param_type
             param_type = type.to_sym
-            param_type = Hash[fields.map { |field| [field.name.to_sym, field.param_type] }] if record?
+            param_type = fields.to_h { |field| [field.name.to_sym, field.param_type] } if record?
             param_type = [param_type] if repeated?
             param_type
           end

data/lib/google/cloud/bigquery/schema.rb

@@ -172,7 +172,7 @@ module Google
         #   schema.param_types
         #
         def param_types
-          Hash[fields.map { |field| [field.name.to_sym, field.param_type] }]
+          fields.to_h { |field| [field.name.to_sym, field.param_type] }
         end
 
         ##

data/lib/google/cloud/bigquery/service.rb

@@ -144,10 +144,11 @@ module Google
 
         ##
         # Gets the specified table resource by full table reference.
-        def get_project_table project_id, dataset_id, table_id
+        def get_project_table project_id, dataset_id, table_id, metadata_view: nil
+          metadata_view = table_metadata_view_type_for metadata_view
           # The get operation is considered idempotent
           execute backoff: true do
-            service.get_table project_id, dataset_id, table_id
+            service.get_table project_id, dataset_id, table_id, view: metadata_view
           end
         end
 
@@ -156,8 +157,8 @@ module Google
         # This method does not return the data in the table,
         # it only returns the table resource,
         # which describes the structure of this table.
-        def get_table dataset_id, table_id
-          get_project_table @project, dataset_id, table_id
+        def get_table dataset_id, table_id, metadata_view: nil
+          get_project_table @project, dataset_id, table_id, metadata_view: metadata_view
         end
 
         ##
@@ -496,6 +497,19 @@ module Google
           ref
         end
 
+        ##
+        # Converts a hash to a Google::Apis::BigqueryV2::DatasetAccessEntry oject.
+        #
+        # @param [Hash<String,String>] dataset_hash Hash for a DatasetAccessEntry.
+        #
+        def self.dataset_access_entry_from_hash dataset_hash
+          params = {
+            dataset: Google::Apis::BigqueryV2::DatasetReference.new(**dataset_hash),
+            target_types: dataset_hash[:target_types]
+          }.delete_if { |_, v| v.nil? }
+          Google::Apis::BigqueryV2::DatasetAccessEntry.new(**params)
+        end
+
         def self.validate_table_ref table_ref
           [:project_id, :dataset_id, :table_id].each do |f|
             raise ArgumentError, "TableReference is missing #{f}" if table_ref.send(f).nil?
@@ -572,6 +586,14 @@ module Google
           raise Google::Cloud::Error.from_error e
         end
 
+        def table_metadata_view_type_for str
+          return nil if str.nil?
+          { "unspecified" => "TABLE_METADATA_VIEW_UNSPECIFIED",
+            "basic" => "BASIC",
+            "storage" => "STORAGE_STATS",
+            "full" => "FULL" }[str.to_s.downcase]
+        end
+
         class Backoff
           class << self
             attr_accessor :retries