google-cloud-bigquery 1.52.1 → 1.61.1

data/lib/google/cloud/bigquery/dataset.rb

@@ -29,6 +29,48 @@ require "google/apis/bigquery_v2"
 module Google
   module Cloud
     module Bigquery
+      module DatasetView
+        # Provides constants for the dataset_view parameter, an optional field
+        # in the GetDatasetRequest used to specify which information about a
+        # BigQuery dataset should be returned in the response. By controlling
+        # this parameter, users can request a partial or full response, which
+        # helps enforce fine-grained access control based on their permissions.
+
+        # Default. Equivalent to `FULL`. `datasets.get` and `datasets.getIamPolicy` permissions required.
+        DATASET_VIEW_UNSPECIFIED = "DATASET_VIEW_UNSPECIFIED".freeze
+
+        # Returns metadata only. `datasets.get` permission required.
+        METADATA = "METADATA".freeze
+
+        # Returns ACLs only. `datasets.getIamPolicy` permission required.
+        ACL = "ACL".freeze
+
+        # Returns metadata and ACLs. `datasets.get` and `datasets.getIamPolicy` permissions required.
+        FULL = "FULL".freeze
+      end
+
+      module UpdateMode
+        # Provides constants for the update_mode parameter, an optional field
+        # in the PatchDatasetRequest and UpdateDatasetRequest used to specify
+        # whether the resource is being updated with full or partial semantics
+        # (metadata, ACLs, or both).
+        # By controlling this parameter, users can request full or partial
+        # update semantics, which helps enforce fine-grained access control
+        # based on their permissions.
+
+        # Default. Equivalent to `UPDATE_FULL`. `datasets.update` and `datasets.setIamPolicy` permissions required.
+        UPDATE_MODE_UNSPECIFIED = "UPDATE_MODE_UNSPECIFIED".freeze
+
+        # Updates both metadata and ACLs. `datasets.update` and `datasets.setIamPolicy` permissions required.
+        UPDATE_FULL = "UPDATE_FULL".freeze
+
+        # Updates only metadata. `datasets.update` permission required.
+        UPDATE_METADATA = "UPDATE_METADATA".freeze
+
+        # Updates only ACLs. `datasets.setIamPolicy` permission required.
+        UPDATE_ACL = "UPDATE_ACL".freeze
+      end
+
       ##
       # # Dataset
       #
@@ -59,12 +101,25 @@ module Google
         # @private A Google API Client Dataset Reference object.
         attr_reader :reference
 
+        ##
+        # @private Access Policy Version for get, update, patch, and insert API calls
+        attr_accessor :access_policy_version
+
+        ##
+        # @private The dataset_view parameter is an optional field in the GetDatasetRequest used to specify which
+        # information about a BigQuery dataset should be returned in the response. By controlling this parameter, users
+        # can request a partial or full response, which helps enforce fine-grained access control based on their
+        # permissions.
+        attr_accessor :dataset_view
+
         ##
         # @private Create an empty Dataset object.
         def initialize
           @service = nil
           @gapi = nil
           @reference = nil
+          @access_policy_version = nil
+          @dataset_view = nil
         end
 
         ##
@@ -192,6 +247,37 @@ module Google
           patch_gapi! :description
         end
 
+        ##
+        # The default collation of the dataset.
+        #
+        # @return [String, nil] The default collation, or `nil` if not present or the object is a
+        #   reference (see {#reference?}).
+        #
+        # @!group Attributes
+        #
+        def default_collation
+          return nil if reference?
+          ensure_full_data!
+          @gapi.default_collation
+        end
+
+        ##
+        # Updates the default collation of the dataset.
+        #
+        # If the dataset is not a full resource representation (see
+        # {#resource_full?}), the full representation will be retrieved before
+        # the update to comply with ETag-based optimistic concurrency control.
+        #
+        # @param [String] new_default_collation The new default collation for the dataset.
+        #
+        # @!group Attributes
+        #
+        def default_collation= new_default_collation
+          reload! unless resource_full?
+          @gapi.update! default_collation: new_default_collation
+          patch_gapi! :default_collation
+        end
+
         ##
         # The default lifetime of all tables in the dataset, in milliseconds.
         #
@@ -1388,6 +1474,12 @@ module Google
         #   The default value is false.
         # @param [String] session_id The ID of an existing session. See also the
         #   `create_session` param and {Job#session_id}.
+        # @param [String] reservation The reservation that job would use. User
+        #    can specify a reservation to execute the job. If reservation is not
+        #    set, reservation is determined based on the rules defined by the
+        #    reservation assignments. The expected format is
+        #    `projects/`project`/locations/`location`/reservations/`reservation``.
+        #
         # @yield [job] a job configuration object
         # @yieldparam [Google::Cloud::Bigquery::QueryJob::Updater] job a job
         #   configuration object for setting additional options for the query.
@@ -1560,7 +1652,8 @@ module Google
                       labels: nil,
                       udfs: nil,
                       create_session: nil,
-                      session_id: nil
+                      session_id: nil,
+                      reservation: nil
           ensure_service!
           options = {
             params: params,
@@ -1583,7 +1676,8 @@ module Google
             labels: labels,
             udfs: udfs,
             create_session: create_session,
-            session_id: session_id
+            session_id: session_id,
+            reservation: reservation
           }
 
           updater = QueryJob::Updater.from_options service, query, options
@@ -1710,6 +1804,14 @@ module Google
         #   `flatten` is false. Optional. The default value is false.
         # @param [String] session_id The ID of an existing session. See the
         #   `create_session` param in {#query_job} and {Job#session_id}.
+        # @param [Boolean] format_options_use_int64_timestamp Output timestamp
+        #   as usec int64. Default is true.
+        # @param [String] reservation The reservation that job would use. User
+        #    can specify a reservation to execute the job. If reservation is not
+        #    set, reservation is determined based on the rules defined by the
+        #    reservation assignments. The expected format is
+        #    `projects/`project`/locations/`location`/reservations/`reservation``.
+        #
         # @yield [job] a job configuration object
         # @yieldparam [Google::Cloud::Bigquery::QueryJob::Updater] job a job
         #   configuration object for setting additional options for the query.
@@ -1864,6 +1966,8 @@ module Google
                   standard_sql: nil,
                   legacy_sql: nil,
                   session_id: nil,
+                  format_options_use_int64_timestamp: true,
+                  reservation: nil,
                   &block
           job = query_job query,
                           params: params,
@@ -1873,11 +1977,12 @@ module Google
                           standard_sql: standard_sql,
                           legacy_sql: legacy_sql,
                           session_id: session_id,
+                          reservation: reservation,
                           &block
           job.wait_until_done!
           ensure_job_succeeded! job
 
-          job.data max: max
+          job.data max: max, format_options_use_int64_timestamp: format_options_use_int64_timestamp
         end
 
         ##
@@ -2091,6 +2196,53 @@ module Google
         #   and the load job will happen in the table created within that session.
         #   Note: This will work only for _SESSION dataset.
         # @param [string] session_id Session ID in which the load job must run.
+        # @param [String] date_format Format used to parse DATE values.
+        #   Supports SQL-style format strings. See
+        #   [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
+        # @param [String] datetime_format Format used to parse DATETIME
+        #   values. Supports SQL-style format strings. See
+        #   [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
+        # @param [String] time_format Format used to parse TIME values.
+        #   Supports SQL-style format strings. See
+        #   [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
+        # @param [String] timestamp_format Format used to parse
+        #   TIMESTAMP values. Supports SQL-style format strings. See
+        #   [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
+        # @param [Array<String>] null_markers A list of strings represented as
+        #   SQL NULL value in a CSV file. null_marker and null_markers can't be
+        #   set at the same time. If null_marker is set, null_markers has to be
+        #   not set. If null_markers is set, null_marker has to be not set. If
+        #   both null_marker and null_markers are set at the same time, a user
+        #   error would be thrown. Any strings listed in null_markers, including
+        #   empty string would be interpreted as SQL NULL. This applies to all
+        #   column types.
+        # @param [String] source_column_match Controls the strategy used to
+        #   match loaded columns to the schema. If not set, a sensible default is
+        #   chosen based on how the schema is provided. If autodetect is used,
+        #   then columns are matched by name. Otherwise, columns are matched by
+        #   position. This is done to keep the behavior backward-compatible.
+        #
+        #   Acceptable values are:
+        #   * `POSITION` - matches by position. This assumes that the columns are
+        #     ordered the same way as the schema.
+        #   * `NAME` - matches by name. This reads the header row as column names
+        #     and reorders columns to match the field names in the schema.
+        # @param [String] time_zone The time zone used when parsing timestamp
+        #   values.
+        # @param [String] reference_file_schema_uri The URI of the reference
+        #   file with the reader schema. This file is only loaded if it is part
+        #   of source URIs, but is not loaded otherwise. It is enabled for the
+        #   following formats: `AVRO`, `PARQUET`, `ORC`.
+        # @param [Boolean] preserve_ascii_control_characters When source_format
+        #   is set to `CSV`, indicates if the embedded ASCII control characters
+        #   (the first 32 characters in the ASCII-table, from `\x00` to `\x1F`)
+        #   are preserved. By default, ASCII control characters are not
+        #   preserved.
+        # @param [String] reservation The reservation that job would use. User
+        #    can specify a reservation to execute the job. If reservation is not
+        #    set, reservation is determined based on the rules defined by the
+        #    reservation assignments. The expected format is
+        #    `projects/`project`/locations/`location`/reservations/`reservation``.
         #
         # @yield [updater] A block for setting the schema and other
         #   options for the destination table. The schema can be omitted if the
@@ -2185,7 +2337,10 @@ module Google
         def load_job table_id, files, format: nil, create: nil, write: nil, projection_fields: nil, jagged_rows: nil,
                      quoted_newlines: nil, encoding: nil, delimiter: nil, ignore_unknown: nil, max_bad_records: nil,
                      quote: nil, skip_leading: nil, schema: nil, job_id: nil, prefix: nil, labels: nil, autodetect: nil,
-                     null_marker: nil, dryrun: nil, create_session: nil, session_id: nil
+                     null_marker: nil, dryrun: nil, create_session: nil, session_id: nil, date_format: nil,
+                     datetime_format: nil, time_format: nil, timestamp_format: nil, null_markers: nil,
+                     source_column_match: nil, time_zone: nil, reference_file_schema_uri: nil,
+                     preserve_ascii_control_characters: nil, reservation: nil
           ensure_service!
 
           updater = load_job_updater table_id,
@@ -2195,7 +2350,13 @@ module Google
                                      max_bad_records: max_bad_records, quote: quote, skip_leading: skip_leading,
                                      dryrun: dryrun, schema: schema, job_id: job_id, prefix: prefix, labels: labels,
                                      autodetect: autodetect, null_marker: null_marker, create_session: create_session,
-                                     session_id: session_id
+                                     session_id: session_id, date_format: date_format, datetime_format: datetime_format,
+                                     time_format: time_format, timestamp_format: timestamp_format,
+                                     null_markers: null_markers, source_column_match: source_column_match,
+                                     time_zone: time_zone, reference_file_schema_uri: reference_file_schema_uri,
+                                     preserve_ascii_control_characters: preserve_ascii_control_characters,
+                                     reservation: reservation
+
 
           yield updater if block_given?
 
@@ -2322,7 +2483,53 @@ module Google
         #   this option. Also note that for most use cases, the block yielded by
         #   this method is a more convenient way to configure the schema.
         # @param [string] session_id Session ID in which the load job must run.
-        #
+        # @param [String] date_format Format used to parse DATE values.
+        #   Supports SQL-style format strings. See
+        #   [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
+        # @param [String] datetime_format Format used to parse DATETIME
+        #   values. Supports SQL-style format strings. See
+        #   [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
+        # @param [String] time_format Format used to parse TIME values.
+        #   Supports SQL-style format strings. See
+        #   [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
+        # @param [String] timestamp_format Format used to parse
+        #   TIMESTAMP values. Supports SQL-style format strings. See
+        #   [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
+        # @param [Array<String>] null_markers A list of strings represented as
+        #   SQL NULL value in a CSV file. null_marker and null_markers can't be
+        #   set at the same time. If null_marker is set, null_markers has to be
+        #   not set. If null_markers is set, null_marker has to be not set. If
+        #   both null_marker and null_markers are set at the same time, a user
+        #   error would be thrown. Any strings listed in null_markers, including
+        #   empty string would be interpreted as SQL NULL. This applies to all
+        #   column types.
+        # @param [String] source_column_match Controls the strategy used to
+        #   match loaded columns to the schema. If not set, a sensible default is
+        #   chosen based on how the schema is provided. If autodetect is used,
+        #   then columns are matched by name. Otherwise, columns are matched by
+        #   position. This is done to keep the behavior backward-compatible.
+        #
+        #   Acceptable values are:
+        #   * `POSITION` - matches by position. This assumes that the columns are
+        #     ordered the same way as the schema.
+        #   * `NAME` - matches by name. This reads the header row as column names
+        #     and reorders columns to match the field names in the schema.
+        # @param [String] time_zone The time zone used when parsing timestamp
+        #   values.
+        # @param [String] reference_file_schema_uri The URI of the reference
+        #   file with the reader schema. This file is only loaded if it is part
+        #   of source URIs, but is not loaded otherwise. It is enabled for the
+        #   following formats: `AVRO`, `PARQUET`, `ORC`.
+        # @param [Boolean] preserve_ascii_control_characters When source_format
+        #   is set to `CSV`, indicates if the embedded ASCII control characters
+        #   (the first 32 characters in the ASCII-table, from `\x00` to `\x1F`)
+        #   are preserved. By default, ASCII control characters are not
+        #   preserved.
+        # @param [String] reservation The reservation that job would use. User
+        #    can specify a reservation to execute the job. If reservation is not
+        #    set, reservation is determined based on the rules defined by the
+        #    reservation assignments. The expected format is
+        #    `projects/`project`/locations/`location`/reservations/`reservation``.
         #
         # @yield [updater] A block for setting the schema of the destination
         #   table and other options for the load job. The schema can be omitted
@@ -2415,13 +2622,22 @@ module Google
         #
         def load table_id, files, format: nil, create: nil, write: nil, projection_fields: nil, jagged_rows: nil,
                  quoted_newlines: nil, encoding: nil, delimiter: nil, ignore_unknown: nil, max_bad_records: nil,
-                 quote: nil, skip_leading: nil, schema: nil, autodetect: nil, null_marker: nil, session_id: nil, &block
+                 quote: nil, skip_leading: nil, schema: nil, autodetect: nil, null_marker: nil, session_id: nil,
+                 date_format: nil, datetime_format: nil, time_format: nil, timestamp_format: nil,
+                 null_markers: nil, source_column_match: nil, time_zone: nil, reference_file_schema_uri: nil,
+                 preserve_ascii_control_characters: nil,
+                 reservation: nil, &block
           job = load_job table_id, files,
                          format: format, create: create, write: write, projection_fields: projection_fields,
                          jagged_rows: jagged_rows, quoted_newlines: quoted_newlines, encoding: encoding,
                          delimiter: delimiter, ignore_unknown: ignore_unknown, max_bad_records: max_bad_records,
                          quote: quote, skip_leading: skip_leading, schema: schema, autodetect: autodetect,
-                         null_marker: null_marker, session_id: session_id, &block
+                         null_marker: null_marker, session_id: session_id, date_format: date_format,
+                         datetime_format: datetime_format, time_format: time_format, timestamp_format: timestamp_format,
+                         null_markers: null_markers, source_column_match: source_column_match, time_zone: time_zone,
+                         reference_file_schema_uri: reference_file_schema_uri,
+                         preserve_ascii_control_characters: preserve_ascii_control_characters,
+                         reservation: reservation, &block
 
           job.wait_until_done!
           ensure_job_succeeded! job
@@ -2444,7 +2660,8 @@ module Google
         #
         def reload!
           ensure_service!
-          @gapi = service.get_project_dataset project_id, dataset_id
+          @gapi = service.get_project_dataset project_id, dataset_id, access_policy_version: @access_policy_version,
+            dataset_view: @dataset_view
           @reference = nil
           @exists = nil
           self
@@ -2573,10 +2790,12 @@ module Google
 
         ##
         # @private New Dataset from a Google API Client object.
-        def self.from_gapi gapi, conn
+        def self.from_gapi gapi, conn, access_policy_version: nil, dataset_view: nil
           new.tap do |f|
             f.gapi = gapi
             f.service = conn
+            f.access_policy_version = access_policy_version
+            f.dataset_view = dataset_view
           end
         end
 
@@ -2897,10 +3116,25 @@ module Google
         def patch_gapi! *attributes
           return if attributes.empty?
           ensure_service!
+
           patch_args = attributes.to_h { |attr| [attr, @gapi.send(attr)] }
+
+          update_mode = nil
+          has_access_key = patch_args.key? :access
+          other_keys_exist = (patch_args.keys - [:access]).any?
+
+          if has_access_key && !other_keys_exist
+            update_mode = UpdateMode::UPDATE_ACL
+          elsif !has_access_key && other_keys_exist
+            update_mode = UpdateMode::UPDATE_METADATA
+          elsif has_access_key && other_keys_exist
+            update_mode = UpdateMode::FULL
+          end
+
           patch_gapi = Google::Apis::BigqueryV2::Dataset.new(**patch_args)
           patch_gapi.etag = etag if etag
-          @gapi = service.patch_dataset dataset_id, patch_gapi
+          @gapi = service.patch_dataset dataset_id, patch_gapi, access_policy_version: @access_policy_version,
+update_mode: update_mode
         end
 
         ##
@@ -2921,7 +3155,7 @@ module Google
           end
         end
 
-        def load_job_gapi table_id, dryrun, job_id: nil, prefix: nil
+        def load_job_gapi table_id, dryrun, job_id: nil, prefix: nil, reservation: nil
           job_ref = service.job_ref_from job_id, prefix
           Google::Apis::BigqueryV2::Job.new(
             job_reference: job_ref,
@@ -2933,43 +3167,63 @@ module Google
                   table_id:   table_id
                 )
               ),
-              dry_run: dryrun
+              dry_run: dryrun,
+              reservation: reservation
             )
           )
         end
 
         def load_job_csv_options! job, jagged_rows: nil, quoted_newlines: nil, delimiter: nil, quote: nil,
-                                  skip_leading: nil, null_marker: nil
+                                  skip_leading: nil, null_marker: nil, null_markers: nil, source_column_match: nil,
+                                  preserve_ascii_control_characters: nil
           job.jagged_rows = jagged_rows unless jagged_rows.nil?
           job.quoted_newlines = quoted_newlines unless quoted_newlines.nil?
           job.delimiter = delimiter unless delimiter.nil?
           job.null_marker = null_marker unless null_marker.nil?
           job.quote = quote unless quote.nil?
           job.skip_leading = skip_leading unless skip_leading.nil?
+          job.null_markers = null_markers unless null_markers.nil?
+          job.source_column_match = source_column_match unless source_column_match.nil?
+          job.preserve_ascii_control_characters = preserve_ascii_control_characters unless
+            preserve_ascii_control_characters.nil?
         end
 
         def load_job_file_options! job, format: nil, projection_fields: nil, jagged_rows: nil, quoted_newlines: nil,
                                    encoding: nil, delimiter: nil, ignore_unknown: nil, max_bad_records: nil, quote: nil,
-                                   skip_leading: nil, null_marker: nil
+                                   skip_leading: nil, null_marker: nil, date_format: nil, datetime_format: nil,
+                                   time_format: nil, timestamp_format: nil, null_markers: nil, source_column_match: nil,
+                                   time_zone: nil, reference_file_schema_uri: nil,
+                                   preserve_ascii_control_characters: nil
           job.format = format unless format.nil?
           job.projection_fields = projection_fields unless projection_fields.nil?
           job.encoding = encoding unless encoding.nil?
           job.ignore_unknown = ignore_unknown unless ignore_unknown.nil?
           job.max_bad_records = max_bad_records unless max_bad_records.nil?
+          job.date_format = date_format unless date_format.nil?
+          job.datetime_format = datetime_format unless datetime_format.nil?
+          job.time_format = time_format unless time_format.nil?
+          job.timestamp_format = timestamp_format unless timestamp_format.nil?
+          job.time_zone = time_zone unless time_zone.nil?
+          job.reference_file_schema_uri = reference_file_schema_uri unless reference_file_schema_uri.nil?
           load_job_csv_options! job, jagged_rows:     jagged_rows,
                                      quoted_newlines: quoted_newlines,
                                      delimiter:       delimiter,
                                      quote:           quote,
                                      skip_leading:    skip_leading,
-                                     null_marker:     null_marker
+                                     null_marker:     null_marker,
+                                     null_markers:    null_markers,
+                                     source_column_match: source_column_match,
+                                     preserve_ascii_control_characters: preserve_ascii_control_characters
         end
 
         def load_job_updater table_id, format: nil, create: nil, write: nil, projection_fields: nil, jagged_rows: nil,
                              quoted_newlines: nil, encoding: nil, delimiter: nil, ignore_unknown: nil,
                              max_bad_records: nil, quote: nil, skip_leading: nil, dryrun: nil, schema: nil, job_id: nil,
                              prefix: nil, labels: nil, autodetect: nil, null_marker: nil, create_session: nil,
-                             session_id: nil
-          new_job = load_job_gapi table_id, dryrun, job_id: job_id, prefix: prefix
+                             session_id: nil, date_format: nil, datetime_format: nil, time_format: nil,
+                             timestamp_format: nil, null_markers: nil, source_column_match: nil, time_zone: nil,
+                             reference_file_schema_uri: nil, preserve_ascii_control_characters: nil, reservation: nil
+          new_job = load_job_gapi table_id, dryrun, job_id: job_id, prefix: prefix, reservation: reservation
           LoadJob::Updater.new(new_job).tap do |job|
             job.location = location if location # may be dataset reference
             job.create = create unless create.nil?
@@ -2989,7 +3243,16 @@ module Google
                                         max_bad_records:   max_bad_records,
                                         quote:             quote,
                                         skip_leading:      skip_leading,
-                                        null_marker:       null_marker
+                                        null_marker:       null_marker,
+                                        date_format:       date_format,
+                                        datetime_format:  datetime_format,
+                                        time_format:       time_format,
+                                        timestamp_format:  timestamp_format,
+                                        null_markers:      null_markers,
+                                        source_column_match: source_column_match,
+                                        time_zone:         time_zone,
+                                        reference_file_schema_uri: reference_file_schema_uri,
+                                        preserve_ascii_control_characters: preserve_ascii_control_characters
           end
         end