google-cloud-bigquery 1.52.1 → 1.61.1

data/lib/google/cloud/bigquery/external/csv_source.rb

@@ -367,6 +367,224 @@ module Google
             @gapi.csv_options.skip_leading_rows = row_count
           end
 
+          ##
+          # Specifies a string that represents a null value in a CSV file. For
+          # example, if you specify `\N`, BigQuery interprets `\N` as a null value when
+          # querying a CSV file. The default value is the empty string. If you set this
+          # property to a custom value, BigQuery throws an error if an empty string is
+          # present for all data types except for STRING and BYTE. For STRING and BYTE
+          # columns, BigQuery interprets the empty string as an empty value.
+          #
+          # @return [String, nil] The null marker string. `nil` if not set.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   csv_url = "gs://bucket/path/to/data.csv"
+          #   csv_table = bigquery.external csv_url do |csv|
+          #     csv.null_marker = "\N"
+          #   end
+          #
+          #   csv_table.null_marker #=> "\N"
+          #
+          def null_marker
+            @gapi.csv_options.null_marker
+          end
+
+          ##
+          # Sets a string that represents a null value in a CSV file. For
+          # example, if you specify `\N`, BigQuery interprets `\N` as a null value when
+          # querying a CSV file. The default value is the empty string. If you set this
+          # property to a custom value, BigQuery throws an error if an empty string is
+          # present for all data types except for STRING and BYTE. For STRING and BYTE
+          # columns, BigQuery interprets the empty string as an empty value.
+          #
+          # @param [String, nil] null_marker The null marker string. `nil` to unset.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   csv_url = "gs://bucket/path/to/data.csv"
+          #   csv_table = bigquery.external csv_url do |csv|
+          #     csv.null_marker = "\N"
+          #   end
+          #
+          #   csv_table.null_marker #=> "\N"
+          #
+          def null_marker= null_marker
+            frozen_check!
+            @gapi.csv_options.null_marker = null_marker
+          end
+
+          ##
+          # The list of strings represented as SQL NULL value in a CSV file.
+          # null_marker and null_markers can't be set at the same time. If null_marker is
+          # set, null_markers has to be not set. If null_markers is set, null_marker has
+          # to be not set. If both null_marker and null_markers are set at the same time,
+          # a user error would be thrown. Any strings listed in null_markers, including
+          # empty string would be interpreted as SQL NULL. This applies to all column
+          # types.
+          #
+          # @return [Array<String>] The array of null marker strings.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   csv_url = "gs://bucket/path/to/data.csv"
+          #   csv_table = bigquery.external csv_url do |csv|
+          #     csv.null_markers = ["\N", "NULL"]
+          #   end
+          #
+          #   csv_table.null_markers #=> ["\N", "NULL"]
+          #
+          def null_markers
+            @gapi.csv_options.null_markers || []
+          end
+
+          ##
+          # Sets the list of strings represented as SQL NULL value in a CSV file.
+          # null_marker and null_markers can't be set at the same time. If null_marker is
+          # set, null_markers has to be not set. If null_markers is set, null_marker has
+          # to be not set. If both null_marker and null_markers are set at the same time,
+          # a user error would be thrown. Any strings listed in null_markers, including
+          # empty string would be interpreted as SQL NULL. This applies to all column
+          # types.
+          #
+          # @param [Array<String>] null_markers The array of null marker strings.
+          #
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   csv_url = "gs://bucket/path/to/data.csv"
+          #   csv_table = bigquery.external csv_url do |csv|
+          #     csv.null_markers = ["\N", "NULL"]
+          #   end
+          #
+          #   csv_table.null_markers #=> ["\N", "NULL"]
+          #
+          def null_markers= null_markers
+            frozen_check!
+            @gapi.csv_options.null_markers = null_markers
+          end
+
+          # Controls the strategy used to match loaded columns to the schema.
+          # If not set, a sensible default is chosen based on how the schema is
+          # provided. If autodetect is used, then columns are matched by name.
+          # Otherwise, columns are matched by position. This is done to keep the
+          # behavior backward-compatible.
+          #
+          # Acceptable values are:
+          # - `POSITION`: matches by position. Assumes columns are ordered the
+          #   same way as the schema.
+          # - `NAME`: matches by name. Reads the header row as column names and
+          #   reorders columns to match the schema.
+          #
+          # @return [String, nil] The source column match value. `nil` if not set.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   csv_url = "gs://bucket/path/to/data.csv"
+          #   csv_table = bigquery.external csv_url do |csv|
+          #     csv.source_column_match = "NAME"
+          #   end
+          #
+          #   csv_table.source_column_match #=> "NAME"
+          #
+          def source_column_match
+            @gapi.csv_options.source_column_match
+          end
+
+          # Sets the strategy used to match loaded columns to the schema.
+          # If not set, a sensible default is chosen based on how the schema is
+          # provided. If autodetect is used, then columns are matched by name.
+          # Otherwise, columns are matched by position. This is done to keep the
+          # behavior backward-compatible. Optional.
+          #
+          # Acceptable values are:
+          # - `POSITION`: matches by position. Assumes columns are ordered the
+          #   same way as the schema.
+          # - `NAME`: matches by name. Reads the header row as column names and
+          #   reorders columns to match the schema.
+          #
+          # @param [String, nil] source_column_match The new source column match value. `nil` to unset.
+          #
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   csv_url = "gs://bucket/path/to/data.csv"
+          #   csv_table = bigquery.external csv_url do |csv|
+          #     csv.source_column_match = "NAME"
+          #   end
+          #
+          #   csv_table.source_column_match #=> "NAME"
+          #
+          def source_column_match= source_column_match
+            frozen_check!
+            @gapi.csv_options.source_column_match = source_column_match
+          end
+
+          # Indicates if the embedded ASCII control characters (the first 32
+          # characters in the ASCII-table, from `\x00` to `\x1F`) are preserved.
+          # By default, ASCII control characters are not preserved.
+          #
+          # @return [Boolean, nil] whether or not ASCII control characters are
+          #   preserved. `nil` if not set.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   csv_url = "gs://bucket/path/to/data.csv"
+          #   csv_table = bigquery.external csv_url do |csv|
+          #     csv.preserve_ascii_control_characters = true
+          #   end
+          #
+          #   csv_table.preserve_ascii_control_characters #=> true
+          #
+          def preserve_ascii_control_characters
+            @gapi.csv_options.preserve_ascii_control_characters
+          end
+
+          # Sets whether the embedded ASCII control characters (the first 32
+          # characters in the ASCII-table, from `\x00` to `\x1F`) are preserved.
+          # By default, ASCII control characters are not preserved.
+          #
+          # @param [Boolean, nil] val whether or not ASCII control characters
+          #    are preserved. `nil` to unset.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   csv_url = "gs://bucket/path/to/data.csv"
+          #   csv_table = bigquery.external csv_url do |csv|
+          #     csv.preserve_ascii_control_characters = true
+          #   end
+          #
+          #   csv_table.preserve_ascii_control_characters #=> true
+          #
+          def preserve_ascii_control_characters= val
+            frozen_check!
+            @gapi.csv_options.preserve_ascii_control_characters = val
+          end
+
           ##
           # The schema for the data.
           #

data/lib/google/cloud/bigquery/external/data_source.rb

@@ -504,6 +504,49 @@ module Google
             @gapi.max_bad_records = new_max_bad_records
           end
 
+          ##
+          # The URI of a reference file with the table schema. This is enabled
+          # for the following formats: `AVRO`, `PARQUET`, `ORC`.
+          #
+          # @return [String, nil] The URI. `nil` if not set.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   external_data = bigquery.external "gs://bucket/path/to/data.avro", format: :avro do |ext|
+          #     ext.reference_file_schema_uri = "gs://bucket/path/to/schema.json"
+          #   end
+          #
+          #   external_data.reference_file_schema_uri #=> "gs://bucket/path/to/schema.json"
+          #
+          def reference_file_schema_uri
+            @gapi.reference_file_schema_uri
+          end
+
+          ##
+          # Sets the URI of a reference file with the table schema. This is
+          # enabled for the following formats: `AVRO`, `PARQUET`, `ORC`.
+          #
+          # @param [String, nil] uri The URI. `nil` to unset.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   external_data = bigquery.external "gs://bucket/path/to/data.avro", format: :avro do |ext|
+          #     ext.reference_file_schema_uri = "gs://bucket/path/to/schema.json"
+          #   end
+          #
+          #   external_data.reference_file_schema_uri #=> "gs://bucket/path/to/schema.json"
+          #
+          def reference_file_schema_uri= uri
+            frozen_check!
+            @gapi.reference_file_schema_uri = uri
+          end
+
           ###
           # Checks if hive partitioning options are set.
           #
@@ -744,6 +787,227 @@ module Google
             @gapi.hive_partitioning_options.source_uri_prefix = source_uri_prefix
           end
 
+          ##
+          # Time zone used when parsing timestamp values that do not have specific
+          # time zone information (e.g. `2024-04-20 12:34:56`). The expected format
+          # is an IANA timezone string (e.g. `America/Los_Angeles`).
+          #
+          # @return [String, nil] The IANA time zone name. `nil` if not set.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   external_data = bigquery.external "gs://bucket/path/to/data.csv" do |ext|
+          #     ext.time_zone = "America/Los_Angeles"
+          #   end
+          #
+          #   external_data.time_zone #=> "America/Los_Angeles"
+          #
+          def time_zone
+            @gapi.time_zone
+          end
+
+          ##
+          # Sets the time zone used when parsing timestamp values that do not have
+          # specific time zone information (e.g. `2024-04-20 12:34:56`). The expected
+          # format is an IANA timezone string (e.g. `America/Los_Angeles`).
+          #
+          # @param [String, nil] time_zone The IANA time zone name. `nil` to unset.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   external_data = bigquery.external "gs://bucket/path/to/data.csv" do |ext|
+          #     ext.time_zone = "America/Los_Angeles"
+          #   end
+          #
+          #   external_data.time_zone #=> "America/Los_Angeles"
+          #
+          def time_zone= time_zone
+            frozen_check!
+            @gapi.time_zone = time_zone
+          end
+
+          ##
+          # Format used to parse TIME values. Supports SQL-style values. See
+          # [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
+          #
+          # @return [String, nil] The time format pattern. `nil` if not set.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   external_data = bigquery.external "gs://bucket/path/to/data.csv" do |ext|
+          #     ext.time_format = "HH24:MI:SS"
+          #   end
+          #
+          #   external_data.time_format #=> "HH24:MI:SS"
+          #
+          def time_format
+            @gapi.time_format
+          end
+
+          ##
+          # Sets the format used to parse TIME values. Supports SQL-style See
+          # [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
+          # values.
+          #
+          # @param [String, nil] time_format The time format pattern. `nil` if not set.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   external_data = bigquery.external "gs://bucket/path/to/data.csv" do |ext|
+          #     ext.time_format = "HH24:MI:SS"
+          #   end
+          #
+          #   external_data.time_format #=> "HH24:MI:SS"
+          #
+          def time_format= time_format
+            frozen_check!
+            @gapi.time_format = time_format
+          end
+
+          ##
+          # Format used to parse TIMESTAMP values. Supports SQL-style values. See
+          # [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
+          #
+          # @return [String, nil] The timestamp format pattern. `nil` if not set.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   external_data = bigquery.external "gs://bucket/path/to/data.csv" do |ext|
+          #     ext.timestamp_format = "YYYY-MM-DD HH24:MI:SS.FF3 TZH"
+          #   end
+          #
+          #   external_data.timestamp_format #=> "YYYY-MM-DD HH24:MI:SS.FF3 TZH"
+          #
+          def timestamp_format
+            @gapi.timestamp_format
+          end
+
+          ##
+          # Sets the format used to parse TIMESTAMP values. Supports SQL-style
+          # values. See
+          # [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
+          #
+          # @param [String, nil] timestamp_format The timestamp format pattern. `nil` to unset.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   external_data = bigquery.external "gs://bucket/path/to/data.csv" do |ext|
+          #     ext.timestamp_format = "YYYY-MM-DD HH24:MI:SS.FF3 TZH"
+          #   end
+          #
+          #   external_data.timestamp_format #=> "YYYY-MM-DD HH24:MI:SS.FF3 TZH"
+          #
+          def timestamp_format= timestamp_format
+            frozen_check!
+            @gapi.timestamp_format = timestamp_format
+          end
+
+          ##
+          # Format used to parse DATETIME values. Supports SQL-style values. See
+          # [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
+          #
+          # @return [String, nil] The datetime format pattern. `nil` if not set.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   external_data = bigquery.external "gs://bucket/path/to/data.csv" do |ext|
+          #     ext.datetime_format = "YYYY-MM-DD HH24:MI:SS"
+          #   end
+          #
+          #   external_data.datetime_format #=> "YYYY-MM-DD HH24:MI:SS"
+          #
+          def datetime_format
+            @gapi.datetime_format
+          end
+
+          ##
+          # Sets the format used to parse DATETIME values. Supports SQL-style
+          # values. See
+          # [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
+          #
+          # @param [String, nil] datetime_format The datetime format pattern. `nil` to unset.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   external_data = bigquery.external "gs://bucket/path/to/data.csv" do |ext|
+          #     ext.datetime_format = "YYYY-MM-DD HH24:MI:SS"
+          #   end
+          #
+          #   external_data.datetime_format #=> "YYYY-MM-DD HH24:MI:SS"
+          #
+          def datetime_format= datetime_format
+            frozen_check!
+            @gapi.datetime_format = datetime_format
+          end
+
+          ##
+          # Format used to parse DATE values. Supports SQL-style values. See
+          # [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
+          #
+          # @return [String, nil] The date format pattern. `nil` if not set.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   external_data = bigquery.external "gs://bucket/path/to/data.csv" do |ext|
+          #     ext.date_format = "YYYY-MM-DD"
+          #   end
+          #
+          #   external_data.date_format #=> "YYYY-MM-DD"
+          #
+          def date_format
+            @gapi.date_format
+          end
+
+          ##
+          # Sets the format used to parse DATE values. Supports SQL-style
+          # values. See
+          # [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
+          #
+          # @param [String, nil] date_format The date format pattern. `nil` to unset.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #
+          #   external_data = bigquery.external "gs://bucket/path/to/data.csv" do |ext|
+          #     ext.date_format = "YYYY-MM-DD"
+          #   end
+          #
+          #   external_data.date_format #=> "YYYY-MM-DD"
+          #
+          def date_format= date_format
+            frozen_check!
+            @gapi.date_format = date_format
+          end
+
           ##
           # @private Google API Client object.
           def to_gapi

data/lib/google/cloud/bigquery/extract_job.rb

@@ -280,7 +280,8 @@ module Google
               job_reference: job_ref,
               configuration: Google::Apis::BigqueryV2::JobConfiguration.new(
                 extract: extract_config,
-                dry_run: options[:dryrun]
+                dry_run: options[:dryrun],
+                reservation: options[:reservation]
               )
             )
 
@@ -438,6 +439,18 @@ module Google
             @gapi.configuration.extract.use_avro_logical_types = value
           end
 
+          ##
+          # Sets the reservation that job would use. User can specify a reservation
+          # to execute the job. If reservation is not set, reservation is determined
+          # based on the rules defined by the reservation assignments. The expected
+          # format is `projects/`project`/locations/`location`/reservations/`reservation``.
+          # @param [String] value The reservation name.
+          #
+          # @!group Attributes
+          def reservation= value
+            @gapi.configuration.update! reservation: value
+          end
+
           def cancel
             raise "not implemented in #{self.class}"
           end

data/lib/google/cloud/bigquery/job.rb

@@ -327,8 +327,8 @@ module Google
         end
 
         ##
-        # The last error for the job, if any errors have occurred. Returns a
-        # hash.
+        # Output only. Final error result of the job. If present, indicates that
+        # the job has completed and was unsuccessful.
         #
         # @see https://cloud.google.com/bigquery/docs/reference/v2/jobs Jobs API
         #   reference
@@ -346,8 +346,10 @@ module Google
         end
 
         ##
-        # The errors for the job, if any errors have occurred. Returns an array
-        # of hash objects. See {#error}.
+        # Output only. The first errors encountered during the running of the
+        # job. The final message includes the number of errors that caused the
+        # process to stop. Errors here do not necessarily mean that the job has
+        # not completed or was unsuccessful.
         #
         # @return [Array<Hash>, nil] Returns an array of hashes containing
         #   `reason` and `message` keys: