google-cloud-bigquery 1.25.0 → 1.30.0

data/lib/google/cloud/bigquery/extract_job.rb

@@ -103,8 +103,7 @@ module Google
         #   table extraction.
         def compression?
           return false unless table?
-          val = @gapi.configuration.extract.compression
-          val == "GZIP"
+          @gapi.configuration.extract.compression == "GZIP"
         end
 
         ##
@@ -117,8 +116,7 @@ module Google
         #
         def json?
           return false unless table?
-          val = @gapi.configuration.extract.destination_format
-          val == "NEWLINE_DELIMITED_JSON"
+          @gapi.configuration.extract.destination_format == "NEWLINE_DELIMITED_JSON"
         end
 
         ##
@@ -146,8 +144,7 @@ module Google
         #
         def avro?
           return false unless table?
-          val = @gapi.configuration.extract.destination_format
-          val == "AVRO"
+          @gapi.configuration.extract.destination_format == "AVRO"
         end
 
         ##
@@ -173,8 +170,7 @@ module Google
         #
         def ml_xgboost_booster?
           return false unless model?
-          val = @gapi.configuration.extract.destination_format
-          val == "ML_XGBOOST_BOOSTER"
+          @gapi.configuration.extract.destination_format == "ML_XGBOOST_BOOSTER"
         end
 
         ##
@@ -250,6 +246,7 @@ module Google
           ##
           # @private Create an Updater object.
           def initialize gapi
+            super()
             @gapi = gapi
           end
 
@@ -267,9 +264,10 @@ module Google
             extract_config = Google::Apis::BigqueryV2::JobConfigurationExtract.new(
               destination_uris: Array(storage_urls)
             )
-            if source.is_a? Google::Apis::BigqueryV2::TableReference
+            case source
+            when Google::Apis::BigqueryV2::TableReference
               extract_config.source_table = source
-            elsif source.is_a? Google::Apis::BigqueryV2::ModelReference
+            when Google::Apis::BigqueryV2::ModelReference
               extract_config.source_model = source
             end
             job = Google::Apis::BigqueryV2::Job.new(

data/lib/google/cloud/bigquery/job.rb

@@ -215,6 +215,17 @@ module Google
           @gapi.statistics.parent_job_id
         end
 
+        ##
+        # An array containing the job resource usage breakdown by reservation, if present. Reservation usage statistics
+        # are only reported for jobs that are executed within reservations.  On-demand jobs do not report this data.
+        #
+        # @return [Array<Google::Cloud::Bigquery::Job::ReservationUsage>, nil] The reservation usage, if present.
+        #
+        def reservation_usage
+          return nil unless @gapi.statistics.reservation_usage
+          Array(@gapi.statistics.reservation_usage).map { |g| ReservationUsage.from_gapi g }
+        end
+
         ##
         # The statistics including stack frames for a child job of a script.
         #
@@ -489,6 +500,30 @@ module Google
           end
         end
 
+        ##
+        # Represents Job resource usage breakdown by reservation.
+        #
+        # @attr_reader [String] name The reservation name or "unreserved" for on-demand resources usage.
+        # @attr_reader [Fixnum] slot_ms The slot-milliseconds the job spent in the given reservation.
+        #
+        class ReservationUsage
+          attr_reader :name
+          attr_reader :slot_ms
+
+          ##
+          # @private Creates a new ReservationUsage instance.
+          def initialize name, slot_ms
+            @name = name
+            @slot_ms = slot_ms
+          end
+
+          ##
+          # @private New ReservationUsage from a statistics.reservation_usage value.
+          def self.from_gapi gapi
+            new gapi.name, gapi.slot_ms
+          end
+        end
+
         ##
         # Represents statistics for a child job of a script.
         #
@@ -537,7 +572,8 @@ module Google
         #   end
         #
         class ScriptStatistics
-          attr_reader :evaluation_kind, :stack_frames
+          attr_reader :evaluation_kind
+          attr_reader :stack_frames
 
           ##
           # @private Creates a new ScriptStatistics instance.
@@ -547,7 +583,7 @@ module Google
           end
 
           ##
-          # @private New ScriptStatistics from a statistics.script_statistics object.
+          # @private New ScriptStatistics from a statistics.script_statistics value.
           def self.from_gapi gapi
             frames = Array(gapi.stack_frames).map { |g| ScriptStackFrame.from_gapi g }
             new gapi.evaluation_kind, frames
@@ -602,7 +638,11 @@ module Google
         #   end
         #
         class ScriptStackFrame
-          attr_reader :start_line, :start_column, :end_line, :end_column, :text
+          attr_reader :start_line
+          attr_reader :start_column
+          attr_reader :end_line
+          attr_reader :end_column
+          attr_reader :text
 
           ##
           # @private Creates a new ScriptStackFrame instance.

data/lib/google/cloud/bigquery/job/list.rb

@@ -72,8 +72,8 @@ module Google
             return nil unless next?
             ensure_service!
             next_kwargs = @kwargs.merge token: token
-            next_gapi = @service.list_jobs next_kwargs
-            self.class.from_gapi next_gapi, @service, next_kwargs
+            next_gapi = @service.list_jobs(**next_kwargs)
+            self.class.from_gapi next_gapi, @service, **next_kwargs
           end
 
           ##
@@ -121,12 +121,12 @@ module Google
           #     puts job.state
           #   end
           #
-          def all request_limit: nil
+          def all request_limit: nil, &block
             request_limit = request_limit.to_i if request_limit
             return enum_for :all, request_limit: request_limit unless block_given?
             results = self
             loop do
-              results.each { |r| yield r }
+              results.each(&block)
               if request_limit
                 request_limit -= 1
                 break if request_limit.negative?

data/lib/google/cloud/bigquery/load_job.rb

@@ -37,8 +37,8 @@ module Google
       #   bigquery = Google::Cloud::Bigquery.new
       #   dataset = bigquery.dataset "my_dataset"
       #
-      #   gs_url = "gs://my-bucket/file-name.csv"
-      #   load_job = dataset.load_job "my_new_table", gs_url do |schema|
+      #   gcs_uri = "gs://my-bucket/file-name.csv"
+      #   load_job = dataset.load_job "my_new_table", gcs_uri do |schema|
       #     schema.string "first_name", mode: :required
       #     schema.record "cities_lived", mode: :repeated do |nested_schema|
       #       nested_schema.string "place", mode: :required
@@ -112,8 +112,7 @@ module Google
         #   `false` otherwise.
         #
         def iso8859_1?
-          val = @gapi.configuration.load.encoding
-          val == "ISO-8859-1"
+          @gapi.configuration.load.encoding == "ISO-8859-1"
         end
 
         ##
@@ -195,8 +194,7 @@ module Google
         #   `NEWLINE_DELIMITED_JSON`, `false` otherwise.
         #
         def json?
-          val = @gapi.configuration.load.source_format
-          val == "NEWLINE_DELIMITED_JSON"
+          @gapi.configuration.load.source_format == "NEWLINE_DELIMITED_JSON"
         end
 
         ##
@@ -218,8 +216,27 @@ module Google
         #   `false` otherwise.
         #
         def backup?
-          val = @gapi.configuration.load.source_format
-          val == "DATASTORE_BACKUP"
+          @gapi.configuration.load.source_format == "DATASTORE_BACKUP"
+        end
+
+        ##
+        # Checks if the source format is ORC.
+        #
+        # @return [Boolean] `true` when the source format is `ORC`,
+        #   `false` otherwise.
+        #
+        def orc?
+          @gapi.configuration.load.source_format == "ORC"
+        end
+
+        ##
+        # Checks if the source format is Parquet.
+        #
+        # @return [Boolean] `true` when the source format is `PARQUET`,
+        #   `false` otherwise.
+        #
+        def parquet?
+          @gapi.configuration.load.source_format == "PARQUET"
         end
 
         ##
@@ -347,6 +364,58 @@ module Google
           nil
         end
 
+        ###
+        # Checks if hive partitioning options are set.
+        #
+        # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
+        #
+        # @return [Boolean] `true` when hive partitioning options are set, or `false` otherwise.
+        #
+        # @!group Attributes
+        #
+        def hive_partitioning?
+          !@gapi.configuration.load.hive_partitioning_options.nil?
+        end
+
+        ###
+        # The mode of hive partitioning to use when reading data. The following modes are supported:
+        #
+        #   1. `AUTO`: automatically infer partition key name(s) and type(s).
+        #   2. `STRINGS`: automatically infer partition key name(s). All types are interpreted as strings.
+        #   3. `CUSTOM`: partition key schema is encoded in the source URI prefix.
+        #
+        # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
+        #
+        # @return [String, nil] The mode of hive partitioning, or `nil` if not set.
+        #
+        # @!group Attributes
+        #
+        def hive_partitioning_mode
+          @gapi.configuration.load.hive_partitioning_options.mode if hive_partitioning?
+        end
+
+        ###
+        # The common prefix for all source uris when hive partition detection is requested. The prefix must end
+        # immediately before the partition key encoding begins. For example, consider files following this data layout:
+        #
+        # ```
+        # gs://bucket/path_to_table/dt=2019-01-01/country=BR/id=7/file.avro
+        # gs://bucket/path_to_table/dt=2018-12-31/country=CA/id=3/file.avro
+        # ```
+        #
+        # When hive partitioning is requested with either `AUTO` or `STRINGS` mode, the common prefix can be either of
+        # `gs://bucket/path_to_table` or `gs://bucket/path_to_table/` (trailing slash does not matter).
+        #
+        # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
+        #
+        # @return [String, nil] The common prefix for all source uris, or `nil` if not set.
+        #
+        # @!group Attributes
+        #
+        def hive_partitioning_source_uri_prefix
+          @gapi.configuration.load.hive_partitioning_options.source_uri_prefix if hive_partitioning?
+        end
+
         ###
         # Checks if the destination table will be range partitioned. See [Creating and using integer range partitioned
         # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
@@ -537,6 +606,7 @@ module Google
           ##
           # @private Create an Updater object.
           def initialize gapi
+            super()
             @updates = []
             @gapi = gapi
             @schema = nil
@@ -703,9 +773,18 @@ module Google
           end
 
           ##
-          # Adds a numeric number field to the schema. Numeric is a
-          # fixed-precision numeric type with 38 decimal digits, 9 that follow
-          # the decimal point.
+          # Adds a numeric number field to the schema. `NUMERIC` is a decimal
+          # type with fixed precision and scale. Precision is the number of
+          # digits that the number contains. Scale is how many of these
+          # digits appear after the decimal point. It supports:
+          #
+          # Precision: 38
+          # Scale: 9
+          # Min: -9.9999999999999999999999999999999999999E+28
+          # Max: 9.9999999999999999999999999999999999999E+28
+          #
+          # This type can represent decimal fractions exactly, and is suitable
+          # for financial calculations.
           #
           # See {Schema#numeric}
           #
@@ -732,6 +811,45 @@ module Google
             schema.numeric name, description: description, mode: mode
           end
 
+          ##
+          # Adds a bignumeric number field to the schema. `BIGNUMERIC` is a
+          # decimal type with fixed precision and scale. Precision is the
+          # number of digits that the number contains. Scale is how many of
+          # these digits appear after the decimal point. It supports:
+          #
+          # Precision: 76.76 (the 77th digit is partial)
+          # Scale: 38
+          # Min: -5.7896044618658097711785492504343953926634992332820282019728792003956564819968E+38
+          # Max: 5.7896044618658097711785492504343953926634992332820282019728792003956564819967E+38
+          #
+          # This type can represent decimal fractions exactly, and is suitable
+          # for financial calculations.
+          #
+          # See {Schema#bignumeric}
+          #
+          # @param [String] name The field name. The name must contain only
+          #   letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
+          #   start with a letter or underscore. The maximum length is 128
+          #   characters.
+          # @param [String] description A description of the field.
+          # @param [Symbol] mode The field's mode. The possible values are
+          #   `:nullable`, `:required`, and `:repeated`. The default value is
+          #   `:nullable`.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #   job = dataset.load_job "my_table", "gs://abc/file" do |schema|
+          #     schema.bignumeric "total_cost", mode: :required
+          #   end
+          #
+          # @!group Schema
+          def bignumeric name, description: nil, mode: :nullable
+            schema.bignumeric name, description: description, mode: mode
+          end
+
           ##
           # Adds a boolean field to the schema.
           #
@@ -1326,6 +1444,89 @@ module Google
             @gapi.configuration.update! labels: val
           end
 
+          ##
+          # Sets the mode of hive partitioning to use when reading data. The following modes are supported:
+          #
+          #   1. `auto`: automatically infer partition key name(s) and type(s).
+          #   2. `strings`: automatically infer partition key name(s). All types are interpreted as strings.
+          #   3. `custom`: partition key schema is encoded in the source URI prefix.
+          #
+          # Not all storage formats support hive partitioning. Requesting hive partitioning on an unsupported format
+          # will lead to an error. Currently supported types include: `avro`, `csv`, `json`, `orc` and `parquet`.
+          #
+          # See {#format=} and {#hive_partitioning_source_uri_prefix=}.
+          #
+          # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
+          #
+          # @param [String, Symbol] mode The mode of hive partitioning to use when reading data.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
+          #     job.format = :parquet
+          #     job.hive_partitioning_mode = :auto
+          #     job.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #
+          #   load_job.wait_until_done!
+          #   load_job.done? #=> true
+          #
+          # @!group Attributes
+          #
+          def hive_partitioning_mode= mode
+            @gapi.configuration.load.hive_partitioning_options ||= Google::Apis::BigqueryV2::HivePartitioningOptions.new
+            @gapi.configuration.load.hive_partitioning_options.mode = mode.to_s.upcase
+          end
+
+          ##
+          # Sets the common prefix for all source uris when hive partition detection is requested. The prefix must end
+          # immediately before the partition key encoding begins. For example, consider files following this data
+          # layout:
+          #
+          # ```
+          # gs://bucket/path_to_table/dt=2019-01-01/country=BR/id=7/file.avro
+          # gs://bucket/path_to_table/dt=2018-12-31/country=CA/id=3/file.avro
+          # ```
+          #
+          # When hive partitioning is requested with either `AUTO` or `STRINGS` mode, the common prefix can be either of
+          # `gs://bucket/path_to_table` or `gs://bucket/path_to_table/` (trailing slash does not matter).
+          #
+          # See {#hive_partitioning_mode=}.
+          #
+          # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
+          #
+          # @param [String] source_uri_prefix The common prefix for all source uris.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #
+          #   gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
+          #   source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
+          #     job.format = :parquet
+          #     job.hive_partitioning_mode = :auto
+          #     job.hive_partitioning_source_uri_prefix = source_uri_prefix
+          #   end
+          #
+          #   load_job.wait_until_done!
+          #   load_job.done? #=> true
+          #
+          # @!group Attributes
+          #
+          def hive_partitioning_source_uri_prefix= source_uri_prefix
+            @gapi.configuration.load.hive_partitioning_options ||= Google::Apis::BigqueryV2::HivePartitioningOptions.new
+            @gapi.configuration.load.hive_partitioning_options.source_uri_prefix = source_uri_prefix
+          end
+
           ##
           # Sets the field on which to range partition the table. See [Creating and using integer range partitioned
           # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
@@ -1345,8 +1546,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.schema do |schema|
           #       schema.integer "my_table_id", mode: :required
           #       schema.string "my_table_data", mode: :required
@@ -1386,8 +1587,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.schema do |schema|
           #       schema.integer "my_table_id", mode: :required
           #       schema.string "my_table_data", mode: :required
@@ -1427,8 +1628,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.schema do |schema|
           #       schema.integer "my_table_id", mode: :required
           #       schema.string "my_table_data", mode: :required
@@ -1468,8 +1669,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.schema do |schema|
           #       schema.integer "my_table_id", mode: :required
           #       schema.string "my_table_data", mode: :required
@@ -1510,8 +1711,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.time_partitioning_type = "DAY"
           #   end
           #
@@ -1549,8 +1750,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.time_partitioning_type  = "DAY"
           #     job.time_partitioning_field = "dob"
           #     job.schema do |schema|
@@ -1585,8 +1786,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.time_partitioning_type = "DAY"
           #     job.time_partitioning_expiration = 86_400
           #   end
@@ -1645,8 +1846,8 @@ module Google
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
           #
-          #   gs_url = "gs://my-bucket/file-name.csv"
-          #   load_job = dataset.load_job "my_new_table", gs_url do |job|
+          #   gcs_uri = "gs://my-bucket/file-name.csv"
+          #   load_job = dataset.load_job "my_new_table", gcs_uri do |job|
           #     job.time_partitioning_type  = "DAY"
           #     job.time_partitioning_field = "dob"
           #     job.schema do |schema|