google-cloud-bigquery 1.14.0 → 1.42.0

data/lib/google/cloud/bigquery/argument.rb

@@ -0,0 +1,197 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "google/cloud/bigquery/standard_sql"
+
+module Google
+  module Cloud
+    module Bigquery
+      ##
+      # # Argument
+      #
+      # Input/output argument of a function or a stored procedure. See {Routine}.
+      #
+      # @example
+      #   require "google/cloud/bigquery"
+      #
+      #   bigquery = Google::Cloud::Bigquery.new
+      #   dataset = bigquery.dataset "my_dataset"
+      #   routine = dataset.create_routine "my_routine" do |r|
+      #     r.routine_type = "SCALAR_FUNCTION"
+      #     r.language = :SQL
+      #     r.body = "(SELECT SUM(IF(elem.name = \"foo\",elem.val,null)) FROM UNNEST(arr) AS elem)"
+      #     r.arguments = [
+      #       Google::Cloud::Bigquery::Argument.new(
+      #         name: "arr",
+      #         argument_kind: "FIXED_TYPE",
+      #         data_type: Google::Cloud::Bigquery::StandardSql::DataType.new(
+      #           type_kind: "ARRAY",
+      #           array_element_type: Google::Cloud::Bigquery::StandardSql::DataType.new(
+      #             type_kind: "STRUCT",
+      #             struct_type: Google::Cloud::Bigquery::StandardSql::StructType.new(
+      #               fields: [
+      #                 Google::Cloud::Bigquery::StandardSql::Field.new(
+      #                   name: "name",
+      #                   type: Google::Cloud::Bigquery::StandardSql::DataType.new(type_kind: "STRING")
+      #                 ),
+      #                 Google::Cloud::Bigquery::StandardSql::Field.new(
+      #                   name: "val",
+      #                   type: Google::Cloud::Bigquery::StandardSql::DataType.new(type_kind: "INT64")
+      #                 )
+      #               ]
+      #             )
+      #           )
+      #         )
+      #       )
+      #     ]
+      #   end
+      #
+      class Argument
+        ##
+        # Creates a new, immutable Argument object.
+        #
+        # @overload initialize(data_type, kind, mode, name)
+        #   @param [StandardSql::DataType, String] data_type The data type of the argument. Required unless
+        #     {#argument_kind} is `ANY_TYPE`.
+        #   @param [String] argument_kind The kind of argument. Optional. Defaults to `FIXED_TYPE`.
+        #
+        #     * `FIXED_TYPE` - The argument is a variable with fully specified type, which can be a struct or an array,
+        #       but not a table.
+        #     * `ANY_TYPE` - The argument is any type, including struct or array, but not a table.
+        #
+        #     To be added: `FIXED_TABLE`, `ANY_TABLE`.
+        #   @param [String] mode Specifies whether the argument is input or output. Optional. Can be set for procedures
+        #     only.
+        #
+        #     * IN - The argument is input-only.
+        #     * OUT - The argument is output-only.
+        #     * INOUT - The argument is both an input and an output.
+        #   @param [String] name The name of the argument. Optional. Can be absent for a function return argument.
+        #
+        def initialize **kwargs
+          kwargs[:data_type] = StandardSql::DataType.gapi_from_string_or_data_type kwargs[:data_type]
+          @gapi = Google::Apis::BigqueryV2::Argument.new(**kwargs)
+        end
+
+        ##
+        # The data type of the argument. Required unless {#argument_kind} is `ANY_TYPE`.
+        #
+        # @return [StandardSql::DataType] The data type.
+        #
+        def data_type
+          StandardSql::DataType.from_gapi @gapi.data_type
+        end
+
+        ##
+        # The kind of argument. Optional. Defaults to `FIXED_TYPE`.
+        #
+        # * `FIXED_TYPE` - The argument is a variable with fully specified type, which can be a struct or an array, but
+        #   not a table.
+        # * `ANY_TYPE` - The argument is any type, including struct or array, but not a table.
+        #
+        # To be added: `FIXED_TABLE`, `ANY_TABLE`.
+        #
+        # @return [String] The upper case kind of argument.
+        #
+        def argument_kind
+          @gapi.argument_kind
+        end
+
+        ##
+        # Checks if the value of {#argument_kind} is `FIXED_TYPE`. The default is `true`.
+        #
+        # @return [Boolean] `true` when `FIXED_TYPE`, `false` otherwise.
+        #
+        def fixed_type?
+          return true if @gapi.argument_kind.nil?
+          @gapi.argument_kind == "FIXED_TYPE"
+        end
+
+        ##
+        # Checks if the value of {#argument_kind} is `ANY_TYPE`. The default is `false`.
+        #
+        # @return [Boolean] `true` when `ANY_TYPE`, `false` otherwise.
+        #
+        def any_type?
+          @gapi.argument_kind == "ANY_TYPE"
+        end
+
+        ##
+        # Specifies whether the argument is input or output. Optional. Can be set for procedures only.
+        #
+        # * IN - The argument is input-only.
+        # * OUT - The argument is output-only.
+        # * INOUT - The argument is both an input and an output.
+        #
+        # @return [String] The upper case input/output mode of the argument.
+        #
+        def mode
+          @gapi.mode
+        end
+
+        ##
+        # Checks if the value of {#mode} is `IN`. Can be set for procedures only. The default is `false`.
+        #
+        # @return [Boolean] `true` when `IN`, `false` otherwise.
+        #
+        def in?
+          @gapi.mode == "IN"
+        end
+
+        ##
+        # Checks if the value of {#mode} is `OUT`. Can be set for procedures only. The default is `false`.
+        #
+        # @return [Boolean] `true` when `OUT`, `false` otherwise.
+        #
+        def out?
+          @gapi.mode == "OUT"
+        end
+
+        ##
+        # Checks if the value of {#mode} is `INOUT`. Can be set for procedures only. The default is `false`.
+        #
+        # @return [Boolean] `true` when `INOUT`, `false` otherwise.
+        #
+        def inout?
+          @gapi.mode == "INOUT"
+        end
+
+        ##
+        #
+        # The name of the argument. Optional. Can be absent for a function return argument.
+        #
+        # @return [String] The name of the argument.
+        #
+        def name
+          @gapi.name
+        end
+
+        ##
+        # @private
+        def to_gapi
+          @gapi
+        end
+
+        ##
+        # @private New Argument from a Google API Client object.
+        def self.from_gapi gapi
+          new.tap do |a|
+            a.instance_variable_set :@gapi, gapi
+          end
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/bigquery/convert.rb

@@ -23,30 +23,29 @@ require "date"
 module Google
   module Cloud
     module Bigquery
-      # rubocop:disable Metrics/ModuleLength
-
       ##
       # @private
       #
-      # Internal conversion of raw data values to/from Bigquery values
+      # Internal conversion of raw data values to/from BigQuery values
+      #
+      #   | BigQuery     | Ruby                                 | Notes                                              |
+      #   |--------------|--------------------------------------|----------------------------------------------------|
+      #   | `BOOL`       | `true`/`false`                       |                                                    |
+      #   | `INT64`      | `Integer`                            |                                                    |
+      #   | `FLOAT64`    | `Float`                              |                                                    |
+      #   | `NUMERIC`    | `BigDecimal`                         | `BigDecimal` values will be rounded to scale 9.    |
+      #   | `BIGNUMERIC` | converted to `BigDecimal`            | Pass data as `String`; map query params in `types`.|
+      #   | `STRING`     | `String`                             |                                                    |
+      #   | `DATETIME`   | `DateTime`                           | `DATETIME` does not support time zone.             |
+      #   | `DATE`       | `Date`                               |                                                    |
+      #   | `GEOGRAPHY`  | `String`                             |                                                    |
+      #   | `TIMESTAMP`  | `Time`                               |                                                    |
+      #   | `TIME`       | `Google::Cloud::BigQuery::Time`      |                                                    |
+      #   | `BYTES`      | `File`, `IO`, `StringIO`, or similar |                                                    |
+      #   | `ARRAY`      | `Array`                              | Nested arrays, `nil` values are not supported.     |
+      #   | `STRUCT`     | `Hash`                               | Hash keys may be strings or symbols.               |
       #
-      # | BigQuery    | Ruby           | Notes  |
-      # |-------------|----------------|---|
-      # | `BOOL`      | `true`/`false` | |
-      # | `INT64`     | `Integer`      | |
-      # | `FLOAT64`   | `Float`        | |
-      # | `NUMERIC`   | `BigDecimal`   | Will be rounded to 9 decimal places |
-      # | `STRING`    | `String`       | |
-      # | `DATETIME`  | `DateTime`  | `DATETIME` does not support time zone. |
-      # | `DATE`      | `Date`         | |
-      # | `TIMESTAMP` | `Time`         | |
-      # | `TIME`      | `Google::Cloud::BigQuery::Time` | |
-      # | `BYTES`     | `File`, `IO`, `StringIO`, or similar | |
-      # | `ARRAY` | `Array` | Nested arrays, `nil` values are not supported. |
-      # | `STRUCT`    | `Hash`        | Hash keys may be strings or symbols. |
       module Convert
-        ##
-        # @private
         def self.format_rows rows, fields
           Array(rows).map do |row|
             # convert TableRow to hash to handle nested TableCell values
@@ -54,13 +53,10 @@ module Google
           end
         end
 
-        ##
-        # @private
         def self.format_row row, fields
-          row_pairs = fields.zip(row[:f]).map do |f, v|
+          fields.zip(row[:f]).to_h do |f, v|
             [f.name.to_sym, format_value(v, f)]
           end
-          Hash[row_pairs]
         end
 
         # rubocop:disable all
@@ -75,11 +71,7 @@ module Google
           elsif Array === value[:v]
             value[:v].map { |v| format_value v, field }
           elsif Hash === value[:v]
-            if value[:v].empty?
-              nil
-            else
-              format_row value[:v], field.fields
-            end
+            format_row value[:v], field.fields
           elsif field.type == "STRING"
             String value[:v]
           elsif field.type == "INTEGER"
@@ -96,6 +88,8 @@ module Google
             end
           elsif field.type == "NUMERIC"
             BigDecimal value[:v]
+          elsif field.type == "BIGNUMERIC"
+            BigDecimal value[:v]
           elsif field.type == "BOOLEAN"
             (value[:v] == "true" ? true : (value[:v] == "false" ? false : nil))
           elsif field.type == "BYTES"
@@ -108,135 +102,118 @@ module Google
             ::Time.parse("#{value[:v]} UTC").to_datetime
           elsif field.type == "DATE"
             Date.parse value[:v]
+          elsif field.type == "GEOGRAPHY"
+            String value[:v]
           else
             value[:v]
           end
         end
 
-        ##
-        # @private
-        def self.to_query_param value
-          if TrueClass === value
-            return Google::Apis::BigqueryV2::QueryParameter.new(
-              parameter_type:  Google::Apis::BigqueryV2::QueryParameterType.new(
-                type: "BOOL"),
-              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
-                value: true)
-            )
-          elsif FalseClass === value
-            return Google::Apis::BigqueryV2::QueryParameter.new(
-              parameter_type:  Google::Apis::BigqueryV2::QueryParameterType.new(
-                type: "BOOL"),
-              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
-                value: false)
-            )
-          elsif Integer === value
-            return Google::Apis::BigqueryV2::QueryParameter.new(
-              parameter_type:  Google::Apis::BigqueryV2::QueryParameterType.new(
-                type: "INT64"),
-              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
-                value: value)
-            )
-          elsif Float === value
-            return Google::Apis::BigqueryV2::QueryParameter.new(
-              parameter_type:  Google::Apis::BigqueryV2::QueryParameterType.new(
-                type: "FLOAT64"),
-              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
-                value: value)
-            )
-          elsif BigDecimal === value
-            # Round to precision of 9
-            value_str = value.finite? ? value.round(9).to_s("F") : value.to_s
-            return Google::Apis::BigqueryV2::QueryParameter.new(
-              parameter_type:  Google::Apis::BigqueryV2::QueryParameterType.new(
-                type: "NUMERIC"),
-              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
-                value: value_str)
-            )
-          elsif String === value
-            return Google::Apis::BigqueryV2::QueryParameter.new(
-              parameter_type:  Google::Apis::BigqueryV2::QueryParameterType.new(
-                type: "STRING"),
-              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
-                value: value)
-            )
-          elsif DateTime === value
-            return Google::Apis::BigqueryV2::QueryParameter.new(
-              parameter_type:  Google::Apis::BigqueryV2::QueryParameterType.new(
-                type: "DATETIME"),
-              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
-                value: value.strftime("%Y-%m-%d %H:%M:%S.%6N"))
-            )
-          elsif Date === value
-            return Google::Apis::BigqueryV2::QueryParameter.new(
-              parameter_type:  Google::Apis::BigqueryV2::QueryParameterType.new(
-                type: "DATE"),
-              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
-                value: value.to_s)
-            )
-          elsif ::Time === value
-            return Google::Apis::BigqueryV2::QueryParameter.new(
-              parameter_type:  Google::Apis::BigqueryV2::QueryParameterType.new(
-                type: "TIMESTAMP"),
-              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
-                value: value.strftime("%Y-%m-%d %H:%M:%S.%6N%:z"))
-            )
-          elsif Bigquery::Time === value
-            return Google::Apis::BigqueryV2::QueryParameter.new(
-              parameter_type:  Google::Apis::BigqueryV2::QueryParameterType.new(
-              type: "TIME"),
-              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
-                value: value.value)
-            )
-          elsif value.respond_to?(:read) && value.respond_to?(:rewind)
-            value.rewind
-            return Google::Apis::BigqueryV2::QueryParameter.new(
-              parameter_type:  Google::Apis::BigqueryV2::QueryParameterType.new(
-                type: "BYTES"),
-              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
-                value: Base64.strict_encode64(
-                  value.read.force_encoding("ASCII-8BIT")))
-            )
-          elsif Array === value
-            array_params = value.map { |param| Convert.to_query_param param }
-            return Google::Apis::BigqueryV2::QueryParameter.new(
-              parameter_type: Google::Apis::BigqueryV2::QueryParameterType.new(
-                type: "ARRAY",
-                array_type: array_params.first.parameter_type
-              ),
-              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
-                array_values: array_params.map(&:parameter_value)
-              )
-            )
-          elsif Hash === value
-            struct_pairs = value.map do |name, param|
-              struct_param = Convert.to_query_param param
-              [Google::Apis::BigqueryV2::QueryParameterType::StructType.new(
-                name: String(name),
-                type: struct_param.parameter_type
-              ), struct_param.parameter_value]
+        # rubocop:enable all
+
+        def self.to_query_param_value value, type = nil
+          return Google::Apis::BigqueryV2::QueryParameterValue.new value: nil if value.nil?
+
+          json_value = to_json_value value, type
+
+          case json_value
+          when Array
+            type = extract_array_type type
+            array_values = json_value.map { |v| to_query_param_value v, type }
+            Google::Apis::BigqueryV2::QueryParameterValue.new array_values: array_values
+          when Hash
+            struct_values = json_value.to_h do |k, v|
+              [String(k), to_query_param_value(v, type)]
             end
-            struct_values = Hash[struct_pairs.map do |type, value|
-              [type.name.to_sym, value]
-            end]
+            Google::Apis::BigqueryV2::QueryParameterValue.new struct_values: struct_values
+          else
+            # Everything else is converted to a string, per the API expectations.
+            Google::Apis::BigqueryV2::QueryParameterValue.new value: json_value.to_s
+          end
+        end
 
-            return Google::Apis::BigqueryV2::QueryParameter.new(
-              parameter_type: Google::Apis::BigqueryV2::QueryParameterType.new(
-                type: "STRUCT",
-                struct_types: struct_pairs.map(&:first)
-              ),
-              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
-                struct_values: struct_values
-              )
+        def self.to_query_param_type type
+          case type
+          when Array
+            Google::Apis::BigqueryV2::QueryParameterType.new(
+              type: "ARRAY".freeze,
+              array_type: to_query_param_type(type.first)
+            )
+          when Hash
+            Google::Apis::BigqueryV2::QueryParameterType.new(
+              type: "STRUCT".freeze,
+              struct_types: type.map do |key, val|
+                Google::Apis::BigqueryV2::QueryParameterType::StructType.new(
+                  name: String(key),
+                  type: to_query_param_type(val)
+                )
+              end
             )
           else
-            raise "A query parameter of type #{value.class} is not supported."
+            Google::Apis::BigqueryV2::QueryParameterType.new type: type.to_s.freeze
           end
         end
 
-        ##
-        # @private
-        def self.to_json_value value
+        # rubocop:disable Lint/DuplicateBranch
+        # rubocop:disable Metrics/CyclomaticComplexity
+        # rubocop:disable Style/GuardClause
+
+        def self.default_query_param_type_for param
+          raise ArgumentError, "nil params are not supported, must assign optional type" if param.nil?
+
+          case param
+          when String
+            :STRING
+          when Symbol
+            :STRING
+          when TrueClass
+            :BOOL
+          when FalseClass
+            :BOOL
+          when Integer
+            :INT64
+          when BigDecimal
+            :NUMERIC
+          when Numeric
+            :FLOAT64
+          when ::Time
+            :TIMESTAMP
+          when Bigquery::Time
+            :TIME
+          when DateTime
+            :DATETIME
+          when Date
+            :DATE
+          when Array
+            if param.empty?
+              raise ArgumentError, "Cannot determine type for empty array values"
+            end
+            non_nil_values = param.compact.map { |p| default_query_param_type_for p }.compact
+            if non_nil_values.empty?
+              raise ArgumentError, "Cannot determine type for array of nil values"
+            end
+            if non_nil_values.uniq.count > 1
+              raise ArgumentError, "Cannot determine type for array of different types of values"
+            end
+            [non_nil_values.first]
+          when Hash
+            param.transform_values do |value|
+              default_query_param_type_for value
+            end
+          else
+            if param.respond_to?(:read) && param.respond_to?(:rewind)
+              :BYTES
+            else
+              raise "A query parameter of type #{param.class} is not supported"
+            end
+          end
+        end
+
+        # rubocop:enable Lint/DuplicateBranch
+        # rubocop:enable Metrics/CyclomaticComplexity
+        # rubocop:enable Style/GuardClause
+
+        def self.to_json_value value, type = nil
           if DateTime === value
             value.strftime "%Y-%m-%d %H:%M:%S.%6N"
           elsif Date === value
@@ -245,30 +222,49 @@ module Google
             value.strftime "%Y-%m-%d %H:%M:%S.%6N%:z"
           elsif Bigquery::Time === value
             value.value
+          elsif BigDecimal === value
+            if value.finite?
+              # Round to precision of 9 unless explicit `BIGNUMERIC`
+              bigdecimal = type == :BIGNUMERIC ? value : value.round(9)
+              bigdecimal.to_s "F"
+            else
+              value.to_s
+            end
           elsif value.respond_to?(:read) && value.respond_to?(:rewind)
             value.rewind
-            Base64.strict_encode64(value.read.force_encoding("ASCII-8BIT"))
+            Base64.strict_encode64 value.read.force_encoding("ASCII-8BIT")
           elsif Array === value
-            value.map { |v| to_json_value v }
+            type = extract_array_type type
+            value.map { |x| to_json_value x, type }
           elsif Hash === value
-            Hash[value.map { |k, v| [k.to_s, to_json_value(v)] }]
+            value.to_h { |k, v| [k.to_s, to_json_value(v, type)] }
           else
             value
           end
         end
 
-        # rubocop:enable all
+        def self.to_query_param param, type = nil
+          type ||= default_query_param_type_for param
 
-        ##
-        # @private
-        def self.to_json_rows rows
-          rows.map { |row| to_json_row row }
+          Google::Apis::BigqueryV2::QueryParameter.new(
+            parameter_type:  to_query_param_type(type),
+            parameter_value: to_query_param_value(param, type)
+          )
         end
 
         ##
-        # @private
+        # Lists are specified by providing the type code in an array. For example, an array of integers are specified as
+        # `[:INT64]`. Extracts the symbol/hash.
+        def self.extract_array_type type
+          return nil if type.nil?
+          unless type.is_a?(Array) && type.count == 1 && (type.first.is_a?(Symbol) || type.first.is_a?(Hash))
+            raise ArgumentError, "types Array #{type.inspect} should include only a single symbol or hash element."
+          end
+          type.first
+        end
+
         def self.to_json_row row
-          Hash[row.map { |k, v| [k.to_s, to_json_value(v)] }]
+          row.to_h { |k, v| [k.to_s, to_json_value(v)] }
         end
 
         def self.resolve_legacy_sql standard_sql, legacy_sql
@@ -278,8 +274,6 @@ module Google
         end
 
         ##
-        # @private
-        #
         # Converts create disposition strings to API values.
         #
         # @return [String] API representation of create disposition.
@@ -298,8 +292,6 @@ module Google
         end
 
         ##
-        # @private
-        #
         # Converts write disposition strings to API values.
         #
         # @return [String] API representation of write disposition.
@@ -320,8 +312,6 @@ module Google
         end
 
         ##
-        # @private
-        #
         # Converts source format strings to API values.
         #
         # @return [String] API representation of source format.
@@ -335,15 +325,15 @@ module Google
             "parquet"                => "PARQUET",
             "datastore"              => "DATASTORE_BACKUP",
             "backup"                 => "DATASTORE_BACKUP",
-            "datastore_backup"       => "DATASTORE_BACKUP"
+            "datastore_backup"       => "DATASTORE_BACKUP",
+            "ml_tf_saved_model"      => "ML_TF_SAVED_MODEL",
+            "ml_xgboost_booster"     => "ML_XGBOOST_BOOSTER"
           }[format.to_s.downcase]
           return val unless val.nil?
           format
         end
 
         ##
-        # @private
-        #
         # Converts file paths into source format by extension.
         #
         # @return [String] API representation of source format.
@@ -354,8 +344,6 @@ module Google
         end
 
         ##
-        # @private
-        #
         # Converts file path into source format by extension.
         #
         # @return [String] API representation of source format.
@@ -370,8 +358,6 @@ module Google
         end
 
         ##
-        # @private
-        #
         # Converts a primitive time value in milliseconds to a Ruby Time object.
         #
         # @return [Time, nil] The Ruby Time object, or nil if the given argument
@@ -382,19 +368,15 @@ module Google
         end
 
         ##
-        # @private
-        #
         # Converts a Ruby Time object to a primitive time value in milliseconds.
         #
         # @return [Integer, nil] The primitive time value in milliseconds, or
         #   nil if the given argument is nil.
         def self.time_to_millis time_obj
           return nil unless time_obj
-          (time_obj.to_i * 1000) + (time_obj.nsec / 1000000)
+          (time_obj.to_i * 1000) + (time_obj.nsec / 1_000_000)
         end
       end
-
-      # rubocop:enable Metrics/ModuleLength
     end
   end
 end