google-cloud-bigquery 0.23.0 → 0.24.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b8c37453c0300ca5c4bddd798fb4e34dbb0afeb7
-  data.tar.gz: f9a9842d943e303c92e8f90a95403d05d8f5ba73
+  metadata.gz: e8cba429d574299afb4e2a230f071e13417c954c
+  data.tar.gz: ca363404d3ec1d319ca4aa3738170a924c3f20fb
 SHA512:
-  metadata.gz: e0046330e19fede666ad09614981e0010787c52b59a1cf5675bf994e04b08f64033ac565789d180e34c26f59d5cd72e8d35e4fe722a42407ad92625fa2887280
-  data.tar.gz: 2989d1c9d3cc203561c26181b8549eb25ffadbff7a0aa941a4ece3a2b628778949b29225999193b46efcf3b92c9f6afb7b2a7322d1af1a6ce3d742dd23e3d6ab
+  metadata.gz: c4aa08940ba60adcc8058e4c1787227d0b40314db0c92e1c40a01dada7a5a4d973311430591d3e786dced51ce7ae65bb554220041ef6c36a5cfc27e930b866b4
+  data.tar.gz: 8ce5bc5f122cb1b90f617ed6c334df9ad7eade436e7727771ff4bad4687e4d712a97c0d4f64db4cafcb55a93e0ee72bc869f58387ac846816968e06f37a09b4b

data/lib/google/cloud/bigquery.rb

@@ -75,7 +75,7 @@ module Google
     # dataset = bigquery.dataset "samples"
     # table = dataset.table "shakespeare"
     #
-    # table.headers #=> ["word", "word_count", "corpus", "corpus_date"]
+    # table.headers #=> [:word, :word_count, :corpus, :corpus_date]
     # table.rows_count #=> 164656
     # ```
     #
@@ -95,33 +95,12 @@ module Google
     # explained in [Querying
     # Data](https://cloud.google.com/bigquery/querying-data).
     #
-    # ### Legacy SQL (formerly BigQuery SQL)
-    #
-    # Before version 2.0, BigQuery executed queries using a non-standard SQL
-    # dialect known as BigQuery SQL. This variant is still the default, and will
-    # be used unless you pass the flag `standard_sql: true` with your query.
-    # (If you get an SQL syntax error with a query that may be written in
-    # standard SQL, be sure that you are passing this option.)
-    #
-    # ```ruby
-    # require "google/cloud/bigquery"
-    #
-    # bigquery = Google::Cloud::Bigquery.new
-    #
-    # sql = "SELECT TOP(word, 50) as word, COUNT(*) as count " \
-    #       "FROM [publicdata:samples.shakespeare]"
-    # data = bigquery.query sql
-    # ```
-    #
-    # Notice that in legacy SQL, a fully-qualified table name uses the following
-    # format: `[my-dashed-project:dataset1.tableName]`.
-    #
     # ### Standard SQL
     #
     # Standard SQL is the preferred SQL dialect for querying data stored in
     # BigQuery. It is compliant with the SQL 2011 standard, and has extensions
-    # that support querying nested and repeated data. It has several advantages
-    # over legacy SQL, including:
+    # that support querying nested and repeated data. This is the default
+    # syntax. It has several advantages over Legacy SQL, including:
     #
     # * Composability using `WITH` clauses and SQL functions
     # * Subqueries in the `SELECT` list and `WHERE` clause
@@ -136,8 +115,7 @@ module Google
     # For examples that demonstrate some of these features, see [Standard SQL
     # highlights](https://cloud.google.com/bigquery/docs/reference/standard-sql/migrating-from-legacy-sql#standard_sql_highlights).
     #
-    # Legacy SQL is still the default. To use standard SQL instead, pass the
-    # option `standard_sql: true` with your query.
+    # Standard SQL is the default.
     #
     # ```ruby
     # require "google/cloud/bigquery"
@@ -147,12 +125,35 @@ module Google
     # sql = "SELECT word, SUM(word_count) AS word_count " \
     #       "FROM `bigquery-public-data.samples.shakespeare`" \
     #       "WHERE word IN ('me', 'I', 'you') GROUP BY word"
-    # data = bigquery.query sql, standard_sql: true
+    # data = bigquery.query sql
+    # ```
+    #
+    # Notice that in standard SQL, a fully-qualified table name uses the
+    # following format: <code>`my-dashed-project.dataset1.tableName`</code>.
+    #
+    # ### Legacy SQL (formerly BigQuery SQL)
+    #
+    # Before version 2.0, BigQuery executed queries using a non-standard SQL
+    # dialect known as BigQuery SQL. This variant is optional, and can be
+    # enabled by passing the flag `legacy_sql: true` with your query. (If you
+    # get an SQL syntax error with a query that may be written in standard SQL,
+    # be sure that you are passing this option.)
+    #
+    # To use legacy SQL, pass the option `legacy_sql: true` with your query.
+    #
+    # ```ruby
+    # require "google/cloud/bigquery"
+    #
+    # bigquery = Google::Cloud::Bigquery.new
+    #
+    # sql = "SELECT TOP(word, 50) as word, COUNT(*) as count " \
+    #       "FROM [publicdata:samples.shakespeare]"
+    # data = bigquery.query sql, legacy_sql: true
     # ```
     #
-    # Notice that in standard SQL, the format for a fully-qualified table name
-    # uses back-ticks instead of brackets, and a dot instead of a semi-colon:
-    # <code>`my-dashed-project.dataset1.tableName`</code>.
+    # Notice that in legacy SQL, a fully-qualified table name uses brackets
+    # instead of back-ticks, and a semi-colon instead of a dot to separate the
+    # project and the dataset: `[my-dashed-project:dataset1.tableName]`.
     #
     # #### Query parameters
     #
@@ -166,7 +167,7 @@ module Google
     #
     # sql = "SELECT word, SUM(word_count) AS word_count " \
     #       "FROM `bigquery-public-data.samples.shakespeare`" \
-    #       "WHERE word IN (@words) GROUP BY word"
+    #       "WHERE word IN UNNEST(@words) GROUP BY word"
     # data = bigquery.query sql, params: { words: ['me', 'I', 'you'] }
     # ```
     #
@@ -244,7 +245,7 @@ module Google
     # job.wait_until_done!
     # if !job.failed?
     #   job.query_results.each do |row|
-    #     puts row["word"]
+    #     puts row[:word]
     #   end
     # end
     # ```

data/lib/google/cloud/bigquery/convert.rb

@@ -0,0 +1,258 @@
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "google/apis/bigquery_v2"
+require "stringio"
+require "base64"
+require "time"
+require "date"
+
+module Google
+  module Cloud
+    module Bigquery
+      # rubocop:disable all
+
+      ##
+      # @private
+      #
+      # Internal conversion of raw data values to/from Bigquery values
+      #
+      # | BigQuery    | Ruby           | Notes  |
+      # |-------------|----------------|---|
+      # | `BOOL`      | `true`/`false` | |
+      # | `INT64`     | `Integer`      | |
+      # | `FLOAT64`   | `Float`        | |
+      # | `STRING`    | `STRING`       | |
+      # | `DATETIME`  | `DateTime`  | `DATETIME` does not support time zone. |
+      # | `DATE`      | `Date`         | |
+      # | `TIMESTAMP` | `Time`         | |
+      # | `TIME`      | `Google::Cloud::BigQuery::Time` | |
+      # | `BYTES`     | `File`, `IO`, `StringIO`, or similar | |
+      # | `ARRAY` | `Array` | Nested arrays, `nil` values are not supported. |
+      # | `STRUCT`    | `Hash`        | Hash keys may be strings or symbols. |
+
+      module Convert
+        ##
+        # @private
+        def self.format_rows rows, fields
+          Array(rows).map do |row|
+            # convert TableRow to hash to handle nested TableCell values
+            format_row row.to_h, fields
+          end
+        end
+
+        ##
+        # @private
+        def self.format_row row, fields
+          row_pairs = fields.zip(row[:f]).map do |f, v|
+            [f.name.to_sym, format_value(v, f)]
+          end
+          Hash[row_pairs]
+        end
+
+        def self.format_value value, field
+          if value.nil?
+            nil
+          elsif value.empty?
+            nil
+          elsif value[:v].nil?
+            nil
+          elsif Array === value[:v]
+            value[:v].map { |v| format_value v, field }
+          elsif Hash === value[:v]
+            if value[:v].empty?
+              nil
+            else
+              format_row value[:v], field.fields
+            end
+          elsif field.type == "STRING"
+            String value[:v]
+          elsif field.type == "INTEGER"
+            Integer value[:v]
+          elsif field.type == "FLOAT"
+            Float value[:v]
+          elsif field.type == "BOOLEAN"
+            (value[:v] == "true" ? true : (value[:v] == "false" ? false : nil))
+          elsif field.type == "BYTES"
+            StringIO.new Base64.decode64 value[:v]
+          elsif field.type == "TIMESTAMP"
+            ::Time.at Float(value[:v])
+          elsif field.type == "TIME"
+            Bigquery::Time.new value[:v]
+          elsif field.type == "DATETIME"
+            ::Time.parse("#{value[:v]} UTC").to_datetime
+          elsif field.type == "DATE"
+            Date.parse value[:v]
+          else
+            value[:v]
+          end
+        end
+
+        ##
+        # @private
+        def self.to_query_param value
+          if TrueClass === value
+            return Google::Apis::BigqueryV2::QueryParameter.new(
+              parameter_type:  Google::Apis::BigqueryV2::QueryParameterType.new(
+                type: "BOOL"),
+              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
+                value: true)
+            )
+          elsif FalseClass === value
+            return Google::Apis::BigqueryV2::QueryParameter.new(
+              parameter_type:  Google::Apis::BigqueryV2::QueryParameterType.new(
+                type: "BOOL"),
+              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
+                value: false)
+            )
+          elsif Integer === value
+            return Google::Apis::BigqueryV2::QueryParameter.new(
+              parameter_type:  Google::Apis::BigqueryV2::QueryParameterType.new(
+                type: "INT64"),
+              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
+                value: value)
+            )
+          elsif Float === value
+            return Google::Apis::BigqueryV2::QueryParameter.new(
+              parameter_type:  Google::Apis::BigqueryV2::QueryParameterType.new(
+                type: "FLOAT64"),
+              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
+                value: value)
+            )
+          elsif String === value
+            return Google::Apis::BigqueryV2::QueryParameter.new(
+              parameter_type:  Google::Apis::BigqueryV2::QueryParameterType.new(
+                type: "STRING"),
+              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
+                value: value)
+            )
+          elsif DateTime === value
+            return Google::Apis::BigqueryV2::QueryParameter.new(
+              parameter_type:  Google::Apis::BigqueryV2::QueryParameterType.new(
+                type: "DATETIME"),
+              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
+                value: value.strftime("%Y-%m-%d %H:%M:%S.%6N"))
+            )
+          elsif Date === value
+            return Google::Apis::BigqueryV2::QueryParameter.new(
+              parameter_type:  Google::Apis::BigqueryV2::QueryParameterType.new(
+                type: "DATE"),
+              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
+                value: value.to_s)
+            )
+          elsif ::Time === value
+            return Google::Apis::BigqueryV2::QueryParameter.new(
+              parameter_type:  Google::Apis::BigqueryV2::QueryParameterType.new(
+                type: "TIMESTAMP"),
+              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
+                value: value.strftime("%Y-%m-%d %H:%M:%S.%6N%:z"))
+            )
+          elsif Bigquery::Time === value
+            return Google::Apis::BigqueryV2::QueryParameter.new(
+              parameter_type:  Google::Apis::BigqueryV2::QueryParameterType.new(
+              type: "TIME"),
+              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
+                value: value.value)
+            )
+          elsif value.respond_to?(:read) && value.respond_to?(:rewind)
+            value.rewind
+            return Google::Apis::BigqueryV2::QueryParameter.new(
+              parameter_type:  Google::Apis::BigqueryV2::QueryParameterType.new(
+                type: "BYTES"),
+              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
+                value: Base64.strict_encode64(
+                  value.read.force_encoding("ASCII-8BIT")))
+            )
+          elsif Array === value
+            array_params = value.map { |param| Convert.to_query_param param }
+            return Google::Apis::BigqueryV2::QueryParameter.new(
+              parameter_type: Google::Apis::BigqueryV2::QueryParameterType.new(
+                type: "ARRAY",
+                array_type: array_params.first.parameter_type
+              ),
+              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
+                array_values: array_params.map(&:parameter_value)
+              )
+            )
+          elsif Hash === value
+            struct_pairs = value.map do |name, param|
+              struct_param = Convert.to_query_param param
+              [Google::Apis::BigqueryV2::QueryParameterType::StructType.new(
+                name: String(name),
+                type: struct_param.parameter_type
+              ), struct_param.parameter_value]
+            end
+            struct_values = Hash[struct_pairs.map do |type, value|
+              [type.name.to_sym, value]
+            end]
+
+            return Google::Apis::BigqueryV2::QueryParameter.new(
+              parameter_type: Google::Apis::BigqueryV2::QueryParameterType.new(
+                type: "STRUCT",
+                struct_types: struct_pairs.map(&:first)
+              ),
+              parameter_value: Google::Apis::BigqueryV2::QueryParameterValue.new(
+                struct_values: struct_values
+              )
+            )
+          else
+            fail "A query parameter of type #{value.class} is not supported."
+          end
+        end
+
+        ##
+        # @private
+        def self.to_json_rows rows
+          rows.map { |row| to_json_row row }
+        end
+        ##
+        # @private
+        def self.to_json_row row
+          Hash[row.map { |k, v| [k.to_s, to_json_value(v)] }]
+        end
+        ##
+        # @private
+        def self.to_json_value value
+          if DateTime === value
+            value.strftime "%Y-%m-%d %H:%M:%S.%6N"
+          elsif Date === value
+            value.to_s
+          elsif ::Time === value
+            value.strftime "%Y-%m-%d %H:%M:%S.%6N%:z"
+          elsif Bigquery::Time === value
+            value.value
+          elsif value.respond_to?(:read) && value.respond_to?(:rewind)
+            value.rewind
+            Base64.strict_encode64(value.read.force_encoding("ASCII-8BIT"))
+          elsif Array === value
+            value.map { |v| to_json_value v }
+          elsif Hash === value
+            Hash[value.map { |k, v| [k.to_s, to_json_value(v)] }]
+          else
+            value
+          end
+        end
+
+        def self.resolve_legacy_sql standard_sql, legacy_sql
+          return !standard_sql unless standard_sql.nil?
+          return legacy_sql unless legacy_sql.nil?
+          false
+        end
+
+        # rubocop:enable all
+      end
+    end
+  end
+end

data/lib/google/cloud/bigquery/data.rb

@@ -65,6 +65,24 @@ module Google
           nil
         end
 
+        ##
+        # The schema of the data.
+        def schema
+          table.schema
+        end
+
+        ##
+        # The fields of the data.
+        def fields
+          schema.fields
+        end
+
+        ##
+        # The name of the columns in the data.
+        def headers
+          schema.headers
+        end
+
         ##
         # Whether there is a next page of data.
         #
@@ -177,18 +195,11 @@ module Google
           end
         end
 
-        ##
-        # Represents Table Data as a list of positional values (array of
-        # arrays). No type conversion is made, e.g. numbers are formatted as
-        # strings.
-        def raw
-          Array(gapi.rows).map { |row| row.f.map(&:v) }
-        end
-
         ##
         # @private New Data from a response object.
         def self.from_gapi gapi, table
-          formatted_rows = format_rows gapi.rows, table.fields
+          formatted_rows = Convert.format_rows(gapi.rows,
+                                               table.gapi.schema.fields)
 
           data = new formatted_rows
           data.table = table
@@ -196,41 +207,6 @@ module Google
           data
         end
 
-        # rubocop:disable all
-        # Disabled rubocop because this implementation will not last.
-
-        def self.format_rows rows, fields
-          headers = Array(fields).map { |f| f.name }
-          field_types = Array(fields).map { |f| f.type }
-
-          Array(rows).map do |row|
-            values = row.f.map { |f| f.v }
-            formatted_values = format_values field_types, values
-            Hash[headers.zip formatted_values]
-          end
-        end
-
-        def self.format_values field_types, values
-          field_types.zip(values).map do |type, value|
-            begin
-              if value.nil?
-                nil
-              elsif type == "INTEGER"
-                Integer value
-              elsif type == "FLOAT"
-                Float value
-              elsif type == "BOOLEAN"
-                (value == "true" ? true : (value == "false" ? false : nil))
-              else
-                value
-              end
-            rescue
-              value
-            end
-          end
-        end
-        # rubocop:enable all
-
         protected
 
         ##