google-cloud-bigquery 0.20.0

data/lib/google/cloud/bigquery/query_data.rb

@@ -0,0 +1,238 @@
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "google/cloud/bigquery/service"
+require "google/cloud/bigquery/data"
+
+module Google
+  module Cloud
+    module Bigquery
+      ##
+      # # QueryData
+      #
+      # Represents Data returned from a query a a list of name/value pairs.
+      class QueryData < Data
+        ##
+        # @private The Service object.
+        attr_accessor :service
+
+        # @private
+        def initialize arr = []
+          @job = nil
+          super
+        end
+
+        # The total number of bytes processed for this query.
+        def total_bytes
+          Integer @gapi.total_bytes_processed
+        rescue
+          nil
+        end
+
+        # Whether the query has completed or not. When data is present this will
+        # always be `true`. When `false`, `total` will not be available.
+        def complete?
+          @gapi.job_complete
+        end
+
+        # Whether the query result was fetched from the query cache.
+        def cache_hit?
+          @gapi.cache_hit
+        end
+
+        ##
+        # The schema of the data.
+        def schema
+          Schema.from_gapi(@gapi.schema).freeze
+        end
+
+        ##
+        # The fields of the data.
+        def fields
+          f = schema.fields
+          f = f.to_hash if f.respond_to? :to_hash
+          f = [] if f.nil?
+          f
+        end
+
+        ##
+        # The name of the columns in the data.
+        def headers
+          fields.map(&:name)
+        end
+
+        ##
+        # Whether there is a next page of query data.
+        #
+        # @return [Boolean]
+        #
+        # @example
+        #   require "google/cloud"
+        #
+        #   gcloud = Google::Cloud.new
+        #   bigquery = gcloud.bigquery
+        #   job = bigquery.job "my_job"
+        #
+        #   data = job.query_results
+        #   if data.next?
+        #     next_data = data.next
+        #   end
+        #
+        def next?
+          !token.nil?
+        end
+
+        ##
+        # Retrieve the next page of query data.
+        #
+        # @return [QueryData]
+        #
+        # @example
+        #   require "google/cloud"
+        #
+        #   gcloud = Google::Cloud.new
+        #   bigquery = gcloud.bigquery
+        #   job = bigquery.job "my_job"
+        #
+        #   data = job.query_results
+        #   if data.next?
+        #     next_data = data.next
+        #   end
+        #
+        def next
+          return nil unless next?
+          ensure_service!
+          gapi = service.job_query_results job_id, token: token
+          QueryData.from_gapi gapi, service
+        end
+
+        ##
+        # Retrieves all rows by repeatedly loading {#next} until {#next?}
+        # returns `false`. Calls the given block once for each row, which is
+        # passed as the parameter.
+        #
+        # An Enumerator is returned if no block is given.
+        #
+        # This method may make several API calls until all rows are retrieved.
+        # Be sure to use as narrow a search criteria as possible. Please use
+        # with caution.
+        #
+        # @param [Integer] request_limit The upper limit of API requests to make
+        #   to load all data. Default is no limit.
+        # @yield [row] The block for accessing each row of data.
+        # @yieldparam [Hash] row The row object.
+        #
+        # @return [Enumerator]
+        #
+        # @example Iterating each row by passing a block:
+        #   require "google/cloud"
+        #
+        #   gcloud = Google::Cloud.new
+        #   bigquery = gcloud.bigquery
+        #   job = bigquery.job "my_job"
+        #
+        #   data = job.query_results
+        #   data.all do |row|
+        #     puts row["word"]
+        #   end
+        #
+        # @example Using the enumerator by not passing a block:
+        #   require "google/cloud"
+        #
+        #   gcloud = Google::Cloud.new
+        #   bigquery = gcloud.bigquery
+        #   job = bigquery.job "my_job"
+        #
+        #   data = job.query_results
+        #   words = data.all.map do |row|
+        #     row["word"]
+        #   end
+        #
+        # @example Limit the number of API calls made:
+        #   require "google/cloud"
+        #
+        #   gcloud = Google::Cloud.new
+        #   bigquery = gcloud.bigquery
+        #   job = bigquery.job "my_job"
+        #
+        #   data = job.query_results
+        #   data.all(request_limit: 10) do |row|
+        #     puts row["word"]
+        #   end
+        #
+        def all request_limit: nil
+          request_limit = request_limit.to_i if request_limit
+          unless block_given?
+            return enum_for(:all, request_limit: request_limit)
+          end
+          results = self
+          loop do
+            results.each { |r| yield r }
+            if request_limit
+              request_limit -= 1
+              break if request_limit < 0
+            end
+            break unless results.next?
+            results = results.next
+          end
+        end
+
+        ##
+        # The BigQuery {Job} that was created to run the query.
+        def job
+          return @job if @job
+          return nil unless job?
+          ensure_service!
+          gapi = service.get_job job_id
+          @job = Job.from_gapi gapi, service
+        rescue Google::Cloud::NotFoundError
+          nil
+        end
+
+        ##
+        # @private New Data from a response object.
+        def self.from_gapi gapi, service
+          if gapi.schema.nil?
+            formatted_rows = []
+          else
+            formatted_rows = format_rows gapi.rows,
+                                         gapi.schema.fields
+          end
+
+          data = new formatted_rows
+          data.gapi = gapi
+          data.service = service
+          data
+        end
+
+        protected
+
+        ##
+        # Raise an error unless an active connection is available.
+        def ensure_service!
+          fail "Must have active connection" unless service
+        end
+
+        def job?
+          @gapi.job_reference && @gapi.job_reference.job_id
+        end
+
+        def job_id
+          @gapi.job_reference.job_id
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/bigquery/query_job.rb

@@ -0,0 +1,139 @@
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "google/cloud/bigquery/service"
+
+module Google
+  module Cloud
+    module Bigquery
+      ##
+      # # QueryJob
+      #
+      # A {Job} subclass representing a query operation that may be performed
+      # on a {Table}. A QueryJob instance is created when you call
+      # {Project#query_job}, {Dataset#query_job}, or {View#data}.
+      #
+      # @see https://cloud.google.com/bigquery/querying-data Querying Data
+      # @see https://cloud.google.com/bigquery/docs/reference/v2/jobs Jobs API
+      #   reference
+      #
+      class QueryJob < Job
+        ##
+        # Checks if the priority for the query is `BATCH`.
+        def batch?
+          val = @gapi.configuration.query.priority
+          val == "BATCH"
+        end
+
+        ##
+        # Checks if the priority for the query is `INTERACTIVE`.
+        def interactive?
+          val = @gapi.configuration.query.priority
+          return true if val.nil?
+          val == "INTERACTIVE"
+        end
+
+        ##
+        # Checks if the the query job allows arbitrarily large results at a
+        # slight cost to performance.
+        def large_results?
+          val = @gapi.configuration.query.allow_large_results
+          return false if val.nil?
+          val
+        end
+
+        ##
+        # Checks if the query job looks for an existing result in the query
+        # cache. For more information, see [Query
+        # Caching](https://cloud.google.com/bigquery/querying-data#querycaching).
+        def cache?
+          val = @gapi.configuration.query.use_query_cache
+          return false if val.nil?
+          val
+        end
+
+        ##
+        # Checks if the query job flattens nested and repeated fields in the
+        # query results. The default is `true`. If the value is `false`,
+        # #large_results? should return `true`.
+        def flatten?
+          val = @gapi.configuration.query.flatten_results
+          return true if val.nil?
+          val
+        end
+
+        ##
+        # Checks if the query results are from the query cache.
+        def cache_hit?
+          @gapi.statistics.query.cache_hit
+        end
+
+        ##
+        # The number of bytes processed by the query.
+        def bytes_processed
+          Integer @gapi.statistics.query.total_bytes_processed
+        rescue
+          nil
+        end
+
+        ##
+        # The table in which the query results are stored.
+        def destination
+          table = @gapi.configuration.query.destination_table
+          return nil unless table
+          retrieve_table table.project_id,
+                         table.dataset_id,
+                         table.table_id
+        end
+
+        ##
+        # Retrieves the query results for the job.
+        #
+        # @param [String] token Page token, returned by a previous call,
+        #   identifying the result set.
+        # @param [Integer] max Maximum number of results to return.
+        # @param [Integer] start Zero-based index of the starting row to read.
+        # @param [Integer] timeout How long to wait for the query to complete,
+        #   in milliseconds, before returning. Default is 10,000 milliseconds
+        #   (10 seconds).
+        #
+        # @return [Google::Cloud::Bigquery::QueryData]
+        #
+        # @example
+        #   require "google/cloud"
+        #
+        #   gcloud = Google::Cloud.new
+        #   bigquery = gcloud.bigquery
+        #
+        #   q = "SELECT word FROM publicdata:samples.shakespeare"
+        #   job = bigquery.query_job q
+        #
+        #   job.wait_until_done!
+        #   data = job.query_results
+        #   data.each do |row|
+        #     puts row["word"]
+        #   end
+        #   data = data.next if data.next?
+        #
+        def query_results token: nil, max: nil, start: nil, timeout: nil
+          ensure_service!
+          options = { token: token, max: max, start: start, timeout: timeout }
+          gapi = service.job_query_results job_id, options
+          QueryData.from_gapi gapi, service
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/bigquery/schema.rb

@@ -0,0 +1,361 @@
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+module Google
+  module Cloud
+    module Bigquery
+      ##
+      # # Table Schema
+      #
+      # A builder for BigQuery table schemas, passed to block arguments to
+      # {Dataset#create_table} and {Table#schema}. Supports nested and
+      # repeated fields via a nested block.
+      #
+      # @see https://cloud.google.com/bigquery/preparing-data-for-bigquery
+      #   Preparing Data for BigQuery
+      #
+      # @example
+      #   require "google/cloud"
+      #
+      #   gcloud = Google::Cloud.new
+      #   bigquery = gcloud.bigquery
+      #   dataset = bigquery.dataset "my_dataset"
+      #   table = dataset.create_table "my_table"
+      #
+      #   table.schema do |schema|
+      #     schema.string "first_name", mode: :required
+      #     schema.record "cities_lived", mode: :repeated do |cities_lived|
+      #       cities_lived.string "place", mode: :required
+      #       cities_lived.integer "number_of_years", mode: :required
+      #     end
+      #   end
+      #
+      class Schema
+        def initialize
+          @nested = nil
+        end
+
+        def fields
+          @fields ||= @gapi.fields.map { |f| Field.from_gapi f }
+        end
+
+        def fields= new_fields
+          @gapi.fields = Array(new_fields).map(&:to_gapi)
+          @fields = @gapi.fields.map { |f| Field.from_gapi f }
+        end
+
+        def empty?
+          fields.empty?
+        end
+
+        # @private
+        def changed?
+          return false if frozen?
+          check_for_mutated_schema!
+          @original_json != @gapi.to_json
+        end
+
+        # @private
+        def freeze
+          @gapi = @gapi.dup.freeze
+          @gapi.fields.freeze
+          @fields = @gapi.fields.map { |f| Field.from_gapi(f).freeze }
+          @fields.freeze
+          super
+        end
+
+        ##
+        # @private Make sure any changes are saved.
+        def check_for_mutated_schema!
+          return if frozen?
+          return if @gapi.frozen?
+          return if @fields.nil?
+          gapi_fields = Array(@fields).map(&:to_gapi)
+          @gapi.update! fields: gapi_fields
+        end
+
+        # @private
+        def self.from_gapi gapi
+          gapi ||= Google::Apis::BigqueryV2::TableSchema.new fields: []
+          gapi.fields ||= []
+          new.tap do |s|
+            s.instance_variable_set :@gapi, gapi
+            s.instance_variable_set :@original_json, gapi.to_json
+          end
+        end
+
+        # @private
+        def to_gapi
+          check_for_mutated_schema!
+          @gapi
+        end
+
+        # @private
+        def == other
+          return false unless other.is_a? Schema
+          to_gapi.to_h == other.to_gapi.to_h
+        end
+
+        ##
+        # Adds a string field to the schema.
+        #
+        # @param [String] name The field name. The name must contain only
+        #   letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
+        #   start with a letter or underscore. The maximum length is 128
+        #   characters.
+        # @param [String] description A description of the field.
+        # @param [Symbol] mode The field's mode. The possible values are
+        #   `:nullable`, `:required`, and `:repeated`. The default value is
+        #   `:nullable`.
+        def string name, description: nil, mode: :nullable
+          add_field name, :string, nil, description: description, mode: mode
+        end
+
+        ##
+        # Adds an integer field to the schema.
+        #
+        # @param [String] name The field name. The name must contain only
+        #   letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
+        #   start with a letter or underscore. The maximum length is 128
+        #   characters.
+        # @param [String] description A description of the field.
+        # @param [Symbol] mode The field's mode. The possible values are
+        #   `:nullable`, `:required`, and `:repeated`. The default value is
+        #   `:nullable`.
+        def integer name, description: nil, mode: :nullable
+          add_field name, :integer, nil, description: description, mode: mode
+        end
+
+        ##
+        # Adds a floating-point number field to the schema.
+        #
+        # @param [String] name The field name. The name must contain only
+        #   letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
+        #   start with a letter or underscore. The maximum length is 128
+        #   characters.
+        # @param [String] description A description of the field.
+        # @param [Symbol] mode The field's mode. The possible values are
+        #   `:nullable`, `:required`, and `:repeated`. The default value is
+        #   `:nullable`.
+        def float name, description: nil, mode: :nullable
+          add_field name, :float, nil, description: description, mode: mode
+        end
+
+        ##
+        # Adds a boolean field to the schema.
+        #
+        # @param [String] name The field name. The name must contain only
+        #   letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
+        #   start with a letter or underscore. The maximum length is 128
+        #   characters.
+        # @param [String] description A description of the field.
+        # @param [Symbol] mode The field's mode. The possible values are
+        #   `:nullable`, `:required`, and `:repeated`. The default value is
+        #   `:nullable`.
+        def boolean name, description: nil, mode: :nullable
+          add_field name, :boolean, nil, description: description, mode: mode
+        end
+
+        ##
+        # Adds a timestamp field to the schema.
+        #
+        # @param [String] name The field name. The name must contain only
+        #   letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
+        #   start with a letter or underscore. The maximum length is 128
+        #   characters.
+        # @param [String] description A description of the field.
+        # @param [Symbol] mode The field's mode. The possible values are
+        #   `:nullable`, `:required`, and `:repeated`. The default value is
+        #   `:nullable`.
+        def timestamp name, description: nil, mode: :nullable
+          add_field name, :timestamp, nil, description: description, mode: mode
+        end
+
+        ##
+        # Adds a record field to the schema. A block must be passed describing
+        # the nested fields of the record. For more information about nested
+        # and repeated records, see [Preparing Data for BigQuery
+        # ](https://cloud.google.com/bigquery/preparing-data-for-bigquery).
+        #
+        # @param [String] name The field name. The name must contain only
+        #   letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
+        #   start with a letter or underscore. The maximum length is 128
+        #   characters.
+        # @param [String] description A description of the field.
+        # @param [Symbol] mode The field's mode. The possible values are
+        #   `:nullable`, `:required`, and `:repeated`. The default value is
+        #   `:nullable`.
+        # @yield [nested_schema] a block for setting the nested schema
+        # @yieldparam [Schema] nested_schema the object accepting the
+        #   nested schema
+        #
+        # @example
+        #   require "google/cloud"
+        #
+        #   gcloud = Google::Cloud.new
+        #   bigquery = gcloud.bigquery
+        #   dataset = bigquery.dataset "my_dataset"
+        #   table = dataset.create_table "my_table"
+        #
+        #   table.schema do |schema|
+        #     schema.string "first_name", mode: :required
+        #     schema.record "cities_lived", mode: :repeated do |cities_lived|
+        #       cities_lived.string "place", mode: :required
+        #       cities_lived.integer "number_of_years", mode: :required
+        #     end
+        #   end
+        #
+        def record name, description: nil, mode: nil
+          fail ArgumentError, "nested RECORD type is not permitted" if @nested
+          fail ArgumentError, "a block is required" unless block_given?
+          empty_schema = Google::Apis::BigqueryV2::TableSchema.new fields: []
+          nested_schema = self.class.from_gapi(empty_schema).tap do |s|
+            s.instance_variable_set :@nested, true
+          end
+          yield nested_schema
+          add_field name, :record, nested_schema.fields,
+                    description: description, mode: mode
+        end
+
+        protected
+
+        def add_field name, type, nested_fields, description: nil,
+                      mode: :nullable
+          # Remove any existing field of this name
+          fields.reject! { |f| f.name == name }
+          fields << Field.new(name, type, description: description,
+                                          mode: mode, fields: nested_fields)
+        end
+
+        class Field
+          # @private
+          MODES = %w( NULLABLE REQUIRED REPEATED )
+
+          # @private
+          TYPES = %w( STRING INTEGER FLOAT BOOLEAN TIMESTAMP RECORD )
+
+          def initialize name, type, description: nil,
+                         mode: :nullable, fields: nil
+            @gapi = Google::Apis::BigqueryV2::TableFieldSchema.new
+            @gapi.update! name: name
+            @gapi.update! type: verify_type(type)
+            @gapi.update! description: description if description
+            @gapi.update! mode: verify_mode(mode) if mode
+            if fields
+              @fields = fields
+              check_for_changed_fields!
+            end
+            @original_json = @gapi.to_json
+          end
+
+          def name
+            @gapi.name
+          end
+
+          def name= new_name
+            @gapi.update! name: new_name
+          end
+
+          def type
+            @gapi.type
+          end
+
+          def type= new_type
+            @gapi.update! type: verify_type(new_type)
+          end
+
+          def description
+            @gapi.description
+          end
+
+          def description= new_description
+            @gapi.update! description: new_description
+          end
+
+          def mode
+            @gapi.mode
+          end
+
+          def mode= new_mode
+            @gapi.update! mode: verify_mode(new_mode)
+          end
+
+          def fields
+            @fields ||= Array(@gapi.fields).map { |f| Field.from_gapi f }
+          end
+
+          def fields= new_fields
+            @fields = new_fields
+          end
+
+          ##
+          # @private Make sure any fields are saved.
+          def check_for_changed_fields!
+            return if frozen?
+            fields.each(&:check_for_changed_fields!)
+            gapi_fields = Array(fields).map(&:to_gapi)
+            gapi_fields = nil if gapi_fields.empty?
+            @gapi.update! fields: gapi_fields
+          end
+
+          # @private
+          def changed?
+            @original_json == to_gapi.to_json
+          end
+
+          # @private
+          def self.from_gapi gapi
+            new("to-be-replaced", "STRING").tap do |f|
+              f.instance_variable_set :@gapi, gapi
+              f.instance_variable_set :@original_json, gapi.to_json
+            end
+          end
+
+          # @private
+          def to_gapi
+            # make sure any changes are saved.
+            check_for_changed_fields!
+            @gapi
+          end
+
+          # @private
+          def == other
+            return false unless other.is_a? Field
+            to_gapi.to_h == other.to_gapi.to_h
+          end
+
+          protected
+
+          def verify_type type
+            upcase_type = type.to_s.upcase
+            unless TYPES.include? upcase_type
+              fail ArgumentError,
+                   "Type '#{upcase_type}' not found in #{TYPES.inspect}"
+            end
+            upcase_type
+          end
+
+          def verify_mode mode
+            upcase_mode = mode.to_s.upcase
+            unless MODES.include? upcase_mode
+              fail ArgumentError "Unable to determine mode for '#{mode}'"
+            end
+            upcase_mode
+          end
+        end
+      end
+    end
+  end
+end