gcloud 0.12.2 → 0.20.0

data/lib/gcloud/bigquery/query_data.rb

@@ -1,234 +0,0 @@
-# Copyright 2015 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-require "gcloud/bigquery/service"
-require "gcloud/bigquery/data"
-
-module Gcloud
-  module Bigquery
-    ##
-    # # QueryData
-    #
-    # Represents Data returned from a query a a list of name/value pairs.
-    class QueryData < Data
-      ##
-      # @private The Service object.
-      attr_accessor :service
-
-      # @private
-      def initialize arr = []
-        @job = nil
-        super
-      end
-
-      # The total number of bytes processed for this query.
-      def total_bytes
-        Integer @gapi.total_bytes_processed
-      rescue
-        nil
-      end
-
-      # Whether the query has completed or not. When data is present this will
-      # always be `true`. When `false`, `total` will not be available.
-      def complete?
-        @gapi.job_complete
-      end
-
-      # Whether the query result was fetched from the query cache.
-      def cache_hit?
-        @gapi.cache_hit
-      end
-
-      ##
-      # The schema of the data.
-      def schema
-        Schema.from_gapi(@gapi.schema).freeze
-      end
-
-      ##
-      # The fields of the data.
-      def fields
-        f = schema.fields
-        f = f.to_hash if f.respond_to? :to_hash
-        f = [] if f.nil?
-        f
-      end
-
-      ##
-      # The name of the columns in the data.
-      def headers
-        fields.map(&:name)
-      end
-
-      ##
-      # Whether there is a next page of query data.
-      #
-      # @return [Boolean]
-      #
-      # @example
-      #   require "gcloud"
-      #
-      #   gcloud = Gcloud.new
-      #   bigquery = gcloud.bigquery
-      #   job = bigquery.job "my_job"
-      #
-      #   data = job.query_results
-      #   if data.next?
-      #     next_data = data.next
-      #   end
-      #
-      def next?
-        !token.nil?
-      end
-
-      ##
-      # Retrieve the next page of query data.
-      #
-      # @return [QueryData]
-      #
-      # @example
-      #   require "gcloud"
-      #
-      #   gcloud = Gcloud.new
-      #   bigquery = gcloud.bigquery
-      #   job = bigquery.job "my_job"
-      #
-      #   data = job.query_results
-      #   if data.next?
-      #     next_data = data.next
-      #   end
-      #
-      def next
-        return nil unless next?
-        ensure_service!
-        gapi = service.job_query_results job_id, token: token
-        QueryData.from_gapi gapi, service
-      end
-
-      ##
-      # Retrieves all rows by repeatedly loading {#next} until {#next?} returns
-      # `false`. Calls the given block once for each row, which is passed as the
-      # parameter.
-      #
-      # An Enumerator is returned if no block is given.
-      #
-      # This method may make several API calls until all rows are retrieved. Be
-      # sure to use as narrow a search criteria as possible. Please use with
-      # caution.
-      #
-      # @param [Integer] request_limit The upper limit of API requests to make
-      #   to load all data. Default is no limit.
-      # @yield [row] The block for accessing each row of data.
-      # @yieldparam [Hash] row The row object.
-      #
-      # @return [Enumerator]
-      #
-      # @example Iterating each row by passing a block:
-      #   require "gcloud"
-      #
-      #   gcloud = Gcloud.new
-      #   bigquery = gcloud.bigquery
-      #   job = bigquery.job "my_job"
-      #
-      #   data = job.query_results
-      #   data.all do |row|
-      #     puts row["word"]
-      #   end
-      #
-      # @example Using the enumerator by not passing a block:
-      #   require "gcloud"
-      #
-      #   gcloud = Gcloud.new
-      #   bigquery = gcloud.bigquery
-      #   job = bigquery.job "my_job"
-      #
-      #   data = job.query_results
-      #   words = data.all.map do |row|
-      #     row["word"]
-      #   end
-      #
-      # @example Limit the number of API calls made:
-      #   require "gcloud"
-      #
-      #   gcloud = Gcloud.new
-      #   bigquery = gcloud.bigquery
-      #   job = bigquery.job "my_job"
-      #
-      #   data = job.query_results
-      #   data.all(request_limit: 10) do |row|
-      #     puts row["word"]
-      #   end
-      #
-      def all request_limit: nil
-        request_limit = request_limit.to_i if request_limit
-        return enum_for(:all, request_limit: request_limit) unless block_given?
-        results = self
-        loop do
-          results.each { |r| yield r }
-          if request_limit
-            request_limit -= 1
-            break if request_limit < 0
-          end
-          break unless results.next?
-          results = results.next
-        end
-      end
-
-      ##
-      # The BigQuery {Job} that was created to run the query.
-      def job
-        return @job if @job
-        return nil unless job?
-        ensure_service!
-        gapi = service.get_job job_id
-        @job = Job.from_gapi gapi, service
-      rescue Gcloud::NotFoundError
-        nil
-      end
-
-      ##
-      # @private New Data from a response object.
-      def self.from_gapi gapi, service
-        if gapi.schema.nil?
-          formatted_rows = []
-        else
-          formatted_rows = format_rows gapi.rows,
-                                       gapi.schema.fields
-        end
-
-        data = new formatted_rows
-        data.gapi = gapi
-        data.service = service
-        data
-      end
-
-      protected
-
-      ##
-      # Raise an error unless an active connection is available.
-      def ensure_service!
-        fail "Must have active connection" unless service
-      end
-
-      def job?
-        @gapi.job_reference && @gapi.job_reference.job_id
-      end
-
-      def job_id
-        @gapi.job_reference.job_id
-      end
-    end
-  end
-end

data/lib/gcloud/bigquery/query_job.rb

@@ -1,137 +0,0 @@
-# Copyright 2015 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-require "gcloud/bigquery/service"
-
-module Gcloud
-  module Bigquery
-    ##
-    # # QueryJob
-    #
-    # A {Job} subclass representing a query operation that may be performed
-    # on a {Table}. A QueryJob instance is created when you call
-    # {Project#query_job}, {Dataset#query_job}, or {View#data}.
-    #
-    # @see https://cloud.google.com/bigquery/querying-data Querying Data
-    # @see https://cloud.google.com/bigquery/docs/reference/v2/jobs Jobs API
-    #   reference
-    #
-    class QueryJob < Job
-      ##
-      # Checks if the priority for the query is `BATCH`.
-      def batch?
-        val = @gapi.configuration.query.priority
-        val == "BATCH"
-      end
-
-      ##
-      # Checks if the priority for the query is `INTERACTIVE`.
-      def interactive?
-        val = @gapi.configuration.query.priority
-        return true if val.nil?
-        val == "INTERACTIVE"
-      end
-
-      ##
-      # Checks if the the query job allows arbitrarily large results at a slight
-      # cost to performance.
-      def large_results?
-        val = @gapi.configuration.query.allow_large_results
-        return false if val.nil?
-        val
-      end
-
-      ##
-      # Checks if the query job looks for an existing result in the query cache.
-      # For more information, see [Query
-      # Caching](https://cloud.google.com/bigquery/querying-data#querycaching).
-      def cache?
-        val = @gapi.configuration.query.use_query_cache
-        return false if val.nil?
-        val
-      end
-
-      ##
-      # Checks if the query job flattens nested and repeated fields in the query
-      # results. The default is `true`. If the value is `false`, #large_results?
-      # should return `true`.
-      def flatten?
-        val = @gapi.configuration.query.flatten_results
-        return true if val.nil?
-        val
-      end
-
-      ##
-      # Checks if the query results are from the query cache.
-      def cache_hit?
-        @gapi.statistics.query.cache_hit
-      end
-
-      ##
-      # The number of bytes processed by the query.
-      def bytes_processed
-        Integer @gapi.statistics.query.total_bytes_processed
-      rescue
-        nil
-      end
-
-      ##
-      # The table in which the query results are stored.
-      def destination
-        table = @gapi.configuration.query.destination_table
-        return nil unless table
-        retrieve_table table.project_id,
-                       table.dataset_id,
-                       table.table_id
-      end
-
-      ##
-      # Retrieves the query results for the job.
-      #
-      # @param [String] token Page token, returned by a previous call,
-      #   identifying the result set.
-      # @param [Integer] max Maximum number of results to return.
-      # @param [Integer] start Zero-based index of the starting row to read.
-      # @param [Integer] timeout How long to wait for the query to complete, in
-      #   milliseconds, before returning. Default is 10,000 milliseconds (10
-      #   seconds).
-      #
-      # @return [Gcloud::Bigquery::QueryData]
-      #
-      # @example
-      #   require "gcloud"
-      #
-      #   gcloud = Gcloud.new
-      #   bigquery = gcloud.bigquery
-      #
-      #   q = "SELECT word FROM publicdata:samples.shakespeare"
-      #   job = bigquery.query_job q
-      #
-      #   job.wait_until_done!
-      #   data = job.query_results
-      #   data.each do |row|
-      #     puts row["word"]
-      #   end
-      #   data = data.next if data.next?
-      #
-      def query_results token: nil, max: nil, start: nil, timeout: nil
-        ensure_service!
-        options = { token: token, max: max, start: start, timeout: timeout }
-        gapi = service.job_query_results job_id, options
-        QueryData.from_gapi gapi, service
-      end
-    end
-  end
-end

data/lib/gcloud/bigquery/schema.rb

@@ -1,359 +0,0 @@
-# Copyright 2015 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-module Gcloud
-  module Bigquery
-    ##
-    # # Table Schema
-    #
-    # A builder for BigQuery table schemas, passed to block arguments to
-    # {Dataset#create_table} and {Table#schema}. Supports nested and
-    # repeated fields via a nested block.
-    #
-    # @see https://cloud.google.com/bigquery/preparing-data-for-bigquery
-    #   Preparing Data for BigQuery
-    #
-    # @example
-    #   require "gcloud"
-    #
-    #   gcloud = Gcloud.new
-    #   bigquery = gcloud.bigquery
-    #   dataset = bigquery.dataset "my_dataset"
-    #   table = dataset.create_table "my_table"
-    #
-    #   table.schema do |schema|
-    #     schema.string "first_name", mode: :required
-    #     schema.record "cities_lived", mode: :repeated do |cities_lived|
-    #       cities_lived.string "place", mode: :required
-    #       cities_lived.integer "number_of_years", mode: :required
-    #     end
-    #   end
-    #
-    class Schema
-      def initialize
-        @nested = nil
-      end
-
-      def fields
-        @fields ||= @gapi.fields.map { |f| Field.from_gapi f }
-      end
-
-      def fields= new_fields
-        @gapi.fields = Array(new_fields).map(&:to_gapi)
-        @fields = @gapi.fields.map { |f| Field.from_gapi f }
-      end
-
-      def empty?
-        fields.empty?
-      end
-
-      # @private
-      def changed?
-        return false if frozen?
-        check_for_mutated_schema!
-        @original_json != @gapi.to_json
-      end
-
-      # @private
-      def freeze
-        @gapi = @gapi.dup.freeze
-        @gapi.fields.freeze
-        @fields = @gapi.fields.map { |f| Field.from_gapi(f).freeze }
-        @fields.freeze
-        super
-      end
-
-      ##
-      # @private Make sure any changes are saved.
-      def check_for_mutated_schema!
-        return if frozen?
-        return if @gapi.frozen?
-        return if @fields.nil?
-        gapi_fields = Array(@fields).map(&:to_gapi)
-        @gapi.update! fields: gapi_fields
-      end
-
-      # @private
-      def self.from_gapi gapi
-        gapi ||= Google::Apis::BigqueryV2::TableSchema.new fields: []
-        gapi.fields ||= []
-        new.tap do |s|
-          s.instance_variable_set :@gapi, gapi
-          s.instance_variable_set :@original_json, gapi.to_json
-        end
-      end
-
-      # @private
-      def to_gapi
-        check_for_mutated_schema!
-        @gapi
-      end
-
-      # @private
-      def == other
-        return false unless other.is_a? Schema
-        to_gapi.to_h == other.to_gapi.to_h
-      end
-
-      ##
-      # Adds a string field to the schema.
-      #
-      # @param [String] name The field name. The name must contain only
-      #   letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
-      #   start with a letter or underscore. The maximum length is 128
-      #   characters.
-      # @param [String] description A description of the field.
-      # @param [Symbol] mode The field's mode. The possible values are
-      #   `:nullable`, `:required`, and `:repeated`. The default value is
-      #   `:nullable`.
-      def string name, description: nil, mode: :nullable
-        add_field name, :string, nil, description: description, mode: mode
-      end
-
-      ##
-      # Adds an integer field to the schema.
-      #
-      # @param [String] name The field name. The name must contain only
-      #   letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
-      #   start with a letter or underscore. The maximum length is 128
-      #   characters.
-      # @param [String] description A description of the field.
-      # @param [Symbol] mode The field's mode. The possible values are
-      #   `:nullable`, `:required`, and `:repeated`. The default value is
-      #   `:nullable`.
-      def integer name, description: nil, mode: :nullable
-        add_field name, :integer, nil, description: description, mode: mode
-      end
-
-      ##
-      # Adds a floating-point number field to the schema.
-      #
-      # @param [String] name The field name. The name must contain only
-      #   letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
-      #   start with a letter or underscore. The maximum length is 128
-      #   characters.
-      # @param [String] description A description of the field.
-      # @param [Symbol] mode The field's mode. The possible values are
-      #   `:nullable`, `:required`, and `:repeated`. The default value is
-      #   `:nullable`.
-      def float name, description: nil, mode: :nullable
-        add_field name, :float, nil, description: description, mode: mode
-      end
-
-      ##
-      # Adds a boolean field to the schema.
-      #
-      # @param [String] name The field name. The name must contain only
-      #   letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
-      #   start with a letter or underscore. The maximum length is 128
-      #   characters.
-      # @param [String] description A description of the field.
-      # @param [Symbol] mode The field's mode. The possible values are
-      #   `:nullable`, `:required`, and `:repeated`. The default value is
-      #   `:nullable`.
-      def boolean name, description: nil, mode: :nullable
-        add_field name, :boolean, nil, description: description, mode: mode
-      end
-
-      ##
-      # Adds a timestamp field to the schema.
-      #
-      # @param [String] name The field name. The name must contain only
-      #   letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
-      #   start with a letter or underscore. The maximum length is 128
-      #   characters.
-      # @param [String] description A description of the field.
-      # @param [Symbol] mode The field's mode. The possible values are
-      #   `:nullable`, `:required`, and `:repeated`. The default value is
-      #   `:nullable`.
-      def timestamp name, description: nil, mode: :nullable
-        add_field name, :timestamp, nil, description: description, mode: mode
-      end
-
-      ##
-      # Adds a record field to the schema. A block must be passed describing
-      # the nested fields of the record. For more information about nested
-      # and repeated records, see [Preparing Data for BigQuery
-      # ](https://cloud.google.com/bigquery/preparing-data-for-bigquery).
-      #
-      # @param [String] name The field name. The name must contain only
-      #   letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
-      #   start with a letter or underscore. The maximum length is 128
-      #   characters.
-      # @param [String] description A description of the field.
-      # @param [Symbol] mode The field's mode. The possible values are
-      #   `:nullable`, `:required`, and `:repeated`. The default value is
-      #   `:nullable`.
-      # @yield [nested_schema] a block for setting the nested schema
-      # @yieldparam [Schema] nested_schema the object accepting the nested
-      #   schema
-      #
-      # @example
-      #   require "gcloud"
-      #
-      #   gcloud = Gcloud.new
-      #   bigquery = gcloud.bigquery
-      #   dataset = bigquery.dataset "my_dataset"
-      #   table = dataset.create_table "my_table"
-      #
-      #   table.schema do |schema|
-      #     schema.string "first_name", mode: :required
-      #     schema.record "cities_lived", mode: :repeated do |cities_lived|
-      #       cities_lived.string "place", mode: :required
-      #       cities_lived.integer "number_of_years", mode: :required
-      #     end
-      #   end
-      #
-      def record name, description: nil, mode: nil
-        fail ArgumentError, "nested RECORD type is not permitted" if @nested
-        fail ArgumentError, "a block is required" unless block_given?
-        empty_schema = Google::Apis::BigqueryV2::TableSchema.new fields: []
-        nested_schema = self.class.from_gapi(empty_schema).tap do |s|
-          s.instance_variable_set :@nested, true
-        end
-        yield nested_schema
-        add_field name, :record, nested_schema.fields,
-                  description: description, mode: mode
-      end
-
-      protected
-
-      def add_field name, type, nested_fields, description: nil,
-                    mode: :nullable
-        # Remove any existing field of this name
-        fields.reject! { |f| f.name == name }
-        fields << Field.new(name, type, description: description,
-                                        mode: mode, fields: nested_fields)
-      end
-
-      class Field
-        # @private
-        MODES = %w( NULLABLE REQUIRED REPEATED )
-
-        # @private
-        TYPES = %w( STRING INTEGER FLOAT BOOLEAN TIMESTAMP RECORD )
-
-        def initialize name, type, description: nil,
-                       mode: :nullable, fields: nil
-          @gapi = Google::Apis::BigqueryV2::TableFieldSchema.new
-          @gapi.update! name: name
-          @gapi.update! type: verify_type(type)
-          @gapi.update! description: description if description
-          @gapi.update! mode: verify_mode(mode) if mode
-          if fields
-            @fields = fields
-            check_for_changed_fields!
-          end
-          @original_json = @gapi.to_json
-        end
-
-        def name
-          @gapi.name
-        end
-
-        def name= new_name
-          @gapi.update! name: new_name
-        end
-
-        def type
-          @gapi.type
-        end
-
-        def type= new_type
-          @gapi.update! type: verify_type(new_type)
-        end
-
-        def description
-          @gapi.description
-        end
-
-        def description= new_description
-          @gapi.update! description: new_description
-        end
-
-        def mode
-          @gapi.mode
-        end
-
-        def mode= new_mode
-          @gapi.update! mode: verify_mode(new_mode)
-        end
-
-        def fields
-          @fields ||= Array(@gapi.fields).map { |f| Field.from_gapi f }
-        end
-
-        def fields= new_fields
-          @fields = new_fields
-        end
-
-        ##
-        # @private Make sure any fields are saved.
-        def check_for_changed_fields!
-          return if frozen?
-          fields.each(&:check_for_changed_fields!)
-          gapi_fields = Array(fields).map(&:to_gapi)
-          gapi_fields = nil if gapi_fields.empty?
-          @gapi.update! fields: gapi_fields
-        end
-
-        # @private
-        def changed?
-          @original_json == to_gapi.to_json
-        end
-
-        # @private
-        def self.from_gapi gapi
-          new("to-be-replaced", "STRING").tap do |f|
-            f.instance_variable_set :@gapi, gapi
-            f.instance_variable_set :@original_json, gapi.to_json
-          end
-        end
-
-        # @private
-        def to_gapi
-          # make sure any changes are saved.
-          check_for_changed_fields!
-          @gapi
-        end
-
-        # @private
-        def == other
-          return false unless other.is_a? Field
-          to_gapi.to_h == other.to_gapi.to_h
-        end
-
-        protected
-
-        def verify_type type
-          upcase_type = type.to_s.upcase
-          unless TYPES.include? upcase_type
-            fail ArgumentError,
-                 "Type '#{upcase_type}' not found in #{TYPES.inspect}"
-          end
-          upcase_type
-        end
-
-        def verify_mode mode
-          upcase_mode = mode.to_s.upcase
-          unless MODES.include? upcase_mode
-            fail ArgumentError "Unable to determine mode for '#{mode}'"
-          end
-          upcase_mode
-        end
-      end
-    end
-  end
-end