gcloud 0.12.2 → 0.20.0

data/lib/gcloud/bigquery/dataset/list.rb

@@ -1,169 +0,0 @@
-# Copyright 2015 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-require "delegate"
-
-module Gcloud
-  module Bigquery
-    class Dataset
-      ##
-      # Dataset::List is a special case Array with additional values.
-      class List < DelegateClass(::Array)
-        ##
-        # If not empty, indicates that there are more records that match
-        # the request and this value should be passed to continue.
-        attr_accessor :token
-
-        # A hash of this page of results.
-        attr_accessor :etag
-
-        ##
-        # @private Create a new Dataset::List with an array of datasets.
-        def initialize arr = []
-          super arr
-        end
-
-        ##
-        # Whether there is a next page of datasets.
-        #
-        # @return [Boolean]
-        #
-        # @example
-        #   require "gcloud"
-        #
-        #   gcloud = Gcloud.new
-        #   bigquery = gcloud.bigquery
-        #
-        #   datasets = bigquery.datasets
-        #   if datasets.next?
-        #     next_datasets = datasets.next
-        #   end
-        def next?
-          !token.nil?
-        end
-
-        ##
-        # Retrieve the next page of datasets.
-        #
-        # @return [Dataset::List]
-        #
-        # @example
-        #   require "gcloud"
-        #
-        #   gcloud = Gcloud.new
-        #   bigquery = gcloud.bigquery
-        #
-        #   datasets = bigquery.datasets
-        #   if datasets.next?
-        #     next_datasets = datasets.next
-        #   end
-        def next
-          return nil unless next?
-          ensure_service!
-          options = { all: @hidden, token: token, max: @max }
-          gapi = @service.list_datasets options
-          self.class.from_gapi gapi, @service, @hidden, @max
-        end
-
-        ##
-        # Retrieves all datasets by repeatedly loading {#next} until {#next?}
-        # returns `false`. Calls the given block once for each dataset, which is
-        # passed as the parameter.
-        #
-        # An Enumerator is returned if no block is given.
-        #
-        # This method may make several API calls until all datasets are
-        # retrieved. Be sure to use as narrow a search criteria as possible.
-        # Please use with caution.
-        #
-        # @param [Integer] request_limit The upper limit of API requests to make
-        #   to load all datasets. Default is no limit.
-        # @yield [dataset] The block for accessing each dataset.
-        # @yieldparam [Dataset] dataset The dataset object.
-        #
-        # @return [Enumerator]
-        #
-        # @example Iterating each result by passing a block:
-        #   require "gcloud"
-        #
-        #   gcloud = Gcloud.new
-        #   bigquery = gcloud.bigquery
-        #
-        #   bigquery.datasets.all do |dataset|
-        #     puts dataset.name
-        #   end
-        #
-        # @example Using the enumerator by not passing a block:
-        #   require "gcloud"
-        #
-        #   gcloud = Gcloud.new
-        #   bigquery = gcloud.bigquery
-        #
-        #   all_names = bigquery.datasets.all.map do |dataset|
-        #     dataset.name
-        #   end
-        #
-        # @example Limit the number of API calls made:
-        #   require "gcloud"
-        #
-        #   gcloud = Gcloud.new
-        #   bigquery = gcloud.bigquery
-        #
-        #   bigquery.datasets.all(request_limit: 10) do |dataset|
-        #     puts dataset.name
-        #   end
-        #
-        def all request_limit: nil
-          request_limit = request_limit.to_i if request_limit
-          unless block_given?
-            return enum_for(:all, request_limit: request_limit)
-          end
-          results = self
-          loop do
-            results.each { |r| yield r }
-            if request_limit
-              request_limit -= 1
-              break if request_limit < 0
-            end
-            break unless results.next?
-            results = results.next
-          end
-        end
-
-        ##
-        # @private New Dataset::List from a response object.
-        def self.from_gapi gapi_list, service, hidden = nil, max = nil
-          datasets = List.new(Array(gapi_list.datasets).map do |gapi_object|
-            Dataset.from_gapi gapi_object, service
-          end)
-          datasets.instance_variable_set :@token,   gapi_list.next_page_token
-          datasets.instance_variable_set :@etag,    gapi_list.etag
-          datasets.instance_variable_set :@service, service
-          datasets.instance_variable_set :@hidden,  hidden
-          datasets.instance_variable_set :@max,     max
-          datasets
-        end
-
-        protected
-
-        ##
-        # Raise an error unless an active service is available.
-        def ensure_service!
-          fail "Must have active connection" unless @service
-        end
-      end
-    end
-  end
-end

data/lib/gcloud/bigquery/extract_job.rb

@@ -1,117 +0,0 @@
-# Copyright 2015 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-module Gcloud
-  module Bigquery
-    ##
-    # # ExtractJob
-    #
-    # A {Job} subclass representing an export operation that may be performed
-    # on a {Table}. A ExtractJob instance is created when you call
-    # {Table#extract}.
-    #
-    # @see https://cloud.google.com/bigquery/exporting-data-from-bigquery
-    #   Exporting Data From BigQuery
-    # @see https://cloud.google.com/bigquery/docs/reference/v2/jobs Jobs API
-    #   reference
-    #
-    class ExtractJob < Job
-      ##
-      # The URI or URIs representing the Google Cloud Storage files to which
-      # the data is exported.
-      def destinations
-        Array @gapi.configuration.extract.destination_uris
-      end
-
-      ##
-      # The table from which the data is exported. This is the table upon
-      # which {Table#extract} was called. Returns a {Table} instance.
-      def source
-        table = @gapi.configuration.extract.source_table
-        return nil unless table
-        retrieve_table table.project_id,
-                       table.dataset_id,
-                       table.table_id
-      end
-
-      ##
-      # Checks if the export operation compresses the data using gzip. The
-      # default is `false`.
-      def compression?
-        val = @gapi.configuration.extract.compression
-        val == "GZIP"
-      end
-
-      ##
-      # Checks if the destination format for the data is [newline-delimited
-      # JSON](http://jsonlines.org/). The default is `false`.
-      def json?
-        val = @gapi.configuration.extract.destination_format
-        val == "NEWLINE_DELIMITED_JSON"
-      end
-
-      ##
-      # Checks if the destination format for the data is CSV. Tables with nested
-      # or repeated fields cannot be exported as CSV. The default is `true`.
-      def csv?
-        val = @gapi.configuration.extract.destination_format
-        return true if val.nil?
-        val == "CSV"
-      end
-
-      ##
-      # Checks if the destination format for the data is
-      # [Avro](http://avro.apache.org/). The default is `false`.
-      def avro?
-        val = @gapi.configuration.extract.destination_format
-        val == "AVRO"
-      end
-
-      ##
-      # The symbol the operation uses to delimit fields in the exported data.
-      # The default is a comma (,).
-      def delimiter
-        val = @gapi.configuration.extract.field_delimiter
-        val = "," if val.nil?
-        val
-      end
-
-      ##
-      # Checks if the exported data contains a header row. The default is
-      # `true`.
-      def print_header?
-        val = @gapi.configuration.extract.print_header
-        val = true if val.nil?
-        val
-      end
-
-      ##
-      # The count of files per destination URI or URI pattern specified in
-      # {#destinations}. Returns an Array of values in the same order as the URI
-      # patterns.
-      def destinations_file_counts
-        Array @gapi.statistics.extract.destination_uri_file_counts
-      end
-
-      ##
-      # The count of files per destination URI or URI pattern specified in
-      # {#destinations}. Returns a Hash with the URI patterns as keys and the
-      # counts as values.
-      def destinations_counts
-        Hash[destinations.zip destinations_file_counts]
-      end
-    end
-  end
-end

data/lib/gcloud/bigquery/insert_response.rb

@@ -1,81 +0,0 @@
-# Copyright 2015 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-require "json"
-
-module Gcloud
-  module Bigquery
-    ##
-    # InsertResponse
-    class InsertResponse
-      # @private
-      def initialize rows, gapi
-        @rows = rows
-        @gapi = gapi
-      end
-
-      def success?
-        error_count.zero?
-      end
-
-      def insert_count
-        @rows.count - error_count
-      end
-
-      def error_count
-        Array(@gapi.insert_errors).count
-      end
-
-      def insert_errors
-        Array(@gapi.insert_errors).map do |ie|
-          row = @rows[ie.index]
-          errors = ie.errors.map { |e| JSON.parse e.to_json }
-          InsertError.new row, errors
-        end
-      end
-
-      def error_rows
-        Array(@gapi.insert_errors).map do |ie|
-          @rows[ie.index]
-        end
-      end
-
-      def errors_for row
-        ie = insert_errors.detect { |e| e.row == row }
-        return ie.errors if ie
-        []
-      end
-
-      # @private New InsertResponse from the inserted rows and a
-      # Google::Apis::BigqueryV2::InsertAllTableDataResponse object.
-      def self.from_gapi rows, gapi
-        new rows, gapi
-      end
-
-      ##
-      # InsertError
-      class InsertError
-        attr_reader :row
-        attr_reader :errors
-
-        # @private
-        def initialize row, errors
-          @row = row
-          @errors = errors
-        end
-      end
-    end
-  end
-end

data/lib/gcloud/bigquery/job.rb

@@ -1,299 +0,0 @@
-# Copyright 2015 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-require "gcloud/errors"
-require "gcloud/bigquery/service"
-require "gcloud/bigquery/query_data"
-require "gcloud/bigquery/job/list"
-require "json"
-
-module Gcloud
-  module Bigquery
-    ##
-    # # Job
-    #
-    # Represents a generic Job that may be performed on a {Table}.
-    #
-    # The subclasses of Job represent the specific BigQuery job types:
-    # {CopyJob}, {ExtractJob}, {LoadJob}, and {QueryJob}.
-    #
-    # A job instance is created when you call {Project#query_job},
-    # {Dataset#query_job}, {Table#copy}, {Table#extract}, {Table#load}, or
-    # {View#data}.
-    #
-    # @see https://cloud.google.com/bigquery/docs/managing_jobs_datasets_projects
-    #   Managing Jobs, Datasets, and Projects
-    # @see https://cloud.google.com/bigquery/docs/reference/v2/jobs Jobs API
-    #   reference
-    #
-    # @example
-    #   require "gcloud"
-    #
-    #   gcloud = Gcloud.new
-    #   bigquery = gcloud.bigquery
-    #
-    #   q = "SELECT COUNT(word) as count FROM publicdata:samples.shakespeare"
-    #   job = bigquery.query_job q
-    #
-    #   job.wait_until_done!
-    #
-    #   if job.failed?
-    #     puts job.error
-    #   else
-    #     puts job.query_results.first
-    #   end
-    #
-    class Job
-      ##
-      # @private The Service object.
-      attr_accessor :service
-
-      ##
-      # @private The Google API Client object.
-      attr_accessor :gapi
-
-      ##
-      # @private Create an empty Job object.
-      def initialize
-        @service = nil
-        @gapi = {}
-      end
-
-      ##
-      # The ID of the job.
-      def job_id
-        @gapi.job_reference.job_id
-      end
-
-      ##
-      # The ID of the project containing the job.
-      def project_id
-        @gapi.job_reference.project_id
-      end
-
-      ##
-      # The current state of the job. The possible values are `PENDING`,
-      # `RUNNING`, and `DONE`. A `DONE` state does not mean that the job
-      # completed successfully. Use {#failed?} to discover if an error occurred
-      # or if the job was successful.
-      def state
-        return nil if @gapi.status.nil?
-        @gapi.status.state
-      end
-
-      ##
-      # Checks if the job's state is `RUNNING`.
-      def running?
-        return false if state.nil?
-        "running".casecmp(state).zero?
-      end
-
-      ##
-      # Checks if the job's state is `PENDING`.
-      def pending?
-        return false if state.nil?
-        "pending".casecmp(state).zero?
-      end
-
-      ##
-      # Checks if the job's state is `DONE`. When `true`, the job has stopped
-      # running. However, a `DONE` state does not mean that the job completed
-      # successfully.  Use {#failed?} to detect if an error occurred or if the
-      # job was successful.
-      def done?
-        return false if state.nil?
-        "done".casecmp(state).zero?
-      end
-
-      ##
-      # Checks if an error is present.
-      def failed?
-        !error.nil?
-      end
-
-      ##
-      # The time when the job was created.
-      def created_at
-        Time.at(Integer(@gapi.statistics.creation_time) / 1000.0)
-      rescue
-        nil
-      end
-
-      ##
-      # The time when the job was started.
-      # This field is present after the job's state changes from `PENDING`
-      # to either `RUNNING` or `DONE`.
-      def started_at
-        Time.at(Integer(@gapi.statistics.start_time) / 1000.0)
-      rescue
-        nil
-      end
-
-      ##
-      # The time when the job ended.
-      # This field is present when the job's state is `DONE`.
-      def ended_at
-        Time.at(Integer(@gapi.statistics.end_time) / 1000.0)
-      rescue
-        nil
-      end
-
-      ##
-      # The configuration for the job. Returns a hash.
-      #
-      # @see https://cloud.google.com/bigquery/docs/reference/v2/jobs Jobs API
-      #   reference
-      def configuration
-        JSON.parse @gapi.configuration.to_json
-      end
-      alias_method :config, :configuration
-
-      ##
-      # The statistics for the job. Returns a hash.
-      #
-      # @see https://cloud.google.com/bigquery/docs/reference/v2/jobs Jobs API
-      #   reference
-      def statistics
-        JSON.parse @gapi.statistics.to_json
-      end
-      alias_method :stats, :statistics
-
-      ##
-      # The job's status. Returns a hash. The values contained in the hash are
-      # also exposed by {#state}, {#error}, and {#errors}.
-      def status
-        JSON.parse @gapi.status.to_json
-      end
-
-      ##
-      # The last error for the job, if any errors have occurred. Returns a
-      # hash.
-      #
-      # @see https://cloud.google.com/bigquery/docs/reference/v2/jobs Jobs API
-      #   reference
-      #
-      # @return [Hash] Returns a hash containing `reason` and `message` keys:
-      #
-      #   {
-      #     "reason"=>"notFound",
-      #     "message"=>"Not found: Table publicdata:samples.BAD_ID"
-      #   }
-      #
-      def error
-        return nil if @gapi.status.nil?
-        return nil if @gapi.status.error_result.nil?
-        JSON.parse @gapi.status.error_result.to_json
-      end
-
-      ##
-      # The errors for the job, if any errors have occurred. Returns an array
-      # of hash objects. See {#error}.
-      def errors
-        return [] if @gapi.status.nil?
-        Array(@gapi.status.errors).map { |e| JSON.parse e.to_json }
-      end
-
-      ##
-      # Created a new job with the current configuration.
-      def rerun!
-        ensure_service!
-        gapi = service.insert_job @gapi.configuration
-        Job.from_gapi gapi, service
-      end
-
-      ##
-      # Reloads the job with current data from the BigQuery service.
-      def reload!
-        ensure_service!
-        gapi = service.get_job job_id
-        @gapi = gapi
-      end
-      alias_method :refresh!, :reload!
-
-      ##
-      # Refreshes the job until the job is `DONE`.
-      # The delay between refreshes will incrementally increase.
-      #
-      # @example
-      #   require "gcloud"
-      #
-      #   gcloud = Gcloud.new
-      #   bigquery = gcloud.bigquery
-      #   dataset = bigquery.dataset "my_dataset"
-      #   table = dataset.table "my_table"
-      #
-      #   extract_job = table.extract "gs://my-bucket/file-name.json",
-      #                               format: "json"
-      #   extract_job.wait_until_done!
-      #   extract_job.done? #=> true
-      def wait_until_done!
-        backoff = ->(retries) { sleep 2 * retries + 5 }
-        retries = 0
-        until done?
-          backoff.call retries
-          retries += 1
-          reload!
-        end
-      end
-
-      ##
-      # @private New Job from a Google API Client object.
-      def self.from_gapi gapi, conn
-        klass = klass_for gapi
-        klass.new.tap do |f|
-          f.gapi = gapi
-          f.service = conn
-        end
-      end
-
-      protected
-
-      ##
-      # Raise an error unless an active connection is available.
-      def ensure_service!
-        fail "Must have active connection" unless service
-      end
-
-      ##
-      # Get the subclass for a job type
-      def self.klass_for gapi
-        if gapi.configuration.copy
-          return CopyJob
-        elsif gapi.configuration.extract
-          return ExtractJob
-        elsif gapi.configuration.load
-          return LoadJob
-        elsif gapi.configuration.query
-          return QueryJob
-        end
-        Job
-      end
-
-      def retrieve_table project_id, dataset_id, table_id
-        ensure_service!
-        gapi = service.get_project_table project_id, dataset_id, table_id
-        Table.from_gapi gapi, service
-      rescue Gcloud::NotFoundError
-        nil
-      end
-    end
-  end
-end
-
-# We need Job to be defined before loading these.
-require "gcloud/bigquery/copy_job"
-require "gcloud/bigquery/extract_job"
-require "gcloud/bigquery/load_job"
-require "gcloud/bigquery/query_job"