gcloud 0.10.0 → 0.11.0

data/lib/gcloud/bigquery/table.rb

@@ -409,7 +409,7 @@ module Gcloud
       #
       # @return [Gcloud::Bigquery::Data]
       #
-      # @example
+      # @example Paginate rows of data: (See {Data#next})
       #   require "gcloud"
       #
       #   gcloud = Gcloud.new
@@ -421,7 +421,22 @@ module Gcloud
       #   data.each do |row|
       #     puts row["first_name"]
       #   end
-      #   more_data = table.data token: data.token
+      #   if data.next?
+      #     more_data = data.next if data.next?
+      #   end
+      #
+      # @example Retrieve all rows of data: (See {Data#all})
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #   dataset = bigquery.dataset "my_dataset"
+      #   table = dataset.table "my_table"
+      #
+      #   data = table.data
+      #   data.all do |row|
+      #     puts row["first_name"]
+      #   end
       #
       # @!group Data
       #

data/lib/gcloud/bigquery/table/list.rb

@@ -29,26 +29,155 @@ module Gcloud
         # A hash of this page of results.
         attr_accessor :etag
 
-        # Total number of jobs in this collection.
+        # Total number of tables in this collection.
         attr_accessor :total
 
         ##
-        # Create a new Table::List with an array of jobs.
+        # @private Create a new Table::List with an array of tables.
         def initialize arr = []
           super arr
         end
 
+        ##
+        # Whether there is a next page of tables.
+        #
+        # @return [Boolean]
+        #
+        # @example
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   bigquery = gcloud.bigquery
+        #   dataset = bigquery.dataset "my_dataset"
+        #
+        #   tables = dataset.tables
+        #   if tables.next?
+        #     next_tables = tables.next
+        #   end
+        #
+        def next?
+          !token.nil?
+        end
+
+        ##
+        # Retrieve the next page of tables.
+        #
+        # @return [Table::List]
+        #
+        # @example
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   bigquery = gcloud.bigquery
+        #   dataset = bigquery.dataset "my_dataset"
+        #
+        #   tables = dataset.tables
+        #   if tables.next?
+        #     next_tables = tables.next
+        #   end
+        #
+        def next
+          return nil unless next?
+          ensure_connection!
+          options = { token: token, max: @max }
+          resp = @connection.list_tables @dataset_id, options
+          if resp.success?
+            self.class.from_response resp, @connection, @dataset_id, @max
+          else
+            fail ApiError.from_response(resp)
+          end
+        end
+
+        ##
+        # Retrieves all tables by repeatedly loading {#next} until {#next?}
+        # returns `false`. Calls the given block once for each table, which is
+        # passed as the parameter.
+        #
+        # An Enumerator is returned if no block is given.
+        #
+        # This method may make several API calls until all tables are retrieved.
+        # Be sure to use as narrow a search criteria as possible. Please use
+        # with caution.
+        #
+        # @param [Integer] request_limit The upper limit of API requests to make
+        #   to load all tables. Default is no limit.
+        # @yield [table] The block for accessing each table.
+        # @yieldparam [Table] table The table object.
+        #
+        # @return [Enumerator]
+        #
+        # @example Iterating each result by passing a block:
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   bigquery = gcloud.bigquery
+        #   dataset = bigquery.dataset "my_dataset"
+        #
+        #   dataset.tables.all do |table|
+        #     puts table.name
+        #   end
+        #
+        # @example Using the enumerator by not passing a block:
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   bigquery = gcloud.bigquery
+        #   dataset = bigquery.dataset "my_dataset"
+        #
+        #   all_names = dataset.tables.all.map do |table|
+        #     table.name
+        #   end
+        #
+        # @example Limit the number of API requests made:
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   bigquery = gcloud.bigquery
+        #   dataset = bigquery.dataset "my_dataset"
+        #
+        #   dataset.tables.all(request_limit: 10) do |table|
+        #     puts table.name
+        #   end
+        #
+        def all request_limit: nil
+          request_limit = request_limit.to_i if request_limit
+          unless block_given?
+            return enum_for(:all, request_limit: request_limit)
+          end
+          results = self
+          loop do
+            results.each { |r| yield r }
+            if request_limit
+              request_limit -= 1
+              break if request_limit < 0
+            end
+            break unless results.next?
+            results = results.next
+          end
+        end
+
         ##
         # @private New Table::List from a response object.
-        def self.from_response resp, conn
+        def self.from_response resp, conn, dataset_id = nil, max = nil
           tables = List.new(Array(resp.data["tables"]).map do |gapi_object|
             Table.from_gapi gapi_object, conn
           end)
           tables.instance_variable_set "@token", resp.data["nextPageToken"]
           tables.instance_variable_set "@etag",  resp.data["etag"]
           tables.instance_variable_set "@total", resp.data["totalItems"]
+          tables.instance_variable_set "@connection", conn
+          tables.instance_variable_set "@dataset_id", dataset_id
+          tables.instance_variable_set "@max",        max
           tables
         end
+
+        protected
+
+        ##
+        # Raise an error unless an active connection is available.
+        def ensure_connection!
+          fail "Must have active connection" unless @connection
+        end
       end
     end
   end

data/lib/gcloud/datastore.rb

@@ -205,9 +205,8 @@ module Gcloud
   #
   # ### Paginating records
   #
-  # All records may not return at once, requiring multiple calls to Datastore
-  # to return them all. The returned records will have a <tt>cursor</tt> if
-  # there are more available.
+  # All records may not return at once, but multiple calls can be made to
+  # Datastore to return them all.
   #
   # ```ruby
   # require "gcloud"
@@ -215,22 +214,10 @@ module Gcloud
   # gcloud = Gcloud.new
   # datastore = gcloud.datastore
   #
-  # task_list_key = datastore.key "TaskList", "default"
-  #
-  # query = datastore.query("Task").
-  #   ancestor(task_list_key)
-  # all_tasks = []
-  # tmp_tasks = datastore.run query
-  # while tmp_tasks.any? do
-  #   tmp_tasks.each do |task|
-  #     all_tasks << task
-  #   end
-  #   # break loop if no more tasks available
-  #   break if tmp_tasks.cursor.nil?
-  #   # set cursor on the query
-  #   query = query.cursor tmp_tasks.cursor
-  #   # query for more records
-  #   tmp_tasks = datastore.run query
+  # query = datastore.query("Task")
+  # tasks = datastore.run query
+  # tasks.all do |task|
+  #   puts t["description"]
   # end
   # ```
   #
@@ -520,6 +507,30 @@ module Gcloud
   # end
   # ```
   #
+  # ## Configuring Backoff
+  #
+  # The {Gcloud::Backoff} class allows users to globally configure how Cloud API
+  # requests are automatically retried in the case of some errors, such as a
+  # `500` or `503` status code, or a specific internal error code such as
+  # `rateLimitExceeded`.
+  #
+  # If an API call fails, the response will be inspected to see if the call
+  # should be retried. If the response matches the criteria, then the request
+  # will be retried after a delay. If another error occurs, the delay will be
+  # increased incrementally before a subsequent attempt. The first retry will be
+  # delayed one second, the second retry two seconds, and so on.
+  #
+  # ```ruby
+  # require "gcloud"
+  # require "gcloud/backoff"
+  #
+  # Gcloud::Backoff.retries = 5 # Raise the maximum number of retries from 3
+  # ```
+  #
+  # See the [Datastore error
+  # codes](https://cloud.google.com/datastore/docs/concepts/errors#error_codes)
+  # for a list of error conditions.
+  #
   # ## The Datastore Emulator
   #
   # As of this release, the Datastore emulator that is part of the gcloud SDK is

data/lib/gcloud/datastore/dataset.rb

@@ -339,12 +339,9 @@ module Gcloud
       def find_all *keys, consistency: nil
         ensure_service!
         check_consistency! consistency
-        lookup_res = service.lookup(*keys.map(&:to_grpc),
+        lookup_res = service.lookup(*Array(keys).flatten.map(&:to_grpc),
                                     consistency: consistency)
-        entities = to_gcloud_entities lookup_res.found
-        deferred = to_gcloud_keys lookup_res.deferred
-        missing  = to_gcloud_entities lookup_res.missing
-        LookupResults.new entities, deferred, missing
+        LookupResults.from_grpc lookup_res, service, consistency
       rescue GRPC::BadStatus => e
         raise Gcloud::Error.from_error(e)
       end
@@ -663,23 +660,6 @@ module Gcloud
         fail "Must have active connection to service" unless service
       end
 
-      ##
-      # Convenience method to convert GRPC entities to Gcloud entities.
-      def to_gcloud_entities grpc_entity_results
-        # Entities are nested in an object.
-        Array(grpc_entity_results).map do |result|
-          # TODO: Make this return an EntityResult with cursor...
-          Entity.from_grpc result.entity
-        end
-      end
-
-      ##
-      # Convenience method to convert GRPC keys to Gcloud keys.
-      def to_gcloud_keys grpc_keys
-        # Keys are not nested in an object like entities are.
-        Array(grpc_keys).map { |key| Key.from_grpc key }
-      end
-
       def check_consistency! consistency
         fail(ArgumentError,
              format("Consistency must be :eventual or :strong, not %s.",

data/lib/gcloud/datastore/dataset/lookup_results.rb

@@ -53,11 +53,167 @@ module Gcloud
         attr_accessor :missing
 
         ##
-        # Create a new LookupResults with an array of values.
-        def initialize arr = [], deferred = [], missing = []
+        # @private Create a new LookupResults with an array of values.
+        def initialize arr = []
           super arr
-          @deferred = deferred
-          @missing = missing
+        end
+
+        ##
+        # Whether there are more results available.
+        #
+        # @return [Boolean]
+        #
+        # @example
+        #   gcloud = Gcloud.new
+        #   datastore = gcloud.datastore
+        #
+        #   task_key1 = datastore.key "Task", "sampleTask1"
+        #   task_key2 = datastore.key "Task", "sampleTask2"
+        #   tasks = datastore.find_all task_key1, task_key2
+        #   if tasks.next?
+        #     next_tasks = tasks.next
+        #   end
+        #
+        def next?
+          Array(@deferred).any?
+        end
+
+        ##
+        # Retrieve the next page of results.
+        #
+        # @return [LookupResults]
+        #
+        # @example
+        #   gcloud = Gcloud.new
+        #   datastore = gcloud.datastore
+        #
+        #   task_key1 = datastore.key "Task", "sampleTask1"
+        #   task_key2 = datastore.key "Task", "sampleTask2"
+        #   tasks = datastore.find_all task_key1, task_key2
+        #   if tasks.next?
+        #     next_tasks = tasks.next
+        #   end
+        #
+        def next
+          return nil unless next?
+          ensure_service!
+          lookup_res = @service.lookup(*Array(@deferred).flatten.map(&:to_grpc),
+                                       consistency: @consistency,
+                                       transaction: @transaction)
+          self.class.from_grpc lookup_res, @service, @consistency
+        rescue GRPC::BadStatus => e
+          raise Gcloud::Error.from_error(e)
+        end
+
+        ##
+        # Retrieves all lookup results by repeatedly loading {#next} until
+        # {#next?} returns `false`. Calls the given block once for each result,
+        # which is passed as the parameter.
+        #
+        # An Enumerator is returned if no block is given.
+        #
+        # This method may make several API calls until all lookup results are
+        # retrieved. Be sure to use as narrow a search criteria as possible.
+        # Please use with caution.
+        #
+        # @param [Integer] request_limit The upper limit of API requests to make
+        #   to load all lookup results. Default is no limit.
+        # @yield [result] The block for accessing each lookup result.
+        # @yieldparam [Entity] result The lookup result object.
+        #
+        # @return [Enumerator]
+        #
+        # @example Iterating each result by passing a block:
+        #   gcloud = Gcloud.new
+        #   datastore = gcloud.datastore
+        #
+        #   task_key1 = datastore.key "Task", "sampleTask1"
+        #   task_key2 = datastore.key "Task", "sampleTask2"
+        #   tasks = datastore.find_all task_key1, task_key2
+        #   tasks.all do |task|
+        #     puts "Task #{task.key.id} (#cursor)"
+        #   end
+        #
+        # @example Using the enumerator by not passing a block:
+        #   gcloud = Gcloud.new
+        #   datastore = gcloud.datastore
+        #
+        #   task_key1 = datastore.key "Task", "sampleTask1"
+        #   task_key2 = datastore.key "Task", "sampleTask2"
+        #   tasks = datastore.find_all task_key1, task_key2
+        #   all_keys = tasks.all.map(&:key).each do |task|
+        #     task.key
+        #   end
+        #
+        # @example Limit the number of API calls made:
+        #   gcloud = Gcloud.new
+        #   datastore = gcloud.datastore
+        #
+        #   task_key1 = datastore.key "Task", "sampleTask1"
+        #   task_key2 = datastore.key "Task", "sampleTask2"
+        #   tasks = datastore.find_all task_key1, task_key2
+        #   tasks.all(request_limit: 10) do |task|
+        #     puts "Task #{task.key.id} (#cursor)"
+        #   end
+        #
+        def all request_limit: nil
+          request_limit = request_limit.to_i if request_limit
+          unless block_given?
+            return enum_for(:all, request_limit: request_limit)
+          end
+          results = self
+          loop do
+            results.each { |r| yield r }
+            if request_limit
+              request_limit -= 1
+              break if request_limit < 0
+            end
+            break unless results.next?
+            results = results.next
+          end
+        end
+
+        ##
+        # @private New Dataset::LookupResults from a
+        # Google::Dataset::V1beta3::LookupResponse object.
+        def self.from_grpc lookup_res, service, consistency = nil, tx = nil
+          entities = to_gcloud_entities lookup_res.found
+          deferred = to_gcloud_keys lookup_res.deferred
+          missing  = to_gcloud_entities lookup_res.missing
+          new(entities).tap do |lr|
+            lr.instance_variable_set :@service,     service
+            lr.instance_variable_set :@consistency, consistency
+            lr.instance_variable_set :@transaction, tx
+            lr.instance_variable_set :@deferred,    deferred
+            lr.instance_variable_set :@missing,     missing
+          end
+        end
+
+        protected
+
+        ##
+        # @private Raise an error unless an active connection to the service is
+        # available.
+        def ensure_service!
+          msg = "Must have active connection to datastore service to get next"
+          fail msg if @service.nil?
+        end
+
+        ##
+        # Convenience method to convert GRPC entities to Gcloud entities.
+        def self.to_gcloud_entities grpc_entity_results
+          # Entities are nested in an object.
+          Array(grpc_entity_results).map do |result|
+            # TODO: Make this return an EntityResult with cursor...
+            Entity.from_grpc result.entity
+          end
+        end
+
+        ##
+        # Convenience method to convert GRPC keys to Gcloud keys.
+        def self.to_gcloud_keys grpc_keys
+          # Keys are not nested in an object like entities are.
+          Array(grpc_keys).map { |key| Key.from_grpc key }
         end
       end
     end