gcloud 0.10.0 → 0.11.0

data/lib/gcloud/bigquery/dataset.rb

@@ -502,23 +502,15 @@ module Gcloud
       #     puts table.name
       #   end
       #
-      # @example With pagination: (See {Dataset::List#token})
+      # @example Retrieve all tables: (See {Table::List#all})
       #   require "gcloud"
       #
       #   gcloud = Gcloud.new
       #   bigquery = gcloud.bigquery
       #   dataset = bigquery.dataset "my_dataset"
-      #
-      #   all_tables = []
-      #   tmp_tables = dataset.tables
-      #   while tmp_tables.any? do
-      #     tmp_tables.each do |table|
-      #       all_tables << table
-      #     end
-      #     # break loop if no more tables available
-      #     break if tmp_tables.token.nil?
-      #     # get the next group of tables
-      #     tmp_tables = dataset.tables token: tmp_tables.token
+      #   tables = dataset.tables
+      #   tables.all do |table|
+      #     puts table.name
       #   end
       #
       # @!group Table
@@ -528,7 +520,7 @@ module Gcloud
         options = { token: token, max: max }
         resp = connection.list_tables dataset_id, options
         if resp.success?
-          Table::List.from_response resp, connection
+          Table::List.from_response resp, connection, dataset_id, max
         else
           fail ApiError.from_response(resp)
         end

data/lib/gcloud/bigquery/dataset/list.rb

@@ -30,21 +30,143 @@ module Gcloud
         attr_accessor :etag
 
         ##
-        # Create a new Dataset::List with an array of datasets.
+        # @private Create a new Dataset::List with an array of datasets.
         def initialize arr = []
           super arr
         end
 
+        ##
+        # Whether there is a next page of datasets.
+        #
+        # @return [Boolean]
+        #
+        # @example
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   bigquery = gcloud.bigquery
+        #
+        #   datasets = bigquery.datasets
+        #   if datasets.next?
+        #     next_datasets = datasets.next
+        #   end
+        def next?
+          !token.nil?
+        end
+
+        ##
+        # Retrieve the next page of datasets.
+        #
+        # @return [Dataset::List]
+        #
+        # @example
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   bigquery = gcloud.bigquery
+        #
+        #   datasets = bigquery.datasets
+        #   if datasets.next?
+        #     next_datasets = datasets.next
+        #   end
+        def next
+          return nil unless next?
+          ensure_connection!
+          options = { all: @hidden, token: token, max: @max }
+          resp = @connection.list_datasets options
+          if resp.success?
+            self.class.from_response resp, @connection
+          else
+            fail ApiError.from_response(resp)
+          end
+        end
+
+        ##
+        # Retrieves all datasets by repeatedly loading {#next} until {#next?}
+        # returns `false`. Calls the given block once for each dataset, which is
+        # passed as the parameter.
+        #
+        # An Enumerator is returned if no block is given.
+        #
+        # This method may make several API calls until all datasets are
+        # retrieved. Be sure to use as narrow a search criteria as possible.
+        # Please use with caution.
+        #
+        # @param [Integer] request_limit The upper limit of API requests to make
+        #   to load all datasets. Default is no limit.
+        # @yield [dataset] The block for accessing each dataset.
+        # @yieldparam [Dataset] dataset The dataset object.
+        #
+        # @return [Enumerator]
+        #
+        # @example Iterating each result by passing a block:
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   bigquery = gcloud.bigquery
+        #
+        #   bigquery.datasets.all do |dataset|
+        #     puts dataset.name
+        #   end
+        #
+        # @example Using the enumerator by not passing a block:
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   bigquery = gcloud.bigquery
+        #
+        #   all_names = bigquery.datasets.all.map do |dataset|
+        #     dataset.name
+        #   end
+        #
+        # @example Limit the number of API calls made:
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   bigquery = gcloud.bigquery
+        #
+        #   bigquery.datasets.all(request_limit: 10) do |dataset|
+        #     puts dataset.name
+        #   end
+        #
+        def all request_limit: nil
+          request_limit = request_limit.to_i if request_limit
+          unless block_given?
+            return enum_for(:all, request_limit: request_limit)
+          end
+          results = self
+          loop do
+            results.each { |r| yield r }
+            if request_limit
+              request_limit -= 1
+              break if request_limit < 0
+            end
+            break unless results.next?
+            results = results.next
+          end
+        end
+
         ##
         # @private New Dataset::List from a response object.
-        def self.from_response resp, conn
+        def self.from_response resp, conn, hidden = nil, max = nil
           datasets = List.new(Array(resp.data["datasets"]).map do |gapi_object|
             Dataset.from_gapi gapi_object, conn
           end)
           datasets.instance_variable_set "@token", resp.data["nextPageToken"]
           datasets.instance_variable_set "@etag",  resp.data["etag"]
+          datasets.instance_variable_set "@connection", conn
+          datasets.instance_variable_set "@hidden",     hidden
+          datasets.instance_variable_set "@max",        max
           datasets
         end
+
+        protected
+
+        ##
+        # Raise an error unless an active connection is available.
+        def ensure_connection!
+          fail "Must have active connection" unless @connection
+        end
       end
     end
   end

data/lib/gcloud/bigquery/job/list.rb

@@ -33,22 +33,145 @@ module Gcloud
         attr_accessor :total
 
         ##
-        # Create a new Job::List with an array of jobs.
+        # @private Create a new Job::List with an array of jobs.
         def initialize arr = []
           super arr
         end
 
+        ##
+        # Whether there is a next page of jobs.
+        #
+        # @return [Boolean]
+        #
+        # @example
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   bigquery = gcloud.bigquery
+        #
+        #   jobs = bigquery.jobs
+        #   if jobs.next?
+        #     next_jobs = jobs.next
+        #   end
+        def next?
+          !token.nil?
+        end
+
+        ##
+        # Retrieve the next page of jobs.
+        #
+        # @return [Job::List]
+        #
+        # @example
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   bigquery = gcloud.bigquery
+        #
+        #   jobs = bigquery.jobs
+        #   if jobs.next?
+        #     next_jobs = jobs.next
+        #   end
+        def next
+          return nil unless next?
+          ensure_connection!
+          options = { all: @hidden, token: token, max: @max, filter: @filter }
+          resp = @connection.list_jobs options
+          if resp.success?
+            self.class.from_response resp, @connection, @hidden, @max, @filter
+          else
+            fail ApiError.from_response(resp)
+          end
+        end
+
+        ##
+        # Retrieves all jobs by repeatedly loading {#next} until {#next?}
+        # returns `false`. Calls the given block once for each job, which is
+        # passed as the parameter.
+        #
+        # An Enumerator is returned if no block is given.
+        #
+        # This method may make several API calls until all jobs are retrieved.
+        # Be sure to use as narrow a search criteria as possible. Please use
+        # with caution.
+        #
+        # @param [Integer] request_limit The upper limit of API requests to make
+        #   to load all jobs. Default is no limit.
+        # @yield [job] The block for accessing each job.
+        # @yieldparam [Job] job The job object.
+        #
+        # @return [Enumerator]
+        #
+        # @example Iterating each job by passing a block:
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   bigquery = gcloud.bigquery
+        #
+        #   bigquery.jobs.all do |job|
+        #     puts job.state
+        #   end
+        #
+        # @example Using the enumerator by not passing a block:
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   bigquery = gcloud.bigquery
+        #
+        #   all_states = bigquery.jobs.all.map do |job|
+        #     job.state
+        #   end
+        #
+        # @example Limit the number of API calls made:
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   bigquery = gcloud.bigquery
+        #
+        #   bigquery.jobs.all(request_limit: 10) do |job|
+        #     puts job.state
+        #   end
+        #
+        def all request_limit: nil
+          request_limit = request_limit.to_i if request_limit
+          unless block_given?
+            return enum_for(:all, request_limit: request_limit)
+          end
+          results = self
+          loop do
+            results.each { |r| yield r }
+            if request_limit
+              request_limit -= 1
+              break if request_limit < 0
+            end
+            break unless results.next?
+            results = results.next
+          end
+        end
+
         ##
         # @private New Job::List from a response object.
-        def self.from_response resp, conn
+        def self.from_response resp, conn, hidden = nil, max = nil, filter = nil
           jobs = List.new(Array(resp.data["jobs"]).map do |gapi_object|
             Job.from_gapi gapi_object, conn
           end)
           jobs.instance_variable_set "@token", resp.data["nextPageToken"]
           jobs.instance_variable_set "@etag",  resp.data["etag"]
           jobs.instance_variable_set "@total", resp.data["totalItems"]
+          jobs.instance_variable_set "@connection", conn
+          jobs.instance_variable_set "@hidden",     hidden
+          jobs.instance_variable_set "@max",        max
+          jobs.instance_variable_set "@filter",     filter
           jobs
         end
+
+        protected
+
+        ##
+        # Raise an error unless an active connection is available.
+        def ensure_connection!
+          fail "Must have active connection" unless @connection
+        end
       end
     end
   end

data/lib/gcloud/bigquery/project.rb

@@ -210,6 +210,17 @@ module Gcloud
       #     puts row["name"]
       #   end
       #
+      # @example Retrieve all rows: (See {QueryData#all})
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #
+      #   data = bigquery.query "SELECT name FROM [my_proj:my_data.my_table]"
+      #   data.all do |row|
+      #     puts row["name"]
+      #   end
+      #
       def query query, max: nil, timeout: 10000, dryrun: nil, cache: true,
                 dataset: nil, project: nil
         ensure_connection!
@@ -352,30 +363,23 @@ module Gcloud
       #     puts dataset.name
       #   end
       #
-      # @example Retrieve all datasets, including hidden ones, with `:all`:
+      # @example Retrieve hidden datasets with the `all` optional arg:
       #   require "gcloud"
       #
       #   gcloud = Gcloud.new
       #   bigquery = gcloud.bigquery
       #
-      #   all_datasets = bigquery.datasets, all: true
+      #   all_datasets = bigquery.datasets all: true
       #
-      # @example With pagination: (See {Dataset::List#token})
+      # @example Retrieve all datasets: (See {Dataset::List#all})
       #   require "gcloud"
       #
       #   gcloud = Gcloud.new
       #   bigquery = gcloud.bigquery
       #
-      #   all_datasets = []
-      #   tmp_datasets = bigquery.datasets
-      #   while tmp_datasets.any? do
-      #     tmp_datasets.each do |dataset|
-      #       all_datasets << dataset
-      #     end
-      #     # break loop if no more datasets available
-      #     break if tmp_datasets.token.nil?
-      #     # get the next group of datasets
-      #     tmp_datasets = bigquery.datasets token: tmp_datasets.token
+      #   datasets = bigquery.datasets
+      #   datasets.all do |dataset|
+      #     puts dataset.name
       #   end
       #
       def datasets all: nil, token: nil, max: nil
@@ -383,7 +387,7 @@ module Gcloud
         options = { all: all, token: token, max: max }
         resp = connection.list_datasets options
         if resp.success?
-          Dataset::List.from_response resp, connection
+          Dataset::List.from_response resp, connection, all, max
         else
           fail ApiError.from_response(resp)
         end
@@ -442,31 +446,30 @@ module Gcloud
       #   bigquery = gcloud.bigquery
       #
       #   jobs = bigquery.jobs
+      #   jobs.each do |job|
+      #     # process job
+      #   end
       #
-      # @example Retrieve only running jobs using the `:filter` option:
+      # @example Retrieve only running jobs using the `filter` optional arg:
       #   require "gcloud"
       #
       #   gcloud = Gcloud.new
       #   bigquery = gcloud.bigquery
       #
       #   running_jobs = bigquery.jobs filter: "running"
+      #   running_jobs.each do |job|
+      #     # process job
+      #   end
       #
-      # @example With pagination: (See {Job::List#token})
+      # @example Retrieve all jobs: (See {Job::List#all})
       #   require "gcloud"
       #
       #   gcloud = Gcloud.new
       #   bigquery = gcloud.bigquery
       #
-      #   all_jobs = []
-      #   tmp_jobs = bigquery.jobs
-      #   while tmp_jobs.any? do
-      #     tmp_jobs.each do |job|
-      #       all_jobs << job
-      #     end
-      #     # break loop if no more jobs available
-      #     break if tmp_jobs.token.nil?
-      #     # get the next group of jobs
-      #     tmp_jobs = bigquery.jobs token: tmp_jobs.token
+      #   jobs = bigquery.jobs
+      #   jobs.all do |job|
+      #     # process job
       #   end
       #
       def jobs all: nil, token: nil, max: nil, filter: nil
@@ -474,7 +477,7 @@ module Gcloud
         options = { all: all, token: token, max: max, filter: filter }
         resp = connection.list_jobs options
         if resp.success?
-          Job::List.from_response resp, connection
+          Job::List.from_response resp, connection, all, max, filter
         else
           fail ApiError.from_response(resp)
         end

data/lib/gcloud/bigquery/query_data.rb

@@ -73,11 +73,43 @@ module Gcloud
       end
 
       ##
-      # Is there a next page of data?
+      # Whether there is a next page of query data.
+      #
+      # @return [Boolean]
+      #
+      # @example
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #   job = bigquery.job "my_job"
+      #
+      #   data = job.query_results
+      #   if data.next?
+      #     next_data = data.next
+      #   end
+      #
       def next?
         !token.nil?
       end
 
+      ##
+      # Retrieve the next page of query data.
+      #
+      # @return [QueryData]
+      #
+      # @example
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #   job = bigquery.job "my_job"
+      #
+      #   data = job.query_results
+      #   if data.next?
+      #     next_data = data.next
+      #   end
+      #
       def next
         return nil unless next?
         ensure_connection!
@@ -89,6 +121,75 @@ module Gcloud
         end
       end
 
+      ##
+      # Retrieves all rows by repeatedly loading {#next} until {#next?} returns
+      # `false`. Calls the given block once for each row, which is passed as the
+      # parameter.
+      #
+      # An Enumerator is returned if no block is given.
+      #
+      # This method may make several API calls until all rows are retrieved. Be
+      # sure to use as narrow a search criteria as possible. Please use with
+      # caution.
+      #
+      # @param [Integer] request_limit The upper limit of API requests to make
+      #   to load all data. Default is no limit.
+      # @yield [row] The block for accessing each row of data.
+      # @yieldparam [Hash] row The row object.
+      #
+      # @return [Enumerator]
+      #
+      # @example Iterating each row by passing a block:
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #   job = bigquery.job "my_job"
+      #
+      #   data = job.query_results
+      #   data.all do |row|
+      #     puts row["word"]
+      #   end
+      #
+      # @example Using the enumerator by not passing a block:
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #   job = bigquery.job "my_job"
+      #
+      #   data = job.query_results
+      #   words = data.all.map do |row|
+      #     row["word"]
+      #   end
+      #
+      # @example Limit the number of API calls made:
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #   job = bigquery.job "my_job"
+      #
+      #   data = job.query_results
+      #   data.all(request_limit: 10) do |row|
+      #     puts row["word"]
+      #   end
+      #
+      def all request_limit: nil
+        request_limit = request_limit.to_i if request_limit
+        return enum_for(:all, request_limit: request_limit) unless block_given?
+        results = self
+        loop do
+          results.each { |r| yield r }
+          if request_limit
+            request_limit -= 1
+            break if request_limit < 0
+          end
+          break unless results.next?
+          results = results.next
+        end
+      end
+
       ##
       # The BigQuery {Job} that was created to run the query.
       def job