gcloud 0.10.0 → 0.11.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZDg1NjYxMTkxZjIxYmUzNmMyZGMzNDNlY2Y1MGZiZWYwNWYzZTdjOQ==
+    MTIzYjc5OWY0M2E1MGJkMTVlNWI5NmM4YTdkZDM3ZWRmZTgyZDRiZQ==
   data.tar.gz: !binary |-
-    ZDQwZjAxZGRmZmY4MmNlNGE1NmJiMTc3MTFkYWVmODkzNmFkZmU4Yw==
+    Nzg1ZTExY2RhMDRlMDUzMzkyY2FkYTI4ODUyMDc3NTEyMjIzM2QyMw==
 SHA512:
   metadata.gz: !binary |-
-    YzkwYWVjYWZhYTJjMjY2NWM0ZWFjYmNkZDI3YzBiZjc4YTcwOTVjYjhkZTBk
-    ZmVjMDE4YmQ5Mjc3YmViMzZlNWExOTllMzRmYTE0NTJiMDNkYjI2MDlmMmJk
-    OWU0N2UzODJlYWI3NjU5NDAwODQ2MjAzMDZkODAwMjVmYTEzYTk=
+    YmFjZmQyMjU4NjRhZGM0Zjc4NDYxYmVjYTQyNzhhZGI1ZTAyZjBhYTBjMDZm
+    N2U5OWZkOTYzZDk1NjEyNzk0MjJiMTU1OTVlYjZiN2Q1MjhjMTMwYjg2M2U3
+    OWI2NWYxMjlmZGE2NDM2MDZjOWExYzIyNDhiMjEzOTE4ZWRjMjY=
   data.tar.gz: !binary |-
-    ZTEzMjg3Y2M5ZTAzNjk5MGRjYzZjOTE2ZGQyZjljOTljYjI2NDczMjhkMTgw
-    OWZiZTZiNTg1YjI3ODU3NGY3NzIxMTBjN2Y2YmI1N2IzNTViMjc4MTNhOTFl
-    MWZjMzFkZjFkOTM1NDM2YjU4MWM5ODEwNDdmMWJjNDVhOWQ3MGY=
+    ZjgzNWI3NjA3ZWFkNmJjZWUxYWM1M2M3MGIxMzdmOTcxMTdlZjIzZjRiOGU3
+    Yzk1MWQ0ZDQzYjg0N2Y1MDI2MzhmM2VlNTA0NWY2NWYyMDAzMjU3YmE5YzM3
+    ZWI3MTkyMDU2M2YyYzMyYWQ0MTg0YWMwYzc4NWNlNTBmMThlNzU=

data/CHANGELOG.md

@@ -1,5 +1,41 @@
 # Release History
 
+### 0.11.0 / 2016-06-13
+
+#### Changes
+
+* Add backoff to all requests (dolzenko)
+* BigQuery
+  * Add `#all` to `Data`
+  * Add `#all`, `#next` and `#next?` to `Dataset::List`, `Job::List`,
+    `QueryData`, `Table::List` and `Dataset::LookupResults`
+  * `#all` methods now return lazy enumerator with `request_limit`
+* Datastore
+  * Add `#cursor_for`, `#each_with_cursor` and `#all_with_cursor` to
+    `Dataset::QueryResults`
+  * `#all` and `#all_with_cursor` methods now return lazy enumerator with
+    `request_limit`
+* DNS
+  * Add `#all` to `Change::List` and  `Zone::List`
+  * `#all` methods now return lazy enumerator with `request_limit`
+* Logging
+  * `#all` methods now return lazy enumerator with `request_limit`
+* Pub/Sub
+  * Fix bug when publishing multi-byte strings
+  * Add support for IO-ish objects
+  * Add `#all`, `#next` and `#next?` to `Subscription::List` and `Topic::List`
+  * `#all` methods now return lazy enumerator with `request_limit`
+* Resource Manager
+  * `#all` methods now return lazy enumerator with `request_limit`
+* Storage
+  * Breaking Change: Remove `retries` option from `Bucket#delete` and
+    `Project#create_bucket` (configure in `Backoff` instead)
+  * Add support for customer-supplied encryption keys
+  * Fix issue verifying large files (Aguasvivas22)
+  * `#all` methods now return lazy enumerator with `request_limit`
+* Vision
+  * Add support for IO-ish objects
+
 ### 0.10.0 / 2016-05-19
 
 #### Major Changes

data/lib/gcloud/backoff.rb

@@ -33,26 +33,26 @@ module Gcloud
       ##
       # The number of times a retriable API call should be retried.
       #
-      # The default value is 3.
+      # The default value is `3`.
       attr_accessor :retries
 
       ##
       # The GRPC Status Codes that should be retried.
       #
-      # The default values are 14.
+      # The default values are `14`.
       attr_accessor :grpc_codes
 
       ##
       # The HTTP Status Codes that should be retried.
       #
-      # The default values are 500 and 503.
+      # The default values are `500` and `503`.
       attr_accessor :http_codes
 
       ##
       # The Google API error reasons that should be retried.
       #
-      # The default values are rateLimitExceeded and
-      # userRateLimitExceeded.
+      # The default values are `rateLimitExceeded` and
+      # `userRateLimitExceeded`.
       attr_accessor :reasons
 
       ##

data/lib/gcloud/bigquery.rb

@@ -386,6 +386,30 @@ module Gcloud
   # BigQuery](https://cloud.google.com/bigquery/exporting-data-from-bigquery)
   # for details.
   #
+  # ## Configuring Backoff
+  #
+  # The {Gcloud::Backoff} class allows users to globally configure how Cloud API
+  # requests are automatically retried in the case of some errors, such as a
+  # `500` or `503` status code, or a specific internal error code such as
+  # `rateLimitExceeded`.
+  #
+  # If an API call fails, the response will be inspected to see if the call
+  # should be retried. If the response matches the criteria, then the request
+  # will be retried after a delay. If another error occurs, the delay will be
+  # increased incrementally before a subsequent attempt. The first retry will be
+  # delayed one second, the second retry two seconds, and so on.
+  #
+  # ```ruby
+  # require "gcloud"
+  # require "gcloud/backoff"
+  #
+  # Gcloud::Backoff.retries = 5 # Raise the maximum number of retries from 3
+  # ```
+  #
+  # See the [BigQuery error
+  # table](https://cloud.google.com/bigquery/troubleshooting-errors#errortable)
+  # for a list of error conditions.
+  #
   module Bigquery
   end
 end

data/lib/gcloud/bigquery/connection.rb

@@ -15,6 +15,7 @@
 
 require "pathname"
 require "gcloud/version"
+require "gcloud/backoff"
 require "google/api_client"
 require "digest/md5"
 
@@ -50,7 +51,7 @@ module Gcloud
                    maxResults: options.delete(:max)
                  }.delete_if { |_, v| v.nil? }
 
-        @client.execute(
+        execute(
           api_method: @bigquery.datasets.list,
           parameters: params
         )
@@ -59,7 +60,7 @@ module Gcloud
       ##
       # Returns the dataset specified by datasetID.
       def get_dataset dataset_id
-        @client.execute(
+        execute(
           api_method: @bigquery.datasets.get,
           parameters: { projectId: @project, datasetId: dataset_id }
         )
@@ -68,7 +69,7 @@ module Gcloud
       ##
       # Creates a new empty dataset.
       def insert_dataset dataset_id, options = {}
-        @client.execute(
+        execute(
           api_method: @bigquery.datasets.insert,
           parameters: { projectId: @project },
           body_object: insert_dataset_request(dataset_id, options)
@@ -81,7 +82,7 @@ module Gcloud
       def patch_dataset dataset_id, options = {}
         project_id = options[:project_id] || @project
 
-        @client.execute(
+        execute(
           api_method: @bigquery.datasets.patch,
           parameters: { projectId: project_id, datasetId: dataset_id },
           body_object: patch_dataset_request(options)
@@ -95,7 +96,7 @@ module Gcloud
       # Immediately after deletion, you can create another dataset with
       # the same name.
       def delete_dataset dataset_id, force = nil
-        @client.execute(
+        execute(
           api_method: @bigquery.datasets.delete,
           parameters: { projectId: @project, datasetId: dataset_id,
                         deleteContents: force
@@ -113,14 +114,14 @@ module Gcloud
                    maxResults: options.delete(:max)
                  }.delete_if { |_, v| v.nil? }
 
-        @client.execute(
+        execute(
           api_method: @bigquery.tables.list,
           parameters: params
         )
       end
 
       def get_project_table project_id, dataset_id, table_id
-        @client.execute(
+        execute(
           api_method: @bigquery.tables.get,
           parameters: { projectId: project_id, datasetId: dataset_id,
                         tableId: table_id }
@@ -139,7 +140,7 @@ module Gcloud
       ##
       # Creates a new, empty table in the dataset.
       def insert_table dataset_id, table_id, options = {}
-        @client.execute(
+        execute(
           api_method: @bigquery.tables.insert,
           parameters: { projectId: @project, datasetId: dataset_id },
           body_object: insert_table_request(dataset_id, table_id, options)
@@ -150,7 +151,7 @@ module Gcloud
       # Updates information in an existing table, replacing fields that
       # are provided in the submitted table resource.
       def patch_table dataset_id, table_id, options = {}
-        @client.execute(
+        execute(
           api_method: @bigquery.tables.patch,
           parameters: { projectId: @project, datasetId: dataset_id,
                         tableId: table_id },
@@ -162,7 +163,7 @@ module Gcloud
       # Deletes the table specified by tableId from the dataset.
       # If the table contains data, all the data will be deleted.
       def delete_table dataset_id, table_id
-        @client.execute(
+        execute(
           api_method: @bigquery.tables.delete,
           parameters: { projectId: @project, datasetId: dataset_id,
                         tableId: table_id }
@@ -179,14 +180,14 @@ module Gcloud
                    startIndex: options.delete(:start)
                  }.delete_if { |_, v| v.nil? }
 
-        @client.execute(
+        execute(
           api_method: @bigquery.tabledata.list,
           parameters: params
         )
       end
 
       def insert_tabledata dataset_id, table_id, rows, options = {}
-        @client.execute(
+        execute(
           api_method: @bigquery.tabledata.insert_all,
           parameters: { projectId: @project,
                         datasetId: dataset_id,
@@ -199,7 +200,7 @@ module Gcloud
       # Lists all jobs in the specified project to which you have
       # been granted the READER job role.
       def list_jobs options = {}
-        @client.execute(
+        execute(
           api_method: @bigquery.jobs.list,
           parameters: list_jobs_params(options)
         )
@@ -208,14 +209,14 @@ module Gcloud
       ##
       # Returns the job specified by jobID.
       def get_job job_id
-        @client.execute(
+        execute(
           api_method: @bigquery.jobs.get,
           parameters: { projectId: @project, jobId: job_id }
         )
       end
 
       def insert_job config
-        @client.execute(
+        execute(
           api_method: @bigquery.jobs.insert,
           parameters: { projectId: @project },
           body_object: { "configuration" => config }
@@ -223,7 +224,7 @@ module Gcloud
       end
 
       def query_job query, options = {}
-        @client.execute(
+        execute(
           api_method: @bigquery.jobs.insert,
           parameters: { projectId: @project },
           body_object: query_table_config(query, options)
@@ -231,7 +232,7 @@ module Gcloud
       end
 
       def query query, options = {}
-        @client.execute(
+        execute(
           api_method: @bigquery.jobs.query,
           parameters: { projectId: @project },
           body_object: query_config(query, options)
@@ -248,14 +249,14 @@ module Gcloud
                    timeoutMs: options.delete(:timeout)
                  }.delete_if { |_, v| v.nil? }
 
-        @client.execute(
+        execute(
           api_method: @bigquery.jobs.get_query_results,
           parameters: params
         )
       end
 
       def copy_table source, target, options = {}
-        @client.execute(
+        execute(
           api_method: @bigquery.jobs.insert,
           parameters: { projectId: @project },
           body_object: copy_table_config(source, target, options)
@@ -263,7 +264,7 @@ module Gcloud
       end
 
       def link_table table, urls, options = {}
-        @client.execute(
+        execute(
           api_method: @bigquery.jobs.insert,
           parameters: { projectId: @project },
           body_object: link_table_config(table, urls, options)
@@ -271,7 +272,7 @@ module Gcloud
       end
 
       def extract_table table, storage_files, options = {}
-        @client.execute(
+        execute(
           api_method: @bigquery.jobs.insert,
           parameters: { projectId: @project },
           body_object: extract_table_config(table, storage_files, options)
@@ -279,7 +280,7 @@ module Gcloud
       end
 
       def load_table table, storage_url, options = {}
-        @client.execute(
+        execute(
           api_method: @bigquery.jobs.insert,
           parameters: { projectId: @project },
           body_object: load_table_config(table, storage_url,
@@ -290,7 +291,7 @@ module Gcloud
       def load_multipart table, file, options = {}
         media = load_media file
 
-        @client.execute(
+        execute(
           api_method: @bigquery.jobs.insert,
           media: media,
           parameters: { projectId: @project, uploadType: "multipart" },
@@ -301,14 +302,14 @@ module Gcloud
       def load_resumable table, file, chunk_size = nil, options = {}
         media = load_media file, chunk_size
 
-        result = @client.execute(
+        result = execute(
           api_method: @bigquery.jobs.insert,
           media: media,
           parameters: { projectId: @project, uploadType: "resumable" },
           body_object: load_table_config(table, nil, file, options)
         )
         upload = result.resumable_upload
-        result = @client.execute upload while upload.resumable?
+        result = execute upload while upload.resumable?
         result
       end
 
@@ -612,6 +613,12 @@ module Gcloud
         media.chunk_size = chunk_size unless chunk_size.nil?
         media
       end
+
+      def execute options
+        Gcloud::Backoff.new.execute_gapi do
+          @client.execute options
+        end
+      end
     end
   end
 end

data/lib/gcloud/bigquery/data.rb

@@ -61,17 +61,115 @@ module Gcloud
       end
 
       ##
-      # Is there a next page of data?
+      # Whether there is a next page of data.
+      #
+      # @return [Boolean]
+      #
+      # @example
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #   table = dataset.table "my_table"
+      #
+      #   data = table.data
+      #   if data.next?
+      #     next_data = data.next
+      #   end
+      #
       def next?
         !token.nil?
       end
 
+      ##
+      # Retrieve the next page of data.
+      #
+      # @return [Data]
+      #
+      # @example
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #   table = dataset.table "my_table"
+      #
+      #   data = table.data
+      #   if data.next?
+      #     next_data = data.next
+      #   end
+      #
       def next
         return nil unless next?
         ensure_table!
         table.data token: token
       end
 
+      ##
+      # Retrieves all rows by repeatedly loading {#next} until {#next?} returns
+      # `false`. Calls the given block once for each row, which is passed as the
+      # parameter.
+      #
+      # An Enumerator is returned if no block is given.
+      #
+      # This method may make several API calls until all rows are retrieved. Be
+      # sure to use as narrow a search criteria as possible. Please use with
+      # caution.
+      #
+      # @param [Integer] request_limit The upper limit of API requests to make
+      #   to load all data. Default is no limit.
+      # @yield [row] The block for accessing each row of data.
+      # @yieldparam [Hash] row The row object.
+      #
+      # @return [Enumerator]
+      #
+      # @example Iterating each rows by passing a block:
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #   table = dataset.table "my_table"
+      #
+      #   table.data.all do |row|
+      #     puts row["word"]
+      #   end
+      #
+      # @example Using the enumerator by not passing a block:
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #   table = dataset.table "my_table"
+      #
+      #   words = table.data.all.map do |row|
+      #     row["word"]
+      #   end
+      #
+      # @example Limit the number of API calls made:
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #   table = dataset.table "my_table"
+      #
+      #   table.data.all(request_limit: 10) do |row|
+      #     puts row["word"]
+      #   end
+      #
+      def all request_limit: nil
+        request_limit = request_limit.to_i if request_limit
+        return enum_for(:all, request_limit: request_limit) unless block_given?
+        results = self
+        loop do
+          results.each { |r| yield r }
+          if request_limit
+            request_limit -= 1
+            break if request_limit < 0
+          end
+          break unless results.next?
+          results = results.next
+        end
+      end
+
       ##
       # Represents Table Data as a list of positional values (array of arrays).
       # No type conversion is made, e.g. numbers are formatted as strings.