gcloud 0.11.0 → 0.12.0

data/lib/gcloud/datastore.rb

@@ -39,6 +39,9 @@ module Gcloud
   #   The default scope is:
   #
   #   * `https://www.googleapis.com/auth/datastore`
+  # @param [Integer] retries Number of times to retry requests on server error.
+  #   The default value is `3`. Optional.
+  # @param [Integer] timeout Default timeout to use in requests. Optional.
   #
   # @return [Gcloud::Datastore::Dataset]
   #
@@ -57,19 +60,28 @@ module Gcloud
   #
   #   datastore.save task
   #
-  def self.datastore project = nil, keyfile = nil, scope: nil
+  def self.datastore project = nil, keyfile = nil, scope: nil, retries: nil,
+                     timeout: nil
     project ||= Gcloud::Datastore::Dataset.default_project
+    project = project.to_s # Always cast to a string
+    fail ArgumentError, "project is missing" if project.empty?
+
     if ENV["DATASTORE_EMULATOR_HOST"]
-      ds = Gcloud::Datastore::Dataset.new project, :this_channel_is_insecure
-      ds.service.host = ENV["DATASTORE_EMULATOR_HOST"]
-      return ds
+      return Gcloud::Datastore::Dataset.new(
+        Gcloud::Datastore::Service.new(
+          project, :this_channel_is_insecure,
+          host: ENV["DATASTORE_EMULATOR_HOST"], retries: retries))
     end
+
     if keyfile.nil?
       credentials = Gcloud::Datastore::Credentials.default scope: scope
     else
       credentials = Gcloud::Datastore::Credentials.new keyfile, scope: scope
     end
-    Gcloud::Datastore::Dataset.new project, credentials
+
+    Gcloud::Datastore::Dataset.new(
+      Gcloud::Datastore::Service.new(
+        project, credentials, retries: retries, timeout: timeout))
   end
 
   ##
@@ -80,7 +92,7 @@ module Gcloud
   # relational databases, but there are some key differences to be aware of to
   # make the most of using Datastore.
   #
-  # Gcloud's goal is to provide a API that is familiar and comfortable to
+  # The goal of gcloud-ruby is to provide a API that is comfortable to
   # Rubyists. Authentication is handled by {Gcloud#datastore}. You can provide
   # the project and credential information to connect to the Datastore service,
   # or if you are running on Google Compute Engine this configuration is taken
@@ -507,43 +519,43 @@ module Gcloud
   # end
   # ```
   #
-  # ## Configuring Backoff
+  # ## Configuring retries and timeout
   #
-  # The {Gcloud::Backoff} class allows users to globally configure how Cloud API
-  # requests are automatically retried in the case of some errors, such as a
-  # `500` or `503` status code, or a specific internal error code such as
-  # `rateLimitExceeded`.
+  # You can configure how many times API requests may be automatically retried.
+  # When an API request fails, the response will be inspected to see if the
+  # request meets criteria indicating that it may succeed on retry, such as
+  # `500` and `503` status codes or a specific internal error code such as
+  # `rateLimitExceeded`. If it meets the criteria, the request will be retried
+  # after a delay. If another error occurs, the delay will be increased before a
+  # subsequent attempt, until the `retries` limit is reached.
   #
-  # If an API call fails, the response will be inspected to see if the call
-  # should be retried. If the response matches the criteria, then the request
-  # will be retried after a delay. If another error occurs, the delay will be
-  # increased incrementally before a subsequent attempt. The first retry will be
-  # delayed one second, the second retry two seconds, and so on.
+  # You can also set the request `timeout` value in seconds.
   #
   # ```ruby
   # require "gcloud"
-  # require "gcloud/backoff"
   #
-  # Gcloud::Backoff.retries = 5 # Raise the maximum number of retries from 3
+  # gcloud = Gcloud.new
+  # datastore = gcloud.datastore retries: 10, timeout: 120
   # ```
   #
   # See the [Datastore error
   # codes](https://cloud.google.com/datastore/docs/concepts/errors#error_codes)
   # for a list of error conditions.
   #
-  # ## The Datastore Emulator
+  # ## The Cloud Datastore Emulator
   #
-  # As of this release, the Datastore emulator that is part of the gcloud SDK is
-  # no longer compatible with gcloud-ruby. This is because the gcloud SDK's
-  # Datastore emulator does not yet support gRPC as a transport layer.
+  # As of this release, the Cloud Datastore emulator that is part of the gcloud
+  # SDK is no longer compatible with gcloud-ruby. This is because the gcloud
+  # SDK's Cloud Datastore emulator does not yet support gRPC as a transport
+  # layer.
   #
-  # A gRPC-compatible emulator is available until the gcloud SDK Datastore
+  # A gRPC-compatible emulator is available until the gcloud SDK Cloud Datastore
   # emulator supports gRPC. To use it you must [download the gRPC
-  # emulator](https://storage.googleapis.com/gcd/tools/gcd-grpc-1.0.0.zip) and
-  # use the `gcd.sh` script.
+  # emulator](https://storage.googleapis.com/gcd/tools/cloud-datastore-emulator-1.1.1.zip)
+  # and use the `cloud_datastore_emulator` script.
   #
-  # When you run the gRPC emulator you will see a message similar to the
-  # following printed:
+  # When you run the Cloud Datastore emulator you will see a message similar to
+  # the following printed:
   #
   # ```
   # If you are using a library that supports the DATASTORE_EMULATOR_HOST

data/lib/gcloud/datastore/commit.rb

@@ -29,8 +29,8 @@ module Gcloud
     #     c.delete entity1, entity2
     #   end
     #
-    # @see {Gcloud::Datastore::Dataset#commit}
-    # @see {Gcloud::Datastore::Transaction#commit}
+    # See {Gcloud::Datastore::Dataset#commit} and
+    # {Gcloud::Datastore::Transaction#commit}.
     class Commit
       ##
       # @private Create a new Commit object.

data/lib/gcloud/datastore/dataset.rb

@@ -60,10 +60,8 @@ module Gcloud
       # @private Creates a new Dataset instance.
       #
       # See {Gcloud#datastore}
-      def initialize project, credentials
-        project = project.to_s # Always cast to a string
-        fail ArgumentError, "project is missing" if project.empty?
-        @service = Service.new project, credentials
+      def initialize service
+        @service = service
       end
 
       ##
@@ -106,15 +104,14 @@ module Gcloud
       #
       def allocate_ids incomplete_key, count = 1
         if incomplete_key.complete?
-          fail Gcloud::Datastore::Error, "An incomplete key must be provided."
+          fail Gcloud::Datastore::KeyError,
+               "An incomplete key must be provided."
         end
 
         ensure_service!
         incomplete_keys = count.times.map { incomplete_key.to_grpc }
         allocate_res = service.allocate_ids(*incomplete_keys)
         allocate_res.keys.map { |key| Key.from_grpc key }
-      rescue GRPC::BadStatus => e
-        raise Gcloud::Error.from_error(e)
       end
 
       ##
@@ -275,8 +272,6 @@ module Gcloud
         end
         entities.each { |e| e.key.freeze unless e.persisted? }
         entities
-      rescue GRPC::BadStatus => e
-        raise Gcloud::Error.from_error(e)
       end
 
       ##
@@ -342,8 +337,6 @@ module Gcloud
         lookup_res = service.lookup(*Array(keys).flatten.map(&:to_grpc),
                                     consistency: consistency)
         LookupResults.from_grpc lookup_res, service, consistency
-      rescue GRPC::BadStatus => e
-        raise Gcloud::Error.from_error(e)
       end
       alias_method :lookup, :find_all
 
@@ -399,8 +392,6 @@ module Gcloud
         query_res = service.run_query query.to_grpc, namespace,
                                       consistency: consistency
         QueryResults.from_grpc query_res, service, namespace, query.to_grpc.dup
-      rescue GRPC::BadStatus => e
-        raise Gcloud::Error.from_error(e)
       end
       alias_method :run_query, :run
 
@@ -459,15 +450,13 @@ module Gcloud
         begin
           yield tx
           tx.commit
-        rescue => e
+        rescue
           begin
             tx.rollback
-          rescue => re
-            msg = "Transaction failed to commit and rollback."
-            raise TransactionError.new(msg, commit_error: e, rollback_error: re)
+          rescue
+            raise TransactionError, "Transaction failed to commit and rollback."
           end
-          raise TransactionError.new("Transaction failed to commit.",
-                                     commit_error: e)
+          raise TransactionError, "Transaction failed to commit."
         end
       end
 

data/lib/gcloud/datastore/dataset/lookup_results.rb

@@ -101,8 +101,6 @@ module Gcloud
                                        consistency: @consistency,
                                        transaction: @transaction)
           self.class.from_grpc lookup_res, @service, @consistency
-        rescue GRPC::BadStatus => e
-          raise Gcloud::Error.from_error(e)
         end
 
         ##
@@ -200,7 +198,7 @@ module Gcloud
         end
 
         ##
-        # Convenience method to convert GRPC entities to Gcloud entities.
+        # Convenience method to convert GRPC entities to gcloud-ruby entities.
         def self.to_gcloud_entities grpc_entity_results
           # Entities are nested in an object.
           Array(grpc_entity_results).map do |result|
@@ -210,7 +208,7 @@ module Gcloud
         end
 
         ##
-        # Convenience method to convert GRPC keys to Gcloud keys.
+        # Convenience method to convert GRPC keys to gcloud-ruby keys.
         def self.to_gcloud_keys grpc_keys
           # Keys are not nested in an object like entities are.
           Array(grpc_keys).map { |key| Key.from_grpc key }

data/lib/gcloud/datastore/dataset/query_results.rb

@@ -160,8 +160,6 @@ module Gcloud
           query.start_cursor = cursor.to_grpc # should always be a Cursor...
           query_res = service.run_query query, namespace
           self.class.from_grpc query_res, service, namespace, query
-        rescue GRPC::BadStatus => e
-          raise Gcloud::Error.from_error(e)
         end
 
         ##

data/lib/gcloud/datastore/entity.rb

@@ -350,6 +350,12 @@ module Gcloud
         end
       end
 
+      ##
+      # The number of bytes the Entity will take to serialize during API calls.
+      def serialized_size
+        to_grpc.to_proto.length
+      end
+
       ##
       # @private Convert the Entity to a Google::Datastore::V1beta3::Entity
       # object.
@@ -415,7 +421,7 @@ module Gcloud
           next if value.nil?
           @_exclude_indexes[name] = value.exclude_from_indexes
           unless value.array_value.nil?
-            exclude = value.array_value.values.map &:exclude_from_indexes
+            exclude = value.array_value.values.map(&:exclude_from_indexes)
             @_exclude_indexes[name] = exclude
           end
         end

data/lib/gcloud/datastore/errors.rb

@@ -18,46 +18,24 @@ require "gcloud/errors"
 module Gcloud
   module Datastore
     ##
-    # # Datastore Error
+    # # KeyError
     #
-    # Base Datastore exception class.
-    class Error < Gcloud::Error
-    end
-
-    ##
-    # # KeyfileError
-    #
-    # Raised when a keyfile is not correct.
-    class KeyfileError < Gcloud::Datastore::Error
+    # Raised when a key is not correct.
+    class KeyError < Gcloud::Error
     end
 
     ##
     # # PropertyError
     #
     # Raised when a property is not correct.
-    class PropertyError < Gcloud::Datastore::Error
+    class PropertyError < Gcloud::Error
     end
 
     ##
     # # TransactionError
     #
     # General error for Transaction problems.
-    class TransactionError < Gcloud::Datastore::Error
-      ##
-      # An error that occurred within the transaction. (optional)
-      attr_reader :commit_error
-      alias_method :inner, :commit_error # backwards compatibility
-
-      ##
-      # An error that occurred within the transaction. (optional)
-      attr_reader :rollback_error
-
-      # @private
-      def initialize message, commit_error: nil, rollback_error: nil
-        super(message)
-        @commit_error   = commit_error
-        @rollback_error = rollback_error
-      end
+    class TransactionError < Gcloud::Error
     end
   end
 end

data/lib/gcloud/datastore/grpc_utils.rb

@@ -16,6 +16,7 @@
 require "gcloud/grpc_utils"
 require "gcloud/datastore/errors"
 require "stringio"
+require "base64"
 
 module Gcloud
   ##
@@ -102,7 +103,7 @@ module Gcloud
         v.string_value = value
       elsif Array === value
         v.array_value = Google::Datastore::V1beta3::ArrayValue.new(
-          values: value.map { |v| to_value v }
+          values: value.map { |val| to_value val }
         )
       elsif value.respond_to? :to_time
         v.timestamp_value = Google::Protobuf::Timestamp.new(
@@ -120,11 +121,11 @@ module Gcloud
     end
 
     def self.encode_bytes bytes
-      Array(bytes.to_s).pack("m").chomp.encode("ASCII-8BIT")
+      Base64.strict_encode64(bytes.to_s).encode("ASCII-8BIT")
     end
 
     def self.decode_bytes bytes
-      bytes.to_s.unpack("m").first.force_encoding Encoding::ASCII_8BIT
+      Base64.decode64(bytes.to_s).force_encoding Encoding::ASCII_8BIT
     end
     # rubocop:enable all
   end

data/lib/gcloud/datastore/key.rb

@@ -232,6 +232,12 @@ module Gcloud
         kind.nil? || (id.nil? && (name.nil? || name.empty?))
       end
 
+      ##
+      # The number of bytes the Key will take to serialize during API calls.
+      def serialized_size
+        to_grpc.to_proto.length
+      end
+
       ##
       # @private Convert the Key to a Google::Datastore::V1beta3::Key object.
       def to_grpc

data/lib/gcloud/datastore/service.rb

@@ -23,14 +23,16 @@ module Gcloud
     # @private Represents the gRPC Datastore service, including all the API
     # methods.
     class Service
-      attr_accessor :project, :credentials, :host
+      attr_accessor :project, :credentials, :host, :retries, :timeout
 
       ##
       # Creates a new Service instance.
-      def initialize project, credentials
+      def initialize project, credentials, host: nil, retries: nil, timeout: nil
         @project = project
         @credentials = credentials
-        @host = "datastore.googleapis.com"
+        @host = host || "datastore.googleapis.com"
+        @retries = retries
+        @timeout = timeout
       end
 
       def creds
@@ -42,7 +44,7 @@ module Gcloud
       def datastore
         return mocked_datastore if mocked_datastore
         @datastore ||= Google::Datastore::V1beta3::Datastore::Stub.new(
-          host, creds)
+          host, creds, timeout: timeout)
       end
       attr_accessor :mocked_datastore
 
@@ -59,7 +61,7 @@ module Gcloud
           keys: incomplete_keys
         )
 
-        backoff { datastore.allocate_ids allocate_req }
+        execute { datastore.allocate_ids allocate_req }
       end
 
       ##
@@ -71,7 +73,7 @@ module Gcloud
         )
         lookup_req.read_options = generate_read_options consistency, transaction
 
-        backoff { datastore.lookup lookup_req }
+        execute { datastore.lookup lookup_req }
       end
 
       # Query for entities.
@@ -90,7 +92,7 @@ module Gcloud
         run_req.partition_id = Google::Datastore::V1beta3::PartitionId.new(
           namespace_id: namespace) if namespace
 
-        backoff { datastore.run_query run_req }
+        execute { datastore.run_query run_req }
       end
 
       ##
@@ -100,7 +102,7 @@ module Gcloud
           project_id: project
         )
 
-        backoff { datastore.begin_transaction tx_req }
+        execute { datastore.begin_transaction tx_req }
       end
 
       ##
@@ -117,7 +119,7 @@ module Gcloud
           commit_req.transaction = transaction
         end
 
-        backoff { datastore.commit commit_req }
+        execute { datastore.commit commit_req }
       end
 
       ##
@@ -128,17 +130,21 @@ module Gcloud
           transaction: transaction
         )
 
-        backoff { datastore.rollback rb_req }
+        execute { datastore.rollback rb_req }
       end
 
       def inspect
         "#{self.class}(#{@project})"
       end
 
-      def backoff options = {}
-        Gcloud::Backoff.new(options).execute_grpc do
+      ##
+      # Performs backoff and error handling
+      def execute
+        Gcloud::Backoff.new(retries: retries).execute_grpc do
           yield
         end
+      rescue GRPC::BadStatus => e
+        raise Gcloud::Error.from_error(e)
       end
 
       protected