gcloud 0.8.2 → 0.9.0

data/lib/gcloud/datastore/properties.rb

@@ -71,6 +71,17 @@ module Gcloud
       end
       alias_method :to_hash, :to_h
 
+      def to_grpc
+        Hash[@hash.map { |(k, v)| [k.to_s, GRPCUtils.to_value(v)] }]
+      end
+
+      def self.from_grpc grpc_map
+        # For some reason Google::Protobuf::Map#map isn't returning the value.
+        # It returns nil every time. COnvert to Hash to get actual objects.
+        grpc_hash = GRPCUtils.map_to_hash grpc_map
+        new Hash[grpc_hash.map { |(k, v)| [k.to_s, GRPCUtils.from_value(v)] }]
+      end
+
       protected
 
       ##
@@ -82,7 +93,7 @@ module Gcloud
       end
 
       # rubocop:disable all
-      # Disabled rubocop because this needs to match Proto.to_proto_value
+      # Disabled rubocop because this needs to match GRPCUtils.to_value
 
       ##
       # Ensures the value is a type that can be persisted,
@@ -100,6 +111,8 @@ module Gcloud
           return value
         elsif value.respond_to?(:to_time)
           return value
+        elsif value.respond_to?(:to_hash) && value.keys.sort == [:latitude, :longitude]
+          return value
         elsif value.respond_to?(:read) && value.respond_to?(:rewind)
           # Always convert an IO object to a StringIO when storing.
           value.rewind

data/lib/gcloud/datastore/query.rb

@@ -15,7 +15,6 @@
 
 require "gcloud/datastore/entity"
 require "gcloud/datastore/key"
-require "gcloud/datastore/proto"
 
 module Gcloud
   module Datastore
@@ -24,12 +23,19 @@ module Gcloud
     #
     # Represents the search criteria against a Datastore.
     #
+    # @see https://cloud.google.com/datastore/docs/concepts/queries Datastore
+    #   Queries
+    # @see https://cloud.google.com/datastore/docs/concepts/metadataqueries
+    #   Datastore Metadata
+    #
     # @example
     #   query = Gcloud::Datastore::Query.new
     #   query.kind("Task").
-    #     where("completed", "=", true)
+    #     where("done", "=", false).
+    #     where("priority", ">=", 4).
+    #     order("priority", :desc)
     #
-    #   entities = dataset.run query
+    #   tasks = datastore.run query
     #
     class Query
       ##
@@ -39,22 +45,28 @@ module Gcloud
       #   query = Gcloud::Datastore::Query.new
       #
       def initialize
-        @_query = Proto::Query.new
+        @grpc = Google::Datastore::V1beta3::Query.new
       end
 
       ##
       # Add the kind of entities to query.
       #
+      # Special entity kinds such as `__namespace__`, `__kind__`, and
+      # `__property__` can be used for [metadata
+      # queries](https://cloud.google.com/datastore/docs/concepts/metadataqueries).
+      #
       # @example
       #   query = Gcloud::Datastore::Query.new
       #   query.kind "Task"
       #
-      #   all_tasks = dataset.run query
+      #   tasks = datastore.run query
       #
       def kind *kinds
-        @_query.kind ||= Proto::KindExpression.new
-        @_query.kind.name ||= []
-        @_query.kind.name |= kinds
+        kinds.each do |kind|
+          grpc_kind = Google::Datastore::V1beta3::KindExpression.new(name: kind)
+          @grpc.kind << grpc_kind
+        end
+
         self
       end
 
@@ -64,21 +76,72 @@ module Gcloud
       # @example
       #   query = Gcloud::Datastore::Query.new
       #   query.kind("Task").
-      #     where("completed", "=", true)
+      #     where("done", "=", false)
+      #
+      #   tasks = datastore.run query
+      #
+      # @example Add a composite property filter:
+      #   query = Gcloud::Datastore::Query.new
+      #   query.kind("Task").
+      #     where("done", "=", false).
+      #     where("priority", ">=", 4)
+      #
+      #   tasks = datastore.run query
+      #
+      # @example Add an inequality filter on a **single** property only:
+      #   query = Gcloud::Datastore::Query.new
+      #   query.kind("Task").
+      #     where("created", ">=", Time.utc(1990, 1, 1)).
+      #     where("created", "<", Time.utc(2000, 1, 1))
+      #
+      #   tasks = datastore.run query
+      #
+      # @example Add a composite filter on an array property:
+      #   query = Gcloud::Datastore::Query.new
+      #   query.kind("Task").
+      #     where("tag", "=", "fun").
+      #     where("tag", "=", "programming")
       #
-      #   completed_tasks = dataset.run query
+      #   tasks = datastore.run query
+      #
+      # @example Add an inequality filter on an array property :
+      #   query = Gcloud::Datastore::Query.new
+      #   query.kind("Task").
+      #     where("tag", ">", "learn").
+      #     where("tag", "<", "math")
+      #
+      #   tasks = datastore.run query
+      #
+      # @example Add a key filter using the special property `__key__`:
+      #   query = Gcloud::Datastore::Query.new
+      #   query.kind("Task").
+      #     where("__key__", ">", datastore.key("Task", "someTask"))
+      #
+      #   tasks = datastore.run query
+      #
+      # @example Add a key filter to a *kindless* query:
+      #   last_seen_key = datastore.key "Task", "a"
+      #   query = Gcloud::Datastore::Query.new
+      #   query.where("__key__", ">", last_seen_key)
+      #
+      #   tasks = datastore.run query
       #
       def where name, operator, value
-        # Initialize filter
-        @_query.filter ||= Proto.new_filter.tap do |f|
-          f.composite_filter = Proto.new_composite_filter
-        end
-        # Create new property filter
-        filter = Proto.new_filter.tap do |f|
-          f.property_filter = Proto.new_property_filter name, operator, value
-        end
-        # Add new property filter to the list
-        @_query.filter.composite_filter.filter << filter
+        @grpc.filter ||= Google::Datastore::V1beta3::Filter.new(
+          composite_filter: Google::Datastore::V1beta3::CompositeFilter.new(
+            op: :AND
+          )
+        )
+        @grpc.filter.composite_filter.filters << \
+          Google::Datastore::V1beta3::Filter.new(
+            property_filter: Google::Datastore::V1beta3::PropertyFilter.new(
+              property: Google::Datastore::V1beta3::PropertyReference.new(
+                name: name),
+              op: GRPCUtils.to_prop_filter_op(operator),
+              value: GRPCUtils.to_value(value)
+            )
+          )
+
         self
       end
       alias_method :filter, :where
@@ -87,11 +150,13 @@ module Gcloud
       # Add a filter for entities that inherit from a key.
       #
       # @example
+      #   task_list_key = datastore.key "TaskList", "default"
+      #
       #   query = Gcloud::Datastore::Query.new
       #   query.kind("Task").
-      #     ancestor(parent.key)
+      #     ancestor(task_list_key)
       #
-      #   completed_tasks = dataset.run query
+      #   tasks = datastore.run query
       #
       def ancestor parent
         # Use key if given an entity
@@ -105,20 +170,44 @@ module Gcloud
       # To sort in descending order, provide a second argument
       # of a string or symbol that starts with "d".
       #
-      # @example
+      # @example With ascending sort order:
+      #   query = Gcloud::Datastore::Query.new
+      #   query.kind("Task").
+      #     order("created")
+      #
+      #   tasks = datastore.run query
+      #
+      # @example With descending sort order:
+      #   query = Gcloud::Datastore::Query.new
+      #   query.kind("Task").
+      #     order("created", :desc)
+      #
+      #   tasks = datastore.run query
+      #
+      # @example With multiple sort orders:
+      #   query = Gcloud::Datastore::Query.new
+      #   query.kind("Task").
+      #     order("priority", :desc).
+      #     order("created")
+      #
+      #   tasks = datastore.run query
+      #
+      # @example A property used in an inequality filter must be ordered first:
       #   query = Gcloud::Datastore::Query.new
       #   query.kind("Task").
-      #     order("due", :desc)
+      #     where("priority", ">", 3).
+      #     order("priority").
+      #     order("created")
       #
-      #   sorted_tasks = dataset.run query
+      #   tasks = datastore.run query
       #
       def order name, direction = :asc
-        @_query.order ||= []
-        po = Proto::PropertyOrder.new
-        po.property = Proto::PropertyReference.new
-        po.property.name = name
-        po.direction = Proto.to_prop_order_direction direction
-        @_query.order << po
+        @grpc.order << Google::Datastore::V1beta3::PropertyOrder.new(
+          property: Google::Datastore::V1beta3::PropertyReference.new(
+            name: name),
+          direction: prop_order_direction(direction)
+        )
+
         self
       end
 
@@ -128,12 +217,13 @@ module Gcloud
       # @example
       #   query = Gcloud::Datastore::Query.new
       #   query.kind("Task").
-      #     limit(10)
+      #     limit(5)
       #
-      #   paginated_tasks = dataset.run query
+      #   tasks = datastore.run query
       #
       def limit num
-        @_query.limit = num
+        @grpc.limit = Google::Protobuf::Int32Value.new(value: num)
+
         self
       end
 
@@ -143,13 +233,14 @@ module Gcloud
       # @example
       #   query = Gcloud::Datastore::Query.new
       #   query.kind("Task").
-      #     limit(10).
-      #     offset(20)
+      #     limit(5).
+      #     offset(10)
       #
-      #   paginated_tasks = dataset.run query
+      #   tasks = datastore.run query
       #
       def offset num
-        @_query.offset = num
+        @grpc.offset = num
+
         self
       end
 
@@ -159,13 +250,20 @@ module Gcloud
       # @example
       #   query = Gcloud::Datastore::Query.new
       #   query.kind("Task").
-      #     limit(10).
-      #     cursor(task_cursor)
+      #     limit(page_size).
+      #     start(page_cursor)
       #
-      #   paginated_tasks = dataset.run query
+      #   tasks = datastore.run query
       #
       def start cursor
-        @_query.start_cursor = Proto.decode_cursor cursor
+        if cursor.is_a? Cursor
+          @grpc.start_cursor = cursor.to_grpc
+        elsif cursor.is_a? String
+          @grpc.start_cursor = GRPCUtils.decode_bytes cursor
+        else
+          fail ArgumentError, "Can't set a cursor using a #{cursor.class}."
+        end
+
         self
       end
       alias_method :cursor, :start
@@ -176,13 +274,30 @@ module Gcloud
       # @example
       #   query = Gcloud::Datastore::Query.new
       #   query.kind("Task").
-      #     select("completed", "due")
+      #     select("priority", "percent_complete")
+      #
+      #   priorities = []
+      #   percent_completes = []
+      #   datastore.run(query).each do |task|
+      #     priorities << task["priority"]
+      #     percent_completes << task["percent_complete"]
+      #   end
+      #
+      # @example A keys-only query using the special property `__key__`:
+      #   query = Gcloud::Datastore::Query.new
+      #   query.kind("Task").
+      #     select("__key__")
       #
-      #   partial_tasks = dataset.run query
+      #   keys = datastore.run(query).map(&:key)
       #
       def select *names
-        @_query.projection ||= []
-        @_query.projection += Proto.new_property_expressions(*names)
+        names.each do |name|
+          grpc_projection = Google::Datastore::V1beta3::Projection.new(
+            property: Google::Datastore::V1beta3::PropertyReference.new(
+              name: name))
+          @grpc.projection << grpc_projection
+        end
+
         self
       end
       alias_method :projection, :select
@@ -193,19 +308,40 @@ module Gcloud
       # @example
       #   query = Gcloud::Datastore::Query.new
       #   query.kind("Task").
-      #     group_by("completed")
+      #     distinct_on("type", "priority").
+      #     order("type").
+      #     order("priority")
       #
-      #   grouped_tasks = dataset.run query
+      #   tasks = datastore.run query
       #
       def group_by *names
-        @_query.group_by ||= []
-        @_query.group_by += Proto.new_property_references(*names)
+        names.each do |name|
+          grpc_property = Google::Datastore::V1beta3::PropertyReference.new(
+            name: name)
+          @grpc.distinct_on << grpc_property
+        end
+
         self
       end
+      alias_method :distinct_on, :group_by
 
       # @private
-      def to_proto
-        @_query
+      def to_grpc
+        @grpc
+      end
+
+      protected
+
+      ##
+      # @private Get the property order direction for a string.
+      def prop_order_direction direction
+        if direction.to_s.downcase.start_with? "a"
+          :ASCENDING
+        elsif direction.to_s.downcase.start_with? "d"
+          :DESCENDING
+        else
+          :DIRECTION_UNSPECIFIED
+        end
       end
     end
   end

data/lib/gcloud/datastore/service.rb

@@ -0,0 +1,161 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "gcloud/datastore/credentials"
+require "google/datastore/v1beta3/datastore_services"
+require "gcloud/backoff"
+
+module Gcloud
+  module Datastore
+    ##
+    # @private Represents the gRPC Datastore service, including all the API
+    # methods.
+    class Service
+      attr_accessor :project, :credentials, :host
+
+      ##
+      # Creates a new Service instance.
+      def initialize project, credentials
+        @project = project
+        @credentials = credentials
+        @host = "datastore.googleapis.com"
+      end
+
+      def creds
+        return credentials if insecure?
+        GRPC::Core::ChannelCredentials.new.compose \
+          GRPC::Core::CallCredentials.new credentials.client.updater_proc
+      end
+
+      def datastore
+        return mocked_datastore if mocked_datastore
+        @datastore ||= Google::Datastore::V1beta3::Datastore::Stub.new(
+          host, creds)
+      end
+      attr_accessor :mocked_datastore
+
+      def insecure?
+        credentials == :this_channel_is_insecure
+      end
+
+      ##
+      # Allocate IDs for incomplete keys.
+      # (This is useful for referencing an entity before it is inserted.)
+      def allocate_ids *incomplete_keys
+        allocate_req = Google::Datastore::V1beta3::AllocateIdsRequest.new(
+          project_id: project,
+          keys: incomplete_keys
+        )
+
+        backoff { datastore.allocate_ids allocate_req }
+      end
+
+      ##
+      # Look up entities by keys.
+      def lookup *keys, consistency: nil, transaction: nil
+        lookup_req = Google::Datastore::V1beta3::LookupRequest.new(
+          project_id: project,
+          keys: keys
+        )
+        lookup_req.read_options = generate_read_options consistency, transaction
+
+        backoff { datastore.lookup lookup_req }
+      end
+
+      # Query for entities.
+      def run_query query, namespace = nil, consistency: nil, transaction: nil
+        run_req = Google::Datastore::V1beta3::RunQueryRequest.new(
+          project_id: project)
+        if query.is_a? Google::Datastore::V1beta3::Query
+          run_req["query"] = query
+        elsif query.is_a? Google::Datastore::V1beta3::GqlQuery
+          run_req["gql_query"] = query
+        else
+          fail ArgumentError, "Unable to query with a #{query.class} object."
+        end
+        run_req.read_options = generate_read_options consistency, transaction
+
+        run_req.partition_id = Google::Datastore::V1beta3::PartitionId.new(
+          namespace_id: namespace) if namespace
+
+        backoff { datastore.run_query run_req }
+      end
+
+      ##
+      # Begin a new transaction.
+      def begin_transaction
+        tx_req = Google::Datastore::V1beta3::BeginTransactionRequest.new(
+          project_id: project
+        )
+
+        backoff { datastore.begin_transaction tx_req }
+      end
+
+      ##
+      # Commit a transaction, optionally creating, deleting or modifying
+      # some entities.
+      def commit mutations, transaction: nil
+        commit_req = Google::Datastore::V1beta3::CommitRequest.new(
+          project_id: project,
+          mode: :NON_TRANSACTIONAL,
+          mutations: mutations
+        )
+        if transaction
+          commit_req.mode = :TRANSACTIONAL
+          commit_req.transaction = transaction
+        end
+
+        backoff { datastore.commit commit_req }
+      end
+
+      ##
+      # Roll back a transaction.
+      def rollback transaction
+        rb_req = Google::Datastore::V1beta3::RollbackRequest.new(
+          project_id: project,
+          transaction: transaction
+        )
+
+        backoff { datastore.rollback rb_req }
+      end
+
+      def inspect
+        "#{self.class}(#{@project})"
+      end
+
+      def backoff options = {}
+        Gcloud::Backoff.new(options).execute_grpc do
+          yield
+        end
+      end
+
+      protected
+
+      def generate_read_options consistency, transaction
+        if consistency == :eventual
+          return Google::Datastore::V1beta3::ReadOptions.new(
+            read_consistency: :EVENTUAL)
+        elsif consistency == :strong
+          return  Google::Datastore::V1beta3::ReadOptions.new(
+            read_consistency: :STRONG)
+        elsif transaction
+          return  Google::Datastore::V1beta3::ReadOptions.new(
+            transaction: transaction)
+        end
+        nil
+      end
+    end
+  end
+end