gcloud 0.0.7 → 0.1.0

data/lib/gcloud/datastore.rb

@@ -0,0 +1,66 @@
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "gcloud"
+require "gcloud/datastore/errors"
+require "gcloud/datastore/dataset"
+require "gcloud/datastore/transaction"
+require "gcloud/datastore/credentials"
+
+##
+# Google Cloud Datastore
+module Gcloud
+  ##
+  # Create a new Gcloud::Datastore::Dataset.
+  #
+  #   entity = Gcloud::Datastore::Entity.new
+  #   entity.key = Gcloud::Datastore::Key.new "Task"
+  #   entity["description"] = "Get started with Google Cloud"
+  #   entity["completed"] = false
+  #
+  #   dataset = Gcloud.datastore "my-todo-project",
+  #                              "/path/to/keyfile.json"
+  #   dataset.save entity
+  #
+  # @param dataset_id [String] the dataset identifier for the Datastore
+  # you are connecting to.
+  # @param keyfile [String] the path to the keyfile you downloaded from
+  # Google Cloud. The file must readable.
+  # @return [Gcloud::Datastore::Dataset] new dataset.
+  #
+  # See Gcloud::Datastore::Dataset
+  def self.datastore project = ENV["DATASTORE_PROJECT"],
+                     keyfile = nil
+    if keyfile.nil?
+      credentials = Gcloud::Datastore::Credentials.default
+    else
+      credentials = Gcloud::Datastore::Credentials.new keyfile
+    end
+    Gcloud::Datastore::Dataset.new project, credentials
+  end
+
+  ##
+  # Google Cloud Datastore
+  #
+  #   dataset = Gcloud.datastore "my-todo-project",
+  #                              "/path/to/keyfile.json"
+  #   entity = dataset.find "Task", "start"
+  #   entity["completed"] = true
+  #   dataset.save entity
+  #
+  #
+  # See Gcloud::Datastore::Dataset
+  module Datastore
+  end
+end

data/lib/gcloud/datastore/connection.rb

@@ -0,0 +1,177 @@
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "faraday"
+
+module Gcloud
+  module Datastore
+    ##
+    # Represent the HTTP connection to the Datastore,
+    # as well as the Datastore API calls.
+    #
+    # This class only deals with Protocol Buffer objects,
+    # and is not part of the public API.
+    class Connection #:nodoc:
+      API_VERSION = "v1beta2"
+      API_URL = "https://www.googleapis.com"
+
+      ##
+      # The project/dataset_id connected to.
+      attr_accessor :dataset_id
+
+      ##
+      # The Credentials object for signing HTTP requests.
+      attr_accessor :credentials #:nodoc:
+
+      ##
+      # Create a new Connection instance.
+      #
+      #   conn = Gcloud::Datastore.Connection.new "my-todo-project",
+      #     Gcloud::Datastore::Credentials.new("/path/to/keyfile.json")
+      def initialize dataset_id, credentials
+        @dataset_id = dataset_id
+        @credentials = credentials
+      end
+
+      ##
+      # Allocate IDs for incomplete keys.
+      # (This is useful for referencing an entity before it is inserted.)
+      def allocate_ids *incomplete_keys
+        allocate_ids = Proto::AllocateIdsRequest.new.tap do |ai|
+          ai.key = incomplete_keys
+        end
+
+        rpc_response = rpc("allocateIds", allocate_ids)
+        Proto::AllocateIdsResponse.decode rpc_response
+      end
+
+      ##
+      # Look up entities by keys.
+      def lookup *keys
+        lookup = Proto::LookupRequest.new
+        lookup.key = keys
+
+        Proto::LookupResponse.decode rpc("lookup", lookup)
+      end
+
+      # Query for entities.
+      def run_query query
+        run_query = Proto::RunQueryRequest.new.tap do |rq|
+          rq.query = query
+        end
+
+        Proto::RunQueryResponse.decode rpc("runQuery", run_query)
+      end
+
+      ##
+      # Begin a new transaction.
+      def begin_transaction
+        tx_request = Proto::BeginTransactionRequest.new
+
+        response_rpc = rpc "beginTransaction", tx_request
+        Proto::BeginTransactionResponse.decode response_rpc
+      end
+
+      ##
+      # Commit a transaction, optionally creating, deleting or modifying
+      # some entities.
+      def commit mutation, transaction = nil
+        mode = transaction ? Proto::CommitRequest::Mode::TRANSACTIONAL :
+                             Proto::CommitRequest::Mode::NON_TRANSACTIONAL
+        commit = Proto::CommitRequest.new.tap do |c|
+          c.mutation = mutation
+          c.mode = mode
+          c.transaction = transaction
+        end
+
+        Proto::CommitResponse.decode rpc("commit", commit)
+      end
+
+      ##
+      # Roll back a transaction.
+      def rollback transaction
+        rollback = Proto::RollbackRequest.new.tap do |r|
+          r.transaction = transaction
+        end
+
+        Proto::RollbackResponse.decode rpc("rollback", rollback)
+      end
+
+      ##
+      # The default HTTP headers to be sent on all API calls.
+      def default_http_headers #:nodoc:
+        @default_http_headers ||= {
+          "User-Agent"   => "gcloud-node/#{Gcloud::VERSION}",
+          "Content-Type" => "application/x-protobuf" }
+      end
+      ##
+      # Update the default HTTP headers.
+      attr_writer :default_http_headers #:nodoc:
+
+      ##
+      # The HTTP object that makes calls to Datastore.
+      # This must be a Faraday object.
+      def http #:nodoc:
+        @http ||= Faraday.new url: http_host
+      end
+      ##
+      # Update the HTTP object.
+      attr_writer :http #:nodoc:
+
+      ##
+      # The Datastore API URL.
+      def http_host
+        @http_host || ENV["DATASTORE_HOST"] || API_URL
+      end
+
+      ##
+      # Update the Datastore API URL.
+      def http_host= new_http_host
+        @http = nil # Reset the HTTP connection when host is set
+        @http_host = new_http_host
+      end
+
+      protected
+
+      ##
+      # Convenience method for making an API call to Datastore.
+      # Requests will be signed with the credentials object.
+      def rpc proto_method, proto_request
+        response = http.post(rpc_path proto_method) do |req|
+          req.headers.merge! default_http_headers
+          req.body = get_proto_request_body proto_request
+
+          @credentials.sign_http_request req
+        end
+
+        fail ApiError.new(proto_method, response) unless response.success?
+
+        response.body
+      end
+
+      ## Generates the HTTP Path value for the API call.
+      def rpc_path proto_method
+        "/datastore/#{API_VERSION}/datasets/#{dataset_id}/#{proto_method}"
+      end
+
+      ##
+      # Convenience method for encoding a Beefcake object to a string.
+      def get_proto_request_body proto_request
+        proto_request_body = ""
+        proto_request.encode proto_request_body
+        proto_request_body
+      end
+    end
+  end
+end

data/lib/gcloud/datastore/credentials.rb

@@ -0,0 +1,41 @@
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "gcloud/credentials"
+
+module Gcloud
+  module Datastore
+    ##
+    # Authentication credentials to Google Cloud.
+    # The most common way to create this object is to provide the path
+    # to the JSON keyfile downloaded from Google Cloud.
+    #
+    # https://developers.google.com/accounts/docs/application-default-credentials
+    class Credentials < Gcloud::Credentials #:nodoc:
+      SCOPE = ["https://www.googleapis.com/auth/datastore",
+               "https://www.googleapis.com/auth/userinfo.email"]
+      ENV_VARS = ["DATASTORE_KEYFILE"]
+
+      ##
+      # Sign Oauth2 API calls.
+      def sign_http_request request #:nodoc:
+        if @client
+          @client.fetch_access_token! if @client.expired?
+          @client.generate_authenticated_request request: request
+        end
+        request
+      end
+    end
+  end
+end

data/lib/gcloud/datastore/dataset.rb

@@ -0,0 +1,259 @@
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "gcloud/datastore/connection"
+require "gcloud/datastore/credentials"
+require "gcloud/datastore/entity"
+require "gcloud/datastore/key"
+require "gcloud/datastore/query"
+require "gcloud/datastore/dataset/lookup_results"
+require "gcloud/datastore/dataset/query_results"
+
+module Gcloud
+  module Datastore
+    ##
+    # Dataset is the data saved in a project's Datastore.
+    # Dataset is analogous to a database in relational database world.
+    #
+    # Gcloud::Datastore::Dataset is the main object for interacting with
+    # Google Datastore. Gcloud::Datastore::Entity objects are created,
+    # read, updated, and deleted by Gcloud::Datastore::Dataset.
+    #
+    #   dataset = Gcloud.datastore "my-todo-project",
+    #                              "/path/to/keyfile.json"
+    #
+    #   query = Gcloud::Datastore::Query.new.kind("Task").
+    #     where("completed", "=", true)
+    #
+    #   tasks = dataset.run query
+    #
+    # See Gcloud.datastore
+    class Dataset
+      attr_accessor :connection #:nodoc:
+
+      ##
+      # Creates a new Dataset instance.
+      #
+      # See Gcloud.datastore
+      def initialize project, credentials #:nodoc:
+        @connection = Connection.new project, credentials
+      end
+
+      ##
+      # The project connected to.
+      def project
+        connection.dataset_id
+      end
+
+      ##
+      # Generate IDs for a Key before creating an entity.
+      #
+      #   dataset = Gcloud.datastore
+      #   empty_key = Gcloud::Datastore::Key.new "Task"
+      #   task_keys = dataset.allocate_ids empty_key, 5
+      def allocate_ids incomplete_key, count = 1
+        if incomplete_key.complete?
+          fail Gcloud::Datastore::Error, "An incomplete key must be provided."
+        end
+
+        incomplete_keys = count.times.map { incomplete_key.to_proto }
+        response = connection.allocate_ids(*incomplete_keys)
+        Array(response.key).map do |key|
+          Key.from_proto key
+        end
+      end
+
+      ##
+      # Persist entities to the Datastore.
+      #
+      #   dataset = Gcloud.datastore
+      #   dataset.save task1, task2
+      def save *entities
+        mutation = Proto.new_mutation
+        save_entities_to_mutation entities, mutation
+        response = connection.commit mutation
+        auto_id_assign_ids response.mutation_result.insert_auto_id_key
+        entities
+      end
+
+      ##
+      # Retrieve an entity by providing key information.
+      # Either a Key object or kind and id/name can be provided.
+      #
+      #   dataset = Gcloud.datastore
+      #   key = Gcloud::Datastore::Key.new "Task", 123456
+      #   task = dataset.find key
+      #
+      #   dataset = Gcloud.datastore
+      #   task = dataset.find "Task", 123456
+      def find key_or_kind, id_or_name = nil
+        key = key_or_kind
+        key = Key.new key_or_kind, id_or_name unless key_or_kind.is_a? Key
+        find_all(key).first
+      end
+      alias_method :get, :find
+
+      ##
+      # Retrieve the entities for the provided keys.
+      #
+      #   dataset = Gcloud.datastore
+      #   key1 = Gcloud::Datastore::Key.new "Task", 123456
+      #   key2 = Gcloud::Datastore::Key.new "Task", 987654
+      #   tasks = dataset.find_all key1, key2
+      #
+      # The entities returned from the lookup are returned in a
+      # Dataset::LookupResults object.
+      def find_all *keys
+        response = connection.lookup(*keys.map(&:to_proto))
+        entities = to_gcloud_entities response.found
+        deferred = to_gcloud_keys response.deferred
+        missing  = to_gcloud_entities response.missing
+        LookupResults.new entities, deferred, missing
+      end
+      alias_method :lookup, :find_all
+
+      ##
+      # Remove entities from the Datastore.
+      # Accepts Entity and Key objects.
+      #
+      #   dataset = Gcloud.datastore
+      #   dataset.delete task1, task2
+      def delete *entities_or_keys
+        keys = entities_or_keys.map do |e_or_k|
+          e_or_k.respond_to?(:key) ? e_or_k.key.to_proto : e_or_k.to_proto
+        end
+        mutation = Proto.new_mutation.tap do |m|
+          m.delete = keys
+        end
+        connection.commit mutation
+        true
+      end
+
+      ##
+      # Retrieve entities specified by a Query.
+      #
+      #   query = Gcloud::Datastore::Query.new.kind("Task").
+      #     where("completed", "=", true)
+      #   tasks = dataset.run query
+      #
+      # The entities returned from the query are returned in a
+      # Dataset::QueryResults object.
+      def run query
+        response = connection.run_query query.to_proto
+        entities = to_gcloud_entities response.batch.entity_result
+        cursor = Proto.encode_cursor response.batch.end_cursor
+        more_results = Proto.to_more_results_string response.batch.more_results
+        QueryResults.new entities, cursor, more_results
+      end
+      alias_method :run_query, :run
+
+      ##
+      # Runs the given block in a database transaction.
+      # If no block is given the transaction object is returned.
+      #
+      #   user = Gcloud::Datastore::Entity.new
+      #   user.key = Gcloud::Datastore::Key.new "User", "username"
+      #   user["name"] = "Test"
+      #   user["email"] = "test@example.net"
+      #
+      #   dataset.transaction do |tx|
+      #     if tx.find(user.key).nil?
+      #       tx.save user
+      #     end
+      #   end
+      #
+      # Alternatively, you can manually commit or rollback by
+      # using the returned transaction object.
+      #
+      #   user = Gcloud::Datastore::Entity.new
+      #   user.key = Gcloud::Datastore::Key.new "User", "username"
+      #   user["name"] = "Test"
+      #   user["email"] = "test@example.net"
+      #
+      #   tx = dataset.transaction
+      #   begin
+      #     if tx.find(user.key).nil?
+      #       tx.save user
+      #     end
+      #     tx.commit
+      #   rescue
+      #     tx.rollback
+      #   end
+      def transaction
+        tx = Transaction.new connection
+        return tx unless block_given?
+
+        begin
+          yield tx
+          tx.commit
+        rescue => e
+          tx.rollback
+          raise TransactionError.new("Transaction failed to commit.", e)
+        end
+      end
+
+      protected
+
+      ##
+      # Convenince method to convert proto entities to Gcloud entities.
+      def to_gcloud_entities proto_results
+        # Entities are nested in an object.
+        Array(proto_results).map do |result|
+          Entity.from_proto result.entity
+        end
+      end
+
+      ##
+      # Convenince method to convert proto keys to Gcloud keys.
+      def to_gcloud_keys proto_results
+        # Keys are not nested in an object like entities are.
+        Array(proto_results).map do |key|
+          Key.from_proto key
+        end
+      end
+
+      ##
+      # Save a key to be given an ID when comitted.
+      def auto_id_register entity #:nodoc:
+        @_auto_id_entities ||= []
+        @_auto_id_entities << entity
+      end
+
+      ##
+      # Update saved keys with new IDs post-commit.
+      def auto_id_assign_ids auto_ids #:nodoc:
+        @_auto_id_entities ||= []
+        Array(auto_ids).each_with_index do |key, index|
+          entity = @_auto_id_entities[index]
+          entity.key = Key.from_proto key
+        end
+        @_auto_id_entities = []
+      end
+
+      ##
+      # Add entities to a Mutation, and register they key to be
+      # updated with an auto ID if needed.
+      def save_entities_to_mutation entities, mutation #:nodoc:
+        entities.each do |entity|
+          if entity.key.id.nil? && entity.key.name.nil?
+            mutation.insert_auto_id << entity.to_proto
+            auto_id_register entity
+          else
+            mutation.upsert << entity.to_proto
+          end
+        end
+      end
+    end
+  end
+end