google-cloud-spanner 0.21.0

data/lib/google/cloud/spanner/instance/list.rb

@@ -0,0 +1,167 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "delegate"
+
+module Google
+  module Cloud
+    module Spanner
+      class Instance
+        ##
+        # Instance::List is a special case Array with additional
+        # values.
+        class List < DelegateClass(::Array)
+          ##
+          # If not empty, indicates that there are more records that match
+          # the request and this value should be passed to continue.
+          attr_accessor :token
+
+          ##
+          # @private Create a new Instance::List with an array of
+          # Instance instances.
+          def initialize arr = []
+            super arr
+          end
+
+          ##
+          # Whether there is a next page of instances.
+          #
+          # @return [Boolean]
+          #
+          # @example
+          #   require "google/cloud/spanner"
+          #
+          #   spanner = Google::Cloud::Spanner.new
+          #
+          #   instances = spanner.instances
+          #   if instances.next?
+          #     next_instances = instances.next
+          #   end
+          def next?
+            !token.nil?
+          end
+
+          ##
+          # Retrieve the next page of instances.
+          #
+          # @return [Instance::List] The list of instances.
+          #
+          # @example
+          #   require "google/cloud/spanner"
+          #
+          #   spanner = Google::Cloud::Spanner.new
+          #
+          #   instances = spanner.instances
+          #   if instances.next?
+          #     next_instances = instances.next
+          #   end
+          def next
+            return nil unless next?
+            ensure_service!
+            options = { token: token, max: @max }
+            grpc = @service.list_instances options
+            self.class.from_grpc grpc, @service, @max
+          end
+
+          ##
+          # Retrieves remaining results by repeatedly invoking {#next} until
+          # {#next?} returns `false`. Calls the given block once for each
+          # result, which is passed as the argument to the block.
+          #
+          # An Enumerator is returned if no block is given.
+          #
+          # This method will make repeated API calls until all remaining results
+          # are retrieved. (Unlike `#each`, for example, which merely iterates
+          # over the results returned by a single API call.) Use with caution.
+          #
+          # @param [Integer] request_limit The upper limit of API requests to
+          #   make to load all instances. Default is no limit.
+          # @yield [instance] The block for accessing each instance.
+          # @yieldparam [Instance] instance The instance object.
+          #
+          # @return [Enumerator]
+          #
+          # @example Iterating each instance by passing a block:
+          #   require "google/cloud/spanner"
+          #
+          #   spanner = Google::Cloud::Spanner.new
+          #
+          #   spanner.instances.all do |instance|
+          #     puts instance.instance_id
+          #   end
+          #
+          # @example Using the enumerator by not passing a block:
+          #   require "google/cloud/spanner"
+          #
+          #   spanner = Google::Cloud::Spanner.new
+          #
+          #   all_instance_ids = spanner.instances.all.map do |instance|
+          #     instance.instance_id
+          #   end
+          #
+          # @example Limit the number of API calls made:
+          #   require "google/cloud/spanner"
+          #
+          #   spanner = Google::Cloud::Spanner.new
+          #
+          #   spanner.instances.all(request_limit: 10) do |instance|
+          #     puts instance.instance_id
+          #   end
+          #
+          def all request_limit: nil
+            request_limit = request_limit.to_i if request_limit
+            unless block_given?
+              return enum_for(:all, request_limit: request_limit)
+            end
+            results = self
+            loop do
+              results.each { |r| yield r }
+              if request_limit
+                request_limit -= 1
+                break if request_limit < 0
+              end
+              break unless results.next?
+              results = results.next
+            end
+          end
+
+          ##
+          # @private New Instance::List from a
+          # Google::Spanner::Admin::Instance::V1::ListInstancesResponse
+          # object.
+          def self.from_grpc grpc, service, max = nil
+            instances = List.new(Array(grpc.instances).map do |instance|
+              Instance.from_grpc instance, service
+            end)
+            token = grpc.next_page_token
+            token = nil if token == ""
+            instances.instance_variable_set :@token,   token
+            instances.instance_variable_set :@service, service
+            instances.instance_variable_set :@max,     max
+            instances
+          end
+
+          protected
+
+          ##
+          # Raise an error unless an active service is available.
+          def ensure_service!
+            fail "Must have active connection" unless @service
+          end
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/spanner/policy.rb

@@ -0,0 +1,201 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "google/cloud/spanner/errors"
+
+module Google
+  module Cloud
+    module Spanner
+      ##
+      # # Policy
+      #
+      # Represents a Cloud IAM Policy for the Spanner service.
+      #
+      # A common pattern for updating a resource's metadata, such as its Policy,
+      # is to read the current data from the service, update the data locally,
+      # and then send the modified data for writing. This pattern may result in
+      # a conflict if two or more processes attempt the sequence simultaneously.
+      # IAM solves this problem with the {Google::Cloud::Spanner::Policy#etag}
+      # property, which is used to verify whether the policy has changed since
+      # the last request. When you make a request to with an `etag` value, Cloud
+      # IAM compares the `etag` value in the request with the existing `etag`
+      # value associated with the policy. It writes the policy only if the
+      # `etag` values match.
+      #
+      # When you update a policy, first read the policy (and its current `etag`)
+      # from the service, then modify the policy locally, and then write the
+      # modified policy to the service. See
+      # {Google::Cloud::Spanner::Instance#policy} and
+      # {Google::Cloud::Spanner::Instance#policy=} and
+      # {Google::Cloud::Spanner::Database#policy} and
+      # {Google::Cloud::Spanner::Database#policy=}.
+      #
+      # @see https://cloud.google.com/iam/docs/managing-policies Managing
+      #   policies
+      # @see https://cloud.google.com/spanner/reference/rpc/google.iam.v1#google.iam.v1.Policy
+      #   google.iam.v1.IAMPolicy
+      #
+      # @attr [String] etag Used to verify whether the policy has changed since
+      #   the last request. The policy will be written only if the `etag` values
+      #   match.
+      # @attr [Hash{String => Array<String>}] roles The bindings that associate
+      #   roles with an array of members. See [Understanding
+      #   Roles](https://cloud.google.com/iam/docs/understanding-roles) for a
+      #   listing of primitive and curated roles.
+      #   See [Binding](https://cloud.google.com/spanner/reference/rpc/google.iam.v1#google.iam.v1.Binding)
+      #   for a listing of values and patterns for members.
+      #
+      # @example
+      #   require "google/cloud/spanner"
+      #
+      #   spanner = Google::Cloud::Spanner.new
+      #   instance = spanner.instance "my-instance"
+      #
+      #   policy = instance.policy do |p|
+      #     p.remove "roles/owner", "user:owner@example.com"
+      #     p.add "roles/owner", "user:newowner@example.com"
+      #     p.roles["roles/viewer"] = ["allUsers"]
+      #   end
+      #
+      class Policy
+        attr_reader :etag, :roles
+
+        ##
+        # @private Creates a Policy object.
+        def initialize etag, roles
+          @etag = etag
+          @roles = roles
+        end
+
+        ##
+        # Convenience method for adding a member to a binding on this policy.
+        # See [Understanding
+        # Roles](https://cloud.google.com/iam/docs/understanding-roles) for a
+        # listing of primitive and curated roles.
+        # See [Binding](https://cloud.google.com/spanner/reference/rpc/google.iam.v1#google.iam.v1.Binding)
+        # for a listing of values and patterns for members.
+        #
+        # @param [String] role_name A Cloud IAM role, such as
+        #   `"roles/spanner.admin"`.
+        # @param [String] member A Cloud IAM identity, such as
+        #   `"user:owner@example.com"`.
+        #
+        # @example
+        #   require "google/cloud/spanner"
+        #
+        #   spanner = Google::Cloud::Spanner.new
+        #   instance = spanner.instance "my-instance"
+        #
+        #   policy = instance.policy do |p|
+        #     p.add "roles/owner", "user:newowner@example.com"
+        #   end
+        #
+        def add role_name, member
+          role(role_name) << member
+        end
+
+        ##
+        # Convenience method for removing a member from a binding on this
+        # policy. See [Understanding
+        # Roles](https://cloud.google.com/iam/docs/understanding-roles) for a
+        # listing of primitive and curated roles. See
+        # [Binding](https://cloud.google.com/spanner/reference/rpc/google.iam.v1#google.iam.v1.Binding)
+        # for a listing of values and patterns for members.
+        #
+        # @param [String] role_name A Cloud IAM role, such as
+        #   `"roles/spanner.admin"`.
+        # @param [String] member A Cloud IAM identity, such as
+        #   `"user:owner@example.com"`.
+        #
+        # @example
+        #   require "google/cloud/spanner"
+        #
+        #   spanner = Google::Cloud::Spanner.new
+        #   instance = spanner.instance "my-instance"
+        #
+        #   policy = instance.policy do |p|
+        #     p.remove "roles/owner", "user:owner@example.com"
+        #   end
+        #
+        def remove role_name, member
+          role(role_name).delete member
+        end
+
+        ##
+        # Convenience method returning the array of members bound to a role in
+        # this policy, or an empty array if no value is present for the role in
+        # {#roles}. See [Understanding
+        # Roles](https://cloud.google.com/iam/docs/understanding-roles) for a
+        # listing of primitive and curated roles. See
+        # [Binding](https://cloud.google.com/spanner/reference/rpc/google.iam.v1#google.iam.v1.Binding)
+        # for a listing of values and patterns for members.
+        #
+        # @return [Array<String>] The members strings, or an empty array.
+        #
+        # @example
+        #   require "google/cloud/spanner"
+        #
+        #   spanner = Google::Cloud::Spanner.new
+        #   instance = spanner.instance "my-instance"
+        #
+        #   policy = instance.policy do |p|
+        #     p.role("roles/viewer") << "user:viewer@example.com"
+        #   end
+        #
+        def role role_name
+          roles[role_name] ||= []
+        end
+
+        ##
+        # Returns a deep copy of the policy.
+        #
+        # @return [Policy]
+        #
+        def deep_dup
+          dup.tap do |p|
+            roles_dup = p.roles.each_with_object({}) do |(k, v), memo|
+              memo[k] = v.dup rescue value
+            end
+            p.instance_variable_set "@roles", roles_dup
+          end
+        end
+
+        ##
+        # @private Convert the Policy to a Google::Iam::V1::Policy object.
+        def to_grpc
+          Google::Iam::V1::Policy.new(
+            etag: etag,
+            bindings: roles.keys.map do |role_name|
+              next if roles[role_name].empty?
+              Google::Iam::V1::Binding.new(
+                role: role_name,
+                members: roles[role_name]
+              )
+            end
+          )
+        end
+
+        ##
+        # @private New Policy from a Google::Iam::V1::Policy object.
+        def self.from_grpc grpc
+          roles = grpc.bindings.each_with_object({}) do |binding, memo|
+            memo[binding.role] = binding.members.to_a
+          end
+          new grpc.etag, roles
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/spanner/pool.rb

@@ -0,0 +1,279 @@
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "thread"
+require "concurrent"
+require "google/cloud/spanner/errors"
+require "google/cloud/spanner/session"
+
+module Google
+  module Cloud
+    module Spanner
+      ##
+      # @private
+      #
+      # # Pool
+      #
+      # Implements a pool for managing and reusing
+      # {Google::Cloud::Spanner::Session} instances.
+      #
+      class Pool
+        attr_accessor :all_sessions, :session_queue, :transaction_queue
+
+        def initialize client, min: 10, max: 100, keepalive: 1800,
+                       write_ratio: 0.3, fail: true
+          @client = client
+          @min = min
+          @max = max
+          @keepalive = keepalive
+          @write_ratio = write_ratio
+          @write_ratio = 0 if write_ratio < 0
+          @write_ratio = 1 if write_ratio > 1
+          @fail = fail
+
+          @mutex = Mutex.new
+          @resource = ConditionVariable.new
+
+          # initialize pool and availability queue
+          init
+        end
+
+        def with_session
+          session = checkout_session
+          begin
+            yield session
+          ensure
+            checkin_session session
+          end
+        end
+
+        def checkout_session
+          action = nil
+          @mutex.synchronize do
+            loop do
+              fail ClientClosedError if @closed
+
+              read_session = session_queue.shift
+              return read_session if read_session
+              write_transaction = transaction_queue.shift
+              return write_transaction.session if write_transaction
+
+              if can_allocate_more_sessions?
+                @new_sessions_in_process += 1
+                action = :new
+                break
+              end
+
+              fail SessionLimitError if @fail
+
+              @resource.wait @mutex
+            end
+          end
+
+          return new_session! if action == :new
+        end
+
+        def checkin_session session
+          unless all_sessions.include? session
+            fail ArgumentError, "Cannot checkin session"
+          end
+
+          @mutex.synchronize do
+            session_queue.push session
+
+            @resource.signal
+          end
+
+          nil
+        end
+
+        def with_transaction
+          tx = checkout_transaction
+          begin
+            yield tx
+          ensure
+            future do
+              # Create and checkin a new transaction
+              tx = tx.session.create_transaction
+              checkin_transaction tx
+            end
+          end
+        end
+
+        def checkout_transaction
+          action = nil
+          @mutex.synchronize do
+            loop do
+              fail ClientClosedError if @closed
+
+              write_transaction = transaction_queue.shift
+              return write_transaction if write_transaction
+              read_session = session_queue.shift
+              if read_session
+                action = read_session
+                break
+              end
+
+              if can_allocate_more_sessions?
+                @new_sessions_in_process += 1
+                action = :new
+                break
+              end
+
+              fail SessionLimitError if @fail
+
+              @resource.wait @mutex
+            end
+          end
+          if action.is_a? Google::Cloud::Spanner::Session
+            return action.create_transaction
+          end
+          return new_transaction! if action == :new
+        end
+
+        def checkin_transaction tx
+          unless all_sessions.include? tx.session
+            fail ArgumentError, "Cannot checkin session"
+          end
+
+          @mutex.synchronize do
+            transaction_queue.push tx
+
+            @resource.signal
+          end
+
+          nil
+        end
+
+        def reset
+          close
+          init
+
+          true
+        end
+
+        def close
+          @mutex.synchronize do
+            @closed = true
+          end
+          @keepalive_task.shutdown
+          # Unblock all waiting threads
+          @resource.broadcast
+          # Delete all sessions
+          @mutex.synchronize do
+            @all_sessions.each { |s| future { s.release! } }
+            @all_sessions = []
+            @session_queue = []
+            @transaction_queue = []
+          end
+          # shutdown existing thread pool
+          @thread_pool.shutdown
+
+          true
+        end
+
+        def keepalive_or_release!
+          to_keepalive = []
+          to_release = []
+
+          @mutex.synchronize do
+            available_count = session_queue.count + transaction_queue.count
+            release_count = @min - available_count
+            release_count = 0 if release_count < 0
+
+            to_keepalive += (session_queue + transaction_queue).select do |x|
+              x.idle_since? @keepalive
+            end
+
+            # Remove a random portion of the sessions and transactions
+            to_release = to_keepalive.sample release_count
+            to_keepalive -= to_release
+
+            # Remove those to be released from circulation
+            @all_sessions -= to_release.map(&:session)
+            @session_queue -= to_release
+            @transaction_queue -= to_release
+          end
+
+          to_release.each { |x| future { x.release! } }
+          to_keepalive.each { |x| future { x.keepalive! } }
+        end
+
+        private
+
+        def init
+          # init the thread pool
+          @thread_pool = Concurrent::FixedThreadPool.new(
+            [2, Concurrent.processor_count].max * 2,
+            fallback_policy: :caller_runs
+          )
+          # init the queues
+          @new_sessions_in_process = @min.to_i
+          @all_sessions = []
+          @session_queue = []
+          @transaction_queue = []
+          # init the keepalive task
+          create_keepalive_task!
+          # init session queue
+          num_transactions = (@min * @write_ratio).round
+          num_sessions = @min.to_i - num_transactions
+          num_sessions.times.each do
+            future { checkin_session new_session! }
+          end
+          # init transaction queue
+          num_transactions.times.each do
+            future { checkin_transaction new_transaction! }
+          end
+        end
+
+        def new_session!
+          session = @client.create_new_session
+
+          @mutex.synchronize do
+            # don't add if the pool is closed
+            return session.release! if @closed
+
+            @new_sessions_in_process -= 1
+            all_sessions << session
+          end
+          session
+        end
+
+        def new_transaction!
+          new_session!.create_transaction
+        end
+
+        def can_allocate_more_sessions?
+          # This is expected to be called from within a synchronize block
+          all_sessions.size + @new_sessions_in_process < @max
+        end
+
+        def create_keepalive_task!
+          @keepalive_task = Concurrent::TimerTask.new(execution_interval: 300,
+                                                      timeout_interval: 60) do
+            keepalive_or_release!
+          end
+          @keepalive_task.execute
+        end
+
+        def future
+          Concurrent::Future.new(executor: @thread_pool) do
+            yield
+          end.execute
+        end
+      end
+    end
+  end
+end