google-cloud-firestore 0.22.0 → 0.23.0

data/lib/google/cloud/firestore/v1beta1/firestore_client_config.json

@@ -80,12 +80,12 @@
           "retry_params_name": "default"
         },
         "Write": {
-          "timeout_millis": 300000,
+          "timeout_millis": 86400000,
           "retry_codes_name": "non_idempotent",
           "retry_params_name": "streaming"
         },
         "Listen": {
-          "timeout_millis": 300000,
+          "timeout_millis": 86400000,
           "retry_codes_name": "idempotent",
           "retry_params_name": "streaming"
         },

data/lib/google/cloud/firestore/version.rb

@@ -16,7 +16,7 @@
 module Google
   module Cloud
     module Firestore
-      VERSION = "0.22.0".freeze
+      VERSION = "0.23.0".freeze
     end
   end
 end

data/lib/google/cloud/firestore/watch/enumerator_queue.rb

@@ -0,0 +1,47 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "thread"
+
+module Google
+  module Cloud
+    module Firestore
+      # @private
+      module Watch
+        # @private
+        class EnumeratorQueue
+          def initialize sentinel = nil
+            @queue    = Queue.new
+            @sentinel = sentinel
+          end
+
+          def push obj
+            @queue.push obj
+          end
+
+          def each
+            return enum_for(:each) unless block_given?
+
+            loop do
+              obj = @queue.pop
+              break if obj.equal? @sentinel
+              yield obj
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/firestore/watch/inventory.rb

@@ -0,0 +1,280 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "google/cloud/firestore/v1beta1"
+require "google/cloud/firestore/convert"
+require "google/cloud/firestore/document_reference"
+require "google/cloud/firestore/document_snapshot"
+require "google/cloud/firestore/document_change"
+require "google/cloud/firestore/query_snapshot"
+require "google/cloud/firestore/watch/order"
+require "rbtree"
+
+module Google
+  module Cloud
+    module Firestore
+      ##
+      # @private
+      module Watch
+        # @private Collects changes and produces a QuerySnapshot.
+        # Uses RBTree to hold a sorted list of DocumentSnapshot objects and to
+        # make inserting and removing objects much more efficent.
+        class Inventory
+          attr_accessor :current
+          attr_reader :resume_token, :read_time
+
+          def initialize client, query
+            @client = client
+            @query = query
+            @pending = {
+              add: [],
+              delete: []
+            }
+            @current = nil
+            @resume_token = nil
+            @read_time = nil
+            @tree = RBTree.new
+            @tree.readjust(&method(:query_comparison_proc))
+            @old_order = nil
+
+            # TODO: Remove this when done benchmarking
+            @comp_proc_counter = 0
+          end
+
+          def current?
+            @current
+          end
+
+          def add doc_grpc
+            @pending[:add] << doc_grpc
+          end
+
+          def delete doc_path
+            @pending[:delete] << doc_path
+          end
+
+          def pending?
+            @pending[:add].any? || @pending[:delete].any?
+          end
+
+          def clear_pending
+            @pending[:add].clear
+            @pending[:delete].clear
+          end
+
+          def size
+            @tree.size
+          end
+          alias count size
+
+          def size_with_pending
+            count_with_pending_tree = @tree.dup
+            apply_pending_changes_to_tree @pending, count_with_pending_tree
+            count_with_pending_tree.size
+          end
+          alias count_with_pending size_with_pending
+
+          def restart
+            # clears all but query, resume token, read time, and old order
+            clear_pending
+
+            @current = nil
+
+            @tree.clear
+          end
+
+          def reset
+            restart
+
+            # clears the resume token and read time, but not query and old order
+            @resume_token = nil
+            @read_time = nil
+          end
+
+          # TODO: Remove this when done benchmarking
+          def reset_comp_proc_counter!
+            old_count = @comp_proc_counter
+            @comp_proc_counter = 0
+            old_count
+          end
+
+          def persist resume_token, read_time
+            @resume_token = resume_token
+            @read_time = read_time
+
+            apply_pending_changes_to_tree @pending, @tree
+            clear_pending
+          end
+
+          def changes?
+            # Act like there are changes if we have never run before
+            return true if @old_order.nil?
+            added_paths, deleted_paths, changed_paths = \
+              change_paths current_order, @old_order
+            added_paths.any? || deleted_paths.any? || changed_paths.any?
+          end
+
+          def current_docs
+            @tree.keys
+          end
+
+          def order_for docs
+            Hash[docs.map { |doc| [doc.path, doc.updated_at] }]
+          end
+
+          def current_order
+            order_for current_docs
+          end
+
+          def build_query_snapshot
+            # If this is the first time building, set to empty hash
+            @old_order ||= {}
+
+            # Get the new set of documents, changes, order
+            docs = current_docs
+            new_order = order_for docs
+            changes = build_changes new_order, @old_order
+            @old_order = new_order
+
+            QuerySnapshot.from_docs @query, docs, changes, @read_time
+          end
+
+          protected
+
+          def query_comparison_proc a, b
+            # TODO: Remove this when done benchmarking
+            @comp_proc_counter += 1
+
+            return Order.compare_field_values a.ref, b.ref if @query.nil?
+
+            @directions ||= @query.query.order_by.map(&:direction)
+
+            a_comps = a.query_comparisons_for @query.query
+            b_comps = b.query_comparisons_for @query.query
+            @directions.zip(a_comps, b_comps).each do |dir, a_comp, b_comp|
+              comp = a_comp <=> b_comp
+              comp = 0 - comp if dir == :DESCENDING
+              return comp unless comp.zero?
+            end
+
+            # Compare paths when everything else is equal
+            ref_comp = Order.compare_field_values a.ref, b.ref
+            ref_comp = 0 - ref_comp if @directions.last == :DESCENDING
+            ref_comp
+          end
+
+          def apply_pending_changes_to_tree pending, tree
+            # Remove the deleted documents
+            pending[:delete].each do |doc_path|
+              remove_doc_from_tree doc_path, tree
+            end
+
+            # Add/update the changed documents
+            pending[:add].each do |doc_grpc|
+              removed_doc = remove_doc_from_tree doc_grpc.name, tree
+              added_doc = DocumentSnapshot.from_document(
+                doc_grpc, @client, read_at: read_time
+              )
+
+              if removed_doc && removed_doc.updated_at >= added_doc.updated_at
+                # Restore the removed doc if the added doc isn't newer
+                added_doc = removed_doc
+              end
+
+              add_doc_to_tree added_doc, tree
+            end
+          end
+
+          def change_paths new_order, old_order
+            added_paths = new_order.keys - old_order.keys
+            deleted_paths = old_order.keys - new_order.keys
+            new_hash = new_order.dup.delete_if do |path, _updated_at|
+              added_paths.include? path
+            end
+            old_hash = old_order.dup.delete_if do |path, _updated_at|
+              deleted_paths.include? path
+            end
+            changed_paths = (new_hash.to_a - old_hash.to_a).map(&:first)
+
+            [added_paths, deleted_paths, changed_paths]
+          end
+
+          def build_changes new_order, old_order
+            new_paths = new_order.keys
+            old_paths = old_order.keys
+            added_paths, deleted_paths, changed_paths = \
+              change_paths new_order, old_order
+
+            changes = deleted_paths.map do |doc_path|
+              build_deleted_doc_change doc_path, old_paths
+            end
+            changes += added_paths.map do |doc_path|
+              build_added_doc_change doc_path, new_paths
+            end
+            changes += changed_paths.map do |doc_path|
+              build_modified_doc_change doc_path, new_paths, old_paths
+            end
+            changes
+          end
+
+          def build_deleted_doc_change doc_path, old_paths
+            doc_ref = DocumentReference.from_path doc_path, @client
+            doc_snp = DocumentSnapshot.missing doc_ref
+            old_index = get_index_from_order_array doc_path, old_paths
+            DocumentChange.from_doc doc_snp, old_index, nil
+          end
+
+          def build_added_doc_change doc_path, new_paths
+            doc_snp = get_doc_from_tree doc_path, @tree
+            new_index = get_index_from_order_array doc_path, new_paths
+            DocumentChange.from_doc doc_snp, nil, new_index
+          end
+
+          def build_modified_doc_change doc_path, new_paths, old_paths
+            doc_snp = get_doc_from_tree doc_path, @tree
+            old_index = get_index_from_order_array doc_path, old_paths
+            new_index = get_index_from_order_array doc_path, new_paths
+            DocumentChange.from_doc doc_snp, old_index, new_index
+          end
+
+          def get_index_from_order_array doc_path, order_array
+            order_array.index doc_path
+          end
+
+          def get_doc_from_tree doc_path, tree
+            tree.key doc_path
+          end
+
+          def add_doc_to_tree doc_snp, tree
+            tree[doc_snp] = doc_snp.path
+          end
+
+          def remove_doc_from_tree doc_path, tree
+            # Remove old snapshot
+            old_snp = tree.key doc_path
+            tree.delete old_snp unless old_snp.nil?
+            old_snp
+          end
+
+          def type_from_indexes old_index, new_index
+            return :removed if new_index.nil?
+            return :added if old_index.nil?
+            :modified
+          end
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/firestore/watch/listener.rb

@@ -0,0 +1,298 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "google/cloud/firestore/v1beta1"
+require "google/cloud/firestore/convert"
+require "google/cloud/firestore/watch/enumerator_queue"
+require "google/cloud/firestore/watch/inventory"
+require "monitor"
+require "thread"
+
+module Google
+  module Cloud
+    module Firestore
+      ##
+      # @private
+      module Watch
+        ##
+        # @private
+        class Listener
+          include MonitorMixin
+
+          def self.for_doc_ref doc_ref, &callback
+            raise ArgumentError if doc_ref.nil?
+            raise ArgumentError if callback.nil?
+
+            init_listen_req = Google::Firestore::V1beta1::ListenRequest.new(
+              database: doc_ref.client.path,
+              add_target: Google::Firestore::V1beta1::Target.new(
+                documents: \
+                  Google::Firestore::V1beta1::Target::DocumentsTarget.new(
+                    documents: [doc_ref.path]
+                  )
+              )
+            )
+
+            new nil, doc_ref, doc_ref.client, init_listen_req, &callback
+          end
+
+          def self.for_query query, &callback
+            raise ArgumentError if query.nil?
+            raise ArgumentError if callback.nil?
+
+            init_listen_req = Google::Firestore::V1beta1::ListenRequest.new(
+              database: query.client.path,
+              add_target: Google::Firestore::V1beta1::Target.new(
+                query: Google::Firestore::V1beta1::Target::QueryTarget.new(
+                  parent: query.parent_path,
+                  structured_query: query.query
+                )
+              )
+            )
+
+            new query, nil, query.client, init_listen_req, &callback
+          end
+
+          def initialize query, doc_ref, client, init_listen_req, &callback
+            @query = query
+            @doc_ref = doc_ref
+            @client = client
+            @init_listen_req = init_listen_req
+            @callback = callback
+
+            super() # to init MonitorMixin
+          end
+
+          def start
+            synchronize { start_listening! }
+            self
+          end
+
+          def stop
+            synchronize do
+              @stopped = true
+              @request_queue.push self if @request_queue
+            end
+          end
+
+          ##
+          # Whether the client has stopped listening for changes.
+          #
+          # @example
+          #   require "google/cloud/firestore"
+          #
+          #   firestore = Google::Cloud::Firestore.new
+          #
+          #   # Create a query
+          #   query = firestore.col(:cities).order(:population, :desc)
+          #
+          #   listener = query.listen do |snapshot|
+          #     puts "The query snapshot has #{snapshot.docs.count} documents "
+          #     puts "and has #{snapshot.changes.count} changes."
+          #   end
+          #
+          #   # Checks if the listener is stopped.
+          #   listener.stopped? #=> false
+          #
+          #   # When ready, stop the listen operation and close the stream.
+          #   listener.stop
+          #
+          #   # Checks if the listener is stopped.
+          #   listener.stopped? #=> true
+          #
+          def stopped?
+            synchronize { @stopped }
+          end
+
+          private
+
+          def send_callback query_snp
+            @callback.call query_snp
+          end
+
+          def start_listening!
+            # create new background thread to handle the stream's enumerator
+            @background_thread = Thread.new { background_run }
+          end
+
+          # @private
+          class RestartStream < StandardError; end
+
+          # rubocop:disable all
+
+          def background_run
+            # Don't allow a stream to restart if already stopped
+            return if synchronize { @stopped }
+
+            @backoff ||= { current: 0, delay: 1.0, max: 5, mod: 1.3 }
+
+            # Reuse inventory if one already exists
+            # Even though this uses an @var, no need to synchronize
+            @inventory ||= Inventory.new(@client, @query)
+            @inventory.restart
+
+            # Send stop if already running
+            synchronize do
+              @request_queue.push self if @request_queue
+            end
+
+            # Customize the provided initial listen request
+            init_listen_req = @init_listen_req.dup.tap do |req|
+              req.add_target.resume_token = String(@inventory.resume_token)
+              req.add_target.target_id = 0x42
+            end
+
+            # Always create a new enum queue
+            synchronize do
+              @request_queue = EnumeratorQueue.new self
+              @request_queue.push init_listen_req
+            end
+
+            # Not an @var, we get a new enum each time
+            enum = synchronize do
+              @client.service.listen @request_queue.each
+            end
+
+            loop do
+
+              # Break loop, close thread if stopped
+              break if synchronize { @stopped }
+
+              begin
+                # Cannot syncronize the enumerator, causes deadlock
+                response = enum.next
+
+                case response.response_type
+                when :target_change
+                  case response.target_change.target_change_type
+                  when :NO_CHANGE
+                    # No change has occurred. Used only to send an updated
+                    # +resume_token+.
+
+                    @inventory.persist(
+                      response.target_change.resume_token,
+                      Convert.timestamp_to_time(
+                        response.target_change.read_time
+                      )
+                    )
+
+                    if @inventory.current? && @inventory.changes?
+                      synchronize do
+                        send_callback @inventory.build_query_snapshot
+                      end
+                    end
+                  when :CURRENT
+                    # The targets reflect all changes committed before the targets
+                    # were added to the stream.
+                    #
+                    # This will be sent after or with a +read_time+ that is
+                    # greater than or equal to the time at which the targets were
+                    # added.
+                    #
+                    # Listeners can wait for this change if read-after-write
+                    # semantics are desired.
+
+                    @inventory.persist(
+                      response.target_change.resume_token,
+                      Convert.timestamp_to_time(
+                        response.target_change.read_time
+                      )
+                    )
+
+                    @inventory.current = true
+                  when :RESET
+                    # The targets have been reset, and a new initial state for the
+                    # targets will be returned in subsequent changes.
+                    #
+                    # After the initial state is complete, +CURRENT+ will be
+                    # returned even if the target was previously indicated to be
+                    # +CURRENT+.
+
+                    @inventory.reset
+                    raise RestartStream # Raise to restart the stream
+                  end
+                when :document_change
+                  # A {Google::Firestore::V1beta1::Document Document} has changed.
+
+                  if response.document_change.removed_target_ids.any?
+                    @inventory.delete response.document_change.document.name
+                  else
+                    @inventory.add response.document_change.document
+                  end
+                when :document_delete
+                  # A {Google::Firestore::V1beta1::Document Document} has been
+                  # deleted.
+
+                  @inventory.delete response.document_delete.document
+                when :document_remove
+                  # A {Google::Firestore::V1beta1::Document Document} has been
+                  # removed from a target (because it is no longer relevant to
+                  # that target).
+
+                  @inventory.delete response.document_remove.document
+                when :filter
+                  # A filter to apply to the set of documents previously returned
+                  # for the given target.
+                  #
+                  # Returned when documents may have been removed from the given
+                  # target, but the exact documents are unknown.
+
+                  if response.filter.count != @inventory.count_with_pending
+                    @inventory.reset
+                    raise RestartStream # Raise to restart the stream
+                  end
+                end
+              rescue StopIteration
+                break
+              end
+
+              # Reset backoff values when completed without an error
+              @backoff[:current] = 0
+              @backoff[:delay] = 1.0
+            end
+
+            # Has the loop broken but we aren't stopped?
+            # Could be GRPC has thrown an internal error, so restart.
+            raise RestartStream unless synchronize { @stopped }
+
+            # We must be stopped, tell the stream to quit.
+            @request_queue.push self
+          rescue GRPC::Cancelled, GRPC::DeadlineExceeded, GRPC::Internal,
+                 GRPC::ResourceExhausted, GRPC::Unauthenticated,
+                 GRPC::Unavailable, GRPC::Core::CallError
+            # Restart the stream with an incremental back for a retriable error.
+            # Also when GRPC raises the internal CallError.
+
+            # Re-raise if retried more than the max
+            raise err if @backoff[:current] > @backoff[:max]
+
+            # Sleep with incremental backoff before restarting
+            sleep @backoff[:delay]
+
+            # Update increment backoff delay and retry counter
+            @backoff[:delay] *= @backoff[:mod]
+            @backoff[:current] += 1
+
+            retry
+          rescue RestartStream
+            retry
+          rescue StandardError => e
+            raise Google::Cloud::Error.from_error(e)
+          end
+        end
+      end
+    end
+  end
+end