google-cloud-firestore 2.7.2 → 2.15.0

data/lib/google/cloud/firestore/bulk_writer_operation.rb

@@ -0,0 +1,126 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "concurrent"
+require "google/cloud/firestore/bulk_writer_exception"
+
+
+module Google
+  module Cloud
+    module Firestore
+      ##
+      #
+      # @private A BulkWriterOperation object refers to a write operation and contains
+      # all the necessary information for a specific write task, including meta
+      # information like the current number of attempts
+      #
+      class BulkWriterOperation
+        attr_reader :retry_time
+        attr_reader :result
+        attr_reader :completion_event
+        attr_reader :write
+
+        ##
+        # Initialize the object
+        def initialize write, retries
+          @write = write
+          @failed_attempts = 0
+          @retries = retries
+          @retry_time = Time.now
+          @completion_event = Concurrent::Event.new
+        end
+
+        ##
+        # Processing to be done when the response is a success.
+        # Updates the result and set the completion event.
+        #
+        # @param [Google::Cloud::Firestore::V1::WriteResult] result The result returned in the response.
+        def on_success result
+          begin
+            @result = WriteResult.new result
+          rescue StandardError => e
+            raise BulkWriterOperationError, e
+          ensure
+            @completion_event.set
+          end
+        end
+
+        ##
+        # Processing to be done when the response is a failure.
+        # Updates the failure attempts. If the retry count reaches
+        # the upper threshold, operations will be marked
+        # as failure and the completion event will be set.
+        #
+        # @param [Google::Rpc::Status] status The status received in the response.
+        #
+        def on_failure status
+          @failed_attempts += 1
+          if @failed_attempts == @retries + 1
+            begin
+              @result = BulkWriterException.new status
+            rescue StandardError => e
+              raise BulkWriterOperationError, e
+            ensure
+              @completion_event.set
+            end
+          else
+            backoff_duration
+          end
+        end
+
+        ##
+        # Exponentially increases the waiting time for retry.
+        #
+        def backoff_duration
+          @retry_time = Time.now + (@failed_attempts**2)
+        end
+
+        ##
+        # Represents the result of applying a write.
+        #
+        # @example
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   # Set the data for NYC
+        #   result = bw.set("cities/NYC", { name: "New York City" })
+        #
+        #   result.wait!
+        #
+        #   puts result.value
+        #
+        class WriteResult
+          ##
+          # The last update time of the document after applying the write. Set to
+          # nil for a +delete+ mutation.
+          #
+          # If the write did not actually change the document, this will be
+          # the previous update_time.
+          #
+          # @return [Time] The last update time.
+          attr_reader :update_time
+
+          ##
+          # @private
+          def initialize result
+            @update_time = Convert.timestamp_to_time result.update_time
+          end
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/firestore/bulk_writer_scheduler.rb

@@ -0,0 +1,164 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "concurrent"
+require "google/cloud/firestore/errors"
+require "google/cloud/firestore/bulk_writer_operation"
+require "google/cloud/firestore/rate_limiter"
+require "google/cloud/firestore/bulk_commit_batch"
+require "google/cloud/firestore/bulk_writer_exception"
+require "google/cloud/firestore/bulk_writer_scheduler"
+
+
+module Google
+  module Cloud
+    module Firestore
+      ##
+      #
+      # @private Accumulate BulkWriterOperations from the BulkWriter, schedules them
+      # in accordance with 555 rule and retry the failed operations from the BulkCommitBatch.
+      #
+      class BulkWriterScheduler
+        MAX_BATCH_SIZE = 20
+        BATCH_THREAD_COUNT = 4
+
+        ##
+        # Initialize the attributes and start the schedule_operations job
+        #
+        def initialize client, service, batch_threads
+          @client = client
+          @service = service
+          @rate_limiter = RateLimiter.new
+          @buffered_operations = []
+          @batch_threads = (batch_threads || BATCH_THREAD_COUNT).to_i
+          @batch_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @batch_threads,
+                                                                  max_queue: 0,
+                                                                  auto_terminate: true
+          @retry_operations = []
+          @mutex = Mutex.new
+          start_scheduling_operations
+        end
+
+        def start_scheduling_operations
+          Concurrent::Promises.future_on @batch_thread_pool do
+            begin
+              schedule_operations
+            rescue StandardError
+              # TODO: Log the error when logging is available
+              retry
+            end
+          end
+        end
+
+        def add_operation operation
+          @mutex.synchronize { @buffered_operations << operation }
+        end
+
+        ##
+        # Closes the scheduler object.
+        # Waits for the enqueued tasks to complete
+        # before closing down.
+        #
+        # @return [nil]
+        def close
+          @mutex.synchronize do
+            @batch_thread_pool.shutdown
+            @batch_thread_pool.wait_for_termination 1
+            @batch_thread_pool.kill unless @batch_thread_pool.shutdown?
+          end
+        end
+
+        private
+
+        ##
+        # @private Adds failed operations in the retry heap.
+        #
+        def post_commit_batch bulk_commit_batch
+          @mutex.synchronize do
+            bulk_commit_batch.operations.each do |operation|
+              unless operation.completion_event.set?
+                @retry_operations << operation
+              end
+            end
+            @retry_operations.sort_by!(&:retry_time)
+          end
+        end
+
+        ##
+        # @private Commits a batch of scheduled operations.
+        # Batch size <= 20 to match the constraint of request size < 9.8 MB
+        #
+        # @return [nil]
+        def commit_batch bulk_commit_batch
+          begin
+            Concurrent::Promises.future_on @batch_thread_pool, bulk_commit_batch do |batch|
+              begin
+                batch.commit
+              rescue StandardError
+                # TODO: Log the errors while committing a batch
+              ensure
+                post_commit_batch bulk_commit_batch
+              end
+            end
+          rescue StandardError => e
+            post_commit_batch bulk_commit_batch
+            raise BulkWriterSchedulerError, e.message
+          end
+        end
+
+        ##
+        # @private Schedule the enqueued operations in batches.
+        #
+        # @return [nil]
+        def schedule_operations
+          loop do
+            break if @batch_thread_pool.shuttingdown?
+            dequeue_retry_operations
+            batch_size = [MAX_BATCH_SIZE, @buffered_operations.length].min
+            if batch_size.zero?
+              sleep 0.001
+              next
+            end
+            @rate_limiter.wait_for_tokens batch_size
+            operations = dequeue_buffered_operations batch_size
+            commit_batch BulkCommitBatch.new(@service, operations)
+          end
+        end
+
+        ##
+        # @private Removes BulkWriterOperations from the buffered queue to scheduled in
+        # the current batch
+        #
+        def dequeue_buffered_operations size
+          @mutex.synchronize do
+            @buffered_operations.shift size
+          end
+        end
+
+        ##
+        # @private Removes BulkWriterOperations from the retry queue to scheduled in
+        # the current batch
+        #
+        def dequeue_retry_operations
+          @mutex.synchronize do
+            while @retry_operations.length.positive? && @retry_operations.first.retry_time <= Time.now
+              @buffered_operations << @retry_operations.shift
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/firestore/client.rb

@@ -23,6 +23,8 @@ require "google/cloud/firestore/document_snapshot"
 require "google/cloud/firestore/collection_group"
 require "google/cloud/firestore/batch"
 require "google/cloud/firestore/transaction"
+require "google/cloud/firestore/bulk_writer"
+require "google/cloud/firestore/filter"
 
 module Google
   module Cloud
@@ -69,7 +71,7 @@ module Google
         #
         # @return [String] database identifier.
         def database_id
-          "(default)"
+          service.database
         end
 
         ##
@@ -85,6 +87,9 @@ module Google
         ##
         # Retrieves an enumerator for the root collections.
         #
+        # @param [Time] read_time Reads documents as they were at the given time.
+        #   This may not be older than 270 seconds. Optional
+        #
         # @yield [collections] The block for accessing the collections.
         # @yieldparam [CollectionReference] collection A collection reference object.
         #
@@ -101,10 +106,21 @@ module Google
         #     puts col.collection_id
         #   end
         #
-        def cols &block
+        # @example
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   read_time = Time.now
+        #
+        #   # Get the root collections
+        #   firestore.cols(read_time: read_time).each do |col|
+        #     puts col.collection_id
+        #   end
+        #
+        def cols read_time: nil, &block
           ensure_service!
-          grpc = service.list_collections "#{path}/documents"
-          cols_enum = CollectionReferenceList.from_grpc(grpc, self, "#{path}/documents").all
+          grpc = service.list_collections "#{path}/documents", read_time: read_time
+          cols_enum = CollectionReferenceList.from_grpc(grpc, self, "#{path}/documents", read_time: read_time).all
           cols_enum.each(&block) if block_given?
           cols_enum
         end
@@ -171,6 +187,56 @@ module Google
         end
         alias collection_group col_group
 
+        ##
+        # Creates a filter object.
+        #
+        # @param field [FieldPath, String, Symbol] A field path to filter
+        #   results with.
+        #   If a {FieldPath} object is not provided then the field will be
+        #   treated as a dotted string, meaning the string represents individual
+        #   fields joined by ".". Fields containing `~`, `*`, `/`, `[`, `]`, and
+        #   `.` cannot be in a dotted string, and should provided using a
+        #   {FieldPath} object instead.
+        #
+        # @param operator [String, Symbol] The operation to compare the field
+        #   to. Acceptable values include:
+        #   * less than: `<`, `lt`
+        #   * less than or equal: `<=`, `lte`
+        #   * greater than: `>`, `gt`
+        #   * greater than or equal: `>=`, `gte`
+        #   * equal: `=`, `==`, `eq`, `eql`, `is`
+        #   * not equal: `!=`
+        #   * in: `in`
+        #   * not in: `not-in`, `not_in`
+        #   * array contains: `array-contains`, `array_contains`
+        #
+        # @param value [Object] The value to compare the property to. Defaults to nil.
+        #   Possible values are:
+        #   * Integer
+        #   * Float/BigDecimal
+        #   * String
+        #   * Boolean
+        #   * Array
+        #   * Date/Time
+        #   * StringIO
+        #   * Google::Cloud::Datastore::Key
+        #   * Google::Cloud::Datastore::Entity
+        #   * nil
+        #
+        # @return [Google::Cloud::Firestore::Filter] New filter object.
+        #
+        # @example
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #
+        #   # Create a filter
+        #   filter = firestore.filter(:population, :>=, 1000000)
+        #
+        def filter field, operator, value
+          Filter.new field, operator, value
+        end
+
         ##
         # Retrieves a document reference.
         #
@@ -217,6 +283,8 @@ module Google
         #   individual fields joined by ".". Fields containing `~`, `*`, `/`,
         #   `[`, `]`, and `.` cannot be in a dotted string, and should provided
         #   using a {FieldPath} object instead. (See {#field_path}.)
+        # @param [Time] read_time Reads documents as they were at the given time.
+        #   This may not be older than 270 seconds. Optional
         #
         # @yield [documents] The block for accessing the document snapshots.
         # @yieldparam [DocumentSnapshot] document A document snapshot.
@@ -245,11 +313,24 @@ module Google
         #     puts "#{city.document_id} has #{city[:population]} residents."
         #   end
         #
-        def get_all *docs, field_mask: nil
+        # @example Get docs using a read_time:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #
+        #   read_time = Time.now
+        #
+        #   # Get and print city documents
+        #   cities = ["cities/NYC", "cities/SF", "cities/LA"]
+        #   firestore.get_all(cities, read_time: read_time).each do |city|
+        #     puts "#{city.document_id} has #{city[:population]} residents."
+        #   end
+        #
+        def get_all *docs, field_mask: nil, read_time: nil
           ensure_service!
 
           unless block_given?
-            return enum_for :get_all, *docs, field_mask: field_mask
+            return enum_for :get_all, *docs, field_mask: field_mask, read_time: read_time
           end
 
           doc_paths = Array(docs).flatten.map do |doc_path|
@@ -264,7 +345,7 @@ module Google
           end
           mask = nil if mask.empty?
 
-          results = service.get_documents doc_paths, mask: mask
+          results = service.get_documents doc_paths, mask: mask, read_time: read_time
           results.each do |result|
             next if result.result.nil?
             yield DocumentSnapshot.from_batch_result result, self
@@ -668,13 +749,83 @@ module Google
           end
         end
 
+        ##
+        # Create a transaction to perform multiple reads that are
+        # executed atomically at a single logical point in time in a database.
+        #
+        # All changes are accumulated in memory until the block completes.
+        # Transactions will be automatically retried when documents change
+        # before the transaction is committed. See {Transaction}.
+        #
+        # @see https://firebase.google.com/docs/firestore/manage-data/transactions
+        #   Transactions and Batched Writes
+        #
+        # @param [Time] read_time The maximum number of retries for
+        #   transactions failed due to errors. Default is 5. Optional.
+        #
+        # @yield [transaction] The block for reading data.
+        # @yieldparam [Transaction] transaction The transaction object for
+        #   making changes.
+        #
+        # @return [Object] The return value of the provided
+        #   yield block
+        #
+        # @example Read only transaction with read time
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #
+        #   # Get a document reference
+        #   nyc_ref = firestore.doc "cities/NYC"
+        #
+        #   read_time = Time.now
+        #
+        #   firestore.read_only_transaction(read_time: read_time) do |tx|
+        #     # Get a document snapshot
+        #     nyc_snap = tx.get nyc_ref
+        #   end
+        #
+        def read_only_transaction read_time: nil
+          transaction = Transaction.from_client self, read_time: read_time, read_only: true
+          yield transaction
+        end
+
+        ##
+        # Create a bulk writer to perform multiple writes that are
+        # executed parallely.
+        #
+        # @param [Integer] request_threads The number of threads used for handling
+        #  requests. Default is 2. Optional.
+        # @param [Integer] batch_threads The number of threads used for processing
+        #  batches. Default is 4. Optional.
+        # @param [Integer] retries The number of times a failed write request will
+        # be retried (with exponential delay) before being marked as failure. Max
+        # attempts are 15. Optional
+        #
+        # @return [Google::Cloud::Firestore::BulkWriter] Returns an object of
+        #   bulk writer.
+        #
+        # @example Initializing a BulkWriter with all the configurations.
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #
+        #   bw = firestore.bulk_writer
+        #
+        #   bulk_write_result = bw.create "doc_ref", request_threads: 4, batch_threads: 10, retries: 10
+        #
+        def bulk_writer request_threads: nil, batch_threads: nil, retries: nil
+          BulkWriter.new self, @service, request_threads: request_threads,
+                         batch_threads: batch_threads, retries: retries
+        end
+
         # @!endgroup
 
         # @private
-        def list_documents parent, collection_id, token: nil, max: nil
+        def list_documents parent, collection_id, token: nil, max: nil, read_time: nil
           ensure_service!
-          grpc = service.list_documents parent, collection_id, token: token, max: max
-          DocumentReference::List.from_grpc grpc, self, parent, collection_id
+          grpc = service.list_documents parent, collection_id, token: token, max: max, read_time: read_time
+          DocumentReference::List.from_grpc grpc, self, parent, collection_id, read_time: read_time
         end
 
         protected

data/lib/google/cloud/firestore/collection_group.rb

@@ -48,6 +48,8 @@ module Google
         #
         # @param [Integer] partition_count The desired maximum number of partition points. The number must be strictly
         #   positive. The actual number of partitions returned may be fewer.
+        # @param [Time] read_time Reads documents as they were at the given time.
+        #   This may not be older than 270 seconds. Optional
         #
         # @return [Array<QueryPartition>] An ordered array of query partitions.
         #
@@ -62,7 +64,20 @@ module Google
         #
         #   queries = partitions.map(&:to_query)
         #
-        def partitions partition_count
+        # @example partition with read time
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #
+        #   col_group = firestore.col_group "cities"
+        #
+        #   read_time = Time.now
+        #
+        #   partitions = col_group.partitions 3, read_time: read_time
+        #
+        #   queries = partitions.map(&:to_query)
+        #
+        def partitions partition_count, read_time: nil
           ensure_service!
 
           raise ArgumentError, "partition_count must be > 0" unless partition_count.positive?
@@ -75,7 +90,7 @@ module Google
 
           grpc_partitions = if partition_count.positive?
                               # Retrieve all pages, since cursor order is not guaranteed and they must be sorted.
-                              list_all partition_count, query_with_default_order
+                              list_all partition_count, query_with_default_order, read_time
                             else
                               [] # Ensure that a single, empty QueryPartition is returned.
                             end
@@ -118,11 +133,12 @@ module Google
 
         protected
 
-        def list_all partition_count, query_with_default_order
+        def list_all partition_count, query_with_default_order, read_time
           grpc_partitions = []
           token = nil
           loop do
-            grpc = service.partition_query parent_path, query_with_default_order.query, partition_count, token: token
+            grpc = service.partition_query parent_path, query_with_default_order.query, partition_count,
+                                           token: token, read_time: read_time
             grpc_partitions += Array(grpc.partitions)
             token = grpc.next_page_token
             token = nil if token == ""

data/lib/google/cloud/firestore/collection_reference.rb

@@ -147,6 +147,8 @@ module Google
         # @param [String] token A previously-returned page token representing
         #   part of the larger set of results to view.
         # @param [Integer] max Maximum number of results to return.
+        # @param [Time] read_time Reads documents as they were at the given time.
+        #   This may not be older than 270 seconds. Optional
         #
         # @return [Array<DocumentReference>] An array of document references.
         #
@@ -161,10 +163,23 @@ module Google
         #     puts doc_ref.document_id
         #   end
         #
-        def list_documents token: nil, max: nil
+        # @example List documents with read time
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #
+        #   read_time = Time.now
+        #
+        #   col = firestore.col "cities"
+        #
+        #   col.list_documents(read_time: read_time).each do |doc_ref|
+        #     puts doc_ref.document_id
+        #   end
+        #
+        def list_documents token: nil, max: nil, read_time: nil
           ensure_client!
           client.list_documents \
-            parent_path, collection_id, token: token, max: max
+            parent_path, collection_id, token: token, max: max, read_time: read_time
         end
 
         ##

data/lib/google/cloud/firestore/collection_reference_list.rb

@@ -52,8 +52,8 @@ module Google
         def next
           return nil unless next?
           ensure_service!
-          grpc = @client.service.list_collections @parent, token: token, max: @max
-          self.class.from_grpc grpc, @client, @parent, max: @max
+          grpc = @client.service.list_collections @parent, token: token, max: @max, read_time: @read_time
+          self.class.from_grpc grpc, @client, @parent, max: @max, read_time: @read_time
         end
 
         ##
@@ -110,7 +110,7 @@ module Google
         ##
         # @private New CollectionReference::List from a `Google::Cloud::Firestore::V1::ListCollectionIdsResponse`
         # object.
-        def self.from_grpc grpc, client, parent, max: nil
+        def self.from_grpc grpc, client, parent, max: nil, read_time: nil
           raise ArgumentError, "parent is required" unless parent
           cols = CollectionReferenceList.new(Array(grpc.collection_ids).map do |collection_id|
             CollectionReference.from_path "#{parent}/#{collection_id}", client
@@ -121,6 +121,7 @@ module Google
           cols.instance_variable_set :@client, client
           cols.instance_variable_set :@parent, parent
           cols.instance_variable_set :@max, max
+          cols.instance_variable_set :@read_time, read_time
           cols
         end
 

data/lib/google/cloud/firestore/convert.rb

@@ -365,7 +365,7 @@ module Google
               write.current_document = \
                 Google::Cloud::Firestore::V1::Precondition.new({
                   exists: exists, update_time: time_to_timestamp(update_time)
-                }.delete_if { |_, v| v.nil? })
+                }.compact)
             end
 
             write
@@ -381,12 +381,12 @@ module Google
                 return val if val
               end
             when Hash
-              obj.each do |_k, v|
+              obj.each_value do |v|
                 val = field_value_nested? v, field_value_type
                 return val if val
               end
             end
-            nil
+            nil # rubocop:disable Style/ReturnNilInPredicateMethodDefinition
           end
 
           def remove_field_value_from obj, field_value_type = nil
@@ -498,7 +498,6 @@ module Google
               dup_hash = dup_hash[field]
             end
             prev_hash[last_field] = dup_hash
-            prev_hash.delete_if { |_k, v| v.nil? }
             ret_hash
           end
 
@@ -536,13 +535,13 @@ module Google
           ESCAPED_FIELD_PATH = /\A`(.*)`\z/.freeze
 
           def build_hash_from_field_paths_and_values pairs
-            pairs.each do |field_path, _value|
-              raise ArgumentError unless field_path.is_a? FieldPath
+            pairs.each do |pair|
+              raise ArgumentError unless pair.first.is_a? FieldPath
             end
 
             dup_hash = {}
 
-            pairs.each do |field_path, value|
+            pairs.each do |(field_path, value)|
               tmp_dup = dup_hash
               last_field = nil
               field_path.fields.map(&:to_sym).each do |field|