google-cloud-firestore 2.12.0 → 2.13.1

data/lib/google/cloud/firestore/bulk_writer_operation.rb

@@ -0,0 +1,126 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "concurrent"
+require "google/cloud/firestore/bulk_writer_exception"
+
+
+module Google
+  module Cloud
+    module Firestore
+      ##
+      #
+      # @private A BulkWriterOperation object refers to a write operation and contains
+      # all the necessary information for a specific write task, including meta
+      # information like the current number of attempts
+      #
+      class BulkWriterOperation
+        attr_reader :retry_time
+        attr_reader :result
+        attr_reader :completion_event
+        attr_reader :write
+
+        ##
+        # Initialize the object
+        def initialize write, retries
+          @write = write
+          @failed_attempts = 0
+          @retries = retries
+          @retry_time = Time.now
+          @completion_event = Concurrent::Event.new
+        end
+
+        ##
+        # Processing to be done when the response is a success.
+        # Updates the result and set the completion event.
+        #
+        # @param [Google::Cloud::Firestore::V1::WriteResult] result The result returned in the response.
+        def on_success result
+          begin
+            @result = WriteResult.new result
+          rescue StandardError => e
+            raise BulkWriterOperationError, e
+          ensure
+            @completion_event.set
+          end
+        end
+
+        ##
+        # Processing to be done when the response is a failure.
+        # Updates the failure attempts. If the retry count reaches
+        # the upper threshold, operations will be marked
+        # as failure and the completion event will be set.
+        #
+        # @param [Google::Rpc::Status] status The status received in the response.
+        #
+        def on_failure status
+          @failed_attempts += 1
+          if @failed_attempts == @retries + 1
+            begin
+              @result = BulkWriterException.new status
+            rescue StandardError => e
+              raise BulkWriterOperationError, e
+            ensure
+              @completion_event.set
+            end
+          else
+            backoff_duration
+          end
+        end
+
+        ##
+        # Exponentially increases the waiting time for retry.
+        #
+        def backoff_duration
+          @retry_time = Time.now + (@failed_attempts**2)
+        end
+
+        ##
+        # Represents the result of applying a write.
+        #
+        # @example
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   # Set the data for NYC
+        #   result = bw.set("cities/NYC", { name: "New York City" })
+        #
+        #   result.wait!
+        #
+        #   puts result.value
+        #
+        class WriteResult
+          ##
+          # The last update time of the document after applying the write. Set to
+          # nil for a +delete+ mutation.
+          #
+          # If the write did not actually change the document, this will be
+          # the previous update_time.
+          #
+          # @return [Time] The last update time.
+          attr_reader :update_time
+
+          ##
+          # @private
+          def initialize result
+            @update_time = Convert.timestamp_to_time result.update_time
+          end
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/firestore/bulk_writer_scheduler.rb

@@ -0,0 +1,164 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "concurrent"
+require "google/cloud/firestore/errors"
+require "google/cloud/firestore/bulk_writer_operation"
+require "google/cloud/firestore/rate_limiter"
+require "google/cloud/firestore/bulk_commit_batch"
+require "google/cloud/firestore/bulk_writer_exception"
+require "google/cloud/firestore/bulk_writer_scheduler"
+
+
+module Google
+  module Cloud
+    module Firestore
+      ##
+      #
+      # @private Accumulate BulkWriterOperations from the BulkWriter, schedules them
+      # in accordance with 555 rule and retry the failed operations from the BulkCommitBatch.
+      #
+      class BulkWriterScheduler
+        MAX_BATCH_SIZE = 20
+        BATCH_THREAD_COUNT = 4
+
+        ##
+        # Initialize the attributes and start the schedule_operations job
+        #
+        def initialize client, service, batch_threads
+          @client = client
+          @service = service
+          @rate_limiter = RateLimiter.new
+          @buffered_operations = []
+          @batch_threads = (batch_threads || BATCH_THREAD_COUNT).to_i
+          @batch_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @batch_threads,
+                                                                  max_queue: 0,
+                                                                  auto_terminate: true
+          @retry_operations = []
+          @mutex = Mutex.new
+          start_scheduling_operations
+        end
+
+        def start_scheduling_operations
+          Concurrent::Promises.future_on @batch_thread_pool do
+            begin
+              schedule_operations
+            rescue StandardError
+              # TODO: Log the error when logging is available
+              retry
+            end
+          end
+        end
+
+        def add_operation operation
+          @mutex.synchronize { @buffered_operations << operation }
+        end
+
+        ##
+        # Closes the scheduler object.
+        # Waits for the enqueued tasks to complete
+        # before closing down.
+        #
+        # @return [nil]
+        def close
+          @mutex.synchronize do
+            @batch_thread_pool.shutdown
+            @batch_thread_pool.wait_for_termination 1
+            @batch_thread_pool.kill unless @batch_thread_pool.shutdown?
+          end
+        end
+
+        private
+
+        ##
+        # @private Adds failed operations in the retry heap.
+        #
+        def post_commit_batch bulk_commit_batch
+          @mutex.synchronize do
+            bulk_commit_batch.operations.each do |operation|
+              unless operation.completion_event.set?
+                @retry_operations << operation
+              end
+            end
+            @retry_operations.sort_by!(&:retry_time)
+          end
+        end
+
+        ##
+        # @private Commits a batch of scheduled operations.
+        # Batch size <= 20 to match the constraint of request size < 9.8 MB
+        #
+        # @return [nil]
+        def commit_batch bulk_commit_batch
+          begin
+            Concurrent::Promises.future_on @batch_thread_pool, bulk_commit_batch do |batch|
+              begin
+                batch.commit
+              rescue StandardError
+                # TODO: Log the errors while committing a batch
+              ensure
+                post_commit_batch bulk_commit_batch
+              end
+            end
+          rescue StandardError => e
+            post_commit_batch bulk_commit_batch
+            raise BulkWriterSchedulerError, e.message
+          end
+        end
+
+        ##
+        # @private Schedule the enqueued operations in batches.
+        #
+        # @return [nil]
+        def schedule_operations
+          loop do
+            break if @batch_thread_pool.shuttingdown?
+            dequeue_retry_operations
+            batch_size = [MAX_BATCH_SIZE, @buffered_operations.length].min
+            if batch_size.zero?
+              sleep 0.001
+              next
+            end
+            @rate_limiter.wait_for_tokens batch_size
+            operations = dequeue_buffered_operations batch_size
+            commit_batch BulkCommitBatch.new(@service, operations)
+          end
+        end
+
+        ##
+        # @private Removes BulkWriterOperations from the buffered queue to scheduled in
+        # the current batch
+        #
+        def dequeue_buffered_operations size
+          @mutex.synchronize do
+            @buffered_operations.shift size
+          end
+        end
+
+        ##
+        # @private Removes BulkWriterOperations from the retry queue to scheduled in
+        # the current batch
+        #
+        def dequeue_retry_operations
+          @mutex.synchronize do
+            while @retry_operations.length.positive? && @retry_operations.first.retry_time <= Time.now
+              @buffered_operations << @retry_operations.shift
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/firestore/client.rb

@@ -23,6 +23,7 @@ require "google/cloud/firestore/document_snapshot"
 require "google/cloud/firestore/collection_group"
 require "google/cloud/firestore/batch"
 require "google/cloud/firestore/transaction"
+require "google/cloud/firestore/bulk_writer"
 require "google/cloud/firestore/filter"
 
 module Google
@@ -789,6 +790,35 @@ module Google
           yield transaction
         end
 
+        ##
+        # Create a bulk writer to perform multiple writes that are
+        # executed parallely.
+        #
+        # @param [Integer] request_threads The number of threads used for handling
+        #  requests. Default is 2. Optional.
+        # @param [Integer] batch_threads The number of threads used for processing
+        #  batches. Default is 4. Optional.
+        # @param [Integer] retries The number of times a failed write request will
+        # be retried (with exponential delay) before being marked as failure. Max
+        # attempts are 15. Optional
+        #
+        # @return [Google::Cloud::Firestore::BulkWriter] Returns an object of
+        #   bulk writer.
+        #
+        # @example Initializing a BulkWriter with all the configurations.
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #
+        #   bw = firestore.bulk_writer
+        #
+        #   bulk_write_result = bw.create "doc_ref", request_threads: 4, batch_threads: 10, retries: 10
+        #
+        def bulk_writer request_threads: nil, batch_threads: nil, retries: nil
+          BulkWriter.new self, @service, request_threads: request_threads,
+                         batch_threads: batch_threads, retries: retries
+        end
+
         # @!endgroup
 
         # @private

data/lib/google/cloud/firestore/convert.rb

@@ -365,7 +365,7 @@ module Google
               write.current_document = \
                 Google::Cloud::Firestore::V1::Precondition.new({
                   exists: exists, update_time: time_to_timestamp(update_time)
-                }.delete_if { |_, v| v.nil? })
+                }.compact)
             end
 
             write

data/lib/google/cloud/firestore/errors.rb

@@ -0,0 +1,60 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "google/cloud/errors"
+
+module Google
+  module Cloud
+    module Firestore
+      ##
+      # Indicates that the an error was reported while scheduling
+      # BulkWriter operations.
+      #
+      class BulkWriterSchedulerError < Google::Cloud::Error
+        def initialize message
+          super "BulkWriterSchedulerError : #{message}"
+        end
+      end
+
+      ##
+      # Indicates that the an error was reported while committing a
+      # batch of operations.
+      #
+      class BulkCommitBatchError < Google::Cloud::Error
+        def initialize message
+          super "BulkCommitBatchError : #{message}"
+        end
+      end
+
+      ##
+      # Indicates that the an error was reported while parsing response for
+      # BulkWriterOperation.
+      #
+      class BulkWriterOperationError < Google::Cloud::Error
+        def initialize message
+          super "BulkWriterOperationError : #{message}"
+        end
+      end
+
+      ##
+      # Indicates that the an error was reported in BulkWriter.
+      #
+      class BulkWriterError < Google::Cloud::Error
+        def initialize message
+          super "BulkWriterError : #{message}"
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/firestore/promise/future.rb

@@ -0,0 +1,97 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Google
+  module Cloud
+    module Firestore
+      module Promise
+        ##
+        # # Future
+        #
+        # A Future object represents a value which will become available in future.
+        # May reject with a reason instead, e.g. when the tasks raises an exception.
+        #
+        class Future
+          ##
+          # Initialize the future object
+          #
+          def initialize future
+            @future = future
+          end
+
+          # Is it in fulfilled state?
+          #
+          # @return [Boolean]
+          def fulfilled?
+            @future.fulfilled?
+          end
+
+          # Is it in rejected state?
+          #
+          # @return [Boolean]
+          def rejected?
+            @future.rejected?
+          end
+
+          ##
+          # Method waits for the timeout duration and return the value of the future if
+          # fulfilled, timeout value in case of timeout and nil in case of rejection.
+          #
+          # @param [Integer] timeout the maximum time in seconds to wait
+          # @param [Object] timeout_value a value returned by the method when it times out
+          # @return [Object, nil, timeout_value] the value of the Future when fulfilled,
+          #  timeout_value on timeout, nil on rejection.
+          def value timeout = nil, timeout_value = nil
+            @future.value timeout, timeout_value
+          end
+
+          # Returns reason of future's rejection.
+          #
+          # @return [Object, timeout_value] the reason, or timeout_value on timeout, or nil on fulfillment.
+          def reason timeout = nil, timeout_value = nil
+            @future.reason timeout, timeout_value
+          end
+
+          ##
+          # Method waits for the timeout duration and raise exception on rejection
+          #
+          # @param [Integer] timeout the maximum time in seconds to wait
+          def wait! timeout = nil
+            @future.wait! timeout
+          end
+
+          ##
+          # Chains the task to be executed synchronously after it fulfills. Does not run
+          # the task if it rejects. It will resolve though, triggering any dependent futures.
+          #
+          # @return [Future]
+          # @yield [reason, *args] to the task.
+          def then *args, &task
+            Future.new @future.then(*args, &task)
+          end
+
+          # Chains the task to be executed synchronously on executor after it rejects. Does
+          # not run the task if it fulfills. It will resolve though, triggering any
+          # dependent futures.
+          #
+          # @return [Future]
+          # @yield [reason, *args] to the task.
+          def rescue *args, &task
+            Future.new @future.rescue(*args, &task)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/firestore/rate_limiter.rb

@@ -0,0 +1,80 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+module Google
+  module Cloud
+    module Firestore
+      ##
+      # @private Implements 5/5/5 ramp-up via Token Bucket algorithm.
+      #
+      # 5/5/5 is a ramp up strategy that starts with a budget of 500 operations per
+      # second. Additionally, every 5 minutes, the maximum budget can increase by
+      # 50%. Thus, at 5:01 into a long bulk-writing process, the maximum budget
+      # becomes 750 operations per second. At 10:01, the budget becomes 1,125
+      # operations per second.
+      #
+      class RateLimiter
+        DEFAULT_STARTING_MAXIMUM_OPS_PER_SECOND = 500.0
+        DEFAULT_PHASE_LENGTH = 300.0
+
+        attr_reader :bandwidth
+
+        ##
+        # Initialize the object
+        def initialize starting_ops: nil, phase_length: nil
+          @start_time = time
+          @last_fetched = time
+          @bandwidth = (starting_ops || DEFAULT_STARTING_MAXIMUM_OPS_PER_SECOND).to_f
+          @phase_length = phase_length || DEFAULT_PHASE_LENGTH
+        end
+
+        ##
+        # Wait till the number of tokens is available
+        # Assumes that the bandwidth is distributed evenly across the entire second.
+        #
+        # Example - If the limit is 500 qps, then it has been further broken down to 2e+6 nsec
+        # per query
+        #
+        # @return [nil]
+        def wait_for_tokens size
+          available_time = @last_fetched + (size / @bandwidth)
+          waiting_time = [0, available_time - time].max
+          sleep waiting_time
+          @last_fetched = time
+          increase_bandwidth
+        end
+
+        private
+
+        ##
+        # Returns time elapsed since epoch.
+        #
+        # @return [Float] Float denoting time elapsed since epoch
+        def time
+          Time.now.to_f
+        end
+
+        ##
+        # Increase the bandwidth as per 555 rule
+        #
+        # @return [nil]
+        def increase_bandwidth
+          intervals = (time - @start_time) / @phase_length
+          @bandwidth = (DEFAULT_STARTING_MAXIMUM_OPS_PER_SECOND * (1.5**intervals.floor)).to_f
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/firestore/service.rb

@@ -186,6 +186,18 @@ module Google
           )
         end
 
+        ##
+        # Makes the BatchWrite API call. Contains the list of write operations to be processed.
+        #
+        # @return [::Google::Cloud::Firestore::V1::BatchWriteResponse]
+        def batch_write writes
+          batch_write_req = {
+            database: database_path,
+            writes: writes
+          }
+          firestore.batch_write batch_write_req, call_options(parent: database_path)
+        end
+
         def database_path project_id: project, database_id: database
           # Originally used V1::FirestoreClient.database_root_path until it was removed in #5405.
           "projects/#{project_id}/databases/#{database_id}"
@@ -222,7 +234,7 @@ module Google
           Gapic::CallOptions.new(**{
             metadata:   default_headers(parent),
             page_token: token
-          }.delete_if { |_, v| v.nil? })
+          }.compact)
         end
 
         def document_mask mask

data/lib/google/cloud/firestore/version.rb

@@ -16,7 +16,7 @@
 module Google
   module Cloud
     module Firestore
-      VERSION = "2.12.0".freeze
+      VERSION = "2.13.1".freeze
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: google-cloud-firestore
 version: !ruby/object:Gem::Version
-  version: 2.12.0
+  version: 2.13.1
 platform: ruby
 authors:
 - Google Inc
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-04-20 00:00:00.000000000 Z
+date: 2023-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-cloud-core
@@ -228,6 +228,11 @@ files:
 - lib/google/cloud/firestore/aggregate_query.rb
 - lib/google/cloud/firestore/aggregate_query_snapshot.rb
 - lib/google/cloud/firestore/batch.rb
+- lib/google/cloud/firestore/bulk_commit_batch.rb
+- lib/google/cloud/firestore/bulk_writer.rb
+- lib/google/cloud/firestore/bulk_writer_exception.rb
+- lib/google/cloud/firestore/bulk_writer_operation.rb
+- lib/google/cloud/firestore/bulk_writer_scheduler.rb
 - lib/google/cloud/firestore/client.rb
 - lib/google/cloud/firestore/collection_group.rb
 - lib/google/cloud/firestore/collection_reference.rb
@@ -240,14 +245,17 @@ files:
 - lib/google/cloud/firestore/document_reference.rb
 - lib/google/cloud/firestore/document_reference/list.rb
 - lib/google/cloud/firestore/document_snapshot.rb
+- lib/google/cloud/firestore/errors.rb
 - lib/google/cloud/firestore/field_path.rb
 - lib/google/cloud/firestore/field_value.rb
 - lib/google/cloud/firestore/filter.rb
 - lib/google/cloud/firestore/generate.rb
+- lib/google/cloud/firestore/promise/future.rb
 - lib/google/cloud/firestore/query.rb
 - lib/google/cloud/firestore/query_listener.rb
 - lib/google/cloud/firestore/query_partition.rb
 - lib/google/cloud/firestore/query_snapshot.rb
+- lib/google/cloud/firestore/rate_limiter.rb
 - lib/google/cloud/firestore/resource_path.rb
 - lib/google/cloud/firestore/service.rb
 - lib/google/cloud/firestore/transaction.rb