google-cloud-firestore 2.11.0 → 2.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c8fe0d4f3e79da21256a4495d0a19da58febfaf27298ea1735fef2042c8c7281
-  data.tar.gz: 10e4929cd461e369991e153356d3878add4b179c972715880734456b566832ca
+  metadata.gz: 99ee040593be1625729cd2eae0d3a8bda03185c53692bec5fdfe18a32c2021f1
+  data.tar.gz: 26ff29bf53e07708590e92e9db03955ce1c1c60d671f7cf7fbd4dad52b3b8eab
 SHA512:
-  metadata.gz: 7491e2f8033a2e2033e37e77b16e0401f9ecd77e9e357ecf47c72ab55204df6a7b8aa0a8bdf508d58215c3f6e3818a3ffaa7eebd0cc425e9355fe987df1bc478
-  data.tar.gz: 0f92c886ee8ba0043782aae64166947efd7d5ea1ccd0968fd21fa1bc2e7936f2eba3cb4805853a569d3631e221415fb2649a9c6a5aac101c87cdab1976558472
+  metadata.gz: 56a3248b0d03ec14935cc78aa2e0ca6fcf63b810f1800dc96ba3d19b56c1546e914e7449c30cf56b8460d81c1b2e2210bdfaaf035405cf8c7e9909d05d7a9fba
+  data.tar.gz: 3cb3a0a89e2ef1d3707e4f30097e2019d62fee21354e6f1575cf29b5bd61121415945a2ff23735c8611af849c5399b6f30f01e58bd1c03f591baa55359f7cd1f

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Release History
 
+### 2.13.0 (2023-05-10)
+
+#### Features
+
+* Added support for bulk writer ([#21426](https://github.com/googleapis/google-cloud-ruby/issues/21426)) 
+
+### 2.12.0 (2023-04-20)
+
+#### Features
+
+* Add support for OR query ([#20920](https://github.com/googleapis/google-cloud-ruby/issues/20920)) 
+
 ### 2.11.0 (2023-02-23)
 
 #### Features

data/lib/google/cloud/firestore/bulk_commit_batch.rb

@@ -0,0 +1,73 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+module Google
+  module Cloud
+    module Firestore
+      ##
+      #
+      # @private Accumulate write operations to be sent in a batch. Use this for higher
+      # volumes (e.g., via `BulkWriter`) and when the order of operations
+      # within a given batch is unimportant.
+      #
+      # Because the order in which individual write operations are applied to the database
+      # is not guaranteed, `batch_write` RPCs can never contain multiple operations
+      # to the same document. In practice, the BulkWriter class handle this case.
+      #
+      class BulkCommitBatch
+        attr_reader :operations
+
+        ##
+        # Initialize the object
+        def initialize service, operations
+          @service = service
+          @operations = operations
+        end
+
+        ##
+        # Updates the operation based on the result received from the API request.
+        #
+        # @param [Google::Cloud::Firestore::V1::BatchWriteResponse] responses
+        #
+        # @return [nil]
+        #
+        def parse_results responses
+          @operations.zip responses.write_results, responses.status do |operation, write_result, status|
+            begin
+              status&.code&.zero? ? operation.on_success(write_result) : operation.on_failure(status)
+            rescue StandardError
+              # TODO: Log the error while parsing response
+            end
+          end
+        end
+
+        ##
+        # Makes the BatchWrite API request with all the operations in the batch and
+        # parses the results for each operation.
+        #
+        # @return [nil]
+        #
+        def commit
+          begin
+            responses = @service.batch_write @operations.map(&:write)
+            parse_results responses
+          rescue StandardError => e
+            raise BulkCommitBatchError, e
+          end
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/firestore/bulk_writer.rb

@@ -0,0 +1,558 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "concurrent"
+require "google/cloud/firestore/rate_limiter"
+require "google/cloud/firestore/bulk_commit_batch"
+require "google/cloud/firestore/promise/future"
+require "google/cloud/firestore/bulk_writer_operation"
+require "google/cloud/firestore/bulk_writer_exception"
+require "google/cloud/firestore/bulk_writer_scheduler"
+
+
+module Google
+  module Cloud
+    module Firestore
+      ##
+      # #  BulkWriter
+      #
+      # Accumulate and efficiently sends large amounts of document write
+      # operations to the server.
+      #
+      # BulkWriter can handle large data migrations or updates, buffering records
+      # in memory and submitting them to the server in batches of 20.
+      #
+      # The submission of batches is internally parallelized with a ThreadPoolExecutor.
+      #
+      # @example Create a BulkWriter and add a write request:
+      #   require "google/cloud/firestore"
+      #
+      #   firestore = Google::Cloud::Firestore.new
+      #   bw = firestore.bulk_writer
+      #
+      #   bw.create("cities/NYC", { name: "New York City" })
+      #
+      #   bw.flush
+      #   bw.close
+      #
+      class BulkWriter
+        MAX_RETRY_ATTEMPTS = 10
+
+        ##
+        # Initialize the attributes and start the schedule_operations job
+        #
+        def initialize client, service,
+                       request_threads: nil,
+                       batch_threads: nil,
+                       retries: nil
+          @client = client
+          @service = service
+          @closed = false
+          @flush = false
+          @request_threads = (request_threads || 2).to_i
+          @write_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @request_threads,
+                                                                  max_queue: 0
+          @mutex = Mutex.new
+          @scheduler = BulkWriterScheduler.new client, service, batch_threads
+          @doc_refs = Set.new
+          @retries = [retries || MAX_RETRY_ATTEMPTS, MAX_RETRY_ATTEMPTS].min
+          @request_results = []
+        end
+
+        ##
+        # Creates a document with the provided data (fields and values).
+        #
+        # The operation will fail if the document already exists.
+        #
+        # @param [String, DocumentReference] doc A string representing the
+        #   path of the document, or a document reference object.
+        # @param [Hash] data The document's fields and values.
+        #
+        # @return [Google::Cloud::Firestore::Promise::Future] Denoting the future value of
+        #   write operation.
+        #
+        # @example Create a document using a document path:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   bw.create("cities/NYC", { name: "New York City" })
+        #
+        # @example Create a document using a document reference:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   # Get a document reference
+        #   nyc_ref = firestore.doc "cities/NYC"
+        #
+        #   bw.create(nyc_ref, { name: "New York City" })
+        #
+        # @example Create a document and set a field to server_time:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   # Get a document reference
+        #   nyc_ref = firestore.doc "cities/NYC"
+        #
+        #   bw.create(nyc_ref, { name: "New York City",
+        #                          updated_at: firestore.field_server_time })
+        #
+        # @example Get the value of write operation:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   # Get a document reference
+        #   nyc_ref = firestore.doc "cities/NYC"
+        #
+        #   result = bw.create(nyc_ref, { name: "New York City",
+        #                                 updated_at: firestore.field_server_time })
+        #
+        #   bw.close
+        #
+        #   puts result.value
+        #
+        def create doc, data
+          doc_path = coalesce_doc_path_argument doc
+          pre_add_operation doc_path
+
+          write = Convert.write_for_create doc_path, data
+
+          create_and_enqueue_operation write
+        end
+
+        ##
+        # Writes the provided data (fields and values) to the provided document.
+        # If the document does not exist, it will be created. By default, the
+        # provided data overwrites existing data, but the provided data can be
+        # merged into the existing document using the `merge` argument.
+        #
+        # If you're not sure whether the document exists, use the `merge`
+        # argument to merge the new data with any existing document data to
+        # avoid overwriting entire documents.
+        #
+        # @param [String, DocumentReference] doc A string representing the
+        #   path of the document, or a document reference object.
+        # @param [Hash] data The document's fields and values.
+        # @param [Boolean, FieldPath, String, Symbol] merge When
+        #   `true`, all provided data is merged with the existing document data.
+        #   When the argument is one or more field path, only the data for
+        #   fields in this argument is merged with the existing document data.
+        #   The default is to not merge, but to instead overwrite the existing
+        #   document data.
+        #
+        # @return [Google::Cloud::Firestore::Promise::Future] Denoting the future value of
+        #   write operation.
+        #
+        # @example Set a document using a document path:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   # Update a document
+        #   bw.set("cities/NYC", { name: "New York City" })
+        #
+        # @example Create a document using a document reference:
+        #   require "google/cloud/firestore"
+        #
+        #   bw = firestore.bulk_writer
+        #
+        #   # Get a document reference
+        #   nyc_ref = firestore.doc "cities/NYC"
+        #
+        #   # Update a document
+        #   bw.set(nyc_ref, { name: "New York City" })
+        #
+        # @example Set a document and merge all data:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   bw.set("cities/NYC", { name: "New York City" }, merge: true)
+        #
+        # @example Set a document and merge only name:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   bw.set("cities/NYC", { name: "New York City" }, merge: :name)
+        #
+        # @example Set a document and deleting a field using merge:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   # Get a document reference
+        #   nyc_ref = firestore.doc "cities/NYC"
+        #
+        #   nyc_data = { name: "New York City",
+        #                trash: firestore.field_delete }
+        #
+        #   bw.set(nyc_ref, nyc_data, merge: true)
+        #
+        # @example Set a document and set a field to server_time:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   # Get a document reference
+        #   nyc_ref = firestore.doc "cities/NYC"
+        #
+        #   nyc_data = { name: "New York City",
+        #                updated_at: firestore.field_server_time }
+        #
+        #   bw.set(nyc_ref, nyc_data, merge: true)
+        #
+        # @example Get the value of write operation:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   # Get a document reference
+        #   nyc_ref = firestore.doc "cities/NYC"
+        #
+        #   nyc_data = { name: "New York City",
+        #                updated_at: firestore.field_server_time }
+        #
+        #   result = bw.set(nyc_ref, nyc_data)
+        #
+        #   bw.close
+        #
+        #   puts result.value
+        #
+        def set doc, data, merge: nil
+          doc_path = coalesce_doc_path_argument doc
+          pre_add_operation doc_path
+
+          write = Convert.write_for_set doc_path, data, merge: merge
+
+          create_and_enqueue_operation write
+        end
+
+        ##
+        # Updates the document with the provided data (fields and values). The
+        # provided data is merged into the existing document data.
+        #
+        # The operation will fail if the document does not exist.
+        #
+        # @param [String, DocumentReference] doc A string representing the
+        #   path of the document, or a document reference object.
+        # @param [Hash<FieldPath|String|Symbol, Object>] data The document's
+        #   fields and values.
+        #
+        #   The top-level keys in the data hash are considered field paths, and
+        #   can either be a FieldPath object, or a string representing the
+        #   nested fields. In other words the string represents individual
+        #   fields joined by ".". Fields containing `~`, `*`, `/`, `[`, `]`, and
+        #   `.` cannot be in a dotted string, and should provided using a
+        #   {FieldPath} object instead.
+        # @param [Time] update_time When set, the document must have been last
+        #   updated at that time. Optional.
+        #
+        # @return [Google::Cloud::Firestore::Promise::Future] Denoting the future value of
+        #   write operation.
+        #
+        # @example Update a document using a document path:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   bw.update("cities/NYC", { name: "New York City" })
+        #
+        # @example Directly update a deeply-nested field with a `FieldPath`:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   nested_field_path = firestore.field_path :favorites, :food
+        #
+        #   bw.update("users/frank", { nested_field_path => "Pasta" })
+        #
+        # @example Update a document using a document reference:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   # Get a document reference
+        #   nyc_ref = firestore.doc "cities/NYC"
+        #
+        #   bw.update(nyc_ref, { name: "New York City" })
+        #
+        # @example Update a document using the `update_time` precondition:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   last_updated_at = Time.now - 42 # 42 seconds ago
+        #
+        #   bw.update("cities/NYC", { name: "New York City" },
+        #              update_time: last_updated_at)
+        #
+        # @example Update a document and deleting a field:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   # Get a document reference
+        #   nyc_ref = firestore.doc "cities/NYC"
+        #
+        #   nyc_data = { name: "New York City",
+        #                trash: firestore.field_delete }
+        #
+        #   bw.update(nyc_ref, nyc_data)
+        #
+        # @example Update a document and set a field to server_time:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   # Get a document reference
+        #   nyc_ref = firestore.doc "cities/NYC"
+        #
+        #   nyc_data = { name: "New York City",
+        #                updated_at: firestore.field_server_time }
+        #
+        #   bw.update(nyc_ref, nyc_data)
+        #
+        # @example Get the value of write operation:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   # Get a document reference
+        #   nyc_ref = firestore.doc "cities/NYC"
+        #
+        #   nyc_data = { name: "New York City",
+        #                updated_at: firestore.field_server_time }
+        #
+        #   result = bw.update(nyc_ref, nyc_data)
+        #
+        #   bw.close
+        #
+        #   puts result.value
+        #
+        def update doc, data, update_time: nil
+          doc_path = coalesce_doc_path_argument doc
+          pre_add_operation doc_path
+
+          write = Convert.write_for_update doc_path, data, update_time: update_time
+
+          create_and_enqueue_operation write
+        end
+
+        ##
+        # Deletes a document from the database.
+        #
+        # @param [String, DocumentReference] doc A string representing the
+        #   path of the document, or a document reference object.
+        # @param [Boolean] exists Whether the document must exist. When `true`,
+        #   the document must exist or an error is raised. Default is `false`.
+        #   Optional.
+        # @param [Time] update_time When set, the document must have been last
+        #   updated at that time. Optional.
+        #
+        # @return [Google::Cloud::Firestore::Promise::Future] Denoting the future value of
+        #   write operation.
+        #
+        # @example Delete a document using a document path:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   # Delete a document
+        #   bw.delete "cities/NYC"
+        #
+        # @example Delete a document using a document reference:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   # Get a document reference
+        #   nyc_ref = firestore.doc "cities/NYC"
+        #
+        #   # Delete a document
+        #   bw.delete nyc_ref
+        #
+        # @example Delete a document using `exists`:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   # Delete a document
+        #   bw.delete "cities/NYC", exists: true
+        #
+        # @example Delete a document using the `update_time` precondition:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   last_updated_at = Time.now - 42 # 42 seconds ago
+        #
+        #   # Delete a document
+        #   bw.delete "cities/NYC", update_time: last_updated_at
+        #
+        # @example Get the value of write operation:
+        #   require "google/cloud/firestore"
+        #
+        #   firestore = Google::Cloud::Firestore.new
+        #   bw = firestore.bulk_writer
+        #
+        #   last_updated_at = Time.now - 42 # 42 seconds ago
+        #
+        #   # Delete a document
+        #   result = bw.delete "cities/NYC", update_time: last_updated_at
+        #
+        #   bw.close
+        #
+        #   puts result.value
+        #
+        def delete doc, exists: nil, update_time: nil
+          doc_path = coalesce_doc_path_argument doc
+          pre_add_operation doc_path
+
+          write = Convert.write_for_delete doc_path, exists: exists, update_time: update_time
+
+          create_and_enqueue_operation write
+        end
+
+        ##
+        # Flushes all the current operation before enqueuing new operations.
+        #
+        # @return [nil]
+        def flush
+          @mutex.synchronize { @flush = true }
+          @request_results.each do |result|
+            begin
+              result.wait!
+            rescue StandardError
+              # Ignored
+            end
+          end
+          @mutex.synchronize do
+            @doc_refs = Set.new
+            @flush = false
+          end
+        end
+
+        ##
+        # Closes the BulkWriter object for new operations.
+        # Existing operations will be flushed and the threadpool will shutdown.
+        #
+        # @return [nil]
+        def close
+          @mutex.synchronize { @closed = true }
+          flush
+          @mutex.synchronize do
+            @write_thread_pool.shutdown
+            @scheduler.close
+          end
+        end
+
+        private
+
+        ##
+        # @private The client the Cloud Firestore BulkWriter belongs to.
+        #
+        # @return [Client] firestore client.
+        def firestore
+          @client
+        end
+        alias client firestore
+
+        ##
+        # @private Checks if the BulkWriter is accepting write requests
+        def accepting_request?
+          unless @closed || @flush
+            return true
+          end
+          false
+        end
+
+        ##
+        # @private Sanity checks before adding a write request in the BulkWriter
+        def pre_add_operation doc_path
+          @mutex.synchronize do
+            unless accepting_request?
+              raise BulkWriterError, "Not accepting responses for now. Either closed or in flush state"
+            end
+            if @doc_refs.include? doc_path
+              raise BulkWriterError, "Already contains mutations for this document"
+            end
+            @doc_refs.add doc_path
+          end
+        end
+
+        ##
+        # @private Creates a BulkWriterOperation
+        #
+        def create_operation write
+          BulkWriterOperation.new write, @retries
+        end
+
+        ##
+        # @private Adds a BulkWriterOperation to the scheduler.
+        def enqueue_operation operation
+          @mutex.synchronize { @scheduler.add_operation operation }
+        end
+
+        ##
+        # @private Creates a BulkWriterOperation and adds it in the scheduler.
+        #
+        def create_and_enqueue_operation write
+          operation = create_operation write
+          enqueue_operation operation
+          future = Concurrent::Promises.future_on @write_thread_pool, operation do |bulk_writer_operation|
+            bulk_writer_operation.completion_event.wait
+            raise bulk_writer_operation.result if bulk_writer_operation.result.is_a? BulkWriterException
+            bulk_writer_operation.result
+          end
+          @mutex.synchronize { @request_results << future }
+          Promise::Future.new future
+        end
+
+        ##
+        # @private
+        def coalesce_doc_path_argument doc_path
+          return doc_path.path if doc_path.respond_to? :path
+
+          client.doc(doc_path).path
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/firestore/bulk_writer_exception.rb

@@ -0,0 +1,40 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+module Google
+  module Cloud
+    module Firestore
+      ##
+      # #  BulkWriterException
+      #
+      # A BulkWriterException object refers to the error that will be thrown
+      # in case a BulkWriterOperation fails in all the attempts.
+      #
+      class BulkWriterException < StandardError
+        attr_reader :status
+        attr_reader :message
+        attr_reader :details
+
+        def initialize status
+          @status = status.code
+          @message = status.message
+          @details = status.details
+
+          super status.message
+        end
+      end
+    end
+  end
+end