elasticgraph-indexer 0.19.2.1 → 1.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 38ea0f67fa2d9c0972b621a8e6d018b8104b1ffcff7756b3071bef3d73e7d444
-  data.tar.gz: d77f2dea3a7f5e381bcf72693be14711a5de11c6132c2e5c0698528b374c29e9
+  metadata.gz: e5788fea72c47b1c983ffc469b17c39d606e655c6ceb465db62b31c2eb59a2a9
+  data.tar.gz: e4a0a9d49a9bb6ae35d0dd0a688289c103e70db9613029f0ed10224e0b474309
 SHA512:
-  metadata.gz: 188b88d42484f245fa3be6aeebff1f7d36f4ad3cbdb1e456a66b322735037cfd016ae592632ddbc05b42890b9043115eca0c1182c6306f20c1c262deb34c1a69
-  data.tar.gz: deb8433ce4429975cb3d6abd76eeb915c8a79e0ac31556ad26f4dde6c8c3e393bebdf4fcd3d5d920510b18f94c01d260ba4636ed9c23b43d740d63bb521dbdd2
+  metadata.gz: 1e214cebc685a4f948ee862a87e65b59b6ac9bbb87bcfa6ce778e4789f85eb6b2712e653386d0c1de7e0a7af6c42423f09c41ddf98fdcb6a69d5e17fe5f7f119
+  data.tar.gz: 4bee1404c3406dc042b5641d62727d651cdd4985937e36bae182e0ae3111a66b80a49a4afe4625c23f81514b4c39ea0ab711a5254a08329807022695a3f12f55

data/lib/elastic_graph/indexer/datastore_indexing_router.rb

@@ -198,27 +198,11 @@ module ElasticGraph
         client_names_and_results = Support::Threading.parallel_map(ops_by_client_name) do |(client_name, all_ops)|
           # @type block: [::String, ::Symbol, ::Array[untyped] | ::Hash[_Operation, ::Array[::Integer]]]
 
-          ops, unversioned_ops = all_ops.partition(&:versioned?)
+          ops, unversioned_ops = all_ops.partition(&:versioned?) # : [::Array[Operation::Update], ::Array[Operation::Update]]
 
           msearch_response =
             if (client = @datastore_clients_by_name[client_name]) && ops.any?
               body = ops.flat_map do |op|
-                # We only care about the source versions, but the way we get it varies.
-                include_version =
-                  if op.destination_index_def.use_updates_for_indexing?
-                    # @type var op: Operation::Update
-                    {_source: {includes: [
-                      "__versions.#{op.update_target.relationship}",
-                      # The update_data script before ElasticGraph v0.8 used __sourceVersions[type] instead of __versions[relationship].
-                      # To be backwards-compatible we need to fetch the data at both paths.
-                      #
-                      # TODO: Drop this when we no longer need to maintain backwards-compatibility.
-                      "__sourceVersions.#{op.event.fetch("type")}"
-                    ]}}
-                  else
-                    {version: true, _source: false}
-                  end
-
                 [
                   # Note: we intentionally search the entire index expression, not just an individual index based on a rollover timestamp.
                   # And we intentionally do NOT provide a routing value--we want to find the version, no matter what shard the document
@@ -229,8 +213,12 @@ module ElasticGraph
                   # a different shard and index than what the operation is targeted at. We want to search across all of them
                   # so that we will find it, regardless of where it lives.
                   {index: op.destination_index_def.index_expression_for_search},
-                  # Filter to the documents matching the id.
-                  {query: {ids: {values: [op.doc_id]}}}.merge(include_version)
+                  {
+                    # Filter to the documents matching the id.
+                    query: {ids: {values: [op.doc_id]}},
+                    # We only care about the version.
+                    _source: {includes: ["__versions.#{op.update_target.relationship}"]}
+                  }
                 ]
               end
 
@@ -244,7 +232,7 @@ module ElasticGraph
 
           if errors.empty?
             # We assume the size of the ops and the other array is the same and it cannot have `nil`.
-            zip = ops.zip(msearch_response.fetch("responses")) # : ::Array[[_Operation, ::Hash[::String, ::Hash[::String, untyped]]]]
+            zip = ops.zip(msearch_response.fetch("responses")) # : ::Array[[Operation::Update, ::Hash[::String, ::Hash[::String, untyped]]]]
 
             versions_by_op = zip.to_h do |(op, response)|
               hits = response.fetch("hits").fetch("hits")
@@ -260,25 +248,15 @@ module ElasticGraph
                 })
               end
 
-              if op.destination_index_def.use_updates_for_indexing?
-                # @type var op: Operation::Update
-                versions = hits.filter_map do |hit|
-                  hit.dig("_source", "__versions", op.update_target.relationship, hit.fetch("_id")) ||
-                    # The update_data script before ElasticGraph v0.8 used __sourceVersions[type] instead of __versions[relationship].
-                    # To be backwards-compatible we need to fetch the data at both paths.
-                    #
-                    # TODO: Drop this when we no longer need to maintain backwards-compatibility.
-                    hit.dig("_source", "__sourceVersions", op.event.fetch("type"), hit.fetch("_id"))
-                end
-
-                [op, versions.uniq]
-              else
-                [op, hits.map { |h| h.fetch("_version") }.uniq]
+              versions = hits.filter_map do |hit|
+                hit.dig("_source", "__versions", op.update_target.relationship, hit.fetch("_id"))
               end
+
+              [op, versions.uniq]
             end
 
             unversioned_ops_hash = unversioned_ops.to_h do |op|
-              [op, []] # : [_Operation, ::Array[::Integer]]
+              [op, []] # : [Operation::Update, ::Array[::Integer]]
             end
 
             [client_name, :success, versions_by_op.merge(unversioned_ops_hash)]

data/lib/elastic_graph/indexer/operation/factory.rb

@@ -10,7 +10,6 @@ require "elastic_graph/constants"
 require "elastic_graph/indexer/event_id"
 require "elastic_graph/indexer/failed_event_error"
 require "elastic_graph/indexer/operation/update"
-require "elastic_graph/indexer/operation/upsert"
 require "elastic_graph/indexer/record_preparer"
 require "elastic_graph/json_schema/validator_factory"
 require "elastic_graph/support/memoizable_data"
@@ -140,23 +139,6 @@ module ElasticGraph
         end
 
         def build_all_operations_for(event, record_preparer)
-          upsert_operations(event, record_preparer) + update_operations(event, record_preparer)
-        end
-
-        def upsert_operations(event, record_preparer)
-          type = event.fetch("type") do
-            # This key should only be missing on invalid events. We still want to build operations
-            # for the event (to put it in the `FailedEventError`) but in this case we can't build
-            # any because we don't know what indices to target.
-            return []
-          end
-
-          index_definitions_for(type).reject(&:use_updates_for_indexing?).map do |index_definition|
-            Upsert.new(event, index_definition, record_preparer)
-          end
-        end
-
-        def update_operations(event, record_preparer)
           # If `type` is missing or is not a known type (as indicated by `runtime_metadata` being nil)
           # then we can't build a derived indexing type update operation. That case will only happen when we build
           # operations for an `FailedEventError` rather than to execute.

data/lib/elastic_graph/indexer/operation/update.rb

@@ -27,8 +27,6 @@ module ElasticGraph
           update_target:,
           destination_index_mapping:
         )
-          return [] if update_target.for_normal_indexing? && !destination_index_def.use_updates_for_indexing?
-
           prepared_record = record_preparer.prepare_for_index(event["type"], event["record"] || {"id" => event["id"]})
 
           Support::HashUtil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elasticgraph-indexer
 version: !ruby/object:Gem::Version
-  version: 0.19.2.1
+  version: 1.0.0.rc1
 platform: ruby
 authors:
 - Myron Marston
@@ -9,7 +9,7 @@ authors:
 - Block Engineering
 bindir: bin
 cert_chain: []
-date: 2025-04-24 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: elasticgraph-datastore_core
@@ -17,56 +17,56 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.19.2.1
+        version: 1.0.0.rc1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.19.2.1
+        version: 1.0.0.rc1
 - !ruby/object:Gem::Dependency
   name: elasticgraph-json_schema
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.19.2.1
+        version: 1.0.0.rc1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.19.2.1
+        version: 1.0.0.rc1
 - !ruby/object:Gem::Dependency
   name: elasticgraph-schema_artifacts
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.19.2.1
+        version: 1.0.0.rc1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.19.2.1
+        version: 1.0.0.rc1
 - !ruby/object:Gem::Dependency
   name: elasticgraph-support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.19.2.1
+        version: 1.0.0.rc1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.19.2.1
+        version: 1.0.0.rc1
 - !ruby/object:Gem::Dependency
   name: hashdiff
   requirement: !ruby/object:Gem::Requirement
@@ -93,56 +93,56 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.19.2.1
+        version: 1.0.0.rc1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.19.2.1
+        version: 1.0.0.rc1
 - !ruby/object:Gem::Dependency
   name: elasticgraph-elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.19.2.1
+        version: 1.0.0.rc1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.19.2.1
+        version: 1.0.0.rc1
 - !ruby/object:Gem::Dependency
   name: elasticgraph-opensearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.19.2.1
+        version: 1.0.0.rc1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.19.2.1
+        version: 1.0.0.rc1
 - !ruby/object:Gem::Dependency
   name: elasticgraph-schema_definition
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.19.2.1
+        version: 1.0.0.rc1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.19.2.1
+        version: 1.0.0.rc1
 email:
 - myron@squareup.com
 executables: []
@@ -165,7 +165,6 @@ files:
 - lib/elastic_graph/indexer/operation/factory.rb
 - lib/elastic_graph/indexer/operation/result.rb
 - lib/elastic_graph/indexer/operation/update.rb
-- lib/elastic_graph/indexer/operation/upsert.rb
 - lib/elastic_graph/indexer/processor.rb
 - lib/elastic_graph/indexer/record_preparer.rb
 - lib/elastic_graph/indexer/spec_support/event_matcher.rb
@@ -175,10 +174,10 @@ licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/block/elasticgraph/issues
-  changelog_uri: https://github.com/block/elasticgraph/releases/tag/v0.19.2.1
-  documentation_uri: https://block.github.io/elasticgraph/api-docs/v0.19.2.1/
+  changelog_uri: https://github.com/block/elasticgraph/releases/tag/v1.0.0.rc1
+  documentation_uri: https://block.github.io/elasticgraph/api-docs/v1.0.0.rc1/
   homepage_uri: https://block.github.io/elasticgraph/
-  source_code_uri: https://github.com/block/elasticgraph/tree/v0.19.2.1/elasticgraph-indexer
+  source_code_uri: https://github.com/block/elasticgraph/tree/v1.0.0.rc1/elasticgraph-indexer
   gem_category: core
 rdoc_options: []
 require_paths:
@@ -187,7 +186,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.2'
+      version: '3.4'
   - - "<"
     - !ruby/object:Gem::Version
       version: '3.5'
@@ -197,7 +196,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 3.6.7
 specification_version: 4
 summary: ElasticGraph gem that provides APIs to robustly index data into a datastore.
 test_files: []

data/lib/elastic_graph/indexer/operation/upsert.rb

@@ -1,71 +0,0 @@
-# Copyright 2024 - 2025 Block, Inc.
-#
-# Use of this source code is governed by an MIT-style
-# license that can be found in the LICENSE file or at
-# https://opensource.org/licenses/MIT.
-#
-# frozen_string_literal: true
-
-require "elastic_graph/indexer/operation/result"
-require "elastic_graph/support/hash_util"
-require "elastic_graph/support/memoizable_data"
-
-module ElasticGraph
-  class Indexer
-    module Operation
-      Upsert = Support::MemoizableData.define(:event, :destination_index_def, :record_preparer) do
-        # @implements Upsert
-
-        def to_datastore_bulk
-          @to_datastore_bulk ||= [{index: metadata}, prepared_record]
-        end
-
-        def categorize(response)
-          index = response.fetch("index")
-          status = index.fetch("status")
-
-          case status
-          when 200..299
-            Result.success_of(self)
-          when 409
-            Result.noop_of(self, index.fetch("error").fetch("reason"))
-          else
-            Result.failure_of(self, index.fetch("error").fetch("reason"))
-          end
-        end
-
-        def doc_id
-          @doc_id ||= event.fetch("id")
-        end
-
-        def type
-          :upsert
-        end
-
-        def description
-          "#{event.fetch("type")} upsert"
-        end
-
-        def versioned?
-          true
-        end
-
-        private
-
-        def metadata
-          @metadata ||= {
-            _index: destination_index_def.index_name_for_writes(prepared_record),
-            _id: doc_id,
-            version: event.fetch("version"),
-            version_type: "external",
-            routing: destination_index_def.routing_value_for_prepared_record(prepared_record)
-          }.compact
-        end
-
-        def prepared_record
-          @prepared_record ||= record_preparer.prepare_for_index(event.fetch("type"), event.fetch("record"))
-        end
-      end
-    end
-  end
-end