elasticgraph-indexer 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7a8d580572ec508d9df0131448f19398a5850aaee3cd1e36b4530be67bee190f
-  data.tar.gz: 7db60416bac8d71edad120569e8abfe55e2884b6844d140109e485b8c9d66d10
+  metadata.gz: '0308b6d8490bd282e073ec4ca59840aa95fb5fa0617aeeb0f6abf8c43d767a0b'
+  data.tar.gz: f8fa720ceb8cfed471671a155069dbddd9857ea91cd6a23af787ead0e7e221fc
 SHA512:
-  metadata.gz: 14b344d1591976df42dd53257a03911e158f9913974c7c2e606c8ea1222989a6552154f615606d0447e97aebef4097f6710600a0af49f453e821c519c89ee146
-  data.tar.gz: 45d6e1a8c27040378857d7b0c7a7411b669c9f9c6e4ba31ee7e3307cfd71f48780c842e1f505d4d3b19e1c63fdfdcdee2db2dfb30b9e5feb40a122796a991dbe
+  metadata.gz: 70d14ee0336b0d99c2c470b6acddd04e6d7c34d7ab664bea6bf9b87a1a140fa81826feca1d69f437105f833bed0a1ea069fe08b5841b524727cdfbc348851542
+  data.tar.gz: eec5e893caf16c8bcb888f2bba16122f45b52a006b3350032a33598d42f8384b5cf683a79d0e88d7481b2efb27cbf341b4c12f17fc4843e392ef874803aeef81

data/lib/elastic_graph/indexer/datastore_indexing_router.rb

@@ -10,6 +10,7 @@ require "elastic_graph/constants"
 require "elastic_graph/errors"
 require "elastic_graph/indexer/event_id"
 require "elastic_graph/indexer/indexing_failures_error"
+require "elastic_graph/support/opaque_id"
 require "elastic_graph/support/threading"
 
 module ElasticGraph
@@ -188,7 +189,14 @@ module ElasticGraph
                 ]
               end
 
-              client.msearch(body: body)
+              headers = {
+                OPAQUE_ID_HEADER => Support::OpaqueID.build_header(opaque_id_parts_for_source_event_versions(ops))
+              }.compact # : ::Hash[::String, ::String]
+
+              client.msearch(
+                body: body,
+                headers: headers
+              )
             else
               # The named client doesn't exist, so we don't have any versions for the docs.
               {"responses" => ops.map { |op| {"hits" => {"hits" => _ = []}} }}
@@ -256,6 +264,22 @@ module ElasticGraph
             "for document versions:\n\n#{failures.join("\n")}"
         end
       end
+
+      private
+
+      def opaque_id_parts_for_source_event_versions(operations)
+        type_counts = operations
+          .group_by { |op| op.event.fetch("type") }
+          .sort_by(&:first)
+          .map { |type_name, ops| "#{type_name}:#{ops.size}" }
+
+        [
+          "elasticgraph-indexer",
+          "purpose=source_event_versions",
+          "operation_count=#{operations.size}",
+          "type_counts=#{type_counts.join(",")}"
+        ]
+      end
     end
   end
 end

data/lib/elastic_graph/indexer/operation/update.rb

@@ -27,7 +27,11 @@ module ElasticGraph
           update_target:,
           destination_index_mapping:
         )
-          prepared_record = record_preparer.prepare_for_index(event["type"], event["record"] || {"id" => event["id"]})
+          prepared_record = record_preparer.prepare_for_index(
+            event["type"],
+            event["record"] || {"id" => event["id"]},
+            destination_index_mapping.fetch("properties")
+          )
 
           Support::HashUtil
             .fetch_leaf_values_at_path(prepared_record, update_target.id_source.split("."))

data/lib/elastic_graph/indexer/record_preparer.rb

@@ -50,10 +50,6 @@ module ElasticGraph
               {} # : ::Hash[::String, untyped]
             end # : ::Hash[::String, untyped]
 
-            required_fields = type_def.fetch("required") do
-              [] # : ::Array[::String]
-            end # : ::Array[::String]
-
             eg_meta_by_field_name = properties.filter_map do |prop_name, prop|
               eg_meta = prop["ElasticGraph"]
               [prop_name, eg_meta] if eg_meta
@@ -61,7 +57,6 @@ module ElasticGraph
 
             TypeMetadata.new(
               name: type,
-              requires_typename: required_fields.include?("__typename"),
               eg_meta_by_field_name: eg_meta_by_field_name
             )
           end
@@ -76,8 +71,8 @@ module ElasticGraph
       # still build operations for it, and the operations require a `RecordPreparer`, but we do
       # not send them to the datastore.
       module Identity
-        def self.prepare_for_index(type_name, record)
-          record
+        def self.prepare_for_index(type_name, value, mapping_properties)
+          value
         end
       end
 
@@ -86,10 +81,6 @@ module ElasticGraph
         @eg_meta_by_field_name_by_concrete_type = type_metas.to_h do |meta|
           [meta.name, meta.eg_meta_by_field_name]
         end
-
-        @types_requiring_typename = type_metas.filter_map do |meta|
-          meta.name if meta.requires_typename
-        end.to_set
       end
 
       # Prepares the given payload for being indexed into the named index.
@@ -104,27 +95,21 @@ module ElasticGraph
       # the set union of fields and this will take care of dropping any unsupported
       # fields before we attempt to index the record.
       #
-      # Note: this method does not mutate the given `record`. Instead it returns a
+      # Note: this method does not mutate the given `value`. Instead it returns a
       # copy with any updates applied to it.
-      def prepare_for_index(type_name, record)
-        prepare_value_for_indexing(record, type_name)
-      end
-
-      private
-
-      def prepare_value_for_indexing(value, type_name)
+      def prepare_for_index(type_name, value, mapping_properties)
         type_name = type_name.delete_suffix("!")
 
-        return nil if value.nil?
+        return (_ = nil) if value.nil? # Steep 2.0 narrows to nil here but can't see it satisfies T
 
         if (preparer = @indexing_preparer_by_scalar_type_name[type_name])
           return (_ = preparer).prepare_for_indexing(value)
         end
 
-        case value
+        _ = case value # Steep 2.0 can't narrow generic T through case/when branches
         when ::Array
           element_type_name = type_name.delete_prefix("[").delete_suffix("]")
-          value.map { |v| prepare_value_for_indexing(v, element_type_name) }
+          value.map { |v| prepare_for_index(element_type_name, v, mapping_properties) }
         when ::Hash
           # `@eg_meta_by_field_name_by_concrete_type` does not have abstract types in it (e.g. type unions).
           # Instead, it'll have each concrete subtype in it.
@@ -133,15 +118,30 @@ module ElasticGraph
           # what the concrete subtype is. `__typename` is required on abstract types and indicates that.
           eg_meta_by_field_name = @eg_meta_by_field_name_by_concrete_type.fetch(value["__typename"] || type_name)
 
-          value.filter_map do |field_name, field_value|
+          # We only want to consider __typename if it's in the per-record mapping in order to determine
+          # whether __typename is required on records. When it's a constant_keyword it exists at the index
+          # level and therefore should be ignored for this purpose.
+          typename_type = mapping_properties&.dig("__typename", "type")
+          typename_in_record_mapping = typename_type && typename_type != "constant_keyword"
+
+          prepared_fields = value.filter_map do |field_name, field_value|
             if field_name == "__typename"
-              # We only want to include __typename if it we're dealing with a type that requires it.
-              # (This is the case for an abstract type, so it can differentiate between which subtype we have
-              [field_name, field_value] if @types_requiring_typename.include?(type_name)
+              # Only include __typename if the index mapping has it at this position.
+              [field_name, field_value] if typename_in_record_mapping
             elsif (eg_meta = eg_meta_by_field_name[field_name])
-              [eg_meta.fetch("nameInIndex"), prepare_value_for_indexing(field_value, eg_meta.fetch("type"))]
+              name_in_index = eg_meta.fetch("nameInIndex")
+              nested_mapping_properties = mapping_properties&.dig(name_in_index, "properties")
+              [name_in_index, prepare_for_index(eg_meta.fetch("type"), field_value, nested_mapping_properties)]
             end
           end.to_h
+
+          # Inject __typename if the mapping requires it but it's absent from the record
+          # (e.g. for a concrete type indexed in a mixed-type index).
+          if typename_in_record_mapping && !value.key?("__typename")
+            prepared_fields["__typename"] = type_name
+          end
+
+          prepared_fields
         else
           # We won't have a registered preparer for enum types, since those aren't dumped in
           # runtime metadata `scalar_types_by_name`, and we can just return the value as-is in
@@ -153,8 +153,6 @@ module ElasticGraph
       TypeMetadata = ::Data.define(
         # The name of the type this metadata object is for.
         :name,
-        # Indicates if this type requires a `__typename` field.
-        :requires_typename,
         # The per-field ElasticGraph metadata, keyed by field name.
         :eg_meta_by_field_name
       )

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elasticgraph-indexer
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Myron Marston
@@ -17,42 +17,42 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 1.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: elasticgraph-schema_artifacts
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 1.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: elasticgraph-support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 1.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: hashdiff
   requirement: !ruby/object:Gem::Requirement
@@ -79,56 +79,56 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 1.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: elasticgraph-elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 1.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: elasticgraph-opensearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 1.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: elasticgraph-schema_definition
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 1.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 1.2.0
 email:
 - myron@squareup.com
 executables: []
@@ -161,10 +161,10 @@ licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/block/elasticgraph/issues
-  changelog_uri: https://github.com/block/elasticgraph/releases/tag/v1.1.0
-  documentation_uri: https://block.github.io/elasticgraph/api-docs/v1.1.0/
+  changelog_uri: https://github.com/block/elasticgraph/releases/tag/v1.2.0
+  documentation_uri: https://block.github.io/elasticgraph/api-docs/v1.2.0/
   homepage_uri: https://block.github.io/elasticgraph/
-  source_code_uri: https://github.com/block/elasticgraph/tree/v1.1.0/elasticgraph-indexer
+  source_code_uri: https://github.com/block/elasticgraph/tree/v1.2.0/elasticgraph-indexer
   gem_category: core
 rdoc_options: []
 require_paths:
@@ -183,7 +183,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.6
 specification_version: 4
 summary: Indexes ElasticGraph data into a datastore.
 test_files: []