elasticgraph-indexer 0.18.0.0

data/lib/elastic_graph/indexer/indexing_failures_error.rb

@@ -0,0 +1,28 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/error"
+
+module ElasticGraph
+  class Indexer
+    class IndexingFailuresError < Error
+      # Builds an `IndexingFailuresError` with a nicely formatted message for the given array of `FailedEventError`.
+      def self.for(failures:, events:)
+        summary = "Got #{failures.size} failure(s) from #{events.size} event(s):"
+        failure_details = failures.map.with_index { |failure, index| "#{index + 1}) #{failure.message}" }
+
+        message_ids = failures.filter_map { |f| f.event["message_id"] }.uniq
+        if message_ids.any?
+          message_details = "These failures came from #{message_ids.size} message(s): #{message_ids.join(", ")}."
+        end
+
+        new([summary, failure_details, message_details].compact.join("\n\n"))
+      end
+    end
+  end
+end

data/lib/elastic_graph/indexer/indexing_preparers/integer.rb

@@ -0,0 +1,41 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+module ElasticGraph
+  class Indexer
+    module IndexingPreparers
+      class Integer
+        # Here we coerce an integer-valued float like `3.0` to a true integer (e.g. `3`).
+        # This is necessary because:
+        #
+        #   1. If a field is an integer in the datastore mapping, it does not tolerate it coming in
+        #      as a float, even if it is integer-valued.
+        #   2. While we use JSON schema to validate event payloads before we get here, JSON schema
+        #      cannot consistently enforce that we receive true integers for int fields.
+        #
+        # As https://json-schema.org/understanding-json-schema/reference/numeric.html#integer explains:
+        #
+        # > **Warning**
+        # >
+        # > The precise treatment of the “integer” type may depend on the implementation of your
+        # > JSON Schema validator. JavaScript (and thus also JSON) does not have distinct types
+        # > for integers and floating-point values. Therefore, JSON Schema can not use type alone
+        # > to distinguish between integers and non-integers. The JSON Schema specification
+        # > recommends, but does not require, that validators use the mathematical value to
+        # > determine whether a number is an integer, and not the type alone. Therefore, there
+        # > is some disagreement between validators on this point. For example, a JavaScript-based
+        # > validator may accept 1.0 as an integer, whereas the Python-based jsonschema does not.
+        def self.prepare_for_indexing(value)
+          integer = value.to_i
+          return integer if value == integer
+          raise IndexOperationError, "Cannot safely coerce `#{value.inspect}` to an integer"
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/indexer/indexing_preparers/no_op.rb

@@ -0,0 +1,19 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+module ElasticGraph
+  class Indexer
+    module IndexingPreparers
+      class NoOp
+        def self.prepare_for_indexing(value)
+          value
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/indexer/indexing_preparers/untyped.rb

@@ -0,0 +1,22 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/support/untyped_encoder"
+
+module ElasticGraph
+  class Indexer
+    module IndexingPreparers
+      class Untyped
+        # Converts the given untyped value to a String so it can be indexed in a `keyword` field.
+        def self.prepare_for_indexing(value)
+          Support::UntypedEncoder.encode(value)
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/indexer/operation/count_accumulator.rb

@@ -0,0 +1,166 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/constants"
+
+module ElasticGraph
+  class Indexer
+    module Operation
+      # Responsible for maintaining state and accumulating list counts while we traverse the `data` we are preparing
+      # to update in the index. Much of the complexity here is due to the fact that we have 3 kinds of list fields:
+      # scalar lists, embedded object lists, and `nested` object lists.
+      #
+      # The Elasticsearch/OpenSearch `nested` type[^1] indexes objects of this type as separate hidden documents. As a result,
+      # each `nested` object type gets its own `__counts` field. In contrast, embedded object lists get flattened into separate
+      # entries (one per field path) in a flat map (with `dot_separated_path: values_at_path` entries) at the document root.
+      #
+      # We mirror this structure with our `__counts`: each document (either a root document, or a hidden `nested` document)
+      # gets its own `__counts` field, so we essentially have multiple "count parents". Each `__counts` field is a map,
+      # keyed by field paths, and containing the number of list elements at that field path after the flattening has
+      # occurred.
+      #
+      # The index mapping defines where the `__counts` fields go. This abstraction uses the mapping to determine when
+      # it needs to create a new "count parent".
+      #
+      # Note: instances of this class are "shallow immutable" (none of the attributes of an instance can be reassigned)
+      # but the `counts` attribute is itself a mutable hash--we use it to accumulate the list counts as we traverse the
+      # structure.
+      #
+      # [^1]: https://www.elastic.co/guide/en/elasticsearch/reference/8.9/nested.html
+      CountAccumulator = ::Data.define(
+        # Hash containing the counts we have accumulated so far. This hash gets mutated as we accumulate,
+        # and multiple accumulator instances share the same hash instance. However, a new `counts` hash will
+        # be created when we reach a new parent.
+        :counts,
+        # String describing our current location in the traversed structure relative to the current parent.
+        # This gets replaced on new accumulator instances as we traverse the data structure.
+        :path_from_parent,
+        # String describing our current location in the traversed structure relative to the overall document root.
+        # This gets replaced on new accumulator instances as we traverse the data structure.
+        :path_from_root,
+        # The index mapping at the current level of the structure when this accumulator instance was created.
+        # As we traverse new levels of the data structure, new `CountAccumulator` instances will be created with
+        # the `mapping` updated to reflect the new level of the structure we are at.
+        :mapping,
+        # Set of field paths to subfields of `LIST_COUNTS_FIELD` for the current source relationship.
+        # This will be used to determine which subfields of the `LIST_COUNTS_FIELD` are populated.
+        :list_counts_field_paths_for_source,
+        # Indicates if our current path is underneath a list; if so, `maybe_increment` will increment when called.
+        :has_list_ancestor
+      ) do
+        # @implements CountAccumulator
+        def self.merge_list_counts_into(params, mapping:, list_counts_field_paths_for_source:)
+          # Here we compute the counts of our list elements so that we can index it.
+          data = compute_list_counts_of(params.fetch("data"), CountAccumulator.new_parent(
+            # We merge in `type: nested` since the `nested` type indicates a new count accumulator parent and we want that applied at the root.
+            mapping.merge("type" => "nested"),
+            list_counts_field_paths_for_source
+          ))
+
+          # The root `__counts` field needs special handling due to our `sourced_from` feature. Anything in `data`
+          # will overwrite what's in the specified fields when the script executes, but since there could be list
+          # fields from multiple sources, we need `__counts` to get merged properly. So here we "promote" it from
+          # `data.__counts` to being a root-level parameter.
+          params.merge(
+            "data" => data.except(LIST_COUNTS_FIELD),
+            LIST_COUNTS_FIELD => data[LIST_COUNTS_FIELD]
+          )
+        end
+
+        def self.compute_list_counts_of(value, parent_accumulator)
+          case value
+          when nil
+            value
+          when ::Hash
+            parent_accumulator.maybe_increment
+            parent_accumulator.process_hash(value) do |key, subvalue, accumulator|
+              [key, compute_list_counts_of(subvalue, accumulator[key])]
+            end
+          when ::Array
+            parent_accumulator.process_list(value) do |element, accumulator|
+              compute_list_counts_of(element, accumulator)
+            end
+          else
+            parent_accumulator.maybe_increment
+            value
+          end
+        end
+
+        # Creates an initially empty accumulator instance for a new parent (either at the overall document
+        # root are at the root of a `nested` object).
+        def self.new_parent(mapping, list_counts_field_paths_for_source, path_from_root: nil)
+          count_field_prefix = path_from_root ? "#{path_from_root}.#{LIST_COUNTS_FIELD}." : "#{LIST_COUNTS_FIELD}."
+
+          initial_counts = (mapping.dig("properties", LIST_COUNTS_FIELD, "properties") || {}).filter_map do |field, _|
+            [field, 0] if list_counts_field_paths_for_source.include?(count_field_prefix + field)
+          end.to_h
+
+          new(initial_counts, nil, path_from_root, mapping, list_counts_field_paths_for_source, false)
+        end
+
+        # Processes the given hash, beginning a new parent if need. A new parent is needed if the
+        # current mapping has a `__counts` field.
+        #
+        # Yields repeatedly (once per hash entry). We yield the entry key/value, and an accumulator
+        # instance (either the current `self` or a new parent).
+        #
+        # Afterwards, merges the resulting `__counts` into the hash before it's returned, as needed.
+        def process_hash(hash)
+          mapping_type = mapping["type"]
+
+          # As we traverse through the JSON object structure, we also have to traverse through the
+          # condenseed mapping. Doing this requires that the `properties` of the index mapping
+          # match the fields of the JSON data structure. However, Elasticsearch/OpenSearch have a number of field
+          # types which can be represented as a JSON object in an indexing call, but which have no
+          # `properties` in the mapping. We can't successfully traverse through the JSON data and the
+          # mapping when we encounter these field types (since the mapping has no record of the
+          # subfields) so we must treat these types as a special case; we can't proceed, and we won't
+          # have any lists to count, anyway.
+          return hash if DATASTORE_PROPERTYLESS_OBJECT_TYPES.include?(mapping_type)
+
+          # THe `nested` type indicates a new document level, so if it's not `nested`, we should process the hash without making a new parent.
+          return hash.to_h { |key, value| yield key, value, self } unless mapping_type == "nested"
+
+          # ...but otherwise, we should make a new parent.
+          new_parent = CountAccumulator.new_parent(mapping, list_counts_field_paths_for_source, path_from_root: path_from_root)
+          updated_hash = hash.to_h { |key, value| yield key, value, new_parent }
+
+          # If we have a LIST_COUNTS_FIELD at this level of our mapping, we should merge in the counts hash from the new parent.
+          if mapping.dig("properties", LIST_COUNTS_FIELD)
+            updated_hash.merge(LIST_COUNTS_FIELD => new_parent.counts)
+          else
+            updated_hash
+          end
+        end
+
+        # Processes the given list, tracking the fact that subpaths have a list ancestor.
+        def process_list(list)
+          child_accumulator = with(has_list_ancestor: true)
+          list.map { |value| yield value, child_accumulator }
+        end
+
+        # Increments the count at the current `path_from_parent` in the current parent's counts hash if we are under a list.
+        def maybe_increment
+          return unless has_list_ancestor
+
+          key = path_from_parent.to_s
+          counts[key] = counts.fetch(key) + 1
+        end
+
+        # Creates a "child" accumulator at the given subpath. Should be used as we traverse the data structure.
+        def [](subpath)
+          with(
+            path_from_parent: path_from_parent ? "#{path_from_parent}#{LIST_COUNTS_FIELD_PATH_KEY_SEPARATOR}#{subpath}" : subpath,
+            path_from_root: path_from_root ? "#{path_from_root}.#{subpath}" : subpath,
+            mapping: mapping.fetch("properties").fetch(subpath)
+          )
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/indexer/operation/factory.rb

@@ -0,0 +1,226 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/constants"
+require "elastic_graph/indexer/event_id"
+require "elastic_graph/indexer/failed_event_error"
+require "elastic_graph/indexer/operation/update"
+require "elastic_graph/indexer/operation/upsert"
+require "elastic_graph/indexer/record_preparer"
+require "elastic_graph/json_schema/validator_factory"
+require "elastic_graph/support/memoizable_data"
+
+module ElasticGraph
+  class Indexer
+    module Operation
+      class Factory < Support::MemoizableData.define(
+        :schema_artifacts,
+        :index_definitions_by_graphql_type,
+        :record_preparer_factory,
+        :logger,
+        :skip_derived_indexing_type_updates,
+        :configure_record_validator
+      )
+        def build(event)
+          event = prepare_event(event)
+
+          selected_json_schema_version = select_json_schema_version(event) { |failure| return failure }
+
+          # Because the `select_json_schema_version` picks the closest-matching json schema version, the incoming
+          # event might not match the expected json_schema_version value in the json schema (which is a `const` field).
+          # This is by design, since we're picking a schema based on best-effort, so to avoid that by-design validation error,
+          # performing the envelope validation on a "patched" version of the event.
+          event_with_patched_envelope = event.merge({JSON_SCHEMA_VERSION_KEY => selected_json_schema_version})
+
+          if (error_message = validator(EVENT_ENVELOPE_JSON_SCHEMA_NAME, selected_json_schema_version).validate_with_error_message(event_with_patched_envelope))
+            return build_failed_result(event, "event payload", error_message)
+          end
+
+          failed_result = validate_record_returning_failure(event, selected_json_schema_version)
+          failed_result || BuildResult.success(build_all_operations_for(
+            event,
+            record_preparer_factory.for_json_schema_version(selected_json_schema_version)
+          ))
+        end
+
+        private
+
+        def select_json_schema_version(event)
+          available_json_schema_versions = schema_artifacts.available_json_schema_versions
+
+          requested_json_schema_version = event[JSON_SCHEMA_VERSION_KEY]
+
+          # First check that a valid value has been requested (a positive integer)
+          if !event.key?(JSON_SCHEMA_VERSION_KEY)
+            yield build_failed_result(event, JSON_SCHEMA_VERSION_KEY, "Event lacks a `#{JSON_SCHEMA_VERSION_KEY}`")
+          elsif !requested_json_schema_version.is_a?(Integer) || requested_json_schema_version < 1
+            yield build_failed_result(event, JSON_SCHEMA_VERSION_KEY, "#{JSON_SCHEMA_VERSION_KEY} (#{requested_json_schema_version}) must be a positive integer.")
+          end
+
+          # The requested version might not necessarily be available (if the publisher is deployed ahead of the indexer, or an old schema
+          # version is removed prematurely, or an indexer deployment is rolled back). So the behavior is to always pick the closest-available
+          # version. If there's an exact match, great. Even if not an exact match, if the incoming event payload conforms to the closest match,
+          # the event can still be indexed.
+          #
+          # This min_by block will take the closest version in the list. If a tie occurs, the first value in the list wins. The desired
+          # behavior is in the event of a tie (highly unlikely, there shouldn't be a gap in available json schema versions), the higher version
+          # should be selected. So to get that behavior, the list is sorted in descending order.
+          #
+          selected_json_schema_version = available_json_schema_versions.sort.reverse.min_by { |it| (requested_json_schema_version - it).abs }
+
+          if selected_json_schema_version != requested_json_schema_version
+            logger.info({
+              "message_type" => "ElasticGraphMissingJSONSchemaVersion",
+              "message_id" => event["message_id"],
+              "event_id" => EventID.from_event(event),
+              "event_type" => event["type"],
+              "requested_json_schema_version" => requested_json_schema_version,
+              "selected_json_schema_version" => selected_json_schema_version
+            })
+          end
+
+          if selected_json_schema_version.nil?
+            yield build_failed_result(
+              event, JSON_SCHEMA_VERSION_KEY,
+              "Failed to select json schema version. Requested version: #{event[JSON_SCHEMA_VERSION_KEY]}. \
+              Available json schema versions: #{available_json_schema_versions.sort.join(", ")}"
+            )
+          end
+
+          selected_json_schema_version
+        end
+
+        def validator(type, selected_json_schema_version)
+          factory = validator_factories_by_version[selected_json_schema_version]
+          factory.validator_for(type)
+        end
+
+        def validator_factories_by_version
+          @validator_factories_by_version ||= ::Hash.new do |hash, json_schema_version|
+            factory = JSONSchema::ValidatorFactory.new(
+              schema: schema_artifacts.json_schemas_for(json_schema_version),
+              sanitize_pii: true
+            )
+            factory = configure_record_validator.call(factory) if configure_record_validator
+            hash[json_schema_version] = factory
+          end
+        end
+
+        # This copies the `id` from event into the actual record
+        # This is necessary because we want to index `id` as part of the record so that the datastore will include `id` in returned search payloads.
+        def prepare_event(event)
+          return event unless event["record"].is_a?(::Hash) && event["id"]
+          event.merge("record" => event["record"].merge("id" => event.fetch("id")))
+        end
+
+        def validate_record_returning_failure(event, selected_json_schema_version)
+          record = event.fetch("record")
+          graphql_type_name = event.fetch("type")
+          validator = validator(graphql_type_name, selected_json_schema_version)
+
+          if (error_message = validator.validate_with_error_message(record))
+            build_failed_result(event, "#{graphql_type_name} record", error_message)
+          end
+        end
+
+        def build_failed_result(event, payload_description, validation_message)
+          message = "Malformed #{payload_description}. #{validation_message}"
+
+          # Here we use the `RecordPreparer::Identity` record preparer because we may not have a valid JSON schema
+          # version number in this case (which is usually required to get a `RecordPreparer` from the factory), and
+          # we won't wind up using the record preparer for real on these operations, anyway.
+          operations = build_all_operations_for(event, RecordPreparer::Identity)
+
+          BuildResult.failure(FailedEventError.new(event: event, operations: operations.to_set, main_message: message))
+        end
+
+        def build_all_operations_for(event, record_preparer)
+          upsert_operations(event, record_preparer) + update_operations(event, record_preparer)
+        end
+
+        def upsert_operations(event, record_preparer)
+          type = event.fetch("type") do
+            # This key should only be missing on invalid events. We still want to build operations
+            # for the event (to put it in the `FailedEventError`) but in this case we can't build
+            # any because we don't know what indices to target.
+            return []
+          end
+
+          index_definitions_for(type).reject(&:use_updates_for_indexing?).map do |index_definition|
+            Upsert.new(event, index_definition, record_preparer)
+          end
+        end
+
+        def update_operations(event, record_preparer)
+          # If `type` is missing or is not a known type (as indicated by `runtime_metadata` being nil)
+          # then we can't build a derived indexing type update operation. That case will only happen when we build
+          # operations for an `FailedEventError` rather than to execute.
+          return [] unless (type = event["type"])
+          return [] unless (runtime_metadata = schema_artifacts.runtime_metadata.object_types_by_name[type])
+
+          runtime_metadata.update_targets.flat_map do |update_target|
+            ids_to_skip = skip_derived_indexing_type_updates.fetch(update_target.type, ::Set.new)
+
+            index_definitions_for(update_target.type).flat_map do |destination_index_def|
+              operations = Update.operations_for(
+                event: event,
+                destination_index_def: destination_index_def,
+                record_preparer: record_preparer,
+                update_target: update_target,
+                destination_index_mapping: schema_artifacts.index_mappings_by_index_def_name.fetch(destination_index_def.name)
+              )
+
+              operations.reject do |op|
+                ids_to_skip.include?(op.doc_id).tap do |skipped|
+                  if skipped
+                    logger.info({
+                      "message_type" => "SkippingUpdate",
+                      "message_id" => event["message_id"],
+                      "update_target" => update_target.type,
+                      "id" => op.doc_id,
+                      "event_id" => EventID.from_event(event).to_s
+                    })
+                  end
+                end
+              end
+            end
+          end
+        end
+
+        def index_definitions_for(type)
+          # If `type` is missing or is not a known type (as indicated by not being in this hash)
+          # then we return an empty list. That case will only happen when we build
+          # operations for an `FailedEventError` rather than to execute.
+          index_definitions_by_graphql_type[type] || []
+        end
+
+        # :nocov: -- this should not be called. Instead, it exists to guard against wrongly raising an error from this class.
+        def raise(*args)
+          super("`raise` was called on `Operation::Factory`, but should not. Instead, use " \
+            "`yield build_failed_result(...)` so that we can accumulate all invalid events and allow " \
+            "the valid events to still be processed.")
+        end
+        # :nocov:
+
+        # Return value from `build` that indicates what happened.
+        # - If it was successful, `operations` will be a non-empty array of operations and `failed_event_error` will be nil.
+        # - If there was a validation issue, `operations` will be an empty array and `failed_event_error` will be non-nil.
+        BuildResult = ::Data.define(:operations, :failed_event_error) do
+          # @implements BuildResult
+          def self.success(operations)
+            new(operations, nil)
+          end
+
+          def self.failure(failed_event_error)
+            new([], failed_event_error)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/indexer/operation/result.rb

@@ -0,0 +1,76 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/indexer/event_id"
+
+module ElasticGraph
+  class Indexer
+    module Operation
+      # Describes the result of an operation.
+      # :category value will be one of: [:success, :noop, :failure]
+      Result = ::Data.define(:category, :operation, :description) do
+        # @implements Result
+        def self.success_of(operation)
+          Result.new(
+            category: :success,
+            operation: operation,
+            description: nil
+          )
+        end
+
+        def self.noop_of(operation, description)
+          Result.new(
+            category: :noop,
+            operation: operation,
+            description: description
+          )
+        end
+
+        def self.failure_of(operation, description)
+          Result.new(
+            category: :failure,
+            operation: operation,
+            description: description
+          )
+        end
+
+        def operation_type
+          operation.type
+        end
+
+        def event
+          operation.event
+        end
+
+        def event_id
+          EventID.from_event(event)
+        end
+
+        def summary
+          # :nocov: -- `description == nil` case is not covered; not simple to test.
+          suffix = description ? "--#{description}" : nil
+          # :nocov:
+          "<#{operation.description} #{event_id} #{category}#{suffix}>"
+        end
+
+        def inspect
+          parts = [
+            self.class.name,
+            operation_type.inspect,
+            category.inspect,
+            event_id,
+            description
+          ].compact
+
+          "#<#{parts.join(" ")}>"
+        end
+        alias_method :to_s, :inspect
+      end
+    end
+  end
+end