elasticgraph-schema_definition 0.18.0.0

data/lib/elastic_graph/schema_definition/indexing/index.rb

@@ -0,0 +1,318 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/schema_artifacts/runtime_metadata/index_definition"
+require "elastic_graph/schema_artifacts/runtime_metadata/index_field"
+require "elastic_graph/schema_definition/indexing/derived_indexed_type"
+require "elastic_graph/schema_definition/indexing/list_counts_mapping"
+require "elastic_graph/schema_definition/indexing/rollover_config"
+require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect"
+require "elastic_graph/schema_definition/schema_elements/field_path"
+require "elastic_graph/support/hash_util"
+
+module ElasticGraph
+  module SchemaDefinition
+    # Contains schema definition logic specific to indexing (such as JSON schema and mapping generation).
+    module Indexing
+      # Represents an index in a datastore. Defined within an indexed type. Modeled as a separate object to facilitate
+      # further customization of the index.
+      #
+      # @!attribute [r] name
+      #   @return [String] name of the index
+      # @!attribute [r] default_sort_pairs
+      #   @return [Array<(String, Symbol)>] (field name, direction) pairs for the default sort
+      # @!attribute [r] settings
+      #   @return [Hash<(String, Object)>] datastore settings for the index
+      # @!attribute [r] schema_def_state
+      #   @return [State] schema definition state
+      # @!attribute [r] indexed_type
+      #   @return [SchemaElements::ObjectType, SchemaElements::InterfaceType, SchemaElements::UnionType] type backed by this index
+      # @!attribute [r] routing_field_path
+      #   @return [Array<String>] path to the field used for shard routing
+      # @!attribute [r] rollover_config
+      #   @return [RolloverConfig, nil] rollover configuration for the index
+      class Index < Struct.new(:name, :default_sort_pairs, :settings, :schema_def_state, :indexed_type, :routing_field_path, :rollover_config)
+        include Mixins::HasReadableToSAndInspect.new { |i| i.name }
+
+        # @param name [String] name of the index
+        # @param settings [Hash<(String, Object)>] datastore settings for the index
+        # @param schema_def_state [State] schema definition state
+        # @param indexed_type [SchemaElements::ObjectType, SchemaElements::InterfaceType, SchemaElements::UnionType] type backed by this index
+        # @yield [Index] the index, for further customization
+        # @api private
+        def initialize(name, settings, schema_def_state, indexed_type)
+          if name.include?(ROLLOVER_INDEX_INFIX_MARKER)
+            raise SchemaError, "`#{name}` is an invalid index definition name since it contains " \
+              "`#{ROLLOVER_INDEX_INFIX_MARKER}` which ElasticGraph treats as special."
+          end
+
+          settings = DEFAULT_SETTINGS.merge(Support::HashUtil.flatten_and_stringify_keys(settings, prefix: "index"))
+
+          super(name, [], settings, schema_def_state, indexed_type, [], nil)
+
+          # `id` is the field Elasticsearch/OpenSearch use for routing by default:
+          # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-routing-field.html
+          # By using it here, it will cause queries to pass a `routing` parameter when
+          # searching with id filtering on an index that does not use custom shard routing, giving
+          # us a nice efficiency boost.
+          self.routing_field_path = public_field_path("id", explanation: "indexed types must have an `id` field")
+
+          yield self if block_given?
+        end
+
+        # Specifies how documents in this index should sort by default, when no `orderBy` argument is provided to the GraphQL query.
+        #
+        # @note the field name strings can be a dot-separated nested fields, but all referenced
+        #   fields must exist when this is called.
+        #
+        # @param field_name_direction_pairs [Array<(String, Symbol)>] pairs of field names and `:asc` or `:desc`
+        # @return [void]
+        #
+        # @example Sort on `name` (ascending) with `createdAt` (descending) as a tie-breaker
+        #   ElasticGraph.define_schema do |schema|
+        #     schema.object_type "Campaign" do |t|
+        #       t.field "id", "ID!"
+        #       t.field "name", "String"
+        #       t.field "createdAt", "DateTime"
+        #
+        #       t.index "campaigns"do |i|
+        #         i.default_sort "name", :asc, "createdAt", :desc
+        #       end
+        #     end
+        #   end
+        def default_sort(*field_name_direction_pairs)
+          self.default_sort_pairs = field_name_direction_pairs
+        end
+
+        # Causes this index to "rollover" at the provided `frequency` based on the value of the provided `timestamp_field_path_name`.
+        # This is particularly useful for time-series data. Partitioning the data into `hourly`, `daily`, `monthly` or `yearly` buckets
+        # allows for different index configurations, and can be necessary when a dataset is too large to fit in one dataset given
+        # Elasticsearch/OpenSearch limitations on the number of shards in one index. In addition, ElasticGraph optimizes queries which
+        # filter on the timestamp field to target the subset of the indices in which matching documents could reside.
+        #
+        # @note the timestamp field specified here **must be immutable**. To understand why, consider a `:yearly` rollover
+        #   index used for data based on `createdAt`; if ElasticGraph ingests record `123` with a createdAt of `2023-12-31T23:59:59Z`, it
+        #   will be indexed in the `2023` index. Later if it receives an update event for record `123` with a `createdAt` of
+        #   `2024-01-01T00:00:00Z` (a mere one second later!), ElasticGraph will store the new version of the payment in the `2024` index,
+        #   and leave the old copy of the payment in the `2023` index unchanged. It’ll have duplicates for that document.
+        # @note changing the `rollover` configuration on an existing index that already has data will result in duplicate documents
+        #
+        # @param frequency [:yearly, :monthly, :daily, :hourly] how often to rollover the index
+        # @param timestamp_field_path_name [String] dot-separated path to the timestamp field used for rollover. Note: all referenced
+        #   fields must exist when this is called.
+        # @return [void]
+        #
+        # @example Define a `campaigns` index to rollover yearly based on `createdAt`
+        #   ElasticGraph.define_schema do |schema|
+        #     schema.object_type "Campaign" do |t|
+        #       t.field "id", "ID!"
+        #       t.field "name", "String"
+        #       t.field "createdAt", "DateTime"
+        #
+        #       t.index "campaigns"do |i|
+        #         i.rollover :yearly, "createdAt"
+        #       end
+        #     end
+        #   end
+        def rollover(frequency, timestamp_field_path_name)
+          timestamp_field_path = public_field_path(timestamp_field_path_name, explanation: "it is referenced as an index `rollover` field")
+
+          unless date_and_datetime_types.include?(timestamp_field_path.type.fully_unwrapped.name)
+            date_or_datetime_description = date_and_datetime_types.map { |t| "`#{t}`" }.join(" or ")
+            raise SchemaError, "rollover field `#{timestamp_field_path.full_description}` cannot be used for rollover since it is not a #{date_or_datetime_description} field."
+          end
+
+          if timestamp_field_path.type.list?
+            raise SchemaError, "rollover field `#{timestamp_field_path.full_description}` cannot be used for rollover since it is a list field."
+          end
+
+          timestamp_field_path.path_parts.each { |f| f.json_schema nullable: false }
+
+          self.rollover_config = RolloverConfig.new(
+            frequency: frequency,
+            timestamp_field_path: timestamp_field_path
+          )
+        end
+
+        # Configures the index to [route documents to shards](https://www.elastic.co/guide/en/elasticsearch/reference/8.15/mapping-routing-field.html)
+        # based on the specified field. ElasticGraph optimizes queries that filter on the shard routing field so that they only run on a
+        # subset of nodes instead of all nodes. This can make a big difference in query performance if queries usually filter on a certain
+        # field. Using an appropriate field for shard routing is often essential for horizontal scaling, as it avoids having every query
+        # hit every node, allowing additional nodes to increase query throughput.
+        #
+        # @note it is essential that the shards are well-balanced. If the data’s distribution is lopsided, using this feature can make
+        #   performance worse.
+        # @note the routing field specified here **must be immutable**. If ElasticGraph receives an updated version of a document with a
+        #   different routing value, it’ll write the new version of the document to a different shard and leave the copy on the old shard
+        #   unchanged, leading to duplicates.
+        # @note changing the shard routing configuration on an existing index that already has data will result in duplicate documents
+        #
+        # @param routing_field_path_name [String] dot-separated path to the field used for shard routing. Note: all referenced
+        #   fields must exist when this is called.
+        # @return [void]
+        #
+        # @example Define a `campaigns` index to shard on `organizationId`
+        #   ElasticGraph.define_schema do |schema|
+        #     schema.object_type "Campaign" do |t|
+        #       t.field "id", "ID!"
+        #       t.field "name", "String"
+        #       t.field "organizationId", "ID"
+        #
+        #       t.index "campaigns"do |i|
+        #         i.route_with "organizationId"
+        #       end
+        #     end
+        #   end
+        def route_with(routing_field_path_name)
+          routing_field_path = public_field_path(routing_field_path_name, explanation: "it is referenced as an index `route_with` field")
+
+          unless routing_field_path.type.leaf?
+            raise SchemaError, "shard routing field `#{routing_field_path.full_description}` cannot be used for routing since it is not a leaf field."
+          end
+
+          self.routing_field_path = routing_field_path
+
+          routing_field_path.path_parts[0..-2].each { |f| f.json_schema nullable: false }
+          routing_field_path.last_part.json_schema nullable: false, pattern: HAS_NON_WHITE_SPACE_REGEX
+          indexed_type.append_to_documentation "For more performant queries on this type, please filter on `#{routing_field_path_name}` if possible."
+        end
+
+        # @see #route_with
+        # @return [Boolean] whether or not this index uses custom shard routing
+        def uses_custom_routing?
+          routing_field_path.path_in_index != "id"
+        end
+
+        # @return [Hash<String, Object>] datastore configuration for this index for when it does not use rollover
+        def to_index_config
+          {
+            "aliases" => {},
+            "mappings" => mappings,
+            "settings" => settings
+          }.compact
+        end
+
+        # @return [Hash<String, Object>] datastore configuration for the index template that will be defined if rollover is used
+        def to_index_template_config
+          {
+            "index_patterns" => ["#{name}#{ROLLOVER_INDEX_INFIX_MARKER}*"],
+            "template" => {
+              "aliases" => {},
+              "mappings" => mappings,
+              "settings" => settings
+            }
+          }
+        end
+
+        # @return [SchemaArtifacts::RuntimeMetadata::IndexDefinition] runtime metadata for this index
+        def runtime_metadata
+          SchemaArtifacts::RuntimeMetadata::IndexDefinition.new(
+            route_with: routing_field_path.path_in_index,
+            rollover: rollover_config&.runtime_metadata,
+            current_sources: indexed_type.current_sources,
+            fields_by_path: indexed_type.index_field_runtime_metadata_tuples.to_h,
+            default_sort_fields: default_sort_pairs.each_slice(2).map do |(graphql_field_path_name, direction)|
+              SchemaArtifacts::RuntimeMetadata::SortField.new(
+                field_path: public_field_path(graphql_field_path_name, explanation: "it is referenced as an index `default_sort` field").path_in_index,
+                direction: direction
+              )
+            end
+          )
+        end
+
+        private
+
+        # A regex that requires at least one non-whitespace character.
+        # Note: this does not use the `/S` character class because it's recommended to use a small subset
+        # of Regex syntax:
+        #
+        # > The regular expression syntax used is from JavaScript (ECMA 262, specifically). However, that
+        # > complete syntax is not widely supported, therefore it is recommended that you stick to the subset
+        # > of that syntax described below.
+        #
+        # (From https://json-schema.org/understanding-json-schema/reference/regular_expressions.html)
+        HAS_NON_WHITE_SPACE_REGEX = "[^ \t\n]+"
+
+        DEFAULT_SETTINGS = {
+          "index.mapping.ignore_malformed" => false,
+          "index.mapping.coerce" => false,
+          "index.number_of_replicas" => 1,
+          "index.number_of_shards" => 1
+        }
+
+        def mappings
+          field_mappings = indexed_type
+            .to_indexing_field_type
+            .to_mapping
+            .except("type") # `type` is invalid at the mapping root because it always has to be an object.
+            .then { |mapping| ListCountsMapping.merged_into(mapping, for_type: indexed_type) }
+            .then do |fm|
+              Support::HashUtil.deep_merge(fm, {"properties" => {
+                "__sources" => {"type" => "keyword"},
+                "__versions" => {
+                  "type" => "object",
+                  # __versions is map keyed by relationship name, with values that are maps keyed by id. Since it's not
+                  # a static object with known fields, we need to use dynamic here. Passing `false` allows some level
+                  # of dynamicness. As per https://www.elastic.co/guide/en/elasticsearch/reference/8.7/dynamic.html#dynamic-parameters:
+                  #
+                  # > New fields are ignored. These fields will not be indexed or searchable, but will still appear in the _source
+                  # > field of returned hits. These fields will not be added to the mapping, and new fields must be added explicitly.
+                  #
+                  # We need `__versions` to be in `_source` (so that our update scripts can operate on it), but
+                  # have no need for it to be searchable (as it's just an internal data structure used for indexing).
+                  #
+                  # Note: we intentionally set false as a string here, because that's how the datastore echoes it back
+                  # to us when you query the mapping (even if you set it as a boolean). Our checks for index mapping
+                  # consistency fail validation if we set it as a boolean since the datastore doesn't echo it back as
+                  # a boolean.
+                  "dynamic" => "false"
+                }
+              }})
+            end
+
+          {"dynamic" => "strict"}.merge(field_mappings).tap do |hash|
+            # If we are using custom shard routing, we want to require a `routing` value to be provided
+            # in every single index, get, delete or update request; otherwise the request might be
+            # made against the wrong shard.
+            hash["_routing"] = {"required" => true} if uses_custom_routing?
+            hash["_size"] = {"enabled" => true} if schema_def_state.index_document_sizes?
+          end
+        end
+
+        def public_field_path(public_path_string, explanation:)
+          parent_is_not_list = ->(parent_field) { !parent_field.type.list? }
+          resolver = SchemaElements::FieldPath::Resolver.new
+          resolved_path = resolver.resolve_public_path(indexed_type, public_path_string, &parent_is_not_list)
+          return resolved_path if resolved_path
+
+          path_parts = public_path_string.split(".")
+          error_msg = "Field `#{indexed_type.name}.#{public_path_string}` cannot be resolved, but #{explanation}."
+
+          # If it is a nested field path, the problem could be that a type has been referenced which does not exist, so mention that.
+          if path_parts.size > 1
+            error_msg += " Verify that all fields and types referenced by `#{public_path_string}` are defined."
+          end
+
+          # If the first part of the path doesn't resolve, the problem could be that the field is defined after the `index` call
+          # but it needs to be defined before it, so mention that.
+          if resolver.resolve_public_path(indexed_type, path_parts.first, &parent_is_not_list).nil?
+            error_msg += " Note: the `#{indexed_type.name}.#{path_parts.first}` definition must come before the `index` call."
+          end
+
+          raise SchemaError, error_msg
+        end
+
+        def date_and_datetime_types
+          @date_and_datetime_types ||= %w[Date DateTime].map do |type|
+            schema_def_state.type_namer.name_for(type)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/schema_definition/indexing/json_schema_field_metadata.rb

@@ -0,0 +1,34 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+module ElasticGraph
+  module SchemaDefinition
+    module Indexing
+      # @!parse class JSONSchemaFieldMetadata; end
+      JSONSchemaFieldMetadata = ::Data.define(:type, :name_in_index)
+
+      # Metadata about an ElasticGraph field that needs to be stored in our versioned JSON schemas
+      # alongside the JSON schema fields.
+      #
+      # @!attribute [r] type
+      #   @return [String] name of the ElasticGraph type for this field
+      # @!attribute [r] name_in_index
+      #   @return [String] name of the field in the index
+      #
+      # @api private
+      class JSONSchemaFieldMetadata < ::Data
+        # @return [Hash<String, String>] hash form of the metadata that can be dumped in JSON schema
+        def to_dumpable_hash
+          {"type" => type, "nameInIndex" => name_in_index}
+        end
+
+        # @dynamic initialize, type, name_in_index
+      end
+    end
+  end
+end

data/lib/elastic_graph/schema_definition/indexing/json_schema_with_metadata.rb

@@ -0,0 +1,234 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/constants"
+
+module ElasticGraph
+  module SchemaDefinition
+    module Indexing
+      # Represents the result of merging a JSON schema with metadata. The result includes both
+      # the merged JSON schema and a list of `failed_fields` indicating which fields metadata
+      # could not be determined for.
+      #
+      # @private
+      class JSONSchemaWithMetadata < ::Data.define(
+        # The JSON schema.
+        :json_schema,
+        # A set of fields (in the form `Type.field`) that were needed but not found.
+        :missing_fields,
+        # A set of type names that were needed but not found.
+        :missing_types,
+        # A set of `DeprecatedElement` objects that create conflicting definitions.
+        :definition_conflicts,
+        # A set of fields that have been deleted but that must be retained (e.g. for custom shard routing or rollover)
+        :missing_necessary_fields
+      )
+        def json_schema_version
+          json_schema.fetch(JSON_SCHEMA_VERSION_KEY)
+        end
+
+        # Responsible for building `JSONSchemaWithMetadata` instances.
+        #
+        # @private
+        class Merger
+          # @dynamic unused_deprecated_elements
+          attr_reader :unused_deprecated_elements
+
+          def initialize(schema_def_results)
+            @field_metadata_by_type_and_field_name = schema_def_results.json_schema_field_metadata_by_type_and_field_name
+            @renamed_types_by_old_name = schema_def_results.state.renamed_types_by_old_name
+            @deleted_types_by_old_name = schema_def_results.state.deleted_types_by_old_name
+            @renamed_fields_by_type_name_and_old_field_name = schema_def_results.state.renamed_fields_by_type_name_and_old_field_name
+            @deleted_fields_by_type_name_and_old_field_name = schema_def_results.state.deleted_fields_by_type_name_and_old_field_name
+            @state = schema_def_results.state
+            @derived_indexing_type_names = schema_def_results.derived_indexing_type_names
+
+            @unused_deprecated_elements = (
+              @renamed_types_by_old_name.values +
+              @deleted_types_by_old_name.values +
+              @renamed_fields_by_type_name_and_old_field_name.values.flat_map(&:values) +
+              @deleted_fields_by_type_name_and_old_field_name.values.flat_map(&:values)
+            ).to_set
+          end
+
+          def merge_metadata_into(json_schema)
+            missing_fields = ::Set.new
+            missing_types = ::Set.new
+            definition_conflicts = ::Set.new
+            old_type_name_by_current_name = {} # : ::Hash[String, String]
+
+            defs = json_schema.fetch("$defs").to_h do |type_name, type_def|
+              if type_name != EVENT_ENVELOPE_JSON_SCHEMA_NAME && (properties = type_def["properties"])
+                current_type_name = determine_current_type_name(
+                  type_name,
+                  missing_types: missing_types,
+                  definition_conflicts: definition_conflicts
+                )
+
+                if current_type_name
+                  old_type_name_by_current_name[current_type_name] = type_name
+                end
+
+                properties = properties.to_h do |field_name, prop|
+                  unless field_name == "__typename"
+                    field_metadata = current_type_name&.then do |name|
+                      field_metadata_for(
+                        name,
+                        field_name,
+                        missing_fields: missing_fields,
+                        definition_conflicts: definition_conflicts
+                      )
+                    end
+
+                    prop = prop.merge({"ElasticGraph" => field_metadata&.to_dumpable_hash})
+                  end
+
+                  [field_name, prop]
+                end
+
+                type_def = type_def.merge({"properties" => properties})
+              end
+
+              [type_name, type_def]
+            end
+
+            json_schema = json_schema.merge("$defs" => defs)
+
+            JSONSchemaWithMetadata.new(
+              json_schema: json_schema,
+              missing_fields: missing_fields,
+              missing_types: missing_types,
+              definition_conflicts: definition_conflicts,
+              missing_necessary_fields: identify_missing_necessary_fields(json_schema, old_type_name_by_current_name)
+            )
+          end
+
+          private
+
+          # Given a historical `type_name`, determines (and returns) the current name for that type.
+          def determine_current_type_name(type_name, missing_types:, definition_conflicts:)
+            exists_currently = @field_metadata_by_type_and_field_name.key?(type_name)
+            deleted = @deleted_types_by_old_name[type_name]&.tap { |elem| @unused_deprecated_elements.delete(elem) }
+            renamed = @renamed_types_by_old_name[type_name]&.tap { |elem| @unused_deprecated_elements.delete(elem) }
+
+            if [exists_currently, deleted, renamed].count(&:itself) > 1
+              definition_conflicts.merge([deleted, renamed].compact)
+            end
+
+            return type_name if exists_currently
+            return nil if deleted
+            return renamed.name if renamed
+
+            missing_types << type_name
+            nil
+          end
+
+          # Given a historical `type_name` and `field_name` determines (and returns) the field metadata for it.
+          def field_metadata_for(type_name, field_name, missing_fields:, definition_conflicts:)
+            full_name = "#{type_name}.#{field_name}"
+
+            current_meta = @field_metadata_by_type_and_field_name.dig(type_name, field_name)
+            deleted = @deleted_fields_by_type_name_and_old_field_name.dig(type_name, field_name)&.tap do |elem|
+              @unused_deprecated_elements.delete(elem)
+            end
+            renamed = @renamed_fields_by_type_name_and_old_field_name.dig(type_name, field_name)&.tap do |elem|
+              @unused_deprecated_elements.delete(elem)
+            end
+
+            if [current_meta, deleted, renamed].count(&:itself) > 1
+              definition_conflicts.merge([deleted, renamed].compact.map { |elem| elem.with(name: full_name) })
+            end
+
+            return current_meta if current_meta
+            return nil if deleted
+            return @field_metadata_by_type_and_field_name.dig(type_name, renamed.name) if renamed
+
+            missing_fields << full_name
+            nil
+          end
+
+          def identify_missing_necessary_fields(json_schema, old_type_name_by_current_name)
+            json_schema_resolver = JSONSchemaResolver.new(@state, json_schema, old_type_name_by_current_name)
+            version = json_schema.fetch(JSON_SCHEMA_VERSION_KEY)
+
+            types_to_check = @state.object_types_by_name.values.select do |type|
+              type.indexed? && !@derived_indexing_type_names.include?(type.name)
+            end
+
+            types_to_check.flat_map do |object_type|
+              object_type.indices.flat_map do |index_def|
+                identify_missing_necessary_fields_for_index_def(object_type, index_def, json_schema_resolver, version)
+              end
+            end
+          end
+
+          def identify_missing_necessary_fields_for_index_def(object_type, index_def, json_schema_resolver, json_schema_version)
+            {
+              "routing" => index_def.routing_field_path,
+              "rollover" => index_def.rollover_config&.timestamp_field_path
+            }.compact.filter_map do |field_type, field_path|
+              if json_schema_resolver.necessary_path_missing?(field_path)
+                # The JSON schema v # {json_schema_version} artifact has no field that maps to the #{field_type} path of `#{field_path.fully_qualified_path_in_index}`.
+
+                MissingNecessaryField.new(
+                  field_type: field_type,
+                  fully_qualified_path: field_path.fully_qualified_path_in_index
+                )
+              end
+            end
+          end
+
+          class JSONSchemaResolver
+            def initialize(state, json_schema, old_type_name_by_current_name)
+              @state = state
+              @old_type_name_by_current_name = old_type_name_by_current_name
+              @meta_by_old_type_and_name_in_index = ::Hash.new do |hash, type_name|
+                properties = json_schema.fetch("$defs").fetch(type_name).fetch("properties")
+
+                hash[type_name] = properties.filter_map do |name, prop|
+                  if (metadata = prop["ElasticGraph"])
+                    [metadata.fetch("nameInIndex"), metadata]
+                  end
+                end.to_h
+              end
+            end
+
+            # Indicates if the given `field_path` is (1) necessary and (2) missing from the JSON schema, indicating a problem.
+            #
+            # - Returns `false` is the given `field_path` is present in the JSON schema.
+            # - Returns `false` is the parent type of `field_path` has not been retained in this JSON schema version
+            #   (in that case, the field path is not necessary).
+            # - Otherwise, returns `true` since the field path is both necessary and missing.
+            def necessary_path_missing?(field_path)
+              parent_type = field_path.first_part.parent_type.name
+
+              field_path.path_parts.any? do |path_part|
+                necessary_path_part_missing?(parent_type, path_part.name_in_index) do |meta|
+                  parent_type = @state.type_ref(meta.fetch("type")).fully_unwrapped.name
+                end
+              end
+            end
+
+            private
+
+            def necessary_path_part_missing?(parent_type, name_in_index)
+              old_type_name = @old_type_name_by_current_name[parent_type]
+              return false unless old_type_name
+
+              meta = @meta_by_old_type_and_name_in_index.dig(old_type_name, name_in_index)
+              yield meta if meta
+              !meta
+            end
+          end
+        end
+
+        MissingNecessaryField = ::Data.define(:field_type, :fully_qualified_path)
+      end
+    end
+  end
+end

data/lib/elastic_graph/schema_definition/indexing/list_counts_mapping.rb

@@ -0,0 +1,53 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/constants"
+require "elastic_graph/support/hash_util"
+
+module ElasticGraph
+  module SchemaDefinition
+    module Indexing
+      # To support filtering on the `count` of a list field, we need to index the counts as we ingest
+      # events. This is responsible for defining the mapping for the special `__counts` field in which
+      # we store the list counts.
+      #
+      # @private
+      module ListCountsMapping
+        # Builds the `__counts` field mapping for the given `for_type`. Returns a new `mapping_hash` with
+        # the extra `__counts` field merged into it.
+        def self.merged_into(mapping_hash, for_type:)
+          counts_properties = for_type.indexing_fields_by_name_in_index.values.flat_map do |field|
+            field.paths_to_lists_for_count_indexing.map do |path|
+              # We chose the `integer` type here because:
+              #
+              # - While we expect datasets with more documents than the max integer value (~2B), we don't expect
+              #   individual documents to have any list fields with more elements than can fit in an integer.
+              # - Using `long` would allow for much larger counts, but we don't want to take up double the
+              #   storage space for this.
+              #
+              # Note that `new_list_filter_input_type` (in `schema_definition/factory.rb`) relies on this, and
+              # has chosen to use `IntFilterInput` (rather than `JsonSafeLongFilterInput`) for filtering these count values.
+              # If we change the mapping type here, we should re-evaluate the filter used there.
+              [path, {"type" => "integer"}]
+            end
+          end.to_h
+
+          return mapping_hash if counts_properties.empty?
+
+          Support::HashUtil.deep_merge(mapping_hash, {
+            "properties" => {
+              LIST_COUNTS_FIELD => {
+                "properties" => counts_properties
+              }
+            }
+          })
+        end
+      end
+    end
+  end
+end