elasticgraph-schema_definition 0.18.0.0

data/lib/elastic_graph/schema_definition/indexing/field.rb

@@ -0,0 +1,181 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/constants"
+require "elastic_graph/schema_definition/indexing/json_schema_field_metadata"
+require "elastic_graph/schema_definition/indexing/list_counts_mapping"
+require "elastic_graph/support/hash_util"
+require "elastic_graph/support/memoizable_data"
+
+module ElasticGraph
+  module SchemaDefinition
+    module Indexing
+      # Represents a field in a JSON document during indexing.
+      #
+      # @api private
+      class Field < Support::MemoizableData.define(
+        :name,
+        :name_in_index,
+        :type,
+        :json_schema_layers,
+        :indexing_field_type,
+        :accuracy_confidence,
+        :json_schema_customizations,
+        :mapping_customizations,
+        :source,
+        :runtime_field_script
+      )
+        # JSON schema overrides that automatically apply to specific mapping types so that the JSON schema
+        # validation will reject values which cannot be indexed into fields of a specific mapping type.
+        #
+        # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/number.html Elasticsearch numeric field type documentation
+        # @note We don't handle `integer` here because it's the default numeric type (handled by our definition of the `Int` scalar type).
+        # @note Likewise, we don't handle `long` here because a custom scalar type must be used for that since GraphQL's `Int` type can't handle long values.
+        JSON_SCHEMA_OVERRIDES_BY_MAPPING_TYPE = {
+          "byte" => {"minimum" => -(2**7), "maximum" => (2**7) - 1},
+          "short" => {"minimum" => -(2**15), "maximum" => (2**15) - 1},
+          "keyword" => {"maxLength" => DEFAULT_MAX_KEYWORD_LENGTH},
+          "text" => {"maxLength" => DEFAULT_MAX_TEXT_LENGTH}
+        }
+
+        # @return [Hash<String, Object>] the mapping for this field. The returned hash should be composed entirely
+        #   of Ruby primitives that, when converted to a JSON string, match the structure required by
+        #   [Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html).
+        def mapping
+          @mapping ||= begin
+            raw_mapping = indexing_field_type
+              .to_mapping
+              .merge(Support::HashUtil.stringify_keys(mapping_customizations))
+
+            if (object_type = type.fully_unwrapped.as_object_type) && type.list? && mapping_customizations[:type] == "nested"
+              # If it's an object list field using the `nested` type, we need to add a `__counts` field to
+              # the mapping for all of its subfields which are lists.
+              ListCountsMapping.merged_into(raw_mapping, for_type: object_type)
+            else
+              raw_mapping
+            end
+          end
+        end
+
+        # @return [Hash<String, Object>] the JSON schema definition for this field. The returned object should
+        #   be composed entirely of Ruby primitives that, when converted to a JSON string, match the
+        #   requirements of [the JSON schema spec](https://json-schema.org/).
+        def json_schema
+          json_schema_layers
+            .reverse # resolve layers from innermost to outermost wrappings
+            .reduce(inner_json_schema) { |acc, layer| process_layer(layer, acc) }
+            .merge(outer_json_schema_customizations)
+            .then { |h| Support::HashUtil.stringify_keys(h) }
+        end
+
+        # @return [JSONSchemaFieldMetadata] additional ElasticGraph metadata to be stored in the JSON schema for this field.
+        def json_schema_metadata
+          JSONSchemaFieldMetadata.new(type: type.name, name_in_index: name_in_index)
+        end
+
+        # Builds a hash containing the mapping for the provided fields, normalizing it in the same way that the
+        # datastore does so that consistency checks between our index configuration and what's in the datastore
+        # work properly.
+        #
+        # @param fields [Array<Field>] fields to generate a mapping hash from
+        # @return [Hash<String, Object>] generated mapping hash
+        def self.normalized_mapping_hash_for(fields)
+          # When an object field has `properties`, the datastore normalizes the mapping by dropping
+          # the `type => object` (it's implicit, as `properties` are only valid on an object...).
+          # OTOH, when there are no properties, the datastore normalizes the mapping by dropping the
+          # empty `properties` entry and instead returning `type => object`.
+          return {"type" => "object"} if fields.empty?
+
+          # Partition the fields into runtime fields and normal fields based on the presence of runtime_script
+          runtime_fields, normal_fields = fields.partition(&:runtime_field_script)
+
+          mapping_hash = {
+            "properties" => normal_fields.to_h { |f| [f.name_in_index, f.mapping] }
+          }
+          unless runtime_fields.empty?
+            mapping_hash["runtime"] = runtime_fields.to_h do |f|
+              [f.name_in_index, f.mapping.merge({"script" => {"source" => f.runtime_field_script}})]
+            end
+          end
+
+          mapping_hash
+        end
+
+        private
+
+        def inner_json_schema
+          user_specified_customizations =
+            if user_specified_json_schema_customizations_go_on_outside?
+              {} # : ::Hash[::String, untyped]
+            else
+              Support::HashUtil.stringify_keys(json_schema_customizations)
+            end
+
+          customizations_from_mapping = JSON_SCHEMA_OVERRIDES_BY_MAPPING_TYPE[mapping["type"]] || {}
+          customizations = customizations_from_mapping.merge(user_specified_customizations)
+          customizations = indexing_field_type.format_field_json_schema_customizations(customizations)
+
+          ref = {"$ref" => "#/$defs/#{type.unwrapped_name}"}
+          return ref if customizations.empty?
+
+          # Combine any customizations with type ref under an "allOf" subschema:
+          # All of these properties must hold true for the type to be valid.
+          #
+          # Note that if we simply combine the customizations with the `$ref`
+          # at the same level, it will not work, because other subschema
+          # properties are ignored when they are in the same object as a `$ref`:
+          # https://github.com/json-schema-org/JSON-Schema-Test-Suite/blob/2.0.0/tests/draft7/ref.json#L165-L168
+          {"allOf" => [ref, customizations]}
+        end
+
+        def outer_json_schema_customizations
+          return {} unless user_specified_json_schema_customizations_go_on_outside?
+          Support::HashUtil.stringify_keys(json_schema_customizations)
+        end
+
+        # Indicates if the user-specified JSON schema customizations should go on the inside
+        # (where they normally go) or on the outside. They only go on the outside when it's
+        # an array field, because then they apply to the array itself instead of the items in the
+        # array.
+        def user_specified_json_schema_customizations_go_on_outside?
+          json_schema_layers.include?(:array)
+        end
+
+        def process_layer(layer, schema)
+          case layer
+          when :nullable
+            make_nullable(schema)
+          when :array
+            make_array(schema)
+          else
+            # :nocov: - layer is only ever `:nullable` or `:array` so we never get here
+            schema
+            # :nocov:
+          end
+        end
+
+        def make_nullable(schema)
+          # Here we use "anyOf" to ensure that JSON can either match the schema OR null.
+          #
+          # (Using "oneOf" would mean that if we had a schema that also allowed null,
+          # null would never be allowed, since "oneOf" must match exactly one subschema).
+          {
+            "anyOf" => [
+              schema,
+              {"type" => "null"}
+            ]
+          }
+        end
+
+        def make_array(schema)
+          {"type" => "array", "items" => schema}
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/schema_definition/indexing/field_reference.rb

@@ -0,0 +1,51 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+module ElasticGraph
+  module SchemaDefinition
+    module Indexing
+      # @!parse class FieldReference < ::Data; end
+      FieldReference = ::Data.define(
+        :name,
+        :name_in_index,
+        :type,
+        :mapping_options,
+        :json_schema_options,
+        :accuracy_confidence,
+        :source,
+        :runtime_field_script
+      )
+
+      # A lazy reference to a {Field}. It contains all attributes needed to build a {Field}, but the referenced `type` may not be
+      # resolvable yet (which is why this exists).
+      #
+      # @api private
+      class FieldReference < ::Data
+        # @return [Field, nil] the {Field} this reference resolves to (if it can be resolved)
+        def resolve
+          return nil unless (resolved_type = type.fully_unwrapped.resolved)
+
+          Indexing::Field.new(
+            name: name,
+            name_in_index: name_in_index,
+            type: type,
+            json_schema_layers: type.json_schema_layers,
+            indexing_field_type: resolved_type.to_indexing_field_type,
+            accuracy_confidence: accuracy_confidence,
+            json_schema_customizations: json_schema_options,
+            mapping_customizations: mapping_options,
+            source: source,
+            runtime_field_script: runtime_field_script
+          )
+        end
+
+        # @dynamic initialize, with, name, name_in_index, type, mapping_options, json_schema_options, accuracy_confidence, source, runtime_field_script
+      end
+    end
+  end
+end

data/lib/elastic_graph/schema_definition/indexing/field_type/enum.rb

@@ -0,0 +1,65 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+module ElasticGraph
+  module SchemaDefinition
+    module Indexing
+      # Contains implementation logic for the different types of indexing fields.
+      #
+      # @api private
+      module FieldType
+        # @!parse class Enum < ::Data; end
+        Enum = ::Data.define(:enum_value_names)
+
+        # Responsible for the JSON schema and mapping of a {SchemaElements::EnumType}.
+        #
+        # @!attribute [r] enum_value_names
+        #   @return [Array<String>] list of names of values in this enum type.
+        #
+        # @api private
+        class Enum < ::Data
+          # @return [Hash<String, ::Object>] the JSON schema for this enum type.
+          def to_json_schema
+            {"type" => "string", "enum" => enum_value_names}
+          end
+
+          # @return [Hash<String, ::Object>] the datastore mapping for this enum type.
+          def to_mapping
+            {"type" => "keyword"}
+          end
+
+          # @return [Hash<String, ::Object>] additional ElasticGraph metadata to put in the JSON schema for this enum type.
+          def json_schema_field_metadata_by_field_name
+            {}
+          end
+
+          # @param customizations [Hash<String, ::Object>] JSON schema customizations
+          # @return [Hash<String, ::Object>] formatted customizations.
+          def format_field_json_schema_customizations(customizations)
+            # Since an enum type already restricts the values to a small set of allowed values, we do not need to keep
+            # other customizations (such as the `maxLength` field customization EG automatically applies to fields
+            # indexed as a `keyword`--we don't allow enum values to exceed that length, anyway).
+            #
+            # It's desirable to restrict what customizations are applied because when a publisher uses the JSON schema
+            # to generate code using a library such as https://github.com/pwall567/json-kotlin-schema-codegen, we found
+            # that the presence of extra field customizations inhibits the library's ability to generate code in the way
+            # we want (it causes the type of the enum to change since the JSON schema changes from a direct `$ref` to
+            # being wrapped in an `allOf`).
+            #
+            # However, we still want to apply `enum` customizations--this allows a user to "narrow" the set of allowed
+            # values for a field. For example, a `Currency` enum could contain every currency, and a user may want to
+            # restrict a specific `currency` field to a subset of currencies (e.g. to just USD, CAD, and EUR).
+            customizations.slice("enum")
+          end
+
+          # @dynamic initialize, enum_value_names
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/schema_definition/indexing/field_type/object.rb

@@ -0,0 +1,113 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/error"
+require "elastic_graph/support/hash_util"
+require "elastic_graph/support/memoizable_data"
+
+module ElasticGraph
+  module SchemaDefinition
+    module Indexing
+      module FieldType
+        # Responsible for the JSON schema and mapping of a {SchemaElements::ObjectType}.
+        #
+        # @!attribute [r] type_name
+        #   @return [String] name of the object type
+        # @!attribute [r] subfields
+        #   @return [Array<Field>] the subfields of this object type
+        # @!attribute [r] mapping_options
+        #   @return [Hash<String, ::Object>] options to be included in the mapping
+        # @!attribute [r] json_schema_options
+        #   @return [Hash<String, ::Object>] options to be included in the JSON schema
+        #
+        # @api private
+        class Object < Support::MemoizableData.define(:type_name, :subfields, :mapping_options, :json_schema_options)
+          # @return [Hash<String, ::Object>] the datastore mapping for this object type.
+          def to_mapping
+            @to_mapping ||= begin
+              base_mapping = Field.normalized_mapping_hash_for(subfields)
+              # When a custom mapping type is used, we need to omit `properties`, because custom mapping
+              # types generally don't use `properties` (and if you need to use `properties` with a custom
+              # type, you're responsible for defining the properties).
+              base_mapping = base_mapping.except("properties") if (mapping_options[:type] || "object") != "object"
+              base_mapping.merge(Support::HashUtil.stringify_keys(mapping_options))
+            end
+          end
+
+          # @return [Hash<String, ::Object>] the JSON schema for this object type.
+          def to_json_schema
+            @to_json_schema ||=
+              if json_schema_options.empty?
+                # Fields that are `sourced_from` an alternate type must not be included in this types JSON schema,
+                # since events of this type won't include them.
+                other_source_subfields, json_schema_candidate_subfields = subfields.partition(&:source)
+                validate_sourced_fields_have_no_json_schema_overrides(other_source_subfields)
+                json_schema_subfields = json_schema_candidate_subfields.reject(&:runtime_field_script)
+
+                {
+                  "type" => "object",
+                  "properties" => json_schema_subfields.to_h { |f| [f.name, f.json_schema] }.merge(json_schema_typename_field),
+                  # Note: `__typename` is intentionally not included in the `required` list. If `__typename` is present
+                  # we want it validated (as we do by merging in `json_schema_typename_field`) but we only want
+                  # to require it in the context of a union type. The union's json schema requires the field.
+                  "required" => json_schema_subfields.map(&:name).freeze
+                }.freeze
+              else
+                Support::HashUtil.stringify_keys(json_schema_options)
+              end
+          end
+
+          # @return [Hash<String, ::Object>] additional ElasticGraph metadata to put in the JSON schema for this object type.
+          def json_schema_field_metadata_by_field_name
+            subfields.to_h { |f| [f.name, f.json_schema_metadata] }
+          end
+
+          # @param customizations [Hash<String, ::Object>] JSON schema customizations
+          # @return [Hash<String, ::Object>] formatted customizations.
+          def format_field_json_schema_customizations(customizations)
+            customizations
+          end
+
+          private
+
+          def after_initialize
+            subfields.freeze
+          end
+
+          # Returns a __typename property which we use for union types.
+          #
+          # This must always be set to the name of the type (thus the const value).
+          #
+          # We also add a "default" value. This does not impact validation, but rather
+          # aids tools like our kotlin codegen to save publishers from having to set the
+          # property explicitly when creating events.
+          def json_schema_typename_field
+            {
+              "__typename" => {
+                "type" => "string",
+                "const" => type_name,
+                "default" => type_name
+              }
+            }
+          end
+
+          def validate_sourced_fields_have_no_json_schema_overrides(other_source_subfields)
+            problem_fields = other_source_subfields.reject { |f| f.json_schema_customizations.empty? }
+            return if problem_fields.empty?
+
+            field_descriptions = problem_fields.map(&:name).sort.map { |f| "`#{f}`" }.join(", ")
+            raise SchemaError,
+              "`#{type_name}` has #{problem_fields.size} field(s) (#{field_descriptions}) that are `sourced_from` " \
+              "another type and also have JSON schema customizations. Instead, put the JSON schema " \
+              "customizations on the source type's field definitions."
+          end
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/schema_definition/indexing/field_type/scalar.rb

@@ -0,0 +1,51 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/support/hash_util"
+
+module ElasticGraph
+  module SchemaDefinition
+    module Indexing
+      module FieldType
+        # @!parse class Scalar < ::Data; end
+        Scalar = ::Data.define(:scalar_type)
+
+        # Responsible for the JSON schema and mapping of a {SchemaElements::ScalarType}.
+        #
+        # @!attribute [r] scalar_type
+        #   @return [SchemaElements::ScalarType] the scalar type
+        #
+        # @api private
+        class Scalar < ::Data
+          # @return [Hash<String, ::Object>] the datastore mapping for this scalar type.
+          def to_mapping
+            Support::HashUtil.stringify_keys(scalar_type.mapping_options)
+          end
+
+          # @return [Hash<String, ::Object>] the JSON schema for this scalar type.
+          def to_json_schema
+            Support::HashUtil.stringify_keys(scalar_type.json_schema_options)
+          end
+
+          # @return [Hash<String, ::Object>] additional ElasticGraph metadata to put in the JSON schema for this scalar type.
+          def json_schema_field_metadata_by_field_name
+            {}
+          end
+
+          # @param customizations [Hash<String, ::Object>] JSON schema customizations
+          # @return [Hash<String, ::Object>] formatted customizations.
+          def format_field_json_schema_customizations(customizations)
+            customizations
+          end
+
+          # @dynamic initialize, scalar_type
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/schema_definition/indexing/field_type/union.rb

@@ -0,0 +1,70 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/schema_definition/indexing/field_type/object"
+require "elastic_graph/support/hash_util"
+
+module ElasticGraph
+  module SchemaDefinition
+    module Indexing
+      module FieldType
+        # Responsible for the JSON schema and mapping of a {SchemaElements::UnionType}.
+        #
+        # @note In JSON schema, we model this with a `oneOf`, and a `__typename` field on each subtype.
+        # @note Within the mapping, we have a single object type that has a set union of the properties
+        #   of the subtypes (and also a `__typename` keyword field).
+        #
+        # @!attribute [r] subtypes_by_name
+        #   @return [Hash<String, Object>] the subtypes of the union, keyed by name.
+        #
+        # @api private
+        class Union < ::Data.define(:subtypes_by_name)
+          # @return [Hash<String, ::Object>] the JSON schema for this union type.
+          def to_json_schema
+            subtype_json_schemas = subtypes_by_name.keys.map { |name| {"$ref" => "#/$defs/#{name}"} }
+
+            # A union type can represent multiple subtypes, referenced by the "anyOf" clause below.
+            # We also add a requirement for the presence of __typename to indicate which type
+            # is being referenced (this property is pre-defined on the type itself as a constant).
+            #
+            # Note: Although both "oneOf" and "anyOf" keywords are valid for combining schemas
+            # to form a union, and validate equivalently when no object can satisfy multiple of the
+            # subschemas (which is the case here given the __typename requirements are mutually
+            # exclusive), we chose to use "oneOf" here because it works better with this library:
+            # https://github.com/pwall567/json-kotlin-schema-codegen
+            {
+              "required" => %w[__typename],
+              "oneOf" => subtype_json_schemas
+            }
+          end
+
+          # @return [Hash<String, ::Object>] the datastore mapping for this union type.
+          def to_mapping
+            mapping_subfields = subtypes_by_name.values.map(&:subfields).reduce([], :union)
+
+            Support::HashUtil.deep_merge(
+              Field.normalized_mapping_hash_for(mapping_subfields),
+              {"properties" => {"__typename" => {"type" => "keyword"}}}
+            )
+          end
+
+          # @return [Hash<String, ::Object>] additional ElasticGraph metadata to put in the JSON schema for this union type.
+          def json_schema_field_metadata_by_field_name
+            {}
+          end
+
+          # @param customizations [Hash<String, ::Object>] JSON schema customizations
+          # @return [Hash<String, ::Object>] formatted customizations.
+          def format_field_json_schema_customizations(customizations)
+            customizations
+          end
+        end
+      end
+    end
+  end
+end