elasticgraph-indexer 1.0.2 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 96459aae32411712ac8a1fe13289cf2e411a4ac8c7a38d2b868712380149fcf1
-  data.tar.gz: d73d0c3420d55399b6e4111606ba8ba154a19067efb15eb0b006f30e86f664af
+  metadata.gz: 7a8d580572ec508d9df0131448f19398a5850aaee3cd1e36b4530be67bee190f
+  data.tar.gz: 7db60416bac8d71edad120569e8abfe55e2884b6844d140109e485b8c9d66d10
 SHA512:
-  metadata.gz: ab47eea78b85a3e0e6252eed24cc775402be7527778a8cb15bde35c0b76451af057baab6b37b43c02957ee13138ff70e11078625bcb48834a050c3c1045cddea
-  data.tar.gz: c47921bbfce1a736c714ad5f696c6692460b59c545a15af23a575a85c192e41ad27ef671fe7e4bb6309a2328f278b5a28a7b96dc781f13ef96adaa9f8c554e7e
+  metadata.gz: 14b344d1591976df42dd53257a03911e158f9913974c7c2e606c8ea1222989a6552154f615606d0447e97aebef4097f6710600a0af49f453e821c519c89ee146
+  data.tar.gz: 45d6e1a8c27040378857d7b0c7a7411b669c9f9c6e4ba31ee7e3307cfd71f48780c842e1f505d4d3b19e1c63fdfdcdee2db2dfb30b9e5feb40a122796a991dbe

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2024 - 2025 Block, Inc.
+Copyright (c) 2024 - 2026 Block, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/lib/elastic_graph/indexer/config.rb

@@ -1,4 +1,4 @@
-# Copyright 2024 - 2025 Block, Inc.
+# Copyright 2024 - 2026 Block, Inc.
 #
 # Use of this source code is governed by an MIT-style
 # license that can be found in the LICENSE file or at

data/lib/elastic_graph/indexer/datastore_indexing_router.rb

@@ -1,4 +1,4 @@
-# Copyright 2024 - 2025 Block, Inc.
+# Copyright 2024 - 2026 Block, Inc.
 #
 # Use of this source code is governed by an MIT-style
 # license that can be found in the LICENSE file or at
@@ -8,9 +8,7 @@
 
 require "elastic_graph/constants"
 require "elastic_graph/errors"
-require "elastic_graph/datastore_core/index_config_normalizer"
 require "elastic_graph/indexer/event_id"
-require "elastic_graph/indexer/hash_differ"
 require "elastic_graph/indexer/indexing_failures_error"
 require "elastic_graph/support/threading"
 
@@ -18,38 +16,12 @@ module ElasticGraph
   class Indexer
     # Responsible for routing datastore indexing requests to the appropriate cluster and index.
     class DatastoreIndexingRouter
-      # In this class, we internally cache the datastore mapping for an index definition, so that we don't have to
-      # fetch the mapping from the datastore on each call to `bulk`. It rarely changes and ElasticGraph is designed so that
-      # mapping updates are applied before deploying the indexer with a new mapping.
-      #
-      # However, if an engineer forgets to apply a mapping update before deploying, they'll run into "mappings are incomplete"
-      # errors. They can updated the mapping to fix it, but the use of caching in this class could mean that the fix doesn't
-      # necessarily work right away. The app would have to be deployed or restarted so that the caches are cleared. That could
-      # be annoying.
-      #
-      # To address this issue, we're adding an expiration on the caching of the index mappings. Re-fetching the index
-      # mapping once every few minutes is no big deal and will allow the indexer to recover on its own after a mapping
-      # update has been applied without requiring a deploy or a restart.
-      #
-      # The expiration is a range so that, when we have many processes running, and they all started around the same time,
-      # (say, after a deploy!), they don't all expire their caches in sync, leading to spiky load on the datastore. Instead,
-      # the random distribution of expiration times will spread out the load.
-      MAPPING_CACHE_MAX_AGE_IN_MS_RANGE = (5 * 60 * 1000)..(10 * 60 * 1000)
-
       def initialize(
         datastore_clients_by_name:,
-        mappings_by_index_def_name:,
-        monotonic_clock:,
         logger:
       )
         @datastore_clients_by_name = datastore_clients_by_name
         @logger = logger
-        @monotonic_clock = monotonic_clock
-        @cached_mappings = {}
-
-        @mappings_by_index_def_name = mappings_by_index_def_name.transform_values do |mappings|
-          DatastoreCore::IndexConfigNormalizer.normalize_mappings(mappings)
-        end
       end
 
       # Proxies `client#bulk` by converting `operations` to their bulk
@@ -70,17 +42,11 @@ module ElasticGraph
       #
       # It is the caller's responsibility to deal with any returned failures as this method does not
       # raise an exception in that case.
-      #
-      # Note: before any operations are attempted, the datastore indices are validated for consistency
-      # with the mappings we expect, meaning that no bulk operations will be attempted if that is not up-to-date.
       def bulk(operations, refresh: false)
-        # Before writing these operations, verify their destination index mapping are consistent.
-        validate_mapping_completeness_of!(:accessible_cluster_names_to_index_into, *operations.map(&:destination_index_def).uniq)
-
         ops_by_client = ::Hash.new { |h, k| h[k] = [] } # : ::Hash[DatastoreCore::_Client, ::Array[_Operation]]
         unsupported_ops = ::Set.new # : ::Set[_Operation]
 
-        operations.reject { |op| op.to_datastore_bulk.empty? }.each do |op|
+        operations.each do |op|
           # Note: this intentionally does not use `accessible_cluster_names_to_index_into`.
           # We want to fail with clear error if any clusters are inaccessible instead of silently ignoring
           # the named cluster. The `IndexingFailuresError` provides a clear error.
@@ -290,108 +256,6 @@ module ElasticGraph
             "for document versions:\n\n#{failures.join("\n")}"
         end
       end
-
-      # Queries the datastore mapping(s) for the given index definition(s) to verify that they are up-to-date
-      # with our schema artifacts, raising an error if the datastore mappings are missing fields that we
-      # expect. (Extra fields are allowed, though--we'll just ignore them).
-      #
-      # This is intended for use when you want a strong guarantee before proceeding that the indices are current,
-      # such as before indexing data, or after applying index updates (to "prove" that everything is how it should
-      # be).
-      #
-      # This correctly queries the datastore clusters specified via `index_into_clusters` in config,
-      # but ignores clusters specified via `query_cluster` (since this isn't intended to be used as part
-      # of the query flow).
-      #
-      # For a rollover template, this takes care of verifying the template itself and also any indices that originated
-      # from the template.
-      #
-      # Note also that this caches the datastore mappings, since this is intended to be used to verify an index
-      # before we index data into it, and we do not want to impose a huge performance penalty on that process (requiring
-      # multiple datastore requests before we index each document...). In general, the index mapping only changes
-      # when we make it change, and we deploy and restart ElasticGraph after any index mapping changes, so we do not
-      # need to worry about it mutating during the lifetime of a single process (particularly given the expense of doing
-      # so).
-      def validate_mapping_completeness_of!(index_cluster_name_method, *index_definitions)
-        diffs_by_cluster_and_index_name =
-          index_definitions.reduce({}) do |accum, index_def| # $ ::Hash[[::String, ::String], ::String]
-            accum.merge(mapping_diffs_for(index_def, index_cluster_name_method))
-          end
-
-        if diffs_by_cluster_and_index_name.any?
-          formatted_diffs = diffs_by_cluster_and_index_name.map do |(cluster_name, index_name), diff|
-            <<~EOS
-              On cluster `#{cluster_name}` and index/template `#{index_name}`:
-              #{diff}
-            EOS
-          end.join("\n\n")
-
-          raise Errors::ConfigError, "Datastore index mappings are incomplete compared to the current schema. " \
-            "The diff below uses the datastore index mapping as the base, and shows the expected mapping as a diff. " \
-            "\n\n#{formatted_diffs}"
-        end
-      end
-
-      private
-
-      def mapping_diffs_for(index_definition, index_cluster_name_method)
-        expected_mapping = @mappings_by_index_def_name.fetch(index_definition.name)
-
-        index_definition.public_send(index_cluster_name_method).flat_map do |cluster_name|
-          datastore_client = datastore_client_named(cluster_name)
-
-          cached_mappings_for(index_definition, datastore_client).filter_map do |index, mapping_in_index|
-            if (diff = HashDiffer.diff(mapping_in_index, expected_mapping, ignore_ops: [:-]))
-              [[cluster_name, index.name], diff]
-            end
-          end
-        end.to_h
-      end
-
-      def cached_mappings_for(index_definition, datastore_client)
-        key = [datastore_client, index_definition] # : [DatastoreCore::_Client, DatastoreCore::indexDefinition]
-        cached_mapping = @cached_mappings[key] ||= new_cached_mapping(fetch_mappings_from_datastore(index_definition, datastore_client))
-
-        return cached_mapping.mappings if @monotonic_clock.now_in_ms < cached_mapping.expires_at
-
-        begin
-          fetch_mappings_from_datastore(index_definition, datastore_client).tap do |mappings|
-            @logger.info "Mapping cache expired for #{index_definition.name}; cleared it from the cache and re-fetched the mapping."
-            @cached_mappings[key] = new_cached_mapping(mappings)
-          end
-        rescue => e
-          @logger.warn <<~EOS
-            Mapping cache expired for #{index_definition.name}; attempted to re-fetch it but got an error[1]. Will continue using expired mapping information for now.
-
-            [1] #{e.class}: #{e.message}
-            #{e.backtrace.join("\n")}
-          EOS
-
-          # Update the cached mapping so that the expiration is reset.
-          @cached_mappings[key] = new_cached_mapping(cached_mapping.mappings)
-
-          cached_mapping.mappings
-        end
-      end
-
-      def fetch_mappings_from_datastore(index_definition, datastore_client)
-        # We need to also check any related indices...
-        indices_to_check = [index_definition] + index_definition.related_rollover_indices(datastore_client)
-
-        indices_to_check.to_h do |index|
-          [index, index.mappings_in_datastore(datastore_client)]
-        end
-      end
-
-      def new_cached_mapping(mappings)
-        CachedMapping.new(mappings, @monotonic_clock.now_in_ms + rand(MAPPING_CACHE_MAX_AGE_IN_MS_RANGE).to_i)
-      end
-
-      def datastore_client_named(name)
-        @datastore_clients_by_name.fetch(name)
-      end
-
-      CachedMapping = ::Data.define(:mappings, :expires_at)
     end
   end
 end

data/lib/elastic_graph/indexer/event_id.rb

@@ -1,4 +1,4 @@
-# Copyright 2024 - 2025 Block, Inc.
+# Copyright 2024 - 2026 Block, Inc.
 #
 # Use of this source code is governed by an MIT-style
 # license that can be found in the LICENSE file or at

data/lib/elastic_graph/indexer/failed_event_error.rb

@@ -1,4 +1,4 @@
-# Copyright 2024 - 2025 Block, Inc.
+# Copyright 2024 - 2026 Block, Inc.
 #
 # Use of this source code is governed by an MIT-style
 # license that can be found in the LICENSE file or at

data/lib/elastic_graph/indexer/hash_differ.rb

@@ -1,4 +1,4 @@
-# Copyright 2024 - 2025 Block, Inc.
+# Copyright 2024 - 2026 Block, Inc.
 #
 # Use of this source code is governed by an MIT-style
 # license that can be found in the LICENSE file or at

data/lib/elastic_graph/indexer/indexing_failures_error.rb

@@ -1,4 +1,4 @@
-# Copyright 2024 - 2025 Block, Inc.
+# Copyright 2024 - 2026 Block, Inc.
 #
 # Use of this source code is governed by an MIT-style
 # license that can be found in the LICENSE file or at

data/lib/elastic_graph/indexer/indexing_preparers/date_time.rb

@@ -0,0 +1,41 @@
+# Copyright 2024 - 2026 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/constants"
+require "time"
+
+module ElasticGraph
+  class Indexer
+    module IndexingPreparers
+      class DateTime
+        # Here we normalize DateTime strings to consistent 3-digit millisecond precision.
+        # This is critical for min/max value tracking which uses string comparison via
+        # Painless's `.compareTo()` method.
+        #
+        # Without consistent precision, string comparison produces incorrect results:
+        #   "2025-12-19T04:15:47.53Z" vs "2025-12-19T04:15:47.531Z"
+        #                        ^                              ^
+        #                       'Z' (ASCII 90)  >  '1' (ASCII 49)
+        #
+        # This would incorrectly determine that `.53Z > .531Z`.
+        #
+        # By normalizing all DateTime values to 3-digit precision, we ensure:
+        #   "2025-12-19T04:15:47.530Z" < "2025-12-19T04:15:47.531Z"
+        def self.prepare_for_indexing(value)
+          return value if value.nil?
+          time = ::Time.iso8601(value)
+          time.getutc.iso8601(DATE_TIME_PRECISION)
+        rescue ::ArgumentError, ::TypeError
+          # If the value is not a valid ISO8601 string, return it as-is
+          # and let the datastore reject it if necessary.
+          value
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/indexer/indexing_preparers/integer.rb

@@ -1,4 +1,4 @@
-# Copyright 2024 - 2025 Block, Inc.
+# Copyright 2024 - 2026 Block, Inc.
 #
 # Use of this source code is governed by an MIT-style
 # license that can be found in the LICENSE file or at

data/lib/elastic_graph/indexer/indexing_preparers/no_op.rb

@@ -1,4 +1,4 @@
-# Copyright 2024 - 2025 Block, Inc.
+# Copyright 2024 - 2026 Block, Inc.
 #
 # Use of this source code is governed by an MIT-style
 # license that can be found in the LICENSE file or at

data/lib/elastic_graph/indexer/indexing_preparers/untyped.rb

@@ -1,4 +1,4 @@
-# Copyright 2024 - 2025 Block, Inc.
+# Copyright 2024 - 2026 Block, Inc.
 #
 # Use of this source code is governed by an MIT-style
 # license that can be found in the LICENSE file or at

data/lib/elastic_graph/indexer/operation/count_accumulator.rb

@@ -1,4 +1,4 @@
-# Copyright 2024 - 2025 Block, Inc.
+# Copyright 2024 - 2026 Block, Inc.
 #
 # Use of this source code is governed by an MIT-style
 # license that can be found in the LICENSE file or at

data/lib/elastic_graph/indexer/operation/factory.rb

@@ -1,4 +1,4 @@
-# Copyright 2024 - 2025 Block, Inc.
+# Copyright 2024 - 2026 Block, Inc.
 #
 # Use of this source code is governed by an MIT-style
 # license that can be found in the LICENSE file or at

data/lib/elastic_graph/indexer/operation/result.rb

@@ -1,4 +1,4 @@
-# Copyright 2024 - 2025 Block, Inc.
+# Copyright 2024 - 2026 Block, Inc.
 #
 # Use of this source code is governed by an MIT-style
 # license that can be found in the LICENSE file or at

data/lib/elastic_graph/indexer/operation/update.rb

@@ -1,4 +1,4 @@
-# Copyright 2024 - 2025 Block, Inc.
+# Copyright 2024 - 2026 Block, Inc.
 #
 # Use of this source code is governed by an MIT-style
 # license that can be found in the LICENSE file or at

data/lib/elastic_graph/indexer/processor.rb

@@ -1,4 +1,4 @@
-# Copyright 2024 - 2025 Block, Inc.
+# Copyright 2024 - 2026 Block, Inc.
 #
 # Use of this source code is governed by an MIT-style
 # license that can be found in the LICENSE file or at

data/lib/elastic_graph/indexer/record_preparer.rb

@@ -1,4 +1,4 @@
-# Copyright 2024 - 2025 Block, Inc.
+# Copyright 2024 - 2026 Block, Inc.
 #
 # Use of this source code is governed by an MIT-style
 # license that can be found in the LICENSE file or at

data/lib/elastic_graph/indexer/spec_support/event_matcher.rb

@@ -1,4 +1,4 @@
-# Copyright 2024 - 2025 Block, Inc.
+# Copyright 2024 - 2026 Block, Inc.
 #
 # Use of this source code is governed by an MIT-style
 # license that can be found in the LICENSE file or at

data/lib/elastic_graph/indexer/test_support/converters.rb

@@ -1,4 +1,4 @@
-# Copyright 2024 - 2025 Block, Inc.
+# Copyright 2024 - 2026 Block, Inc.
 #
 # Use of this source code is governed by an MIT-style
 # license that can be found in the LICENSE file or at

data/lib/elastic_graph/indexer.rb

@@ -1,4 +1,4 @@
-# Copyright 2024 - 2025 Block, Inc.
+# Copyright 2024 - 2026 Block, Inc.
 #
 # Use of this source code is governed by an MIT-style
 # license that can be found in the LICENSE file or at
@@ -47,8 +47,6 @@ module ElasticGraph
         require "elastic_graph/indexer/datastore_indexing_router"
         DatastoreIndexingRouter.new(
           datastore_clients_by_name: datastore_core.clients_by_name,
-          mappings_by_index_def_name: schema_artifacts.index_mappings_by_index_def_name,
-          monotonic_clock: monotonic_clock,
           logger: datastore_core.logger
         )
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elasticgraph-indexer
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.1.0
 platform: ruby
 authors:
 - Myron Marston
@@ -17,42 +17,42 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: 1.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: 1.1.0
 - !ruby/object:Gem::Dependency
   name: elasticgraph-schema_artifacts
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: 1.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: 1.1.0
 - !ruby/object:Gem::Dependency
   name: elasticgraph-support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: 1.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: 1.1.0
 - !ruby/object:Gem::Dependency
   name: hashdiff
   requirement: !ruby/object:Gem::Requirement
@@ -79,56 +79,56 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: 1.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: 1.1.0
 - !ruby/object:Gem::Dependency
   name: elasticgraph-elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: 1.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: 1.1.0
 - !ruby/object:Gem::Dependency
   name: elasticgraph-opensearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: 1.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: 1.1.0
 - !ruby/object:Gem::Dependency
   name: elasticgraph-schema_definition
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: 1.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: 1.1.0
 email:
 - myron@squareup.com
 executables: []
@@ -144,6 +144,7 @@ files:
 - lib/elastic_graph/indexer/failed_event_error.rb
 - lib/elastic_graph/indexer/hash_differ.rb
 - lib/elastic_graph/indexer/indexing_failures_error.rb
+- lib/elastic_graph/indexer/indexing_preparers/date_time.rb
 - lib/elastic_graph/indexer/indexing_preparers/integer.rb
 - lib/elastic_graph/indexer/indexing_preparers/no_op.rb
 - lib/elastic_graph/indexer/indexing_preparers/untyped.rb
@@ -160,10 +161,10 @@ licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/block/elasticgraph/issues
-  changelog_uri: https://github.com/block/elasticgraph/releases/tag/v1.0.2
-  documentation_uri: https://block.github.io/elasticgraph/api-docs/v1.0.2/
+  changelog_uri: https://github.com/block/elasticgraph/releases/tag/v1.1.0
+  documentation_uri: https://block.github.io/elasticgraph/api-docs/v1.1.0/
   homepage_uri: https://block.github.io/elasticgraph/
-  source_code_uri: https://github.com/block/elasticgraph/tree/v1.0.2/elasticgraph-indexer
+  source_code_uri: https://github.com/block/elasticgraph/tree/v1.1.0/elasticgraph-indexer
   gem_category: core
 rdoc_options: []
 require_paths:
@@ -175,14 +176,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '3.4'
   - - "<"
     - !ruby/object:Gem::Version
-      version: '3.5'
+      version: '4.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.3
 specification_version: 4
 summary: Indexes ElasticGraph data into a datastore.
 test_files: []