elasticgraph-indexer 1.0.0 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1c834d6b276695fadebbfe58e0f090fc437e26c1e9d9a5b3fdec733f653e3966
-  data.tar.gz: b4b98c05d097e30612a5b2c8bc1e0619d4b6eb3584e509db6dd1961e6dc0b23d
+  metadata.gz: 96459aae32411712ac8a1fe13289cf2e411a4ac8c7a38d2b868712380149fcf1
+  data.tar.gz: d73d0c3420d55399b6e4111606ba8ba154a19067efb15eb0b006f30e86f664af
 SHA512:
-  metadata.gz: dc6eb14787a25fd7ce76af3f76963f40c371d20f7bd78a1947db662457dfdf79e16c0a6ef2b5694d0a1a66f59d6b963cf26ad88341b735d65bc484ecdea07c64
-  data.tar.gz: d53e1a9f8304955b6ff6996e1485882fe33a4af5d705a8fb9f48e87d56558431fbd558d7d6f76ffb9d14ca587c4e45360c2d95476311ce733f0a2c7b66fab198
+  metadata.gz: ab47eea78b85a3e0e6252eed24cc775402be7527778a8cb15bde35c0b76451af057baab6b37b43c02957ee13138ff70e11078625bcb48834a050c3c1045cddea
+  data.tar.gz: c47921bbfce1a736c714ad5f696c6692460b59c545a15af23a575a85c192e41ad27ef671fe7e4bb6309a2328f278b5a28a7b96dc781f13ef96adaa9f8c554e7e

data/README.md

@@ -14,9 +14,6 @@ graph LR;
     elasticgraph-datastore_core["elasticgraph-datastore_core"];
     elasticgraph-indexer --> elasticgraph-datastore_core;
     class elasticgraph-datastore_core otherEgGemStyle;
-    elasticgraph-json_schema["elasticgraph-json_schema"];
-    elasticgraph-indexer --> elasticgraph-json_schema;
-    class elasticgraph-json_schema otherEgGemStyle;
     elasticgraph-schema_artifacts["elasticgraph-schema_artifacts"];
     elasticgraph-indexer --> elasticgraph-schema_artifacts;
     class elasticgraph-schema_artifacts otherEgGemStyle;

data/lib/elastic_graph/indexer/config.rb

@@ -6,43 +6,53 @@
 #
 # frozen_string_literal: true
 
+require "elastic_graph/support/config"
 require "elastic_graph/errors"
-require "elastic_graph/indexer/event_id"
 
 module ElasticGraph
   class Indexer
-    class Config < ::Data.define(
-      # Map of indexing latency thresholds (in milliseconds), keyed by the name of
-      # the indexing latency metric. When an event is indexed with an indexing latency
-      # exceeding the threshold, a warning with the event type, id, and version will
-      # be logged, so the issue can be investigated.
-      :latency_slo_thresholds_by_timestamp_in_ms,
-      # Setting that can be used to specify some derived indexing type updates that should be skipped. This
-      # setting should be a map keyed by the name of the derived indexing type, and the values should be sets
-      # of ids. This can be useful when you have a "hot spot" of a single derived document that is
-      # receiving a ton of updates. During a backfill (or whatever) you may want to skip the derived
-      # type updates.
-      :skip_derived_indexing_type_updates
-    )
-      def self.from_parsed_yaml(hash)
-        hash = hash.fetch("indexer")
-        extra_keys = hash.keys - EXPECTED_KEYS
+    class Config < Support::Config.define(:latency_slo_thresholds_by_timestamp_in_ms, :skip_derived_indexing_type_updates)
+      json_schema at: "indexer",
+        optional: false,
+        description: "Configuration for indexing operations and metrics used by `elasticgraph-indexer`.",
+        properties: {
+          latency_slo_thresholds_by_timestamp_in_ms: {
+            description: "Map of indexing latency thresholds (in milliseconds), keyed by the name of " \
+              "the indexing latency metric. When an event is indexed with an indexing latency " \
+              "exceeding the threshold, a warning with the event type, id, and version will " \
+              "be logged, so the issue can be investigated.",
+            type: "object",
+            patternProperties: {/.+/.source => {type: "integer", minimum: 0}},
+            default: {}, # : untyped
+            examples: [
+              {}, # : untyped
+              {"ingested_from_topic_at" => 10000, "entity_updated_at" => 15000}
+            ]
+          },
+          skip_derived_indexing_type_updates: {
+            description: "Setting that can be used to specify some derived indexing type updates that should be skipped. This " \
+              "setting should be a map keyed by the name of the derived indexing type, and the values should be sets " \
+              'of ids. This can be useful when you have a "hot spot" of a single derived document that is ' \
+              "receiving a ton of updates. During a backfill (or whatever) you may want to skip the derived " \
+              "type updates.",
+            type: "object",
+            patternProperties: {/^[A-Z]\w*$/.source => {type: "array", items: {type: "string", minLength: 1}}},
+            default: {}, # : untyped
+            examples: [
+              {}, # : untyped
+              {"WidgetWorkspace" => ["ABC12345678"]}
+            ]
+          }
+        }
 
-        unless extra_keys.empty?
-          raise Errors::ConfigError, "Unknown `indexer` config settings: #{extra_keys.join(", ")}"
-        end
+      private
 
-        new(
-          latency_slo_thresholds_by_timestamp_in_ms: hash.fetch("latency_slo_thresholds_by_timestamp_in_ms"),
-          skip_derived_indexing_type_updates: (hash["skip_derived_indexing_type_updates"] || {}).transform_values(&:to_set)
-        )
+      def convert_values(skip_derived_indexing_type_updates:, latency_slo_thresholds_by_timestamp_in_ms:)
+        {
+          skip_derived_indexing_type_updates: skip_derived_indexing_type_updates.transform_values(&:to_set),
+          latency_slo_thresholds_by_timestamp_in_ms: latency_slo_thresholds_by_timestamp_in_ms
+        }
       end
-
-      EXPECTED_KEYS = members.map(&:to_s)
     end
-
-    # Steep weirdly expects them here...
-    # @dynamic initialize, config, datastore_core, schema_artifacts, datastore_router
-    # @dynamic record_preparer, processor, operation_factory
   end
 end

data/lib/elastic_graph/indexer/operation/factory.rb

@@ -11,7 +11,7 @@ require "elastic_graph/indexer/event_id"
 require "elastic_graph/indexer/failed_event_error"
 require "elastic_graph/indexer/operation/update"
 require "elastic_graph/indexer/record_preparer"
-require "elastic_graph/json_schema/validator_factory"
+require "elastic_graph/support/json_schema/validator_factory"
 require "elastic_graph/support/memoizable_data"
 
 module ElasticGraph
@@ -70,7 +70,7 @@ module ElasticGraph
           # behavior is in the event of a tie (highly unlikely, there shouldn't be a gap in available json schema versions), the higher version
           # should be selected. So to get that behavior, the list is sorted in descending order.
           #
-          selected_json_schema_version = available_json_schema_versions.sort.reverse.min_by { |it| (requested_json_schema_version - it).abs }
+          selected_json_schema_version = available_json_schema_versions.sort.reverse.min_by { |version| (requested_json_schema_version - version).abs }
 
           if selected_json_schema_version != requested_json_schema_version
             logger.info({
@@ -95,13 +95,13 @@ module ElasticGraph
         end
 
         def validator(type, selected_json_schema_version)
-          factory = validator_factories_by_version[selected_json_schema_version] # : JSONSchema::ValidatorFactory
+          factory = validator_factories_by_version[selected_json_schema_version] # : Support::JSONSchema::ValidatorFactory
           factory.validator_for(type)
         end
 
         def validator_factories_by_version
           @validator_factories_by_version ||= ::Hash.new do |hash, json_schema_version|
-            factory = JSONSchema::ValidatorFactory.new(
+            factory = Support::JSONSchema::ValidatorFactory.new(
               schema: schema_artifacts.json_schemas_for(json_schema_version),
               sanitize_pii: true
             )

data/lib/elastic_graph/indexer.rb

@@ -21,7 +21,7 @@ module ElasticGraph
     # `from_yaml_file(file_name, &block)` is also available (via `Support::FromYamlFile`).
     def self.from_parsed_yaml(parsed_yaml, &datastore_client_customization_block)
       new(
-        config: Indexer::Config.from_parsed_yaml(parsed_yaml),
+        config: Indexer::Config.from_parsed_yaml(parsed_yaml) || Indexer::Config.new,
         datastore_core: DatastoreCore.from_parsed_yaml(parsed_yaml, &datastore_client_customization_block)
       )
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elasticgraph-indexer
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.2
 platform: ruby
 authors:
 - Myron Marston
@@ -17,56 +17,42 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
-- !ruby/object:Gem::Dependency
-  name: elasticgraph-json_schema
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 1.0.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.2
 - !ruby/object:Gem::Dependency
   name: elasticgraph-schema_artifacts
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.2
 - !ruby/object:Gem::Dependency
   name: elasticgraph-support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.2
 - !ruby/object:Gem::Dependency
   name: hashdiff
   requirement: !ruby/object:Gem::Requirement
@@ -74,6 +60,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -81,62 +70,65 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.1
 - !ruby/object:Gem::Dependency
   name: elasticgraph-admin
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.2
 - !ruby/object:Gem::Dependency
   name: elasticgraph-elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.2
 - !ruby/object:Gem::Dependency
   name: elasticgraph-opensearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.2
 - !ruby/object:Gem::Dependency
   name: elasticgraph-schema_definition
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.2
 email:
 - myron@squareup.com
 executables: []
@@ -168,10 +160,10 @@ licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/block/elasticgraph/issues
-  changelog_uri: https://github.com/block/elasticgraph/releases/tag/v1.0.0
-  documentation_uri: https://block.github.io/elasticgraph/api-docs/v1.0.0/
+  changelog_uri: https://github.com/block/elasticgraph/releases/tag/v1.0.2
+  documentation_uri: https://block.github.io/elasticgraph/api-docs/v1.0.2/
   homepage_uri: https://block.github.io/elasticgraph/
-  source_code_uri: https://github.com/block/elasticgraph/tree/v1.0.0/elasticgraph-indexer
+  source_code_uri: https://github.com/block/elasticgraph/tree/v1.0.2/elasticgraph-indexer
   gem_category: core
 rdoc_options: []
 require_paths: