elasticgraph-indexer 1.0.0 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +0 -3
  3. data/lib/elastic_graph/indexer/config.rb +40 -30
  4. data/lib/elastic_graph/indexer/operation/factory.rb +3 -3
  5. data/lib/elastic_graph/indexer.rb +1 -1
  6. metadata +18 -32
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 1c834d6b276695fadebbfe58e0f090fc437e26c1e9d9a5b3fdec733f653e3966
4
- data.tar.gz: b4b98c05d097e30612a5b2c8bc1e0619d4b6eb3584e509db6dd1961e6dc0b23d
3
+ metadata.gz: 50d15365cf8ef27c7ec418e662b15c2693d726811825ad81e1765bef3dec58d1
4
+ data.tar.gz: 61d3e54674ce415e5fd8ba21e850a15c80f43e4725fe29f99a961c03d8c61a38
5
5
  SHA512:
6
- metadata.gz: dc6eb14787a25fd7ce76af3f76963f40c371d20f7bd78a1947db662457dfdf79e16c0a6ef2b5694d0a1a66f59d6b963cf26ad88341b735d65bc484ecdea07c64
7
- data.tar.gz: d53e1a9f8304955b6ff6996e1485882fe33a4af5d705a8fb9f48e87d56558431fbd558d7d6f76ffb9d14ca587c4e45360c2d95476311ce733f0a2c7b66fab198
6
+ metadata.gz: fdcd4764e9070c5379e1f24d895cd760adc6611cd750e17dc6f3c7d441c1ccfd0cb72dea515ad2647b4aa85c583d7d785790071027250aee337eea085159d428
7
+ data.tar.gz: 2a8908cb7647715cb668f79d9429ff2c47315539f97f354697bef77eb6202a5beec5bf4b998f5188ab380d5892096236166931c2aa6b82cba8ff64fdaea10997
data/README.md CHANGED
@@ -14,9 +14,6 @@ graph LR;
14
14
  elasticgraph-datastore_core["elasticgraph-datastore_core"];
15
15
  elasticgraph-indexer --> elasticgraph-datastore_core;
16
16
  class elasticgraph-datastore_core otherEgGemStyle;
17
- elasticgraph-json_schema["elasticgraph-json_schema"];
18
- elasticgraph-indexer --> elasticgraph-json_schema;
19
- class elasticgraph-json_schema otherEgGemStyle;
20
17
  elasticgraph-schema_artifacts["elasticgraph-schema_artifacts"];
21
18
  elasticgraph-indexer --> elasticgraph-schema_artifacts;
22
19
  class elasticgraph-schema_artifacts otherEgGemStyle;
data/lib/elastic_graph/indexer/config.rb CHANGED
@@ -6,43 +6,53 @@
6
6
  #
7
7
  # frozen_string_literal: true
8
8
 
9
+ require "elastic_graph/support/config"
9
10
  require "elastic_graph/errors"
10
- require "elastic_graph/indexer/event_id"
11
11
 
12
12
  module ElasticGraph
13
13
  class Indexer
14
- class Config < ::Data.define(
15
- # Map of indexing latency thresholds (in milliseconds), keyed by the name of
16
- # the indexing latency metric. When an event is indexed with an indexing latency
17
- # exceeding the threshold, a warning with the event type, id, and version will
18
- # be logged, so the issue can be investigated.
19
- :latency_slo_thresholds_by_timestamp_in_ms,
20
- # Setting that can be used to specify some derived indexing type updates that should be skipped. This
21
- # setting should be a map keyed by the name of the derived indexing type, and the values should be sets
22
- # of ids. This can be useful when you have a "hot spot" of a single derived document that is
23
- # receiving a ton of updates. During a backfill (or whatever) you may want to skip the derived
24
- # type updates.
25
- :skip_derived_indexing_type_updates
26
- )
27
- def self.from_parsed_yaml(hash)
28
- hash = hash.fetch("indexer")
29
- extra_keys = hash.keys - EXPECTED_KEYS
14
+ class Config < Support::Config.define(:latency_slo_thresholds_by_timestamp_in_ms, :skip_derived_indexing_type_updates)
15
+ json_schema at: "indexer",
16
+ optional: false,
17
+ description: "Configuration for indexing operations and metrics used by `elasticgraph-indexer`.",
18
+ properties: {
19
+ latency_slo_thresholds_by_timestamp_in_ms: {
20
+ description: "Map of indexing latency thresholds (in milliseconds), keyed by the name of " \
21
+ "the indexing latency metric. When an event is indexed with an indexing latency " \
22
+ "exceeding the threshold, a warning with the event type, id, and version will " \
23
+ "be logged, so the issue can be investigated.",
24
+ type: "object",
25
+ patternProperties: {/.+/.source => {type: "integer", minimum: 0}},
26
+ default: {}, # : untyped
27
+ examples: [
28
+ {}, # : untyped
29
+ {"ingested_from_topic_at" => 10000, "entity_updated_at" => 15000}
30
+ ]
31
+ },
32
+ skip_derived_indexing_type_updates: {
33
+ description: "Setting that can be used to specify some derived indexing type updates that should be skipped. This " \
34
+ "setting should be a map keyed by the name of the derived indexing type, and the values should be sets " \
35
+ 'of ids. This can be useful when you have a "hot spot" of a single derived document that is ' \
36
+ "receiving a ton of updates. During a backfill (or whatever) you may want to skip the derived " \
37
+ "type updates.",
38
+ type: "object",
39
+ patternProperties: {/^[A-Z]\w*$/.source => {type: "array", items: {type: "string", minLength: 1}}},
40
+ default: {}, # : untyped
41
+ examples: [
42
+ {}, # : untyped
43
+ {"WidgetWorkspace" => ["ABC12345678"]}
44
+ ]
45
+ }
46
+ }
30
47
 
31
- unless extra_keys.empty?
32
- raise Errors::ConfigError, "Unknown `indexer` config settings: #{extra_keys.join(", ")}"
33
- end
48
+ private
34
49
 
35
- new(
36
- latency_slo_thresholds_by_timestamp_in_ms: hash.fetch("latency_slo_thresholds_by_timestamp_in_ms"),
37
- skip_derived_indexing_type_updates: (hash["skip_derived_indexing_type_updates"] || {}).transform_values(&:to_set)
38
- )
50
+ def convert_values(skip_derived_indexing_type_updates:, latency_slo_thresholds_by_timestamp_in_ms:)
51
+ {
52
+ skip_derived_indexing_type_updates: skip_derived_indexing_type_updates.transform_values(&:to_set),
53
+ latency_slo_thresholds_by_timestamp_in_ms: latency_slo_thresholds_by_timestamp_in_ms
54
+ }
39
55
  end
40
-
41
- EXPECTED_KEYS = members.map(&:to_s)
42
56
  end
43
-
44
- # Steep weirdly expects them here...
45
- # @dynamic initialize, config, datastore_core, schema_artifacts, datastore_router
46
- # @dynamic record_preparer, processor, operation_factory
47
57
  end
48
58
  end
data/lib/elastic_graph/indexer/operation/factory.rb CHANGED
@@ -11,7 +11,7 @@ require "elastic_graph/indexer/event_id"
11
11
  require "elastic_graph/indexer/failed_event_error"
12
12
  require "elastic_graph/indexer/operation/update"
13
13
  require "elastic_graph/indexer/record_preparer"
14
- require "elastic_graph/json_schema/validator_factory"
14
+ require "elastic_graph/support/json_schema/validator_factory"
15
15
  require "elastic_graph/support/memoizable_data"
16
16
 
17
17
  module ElasticGraph
@@ -95,13 +95,13 @@ module ElasticGraph
95
95
  end
96
96
 
97
97
  def validator(type, selected_json_schema_version)
98
- factory = validator_factories_by_version[selected_json_schema_version] # : JSONSchema::ValidatorFactory
98
+ factory = validator_factories_by_version[selected_json_schema_version] # : Support::JSONSchema::ValidatorFactory
99
99
  factory.validator_for(type)
100
100
  end
101
101
 
102
102
  def validator_factories_by_version
103
103
  @validator_factories_by_version ||= ::Hash.new do |hash, json_schema_version|
104
- factory = JSONSchema::ValidatorFactory.new(
104
+ factory = Support::JSONSchema::ValidatorFactory.new(
105
105
  schema: schema_artifacts.json_schemas_for(json_schema_version),
106
106
  sanitize_pii: true
107
107
  )
data/lib/elastic_graph/indexer.rb CHANGED
@@ -21,7 +21,7 @@ module ElasticGraph
21
21
  # `from_yaml_file(file_name, &block)` is also available (via `Support::FromYamlFile`).
22
22
  def self.from_parsed_yaml(parsed_yaml, &datastore_client_customization_block)
23
23
  new(
24
- config: Indexer::Config.from_parsed_yaml(parsed_yaml),
24
+ config: Indexer::Config.from_parsed_yaml(parsed_yaml) || Indexer::Config.new,
25
25
  datastore_core: DatastoreCore.from_parsed_yaml(parsed_yaml, &datastore_client_customization_block)
26
26
  )
27
27
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: elasticgraph-indexer
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0
4
+ version: 1.0.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Myron Marston
@@ -17,56 +17,42 @@ dependencies:
17
17
  requirements:
18
18
  - - '='
19
19
  - !ruby/object:Gem::Version
20
- version: 1.0.0
20
+ version: 1.0.1
21
21
  type: :runtime
22
22
  prerelease: false
23
23
  version_requirements: !ruby/object:Gem::Requirement
24
24
  requirements:
25
25
  - - '='
26
26
  - !ruby/object:Gem::Version
27
- version: 1.0.0
28
- - !ruby/object:Gem::Dependency
29
- name: elasticgraph-json_schema
30
- requirement: !ruby/object:Gem::Requirement
31
- requirements:
32
- - - '='
33
- - !ruby/object:Gem::Version
34
- version: 1.0.0
35
- type: :runtime
36
- prerelease: false
37
- version_requirements: !ruby/object:Gem::Requirement
38
- requirements:
39
- - - '='
40
- - !ruby/object:Gem::Version
41
- version: 1.0.0
27
+ version: 1.0.1
42
28
  - !ruby/object:Gem::Dependency
43
29
  name: elasticgraph-schema_artifacts
44
30
  requirement: !ruby/object:Gem::Requirement
45
31
  requirements:
46
32
  - - '='
47
33
  - !ruby/object:Gem::Version
48
- version: 1.0.0
34
+ version: 1.0.1
49
35
  type: :runtime
50
36
  prerelease: false
51
37
  version_requirements: !ruby/object:Gem::Requirement
52
38
  requirements:
53
39
  - - '='
54
40
  - !ruby/object:Gem::Version
55
- version: 1.0.0
41
+ version: 1.0.1
56
42
  - !ruby/object:Gem::Dependency
57
43
  name: elasticgraph-support
58
44
  requirement: !ruby/object:Gem::Requirement
59
45
  requirements:
60
46
  - - '='
61
47
  - !ruby/object:Gem::Version
62
- version: 1.0.0
48
+ version: 1.0.1
63
49
  type: :runtime
64
50
  prerelease: false
65
51
  version_requirements: !ruby/object:Gem::Requirement
66
52
  requirements:
67
53
  - - '='
68
54
  - !ruby/object:Gem::Version
69
- version: 1.0.0
55
+ version: 1.0.1
70
56
  - !ruby/object:Gem::Dependency
71
57
  name: hashdiff
72
58
  requirement: !ruby/object:Gem::Requirement
@@ -87,56 +73,56 @@ dependencies:
87
73
  requirements:
88
74
  - - '='
89
75
  - !ruby/object:Gem::Version
90
- version: 1.0.0
76
+ version: 1.0.1
91
77
  type: :development
92
78
  prerelease: false
93
79
  version_requirements: !ruby/object:Gem::Requirement
94
80
  requirements:
95
81
  - - '='
96
82
  - !ruby/object:Gem::Version
97
- version: 1.0.0
83
+ version: 1.0.1
98
84
  - !ruby/object:Gem::Dependency
99
85
  name: elasticgraph-elasticsearch
100
86
  requirement: !ruby/object:Gem::Requirement
101
87
  requirements:
102
88
  - - '='
103
89
  - !ruby/object:Gem::Version
104
- version: 1.0.0
90
+ version: 1.0.1
105
91
  type: :development
106
92
  prerelease: false
107
93
  version_requirements: !ruby/object:Gem::Requirement
108
94
  requirements:
109
95
  - - '='
110
96
  - !ruby/object:Gem::Version
111
- version: 1.0.0
97
+ version: 1.0.1
112
98
  - !ruby/object:Gem::Dependency
113
99
  name: elasticgraph-opensearch
114
100
  requirement: !ruby/object:Gem::Requirement
115
101
  requirements:
116
102
  - - '='
117
103
  - !ruby/object:Gem::Version
118
- version: 1.0.0
104
+ version: 1.0.1
119
105
  type: :development
120
106
  prerelease: false
121
107
  version_requirements: !ruby/object:Gem::Requirement
122
108
  requirements:
123
109
  - - '='
124
110
  - !ruby/object:Gem::Version
125
- version: 1.0.0
111
+ version: 1.0.1
126
112
  - !ruby/object:Gem::Dependency
127
113
  name: elasticgraph-schema_definition
128
114
  requirement: !ruby/object:Gem::Requirement
129
115
  requirements:
130
116
  - - '='
131
117
  - !ruby/object:Gem::Version
132
- version: 1.0.0
118
+ version: 1.0.1
133
119
  type: :development
134
120
  prerelease: false
135
121
  version_requirements: !ruby/object:Gem::Requirement
136
122
  requirements:
137
123
  - - '='
138
124
  - !ruby/object:Gem::Version
139
- version: 1.0.0
125
+ version: 1.0.1
140
126
  email:
141
127
  - myron@squareup.com
142
128
  executables: []
@@ -168,10 +154,10 @@ licenses:
168
154
  - MIT
169
155
  metadata:
170
156
  bug_tracker_uri: https://github.com/block/elasticgraph/issues
171
- changelog_uri: https://github.com/block/elasticgraph/releases/tag/v1.0.0
172
- documentation_uri: https://block.github.io/elasticgraph/api-docs/v1.0.0/
157
+ changelog_uri: https://github.com/block/elasticgraph/releases/tag/v1.0.1
158
+ documentation_uri: https://block.github.io/elasticgraph/api-docs/v1.0.1/
173
159
  homepage_uri: https://block.github.io/elasticgraph/
174
- source_code_uri: https://github.com/block/elasticgraph/tree/v1.0.0/elasticgraph-indexer
160
+ source_code_uri: https://github.com/block/elasticgraph/tree/v1.0.1/elasticgraph-indexer
175
161
  gem_category: core
176
162
  rdoc_options: []
177
163
  require_paths: