elasticgraph-indexer 0.19.1.1 → 0.19.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE.txt +1 -1
- data/lib/elastic_graph/indexer/config.rb +1 -1
- data/lib/elastic_graph/indexer/datastore_indexing_router.rb +21 -8
- data/lib/elastic_graph/indexer/event_id.rb +1 -1
- data/lib/elastic_graph/indexer/failed_event_error.rb +1 -1
- data/lib/elastic_graph/indexer/hash_differ.rb +1 -1
- data/lib/elastic_graph/indexer/indexing_failures_error.rb +1 -1
- data/lib/elastic_graph/indexer/indexing_preparers/integer.rb +1 -1
- data/lib/elastic_graph/indexer/indexing_preparers/no_op.rb +1 -1
- data/lib/elastic_graph/indexer/indexing_preparers/untyped.rb +1 -1
- data/lib/elastic_graph/indexer/operation/count_accumulator.rb +1 -1
- data/lib/elastic_graph/indexer/operation/factory.rb +1 -1
- data/lib/elastic_graph/indexer/operation/result.rb +1 -1
- data/lib/elastic_graph/indexer/operation/update.rb +2 -2
- data/lib/elastic_graph/indexer/operation/upsert.rb +1 -1
- data/lib/elastic_graph/indexer/processor.rb +1 -1
- data/lib/elastic_graph/indexer/record_preparer.rb +2 -2
- data/lib/elastic_graph/indexer/spec_support/event_matcher.rb +1 -1
- data/lib/elastic_graph/indexer/test_support/converters.rb +1 -1
- data/lib/elastic_graph/indexer.rb +1 -1
- metadata +28 -26
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: eeaf2e7261d563815bb70e4fdbc73ad42f2abe03e94c212f32811a4a7a37ce0b
|
|
4
|
+
data.tar.gz: c76f9406c811e55b736b3ff211eb271d3f709e8be1b209cf0531192670868410
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4f60f5c8688adaf0b53c9e96bc933de69ac5e9b2eaf278367b15f3c109318cc83a13636323e5757a920171883cbde8c485de594f2cced44d280654988a882545
|
|
7
|
+
data.tar.gz: a5fd9fc942442cfc66ba6d4c6e92fb9ddbfce00554b3e449f12832a8f8acd1c3c9bb5f5c0b5ccafa4d545f85763440adab099f44585c7614dd28a6f5d9392594
|
data/LICENSE.txt
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2024 Block, Inc.
|
|
1
|
+
# Copyright 2024 - 2025 Block, Inc.
|
|
2
2
|
#
|
|
3
3
|
# Use of this source code is governed by an MIT-style
|
|
4
4
|
# license that can be found in the LICENSE file or at
|
|
@@ -186,15 +186,18 @@ module ElasticGraph
|
|
|
186
186
|
# This nested structure is necessary because a single operation can target more than one datastore
|
|
187
187
|
# cluster, and a document may have different source event versions in different datastore clusters.
|
|
188
188
|
def source_event_versions_in_index(operations)
|
|
189
|
-
ops_by_client_name =
|
|
189
|
+
ops_by_client_name = ::Hash.new { |h, k| h[k] = [] } # : ::Hash[::String, ::Array[_Operation]]
|
|
190
|
+
operations.each do |op|
|
|
190
191
|
# Note: this intentionally does not use `accessible_cluster_names_to_index_into`.
|
|
191
192
|
# We want to fail with clear error if any clusters are inaccessible instead of silently ignoring
|
|
192
193
|
# the named cluster. The `IndexingFailuresError` provides a clear error.
|
|
193
194
|
cluster_names = op.destination_index_def.clusters_to_index_into
|
|
194
|
-
cluster_names.each { |cluster_name|
|
|
195
|
+
cluster_names.each { |cluster_name| ops_by_client_name[cluster_name] << op }
|
|
195
196
|
end
|
|
196
197
|
|
|
197
198
|
client_names_and_results = Support::Threading.parallel_map(ops_by_client_name) do |(client_name, all_ops)|
|
|
199
|
+
# @type block: [::String, ::Symbol, ::Array[untyped] | ::Hash[_Operation, ::Array[::Integer]]]
|
|
200
|
+
|
|
198
201
|
ops, unversioned_ops = all_ops.partition(&:versioned?)
|
|
199
202
|
|
|
200
203
|
msearch_response =
|
|
@@ -203,6 +206,7 @@ module ElasticGraph
|
|
|
203
206
|
# We only care about the source versions, but the way we get it varies.
|
|
204
207
|
include_version =
|
|
205
208
|
if op.destination_index_def.use_updates_for_indexing?
|
|
209
|
+
# @type var op: Operation::Update
|
|
206
210
|
{_source: {includes: [
|
|
207
211
|
"__versions.#{op.update_target.relationship}",
|
|
208
212
|
# The update_data script before ElasticGraph v0.8 used __sourceVersions[type] instead of __versions[relationship].
|
|
@@ -239,7 +243,10 @@ module ElasticGraph
|
|
|
239
243
|
errors = msearch_response.fetch("responses").select { |res| res["error"] }
|
|
240
244
|
|
|
241
245
|
if errors.empty?
|
|
242
|
-
|
|
246
|
+
# We assume the size of the ops and the other array is the same and it cannot have `nil`.
|
|
247
|
+
zip = ops.zip(msearch_response.fetch("responses")) # : ::Array[[_Operation, ::Hash[::String, ::Hash[::String, untyped]]]]
|
|
248
|
+
|
|
249
|
+
versions_by_op = zip.to_h do |(op, response)|
|
|
243
250
|
hits = response.fetch("hits").fetch("hits")
|
|
244
251
|
|
|
245
252
|
if hits.size > 1
|
|
@@ -254,6 +261,7 @@ module ElasticGraph
|
|
|
254
261
|
end
|
|
255
262
|
|
|
256
263
|
if op.destination_index_def.use_updates_for_indexing?
|
|
264
|
+
# @type var op: Operation::Update
|
|
257
265
|
versions = hits.filter_map do |hit|
|
|
258
266
|
hit.dig("_source", "__versions", op.update_target.relationship, hit.fetch("_id")) ||
|
|
259
267
|
# The update_data script before ElasticGraph v0.8 used __sourceVersions[type] instead of __versions[relationship].
|
|
@@ -290,7 +298,11 @@ module ElasticGraph
|
|
|
290
298
|
end
|
|
291
299
|
|
|
292
300
|
if failures.empty?
|
|
293
|
-
|
|
301
|
+
# All results are success and the third element of the tuple is a hash.
|
|
302
|
+
# Assign the results to narrow down the type.
|
|
303
|
+
success_results = client_names_and_results # : ::Array[[::String, ::Symbol, ::Hash[_Operation, ::Array[::Integer]]]]
|
|
304
|
+
|
|
305
|
+
success_results.each_with_object(_ = {}) do |(client_name, _success_or_failure, results), accum|
|
|
294
306
|
results.each do |op, version|
|
|
295
307
|
(accum[op] ||= {})[client_name] = version
|
|
296
308
|
end
|
|
@@ -323,9 +335,10 @@ module ElasticGraph
|
|
|
323
335
|
# need to worry about it mutating during the lifetime of a single process (particularly given the expense of doing
|
|
324
336
|
# so).
|
|
325
337
|
def validate_mapping_completeness_of!(index_cluster_name_method, *index_definitions)
|
|
326
|
-
diffs_by_cluster_and_index_name =
|
|
327
|
-
|
|
328
|
-
|
|
338
|
+
diffs_by_cluster_and_index_name =
|
|
339
|
+
index_definitions.reduce({}) do |accum, index_def| # $ ::Hash[[::String, ::String], ::String]
|
|
340
|
+
accum.merge(mapping_diffs_for(index_def, index_cluster_name_method))
|
|
341
|
+
end
|
|
329
342
|
|
|
330
343
|
if diffs_by_cluster_and_index_name.any?
|
|
331
344
|
formatted_diffs = diffs_by_cluster_and_index_name.map do |(cluster_name, index_name), diff|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2024 Block, Inc.
|
|
1
|
+
# Copyright 2024 - 2025 Block, Inc.
|
|
2
2
|
#
|
|
3
3
|
# Use of this source code is governed by an MIT-style
|
|
4
4
|
# license that can be found in the LICENSE file or at
|
|
@@ -32,7 +32,7 @@ module ElasticGraph
|
|
|
32
32
|
prepared_record = record_preparer.prepare_for_index(event["type"], event["record"] || {"id" => event["id"]})
|
|
33
33
|
|
|
34
34
|
Support::HashUtil
|
|
35
|
-
.fetch_leaf_values_at_path(prepared_record, update_target.id_source)
|
|
35
|
+
.fetch_leaf_values_at_path(prepared_record, update_target.id_source.split("."))
|
|
36
36
|
.reject { |id| id.to_s.strip.empty? }
|
|
37
37
|
.uniq
|
|
38
38
|
.map { |doc_id| new(event, prepared_record, destination_index_def, update_target, doc_id, destination_index_mapping) }
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2024 Block, Inc.
|
|
1
|
+
# Copyright 2024 - 2025 Block, Inc.
|
|
2
2
|
#
|
|
3
3
|
# Use of this source code is governed by an MIT-style
|
|
4
4
|
# license that can be found in the LICENSE file or at
|
|
@@ -19,7 +19,7 @@ module ElasticGraph
|
|
|
19
19
|
scalar_types_by_name = schema_artifacts.runtime_metadata.scalar_types_by_name
|
|
20
20
|
indexing_preparer_by_scalar_type_name = ::Hash.new do |hash, type_name|
|
|
21
21
|
hash[type_name] = scalar_types_by_name[type_name]&.load_indexing_preparer&.extension_class
|
|
22
|
-
end
|
|
22
|
+
end # : ::Hash[::String, SchemaArtifacts::RuntimeMetadata::extensionClass?]
|
|
23
23
|
|
|
24
24
|
@preparers_by_json_schema_version = ::Hash.new do |hash, version|
|
|
25
25
|
hash[version] = RecordPreparer.new(
|
metadata
CHANGED
|
@@ -1,16 +1,15 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: elasticgraph-indexer
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.19.
|
|
4
|
+
version: 0.19.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Myron Marston
|
|
8
8
|
- Ben VandenBos
|
|
9
9
|
- Block Engineering
|
|
10
|
-
autorequire:
|
|
11
10
|
bindir: bin
|
|
12
11
|
cert_chain: []
|
|
13
|
-
date: 2025-
|
|
12
|
+
date: 2025-04-09 00:00:00.000000000 Z
|
|
14
13
|
dependencies:
|
|
15
14
|
- !ruby/object:Gem::Dependency
|
|
16
15
|
name: elasticgraph-datastore_core
|
|
@@ -18,56 +17,56 @@ dependencies:
|
|
|
18
17
|
requirements:
|
|
19
18
|
- - '='
|
|
20
19
|
- !ruby/object:Gem::Version
|
|
21
|
-
version: 0.19.
|
|
20
|
+
version: 0.19.2.0
|
|
22
21
|
type: :runtime
|
|
23
22
|
prerelease: false
|
|
24
23
|
version_requirements: !ruby/object:Gem::Requirement
|
|
25
24
|
requirements:
|
|
26
25
|
- - '='
|
|
27
26
|
- !ruby/object:Gem::Version
|
|
28
|
-
version: 0.19.
|
|
27
|
+
version: 0.19.2.0
|
|
29
28
|
- !ruby/object:Gem::Dependency
|
|
30
29
|
name: elasticgraph-json_schema
|
|
31
30
|
requirement: !ruby/object:Gem::Requirement
|
|
32
31
|
requirements:
|
|
33
32
|
- - '='
|
|
34
33
|
- !ruby/object:Gem::Version
|
|
35
|
-
version: 0.19.
|
|
34
|
+
version: 0.19.2.0
|
|
36
35
|
type: :runtime
|
|
37
36
|
prerelease: false
|
|
38
37
|
version_requirements: !ruby/object:Gem::Requirement
|
|
39
38
|
requirements:
|
|
40
39
|
- - '='
|
|
41
40
|
- !ruby/object:Gem::Version
|
|
42
|
-
version: 0.19.
|
|
41
|
+
version: 0.19.2.0
|
|
43
42
|
- !ruby/object:Gem::Dependency
|
|
44
43
|
name: elasticgraph-schema_artifacts
|
|
45
44
|
requirement: !ruby/object:Gem::Requirement
|
|
46
45
|
requirements:
|
|
47
46
|
- - '='
|
|
48
47
|
- !ruby/object:Gem::Version
|
|
49
|
-
version: 0.19.
|
|
48
|
+
version: 0.19.2.0
|
|
50
49
|
type: :runtime
|
|
51
50
|
prerelease: false
|
|
52
51
|
version_requirements: !ruby/object:Gem::Requirement
|
|
53
52
|
requirements:
|
|
54
53
|
- - '='
|
|
55
54
|
- !ruby/object:Gem::Version
|
|
56
|
-
version: 0.19.
|
|
55
|
+
version: 0.19.2.0
|
|
57
56
|
- !ruby/object:Gem::Dependency
|
|
58
57
|
name: elasticgraph-support
|
|
59
58
|
requirement: !ruby/object:Gem::Requirement
|
|
60
59
|
requirements:
|
|
61
60
|
- - '='
|
|
62
61
|
- !ruby/object:Gem::Version
|
|
63
|
-
version: 0.19.
|
|
62
|
+
version: 0.19.2.0
|
|
64
63
|
type: :runtime
|
|
65
64
|
prerelease: false
|
|
66
65
|
version_requirements: !ruby/object:Gem::Requirement
|
|
67
66
|
requirements:
|
|
68
67
|
- - '='
|
|
69
68
|
- !ruby/object:Gem::Version
|
|
70
|
-
version: 0.19.
|
|
69
|
+
version: 0.19.2.0
|
|
71
70
|
- !ruby/object:Gem::Dependency
|
|
72
71
|
name: hashdiff
|
|
73
72
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -75,6 +74,9 @@ dependencies:
|
|
|
75
74
|
- - "~>"
|
|
76
75
|
- !ruby/object:Gem::Version
|
|
77
76
|
version: '1.1'
|
|
77
|
+
- - ">="
|
|
78
|
+
- !ruby/object:Gem::Version
|
|
79
|
+
version: 1.1.2
|
|
78
80
|
type: :runtime
|
|
79
81
|
prerelease: false
|
|
80
82
|
version_requirements: !ruby/object:Gem::Requirement
|
|
@@ -82,63 +84,65 @@ dependencies:
|
|
|
82
84
|
- - "~>"
|
|
83
85
|
- !ruby/object:Gem::Version
|
|
84
86
|
version: '1.1'
|
|
87
|
+
- - ">="
|
|
88
|
+
- !ruby/object:Gem::Version
|
|
89
|
+
version: 1.1.2
|
|
85
90
|
- !ruby/object:Gem::Dependency
|
|
86
91
|
name: elasticgraph-admin
|
|
87
92
|
requirement: !ruby/object:Gem::Requirement
|
|
88
93
|
requirements:
|
|
89
94
|
- - '='
|
|
90
95
|
- !ruby/object:Gem::Version
|
|
91
|
-
version: 0.19.
|
|
96
|
+
version: 0.19.2.0
|
|
92
97
|
type: :development
|
|
93
98
|
prerelease: false
|
|
94
99
|
version_requirements: !ruby/object:Gem::Requirement
|
|
95
100
|
requirements:
|
|
96
101
|
- - '='
|
|
97
102
|
- !ruby/object:Gem::Version
|
|
98
|
-
version: 0.19.
|
|
103
|
+
version: 0.19.2.0
|
|
99
104
|
- !ruby/object:Gem::Dependency
|
|
100
105
|
name: elasticgraph-elasticsearch
|
|
101
106
|
requirement: !ruby/object:Gem::Requirement
|
|
102
107
|
requirements:
|
|
103
108
|
- - '='
|
|
104
109
|
- !ruby/object:Gem::Version
|
|
105
|
-
version: 0.19.
|
|
110
|
+
version: 0.19.2.0
|
|
106
111
|
type: :development
|
|
107
112
|
prerelease: false
|
|
108
113
|
version_requirements: !ruby/object:Gem::Requirement
|
|
109
114
|
requirements:
|
|
110
115
|
- - '='
|
|
111
116
|
- !ruby/object:Gem::Version
|
|
112
|
-
version: 0.19.
|
|
117
|
+
version: 0.19.2.0
|
|
113
118
|
- !ruby/object:Gem::Dependency
|
|
114
119
|
name: elasticgraph-opensearch
|
|
115
120
|
requirement: !ruby/object:Gem::Requirement
|
|
116
121
|
requirements:
|
|
117
122
|
- - '='
|
|
118
123
|
- !ruby/object:Gem::Version
|
|
119
|
-
version: 0.19.
|
|
124
|
+
version: 0.19.2.0
|
|
120
125
|
type: :development
|
|
121
126
|
prerelease: false
|
|
122
127
|
version_requirements: !ruby/object:Gem::Requirement
|
|
123
128
|
requirements:
|
|
124
129
|
- - '='
|
|
125
130
|
- !ruby/object:Gem::Version
|
|
126
|
-
version: 0.19.
|
|
131
|
+
version: 0.19.2.0
|
|
127
132
|
- !ruby/object:Gem::Dependency
|
|
128
133
|
name: elasticgraph-schema_definition
|
|
129
134
|
requirement: !ruby/object:Gem::Requirement
|
|
130
135
|
requirements:
|
|
131
136
|
- - '='
|
|
132
137
|
- !ruby/object:Gem::Version
|
|
133
|
-
version: 0.19.
|
|
138
|
+
version: 0.19.2.0
|
|
134
139
|
type: :development
|
|
135
140
|
prerelease: false
|
|
136
141
|
version_requirements: !ruby/object:Gem::Requirement
|
|
137
142
|
requirements:
|
|
138
143
|
- - '='
|
|
139
144
|
- !ruby/object:Gem::Version
|
|
140
|
-
version: 0.19.
|
|
141
|
-
description:
|
|
145
|
+
version: 0.19.2.0
|
|
142
146
|
email:
|
|
143
147
|
- myron@squareup.com
|
|
144
148
|
executables: []
|
|
@@ -171,12 +175,11 @@ licenses:
|
|
|
171
175
|
- MIT
|
|
172
176
|
metadata:
|
|
173
177
|
bug_tracker_uri: https://github.com/block/elasticgraph/issues
|
|
174
|
-
changelog_uri: https://github.com/block/elasticgraph/releases/tag/v0.19.
|
|
175
|
-
documentation_uri: https://block.github.io/elasticgraph/docs/
|
|
178
|
+
changelog_uri: https://github.com/block/elasticgraph/releases/tag/v0.19.2.0
|
|
179
|
+
documentation_uri: https://block.github.io/elasticgraph/api-docs/v0.19.2.0/
|
|
176
180
|
homepage_uri: https://block.github.io/elasticgraph/
|
|
177
|
-
source_code_uri: https://github.com/block/elasticgraph/tree/v0.19.
|
|
181
|
+
source_code_uri: https://github.com/block/elasticgraph/tree/v0.19.2.0/elasticgraph-indexer
|
|
178
182
|
gem_category: core
|
|
179
|
-
post_install_message:
|
|
180
183
|
rdoc_options: []
|
|
181
184
|
require_paths:
|
|
182
185
|
- lib
|
|
@@ -194,8 +197,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
194
197
|
- !ruby/object:Gem::Version
|
|
195
198
|
version: '0'
|
|
196
199
|
requirements: []
|
|
197
|
-
rubygems_version: 3.
|
|
198
|
-
signing_key:
|
|
200
|
+
rubygems_version: 3.6.2
|
|
199
201
|
specification_version: 4
|
|
200
202
|
summary: ElasticGraph gem that provides APIs to robustly index data into a datastore.
|
|
201
203
|
test_files: []
|