elasticgraph-indexer 0.18.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,98 @@
1
+ # Copyright 2024 Block, Inc.
2
+ #
3
+ # Use of this source code is governed by an MIT-style
4
+ # license that can be found in the LICENSE file or at
5
+ # https://opensource.org/licenses/MIT.
6
+ #
7
+ # frozen_string_literal: true
8
+
9
+ require "elastic_graph/datastore_core"
10
+ require "elastic_graph/indexer/config"
11
+ require "elastic_graph/support/from_yaml_file"
12
+
13
+ module ElasticGraph
14
+ class Indexer
15
+ extend Support::FromYamlFile
16
+
17
+ # @dynamic config, datastore_core, schema_artifacts, logger
18
+ attr_reader :config, :datastore_core, :schema_artifacts, :logger
19
+
20
+ # A factory method that builds an Indexer instance from the given parsed YAML config.
21
+ # `from_yaml_file(file_name, &block)` is also available (via `Support::FromYamlFile`).
22
+ def self.from_parsed_yaml(parsed_yaml, &datastore_client_customization_block)
23
+ new(
24
+ config: Indexer::Config.from_parsed_yaml(parsed_yaml),
25
+ datastore_core: DatastoreCore.from_parsed_yaml(parsed_yaml, for_context: :indexer, &datastore_client_customization_block)
26
+ )
27
+ end
28
+
29
+ def initialize(
30
+ config:,
31
+ datastore_core:,
32
+ datastore_router: nil,
33
+ monotonic_clock: nil,
34
+ clock: nil
35
+ )
36
+ @config = config
37
+ @datastore_core = datastore_core
38
+ @logger = datastore_core.logger
39
+ @datastore_router = datastore_router
40
+ @schema_artifacts = @datastore_core.schema_artifacts
41
+ @monotonic_clock = monotonic_clock
42
+ @clock = clock || ::Time
43
+ end
44
+
45
+ def datastore_router
46
+ @datastore_router ||= begin
47
+ require "elastic_graph/indexer/datastore_indexing_router"
48
+ DatastoreIndexingRouter.new(
49
+ datastore_clients_by_name: datastore_core.clients_by_name,
50
+ mappings_by_index_def_name: schema_artifacts.index_mappings_by_index_def_name,
51
+ monotonic_clock: monotonic_clock,
52
+ logger: datastore_core.logger
53
+ )
54
+ end
55
+ end
56
+
57
+ def record_preparer_factory
58
+ @record_preparer_factory ||= begin
59
+ require "elastic_graph/indexer/record_preparer"
60
+ RecordPreparer::Factory.new(schema_artifacts)
61
+ end
62
+ end
63
+
64
+ def processor
65
+ @processor ||= begin
66
+ require "elastic_graph/indexer/processor"
67
+ Processor.new(
68
+ datastore_router: datastore_router,
69
+ operation_factory: operation_factory,
70
+ indexing_latency_slo_thresholds_by_timestamp_in_ms: config.latency_slo_thresholds_by_timestamp_in_ms,
71
+ clock: @clock,
72
+ logger: datastore_core.logger
73
+ )
74
+ end
75
+ end
76
+
77
+ def operation_factory
78
+ @operation_factory ||= begin
79
+ require "elastic_graph/indexer/operation/factory"
80
+ Operation::Factory.new(
81
+ schema_artifacts: schema_artifacts,
82
+ index_definitions_by_graphql_type: datastore_core.index_definitions_by_graphql_type,
83
+ record_preparer_factory: record_preparer_factory,
84
+ logger: datastore_core.logger,
85
+ skip_derived_indexing_type_updates: config.skip_derived_indexing_type_updates,
86
+ configure_record_validator: nil
87
+ )
88
+ end
89
+ end
90
+
91
+ def monotonic_clock
92
+ @monotonic_clock ||= begin
93
+ require "elastic_graph/support/monotonic_clock"
94
+ Support::MonotonicClock.new
95
+ end
96
+ end
97
+ end
98
+ end
metadata ADDED
@@ -0,0 +1,454 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: elasticgraph-indexer
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.18.0.0
5
+ platform: ruby
6
+ authors:
7
+ - Myron Marston
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2024-08-27 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: rubocop-factory_bot
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '2.26'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '2.26'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rubocop-rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '0.6'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '0.6'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rubocop-rspec
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '3.0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '3.0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: standard
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: 1.39.0
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: 1.39.0
69
+ - !ruby/object:Gem::Dependency
70
+ name: steep
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '1.7'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '1.7'
83
+ - !ruby/object:Gem::Dependency
84
+ name: coderay
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '1.1'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '1.1'
97
+ - !ruby/object:Gem::Dependency
98
+ name: flatware-rspec
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: 2.3.2
104
+ - - "<"
105
+ - !ruby/object:Gem::Version
106
+ version: '3.0'
107
+ type: :development
108
+ prerelease: false
109
+ version_requirements: !ruby/object:Gem::Requirement
110
+ requirements:
111
+ - - ">="
112
+ - !ruby/object:Gem::Version
113
+ version: 2.3.2
114
+ - - "<"
115
+ - !ruby/object:Gem::Version
116
+ version: '3.0'
117
+ - !ruby/object:Gem::Dependency
118
+ name: rspec
119
+ requirement: !ruby/object:Gem::Requirement
120
+ requirements:
121
+ - - "~>"
122
+ - !ruby/object:Gem::Version
123
+ version: '3.13'
124
+ type: :development
125
+ prerelease: false
126
+ version_requirements: !ruby/object:Gem::Requirement
127
+ requirements:
128
+ - - "~>"
129
+ - !ruby/object:Gem::Version
130
+ version: '3.13'
131
+ - !ruby/object:Gem::Dependency
132
+ name: super_diff
133
+ requirement: !ruby/object:Gem::Requirement
134
+ requirements:
135
+ - - ">="
136
+ - !ruby/object:Gem::Version
137
+ version: 0.12.1
138
+ type: :development
139
+ prerelease: false
140
+ version_requirements: !ruby/object:Gem::Requirement
141
+ requirements:
142
+ - - ">="
143
+ - !ruby/object:Gem::Version
144
+ version: 0.12.1
145
+ - !ruby/object:Gem::Dependency
146
+ name: simplecov
147
+ requirement: !ruby/object:Gem::Requirement
148
+ requirements:
149
+ - - "~>"
150
+ - !ruby/object:Gem::Version
151
+ version: '0.22'
152
+ type: :development
153
+ prerelease: false
154
+ version_requirements: !ruby/object:Gem::Requirement
155
+ requirements:
156
+ - - "~>"
157
+ - !ruby/object:Gem::Version
158
+ version: '0.22'
159
+ - !ruby/object:Gem::Dependency
160
+ name: simplecov-console
161
+ requirement: !ruby/object:Gem::Requirement
162
+ requirements:
163
+ - - ">="
164
+ - !ruby/object:Gem::Version
165
+ version: 0.9.1
166
+ - - "<"
167
+ - !ruby/object:Gem::Version
168
+ version: '1.0'
169
+ type: :development
170
+ prerelease: false
171
+ version_requirements: !ruby/object:Gem::Requirement
172
+ requirements:
173
+ - - ">="
174
+ - !ruby/object:Gem::Version
175
+ version: 0.9.1
176
+ - - "<"
177
+ - !ruby/object:Gem::Version
178
+ version: '1.0'
179
+ - !ruby/object:Gem::Dependency
180
+ name: httpx
181
+ requirement: !ruby/object:Gem::Requirement
182
+ requirements:
183
+ - - ">="
184
+ - !ruby/object:Gem::Version
185
+ version: 1.2.6
186
+ - - "<"
187
+ - !ruby/object:Gem::Version
188
+ version: '2.0'
189
+ type: :development
190
+ prerelease: false
191
+ version_requirements: !ruby/object:Gem::Requirement
192
+ requirements:
193
+ - - ">="
194
+ - !ruby/object:Gem::Version
195
+ version: 1.2.6
196
+ - - "<"
197
+ - !ruby/object:Gem::Version
198
+ version: '2.0'
199
+ - !ruby/object:Gem::Dependency
200
+ name: method_source
201
+ requirement: !ruby/object:Gem::Requirement
202
+ requirements:
203
+ - - "~>"
204
+ - !ruby/object:Gem::Version
205
+ version: '1.1'
206
+ type: :development
207
+ prerelease: false
208
+ version_requirements: !ruby/object:Gem::Requirement
209
+ requirements:
210
+ - - "~>"
211
+ - !ruby/object:Gem::Version
212
+ version: '1.1'
213
+ - !ruby/object:Gem::Dependency
214
+ name: rspec-retry
215
+ requirement: !ruby/object:Gem::Requirement
216
+ requirements:
217
+ - - "~>"
218
+ - !ruby/object:Gem::Version
219
+ version: '0.6'
220
+ type: :development
221
+ prerelease: false
222
+ version_requirements: !ruby/object:Gem::Requirement
223
+ requirements:
224
+ - - "~>"
225
+ - !ruby/object:Gem::Version
226
+ version: '0.6'
227
+ - !ruby/object:Gem::Dependency
228
+ name: vcr
229
+ requirement: !ruby/object:Gem::Requirement
230
+ requirements:
231
+ - - ">="
232
+ - !ruby/object:Gem::Version
233
+ version: 6.3.1
234
+ - - "<"
235
+ - !ruby/object:Gem::Version
236
+ version: 7.0.0
237
+ type: :development
238
+ prerelease: false
239
+ version_requirements: !ruby/object:Gem::Requirement
240
+ requirements:
241
+ - - ">="
242
+ - !ruby/object:Gem::Version
243
+ version: 6.3.1
244
+ - - "<"
245
+ - !ruby/object:Gem::Version
246
+ version: 7.0.0
247
+ - !ruby/object:Gem::Dependency
248
+ name: factory_bot
249
+ requirement: !ruby/object:Gem::Requirement
250
+ requirements:
251
+ - - "~>"
252
+ - !ruby/object:Gem::Version
253
+ version: '6.4'
254
+ type: :development
255
+ prerelease: false
256
+ version_requirements: !ruby/object:Gem::Requirement
257
+ requirements:
258
+ - - "~>"
259
+ - !ruby/object:Gem::Version
260
+ version: '6.4'
261
+ - !ruby/object:Gem::Dependency
262
+ name: faker
263
+ requirement: !ruby/object:Gem::Requirement
264
+ requirements:
265
+ - - "~>"
266
+ - !ruby/object:Gem::Version
267
+ version: '3.4'
268
+ type: :development
269
+ prerelease: false
270
+ version_requirements: !ruby/object:Gem::Requirement
271
+ requirements:
272
+ - - "~>"
273
+ - !ruby/object:Gem::Version
274
+ version: '3.4'
275
+ - !ruby/object:Gem::Dependency
276
+ name: elasticgraph-datastore_core
277
+ requirement: !ruby/object:Gem::Requirement
278
+ requirements:
279
+ - - '='
280
+ - !ruby/object:Gem::Version
281
+ version: 0.18.0.0
282
+ type: :runtime
283
+ prerelease: false
284
+ version_requirements: !ruby/object:Gem::Requirement
285
+ requirements:
286
+ - - '='
287
+ - !ruby/object:Gem::Version
288
+ version: 0.18.0.0
289
+ - !ruby/object:Gem::Dependency
290
+ name: elasticgraph-json_schema
291
+ requirement: !ruby/object:Gem::Requirement
292
+ requirements:
293
+ - - '='
294
+ - !ruby/object:Gem::Version
295
+ version: 0.18.0.0
296
+ type: :runtime
297
+ prerelease: false
298
+ version_requirements: !ruby/object:Gem::Requirement
299
+ requirements:
300
+ - - '='
301
+ - !ruby/object:Gem::Version
302
+ version: 0.18.0.0
303
+ - !ruby/object:Gem::Dependency
304
+ name: elasticgraph-schema_artifacts
305
+ requirement: !ruby/object:Gem::Requirement
306
+ requirements:
307
+ - - '='
308
+ - !ruby/object:Gem::Version
309
+ version: 0.18.0.0
310
+ type: :runtime
311
+ prerelease: false
312
+ version_requirements: !ruby/object:Gem::Requirement
313
+ requirements:
314
+ - - '='
315
+ - !ruby/object:Gem::Version
316
+ version: 0.18.0.0
317
+ - !ruby/object:Gem::Dependency
318
+ name: elasticgraph-support
319
+ requirement: !ruby/object:Gem::Requirement
320
+ requirements:
321
+ - - '='
322
+ - !ruby/object:Gem::Version
323
+ version: 0.18.0.0
324
+ type: :runtime
325
+ prerelease: false
326
+ version_requirements: !ruby/object:Gem::Requirement
327
+ requirements:
328
+ - - '='
329
+ - !ruby/object:Gem::Version
330
+ version: 0.18.0.0
331
+ - !ruby/object:Gem::Dependency
332
+ name: hashdiff
333
+ requirement: !ruby/object:Gem::Requirement
334
+ requirements:
335
+ - - "~>"
336
+ - !ruby/object:Gem::Version
337
+ version: '1.1'
338
+ type: :runtime
339
+ prerelease: false
340
+ version_requirements: !ruby/object:Gem::Requirement
341
+ requirements:
342
+ - - "~>"
343
+ - !ruby/object:Gem::Version
344
+ version: '1.1'
345
+ - !ruby/object:Gem::Dependency
346
+ name: elasticgraph-admin
347
+ requirement: !ruby/object:Gem::Requirement
348
+ requirements:
349
+ - - '='
350
+ - !ruby/object:Gem::Version
351
+ version: 0.18.0.0
352
+ type: :development
353
+ prerelease: false
354
+ version_requirements: !ruby/object:Gem::Requirement
355
+ requirements:
356
+ - - '='
357
+ - !ruby/object:Gem::Version
358
+ version: 0.18.0.0
359
+ - !ruby/object:Gem::Dependency
360
+ name: elasticgraph-elasticsearch
361
+ requirement: !ruby/object:Gem::Requirement
362
+ requirements:
363
+ - - '='
364
+ - !ruby/object:Gem::Version
365
+ version: 0.18.0.0
366
+ type: :development
367
+ prerelease: false
368
+ version_requirements: !ruby/object:Gem::Requirement
369
+ requirements:
370
+ - - '='
371
+ - !ruby/object:Gem::Version
372
+ version: 0.18.0.0
373
+ - !ruby/object:Gem::Dependency
374
+ name: elasticgraph-opensearch
375
+ requirement: !ruby/object:Gem::Requirement
376
+ requirements:
377
+ - - '='
378
+ - !ruby/object:Gem::Version
379
+ version: 0.18.0.0
380
+ type: :development
381
+ prerelease: false
382
+ version_requirements: !ruby/object:Gem::Requirement
383
+ requirements:
384
+ - - '='
385
+ - !ruby/object:Gem::Version
386
+ version: 0.18.0.0
387
+ - !ruby/object:Gem::Dependency
388
+ name: elasticgraph-schema_definition
389
+ requirement: !ruby/object:Gem::Requirement
390
+ requirements:
391
+ - - '='
392
+ - !ruby/object:Gem::Version
393
+ version: 0.18.0.0
394
+ type: :development
395
+ prerelease: false
396
+ version_requirements: !ruby/object:Gem::Requirement
397
+ requirements:
398
+ - - '='
399
+ - !ruby/object:Gem::Version
400
+ version: 0.18.0.0
401
+ description:
402
+ email:
403
+ - myron@squareup.com
404
+ executables: []
405
+ extensions: []
406
+ extra_rdoc_files: []
407
+ files:
408
+ - LICENSE.txt
409
+ - README.md
410
+ - elasticgraph-indexer.gemspec
411
+ - lib/elastic_graph/indexer.rb
412
+ - lib/elastic_graph/indexer/config.rb
413
+ - lib/elastic_graph/indexer/datastore_indexing_router.rb
414
+ - lib/elastic_graph/indexer/event_id.rb
415
+ - lib/elastic_graph/indexer/failed_event_error.rb
416
+ - lib/elastic_graph/indexer/hash_differ.rb
417
+ - lib/elastic_graph/indexer/indexing_failures_error.rb
418
+ - lib/elastic_graph/indexer/indexing_preparers/integer.rb
419
+ - lib/elastic_graph/indexer/indexing_preparers/no_op.rb
420
+ - lib/elastic_graph/indexer/indexing_preparers/untyped.rb
421
+ - lib/elastic_graph/indexer/operation/count_accumulator.rb
422
+ - lib/elastic_graph/indexer/operation/factory.rb
423
+ - lib/elastic_graph/indexer/operation/result.rb
424
+ - lib/elastic_graph/indexer/operation/update.rb
425
+ - lib/elastic_graph/indexer/operation/upsert.rb
426
+ - lib/elastic_graph/indexer/processor.rb
427
+ - lib/elastic_graph/indexer/record_preparer.rb
428
+ - lib/elastic_graph/indexer/spec_support/event_matcher.rb
429
+ - lib/elastic_graph/indexer/test_support/converters.rb
430
+ homepage:
431
+ licenses:
432
+ - MIT
433
+ metadata:
434
+ gem_category: core
435
+ post_install_message:
436
+ rdoc_options: []
437
+ require_paths:
438
+ - lib
439
+ required_ruby_version: !ruby/object:Gem::Requirement
440
+ requirements:
441
+ - - "~>"
442
+ - !ruby/object:Gem::Version
443
+ version: '3.2'
444
+ required_rubygems_version: !ruby/object:Gem::Requirement
445
+ requirements:
446
+ - - ">="
447
+ - !ruby/object:Gem::Version
448
+ version: '0'
449
+ requirements: []
450
+ rubygems_version: 3.5.9
451
+ signing_key:
452
+ specification_version: 4
453
+ summary: ElasticGraph gem that provides APIs to robustly index data into a datastore.
454
+ test_files: []