elasticgraph-indexer 0.18.0.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,98 @@
1
+ # Copyright 2024 Block, Inc.
2
+ #
3
+ # Use of this source code is governed by an MIT-style
4
+ # license that can be found in the LICENSE file or at
5
+ # https://opensource.org/licenses/MIT.
6
+ #
7
+ # frozen_string_literal: true
8
+
9
+ require "elastic_graph/datastore_core"
10
+ require "elastic_graph/indexer/config"
11
+ require "elastic_graph/support/from_yaml_file"
12
+
13
+ module ElasticGraph
14
+ class Indexer
15
+ extend Support::FromYamlFile
16
+
17
+ # @dynamic config, datastore_core, schema_artifacts, logger
18
+ attr_reader :config, :datastore_core, :schema_artifacts, :logger
19
+
20
+ # A factory method that builds an Indexer instance from the given parsed YAML config.
21
+ # `from_yaml_file(file_name, &block)` is also available (via `Support::FromYamlFile`).
22
+ def self.from_parsed_yaml(parsed_yaml, &datastore_client_customization_block)
23
+ new(
24
+ config: Indexer::Config.from_parsed_yaml(parsed_yaml),
25
+ datastore_core: DatastoreCore.from_parsed_yaml(parsed_yaml, for_context: :indexer, &datastore_client_customization_block)
26
+ )
27
+ end
28
+
29
+ def initialize(
30
+ config:,
31
+ datastore_core:,
32
+ datastore_router: nil,
33
+ monotonic_clock: nil,
34
+ clock: nil
35
+ )
36
+ @config = config
37
+ @datastore_core = datastore_core
38
+ @logger = datastore_core.logger
39
+ @datastore_router = datastore_router
40
+ @schema_artifacts = @datastore_core.schema_artifacts
41
+ @monotonic_clock = monotonic_clock
42
+ @clock = clock || ::Time
43
+ end
44
+
45
+ def datastore_router
46
+ @datastore_router ||= begin
47
+ require "elastic_graph/indexer/datastore_indexing_router"
48
+ DatastoreIndexingRouter.new(
49
+ datastore_clients_by_name: datastore_core.clients_by_name,
50
+ mappings_by_index_def_name: schema_artifacts.index_mappings_by_index_def_name,
51
+ monotonic_clock: monotonic_clock,
52
+ logger: datastore_core.logger
53
+ )
54
+ end
55
+ end
56
+
57
+ def record_preparer_factory
58
+ @record_preparer_factory ||= begin
59
+ require "elastic_graph/indexer/record_preparer"
60
+ RecordPreparer::Factory.new(schema_artifacts)
61
+ end
62
+ end
63
+
64
+ def processor
65
+ @processor ||= begin
66
+ require "elastic_graph/indexer/processor"
67
+ Processor.new(
68
+ datastore_router: datastore_router,
69
+ operation_factory: operation_factory,
70
+ indexing_latency_slo_thresholds_by_timestamp_in_ms: config.latency_slo_thresholds_by_timestamp_in_ms,
71
+ clock: @clock,
72
+ logger: datastore_core.logger
73
+ )
74
+ end
75
+ end
76
+
77
+ def operation_factory
78
+ @operation_factory ||= begin
79
+ require "elastic_graph/indexer/operation/factory"
80
+ Operation::Factory.new(
81
+ schema_artifacts: schema_artifacts,
82
+ index_definitions_by_graphql_type: datastore_core.index_definitions_by_graphql_type,
83
+ record_preparer_factory: record_preparer_factory,
84
+ logger: datastore_core.logger,
85
+ skip_derived_indexing_type_updates: config.skip_derived_indexing_type_updates,
86
+ configure_record_validator: nil
87
+ )
88
+ end
89
+ end
90
+
91
+ def monotonic_clock
92
+ @monotonic_clock ||= begin
93
+ require "elastic_graph/support/monotonic_clock"
94
+ Support::MonotonicClock.new
95
+ end
96
+ end
97
+ end
98
+ end
metadata ADDED
@@ -0,0 +1,454 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: elasticgraph-indexer
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.18.0.0
5
+ platform: ruby
6
+ authors:
7
+ - Myron Marston
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2024-08-27 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: rubocop-factory_bot
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '2.26'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '2.26'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rubocop-rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '0.6'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '0.6'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rubocop-rspec
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '3.0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '3.0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: standard
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: 1.39.0
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: 1.39.0
69
+ - !ruby/object:Gem::Dependency
70
+ name: steep
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '1.7'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '1.7'
83
+ - !ruby/object:Gem::Dependency
84
+ name: coderay
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '1.1'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '1.1'
97
+ - !ruby/object:Gem::Dependency
98
+ name: flatware-rspec
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: 2.3.2
104
+ - - "<"
105
+ - !ruby/object:Gem::Version
106
+ version: '3.0'
107
+ type: :development
108
+ prerelease: false
109
+ version_requirements: !ruby/object:Gem::Requirement
110
+ requirements:
111
+ - - ">="
112
+ - !ruby/object:Gem::Version
113
+ version: 2.3.2
114
+ - - "<"
115
+ - !ruby/object:Gem::Version
116
+ version: '3.0'
117
+ - !ruby/object:Gem::Dependency
118
+ name: rspec
119
+ requirement: !ruby/object:Gem::Requirement
120
+ requirements:
121
+ - - "~>"
122
+ - !ruby/object:Gem::Version
123
+ version: '3.13'
124
+ type: :development
125
+ prerelease: false
126
+ version_requirements: !ruby/object:Gem::Requirement
127
+ requirements:
128
+ - - "~>"
129
+ - !ruby/object:Gem::Version
130
+ version: '3.13'
131
+ - !ruby/object:Gem::Dependency
132
+ name: super_diff
133
+ requirement: !ruby/object:Gem::Requirement
134
+ requirements:
135
+ - - ">="
136
+ - !ruby/object:Gem::Version
137
+ version: 0.12.1
138
+ type: :development
139
+ prerelease: false
140
+ version_requirements: !ruby/object:Gem::Requirement
141
+ requirements:
142
+ - - ">="
143
+ - !ruby/object:Gem::Version
144
+ version: 0.12.1
145
+ - !ruby/object:Gem::Dependency
146
+ name: simplecov
147
+ requirement: !ruby/object:Gem::Requirement
148
+ requirements:
149
+ - - "~>"
150
+ - !ruby/object:Gem::Version
151
+ version: '0.22'
152
+ type: :development
153
+ prerelease: false
154
+ version_requirements: !ruby/object:Gem::Requirement
155
+ requirements:
156
+ - - "~>"
157
+ - !ruby/object:Gem::Version
158
+ version: '0.22'
159
+ - !ruby/object:Gem::Dependency
160
+ name: simplecov-console
161
+ requirement: !ruby/object:Gem::Requirement
162
+ requirements:
163
+ - - ">="
164
+ - !ruby/object:Gem::Version
165
+ version: 0.9.1
166
+ - - "<"
167
+ - !ruby/object:Gem::Version
168
+ version: '1.0'
169
+ type: :development
170
+ prerelease: false
171
+ version_requirements: !ruby/object:Gem::Requirement
172
+ requirements:
173
+ - - ">="
174
+ - !ruby/object:Gem::Version
175
+ version: 0.9.1
176
+ - - "<"
177
+ - !ruby/object:Gem::Version
178
+ version: '1.0'
179
+ - !ruby/object:Gem::Dependency
180
+ name: httpx
181
+ requirement: !ruby/object:Gem::Requirement
182
+ requirements:
183
+ - - ">="
184
+ - !ruby/object:Gem::Version
185
+ version: 1.2.6
186
+ - - "<"
187
+ - !ruby/object:Gem::Version
188
+ version: '2.0'
189
+ type: :development
190
+ prerelease: false
191
+ version_requirements: !ruby/object:Gem::Requirement
192
+ requirements:
193
+ - - ">="
194
+ - !ruby/object:Gem::Version
195
+ version: 1.2.6
196
+ - - "<"
197
+ - !ruby/object:Gem::Version
198
+ version: '2.0'
199
+ - !ruby/object:Gem::Dependency
200
+ name: method_source
201
+ requirement: !ruby/object:Gem::Requirement
202
+ requirements:
203
+ - - "~>"
204
+ - !ruby/object:Gem::Version
205
+ version: '1.1'
206
+ type: :development
207
+ prerelease: false
208
+ version_requirements: !ruby/object:Gem::Requirement
209
+ requirements:
210
+ - - "~>"
211
+ - !ruby/object:Gem::Version
212
+ version: '1.1'
213
+ - !ruby/object:Gem::Dependency
214
+ name: rspec-retry
215
+ requirement: !ruby/object:Gem::Requirement
216
+ requirements:
217
+ - - "~>"
218
+ - !ruby/object:Gem::Version
219
+ version: '0.6'
220
+ type: :development
221
+ prerelease: false
222
+ version_requirements: !ruby/object:Gem::Requirement
223
+ requirements:
224
+ - - "~>"
225
+ - !ruby/object:Gem::Version
226
+ version: '0.6'
227
+ - !ruby/object:Gem::Dependency
228
+ name: vcr
229
+ requirement: !ruby/object:Gem::Requirement
230
+ requirements:
231
+ - - ">="
232
+ - !ruby/object:Gem::Version
233
+ version: 6.3.1
234
+ - - "<"
235
+ - !ruby/object:Gem::Version
236
+ version: 7.0.0
237
+ type: :development
238
+ prerelease: false
239
+ version_requirements: !ruby/object:Gem::Requirement
240
+ requirements:
241
+ - - ">="
242
+ - !ruby/object:Gem::Version
243
+ version: 6.3.1
244
+ - - "<"
245
+ - !ruby/object:Gem::Version
246
+ version: 7.0.0
247
+ - !ruby/object:Gem::Dependency
248
+ name: factory_bot
249
+ requirement: !ruby/object:Gem::Requirement
250
+ requirements:
251
+ - - "~>"
252
+ - !ruby/object:Gem::Version
253
+ version: '6.4'
254
+ type: :development
255
+ prerelease: false
256
+ version_requirements: !ruby/object:Gem::Requirement
257
+ requirements:
258
+ - - "~>"
259
+ - !ruby/object:Gem::Version
260
+ version: '6.4'
261
+ - !ruby/object:Gem::Dependency
262
+ name: faker
263
+ requirement: !ruby/object:Gem::Requirement
264
+ requirements:
265
+ - - "~>"
266
+ - !ruby/object:Gem::Version
267
+ version: '3.4'
268
+ type: :development
269
+ prerelease: false
270
+ version_requirements: !ruby/object:Gem::Requirement
271
+ requirements:
272
+ - - "~>"
273
+ - !ruby/object:Gem::Version
274
+ version: '3.4'
275
+ - !ruby/object:Gem::Dependency
276
+ name: elasticgraph-datastore_core
277
+ requirement: !ruby/object:Gem::Requirement
278
+ requirements:
279
+ - - '='
280
+ - !ruby/object:Gem::Version
281
+ version: 0.18.0.0
282
+ type: :runtime
283
+ prerelease: false
284
+ version_requirements: !ruby/object:Gem::Requirement
285
+ requirements:
286
+ - - '='
287
+ - !ruby/object:Gem::Version
288
+ version: 0.18.0.0
289
+ - !ruby/object:Gem::Dependency
290
+ name: elasticgraph-json_schema
291
+ requirement: !ruby/object:Gem::Requirement
292
+ requirements:
293
+ - - '='
294
+ - !ruby/object:Gem::Version
295
+ version: 0.18.0.0
296
+ type: :runtime
297
+ prerelease: false
298
+ version_requirements: !ruby/object:Gem::Requirement
299
+ requirements:
300
+ - - '='
301
+ - !ruby/object:Gem::Version
302
+ version: 0.18.0.0
303
+ - !ruby/object:Gem::Dependency
304
+ name: elasticgraph-schema_artifacts
305
+ requirement: !ruby/object:Gem::Requirement
306
+ requirements:
307
+ - - '='
308
+ - !ruby/object:Gem::Version
309
+ version: 0.18.0.0
310
+ type: :runtime
311
+ prerelease: false
312
+ version_requirements: !ruby/object:Gem::Requirement
313
+ requirements:
314
+ - - '='
315
+ - !ruby/object:Gem::Version
316
+ version: 0.18.0.0
317
+ - !ruby/object:Gem::Dependency
318
+ name: elasticgraph-support
319
+ requirement: !ruby/object:Gem::Requirement
320
+ requirements:
321
+ - - '='
322
+ - !ruby/object:Gem::Version
323
+ version: 0.18.0.0
324
+ type: :runtime
325
+ prerelease: false
326
+ version_requirements: !ruby/object:Gem::Requirement
327
+ requirements:
328
+ - - '='
329
+ - !ruby/object:Gem::Version
330
+ version: 0.18.0.0
331
+ - !ruby/object:Gem::Dependency
332
+ name: hashdiff
333
+ requirement: !ruby/object:Gem::Requirement
334
+ requirements:
335
+ - - "~>"
336
+ - !ruby/object:Gem::Version
337
+ version: '1.1'
338
+ type: :runtime
339
+ prerelease: false
340
+ version_requirements: !ruby/object:Gem::Requirement
341
+ requirements:
342
+ - - "~>"
343
+ - !ruby/object:Gem::Version
344
+ version: '1.1'
345
+ - !ruby/object:Gem::Dependency
346
+ name: elasticgraph-admin
347
+ requirement: !ruby/object:Gem::Requirement
348
+ requirements:
349
+ - - '='
350
+ - !ruby/object:Gem::Version
351
+ version: 0.18.0.0
352
+ type: :development
353
+ prerelease: false
354
+ version_requirements: !ruby/object:Gem::Requirement
355
+ requirements:
356
+ - - '='
357
+ - !ruby/object:Gem::Version
358
+ version: 0.18.0.0
359
+ - !ruby/object:Gem::Dependency
360
+ name: elasticgraph-elasticsearch
361
+ requirement: !ruby/object:Gem::Requirement
362
+ requirements:
363
+ - - '='
364
+ - !ruby/object:Gem::Version
365
+ version: 0.18.0.0
366
+ type: :development
367
+ prerelease: false
368
+ version_requirements: !ruby/object:Gem::Requirement
369
+ requirements:
370
+ - - '='
371
+ - !ruby/object:Gem::Version
372
+ version: 0.18.0.0
373
+ - !ruby/object:Gem::Dependency
374
+ name: elasticgraph-opensearch
375
+ requirement: !ruby/object:Gem::Requirement
376
+ requirements:
377
+ - - '='
378
+ - !ruby/object:Gem::Version
379
+ version: 0.18.0.0
380
+ type: :development
381
+ prerelease: false
382
+ version_requirements: !ruby/object:Gem::Requirement
383
+ requirements:
384
+ - - '='
385
+ - !ruby/object:Gem::Version
386
+ version: 0.18.0.0
387
+ - !ruby/object:Gem::Dependency
388
+ name: elasticgraph-schema_definition
389
+ requirement: !ruby/object:Gem::Requirement
390
+ requirements:
391
+ - - '='
392
+ - !ruby/object:Gem::Version
393
+ version: 0.18.0.0
394
+ type: :development
395
+ prerelease: false
396
+ version_requirements: !ruby/object:Gem::Requirement
397
+ requirements:
398
+ - - '='
399
+ - !ruby/object:Gem::Version
400
+ version: 0.18.0.0
401
+ description:
402
+ email:
403
+ - myron@squareup.com
404
+ executables: []
405
+ extensions: []
406
+ extra_rdoc_files: []
407
+ files:
408
+ - LICENSE.txt
409
+ - README.md
410
+ - elasticgraph-indexer.gemspec
411
+ - lib/elastic_graph/indexer.rb
412
+ - lib/elastic_graph/indexer/config.rb
413
+ - lib/elastic_graph/indexer/datastore_indexing_router.rb
414
+ - lib/elastic_graph/indexer/event_id.rb
415
+ - lib/elastic_graph/indexer/failed_event_error.rb
416
+ - lib/elastic_graph/indexer/hash_differ.rb
417
+ - lib/elastic_graph/indexer/indexing_failures_error.rb
418
+ - lib/elastic_graph/indexer/indexing_preparers/integer.rb
419
+ - lib/elastic_graph/indexer/indexing_preparers/no_op.rb
420
+ - lib/elastic_graph/indexer/indexing_preparers/untyped.rb
421
+ - lib/elastic_graph/indexer/operation/count_accumulator.rb
422
+ - lib/elastic_graph/indexer/operation/factory.rb
423
+ - lib/elastic_graph/indexer/operation/result.rb
424
+ - lib/elastic_graph/indexer/operation/update.rb
425
+ - lib/elastic_graph/indexer/operation/upsert.rb
426
+ - lib/elastic_graph/indexer/processor.rb
427
+ - lib/elastic_graph/indexer/record_preparer.rb
428
+ - lib/elastic_graph/indexer/spec_support/event_matcher.rb
429
+ - lib/elastic_graph/indexer/test_support/converters.rb
430
+ homepage:
431
+ licenses:
432
+ - MIT
433
+ metadata:
434
+ gem_category: core
435
+ post_install_message:
436
+ rdoc_options: []
437
+ require_paths:
438
+ - lib
439
+ required_ruby_version: !ruby/object:Gem::Requirement
440
+ requirements:
441
+ - - "~>"
442
+ - !ruby/object:Gem::Version
443
+ version: '3.2'
444
+ required_rubygems_version: !ruby/object:Gem::Requirement
445
+ requirements:
446
+ - - ">="
447
+ - !ruby/object:Gem::Version
448
+ version: '0'
449
+ requirements: []
450
+ rubygems_version: 3.5.9
451
+ signing_key:
452
+ specification_version: 4
453
+ summary: ElasticGraph gem that provides APIs to robustly index data into a datastore.
454
+ test_files: []