deimos-ruby 1.16.3 → 1.16.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +3 -3
- data/.gitignore +1 -0
- data/.rubocop.yml +20 -14
- data/.rubocop_todo.yml +364 -0
- data/.ruby-version +2 -1
- data/CHANGELOG.md +9 -0
- data/Gemfile +6 -0
- data/README.md +7 -1
- data/Steepfile +6 -0
- data/deimos-ruby.gemspec +3 -2
- data/lib/deimos/active_record_consume/batch_consumption.rb +7 -2
- data/lib/deimos/active_record_consume/batch_slicer.rb +2 -0
- data/lib/deimos/active_record_consume/message_consumption.rb +8 -4
- data/lib/deimos/active_record_consumer.rb +7 -4
- data/lib/deimos/active_record_producer.rb +3 -0
- data/lib/deimos/backends/base.rb +4 -2
- data/lib/deimos/backends/kafka.rb +1 -0
- data/lib/deimos/backends/kafka_async.rb +1 -0
- data/lib/deimos/config/configuration.rb +4 -0
- data/lib/deimos/config/phobos_config.rb +2 -1
- data/lib/deimos/consume/batch_consumption.rb +8 -1
- data/lib/deimos/consume/message_consumption.rb +4 -1
- data/lib/deimos/instrumentation.rb +11 -4
- data/lib/deimos/kafka_message.rb +1 -0
- data/lib/deimos/kafka_source.rb +5 -0
- data/lib/deimos/kafka_topic_info.rb +4 -0
- data/lib/deimos/message.rb +19 -2
- data/lib/deimos/metrics/datadog.rb +2 -1
- data/lib/deimos/metrics/mock.rb +2 -2
- data/lib/deimos/metrics/provider.rb +6 -0
- data/lib/deimos/monkey_patches/phobos_cli.rb +1 -1
- data/lib/deimos/monkey_patches/phobos_producer.rb +1 -0
- data/lib/deimos/producer.rb +12 -6
- data/lib/deimos/schema_backends/base.rb +31 -17
- data/lib/deimos/schema_backends/mock.rb +2 -2
- data/lib/deimos/schema_class/base.rb +9 -5
- data/lib/deimos/schema_class/enum.rb +4 -2
- data/lib/deimos/schema_class/record.rb +5 -5
- data/lib/deimos/shared_config.rb +6 -2
- data/lib/deimos/test_helpers.rb +21 -4
- data/lib/deimos/tracing/datadog.rb +1 -1
- data/lib/deimos/tracing/mock.rb +4 -3
- data/lib/deimos/tracing/provider.rb +5 -0
- data/lib/deimos/utils/db_poller.rb +9 -1
- data/lib/deimos/utils/db_producer.rb +14 -2
- data/lib/deimos/utils/deadlock_retry.rb +3 -0
- data/lib/deimos/utils/inline_consumer.rb +14 -6
- data/lib/deimos/utils/lag_reporter.rb +11 -0
- data/lib/deimos/utils/schema_controller_mixin.rb +8 -0
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +3 -2
- data/lib/generators/deimos/active_record_generator.rb +1 -1
- data/lib/generators/deimos/db_backend_generator.rb +1 -0
- data/lib/generators/deimos/db_poller_generator.rb +1 -0
- data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt +1 -1
- data/lib/generators/deimos/schema_class_generator.rb +13 -4
- data/rbs_collection.lock.yaml +176 -0
- data/rbs_collection.yaml +15 -0
- data/sig/avro.rbs +14 -0
- data/sig/defs.rbs +1867 -0
- data/sig/fig_tree.rbs +2 -0
- data/spec/consumer_spec.rb +14 -14
- data/spec/generators/schema_class/my_schema_spec.rb +3 -3
- data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb +1 -1
- data/spec/producer_spec.rb +1 -1
- data/spec/schemas/my_namespace/my_schema_with_complex_type.rb +3 -3
- data/spec/snapshots/consumers-no-nest.snap +1 -1
- data/spec/snapshots/consumers.snap +1 -1
- data/spec/snapshots/consumers_and_producers-no-nest.snap +3 -3
- data/spec/snapshots/consumers_and_producers.snap +3 -3
- data/spec/snapshots/consumers_circular-no-nest.snap +1 -1
- data/spec/snapshots/consumers_circular.snap +1 -1
- data/spec/snapshots/consumers_complex_types-no-nest.snap +1 -1
- data/spec/snapshots/consumers_complex_types.snap +1 -1
- data/spec/snapshots/consumers_nested-no-nest.snap +1 -1
- data/spec/snapshots/consumers_nested.snap +1 -1
- data/spec/snapshots/namespace_folders.snap +3 -3
- data/spec/snapshots/producers_with_key-no-nest.snap +1 -1
- data/spec/snapshots/producers_with_key.snap +1 -1
- metadata +39 -21
- data/.gemfiles/avro_turf-0.gemfile +0 -3
- data/.gemfiles/avro_turf-1.gemfile +0 -3
- data/.ruby-gemset +0 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 021fca59477fbfa890fa77ae867cd3a006b65e2c0235d1bb47ccdd1780e51d9c
|
4
|
+
data.tar.gz: 8330ba3d8f90567c7abbfd91298dd04d80f9507f702a0d4ac45300f68e5d224a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ab861eb37e8590664f47e0e11082ed5aac344a730c009c5261edcbdcbfcd00a39bc8e1f44bf2be5c05b1eb587ba15ad32e77edfd80ca0789da05049c91dd20e9
|
7
|
+
data.tar.gz: 851b9b844d11be5d90d3b4e652a648e7ce1e8e90e1ddd96f25a34d690cb89e2503302a80ad9cb891cbaa30949a0394e68337b34de86cf848a6815730d1548ca0
|
data/.github/workflows/ci.yml
CHANGED
@@ -25,18 +25,18 @@ jobs:
|
|
25
25
|
ruby-version: 2.7
|
26
26
|
bundler-cache: true
|
27
27
|
|
28
|
+
- name: Bundle install
|
29
|
+
run: bundle install --with development
|
30
|
+
|
28
31
|
- name: Rubocop
|
29
32
|
run: bundle exec rubocop --format progress
|
30
33
|
|
31
34
|
build:
|
32
35
|
runs-on: ubuntu-latest
|
33
|
-
env:
|
34
|
-
BUNDLE_GEMFILE: ${{ github.workspace }}/.gemfiles/${{ matrix.gemfile }}.gemfile
|
35
36
|
strategy:
|
36
37
|
fail-fast: false
|
37
38
|
matrix:
|
38
39
|
ruby: [ '2.6', '2.7', '3.0', '3.1' ]
|
39
|
-
gemfile: [ 'avro_turf-0', 'avro_turf-1' ]
|
40
40
|
|
41
41
|
steps:
|
42
42
|
- uses: actions/checkout@v2
|
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
@@ -1,7 +1,9 @@
|
|
1
|
+
inherit_from: .rubocop_todo.yml
|
2
|
+
|
1
3
|
require: rubocop-rspec
|
2
4
|
|
3
5
|
AllCops:
|
4
|
-
TargetRubyVersion: 2.
|
6
|
+
TargetRubyVersion: 2.5
|
5
7
|
Exclude:
|
6
8
|
- lib/deimos/monkey_patches/*.rb
|
7
9
|
- vendor/**/*
|
@@ -39,8 +41,9 @@ Layout/EmptyLinesAroundBlockBody:
|
|
39
41
|
Layout/LineLength:
|
40
42
|
Max: 120
|
41
43
|
Severity: refactor
|
42
|
-
|
43
|
-
|
44
|
+
# TODO: uncomment the files below once rubocop is fixed
|
45
|
+
# Exclude:
|
46
|
+
# - 'spec/**/*'
|
44
47
|
|
45
48
|
# foo = if expression
|
46
49
|
# 'bar'
|
@@ -149,18 +152,20 @@ Style/CollectionMethods:
|
|
149
152
|
Style/DateTime:
|
150
153
|
AllowCoercion: true
|
151
154
|
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
- 'app/
|
156
|
-
- '
|
155
|
+
# TODO: uncomment the lines below once rubocop is fixed
|
156
|
+
#Style/Documentation:
|
157
|
+
# Exclude:
|
158
|
+
# - 'app/controllers/**/*'
|
159
|
+
# - 'app/helpers/**/*'
|
160
|
+
# - 'db/**/*'
|
157
161
|
|
158
162
|
# Force documentation for public methods and classes
|
159
163
|
Style/DocumentationMethod:
|
160
164
|
Enabled: true
|
161
|
-
|
162
|
-
|
163
|
-
- '
|
165
|
+
# TODO: uncomment the file below once rubocop is fixed
|
166
|
+
# Exclude:
|
167
|
+
# - 'app/controllers/**/*'
|
168
|
+
# - 'db/**/*'
|
164
169
|
|
165
170
|
# Allow else with just nil in it
|
166
171
|
Style/EmptyElse:
|
@@ -231,9 +236,10 @@ Style/MethodCallWithArgsParentheses:
|
|
231
236
|
- before_destroy
|
232
237
|
- after_destroy
|
233
238
|
- queue_as
|
234
|
-
|
235
|
-
|
236
|
-
- '
|
239
|
+
# TODO: uncomment the file below once rubocop is fixed
|
240
|
+
# Exclude:
|
241
|
+
# - 'bin/**/*'
|
242
|
+
# - 'Gemfile'
|
237
243
|
|
238
244
|
# Do not allow "end.something"
|
239
245
|
Style/MethodCalledOnDoEndBlock:
|
data/.rubocop_todo.yml
ADDED
@@ -0,0 +1,364 @@
|
|
1
|
+
# This configuration was generated by
|
2
|
+
# `rubocop --auto-gen-config --auto-gen-only-exclude`
|
3
|
+
# on 2022-09-08 19:46:52 UTC using RuboCop version 0.90.0.
|
4
|
+
# The point is for the user to remove these configuration records
|
5
|
+
# one by one as the offenses are removed from the code base.
|
6
|
+
# Note that changes in the inspected code, or installation of new
|
7
|
+
# versions of RuboCop, may require this file to be generated again.
|
8
|
+
|
9
|
+
# Offense count: 1
|
10
|
+
# Configuration parameters: Include.
|
11
|
+
# Include: **/*.gemspec
|
12
|
+
Gemspec/RequiredRubyVersion:
|
13
|
+
Exclude:
|
14
|
+
- 'deimos-ruby.gemspec'
|
15
|
+
|
16
|
+
# Offense count: 4
|
17
|
+
# Cop supports --auto-correct.
|
18
|
+
Layout/EmptyLines:
|
19
|
+
Exclude:
|
20
|
+
- 'spec/schemas/my_namespace/generated.rb'
|
21
|
+
- 'spec/schemas/my_namespace/my_nested_schema.rb'
|
22
|
+
- 'spec/schemas/my_namespace/my_schema_with_circular_reference.rb'
|
23
|
+
- 'spec/schemas/my_namespace/my_schema_with_complex_type.rb'
|
24
|
+
|
25
|
+
# Offense count: 32
|
26
|
+
# Cop supports --auto-correct.
|
27
|
+
# Configuration parameters: EnforcedStyle.
|
28
|
+
# SupportedStyles: empty_lines, empty_lines_except_namespace, empty_lines_special, no_empty_lines, beginning_only, ending_only
|
29
|
+
Layout/EmptyLinesAroundClassBody:
|
30
|
+
Enabled: false
|
31
|
+
|
32
|
+
# Offense count: 1
|
33
|
+
# Cop supports --auto-correct.
|
34
|
+
Layout/EmptyLinesAroundMethodBody:
|
35
|
+
Exclude:
|
36
|
+
- 'lib/generators/deimos/schema_class_generator.rb'
|
37
|
+
|
38
|
+
# Offense count: 42
|
39
|
+
# Cop supports --auto-correct.
|
40
|
+
# Configuration parameters: EnforcedStyleAlignWith, Severity.
|
41
|
+
# SupportedStylesAlignWith: keyword, variable, start_of_line
|
42
|
+
Layout/EndAlignment:
|
43
|
+
Enabled: false
|
44
|
+
|
45
|
+
# Offense count: 4
|
46
|
+
# Cop supports --auto-correct.
|
47
|
+
# Configuration parameters: EnforcedStyle, IndentationWidth.
|
48
|
+
# SupportedStyles: special_inside_parentheses, consistent, align_braces
|
49
|
+
Layout/FirstHashElementIndentation:
|
50
|
+
Exclude:
|
51
|
+
- 'spec/producer_spec.rb'
|
52
|
+
|
53
|
+
# Offense count: 19
|
54
|
+
# Cop supports --auto-correct.
|
55
|
+
# Configuration parameters: AllowMultipleStyles, EnforcedHashRocketStyle, EnforcedColonStyle, EnforcedLastArgumentHashStyle.
|
56
|
+
# SupportedHashRocketStyles: key, separator, table
|
57
|
+
# SupportedColonStyles: key, separator, table
|
58
|
+
# SupportedLastArgumentHashStyles: always_inspect, always_ignore, ignore_implicit, ignore_explicit
|
59
|
+
Layout/HashAlignment:
|
60
|
+
Exclude:
|
61
|
+
- 'spec/consumer_spec.rb'
|
62
|
+
- 'spec/generators/schema_class/my_schema_with_circular_reference_spec.rb'
|
63
|
+
|
64
|
+
# Offense count: 21
|
65
|
+
# Cop supports --auto-correct.
|
66
|
+
# Configuration parameters: Width, IgnoredPatterns.
|
67
|
+
Layout/IndentationWidth:
|
68
|
+
Enabled: false
|
69
|
+
|
70
|
+
# Offense count: 9
|
71
|
+
# Cop supports --auto-correct.
|
72
|
+
# Configuration parameters: AllowForAlignment, EnforcedStyleForExponentOperator.
|
73
|
+
# SupportedStylesForExponentOperator: space, no_space
|
74
|
+
Layout/SpaceAroundOperators:
|
75
|
+
Exclude:
|
76
|
+
- 'spec/generators/schema_class/my_schema_with_circular_reference_spec.rb'
|
77
|
+
- 'spec/generators/schema_class_generator_spec.rb'
|
78
|
+
- 'spec/schemas/my_namespace/my_schema_with_complex_type.rb'
|
79
|
+
|
80
|
+
# Offense count: 1
|
81
|
+
# Cop supports --auto-correct.
|
82
|
+
# Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBraces, SpaceBeforeBlockParameters.
|
83
|
+
# SupportedStyles: space, no_space
|
84
|
+
# SupportedStylesForEmptyBraces: space, no_space
|
85
|
+
Layout/SpaceInsideBlockBraces:
|
86
|
+
Exclude:
|
87
|
+
- 'spec/generators/schema_class_generator_spec.rb'
|
88
|
+
|
89
|
+
# Offense count: 1
|
90
|
+
# Cop supports --auto-correct.
|
91
|
+
# Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBraces.
|
92
|
+
# SupportedStyles: space, no_space, compact
|
93
|
+
# SupportedStylesForEmptyBraces: space, no_space
|
94
|
+
Layout/SpaceInsideHashLiteralBraces:
|
95
|
+
Exclude:
|
96
|
+
- 'spec/active_record_producer_spec.rb'
|
97
|
+
|
98
|
+
# Offense count: 1
|
99
|
+
# Cop supports --auto-correct.
|
100
|
+
# Configuration parameters: EnforcedStyle.
|
101
|
+
# SupportedStyles: space, no_space
|
102
|
+
Layout/SpaceInsideParens:
|
103
|
+
Exclude:
|
104
|
+
- 'lib/deimos.rb'
|
105
|
+
|
106
|
+
# Offense count: 23
|
107
|
+
# Cop supports --auto-correct.
|
108
|
+
# Configuration parameters: AllowInHeredoc.
|
109
|
+
Layout/TrailingWhitespace:
|
110
|
+
Exclude:
|
111
|
+
- 'spec/schemas/my_namespace/generated.rb'
|
112
|
+
- 'spec/schemas/my_namespace/my_nested_schema.rb'
|
113
|
+
- 'spec/schemas/my_namespace/my_schema_with_circular_reference.rb'
|
114
|
+
- 'spec/schemas/my_namespace/my_schema_with_complex_type.rb'
|
115
|
+
|
116
|
+
# Offense count: 5
|
117
|
+
Lint/MissingSuper:
|
118
|
+
Exclude:
|
119
|
+
- 'lib/deimos/active_record_consumer.rb'
|
120
|
+
- 'lib/deimos/metrics/datadog.rb'
|
121
|
+
- 'lib/deimos/schema_class/enum.rb'
|
122
|
+
- 'lib/deimos/tracing/datadog.rb'
|
123
|
+
- 'lib/deimos/tracing/mock.rb'
|
124
|
+
|
125
|
+
# Offense count: 1
|
126
|
+
# Cop supports --auto-correct.
|
127
|
+
Lint/RedundantCopDisableDirective:
|
128
|
+
Exclude:
|
129
|
+
- 'spec/spec_helper.rb'
|
130
|
+
|
131
|
+
# Offense count: 3
|
132
|
+
# Cop supports --auto-correct.
|
133
|
+
# Configuration parameters: AllowUnusedKeywordArguments, IgnoreEmptyMethods, IgnoreNotImplementedMethods.
|
134
|
+
Lint/UnusedMethodArgument:
|
135
|
+
Exclude:
|
136
|
+
- 'lib/deimos/tracing/mock.rb'
|
137
|
+
- 'spec/schemas/my_namespace/my_schema_with_circular_reference.rb'
|
138
|
+
|
139
|
+
# Offense count: 1
|
140
|
+
Lint/UselessAssignment:
|
141
|
+
Exclude:
|
142
|
+
- 'spec/generators/schema_class_generator_spec.rb'
|
143
|
+
|
144
|
+
# Offense count: 19
|
145
|
+
# Configuration parameters: IgnoredMethods, Max.
|
146
|
+
Metrics/AbcSize:
|
147
|
+
Exclude:
|
148
|
+
- 'lib/deimos/active_record_consume/message_consumption.rb'
|
149
|
+
- 'lib/deimos/config/phobos_config.rb'
|
150
|
+
- 'lib/deimos/instrumentation.rb'
|
151
|
+
- 'lib/deimos/kafka_source.rb'
|
152
|
+
- 'lib/deimos/kafka_topic_info.rb'
|
153
|
+
- 'lib/deimos/producer.rb'
|
154
|
+
- 'lib/deimos/schema_backends/avro_schema_coercer.rb'
|
155
|
+
- 'lib/deimos/test_helpers.rb'
|
156
|
+
- 'lib/deimos/utils/db_poller.rb'
|
157
|
+
- 'lib/deimos/utils/db_producer.rb'
|
158
|
+
- 'lib/deimos/utils/inline_consumer.rb'
|
159
|
+
- 'lib/deimos/utils/schema_controller_mixin.rb'
|
160
|
+
- 'lib/generators/deimos/schema_class_generator.rb'
|
161
|
+
|
162
|
+
# Offense count: 1
|
163
|
+
# Configuration parameters: CountComments, Max, CountAsOne, ExcludedMethods.
|
164
|
+
Metrics/MethodLength:
|
165
|
+
Exclude:
|
166
|
+
- 'lib/deimos/config/phobos_config.rb'
|
167
|
+
|
168
|
+
# Offense count: 5
|
169
|
+
# Configuration parameters: CountComments, Max, CountAsOne.
|
170
|
+
Metrics/ModuleLength:
|
171
|
+
Exclude:
|
172
|
+
- 'lib/deimos/test_helpers.rb'
|
173
|
+
- 'spec/active_record_batch_consumer_spec.rb'
|
174
|
+
- 'spec/batch_consumer_spec.rb'
|
175
|
+
- 'spec/kafka_source_spec.rb'
|
176
|
+
- 'spec/producer_spec.rb'
|
177
|
+
|
178
|
+
# Offense count: 6
|
179
|
+
# Configuration parameters: IgnoredMethods, Max.
|
180
|
+
Metrics/PerceivedComplexity:
|
181
|
+
Exclude:
|
182
|
+
- 'lib/deimos/config/phobos_config.rb'
|
183
|
+
- 'lib/deimos/consume/batch_consumption.rb'
|
184
|
+
- 'lib/deimos/kafka_source.rb'
|
185
|
+
- 'lib/deimos/schema_backends/avro_schema_coercer.rb'
|
186
|
+
- 'lib/deimos/test_helpers.rb'
|
187
|
+
|
188
|
+
# Offense count: 12
|
189
|
+
# Configuration parameters: Prefixes.
|
190
|
+
# Prefixes: when, with, without
|
191
|
+
RSpec/ContextWording:
|
192
|
+
Exclude:
|
193
|
+
- 'spec/generators/schema_class_generator_spec.rb'
|
194
|
+
|
195
|
+
# Offense count: 4
|
196
|
+
# Configuration parameters: Max.
|
197
|
+
RSpec/ExampleLength:
|
198
|
+
Exclude:
|
199
|
+
- 'spec/config/configuration_spec.rb'
|
200
|
+
- 'spec/utils/db_producer_spec.rb'
|
201
|
+
|
202
|
+
# Offense count: 6
|
203
|
+
# Configuration parameters: Max.
|
204
|
+
RSpec/MultipleExpectations:
|
205
|
+
Exclude:
|
206
|
+
- 'spec/active_record_consumer_spec.rb'
|
207
|
+
- 'spec/config/configuration_spec.rb'
|
208
|
+
- 'spec/kafka_topic_info_spec.rb'
|
209
|
+
- 'spec/utils/db_poller_spec.rb'
|
210
|
+
- 'spec/utils/db_producer_spec.rb'
|
211
|
+
|
212
|
+
# Offense count: 1
|
213
|
+
Style/Documentation:
|
214
|
+
Exclude:
|
215
|
+
- 'spec/**/*'
|
216
|
+
- 'test/**/*'
|
217
|
+
- 'regenerate_test_schema_classes.rb'
|
218
|
+
|
219
|
+
# Offense count: 1
|
220
|
+
# Configuration parameters: RequireForNonPublicMethods.
|
221
|
+
Style/DocumentationMethod:
|
222
|
+
Exclude:
|
223
|
+
- 'spec/**/*'
|
224
|
+
- 'test/**/*'
|
225
|
+
- 'regenerate_test_schema_classes.rb'
|
226
|
+
|
227
|
+
# Offense count: 1
|
228
|
+
# Cop supports --auto-correct.
|
229
|
+
# Configuration parameters: EnforcedStyle.
|
230
|
+
# SupportedStyles: compact, expanded
|
231
|
+
Style/EmptyMethod:
|
232
|
+
Exclude:
|
233
|
+
- 'regenerate_test_schema_classes.rb'
|
234
|
+
|
235
|
+
# Offense count: 2
|
236
|
+
# Cop supports --auto-correct.
|
237
|
+
Style/ExplicitBlockArgument:
|
238
|
+
Exclude:
|
239
|
+
- 'lib/deimos/metrics/datadog.rb'
|
240
|
+
- 'lib/deimos/utils/deadlock_retry.rb'
|
241
|
+
|
242
|
+
# Offense count: 2
|
243
|
+
# Cop supports --auto-correct.
|
244
|
+
# Configuration parameters: EnforcedStyle.
|
245
|
+
# SupportedStyles: always, always_true, never
|
246
|
+
Style/FrozenStringLiteralComment:
|
247
|
+
Exclude:
|
248
|
+
- 'regenerate_test_schema_classes.rb'
|
249
|
+
- 'spec/generators/schema_class/my_schema_spec.rb'
|
250
|
+
|
251
|
+
# Offense count: 10
|
252
|
+
# Cop supports --auto-correct.
|
253
|
+
Style/GlobalStdStream:
|
254
|
+
Exclude:
|
255
|
+
- 'lib/deimos/config/configuration.rb'
|
256
|
+
- 'lib/deimos/config/phobos_config.rb'
|
257
|
+
- 'lib/deimos/metrics/mock.rb'
|
258
|
+
- 'lib/deimos/test_helpers.rb'
|
259
|
+
- 'lib/deimos/tracing/mock.rb'
|
260
|
+
- 'lib/deimos/utils/db_producer.rb'
|
261
|
+
- 'lib/tasks/deimos.rake'
|
262
|
+
- 'spec/rake_spec.rb'
|
263
|
+
|
264
|
+
# Offense count: 1
|
265
|
+
# Cop supports --auto-correct.
|
266
|
+
Style/HashEachMethods:
|
267
|
+
Exclude:
|
268
|
+
- 'lib/generators/deimos/schema_class_generator.rb'
|
269
|
+
|
270
|
+
# Offense count: 31
|
271
|
+
# Cop supports --auto-correct.
|
272
|
+
# Configuration parameters: IgnoreMacros, IgnoredMethods, IgnoredPatterns, IncludedMacros, AllowParenthesesInMultilineCall, AllowParenthesesInChaining, AllowParenthesesInCamelCaseMethod, EnforcedStyle.
|
273
|
+
# SupportedStyles: require_parentheses, omit_parentheses
|
274
|
+
Style/MethodCallWithArgsParentheses:
|
275
|
+
Exclude:
|
276
|
+
- 'lib/deimos/utils/db_poller.rb'
|
277
|
+
- 'lib/deimos/utils/inline_consumer.rb'
|
278
|
+
- 'spec/config/configuration_spec.rb'
|
279
|
+
- 'spec/deimos_spec.rb'
|
280
|
+
- 'spec/generators/schema_class_generator_spec.rb'
|
281
|
+
|
282
|
+
# Offense count: 1
|
283
|
+
Style/OptionalBooleanParameter:
|
284
|
+
Exclude:
|
285
|
+
- 'lib/deimos/test_helpers.rb'
|
286
|
+
|
287
|
+
# Offense count: 1
|
288
|
+
# Cop supports --auto-correct.
|
289
|
+
Style/RedundantAssignment:
|
290
|
+
Exclude:
|
291
|
+
- 'spec/schemas/my_namespace/my_schema_with_circular_reference.rb'
|
292
|
+
|
293
|
+
# Offense count: 2
|
294
|
+
# Cop supports --auto-correct.
|
295
|
+
Style/RedundantBegin:
|
296
|
+
Exclude:
|
297
|
+
- 'lib/deimos.rb'
|
298
|
+
- 'lib/deimos/utils/db_producer.rb'
|
299
|
+
|
300
|
+
# Offense count: 1
|
301
|
+
# Cop supports --auto-correct.
|
302
|
+
Style/SingleArgumentDig:
|
303
|
+
Exclude:
|
304
|
+
- 'lib/deimos/active_record_consume/batch_slicer.rb'
|
305
|
+
|
306
|
+
# Offense count: 1
|
307
|
+
# Cop supports --auto-correct.
|
308
|
+
# Configuration parameters: AllowIfMethodIsEmpty.
|
309
|
+
Style/SingleLineMethods:
|
310
|
+
Exclude:
|
311
|
+
- 'regenerate_test_schema_classes.rb'
|
312
|
+
|
313
|
+
# Offense count: 5
|
314
|
+
# Cop supports --auto-correct.
|
315
|
+
Style/StringConcatenation:
|
316
|
+
Exclude:
|
317
|
+
- 'spec/producer_spec.rb'
|
318
|
+
- 'spec/utils/schema_controller_mixin_spec.rb'
|
319
|
+
|
320
|
+
# Offense count: 10
|
321
|
+
# Cop supports --auto-correct.
|
322
|
+
# Configuration parameters: EnforcedStyle, ConsistentQuotesInMultiline.
|
323
|
+
# SupportedStyles: single_quotes, double_quotes
|
324
|
+
Style/StringLiterals:
|
325
|
+
Exclude:
|
326
|
+
- 'lib/generators/deimos/schema_class_generator.rb'
|
327
|
+
- 'lib/tasks/deimos.rake'
|
328
|
+
- 'regenerate_test_schema_classes.rb'
|
329
|
+
- 'spec/schemas/my_namespace/my_schema_with_complex_type.rb'
|
330
|
+
- 'spec/spec_helper.rb'
|
331
|
+
|
332
|
+
# Offense count: 1
|
333
|
+
# Cop supports --auto-correct.
|
334
|
+
# Configuration parameters: EnforcedStyle, AllowSafeAssignment.
|
335
|
+
# SupportedStyles: require_parentheses, require_no_parentheses, require_parentheses_when_complex
|
336
|
+
Style/TernaryParentheses:
|
337
|
+
Exclude:
|
338
|
+
- 'lib/deimos/config/phobos_config.rb'
|
339
|
+
|
340
|
+
# Offense count: 21
|
341
|
+
# Cop supports --auto-correct.
|
342
|
+
Style/TrailingBodyOnModule:
|
343
|
+
Enabled: false
|
344
|
+
|
345
|
+
# Offense count: 1
|
346
|
+
# Cop supports --auto-correct.
|
347
|
+
# Configuration parameters: EnforcedStyleForMultiline.
|
348
|
+
# SupportedStylesForMultiline: comma, consistent_comma, no_comma
|
349
|
+
Style/TrailingCommaInHashLiteral:
|
350
|
+
Exclude:
|
351
|
+
- 'spec/config/configuration_spec.rb'
|
352
|
+
|
353
|
+
# Offense count: 8
|
354
|
+
# Cop supports --auto-correct.
|
355
|
+
# Configuration parameters: AutoCorrect, Max, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
|
356
|
+
# URISchemes: http, https
|
357
|
+
Layout/LineLength:
|
358
|
+
Exclude:
|
359
|
+
- 'lib/deimos/producer.rb'
|
360
|
+
- 'lib/deimos/utils/db_poller.rb'
|
361
|
+
- 'lib/deimos/utils/db_producer.rb'
|
362
|
+
- 'spec/active_record_batch_consumer_spec.rb'
|
363
|
+
- 'spec/generators/schema_class/my_schema_with_circular_reference_spec.rb'
|
364
|
+
- 'spec/generators/schema_class/my_schema_with_complex_types_spec.rb'
|
data/.ruby-version
CHANGED
@@ -1 +1,2 @@
|
|
1
|
-
|
1
|
+
3.1.0
|
2
|
+
|
data/CHANGELOG.md
CHANGED
@@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
7
7
|
|
8
8
|
## UNRELEASED
|
9
9
|
|
10
|
+
# 1.16.5 - 2022-09-30
|
11
|
+
|
12
|
+
- Fix the linting step in the CI
|
13
|
+
|
14
|
+
# 1.16.4 - 2022-09-09
|
15
|
+
|
16
|
+
- Now generates RBS types.
|
17
|
+
- Use `allocate` instead of `new` in `tombstone` to avoid issues with required fields in `initialize`.
|
18
|
+
|
10
19
|
# 1.16.3 - 2022-09-08
|
11
20
|
|
12
21
|
- Add the `tombstone` method to schema classes.
|
data/Gemfile
CHANGED
@@ -4,3 +4,9 @@ source 'https://rubygems.org'
|
|
4
4
|
|
5
5
|
# Specify your gem's dependencies in boilerplate.gemspec
|
6
6
|
gemspec
|
7
|
+
|
8
|
+
if !ENV['CI'] || ENV['CI'] == ''
|
9
|
+
# TODO: once all PRs are merged, add this to gemspec as a development dependency
|
10
|
+
gem 'sord', git: 'git@github.com:dorner/sord.git', ref: 'local-develop'
|
11
|
+
end
|
12
|
+
|
data/README.md
CHANGED
@@ -1198,10 +1198,16 @@ decoded = Deimos.decode(schema: 'MySchema', namespace: 'com.my-namespace', paylo
|
|
1198
1198
|
|
1199
1199
|
Bug reports and pull requests are welcome on GitHub at https://github.com/flipp-oss/deimos .
|
1200
1200
|
|
1201
|
+
You can/should re-generate RBS types when methods or classes change by running the following:
|
1202
|
+
|
1203
|
+
rbs collection install # if you haven't done it
|
1204
|
+
rbs collection update
|
1205
|
+
bundle exec sord --hide-private --no-sord-comments sig/defs.rbs --tags 'override:Override'
|
1206
|
+
|
1201
1207
|
### Linting
|
1202
1208
|
|
1203
1209
|
Deimos uses Rubocop to lint the code. Please run Rubocop on your code
|
1204
|
-
before submitting a PR.
|
1210
|
+
before submitting a PR. The GitHub CI will also run rubocop on your pull request.
|
1205
1211
|
|
1206
1212
|
---
|
1207
1213
|
<p align="center">
|
data/Steepfile
ADDED
data/deimos-ruby.gemspec
CHANGED
@@ -18,11 +18,11 @@ Gem::Specification.new do |spec|
|
|
18
18
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
19
|
spec.require_paths = ['lib']
|
20
20
|
|
21
|
-
spec.add_runtime_dependency('avro_turf', '>=
|
21
|
+
spec.add_runtime_dependency('avro_turf', '>= 1.4', '< 2')
|
22
|
+
spec.add_runtime_dependency('fig_tree', '~> 0.0.2')
|
22
23
|
spec.add_runtime_dependency('phobos', '>= 1.9', '< 3.0')
|
23
24
|
spec.add_runtime_dependency('ruby-kafka', '< 2')
|
24
25
|
spec.add_runtime_dependency('sigurd', '>= 0.1.0', '< 1.0')
|
25
|
-
spec.add_runtime_dependency('fig_tree', '~> 0.0.2')
|
26
26
|
|
27
27
|
spec.add_development_dependency('activerecord-import')
|
28
28
|
spec.add_development_dependency('avro', '~> 1.9')
|
@@ -43,4 +43,5 @@ Gem::Specification.new do |spec|
|
|
43
43
|
spec.add_development_dependency('rubocop', '0.89.0')
|
44
44
|
spec.add_development_dependency('rubocop-rspec', '1.42.0')
|
45
45
|
spec.add_development_dependency('sqlite3', '~> 1.3')
|
46
|
+
spec.add_development_dependency('steep', '~> 1.0')
|
46
47
|
end
|
@@ -15,8 +15,9 @@ module Deimos
|
|
15
15
|
# If two messages in a batch have the same key, we cannot process them
|
16
16
|
# in the same operation as they would interfere with each other. Thus
|
17
17
|
# they are split
|
18
|
-
# @param payloads [Array<Hash
|
18
|
+
# @param payloads [Array<Hash,Deimos::SchemaClass::Record>] Decoded payloads
|
19
19
|
# @param metadata [Hash] Information about batch, including keys.
|
20
|
+
# @return [void]
|
20
21
|
def consume_batch(payloads, metadata)
|
21
22
|
messages = payloads.
|
22
23
|
zip(metadata[:keys]).
|
@@ -59,6 +60,7 @@ module Deimos
|
|
59
60
|
# All messages are split into slices containing only unique keys, and
|
60
61
|
# each slice is handles as its own batch.
|
61
62
|
# @param messages [Array<Message>] List of messages.
|
63
|
+
# @return [void]
|
62
64
|
def uncompacted_update(messages)
|
63
65
|
BatchSlicer.
|
64
66
|
slice(messages).
|
@@ -69,6 +71,7 @@ module Deimos
|
|
69
71
|
# All messages with payloads are passed to upsert_records.
|
70
72
|
# All tombstones messages are passed to remove_records.
|
71
73
|
# @param messages [Array<Message>] List of messages.
|
74
|
+
# @return [void]
|
72
75
|
def update_database(messages)
|
73
76
|
# Find all upserted records (i.e. that have a payload) and all
|
74
77
|
# deleted record (no payload)
|
@@ -81,6 +84,7 @@ module Deimos
|
|
81
84
|
# Upsert any non-deleted records
|
82
85
|
# @param messages [Array<Message>] List of messages for a group of
|
83
86
|
# records to either be updated or inserted.
|
87
|
+
# @return [void]
|
84
88
|
def upsert_records(messages)
|
85
89
|
key_cols = key_columns(messages)
|
86
90
|
|
@@ -119,6 +123,7 @@ module Deimos
|
|
119
123
|
# Delete any records with a tombstone.
|
120
124
|
# @param messages [Array<Message>] List of messages for a group of
|
121
125
|
# deleted records.
|
126
|
+
# @return [void]
|
122
127
|
def remove_records(messages)
|
123
128
|
clause = deleted_query(messages)
|
124
129
|
|
@@ -128,7 +133,7 @@ module Deimos
|
|
128
133
|
# Create an ActiveRecord relation that matches all of the passed
|
129
134
|
# records. Used for bulk deletion.
|
130
135
|
# @param records [Array<Message>] List of messages.
|
131
|
-
# @return ActiveRecord::Relation Matching relation.
|
136
|
+
# @return [ActiveRecord::Relation] Matching relation.
|
132
137
|
def deleted_query(records)
|
133
138
|
keys = records.
|
134
139
|
map { |m| record_key(m.key)[@klass.primary_key] }.
|
@@ -11,6 +11,8 @@ module Deimos
|
|
11
11
|
# slices that maintain the correct order.
|
12
12
|
# E.g. Given messages A1, A2, B1, C1, C2, C3, they will be sliced as:
|
13
13
|
# [[A1, B1, C1], [A2, C2], [C3]]
|
14
|
+
# @param messages [Array<Message>]
|
15
|
+
# @return [Array<Array<Message>>]
|
14
16
|
def self.slice(messages)
|
15
17
|
ops = messages.group_by(&:key)
|
16
18
|
|
@@ -8,8 +8,8 @@ module Deimos
|
|
8
8
|
# Find the record specified by the given payload and key.
|
9
9
|
# Default is to use the primary key column and the value of the first
|
10
10
|
# field in the key.
|
11
|
-
# @param klass [Class
|
12
|
-
# @param _payload [Hash
|
11
|
+
# @param klass [Class<ActiveRecord::Base>]
|
12
|
+
# @param _payload [Hash,Deimos::SchemaClass::Record]
|
13
13
|
# @param key [Object]
|
14
14
|
# @return [ActiveRecord::Base]
|
15
15
|
def fetch_record(klass, _payload, key)
|
@@ -19,14 +19,16 @@ module Deimos
|
|
19
19
|
|
20
20
|
# Assign a key to a new record.
|
21
21
|
# @param record [ActiveRecord::Base]
|
22
|
-
# @param _payload [Hash
|
22
|
+
# @param _payload [Hash,Deimos::SchemaClass::Record]
|
23
23
|
# @param key [Object]
|
24
|
+
# @return [void]
|
24
25
|
def assign_key(record, _payload, key)
|
25
26
|
record[record.class.primary_key] = key
|
26
27
|
end
|
27
28
|
|
28
|
-
# @param payload [Hash
|
29
|
+
# @param payload [Hash,Deimos::SchemaClass::Record] Decoded payloads
|
29
30
|
# @param metadata [Hash] Information about batch, including keys.
|
31
|
+
# @return [void]
|
30
32
|
def consume(payload, metadata)
|
31
33
|
unless self.process_message?(payload)
|
32
34
|
Deimos.config.logger.debug(
|
@@ -64,6 +66,7 @@ module Deimos
|
|
64
66
|
end
|
65
67
|
|
66
68
|
# @param record [ActiveRecord::Base]
|
69
|
+
# @return [void]
|
67
70
|
def save_record(record)
|
68
71
|
record.created_at ||= Time.zone.now if record.respond_to?(:created_at)
|
69
72
|
record.updated_at = Time.zone.now if record.respond_to?(:updated_at)
|
@@ -73,6 +76,7 @@ module Deimos
|
|
73
76
|
# Destroy a record that received a null payload. Override if you need
|
74
77
|
# to do something other than a straight destroy (e.g. mark as archived).
|
75
78
|
# @param record [ActiveRecord::Base]
|
79
|
+
# @return [void]
|
76
80
|
def destroy_record(record)
|
77
81
|
record&.destroy
|
78
82
|
end
|