deimos-ruby 1.16.3 → 1.16.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +3 -3
- data/.gitignore +1 -0
- data/.rubocop.yml +20 -14
- data/.rubocop_todo.yml +364 -0
- data/.ruby-version +2 -1
- data/CHANGELOG.md +9 -0
- data/Gemfile +6 -0
- data/README.md +7 -1
- data/Steepfile +6 -0
- data/deimos-ruby.gemspec +3 -2
- data/lib/deimos/active_record_consume/batch_consumption.rb +7 -2
- data/lib/deimos/active_record_consume/batch_slicer.rb +2 -0
- data/lib/deimos/active_record_consume/message_consumption.rb +8 -4
- data/lib/deimos/active_record_consumer.rb +7 -4
- data/lib/deimos/active_record_producer.rb +3 -0
- data/lib/deimos/backends/base.rb +4 -2
- data/lib/deimos/backends/kafka.rb +1 -0
- data/lib/deimos/backends/kafka_async.rb +1 -0
- data/lib/deimos/config/configuration.rb +4 -0
- data/lib/deimos/config/phobos_config.rb +2 -1
- data/lib/deimos/consume/batch_consumption.rb +8 -1
- data/lib/deimos/consume/message_consumption.rb +4 -1
- data/lib/deimos/instrumentation.rb +11 -4
- data/lib/deimos/kafka_message.rb +1 -0
- data/lib/deimos/kafka_source.rb +5 -0
- data/lib/deimos/kafka_topic_info.rb +4 -0
- data/lib/deimos/message.rb +19 -2
- data/lib/deimos/metrics/datadog.rb +2 -1
- data/lib/deimos/metrics/mock.rb +2 -2
- data/lib/deimos/metrics/provider.rb +6 -0
- data/lib/deimos/monkey_patches/phobos_cli.rb +1 -1
- data/lib/deimos/monkey_patches/phobos_producer.rb +1 -0
- data/lib/deimos/producer.rb +12 -6
- data/lib/deimos/schema_backends/base.rb +31 -17
- data/lib/deimos/schema_backends/mock.rb +2 -2
- data/lib/deimos/schema_class/base.rb +9 -5
- data/lib/deimos/schema_class/enum.rb +4 -2
- data/lib/deimos/schema_class/record.rb +5 -5
- data/lib/deimos/shared_config.rb +6 -2
- data/lib/deimos/test_helpers.rb +21 -4
- data/lib/deimos/tracing/datadog.rb +1 -1
- data/lib/deimos/tracing/mock.rb +4 -3
- data/lib/deimos/tracing/provider.rb +5 -0
- data/lib/deimos/utils/db_poller.rb +9 -1
- data/lib/deimos/utils/db_producer.rb +14 -2
- data/lib/deimos/utils/deadlock_retry.rb +3 -0
- data/lib/deimos/utils/inline_consumer.rb +14 -6
- data/lib/deimos/utils/lag_reporter.rb +11 -0
- data/lib/deimos/utils/schema_controller_mixin.rb +8 -0
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +3 -2
- data/lib/generators/deimos/active_record_generator.rb +1 -1
- data/lib/generators/deimos/db_backend_generator.rb +1 -0
- data/lib/generators/deimos/db_poller_generator.rb +1 -0
- data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt +1 -1
- data/lib/generators/deimos/schema_class_generator.rb +13 -4
- data/rbs_collection.lock.yaml +176 -0
- data/rbs_collection.yaml +15 -0
- data/sig/avro.rbs +14 -0
- data/sig/defs.rbs +1867 -0
- data/sig/fig_tree.rbs +2 -0
- data/spec/consumer_spec.rb +14 -14
- data/spec/generators/schema_class/my_schema_spec.rb +3 -3
- data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb +1 -1
- data/spec/producer_spec.rb +1 -1
- data/spec/schemas/my_namespace/my_schema_with_complex_type.rb +3 -3
- data/spec/snapshots/consumers-no-nest.snap +1 -1
- data/spec/snapshots/consumers.snap +1 -1
- data/spec/snapshots/consumers_and_producers-no-nest.snap +3 -3
- data/spec/snapshots/consumers_and_producers.snap +3 -3
- data/spec/snapshots/consumers_circular-no-nest.snap +1 -1
- data/spec/snapshots/consumers_circular.snap +1 -1
- data/spec/snapshots/consumers_complex_types-no-nest.snap +1 -1
- data/spec/snapshots/consumers_complex_types.snap +1 -1
- data/spec/snapshots/consumers_nested-no-nest.snap +1 -1
- data/spec/snapshots/consumers_nested.snap +1 -1
- data/spec/snapshots/namespace_folders.snap +3 -3
- data/spec/snapshots/producers_with_key-no-nest.snap +1 -1
- data/spec/snapshots/producers_with_key.snap +1 -1
- metadata +39 -21
- data/.gemfiles/avro_turf-0.gemfile +0 -3
- data/.gemfiles/avro_turf-1.gemfile +0 -3
- data/.ruby-gemset +0 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 021fca59477fbfa890fa77ae867cd3a006b65e2c0235d1bb47ccdd1780e51d9c
|
4
|
+
data.tar.gz: 8330ba3d8f90567c7abbfd91298dd04d80f9507f702a0d4ac45300f68e5d224a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ab861eb37e8590664f47e0e11082ed5aac344a730c009c5261edcbdcbfcd00a39bc8e1f44bf2be5c05b1eb587ba15ad32e77edfd80ca0789da05049c91dd20e9
|
7
|
+
data.tar.gz: 851b9b844d11be5d90d3b4e652a648e7ce1e8e90e1ddd96f25a34d690cb89e2503302a80ad9cb891cbaa30949a0394e68337b34de86cf848a6815730d1548ca0
|
data/.github/workflows/ci.yml
CHANGED
@@ -25,18 +25,18 @@ jobs:
|
|
25
25
|
ruby-version: 2.7
|
26
26
|
bundler-cache: true
|
27
27
|
|
28
|
+
- name: Bundle install
|
29
|
+
run: bundle install --with development
|
30
|
+
|
28
31
|
- name: Rubocop
|
29
32
|
run: bundle exec rubocop --format progress
|
30
33
|
|
31
34
|
build:
|
32
35
|
runs-on: ubuntu-latest
|
33
|
-
env:
|
34
|
-
BUNDLE_GEMFILE: ${{ github.workspace }}/.gemfiles/${{ matrix.gemfile }}.gemfile
|
35
36
|
strategy:
|
36
37
|
fail-fast: false
|
37
38
|
matrix:
|
38
39
|
ruby: [ '2.6', '2.7', '3.0', '3.1' ]
|
39
|
-
gemfile: [ 'avro_turf-0', 'avro_turf-1' ]
|
40
40
|
|
41
41
|
steps:
|
42
42
|
- uses: actions/checkout@v2
|
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
@@ -1,7 +1,9 @@
|
|
1
|
+
inherit_from: .rubocop_todo.yml
|
2
|
+
|
1
3
|
require: rubocop-rspec
|
2
4
|
|
3
5
|
AllCops:
|
4
|
-
TargetRubyVersion: 2.
|
6
|
+
TargetRubyVersion: 2.5
|
5
7
|
Exclude:
|
6
8
|
- lib/deimos/monkey_patches/*.rb
|
7
9
|
- vendor/**/*
|
@@ -39,8 +41,9 @@ Layout/EmptyLinesAroundBlockBody:
|
|
39
41
|
Layout/LineLength:
|
40
42
|
Max: 120
|
41
43
|
Severity: refactor
|
42
|
-
|
43
|
-
|
44
|
+
# TODO: uncomment the files below once rubocop is fixed
|
45
|
+
# Exclude:
|
46
|
+
# - 'spec/**/*'
|
44
47
|
|
45
48
|
# foo = if expression
|
46
49
|
# 'bar'
|
@@ -149,18 +152,20 @@ Style/CollectionMethods:
|
|
149
152
|
Style/DateTime:
|
150
153
|
AllowCoercion: true
|
151
154
|
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
- 'app/
|
156
|
-
- '
|
155
|
+
# TODO: uncomment the lines below once rubocop is fixed
|
156
|
+
#Style/Documentation:
|
157
|
+
# Exclude:
|
158
|
+
# - 'app/controllers/**/*'
|
159
|
+
# - 'app/helpers/**/*'
|
160
|
+
# - 'db/**/*'
|
157
161
|
|
158
162
|
# Force documentation for public methods and classes
|
159
163
|
Style/DocumentationMethod:
|
160
164
|
Enabled: true
|
161
|
-
|
162
|
-
|
163
|
-
- '
|
165
|
+
# TODO: uncomment the file below once rubocop is fixed
|
166
|
+
# Exclude:
|
167
|
+
# - 'app/controllers/**/*'
|
168
|
+
# - 'db/**/*'
|
164
169
|
|
165
170
|
# Allow else with just nil in it
|
166
171
|
Style/EmptyElse:
|
@@ -231,9 +236,10 @@ Style/MethodCallWithArgsParentheses:
|
|
231
236
|
- before_destroy
|
232
237
|
- after_destroy
|
233
238
|
- queue_as
|
234
|
-
|
235
|
-
|
236
|
-
- '
|
239
|
+
# TODO: uncomment the file below once rubocop is fixed
|
240
|
+
# Exclude:
|
241
|
+
# - 'bin/**/*'
|
242
|
+
# - 'Gemfile'
|
237
243
|
|
238
244
|
# Do not allow "end.something"
|
239
245
|
Style/MethodCalledOnDoEndBlock:
|
data/.rubocop_todo.yml
ADDED
@@ -0,0 +1,364 @@
|
|
1
|
+
# This configuration was generated by
|
2
|
+
# `rubocop --auto-gen-config --auto-gen-only-exclude`
|
3
|
+
# on 2022-09-08 19:46:52 UTC using RuboCop version 0.90.0.
|
4
|
+
# The point is for the user to remove these configuration records
|
5
|
+
# one by one as the offenses are removed from the code base.
|
6
|
+
# Note that changes in the inspected code, or installation of new
|
7
|
+
# versions of RuboCop, may require this file to be generated again.
|
8
|
+
|
9
|
+
# Offense count: 1
|
10
|
+
# Configuration parameters: Include.
|
11
|
+
# Include: **/*.gemspec
|
12
|
+
Gemspec/RequiredRubyVersion:
|
13
|
+
Exclude:
|
14
|
+
- 'deimos-ruby.gemspec'
|
15
|
+
|
16
|
+
# Offense count: 4
|
17
|
+
# Cop supports --auto-correct.
|
18
|
+
Layout/EmptyLines:
|
19
|
+
Exclude:
|
20
|
+
- 'spec/schemas/my_namespace/generated.rb'
|
21
|
+
- 'spec/schemas/my_namespace/my_nested_schema.rb'
|
22
|
+
- 'spec/schemas/my_namespace/my_schema_with_circular_reference.rb'
|
23
|
+
- 'spec/schemas/my_namespace/my_schema_with_complex_type.rb'
|
24
|
+
|
25
|
+
# Offense count: 32
|
26
|
+
# Cop supports --auto-correct.
|
27
|
+
# Configuration parameters: EnforcedStyle.
|
28
|
+
# SupportedStyles: empty_lines, empty_lines_except_namespace, empty_lines_special, no_empty_lines, beginning_only, ending_only
|
29
|
+
Layout/EmptyLinesAroundClassBody:
|
30
|
+
Enabled: false
|
31
|
+
|
32
|
+
# Offense count: 1
|
33
|
+
# Cop supports --auto-correct.
|
34
|
+
Layout/EmptyLinesAroundMethodBody:
|
35
|
+
Exclude:
|
36
|
+
- 'lib/generators/deimos/schema_class_generator.rb'
|
37
|
+
|
38
|
+
# Offense count: 42
|
39
|
+
# Cop supports --auto-correct.
|
40
|
+
# Configuration parameters: EnforcedStyleAlignWith, Severity.
|
41
|
+
# SupportedStylesAlignWith: keyword, variable, start_of_line
|
42
|
+
Layout/EndAlignment:
|
43
|
+
Enabled: false
|
44
|
+
|
45
|
+
# Offense count: 4
|
46
|
+
# Cop supports --auto-correct.
|
47
|
+
# Configuration parameters: EnforcedStyle, IndentationWidth.
|
48
|
+
# SupportedStyles: special_inside_parentheses, consistent, align_braces
|
49
|
+
Layout/FirstHashElementIndentation:
|
50
|
+
Exclude:
|
51
|
+
- 'spec/producer_spec.rb'
|
52
|
+
|
53
|
+
# Offense count: 19
|
54
|
+
# Cop supports --auto-correct.
|
55
|
+
# Configuration parameters: AllowMultipleStyles, EnforcedHashRocketStyle, EnforcedColonStyle, EnforcedLastArgumentHashStyle.
|
56
|
+
# SupportedHashRocketStyles: key, separator, table
|
57
|
+
# SupportedColonStyles: key, separator, table
|
58
|
+
# SupportedLastArgumentHashStyles: always_inspect, always_ignore, ignore_implicit, ignore_explicit
|
59
|
+
Layout/HashAlignment:
|
60
|
+
Exclude:
|
61
|
+
- 'spec/consumer_spec.rb'
|
62
|
+
- 'spec/generators/schema_class/my_schema_with_circular_reference_spec.rb'
|
63
|
+
|
64
|
+
# Offense count: 21
|
65
|
+
# Cop supports --auto-correct.
|
66
|
+
# Configuration parameters: Width, IgnoredPatterns.
|
67
|
+
Layout/IndentationWidth:
|
68
|
+
Enabled: false
|
69
|
+
|
70
|
+
# Offense count: 9
|
71
|
+
# Cop supports --auto-correct.
|
72
|
+
# Configuration parameters: AllowForAlignment, EnforcedStyleForExponentOperator.
|
73
|
+
# SupportedStylesForExponentOperator: space, no_space
|
74
|
+
Layout/SpaceAroundOperators:
|
75
|
+
Exclude:
|
76
|
+
- 'spec/generators/schema_class/my_schema_with_circular_reference_spec.rb'
|
77
|
+
- 'spec/generators/schema_class_generator_spec.rb'
|
78
|
+
- 'spec/schemas/my_namespace/my_schema_with_complex_type.rb'
|
79
|
+
|
80
|
+
# Offense count: 1
|
81
|
+
# Cop supports --auto-correct.
|
82
|
+
# Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBraces, SpaceBeforeBlockParameters.
|
83
|
+
# SupportedStyles: space, no_space
|
84
|
+
# SupportedStylesForEmptyBraces: space, no_space
|
85
|
+
Layout/SpaceInsideBlockBraces:
|
86
|
+
Exclude:
|
87
|
+
- 'spec/generators/schema_class_generator_spec.rb'
|
88
|
+
|
89
|
+
# Offense count: 1
|
90
|
+
# Cop supports --auto-correct.
|
91
|
+
# Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBraces.
|
92
|
+
# SupportedStyles: space, no_space, compact
|
93
|
+
# SupportedStylesForEmptyBraces: space, no_space
|
94
|
+
Layout/SpaceInsideHashLiteralBraces:
|
95
|
+
Exclude:
|
96
|
+
- 'spec/active_record_producer_spec.rb'
|
97
|
+
|
98
|
+
# Offense count: 1
|
99
|
+
# Cop supports --auto-correct.
|
100
|
+
# Configuration parameters: EnforcedStyle.
|
101
|
+
# SupportedStyles: space, no_space
|
102
|
+
Layout/SpaceInsideParens:
|
103
|
+
Exclude:
|
104
|
+
- 'lib/deimos.rb'
|
105
|
+
|
106
|
+
# Offense count: 23
|
107
|
+
# Cop supports --auto-correct.
|
108
|
+
# Configuration parameters: AllowInHeredoc.
|
109
|
+
Layout/TrailingWhitespace:
|
110
|
+
Exclude:
|
111
|
+
- 'spec/schemas/my_namespace/generated.rb'
|
112
|
+
- 'spec/schemas/my_namespace/my_nested_schema.rb'
|
113
|
+
- 'spec/schemas/my_namespace/my_schema_with_circular_reference.rb'
|
114
|
+
- 'spec/schemas/my_namespace/my_schema_with_complex_type.rb'
|
115
|
+
|
116
|
+
# Offense count: 5
|
117
|
+
Lint/MissingSuper:
|
118
|
+
Exclude:
|
119
|
+
- 'lib/deimos/active_record_consumer.rb'
|
120
|
+
- 'lib/deimos/metrics/datadog.rb'
|
121
|
+
- 'lib/deimos/schema_class/enum.rb'
|
122
|
+
- 'lib/deimos/tracing/datadog.rb'
|
123
|
+
- 'lib/deimos/tracing/mock.rb'
|
124
|
+
|
125
|
+
# Offense count: 1
|
126
|
+
# Cop supports --auto-correct.
|
127
|
+
Lint/RedundantCopDisableDirective:
|
128
|
+
Exclude:
|
129
|
+
- 'spec/spec_helper.rb'
|
130
|
+
|
131
|
+
# Offense count: 3
|
132
|
+
# Cop supports --auto-correct.
|
133
|
+
# Configuration parameters: AllowUnusedKeywordArguments, IgnoreEmptyMethods, IgnoreNotImplementedMethods.
|
134
|
+
Lint/UnusedMethodArgument:
|
135
|
+
Exclude:
|
136
|
+
- 'lib/deimos/tracing/mock.rb'
|
137
|
+
- 'spec/schemas/my_namespace/my_schema_with_circular_reference.rb'
|
138
|
+
|
139
|
+
# Offense count: 1
|
140
|
+
Lint/UselessAssignment:
|
141
|
+
Exclude:
|
142
|
+
- 'spec/generators/schema_class_generator_spec.rb'
|
143
|
+
|
144
|
+
# Offense count: 19
|
145
|
+
# Configuration parameters: IgnoredMethods, Max.
|
146
|
+
Metrics/AbcSize:
|
147
|
+
Exclude:
|
148
|
+
- 'lib/deimos/active_record_consume/message_consumption.rb'
|
149
|
+
- 'lib/deimos/config/phobos_config.rb'
|
150
|
+
- 'lib/deimos/instrumentation.rb'
|
151
|
+
- 'lib/deimos/kafka_source.rb'
|
152
|
+
- 'lib/deimos/kafka_topic_info.rb'
|
153
|
+
- 'lib/deimos/producer.rb'
|
154
|
+
- 'lib/deimos/schema_backends/avro_schema_coercer.rb'
|
155
|
+
- 'lib/deimos/test_helpers.rb'
|
156
|
+
- 'lib/deimos/utils/db_poller.rb'
|
157
|
+
- 'lib/deimos/utils/db_producer.rb'
|
158
|
+
- 'lib/deimos/utils/inline_consumer.rb'
|
159
|
+
- 'lib/deimos/utils/schema_controller_mixin.rb'
|
160
|
+
- 'lib/generators/deimos/schema_class_generator.rb'
|
161
|
+
|
162
|
+
# Offense count: 1
|
163
|
+
# Configuration parameters: CountComments, Max, CountAsOne, ExcludedMethods.
|
164
|
+
Metrics/MethodLength:
|
165
|
+
Exclude:
|
166
|
+
- 'lib/deimos/config/phobos_config.rb'
|
167
|
+
|
168
|
+
# Offense count: 5
|
169
|
+
# Configuration parameters: CountComments, Max, CountAsOne.
|
170
|
+
Metrics/ModuleLength:
|
171
|
+
Exclude:
|
172
|
+
- 'lib/deimos/test_helpers.rb'
|
173
|
+
- 'spec/active_record_batch_consumer_spec.rb'
|
174
|
+
- 'spec/batch_consumer_spec.rb'
|
175
|
+
- 'spec/kafka_source_spec.rb'
|
176
|
+
- 'spec/producer_spec.rb'
|
177
|
+
|
178
|
+
# Offense count: 6
|
179
|
+
# Configuration parameters: IgnoredMethods, Max.
|
180
|
+
Metrics/PerceivedComplexity:
|
181
|
+
Exclude:
|
182
|
+
- 'lib/deimos/config/phobos_config.rb'
|
183
|
+
- 'lib/deimos/consume/batch_consumption.rb'
|
184
|
+
- 'lib/deimos/kafka_source.rb'
|
185
|
+
- 'lib/deimos/schema_backends/avro_schema_coercer.rb'
|
186
|
+
- 'lib/deimos/test_helpers.rb'
|
187
|
+
|
188
|
+
# Offense count: 12
|
189
|
+
# Configuration parameters: Prefixes.
|
190
|
+
# Prefixes: when, with, without
|
191
|
+
RSpec/ContextWording:
|
192
|
+
Exclude:
|
193
|
+
- 'spec/generators/schema_class_generator_spec.rb'
|
194
|
+
|
195
|
+
# Offense count: 4
|
196
|
+
# Configuration parameters: Max.
|
197
|
+
RSpec/ExampleLength:
|
198
|
+
Exclude:
|
199
|
+
- 'spec/config/configuration_spec.rb'
|
200
|
+
- 'spec/utils/db_producer_spec.rb'
|
201
|
+
|
202
|
+
# Offense count: 6
|
203
|
+
# Configuration parameters: Max.
|
204
|
+
RSpec/MultipleExpectations:
|
205
|
+
Exclude:
|
206
|
+
- 'spec/active_record_consumer_spec.rb'
|
207
|
+
- 'spec/config/configuration_spec.rb'
|
208
|
+
- 'spec/kafka_topic_info_spec.rb'
|
209
|
+
- 'spec/utils/db_poller_spec.rb'
|
210
|
+
- 'spec/utils/db_producer_spec.rb'
|
211
|
+
|
212
|
+
# Offense count: 1
|
213
|
+
Style/Documentation:
|
214
|
+
Exclude:
|
215
|
+
- 'spec/**/*'
|
216
|
+
- 'test/**/*'
|
217
|
+
- 'regenerate_test_schema_classes.rb'
|
218
|
+
|
219
|
+
# Offense count: 1
|
220
|
+
# Configuration parameters: RequireForNonPublicMethods.
|
221
|
+
Style/DocumentationMethod:
|
222
|
+
Exclude:
|
223
|
+
- 'spec/**/*'
|
224
|
+
- 'test/**/*'
|
225
|
+
- 'regenerate_test_schema_classes.rb'
|
226
|
+
|
227
|
+
# Offense count: 1
|
228
|
+
# Cop supports --auto-correct.
|
229
|
+
# Configuration parameters: EnforcedStyle.
|
230
|
+
# SupportedStyles: compact, expanded
|
231
|
+
Style/EmptyMethod:
|
232
|
+
Exclude:
|
233
|
+
- 'regenerate_test_schema_classes.rb'
|
234
|
+
|
235
|
+
# Offense count: 2
|
236
|
+
# Cop supports --auto-correct.
|
237
|
+
Style/ExplicitBlockArgument:
|
238
|
+
Exclude:
|
239
|
+
- 'lib/deimos/metrics/datadog.rb'
|
240
|
+
- 'lib/deimos/utils/deadlock_retry.rb'
|
241
|
+
|
242
|
+
# Offense count: 2
|
243
|
+
# Cop supports --auto-correct.
|
244
|
+
# Configuration parameters: EnforcedStyle.
|
245
|
+
# SupportedStyles: always, always_true, never
|
246
|
+
Style/FrozenStringLiteralComment:
|
247
|
+
Exclude:
|
248
|
+
- 'regenerate_test_schema_classes.rb'
|
249
|
+
- 'spec/generators/schema_class/my_schema_spec.rb'
|
250
|
+
|
251
|
+
# Offense count: 10
|
252
|
+
# Cop supports --auto-correct.
|
253
|
+
Style/GlobalStdStream:
|
254
|
+
Exclude:
|
255
|
+
- 'lib/deimos/config/configuration.rb'
|
256
|
+
- 'lib/deimos/config/phobos_config.rb'
|
257
|
+
- 'lib/deimos/metrics/mock.rb'
|
258
|
+
- 'lib/deimos/test_helpers.rb'
|
259
|
+
- 'lib/deimos/tracing/mock.rb'
|
260
|
+
- 'lib/deimos/utils/db_producer.rb'
|
261
|
+
- 'lib/tasks/deimos.rake'
|
262
|
+
- 'spec/rake_spec.rb'
|
263
|
+
|
264
|
+
# Offense count: 1
|
265
|
+
# Cop supports --auto-correct.
|
266
|
+
Style/HashEachMethods:
|
267
|
+
Exclude:
|
268
|
+
- 'lib/generators/deimos/schema_class_generator.rb'
|
269
|
+
|
270
|
+
# Offense count: 31
|
271
|
+
# Cop supports --auto-correct.
|
272
|
+
# Configuration parameters: IgnoreMacros, IgnoredMethods, IgnoredPatterns, IncludedMacros, AllowParenthesesInMultilineCall, AllowParenthesesInChaining, AllowParenthesesInCamelCaseMethod, EnforcedStyle.
|
273
|
+
# SupportedStyles: require_parentheses, omit_parentheses
|
274
|
+
Style/MethodCallWithArgsParentheses:
|
275
|
+
Exclude:
|
276
|
+
- 'lib/deimos/utils/db_poller.rb'
|
277
|
+
- 'lib/deimos/utils/inline_consumer.rb'
|
278
|
+
- 'spec/config/configuration_spec.rb'
|
279
|
+
- 'spec/deimos_spec.rb'
|
280
|
+
- 'spec/generators/schema_class_generator_spec.rb'
|
281
|
+
|
282
|
+
# Offense count: 1
|
283
|
+
Style/OptionalBooleanParameter:
|
284
|
+
Exclude:
|
285
|
+
- 'lib/deimos/test_helpers.rb'
|
286
|
+
|
287
|
+
# Offense count: 1
|
288
|
+
# Cop supports --auto-correct.
|
289
|
+
Style/RedundantAssignment:
|
290
|
+
Exclude:
|
291
|
+
- 'spec/schemas/my_namespace/my_schema_with_circular_reference.rb'
|
292
|
+
|
293
|
+
# Offense count: 2
|
294
|
+
# Cop supports --auto-correct.
|
295
|
+
Style/RedundantBegin:
|
296
|
+
Exclude:
|
297
|
+
- 'lib/deimos.rb'
|
298
|
+
- 'lib/deimos/utils/db_producer.rb'
|
299
|
+
|
300
|
+
# Offense count: 1
|
301
|
+
# Cop supports --auto-correct.
|
302
|
+
Style/SingleArgumentDig:
|
303
|
+
Exclude:
|
304
|
+
- 'lib/deimos/active_record_consume/batch_slicer.rb'
|
305
|
+
|
306
|
+
# Offense count: 1
|
307
|
+
# Cop supports --auto-correct.
|
308
|
+
# Configuration parameters: AllowIfMethodIsEmpty.
|
309
|
+
Style/SingleLineMethods:
|
310
|
+
Exclude:
|
311
|
+
- 'regenerate_test_schema_classes.rb'
|
312
|
+
|
313
|
+
# Offense count: 5
|
314
|
+
# Cop supports --auto-correct.
|
315
|
+
Style/StringConcatenation:
|
316
|
+
Exclude:
|
317
|
+
- 'spec/producer_spec.rb'
|
318
|
+
- 'spec/utils/schema_controller_mixin_spec.rb'
|
319
|
+
|
320
|
+
# Offense count: 10
|
321
|
+
# Cop supports --auto-correct.
|
322
|
+
# Configuration parameters: EnforcedStyle, ConsistentQuotesInMultiline.
|
323
|
+
# SupportedStyles: single_quotes, double_quotes
|
324
|
+
Style/StringLiterals:
|
325
|
+
Exclude:
|
326
|
+
- 'lib/generators/deimos/schema_class_generator.rb'
|
327
|
+
- 'lib/tasks/deimos.rake'
|
328
|
+
- 'regenerate_test_schema_classes.rb'
|
329
|
+
- 'spec/schemas/my_namespace/my_schema_with_complex_type.rb'
|
330
|
+
- 'spec/spec_helper.rb'
|
331
|
+
|
332
|
+
# Offense count: 1
|
333
|
+
# Cop supports --auto-correct.
|
334
|
+
# Configuration parameters: EnforcedStyle, AllowSafeAssignment.
|
335
|
+
# SupportedStyles: require_parentheses, require_no_parentheses, require_parentheses_when_complex
|
336
|
+
Style/TernaryParentheses:
|
337
|
+
Exclude:
|
338
|
+
- 'lib/deimos/config/phobos_config.rb'
|
339
|
+
|
340
|
+
# Offense count: 21
|
341
|
+
# Cop supports --auto-correct.
|
342
|
+
Style/TrailingBodyOnModule:
|
343
|
+
Enabled: false
|
344
|
+
|
345
|
+
# Offense count: 1
|
346
|
+
# Cop supports --auto-correct.
|
347
|
+
# Configuration parameters: EnforcedStyleForMultiline.
|
348
|
+
# SupportedStylesForMultiline: comma, consistent_comma, no_comma
|
349
|
+
Style/TrailingCommaInHashLiteral:
|
350
|
+
Exclude:
|
351
|
+
- 'spec/config/configuration_spec.rb'
|
352
|
+
|
353
|
+
# Offense count: 8
|
354
|
+
# Cop supports --auto-correct.
|
355
|
+
# Configuration parameters: AutoCorrect, Max, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
|
356
|
+
# URISchemes: http, https
|
357
|
+
Layout/LineLength:
|
358
|
+
Exclude:
|
359
|
+
- 'lib/deimos/producer.rb'
|
360
|
+
- 'lib/deimos/utils/db_poller.rb'
|
361
|
+
- 'lib/deimos/utils/db_producer.rb'
|
362
|
+
- 'spec/active_record_batch_consumer_spec.rb'
|
363
|
+
- 'spec/generators/schema_class/my_schema_with_circular_reference_spec.rb'
|
364
|
+
- 'spec/generators/schema_class/my_schema_with_complex_types_spec.rb'
|
data/.ruby-version
CHANGED
@@ -1 +1,2 @@
|
|
1
|
-
|
1
|
+
3.1.0
|
2
|
+
|
data/CHANGELOG.md
CHANGED
@@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
7
7
|
|
8
8
|
## UNRELEASED
|
9
9
|
|
10
|
+
# 1.16.5 - 2022-09-30
|
11
|
+
|
12
|
+
- Fix the linting step in the CI
|
13
|
+
|
14
|
+
# 1.16.4 - 2022-09-09
|
15
|
+
|
16
|
+
- Now generates RBS types.
|
17
|
+
- Use `allocate` instead of `new` in `tombstone` to avoid issues with required fields in `initialize`.
|
18
|
+
|
10
19
|
# 1.16.3 - 2022-09-08
|
11
20
|
|
12
21
|
- Add the `tombstone` method to schema classes.
|
data/Gemfile
CHANGED
@@ -4,3 +4,9 @@ source 'https://rubygems.org'
|
|
4
4
|
|
5
5
|
# Specify your gem's dependencies in boilerplate.gemspec
|
6
6
|
gemspec
|
7
|
+
|
8
|
+
if !ENV['CI'] || ENV['CI'] == ''
|
9
|
+
# TODO: once all PRs are merged, add this to gemspec as a development dependency
|
10
|
+
gem 'sord', git: 'git@github.com:dorner/sord.git', ref: 'local-develop'
|
11
|
+
end
|
12
|
+
|
data/README.md
CHANGED
@@ -1198,10 +1198,16 @@ decoded = Deimos.decode(schema: 'MySchema', namespace: 'com.my-namespace', paylo
|
|
1198
1198
|
|
1199
1199
|
Bug reports and pull requests are welcome on GitHub at https://github.com/flipp-oss/deimos .
|
1200
1200
|
|
1201
|
+
You can/should re-generate RBS types when methods or classes change by running the following:
|
1202
|
+
|
1203
|
+
rbs collection install # if you haven't done it
|
1204
|
+
rbs collection update
|
1205
|
+
bundle exec sord --hide-private --no-sord-comments sig/defs.rbs --tags 'override:Override'
|
1206
|
+
|
1201
1207
|
### Linting
|
1202
1208
|
|
1203
1209
|
Deimos uses Rubocop to lint the code. Please run Rubocop on your code
|
1204
|
-
before submitting a PR.
|
1210
|
+
before submitting a PR. The GitHub CI will also run rubocop on your pull request.
|
1205
1211
|
|
1206
1212
|
---
|
1207
1213
|
<p align="center">
|
data/Steepfile
ADDED
data/deimos-ruby.gemspec
CHANGED
@@ -18,11 +18,11 @@ Gem::Specification.new do |spec|
|
|
18
18
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
19
|
spec.require_paths = ['lib']
|
20
20
|
|
21
|
-
spec.add_runtime_dependency('avro_turf', '>=
|
21
|
+
spec.add_runtime_dependency('avro_turf', '>= 1.4', '< 2')
|
22
|
+
spec.add_runtime_dependency('fig_tree', '~> 0.0.2')
|
22
23
|
spec.add_runtime_dependency('phobos', '>= 1.9', '< 3.0')
|
23
24
|
spec.add_runtime_dependency('ruby-kafka', '< 2')
|
24
25
|
spec.add_runtime_dependency('sigurd', '>= 0.1.0', '< 1.0')
|
25
|
-
spec.add_runtime_dependency('fig_tree', '~> 0.0.2')
|
26
26
|
|
27
27
|
spec.add_development_dependency('activerecord-import')
|
28
28
|
spec.add_development_dependency('avro', '~> 1.9')
|
@@ -43,4 +43,5 @@ Gem::Specification.new do |spec|
|
|
43
43
|
spec.add_development_dependency('rubocop', '0.89.0')
|
44
44
|
spec.add_development_dependency('rubocop-rspec', '1.42.0')
|
45
45
|
spec.add_development_dependency('sqlite3', '~> 1.3')
|
46
|
+
spec.add_development_dependency('steep', '~> 1.0')
|
46
47
|
end
|
@@ -15,8 +15,9 @@ module Deimos
|
|
15
15
|
# If two messages in a batch have the same key, we cannot process them
|
16
16
|
# in the same operation as they would interfere with each other. Thus
|
17
17
|
# they are split
|
18
|
-
# @param payloads [Array<Hash
|
18
|
+
# @param payloads [Array<Hash,Deimos::SchemaClass::Record>] Decoded payloads
|
19
19
|
# @param metadata [Hash] Information about batch, including keys.
|
20
|
+
# @return [void]
|
20
21
|
def consume_batch(payloads, metadata)
|
21
22
|
messages = payloads.
|
22
23
|
zip(metadata[:keys]).
|
@@ -59,6 +60,7 @@ module Deimos
|
|
59
60
|
# All messages are split into slices containing only unique keys, and
|
60
61
|
# each slice is handles as its own batch.
|
61
62
|
# @param messages [Array<Message>] List of messages.
|
63
|
+
# @return [void]
|
62
64
|
def uncompacted_update(messages)
|
63
65
|
BatchSlicer.
|
64
66
|
slice(messages).
|
@@ -69,6 +71,7 @@ module Deimos
|
|
69
71
|
# All messages with payloads are passed to upsert_records.
|
70
72
|
# All tombstones messages are passed to remove_records.
|
71
73
|
# @param messages [Array<Message>] List of messages.
|
74
|
+
# @return [void]
|
72
75
|
def update_database(messages)
|
73
76
|
# Find all upserted records (i.e. that have a payload) and all
|
74
77
|
# deleted record (no payload)
|
@@ -81,6 +84,7 @@ module Deimos
|
|
81
84
|
# Upsert any non-deleted records
|
82
85
|
# @param messages [Array<Message>] List of messages for a group of
|
83
86
|
# records to either be updated or inserted.
|
87
|
+
# @return [void]
|
84
88
|
def upsert_records(messages)
|
85
89
|
key_cols = key_columns(messages)
|
86
90
|
|
@@ -119,6 +123,7 @@ module Deimos
|
|
119
123
|
# Delete any records with a tombstone.
|
120
124
|
# @param messages [Array<Message>] List of messages for a group of
|
121
125
|
# deleted records.
|
126
|
+
# @return [void]
|
122
127
|
def remove_records(messages)
|
123
128
|
clause = deleted_query(messages)
|
124
129
|
|
@@ -128,7 +133,7 @@ module Deimos
|
|
128
133
|
# Create an ActiveRecord relation that matches all of the passed
|
129
134
|
# records. Used for bulk deletion.
|
130
135
|
# @param records [Array<Message>] List of messages.
|
131
|
-
# @return ActiveRecord::Relation Matching relation.
|
136
|
+
# @return [ActiveRecord::Relation] Matching relation.
|
132
137
|
def deleted_query(records)
|
133
138
|
keys = records.
|
134
139
|
map { |m| record_key(m.key)[@klass.primary_key] }.
|
@@ -11,6 +11,8 @@ module Deimos
|
|
11
11
|
# slices that maintain the correct order.
|
12
12
|
# E.g. Given messages A1, A2, B1, C1, C2, C3, they will be sliced as:
|
13
13
|
# [[A1, B1, C1], [A2, C2], [C3]]
|
14
|
+
# @param messages [Array<Message>]
|
15
|
+
# @return [Array<Array<Message>>]
|
14
16
|
def self.slice(messages)
|
15
17
|
ops = messages.group_by(&:key)
|
16
18
|
|
@@ -8,8 +8,8 @@ module Deimos
|
|
8
8
|
# Find the record specified by the given payload and key.
|
9
9
|
# Default is to use the primary key column and the value of the first
|
10
10
|
# field in the key.
|
11
|
-
# @param klass [Class
|
12
|
-
# @param _payload [Hash
|
11
|
+
# @param klass [Class<ActiveRecord::Base>]
|
12
|
+
# @param _payload [Hash,Deimos::SchemaClass::Record]
|
13
13
|
# @param key [Object]
|
14
14
|
# @return [ActiveRecord::Base]
|
15
15
|
def fetch_record(klass, _payload, key)
|
@@ -19,14 +19,16 @@ module Deimos
|
|
19
19
|
|
20
20
|
# Assign a key to a new record.
|
21
21
|
# @param record [ActiveRecord::Base]
|
22
|
-
# @param _payload [Hash
|
22
|
+
# @param _payload [Hash,Deimos::SchemaClass::Record]
|
23
23
|
# @param key [Object]
|
24
|
+
# @return [void]
|
24
25
|
def assign_key(record, _payload, key)
|
25
26
|
record[record.class.primary_key] = key
|
26
27
|
end
|
27
28
|
|
28
|
-
# @param payload [Hash
|
29
|
+
# @param payload [Hash,Deimos::SchemaClass::Record] Decoded payloads
|
29
30
|
# @param metadata [Hash] Information about batch, including keys.
|
31
|
+
# @return [void]
|
30
32
|
def consume(payload, metadata)
|
31
33
|
unless self.process_message?(payload)
|
32
34
|
Deimos.config.logger.debug(
|
@@ -64,6 +66,7 @@ module Deimos
|
|
64
66
|
end
|
65
67
|
|
66
68
|
# @param record [ActiveRecord::Base]
|
69
|
+
# @return [void]
|
67
70
|
def save_record(record)
|
68
71
|
record.created_at ||= Time.zone.now if record.respond_to?(:created_at)
|
69
72
|
record.updated_at = Time.zone.now if record.respond_to?(:updated_at)
|
@@ -73,6 +76,7 @@ module Deimos
|
|
73
76
|
# Destroy a record that received a null payload. Override if you need
|
74
77
|
# to do something other than a straight destroy (e.g. mark as archived).
|
75
78
|
# @param record [ActiveRecord::Base]
|
79
|
+
# @return [void]
|
76
80
|
def destroy_record(record)
|
77
81
|
record&.destroy
|
78
82
|
end
|