deimos-kafka 1.0.0.pre.beta15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. checksums.yaml +7 -0
  2. data/.circleci/config.yml +74 -0
  3. data/.gitignore +41 -0
  4. data/.gitmodules +0 -0
  5. data/.rspec +1 -0
  6. data/.rubocop.yml +321 -0
  7. data/.ruby-gemset +1 -0
  8. data/.ruby-version +1 -0
  9. data/CHANGELOG.md +9 -0
  10. data/CODE_OF_CONDUCT.md +77 -0
  11. data/Dockerfile +23 -0
  12. data/Gemfile +6 -0
  13. data/Gemfile.lock +165 -0
  14. data/Guardfile +22 -0
  15. data/LICENSE.md +195 -0
  16. data/README.md +742 -0
  17. data/Rakefile +13 -0
  18. data/bin/deimos +4 -0
  19. data/deimos-kafka.gemspec +42 -0
  20. data/docker-compose.yml +71 -0
  21. data/docs/DATABASE_BACKEND.md +147 -0
  22. data/docs/PULL_REQUEST_TEMPLATE.md +34 -0
  23. data/lib/deimos.rb +134 -0
  24. data/lib/deimos/active_record_consumer.rb +81 -0
  25. data/lib/deimos/active_record_producer.rb +64 -0
  26. data/lib/deimos/avro_data_coder.rb +89 -0
  27. data/lib/deimos/avro_data_decoder.rb +36 -0
  28. data/lib/deimos/avro_data_encoder.rb +51 -0
  29. data/lib/deimos/backends/db.rb +27 -0
  30. data/lib/deimos/backends/kafka.rb +27 -0
  31. data/lib/deimos/backends/kafka_async.rb +27 -0
  32. data/lib/deimos/configuration.rb +88 -0
  33. data/lib/deimos/consumer.rb +164 -0
  34. data/lib/deimos/instrumentation.rb +71 -0
  35. data/lib/deimos/kafka_message.rb +27 -0
  36. data/lib/deimos/kafka_source.rb +126 -0
  37. data/lib/deimos/kafka_topic_info.rb +79 -0
  38. data/lib/deimos/message.rb +74 -0
  39. data/lib/deimos/metrics/datadog.rb +47 -0
  40. data/lib/deimos/metrics/mock.rb +39 -0
  41. data/lib/deimos/metrics/provider.rb +38 -0
  42. data/lib/deimos/monkey_patches/phobos_cli.rb +35 -0
  43. data/lib/deimos/monkey_patches/phobos_producer.rb +51 -0
  44. data/lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb +85 -0
  45. data/lib/deimos/monkey_patches/schema_store.rb +19 -0
  46. data/lib/deimos/producer.rb +218 -0
  47. data/lib/deimos/publish_backend.rb +30 -0
  48. data/lib/deimos/railtie.rb +8 -0
  49. data/lib/deimos/schema_coercer.rb +108 -0
  50. data/lib/deimos/shared_config.rb +59 -0
  51. data/lib/deimos/test_helpers.rb +356 -0
  52. data/lib/deimos/tracing/datadog.rb +35 -0
  53. data/lib/deimos/tracing/mock.rb +40 -0
  54. data/lib/deimos/tracing/provider.rb +31 -0
  55. data/lib/deimos/utils/db_producer.rb +95 -0
  56. data/lib/deimos/utils/executor.rb +117 -0
  57. data/lib/deimos/utils/inline_consumer.rb +144 -0
  58. data/lib/deimos/utils/lag_reporter.rb +182 -0
  59. data/lib/deimos/utils/platform_schema_validation.rb +0 -0
  60. data/lib/deimos/utils/signal_handler.rb +68 -0
  61. data/lib/deimos/version.rb +5 -0
  62. data/lib/generators/deimos/db_backend/templates/migration +24 -0
  63. data/lib/generators/deimos/db_backend/templates/rails3_migration +30 -0
  64. data/lib/generators/deimos/db_backend_generator.rb +48 -0
  65. data/lib/tasks/deimos.rake +17 -0
  66. data/spec/active_record_consumer_spec.rb +81 -0
  67. data/spec/active_record_producer_spec.rb +107 -0
  68. data/spec/avro_data_decoder_spec.rb +18 -0
  69. data/spec/avro_data_encoder_spec.rb +37 -0
  70. data/spec/backends/db_spec.rb +35 -0
  71. data/spec/backends/kafka_async_spec.rb +11 -0
  72. data/spec/backends/kafka_spec.rb +11 -0
  73. data/spec/consumer_spec.rb +169 -0
  74. data/spec/deimos_spec.rb +117 -0
  75. data/spec/kafka_source_spec.rb +168 -0
  76. data/spec/kafka_topic_info_spec.rb +88 -0
  77. data/spec/phobos.bad_db.yml +73 -0
  78. data/spec/phobos.yml +73 -0
  79. data/spec/producer_spec.rb +397 -0
  80. data/spec/publish_backend_spec.rb +10 -0
  81. data/spec/schemas/com/my-namespace/MySchema-key.avsc +13 -0
  82. data/spec/schemas/com/my-namespace/MySchema.avsc +18 -0
  83. data/spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc +18 -0
  84. data/spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc +33 -0
  85. data/spec/schemas/com/my-namespace/MySchemaWithId.avsc +28 -0
  86. data/spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc +32 -0
  87. data/spec/schemas/com/my-namespace/Widget.avsc +27 -0
  88. data/spec/schemas/com/my-namespace/WidgetTheSecond.avsc +27 -0
  89. data/spec/spec_helper.rb +207 -0
  90. data/spec/updateable_schema_store_spec.rb +36 -0
  91. data/spec/utils/db_producer_spec.rb +208 -0
  92. data/spec/utils/executor_spec.rb +42 -0
  93. data/spec/utils/lag_reporter_spec.rb +69 -0
  94. data/spec/utils/platform_schema_validation_spec.rb +0 -0
  95. data/spec/utils/signal_handler_spec.rb +16 -0
  96. data/support/deimos-solo.png +0 -0
  97. data/support/deimos-with-name-next.png +0 -0
  98. data/support/deimos-with-name.png +0 -0
  99. data/support/flipp-logo.png +0 -0
  100. metadata +452 -0
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 779ef9db2088a1fe772b1bbb9f242745d083956c6531f1d6dec3f0b0ef1d5022
4
+ data.tar.gz: d1d58c31c340a9ebed48ce815a0ac84f46d051163d573f228651097c023c6527
5
+ SHA512:
6
+ metadata.gz: 873a906b4ed660b11d2e79b5c9f214ebbe5621e3a9d46be3aa264fa5ed90575075f10f3c0207f47a142b5f1d7fe5694d1ee886e9d644f64ce76bd712e0bff036
7
+ data.tar.gz: abc279a885973d6002200792ce3de615d1e36f91c822ec5669d55dd210a0f256cc65eec8f616da6f83fca1c3dd3b71a34d0fe084a905c6551377c4b647d3df9c
@@ -0,0 +1,74 @@
1
+ defaults: &defaults
2
+ parallelism: 1
3
+ working_directory: ~/workspace
4
+ docker:
5
+ - image: ruby:2.6
6
+ environment:
7
+ RAILS_ENV: test
8
+ DB_HOST_IP: 127.0.0.1
9
+ version: 2.1
10
+ jobs:
11
+ build:
12
+ <<: *defaults
13
+ steps:
14
+ - checkout
15
+
16
+ # Restore bundle cache & npm cache
17
+ - restore_cache:
18
+ key: 'rails-{{ checksum "Gemfile.lock" }}'
19
+
20
+ # Bundle install dependencies in /tmp/
21
+ # so Dockerfile does not copy them since
22
+ # its base image is different than CircleCI
23
+ - run:
24
+ name: Bundle install
25
+ command: bundle install --path vendor/bundle --jobs=4 --retry=3
26
+
27
+ # Store bundle cache
28
+ - save_cache:
29
+ key: 'rails-{{ checksum "Gemfile.lock" }}'
30
+ paths:
31
+ - ~/workspace/vendor/bundle
32
+
33
+ - persist_to_workspace:
34
+ root: ~/workspace
35
+ paths:
36
+ - .
37
+
38
+ lint:
39
+ <<: *defaults
40
+ steps:
41
+ - attach_workspace:
42
+ at: ~/workspace
43
+ - run:
44
+ name: Point bundle to vendor/bundle
45
+ command: bundle --path vendor/bundle
46
+ - run: bundle exec rubocop --display-only-fail-level-offenses --fail-level C
47
+
48
+ test-rspec:
49
+ <<: *defaults
50
+ steps:
51
+ - attach_workspace:
52
+ at: ~/workspace
53
+ - run:
54
+ name: Point bundle to vendor/bundle
55
+ command: bundle --path vendor/bundle
56
+ - run: mkdir result
57
+ - run:
58
+ name: Running rspec
59
+ command: bundle exec rspec --format progress --format RspecJunitFormatter -o result/rspec.xml
60
+ when: always
61
+ - store_test_results:
62
+ path: ~/workspace/result
63
+
64
+ workflows:
65
+ version: 2
66
+ build-and-test:
67
+ jobs:
68
+ - build
69
+ - test-rspec:
70
+ requires:
71
+ - build
72
+ - lint:
73
+ requires:
74
+ - build
@@ -0,0 +1,41 @@
1
+ # Created by .ignore support plugin (hsz.mobi)
2
+ ### Ruby template
3
+ *.gem
4
+ *.rbc
5
+ /.config
6
+ /coverage/
7
+ /InstalledFiles
8
+ /pkg/
9
+ /spec/reports/
10
+ /spec/examples.txt
11
+ /test/tmp/
12
+ /test/version_tmp/
13
+ /tmp/
14
+ /log/
15
+
16
+ # Used by dotenv library to load environment variables.
17
+ # .env
18
+
19
+ ## Documentation cache and generated files:
20
+ /.yardoc/
21
+ /_yardoc/
22
+ /doc/
23
+ /rdoc/
24
+
25
+ ## Environment normalization:
26
+ /.bundle/
27
+ /vendor/bundle
28
+ /lib/bundler/man/
29
+
30
+ # for a library or gem, you might want to ignore these files since the code is
31
+ # intended to run in multiple environments; otherwise, check them in:
32
+ # Gemfile.lock
33
+ # .ruby-version
34
+ # .ruby-gemset
35
+
36
+ # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
37
+ .rvmrc
38
+
39
+ test.sqlite3
40
+
41
+ .idea
File without changes
data/.rspec ADDED
@@ -0,0 +1 @@
1
+ --require spec_helper --tag ~kafka_integration --tag ~integration --format documentation
@@ -0,0 +1,321 @@
1
+ require: rubocop-rspec
2
+
3
+ AllCops:
4
+ TargetRubyVersion: 2.3
5
+ Exclude:
6
+ - lib/deimos/monkey_patches/*.rb
7
+ - vendor/**/*
8
+
9
+ # class Plumbus
10
+ # private
11
+ # def smooth; end
12
+ # end
13
+ Layout/AccessModifierIndentation:
14
+ EnforcedStyle: outdent
15
+
16
+ # foo.bar.
17
+ # each do
18
+ # baz
19
+ # end
20
+ Layout/BlockAlignment:
21
+ EnforcedStyleAlignWith: start_of_block
22
+
23
+ # something.
24
+ # method
25
+ #
26
+ # instead of
27
+ #
28
+ # something
29
+ # .method
30
+ Layout/DotPosition:
31
+ EnforcedStyle: trailing
32
+
33
+ # sometimes empty lines can be used for clarity
34
+ Layout/EmptyLinesAroundBlockBody:
35
+ Enabled: false
36
+
37
+ # foo = if expression
38
+ # 'bar'
39
+ # end
40
+ Layout/MultilineAssignmentLayout:
41
+ Enabled: true
42
+ EnforcedStyle: same_line
43
+
44
+ # while myvariable.
45
+ # a.
46
+ # b
47
+ #
48
+ # # do something
49
+ # end
50
+ Layout/MultilineMethodCallIndentation:
51
+ EnforcedStyle: indented
52
+
53
+ # def some_method(arg1=true, arg2=42)
54
+ Layout/SpaceAroundEqualsInParameterDefault:
55
+ EnforcedStyle: no_space
56
+
57
+ # do not allow e.g.
58
+ # if (v = array.grep(/foo/))
59
+ # do_something(v)
60
+ # end
61
+ Lint/AssignmentInCondition:
62
+ AllowSafeAssignment: false
63
+ Severity: convention
64
+
65
+ Lint/UnusedBlockArgument:
66
+ AllowUnusedKeywordArguments: true
67
+
68
+ Lint/UnusedMethodArgument:
69
+ AllowUnusedKeywordArguments: true
70
+
71
+ Metrics/AbcSize:
72
+ Severity: refactor
73
+ Max: 20
74
+
75
+ Metrics/BlockLength:
76
+ Severity: refactor
77
+
78
+ Metrics/ClassLength:
79
+ Severity: refactor
80
+
81
+ Metrics/CyclomaticComplexity:
82
+ Severity: refactor
83
+ Max: 20
84
+
85
+ Metrics/LineLength:
86
+ Max: 100
87
+ Severity: refactor
88
+ Exclude:
89
+ - 'spec/**/*'
90
+
91
+ Metrics/MethodLength:
92
+ Severity: refactor
93
+ Max: 30
94
+
95
+ Metrics/ModuleLength:
96
+ Severity: refactor
97
+
98
+ Metrics/ParameterLists:
99
+ Max: 5
100
+ CountKeywordArgs: false
101
+
102
+ Metrics/PerceivedComplexity:
103
+ Severity: refactor
104
+
105
+ # Use alias_method instead of alias
106
+ Style/Alias:
107
+ EnforcedStyle: prefer_alias_method
108
+
109
+ # Allow "and" or "or" to be used as a statement but not a conditional operator
110
+ Style/AndOr:
111
+ EnforcedStyle: conditionals
112
+
113
+ # Force use of File.open {...} instead of File.open but as a refactor
114
+ Style/AutoResourceCleanup:
115
+ Enabled: true
116
+ Severity: refactor
117
+
118
+ # Do not allow multiline {} blocks unless it is chained with a .
119
+ Style/BlockDelimiters:
120
+ EnforcedStyle: braces_for_chaining
121
+
122
+ # bad
123
+ # some_method(x, y, {a: 1, b: 2})
124
+ # some_method(x, y, {a: 1, b: 2}, a: 1, b: 2)
125
+
126
+ # good
127
+ # some_method(x, y, a: 1, b: 2)
128
+ # some_method(x, y, {a: 1, b: 2}, {a: 1, b: 2})
129
+ Style/BracesAroundHashParameters:
130
+ EnforcedStyle: context_dependent
131
+
132
+ # Enable both this:
133
+ # MyModule::MyClass
134
+ # and this:
135
+ # module MyModule
136
+ # class MyClass
137
+ Style/ClassAndModuleChildren:
138
+ Enabled: false
139
+
140
+ # Don't force "reduce" over "inject"
141
+ Style/CollectionMethods:
142
+ Enabled: true
143
+ AutoCorrect: false
144
+ PreferredMethods:
145
+ collect: map
146
+ collect!: map!
147
+ detect: find
148
+ find_all: select
149
+
150
+ Style/DateTime:
151
+ AllowCoercion: true
152
+
153
+ Style/Documentation:
154
+ Exclude:
155
+ - 'app/controllers/**/*'
156
+ - 'app/helpers/**/*'
157
+ - 'db/**/*'
158
+
159
+ # Force documentation for public methods and classes
160
+ Style/DocumentationMethod:
161
+ Enabled: true
162
+ Exclude:
163
+ - 'app/controllers/**/*'
164
+ - 'db/**/*'
165
+
166
+ # Allow else with just nil in it
167
+ Style/EmptyElse:
168
+ EnforcedStyle: empty
169
+
170
+ # Do not allow one-line methods
171
+ Style/EmptyMethod:
172
+ EnforcedStyle: expanded
173
+
174
+ # One-line bodies are fine without a guard clause
175
+ Style/GuardClause:
176
+ MinBodyLength: 2
177
+
178
+ # Require hash syntax { key: value } in all cases
179
+ Style/HashSyntax:
180
+ EnforcedStyle: ruby19_no_mixed_keys
181
+
182
+ # Allow the following:
183
+ # var x = "foo" +
184
+ # "bar"
185
+ Style/LineEndConcatenation:
186
+ Enabled: false
187
+
188
+ # Require parentheses around all method arguments except for whitelist
189
+ Style/MethodCallWithArgsParentheses:
190
+ Enabled: true
191
+ IgnoredMethods:
192
+ - puts
193
+ - render
194
+ - redirect_to
195
+ - send_data
196
+ - require
197
+ - include
198
+ - require_relative
199
+ - specify
200
+ - example
201
+ - describe
202
+ - it
203
+ - to
204
+ - not_to
205
+ - to_not
206
+ - define
207
+ - expect_with
208
+ - mock_with
209
+ - factory
210
+ - travel_to
211
+ - travel
212
+ - get
213
+ - raise
214
+ - attr_accessor
215
+ - class_attribute
216
+ - before_save
217
+ - after_save
218
+ - before_create
219
+ - after_create
220
+ - before_update
221
+ - after_update
222
+ - before_destroy
223
+ - after_destroy
224
+ - queue_as
225
+ Exclude:
226
+ - 'bin/**/*'
227
+ - 'Gemfile'
228
+
229
+ # Do not allow "end.something"
230
+ Style/MethodCalledOnDoEndBlock:
231
+ Enabled: true
232
+
233
+ Style/OptionHash:
234
+ Enabled: false
235
+
236
+ # Use %i() and %w() instead of []
237
+ Style/PercentLiteralDelimiters:
238
+ PreferredDelimiters:
239
+ '%i': '()'
240
+ '%I': '()'
241
+ '%w': '()'
242
+ '%W': '()'
243
+
244
+ # Allow self.x in all cases - it helps make it obvious when dealing with
245
+ # instance variables
246
+ Style/RedundantSelf:
247
+ Enabled: false
248
+
249
+ # Do not allow single line methods
250
+ Style/SingleLineMethods:
251
+ AllowIfMethodIsEmpty: false
252
+
253
+ # NOTE change this for Ruby < 2.0
254
+ # require %i()
255
+ Style/SymbolArray:
256
+ EnforcedStyle: percent
257
+
258
+ RSpec/AlignLeftLetBrace:
259
+ Enabled: false
260
+
261
+ RSpec/AlignRightLetBrace:
262
+ Enabled: false
263
+
264
+ # Allow allow_any_instance_of().to receive
265
+ RSpec/AnyInstance:
266
+ Enabled: false
267
+
268
+ # Allow describe MyClass, 'some descriptor that isn't a method'
269
+ RSpec/DescribeMethod:
270
+ Enabled: false
271
+
272
+ RSpec/ExampleLength:
273
+ Severity: refactor
274
+ Max: 40
275
+
276
+ # Allow it 'should do something'
277
+ RSpec/ExampleWording:
278
+ Enabled: false
279
+
280
+ # Allow describing specs without only using classes and methods
281
+ RSpec/FilePath:
282
+ Enabled: false
283
+
284
+ # Use before(:each), not before or before(:example)
285
+ RSpec/HookArgument:
286
+ EnforcedStyle: each
287
+
288
+ RSpec/ItBehavesLike:
289
+ EnforcedStyle: it_should_behave_like
290
+
291
+ RSpec/MessageChain:
292
+ Severity: refactor
293
+
294
+ # Allow both "allow" and "expect"
295
+ RSpec/MessageExpectation:
296
+ Enabled: false
297
+
298
+ # Use to receive, not to have_received
299
+ RSpec/MessageSpies:
300
+ Enabled: false
301
+
302
+ RSpec/MultipleExpectations:
303
+ Max: 10
304
+ Severity: refactor
305
+
306
+ # Allow both and_return() and block returns (use these for multi-line)
307
+ RSpec/ReturnFromStub:
308
+ Enabled: false
309
+
310
+ RSpec/SubjectStub:
311
+ Severity: refactor
312
+
313
+ RSpec/ExpectActual:
314
+ Enabled: false
315
+
316
+ RSpec/BeforeAfterAll:
317
+ Enabled: false
318
+
319
+ Security/YAMLLoad:
320
+ Exclude:
321
+ - 'lib/deimos.rb'