deimos-kafka 1.0.0.pre.beta15

Sign up to get free protection for your applications and to get access to all the features.
Files changed (100) hide show
  1. checksums.yaml +7 -0
  2. data/.circleci/config.yml +74 -0
  3. data/.gitignore +41 -0
  4. data/.gitmodules +0 -0
  5. data/.rspec +1 -0
  6. data/.rubocop.yml +321 -0
  7. data/.ruby-gemset +1 -0
  8. data/.ruby-version +1 -0
  9. data/CHANGELOG.md +9 -0
  10. data/CODE_OF_CONDUCT.md +77 -0
  11. data/Dockerfile +23 -0
  12. data/Gemfile +6 -0
  13. data/Gemfile.lock +165 -0
  14. data/Guardfile +22 -0
  15. data/LICENSE.md +195 -0
  16. data/README.md +742 -0
  17. data/Rakefile +13 -0
  18. data/bin/deimos +4 -0
  19. data/deimos-kafka.gemspec +42 -0
  20. data/docker-compose.yml +71 -0
  21. data/docs/DATABASE_BACKEND.md +147 -0
  22. data/docs/PULL_REQUEST_TEMPLATE.md +34 -0
  23. data/lib/deimos.rb +134 -0
  24. data/lib/deimos/active_record_consumer.rb +81 -0
  25. data/lib/deimos/active_record_producer.rb +64 -0
  26. data/lib/deimos/avro_data_coder.rb +89 -0
  27. data/lib/deimos/avro_data_decoder.rb +36 -0
  28. data/lib/deimos/avro_data_encoder.rb +51 -0
  29. data/lib/deimos/backends/db.rb +27 -0
  30. data/lib/deimos/backends/kafka.rb +27 -0
  31. data/lib/deimos/backends/kafka_async.rb +27 -0
  32. data/lib/deimos/configuration.rb +88 -0
  33. data/lib/deimos/consumer.rb +164 -0
  34. data/lib/deimos/instrumentation.rb +71 -0
  35. data/lib/deimos/kafka_message.rb +27 -0
  36. data/lib/deimos/kafka_source.rb +126 -0
  37. data/lib/deimos/kafka_topic_info.rb +79 -0
  38. data/lib/deimos/message.rb +74 -0
  39. data/lib/deimos/metrics/datadog.rb +47 -0
  40. data/lib/deimos/metrics/mock.rb +39 -0
  41. data/lib/deimos/metrics/provider.rb +38 -0
  42. data/lib/deimos/monkey_patches/phobos_cli.rb +35 -0
  43. data/lib/deimos/monkey_patches/phobos_producer.rb +51 -0
  44. data/lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb +85 -0
  45. data/lib/deimos/monkey_patches/schema_store.rb +19 -0
  46. data/lib/deimos/producer.rb +218 -0
  47. data/lib/deimos/publish_backend.rb +30 -0
  48. data/lib/deimos/railtie.rb +8 -0
  49. data/lib/deimos/schema_coercer.rb +108 -0
  50. data/lib/deimos/shared_config.rb +59 -0
  51. data/lib/deimos/test_helpers.rb +356 -0
  52. data/lib/deimos/tracing/datadog.rb +35 -0
  53. data/lib/deimos/tracing/mock.rb +40 -0
  54. data/lib/deimos/tracing/provider.rb +31 -0
  55. data/lib/deimos/utils/db_producer.rb +95 -0
  56. data/lib/deimos/utils/executor.rb +117 -0
  57. data/lib/deimos/utils/inline_consumer.rb +144 -0
  58. data/lib/deimos/utils/lag_reporter.rb +182 -0
  59. data/lib/deimos/utils/platform_schema_validation.rb +0 -0
  60. data/lib/deimos/utils/signal_handler.rb +68 -0
  61. data/lib/deimos/version.rb +5 -0
  62. data/lib/generators/deimos/db_backend/templates/migration +24 -0
  63. data/lib/generators/deimos/db_backend/templates/rails3_migration +30 -0
  64. data/lib/generators/deimos/db_backend_generator.rb +48 -0
  65. data/lib/tasks/deimos.rake +17 -0
  66. data/spec/active_record_consumer_spec.rb +81 -0
  67. data/spec/active_record_producer_spec.rb +107 -0
  68. data/spec/avro_data_decoder_spec.rb +18 -0
  69. data/spec/avro_data_encoder_spec.rb +37 -0
  70. data/spec/backends/db_spec.rb +35 -0
  71. data/spec/backends/kafka_async_spec.rb +11 -0
  72. data/spec/backends/kafka_spec.rb +11 -0
  73. data/spec/consumer_spec.rb +169 -0
  74. data/spec/deimos_spec.rb +117 -0
  75. data/spec/kafka_source_spec.rb +168 -0
  76. data/spec/kafka_topic_info_spec.rb +88 -0
  77. data/spec/phobos.bad_db.yml +73 -0
  78. data/spec/phobos.yml +73 -0
  79. data/spec/producer_spec.rb +397 -0
  80. data/spec/publish_backend_spec.rb +10 -0
  81. data/spec/schemas/com/my-namespace/MySchema-key.avsc +13 -0
  82. data/spec/schemas/com/my-namespace/MySchema.avsc +18 -0
  83. data/spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc +18 -0
  84. data/spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc +33 -0
  85. data/spec/schemas/com/my-namespace/MySchemaWithId.avsc +28 -0
  86. data/spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc +32 -0
  87. data/spec/schemas/com/my-namespace/Widget.avsc +27 -0
  88. data/spec/schemas/com/my-namespace/WidgetTheSecond.avsc +27 -0
  89. data/spec/spec_helper.rb +207 -0
  90. data/spec/updateable_schema_store_spec.rb +36 -0
  91. data/spec/utils/db_producer_spec.rb +208 -0
  92. data/spec/utils/executor_spec.rb +42 -0
  93. data/spec/utils/lag_reporter_spec.rb +69 -0
  94. data/spec/utils/platform_schema_validation_spec.rb +0 -0
  95. data/spec/utils/signal_handler_spec.rb +16 -0
  96. data/support/deimos-solo.png +0 -0
  97. data/support/deimos-with-name-next.png +0 -0
  98. data/support/deimos-with-name.png +0 -0
  99. data/support/flipp-logo.png +0 -0
  100. metadata +452 -0
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 779ef9db2088a1fe772b1bbb9f242745d083956c6531f1d6dec3f0b0ef1d5022
4
+ data.tar.gz: d1d58c31c340a9ebed48ce815a0ac84f46d051163d573f228651097c023c6527
5
+ SHA512:
6
+ metadata.gz: 873a906b4ed660b11d2e79b5c9f214ebbe5621e3a9d46be3aa264fa5ed90575075f10f3c0207f47a142b5f1d7fe5694d1ee886e9d644f64ce76bd712e0bff036
7
+ data.tar.gz: abc279a885973d6002200792ce3de615d1e36f91c822ec5669d55dd210a0f256cc65eec8f616da6f83fca1c3dd3b71a34d0fe084a905c6551377c4b647d3df9c
@@ -0,0 +1,74 @@
1
+ defaults: &defaults
2
+ parallelism: 1
3
+ working_directory: ~/workspace
4
+ docker:
5
+ - image: ruby:2.6
6
+ environment:
7
+ RAILS_ENV: test
8
+ DB_HOST_IP: 127.0.0.1
9
+ version: 2.1
10
+ jobs:
11
+ build:
12
+ <<: *defaults
13
+ steps:
14
+ - checkout
15
+
16
+ # Restore bundle cache & npm cache
17
+ - restore_cache:
18
+ key: 'rails-{{ checksum "Gemfile.lock" }}'
19
+
20
+ # Bundle install dependencies in /tmp/
21
+ # so Dockerfile does not copy them since
22
+ # its base image is different than CircleCI
23
+ - run:
24
+ name: Bundle install
25
+ command: bundle install --path vendor/bundle --jobs=4 --retry=3
26
+
27
+ # Store bundle cache
28
+ - save_cache:
29
+ key: 'rails-{{ checksum "Gemfile.lock" }}'
30
+ paths:
31
+ - ~/workspace/vendor/bundle
32
+
33
+ - persist_to_workspace:
34
+ root: ~/workspace
35
+ paths:
36
+ - .
37
+
38
+ lint:
39
+ <<: *defaults
40
+ steps:
41
+ - attach_workspace:
42
+ at: ~/workspace
43
+ - run:
44
+ name: Point bundle to vendor/bundle
45
+ command: bundle --path vendor/bundle
46
+ - run: bundle exec rubocop --display-only-fail-level-offenses --fail-level C
47
+
48
+ test-rspec:
49
+ <<: *defaults
50
+ steps:
51
+ - attach_workspace:
52
+ at: ~/workspace
53
+ - run:
54
+ name: Point bundle to vendor/bundle
55
+ command: bundle --path vendor/bundle
56
+ - run: mkdir result
57
+ - run:
58
+ name: Running rspec
59
+ command: bundle exec rspec --format progress --format RspecJunitFormatter -o result/rspec.xml
60
+ when: always
61
+ - store_test_results:
62
+ path: ~/workspace/result
63
+
64
+ workflows:
65
+ version: 2
66
+ build-and-test:
67
+ jobs:
68
+ - build
69
+ - test-rspec:
70
+ requires:
71
+ - build
72
+ - lint:
73
+ requires:
74
+ - build
@@ -0,0 +1,41 @@
1
+ # Created by .ignore support plugin (hsz.mobi)
2
+ ### Ruby template
3
+ *.gem
4
+ *.rbc
5
+ /.config
6
+ /coverage/
7
+ /InstalledFiles
8
+ /pkg/
9
+ /spec/reports/
10
+ /spec/examples.txt
11
+ /test/tmp/
12
+ /test/version_tmp/
13
+ /tmp/
14
+ /log/
15
+
16
+ # Used by dotenv library to load environment variables.
17
+ # .env
18
+
19
+ ## Documentation cache and generated files:
20
+ /.yardoc/
21
+ /_yardoc/
22
+ /doc/
23
+ /rdoc/
24
+
25
+ ## Environment normalization:
26
+ /.bundle/
27
+ /vendor/bundle
28
+ /lib/bundler/man/
29
+
30
+ # for a library or gem, you might want to ignore these files since the code is
31
+ # intended to run in multiple environments; otherwise, check them in:
32
+ # Gemfile.lock
33
+ # .ruby-version
34
+ # .ruby-gemset
35
+
36
+ # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
37
+ .rvmrc
38
+
39
+ test.sqlite3
40
+
41
+ .idea
File without changes
data/.rspec ADDED
@@ -0,0 +1 @@
1
+ --require spec_helper --tag ~kafka_integration --tag ~integration --format documentation
@@ -0,0 +1,321 @@
1
+ require: rubocop-rspec
2
+
3
+ AllCops:
4
+ TargetRubyVersion: 2.3
5
+ Exclude:
6
+ - lib/deimos/monkey_patches/*.rb
7
+ - vendor/**/*
8
+
9
+ # class Plumbus
10
+ # private
11
+ # def smooth; end
12
+ # end
13
+ Layout/AccessModifierIndentation:
14
+ EnforcedStyle: outdent
15
+
16
+ # foo.bar.
17
+ # each do
18
+ # baz
19
+ # end
20
+ Layout/BlockAlignment:
21
+ EnforcedStyleAlignWith: start_of_block
22
+
23
+ # something.
24
+ # method
25
+ #
26
+ # instead of
27
+ #
28
+ # something
29
+ # .method
30
+ Layout/DotPosition:
31
+ EnforcedStyle: trailing
32
+
33
+ # sometimes empty lines can be used for clarity
34
+ Layout/EmptyLinesAroundBlockBody:
35
+ Enabled: false
36
+
37
+ # foo = if expression
38
+ # 'bar'
39
+ # end
40
+ Layout/MultilineAssignmentLayout:
41
+ Enabled: true
42
+ EnforcedStyle: same_line
43
+
44
+ # while myvariable.
45
+ # a.
46
+ # b
47
+ #
48
+ # # do something
49
+ # end
50
+ Layout/MultilineMethodCallIndentation:
51
+ EnforcedStyle: indented
52
+
53
+ # def some_method(arg1=true, arg2=42)
54
+ Layout/SpaceAroundEqualsInParameterDefault:
55
+ EnforcedStyle: no_space
56
+
57
+ # do not allow e.g.
58
+ # if (v = array.grep(/foo/))
59
+ # do_something(v)
60
+ # end
61
+ Lint/AssignmentInCondition:
62
+ AllowSafeAssignment: false
63
+ Severity: convention
64
+
65
+ Lint/UnusedBlockArgument:
66
+ AllowUnusedKeywordArguments: true
67
+
68
+ Lint/UnusedMethodArgument:
69
+ AllowUnusedKeywordArguments: true
70
+
71
+ Metrics/AbcSize:
72
+ Severity: refactor
73
+ Max: 20
74
+
75
+ Metrics/BlockLength:
76
+ Severity: refactor
77
+
78
+ Metrics/ClassLength:
79
+ Severity: refactor
80
+
81
+ Metrics/CyclomaticComplexity:
82
+ Severity: refactor
83
+ Max: 20
84
+
85
+ Metrics/LineLength:
86
+ Max: 100
87
+ Severity: refactor
88
+ Exclude:
89
+ - 'spec/**/*'
90
+
91
+ Metrics/MethodLength:
92
+ Severity: refactor
93
+ Max: 30
94
+
95
+ Metrics/ModuleLength:
96
+ Severity: refactor
97
+
98
+ Metrics/ParameterLists:
99
+ Max: 5
100
+ CountKeywordArgs: false
101
+
102
+ Metrics/PerceivedComplexity:
103
+ Severity: refactor
104
+
105
+ # Use alias_method instead of alias
106
+ Style/Alias:
107
+ EnforcedStyle: prefer_alias_method
108
+
109
+ # Allow "and" or "or" to be used as a statement but not a conditional operator
110
+ Style/AndOr:
111
+ EnforcedStyle: conditionals
112
+
113
+ # Force use of File.open {...} instead of File.open but as a refactor
114
+ Style/AutoResourceCleanup:
115
+ Enabled: true
116
+ Severity: refactor
117
+
118
+ # Do not allow multiline {} blocks unless it is chained with a .
119
+ Style/BlockDelimiters:
120
+ EnforcedStyle: braces_for_chaining
121
+
122
+ # bad
123
+ # some_method(x, y, {a: 1, b: 2})
124
+ # some_method(x, y, {a: 1, b: 2}, a: 1, b: 2)
125
+
126
+ # good
127
+ # some_method(x, y, a: 1, b: 2)
128
+ # some_method(x, y, {a: 1, b: 2}, {a: 1, b: 2})
129
+ Style/BracesAroundHashParameters:
130
+ EnforcedStyle: context_dependent
131
+
132
+ # Enable both this:
133
+ # MyModule::MyClass
134
+ # and this:
135
+ # module MyModule
136
+ # class MyClass
137
+ Style/ClassAndModuleChildren:
138
+ Enabled: false
139
+
140
+ # Don't force "reduce" over "inject"
141
+ Style/CollectionMethods:
142
+ Enabled: true
143
+ AutoCorrect: false
144
+ PreferredMethods:
145
+ collect: map
146
+ collect!: map!
147
+ detect: find
148
+ find_all: select
149
+
150
+ Style/DateTime:
151
+ AllowCoercion: true
152
+
153
+ Style/Documentation:
154
+ Exclude:
155
+ - 'app/controllers/**/*'
156
+ - 'app/helpers/**/*'
157
+ - 'db/**/*'
158
+
159
+ # Force documentation for public methods and classes
160
+ Style/DocumentationMethod:
161
+ Enabled: true
162
+ Exclude:
163
+ - 'app/controllers/**/*'
164
+ - 'db/**/*'
165
+
166
+ # Allow else with just nil in it
167
+ Style/EmptyElse:
168
+ EnforcedStyle: empty
169
+
170
+ # Do not allow one-line methods
171
+ Style/EmptyMethod:
172
+ EnforcedStyle: expanded
173
+
174
+ # One-line bodies are fine without a guard clause
175
+ Style/GuardClause:
176
+ MinBodyLength: 2
177
+
178
+ # Require hash syntax { key: value } in all cases
179
+ Style/HashSyntax:
180
+ EnforcedStyle: ruby19_no_mixed_keys
181
+
182
+ # Allow the following:
183
+ # var x = "foo" +
184
+ # "bar"
185
+ Style/LineEndConcatenation:
186
+ Enabled: false
187
+
188
+ # Require parentheses around all method arguments except for whitelist
189
+ Style/MethodCallWithArgsParentheses:
190
+ Enabled: true
191
+ IgnoredMethods:
192
+ - puts
193
+ - render
194
+ - redirect_to
195
+ - send_data
196
+ - require
197
+ - include
198
+ - require_relative
199
+ - specify
200
+ - example
201
+ - describe
202
+ - it
203
+ - to
204
+ - not_to
205
+ - to_not
206
+ - define
207
+ - expect_with
208
+ - mock_with
209
+ - factory
210
+ - travel_to
211
+ - travel
212
+ - get
213
+ - raise
214
+ - attr_accessor
215
+ - class_attribute
216
+ - before_save
217
+ - after_save
218
+ - before_create
219
+ - after_create
220
+ - before_update
221
+ - after_update
222
+ - before_destroy
223
+ - after_destroy
224
+ - queue_as
225
+ Exclude:
226
+ - 'bin/**/*'
227
+ - 'Gemfile'
228
+
229
+ # Do not allow "end.something"
230
+ Style/MethodCalledOnDoEndBlock:
231
+ Enabled: true
232
+
233
+ Style/OptionHash:
234
+ Enabled: false
235
+
236
+ # Use %i() and %w() instead of []
237
+ Style/PercentLiteralDelimiters:
238
+ PreferredDelimiters:
239
+ '%i': '()'
240
+ '%I': '()'
241
+ '%w': '()'
242
+ '%W': '()'
243
+
244
+ # Allow self.x in all cases - it helps make it obvious when dealing with
245
+ # instance variables
246
+ Style/RedundantSelf:
247
+ Enabled: false
248
+
249
+ # Do not allow single line methods
250
+ Style/SingleLineMethods:
251
+ AllowIfMethodIsEmpty: false
252
+
253
+ # NOTE change this for Ruby < 2.0
254
+ # require %i()
255
+ Style/SymbolArray:
256
+ EnforcedStyle: percent
257
+
258
+ RSpec/AlignLeftLetBrace:
259
+ Enabled: false
260
+
261
+ RSpec/AlignRightLetBrace:
262
+ Enabled: false
263
+
264
+ # Allow allow_any_instance_of().to receive
265
+ RSpec/AnyInstance:
266
+ Enabled: false
267
+
268
+ # Allow describe MyClass, 'some descriptor that isn't a method'
269
+ RSpec/DescribeMethod:
270
+ Enabled: false
271
+
272
+ RSpec/ExampleLength:
273
+ Severity: refactor
274
+ Max: 40
275
+
276
+ # Allow it 'should do something'
277
+ RSpec/ExampleWording:
278
+ Enabled: false
279
+
280
+ # Allow describing specs without only using classes and methods
281
+ RSpec/FilePath:
282
+ Enabled: false
283
+
284
+ # Use before(:each), not before or before(:example)
285
+ RSpec/HookArgument:
286
+ EnforcedStyle: each
287
+
288
+ RSpec/ItBehavesLike:
289
+ EnforcedStyle: it_should_behave_like
290
+
291
+ RSpec/MessageChain:
292
+ Severity: refactor
293
+
294
+ # Allow both "allow" and "expect"
295
+ RSpec/MessageExpectation:
296
+ Enabled: false
297
+
298
+ # Use to receive, not to have_received
299
+ RSpec/MessageSpies:
300
+ Enabled: false
301
+
302
+ RSpec/MultipleExpectations:
303
+ Max: 10
304
+ Severity: refactor
305
+
306
+ # Allow both and_return() and block returns (use these for multi-line)
307
+ RSpec/ReturnFromStub:
308
+ Enabled: false
309
+
310
+ RSpec/SubjectStub:
311
+ Severity: refactor
312
+
313
+ RSpec/ExpectActual:
314
+ Enabled: false
315
+
316
+ RSpec/BeforeAfterAll:
317
+ Enabled: false
318
+
319
+ Security/YAMLLoad:
320
+ Exclude:
321
+ - 'lib/deimos.rb'