deimos-temp-fork 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (146) hide show
  1. checksums.yaml +7 -0
  2. data/.circleci/config.yml +83 -0
  3. data/.gitignore +41 -0
  4. data/.gitmodules +0 -0
  5. data/.rspec +1 -0
  6. data/.rubocop.yml +333 -0
  7. data/.ruby-gemset +1 -0
  8. data/.ruby-version +1 -0
  9. data/CHANGELOG.md +349 -0
  10. data/CODE_OF_CONDUCT.md +77 -0
  11. data/Dockerfile +23 -0
  12. data/Gemfile +6 -0
  13. data/Gemfile.lock +286 -0
  14. data/Guardfile +22 -0
  15. data/LICENSE.md +195 -0
  16. data/README.md +1099 -0
  17. data/Rakefile +13 -0
  18. data/bin/deimos +4 -0
  19. data/deimos-ruby.gemspec +44 -0
  20. data/docker-compose.yml +71 -0
  21. data/docs/ARCHITECTURE.md +140 -0
  22. data/docs/CONFIGURATION.md +236 -0
  23. data/docs/DATABASE_BACKEND.md +147 -0
  24. data/docs/INTEGRATION_TESTS.md +52 -0
  25. data/docs/PULL_REQUEST_TEMPLATE.md +35 -0
  26. data/docs/UPGRADING.md +128 -0
  27. data/lib/deimos-temp-fork.rb +95 -0
  28. data/lib/deimos/active_record_consume/batch_consumption.rb +164 -0
  29. data/lib/deimos/active_record_consume/batch_slicer.rb +27 -0
  30. data/lib/deimos/active_record_consume/message_consumption.rb +79 -0
  31. data/lib/deimos/active_record_consume/schema_model_converter.rb +52 -0
  32. data/lib/deimos/active_record_consumer.rb +67 -0
  33. data/lib/deimos/active_record_producer.rb +87 -0
  34. data/lib/deimos/backends/base.rb +32 -0
  35. data/lib/deimos/backends/db.rb +41 -0
  36. data/lib/deimos/backends/kafka.rb +33 -0
  37. data/lib/deimos/backends/kafka_async.rb +33 -0
  38. data/lib/deimos/backends/test.rb +20 -0
  39. data/lib/deimos/batch_consumer.rb +7 -0
  40. data/lib/deimos/config/configuration.rb +381 -0
  41. data/lib/deimos/config/phobos_config.rb +137 -0
  42. data/lib/deimos/consume/batch_consumption.rb +150 -0
  43. data/lib/deimos/consume/message_consumption.rb +94 -0
  44. data/lib/deimos/consumer.rb +104 -0
  45. data/lib/deimos/instrumentation.rb +76 -0
  46. data/lib/deimos/kafka_message.rb +60 -0
  47. data/lib/deimos/kafka_source.rb +128 -0
  48. data/lib/deimos/kafka_topic_info.rb +102 -0
  49. data/lib/deimos/message.rb +79 -0
  50. data/lib/deimos/metrics/datadog.rb +47 -0
  51. data/lib/deimos/metrics/mock.rb +39 -0
  52. data/lib/deimos/metrics/provider.rb +36 -0
  53. data/lib/deimos/monkey_patches/phobos_cli.rb +35 -0
  54. data/lib/deimos/monkey_patches/phobos_producer.rb +51 -0
  55. data/lib/deimos/poll_info.rb +9 -0
  56. data/lib/deimos/producer.rb +224 -0
  57. data/lib/deimos/railtie.rb +8 -0
  58. data/lib/deimos/schema_backends/avro_base.rb +140 -0
  59. data/lib/deimos/schema_backends/avro_local.rb +30 -0
  60. data/lib/deimos/schema_backends/avro_schema_coercer.rb +119 -0
  61. data/lib/deimos/schema_backends/avro_schema_registry.rb +34 -0
  62. data/lib/deimos/schema_backends/avro_validation.rb +21 -0
  63. data/lib/deimos/schema_backends/base.rb +150 -0
  64. data/lib/deimos/schema_backends/mock.rb +42 -0
  65. data/lib/deimos/shared_config.rb +63 -0
  66. data/lib/deimos/test_helpers.rb +360 -0
  67. data/lib/deimos/tracing/datadog.rb +35 -0
  68. data/lib/deimos/tracing/mock.rb +40 -0
  69. data/lib/deimos/tracing/provider.rb +29 -0
  70. data/lib/deimos/utils/db_poller.rb +150 -0
  71. data/lib/deimos/utils/db_producer.rb +243 -0
  72. data/lib/deimos/utils/deadlock_retry.rb +68 -0
  73. data/lib/deimos/utils/inline_consumer.rb +150 -0
  74. data/lib/deimos/utils/lag_reporter.rb +175 -0
  75. data/lib/deimos/utils/schema_controller_mixin.rb +115 -0
  76. data/lib/deimos/version.rb +5 -0
  77. data/lib/generators/deimos/active_record/templates/migration.rb.tt +28 -0
  78. data/lib/generators/deimos/active_record/templates/model.rb.tt +5 -0
  79. data/lib/generators/deimos/active_record_generator.rb +79 -0
  80. data/lib/generators/deimos/db_backend/templates/migration +25 -0
  81. data/lib/generators/deimos/db_backend/templates/rails3_migration +31 -0
  82. data/lib/generators/deimos/db_backend_generator.rb +48 -0
  83. data/lib/generators/deimos/db_poller/templates/migration +11 -0
  84. data/lib/generators/deimos/db_poller/templates/rails3_migration +16 -0
  85. data/lib/generators/deimos/db_poller_generator.rb +48 -0
  86. data/lib/tasks/deimos.rake +34 -0
  87. data/spec/active_record_batch_consumer_spec.rb +481 -0
  88. data/spec/active_record_consume/batch_slicer_spec.rb +42 -0
  89. data/spec/active_record_consume/schema_model_converter_spec.rb +105 -0
  90. data/spec/active_record_consumer_spec.rb +154 -0
  91. data/spec/active_record_producer_spec.rb +85 -0
  92. data/spec/backends/base_spec.rb +10 -0
  93. data/spec/backends/db_spec.rb +54 -0
  94. data/spec/backends/kafka_async_spec.rb +11 -0
  95. data/spec/backends/kafka_spec.rb +11 -0
  96. data/spec/batch_consumer_spec.rb +256 -0
  97. data/spec/config/configuration_spec.rb +248 -0
  98. data/spec/consumer_spec.rb +209 -0
  99. data/spec/deimos_spec.rb +169 -0
  100. data/spec/generators/active_record_generator_spec.rb +56 -0
  101. data/spec/handlers/my_batch_consumer.rb +10 -0
  102. data/spec/handlers/my_consumer.rb +10 -0
  103. data/spec/kafka_listener_spec.rb +55 -0
  104. data/spec/kafka_source_spec.rb +381 -0
  105. data/spec/kafka_topic_info_spec.rb +111 -0
  106. data/spec/message_spec.rb +19 -0
  107. data/spec/phobos.bad_db.yml +73 -0
  108. data/spec/phobos.yml +77 -0
  109. data/spec/producer_spec.rb +498 -0
  110. data/spec/rake_spec.rb +19 -0
  111. data/spec/schema_backends/avro_base_shared.rb +199 -0
  112. data/spec/schema_backends/avro_local_spec.rb +32 -0
  113. data/spec/schema_backends/avro_schema_registry_spec.rb +32 -0
  114. data/spec/schema_backends/avro_validation_spec.rb +24 -0
  115. data/spec/schema_backends/base_spec.rb +33 -0
  116. data/spec/schemas/com/my-namespace/Generated.avsc +71 -0
  117. data/spec/schemas/com/my-namespace/MyNestedSchema.avsc +62 -0
  118. data/spec/schemas/com/my-namespace/MySchema-key.avsc +13 -0
  119. data/spec/schemas/com/my-namespace/MySchema.avsc +18 -0
  120. data/spec/schemas/com/my-namespace/MySchemaCompound-key.avsc +18 -0
  121. data/spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc +18 -0
  122. data/spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc +33 -0
  123. data/spec/schemas/com/my-namespace/MySchemaWithId.avsc +28 -0
  124. data/spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc +32 -0
  125. data/spec/schemas/com/my-namespace/Wibble.avsc +43 -0
  126. data/spec/schemas/com/my-namespace/Widget.avsc +27 -0
  127. data/spec/schemas/com/my-namespace/WidgetTheSecond.avsc +27 -0
  128. data/spec/schemas/com/my-namespace/request/CreateTopic.avsc +11 -0
  129. data/spec/schemas/com/my-namespace/request/Index.avsc +11 -0
  130. data/spec/schemas/com/my-namespace/request/UpdateRequest.avsc +11 -0
  131. data/spec/schemas/com/my-namespace/response/CreateTopic.avsc +11 -0
  132. data/spec/schemas/com/my-namespace/response/Index.avsc +11 -0
  133. data/spec/schemas/com/my-namespace/response/UpdateResponse.avsc +11 -0
  134. data/spec/spec_helper.rb +267 -0
  135. data/spec/utils/db_poller_spec.rb +320 -0
  136. data/spec/utils/db_producer_spec.rb +514 -0
  137. data/spec/utils/deadlock_retry_spec.rb +74 -0
  138. data/spec/utils/inline_consumer_spec.rb +31 -0
  139. data/spec/utils/lag_reporter_spec.rb +76 -0
  140. data/spec/utils/platform_schema_validation_spec.rb +0 -0
  141. data/spec/utils/schema_controller_mixin_spec.rb +84 -0
  142. data/support/deimos-solo.png +0 -0
  143. data/support/deimos-with-name-next.png +0 -0
  144. data/support/deimos-with-name.png +0 -0
  145. data/support/flipp-logo.png +0 -0
  146. metadata +551 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 61d359b10487a52ae086cb17de89ae7c576d16d0b3d282085a8167207eac59d5
4
+ data.tar.gz: 4e27525e4e0d3eb15d121be1a60dd30836d3926dd3ea0e06934d0817bbd81412
5
+ SHA512:
6
+ metadata.gz: 67b0cb1f4966557c1723c8936542b9d0204330804f9317dfe1c647c69f429b689257072f486f4d07951950284098b0608358ada44a8115db7596f7d28012a79e
7
+ data.tar.gz: f6ed7202504debdfdef7568b1b636b426f3826e9edd58bd357083a0d1387af43222505e8d806f46cd64b2ff53ca05e53008d1a26a342b17831b3b4c4bba4613f
@@ -0,0 +1,83 @@
1
+ defaults: &defaults
2
+ parallelism: 1
3
+ working_directory: ~/workspace
4
+ docker:
5
+ - image: ruby:2.6
6
+ environment:
7
+ RAILS_ENV: test
8
+ DB_HOST_IP: 127.0.0.1
9
+ version: 2.1
10
+ jobs:
11
+ build:
12
+ <<: *defaults
13
+ steps:
14
+ - checkout
15
+
16
+ # Restore bundle cache & npm cache
17
+ - restore_cache:
18
+ key: 'rails-{{ checksum "Gemfile.lock" }}'
19
+
20
+ # Bundle install dependencies in /tmp/
21
+ # so Dockerfile does not copy them since
22
+ # its base image is different than CircleCI
23
+ - run:
24
+ name: Install bundler
25
+ command: gem install bundler:2.1.4
26
+ - run:
27
+ name: Bundle install
28
+ command: bundle install --path vendor/bundle --jobs=4 --retry=3
29
+
30
+ # Store bundle cache
31
+ - save_cache:
32
+ key: 'rails-{{ checksum "Gemfile.lock" }}'
33
+ paths:
34
+ - ~/workspace/vendor/bundle
35
+
36
+ - persist_to_workspace:
37
+ root: ~/workspace
38
+ paths:
39
+ - .
40
+
41
+ lint:
42
+ <<: *defaults
43
+ steps:
44
+ - attach_workspace:
45
+ at: ~/workspace
46
+ - run:
47
+ name: Install bundler
48
+ command: gem install bundler:2.1.4
49
+ - run:
50
+ name: Point bundle to vendor/bundle
51
+ command: bundle --path vendor/bundle
52
+ - run: bundle exec rubocop --display-only-fail-level-offenses --fail-level C
53
+
54
+ test-rspec:
55
+ <<: *defaults
56
+ steps:
57
+ - attach_workspace:
58
+ at: ~/workspace
59
+ - run:
60
+ name: Install bundler
61
+ command: gem install bundler:2.1.4
62
+ - run:
63
+ name: Point bundle to vendor/bundle
64
+ command: bundle --path vendor/bundle
65
+ - run: mkdir result
66
+ - run:
67
+ name: Running rspec
68
+ command: bundle exec rspec --format progress --format RspecJunitFormatter -o result/rspec.xml
69
+ when: always
70
+ - store_test_results:
71
+ path: ~/workspace/result
72
+
73
+ workflows:
74
+ version: 2
75
+ build-and-test:
76
+ jobs:
77
+ - build
78
+ - test-rspec:
79
+ requires:
80
+ - build
81
+ - lint:
82
+ requires:
83
+ - build
data/.gitignore ADDED
@@ -0,0 +1,41 @@
1
+ # Created by .ignore support plugin (hsz.mobi)
2
+ ### Ruby template
3
+ *.gem
4
+ *.rbc
5
+ /.config
6
+ /coverage/
7
+ /InstalledFiles
8
+ /pkg/
9
+ /spec/reports/
10
+ /spec/examples.txt
11
+ /test/tmp/
12
+ /test/version_tmp/
13
+ /tmp/
14
+ /log/
15
+
16
+ # Used by dotenv library to load environment variables.
17
+ # .env
18
+
19
+ ## Documentation cache and generated files:
20
+ /.yardoc/
21
+ /_yardoc/
22
+ /doc/
23
+ /rdoc/
24
+
25
+ ## Environment normalization:
26
+ /.bundle/
27
+ /vendor/bundle
28
+ /lib/bundler/man/
29
+
30
+ # for a library or gem, you might want to ignore these files since the code is
31
+ # intended to run in multiple environments; otherwise, check them in:
32
+ # Gemfile.lock
33
+ # .ruby-version
34
+ # .ruby-gemset
35
+
36
+ # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
37
+ .rvmrc
38
+
39
+ test.sqlite3
40
+
41
+ .idea
data/.gitmodules ADDED
File without changes
data/.rspec ADDED
@@ -0,0 +1 @@
1
+ --require spec_helper --tag ~kafka_integration --tag ~integration --format documentation
data/.rubocop.yml ADDED
@@ -0,0 +1,333 @@
1
+ require: rubocop-rspec
2
+
3
+ AllCops:
4
+ TargetRubyVersion: 2.4
5
+ Exclude:
6
+ - lib/deimos/monkey_patches/*.rb
7
+ - vendor/**/*
8
+ - Guardfile
9
+ NewCops: enable
10
+
11
+ # class Plumbus
12
+ # private
13
+ # def smooth; end
14
+ # end
15
+ Layout/AccessModifierIndentation:
16
+ EnforcedStyle: outdent
17
+
18
+ # foo.bar.
19
+ # each do
20
+ # baz
21
+ # end
22
+ Layout/BlockAlignment:
23
+ EnforcedStyleAlignWith: start_of_block
24
+
25
+ # something.
26
+ # method
27
+ #
28
+ # instead of
29
+ #
30
+ # something
31
+ # .method
32
+ Layout/DotPosition:
33
+ EnforcedStyle: trailing
34
+
35
+ # sometimes empty lines can be used for clarity
36
+ Layout/EmptyLinesAroundBlockBody:
37
+ Enabled: false
38
+
39
+ Layout/LineLength:
40
+ Max: 120
41
+ Severity: refactor
42
+ Exclude:
43
+ - 'spec/**/*'
44
+
45
+ # foo = if expression
46
+ # 'bar'
47
+ # end
48
+ Layout/MultilineAssignmentLayout:
49
+ Enabled: true
50
+ EnforcedStyle: same_line
51
+
52
+ # while myvariable.
53
+ # a.
54
+ # b
55
+ #
56
+ # # do something
57
+ # end
58
+ Layout/MultilineMethodCallIndentation:
59
+ EnforcedStyle: indented
60
+
61
+ # def some_method(arg1=true, arg2=42)
62
+ Layout/SpaceAroundEqualsInParameterDefault:
63
+ EnforcedStyle: no_space
64
+
65
+ # do not allow e.g.
66
+ # if (v = array.grep(/foo/))
67
+ # do_something(v)
68
+ # end
69
+ Lint/AssignmentInCondition:
70
+ AllowSafeAssignment: false
71
+ Severity: convention
72
+
73
+ Lint/UnusedBlockArgument:
74
+ AllowUnusedKeywordArguments: true
75
+
76
+ Lint/UnusedMethodArgument:
77
+ AllowUnusedKeywordArguments: true
78
+
79
+ Metrics/AbcSize:
80
+ Severity: refactor
81
+ Max: 25
82
+
83
+ Metrics/BlockLength:
84
+ Enabled: false
85
+
86
+ Metrics/ClassLength:
87
+ Severity: refactor
88
+ Max: 500
89
+
90
+ Metrics/CyclomaticComplexity:
91
+ Severity: refactor
92
+ Max: 20
93
+
94
+ Metrics/MethodLength:
95
+ Severity: refactor
96
+ Max: 50
97
+
98
+ Metrics/ModuleLength:
99
+ Severity: refactor
100
+ Max: 200
101
+
102
+ Metrics/ParameterLists:
103
+ Max: 5
104
+ CountKeywordArgs: false
105
+
106
+ Metrics/PerceivedComplexity:
107
+ Severity: refactor
108
+ Max: 10
109
+
110
+ # Use alias_method instead of alias
111
+ Style/Alias:
112
+ EnforcedStyle: prefer_alias_method
113
+
114
+ # Allow "and" or "or" to be used as a statement but not a conditional operator
115
+ Style/AndOr:
116
+ EnforcedStyle: conditionals
117
+
118
+ # Force use of File.open {...} instead of File.open but as a refactor
119
+ Style/AutoResourceCleanup:
120
+ Enabled: true
121
+ Severity: refactor
122
+
123
+ # Do not allow multiline {} blocks unless it is chained with a .
124
+ Style/BlockDelimiters:
125
+ EnforcedStyle: braces_for_chaining
126
+
127
+ # bad
128
+ # some_method(x, y, {a: 1, b: 2})
129
+ # some_method(x, y, {a: 1, b: 2}, a: 1, b: 2)
130
+
131
+ # Enable both this:
132
+ # MyModule::MyClass
133
+ # and this:
134
+ # module MyModule
135
+ # class MyClass
136
+ Style/ClassAndModuleChildren:
137
+ Enabled: false
138
+
139
+ # Don't force "reduce" over "inject"
140
+ Style/CollectionMethods:
141
+ Enabled: true
142
+ AutoCorrect: false
143
+ PreferredMethods:
144
+ collect: map
145
+ collect!: map!
146
+ detect: find
147
+ find_all: select
148
+
149
+ Style/DateTime:
150
+ AllowCoercion: true
151
+
152
+ Style/Documentation:
153
+ Exclude:
154
+ - 'app/controllers/**/*'
155
+ - 'app/helpers/**/*'
156
+ - 'db/**/*'
157
+
158
+ # Force documentation for public methods and classes
159
+ Style/DocumentationMethod:
160
+ Enabled: true
161
+ Exclude:
162
+ - 'app/controllers/**/*'
163
+ - 'db/**/*'
164
+
165
+ # Allow else with just nil in it
166
+ Style/EmptyElse:
167
+ EnforcedStyle: empty
168
+
169
+ # Do not allow one-line methods
170
+ Style/EmptyMethod:
171
+ EnforcedStyle: expanded
172
+
173
+ # One-line bodies are fine without a guard clause
174
+ Style/GuardClause:
175
+ MinBodyLength: 2
176
+
177
+ # Require hash syntax { key: value } in all cases
178
+ Style/HashSyntax:
179
+ EnforcedStyle: ruby19_no_mixed_keys
180
+
181
+ # We are still unofficially targeting Ruby 2.3
182
+ Style/HashTransformKeys:
183
+ Enabled: false
184
+
185
+ Style/HashTransformValues:
186
+ Enabled: false
187
+
188
+ Style/IfUnlessModifier:
189
+ Enabled: false
190
+
191
+ # Allow the following:
192
+ # var x = "foo" +
193
+ # "bar"
194
+ Style/LineEndConcatenation:
195
+ Enabled: false
196
+
197
+ # Require parentheses around all method arguments except for whitelist
198
+ Style/MethodCallWithArgsParentheses:
199
+ Enabled: true
200
+ IgnoredMethods:
201
+ - puts
202
+ - render
203
+ - redirect_to
204
+ - send_data
205
+ - require
206
+ - include
207
+ - require_relative
208
+ - specify
209
+ - example
210
+ - describe
211
+ - it
212
+ - to
213
+ - not_to
214
+ - to_not
215
+ - define
216
+ - expect_with
217
+ - mock_with
218
+ - factory
219
+ - travel_to
220
+ - travel
221
+ - get
222
+ - raise
223
+ - attr_accessor
224
+ - class_attribute
225
+ - before_save
226
+ - after_save
227
+ - before_create
228
+ - after_create
229
+ - before_update
230
+ - after_update
231
+ - before_destroy
232
+ - after_destroy
233
+ - queue_as
234
+ Exclude:
235
+ - 'bin/**/*'
236
+ - 'Gemfile'
237
+
238
+ # Do not allow "end.something"
239
+ Style/MethodCalledOnDoEndBlock:
240
+ Enabled: true
241
+
242
+ Style/OptionHash:
243
+ Enabled: false
244
+
245
+ # Use %i() and %w() instead of []
246
+ Style/PercentLiteralDelimiters:
247
+ PreferredDelimiters:
248
+ '%i': '()'
249
+ '%I': '()'
250
+ '%w': '()'
251
+ '%W': '()'
252
+
253
+ # Allow self.x in all cases - it helps make it obvious when dealing with
254
+ # instance variables
255
+ Style/RedundantSelf:
256
+ Enabled: false
257
+
258
+ # Do not allow single line methods
259
+ Style/SingleLineMethods:
260
+ AllowIfMethodIsEmpty: false
261
+
262
+ # NOTE change this for Ruby < 2.0
263
+ # require %i()
264
+ Style/SymbolArray:
265
+ EnforcedStyle: percent
266
+
267
+ RSpec/AlignLeftLetBrace:
268
+ Enabled: false
269
+
270
+ RSpec/AlignRightLetBrace:
271
+ Enabled: false
272
+
273
+ # Allow allow_any_instance_of().to receive
274
+ RSpec/AnyInstance:
275
+ Enabled: false
276
+
277
+ # Allow describe MyClass, 'some descriptor that isn't a method'
278
+ RSpec/DescribeMethod:
279
+ Enabled: false
280
+
281
+ RSpec/ExampleLength:
282
+ Severity: refactor
283
+ Max: 40
284
+
285
+ # Allow it 'should do something'
286
+ RSpec/ExampleWording:
287
+ Enabled: false
288
+
289
+ # Allow describing specs without only using classes and methods
290
+ RSpec/FilePath:
291
+ Enabled: false
292
+
293
+ # Use before(:each), not before or before(:example)
294
+ RSpec/HookArgument:
295
+ EnforcedStyle: each
296
+
297
+ RSpec/ItBehavesLike:
298
+ EnforcedStyle: it_should_behave_like
299
+
300
+ RSpec/LeakyConstantDeclaration:
301
+ Enabled: false
302
+
303
+ RSpec/MessageChain:
304
+ Severity: refactor
305
+
306
+ # Allow both "allow" and "expect"
307
+ RSpec/MessageExpectation:
308
+ Enabled: false
309
+
310
+ # Use to receive, not to have_received
311
+ RSpec/MessageSpies:
312
+ Enabled: false
313
+
314
+ RSpec/MultipleExpectations:
315
+ Max: 10
316
+ Severity: refactor
317
+
318
+ # Allow both and_return() and block returns (use these for multi-line)
319
+ RSpec/ReturnFromStub:
320
+ Enabled: false
321
+
322
+ RSpec/SubjectStub:
323
+ Severity: refactor
324
+
325
+ RSpec/ExpectActual:
326
+ Enabled: false
327
+
328
+ RSpec/BeforeAfterAll:
329
+ Enabled: false
330
+
331
+ Security/YAMLLoad:
332
+ Exclude:
333
+ - 'lib/deimos.rb'