deimos-kafka 1.0.0.pre.beta15
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.circleci/config.yml +74 -0
- data/.gitignore +41 -0
- data/.gitmodules +0 -0
- data/.rspec +1 -0
- data/.rubocop.yml +321 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/CHANGELOG.md +9 -0
- data/CODE_OF_CONDUCT.md +77 -0
- data/Dockerfile +23 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +165 -0
- data/Guardfile +22 -0
- data/LICENSE.md +195 -0
- data/README.md +742 -0
- data/Rakefile +13 -0
- data/bin/deimos +4 -0
- data/deimos-kafka.gemspec +42 -0
- data/docker-compose.yml +71 -0
- data/docs/DATABASE_BACKEND.md +147 -0
- data/docs/PULL_REQUEST_TEMPLATE.md +34 -0
- data/lib/deimos.rb +134 -0
- data/lib/deimos/active_record_consumer.rb +81 -0
- data/lib/deimos/active_record_producer.rb +64 -0
- data/lib/deimos/avro_data_coder.rb +89 -0
- data/lib/deimos/avro_data_decoder.rb +36 -0
- data/lib/deimos/avro_data_encoder.rb +51 -0
- data/lib/deimos/backends/db.rb +27 -0
- data/lib/deimos/backends/kafka.rb +27 -0
- data/lib/deimos/backends/kafka_async.rb +27 -0
- data/lib/deimos/configuration.rb +88 -0
- data/lib/deimos/consumer.rb +164 -0
- data/lib/deimos/instrumentation.rb +71 -0
- data/lib/deimos/kafka_message.rb +27 -0
- data/lib/deimos/kafka_source.rb +126 -0
- data/lib/deimos/kafka_topic_info.rb +79 -0
- data/lib/deimos/message.rb +74 -0
- data/lib/deimos/metrics/datadog.rb +47 -0
- data/lib/deimos/metrics/mock.rb +39 -0
- data/lib/deimos/metrics/provider.rb +38 -0
- data/lib/deimos/monkey_patches/phobos_cli.rb +35 -0
- data/lib/deimos/monkey_patches/phobos_producer.rb +51 -0
- data/lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb +85 -0
- data/lib/deimos/monkey_patches/schema_store.rb +19 -0
- data/lib/deimos/producer.rb +218 -0
- data/lib/deimos/publish_backend.rb +30 -0
- data/lib/deimos/railtie.rb +8 -0
- data/lib/deimos/schema_coercer.rb +108 -0
- data/lib/deimos/shared_config.rb +59 -0
- data/lib/deimos/test_helpers.rb +356 -0
- data/lib/deimos/tracing/datadog.rb +35 -0
- data/lib/deimos/tracing/mock.rb +40 -0
- data/lib/deimos/tracing/provider.rb +31 -0
- data/lib/deimos/utils/db_producer.rb +95 -0
- data/lib/deimos/utils/executor.rb +117 -0
- data/lib/deimos/utils/inline_consumer.rb +144 -0
- data/lib/deimos/utils/lag_reporter.rb +182 -0
- data/lib/deimos/utils/platform_schema_validation.rb +0 -0
- data/lib/deimos/utils/signal_handler.rb +68 -0
- data/lib/deimos/version.rb +5 -0
- data/lib/generators/deimos/db_backend/templates/migration +24 -0
- data/lib/generators/deimos/db_backend/templates/rails3_migration +30 -0
- data/lib/generators/deimos/db_backend_generator.rb +48 -0
- data/lib/tasks/deimos.rake +17 -0
- data/spec/active_record_consumer_spec.rb +81 -0
- data/spec/active_record_producer_spec.rb +107 -0
- data/spec/avro_data_decoder_spec.rb +18 -0
- data/spec/avro_data_encoder_spec.rb +37 -0
- data/spec/backends/db_spec.rb +35 -0
- data/spec/backends/kafka_async_spec.rb +11 -0
- data/spec/backends/kafka_spec.rb +11 -0
- data/spec/consumer_spec.rb +169 -0
- data/spec/deimos_spec.rb +117 -0
- data/spec/kafka_source_spec.rb +168 -0
- data/spec/kafka_topic_info_spec.rb +88 -0
- data/spec/phobos.bad_db.yml +73 -0
- data/spec/phobos.yml +73 -0
- data/spec/producer_spec.rb +397 -0
- data/spec/publish_backend_spec.rb +10 -0
- data/spec/schemas/com/my-namespace/MySchema-key.avsc +13 -0
- data/spec/schemas/com/my-namespace/MySchema.avsc +18 -0
- data/spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc +18 -0
- data/spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc +33 -0
- data/spec/schemas/com/my-namespace/MySchemaWithId.avsc +28 -0
- data/spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc +32 -0
- data/spec/schemas/com/my-namespace/Widget.avsc +27 -0
- data/spec/schemas/com/my-namespace/WidgetTheSecond.avsc +27 -0
- data/spec/spec_helper.rb +207 -0
- data/spec/updateable_schema_store_spec.rb +36 -0
- data/spec/utils/db_producer_spec.rb +208 -0
- data/spec/utils/executor_spec.rb +42 -0
- data/spec/utils/lag_reporter_spec.rb +69 -0
- data/spec/utils/platform_schema_validation_spec.rb +0 -0
- data/spec/utils/signal_handler_spec.rb +16 -0
- data/support/deimos-solo.png +0 -0
- data/support/deimos-with-name-next.png +0 -0
- data/support/deimos-with-name.png +0 -0
- data/support/flipp-logo.png +0 -0
- metadata +452 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 779ef9db2088a1fe772b1bbb9f242745d083956c6531f1d6dec3f0b0ef1d5022
|
4
|
+
data.tar.gz: d1d58c31c340a9ebed48ce815a0ac84f46d051163d573f228651097c023c6527
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 873a906b4ed660b11d2e79b5c9f214ebbe5621e3a9d46be3aa264fa5ed90575075f10f3c0207f47a142b5f1d7fe5694d1ee886e9d644f64ce76bd712e0bff036
|
7
|
+
data.tar.gz: abc279a885973d6002200792ce3de615d1e36f91c822ec5669d55dd210a0f256cc65eec8f616da6f83fca1c3dd3b71a34d0fe084a905c6551377c4b647d3df9c
|
@@ -0,0 +1,74 @@
|
|
1
|
+
defaults: &defaults
|
2
|
+
parallelism: 1
|
3
|
+
working_directory: ~/workspace
|
4
|
+
docker:
|
5
|
+
- image: ruby:2.6
|
6
|
+
environment:
|
7
|
+
RAILS_ENV: test
|
8
|
+
DB_HOST_IP: 127.0.0.1
|
9
|
+
version: 2.1
|
10
|
+
jobs:
|
11
|
+
build:
|
12
|
+
<<: *defaults
|
13
|
+
steps:
|
14
|
+
- checkout
|
15
|
+
|
16
|
+
# Restore bundle cache & npm cache
|
17
|
+
- restore_cache:
|
18
|
+
key: 'rails-{{ checksum "Gemfile.lock" }}'
|
19
|
+
|
20
|
+
# Bundle install dependencies in /tmp/
|
21
|
+
# so Dockerfile does not copy them since
|
22
|
+
# its base image is different than CircleCI
|
23
|
+
- run:
|
24
|
+
name: Bundle install
|
25
|
+
command: bundle install --path vendor/bundle --jobs=4 --retry=3
|
26
|
+
|
27
|
+
# Store bundle cache
|
28
|
+
- save_cache:
|
29
|
+
key: 'rails-{{ checksum "Gemfile.lock" }}'
|
30
|
+
paths:
|
31
|
+
- ~/workspace/vendor/bundle
|
32
|
+
|
33
|
+
- persist_to_workspace:
|
34
|
+
root: ~/workspace
|
35
|
+
paths:
|
36
|
+
- .
|
37
|
+
|
38
|
+
lint:
|
39
|
+
<<: *defaults
|
40
|
+
steps:
|
41
|
+
- attach_workspace:
|
42
|
+
at: ~/workspace
|
43
|
+
- run:
|
44
|
+
name: Point bundle to vendor/bundle
|
45
|
+
command: bundle --path vendor/bundle
|
46
|
+
- run: bundle exec rubocop --display-only-fail-level-offenses --fail-level C
|
47
|
+
|
48
|
+
test-rspec:
|
49
|
+
<<: *defaults
|
50
|
+
steps:
|
51
|
+
- attach_workspace:
|
52
|
+
at: ~/workspace
|
53
|
+
- run:
|
54
|
+
name: Point bundle to vendor/bundle
|
55
|
+
command: bundle --path vendor/bundle
|
56
|
+
- run: mkdir result
|
57
|
+
- run:
|
58
|
+
name: Running rspec
|
59
|
+
command: bundle exec rspec --format progress --format RspecJunitFormatter -o result/rspec.xml
|
60
|
+
when: always
|
61
|
+
- store_test_results:
|
62
|
+
path: ~/workspace/result
|
63
|
+
|
64
|
+
workflows:
|
65
|
+
version: 2
|
66
|
+
build-and-test:
|
67
|
+
jobs:
|
68
|
+
- build
|
69
|
+
- test-rspec:
|
70
|
+
requires:
|
71
|
+
- build
|
72
|
+
- lint:
|
73
|
+
requires:
|
74
|
+
- build
|
data/.gitignore
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
# Created by .ignore support plugin (hsz.mobi)
|
2
|
+
### Ruby template
|
3
|
+
*.gem
|
4
|
+
*.rbc
|
5
|
+
/.config
|
6
|
+
/coverage/
|
7
|
+
/InstalledFiles
|
8
|
+
/pkg/
|
9
|
+
/spec/reports/
|
10
|
+
/spec/examples.txt
|
11
|
+
/test/tmp/
|
12
|
+
/test/version_tmp/
|
13
|
+
/tmp/
|
14
|
+
/log/
|
15
|
+
|
16
|
+
# Used by dotenv library to load environment variables.
|
17
|
+
# .env
|
18
|
+
|
19
|
+
## Documentation cache and generated files:
|
20
|
+
/.yardoc/
|
21
|
+
/_yardoc/
|
22
|
+
/doc/
|
23
|
+
/rdoc/
|
24
|
+
|
25
|
+
## Environment normalization:
|
26
|
+
/.bundle/
|
27
|
+
/vendor/bundle
|
28
|
+
/lib/bundler/man/
|
29
|
+
|
30
|
+
# for a library or gem, you might want to ignore these files since the code is
|
31
|
+
# intended to run in multiple environments; otherwise, check them in:
|
32
|
+
# Gemfile.lock
|
33
|
+
# .ruby-version
|
34
|
+
# .ruby-gemset
|
35
|
+
|
36
|
+
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
|
37
|
+
.rvmrc
|
38
|
+
|
39
|
+
test.sqlite3
|
40
|
+
|
41
|
+
.idea
|
data/.gitmodules
ADDED
File without changes
|
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--require spec_helper --tag ~kafka_integration --tag ~integration --format documentation
|
data/.rubocop.yml
ADDED
@@ -0,0 +1,321 @@
|
|
1
|
+
require: rubocop-rspec
|
2
|
+
|
3
|
+
AllCops:
|
4
|
+
TargetRubyVersion: 2.3
|
5
|
+
Exclude:
|
6
|
+
- lib/deimos/monkey_patches/*.rb
|
7
|
+
- vendor/**/*
|
8
|
+
|
9
|
+
# class Plumbus
|
10
|
+
# private
|
11
|
+
# def smooth; end
|
12
|
+
# end
|
13
|
+
Layout/AccessModifierIndentation:
|
14
|
+
EnforcedStyle: outdent
|
15
|
+
|
16
|
+
# foo.bar.
|
17
|
+
# each do
|
18
|
+
# baz
|
19
|
+
# end
|
20
|
+
Layout/BlockAlignment:
|
21
|
+
EnforcedStyleAlignWith: start_of_block
|
22
|
+
|
23
|
+
# something.
|
24
|
+
# method
|
25
|
+
#
|
26
|
+
# instead of
|
27
|
+
#
|
28
|
+
# something
|
29
|
+
# .method
|
30
|
+
Layout/DotPosition:
|
31
|
+
EnforcedStyle: trailing
|
32
|
+
|
33
|
+
# sometimes empty lines can be used for clarity
|
34
|
+
Layout/EmptyLinesAroundBlockBody:
|
35
|
+
Enabled: false
|
36
|
+
|
37
|
+
# foo = if expression
|
38
|
+
# 'bar'
|
39
|
+
# end
|
40
|
+
Layout/MultilineAssignmentLayout:
|
41
|
+
Enabled: true
|
42
|
+
EnforcedStyle: same_line
|
43
|
+
|
44
|
+
# while myvariable.
|
45
|
+
# a.
|
46
|
+
# b
|
47
|
+
#
|
48
|
+
# # do something
|
49
|
+
# end
|
50
|
+
Layout/MultilineMethodCallIndentation:
|
51
|
+
EnforcedStyle: indented
|
52
|
+
|
53
|
+
# def some_method(arg1=true, arg2=42)
|
54
|
+
Layout/SpaceAroundEqualsInParameterDefault:
|
55
|
+
EnforcedStyle: no_space
|
56
|
+
|
57
|
+
# do not allow e.g.
|
58
|
+
# if (v = array.grep(/foo/))
|
59
|
+
# do_something(v)
|
60
|
+
# end
|
61
|
+
Lint/AssignmentInCondition:
|
62
|
+
AllowSafeAssignment: false
|
63
|
+
Severity: convention
|
64
|
+
|
65
|
+
Lint/UnusedBlockArgument:
|
66
|
+
AllowUnusedKeywordArguments: true
|
67
|
+
|
68
|
+
Lint/UnusedMethodArgument:
|
69
|
+
AllowUnusedKeywordArguments: true
|
70
|
+
|
71
|
+
Metrics/AbcSize:
|
72
|
+
Severity: refactor
|
73
|
+
Max: 20
|
74
|
+
|
75
|
+
Metrics/BlockLength:
|
76
|
+
Severity: refactor
|
77
|
+
|
78
|
+
Metrics/ClassLength:
|
79
|
+
Severity: refactor
|
80
|
+
|
81
|
+
Metrics/CyclomaticComplexity:
|
82
|
+
Severity: refactor
|
83
|
+
Max: 20
|
84
|
+
|
85
|
+
Metrics/LineLength:
|
86
|
+
Max: 100
|
87
|
+
Severity: refactor
|
88
|
+
Exclude:
|
89
|
+
- 'spec/**/*'
|
90
|
+
|
91
|
+
Metrics/MethodLength:
|
92
|
+
Severity: refactor
|
93
|
+
Max: 30
|
94
|
+
|
95
|
+
Metrics/ModuleLength:
|
96
|
+
Severity: refactor
|
97
|
+
|
98
|
+
Metrics/ParameterLists:
|
99
|
+
Max: 5
|
100
|
+
CountKeywordArgs: false
|
101
|
+
|
102
|
+
Metrics/PerceivedComplexity:
|
103
|
+
Severity: refactor
|
104
|
+
|
105
|
+
# Use alias_method instead of alias
|
106
|
+
Style/Alias:
|
107
|
+
EnforcedStyle: prefer_alias_method
|
108
|
+
|
109
|
+
# Allow "and" or "or" to be used as a statement but not a conditional operator
|
110
|
+
Style/AndOr:
|
111
|
+
EnforcedStyle: conditionals
|
112
|
+
|
113
|
+
# Force use of File.open {...} instead of File.open but as a refactor
|
114
|
+
Style/AutoResourceCleanup:
|
115
|
+
Enabled: true
|
116
|
+
Severity: refactor
|
117
|
+
|
118
|
+
# Do not allow multiline {} blocks unless it is chained with a .
|
119
|
+
Style/BlockDelimiters:
|
120
|
+
EnforcedStyle: braces_for_chaining
|
121
|
+
|
122
|
+
# bad
|
123
|
+
# some_method(x, y, {a: 1, b: 2})
|
124
|
+
# some_method(x, y, {a: 1, b: 2}, a: 1, b: 2)
|
125
|
+
|
126
|
+
# good
|
127
|
+
# some_method(x, y, a: 1, b: 2)
|
128
|
+
# some_method(x, y, {a: 1, b: 2}, {a: 1, b: 2})
|
129
|
+
Style/BracesAroundHashParameters:
|
130
|
+
EnforcedStyle: context_dependent
|
131
|
+
|
132
|
+
# Enable both this:
|
133
|
+
# MyModule::MyClass
|
134
|
+
# and this:
|
135
|
+
# module MyModule
|
136
|
+
# class MyClass
|
137
|
+
Style/ClassAndModuleChildren:
|
138
|
+
Enabled: false
|
139
|
+
|
140
|
+
# Don't force "reduce" over "inject"
|
141
|
+
Style/CollectionMethods:
|
142
|
+
Enabled: true
|
143
|
+
AutoCorrect: false
|
144
|
+
PreferredMethods:
|
145
|
+
collect: map
|
146
|
+
collect!: map!
|
147
|
+
detect: find
|
148
|
+
find_all: select
|
149
|
+
|
150
|
+
Style/DateTime:
|
151
|
+
AllowCoercion: true
|
152
|
+
|
153
|
+
Style/Documentation:
|
154
|
+
Exclude:
|
155
|
+
- 'app/controllers/**/*'
|
156
|
+
- 'app/helpers/**/*'
|
157
|
+
- 'db/**/*'
|
158
|
+
|
159
|
+
# Force documentation for public methods and classes
|
160
|
+
Style/DocumentationMethod:
|
161
|
+
Enabled: true
|
162
|
+
Exclude:
|
163
|
+
- 'app/controllers/**/*'
|
164
|
+
- 'db/**/*'
|
165
|
+
|
166
|
+
# Allow else with just nil in it
|
167
|
+
Style/EmptyElse:
|
168
|
+
EnforcedStyle: empty
|
169
|
+
|
170
|
+
# Do not allow one-line methods
|
171
|
+
Style/EmptyMethod:
|
172
|
+
EnforcedStyle: expanded
|
173
|
+
|
174
|
+
# One-line bodies are fine without a guard clause
|
175
|
+
Style/GuardClause:
|
176
|
+
MinBodyLength: 2
|
177
|
+
|
178
|
+
# Require hash syntax { key: value } in all cases
|
179
|
+
Style/HashSyntax:
|
180
|
+
EnforcedStyle: ruby19_no_mixed_keys
|
181
|
+
|
182
|
+
# Allow the following:
|
183
|
+
# var x = "foo" +
|
184
|
+
# "bar"
|
185
|
+
Style/LineEndConcatenation:
|
186
|
+
Enabled: false
|
187
|
+
|
188
|
+
# Require parentheses around all method arguments except for whitelist
|
189
|
+
Style/MethodCallWithArgsParentheses:
|
190
|
+
Enabled: true
|
191
|
+
IgnoredMethods:
|
192
|
+
- puts
|
193
|
+
- render
|
194
|
+
- redirect_to
|
195
|
+
- send_data
|
196
|
+
- require
|
197
|
+
- include
|
198
|
+
- require_relative
|
199
|
+
- specify
|
200
|
+
- example
|
201
|
+
- describe
|
202
|
+
- it
|
203
|
+
- to
|
204
|
+
- not_to
|
205
|
+
- to_not
|
206
|
+
- define
|
207
|
+
- expect_with
|
208
|
+
- mock_with
|
209
|
+
- factory
|
210
|
+
- travel_to
|
211
|
+
- travel
|
212
|
+
- get
|
213
|
+
- raise
|
214
|
+
- attr_accessor
|
215
|
+
- class_attribute
|
216
|
+
- before_save
|
217
|
+
- after_save
|
218
|
+
- before_create
|
219
|
+
- after_create
|
220
|
+
- before_update
|
221
|
+
- after_update
|
222
|
+
- before_destroy
|
223
|
+
- after_destroy
|
224
|
+
- queue_as
|
225
|
+
Exclude:
|
226
|
+
- 'bin/**/*'
|
227
|
+
- 'Gemfile'
|
228
|
+
|
229
|
+
# Do not allow "end.something"
|
230
|
+
Style/MethodCalledOnDoEndBlock:
|
231
|
+
Enabled: true
|
232
|
+
|
233
|
+
Style/OptionHash:
|
234
|
+
Enabled: false
|
235
|
+
|
236
|
+
# Use %i() and %w() instead of []
|
237
|
+
Style/PercentLiteralDelimiters:
|
238
|
+
PreferredDelimiters:
|
239
|
+
'%i': '()'
|
240
|
+
'%I': '()'
|
241
|
+
'%w': '()'
|
242
|
+
'%W': '()'
|
243
|
+
|
244
|
+
# Allow self.x in all cases - it helps make it obvious when dealing with
|
245
|
+
# instance variables
|
246
|
+
Style/RedundantSelf:
|
247
|
+
Enabled: false
|
248
|
+
|
249
|
+
# Do not allow single line methods
|
250
|
+
Style/SingleLineMethods:
|
251
|
+
AllowIfMethodIsEmpty: false
|
252
|
+
|
253
|
+
# NOTE change this for Ruby < 2.0
|
254
|
+
# require %i()
|
255
|
+
Style/SymbolArray:
|
256
|
+
EnforcedStyle: percent
|
257
|
+
|
258
|
+
RSpec/AlignLeftLetBrace:
|
259
|
+
Enabled: false
|
260
|
+
|
261
|
+
RSpec/AlignRightLetBrace:
|
262
|
+
Enabled: false
|
263
|
+
|
264
|
+
# Allow allow_any_instance_of().to receive
|
265
|
+
RSpec/AnyInstance:
|
266
|
+
Enabled: false
|
267
|
+
|
268
|
+
# Allow describe MyClass, 'some descriptor that isn't a method'
|
269
|
+
RSpec/DescribeMethod:
|
270
|
+
Enabled: false
|
271
|
+
|
272
|
+
RSpec/ExampleLength:
|
273
|
+
Severity: refactor
|
274
|
+
Max: 40
|
275
|
+
|
276
|
+
# Allow it 'should do something'
|
277
|
+
RSpec/ExampleWording:
|
278
|
+
Enabled: false
|
279
|
+
|
280
|
+
# Allow describing specs without only using classes and methods
|
281
|
+
RSpec/FilePath:
|
282
|
+
Enabled: false
|
283
|
+
|
284
|
+
# Use before(:each), not before or before(:example)
|
285
|
+
RSpec/HookArgument:
|
286
|
+
EnforcedStyle: each
|
287
|
+
|
288
|
+
RSpec/ItBehavesLike:
|
289
|
+
EnforcedStyle: it_should_behave_like
|
290
|
+
|
291
|
+
RSpec/MessageChain:
|
292
|
+
Severity: refactor
|
293
|
+
|
294
|
+
# Allow both "allow" and "expect"
|
295
|
+
RSpec/MessageExpectation:
|
296
|
+
Enabled: false
|
297
|
+
|
298
|
+
# Use to receive, not to have_received
|
299
|
+
RSpec/MessageSpies:
|
300
|
+
Enabled: false
|
301
|
+
|
302
|
+
RSpec/MultipleExpectations:
|
303
|
+
Max: 10
|
304
|
+
Severity: refactor
|
305
|
+
|
306
|
+
# Allow both and_return() and block returns (use these for multi-line)
|
307
|
+
RSpec/ReturnFromStub:
|
308
|
+
Enabled: false
|
309
|
+
|
310
|
+
RSpec/SubjectStub:
|
311
|
+
Severity: refactor
|
312
|
+
|
313
|
+
RSpec/ExpectActual:
|
314
|
+
Enabled: false
|
315
|
+
|
316
|
+
RSpec/BeforeAfterAll:
|
317
|
+
Enabled: false
|
318
|
+
|
319
|
+
Security/YAMLLoad:
|
320
|
+
Exclude:
|
321
|
+
- 'lib/deimos.rb'
|