deimos-kafka 1.0.0.pre.beta15

Sign up to get free protection for your applications and to get access to all the features.
Files changed (100) hide show
  1. checksums.yaml +7 -0
  2. data/.circleci/config.yml +74 -0
  3. data/.gitignore +41 -0
  4. data/.gitmodules +0 -0
  5. data/.rspec +1 -0
  6. data/.rubocop.yml +321 -0
  7. data/.ruby-gemset +1 -0
  8. data/.ruby-version +1 -0
  9. data/CHANGELOG.md +9 -0
  10. data/CODE_OF_CONDUCT.md +77 -0
  11. data/Dockerfile +23 -0
  12. data/Gemfile +6 -0
  13. data/Gemfile.lock +165 -0
  14. data/Guardfile +22 -0
  15. data/LICENSE.md +195 -0
  16. data/README.md +742 -0
  17. data/Rakefile +13 -0
  18. data/bin/deimos +4 -0
  19. data/deimos-kafka.gemspec +42 -0
  20. data/docker-compose.yml +71 -0
  21. data/docs/DATABASE_BACKEND.md +147 -0
  22. data/docs/PULL_REQUEST_TEMPLATE.md +34 -0
  23. data/lib/deimos.rb +134 -0
  24. data/lib/deimos/active_record_consumer.rb +81 -0
  25. data/lib/deimos/active_record_producer.rb +64 -0
  26. data/lib/deimos/avro_data_coder.rb +89 -0
  27. data/lib/deimos/avro_data_decoder.rb +36 -0
  28. data/lib/deimos/avro_data_encoder.rb +51 -0
  29. data/lib/deimos/backends/db.rb +27 -0
  30. data/lib/deimos/backends/kafka.rb +27 -0
  31. data/lib/deimos/backends/kafka_async.rb +27 -0
  32. data/lib/deimos/configuration.rb +88 -0
  33. data/lib/deimos/consumer.rb +164 -0
  34. data/lib/deimos/instrumentation.rb +71 -0
  35. data/lib/deimos/kafka_message.rb +27 -0
  36. data/lib/deimos/kafka_source.rb +126 -0
  37. data/lib/deimos/kafka_topic_info.rb +79 -0
  38. data/lib/deimos/message.rb +74 -0
  39. data/lib/deimos/metrics/datadog.rb +47 -0
  40. data/lib/deimos/metrics/mock.rb +39 -0
  41. data/lib/deimos/metrics/provider.rb +38 -0
  42. data/lib/deimos/monkey_patches/phobos_cli.rb +35 -0
  43. data/lib/deimos/monkey_patches/phobos_producer.rb +51 -0
  44. data/lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb +85 -0
  45. data/lib/deimos/monkey_patches/schema_store.rb +19 -0
  46. data/lib/deimos/producer.rb +218 -0
  47. data/lib/deimos/publish_backend.rb +30 -0
  48. data/lib/deimos/railtie.rb +8 -0
  49. data/lib/deimos/schema_coercer.rb +108 -0
  50. data/lib/deimos/shared_config.rb +59 -0
  51. data/lib/deimos/test_helpers.rb +356 -0
  52. data/lib/deimos/tracing/datadog.rb +35 -0
  53. data/lib/deimos/tracing/mock.rb +40 -0
  54. data/lib/deimos/tracing/provider.rb +31 -0
  55. data/lib/deimos/utils/db_producer.rb +95 -0
  56. data/lib/deimos/utils/executor.rb +117 -0
  57. data/lib/deimos/utils/inline_consumer.rb +144 -0
  58. data/lib/deimos/utils/lag_reporter.rb +182 -0
  59. data/lib/deimos/utils/platform_schema_validation.rb +0 -0
  60. data/lib/deimos/utils/signal_handler.rb +68 -0
  61. data/lib/deimos/version.rb +5 -0
  62. data/lib/generators/deimos/db_backend/templates/migration +24 -0
  63. data/lib/generators/deimos/db_backend/templates/rails3_migration +30 -0
  64. data/lib/generators/deimos/db_backend_generator.rb +48 -0
  65. data/lib/tasks/deimos.rake +17 -0
  66. data/spec/active_record_consumer_spec.rb +81 -0
  67. data/spec/active_record_producer_spec.rb +107 -0
  68. data/spec/avro_data_decoder_spec.rb +18 -0
  69. data/spec/avro_data_encoder_spec.rb +37 -0
  70. data/spec/backends/db_spec.rb +35 -0
  71. data/spec/backends/kafka_async_spec.rb +11 -0
  72. data/spec/backends/kafka_spec.rb +11 -0
  73. data/spec/consumer_spec.rb +169 -0
  74. data/spec/deimos_spec.rb +117 -0
  75. data/spec/kafka_source_spec.rb +168 -0
  76. data/spec/kafka_topic_info_spec.rb +88 -0
  77. data/spec/phobos.bad_db.yml +73 -0
  78. data/spec/phobos.yml +73 -0
  79. data/spec/producer_spec.rb +397 -0
  80. data/spec/publish_backend_spec.rb +10 -0
  81. data/spec/schemas/com/my-namespace/MySchema-key.avsc +13 -0
  82. data/spec/schemas/com/my-namespace/MySchema.avsc +18 -0
  83. data/spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc +18 -0
  84. data/spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc +33 -0
  85. data/spec/schemas/com/my-namespace/MySchemaWithId.avsc +28 -0
  86. data/spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc +32 -0
  87. data/spec/schemas/com/my-namespace/Widget.avsc +27 -0
  88. data/spec/schemas/com/my-namespace/WidgetTheSecond.avsc +27 -0
  89. data/spec/spec_helper.rb +207 -0
  90. data/spec/updateable_schema_store_spec.rb +36 -0
  91. data/spec/utils/db_producer_spec.rb +208 -0
  92. data/spec/utils/executor_spec.rb +42 -0
  93. data/spec/utils/lag_reporter_spec.rb +69 -0
  94. data/spec/utils/platform_schema_validation_spec.rb +0 -0
  95. data/spec/utils/signal_handler_spec.rb +16 -0
  96. data/support/deimos-solo.png +0 -0
  97. data/support/deimos-with-name-next.png +0 -0
  98. data/support/deimos-with-name.png +0 -0
  99. data/support/flipp-logo.png +0 -0
  100. metadata +452 -0
@@ -0,0 +1,208 @@
1
+ # frozen_string_literal: true
2
+
3
+ each_db_config(Deimos::Utils::DbProducer) do
4
+ let(:producer) do
5
+ producer = described_class.new
6
+ allow(producer).to receive(:sleep)
7
+ phobos_producer = instance_double(Phobos::Producer::PublicAPI)
8
+ allow(phobos_producer).to receive(:publish_list)
9
+ allow(producer).to receive(:producer).and_return(phobos_producer)
10
+ producer
11
+ end
12
+
13
+ before(:each) do
14
+ stub_const('Deimos::Utils::DbProducer::BATCH_SIZE', 2)
15
+ end
16
+
17
+ specify '#process_next_messages' do
18
+ expect(producer).to receive(:retrieve_topics).and_return(%w(topic1 topic2))
19
+ expect(producer).to receive(:process_topic).twice
20
+ expect(producer).to receive(:sleep).with(0.5)
21
+ producer.process_next_messages
22
+ end
23
+
24
+ specify '#retrieve_topics' do
25
+ (1..3).each do |i|
26
+ Deimos::KafkaMessage.create!(topic: "topic#{i}",
27
+ key: 'blergkey',
28
+ message: 'blerg')
29
+ end
30
+ expect(producer.retrieve_topics).
31
+ to contain_exactly('topic1', 'topic2', 'topic3')
32
+ end
33
+
34
+ specify '#retrieve_messages' do
35
+ (1..3).each do |i|
36
+ Deimos::KafkaMessage.create!(topic: 'topic1',
37
+ message: 'blah',
38
+ key: "key#{i}")
39
+ end
40
+ stub_const('Deimos::Utils::DbProducer::BATCH_SIZE', 5)
41
+ producer.current_topic = 'topic1'
42
+ messages = producer.retrieve_messages
43
+ expect(messages.size).to eq(3)
44
+ expect(messages).to all(be_a_kind_of(Deimos::KafkaMessage))
45
+ end
46
+
47
+ describe '#process_topic' do
48
+ before(:each) do
49
+ producer.id = 'abc'
50
+ end
51
+
52
+ it 'should do nothing if lock fails' do
53
+ expect(Deimos::KafkaTopicInfo).to receive(:lock).
54
+ with('my-topic', 'abc').and_return(false)
55
+ expect(producer).not_to receive(:retrieve_messages)
56
+ producer.process_topic('my-topic')
57
+ end
58
+
59
+ it 'should complete successfully' do
60
+ messages = (1..4).map do |i|
61
+ Deimos::KafkaMessage.new(
62
+ topic: 'my-topic',
63
+ message: "mess#{i}",
64
+ partition_key: "key#{i}"
65
+ )
66
+ end
67
+ expect(Deimos::KafkaTopicInfo).to receive(:lock).
68
+ with('my-topic', 'abc').and_return(true)
69
+ expect(producer).to receive(:retrieve_messages).ordered.
70
+ and_return(messages[0..1])
71
+ expect(producer).to receive(:produce_messages).ordered.with([
72
+ {
73
+ payload: 'mess1',
74
+ key: nil,
75
+ partition_key: 'key1',
76
+ topic: 'my-topic'
77
+ },
78
+ {
79
+ payload: 'mess2',
80
+ key: nil,
81
+ partition_key: 'key2',
82
+ topic: 'my-topic'
83
+ }
84
+ ])
85
+ expect(producer).to receive(:retrieve_messages).ordered.
86
+ and_return(messages[2..3])
87
+ expect(producer).to receive(:produce_messages).ordered.with([
88
+ {
89
+ payload: 'mess3',
90
+ partition_key: 'key3',
91
+ key: nil,
92
+ topic: 'my-topic'
93
+ },
94
+ {
95
+ payload: 'mess4',
96
+ partition_key: 'key4',
97
+ key: nil,
98
+ topic: 'my-topic'
99
+ }
100
+ ])
101
+ expect(producer).to receive(:retrieve_messages).ordered.
102
+ and_return([])
103
+ expect(Deimos::KafkaTopicInfo).to receive(:heartbeat).
104
+ with('my-topic', 'abc').twice
105
+ expect(Deimos::KafkaTopicInfo).to receive(:clear_lock).
106
+ with('my-topic', 'abc').once
107
+ producer.process_topic('my-topic')
108
+ end
109
+
110
+ it 'should register an error if it gets an error' do
111
+ expect(producer).to receive(:retrieve_messages).and_raise('OH NOES')
112
+ expect(Deimos::KafkaTopicInfo).to receive(:register_error).
113
+ with('my-topic', 'abc')
114
+ expect(producer).not_to receive(:produce_messages)
115
+ producer.process_topic('my-topic')
116
+ end
117
+
118
+ it 'should move on if it gets a partial batch' do
119
+ expect(producer).to receive(:retrieve_messages).ordered.
120
+ and_return([Deimos::KafkaMessage.new(
121
+ topic: 'my-topic',
122
+ message: 'mess1'
123
+ )])
124
+ expect(producer).to receive(:produce_messages).once
125
+ producer.process_topic('my-topic')
126
+ end
127
+
128
+ end
129
+
130
+ example 'Full integration test' do
131
+ (1..4).each do |i|
132
+ (1..2).each do |j|
133
+ Deimos::KafkaMessage.create!(topic: "topic#{j}",
134
+ message: "mess#{i}",
135
+ partition_key: "key#{i}")
136
+ Deimos::KafkaMessage.create!(topic: "topic#{j + 2}",
137
+ key: "key#{i}",
138
+ partition_key: "key#{i}",
139
+ message: "mess#{i}")
140
+ end
141
+ end
142
+ allow(producer).to receive(:produce_messages)
143
+
144
+ producer.process_next_messages
145
+ expect(Deimos::KafkaTopicInfo.count).to eq(4)
146
+ topics = Deimos::KafkaTopicInfo.select('distinct topic').map(&:topic)
147
+ expect(topics).to contain_exactly('topic1', 'topic2', 'topic3', 'topic4')
148
+ expect(Deimos::KafkaMessage.count).to eq(0)
149
+
150
+ expect(producer).to have_received(:produce_messages).with([
151
+ {
152
+ payload: 'mess1',
153
+ partition_key: 'key1',
154
+ key: nil,
155
+ topic: 'topic1'
156
+ },
157
+ {
158
+ payload: 'mess2',
159
+ key: nil,
160
+ partition_key: 'key2',
161
+ topic: 'topic1'
162
+ }
163
+ ])
164
+ expect(producer).to have_received(:produce_messages).with([
165
+ {
166
+ payload: 'mess3',
167
+ key: nil,
168
+ partition_key: 'key3',
169
+ topic: 'topic1'
170
+ },
171
+ {
172
+ payload: 'mess4',
173
+ key: nil,
174
+ partition_key: 'key4',
175
+ topic: 'topic1'
176
+ }
177
+ ])
178
+ expect(producer).to have_received(:produce_messages).with([
179
+ {
180
+ payload: 'mess1',
181
+ key: 'key1',
182
+ partition_key: 'key1',
183
+ topic: 'topic3'
184
+ },
185
+ {
186
+ payload: 'mess2',
187
+ partition_key: 'key2',
188
+ key: 'key2',
189
+ topic: 'topic3'
190
+ }
191
+ ])
192
+ expect(producer).to have_received(:produce_messages).with([
193
+ {
194
+ payload: 'mess3',
195
+ key: 'key3',
196
+ partition_key: 'key3',
197
+ topic: 'topic3'
198
+ },
199
+ {
200
+ payload: 'mess4',
201
+ partition_key: 'key4',
202
+ key: 'key4',
203
+ topic: 'topic3'
204
+ }
205
+ ])
206
+ end
207
+
208
+ end
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'spec_helper'
4
+
5
+ RSpec.describe Deimos::Utils::Executor do
6
+
7
+ let(:executor) { described_class.new(runners) }
8
+ let(:runners) { (1..2).map { |i| TestRunners::TestRunner.new(i) } }
9
+
10
+ it 'starts and stops configured runners' do
11
+ runners.each do |r|
12
+ expect(r.started).to be_falsey
13
+ expect(r.stopped).to be_falsey
14
+ end
15
+ executor.start
16
+ wait_for do
17
+ runners.each do |r|
18
+ expect(r.started).to be_truthy
19
+ expect(r.stopped).to be_falsey
20
+ end
21
+ executor.stop
22
+ runners.each do |r|
23
+ expect(r.started).to be_truthy
24
+ expect(r.stopped).to be_truthy
25
+ end
26
+ end
27
+ end
28
+
29
+ it 'reconnects crashed runners' do
30
+ allow(executor).to receive(:handle_crashed_runner).and_call_original
31
+ allow(executor).to receive(:handle_crashed_runner).and_call_original
32
+ runners.each { |r| r.should_error = true }
33
+ executor.start
34
+ wait_for do
35
+ expect(executor).to have_received(:handle_crashed_runner).with(runners[0], anything, 0).once
36
+ expect(executor).to have_received(:handle_crashed_runner).with(runners[1], anything, 0).once
37
+ runners.each { |r| expect(r.started).to be_truthy }
38
+ executor.stop
39
+ end
40
+ end
41
+
42
+ end
@@ -0,0 +1,69 @@
1
+ # frozen_string_literal: true
2
+
3
+ describe Deimos::Utils::LagReporter do
4
+
5
+ before(:each) do
6
+ kafka_client = instance_double(Kafka::Client)
7
+ allow(kafka_client).to receive(:last_offset_for).and_return(100)
8
+ allow(Phobos).to receive(:create_kafka_client).and_return(kafka_client)
9
+ Deimos.configure { |c| c.report_lag = true }
10
+ end
11
+
12
+ after(:each) do
13
+ described_class.reset
14
+ Deimos.configure { |c| c.report_lag = false }
15
+ end
16
+
17
+ it 'should not report lag before ready' do
18
+ expect(Deimos.config.metrics).not_to receive(:gauge)
19
+ ActiveSupport::Notifications.instrument(
20
+ 'heartbeat.consumer.kafka',
21
+ group_id: 'group1', topic_partitions: { 'my-topic': [1] }
22
+ )
23
+
24
+ end
25
+
26
+ it 'should report lag' do
27
+ expect(Deimos.config.metrics).to receive(:gauge).ordered.twice.
28
+ with('consumer_lag', 95,
29
+ tags: %w(
30
+ consumer_group:group1
31
+ partition:1
32
+ topic:my-topic
33
+ ))
34
+ expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
35
+ with('consumer_lag', 80,
36
+ tags: %w(
37
+ consumer_group:group1
38
+ partition:2
39
+ topic:my-topic
40
+ ))
41
+ expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
42
+ with('consumer_lag', 0,
43
+ tags: %w(
44
+ consumer_group:group1
45
+ partition:2
46
+ topic:my-topic
47
+ ))
48
+ ActiveSupport::Notifications.instrument(
49
+ 'seek.consumer.kafka',
50
+ offset: 5, topic: 'my-topic', group_id: 'group1', partition: 1
51
+ )
52
+ ActiveSupport::Notifications.instrument(
53
+ 'start_process_message.consumer.kafka',
54
+ offset_lag: 80, topic: 'my-topic', group_id: 'group1', partition: 2
55
+ )
56
+ ActiveSupport::Notifications.instrument(
57
+ 'heartbeat.consumer.kafka',
58
+ group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
59
+ )
60
+ ActiveSupport::Notifications.instrument(
61
+ 'start_process_batch.consumer.kafka',
62
+ offset_lag: 0, topic: 'my-topic', group_id: 'group1', partition: 2
63
+ )
64
+ ActiveSupport::Notifications.instrument(
65
+ 'heartbeat.consumer.kafka',
66
+ group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
67
+ )
68
+ end
69
+ end
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ RSpec.describe Deimos::Utils::SignalHandler do
4
+ describe '#run!' do
5
+
6
+ it 'starts and stops the runner' do
7
+ runner = TestRunners::TestRunner.new
8
+ expect(runner).to receive(:start)
9
+ expect(runner).to receive(:stop)
10
+
11
+ signal_handler = described_class.new(runner)
12
+ signal_handler.send(:unblock, described_class::SIGNALS.first)
13
+ signal_handler.run!
14
+ end
15
+ end
16
+ end
Binary file
Binary file
metadata ADDED
@@ -0,0 +1,452 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: deimos-kafka
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.0.0.pre.beta15
5
+ platform: ruby
6
+ authors:
7
+ - Daniel Orner
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2019-07-09 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: avro-patches
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '0.3'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '0.3'
27
+ - !ruby/object:Gem::Dependency
28
+ name: avro_turf
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '0.8'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '0.8'
41
+ - !ruby/object:Gem::Dependency
42
+ name: phobos
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '1.8'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '1.8'
55
+ - !ruby/object:Gem::Dependency
56
+ name: ruby-kafka
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '0.7'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '0.7'
69
+ - !ruby/object:Gem::Dependency
70
+ name: activerecord
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '5.2'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '5.2'
83
+ - !ruby/object:Gem::Dependency
84
+ name: activerecord-import
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ name: bundler
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - "~>"
102
+ - !ruby/object:Gem::Version
103
+ version: '1'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - "~>"
109
+ - !ruby/object:Gem::Version
110
+ version: '1'
111
+ - !ruby/object:Gem::Dependency
112
+ name: ddtrace
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '0.11'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '0.11'
125
+ - !ruby/object:Gem::Dependency
126
+ name: dogstatsd-ruby
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: '4.2'
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: '4.2'
139
+ - !ruby/object:Gem::Dependency
140
+ name: guard
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - "~>"
144
+ - !ruby/object:Gem::Version
145
+ version: '2'
146
+ type: :development
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - "~>"
151
+ - !ruby/object:Gem::Version
152
+ version: '2'
153
+ - !ruby/object:Gem::Dependency
154
+ name: guard-rspec
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - "~>"
158
+ - !ruby/object:Gem::Version
159
+ version: '4'
160
+ type: :development
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - "~>"
165
+ - !ruby/object:Gem::Version
166
+ version: '4'
167
+ - !ruby/object:Gem::Dependency
168
+ name: guard-rubocop
169
+ requirement: !ruby/object:Gem::Requirement
170
+ requirements:
171
+ - - "~>"
172
+ - !ruby/object:Gem::Version
173
+ version: '1'
174
+ type: :development
175
+ prerelease: false
176
+ version_requirements: !ruby/object:Gem::Requirement
177
+ requirements:
178
+ - - "~>"
179
+ - !ruby/object:Gem::Version
180
+ version: '1'
181
+ - !ruby/object:Gem::Dependency
182
+ name: mysql2
183
+ requirement: !ruby/object:Gem::Requirement
184
+ requirements:
185
+ - - "~>"
186
+ - !ruby/object:Gem::Version
187
+ version: '0.5'
188
+ type: :development
189
+ prerelease: false
190
+ version_requirements: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - "~>"
193
+ - !ruby/object:Gem::Version
194
+ version: '0.5'
195
+ - !ruby/object:Gem::Dependency
196
+ name: pg
197
+ requirement: !ruby/object:Gem::Requirement
198
+ requirements:
199
+ - - "~>"
200
+ - !ruby/object:Gem::Version
201
+ version: '1.1'
202
+ type: :development
203
+ prerelease: false
204
+ version_requirements: !ruby/object:Gem::Requirement
205
+ requirements:
206
+ - - "~>"
207
+ - !ruby/object:Gem::Version
208
+ version: '1.1'
209
+ - !ruby/object:Gem::Dependency
210
+ name: rake
211
+ requirement: !ruby/object:Gem::Requirement
212
+ requirements:
213
+ - - "~>"
214
+ - !ruby/object:Gem::Version
215
+ version: '10'
216
+ type: :development
217
+ prerelease: false
218
+ version_requirements: !ruby/object:Gem::Requirement
219
+ requirements:
220
+ - - "~>"
221
+ - !ruby/object:Gem::Version
222
+ version: '10'
223
+ - !ruby/object:Gem::Dependency
224
+ name: rspec
225
+ requirement: !ruby/object:Gem::Requirement
226
+ requirements:
227
+ - - "~>"
228
+ - !ruby/object:Gem::Version
229
+ version: '3'
230
+ type: :development
231
+ prerelease: false
232
+ version_requirements: !ruby/object:Gem::Requirement
233
+ requirements:
234
+ - - "~>"
235
+ - !ruby/object:Gem::Version
236
+ version: '3'
237
+ - !ruby/object:Gem::Dependency
238
+ name: rspec_junit_formatter
239
+ requirement: !ruby/object:Gem::Requirement
240
+ requirements:
241
+ - - "~>"
242
+ - !ruby/object:Gem::Version
243
+ version: '0.3'
244
+ type: :development
245
+ prerelease: false
246
+ version_requirements: !ruby/object:Gem::Requirement
247
+ requirements:
248
+ - - "~>"
249
+ - !ruby/object:Gem::Version
250
+ version: '0.3'
251
+ - !ruby/object:Gem::Dependency
252
+ name: rubocop
253
+ requirement: !ruby/object:Gem::Requirement
254
+ requirements:
255
+ - - "~>"
256
+ - !ruby/object:Gem::Version
257
+ version: '0.72'
258
+ type: :development
259
+ prerelease: false
260
+ version_requirements: !ruby/object:Gem::Requirement
261
+ requirements:
262
+ - - "~>"
263
+ - !ruby/object:Gem::Version
264
+ version: '0.72'
265
+ - !ruby/object:Gem::Dependency
266
+ name: rubocop-rspec
267
+ requirement: !ruby/object:Gem::Requirement
268
+ requirements:
269
+ - - "~>"
270
+ - !ruby/object:Gem::Version
271
+ version: '1.27'
272
+ type: :development
273
+ prerelease: false
274
+ version_requirements: !ruby/object:Gem::Requirement
275
+ requirements:
276
+ - - "~>"
277
+ - !ruby/object:Gem::Version
278
+ version: '1.27'
279
+ - !ruby/object:Gem::Dependency
280
+ name: sqlite3
281
+ requirement: !ruby/object:Gem::Requirement
282
+ requirements:
283
+ - - "~>"
284
+ - !ruby/object:Gem::Version
285
+ version: '1.3'
286
+ type: :development
287
+ prerelease: false
288
+ version_requirements: !ruby/object:Gem::Requirement
289
+ requirements:
290
+ - - "~>"
291
+ - !ruby/object:Gem::Version
292
+ version: '1.3'
293
+ description:
294
+ email:
295
+ - daniel.orner@wishabi.com
296
+ executables:
297
+ - deimos
298
+ extensions: []
299
+ extra_rdoc_files: []
300
+ files:
301
+ - ".circleci/config.yml"
302
+ - ".gitignore"
303
+ - ".gitmodules"
304
+ - ".rspec"
305
+ - ".rubocop.yml"
306
+ - ".ruby-gemset"
307
+ - ".ruby-version"
308
+ - CHANGELOG.md
309
+ - CODE_OF_CONDUCT.md
310
+ - Dockerfile
311
+ - Gemfile
312
+ - Gemfile.lock
313
+ - Guardfile
314
+ - LICENSE.md
315
+ - README.md
316
+ - Rakefile
317
+ - bin/deimos
318
+ - deimos-kafka.gemspec
319
+ - docker-compose.yml
320
+ - docs/DATABASE_BACKEND.md
321
+ - docs/PULL_REQUEST_TEMPLATE.md
322
+ - lib/deimos.rb
323
+ - lib/deimos/active_record_consumer.rb
324
+ - lib/deimos/active_record_producer.rb
325
+ - lib/deimos/avro_data_coder.rb
326
+ - lib/deimos/avro_data_decoder.rb
327
+ - lib/deimos/avro_data_encoder.rb
328
+ - lib/deimos/backends/db.rb
329
+ - lib/deimos/backends/kafka.rb
330
+ - lib/deimos/backends/kafka_async.rb
331
+ - lib/deimos/configuration.rb
332
+ - lib/deimos/consumer.rb
333
+ - lib/deimos/instrumentation.rb
334
+ - lib/deimos/kafka_message.rb
335
+ - lib/deimos/kafka_source.rb
336
+ - lib/deimos/kafka_topic_info.rb
337
+ - lib/deimos/message.rb
338
+ - lib/deimos/metrics/datadog.rb
339
+ - lib/deimos/metrics/mock.rb
340
+ - lib/deimos/metrics/provider.rb
341
+ - lib/deimos/monkey_patches/phobos_cli.rb
342
+ - lib/deimos/monkey_patches/phobos_producer.rb
343
+ - lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb
344
+ - lib/deimos/monkey_patches/schema_store.rb
345
+ - lib/deimos/producer.rb
346
+ - lib/deimos/publish_backend.rb
347
+ - lib/deimos/railtie.rb
348
+ - lib/deimos/schema_coercer.rb
349
+ - lib/deimos/shared_config.rb
350
+ - lib/deimos/test_helpers.rb
351
+ - lib/deimos/tracing/datadog.rb
352
+ - lib/deimos/tracing/mock.rb
353
+ - lib/deimos/tracing/provider.rb
354
+ - lib/deimos/utils/db_producer.rb
355
+ - lib/deimos/utils/executor.rb
356
+ - lib/deimos/utils/inline_consumer.rb
357
+ - lib/deimos/utils/lag_reporter.rb
358
+ - lib/deimos/utils/platform_schema_validation.rb
359
+ - lib/deimos/utils/signal_handler.rb
360
+ - lib/deimos/version.rb
361
+ - lib/generators/deimos/db_backend/templates/migration
362
+ - lib/generators/deimos/db_backend/templates/rails3_migration
363
+ - lib/generators/deimos/db_backend_generator.rb
364
+ - lib/tasks/deimos.rake
365
+ - spec/active_record_consumer_spec.rb
366
+ - spec/active_record_producer_spec.rb
367
+ - spec/avro_data_decoder_spec.rb
368
+ - spec/avro_data_encoder_spec.rb
369
+ - spec/backends/db_spec.rb
370
+ - spec/backends/kafka_async_spec.rb
371
+ - spec/backends/kafka_spec.rb
372
+ - spec/consumer_spec.rb
373
+ - spec/deimos_spec.rb
374
+ - spec/kafka_source_spec.rb
375
+ - spec/kafka_topic_info_spec.rb
376
+ - spec/phobos.bad_db.yml
377
+ - spec/phobos.yml
378
+ - spec/producer_spec.rb
379
+ - spec/publish_backend_spec.rb
380
+ - spec/schemas/com/my-namespace/MySchema-key.avsc
381
+ - spec/schemas/com/my-namespace/MySchema.avsc
382
+ - spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc
383
+ - spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc
384
+ - spec/schemas/com/my-namespace/MySchemaWithId.avsc
385
+ - spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc
386
+ - spec/schemas/com/my-namespace/Widget.avsc
387
+ - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
388
+ - spec/spec_helper.rb
389
+ - spec/updateable_schema_store_spec.rb
390
+ - spec/utils/db_producer_spec.rb
391
+ - spec/utils/executor_spec.rb
392
+ - spec/utils/lag_reporter_spec.rb
393
+ - spec/utils/platform_schema_validation_spec.rb
394
+ - spec/utils/signal_handler_spec.rb
395
+ - support/deimos-solo.png
396
+ - support/deimos-with-name-next.png
397
+ - support/deimos-with-name.png
398
+ - support/flipp-logo.png
399
+ homepage: ''
400
+ licenses:
401
+ - Apache 2.0
402
+ metadata: {}
403
+ post_install_message:
404
+ rdoc_options: []
405
+ require_paths:
406
+ - lib
407
+ required_ruby_version: !ruby/object:Gem::Requirement
408
+ requirements:
409
+ - - ">="
410
+ - !ruby/object:Gem::Version
411
+ version: '0'
412
+ required_rubygems_version: !ruby/object:Gem::Requirement
413
+ requirements:
414
+ - - ">"
415
+ - !ruby/object:Gem::Version
416
+ version: 1.3.1
417
+ requirements: []
418
+ rubygems_version: 3.0.2
419
+ signing_key:
420
+ specification_version: 4
421
+ summary: Kafka libraries for Ruby.
422
+ test_files:
423
+ - spec/active_record_consumer_spec.rb
424
+ - spec/active_record_producer_spec.rb
425
+ - spec/avro_data_decoder_spec.rb
426
+ - spec/avro_data_encoder_spec.rb
427
+ - spec/backends/db_spec.rb
428
+ - spec/backends/kafka_async_spec.rb
429
+ - spec/backends/kafka_spec.rb
430
+ - spec/consumer_spec.rb
431
+ - spec/deimos_spec.rb
432
+ - spec/kafka_source_spec.rb
433
+ - spec/kafka_topic_info_spec.rb
434
+ - spec/phobos.bad_db.yml
435
+ - spec/phobos.yml
436
+ - spec/producer_spec.rb
437
+ - spec/publish_backend_spec.rb
438
+ - spec/schemas/com/my-namespace/MySchema-key.avsc
439
+ - spec/schemas/com/my-namespace/MySchema.avsc
440
+ - spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc
441
+ - spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc
442
+ - spec/schemas/com/my-namespace/MySchemaWithId.avsc
443
+ - spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc
444
+ - spec/schemas/com/my-namespace/Widget.avsc
445
+ - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
446
+ - spec/spec_helper.rb
447
+ - spec/updateable_schema_store_spec.rb
448
+ - spec/utils/db_producer_spec.rb
449
+ - spec/utils/executor_spec.rb
450
+ - spec/utils/lag_reporter_spec.rb
451
+ - spec/utils/platform_schema_validation_spec.rb
452
+ - spec/utils/signal_handler_spec.rb