deimos-temp-fork 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (146) hide show
  1. checksums.yaml +7 -0
  2. data/.circleci/config.yml +83 -0
  3. data/.gitignore +41 -0
  4. data/.gitmodules +0 -0
  5. data/.rspec +1 -0
  6. data/.rubocop.yml +333 -0
  7. data/.ruby-gemset +1 -0
  8. data/.ruby-version +1 -0
  9. data/CHANGELOG.md +349 -0
  10. data/CODE_OF_CONDUCT.md +77 -0
  11. data/Dockerfile +23 -0
  12. data/Gemfile +6 -0
  13. data/Gemfile.lock +286 -0
  14. data/Guardfile +22 -0
  15. data/LICENSE.md +195 -0
  16. data/README.md +1099 -0
  17. data/Rakefile +13 -0
  18. data/bin/deimos +4 -0
  19. data/deimos-ruby.gemspec +44 -0
  20. data/docker-compose.yml +71 -0
  21. data/docs/ARCHITECTURE.md +140 -0
  22. data/docs/CONFIGURATION.md +236 -0
  23. data/docs/DATABASE_BACKEND.md +147 -0
  24. data/docs/INTEGRATION_TESTS.md +52 -0
  25. data/docs/PULL_REQUEST_TEMPLATE.md +35 -0
  26. data/docs/UPGRADING.md +128 -0
  27. data/lib/deimos-temp-fork.rb +95 -0
  28. data/lib/deimos/active_record_consume/batch_consumption.rb +164 -0
  29. data/lib/deimos/active_record_consume/batch_slicer.rb +27 -0
  30. data/lib/deimos/active_record_consume/message_consumption.rb +79 -0
  31. data/lib/deimos/active_record_consume/schema_model_converter.rb +52 -0
  32. data/lib/deimos/active_record_consumer.rb +67 -0
  33. data/lib/deimos/active_record_producer.rb +87 -0
  34. data/lib/deimos/backends/base.rb +32 -0
  35. data/lib/deimos/backends/db.rb +41 -0
  36. data/lib/deimos/backends/kafka.rb +33 -0
  37. data/lib/deimos/backends/kafka_async.rb +33 -0
  38. data/lib/deimos/backends/test.rb +20 -0
  39. data/lib/deimos/batch_consumer.rb +7 -0
  40. data/lib/deimos/config/configuration.rb +381 -0
  41. data/lib/deimos/config/phobos_config.rb +137 -0
  42. data/lib/deimos/consume/batch_consumption.rb +150 -0
  43. data/lib/deimos/consume/message_consumption.rb +94 -0
  44. data/lib/deimos/consumer.rb +104 -0
  45. data/lib/deimos/instrumentation.rb +76 -0
  46. data/lib/deimos/kafka_message.rb +60 -0
  47. data/lib/deimos/kafka_source.rb +128 -0
  48. data/lib/deimos/kafka_topic_info.rb +102 -0
  49. data/lib/deimos/message.rb +79 -0
  50. data/lib/deimos/metrics/datadog.rb +47 -0
  51. data/lib/deimos/metrics/mock.rb +39 -0
  52. data/lib/deimos/metrics/provider.rb +36 -0
  53. data/lib/deimos/monkey_patches/phobos_cli.rb +35 -0
  54. data/lib/deimos/monkey_patches/phobos_producer.rb +51 -0
  55. data/lib/deimos/poll_info.rb +9 -0
  56. data/lib/deimos/producer.rb +224 -0
  57. data/lib/deimos/railtie.rb +8 -0
  58. data/lib/deimos/schema_backends/avro_base.rb +140 -0
  59. data/lib/deimos/schema_backends/avro_local.rb +30 -0
  60. data/lib/deimos/schema_backends/avro_schema_coercer.rb +119 -0
  61. data/lib/deimos/schema_backends/avro_schema_registry.rb +34 -0
  62. data/lib/deimos/schema_backends/avro_validation.rb +21 -0
  63. data/lib/deimos/schema_backends/base.rb +150 -0
  64. data/lib/deimos/schema_backends/mock.rb +42 -0
  65. data/lib/deimos/shared_config.rb +63 -0
  66. data/lib/deimos/test_helpers.rb +360 -0
  67. data/lib/deimos/tracing/datadog.rb +35 -0
  68. data/lib/deimos/tracing/mock.rb +40 -0
  69. data/lib/deimos/tracing/provider.rb +29 -0
  70. data/lib/deimos/utils/db_poller.rb +150 -0
  71. data/lib/deimos/utils/db_producer.rb +243 -0
  72. data/lib/deimos/utils/deadlock_retry.rb +68 -0
  73. data/lib/deimos/utils/inline_consumer.rb +150 -0
  74. data/lib/deimos/utils/lag_reporter.rb +175 -0
  75. data/lib/deimos/utils/schema_controller_mixin.rb +115 -0
  76. data/lib/deimos/version.rb +5 -0
  77. data/lib/generators/deimos/active_record/templates/migration.rb.tt +28 -0
  78. data/lib/generators/deimos/active_record/templates/model.rb.tt +5 -0
  79. data/lib/generators/deimos/active_record_generator.rb +79 -0
  80. data/lib/generators/deimos/db_backend/templates/migration +25 -0
  81. data/lib/generators/deimos/db_backend/templates/rails3_migration +31 -0
  82. data/lib/generators/deimos/db_backend_generator.rb +48 -0
  83. data/lib/generators/deimos/db_poller/templates/migration +11 -0
  84. data/lib/generators/deimos/db_poller/templates/rails3_migration +16 -0
  85. data/lib/generators/deimos/db_poller_generator.rb +48 -0
  86. data/lib/tasks/deimos.rake +34 -0
  87. data/spec/active_record_batch_consumer_spec.rb +481 -0
  88. data/spec/active_record_consume/batch_slicer_spec.rb +42 -0
  89. data/spec/active_record_consume/schema_model_converter_spec.rb +105 -0
  90. data/spec/active_record_consumer_spec.rb +154 -0
  91. data/spec/active_record_producer_spec.rb +85 -0
  92. data/spec/backends/base_spec.rb +10 -0
  93. data/spec/backends/db_spec.rb +54 -0
  94. data/spec/backends/kafka_async_spec.rb +11 -0
  95. data/spec/backends/kafka_spec.rb +11 -0
  96. data/spec/batch_consumer_spec.rb +256 -0
  97. data/spec/config/configuration_spec.rb +248 -0
  98. data/spec/consumer_spec.rb +209 -0
  99. data/spec/deimos_spec.rb +169 -0
  100. data/spec/generators/active_record_generator_spec.rb +56 -0
  101. data/spec/handlers/my_batch_consumer.rb +10 -0
  102. data/spec/handlers/my_consumer.rb +10 -0
  103. data/spec/kafka_listener_spec.rb +55 -0
  104. data/spec/kafka_source_spec.rb +381 -0
  105. data/spec/kafka_topic_info_spec.rb +111 -0
  106. data/spec/message_spec.rb +19 -0
  107. data/spec/phobos.bad_db.yml +73 -0
  108. data/spec/phobos.yml +77 -0
  109. data/spec/producer_spec.rb +498 -0
  110. data/spec/rake_spec.rb +19 -0
  111. data/spec/schema_backends/avro_base_shared.rb +199 -0
  112. data/spec/schema_backends/avro_local_spec.rb +32 -0
  113. data/spec/schema_backends/avro_schema_registry_spec.rb +32 -0
  114. data/spec/schema_backends/avro_validation_spec.rb +24 -0
  115. data/spec/schema_backends/base_spec.rb +33 -0
  116. data/spec/schemas/com/my-namespace/Generated.avsc +71 -0
  117. data/spec/schemas/com/my-namespace/MyNestedSchema.avsc +62 -0
  118. data/spec/schemas/com/my-namespace/MySchema-key.avsc +13 -0
  119. data/spec/schemas/com/my-namespace/MySchema.avsc +18 -0
  120. data/spec/schemas/com/my-namespace/MySchemaCompound-key.avsc +18 -0
  121. data/spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc +18 -0
  122. data/spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc +33 -0
  123. data/spec/schemas/com/my-namespace/MySchemaWithId.avsc +28 -0
  124. data/spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc +32 -0
  125. data/spec/schemas/com/my-namespace/Wibble.avsc +43 -0
  126. data/spec/schemas/com/my-namespace/Widget.avsc +27 -0
  127. data/spec/schemas/com/my-namespace/WidgetTheSecond.avsc +27 -0
  128. data/spec/schemas/com/my-namespace/request/CreateTopic.avsc +11 -0
  129. data/spec/schemas/com/my-namespace/request/Index.avsc +11 -0
  130. data/spec/schemas/com/my-namespace/request/UpdateRequest.avsc +11 -0
  131. data/spec/schemas/com/my-namespace/response/CreateTopic.avsc +11 -0
  132. data/spec/schemas/com/my-namespace/response/Index.avsc +11 -0
  133. data/spec/schemas/com/my-namespace/response/UpdateResponse.avsc +11 -0
  134. data/spec/spec_helper.rb +267 -0
  135. data/spec/utils/db_poller_spec.rb +320 -0
  136. data/spec/utils/db_producer_spec.rb +514 -0
  137. data/spec/utils/deadlock_retry_spec.rb +74 -0
  138. data/spec/utils/inline_consumer_spec.rb +31 -0
  139. data/spec/utils/lag_reporter_spec.rb +76 -0
  140. data/spec/utils/platform_schema_validation_spec.rb +0 -0
  141. data/spec/utils/schema_controller_mixin_spec.rb +84 -0
  142. data/support/deimos-solo.png +0 -0
  143. data/support/deimos-with-name-next.png +0 -0
  144. data/support/deimos-with-name.png +0 -0
  145. data/support/flipp-logo.png +0 -0
  146. metadata +551 -0
@@ -0,0 +1,74 @@
1
+ # frozen_string_literal: true
2
+
3
+ RSpec.describe Deimos::Utils::DeadlockRetry do
4
+ include_context 'with widgets'
5
+
6
+ before(:each) do
7
+ allow(described_class).to receive(:sleep)
8
+ end
9
+
10
+ describe 'deadlock handling' do
11
+ let(:batch) { [{ key: 1, payload: { test_id: 'abc', some_int: 3 } }] }
12
+
13
+ it 'should retry deadlocks 3 times' do
14
+ # Should receive original attempt + 2 retries
15
+ expect(Widget).
16
+ to receive(:create).
17
+ and_raise(ActiveRecord::Deadlocked.new('Lock wait timeout exceeded')).
18
+ exactly(3).times
19
+
20
+ # After 3 tries, should let it bubble up
21
+ expect {
22
+ described_class.wrap do
23
+ Widget.create(test_id: 'abc')
24
+ end
25
+ }.to raise_error(ActiveRecord::Deadlocked)
26
+ end
27
+
28
+ it 'should stop retrying deadlocks after success' do
29
+ allow(Widget).
30
+ to receive(:create).
31
+ with(hash_including(test_id: 'first')).
32
+ and_call_original
33
+
34
+ # Fail on first attempt, succeed on second
35
+ expect(Widget).
36
+ to receive(:create).
37
+ with(hash_including(test_id: 'second')).
38
+ and_raise(ActiveRecord::Deadlocked.new('Deadlock found when trying to get lock')).
39
+ once.
40
+ ordered
41
+
42
+ expect(Widget).
43
+ to receive(:create).
44
+ with(hash_including(test_id: 'second')).
45
+ once.
46
+ ordered.
47
+ and_call_original
48
+
49
+ # Should not raise anything
50
+ described_class.wrap do
51
+ Widget.create(test_id: 'first')
52
+ Widget.create(test_id: 'second')
53
+ end
54
+
55
+ expect(Widget.all).to match_array([
56
+ have_attributes(test_id: 'first'),
57
+ have_attributes(test_id: 'second')
58
+ ])
59
+ end
60
+
61
+ it 'should not retry non-deadlock exceptions' do
62
+ expect(Widget).
63
+ to receive(:create).
64
+ and_raise(ActiveRecord::StatementInvalid.new('Oops!!')).
65
+ once
66
+
67
+ expect {
68
+ described_class.wrap do
69
+ Widget.create(test_id: 'abc')
70
+ end
71
+ }.to raise_error(ActiveRecord::StatementInvalid, 'Oops!!')
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ describe Deimos::Utils::SeekListener do
4
+
5
+ describe '#start_listener' do
6
+ let(:consumer) { instance_double(Kafka::Consumer) }
7
+ let(:handler) { class_double(Deimos::Utils::MessageBankHandler) }
8
+
9
+ before(:each) do
10
+ allow(handler).to receive(:start)
11
+ allow(consumer).to receive(:subscribe)
12
+ allow_any_instance_of(Phobos::Listener).to receive(:create_kafka_consumer).and_return(consumer)
13
+ allow_any_instance_of(Kafka::Client).to receive(:last_offset_for).and_return(100)
14
+ stub_const('Deimos::Utils::SeekListener::MAX_SEEK_RETRIES', 2)
15
+ end
16
+
17
+ it 'should seek offset' do
18
+ allow(consumer).to receive(:seek)
19
+ expect(consumer).to receive(:seek).once
20
+ seek_listener = described_class.new({ handler: handler, group_id: 999, topic: 'test_topic' })
21
+ seek_listener.start_listener
22
+ end
23
+
24
+ it 'should retry on errors when seeking offset' do
25
+ allow(consumer).to receive(:seek).and_raise(StandardError)
26
+ expect(consumer).to receive(:seek).twice
27
+ seek_listener = described_class.new({ handler: handler, group_id: 999, topic: 'test_topic' })
28
+ seek_listener.start_listener
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,76 @@
1
+ # frozen_string_literal: true
2
+
3
+ describe Deimos::Utils::LagReporter do
4
+
5
+ let(:kafka_client) { instance_double(Kafka::Client) }
6
+ let(:partition1_tags) { %w(consumer_group:group1 partition:1 topic:my-topic) }
7
+ let(:partition2_tags) { %w(consumer_group:group1 partition:2 topic:my-topic) }
8
+
9
+ before(:each) do
10
+ allow(kafka_client).to receive(:last_offset_for).and_return(100)
11
+ allow(Phobos).to receive(:create_kafka_client).and_return(kafka_client)
12
+ Deimos.configure { |c| c.consumers.report_lag = true }
13
+ end
14
+
15
+ after(:each) do
16
+ described_class.reset
17
+ Deimos.configure { |c| c.consumers.report_lag = false }
18
+ end
19
+
20
+ it 'should not report lag before ready' do
21
+ expect(Deimos.config.metrics).not_to receive(:gauge)
22
+ ActiveSupport::Notifications.instrument(
23
+ 'heartbeat.consumer.kafka',
24
+ group_id: 'group1', topic_partitions: { 'my-topic': [1] }
25
+ )
26
+ end
27
+
28
+ it 'should report lag' do
29
+ expect(Deimos.config.metrics).to receive(:gauge).ordered.twice.
30
+ with('consumer_lag', 95, tags: partition1_tags)
31
+ expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
32
+ with('consumer_lag', 80, tags: partition2_tags)
33
+ expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
34
+ with('consumer_lag', 0, tags: partition2_tags)
35
+ ActiveSupport::Notifications.instrument(
36
+ 'seek.consumer.kafka',
37
+ offset: 5, topic: 'my-topic', group_id: 'group1', partition: 1
38
+ )
39
+ ActiveSupport::Notifications.instrument(
40
+ 'start_process_message.consumer.kafka',
41
+ offset: 20, topic: 'my-topic', group_id: 'group1', partition: 2
42
+ )
43
+ ActiveSupport::Notifications.instrument(
44
+ 'heartbeat.consumer.kafka',
45
+ group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
46
+ )
47
+ ActiveSupport::Notifications.instrument(
48
+ 'start_process_batch.consumer.kafka',
49
+ last_offset: 100, topic: 'my-topic', group_id: 'group1', partition: 2
50
+ )
51
+ ActiveSupport::Notifications.instrument(
52
+ 'heartbeat.consumer.kafka',
53
+ group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
54
+ )
55
+ end
56
+
57
+ it 'should update lag after heartbeat' do
58
+ expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
59
+ with('consumer_lag', 94, tags: partition2_tags)
60
+ expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
61
+ with('consumer_lag', 95, tags: partition2_tags)
62
+ ActiveSupport::Notifications.instrument(
63
+ 'seek.consumer.kafka',
64
+ offset: 6, topic: 'my-topic', group_id: 'group1', partition: 2
65
+ )
66
+ ActiveSupport::Notifications.instrument(
67
+ 'heartbeat.consumer.kafka',
68
+ group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
69
+ )
70
+ allow(kafka_client).to receive(:last_offset_for).and_return(101)
71
+ ActiveSupport::Notifications.instrument(
72
+ 'heartbeat.consumer.kafka',
73
+ group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
74
+ )
75
+ end
76
+ end
File without changes
@@ -0,0 +1,84 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'deimos/utils/schema_controller_mixin'
4
+ require 'deimos/schema_backends/avro_local'
5
+
6
+ RSpec.describe Deimos::Utils::SchemaControllerMixin, type: :controller do
7
+
8
+ before(:each) do
9
+ Deimos.configure do
10
+ schema.backend(:avro_local)
11
+ end
12
+ end
13
+
14
+ controller(ActionController::Base) do
15
+ include Deimos::Utils::SchemaControllerMixin # rubocop:disable RSpec/DescribedClass
16
+
17
+ request_namespace 'com.my-namespace.request'
18
+ response_namespace 'com.my-namespace.response'
19
+ schemas :index, :show
20
+ schemas create: 'CreateTopic'
21
+ schemas :update, request: 'UpdateRequest', response: 'UpdateResponse'
22
+
23
+ # :nodoc:
24
+ def index
25
+ render_schema({ 'response_id' => payload[:request_id] + ' mom' })
26
+ end
27
+
28
+ # :nodoc:
29
+ def show
30
+ render_schema({ 'response_id' => payload[:request_id] + ' dad' })
31
+ end
32
+
33
+ # :nodoc:
34
+ def create
35
+ render_schema({ 'response_id' => payload[:request_id] + ' bro' })
36
+ end
37
+
38
+ # :nodoc:
39
+ def update
40
+ render_schema({ 'update_response_id' => payload[:update_request_id] + ' sis' })
41
+ end
42
+ end
43
+
44
+ it 'should render the correct response for index' do
45
+ request_backend = Deimos.schema_backend(schema: 'Index',
46
+ namespace: 'com.my-namespace.request')
47
+ response_backend = Deimos.schema_backend(schema: 'Index',
48
+ namespace: 'com.my-namespace.response')
49
+ request.content_type = 'avro/binary'
50
+ get :index, body: request_backend.encode({ 'request_id' => 'hi' })
51
+ expect(response_backend.decode(response.body)).to eq({ 'response_id' => 'hi mom' })
52
+ end
53
+
54
+ it 'should render the correct response for show' do
55
+ request_backend = Deimos.schema_backend(schema: 'Index',
56
+ namespace: 'com.my-namespace.request')
57
+ response_backend = Deimos.schema_backend(schema: 'Index',
58
+ namespace: 'com.my-namespace.response')
59
+ request.content_type = 'avro/binary'
60
+ get :show, params: { id: 1 }, body: request_backend.encode({ 'request_id' => 'hi' })
61
+ expect(response_backend.decode(response.body)).to eq({ 'response_id' => 'hi dad' })
62
+ end
63
+
64
+ it 'should render the correct response for update' do
65
+ request_backend = Deimos.schema_backend(schema: 'UpdateRequest',
66
+ namespace: 'com.my-namespace.request')
67
+ response_backend = Deimos.schema_backend(schema: 'UpdateResponse',
68
+ namespace: 'com.my-namespace.response')
69
+ request.content_type = 'avro/binary'
70
+ post :update, params: { id: 1 }, body: request_backend.encode({ 'update_request_id' => 'hi' })
71
+ expect(response_backend.decode(response.body)).to eq({ 'update_response_id' => 'hi sis' })
72
+ end
73
+
74
+ it 'should render the correct response for create' do
75
+ request_backend = Deimos.schema_backend(schema: 'CreateTopic',
76
+ namespace: 'com.my-namespace.request')
77
+ response_backend = Deimos.schema_backend(schema: 'CreateTopic',
78
+ namespace: 'com.my-namespace.response')
79
+ request.content_type = 'avro/binary'
80
+ post :create, params: { id: 1 }, body: request_backend.encode({ 'request_id' => 'hi' })
81
+ expect(response_backend.decode(response.body)).to eq({ 'response_id' => 'hi bro' })
82
+ end
83
+
84
+ end
Binary file
Binary file
Binary file
Binary file
metadata ADDED
@@ -0,0 +1,551 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: deimos-temp-fork
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Daniel Orner
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2021-08-16 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: avro_turf
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '0.11'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '0.11'
27
+ - !ruby/object:Gem::Dependency
28
+ name: phobos_temp_fork
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: sigurd
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: 0.0.1
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: 0.0.1
55
+ - !ruby/object:Gem::Dependency
56
+ name: fig_tree
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: 0.0.2
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: 0.0.2
69
+ - !ruby/object:Gem::Dependency
70
+ name: activerecord-import
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: avro
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '1.9'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '1.9'
97
+ - !ruby/object:Gem::Dependency
98
+ name: database_cleaner
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - "~>"
102
+ - !ruby/object:Gem::Version
103
+ version: '1.7'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - "~>"
109
+ - !ruby/object:Gem::Version
110
+ version: '1.7'
111
+ - !ruby/object:Gem::Dependency
112
+ name: ddtrace
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '0.11'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '0.11'
125
+ - !ruby/object:Gem::Dependency
126
+ name: dogstatsd-ruby
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: '4.2'
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: '4.2'
139
+ - !ruby/object:Gem::Dependency
140
+ name: guard
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - "~>"
144
+ - !ruby/object:Gem::Version
145
+ version: '2'
146
+ type: :development
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - "~>"
151
+ - !ruby/object:Gem::Version
152
+ version: '2'
153
+ - !ruby/object:Gem::Dependency
154
+ name: guard-rspec
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - "~>"
158
+ - !ruby/object:Gem::Version
159
+ version: '4'
160
+ type: :development
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - "~>"
165
+ - !ruby/object:Gem::Version
166
+ version: '4'
167
+ - !ruby/object:Gem::Dependency
168
+ name: guard-rubocop
169
+ requirement: !ruby/object:Gem::Requirement
170
+ requirements:
171
+ - - "~>"
172
+ - !ruby/object:Gem::Version
173
+ version: '1'
174
+ type: :development
175
+ prerelease: false
176
+ version_requirements: !ruby/object:Gem::Requirement
177
+ requirements:
178
+ - - "~>"
179
+ - !ruby/object:Gem::Version
180
+ version: '1'
181
+ - !ruby/object:Gem::Dependency
182
+ name: mysql2
183
+ requirement: !ruby/object:Gem::Requirement
184
+ requirements:
185
+ - - "~>"
186
+ - !ruby/object:Gem::Version
187
+ version: '0.5'
188
+ type: :development
189
+ prerelease: false
190
+ version_requirements: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - "~>"
193
+ - !ruby/object:Gem::Version
194
+ version: '0.5'
195
+ - !ruby/object:Gem::Dependency
196
+ name: pg
197
+ requirement: !ruby/object:Gem::Requirement
198
+ requirements:
199
+ - - "~>"
200
+ - !ruby/object:Gem::Version
201
+ version: '1.1'
202
+ type: :development
203
+ prerelease: false
204
+ version_requirements: !ruby/object:Gem::Requirement
205
+ requirements:
206
+ - - "~>"
207
+ - !ruby/object:Gem::Version
208
+ version: '1.1'
209
+ - !ruby/object:Gem::Dependency
210
+ name: rails
211
+ requirement: !ruby/object:Gem::Requirement
212
+ requirements:
213
+ - - "~>"
214
+ - !ruby/object:Gem::Version
215
+ version: '6'
216
+ type: :development
217
+ prerelease: false
218
+ version_requirements: !ruby/object:Gem::Requirement
219
+ requirements:
220
+ - - "~>"
221
+ - !ruby/object:Gem::Version
222
+ version: '6'
223
+ - !ruby/object:Gem::Dependency
224
+ name: rake
225
+ requirement: !ruby/object:Gem::Requirement
226
+ requirements:
227
+ - - "~>"
228
+ - !ruby/object:Gem::Version
229
+ version: '13'
230
+ type: :development
231
+ prerelease: false
232
+ version_requirements: !ruby/object:Gem::Requirement
233
+ requirements:
234
+ - - "~>"
235
+ - !ruby/object:Gem::Version
236
+ version: '13'
237
+ - !ruby/object:Gem::Dependency
238
+ name: rspec
239
+ requirement: !ruby/object:Gem::Requirement
240
+ requirements:
241
+ - - "~>"
242
+ - !ruby/object:Gem::Version
243
+ version: '3'
244
+ type: :development
245
+ prerelease: false
246
+ version_requirements: !ruby/object:Gem::Requirement
247
+ requirements:
248
+ - - "~>"
249
+ - !ruby/object:Gem::Version
250
+ version: '3'
251
+ - !ruby/object:Gem::Dependency
252
+ name: rspec_junit_formatter
253
+ requirement: !ruby/object:Gem::Requirement
254
+ requirements:
255
+ - - "~>"
256
+ - !ruby/object:Gem::Version
257
+ version: '0.3'
258
+ type: :development
259
+ prerelease: false
260
+ version_requirements: !ruby/object:Gem::Requirement
261
+ requirements:
262
+ - - "~>"
263
+ - !ruby/object:Gem::Version
264
+ version: '0.3'
265
+ - !ruby/object:Gem::Dependency
266
+ name: rspec-rails
267
+ requirement: !ruby/object:Gem::Requirement
268
+ requirements:
269
+ - - "~>"
270
+ - !ruby/object:Gem::Version
271
+ version: '4'
272
+ type: :development
273
+ prerelease: false
274
+ version_requirements: !ruby/object:Gem::Requirement
275
+ requirements:
276
+ - - "~>"
277
+ - !ruby/object:Gem::Version
278
+ version: '4'
279
+ - !ruby/object:Gem::Dependency
280
+ name: rubocop
281
+ requirement: !ruby/object:Gem::Requirement
282
+ requirements:
283
+ - - '='
284
+ - !ruby/object:Gem::Version
285
+ version: 0.88.0
286
+ type: :development
287
+ prerelease: false
288
+ version_requirements: !ruby/object:Gem::Requirement
289
+ requirements:
290
+ - - '='
291
+ - !ruby/object:Gem::Version
292
+ version: 0.88.0
293
+ - !ruby/object:Gem::Dependency
294
+ name: rubocop-rspec
295
+ requirement: !ruby/object:Gem::Requirement
296
+ requirements:
297
+ - - '='
298
+ - !ruby/object:Gem::Version
299
+ version: 1.42.0
300
+ type: :development
301
+ prerelease: false
302
+ version_requirements: !ruby/object:Gem::Requirement
303
+ requirements:
304
+ - - '='
305
+ - !ruby/object:Gem::Version
306
+ version: 1.42.0
307
+ - !ruby/object:Gem::Dependency
308
+ name: sqlite3
309
+ requirement: !ruby/object:Gem::Requirement
310
+ requirements:
311
+ - - "~>"
312
+ - !ruby/object:Gem::Version
313
+ version: '1.3'
314
+ type: :development
315
+ prerelease: false
316
+ version_requirements: !ruby/object:Gem::Requirement
317
+ requirements:
318
+ - - "~>"
319
+ - !ruby/object:Gem::Version
320
+ version: '1.3'
321
+ description:
322
+ email:
323
+ - daniel.orner@wishabi.com
324
+ executables:
325
+ - deimos
326
+ extensions: []
327
+ extra_rdoc_files: []
328
+ files:
329
+ - ".circleci/config.yml"
330
+ - ".gitignore"
331
+ - ".gitmodules"
332
+ - ".rspec"
333
+ - ".rubocop.yml"
334
+ - ".ruby-gemset"
335
+ - ".ruby-version"
336
+ - CHANGELOG.md
337
+ - CODE_OF_CONDUCT.md
338
+ - Dockerfile
339
+ - Gemfile
340
+ - Gemfile.lock
341
+ - Guardfile
342
+ - LICENSE.md
343
+ - README.md
344
+ - Rakefile
345
+ - bin/deimos
346
+ - deimos-ruby.gemspec
347
+ - docker-compose.yml
348
+ - docs/ARCHITECTURE.md
349
+ - docs/CONFIGURATION.md
350
+ - docs/DATABASE_BACKEND.md
351
+ - docs/INTEGRATION_TESTS.md
352
+ - docs/PULL_REQUEST_TEMPLATE.md
353
+ - docs/UPGRADING.md
354
+ - lib/deimos-temp-fork.rb
355
+ - lib/deimos/active_record_consume/batch_consumption.rb
356
+ - lib/deimos/active_record_consume/batch_slicer.rb
357
+ - lib/deimos/active_record_consume/message_consumption.rb
358
+ - lib/deimos/active_record_consume/schema_model_converter.rb
359
+ - lib/deimos/active_record_consumer.rb
360
+ - lib/deimos/active_record_producer.rb
361
+ - lib/deimos/backends/base.rb
362
+ - lib/deimos/backends/db.rb
363
+ - lib/deimos/backends/kafka.rb
364
+ - lib/deimos/backends/kafka_async.rb
365
+ - lib/deimos/backends/test.rb
366
+ - lib/deimos/batch_consumer.rb
367
+ - lib/deimos/config/configuration.rb
368
+ - lib/deimos/config/phobos_config.rb
369
+ - lib/deimos/consume/batch_consumption.rb
370
+ - lib/deimos/consume/message_consumption.rb
371
+ - lib/deimos/consumer.rb
372
+ - lib/deimos/instrumentation.rb
373
+ - lib/deimos/kafka_message.rb
374
+ - lib/deimos/kafka_source.rb
375
+ - lib/deimos/kafka_topic_info.rb
376
+ - lib/deimos/message.rb
377
+ - lib/deimos/metrics/datadog.rb
378
+ - lib/deimos/metrics/mock.rb
379
+ - lib/deimos/metrics/provider.rb
380
+ - lib/deimos/monkey_patches/phobos_cli.rb
381
+ - lib/deimos/monkey_patches/phobos_producer.rb
382
+ - lib/deimos/poll_info.rb
383
+ - lib/deimos/producer.rb
384
+ - lib/deimos/railtie.rb
385
+ - lib/deimos/schema_backends/avro_base.rb
386
+ - lib/deimos/schema_backends/avro_local.rb
387
+ - lib/deimos/schema_backends/avro_schema_coercer.rb
388
+ - lib/deimos/schema_backends/avro_schema_registry.rb
389
+ - lib/deimos/schema_backends/avro_validation.rb
390
+ - lib/deimos/schema_backends/base.rb
391
+ - lib/deimos/schema_backends/mock.rb
392
+ - lib/deimos/shared_config.rb
393
+ - lib/deimos/test_helpers.rb
394
+ - lib/deimos/tracing/datadog.rb
395
+ - lib/deimos/tracing/mock.rb
396
+ - lib/deimos/tracing/provider.rb
397
+ - lib/deimos/utils/db_poller.rb
398
+ - lib/deimos/utils/db_producer.rb
399
+ - lib/deimos/utils/deadlock_retry.rb
400
+ - lib/deimos/utils/inline_consumer.rb
401
+ - lib/deimos/utils/lag_reporter.rb
402
+ - lib/deimos/utils/schema_controller_mixin.rb
403
+ - lib/deimos/version.rb
404
+ - lib/generators/deimos/active_record/templates/migration.rb.tt
405
+ - lib/generators/deimos/active_record/templates/model.rb.tt
406
+ - lib/generators/deimos/active_record_generator.rb
407
+ - lib/generators/deimos/db_backend/templates/migration
408
+ - lib/generators/deimos/db_backend/templates/rails3_migration
409
+ - lib/generators/deimos/db_backend_generator.rb
410
+ - lib/generators/deimos/db_poller/templates/migration
411
+ - lib/generators/deimos/db_poller/templates/rails3_migration
412
+ - lib/generators/deimos/db_poller_generator.rb
413
+ - lib/tasks/deimos.rake
414
+ - spec/active_record_batch_consumer_spec.rb
415
+ - spec/active_record_consume/batch_slicer_spec.rb
416
+ - spec/active_record_consume/schema_model_converter_spec.rb
417
+ - spec/active_record_consumer_spec.rb
418
+ - spec/active_record_producer_spec.rb
419
+ - spec/backends/base_spec.rb
420
+ - spec/backends/db_spec.rb
421
+ - spec/backends/kafka_async_spec.rb
422
+ - spec/backends/kafka_spec.rb
423
+ - spec/batch_consumer_spec.rb
424
+ - spec/config/configuration_spec.rb
425
+ - spec/consumer_spec.rb
426
+ - spec/deimos_spec.rb
427
+ - spec/generators/active_record_generator_spec.rb
428
+ - spec/handlers/my_batch_consumer.rb
429
+ - spec/handlers/my_consumer.rb
430
+ - spec/kafka_listener_spec.rb
431
+ - spec/kafka_source_spec.rb
432
+ - spec/kafka_topic_info_spec.rb
433
+ - spec/message_spec.rb
434
+ - spec/phobos.bad_db.yml
435
+ - spec/phobos.yml
436
+ - spec/producer_spec.rb
437
+ - spec/rake_spec.rb
438
+ - spec/schema_backends/avro_base_shared.rb
439
+ - spec/schema_backends/avro_local_spec.rb
440
+ - spec/schema_backends/avro_schema_registry_spec.rb
441
+ - spec/schema_backends/avro_validation_spec.rb
442
+ - spec/schema_backends/base_spec.rb
443
+ - spec/schemas/com/my-namespace/Generated.avsc
444
+ - spec/schemas/com/my-namespace/MyNestedSchema.avsc
445
+ - spec/schemas/com/my-namespace/MySchema-key.avsc
446
+ - spec/schemas/com/my-namespace/MySchema.avsc
447
+ - spec/schemas/com/my-namespace/MySchemaCompound-key.avsc
448
+ - spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc
449
+ - spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc
450
+ - spec/schemas/com/my-namespace/MySchemaWithId.avsc
451
+ - spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc
452
+ - spec/schemas/com/my-namespace/Wibble.avsc
453
+ - spec/schemas/com/my-namespace/Widget.avsc
454
+ - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
455
+ - spec/schemas/com/my-namespace/request/CreateTopic.avsc
456
+ - spec/schemas/com/my-namespace/request/Index.avsc
457
+ - spec/schemas/com/my-namespace/request/UpdateRequest.avsc
458
+ - spec/schemas/com/my-namespace/response/CreateTopic.avsc
459
+ - spec/schemas/com/my-namespace/response/Index.avsc
460
+ - spec/schemas/com/my-namespace/response/UpdateResponse.avsc
461
+ - spec/spec_helper.rb
462
+ - spec/utils/db_poller_spec.rb
463
+ - spec/utils/db_producer_spec.rb
464
+ - spec/utils/deadlock_retry_spec.rb
465
+ - spec/utils/inline_consumer_spec.rb
466
+ - spec/utils/lag_reporter_spec.rb
467
+ - spec/utils/platform_schema_validation_spec.rb
468
+ - spec/utils/schema_controller_mixin_spec.rb
469
+ - support/deimos-solo.png
470
+ - support/deimos-with-name-next.png
471
+ - support/deimos-with-name.png
472
+ - support/flipp-logo.png
473
+ homepage: ''
474
+ licenses:
475
+ - Apache-2.0
476
+ metadata: {}
477
+ post_install_message:
478
+ rdoc_options: []
479
+ require_paths:
480
+ - lib
481
+ required_ruby_version: !ruby/object:Gem::Requirement
482
+ requirements:
483
+ - - ">="
484
+ - !ruby/object:Gem::Version
485
+ version: '0'
486
+ required_rubygems_version: !ruby/object:Gem::Requirement
487
+ requirements:
488
+ - - ">="
489
+ - !ruby/object:Gem::Version
490
+ version: '0'
491
+ requirements: []
492
+ rubygems_version: 3.2.3
493
+ signing_key:
494
+ specification_version: 4
495
+ summary: Kafka libraries for Ruby.
496
+ test_files:
497
+ - spec/active_record_batch_consumer_spec.rb
498
+ - spec/active_record_consume/batch_slicer_spec.rb
499
+ - spec/active_record_consume/schema_model_converter_spec.rb
500
+ - spec/active_record_consumer_spec.rb
501
+ - spec/active_record_producer_spec.rb
502
+ - spec/backends/base_spec.rb
503
+ - spec/backends/db_spec.rb
504
+ - spec/backends/kafka_async_spec.rb
505
+ - spec/backends/kafka_spec.rb
506
+ - spec/batch_consumer_spec.rb
507
+ - spec/config/configuration_spec.rb
508
+ - spec/consumer_spec.rb
509
+ - spec/deimos_spec.rb
510
+ - spec/generators/active_record_generator_spec.rb
511
+ - spec/handlers/my_batch_consumer.rb
512
+ - spec/handlers/my_consumer.rb
513
+ - spec/kafka_listener_spec.rb
514
+ - spec/kafka_source_spec.rb
515
+ - spec/kafka_topic_info_spec.rb
516
+ - spec/message_spec.rb
517
+ - spec/phobos.bad_db.yml
518
+ - spec/phobos.yml
519
+ - spec/producer_spec.rb
520
+ - spec/rake_spec.rb
521
+ - spec/schema_backends/avro_base_shared.rb
522
+ - spec/schema_backends/avro_local_spec.rb
523
+ - spec/schema_backends/avro_schema_registry_spec.rb
524
+ - spec/schema_backends/avro_validation_spec.rb
525
+ - spec/schema_backends/base_spec.rb
526
+ - spec/schemas/com/my-namespace/Generated.avsc
527
+ - spec/schemas/com/my-namespace/MyNestedSchema.avsc
528
+ - spec/schemas/com/my-namespace/MySchema-key.avsc
529
+ - spec/schemas/com/my-namespace/MySchema.avsc
530
+ - spec/schemas/com/my-namespace/MySchemaCompound-key.avsc
531
+ - spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc
532
+ - spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc
533
+ - spec/schemas/com/my-namespace/MySchemaWithId.avsc
534
+ - spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc
535
+ - spec/schemas/com/my-namespace/Wibble.avsc
536
+ - spec/schemas/com/my-namespace/Widget.avsc
537
+ - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
538
+ - spec/schemas/com/my-namespace/request/CreateTopic.avsc
539
+ - spec/schemas/com/my-namespace/request/Index.avsc
540
+ - spec/schemas/com/my-namespace/request/UpdateRequest.avsc
541
+ - spec/schemas/com/my-namespace/response/CreateTopic.avsc
542
+ - spec/schemas/com/my-namespace/response/Index.avsc
543
+ - spec/schemas/com/my-namespace/response/UpdateResponse.avsc
544
+ - spec/spec_helper.rb
545
+ - spec/utils/db_poller_spec.rb
546
+ - spec/utils/db_producer_spec.rb
547
+ - spec/utils/deadlock_retry_spec.rb
548
+ - spec/utils/inline_consumer_spec.rb
549
+ - spec/utils/lag_reporter_spec.rb
550
+ - spec/utils/platform_schema_validation_spec.rb
551
+ - spec/utils/schema_controller_mixin_spec.rb