deimos-ruby 1.0.0.pre.beta22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.circleci/config.yml +74 -0
- data/.gitignore +41 -0
- data/.gitmodules +0 -0
- data/.rspec +1 -0
- data/.rubocop.yml +321 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/CHANGELOG.md +32 -0
- data/CODE_OF_CONDUCT.md +77 -0
- data/Dockerfile +23 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +165 -0
- data/Guardfile +22 -0
- data/LICENSE.md +195 -0
- data/README.md +752 -0
- data/Rakefile +13 -0
- data/bin/deimos +4 -0
- data/deimos-kafka.gemspec +42 -0
- data/docker-compose.yml +71 -0
- data/docs/DATABASE_BACKEND.md +147 -0
- data/docs/PULL_REQUEST_TEMPLATE.md +34 -0
- data/lib/deimos/active_record_consumer.rb +81 -0
- data/lib/deimos/active_record_producer.rb +64 -0
- data/lib/deimos/avro_data_coder.rb +89 -0
- data/lib/deimos/avro_data_decoder.rb +36 -0
- data/lib/deimos/avro_data_encoder.rb +51 -0
- data/lib/deimos/backends/db.rb +27 -0
- data/lib/deimos/backends/kafka.rb +27 -0
- data/lib/deimos/backends/kafka_async.rb +27 -0
- data/lib/deimos/configuration.rb +90 -0
- data/lib/deimos/consumer.rb +164 -0
- data/lib/deimos/instrumentation.rb +71 -0
- data/lib/deimos/kafka_message.rb +27 -0
- data/lib/deimos/kafka_source.rb +126 -0
- data/lib/deimos/kafka_topic_info.rb +86 -0
- data/lib/deimos/message.rb +74 -0
- data/lib/deimos/metrics/datadog.rb +47 -0
- data/lib/deimos/metrics/mock.rb +39 -0
- data/lib/deimos/metrics/provider.rb +38 -0
- data/lib/deimos/monkey_patches/phobos_cli.rb +35 -0
- data/lib/deimos/monkey_patches/phobos_producer.rb +51 -0
- data/lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb +85 -0
- data/lib/deimos/monkey_patches/schema_store.rb +19 -0
- data/lib/deimos/producer.rb +218 -0
- data/lib/deimos/publish_backend.rb +30 -0
- data/lib/deimos/railtie.rb +8 -0
- data/lib/deimos/schema_coercer.rb +108 -0
- data/lib/deimos/shared_config.rb +59 -0
- data/lib/deimos/test_helpers.rb +356 -0
- data/lib/deimos/tracing/datadog.rb +35 -0
- data/lib/deimos/tracing/mock.rb +40 -0
- data/lib/deimos/tracing/provider.rb +31 -0
- data/lib/deimos/utils/db_producer.rb +122 -0
- data/lib/deimos/utils/executor.rb +117 -0
- data/lib/deimos/utils/inline_consumer.rb +144 -0
- data/lib/deimos/utils/lag_reporter.rb +182 -0
- data/lib/deimos/utils/platform_schema_validation.rb +0 -0
- data/lib/deimos/utils/signal_handler.rb +68 -0
- data/lib/deimos/version.rb +5 -0
- data/lib/deimos.rb +133 -0
- data/lib/generators/deimos/db_backend/templates/migration +24 -0
- data/lib/generators/deimos/db_backend/templates/rails3_migration +30 -0
- data/lib/generators/deimos/db_backend_generator.rb +48 -0
- data/lib/tasks/deimos.rake +27 -0
- data/spec/active_record_consumer_spec.rb +81 -0
- data/spec/active_record_producer_spec.rb +107 -0
- data/spec/avro_data_decoder_spec.rb +18 -0
- data/spec/avro_data_encoder_spec.rb +37 -0
- data/spec/backends/db_spec.rb +35 -0
- data/spec/backends/kafka_async_spec.rb +11 -0
- data/spec/backends/kafka_spec.rb +11 -0
- data/spec/consumer_spec.rb +169 -0
- data/spec/deimos_spec.rb +120 -0
- data/spec/kafka_source_spec.rb +168 -0
- data/spec/kafka_topic_info_spec.rb +88 -0
- data/spec/phobos.bad_db.yml +73 -0
- data/spec/phobos.yml +73 -0
- data/spec/producer_spec.rb +397 -0
- data/spec/publish_backend_spec.rb +10 -0
- data/spec/schemas/com/my-namespace/MySchema-key.avsc +13 -0
- data/spec/schemas/com/my-namespace/MySchema.avsc +18 -0
- data/spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc +18 -0
- data/spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc +33 -0
- data/spec/schemas/com/my-namespace/MySchemaWithId.avsc +28 -0
- data/spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc +32 -0
- data/spec/schemas/com/my-namespace/Widget.avsc +27 -0
- data/spec/schemas/com/my-namespace/WidgetTheSecond.avsc +27 -0
- data/spec/spec_helper.rb +207 -0
- data/spec/updateable_schema_store_spec.rb +36 -0
- data/spec/utils/db_producer_spec.rb +259 -0
- data/spec/utils/executor_spec.rb +42 -0
- data/spec/utils/lag_reporter_spec.rb +69 -0
- data/spec/utils/platform_schema_validation_spec.rb +0 -0
- data/spec/utils/signal_handler_spec.rb +16 -0
- data/support/deimos-solo.png +0 -0
- data/support/deimos-with-name-next.png +0 -0
- data/support/deimos-with-name.png +0 -0
- data/support/flipp-logo.png +0 -0
- metadata +452 -0
@@ -0,0 +1,30 @@
|
|
1
|
+
class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
|
2
|
+
def self.up
|
3
|
+
create_table :kafka_messages, force: true do |t|
|
4
|
+
t.string :topic, null: false
|
5
|
+
t.binary :message, limit: 10.megabytes
|
6
|
+
t.binary :key
|
7
|
+
t.string :partition_key
|
8
|
+
t.timestamps
|
9
|
+
end
|
10
|
+
|
11
|
+
add_index :kafka_messages, [:topic, :id]
|
12
|
+
|
13
|
+
create_table :kafka_topic_info, force: true do |t| # rubocop:disable Rails/CreateTableWithTimestamps
|
14
|
+
t.string :topic, null: false
|
15
|
+
t.string :locked_by
|
16
|
+
t.datetime :locked_at
|
17
|
+
t.boolean :error, null: false, default: false
|
18
|
+
t.integer :retries, null: false, default: 0
|
19
|
+
end
|
20
|
+
add_index :kafka_topic_info, :topic, unique: true
|
21
|
+
add_index :kafka_topic_info, [:locked_by, :error]
|
22
|
+
add_index :kafka_topic_info, :locked_at
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.down
|
26
|
+
drop_table :kafka_messages
|
27
|
+
drop_table :kafka_topic_info
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'rails/generators'
|
4
|
+
require 'rails/generators/active_record/migration'
|
5
|
+
|
6
|
+
module Deimos
|
7
|
+
module Generators
|
8
|
+
# Generate the database backend migration.
|
9
|
+
class DbBackendGenerator < Rails::Generators::Base
|
10
|
+
include Rails::Generators::Migration
|
11
|
+
if Rails.version < '4'
|
12
|
+
extend(ActiveRecord::Generators::Migration)
|
13
|
+
else
|
14
|
+
include ActiveRecord::Generators::Migration
|
15
|
+
end
|
16
|
+
source_root File.expand_path('db_backend/templates', __dir__)
|
17
|
+
desc 'Add migrations for the database backend'
|
18
|
+
|
19
|
+
# @return [String]
|
20
|
+
def migration_version
|
21
|
+
"[#{ActiveRecord::Migration.current_version}]"
|
22
|
+
rescue StandardError
|
23
|
+
''
|
24
|
+
end
|
25
|
+
|
26
|
+
# @return [String]
|
27
|
+
def db_migrate_path
|
28
|
+
if defined?(Rails.application) && Rails.application
|
29
|
+
paths = Rails.application.config.paths['db/migrate']
|
30
|
+
paths.respond_to?(:to_ary) ? paths.to_ary.first : paths.to_a.first
|
31
|
+
else
|
32
|
+
'db/migrate'
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
# Main method to create all the necessary files
|
37
|
+
def generate
|
38
|
+
if Rails.version < '4'
|
39
|
+
migration_template('rails3_migration',
|
40
|
+
"#{db_migrate_path}/create_db_backend.rb")
|
41
|
+
else
|
42
|
+
migration_template('migration',
|
43
|
+
"#{db_migrate_path}/create_db_backend.rb")
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'phobos'
|
4
|
+
require 'phobos/cli'
|
5
|
+
|
6
|
+
namespace :deimos do
|
7
|
+
desc 'Starts Deimos in the rails environment'
|
8
|
+
task start: :environment do
|
9
|
+
Deimos.configure do |config|
|
10
|
+
config.publish_backend = :kafka_sync if config.publish_backend == :kafka_async
|
11
|
+
end
|
12
|
+
ENV['DEIMOS_RAKE_TASK'] = 'true'
|
13
|
+
STDOUT.sync = true
|
14
|
+
Rails.logger.info('Running deimos:start rake task.')
|
15
|
+
Phobos::CLI::Commands.start(%w(start --skip_config))
|
16
|
+
end
|
17
|
+
|
18
|
+
desc 'Starts the Deimos database producer'
|
19
|
+
task db_producer: :environment do
|
20
|
+
ENV['DEIMOS_RAKE_TASK'] = 'true'
|
21
|
+
STDOUT.sync = true
|
22
|
+
Rails.logger.info('Running deimos:db_producer rake task.')
|
23
|
+
thread_count = ENV['THREADS'].presence || 1
|
24
|
+
Deimos.start_db_backend!(thread_count: thread_count)
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'date'
|
4
|
+
|
5
|
+
# :nodoc:
|
6
|
+
module ActiveRecordProducerTest
|
7
|
+
describe Deimos::ActiveRecordConsumer do
|
8
|
+
|
9
|
+
before(:all) do
|
10
|
+
ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
|
11
|
+
t.string(:test_id)
|
12
|
+
t.integer(:some_int)
|
13
|
+
t.boolean(:some_bool)
|
14
|
+
t.datetime(:some_datetime_int)
|
15
|
+
t.timestamps
|
16
|
+
end
|
17
|
+
|
18
|
+
# :nodoc:
|
19
|
+
class Widget < ActiveRecord::Base
|
20
|
+
end
|
21
|
+
Widget.reset_column_information
|
22
|
+
end
|
23
|
+
|
24
|
+
after(:all) do
|
25
|
+
ActiveRecord::Base.connection.drop_table(:widgets)
|
26
|
+
end
|
27
|
+
|
28
|
+
prepend_before(:each) do
|
29
|
+
|
30
|
+
consumer_class = Class.new(Deimos::ActiveRecordConsumer) do
|
31
|
+
schema 'MySchemaWithDateTimes'
|
32
|
+
namespace 'com.my-namespace'
|
33
|
+
key_config plain: true
|
34
|
+
record_class Widget
|
35
|
+
end
|
36
|
+
stub_const('MyConsumer', consumer_class)
|
37
|
+
|
38
|
+
Time.zone = 'Eastern Time (US & Canada)'
|
39
|
+
end
|
40
|
+
|
41
|
+
it 'should receive events correctly' do
|
42
|
+
travel 1.day do
|
43
|
+
expect(Widget.count).to eq(0)
|
44
|
+
test_consume_message(MyConsumer, {
|
45
|
+
test_id: 'abc',
|
46
|
+
some_int: 3,
|
47
|
+
updated_at: 1.day.ago.to_i,
|
48
|
+
some_datetime_int: Time.zone.now.to_i,
|
49
|
+
timestamp: 2.minutes.ago.to_s
|
50
|
+
}, { call_original: true, key: 5 })
|
51
|
+
|
52
|
+
expect(Widget.count).to eq(1)
|
53
|
+
widget = Widget.last
|
54
|
+
expect(widget.id).to eq(5)
|
55
|
+
expect(widget.test_id).to eq('abc')
|
56
|
+
expect(widget.some_int).to eq(3)
|
57
|
+
expect(widget.some_datetime_int).to eq(Time.zone.now)
|
58
|
+
expect(widget.updated_at).to eq(Time.zone.now)
|
59
|
+
|
60
|
+
# test update
|
61
|
+
test_consume_message(MyConsumer, {
|
62
|
+
test_id: 'abcd',
|
63
|
+
some_int: 3,
|
64
|
+
some_datetime_int: Time.zone.now.to_i,
|
65
|
+
timestamp: 2.minutes.ago.to_s
|
66
|
+
}, { call_original: true, key: 5 })
|
67
|
+
widget = Widget.last
|
68
|
+
expect(widget.id).to eq(5)
|
69
|
+
expect(widget.test_id).to eq('abcd')
|
70
|
+
expect(widget.some_int).to eq(3)
|
71
|
+
|
72
|
+
# test delete
|
73
|
+
test_consume_message(MyConsumer, nil, call_original: true, key: 5)
|
74
|
+
expect(Widget.count).to eq(0)
|
75
|
+
|
76
|
+
end
|
77
|
+
|
78
|
+
end
|
79
|
+
|
80
|
+
end
|
81
|
+
end
|
@@ -0,0 +1,107 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# :nodoc:
|
4
|
+
module ActiveRecordProducerTest
|
5
|
+
describe Deimos::ActiveRecordProducer do
|
6
|
+
|
7
|
+
before(:all) do
|
8
|
+
ActiveRecord::Base.connection.create_table(:widgets) do |t|
|
9
|
+
t.string(:test_id)
|
10
|
+
t.integer(:some_int)
|
11
|
+
t.boolean(:some_bool)
|
12
|
+
t.timestamps
|
13
|
+
end
|
14
|
+
|
15
|
+
# :nodoc:
|
16
|
+
class Widget < ActiveRecord::Base
|
17
|
+
# @return [String]
|
18
|
+
def generated_id
|
19
|
+
'generated_id'
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
after(:all) do
|
25
|
+
ActiveRecord::Base.connection.drop_table(:widgets)
|
26
|
+
end
|
27
|
+
|
28
|
+
prepend_before(:each) do
|
29
|
+
|
30
|
+
producer_class = Class.new(Deimos::ActiveRecordProducer) do
|
31
|
+
schema 'MySchema'
|
32
|
+
namespace 'com.my-namespace'
|
33
|
+
topic 'my-topic'
|
34
|
+
key_config none: true
|
35
|
+
end
|
36
|
+
stub_const('MyProducer', producer_class)
|
37
|
+
|
38
|
+
producer_class = Class.new(Deimos::ActiveRecordProducer) do
|
39
|
+
schema 'MySchemaWithBooleans'
|
40
|
+
namespace 'com.my-namespace'
|
41
|
+
topic 'my-topic-with-boolean'
|
42
|
+
key_config none: true
|
43
|
+
end
|
44
|
+
stub_const('MyBooleanProducer', producer_class)
|
45
|
+
|
46
|
+
producer_class = Class.new(Deimos::ActiveRecordProducer) do
|
47
|
+
schema 'MySchemaWithId'
|
48
|
+
namespace 'com.my-namespace'
|
49
|
+
topic 'my-topic-with-id'
|
50
|
+
key_config none: true
|
51
|
+
record_class Widget
|
52
|
+
|
53
|
+
# :nodoc:
|
54
|
+
def self.generate_payload(attrs, widget)
|
55
|
+
super.merge(message_id: widget.generated_id)
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
59
|
+
stub_const('MyProducerWithID', producer_class)
|
60
|
+
|
61
|
+
producer_class = Class.new(Deimos::ActiveRecordProducer) do
|
62
|
+
schema 'MySchemaWithUniqueId'
|
63
|
+
namespace 'com.my-namespace'
|
64
|
+
topic 'my-topic-with-unique-id'
|
65
|
+
key_config field: :id
|
66
|
+
record_class Widget
|
67
|
+
end
|
68
|
+
stub_const('MyProducerWithUniqueID', producer_class)
|
69
|
+
end
|
70
|
+
|
71
|
+
it 'should send events correctly' do
|
72
|
+
MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 3))
|
73
|
+
expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
|
74
|
+
end
|
75
|
+
|
76
|
+
it 'should coerce values' do
|
77
|
+
MyProducer.send_event(Widget.new(test_id: 'abc', some_int: '3'))
|
78
|
+
MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 4.5))
|
79
|
+
expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
|
80
|
+
expect('my-topic').to have_sent(test_id: 'abc', some_int: 4)
|
81
|
+
expect {
|
82
|
+
MyProducer.send_event(Widget.new(test_id: 'abc', some_int: nil))
|
83
|
+
}.to raise_error(Avro::SchemaValidator::ValidationError)
|
84
|
+
|
85
|
+
MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: nil))
|
86
|
+
MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: true))
|
87
|
+
expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: false)
|
88
|
+
expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: true)
|
89
|
+
end
|
90
|
+
|
91
|
+
it 'should be able to call the record' do
|
92
|
+
widget = Widget.create!(test_id: 'abc2', some_int: 3)
|
93
|
+
MyProducerWithID.send_event(id: widget.id, test_id: 'abc2', some_int: 3)
|
94
|
+
expect('my-topic-with-id').to have_sent(
|
95
|
+
test_id: 'abc2',
|
96
|
+
some_int: 3,
|
97
|
+
message_id: 'generated_id',
|
98
|
+
timestamp: anything
|
99
|
+
)
|
100
|
+
end
|
101
|
+
|
102
|
+
specify '#watched_attributes' do
|
103
|
+
expect(MyProducer.watched_attributes).to eq(%w(test_id some_int))
|
104
|
+
end
|
105
|
+
|
106
|
+
end
|
107
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
describe Deimos::AvroDataDecoder do
|
4
|
+
|
5
|
+
let(:decoder) do
|
6
|
+
decoder = described_class.new(schema: 'MySchema',
|
7
|
+
namespace: 'com.my-namespace')
|
8
|
+
allow(decoder).to(receive(:decode)) { |payload| payload }
|
9
|
+
decoder
|
10
|
+
end
|
11
|
+
|
12
|
+
it 'should decode a key' do
|
13
|
+
# reset stub from TestHelpers
|
14
|
+
allow(described_class).to receive(:new).and_call_original
|
15
|
+
expect(decoder.decode_key({ 'test_id' => '123' }, 'test_id')).to eq('123')
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'avro_turf/messaging'
|
4
|
+
|
5
|
+
describe Deimos::AvroDataEncoder do
|
6
|
+
|
7
|
+
let(:encoder) do
|
8
|
+
encoder = described_class.new(schema: 'MySchema',
|
9
|
+
namespace: 'com.my-namespace')
|
10
|
+
allow(encoder).to(receive(:encode)) { |payload| payload }
|
11
|
+
encoder
|
12
|
+
end
|
13
|
+
|
14
|
+
specify 'generate_key_schema' do
|
15
|
+
expect_any_instance_of(AvroTurf::SchemaStore).
|
16
|
+
to receive(:add_schema).with(
|
17
|
+
'type' => 'record',
|
18
|
+
'name' => 'MySchema_key',
|
19
|
+
'namespace' => 'com.my-namespace',
|
20
|
+
'doc' => 'Key for com.my-namespace.MySchema',
|
21
|
+
'fields' => [
|
22
|
+
{
|
23
|
+
'name' => 'test_id',
|
24
|
+
'type' => 'string'
|
25
|
+
}
|
26
|
+
]
|
27
|
+
)
|
28
|
+
encoder.send(:_generate_key_schema, 'test_id')
|
29
|
+
end
|
30
|
+
|
31
|
+
it 'should encode a key' do
|
32
|
+
# reset stub from TestHelpers
|
33
|
+
allow(described_class).to receive(:new).and_call_original
|
34
|
+
expect(encoder.encode_key('test_id', '123')).to eq('test_id' => '123')
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
each_db_config(Deimos::Backends::Db) do
|
4
|
+
include_context 'with publish_backend'
|
5
|
+
|
6
|
+
it 'should save to the database' do
|
7
|
+
described_class.publish(producer_class: MyProducer, messages: messages)
|
8
|
+
records = Deimos::KafkaMessage.all
|
9
|
+
expect(records.size).to eq(3)
|
10
|
+
expect(records[0].attributes.to_h).to include(
|
11
|
+
'message' => '{"foo"=>1}',
|
12
|
+
'topic' => 'my-topic',
|
13
|
+
'key' => 'foo1'
|
14
|
+
)
|
15
|
+
expect(records[1].attributes.to_h).to include(
|
16
|
+
'message' => '{"foo"=>2}',
|
17
|
+
'topic' => 'my-topic',
|
18
|
+
'key' => 'foo2'
|
19
|
+
)
|
20
|
+
expect(records[2].attributes.to_h).to include(
|
21
|
+
'message' => '{"foo"=>3}',
|
22
|
+
'topic' => 'my-topic',
|
23
|
+
'key' => 'foo3'
|
24
|
+
)
|
25
|
+
end
|
26
|
+
it 'should add to non-keyed messages' do
|
27
|
+
described_class.publish(producer_class: MyNoKeyProducer,
|
28
|
+
messages: messages)
|
29
|
+
expect(Deimos::KafkaMessage.count).to eq(3)
|
30
|
+
described_class.publish(producer_class: MyNoKeyProducer,
|
31
|
+
messages: [messages.first])
|
32
|
+
expect(Deimos::KafkaMessage.count).to eq(4)
|
33
|
+
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
RSpec.describe Deimos::Backends::KafkaAsync do
|
4
|
+
include_context 'with publish_backend'
|
5
|
+
it 'should publish to Kafka asynchronously' do
|
6
|
+
producer = instance_double(Phobos::Producer::ClassMethods::PublicAPI)
|
7
|
+
expect(producer).to receive(:async_publish_list).with(messages.map(&:encoded_hash))
|
8
|
+
expect(described_class).to receive(:producer).and_return(producer)
|
9
|
+
described_class.publish(producer_class: MyProducer, messages: messages)
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
RSpec.describe Deimos::Backends::Kafka do
|
4
|
+
include_context 'with publish_backend'
|
5
|
+
it 'should publish to Kafka synchronously' do
|
6
|
+
producer = instance_double(Phobos::Producer::ClassMethods::PublicAPI)
|
7
|
+
expect(producer).to receive(:publish_list).with(messages.map(&:encoded_hash))
|
8
|
+
expect(described_class).to receive(:producer).and_return(producer)
|
9
|
+
described_class.publish(producer_class: MyProducer, messages: messages)
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,169 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# :nodoc:
|
4
|
+
module ConsumerTest
|
5
|
+
describe Deimos::Consumer do
|
6
|
+
|
7
|
+
prepend_before(:each) do
|
8
|
+
# :nodoc:
|
9
|
+
consumer_class = Class.new(Deimos::Consumer) do
|
10
|
+
schema 'MySchema'
|
11
|
+
namespace 'com.my-namespace'
|
12
|
+
key_config field: 'test_id'
|
13
|
+
|
14
|
+
# :nodoc:
|
15
|
+
def consume(_payload, _metadata)
|
16
|
+
raise 'This should not be called unless call_original is set'
|
17
|
+
end
|
18
|
+
end
|
19
|
+
stub_const('ConsumerTest::MyConsumer', consumer_class)
|
20
|
+
end
|
21
|
+
|
22
|
+
it 'should consume a message' do
|
23
|
+
test_consume_message(MyConsumer,
|
24
|
+
'test_id' => 'foo',
|
25
|
+
'some_int' => 123) do |payload, _metadata|
|
26
|
+
expect(payload['test_id']).to eq('foo')
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
it 'should consume a message on a topic' do
|
31
|
+
test_consume_message('my_consume_topic',
|
32
|
+
'test_id' => 'foo',
|
33
|
+
'some_int' => 123) do |payload, _metadata|
|
34
|
+
expect(payload['test_id']).to eq('foo')
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
it 'should fail on invalid message' do
|
39
|
+
test_consume_invalid_message(MyConsumer, 'invalid' => 'key')
|
40
|
+
end
|
41
|
+
|
42
|
+
it 'should fail on message with extra fields' do
|
43
|
+
test_consume_invalid_message(MyConsumer,
|
44
|
+
'test_id' => 'foo',
|
45
|
+
'some_int' => 123,
|
46
|
+
'extra_field' => 'field name')
|
47
|
+
end
|
48
|
+
|
49
|
+
it 'should not fail when before_consume fails without reraising errors' do
|
50
|
+
Deimos.configure { |config| config.reraise_consumer_errors = false }
|
51
|
+
expect {
|
52
|
+
test_consume_message(
|
53
|
+
MyConsumer,
|
54
|
+
{ 'test_id' => 'foo',
|
55
|
+
'some_int' => 123 },
|
56
|
+
{ skip_expectation: true }
|
57
|
+
) { raise 'OH NOES' }
|
58
|
+
} .not_to raise_error
|
59
|
+
end
|
60
|
+
|
61
|
+
it 'should not fail when consume fails without reraising errors' do
|
62
|
+
Deimos.configure { |config| config.reraise_consumer_errors = false }
|
63
|
+
expect {
|
64
|
+
test_consume_message(
|
65
|
+
MyConsumer,
|
66
|
+
{ 'invalid' => 'key' },
|
67
|
+
{ skip_expectation: true }
|
68
|
+
)
|
69
|
+
} .not_to raise_error
|
70
|
+
end
|
71
|
+
|
72
|
+
it 'should call original' do
|
73
|
+
expect {
|
74
|
+
test_consume_message(MyConsumer,
|
75
|
+
{ 'test_id' => 'foo', 'some_int' => 123 },
|
76
|
+
{ call_original: true })
|
77
|
+
}.to raise_error('This should not be called unless call_original is set')
|
78
|
+
end
|
79
|
+
|
80
|
+
describe 'decode_key' do
|
81
|
+
|
82
|
+
it 'should use the key field in the value if set' do
|
83
|
+
# actual decoding is disabled in test
|
84
|
+
expect(MyConsumer.new.decode_key('test_id' => '123')).to eq('123')
|
85
|
+
expect { MyConsumer.new.decode_key(123) }.to raise_error(NoMethodError)
|
86
|
+
end
|
87
|
+
|
88
|
+
it 'should use the key schema if set' do
|
89
|
+
consumer_class = Class.new(Deimos::Consumer) do
|
90
|
+
schema 'MySchema'
|
91
|
+
namespace 'com.my-namespace'
|
92
|
+
key_config schema: 'MySchema_key'
|
93
|
+
end
|
94
|
+
stub_const('ConsumerTest::MySchemaConsumer', consumer_class)
|
95
|
+
expect(MyConsumer.new.decode_key('test_id' => '123')).to eq('123')
|
96
|
+
expect { MyConsumer.new.decode_key(123) }.to raise_error(NoMethodError)
|
97
|
+
end
|
98
|
+
|
99
|
+
it 'should not encode if plain is set' do
|
100
|
+
consumer_class = Class.new(Deimos::Consumer) do
|
101
|
+
schema 'MySchema'
|
102
|
+
namespace 'com.my-namespace'
|
103
|
+
key_config plain: true
|
104
|
+
end
|
105
|
+
stub_const('ConsumerTest::MyNonEncodedConsumer', consumer_class)
|
106
|
+
expect(MyNonEncodedConsumer.new.decode_key('123')).to eq('123')
|
107
|
+
end
|
108
|
+
|
109
|
+
it 'should error with nothing set' do
|
110
|
+
consumer_class = Class.new(Deimos::Consumer) do
|
111
|
+
schema 'MySchema'
|
112
|
+
namespace 'com.my-namespace'
|
113
|
+
end
|
114
|
+
stub_const('ConsumerTest::MyErrorConsumer', consumer_class)
|
115
|
+
expect { MyErrorConsumer.new.decode_key('123') }.
|
116
|
+
to raise_error('No key config given - if you are not decoding keys, please use `key_config plain: true`')
|
117
|
+
end
|
118
|
+
|
119
|
+
end
|
120
|
+
|
121
|
+
describe 'timestamps' do
|
122
|
+
before(:each) do
|
123
|
+
# :nodoc:
|
124
|
+
consumer_class = Class.new(Deimos::Consumer) do
|
125
|
+
schema 'MySchemaWithDateTimes'
|
126
|
+
namespace 'com.my-namespace'
|
127
|
+
key_config plain: true
|
128
|
+
|
129
|
+
# :nodoc:
|
130
|
+
def consume(_payload, _metadata)
|
131
|
+
raise 'This should not be called unless call_original is set'
|
132
|
+
end
|
133
|
+
end
|
134
|
+
stub_const('ConsumerTest::MyConsumer', consumer_class)
|
135
|
+
stub_consumer(consumer_class)
|
136
|
+
end
|
137
|
+
|
138
|
+
it 'should consume a message' do
|
139
|
+
expect(Deimos.config.metrics).to receive(:histogram).twice
|
140
|
+
test_consume_message('my_consume_topic',
|
141
|
+
'test_id' => 'foo',
|
142
|
+
'some_int' => 123,
|
143
|
+
'updated_at' => Time.now.to_i,
|
144
|
+
'timestamp' => 2.minutes.ago.to_s) do |payload, _metadata|
|
145
|
+
expect(payload['test_id']).to eq('foo')
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
it 'should fail nicely when timestamp wrong format' do
|
150
|
+
expect(Deimos.config.metrics).to receive(:histogram).twice
|
151
|
+
test_consume_message('my_consume_topic',
|
152
|
+
'test_id' => 'foo',
|
153
|
+
'some_int' => 123,
|
154
|
+
'updated_at' => Time.now.to_i,
|
155
|
+
'timestamp' => 'dffdf') do |payload, _metadata|
|
156
|
+
expect(payload['test_id']).to eq('foo')
|
157
|
+
end
|
158
|
+
test_consume_message('my_consume_topic',
|
159
|
+
'test_id' => 'foo',
|
160
|
+
'some_int' => 123,
|
161
|
+
'updated_at' => Time.now.to_i,
|
162
|
+
'timestamp' => '') do |payload, _metadata|
|
163
|
+
expect(payload['test_id']).to eq('foo')
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|