deimos-ruby 1.0.0.pre.beta22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. checksums.yaml +7 -0
  2. data/.circleci/config.yml +74 -0
  3. data/.gitignore +41 -0
  4. data/.gitmodules +0 -0
  5. data/.rspec +1 -0
  6. data/.rubocop.yml +321 -0
  7. data/.ruby-gemset +1 -0
  8. data/.ruby-version +1 -0
  9. data/CHANGELOG.md +32 -0
  10. data/CODE_OF_CONDUCT.md +77 -0
  11. data/Dockerfile +23 -0
  12. data/Gemfile +6 -0
  13. data/Gemfile.lock +165 -0
  14. data/Guardfile +22 -0
  15. data/LICENSE.md +195 -0
  16. data/README.md +752 -0
  17. data/Rakefile +13 -0
  18. data/bin/deimos +4 -0
  19. data/deimos-kafka.gemspec +42 -0
  20. data/docker-compose.yml +71 -0
  21. data/docs/DATABASE_BACKEND.md +147 -0
  22. data/docs/PULL_REQUEST_TEMPLATE.md +34 -0
  23. data/lib/deimos/active_record_consumer.rb +81 -0
  24. data/lib/deimos/active_record_producer.rb +64 -0
  25. data/lib/deimos/avro_data_coder.rb +89 -0
  26. data/lib/deimos/avro_data_decoder.rb +36 -0
  27. data/lib/deimos/avro_data_encoder.rb +51 -0
  28. data/lib/deimos/backends/db.rb +27 -0
  29. data/lib/deimos/backends/kafka.rb +27 -0
  30. data/lib/deimos/backends/kafka_async.rb +27 -0
  31. data/lib/deimos/configuration.rb +90 -0
  32. data/lib/deimos/consumer.rb +164 -0
  33. data/lib/deimos/instrumentation.rb +71 -0
  34. data/lib/deimos/kafka_message.rb +27 -0
  35. data/lib/deimos/kafka_source.rb +126 -0
  36. data/lib/deimos/kafka_topic_info.rb +86 -0
  37. data/lib/deimos/message.rb +74 -0
  38. data/lib/deimos/metrics/datadog.rb +47 -0
  39. data/lib/deimos/metrics/mock.rb +39 -0
  40. data/lib/deimos/metrics/provider.rb +38 -0
  41. data/lib/deimos/monkey_patches/phobos_cli.rb +35 -0
  42. data/lib/deimos/monkey_patches/phobos_producer.rb +51 -0
  43. data/lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb +85 -0
  44. data/lib/deimos/monkey_patches/schema_store.rb +19 -0
  45. data/lib/deimos/producer.rb +218 -0
  46. data/lib/deimos/publish_backend.rb +30 -0
  47. data/lib/deimos/railtie.rb +8 -0
  48. data/lib/deimos/schema_coercer.rb +108 -0
  49. data/lib/deimos/shared_config.rb +59 -0
  50. data/lib/deimos/test_helpers.rb +356 -0
  51. data/lib/deimos/tracing/datadog.rb +35 -0
  52. data/lib/deimos/tracing/mock.rb +40 -0
  53. data/lib/deimos/tracing/provider.rb +31 -0
  54. data/lib/deimos/utils/db_producer.rb +122 -0
  55. data/lib/deimos/utils/executor.rb +117 -0
  56. data/lib/deimos/utils/inline_consumer.rb +144 -0
  57. data/lib/deimos/utils/lag_reporter.rb +182 -0
  58. data/lib/deimos/utils/platform_schema_validation.rb +0 -0
  59. data/lib/deimos/utils/signal_handler.rb +68 -0
  60. data/lib/deimos/version.rb +5 -0
  61. data/lib/deimos.rb +133 -0
  62. data/lib/generators/deimos/db_backend/templates/migration +24 -0
  63. data/lib/generators/deimos/db_backend/templates/rails3_migration +30 -0
  64. data/lib/generators/deimos/db_backend_generator.rb +48 -0
  65. data/lib/tasks/deimos.rake +27 -0
  66. data/spec/active_record_consumer_spec.rb +81 -0
  67. data/spec/active_record_producer_spec.rb +107 -0
  68. data/spec/avro_data_decoder_spec.rb +18 -0
  69. data/spec/avro_data_encoder_spec.rb +37 -0
  70. data/spec/backends/db_spec.rb +35 -0
  71. data/spec/backends/kafka_async_spec.rb +11 -0
  72. data/spec/backends/kafka_spec.rb +11 -0
  73. data/spec/consumer_spec.rb +169 -0
  74. data/spec/deimos_spec.rb +120 -0
  75. data/spec/kafka_source_spec.rb +168 -0
  76. data/spec/kafka_topic_info_spec.rb +88 -0
  77. data/spec/phobos.bad_db.yml +73 -0
  78. data/spec/phobos.yml +73 -0
  79. data/spec/producer_spec.rb +397 -0
  80. data/spec/publish_backend_spec.rb +10 -0
  81. data/spec/schemas/com/my-namespace/MySchema-key.avsc +13 -0
  82. data/spec/schemas/com/my-namespace/MySchema.avsc +18 -0
  83. data/spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc +18 -0
  84. data/spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc +33 -0
  85. data/spec/schemas/com/my-namespace/MySchemaWithId.avsc +28 -0
  86. data/spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc +32 -0
  87. data/spec/schemas/com/my-namespace/Widget.avsc +27 -0
  88. data/spec/schemas/com/my-namespace/WidgetTheSecond.avsc +27 -0
  89. data/spec/spec_helper.rb +207 -0
  90. data/spec/updateable_schema_store_spec.rb +36 -0
  91. data/spec/utils/db_producer_spec.rb +259 -0
  92. data/spec/utils/executor_spec.rb +42 -0
  93. data/spec/utils/lag_reporter_spec.rb +69 -0
  94. data/spec/utils/platform_schema_validation_spec.rb +0 -0
  95. data/spec/utils/signal_handler_spec.rb +16 -0
  96. data/support/deimos-solo.png +0 -0
  97. data/support/deimos-with-name-next.png +0 -0
  98. data/support/deimos-with-name.png +0 -0
  99. data/support/flipp-logo.png +0 -0
  100. metadata +452 -0
@@ -0,0 +1,30 @@
1
+ class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
2
+ def self.up
3
+ create_table :kafka_messages, force: true do |t|
4
+ t.string :topic, null: false
5
+ t.binary :message, limit: 10.megabytes
6
+ t.binary :key
7
+ t.string :partition_key
8
+ t.timestamps
9
+ end
10
+
11
+ add_index :kafka_messages, [:topic, :id]
12
+
13
+ create_table :kafka_topic_info, force: true do |t| # rubocop:disable Rails/CreateTableWithTimestamps
14
+ t.string :topic, null: false
15
+ t.string :locked_by
16
+ t.datetime :locked_at
17
+ t.boolean :error, null: false, default: false
18
+ t.integer :retries, null: false, default: 0
19
+ end
20
+ add_index :kafka_topic_info, :topic, unique: true
21
+ add_index :kafka_topic_info, [:locked_by, :error]
22
+ add_index :kafka_topic_info, :locked_at
23
+ end
24
+
25
+ def self.down
26
+ drop_table :kafka_messages
27
+ drop_table :kafka_topic_info
28
+ end
29
+
30
+ end
@@ -0,0 +1,48 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rails/generators'
4
+ require 'rails/generators/active_record/migration'
5
+
6
+ module Deimos
7
+ module Generators
8
+ # Generate the database backend migration.
9
+ class DbBackendGenerator < Rails::Generators::Base
10
+ include Rails::Generators::Migration
11
+ if Rails.version < '4'
12
+ extend(ActiveRecord::Generators::Migration)
13
+ else
14
+ include ActiveRecord::Generators::Migration
15
+ end
16
+ source_root File.expand_path('db_backend/templates', __dir__)
17
+ desc 'Add migrations for the database backend'
18
+
19
+ # @return [String]
20
+ def migration_version
21
+ "[#{ActiveRecord::Migration.current_version}]"
22
+ rescue StandardError
23
+ ''
24
+ end
25
+
26
+ # @return [String]
27
+ def db_migrate_path
28
+ if defined?(Rails.application) && Rails.application
29
+ paths = Rails.application.config.paths['db/migrate']
30
+ paths.respond_to?(:to_ary) ? paths.to_ary.first : paths.to_a.first
31
+ else
32
+ 'db/migrate'
33
+ end
34
+ end
35
+
36
+ # Main method to create all the necessary files
37
+ def generate
38
+ if Rails.version < '4'
39
+ migration_template('rails3_migration',
40
+ "#{db_migrate_path}/create_db_backend.rb")
41
+ else
42
+ migration_template('migration',
43
+ "#{db_migrate_path}/create_db_backend.rb")
44
+ end
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'phobos'
4
+ require 'phobos/cli'
5
+
6
+ namespace :deimos do
7
+ desc 'Starts Deimos in the rails environment'
8
+ task start: :environment do
9
+ Deimos.configure do |config|
10
+ config.publish_backend = :kafka_sync if config.publish_backend == :kafka_async
11
+ end
12
+ ENV['DEIMOS_RAKE_TASK'] = 'true'
13
+ STDOUT.sync = true
14
+ Rails.logger.info('Running deimos:start rake task.')
15
+ Phobos::CLI::Commands.start(%w(start --skip_config))
16
+ end
17
+
18
+ desc 'Starts the Deimos database producer'
19
+ task db_producer: :environment do
20
+ ENV['DEIMOS_RAKE_TASK'] = 'true'
21
+ STDOUT.sync = true
22
+ Rails.logger.info('Running deimos:db_producer rake task.')
23
+ thread_count = ENV['THREADS'].presence || 1
24
+ Deimos.start_db_backend!(thread_count: thread_count)
25
+ end
26
+
27
+ end
@@ -0,0 +1,81 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'date'
4
+
5
+ # :nodoc:
6
+ module ActiveRecordProducerTest
7
+ describe Deimos::ActiveRecordConsumer do
8
+
9
+ before(:all) do
10
+ ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
11
+ t.string(:test_id)
12
+ t.integer(:some_int)
13
+ t.boolean(:some_bool)
14
+ t.datetime(:some_datetime_int)
15
+ t.timestamps
16
+ end
17
+
18
+ # :nodoc:
19
+ class Widget < ActiveRecord::Base
20
+ end
21
+ Widget.reset_column_information
22
+ end
23
+
24
+ after(:all) do
25
+ ActiveRecord::Base.connection.drop_table(:widgets)
26
+ end
27
+
28
+ prepend_before(:each) do
29
+
30
+ consumer_class = Class.new(Deimos::ActiveRecordConsumer) do
31
+ schema 'MySchemaWithDateTimes'
32
+ namespace 'com.my-namespace'
33
+ key_config plain: true
34
+ record_class Widget
35
+ end
36
+ stub_const('MyConsumer', consumer_class)
37
+
38
+ Time.zone = 'Eastern Time (US & Canada)'
39
+ end
40
+
41
+ it 'should receive events correctly' do
42
+ travel 1.day do
43
+ expect(Widget.count).to eq(0)
44
+ test_consume_message(MyConsumer, {
45
+ test_id: 'abc',
46
+ some_int: 3,
47
+ updated_at: 1.day.ago.to_i,
48
+ some_datetime_int: Time.zone.now.to_i,
49
+ timestamp: 2.minutes.ago.to_s
50
+ }, { call_original: true, key: 5 })
51
+
52
+ expect(Widget.count).to eq(1)
53
+ widget = Widget.last
54
+ expect(widget.id).to eq(5)
55
+ expect(widget.test_id).to eq('abc')
56
+ expect(widget.some_int).to eq(3)
57
+ expect(widget.some_datetime_int).to eq(Time.zone.now)
58
+ expect(widget.updated_at).to eq(Time.zone.now)
59
+
60
+ # test update
61
+ test_consume_message(MyConsumer, {
62
+ test_id: 'abcd',
63
+ some_int: 3,
64
+ some_datetime_int: Time.zone.now.to_i,
65
+ timestamp: 2.minutes.ago.to_s
66
+ }, { call_original: true, key: 5 })
67
+ widget = Widget.last
68
+ expect(widget.id).to eq(5)
69
+ expect(widget.test_id).to eq('abcd')
70
+ expect(widget.some_int).to eq(3)
71
+
72
+ # test delete
73
+ test_consume_message(MyConsumer, nil, call_original: true, key: 5)
74
+ expect(Widget.count).to eq(0)
75
+
76
+ end
77
+
78
+ end
79
+
80
+ end
81
+ end
@@ -0,0 +1,107 @@
1
+ # frozen_string_literal: true
2
+
3
+ # :nodoc:
4
+ module ActiveRecordProducerTest
5
+ describe Deimos::ActiveRecordProducer do
6
+
7
+ before(:all) do
8
+ ActiveRecord::Base.connection.create_table(:widgets) do |t|
9
+ t.string(:test_id)
10
+ t.integer(:some_int)
11
+ t.boolean(:some_bool)
12
+ t.timestamps
13
+ end
14
+
15
+ # :nodoc:
16
+ class Widget < ActiveRecord::Base
17
+ # @return [String]
18
+ def generated_id
19
+ 'generated_id'
20
+ end
21
+ end
22
+ end
23
+
24
+ after(:all) do
25
+ ActiveRecord::Base.connection.drop_table(:widgets)
26
+ end
27
+
28
+ prepend_before(:each) do
29
+
30
+ producer_class = Class.new(Deimos::ActiveRecordProducer) do
31
+ schema 'MySchema'
32
+ namespace 'com.my-namespace'
33
+ topic 'my-topic'
34
+ key_config none: true
35
+ end
36
+ stub_const('MyProducer', producer_class)
37
+
38
+ producer_class = Class.new(Deimos::ActiveRecordProducer) do
39
+ schema 'MySchemaWithBooleans'
40
+ namespace 'com.my-namespace'
41
+ topic 'my-topic-with-boolean'
42
+ key_config none: true
43
+ end
44
+ stub_const('MyBooleanProducer', producer_class)
45
+
46
+ producer_class = Class.new(Deimos::ActiveRecordProducer) do
47
+ schema 'MySchemaWithId'
48
+ namespace 'com.my-namespace'
49
+ topic 'my-topic-with-id'
50
+ key_config none: true
51
+ record_class Widget
52
+
53
+ # :nodoc:
54
+ def self.generate_payload(attrs, widget)
55
+ super.merge(message_id: widget.generated_id)
56
+ end
57
+
58
+ end
59
+ stub_const('MyProducerWithID', producer_class)
60
+
61
+ producer_class = Class.new(Deimos::ActiveRecordProducer) do
62
+ schema 'MySchemaWithUniqueId'
63
+ namespace 'com.my-namespace'
64
+ topic 'my-topic-with-unique-id'
65
+ key_config field: :id
66
+ record_class Widget
67
+ end
68
+ stub_const('MyProducerWithUniqueID', producer_class)
69
+ end
70
+
71
+ it 'should send events correctly' do
72
+ MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 3))
73
+ expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
74
+ end
75
+
76
+ it 'should coerce values' do
77
+ MyProducer.send_event(Widget.new(test_id: 'abc', some_int: '3'))
78
+ MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 4.5))
79
+ expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
80
+ expect('my-topic').to have_sent(test_id: 'abc', some_int: 4)
81
+ expect {
82
+ MyProducer.send_event(Widget.new(test_id: 'abc', some_int: nil))
83
+ }.to raise_error(Avro::SchemaValidator::ValidationError)
84
+
85
+ MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: nil))
86
+ MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: true))
87
+ expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: false)
88
+ expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: true)
89
+ end
90
+
91
+ it 'should be able to call the record' do
92
+ widget = Widget.create!(test_id: 'abc2', some_int: 3)
93
+ MyProducerWithID.send_event(id: widget.id, test_id: 'abc2', some_int: 3)
94
+ expect('my-topic-with-id').to have_sent(
95
+ test_id: 'abc2',
96
+ some_int: 3,
97
+ message_id: 'generated_id',
98
+ timestamp: anything
99
+ )
100
+ end
101
+
102
+ specify '#watched_attributes' do
103
+ expect(MyProducer.watched_attributes).to eq(%w(test_id some_int))
104
+ end
105
+
106
+ end
107
+ end
@@ -0,0 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ describe Deimos::AvroDataDecoder do
4
+
5
+ let(:decoder) do
6
+ decoder = described_class.new(schema: 'MySchema',
7
+ namespace: 'com.my-namespace')
8
+ allow(decoder).to(receive(:decode)) { |payload| payload }
9
+ decoder
10
+ end
11
+
12
+ it 'should decode a key' do
13
+ # reset stub from TestHelpers
14
+ allow(described_class).to receive(:new).and_call_original
15
+ expect(decoder.decode_key({ 'test_id' => '123' }, 'test_id')).to eq('123')
16
+ end
17
+
18
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'avro_turf/messaging'
4
+
5
+ describe Deimos::AvroDataEncoder do
6
+
7
+ let(:encoder) do
8
+ encoder = described_class.new(schema: 'MySchema',
9
+ namespace: 'com.my-namespace')
10
+ allow(encoder).to(receive(:encode)) { |payload| payload }
11
+ encoder
12
+ end
13
+
14
+ specify 'generate_key_schema' do
15
+ expect_any_instance_of(AvroTurf::SchemaStore).
16
+ to receive(:add_schema).with(
17
+ 'type' => 'record',
18
+ 'name' => 'MySchema_key',
19
+ 'namespace' => 'com.my-namespace',
20
+ 'doc' => 'Key for com.my-namespace.MySchema',
21
+ 'fields' => [
22
+ {
23
+ 'name' => 'test_id',
24
+ 'type' => 'string'
25
+ }
26
+ ]
27
+ )
28
+ encoder.send(:_generate_key_schema, 'test_id')
29
+ end
30
+
31
+ it 'should encode a key' do
32
+ # reset stub from TestHelpers
33
+ allow(described_class).to receive(:new).and_call_original
34
+ expect(encoder.encode_key('test_id', '123')).to eq('test_id' => '123')
35
+ end
36
+
37
+ end
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ each_db_config(Deimos::Backends::Db) do
4
+ include_context 'with publish_backend'
5
+
6
+ it 'should save to the database' do
7
+ described_class.publish(producer_class: MyProducer, messages: messages)
8
+ records = Deimos::KafkaMessage.all
9
+ expect(records.size).to eq(3)
10
+ expect(records[0].attributes.to_h).to include(
11
+ 'message' => '{"foo"=>1}',
12
+ 'topic' => 'my-topic',
13
+ 'key' => 'foo1'
14
+ )
15
+ expect(records[1].attributes.to_h).to include(
16
+ 'message' => '{"foo"=>2}',
17
+ 'topic' => 'my-topic',
18
+ 'key' => 'foo2'
19
+ )
20
+ expect(records[2].attributes.to_h).to include(
21
+ 'message' => '{"foo"=>3}',
22
+ 'topic' => 'my-topic',
23
+ 'key' => 'foo3'
24
+ )
25
+ end
26
+ it 'should add to non-keyed messages' do
27
+ described_class.publish(producer_class: MyNoKeyProducer,
28
+ messages: messages)
29
+ expect(Deimos::KafkaMessage.count).to eq(3)
30
+ described_class.publish(producer_class: MyNoKeyProducer,
31
+ messages: [messages.first])
32
+ expect(Deimos::KafkaMessage.count).to eq(4)
33
+
34
+ end
35
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ RSpec.describe Deimos::Backends::KafkaAsync do
4
+ include_context 'with publish_backend'
5
+ it 'should publish to Kafka asynchronously' do
6
+ producer = instance_double(Phobos::Producer::ClassMethods::PublicAPI)
7
+ expect(producer).to receive(:async_publish_list).with(messages.map(&:encoded_hash))
8
+ expect(described_class).to receive(:producer).and_return(producer)
9
+ described_class.publish(producer_class: MyProducer, messages: messages)
10
+ end
11
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ RSpec.describe Deimos::Backends::Kafka do
4
+ include_context 'with publish_backend'
5
+ it 'should publish to Kafka synchronously' do
6
+ producer = instance_double(Phobos::Producer::ClassMethods::PublicAPI)
7
+ expect(producer).to receive(:publish_list).with(messages.map(&:encoded_hash))
8
+ expect(described_class).to receive(:producer).and_return(producer)
9
+ described_class.publish(producer_class: MyProducer, messages: messages)
10
+ end
11
+ end
@@ -0,0 +1,169 @@
1
+ # frozen_string_literal: true
2
+
3
+ # :nodoc:
4
+ module ConsumerTest
5
+ describe Deimos::Consumer do
6
+
7
+ prepend_before(:each) do
8
+ # :nodoc:
9
+ consumer_class = Class.new(Deimos::Consumer) do
10
+ schema 'MySchema'
11
+ namespace 'com.my-namespace'
12
+ key_config field: 'test_id'
13
+
14
+ # :nodoc:
15
+ def consume(_payload, _metadata)
16
+ raise 'This should not be called unless call_original is set'
17
+ end
18
+ end
19
+ stub_const('ConsumerTest::MyConsumer', consumer_class)
20
+ end
21
+
22
+ it 'should consume a message' do
23
+ test_consume_message(MyConsumer,
24
+ 'test_id' => 'foo',
25
+ 'some_int' => 123) do |payload, _metadata|
26
+ expect(payload['test_id']).to eq('foo')
27
+ end
28
+ end
29
+
30
+ it 'should consume a message on a topic' do
31
+ test_consume_message('my_consume_topic',
32
+ 'test_id' => 'foo',
33
+ 'some_int' => 123) do |payload, _metadata|
34
+ expect(payload['test_id']).to eq('foo')
35
+ end
36
+ end
37
+
38
+ it 'should fail on invalid message' do
39
+ test_consume_invalid_message(MyConsumer, 'invalid' => 'key')
40
+ end
41
+
42
+ it 'should fail on message with extra fields' do
43
+ test_consume_invalid_message(MyConsumer,
44
+ 'test_id' => 'foo',
45
+ 'some_int' => 123,
46
+ 'extra_field' => 'field name')
47
+ end
48
+
49
+ it 'should not fail when before_consume fails without reraising errors' do
50
+ Deimos.configure { |config| config.reraise_consumer_errors = false }
51
+ expect {
52
+ test_consume_message(
53
+ MyConsumer,
54
+ { 'test_id' => 'foo',
55
+ 'some_int' => 123 },
56
+ { skip_expectation: true }
57
+ ) { raise 'OH NOES' }
58
+ } .not_to raise_error
59
+ end
60
+
61
+ it 'should not fail when consume fails without reraising errors' do
62
+ Deimos.configure { |config| config.reraise_consumer_errors = false }
63
+ expect {
64
+ test_consume_message(
65
+ MyConsumer,
66
+ { 'invalid' => 'key' },
67
+ { skip_expectation: true }
68
+ )
69
+ } .not_to raise_error
70
+ end
71
+
72
+ it 'should call original' do
73
+ expect {
74
+ test_consume_message(MyConsumer,
75
+ { 'test_id' => 'foo', 'some_int' => 123 },
76
+ { call_original: true })
77
+ }.to raise_error('This should not be called unless call_original is set')
78
+ end
79
+
80
+ describe 'decode_key' do
81
+
82
+ it 'should use the key field in the value if set' do
83
+ # actual decoding is disabled in test
84
+ expect(MyConsumer.new.decode_key('test_id' => '123')).to eq('123')
85
+ expect { MyConsumer.new.decode_key(123) }.to raise_error(NoMethodError)
86
+ end
87
+
88
+ it 'should use the key schema if set' do
89
+ consumer_class = Class.new(Deimos::Consumer) do
90
+ schema 'MySchema'
91
+ namespace 'com.my-namespace'
92
+ key_config schema: 'MySchema_key'
93
+ end
94
+ stub_const('ConsumerTest::MySchemaConsumer', consumer_class)
95
+ expect(MyConsumer.new.decode_key('test_id' => '123')).to eq('123')
96
+ expect { MyConsumer.new.decode_key(123) }.to raise_error(NoMethodError)
97
+ end
98
+
99
+ it 'should not encode if plain is set' do
100
+ consumer_class = Class.new(Deimos::Consumer) do
101
+ schema 'MySchema'
102
+ namespace 'com.my-namespace'
103
+ key_config plain: true
104
+ end
105
+ stub_const('ConsumerTest::MyNonEncodedConsumer', consumer_class)
106
+ expect(MyNonEncodedConsumer.new.decode_key('123')).to eq('123')
107
+ end
108
+
109
+ it 'should error with nothing set' do
110
+ consumer_class = Class.new(Deimos::Consumer) do
111
+ schema 'MySchema'
112
+ namespace 'com.my-namespace'
113
+ end
114
+ stub_const('ConsumerTest::MyErrorConsumer', consumer_class)
115
+ expect { MyErrorConsumer.new.decode_key('123') }.
116
+ to raise_error('No key config given - if you are not decoding keys, please use `key_config plain: true`')
117
+ end
118
+
119
+ end
120
+
121
+ describe 'timestamps' do
122
+ before(:each) do
123
+ # :nodoc:
124
+ consumer_class = Class.new(Deimos::Consumer) do
125
+ schema 'MySchemaWithDateTimes'
126
+ namespace 'com.my-namespace'
127
+ key_config plain: true
128
+
129
+ # :nodoc:
130
+ def consume(_payload, _metadata)
131
+ raise 'This should not be called unless call_original is set'
132
+ end
133
+ end
134
+ stub_const('ConsumerTest::MyConsumer', consumer_class)
135
+ stub_consumer(consumer_class)
136
+ end
137
+
138
+ it 'should consume a message' do
139
+ expect(Deimos.config.metrics).to receive(:histogram).twice
140
+ test_consume_message('my_consume_topic',
141
+ 'test_id' => 'foo',
142
+ 'some_int' => 123,
143
+ 'updated_at' => Time.now.to_i,
144
+ 'timestamp' => 2.minutes.ago.to_s) do |payload, _metadata|
145
+ expect(payload['test_id']).to eq('foo')
146
+ end
147
+ end
148
+
149
+ it 'should fail nicely when timestamp wrong format' do
150
+ expect(Deimos.config.metrics).to receive(:histogram).twice
151
+ test_consume_message('my_consume_topic',
152
+ 'test_id' => 'foo',
153
+ 'some_int' => 123,
154
+ 'updated_at' => Time.now.to_i,
155
+ 'timestamp' => 'dffdf') do |payload, _metadata|
156
+ expect(payload['test_id']).to eq('foo')
157
+ end
158
+ test_consume_message('my_consume_topic',
159
+ 'test_id' => 'foo',
160
+ 'some_int' => 123,
161
+ 'updated_at' => Time.now.to_i,
162
+ 'timestamp' => '') do |payload, _metadata|
163
+ expect(payload['test_id']).to eq('foo')
164
+ end
165
+ end
166
+
167
+ end
168
+ end
169
+ end