deimos-ruby 1.6.4 → 1.7.0.pre.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +9 -0
- data/.rubocop.yml +15 -13
- data/.ruby-version +1 -1
- data/CHANGELOG.md +3 -0
- data/Gemfile.lock +35 -34
- data/README.md +70 -0
- data/Rakefile +1 -1
- data/deimos-ruby.gemspec +0 -1
- data/docs/CONFIGURATION.md +23 -0
- data/lib/deimos/active_record_producer.rb +23 -0
- data/lib/deimos/config/configuration.rb +20 -0
- data/lib/deimos/kafka_topic_info.rb +1 -1
- data/lib/deimos/metrics/provider.rb +0 -2
- data/lib/deimos/poll_info.rb +9 -0
- data/lib/deimos/tracing/provider.rb +0 -2
- data/lib/deimos/utils/db_poller.rb +149 -0
- data/lib/deimos/utils/db_producer.rb +2 -1
- data/lib/deimos/utils/executor.rb +1 -1
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +1 -0
- data/lib/generators/deimos/db_poller/templates/migration +11 -0
- data/lib/generators/deimos/db_poller/templates/rails3_migration +16 -0
- data/lib/generators/deimos/db_poller_generator.rb +48 -0
- data/lib/tasks/deimos.rake +7 -0
- data/spec/active_record_producer_spec.rb +66 -88
- data/spec/consumer_spec.rb +2 -2
- data/spec/producer_spec.rb +3 -3
- data/spec/rake_spec.rb +1 -1
- data/spec/spec_helper.rb +44 -6
- data/spec/utils/db_poller_spec.rb +320 -0
- metadata +13 -19
@@ -2,7 +2,8 @@
|
|
2
2
|
|
3
3
|
module Deimos
|
4
4
|
module Utils
|
5
|
-
# Class which continually polls the
|
5
|
+
# Class which continually polls the kafka_messages table
|
6
|
+
# in the database and sends Kafka messages.
|
6
7
|
class DbProducer
|
7
8
|
include Phobos::Producer
|
8
9
|
attr_accessor :id, :current_topic
|
data/lib/deimos/version.rb
CHANGED
data/lib/deimos.rb
CHANGED
@@ -0,0 +1,11 @@
|
|
1
|
+
class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
|
2
|
+
def change
|
3
|
+
create_table :deimos_poll_info, force: true do |t|
|
4
|
+
t.string :producer, null: false
|
5
|
+
t.datetime :last_sent
|
6
|
+
t.bigint :last_sent_id
|
7
|
+
end
|
8
|
+
|
9
|
+
add_index :deimos_poll_info, [:producer]
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
|
2
|
+
def self.up
|
3
|
+
create_table :deimos_poll_info, force: true do |t|
|
4
|
+
t.string :producer, null: false
|
5
|
+
t.datetime :last_sent
|
6
|
+
t.bigint :last_sent_id
|
7
|
+
end
|
8
|
+
|
9
|
+
add_index :deimos_poll_info, [:producer]
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.down
|
13
|
+
drop_table :deimos_poll_info
|
14
|
+
end
|
15
|
+
|
16
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'rails/generators'
|
4
|
+
require 'rails/generators/active_record/migration'
|
5
|
+
|
6
|
+
module Deimos
|
7
|
+
module Generators
|
8
|
+
# Generate the database backend migration.
|
9
|
+
class DbPollerGenerator < Rails::Generators::Base
|
10
|
+
include Rails::Generators::Migration
|
11
|
+
if Rails.version < '4'
|
12
|
+
extend(ActiveRecord::Generators::Migration)
|
13
|
+
else
|
14
|
+
include ActiveRecord::Generators::Migration
|
15
|
+
end
|
16
|
+
source_root File.expand_path('db_poller/templates', __dir__)
|
17
|
+
desc 'Add migrations for the database poller'
|
18
|
+
|
19
|
+
# @return [String]
|
20
|
+
def migration_version
|
21
|
+
"[#{ActiveRecord::Migration.current_version}]"
|
22
|
+
rescue StandardError
|
23
|
+
''
|
24
|
+
end
|
25
|
+
|
26
|
+
# @return [String]
|
27
|
+
def db_migrate_path
|
28
|
+
if defined?(Rails.application) && Rails.application
|
29
|
+
paths = Rails.application.config.paths['db/migrate']
|
30
|
+
paths.respond_to?(:to_ary) ? paths.to_ary.first : paths.to_a.first
|
31
|
+
else
|
32
|
+
'db/migrate'
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
# Main method to create all the necessary files
|
37
|
+
def generate
|
38
|
+
if Rails.version < '4'
|
39
|
+
migration_template('rails3_migration',
|
40
|
+
"#{db_migrate_path}/create_db_poller.rb")
|
41
|
+
else
|
42
|
+
migration_template('migration',
|
43
|
+
"#{db_migrate_path}/create_db_poller.rb")
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
data/lib/tasks/deimos.rake
CHANGED
@@ -24,4 +24,11 @@ namespace :deimos do
|
|
24
24
|
Deimos.start_db_backend!(thread_count: thread_count)
|
25
25
|
end
|
26
26
|
|
27
|
+
task db_poller: :environment do
|
28
|
+
ENV['DEIMOS_RAKE_TASK'] = 'true'
|
29
|
+
STDOUT.sync = true
|
30
|
+
Rails.logger.info('Running deimos:db_poller rake task.')
|
31
|
+
Deimos::Utils::DbPoller.start!
|
32
|
+
end
|
33
|
+
|
27
34
|
end
|
@@ -1,107 +1,85 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
|
4
|
-
module ActiveRecordProducerTest
|
5
|
-
describe Deimos::ActiveRecordProducer do
|
3
|
+
describe Deimos::ActiveRecordProducer do
|
6
4
|
|
7
|
-
|
8
|
-
ActiveRecord::Base.connection.create_table(:widgets) do |t|
|
9
|
-
t.string(:test_id)
|
10
|
-
t.integer(:some_int)
|
11
|
-
t.boolean(:some_bool)
|
12
|
-
t.timestamps
|
13
|
-
end
|
5
|
+
include_context 'with widgets'
|
14
6
|
|
15
|
-
|
16
|
-
class Widget < ActiveRecord::Base
|
17
|
-
# @return [String]
|
18
|
-
def generated_id
|
19
|
-
'generated_id'
|
20
|
-
end
|
21
|
-
end
|
22
|
-
end
|
7
|
+
prepend_before(:each) do
|
23
8
|
|
24
|
-
|
25
|
-
|
9
|
+
producer_class = Class.new(Deimos::ActiveRecordProducer) do
|
10
|
+
schema 'MySchema'
|
11
|
+
namespace 'com.my-namespace'
|
12
|
+
topic 'my-topic'
|
13
|
+
key_config none: true
|
26
14
|
end
|
15
|
+
stub_const('MyProducer', producer_class)
|
27
16
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
end
|
36
|
-
stub_const('MyProducer', producer_class)
|
37
|
-
|
38
|
-
producer_class = Class.new(Deimos::ActiveRecordProducer) do
|
39
|
-
schema 'MySchemaWithBooleans'
|
40
|
-
namespace 'com.my-namespace'
|
41
|
-
topic 'my-topic-with-boolean'
|
42
|
-
key_config none: true
|
43
|
-
end
|
44
|
-
stub_const('MyBooleanProducer', producer_class)
|
45
|
-
|
46
|
-
producer_class = Class.new(Deimos::ActiveRecordProducer) do
|
47
|
-
schema 'MySchemaWithId'
|
48
|
-
namespace 'com.my-namespace'
|
49
|
-
topic 'my-topic-with-id'
|
50
|
-
key_config none: true
|
51
|
-
record_class Widget
|
17
|
+
producer_class = Class.new(Deimos::ActiveRecordProducer) do
|
18
|
+
schema 'MySchemaWithBooleans'
|
19
|
+
namespace 'com.my-namespace'
|
20
|
+
topic 'my-topic-with-boolean'
|
21
|
+
key_config none: true
|
22
|
+
end
|
23
|
+
stub_const('MyBooleanProducer', producer_class)
|
52
24
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
25
|
+
producer_class = Class.new(Deimos::ActiveRecordProducer) do
|
26
|
+
schema 'MySchemaWithId'
|
27
|
+
namespace 'com.my-namespace'
|
28
|
+
topic 'my-topic-with-id'
|
29
|
+
key_config none: true
|
30
|
+
record_class Widget
|
57
31
|
|
32
|
+
# :nodoc:
|
33
|
+
def self.generate_payload(attrs, widget)
|
34
|
+
super.merge(message_id: widget.generated_id)
|
58
35
|
end
|
59
|
-
stub_const('MyProducerWithID', producer_class)
|
60
36
|
|
61
|
-
producer_class = Class.new(Deimos::ActiveRecordProducer) do
|
62
|
-
schema 'MySchemaWithUniqueId'
|
63
|
-
namespace 'com.my-namespace'
|
64
|
-
topic 'my-topic-with-unique-id'
|
65
|
-
key_config field: :id
|
66
|
-
record_class Widget
|
67
|
-
end
|
68
|
-
stub_const('MyProducerWithUniqueID', producer_class)
|
69
37
|
end
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
38
|
+
stub_const('MyProducerWithID', producer_class)
|
39
|
+
|
40
|
+
producer_class = Class.new(Deimos::ActiveRecordProducer) do
|
41
|
+
schema 'MySchemaWithUniqueId'
|
42
|
+
namespace 'com.my-namespace'
|
43
|
+
topic 'my-topic-with-unique-id'
|
44
|
+
key_config field: :id
|
45
|
+
record_class Widget
|
74
46
|
end
|
47
|
+
stub_const('MyProducerWithUniqueID', producer_class)
|
48
|
+
end
|
75
49
|
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
expect('my-topic').to have_sent(test_id: 'abc', some_int: 4)
|
81
|
-
expect {
|
82
|
-
MyProducer.send_event(Widget.new(test_id: 'abc', some_int: nil))
|
83
|
-
}.to raise_error(Avro::SchemaValidator::ValidationError)
|
84
|
-
|
85
|
-
MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: nil))
|
86
|
-
MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: true))
|
87
|
-
expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: false)
|
88
|
-
expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: true)
|
89
|
-
end
|
50
|
+
it 'should send events correctly' do
|
51
|
+
MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 3))
|
52
|
+
expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
|
53
|
+
end
|
90
54
|
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
55
|
+
it 'should coerce values' do
|
56
|
+
MyProducer.send_event(Widget.new(test_id: 'abc', some_int: '3'))
|
57
|
+
MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 4.5))
|
58
|
+
expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
|
59
|
+
expect('my-topic').to have_sent(test_id: 'abc', some_int: 4)
|
60
|
+
expect {
|
61
|
+
MyProducer.send_event(Widget.new(test_id: 'abc', some_int: nil))
|
62
|
+
}.to raise_error(Avro::SchemaValidator::ValidationError)
|
63
|
+
|
64
|
+
MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: nil))
|
65
|
+
MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: true))
|
66
|
+
expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: false)
|
67
|
+
expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: true)
|
68
|
+
end
|
101
69
|
|
102
|
-
|
103
|
-
|
104
|
-
|
70
|
+
it 'should be able to call the record' do
|
71
|
+
widget = Widget.create!(test_id: 'abc2', some_int: 3)
|
72
|
+
MyProducerWithID.send_event(id: widget.id, test_id: 'abc2', some_int: 3)
|
73
|
+
expect('my-topic-with-id').to have_sent(
|
74
|
+
test_id: 'abc2',
|
75
|
+
some_int: 3,
|
76
|
+
message_id: 'generated_id',
|
77
|
+
timestamp: anything
|
78
|
+
)
|
79
|
+
end
|
105
80
|
|
81
|
+
specify '#watched_attributes' do
|
82
|
+
expect(MyProducer.watched_attributes).to eq(%w(test_id some_int))
|
106
83
|
end
|
84
|
+
|
107
85
|
end
|
data/spec/consumer_spec.rb
CHANGED
@@ -90,7 +90,7 @@ module ConsumerTest
|
|
90
90
|
'some_int' => 123 },
|
91
91
|
{ skip_expectation: true }
|
92
92
|
) { raise 'OH NOES' }
|
93
|
-
}
|
93
|
+
}.not_to raise_error
|
94
94
|
end
|
95
95
|
|
96
96
|
it 'should not fail when consume fails without reraising errors' do
|
@@ -101,7 +101,7 @@ module ConsumerTest
|
|
101
101
|
{ 'invalid' => 'key' },
|
102
102
|
{ skip_expectation: true }
|
103
103
|
)
|
104
|
-
}
|
104
|
+
}.not_to raise_error
|
105
105
|
end
|
106
106
|
|
107
107
|
it 'should call original' do
|
data/spec/producer_spec.rb
CHANGED
@@ -148,7 +148,7 @@ module ProducerTest
|
|
148
148
|
Deimos.disable_producers do
|
149
149
|
raise 'OH NOES'
|
150
150
|
end
|
151
|
-
}
|
151
|
+
}.to raise_error('OH NOES')
|
152
152
|
expect(Deimos).not_to be_producers_disabled
|
153
153
|
end
|
154
154
|
|
@@ -246,7 +246,7 @@ module ProducerTest
|
|
246
246
|
MyNonEncodedProducer.publish_list(
|
247
247
|
[{ 'test_id' => 'foo', 'some_int' => 123 }]
|
248
248
|
)
|
249
|
-
}
|
249
|
+
}.to raise_error('No key given but a key is required! Use `key_config none: true` to avoid using keys.')
|
250
250
|
end
|
251
251
|
|
252
252
|
it 'should allow nil keys if none: true is configured' do
|
@@ -254,7 +254,7 @@ module ProducerTest
|
|
254
254
|
MyNoKeyProducer.publish_list(
|
255
255
|
[{ 'test_id' => 'foo', 'some_int' => 123 }]
|
256
256
|
)
|
257
|
-
}
|
257
|
+
}.not_to raise_error
|
258
258
|
end
|
259
259
|
|
260
260
|
it 'should use a partition key' do
|
data/spec/rake_spec.rb
CHANGED
@@ -9,7 +9,7 @@ if Rake.application.lookup(:environment).nil?
|
|
9
9
|
Rake::Task.define_task(:environment)
|
10
10
|
end
|
11
11
|
|
12
|
-
describe 'Rakefile' do
|
12
|
+
describe 'Rakefile' do
|
13
13
|
it 'should start listeners' do
|
14
14
|
runner = instance_double(Phobos::CLI::Runner)
|
15
15
|
expect(Phobos::CLI::Runner).to receive(:new).and_return(runner)
|
data/spec/spec_helper.rb
CHANGED
@@ -100,9 +100,8 @@ module DbConfigs
|
|
100
100
|
end
|
101
101
|
end
|
102
102
|
|
103
|
-
#
|
104
|
-
def
|
105
|
-
ActiveRecord::Base.establish_connection(options)
|
103
|
+
# :nodoc:
|
104
|
+
def run_db_backend_migration
|
106
105
|
migration_class_name = 'DbBackendMigration'
|
107
106
|
migration_version = '[5.2]'
|
108
107
|
migration = ERB.new(
|
@@ -110,6 +109,24 @@ module DbConfigs
|
|
110
109
|
).result(binding)
|
111
110
|
eval(migration) # rubocop:disable Security/Eval
|
112
111
|
ActiveRecord::Migration.new.run(DbBackendMigration, direction: :up)
|
112
|
+
end
|
113
|
+
|
114
|
+
# :nodoc:
|
115
|
+
def run_db_poller_migration
|
116
|
+
migration_class_name = 'DbPollerMigration'
|
117
|
+
migration_version = '[5.2]'
|
118
|
+
migration = ERB.new(
|
119
|
+
File.read('lib/generators/deimos/db_poller/templates/migration')
|
120
|
+
).result(binding)
|
121
|
+
eval(migration) # rubocop:disable Security/Eval
|
122
|
+
ActiveRecord::Migration.new.run(DbPollerMigration, direction: :up)
|
123
|
+
end
|
124
|
+
|
125
|
+
# Set up the given database.
|
126
|
+
def setup_db(options)
|
127
|
+
ActiveRecord::Base.establish_connection(options)
|
128
|
+
run_db_backend_migration
|
129
|
+
run_db_poller_migration
|
113
130
|
|
114
131
|
ActiveRecord::Base.descendants.each do |klass|
|
115
132
|
klass.reset_sequence_name if klass.respond_to?(:reset_sequence_name)
|
@@ -131,7 +148,7 @@ RSpec.configure do |config|
|
|
131
148
|
config.shared_context_metadata_behavior = :apply_to_host_groups
|
132
149
|
|
133
150
|
config.before(:all) do
|
134
|
-
Time.zone = '
|
151
|
+
Time.zone = 'Eastern Time (US & Canada)'
|
135
152
|
ActiveRecord::Base.logger = Logger.new('/dev/null')
|
136
153
|
ActiveRecord::Base.establish_connection(
|
137
154
|
'adapter' => 'sqlite3',
|
@@ -141,8 +158,6 @@ RSpec.configure do |config|
|
|
141
158
|
config.include Deimos::TestHelpers
|
142
159
|
config.include ActiveSupport::Testing::TimeHelpers
|
143
160
|
config.before(:suite) do
|
144
|
-
Time.zone = 'EST'
|
145
|
-
ActiveRecord::Base.logger = Logger.new('/dev/null')
|
146
161
|
setup_db(DbConfigs::DB_OPTIONS.last)
|
147
162
|
end
|
148
163
|
|
@@ -166,6 +181,29 @@ RSpec.configure do |config|
|
|
166
181
|
end
|
167
182
|
end
|
168
183
|
|
184
|
+
RSpec.shared_context('with widgets') do
|
185
|
+
before(:all) do
|
186
|
+
ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
|
187
|
+
t.string(:test_id)
|
188
|
+
t.integer(:some_int)
|
189
|
+
t.boolean(:some_bool)
|
190
|
+
t.timestamps
|
191
|
+
end
|
192
|
+
|
193
|
+
# :nodoc:
|
194
|
+
class Widget < ActiveRecord::Base
|
195
|
+
# @return [String]
|
196
|
+
def generated_id
|
197
|
+
'generated_id'
|
198
|
+
end
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
after(:all) do
|
203
|
+
ActiveRecord::Base.connection.drop_table(:widgets)
|
204
|
+
end
|
205
|
+
end
|
206
|
+
|
169
207
|
RSpec.shared_context('with DB') do
|
170
208
|
before(:all) do
|
171
209
|
setup_db(self.class.metadata[:db_config] || DbConfigs::DB_OPTIONS.last)
|