deimos-ruby 1.6.4 → 1.7.0.pre.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,7 +2,8 @@
2
2
 
3
3
  module Deimos
4
4
  module Utils
5
- # Class which continually polls the database and sends Kafka messages.
5
+ # Class which continually polls the kafka_messages table
6
+ # in the database and sends Kafka messages.
6
7
  class DbProducer
7
8
  include Phobos::Producer
8
9
  attr_accessor :id, :current_topic
@@ -85,7 +85,7 @@ module Deimos
85
85
  retry unless @signal_to_stop
86
86
  end
87
87
  rescue Exception => e
88
- @logger.error("Failed to run listener (#{e.message}) #{error_metadata(e)}")
88
+ @logger.error("Failed to run executor (#{e.message}) #{error_metadata(e)}")
89
89
  raise e
90
90
  end
91
91
 
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Deimos
4
- VERSION = '1.6.4'
4
+ VERSION = '1.7.0-beta1'
5
5
  end
data/lib/deimos.rb CHANGED
@@ -31,6 +31,7 @@ if defined?(ActiveRecord)
31
31
  require 'deimos/utils/signal_handler.rb'
32
32
  require 'deimos/utils/executor.rb'
33
33
  require 'deimos/utils/db_producer.rb'
34
+ require 'deimos/utils/db_poller'
34
35
  end
35
36
 
36
37
  require 'deimos/utils/inline_consumer'
@@ -0,0 +1,11 @@
1
+ class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
2
+ def change
3
+ create_table :deimos_poll_info, force: true do |t|
4
+ t.string :producer, null: false
5
+ t.datetime :last_sent
6
+ t.bigint :last_sent_id
7
+ end
8
+
9
+ add_index :deimos_poll_info, [:producer]
10
+ end
11
+ end
@@ -0,0 +1,16 @@
1
+ class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
2
+ def self.up
3
+ create_table :deimos_poll_info, force: true do |t|
4
+ t.string :producer, null: false
5
+ t.datetime :last_sent
6
+ t.bigint :last_sent_id
7
+ end
8
+
9
+ add_index :deimos_poll_info, [:producer]
10
+ end
11
+
12
+ def self.down
13
+ drop_table :deimos_poll_info
14
+ end
15
+
16
+ end
@@ -0,0 +1,48 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rails/generators'
4
+ require 'rails/generators/active_record/migration'
5
+
6
+ module Deimos
7
+ module Generators
8
+ # Generate the database backend migration.
9
+ class DbPollerGenerator < Rails::Generators::Base
10
+ include Rails::Generators::Migration
11
+ if Rails.version < '4'
12
+ extend(ActiveRecord::Generators::Migration)
13
+ else
14
+ include ActiveRecord::Generators::Migration
15
+ end
16
+ source_root File.expand_path('db_poller/templates', __dir__)
17
+ desc 'Add migrations for the database poller'
18
+
19
+ # @return [String]
20
+ def migration_version
21
+ "[#{ActiveRecord::Migration.current_version}]"
22
+ rescue StandardError
23
+ ''
24
+ end
25
+
26
+ # @return [String]
27
+ def db_migrate_path
28
+ if defined?(Rails.application) && Rails.application
29
+ paths = Rails.application.config.paths['db/migrate']
30
+ paths.respond_to?(:to_ary) ? paths.to_ary.first : paths.to_a.first
31
+ else
32
+ 'db/migrate'
33
+ end
34
+ end
35
+
36
+ # Main method to create all the necessary files
37
+ def generate
38
+ if Rails.version < '4'
39
+ migration_template('rails3_migration',
40
+ "#{db_migrate_path}/create_db_poller.rb")
41
+ else
42
+ migration_template('migration',
43
+ "#{db_migrate_path}/create_db_poller.rb")
44
+ end
45
+ end
46
+ end
47
+ end
48
+ end
@@ -24,4 +24,11 @@ namespace :deimos do
24
24
  Deimos.start_db_backend!(thread_count: thread_count)
25
25
  end
26
26
 
27
+ task db_poller: :environment do
28
+ ENV['DEIMOS_RAKE_TASK'] = 'true'
29
+ STDOUT.sync = true
30
+ Rails.logger.info('Running deimos:db_poller rake task.')
31
+ Deimos::Utils::DbPoller.start!
32
+ end
33
+
27
34
  end
@@ -1,107 +1,85 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # :nodoc:
4
- module ActiveRecordProducerTest
5
- describe Deimos::ActiveRecordProducer do
3
+ describe Deimos::ActiveRecordProducer do
6
4
 
7
- before(:all) do
8
- ActiveRecord::Base.connection.create_table(:widgets) do |t|
9
- t.string(:test_id)
10
- t.integer(:some_int)
11
- t.boolean(:some_bool)
12
- t.timestamps
13
- end
5
+ include_context 'with widgets'
14
6
 
15
- # :nodoc:
16
- class Widget < ActiveRecord::Base
17
- # @return [String]
18
- def generated_id
19
- 'generated_id'
20
- end
21
- end
22
- end
7
+ prepend_before(:each) do
23
8
 
24
- after(:all) do
25
- ActiveRecord::Base.connection.drop_table(:widgets)
9
+ producer_class = Class.new(Deimos::ActiveRecordProducer) do
10
+ schema 'MySchema'
11
+ namespace 'com.my-namespace'
12
+ topic 'my-topic'
13
+ key_config none: true
26
14
  end
15
+ stub_const('MyProducer', producer_class)
27
16
 
28
- prepend_before(:each) do
29
-
30
- producer_class = Class.new(Deimos::ActiveRecordProducer) do
31
- schema 'MySchema'
32
- namespace 'com.my-namespace'
33
- topic 'my-topic'
34
- key_config none: true
35
- end
36
- stub_const('MyProducer', producer_class)
37
-
38
- producer_class = Class.new(Deimos::ActiveRecordProducer) do
39
- schema 'MySchemaWithBooleans'
40
- namespace 'com.my-namespace'
41
- topic 'my-topic-with-boolean'
42
- key_config none: true
43
- end
44
- stub_const('MyBooleanProducer', producer_class)
45
-
46
- producer_class = Class.new(Deimos::ActiveRecordProducer) do
47
- schema 'MySchemaWithId'
48
- namespace 'com.my-namespace'
49
- topic 'my-topic-with-id'
50
- key_config none: true
51
- record_class Widget
17
+ producer_class = Class.new(Deimos::ActiveRecordProducer) do
18
+ schema 'MySchemaWithBooleans'
19
+ namespace 'com.my-namespace'
20
+ topic 'my-topic-with-boolean'
21
+ key_config none: true
22
+ end
23
+ stub_const('MyBooleanProducer', producer_class)
52
24
 
53
- # :nodoc:
54
- def self.generate_payload(attrs, widget)
55
- super.merge(message_id: widget.generated_id)
56
- end
25
+ producer_class = Class.new(Deimos::ActiveRecordProducer) do
26
+ schema 'MySchemaWithId'
27
+ namespace 'com.my-namespace'
28
+ topic 'my-topic-with-id'
29
+ key_config none: true
30
+ record_class Widget
57
31
 
32
+ # :nodoc:
33
+ def self.generate_payload(attrs, widget)
34
+ super.merge(message_id: widget.generated_id)
58
35
  end
59
- stub_const('MyProducerWithID', producer_class)
60
36
 
61
- producer_class = Class.new(Deimos::ActiveRecordProducer) do
62
- schema 'MySchemaWithUniqueId'
63
- namespace 'com.my-namespace'
64
- topic 'my-topic-with-unique-id'
65
- key_config field: :id
66
- record_class Widget
67
- end
68
- stub_const('MyProducerWithUniqueID', producer_class)
69
37
  end
70
-
71
- it 'should send events correctly' do
72
- MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 3))
73
- expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
38
+ stub_const('MyProducerWithID', producer_class)
39
+
40
+ producer_class = Class.new(Deimos::ActiveRecordProducer) do
41
+ schema 'MySchemaWithUniqueId'
42
+ namespace 'com.my-namespace'
43
+ topic 'my-topic-with-unique-id'
44
+ key_config field: :id
45
+ record_class Widget
74
46
  end
47
+ stub_const('MyProducerWithUniqueID', producer_class)
48
+ end
75
49
 
76
- it 'should coerce values' do
77
- MyProducer.send_event(Widget.new(test_id: 'abc', some_int: '3'))
78
- MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 4.5))
79
- expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
80
- expect('my-topic').to have_sent(test_id: 'abc', some_int: 4)
81
- expect {
82
- MyProducer.send_event(Widget.new(test_id: 'abc', some_int: nil))
83
- }.to raise_error(Avro::SchemaValidator::ValidationError)
84
-
85
- MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: nil))
86
- MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: true))
87
- expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: false)
88
- expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: true)
89
- end
50
+ it 'should send events correctly' do
51
+ MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 3))
52
+ expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
53
+ end
90
54
 
91
- it 'should be able to call the record' do
92
- widget = Widget.create!(test_id: 'abc2', some_int: 3)
93
- MyProducerWithID.send_event(id: widget.id, test_id: 'abc2', some_int: 3)
94
- expect('my-topic-with-id').to have_sent(
95
- test_id: 'abc2',
96
- some_int: 3,
97
- message_id: 'generated_id',
98
- timestamp: anything
99
- )
100
- end
55
+ it 'should coerce values' do
56
+ MyProducer.send_event(Widget.new(test_id: 'abc', some_int: '3'))
57
+ MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 4.5))
58
+ expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
59
+ expect('my-topic').to have_sent(test_id: 'abc', some_int: 4)
60
+ expect {
61
+ MyProducer.send_event(Widget.new(test_id: 'abc', some_int: nil))
62
+ }.to raise_error(Avro::SchemaValidator::ValidationError)
63
+
64
+ MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: nil))
65
+ MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: true))
66
+ expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: false)
67
+ expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: true)
68
+ end
101
69
 
102
- specify '#watched_attributes' do
103
- expect(MyProducer.watched_attributes).to eq(%w(test_id some_int))
104
- end
70
+ it 'should be able to call the record' do
71
+ widget = Widget.create!(test_id: 'abc2', some_int: 3)
72
+ MyProducerWithID.send_event(id: widget.id, test_id: 'abc2', some_int: 3)
73
+ expect('my-topic-with-id').to have_sent(
74
+ test_id: 'abc2',
75
+ some_int: 3,
76
+ message_id: 'generated_id',
77
+ timestamp: anything
78
+ )
79
+ end
105
80
 
81
+ specify '#watched_attributes' do
82
+ expect(MyProducer.watched_attributes).to eq(%w(test_id some_int))
106
83
  end
84
+
107
85
  end
@@ -90,7 +90,7 @@ module ConsumerTest
90
90
  'some_int' => 123 },
91
91
  { skip_expectation: true }
92
92
  ) { raise 'OH NOES' }
93
- } .not_to raise_error
93
+ }.not_to raise_error
94
94
  end
95
95
 
96
96
  it 'should not fail when consume fails without reraising errors' do
@@ -101,7 +101,7 @@ module ConsumerTest
101
101
  { 'invalid' => 'key' },
102
102
  { skip_expectation: true }
103
103
  )
104
- } .not_to raise_error
104
+ }.not_to raise_error
105
105
  end
106
106
 
107
107
  it 'should call original' do
@@ -148,7 +148,7 @@ module ProducerTest
148
148
  Deimos.disable_producers do
149
149
  raise 'OH NOES'
150
150
  end
151
- } .to raise_error('OH NOES')
151
+ }.to raise_error('OH NOES')
152
152
  expect(Deimos).not_to be_producers_disabled
153
153
  end
154
154
 
@@ -246,7 +246,7 @@ module ProducerTest
246
246
  MyNonEncodedProducer.publish_list(
247
247
  [{ 'test_id' => 'foo', 'some_int' => 123 }]
248
248
  )
249
- } .to raise_error('No key given but a key is required! Use `key_config none: true` to avoid using keys.')
249
+ }.to raise_error('No key given but a key is required! Use `key_config none: true` to avoid using keys.')
250
250
  end
251
251
 
252
252
  it 'should allow nil keys if none: true is configured' do
@@ -254,7 +254,7 @@ module ProducerTest
254
254
  MyNoKeyProducer.publish_list(
255
255
  [{ 'test_id' => 'foo', 'some_int' => 123 }]
256
256
  )
257
- } .not_to raise_error
257
+ }.not_to raise_error
258
258
  end
259
259
 
260
260
  it 'should use a partition key' do
data/spec/rake_spec.rb CHANGED
@@ -9,7 +9,7 @@ if Rake.application.lookup(:environment).nil?
9
9
  Rake::Task.define_task(:environment)
10
10
  end
11
11
 
12
- describe 'Rakefile' do # rubocop:disable RSpec/DescribeClass
12
+ describe 'Rakefile' do
13
13
  it 'should start listeners' do
14
14
  runner = instance_double(Phobos::CLI::Runner)
15
15
  expect(Phobos::CLI::Runner).to receive(:new).and_return(runner)
data/spec/spec_helper.rb CHANGED
@@ -100,9 +100,8 @@ module DbConfigs
100
100
  end
101
101
  end
102
102
 
103
- # Set up the given database.
104
- def setup_db(options)
105
- ActiveRecord::Base.establish_connection(options)
103
+ # :nodoc:
104
+ def run_db_backend_migration
106
105
  migration_class_name = 'DbBackendMigration'
107
106
  migration_version = '[5.2]'
108
107
  migration = ERB.new(
@@ -110,6 +109,24 @@ module DbConfigs
110
109
  ).result(binding)
111
110
  eval(migration) # rubocop:disable Security/Eval
112
111
  ActiveRecord::Migration.new.run(DbBackendMigration, direction: :up)
112
+ end
113
+
114
+ # :nodoc:
115
+ def run_db_poller_migration
116
+ migration_class_name = 'DbPollerMigration'
117
+ migration_version = '[5.2]'
118
+ migration = ERB.new(
119
+ File.read('lib/generators/deimos/db_poller/templates/migration')
120
+ ).result(binding)
121
+ eval(migration) # rubocop:disable Security/Eval
122
+ ActiveRecord::Migration.new.run(DbPollerMigration, direction: :up)
123
+ end
124
+
125
+ # Set up the given database.
126
+ def setup_db(options)
127
+ ActiveRecord::Base.establish_connection(options)
128
+ run_db_backend_migration
129
+ run_db_poller_migration
113
130
 
114
131
  ActiveRecord::Base.descendants.each do |klass|
115
132
  klass.reset_sequence_name if klass.respond_to?(:reset_sequence_name)
@@ -131,7 +148,7 @@ RSpec.configure do |config|
131
148
  config.shared_context_metadata_behavior = :apply_to_host_groups
132
149
 
133
150
  config.before(:all) do
134
- Time.zone = 'EST'
151
+ Time.zone = 'Eastern Time (US & Canada)'
135
152
  ActiveRecord::Base.logger = Logger.new('/dev/null')
136
153
  ActiveRecord::Base.establish_connection(
137
154
  'adapter' => 'sqlite3',
@@ -141,8 +158,6 @@ RSpec.configure do |config|
141
158
  config.include Deimos::TestHelpers
142
159
  config.include ActiveSupport::Testing::TimeHelpers
143
160
  config.before(:suite) do
144
- Time.zone = 'EST'
145
- ActiveRecord::Base.logger = Logger.new('/dev/null')
146
161
  setup_db(DbConfigs::DB_OPTIONS.last)
147
162
  end
148
163
 
@@ -166,6 +181,29 @@ RSpec.configure do |config|
166
181
  end
167
182
  end
168
183
 
184
+ RSpec.shared_context('with widgets') do
185
+ before(:all) do
186
+ ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
187
+ t.string(:test_id)
188
+ t.integer(:some_int)
189
+ t.boolean(:some_bool)
190
+ t.timestamps
191
+ end
192
+
193
+ # :nodoc:
194
+ class Widget < ActiveRecord::Base
195
+ # @return [String]
196
+ def generated_id
197
+ 'generated_id'
198
+ end
199
+ end
200
+ end
201
+
202
+ after(:all) do
203
+ ActiveRecord::Base.connection.drop_table(:widgets)
204
+ end
205
+ end
206
+
169
207
  RSpec.shared_context('with DB') do
170
208
  before(:all) do
171
209
  setup_db(self.class.metadata[:db_config] || DbConfigs::DB_OPTIONS.last)