multiple_man 0.8.1 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9ffabcdb730b52881f54be58ff8cb9a5f0da82b7
4
- data.tar.gz: 15a4b0417183785e3234f28cf0c321b4249a8319
3
+ metadata.gz: eff9e5989a0fe4702caf87df4d8a50b0fb5cea03
4
+ data.tar.gz: 04361d38e491938b791dfb98e16727240ee33daa
5
5
  SHA512:
6
- metadata.gz: 1e0d0b0df3f6eee4f9b1aae2f856514d5e063ef64bc3eb277b046c15ea92b4e9100db5f71821d8e06e9d9d8265dcd413da9a08f691ef7d175040f756b673cf9c
7
- data.tar.gz: 9196199a4849489924e1cac04e46dd4e24490b09f89d02b70a5c5398df20c4245e7130148fde0cf9c4aa921168c9518b691a1c0bf7618c1db9f6e2d7eeee115a
6
+ metadata.gz: 4fdf426a23e43e4ee9f13d19d2b940d329b8ead830fe0155043370c0341d7e8b1ef012d8712375f85d9235c9aea2a61e9084b80b6837dd8b37e01f0952c7988a
7
+ data.tar.gz: 092424fa6b574af0eef20f6585b0fe1026cc0baee453eb4d349bb106d9d71ca2f612df050dee82030140dea45f43f017043d784d5bf3ee70c2b2e74ea9fd1ae3
data/README.md CHANGED
@@ -68,6 +68,9 @@ exceptions encountered in an `after_commit` block, meaning
68
68
  that without handling these errors through the configuration,
69
69
  they will be silently ignored.
70
70
 
71
+ Errors will be captured and wrapped in a MultipleMan::Error. The
72
+ cause will be preserved in Exception#cause.
73
+
71
74
  ### Publishing models
72
75
 
73
76
  #### Directly from the model
@@ -111,7 +114,7 @@ By default, MultipleMan will publish all of your models whenever you save a mode
111
114
 
112
115
  ```
113
116
  # Publish all widgets to MultipleMan
114
- Widget.multiple_man_publish
117
+ Widget.multiple_man_publish
115
118
 
116
119
  # Publish a subset of widgets to MultipleMan
117
120
  Widget.where(published: true).multiple_man_publish
@@ -184,6 +187,26 @@ MyModel.multiple_man_publish(:seed)
184
187
 
185
188
  3. Stop the seeder rake task when all of your messages have been processed. You can check your RabbitMQ server
186
189
 
190
+ ## Upgrading to 1.0
191
+
192
+ The major change is that MultipleMan will no longer create a queue per listener.
193
+ There is only 1 queue that will have multiple bindings to the exchange so that
194
+ you have a chance to maintain causal consistency.
195
+
196
+ Assuming you are using vanilla MultipleMan you will need to run both the
197
+ regular worker and the new 'transition_worker' for a short period. The
198
+ transitional worker will connect to your old queues, unbind them and allow them
199
+ to drain. Once the queues are empty you can safely shut the transitional worker
200
+ down and delete the old queues.
201
+
202
+ So for example if you use a Procfile:
203
+
204
+ ```
205
+ multiple_man_worker: rake multiple_man:worker
206
+ # Temporary until old queues are drained
207
+ transition_worker: rake multiple_man:transition_worker
208
+ ```
209
+
187
210
  ## Contributing
188
211
 
189
212
  1. Fork it
data/circle.yml CHANGED
@@ -1,3 +1,3 @@
1
1
  machine:
2
2
  ruby:
3
- version: ruby-2.2.2
3
+ version: ruby-2.2.3
@@ -5,7 +5,7 @@ module MultipleMan
5
5
  proc { queue << RemoveCommand.new(thread_id); reaper.push(channel) }
6
6
  end
7
7
 
8
- def initialize(config, reaper)
8
+ def initialize(_, reaper)
9
9
  @reaper = reaper
10
10
  @queue = Queue.new
11
11
 
@@ -1,5 +1,19 @@
1
1
  module MultipleMan
2
+ def self.configuration
3
+ @configuration ||= Configuration.new
4
+ end
5
+
6
+ def self.configure
7
+ yield(configuration) if block_given?
8
+ end
9
+
2
10
  class Configuration
11
+ attr_reader :subscriber_registry
12
+ attr_accessor :topic_name, :app_name, :connection, :enabled, :error_handler,
13
+ :worker_concurrency, :reraise_errors, :connection_recovery,
14
+ :queue_name, :prefetch_size
15
+
16
+ attr_writer :logger
3
17
 
4
18
  def initialize
5
19
  self.topic_name = "multiple_man"
@@ -7,11 +21,18 @@ module MultipleMan
7
21
  self.enabled = true
8
22
  self.worker_concurrency = 1
9
23
  self.reraise_errors = true
24
+ self.prefetch_size = 100
10
25
  self.connection_recovery = {
11
26
  time_before_reconnect: 0.2,
12
27
  time_between_retries: 0.8,
13
28
  max_retries: 5
14
29
  }
30
+
31
+ @subscriber_registry = Subscribers::Registry.new
32
+ end
33
+
34
+ def queue_name
35
+ @queue_name ||= "#{topic_name}.#{app_name}"
15
36
  end
16
37
 
17
38
  def logger
@@ -22,16 +43,12 @@ module MultipleMan
22
43
  @error_handler = block
23
44
  end
24
45
 
25
- attr_accessor :topic_name, :app_name, :connection, :enabled, :error_handler,
26
- :worker_concurrency, :reraise_errors, :connection_recovery
27
- attr_writer :logger
28
- end
29
-
30
- def self.configuration
31
- @configuration ||= Configuration.new
32
- end
46
+ def listeners
47
+ subscriber_registry.subscriptions
48
+ end
33
49
 
34
- def self.configure
35
- yield(configuration) if block_given?
50
+ def register_listener(listener)
51
+ subscriber_registry.register(listener)
52
+ end
36
53
  end
37
54
  end
@@ -0,0 +1,75 @@
1
+ require 'json'
2
+ require 'active_support/core_ext/hash'
3
+
4
+ module MultipleMan
5
+ module Consumers
6
+ class General
7
+ def initialize(subscribers:, queue:, topic:)
8
+ self.subscribers = subscribers
9
+ @topic = topic
10
+ @queue = queue
11
+ end
12
+
13
+ def listen
14
+ MultipleMan.logger.debug "Starting listeners."
15
+ create_bindings
16
+
17
+ queue.subscribe(manual_ack: true) do |delivery_info, meta_data, payload|
18
+ MultipleMan.logger.debug "Processing message for #{delivery_info.routing_key}."
19
+ message = JSON.parse(payload).with_indifferent_access
20
+ receive(delivery_info, meta_data, message)
21
+ end
22
+ end
23
+
24
+ private
25
+
26
+ attr_reader :subscribers, :topic, :queue
27
+
28
+ def create_bindings
29
+ subscribers.values.each do |subscriber|
30
+ MultipleMan.logger.info "Listening for #{subscriber.listen_to} with routing key #{subscriber.routing_key}."
31
+ queue.bind(topic, routing_key: routing_key_for_subscriber(subscriber))
32
+ end
33
+ end
34
+
35
+ def receive(delivery_info, _, message)
36
+ method = operation(message, delivery_info.routing_key)
37
+ dispatch_subscribers(message, method, delivery_info.routing_key)
38
+ queue.channel.acknowledge(delivery_info.delivery_tag, false)
39
+
40
+ MultipleMan.logger.debug "Successfully processed! #{delivery_info.routing_key}"
41
+ rescue => ex
42
+ begin
43
+ raise ConsumerError
44
+ rescue => wrapped_ex
45
+ MultipleMan.logger.debug "\tError #{wrapped_ex.message} \n#{wrapped_ex.backtrace}"
46
+ MultipleMan.error(wrapped_ex, reraise: false, payload: message, delivery_info: delivery_info)
47
+ queue.channel.nack(delivery_info.delivery_tag)
48
+ end
49
+ end
50
+
51
+ def dispatch_subscribers(message, method, routing_key)
52
+ subscribers.select { |k,s| k.match(routing_key) }
53
+ .values
54
+ .each do |s|
55
+ s.send(method, message)
56
+ end
57
+ end
58
+
59
+ def operation(message, routing_key)
60
+ message['operation'] || routing_key.split('.').last
61
+ end
62
+
63
+ def routing_key_for_subscriber(subscriber)
64
+ subscriber.routing_key
65
+ end
66
+
67
+ def subscribers=(subscribers)
68
+ @subscribers = subscribers.map { |s|
69
+ key = routing_key_for_subscriber(s).gsub('.', '\.').gsub('#', '.*')
70
+ [/^#{key}$/, s]
71
+ }.to_h
72
+ end
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,15 @@
1
+ module MultipleMan::Consumers
2
+ class Seed < General
3
+
4
+ private
5
+
6
+ def operation(_, _)
7
+ "create"
8
+ end
9
+
10
+ def routing_key_for_subscriber(subscriber)
11
+ subscriber.routing_key(:seed)
12
+ end
13
+
14
+ end
15
+ end
@@ -0,0 +1,50 @@
1
+ require 'json'
2
+ require 'active_support/core_ext/hash'
3
+
4
+ module MultipleMan::Consumers
5
+ class Transitional
6
+
7
+ attr_reader :subscription, :queue, :topic
8
+
9
+ def initialize(subscription:, queue:, topic:)
10
+ @subscription = subscription
11
+ @topic = topic
12
+ @queue = queue
13
+ end
14
+
15
+ def listen
16
+ MultipleMan.logger.info "Listening for #{subscription.listen_to} with routing key #{routing_key}."
17
+ queue.unbind(topic, routing_key: routing_key).subscribe(manual_ack: true) do |delivery_info, _, payload|
18
+ process_message(delivery_info, payload)
19
+ end
20
+ end
21
+
22
+ def process_message(delivery_info, payload)
23
+ MultipleMan.logger.debug "Processing message for #{delivery_info.routing_key}."
24
+
25
+ payload = JSON.parse(payload).with_indifferent_access
26
+ subscription.send(operation(delivery_info, payload), payload)
27
+ MultipleMan.logger.debug " Successfully processed!"
28
+ queue.channel.acknowledge(delivery_info.delivery_tag, false)
29
+ rescue => ex
30
+ raise MultipleMan::ConsumerError rescue handle_error($!, delivery_info)
31
+ end
32
+
33
+ def handle_error(ex, delivery_info)
34
+ MultipleMan.logger.error " Error - #{ex.message}\n\n#{ex.backtrace}"
35
+ MultipleMan.error(ex, reraise: false)
36
+
37
+ # Requeue the message
38
+ queue.channel.nack(delivery_info.delivery_tag)
39
+ end
40
+
41
+ def operation(delivery_info, payload)
42
+ payload['operation'] || delivery_info.routing_key.split(".").last
43
+ end
44
+
45
+ def routing_key
46
+ subscription.routing_key
47
+ end
48
+
49
+ end
50
+ end
@@ -1,15 +1,15 @@
1
1
  module MultipleMan
2
2
  module Listener
3
- def Listener.included(base)
3
+ def self.included(base)
4
4
  base.extend(ClassMethods)
5
5
  end
6
6
 
7
- def routing_key(operation=self.operation)
8
- MultipleMan::RoutingKey.new(klass, operation).to_s
7
+ def routing_key(operation = self.operation)
8
+ MultipleMan::RoutingKey.new(listen_to, operation).to_s
9
9
  end
10
10
 
11
- attr_accessor :klass
12
11
  attr_accessor :operation
12
+ attr_accessor :listen_to
13
13
 
14
14
  def create(payload)
15
15
  # noop
@@ -23,16 +23,13 @@ module MultipleMan
23
23
  # noop
24
24
  end
25
25
 
26
- def queue_name
27
- "#{MultipleMan.configuration.topic_name}.#{MultipleMan.configuration.app_name}.#{klass}"
28
- end
29
-
30
26
  module ClassMethods
31
27
  def listen_to(model, operation: '#')
32
- listener = self.new
33
- listener.klass = model
28
+ listener = new
29
+ listener.listen_to = model
34
30
  listener.operation = operation
35
- Subscribers::Registry.register(listener)
31
+ MultipleMan.configuration.register_listener(listener)
32
+ listener
36
33
  end
37
34
  end
38
35
  end
@@ -6,7 +6,7 @@ module MultipleMan
6
6
 
7
7
  module ClassMethods
8
8
  def subscribe(options = {})
9
- Subscribers::Registry.register(Subscribers::ModelSubscriber.new(self, options))
9
+ ::MultipleMan.configuration.register_listener(Subscribers::ModelSubscriber.new(self, options))
10
10
  end
11
11
  end
12
12
  end
@@ -7,19 +7,19 @@ module MultipleMan::Subscribers
7
7
 
8
8
  attr_reader :klass
9
9
 
10
- def create(payload)
10
+ def create(_)
11
11
  # noop
12
12
  end
13
13
 
14
- def update(payload)
14
+ def update(_)
15
15
  # noop
16
16
  end
17
17
 
18
- def destroy(payload)
18
+ def destroy(_)
19
19
  # noop
20
20
  end
21
21
 
22
- def seed(payload)
22
+ def seed(_)
23
23
  # noop
24
24
  end
25
25
 
@@ -27,11 +27,11 @@ module MultipleMan::Subscribers
27
27
  MultipleMan::RoutingKey.new(klass, operation).to_s
28
28
  end
29
29
 
30
- def queue_name
31
- "#{MultipleMan.configuration.topic_name}.#{MultipleMan.configuration.app_name}.#{klass}"
30
+ def listen_to
31
+ klass
32
32
  end
33
33
 
34
- private
34
+ private
35
35
 
36
36
  attr_writer :klass
37
37
  end
@@ -1,12 +1,13 @@
1
1
  module MultipleMan::Subscribers
2
2
  class Registry
3
- @subscriptions = []
4
- class << self
5
- attr_accessor :subscriptions
3
+ attr_reader :subscriptions
6
4
 
7
- def register(subscription)
8
- self.subscriptions << subscription
9
- end
5
+ def initialize
6
+ @subscriptions = []
7
+ end
8
+
9
+ def register(subscription)
10
+ self.subscriptions << subscription
10
11
  end
11
12
  end
12
13
  end
@@ -1,18 +1,58 @@
1
1
  namespace :multiple_man do
2
2
  desc "Run multiple man listeners"
3
- task :worker => :environment do
4
- run_listener(MultipleMan::Listeners::Listener)
3
+ task worker: :environment do
4
+ channel = MultipleMan::Connection.connection.create_channel
5
+ channel.prefetch(MultipleMan.configuration.prefetch_size)
6
+ queue_name = MultipleMan.configuration.queue_name
7
+ queue = channel.queue(queue_name, durable: true, auto_delete: false)
8
+
9
+ run_listener(MultipleMan::Consumers::General, queue)
5
10
  end
6
11
 
7
12
  desc 'Run a seeding listener'
8
13
  task seed: :environment do
9
- run_listener(MultipleMan::Listeners::SeederListener)
14
+ channel = MultipleMan::Connection.connection.create_channel
15
+ channel.prefetch(MultipleMan.configuration.prefetch_size)
16
+ queue_name = MultipleMan.configuration.queue_name + '.seed'
17
+ queue = channel.queue(queue_name, durable: false, auto_delete: true)
18
+
19
+ run_listener(MultipleMan::Consumers::Seed, queue)
20
+ end
21
+
22
+ def run_listener(listener, queue)
23
+ Rails.application.eager_load! if defined?(Rails)
24
+
25
+ subscribers = MultipleMan.configuration.listeners
26
+ topic = MultipleMan.configuration.topic_name
27
+
28
+ listener.new(subscribers: subscribers, queue: queue, topic: topic).listen
29
+
30
+ Signal.trap("INT") { puts "received INT"; exit }
31
+ Signal.trap("QUIT") { puts "received QUIT"; exit }
32
+ Signal.trap("TERM") { puts "received TERM"; exit }
33
+
34
+ sleep
10
35
  end
11
36
 
12
- def run_listener(listener)
13
- Rails.application.eager_load!
37
+ desc 'Run transitional worker'
38
+ task transition_worker: :environment do
39
+ Rails.application.eager_load! if defined?(Rails)
14
40
 
15
- listener.start
41
+ topic = MultipleMan.configuration.topic_name
42
+ app_name = MultipleMan.configuration.app_name
43
+ channel = MultipleMan::Connection.connection.create_channel
44
+ channel.prefetch(MultipleMan.configuration.prefetch_size)
45
+
46
+ MultipleMan.configuration.listeners.each do |listener|
47
+ queue_name = listener.respond_to?(:queue_name) ?
48
+ listener.queue_name :
49
+ "#{topic}.#{app_name}.#{listener.listen_to}"
50
+
51
+ next unless MultipleMan::Connection.connection.queue_exists?(queue_name)
52
+ queue = channel.queue(queue_name, durable: true, auto_delete: false)
53
+
54
+ MultipleMan::Consumers::Transitional.new(subscription: listener, queue: queue, topic: topic).listen
55
+ end
16
56
 
17
57
  Signal.trap("INT") { puts "received INT"; exit }
18
58
  Signal.trap("QUIT") { puts "received QUIT"; exit }
@@ -20,4 +60,5 @@ namespace :multiple_man do
20
60
 
21
61
  sleep
22
62
  end
63
+
23
64
  end
@@ -1,3 +1,3 @@
1
1
  module MultipleMan
2
- VERSION = "0.8.1"
2
+ VERSION = "1.0.0"
3
3
  end
data/lib/multiple_man.rb CHANGED
@@ -2,6 +2,9 @@ require "multiple_man/version"
2
2
  require 'active_support'
3
3
 
4
4
  module MultipleMan
5
+ Error = Class.new(StandardError)
6
+ ConsumerError = Class.new(Error)
7
+
5
8
  require 'multiple_man/railtie' if defined?(Rails)
6
9
 
7
10
  require 'multiple_man/mixins/publisher'
@@ -16,8 +19,9 @@ module MultipleMan
16
19
  require 'multiple_man/payload_generator'
17
20
  require 'multiple_man/connection'
18
21
  require 'multiple_man/routing_key'
19
- require 'multiple_man/listeners/listener'
20
- require 'multiple_man/listeners/seeder_listener'
22
+ require 'multiple_man/consumers/general'
23
+ require 'multiple_man/consumers/transitional'
24
+ require 'multiple_man/consumers/seed'
21
25
  require 'multiple_man/model_populator'
22
26
  require 'multiple_man/identity'
23
27
  require 'multiple_man/publish'
@@ -38,11 +42,14 @@ module MultipleMan
38
42
  end
39
43
 
40
44
  def self.error(ex, options = {})
41
- if configuration.error_handler
42
- configuration.error_handler.call(ex)
43
- raise ex if configuration.reraise_errors && options[:reraise] != false
45
+ raise ex unless configuration.error_handler
46
+
47
+ if configuration.error_handler.arity == 3
48
+ configuration.error_handler.call(ex, options[:payload], options[:delivery_info])
44
49
  else
45
- raise ex
50
+ configuration.error_handler.call(ex)
46
51
  end
52
+
53
+ raise ex if configuration.reraise_errors && options[:reraise] != false
47
54
  end
48
55
  end
data/multiple_man.gemspec CHANGED
@@ -10,13 +10,17 @@ Gem::Specification.new do |spec|
10
10
  spec.email = ["ryan@influitive.com"]
11
11
  spec.description = %q{MultipleMan syncs changes to ActiveRecord models via AMQP}
12
12
  spec.summary = %q{MultipleMan syncs changes to ActiveRecord models via AMQP}
13
- spec.homepage = ""
13
+ spec.homepage = "http://github.com/influitive/multiple_man"
14
14
  spec.license = "MIT"
15
15
 
16
16
  spec.files = `git ls-files`.split($/)
17
17
  spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
18
  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
19
  spec.require_paths = ["lib"]
20
+ spec.required_ruby_version = '>= 2.1'
21
+
22
+ spec.add_runtime_dependency "bunny", '>= 1.2'
23
+ spec.add_runtime_dependency "activesupport", '>= 3.0'
20
24
 
21
25
  spec.add_development_dependency "bundler", "~> 1.3"
22
26
  spec.add_development_dependency "pry"
@@ -25,6 +29,4 @@ Gem::Specification.new do |spec|
25
29
  spec.add_development_dependency 'codeclimate-test-reporter'
26
30
  spec.add_development_dependency 'guard'
27
31
  spec.add_development_dependency 'guard-rspec'
28
- spec.add_runtime_dependency "bunny", '>= 1.2'
29
- spec.add_runtime_dependency "activesupport", '>= 3.0'
30
32
  end
@@ -3,19 +3,18 @@ require 'spec_helper'
3
3
  describe MultipleMan::Connection do
4
4
 
5
5
  let(:mock_bunny) { double(Bunny, open?: true, close: nil) }
6
- let(:mock_channel) { double(Bunny::Channel, close: nil, open?: true, topic: nil) }
6
+ let(:mock_channel) { double(Bunny::Channel, close: nil, open?: true, topic: nil, number: 1) }
7
7
 
8
- describe "connect" do
9
- it "should open a connection and a channel" do
10
- MultipleMan::Connection.should_receive(:connection).and_return(mock_bunny)
11
- mock_bunny.should_receive(:create_channel).once.and_return(mock_channel)
12
-
13
- described_class.connect { }
14
- end
8
+ after do
9
+ Thread.current.thread_variable_set(:multiple_man_current_channel, nil)
10
+ MultipleMan::Connection.reset!
15
11
  end
16
12
 
17
- subject { described_class.new(mock_channel) }
18
-
19
- its(:topic_name) { should == MultipleMan.configuration.topic_name }
13
+ it "should open a connection and a channel" do
14
+ MultipleMan::Connection.should_receive(:connection).and_return(mock_bunny)
15
+ mock_bunny.should_receive(:create_channel).once.and_return(mock_channel)
16
+ expect(mock_channel).to receive(:topic).with(MultipleMan.configuration.topic_name)
20
17
 
18
+ described_class.connect { }
19
+ end
21
20
  end
@@ -0,0 +1,107 @@
1
+ require 'spec_helper'
2
+
3
+ describe MultipleMan::Consumers::General do
4
+ let(:listener1) {
5
+ Class.new do
6
+ include MultipleMan::Listener
7
+
8
+ def initialize
9
+ self.listen_to = 'SomeClass'
10
+ self.operation = '#'
11
+ end
12
+ end
13
+ }
14
+
15
+ let(:listener2) {
16
+ Class.new do
17
+ include MultipleMan::Listener
18
+
19
+ def initialize
20
+ self.listen_to = 'SomeOtherClass'
21
+ self.operation = '#'
22
+ end
23
+ end
24
+ }
25
+
26
+ it "listens to each subscription" do
27
+ subscriptions = [listener1.new, listener2.new]
28
+ queue = double(Bunny::Queue)
29
+
30
+ expect(queue).to receive(:bind).with('some-topic', routing_key: subscriptions.first.routing_key).ordered
31
+ expect(queue).to receive(:bind).with('some-topic', routing_key: subscriptions.last.routing_key).ordered
32
+ expect(queue).to receive(:subscribe).with(manual_ack: true).ordered
33
+
34
+ subject = described_class.new(subscribers: subscriptions, queue: queue, topic: 'some-topic')
35
+
36
+ subject.listen
37
+ end
38
+
39
+ it "sends the correct data" do
40
+ channel = double(Bunny::Channel)
41
+ queue = double(Bunny::Queue, channel: channel).as_null_object
42
+
43
+ subscriber = listener1.new
44
+ subject = described_class.new(subscribers:[subscriber], queue: queue, topic: 'some-topic')
45
+
46
+ expect(channel).to receive(:acknowledge)
47
+ expect(subscriber).to receive(:create).with({"a" => 1, "b" => 2})
48
+
49
+ delivery_info = OpenStruct.new(routing_key: "multiple_man.SomeClass.create")
50
+ payload = '{"a":1,"b":2}'
51
+ allow(queue).to receive(:subscribe).and_yield(delivery_info, double(:meta), payload)
52
+
53
+ subject.listen
54
+ end
55
+
56
+ it "uses the payload to determine the operation if it's available" do
57
+ channel = double(Bunny::Channel).as_null_object
58
+ queue = double(Bunny::Queue, channel: channel).as_null_object
59
+
60
+ subscriber = listener1.new
61
+ subject = described_class.new(subscribers:[subscriber], queue: queue, topic: 'some-topic')
62
+
63
+ delivery_info = OpenStruct.new(routing_key: "multiple_man.SomeClass.some_other_operation")
64
+ payload = '{"operation": "create", "a":1,"b":2}'
65
+ allow(queue).to receive(:subscribe).and_yield(delivery_info, double(:meta), payload)
66
+
67
+ subscriber.should_receive(:create)
68
+
69
+ subject.listen
70
+ end
71
+
72
+ it "sends a nack on failure" do
73
+ allow(MultipleMan.configuration).to receive(:error_handler) { double(:handler).as_null_object}
74
+
75
+ channel = double(Bunny::Channel)
76
+ queue = double(Bunny::Queue, channel: channel).as_null_object
77
+
78
+ delivery_info = OpenStruct.new(routing_key: "multiple_man.SomeClass.create")
79
+ payload = '{"a":1,"b":2}'
80
+ allow(queue).to receive(:subscribe).and_yield(delivery_info, double(:meta), payload)
81
+
82
+ subscriber = listener1.new
83
+ allow(subscriber).to receive(:create).and_raise('anything')
84
+
85
+ expect(channel).to receive(:nack)
86
+ subject = described_class.new(subscribers:[subscriber], queue: queue, topic: 'some-topic')
87
+ subject.listen
88
+ end
89
+
90
+ it 'wraps errors in ConsumerError' do
91
+ channel = double(Bunny::Channel)
92
+ queue = double(Bunny::Queue, channel: channel).as_null_object
93
+
94
+ delivery_info = OpenStruct.new(routing_key: "multiple_man.SomeClass.create")
95
+ payload = '{"a":1,"b":2}'
96
+ allow(queue).to receive(:subscribe).and_yield(delivery_info, double(:meta), payload)
97
+
98
+ subscriber = listener1.new
99
+ allow(subscriber).to receive(:create).and_raise('anything')
100
+
101
+ allow(channel).to receive(:nack)
102
+
103
+ expect(MultipleMan).to receive(:error).with(kind_of(MultipleMan::ConsumerError), kind_of(Hash))
104
+ subject = described_class.new(subscribers:[subscriber], queue: queue, topic: 'some-topic')
105
+ subject.listen
106
+ end
107
+ end
@@ -0,0 +1,66 @@
1
+ require 'spec_helper'
2
+
3
+ describe MultipleMan::Consumers::Seed do
4
+ let(:listener1) {
5
+ Class.new do
6
+ include MultipleMan::Listener
7
+
8
+ def initialize
9
+ self.listen_to = 'SomeClass'
10
+ self.operation = '#'
11
+ end
12
+ end
13
+ }
14
+
15
+ it 'binds for seeding' do
16
+ channel = double(Bunny::Channel).as_null_object
17
+ queue = double(Bunny::Queue, channel: channel)
18
+
19
+ expect(queue).to receive(:bind).with('some-topic', routing_key: "multiple_man.seed.SomeClass")
20
+ expect(queue).to receive(:subscribe).with(manual_ack: true)
21
+
22
+ subject = described_class.new(subscribers: [listener1.new], queue: queue, topic: 'some-topic')
23
+ subject.listen
24
+ end
25
+
26
+ it "sends the correct data" do
27
+ channel = double(Bunny::Channel)
28
+ queue = double(Bunny::Queue, channel: channel).as_null_object
29
+
30
+ subscriber = listener1.new
31
+ subject = described_class.new(subscribers:[subscriber], queue: queue, topic: 'some-topic')
32
+
33
+ expect(channel).to receive(:acknowledge)
34
+ expect(subscriber).to receive(:create).with({"a" => 1, "b" => 2})
35
+
36
+ delivery_info = OpenStruct.new(routing_key: "multiple_man.seed.SomeClass")
37
+ payload = '{"a":1,"b":2}'
38
+ allow(queue).to receive(:subscribe).and_yield(delivery_info, double(:meta), payload)
39
+
40
+ subject.listen
41
+ end
42
+
43
+ let(:group) { Class.new { include MultipleMan::Listener } }
44
+ let(:group_contact) { Class.new { include MultipleMan::Listener } }
45
+
46
+ # BUGFIX:
47
+ it "correctly matches subscribers" do
48
+ channel = double(Bunny::Channel)
49
+ queue = double(Bunny::Queue, channel: channel).as_null_object
50
+
51
+ g = group.listen_to 'Group'
52
+ gc = group_contact.listen_to 'GroupContact'
53
+
54
+ subject = described_class.new(subscribers:[g, gc], queue: queue, topic: 'some-topic')
55
+
56
+ expect(channel).to receive(:acknowledge)
57
+ expect(g).to_not receive(:create)
58
+ expect(gc).to receive(:create).with({"a" => 1, "b" => 2})
59
+
60
+ delivery_info = OpenStruct.new(routing_key: "multiple_man.seed.GroupContact")
61
+ payload = '{"a":1,"b":2}'
62
+ allow(queue).to receive(:subscribe).and_yield(delivery_info, double(:meta), payload)
63
+
64
+ subject.listen
65
+ end
66
+ end
@@ -0,0 +1,33 @@
1
+ require 'spec_helper'
2
+
3
+ describe MultipleMan::Consumers::Transitional do
4
+ let(:listener1) {
5
+ Class.new do
6
+ include MultipleMan::Listener
7
+
8
+ def initialize
9
+ self.listen_to = 'SomeClass'
10
+ self.operation = '#'
11
+ end
12
+ end
13
+ }
14
+
15
+ it 'wraps errors in ConsumerError' do
16
+ channel = double(Bunny::Channel)
17
+ queue = double(Bunny::Queue, channel: channel).as_null_object
18
+
19
+ delivery_info = OpenStruct.new(routing_key: "multiple_man.SomeClass.create")
20
+ payload = '{"a":1,"b":2}'
21
+ allow(queue).to receive(:subscribe).and_yield(delivery_info, double(:meta), payload)
22
+
23
+ subscriber = listener1.new
24
+ allow(subscriber).to receive(:create).and_raise('anything')
25
+
26
+ allow(channel).to receive(:nack)
27
+
28
+ expect(MultipleMan).to receive(:error).with(kind_of(MultipleMan::ConsumerError), kind_of(Hash))
29
+ subject = described_class.new(subscription:subscriber, queue: queue, topic: 'some-topic')
30
+ subject.listen
31
+ end
32
+
33
+ end
@@ -0,0 +1,22 @@
1
+ require 'spec_helper'
2
+
3
+ describe MultipleMan::Subscriber do
4
+ let(:mock_class) do
5
+ Class.new do
6
+
7
+ include MultipleMan::Listener
8
+ end
9
+ end
10
+
11
+ describe "listen_to" do
12
+ it "should register itself" do
13
+ MultipleMan.configuration.should_receive(:register_listener).with(instance_of(mock_class))
14
+ mock_class.listen_to "Model"
15
+ end
16
+
17
+ it "should have a routing key for what it's listening to" do
18
+ listener = mock_class.listen_to "Model"
19
+ expect(listener.routing_key).to eq("multiple_man.Model.#")
20
+ end
21
+ end
22
+ end
@@ -7,7 +7,7 @@ describe MultipleMan::Subscriber do
7
7
 
8
8
  describe "subscribe" do
9
9
  it "should register itself" do
10
- MultipleMan::Subscribers::Registry.should_receive(:register).with(instance_of(MultipleMan::Subscribers::ModelSubscriber))
10
+ expect(MultipleMan.configuration).to receive(:register_listener).with(instance_of(MultipleMan::Subscribers::ModelSubscriber))
11
11
  MockClass.subscribe fields: [:foo, :bar]
12
12
  end
13
13
  end
@@ -8,13 +8,6 @@ describe MultipleMan::Subscribers::Base do
8
8
  described_class.new(MockClass).routing_key.should =~ /\.MockClass\.\#$/
9
9
  end
10
10
 
11
- specify "queue name should be the app name + class" do
12
- MultipleMan.configure do |config|
13
- config.app_name = "test"
14
- end
15
- described_class.new(MockClass).queue_name.should =~ /\.test\.MockClass$/
16
- end
17
-
18
11
  specify "it should be alright to use a string for a class name" do
19
12
  described_class.new("MockClass").routing_key.should =~ /\.MockClass\.\#$/
20
13
  end
@@ -1,11 +1,12 @@
1
1
  require 'spec_helper'
2
2
 
3
- describe MultipleMan::Subscribers::Registry do
4
- describe "register" do
3
+ describe MultipleMan::Subscribers::Registry do
4
+ describe "register" do
5
5
  it "should add a subscriber" do
6
6
  subscription = double(:subscriber)
7
- described_class.register(subscription)
8
- described_class.subscriptions[0].should == subscription
7
+
8
+ subject.register(subscription)
9
+ subject.subscriptions[0].should == subscription
9
10
  end
10
11
  end
11
- end
12
+ end
metadata CHANGED
@@ -1,15 +1,43 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: multiple_man
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.1
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ryan Brunner
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-09-18 00:00:00.000000000 Z
11
+ date: 2016-06-23 00:00:00.000000000 Z
12
12
  dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bunny
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '1.2'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '1.2'
27
+ - !ruby/object:Gem::Dependency
28
+ name: activesupport
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '3.0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '3.0'
13
41
  - !ruby/object:Gem::Dependency
14
42
  name: bundler
15
43
  requirement: !ruby/object:Gem::Requirement
@@ -108,34 +136,6 @@ dependencies:
108
136
  - - ">="
109
137
  - !ruby/object:Gem::Version
110
138
  version: '0'
111
- - !ruby/object:Gem::Dependency
112
- name: bunny
113
- requirement: !ruby/object:Gem::Requirement
114
- requirements:
115
- - - ">="
116
- - !ruby/object:Gem::Version
117
- version: '1.2'
118
- type: :runtime
119
- prerelease: false
120
- version_requirements: !ruby/object:Gem::Requirement
121
- requirements:
122
- - - ">="
123
- - !ruby/object:Gem::Version
124
- version: '1.2'
125
- - !ruby/object:Gem::Dependency
126
- name: activesupport
127
- requirement: !ruby/object:Gem::Requirement
128
- requirements:
129
- - - ">="
130
- - !ruby/object:Gem::Version
131
- version: '3.0'
132
- type: :runtime
133
- prerelease: false
134
- version_requirements: !ruby/object:Gem::Requirement
135
- requirements:
136
- - - ">="
137
- - !ruby/object:Gem::Version
138
- version: '3.0'
139
139
  description: MultipleMan syncs changes to ActiveRecord models via AMQP
140
140
  email:
141
141
  - ryan@influitive.com
@@ -157,9 +157,10 @@ files:
157
157
  - lib/multiple_man/channel_maintenance/reaper.rb
158
158
  - lib/multiple_man/configuration.rb
159
159
  - lib/multiple_man/connection.rb
160
+ - lib/multiple_man/consumers/general.rb
161
+ - lib/multiple_man/consumers/seed.rb
162
+ - lib/multiple_man/consumers/transitional.rb
160
163
  - lib/multiple_man/identity.rb
161
- - lib/multiple_man/listeners/listener.rb
162
- - lib/multiple_man/listeners/seeder_listener.rb
163
164
  - lib/multiple_man/mixins/listener.rb
164
165
  - lib/multiple_man/mixins/publisher.rb
165
166
  - lib/multiple_man/mixins/subscriber.rb
@@ -177,22 +178,24 @@ files:
177
178
  - multiple_man.gemspec
178
179
  - spec/attribute_extractor_spec.rb
179
180
  - spec/connection_spec.rb
181
+ - spec/consumers/general_spec.rb
182
+ - spec/consumers/seed_spec.rb
183
+ - spec/consumers/transitional_spec.rb
180
184
  - spec/identity_spec.rb
181
185
  - spec/integration/ephermal_model_spec.rb
182
- - spec/listeners/listener_spec.rb
183
- - spec/listeners/seeder_listener_spec.rb
184
186
  - spec/logger_spec.rb
187
+ - spec/mixins/listener_spec.rb
188
+ - spec/mixins/publisher_spec.rb
189
+ - spec/mixins/subscriber_spec.rb
185
190
  - spec/model_populator_spec.rb
186
191
  - spec/model_publisher_spec.rb
187
192
  - spec/payload_generator_spec.rb
188
- - spec/publisher_spec.rb
189
193
  - spec/routing_key_spec.rb
190
194
  - spec/spec_helper.rb
191
- - spec/subscriber_spec.rb
192
195
  - spec/subscribers/base_spec.rb
193
196
  - spec/subscribers/model_subscriber_spec.rb
194
197
  - spec/subscribers/registry_spec.rb
195
- homepage: ''
198
+ homepage: http://github.com/influitive/multiple_man
196
199
  licenses:
197
200
  - MIT
198
201
  metadata: {}
@@ -204,7 +207,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
204
207
  requirements:
205
208
  - - ">="
206
209
  - !ruby/object:Gem::Version
207
- version: '0'
210
+ version: '2.1'
208
211
  required_rubygems_version: !ruby/object:Gem::Requirement
209
212
  requirements:
210
213
  - - ">="
@@ -212,25 +215,27 @@ required_rubygems_version: !ruby/object:Gem::Requirement
212
215
  version: '0'
213
216
  requirements: []
214
217
  rubyforge_project:
215
- rubygems_version: 2.4.5
218
+ rubygems_version: 2.4.5.1
216
219
  signing_key:
217
220
  specification_version: 4
218
221
  summary: MultipleMan syncs changes to ActiveRecord models via AMQP
219
222
  test_files:
220
223
  - spec/attribute_extractor_spec.rb
221
224
  - spec/connection_spec.rb
225
+ - spec/consumers/general_spec.rb
226
+ - spec/consumers/seed_spec.rb
227
+ - spec/consumers/transitional_spec.rb
222
228
  - spec/identity_spec.rb
223
229
  - spec/integration/ephermal_model_spec.rb
224
- - spec/listeners/listener_spec.rb
225
- - spec/listeners/seeder_listener_spec.rb
226
230
  - spec/logger_spec.rb
231
+ - spec/mixins/listener_spec.rb
232
+ - spec/mixins/publisher_spec.rb
233
+ - spec/mixins/subscriber_spec.rb
227
234
  - spec/model_populator_spec.rb
228
235
  - spec/model_publisher_spec.rb
229
236
  - spec/payload_generator_spec.rb
230
- - spec/publisher_spec.rb
231
237
  - spec/routing_key_spec.rb
232
238
  - spec/spec_helper.rb
233
- - spec/subscriber_spec.rb
234
239
  - spec/subscribers/base_spec.rb
235
240
  - spec/subscribers/model_subscriber_spec.rb
236
241
  - spec/subscribers/registry_spec.rb
@@ -1,76 +0,0 @@
1
- require 'json'
2
- require 'active_support/core_ext/hash'
3
-
4
- module MultipleMan::Listeners
5
- class Listener
6
-
7
- class << self
8
- def start
9
- MultipleMan.logger.debug "Starting listeners."
10
-
11
- MultipleMan::Subscribers::Registry.subscriptions.each do |subscription|
12
- new(subscription).listen
13
- end
14
- end
15
- end
16
-
17
- delegate :queue_name, to: :subscription
18
-
19
- def initialize(subscription)
20
- self.subscription = subscription
21
- self.init_connection
22
- end
23
-
24
- def init_connection
25
- connection = Bunny.new(MultipleMan.configuration.connection)
26
- connection.start
27
- channel = connection.create_channel(nil, MultipleMan.configuration.worker_concurrency)
28
- channel.prefetch(100)
29
- self.connection = MultipleMan::Connection.new(channel)
30
- end
31
-
32
- attr_accessor :subscription, :connection
33
-
34
- def listen
35
-
36
- MultipleMan.logger.info "Listening for #{subscription.klass} with routing key #{routing_key}."
37
- queue.bind(connection.topic, routing_key: routing_key).subscribe(ack: true) do |delivery_info, meta_data, payload|
38
- process_message(delivery_info, payload)
39
- end
40
- end
41
-
42
- def process_message(delivery_info, payload)
43
- MultipleMan.logger.info "Processing message for #{delivery_info.routing_key}."
44
- begin
45
- payload = JSON.parse(payload).with_indifferent_access
46
- subscription.send(operation(delivery_info, payload), payload)
47
- rescue Exception => ex
48
- handle_error(ex, delivery_info)
49
- else
50
- MultipleMan.logger.debug " Successfully processed!"
51
- queue.channel.acknowledge(delivery_info.delivery_tag, false)
52
- end
53
- end
54
-
55
- def handle_error(ex, delivery_info)
56
- MultipleMan.logger.error " Error - #{ex.message}\n\n#{ex.backtrace}"
57
- MultipleMan.error(ex, reraise: false)
58
-
59
- # Requeue the message
60
- queue.channel.nack(delivery_info.delivery_tag)
61
- end
62
-
63
- def operation(delivery_info, payload)
64
- payload['operation'] || delivery_info.routing_key.split(".").last
65
- end
66
-
67
- def queue
68
- connection.queue(queue_name, durable: true, auto_delete: false)
69
- end
70
-
71
- def routing_key
72
- subscription.routing_key
73
- end
74
-
75
- end
76
- end
@@ -1,16 +0,0 @@
1
- module MultipleMan::Listeners
2
- class SeederListener < Listener
3
- def routing_key
4
- subscription.routing_key(:seed)
5
- end
6
-
7
- # seeds should only ever be a create
8
- def operation(delivery_info, payload)
9
- "create"
10
- end
11
-
12
- def queue
13
- connection.queue(subscription.queue_name + ".seed", auto_delete: true)
14
- end
15
- end
16
- end
@@ -1,75 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe MultipleMan::Listeners::Listener do
4
- class MockClass1; end
5
- class MockClass2; end
6
-
7
- before { MultipleMan::Connection.stub(:connection).and_return(double(Bunny).as_null_object)}
8
-
9
- describe "start" do
10
- it "should listen to each subscription" do
11
- MultipleMan::Subscribers::Registry.stub(:subscriptions).and_return([
12
- mock1 = double(MultipleMan::Subscribers::ModelSubscriber, klass: MockClass1),
13
- mock2 = double(MultipleMan::Subscribers::ModelSubscriber, klass: MockClass2)
14
- ])
15
-
16
- mock_listener = double(described_class)
17
- described_class.should_receive(:new).twice.and_return(mock_listener)
18
-
19
- # Would actually be two seperate objects in reality, this is for
20
- # ease of stubbing.
21
- mock_listener.should_receive(:listen).twice
22
-
23
- described_class.start
24
- end
25
- end
26
-
27
- describe "listen" do
28
- let(:connection_stub) { double(MultipleMan::Connection, queue: queue_stub, topic: 'app') }
29
- let(:queue_stub) { double(Bunny::Queue, bind: bind_stub) }
30
- let(:bind_stub) { double(:bind, subscribe: nil)}
31
-
32
- before { MultipleMan::Connection.stub(:new).and_return(connection_stub) }
33
-
34
- it "should listen to the right topic, and for all updates to a model" do
35
- listener = described_class.new(double(MultipleMan::Subscribers::ModelSubscriber, klass: MockClass1, routing_key: "MockClass1.#", queue_name: "MockClass1"))
36
- queue_stub.should_receive(:bind).with('app', routing_key: "MockClass1.#")
37
- listener.listen
38
- end
39
- end
40
-
41
- specify "process_message should send the correct data" do
42
- connection_stub = double(MultipleMan::Connection).as_null_object
43
- MultipleMan::Connection.stub(:new).and_return(connection_stub)
44
- subscriber = double(MultipleMan::Subscribers::ModelSubscriber, klass: MockClass1, routing_key: "MockClass1.#").as_null_object
45
- listener = described_class.new(subscriber)
46
-
47
- connection_stub.should_receive(:acknowledge)
48
- subscriber.should_receive(:create).with({"a" => 1, "b" => 2})
49
- listener.process_message(OpenStruct.new(routing_key: "app.MockClass1.create"), '{"a":1,"b":2}')
50
- end
51
-
52
- specify "process_message should use the payload to determine the operation if it's available" do
53
- connection_stub = double(MultipleMan::Connection).as_null_object
54
- MultipleMan::Connection.stub(:new).and_return(connection_stub)
55
- subscriber = double(MultipleMan::Subscribers::ModelSubscriber, klass: MockClass1, routing_key: "MockClass1.#").as_null_object
56
- listener = described_class.new(subscriber)
57
-
58
- connection_stub.should_receive(:acknowledge)
59
- subscriber.should_receive(:create)
60
- listener.process_message(OpenStruct.new(routing_key: "some random routing key"), '{"operation":"create","data":{"a":1,"b":2}}')
61
- end
62
-
63
- it "should nack on failure" do
64
- connection_stub = double(MultipleMan::Connection).as_null_object
65
- MultipleMan::Connection.stub(:new).and_return(connection_stub)
66
- subscriber = double(MultipleMan::Subscribers::ModelSubscriber, klass: MockClass1, routing_key: "MockClass1.#").as_null_object
67
- listener = described_class.new(subscriber)
68
-
69
- connection_stub.should_receive(:nack)
70
- MultipleMan.should_receive(:error)
71
- subscriber.should_receive(:create).with({"a" => 1, "b" => 2}).and_raise("fail!")
72
-
73
- listener.process_message(OpenStruct.new(routing_key: "app.MockClass1.create"), '{"a":1,"b":2}')
74
- end
75
- end
@@ -1,22 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe MultipleMan::Listeners::SeederListener do
4
- class MockClass1; end
5
-
6
- before { MultipleMan::Connection.stub(:connection).and_return(double(Bunny).as_null_object)}
7
- let(:connection_stub) { double(MultipleMan::Connection, queue: queue_stub, topic: 'app') }
8
- let(:queue_stub) { double(Bunny::Queue, bind: bind_stub) }
9
- let(:bind_stub) { double(:bind, subscribe: nil)}
10
-
11
- before { MultipleMan::Connection.stub(:new).and_return(connection_stub) }
12
-
13
- it 'listens to seed events' do
14
- listener = described_class.new(double(MultipleMan::Subscribers::ModelSubscriber,
15
- klass: MockClass1,
16
- routing_key: "seed.MockClass1",
17
- queue_name: "MockClass1"))
18
-
19
- queue_stub.should_receive(:bind).with('app', routing_key: "seed.MockClass1")
20
- listener.listen
21
- end
22
- end