action_subscriber 1.0.3-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +21 -0
- data/.rspec +2 -0
- data/Gemfile +5 -0
- data/LICENSE +20 -0
- data/LICENSE.txt +22 -0
- data/README.md +122 -0
- data/Rakefile +8 -0
- data/action_subscriber.gemspec +38 -0
- data/examples/at_least_once.rb +17 -0
- data/examples/at_most_once.rb +15 -0
- data/examples/basic_subscriber.rb +30 -0
- data/examples/message_acknowledgement.rb +19 -0
- data/lib/action_subscriber.rb +93 -0
- data/lib/action_subscriber/base.rb +83 -0
- data/lib/action_subscriber/bunny/subscriber.rb +57 -0
- data/lib/action_subscriber/configuration.rb +68 -0
- data/lib/action_subscriber/default_routing.rb +26 -0
- data/lib/action_subscriber/dsl.rb +83 -0
- data/lib/action_subscriber/march_hare/subscriber.rb +60 -0
- data/lib/action_subscriber/middleware.rb +18 -0
- data/lib/action_subscriber/middleware/active_record/connection_management.rb +17 -0
- data/lib/action_subscriber/middleware/active_record/query_cache.rb +29 -0
- data/lib/action_subscriber/middleware/decoder.rb +33 -0
- data/lib/action_subscriber/middleware/env.rb +65 -0
- data/lib/action_subscriber/middleware/error_handler.rb +16 -0
- data/lib/action_subscriber/middleware/router.rb +17 -0
- data/lib/action_subscriber/middleware/runner.rb +16 -0
- data/lib/action_subscriber/rabbit_connection.rb +40 -0
- data/lib/action_subscriber/railtie.rb +13 -0
- data/lib/action_subscriber/rspec.rb +91 -0
- data/lib/action_subscriber/subscribable.rb +118 -0
- data/lib/action_subscriber/threadpool.rb +29 -0
- data/lib/action_subscriber/version.rb +3 -0
- data/spec/integration/basic_subscriber_spec.rb +42 -0
- data/spec/lib/action_subscriber/base_spec.rb +18 -0
- data/spec/lib/action_subscriber/configuration_spec.rb +32 -0
- data/spec/lib/action_subscriber/dsl_spec.rb +143 -0
- data/spec/lib/action_subscriber/middleware/active_record/connection_management_spec.rb +17 -0
- data/spec/lib/action_subscriber/middleware/active_record/query_cache_spec.rb +49 -0
- data/spec/lib/action_subscriber/middleware/decoder_spec.rb +31 -0
- data/spec/lib/action_subscriber/middleware/env_spec.rb +60 -0
- data/spec/lib/action_subscriber/middleware/error_handler_spec.rb +35 -0
- data/spec/lib/action_subscriber/middleware/router_spec.rb +24 -0
- data/spec/lib/action_subscriber/middleware/runner_spec.rb +6 -0
- data/spec/lib/action_subscriber/subscribable_spec.rb +128 -0
- data/spec/lib/action_subscriber/threadpool_spec.rb +35 -0
- data/spec/spec_helper.rb +26 -0
- data/spec/support/user_subscriber.rb +6 -0
- metadata +255 -0
@@ -0,0 +1,57 @@
|
|
1
|
+
module ActionSubscriber
|
2
|
+
module Bunny
|
3
|
+
module Subscriber
|
4
|
+
def auto_pop!
|
5
|
+
# Because threadpools can be large we want to cap the number
|
6
|
+
# of times we will pop each time we poll the broker
|
7
|
+
times_to_pop = [::ActionSubscriber::Threadpool.ready_size, ::ActionSubscriber.config.times_to_pop].min
|
8
|
+
times_to_pop.times do
|
9
|
+
queues.each do |queue|
|
10
|
+
delivery_info, properties, encoded_payload = queue.pop(queue_subscription_options)
|
11
|
+
next unless encoded_payload # empty queue
|
12
|
+
::ActiveSupport::Notifications.instrument "popped_event.action_subscriber", :payload_size => encoded_payload.bytesize, :queue => queue.name
|
13
|
+
properties = {
|
14
|
+
:channel => queue.channel,
|
15
|
+
:content_type => properties[:content_type],
|
16
|
+
:delivery_tag => delivery_info.delivery_tag,
|
17
|
+
:exchange => delivery_info.exchange,
|
18
|
+
:message_id => nil,
|
19
|
+
:routing_key => delivery_info.routing_key,
|
20
|
+
}
|
21
|
+
env = ::ActionSubscriber::Middleware::Env.new(self, encoded_payload, properties)
|
22
|
+
enqueue_env(env)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def auto_subscribe!
|
28
|
+
queues.each do |queue|
|
29
|
+
queue.channel.prefetch(::ActionSubscriber.config.prefetch) if acknowledge_messages?
|
30
|
+
queue.subscribe(queue_subscription_options) do |delivery_info, properties, encoded_payload|
|
31
|
+
::ActiveSupport::Notifications.instrument "received_event.action_subscriber", :payload_size => encoded_payload.bytesize, :queue => queue.name
|
32
|
+
properties = {
|
33
|
+
:channel => queue.channel,
|
34
|
+
:content_type => properties.content_type,
|
35
|
+
:delivery_tag => delivery_info.delivery_tag,
|
36
|
+
:exchange => delivery_info.exchange,
|
37
|
+
:message_id => properties.message_id,
|
38
|
+
:routing_key => delivery_info.routing_key,
|
39
|
+
}
|
40
|
+
env = ::ActionSubscriber::Middleware::Env.new(self, encoded_payload, properties)
|
41
|
+
enqueue_env(env)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
|
48
|
+
def enqueue_env(env)
|
49
|
+
::ActionSubscriber::Threadpool.pool.async(env) do |env|
|
50
|
+
::ActiveSupport::Notifications.instrument "process_event.action_subscriber", :subscriber => env.subscriber.to_s, :routing_key => env.routing_key do
|
51
|
+
::ActionSubscriber.config.middleware.call(env)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
module ActionSubscriber
|
2
|
+
class Configuration
|
3
|
+
attr_accessor :allow_low_priority_methods,
|
4
|
+
:decoder,
|
5
|
+
:default_exchange,
|
6
|
+
:error_handler,
|
7
|
+
:heartbeat,
|
8
|
+
:timeout,
|
9
|
+
:host,
|
10
|
+
:hosts,
|
11
|
+
:port,
|
12
|
+
:prefetch,
|
13
|
+
:times_to_pop,
|
14
|
+
:threadpool_size
|
15
|
+
|
16
|
+
def initialize
|
17
|
+
self.allow_low_priority_methods = false
|
18
|
+
self.decoder = {
|
19
|
+
'application/json' => lambda { |payload| JSON.parse(payload) },
|
20
|
+
'text/plain' => lambda { |payload| payload.dup }
|
21
|
+
}
|
22
|
+
self.default_exchange = "events"
|
23
|
+
self.error_handler = lambda { |error, env_hash| raise }
|
24
|
+
self.heartbeat = 5
|
25
|
+
self.timeout = 1
|
26
|
+
self.host = 'localhost'
|
27
|
+
self.hosts = []
|
28
|
+
self.port = 5672
|
29
|
+
self.prefetch = 200
|
30
|
+
self.times_to_pop = 8
|
31
|
+
self.threadpool_size = 8
|
32
|
+
end
|
33
|
+
|
34
|
+
##
|
35
|
+
# Instance Methods
|
36
|
+
#
|
37
|
+
def add_decoder(decoders)
|
38
|
+
decoders.each_pair do |content_type, decoder|
|
39
|
+
unless decoder.arity == 1
|
40
|
+
raise "ActionSubscriber decoders must have an arity of 1. The #{content_type} decoder was given with arity of #{decoder.arity}."
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
self.decoder.merge!(decoders)
|
45
|
+
end
|
46
|
+
|
47
|
+
def hosts
|
48
|
+
return @hosts if @hosts.size > 0
|
49
|
+
[ host ]
|
50
|
+
end
|
51
|
+
|
52
|
+
def middleware
|
53
|
+
@middleware ||= Middleware.initialize_stack
|
54
|
+
end
|
55
|
+
|
56
|
+
def inspect
|
57
|
+
inspection_string = <<-INSPECT.strip_heredoc
|
58
|
+
Rabbit Host: #{host}
|
59
|
+
Rabbit Port: #{port}
|
60
|
+
Threadpool Size: #{threadpool_size}
|
61
|
+
Low Priority Subscriber: #{allow_low_priority_methods}
|
62
|
+
Decoders:
|
63
|
+
INSPECT
|
64
|
+
decoder.each_key { |key| inspection_string << " --#{key}\n" }
|
65
|
+
return inspection_string
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module ActionSubscriber
|
2
|
+
module DefaultRouting
|
3
|
+
def queues
|
4
|
+
@_queues ||= []
|
5
|
+
end
|
6
|
+
|
7
|
+
def setup_queue!(method_name, exchange_name)
|
8
|
+
queue_name = queue_name_for_method(method_name)
|
9
|
+
routing_key_name = routing_key_name_for_method(method_name)
|
10
|
+
|
11
|
+
channel = ::ActionSubscriber::RabbitConnection.connection.create_channel
|
12
|
+
exchange = channel.topic(exchange_name)
|
13
|
+
queue = channel.queue(queue_name)
|
14
|
+
queue.bind(exchange, :routing_key => routing_key_name)
|
15
|
+
return queue
|
16
|
+
end
|
17
|
+
|
18
|
+
def setup_queues!
|
19
|
+
exchange_names.each do |exchange_name|
|
20
|
+
subscribable_methods.each do |method_name|
|
21
|
+
queues << setup_queue!(method_name, exchange_name)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
module ActionSubscriber
|
2
|
+
module DSL
|
3
|
+
def at_least_once!
|
4
|
+
@_acknowledge_messages = true
|
5
|
+
@_acknowledge_messages_after_processing = true
|
6
|
+
end
|
7
|
+
|
8
|
+
def at_most_once!
|
9
|
+
@_acknowledge_messages = true
|
10
|
+
@_acknowledge_messages_before_processing = true
|
11
|
+
end
|
12
|
+
|
13
|
+
def acknowledge_messages?
|
14
|
+
!!@_acknowledge_messages
|
15
|
+
end
|
16
|
+
|
17
|
+
def acknowledge_messages_after_processing?
|
18
|
+
!!@_acknowledge_messages_after_processing
|
19
|
+
end
|
20
|
+
|
21
|
+
def acknowledge_messages_before_processing?
|
22
|
+
!!@_acknowledge_messages_before_processing
|
23
|
+
end
|
24
|
+
|
25
|
+
# Explicitly set the name of the exchange
|
26
|
+
#
|
27
|
+
def exchange_names(*names)
|
28
|
+
@_exchange_names ||= []
|
29
|
+
@_exchange_names += names.flatten.map(&:to_s)
|
30
|
+
|
31
|
+
if @_exchange_names.empty?
|
32
|
+
return [ ::ActionSubscriber.config.default_exchange ]
|
33
|
+
else
|
34
|
+
return @_exchange_names.compact.uniq
|
35
|
+
end
|
36
|
+
end
|
37
|
+
alias_method :exchange, :exchange_names
|
38
|
+
|
39
|
+
def manual_acknowledgement!
|
40
|
+
@_acknowledge_messages = true
|
41
|
+
end
|
42
|
+
|
43
|
+
def no_acknowledgement!
|
44
|
+
@_acknowledge_messages = false
|
45
|
+
end
|
46
|
+
|
47
|
+
# Explicitly set the name of a queue for the given method route
|
48
|
+
#
|
49
|
+
# Ex.
|
50
|
+
# queue_for :created, "derp.derp"
|
51
|
+
# queue_for :updated, "foo.bar"
|
52
|
+
#
|
53
|
+
def queue_for(method, queue_name)
|
54
|
+
@_queue_names ||= {}
|
55
|
+
@_queue_names[method] = queue_name
|
56
|
+
end
|
57
|
+
|
58
|
+
def queue_names
|
59
|
+
@_queue_names ||= {}
|
60
|
+
end
|
61
|
+
|
62
|
+
def queue_subscription_options
|
63
|
+
@_queue_subscription_options ||= { :manual_ack => acknowledge_messages? }
|
64
|
+
end
|
65
|
+
|
66
|
+
def remote_application_name(name = nil)
|
67
|
+
@_remote_application_name = name if name
|
68
|
+
@_remote_application_name
|
69
|
+
end
|
70
|
+
alias_method :publisher, :remote_application_name
|
71
|
+
|
72
|
+
# Explicitly set the whole routing key to use for a given method route.
|
73
|
+
#
|
74
|
+
def routing_key_for(method, routing_key_name)
|
75
|
+
@_routing_key_names ||= {}
|
76
|
+
@_routing_key_names[method] = routing_key_name
|
77
|
+
end
|
78
|
+
|
79
|
+
def routing_key_names
|
80
|
+
@_routing_key_names ||= {}
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
module ActionSubscriber
|
2
|
+
module MarchHare
|
3
|
+
module Subscriber
|
4
|
+
def auto_pop!
|
5
|
+
# Because threadpools can be large we want to cap the number
|
6
|
+
# of times we will pop each time we poll the broker
|
7
|
+
times_to_pop = [::ActionSubscriber::Threadpool.ready_size, ::ActionSubscriber.config.times_to_pop].min
|
8
|
+
times_to_pop.times do
|
9
|
+
queues.each do |queue|
|
10
|
+
header, encoded_payload = queue.pop(queue_subscription_options)
|
11
|
+
next unless encoded_payload
|
12
|
+
::ActiveSupport::Notifications.instrument "popped_event.action_subscriber", :payload_size => encoded_payload.bytesize, :queue => queue.name
|
13
|
+
properties = {
|
14
|
+
:channel => queue.channel,
|
15
|
+
:content_type => header.content_type,
|
16
|
+
:delivery_tag => header.delivery_tag,
|
17
|
+
:exchange => header.exchange,
|
18
|
+
:message_id => header.message_id,
|
19
|
+
:routing_key => header.routing_key,
|
20
|
+
}
|
21
|
+
env = ::ActionSubscriber::Middleware::Env.new(self, encoded_payload, properties)
|
22
|
+
enqueue_env(env)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
rescue ::MarchHare::ChannelAlreadyClosed => e
|
27
|
+
# The connection has gone down, we can just try again on the next pop
|
28
|
+
end
|
29
|
+
|
30
|
+
def auto_subscribe!
|
31
|
+
queues.each do |queue|
|
32
|
+
queue.channel.prefetch = ::ActionSubscriber.config.prefetch if acknowledge_messages?
|
33
|
+
queue.subscribe(queue_subscription_options) do |header, encoded_payload|
|
34
|
+
::ActiveSupport::Notifications.instrument "received_event.action_subscriber", :payload_size => encoded_payload.bytesize, :queue => queue.name
|
35
|
+
properties = {
|
36
|
+
:channel => queue.channel,
|
37
|
+
:content_type => header.content_type,
|
38
|
+
:delivery_tag => header.delivery_tag,
|
39
|
+
:exchange => header.exchange,
|
40
|
+
:message_id => header.message_id,
|
41
|
+
:routing_key => header.routing_key,
|
42
|
+
}
|
43
|
+
env = ::ActionSubscriber::Middleware::Env.new(self, encoded_payload, properties)
|
44
|
+
enqueue_env(env)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
private
|
50
|
+
|
51
|
+
def enqueue_env(env)
|
52
|
+
::ActionSubscriber::Threadpool.pool.async(env) do |env|
|
53
|
+
::ActiveSupport::Notifications.instrument "process_event.action_subscriber", :subscriber => env.subscriber.to_s, :routing_key => env.routing_key do
|
54
|
+
::ActionSubscriber.config.middleware.call(env)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
require "action_subscriber/middleware/decoder"
|
2
|
+
require "action_subscriber/middleware/env"
|
3
|
+
require "action_subscriber/middleware/error_handler"
|
4
|
+
require "action_subscriber/middleware/router"
|
5
|
+
require "action_subscriber/middleware/runner"
|
6
|
+
|
7
|
+
module ActionSubscriber
|
8
|
+
module Middleware
|
9
|
+
def self.initialize_stack
|
10
|
+
builder = ::Middleware::Builder.new(:runner_class => ::ActionSubscriber::Middleware::Runner)
|
11
|
+
|
12
|
+
builder.use ErrorHandler
|
13
|
+
builder.use Decoder
|
14
|
+
|
15
|
+
builder
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module ActionSubscriber
|
2
|
+
module Middleware
|
3
|
+
module ActiveRecord
|
4
|
+
class ConnectionManagement
|
5
|
+
def initialize(app)
|
6
|
+
@app = app
|
7
|
+
end
|
8
|
+
|
9
|
+
def call(env)
|
10
|
+
@app.call(env)
|
11
|
+
ensure
|
12
|
+
::ActiveRecord::Base.clear_active_connections!
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module ActionSubscriber
|
2
|
+
module Middleware
|
3
|
+
module ActiveRecord
|
4
|
+
class QueryCache
|
5
|
+
def initialize(app)
|
6
|
+
@app = app
|
7
|
+
end
|
8
|
+
|
9
|
+
def call(env)
|
10
|
+
enabled = ::ActiveRecord::Base.connection.query_cache_enabled
|
11
|
+
connection_id = ::ActiveRecord::Base.connection_id
|
12
|
+
::ActiveRecord::Base.connection.enable_query_cache!
|
13
|
+
|
14
|
+
@app.call(env)
|
15
|
+
ensure
|
16
|
+
restore_query_cache_settings(connection_id, enabled)
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
|
21
|
+
def restore_query_cache_settings(connection_id, enabled)
|
22
|
+
::ActiveRecord::Base.connection_id = connection_id
|
23
|
+
::ActiveRecord::Base.connection.clear_query_cache
|
24
|
+
::ActiveRecord::Base.connection.disable_query_cache! unless enabled
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module ActionSubscriber
|
2
|
+
module Middleware
|
3
|
+
class Decoder
|
4
|
+
attr_reader :env
|
5
|
+
|
6
|
+
def initialize(app)
|
7
|
+
@app = app
|
8
|
+
end
|
9
|
+
|
10
|
+
def call(env)
|
11
|
+
@env = env
|
12
|
+
|
13
|
+
env.payload = decoder? ? decoder.call(encoded_payload) : encoded_payload.dup
|
14
|
+
|
15
|
+
@app.call(env)
|
16
|
+
end
|
17
|
+
|
18
|
+
private
|
19
|
+
|
20
|
+
def decoder
|
21
|
+
ActionSubscriber.config.decoder[env.content_type]
|
22
|
+
end
|
23
|
+
|
24
|
+
def decoder?
|
25
|
+
decoder.present?
|
26
|
+
end
|
27
|
+
|
28
|
+
def encoded_payload
|
29
|
+
env.encoded_payload
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
module ActionSubscriber
|
2
|
+
module Middleware
|
3
|
+
class Env
|
4
|
+
attr_accessor :payload
|
5
|
+
|
6
|
+
attr_reader :content_type,
|
7
|
+
:encoded_payload,
|
8
|
+
:exchange,
|
9
|
+
:message_id,
|
10
|
+
:routing_key,
|
11
|
+
:subscriber
|
12
|
+
|
13
|
+
##
|
14
|
+
# @param subscriber [Class] the class that will handle this message
|
15
|
+
# @param encoded_payload [String] the payload as it was received from RabbitMQ
|
16
|
+
# @param properties [Hash] that must contain the following keys (as symbols)
|
17
|
+
# :channel => RabbitMQ channel for doing acknowledgement
|
18
|
+
# :content_type => String
|
19
|
+
# :delivery_tag => String (the message identifier to send back to rabbitmq for acknowledgement)
|
20
|
+
# :exchange => String
|
21
|
+
# :message_id => String
|
22
|
+
# :routing_key => String
|
23
|
+
|
24
|
+
def initialize(subscriber, encoded_payload, properties)
|
25
|
+
@channel = properties.fetch(:channel)
|
26
|
+
@content_type = properties.fetch(:content_type)
|
27
|
+
@delivery_tag = properties.fetch(:delivery_tag)
|
28
|
+
@encoded_payload = encoded_payload
|
29
|
+
@exchange = properties.fetch(:exchange)
|
30
|
+
@message_id = properties.fetch(:message_id)
|
31
|
+
@routing_key = properties.fetch(:routing_key)
|
32
|
+
@subscriber = subscriber
|
33
|
+
end
|
34
|
+
|
35
|
+
def acknowledge
|
36
|
+
acknowledge_multiple_messages = false
|
37
|
+
@channel.ack(@delivery_tag, acknowledge_multiple_messages)
|
38
|
+
end
|
39
|
+
|
40
|
+
# Return the last element of the routing key to indicate which action
|
41
|
+
# to route the payload to
|
42
|
+
#
|
43
|
+
def action
|
44
|
+
routing_key.split('.').last.to_s
|
45
|
+
end
|
46
|
+
|
47
|
+
def reject
|
48
|
+
requeue_message = true
|
49
|
+
@channel.reject(@delivery_tag, requeue_message)
|
50
|
+
end
|
51
|
+
|
52
|
+
def to_hash
|
53
|
+
{
|
54
|
+
:action => action,
|
55
|
+
:content_type => content_type,
|
56
|
+
:exchange => exchange,
|
57
|
+
:routing_key => routing_key,
|
58
|
+
:payload => payload
|
59
|
+
}
|
60
|
+
end
|
61
|
+
alias_method :to_h, :to_hash
|
62
|
+
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|