msgr 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/msgr.rb +0 -12
- data/lib/msgr/client.rb +74 -81
- data/lib/msgr/connection.rb +82 -82
- data/lib/msgr/dispatcher.rb +23 -6
- data/lib/msgr/logging.rb +2 -2
- data/lib/msgr/message.rb +23 -4
- data/lib/msgr/railtie.rb +0 -21
- data/lib/msgr/routes.rb +14 -2
- data/lib/msgr/version.rb +2 -2
- data/msgr.gemspec +2 -1
- data/spec/msgr/msgr_spec.rb +4 -12
- data/spec/msgr/spec_helper.rb +0 -6
- metadata +17 -10
- data/lib/msgr/binding.rb +0 -61
- data/lib/msgr/message/acknowledge.rb +0 -30
- data/lib/msgr/pool.rb +0 -165
- data/spec/msgr/msgr/client_spec.rb +0 -58
- data/spec/msgr/msgr/pool_spec.rb +0 -68
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ddae04ef1c28f7eb451ee8988da5949ab0c97d6e
|
4
|
+
data.tar.gz: 12490c056d64088711f79d9ddc2d8c227015efc0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 984436fe6ac5ce0783c6a8fabd2039c1d119dcc7c62cb749493818c44c3f6c6c2c6e350efd0555933012b74637e61b140d3dfbf8cf5c7981d2912248eb580e62
|
7
|
+
data.tar.gz: 1fc475e7d83893f7129e4fd6b42736d647cac8b74cd03e93fc1167a331398d44c07ed3964e139e708ff66a19a22e89b17271c066c025b937a3eff07ce1bdd747
|
data/lib/msgr.rb
CHANGED
@@ -1,22 +1,18 @@
|
|
1
1
|
require 'msgr/version'
|
2
|
-
require 'celluloid'
|
3
2
|
require 'active_support'
|
4
3
|
require 'active_support/core_ext/object/blank'
|
5
4
|
require 'active_support/core_ext/module/delegation'
|
6
5
|
require 'active_support/core_ext/string/inflections'
|
7
6
|
require 'active_support/core_ext/hash/reverse_merge'
|
8
7
|
require 'active_support/core_ext/hash/keys'
|
9
|
-
require 'json'
|
10
8
|
|
11
9
|
require 'msgr/logging'
|
12
|
-
require 'msgr/binding'
|
13
10
|
require 'msgr/client'
|
14
11
|
require 'msgr/connection'
|
15
12
|
require 'msgr/consumer'
|
16
13
|
require 'msgr/dispatcher'
|
17
14
|
require 'msgr/errors'
|
18
15
|
require 'msgr/message'
|
19
|
-
require 'msgr/pool'
|
20
16
|
require 'msgr/route'
|
21
17
|
require 'msgr/routes'
|
22
18
|
|
@@ -49,16 +45,8 @@ module Msgr
|
|
49
45
|
@logger = logger
|
50
46
|
end
|
51
47
|
|
52
|
-
def after_load(&block)
|
53
|
-
@after_load_callbacks ||= []
|
54
|
-
@after_load_callbacks << block
|
55
|
-
end
|
56
|
-
|
57
48
|
def start
|
58
49
|
client.start
|
59
|
-
(@after_load_callbacks || []).each do |callback|
|
60
|
-
callback.call client
|
61
|
-
end
|
62
50
|
client
|
63
51
|
end
|
64
52
|
end
|
data/lib/msgr/client.rb
CHANGED
@@ -1,126 +1,119 @@
|
|
1
|
-
require 'bunny'
|
2
|
-
|
3
1
|
module Msgr
|
4
2
|
|
5
3
|
class Client
|
6
|
-
include Celluloid
|
7
4
|
include Logging
|
8
|
-
|
9
|
-
attr_reader :uri
|
5
|
+
attr_reader :uri, :config
|
10
6
|
|
11
7
|
def initialize(config = {})
|
12
|
-
@uri
|
8
|
+
@uri = ::URI.parse config[:uri] ? config.delete(:uri) : 'amqp://localhost/'
|
13
9
|
config[:pass] ||= @uri.password
|
14
10
|
|
15
|
-
@uri.user = config[:user]
|
16
|
-
@uri.scheme = (config[:ssl]
|
17
|
-
@uri.host = config[:host]
|
18
|
-
@uri.port = config[:port]
|
11
|
+
@uri.user = config[:user] ||= @uri.user || 'guest'
|
12
|
+
@uri.scheme = (config[:ssl] ||= @uri.scheme.to_s.downcase == 'amqps') ? 'amqps' : 'amqp'
|
13
|
+
@uri.host = config[:host] ||= @uri.host || '127.0.0.1'
|
14
|
+
@uri.port = config[:port] ||= @uri.port
|
19
15
|
@uri.path = config[:vhost] ||= @uri.path.present? ? @uri.path : '/'
|
20
|
-
config.reject! { |_,v| v.nil? }
|
16
|
+
config.reject! { |_, v| v.nil? }
|
21
17
|
|
22
18
|
@config = config
|
23
|
-
|
19
|
+
@config[:max] ||= 2
|
24
20
|
|
25
|
-
|
26
|
-
|
21
|
+
@mutex = ::Mutex.new
|
22
|
+
@routes = Routes.new
|
23
|
+
@pid ||= ::Process.pid
|
27
24
|
|
28
|
-
|
29
|
-
@routes ||= Routes.new
|
25
|
+
log(:info) { "Created new client on process ##{@pid}..." }
|
30
26
|
end
|
31
27
|
|
32
|
-
def
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
raise StandardError.new 'Client not running.' unless running?
|
38
|
-
log(:info) { 'Reload client.' }
|
39
|
-
|
40
|
-
@connection.release
|
41
|
-
@connection.terminate
|
42
|
-
|
43
|
-
log(:debug) { 'Create new connection.' }
|
44
|
-
new_connection
|
45
|
-
|
46
|
-
log(:info) { 'Client reloaded.' }
|
28
|
+
def running?
|
29
|
+
mutex.synchronize do
|
30
|
+
check_process!
|
31
|
+
connection.running?
|
32
|
+
end
|
47
33
|
end
|
48
34
|
|
49
35
|
def start
|
50
|
-
|
36
|
+
mutex.synchronize do
|
37
|
+
check_process!
|
38
|
+
return if connection.running?
|
51
39
|
|
52
|
-
|
53
|
-
launch
|
40
|
+
log(:info) { "Start on #{uri}..." }
|
54
41
|
|
55
|
-
|
42
|
+
@routes << config[:routing_file] if config[:routing_file].present?
|
43
|
+
@routes.reload
|
44
|
+
connection.bind(@routes)
|
45
|
+
end
|
56
46
|
end
|
57
47
|
|
58
|
-
def stop
|
59
|
-
|
60
|
-
|
48
|
+
def stop
|
49
|
+
mutex.synchronize do
|
50
|
+
check_process!
|
61
51
|
|
62
|
-
|
52
|
+
log(:info) { "Stop on #{uri}..." }
|
63
53
|
|
64
|
-
|
65
|
-
wait_for_graceful_shutdown opts if opts[:timeout]
|
54
|
+
return unless connection.running?
|
66
55
|
|
67
|
-
|
56
|
+
connection.release
|
57
|
+
connection.close
|
58
|
+
dispatcher.shutdown
|
68
59
|
|
69
|
-
|
70
|
-
log(:debug) { 'Delete connection.' }
|
71
|
-
@connection.delete
|
60
|
+
reset
|
72
61
|
end
|
73
|
-
@connection.terminate
|
74
|
-
@bunny.stop
|
75
|
-
|
76
|
-
log(:info) { 'Terminated.' }
|
77
62
|
end
|
78
63
|
|
79
64
|
def publish(payload, opts = {})
|
80
|
-
|
81
|
-
|
65
|
+
mutex.synchronize do
|
66
|
+
check_process!
|
67
|
+
connection.publish payload, opts
|
68
|
+
end
|
82
69
|
end
|
83
70
|
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
log(:warn) { "Could release connection within #{timeout} seconds." }
|
71
|
+
def routes
|
72
|
+
mutex.synchronize do
|
73
|
+
@routes
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def release
|
78
|
+
mutex.synchronize do
|
79
|
+
check_process!
|
80
|
+
return unless running?
|
81
|
+
|
82
|
+
connection.release
|
97
83
|
end
|
98
84
|
end
|
99
85
|
|
100
|
-
|
101
|
-
|
102
|
-
|
86
|
+
private
|
87
|
+
def mutex
|
88
|
+
@mutex
|
89
|
+
end
|
90
|
+
|
91
|
+
def check_process!
|
92
|
+
unless ::Process.pid == @pid
|
93
|
+
log(:warn) { 'Fork detected. Reset internal state...' }
|
103
94
|
|
104
|
-
|
105
|
-
|
106
|
-
|
95
|
+
reset
|
96
|
+
@pid = ::Process.pid
|
97
|
+
end
|
107
98
|
end
|
108
99
|
|
109
|
-
def
|
110
|
-
@
|
111
|
-
|
112
|
-
if @config[:routing_file]
|
113
|
-
routes.files << @config[:routing_file]
|
100
|
+
def connection
|
101
|
+
@connection ||= Connection.new(uri, config, dispatcher).tap do
|
102
|
+
log(:debug) { 'Created new connection..' }
|
114
103
|
end
|
104
|
+
end
|
115
105
|
|
116
|
-
|
106
|
+
def dispatcher
|
107
|
+
@dispatcher ||= Dispatcher.new(config).tap do
|
108
|
+
log(:debug) { 'Created new dispatcher..' }
|
109
|
+
end
|
117
110
|
end
|
118
111
|
|
119
|
-
def
|
120
|
-
@
|
121
|
-
@pool
|
122
|
-
@
|
123
|
-
|
112
|
+
def reset
|
113
|
+
@connection = nil
|
114
|
+
@pool = nil
|
115
|
+
@channel = nil
|
116
|
+
@subscriptions = nil
|
124
117
|
end
|
125
118
|
end
|
126
119
|
end
|
data/lib/msgr/connection.rb
CHANGED
@@ -1,125 +1,125 @@
|
|
1
|
+
require 'bunny'
|
2
|
+
require 'multi_json'
|
3
|
+
|
1
4
|
module Msgr
|
2
5
|
|
3
6
|
class Connection
|
4
|
-
include Celluloid
|
5
7
|
include Logging
|
6
8
|
|
7
|
-
|
8
|
-
finalizer :close
|
9
|
-
|
10
|
-
def initialize(conn, routes, dispatcher, opts = {})
|
11
|
-
@conn = conn
|
12
|
-
@dispatcher = dispatcher
|
13
|
-
@routes = routes
|
14
|
-
@opts = opts
|
9
|
+
EXCHANGE_NAME = 'msgr'
|
15
10
|
|
16
|
-
|
17
|
-
@channel.prefetch(10)
|
11
|
+
attr_reader :uri, :config
|
18
12
|
|
19
|
-
|
13
|
+
def initialize(uri, config, dispatcher)
|
14
|
+
@uri = uri
|
15
|
+
@config = config
|
16
|
+
@dispatcher = dispatcher
|
20
17
|
end
|
21
18
|
|
22
|
-
def
|
23
|
-
|
24
|
-
bind
|
19
|
+
def running?
|
20
|
+
subscriptions.any?
|
25
21
|
end
|
26
22
|
|
27
|
-
def
|
28
|
-
|
29
|
-
|
23
|
+
def publish(payload, opts = {})
|
24
|
+
opts[:routing_key] = opts.delete(:to) if opts[:to]
|
25
|
+
|
26
|
+
begin
|
27
|
+
payload = MultiJson.dump(payload)
|
28
|
+
exchange.publish payload, opts.merge(persistent: true, content_type: 'application/json')
|
29
|
+
rescue => error
|
30
|
+
exchange.publish payload.to_s, opts.merge(persistent: true, content_type: 'application/text')
|
31
|
+
end
|
30
32
|
|
31
|
-
log(:debug) {
|
33
|
+
log(:debug) { "Published message to #{opts[:routing_key]}" }
|
32
34
|
end
|
33
35
|
|
34
|
-
def
|
35
|
-
|
36
|
+
def connection
|
37
|
+
@connection ||= ::Bunny.new(config).tap { |b| b.start }
|
36
38
|
end
|
37
39
|
|
38
|
-
|
39
|
-
|
40
|
-
# receive messages but channel need to be open
|
41
|
-
# to allow further acknowledgments.
|
42
|
-
#
|
43
|
-
def bindings
|
44
|
-
@bindings ||= []
|
40
|
+
def channel
|
41
|
+
@channel ||= connection.create_channel
|
45
42
|
end
|
46
43
|
|
47
|
-
def
|
48
|
-
|
49
|
-
log(:debug) { "Create queue #{queue.name} (durable: #{queue.durable?}, auto_delete: #{queue.auto_delete?})" }
|
50
|
-
end
|
44
|
+
def release
|
45
|
+
subscriptions.each { |subscription| subscription.cancel }
|
51
46
|
end
|
52
47
|
|
53
|
-
def
|
54
|
-
|
55
|
-
|
48
|
+
def subscriptions
|
49
|
+
@subscription ||= []
|
50
|
+
end
|
56
51
|
|
57
|
-
|
52
|
+
def prefix(name)
|
53
|
+
if config[:prefix].present?
|
54
|
+
"#{config[:prefix]}-#{name}"
|
55
|
+
else
|
56
|
+
name
|
58
57
|
end
|
59
|
-
|
60
|
-
@exchange
|
61
58
|
end
|
62
59
|
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
return unless bindings.any?
|
69
|
-
|
70
|
-
log(:debug) { "Release all bindings#{wait ? ' after queues are empty': ''}..." }
|
71
|
-
|
72
|
-
if wait
|
73
|
-
binds = bindings.dup
|
74
|
-
while binds.any?
|
75
|
-
binds.reject! { |b| b.release_if_empty }
|
76
|
-
sleep 1
|
60
|
+
def exchange
|
61
|
+
@exchange ||= channel.topic(prefix(EXCHANGE_NAME), durable: true).tap do |ex|
|
62
|
+
log(:debug) do
|
63
|
+
"Created exchange #{ex.name} (type: #{ex.type}, " \
|
64
|
+
"durable: #{ex.durable?}, auto_delete: #{ex.auto_delete?})"
|
77
65
|
end
|
78
|
-
else
|
79
|
-
bindings.each &:release
|
80
66
|
end
|
81
|
-
|
82
|
-
log(:debug) { 'All bindings released.' }
|
83
67
|
end
|
84
68
|
|
85
|
-
def
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
@exchange.delete if @exchange
|
69
|
+
def queue(name)
|
70
|
+
channel.queue(prefix(name), durable: true).tap do |queue|
|
71
|
+
log(:debug) do
|
72
|
+
"Create queue #{queue.name} (durable: #{queue.durable?}, " \
|
73
|
+
"auto_delete: #{queue.auto_delete?})"
|
74
|
+
end
|
75
|
+
end
|
94
76
|
end
|
95
77
|
|
96
|
-
def
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
begin
|
103
|
-
payload = JSON.generate(payload)
|
104
|
-
exchange.publish payload, opts.merge(persistent: true, content_type: 'application/json')
|
105
|
-
rescue => error
|
106
|
-
exchange.publish payload.to_s, opts.merge(persistent: true, content_type: 'application/text')
|
78
|
+
def bind(routes)
|
79
|
+
if routes.empty?
|
80
|
+
log(:warn) { "No routes to bound to. Bind will have no effect. (#{routes.inspect})" }
|
81
|
+
else
|
82
|
+
bind_all(routes)
|
107
83
|
end
|
108
84
|
end
|
109
85
|
|
110
86
|
def ack(delivery_tag)
|
111
|
-
|
112
|
-
|
87
|
+
channel.ack delivery_tag
|
88
|
+
log(:debug) { "Acked message: #{delivery_tag}" }
|
113
89
|
end
|
114
90
|
|
115
91
|
def reject(delivery_tag, requeue = true)
|
116
|
-
|
117
|
-
|
92
|
+
channel.reject delivery_tag, requeue
|
93
|
+
log(:debug) { "Rejected message: #{delivery_tag}" }
|
118
94
|
end
|
119
95
|
|
120
96
|
def close
|
121
|
-
|
122
|
-
|
97
|
+
channel.close if @channel && @channel.open?
|
98
|
+
connection.close if @connection
|
99
|
+
log(:debug) { 'Closed.' }
|
100
|
+
end
|
101
|
+
|
102
|
+
private
|
103
|
+
def bind_all(routes)
|
104
|
+
routes.each do |route|
|
105
|
+
queue = queue(route.name)
|
106
|
+
|
107
|
+
route.keys.each do |key|
|
108
|
+
log(:debug) { "Bind #{key} to #{queue.name}." }
|
109
|
+
|
110
|
+
queue.bind exchange, routing_key: key
|
111
|
+
end
|
112
|
+
|
113
|
+
subscriptions << queue.subscribe(ack: true) do |*args|
|
114
|
+
begin
|
115
|
+
@dispatcher.call Message.new(self, *args, route)
|
116
|
+
rescue => err
|
117
|
+
log(:error) do
|
118
|
+
"Rescued error from subscribe: #{err.class.name}: #{err}\n#{err.backtrace.join("\n")}"
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
123
|
end
|
124
124
|
end
|
125
125
|
end
|
data/lib/msgr/dispatcher.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'concurrent/cached_thread_pool'
|
2
|
+
|
1
3
|
module Msgr
|
2
4
|
|
3
5
|
# The Dispatcher receives incoming messages,
|
@@ -7,13 +9,18 @@ module Msgr
|
|
7
9
|
class Dispatcher
|
8
10
|
include Logging
|
9
11
|
|
10
|
-
|
11
|
-
dispatch message
|
12
|
+
attr_reader :pool
|
12
13
|
|
13
|
-
|
14
|
-
|
14
|
+
def initialize(config)
|
15
|
+
log(:info) { "Initialize new dispatcher (#{Rails.env})..." }
|
15
16
|
|
16
|
-
|
17
|
+
@pool = ::Concurrent::CachedThreadPool.new(max: config[:max])
|
18
|
+
end
|
19
|
+
|
20
|
+
def call(message)
|
21
|
+
pool.post(message) do |message|
|
22
|
+
dispatch message
|
23
|
+
end
|
17
24
|
end
|
18
25
|
|
19
26
|
def dispatch(message)
|
@@ -22,8 +29,18 @@ module Msgr
|
|
22
29
|
log(:debug) { "Dispatch message to #{consumer_class.name}" }
|
23
30
|
|
24
31
|
consumer_class.new.dispatch message
|
32
|
+
|
33
|
+
# Acknowledge message unless it is already acknowledged.
|
34
|
+
message.ack unless message.acked?
|
25
35
|
rescue => error
|
26
|
-
log(:error)
|
36
|
+
log(:error) do
|
37
|
+
"Dispatcher error: #{error.class.name}: #{error}\n" +
|
38
|
+
error.backtrace.join("\n")
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def shutdown
|
43
|
+
|
27
44
|
end
|
28
45
|
|
29
46
|
def to_s
|
data/lib/msgr/logging.rb
CHANGED
@@ -2,11 +2,11 @@ module Msgr
|
|
2
2
|
|
3
3
|
module Logging
|
4
4
|
def log(level)
|
5
|
-
Msgr.logger.send(level
|
5
|
+
Msgr.logger.send(level) { "#{self.log_name} #{yield}" } if Msgr.logger
|
6
6
|
end
|
7
7
|
|
8
8
|
def log_name
|
9
|
-
"[#{Thread.current.object_id}]
|
9
|
+
"[#{Thread.current.object_id.to_s(16)}] <#{self.class.name}[#{object_id.to_s(16)}]>"
|
10
10
|
end
|
11
11
|
end
|
12
12
|
end
|
data/lib/msgr/message.rb
CHANGED
@@ -1,9 +1,6 @@
|
|
1
1
|
module Msgr
|
2
2
|
|
3
|
-
require 'msgr/message/acknowledge'
|
4
|
-
|
5
3
|
class Message
|
6
|
-
include Acknowledge
|
7
4
|
attr_reader :delivery_info, :metadata, :payload, :route
|
8
5
|
|
9
6
|
def initialize(connection, delivery_info, metadata, payload, route)
|
@@ -14,12 +11,34 @@ module Msgr
|
|
14
11
|
@route = route
|
15
12
|
|
16
13
|
if content_type == 'application/json'
|
17
|
-
@payload =
|
14
|
+
@payload = MultiJson.load(payload)
|
15
|
+
@payload.symbolize_keys! if @payload.respond_to? :symbolize_keys!
|
18
16
|
end
|
19
17
|
end
|
20
18
|
|
21
19
|
def content_type
|
22
20
|
@metadata.content_type
|
23
21
|
end
|
22
|
+
|
23
|
+
# Check if message is already acknowledged.
|
24
|
+
#
|
25
|
+
# @return [Boolean] True if message is acknowledged, false otherwise.
|
26
|
+
# @api public
|
27
|
+
#
|
28
|
+
def acked?
|
29
|
+
@acked ? true : false
|
30
|
+
end
|
31
|
+
|
32
|
+
# Send message acknowledge to broker unless message is
|
33
|
+
# already acknowledged.
|
34
|
+
#
|
35
|
+
# @api public
|
36
|
+
#
|
37
|
+
def ack
|
38
|
+
unless acked?
|
39
|
+
@acked = true
|
40
|
+
@connection.ack(delivery_info.delivery_tag)
|
41
|
+
end
|
42
|
+
end
|
24
43
|
end
|
25
44
|
end
|
data/lib/msgr/railtie.rb
CHANGED
@@ -16,7 +16,6 @@ module Msgr
|
|
16
16
|
initializer 'msgr.start' do
|
17
17
|
config.after_initialize do |app|
|
18
18
|
Msgr.logger = app.config.msgr.logger
|
19
|
-
Celluloid.logger = app.config.msgr.logger
|
20
19
|
|
21
20
|
self.class.load app.config.msgr
|
22
21
|
end
|
@@ -28,26 +27,6 @@ module Msgr
|
|
28
27
|
return unless cfg # no config given -> does not load Msgr
|
29
28
|
|
30
29
|
Msgr.config = cfg
|
31
|
-
|
32
|
-
# later loading for e.g. unicorn
|
33
|
-
if Rails.env.development?
|
34
|
-
Msgr.after_load do |client|
|
35
|
-
setup_autoreload client
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
Msgr.start if cfg[:autostart]
|
40
|
-
end
|
41
|
-
|
42
|
-
def setup_autoreload(client)
|
43
|
-
reloader = ActiveSupport::FileUpdateChecker.new client.routes.files do
|
44
|
-
client.routes.reload
|
45
|
-
client.reload
|
46
|
-
end
|
47
|
-
|
48
|
-
ActionDispatch::Reloader.to_prepare do
|
49
|
-
reloader.execute_if_updated
|
50
|
-
end
|
51
30
|
end
|
52
31
|
|
53
32
|
def parse_config(cfg)
|
data/lib/msgr/routes.rb
CHANGED
@@ -1,14 +1,16 @@
|
|
1
1
|
module Msgr
|
2
2
|
|
3
3
|
class Routes
|
4
|
+
include Logging
|
4
5
|
attr_reader :routes
|
5
|
-
delegate :each, to: :@routes
|
6
|
+
delegate :each, :empty?, :size, :any?, to: :@routes
|
6
7
|
|
7
8
|
def initialize
|
8
9
|
@routes = []
|
9
10
|
end
|
10
11
|
|
11
12
|
def configure(&block)
|
13
|
+
blocks << block
|
12
14
|
instance_eval &block
|
13
15
|
end
|
14
16
|
|
@@ -16,17 +18,27 @@ module Msgr
|
|
16
18
|
@files ||= []
|
17
19
|
end
|
18
20
|
|
21
|
+
def blocks
|
22
|
+
@blocks ||= []
|
23
|
+
end
|
24
|
+
|
19
25
|
def files=(files)
|
20
26
|
@files = Array files
|
21
27
|
end
|
22
28
|
|
29
|
+
def <<(file)
|
30
|
+
files << file
|
31
|
+
end
|
32
|
+
|
23
33
|
def reload
|
24
34
|
routes.clear
|
35
|
+
blocks.each { |block| instance_eval(&block) }
|
36
|
+
files.uniq!
|
25
37
|
files.each do |file|
|
26
38
|
if File.exists? file
|
27
39
|
load file
|
28
40
|
else
|
29
|
-
|
41
|
+
log(:warn) { "Routes file `#{file}` does not exists (anymore)." }
|
30
42
|
end
|
31
43
|
end
|
32
44
|
end
|
data/lib/msgr/version.rb
CHANGED
data/msgr.gemspec
CHANGED
@@ -20,7 +20,8 @@ Gem::Specification.new do |spec|
|
|
20
20
|
|
21
21
|
spec.add_dependency 'activesupport'
|
22
22
|
spec.add_dependency 'bunny', '~> 1.0'
|
23
|
-
spec.add_dependency '
|
23
|
+
spec.add_dependency 'concurrent-ruby'
|
24
|
+
spec.add_dependency 'multi_json'
|
24
25
|
|
25
26
|
spec.add_development_dependency 'bundler', '~> 1.3'
|
26
27
|
end
|
data/spec/msgr/msgr_spec.rb
CHANGED
@@ -8,7 +8,7 @@ end
|
|
8
8
|
|
9
9
|
describe Msgr do
|
10
10
|
before do
|
11
|
-
Msgr.logger = nil
|
11
|
+
Msgr.logger = nil
|
12
12
|
Msgr.logger.level = Logger::Severity::DEBUG if Msgr.logger
|
13
13
|
end
|
14
14
|
|
@@ -23,22 +23,14 @@ describe Msgr do
|
|
23
23
|
end
|
24
24
|
|
25
25
|
after do
|
26
|
-
client.stop
|
26
|
+
client.stop
|
27
27
|
end
|
28
28
|
|
29
29
|
it 'should dispatch published methods to consumer' do
|
30
|
-
expect_any_instance_of(TestConsumer).to receive(:index).
|
30
|
+
expect_any_instance_of(TestConsumer).to receive(:index).seconds.and_call_original
|
31
31
|
|
32
32
|
client.publish 'Payload', to: 'routing.key'
|
33
33
|
|
34
|
-
sleep
|
35
|
-
end
|
36
|
-
|
37
|
-
describe '.after_load' do
|
38
|
-
before { allow_any_instance_of(Msgr::Client).to receive(:launch) }
|
39
|
-
|
40
|
-
it 'should yield the given block when Msgr.start is called' do
|
41
|
-
expect { |cb| Msgr.after_load &cb; Msgr.start }.to yield_with_args
|
42
|
-
end
|
34
|
+
sleep 4
|
43
35
|
end
|
44
36
|
end
|
data/spec/msgr/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: msgr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jan Graichen
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-03-
|
11
|
+
date: 2014-03-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -39,7 +39,21 @@ dependencies:
|
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '1.0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: concurrent-ruby
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: multi_json
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|
44
58
|
requirements:
|
45
59
|
- - ">="
|
@@ -84,7 +98,6 @@ files:
|
|
84
98
|
- gemfiles/Gemfile.rails-3-2
|
85
99
|
- gemfiles/Gemfile.rails-4-0
|
86
100
|
- lib/msgr.rb
|
87
|
-
- lib/msgr/binding.rb
|
88
101
|
- lib/msgr/client.rb
|
89
102
|
- lib/msgr/connection.rb
|
90
103
|
- lib/msgr/consumer.rb
|
@@ -92,8 +105,6 @@ files:
|
|
92
105
|
- lib/msgr/errors.rb
|
93
106
|
- lib/msgr/logging.rb
|
94
107
|
- lib/msgr/message.rb
|
95
|
-
- lib/msgr/message/acknowledge.rb
|
96
|
-
- lib/msgr/pool.rb
|
97
108
|
- lib/msgr/railtie.rb
|
98
109
|
- lib/msgr/route.rb
|
99
110
|
- lib/msgr/routes.rb
|
@@ -148,10 +159,8 @@ files:
|
|
148
159
|
- spec/integration/msgr/railtie_spec.rb
|
149
160
|
- spec/integration/msgr_spec.rb
|
150
161
|
- spec/integration/spec_helper.rb
|
151
|
-
- spec/msgr/msgr/client_spec.rb
|
152
162
|
- spec/msgr/msgr/connection_spec.rb
|
153
163
|
- spec/msgr/msgr/consumer_spec.rb
|
154
|
-
- spec/msgr/msgr/pool_spec.rb
|
155
164
|
- spec/msgr/msgr/route_spec.rb
|
156
165
|
- spec/msgr/msgr/routes_spec.rb
|
157
166
|
- spec/msgr/msgr_spec.rb
|
@@ -231,10 +240,8 @@ test_files:
|
|
231
240
|
- spec/integration/msgr/railtie_spec.rb
|
232
241
|
- spec/integration/msgr_spec.rb
|
233
242
|
- spec/integration/spec_helper.rb
|
234
|
-
- spec/msgr/msgr/client_spec.rb
|
235
243
|
- spec/msgr/msgr/connection_spec.rb
|
236
244
|
- spec/msgr/msgr/consumer_spec.rb
|
237
|
-
- spec/msgr/msgr/pool_spec.rb
|
238
245
|
- spec/msgr/msgr/route_spec.rb
|
239
246
|
- spec/msgr/msgr/routes_spec.rb
|
240
247
|
- spec/msgr/msgr_spec.rb
|
data/lib/msgr/binding.rb
DELETED
@@ -1,61 +0,0 @@
|
|
1
|
-
module Msgr
|
2
|
-
# A single binding
|
3
|
-
class Binding
|
4
|
-
include Logging
|
5
|
-
attr_reader :connection, :route, :subscription, :dispatcher, :queue
|
6
|
-
|
7
|
-
def initialize(connection, route, dispatcher)
|
8
|
-
@connection = connection
|
9
|
-
@route = route
|
10
|
-
@dispatcher = dispatcher
|
11
|
-
|
12
|
-
exchange = connection.exchange
|
13
|
-
@queue = connection.queue route.name
|
14
|
-
|
15
|
-
route.keys.each do |key|
|
16
|
-
log(:debug) { "Bind #{key} to #{@queue.name}." }
|
17
|
-
|
18
|
-
queue.bind exchange, routing_key: key
|
19
|
-
end
|
20
|
-
|
21
|
-
@subscription = queue.subscribe(ack: true) { |*args| call *args }
|
22
|
-
end
|
23
|
-
|
24
|
-
# Called from Bunny Thread Pool. Will create message object from
|
25
|
-
# provided bunny data and dispatch message to connection.
|
26
|
-
#
|
27
|
-
def call(info, metadata, payload)
|
28
|
-
dispatcher.dispatch Message.new(connection, info, metadata, payload, route)
|
29
|
-
rescue => error
|
30
|
-
log(:error) { "Error received within subscribe handler: #{error.inspect}\n#{error.backtrace.join("\n ")}" }
|
31
|
-
end
|
32
|
-
|
33
|
-
# Cancel subscription to not receive any more messages.
|
34
|
-
#
|
35
|
-
def release
|
36
|
-
subscription.cancel if subscription
|
37
|
-
end
|
38
|
-
|
39
|
-
def release_if_empty
|
40
|
-
if queue_empty?
|
41
|
-
release
|
42
|
-
true
|
43
|
-
else
|
44
|
-
false
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
48
|
-
def queue_empty?
|
49
|
-
@queue.message_count == 0
|
50
|
-
end
|
51
|
-
|
52
|
-
def delete
|
53
|
-
release
|
54
|
-
queue.delete
|
55
|
-
end
|
56
|
-
|
57
|
-
def purge
|
58
|
-
queue.purge
|
59
|
-
end
|
60
|
-
end
|
61
|
-
end
|
@@ -1,30 +0,0 @@
|
|
1
|
-
module Msgr
|
2
|
-
class Message
|
3
|
-
|
4
|
-
#
|
5
|
-
#
|
6
|
-
module Acknowledge
|
7
|
-
|
8
|
-
# Check if message is already acknowledged.
|
9
|
-
#
|
10
|
-
# @return [Boolean] True if message is acknowledged, false otherwise.
|
11
|
-
# @api public
|
12
|
-
#
|
13
|
-
def acked?
|
14
|
-
@acked ? true : false
|
15
|
-
end
|
16
|
-
|
17
|
-
# Send message acknowledge to broker unless message is
|
18
|
-
# already acknowledged.
|
19
|
-
#
|
20
|
-
# @api public
|
21
|
-
#
|
22
|
-
def ack
|
23
|
-
unless acked?
|
24
|
-
@acked = true
|
25
|
-
@connection.future(:ack, delivery_info.delivery_tag).value
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
data/lib/msgr/pool.rb
DELETED
@@ -1,165 +0,0 @@
|
|
1
|
-
module Msgr
|
2
|
-
|
3
|
-
class Pool
|
4
|
-
include Celluloid
|
5
|
-
include Logging
|
6
|
-
attr_reader :size
|
7
|
-
|
8
|
-
def initialize(runner_klass, opts = {})
|
9
|
-
@runner_klass = runner_klass
|
10
|
-
@runner_args = opts[:args] ? Array(opts[:args]) : []
|
11
|
-
@size = (opts[:size] || Celluloid.cores).to_i
|
12
|
-
@running = false
|
13
|
-
|
14
|
-
raise ArgumentError.new 'Pool size must be greater zero.' if @size <= 0
|
15
|
-
|
16
|
-
log(:debug) { "Inialize size => #{@size}" }
|
17
|
-
|
18
|
-
every([opts.fetch(:stats_interval, 30).to_i, 1].max) { log_status } if opts[:nostats].nil? || opts[:nostats]
|
19
|
-
end
|
20
|
-
|
21
|
-
def running?
|
22
|
-
@running
|
23
|
-
end
|
24
|
-
|
25
|
-
def idle; @idle ||= [] end
|
26
|
-
def busy; @busy ||= [] end
|
27
|
-
|
28
|
-
def start
|
29
|
-
return if running?
|
30
|
-
|
31
|
-
log(:debug) { 'Spin up worker pool' }
|
32
|
-
@running = true
|
33
|
-
|
34
|
-
idle.clear
|
35
|
-
busy.clear
|
36
|
-
|
37
|
-
@size.times.map do |index|
|
38
|
-
idle << Worker.new_link(Actor.current, index, @runner_klass, @runner_args)
|
39
|
-
end
|
40
|
-
|
41
|
-
log(:debug) { 'Pool ready.' }
|
42
|
-
end
|
43
|
-
|
44
|
-
def log_status
|
45
|
-
log(:info) { "[STATUS] Idle: #{idle.size} Busy: #{busy.size}" }
|
46
|
-
end
|
47
|
-
|
48
|
-
# Request a graceful shutdown of all pool workers.
|
49
|
-
#
|
50
|
-
def stop
|
51
|
-
log(:debug) { 'Graceful shutdown requested.' }
|
52
|
-
|
53
|
-
@running = false
|
54
|
-
idle.each { |worker| worker.terminate }
|
55
|
-
idle.clear
|
56
|
-
|
57
|
-
if busy.any?
|
58
|
-
log(:debug) { "Wait for #{busy.size} workers to terminate." }
|
59
|
-
|
60
|
-
wait :shutdown
|
61
|
-
end
|
62
|
-
|
63
|
-
log(:debug) { 'Graceful shutdown done.' }
|
64
|
-
end
|
65
|
-
alias_method :shutdown, :stop
|
66
|
-
|
67
|
-
# Check if a worker is available.
|
68
|
-
#
|
69
|
-
# @return [Boolean] True if at least on idle worker is available, false otherwise.
|
70
|
-
#
|
71
|
-
def available?
|
72
|
-
idle.any?
|
73
|
-
end
|
74
|
-
|
75
|
-
def messages
|
76
|
-
@message ||= []
|
77
|
-
end
|
78
|
-
|
79
|
-
# Dispatch given message to a worker.
|
80
|
-
#
|
81
|
-
def dispatch(*args)
|
82
|
-
log(:debug) { 'Dispatch message to worker.' }
|
83
|
-
|
84
|
-
fetch_idle_worker.future :dispatch, args
|
85
|
-
end
|
86
|
-
|
87
|
-
# Return an idle worker.
|
88
|
-
#
|
89
|
-
def fetch_idle_worker
|
90
|
-
if (worker = idle.shift)
|
91
|
-
busy << worker
|
92
|
-
worker
|
93
|
-
else
|
94
|
-
wait :worker_done
|
95
|
-
fetch_idle_worker
|
96
|
-
end
|
97
|
-
end
|
98
|
-
|
99
|
-
# Called by worker to indicated it has finished processing.
|
100
|
-
#
|
101
|
-
# @param [Pool::Worker] worker Worker that finished processing.
|
102
|
-
#
|
103
|
-
def executed(worker)
|
104
|
-
busy.delete worker
|
105
|
-
|
106
|
-
if running?
|
107
|
-
idle << worker
|
108
|
-
after(0) { signal :worker_done }
|
109
|
-
else
|
110
|
-
log(:debug) { "Terminate worker. Still #{busy.size} to go..." }
|
111
|
-
|
112
|
-
worker.terminate if worker.alive?
|
113
|
-
if busy.empty?
|
114
|
-
log(:debug) { 'All worker down. Signal :shutdown.' }
|
115
|
-
after(0) { signal :shutdown }
|
116
|
-
end
|
117
|
-
end
|
118
|
-
end
|
119
|
-
|
120
|
-
def to_s
|
121
|
-
"#{self.class.name}[#{@runner_klass}]<#{object_id}>"
|
122
|
-
end
|
123
|
-
|
124
|
-
# Worker actor capsuling worker logic and dispatching
|
125
|
-
# tasks to custom runner object.
|
126
|
-
#
|
127
|
-
class Worker
|
128
|
-
include Celluloid
|
129
|
-
include Logging
|
130
|
-
attr_reader :pool, :index, :runner
|
131
|
-
|
132
|
-
def initialize(pool, index, runner_klass, runner_args)
|
133
|
-
@pool = pool
|
134
|
-
@poolname = pool.to_s
|
135
|
-
@index = index
|
136
|
-
@runner = runner_klass.new *runner_args
|
137
|
-
|
138
|
-
log(:debug) { 'Worker ready.' }
|
139
|
-
end
|
140
|
-
|
141
|
-
# Dispatch given method and argument to custom runner.
|
142
|
-
# Arguments are used to call `#send` on runner instance.
|
143
|
-
#
|
144
|
-
def dispatch(args)
|
145
|
-
log(:debug) { "Dispatch to runner: #{runner.class.name}" }
|
146
|
-
|
147
|
-
# Send method to custom runner.
|
148
|
-
runner.send :call, *args
|
149
|
-
|
150
|
-
rescue => error
|
151
|
-
log(:error) { "Received error from runner: #{error.message}\n#{error.backtrace.join(" \n")}" }
|
152
|
-
ensure
|
153
|
-
if pool.alive?
|
154
|
-
pool.executed Actor.current
|
155
|
-
else
|
156
|
-
terminate
|
157
|
-
end
|
158
|
-
end
|
159
|
-
|
160
|
-
def to_s
|
161
|
-
"#{@poolname}[##{index}]"
|
162
|
-
end
|
163
|
-
end
|
164
|
-
end
|
165
|
-
end
|
@@ -1,58 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Msgr::Client do
|
4
|
-
|
5
|
-
describe '#start' do
|
6
|
-
let(:params) { [] }
|
7
|
-
let(:client) { Msgr::Client.new *params }
|
8
|
-
before { allow_any_instance_of(Msgr::Client).to receive(:launch) }
|
9
|
-
|
10
|
-
context 'with URI' do
|
11
|
-
it 'should pass URI options to bunny (I)' do
|
12
|
-
expect(Bunny).to receive(:new)
|
13
|
-
.with(pass: 'guest', user: 'guest', ssl: false, host: 'localhost', vhost: '/')
|
14
|
-
|
15
|
-
Msgr::Client.new(uri: 'amqp://guest:guest@localhost/').start
|
16
|
-
end
|
17
|
-
|
18
|
-
it 'should pass URI options to bunny (II)' do
|
19
|
-
expect(Bunny).to receive(:new)
|
20
|
-
.with(pass: 'msgr', user: 'abc', ssl: true, host: 'bogus.example.org', vhost: '/rabbit')
|
21
|
-
|
22
|
-
Msgr::Client.new(uri: 'amqps://abc:msgr@bogus.example.org/rabbit').start
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
context 'with options' do
|
27
|
-
it 'should pass options to bunny' do
|
28
|
-
expect(Bunny).to receive(:new)
|
29
|
-
.with(pass: 'guest', user: 'guest', ssl: false, host: 'localhost', vhost: '/')
|
30
|
-
|
31
|
-
Msgr::Client.new(pass: 'guest', user: 'guest', ssl: false, host: 'localhost', vhost: '/').start
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
context 'with URI and options' do
|
36
|
-
it 'should pass merged options to bunny' do
|
37
|
-
expect(Bunny).to receive(:new)
|
38
|
-
.with(pass: 'msgr', user: 'abc', ssl: false, host: 'localhost', vhost: '/joghurt')
|
39
|
-
|
40
|
-
Msgr::Client.new(uri: 'ampq://abc@localhost', pass: 'msgr', vhost: '/joghurt').start
|
41
|
-
end
|
42
|
-
|
43
|
-
it 'should pass prefer hash option' do
|
44
|
-
expect(Bunny).to receive(:new)
|
45
|
-
.with(ssl: true, host: 'a.example.org', vhost: '/', user: 'guest')
|
46
|
-
|
47
|
-
Msgr::Client.new(uri: 'ampq://localhost', ssl: true, host: 'a.example.org').start
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
context 'routes' do
|
52
|
-
let(:params) { [ routing_file: 'config/msgr.rb']}
|
53
|
-
before { client.start }
|
54
|
-
subject { client.routes }
|
55
|
-
its(:files) { should eq ['config/msgr.rb'] }
|
56
|
-
end
|
57
|
-
end
|
58
|
-
end
|
data/spec/msgr/msgr/pool_spec.rb
DELETED
@@ -1,68 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
$shutdown_test_graceful_down = false
|
4
|
-
|
5
|
-
class Runner
|
6
|
-
def call(*_) end
|
7
|
-
|
8
|
-
def shutdown_test
|
9
|
-
sleep 2
|
10
|
-
$shutdown_test_graceful_down = true
|
11
|
-
end
|
12
|
-
end
|
13
|
-
|
14
|
-
describe Msgr::Pool do
|
15
|
-
let(:pool) { Msgr::Pool.new Runner }
|
16
|
-
|
17
|
-
describe '#initialize' do
|
18
|
-
let(:opts) { {} }
|
19
|
-
let(:pool) { Msgr::Pool.new Runner, opts }
|
20
|
-
|
21
|
-
context 'pool size' do
|
22
|
-
let(:opts) { {size: 4} }
|
23
|
-
before { pool }
|
24
|
-
|
25
|
-
it 'should define number of worker actors' do
|
26
|
-
expect(pool.size).to eq 4
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
describe '#start' do
|
32
|
-
let!(:pool) { Msgr::Pool.new Runner, size: 4 }
|
33
|
-
|
34
|
-
it 'should start worker actors' do
|
35
|
-
expect { pool.start }.to change { Celluloid::Actor.all.size }.by(4)
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
describe '#size' do
|
40
|
-
it 'should default to number of available cores' do
|
41
|
-
expect(pool.size).to eq Celluloid.cores
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
describe '#dispatch' do
|
46
|
-
let!(:pool) { Msgr::Pool.new Runner, size: 4 }
|
47
|
-
before { pool.start }
|
48
|
-
|
49
|
-
it 'should dispatch message to runner' do
|
50
|
-
expect_any_instance_of(Runner).to receive(:call).within(10).seconds.once
|
51
|
-
pool.dispatch 5, 3.2, 'hello'
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
describe '#shutdown' do
|
56
|
-
let!(:pool) { Msgr::Pool.new Runner, size: 1 }
|
57
|
-
before do
|
58
|
-
pool.start
|
59
|
-
$shutdown_test_graceful_down = false
|
60
|
-
end
|
61
|
-
|
62
|
-
it 'should do a graceful shutdown of all worker' do
|
63
|
-
pool.dispatch :shutdown_test
|
64
|
-
pool.shutdown
|
65
|
-
expect($shutdown_test_graceful_down).to be_false
|
66
|
-
end
|
67
|
-
end
|
68
|
-
end
|