cloudist 0.4.4 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +2 -6
- data/Gemfile.lock +29 -27
- data/VERSION +1 -1
- data/cloudist.gemspec +9 -21
- data/examples/sandwich_client_with_custom_listener.rb +19 -16
- data/lib/cloudist/application.rb +10 -10
- data/lib/cloudist/core_ext/class.rb +4 -4
- data/lib/cloudist/job.rb +18 -18
- data/lib/cloudist/listener.rb +32 -32
- data/lib/cloudist/message.rb +23 -23
- data/lib/cloudist/messaging.rb +9 -9
- data/lib/cloudist/payload.rb +22 -22
- data/lib/cloudist/payload_old.rb +33 -33
- data/lib/cloudist/publisher.rb +6 -6
- data/lib/cloudist/queues/basic_queue.rb +17 -17
- data/lib/cloudist/queues/job_queue.rb +6 -6
- data/lib/cloudist/queues/reply_queue.rb +5 -5
- data/lib/cloudist/request.rb +5 -5
- data/lib/cloudist/utils.rb +5 -5
- data/lib/cloudist/worker.rb +10 -10
- data/spec/cloudist/basic_queue_spec.rb +10 -10
- data/spec/cloudist/job_spec.rb +4 -4
- data/spec/cloudist/payload_spec_2_spec.rb +16 -16
- data/spec/cloudist/request_spec.rb +13 -13
- data/spec/cloudist/utils_spec.rb +3 -3
- data/spec/cloudist_spec.rb +11 -11
- data/spec/core_ext/string_spec.rb +3 -3
- data/spec/spec_helper.rb +1 -2
- data/spec/support/amqp.rb +11 -11
- metadata +29 -73
data/lib/cloudist/listener.rb
CHANGED
@@ -2,27 +2,27 @@ require "active_support"
|
|
2
2
|
module Cloudist
|
3
3
|
class Listener
|
4
4
|
include ActiveSupport::Callbacks
|
5
|
-
|
5
|
+
|
6
6
|
attr_reader :job_queue_name, :payload
|
7
7
|
class_attribute :job_queue_names
|
8
|
-
|
8
|
+
|
9
9
|
class << self
|
10
10
|
def listen_to(*job_queue_names)
|
11
11
|
self.job_queue_names = job_queue_names.map { |q| Utils.reply_prefix(q) }
|
12
12
|
end
|
13
|
-
|
13
|
+
|
14
14
|
def subscribe(queue_name)
|
15
15
|
raise RuntimeError, "You can't subscribe until EM is running" unless EM.reactor_running?
|
16
|
-
|
16
|
+
|
17
17
|
reply_queue = Cloudist::ReplyQueue.new(queue_name)
|
18
18
|
reply_queue.subscribe do |request|
|
19
19
|
instance = Cloudist.listener_instances[queue_name] ||= new
|
20
20
|
instance.handle_request(request)
|
21
21
|
end
|
22
|
-
|
22
|
+
|
23
23
|
queue_name
|
24
24
|
end
|
25
|
-
|
25
|
+
|
26
26
|
def before(*args, &block)
|
27
27
|
set_callback(:call, :before, *args, &block)
|
28
28
|
end
|
@@ -31,16 +31,16 @@ module Cloudist
|
|
31
31
|
set_callback(:call, :after, *args, &block)
|
32
32
|
end
|
33
33
|
end
|
34
|
-
|
34
|
+
|
35
35
|
define_callbacks :call, :rescuable => true
|
36
|
-
|
36
|
+
|
37
37
|
def handle_request(request)
|
38
38
|
@payload = request.payload
|
39
39
|
key = [payload.message_type.to_s, payload.headers[:event]].compact.join(':')
|
40
|
-
|
40
|
+
|
41
41
|
meth, *args = handle_key(key)
|
42
|
-
|
43
|
-
if meth
|
42
|
+
|
43
|
+
if meth && self.respond_to?(meth)
|
44
44
|
if method(meth).arity <= args.size
|
45
45
|
call(meth, args.first(method(meth).arity))
|
46
46
|
else
|
@@ -48,19 +48,19 @@ module Cloudist
|
|
48
48
|
end
|
49
49
|
end
|
50
50
|
end
|
51
|
-
|
51
|
+
|
52
52
|
def id
|
53
53
|
payload.id
|
54
54
|
end
|
55
|
-
|
55
|
+
|
56
56
|
def data
|
57
57
|
payload.body
|
58
58
|
end
|
59
|
-
|
59
|
+
|
60
60
|
def handle_key(key)
|
61
61
|
key = key.split(':', 2)
|
62
62
|
return [nil, nil] if key.empty?
|
63
|
-
|
63
|
+
|
64
64
|
method_and_args = [key.shift.to_sym]
|
65
65
|
case method_and_args[0]
|
66
66
|
when :event
|
@@ -68,33 +68,33 @@ module Cloudist
|
|
68
68
|
method_and_args = [key.shift]
|
69
69
|
end
|
70
70
|
method_and_args << key
|
71
|
-
|
71
|
+
|
72
72
|
when :progress
|
73
73
|
method_and_args << payload.progress
|
74
74
|
method_and_args << payload.description
|
75
|
-
|
75
|
+
|
76
76
|
when :runtime
|
77
77
|
method_and_args << payload.runtime
|
78
|
-
|
78
|
+
|
79
79
|
when :reply
|
80
|
-
|
80
|
+
|
81
81
|
when :update
|
82
|
-
|
82
|
+
|
83
83
|
when :error
|
84
84
|
# method_and_args << Cloudist::SafeError.new(payload)
|
85
85
|
method_and_args << Hashie::Mash.new(payload.body)
|
86
|
-
|
86
|
+
|
87
87
|
when :log
|
88
88
|
method_and_args << payload.message
|
89
89
|
method_and_args << payload.level
|
90
|
-
|
90
|
+
|
91
91
|
else
|
92
92
|
method_and_args << data if method(method_and_args[0]).arity == 1
|
93
93
|
end
|
94
|
-
|
94
|
+
|
95
95
|
return method_and_args
|
96
96
|
end
|
97
|
-
|
97
|
+
|
98
98
|
def call(meth, args)
|
99
99
|
run_callbacks :call do
|
100
100
|
if args.empty?
|
@@ -104,30 +104,30 @@ module Cloudist
|
|
104
104
|
end
|
105
105
|
end
|
106
106
|
end
|
107
|
-
|
107
|
+
|
108
108
|
def progress(pct)
|
109
109
|
# :noop
|
110
110
|
end
|
111
|
-
|
111
|
+
|
112
112
|
def runtime(seconds)
|
113
113
|
# :noop
|
114
114
|
end
|
115
|
-
|
115
|
+
|
116
116
|
def event(type)
|
117
117
|
# :noop
|
118
118
|
end
|
119
|
-
|
119
|
+
|
120
120
|
def log(message, level)
|
121
121
|
# :noop
|
122
122
|
end
|
123
|
-
|
123
|
+
|
124
124
|
def error(e)
|
125
125
|
# :noop
|
126
126
|
end
|
127
|
-
|
127
|
+
|
128
128
|
end
|
129
|
-
|
129
|
+
|
130
130
|
class GenericListener < Listener
|
131
|
-
|
131
|
+
|
132
132
|
end
|
133
133
|
end
|
data/lib/cloudist/message.rb
CHANGED
@@ -1,49 +1,49 @@
|
|
1
1
|
module Cloudist
|
2
2
|
class Message
|
3
3
|
include Cloudist::Encoding
|
4
|
-
|
4
|
+
|
5
5
|
attr_reader :body, :headers, :id, :timestamp
|
6
|
-
|
6
|
+
|
7
7
|
# Expects body to be decoded
|
8
8
|
def initialize(body, headers = {})
|
9
9
|
@body = Hashie::Mash.new(body.dup)
|
10
|
-
|
10
|
+
|
11
11
|
@id ||= headers[:message_id] || headers[:id] && headers.delete(:id) || UUID.generate
|
12
12
|
@headers = Hashie::Mash.new(headers.dup)
|
13
|
-
|
13
|
+
|
14
14
|
@timestamp = Time.now.utc.to_f
|
15
|
-
|
15
|
+
|
16
16
|
update_headers(headers)
|
17
17
|
end
|
18
|
-
|
18
|
+
|
19
19
|
alias_method :data, :body
|
20
|
-
|
20
|
+
|
21
21
|
def update_headers(new_headers = {})
|
22
22
|
update_headers!
|
23
23
|
headers.merge!(new_headers)
|
24
24
|
end
|
25
|
-
|
25
|
+
|
26
26
|
def update_headers!
|
27
27
|
headers[:ttl] ||= Cloudist::DEFAULT_TTL
|
28
28
|
headers[:timestamp] = timestamp
|
29
29
|
headers[:message_id] ||= id
|
30
30
|
headers[:message_type] = 'message'
|
31
31
|
headers[:queue_name] ||= 'test'
|
32
|
-
|
32
|
+
|
33
33
|
headers.each { |k,v| headers[k] = v.to_s }
|
34
34
|
end
|
35
|
-
|
35
|
+
|
36
36
|
# Convenience method for replying
|
37
37
|
# Constructs a reply message and publishes it
|
38
38
|
def reply(body, reply_headers = {})
|
39
39
|
raise RuntimeError, "Cannot reply to an unpublished message" unless published?
|
40
|
-
|
40
|
+
|
41
41
|
msg = Message.new(body, reply_headers)
|
42
42
|
msg.set_reply_header
|
43
43
|
reply_q = Cloudist::ReplyQueue.new(headers[:queue_name])
|
44
44
|
msg.publish(reply_q)
|
45
45
|
end
|
46
|
-
|
46
|
+
|
47
47
|
# Publishes this message to the exchange or queue
|
48
48
|
# Queue should be a Cloudist::Queue object responding to #publish
|
49
49
|
def publish(queue)
|
@@ -53,45 +53,45 @@ module Cloudist
|
|
53
53
|
update_headers!
|
54
54
|
queue.publish(self)
|
55
55
|
end
|
56
|
-
|
56
|
+
|
57
57
|
def update_published_date!
|
58
58
|
headers[:published_on] = Time.now.utc.to_f
|
59
59
|
end
|
60
|
-
|
60
|
+
|
61
61
|
# This is so we can reply back to the sender
|
62
62
|
def set_queue_name_header(queue)
|
63
63
|
update_headers(:queue_name => queue.name)
|
64
64
|
end
|
65
|
-
|
65
|
+
|
66
66
|
def published?
|
67
67
|
@published ||= !!@headers.published_on
|
68
68
|
end
|
69
|
-
|
69
|
+
|
70
70
|
def created_at
|
71
71
|
headers.timestamp ? Time.at(headers.timestamp.to_f) : timestamp
|
72
72
|
end
|
73
|
-
|
73
|
+
|
74
74
|
def published_at
|
75
75
|
headers[:published_on] ? Time.at(headers[:published_on].to_f) : timestamp
|
76
76
|
end
|
77
|
-
|
77
|
+
|
78
78
|
def latency
|
79
79
|
(published_at.to_f - created_at.to_f)
|
80
80
|
end
|
81
|
-
|
81
|
+
|
82
82
|
def encoded
|
83
83
|
[encode(body), {:headers => headers}]
|
84
84
|
end
|
85
|
-
|
85
|
+
|
86
86
|
def inspect
|
87
87
|
"<#{self.class.name} id=#{id}>"
|
88
88
|
end
|
89
|
-
|
89
|
+
|
90
90
|
private
|
91
|
-
|
91
|
+
|
92
92
|
def set_reply_header
|
93
93
|
headers[:message_type] = 'reply'
|
94
94
|
end
|
95
|
-
|
95
|
+
|
96
96
|
end
|
97
97
|
end
|
data/lib/cloudist/messaging.rb
CHANGED
@@ -1,29 +1,29 @@
|
|
1
1
|
module Cloudist
|
2
2
|
autoload :Singleton, 'singleton'
|
3
|
-
|
3
|
+
|
4
4
|
class Messaging
|
5
5
|
include Singleton
|
6
|
-
|
6
|
+
|
7
7
|
class << self
|
8
|
-
|
8
|
+
|
9
9
|
def active_queues
|
10
10
|
instance.active_queues
|
11
11
|
end
|
12
|
-
|
12
|
+
|
13
13
|
def add_queue(queue)
|
14
14
|
(instance.active_queues ||= {}).merge!({queue.name.to_s => queue})
|
15
15
|
instance.active_queues
|
16
16
|
end
|
17
|
-
|
17
|
+
|
18
18
|
def remove_queue(queue_name)
|
19
19
|
(instance.active_queues ||= {}).delete(queue_name.to_s)
|
20
20
|
instance.active_queues
|
21
21
|
end
|
22
22
|
end
|
23
|
-
|
23
|
+
|
24
24
|
attr_accessor :active_queues
|
25
|
-
|
26
|
-
|
27
|
-
|
25
|
+
|
26
|
+
|
27
|
+
|
28
28
|
end
|
29
29
|
end
|
data/lib/cloudist/payload.rb
CHANGED
@@ -2,81 +2,81 @@ module Cloudist
|
|
2
2
|
class Payload
|
3
3
|
include Utils
|
4
4
|
include Encoding
|
5
|
-
|
5
|
+
|
6
6
|
attr_reader :body, :headers, :amqp_headers, :timestamp
|
7
|
-
|
7
|
+
|
8
8
|
def initialize(body, headers = {})
|
9
9
|
@published = false
|
10
10
|
@timestamp = Time.now.to_f
|
11
|
-
|
11
|
+
|
12
12
|
body = decode(body) if body.is_a?(String)
|
13
13
|
@body = Hashie::Mash.new(decode(body))
|
14
14
|
@headers = Hashie::Mash.new(headers)
|
15
15
|
@amqp_headers = {}
|
16
16
|
# puts "Initialised Payload: #{id}"
|
17
|
-
|
17
|
+
|
18
18
|
parse_headers!
|
19
19
|
end
|
20
|
-
|
20
|
+
|
21
21
|
def find_or_create_id
|
22
|
-
if headers["message_id"]
|
22
|
+
if headers["message_id"]
|
23
23
|
headers.message_id
|
24
24
|
else
|
25
25
|
UUID.generate
|
26
26
|
end
|
27
27
|
end
|
28
|
-
|
28
|
+
|
29
29
|
def id
|
30
30
|
find_or_create_id
|
31
31
|
end
|
32
|
-
|
32
|
+
|
33
33
|
def to_a
|
34
34
|
[encode(body), {:headers => encoded_headers}]
|
35
35
|
end
|
36
|
-
|
36
|
+
|
37
37
|
def parse_headers!
|
38
38
|
headers[:published_on] ||= body.delete("timestamp") || timestamp
|
39
39
|
headers[:message_type] ||= body.delete("message_type") || 'reply'
|
40
|
-
|
40
|
+
|
41
41
|
headers[:ttl] ||= Cloudist::DEFAULT_TTL
|
42
42
|
headers[:message_id] = id
|
43
|
-
|
43
|
+
|
44
44
|
headers[:published_on] = headers[:published_on].to_f
|
45
|
-
|
45
|
+
|
46
46
|
headers[:ttl] = headers[:ttl].to_i rescue -1
|
47
47
|
headers[:ttl] = -1 if headers[:ttl] == 0
|
48
|
-
|
48
|
+
|
49
49
|
# If this payload was received with a timestamp,
|
50
50
|
# we don't want to override it on #timestamp
|
51
51
|
if timestamp > headers[:published_on]
|
52
52
|
@timestamp = headers[:published_on]
|
53
53
|
end
|
54
|
-
|
54
|
+
|
55
55
|
headers
|
56
56
|
end
|
57
|
-
|
57
|
+
|
58
58
|
def encoded_headers
|
59
59
|
h = headers.dup
|
60
60
|
h.each { |k,v| h[k] = v.to_s }
|
61
61
|
return h
|
62
62
|
end
|
63
|
-
|
63
|
+
|
64
64
|
def set_reply_to(queue_name)
|
65
65
|
headers[:reply_to] = reply_prefix(queue_name)
|
66
66
|
end
|
67
|
-
|
67
|
+
|
68
68
|
def reply_to
|
69
69
|
headers.reply_to
|
70
70
|
end
|
71
|
-
|
71
|
+
|
72
72
|
def message_type
|
73
73
|
headers.message_type
|
74
74
|
end
|
75
|
-
|
75
|
+
|
76
76
|
def [](key)
|
77
77
|
self.body[key.to_s]
|
78
78
|
end
|
79
|
-
|
79
|
+
|
80
80
|
def method_missing(meth, *args, &blk)
|
81
81
|
if body.has_key?(meth.to_s)
|
82
82
|
return body[meth]
|
@@ -86,6 +86,6 @@ module Cloudist
|
|
86
86
|
super
|
87
87
|
end
|
88
88
|
end
|
89
|
-
|
89
|
+
|
90
90
|
end
|
91
|
-
end
|
91
|
+
end
|
data/lib/cloudist/payload_old.rb
CHANGED
@@ -1,55 +1,55 @@
|
|
1
1
|
module Cloudist
|
2
2
|
class Payload
|
3
3
|
include Utils
|
4
|
-
|
4
|
+
|
5
5
|
attr_reader :body, :publish_opts, :headers, :timestamp
|
6
6
|
|
7
7
|
def initialize(body, headers = {}, publish_opts = {})
|
8
8
|
@publish_opts, @headers = publish_opts, Hashie::Mash.new(headers)
|
9
9
|
@published = false
|
10
|
-
|
10
|
+
|
11
11
|
body = parse_message(body) if body.is_a?(String)
|
12
|
-
|
12
|
+
|
13
13
|
# raise Cloudist::BadPayload, "Expected Hash for payload" unless body.is_a?(Hash)
|
14
|
-
|
14
|
+
|
15
15
|
@timestamp = Time.now.to_f
|
16
|
-
|
16
|
+
|
17
17
|
@body = body
|
18
18
|
# Hashie::Mash.new(body)
|
19
|
-
|
19
|
+
|
20
20
|
update_headers
|
21
21
|
end
|
22
22
|
|
23
23
|
# Return message formatted as JSON and headers ready for transport in array
|
24
24
|
def formatted
|
25
25
|
update_headers
|
26
|
-
|
26
|
+
|
27
27
|
[encode_message(body), publish_opts]
|
28
28
|
end
|
29
|
-
|
29
|
+
|
30
30
|
def id
|
31
31
|
@id ||= event_hash.to_s
|
32
32
|
end
|
33
|
-
|
33
|
+
|
34
34
|
def id=(new_id)
|
35
35
|
@id = new_id.to_s
|
36
36
|
update_headers
|
37
37
|
end
|
38
|
-
|
38
|
+
|
39
39
|
def frozen?
|
40
40
|
headers.frozen?
|
41
41
|
end
|
42
|
-
|
42
|
+
|
43
43
|
def freeze!
|
44
44
|
headers.freeze
|
45
45
|
body.freeze
|
46
46
|
end
|
47
|
-
|
47
|
+
|
48
48
|
def update_headers
|
49
49
|
headers = extract_custom_headers
|
50
50
|
(publish_opts[:headers] ||= {}).merge!(headers)
|
51
51
|
end
|
52
|
-
|
52
|
+
|
53
53
|
def extract_custom_headers
|
54
54
|
raise StaleHeadersError, "Headers cannot be changed because payload has already been published" if published?
|
55
55
|
headers[:published_on] ||= body.is_a?(Hash) && body.delete(:published_on) || Time.now.utc.to_i
|
@@ -60,19 +60,19 @@ module Cloudist
|
|
60
60
|
|
61
61
|
# this value should be unique for each published/received message pair
|
62
62
|
headers[:message_id] ||= id
|
63
|
-
|
63
|
+
|
64
64
|
# We use JSON for message transport exclusively
|
65
65
|
# headers[:content_type] ||= 'application/json'
|
66
|
-
|
66
|
+
|
67
67
|
# headers[:headers][:message_type] = 'event'
|
68
68
|
# ||= body.delete('message_type') || 'reply'
|
69
|
-
|
69
|
+
|
70
70
|
# headers[:headers] = custom_headers
|
71
|
-
|
71
|
+
|
72
72
|
# some strange behavior with integers makes it better to
|
73
73
|
# convert all amqp headers to strings to avoid any problems
|
74
74
|
headers.each { |k,v| headers[k] = v.to_s }
|
75
|
-
|
75
|
+
|
76
76
|
headers
|
77
77
|
end
|
78
78
|
|
@@ -88,59 +88,59 @@ module Cloudist
|
|
88
88
|
|
89
89
|
h
|
90
90
|
end
|
91
|
-
|
91
|
+
|
92
92
|
def set_reply_to(queue_name)
|
93
93
|
headers["reply_to"] = reply_name(queue_name)
|
94
94
|
set_master_queue_name(queue_name)
|
95
95
|
end
|
96
|
-
|
96
|
+
|
97
97
|
def set_master_queue_name(queue_name)
|
98
|
-
headers[:master_queue] = queue_name
|
98
|
+
headers[:master_queue] = queue_name
|
99
99
|
end
|
100
|
-
|
100
|
+
|
101
101
|
def reply_name(queue_name)
|
102
102
|
# "#{queue_name}.#{id}"
|
103
103
|
Utils.reply_prefix(queue_name)
|
104
104
|
end
|
105
|
-
|
105
|
+
|
106
106
|
def reply_to
|
107
107
|
headers["reply_to"]
|
108
108
|
end
|
109
|
-
|
109
|
+
|
110
110
|
def message_type
|
111
111
|
headers["message_type"]
|
112
112
|
end
|
113
|
-
|
113
|
+
|
114
114
|
def event_hash
|
115
115
|
@event_hash ||= headers["event_hash"] || create_event_hash
|
116
116
|
end
|
117
|
-
|
117
|
+
|
118
118
|
def create_event_hash
|
119
119
|
# s = Time.now.to_s + object_id.to_s + rand(100).to_s
|
120
120
|
# Digest::MD5.hexdigest(s)
|
121
121
|
UUID.generate
|
122
122
|
end
|
123
|
-
|
123
|
+
|
124
124
|
def parse_message(raw)
|
125
125
|
# return { } unless raw
|
126
126
|
# decode_json(raw)
|
127
127
|
decode_message(raw)
|
128
128
|
end
|
129
|
-
|
129
|
+
|
130
130
|
def [](key)
|
131
131
|
body[key]
|
132
132
|
end
|
133
|
-
|
133
|
+
|
134
134
|
def published?
|
135
135
|
@published == true
|
136
136
|
end
|
137
|
-
|
137
|
+
|
138
138
|
def publish
|
139
139
|
return if published?
|
140
140
|
@published = true
|
141
141
|
freeze!
|
142
142
|
end
|
143
|
-
|
143
|
+
|
144
144
|
def method_missing(meth, *args, &blk)
|
145
145
|
if body.is_a?(Hash) && body.has_key?(meth)
|
146
146
|
return body[meth]
|
@@ -150,6 +150,6 @@ module Cloudist
|
|
150
150
|
super
|
151
151
|
end
|
152
152
|
end
|
153
|
-
|
153
|
+
|
154
154
|
end
|
155
|
-
end
|
155
|
+
end
|