cloudist 0.2.1 → 0.4.1
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +15 -11
- data/Gemfile.lock +20 -7
- data/README.md +61 -39
- data/VERSION +1 -1
- data/cloudist.gemspec +50 -16
- data/examples/amqp/Gemfile +3 -0
- data/examples/amqp/Gemfile.lock +12 -0
- data/examples/amqp/amqp_consumer.rb +56 -0
- data/examples/amqp/amqp_publisher.rb +50 -0
- data/examples/queue_message.rb +7 -7
- data/examples/sandwich_client_with_custom_listener.rb +77 -0
- data/examples/sandwich_worker_with_class.rb +22 -7
- data/lib/cloudist.rb +113 -56
- data/lib/cloudist/application.rb +60 -0
- data/lib/cloudist/core_ext/class.rb +139 -0
- data/lib/cloudist/core_ext/kernel.rb +13 -0
- data/lib/cloudist/core_ext/module.rb +11 -0
- data/lib/cloudist/encoding.rb +13 -0
- data/lib/cloudist/errors.rb +2 -0
- data/lib/cloudist/job.rb +21 -18
- data/lib/cloudist/listener.rb +108 -54
- data/lib/cloudist/message.rb +97 -0
- data/lib/cloudist/messaging.rb +29 -0
- data/lib/cloudist/payload.rb +45 -105
- data/lib/cloudist/payload_old.rb +155 -0
- data/lib/cloudist/publisher.rb +7 -2
- data/lib/cloudist/queue.rb +152 -0
- data/lib/cloudist/queues/basic_queue.rb +83 -53
- data/lib/cloudist/queues/job_queue.rb +13 -24
- data/lib/cloudist/queues/reply_queue.rb +13 -21
- data/lib/cloudist/request.rb +33 -7
- data/lib/cloudist/worker.rb +9 -2
- data/lib/cloudist_old.rb +300 -0
- data/lib/em/em_timer_utils.rb +55 -0
- data/lib/em/iterator.rb +27 -0
- data/spec/cloudist/message_spec.rb +91 -0
- data/spec/cloudist/messaging_spec.rb +19 -0
- data/spec/cloudist/payload_spec.rb +10 -4
- data/spec/cloudist/payload_spec_2_spec.rb +78 -0
- data/spec/cloudist/queue_spec.rb +16 -0
- data/spec/cloudist_spec.rb +49 -45
- data/spec/spec_helper.rb +0 -1
- data/spec/support/amqp.rb +16 -0
- metadata +112 -102
- data/examples/extending_values.rb +0 -44
- data/examples/sandwich_client.rb +0 -57
- data/lib/cloudist/callback.rb +0 -16
- data/lib/cloudist/callback_methods.rb +0 -19
- data/lib/cloudist/callbacks/error_callback.rb +0 -14
@@ -0,0 +1,97 @@
|
|
1
|
+
module Cloudist
|
2
|
+
class Message
|
3
|
+
include Cloudist::Encoding
|
4
|
+
|
5
|
+
attr_reader :body, :headers, :id, :timestamp
|
6
|
+
|
7
|
+
# Expects body to be decoded
|
8
|
+
def initialize(body, headers = {})
|
9
|
+
@body = Hashie::Mash.new(body.dup)
|
10
|
+
|
11
|
+
@id ||= headers[:message_id] || headers[:id] && headers.delete(:id) || UUID.generate
|
12
|
+
@headers = Hashie::Mash.new(headers.dup)
|
13
|
+
|
14
|
+
@timestamp = Time.now.utc.to_f
|
15
|
+
|
16
|
+
update_headers(headers)
|
17
|
+
end
|
18
|
+
|
19
|
+
alias_method :data, :body
|
20
|
+
|
21
|
+
def update_headers(new_headers = {})
|
22
|
+
update_headers!
|
23
|
+
headers.merge!(new_headers)
|
24
|
+
end
|
25
|
+
|
26
|
+
def update_headers!
|
27
|
+
headers[:ttl] ||= Cloudist::DEFAULT_TTL
|
28
|
+
headers[:timestamp] = timestamp
|
29
|
+
headers[:message_id] ||= id
|
30
|
+
headers[:message_type] = 'message'
|
31
|
+
headers[:queue_name] ||= 'test'
|
32
|
+
|
33
|
+
headers.each { |k,v| headers[k] = v.to_s }
|
34
|
+
end
|
35
|
+
|
36
|
+
# Convenience method for replying
|
37
|
+
# Constructs a reply message and publishes it
|
38
|
+
def reply(body, reply_headers = {})
|
39
|
+
raise RuntimeError, "Cannot reply to an unpublished message" unless published?
|
40
|
+
|
41
|
+
msg = Message.new(body, reply_headers)
|
42
|
+
msg.set_reply_header
|
43
|
+
reply_q = Cloudist::ReplyQueue.new(headers[:queue_name])
|
44
|
+
msg.publish(reply_q)
|
45
|
+
end
|
46
|
+
|
47
|
+
# Publishes this message to the exchange or queue
|
48
|
+
# Queue should be a Cloudist::Queue object responding to #publish
|
49
|
+
def publish(queue)
|
50
|
+
raise ArgumentError, "Publish expects a Cloudist::Queue instance" unless queue.is_a?(Cloudist::Queue)
|
51
|
+
set_queue_name_header(queue)
|
52
|
+
update_published_date!
|
53
|
+
update_headers!
|
54
|
+
queue.publish(self)
|
55
|
+
end
|
56
|
+
|
57
|
+
def update_published_date!
|
58
|
+
headers[:published_on] = Time.now.utc.to_f
|
59
|
+
end
|
60
|
+
|
61
|
+
# This is so we can reply back to the sender
|
62
|
+
def set_queue_name_header(queue)
|
63
|
+
update_headers(:queue_name => queue.name)
|
64
|
+
end
|
65
|
+
|
66
|
+
def published?
|
67
|
+
@published ||= !!@headers.published_on
|
68
|
+
end
|
69
|
+
|
70
|
+
def created_at
|
71
|
+
headers.timestamp ? Time.at(headers.timestamp.to_f) : timestamp
|
72
|
+
end
|
73
|
+
|
74
|
+
def published_at
|
75
|
+
headers[:published_on] ? Time.at(headers[:published_on].to_f) : timestamp
|
76
|
+
end
|
77
|
+
|
78
|
+
def latency
|
79
|
+
(published_at.to_f - created_at.to_f)
|
80
|
+
end
|
81
|
+
|
82
|
+
def encoded
|
83
|
+
[encode(body), {:headers => headers}]
|
84
|
+
end
|
85
|
+
|
86
|
+
def inspect
|
87
|
+
"<#{self.class.name} id=#{id}>"
|
88
|
+
end
|
89
|
+
|
90
|
+
private
|
91
|
+
|
92
|
+
def set_reply_header
|
93
|
+
headers[:message_type] = 'reply'
|
94
|
+
end
|
95
|
+
|
96
|
+
end
|
97
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module Cloudist
|
2
|
+
autoload :Singleton, 'singleton'
|
3
|
+
|
4
|
+
class Messaging
|
5
|
+
include Singleton
|
6
|
+
|
7
|
+
class << self
|
8
|
+
|
9
|
+
def active_queues
|
10
|
+
instance.active_queues
|
11
|
+
end
|
12
|
+
|
13
|
+
def add_queue(queue)
|
14
|
+
(instance.active_queues ||= {}).merge!({queue.name.to_s => queue})
|
15
|
+
instance.active_queues
|
16
|
+
end
|
17
|
+
|
18
|
+
def remove_queue(queue_name)
|
19
|
+
(instance.active_queues ||= {}).delete(queue_name.to_s)
|
20
|
+
instance.active_queues
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
attr_accessor :active_queues
|
25
|
+
|
26
|
+
|
27
|
+
|
28
|
+
end
|
29
|
+
end
|
data/lib/cloudist/payload.rb
CHANGED
@@ -1,147 +1,87 @@
|
|
1
1
|
module Cloudist
|
2
|
-
DEFAULT_TTL = 300
|
3
|
-
|
4
2
|
class Payload
|
5
3
|
include Utils
|
4
|
+
include Encoding
|
6
5
|
|
7
|
-
attr_reader :body, :
|
8
|
-
|
9
|
-
def initialize(body, headers = {}
|
10
|
-
@publish_opts, @headers = publish_opts, headers
|
6
|
+
attr_reader :body, :headers, :amqp_headers, :timestamp
|
7
|
+
|
8
|
+
def initialize(body, headers = {})
|
11
9
|
@published = false
|
10
|
+
@timestamp = Time.now.to_f
|
12
11
|
|
13
|
-
body =
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
# HashWithIndifferentAccess.new(body)
|
19
|
-
update_headers
|
20
|
-
end
|
21
|
-
|
22
|
-
# Return message formatted as JSON and headers ready for transport in array
|
23
|
-
def formatted
|
24
|
-
update_headers
|
12
|
+
body = decode(body) if body.is_a?(String)
|
13
|
+
@body = Hashie::Mash.new(decode(body))
|
14
|
+
@headers = Hashie::Mash.new(headers)
|
15
|
+
@amqp_headers = {}
|
16
|
+
# puts "Initialised Payload: #{id}"
|
25
17
|
|
26
|
-
|
27
|
-
end
|
28
|
-
|
29
|
-
def id
|
30
|
-
@id ||= event_hash.to_s
|
31
|
-
end
|
32
|
-
|
33
|
-
def id=(new_id)
|
34
|
-
@id = new_id.to_s
|
35
|
-
update_headers
|
18
|
+
parse_headers!
|
36
19
|
end
|
37
20
|
|
38
|
-
def
|
39
|
-
headers.
|
21
|
+
def find_or_create_id
|
22
|
+
if headers["message_id"].present?
|
23
|
+
headers.message_id
|
24
|
+
else
|
25
|
+
UUID.generate
|
26
|
+
end
|
40
27
|
end
|
41
28
|
|
42
|
-
def
|
43
|
-
|
44
|
-
body.freeze
|
29
|
+
def id
|
30
|
+
find_or_create_id
|
45
31
|
end
|
46
32
|
|
47
|
-
def
|
48
|
-
headers
|
49
|
-
(publish_opts[:headers] ||= {}).merge!(headers)
|
33
|
+
def to_a
|
34
|
+
[encode(body), {:headers => encoded_headers}]
|
50
35
|
end
|
51
36
|
|
52
|
-
def
|
53
|
-
|
54
|
-
headers[:
|
55
|
-
headers[:ttl] ||= body.is_a?(Hash) && body.delete('ttl') || Cloudist::DEFAULT_TTL
|
56
|
-
|
57
|
-
# this is the event hash that gets transferred through various publish/reply actions
|
58
|
-
headers[:event_hash] ||= id
|
59
|
-
|
60
|
-
# this value should be unique for each published/received message pair
|
61
|
-
headers[:message_id] ||= id
|
37
|
+
def parse_headers!
|
38
|
+
headers[:published_on] ||= body.delete("timestamp") || timestamp
|
39
|
+
headers[:message_type] ||= body.delete("message_type") || 'reply'
|
62
40
|
|
63
|
-
|
64
|
-
|
41
|
+
headers[:ttl] ||= Cloudist::DEFAULT_TTL
|
42
|
+
headers[:message_id] = id
|
65
43
|
|
66
|
-
|
67
|
-
# ||= body.delete('message_type') || 'reply'
|
44
|
+
headers[:published_on] = headers[:published_on].to_f
|
68
45
|
|
69
|
-
|
46
|
+
headers[:ttl] = headers[:ttl].to_i rescue -1
|
47
|
+
headers[:ttl] = -1 if headers[:ttl] == 0
|
70
48
|
|
71
|
-
#
|
72
|
-
#
|
73
|
-
|
49
|
+
# If this payload was received with a timestamp,
|
50
|
+
# we don't want to override it on #timestamp
|
51
|
+
if timestamp > headers[:published_on]
|
52
|
+
@timestamp = headers[:published_on]
|
53
|
+
end
|
74
54
|
|
75
55
|
headers
|
76
56
|
end
|
77
|
-
|
78
|
-
def
|
79
|
-
return { } unless headers
|
80
|
-
|
57
|
+
|
58
|
+
def encoded_headers
|
81
59
|
h = headers.dup
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
h[:ttl] = h[:ttl].to_i rescue -1
|
86
|
-
h[:ttl] = -1 if h[:ttl] == 0
|
87
|
-
|
88
|
-
h
|
60
|
+
h.each { |k,v| h[k] = v.to_s }
|
61
|
+
return h
|
89
62
|
end
|
90
63
|
|
91
64
|
def set_reply_to(queue_name)
|
92
|
-
headers[:reply_to] =
|
93
|
-
set_master_queue_name(queue_name)
|
94
|
-
end
|
95
|
-
|
96
|
-
def set_master_queue_name(queue_name)
|
97
|
-
headers[:master_queue] = queue_name
|
98
|
-
end
|
99
|
-
|
100
|
-
def reply_name(queue_name)
|
101
|
-
# "#{queue_name}.#{id}"
|
102
|
-
Utils.reply_prefix(queue_name)
|
65
|
+
headers[:reply_to] = reply_prefix(queue_name)
|
103
66
|
end
|
104
67
|
|
105
68
|
def reply_to
|
106
|
-
headers
|
69
|
+
headers.reply_to
|
107
70
|
end
|
108
71
|
|
109
72
|
def message_type
|
110
|
-
headers
|
111
|
-
end
|
112
|
-
|
113
|
-
def event_hash
|
114
|
-
@event_hash ||= headers[:event_hash] || body.is_a?(Hash) && body.delete(:event_hash) || create_event_hash
|
115
|
-
end
|
116
|
-
|
117
|
-
def create_event_hash
|
118
|
-
s = Time.now.to_s + object_id.to_s + rand(100).to_s
|
119
|
-
Digest::MD5.hexdigest(s)
|
120
|
-
end
|
121
|
-
|
122
|
-
def parse_message(raw)
|
123
|
-
# return { } unless raw
|
124
|
-
# decode_json(raw)
|
125
|
-
decode_message(raw)
|
73
|
+
headers.message_type
|
126
74
|
end
|
127
75
|
|
128
76
|
def [](key)
|
129
|
-
body[key]
|
130
|
-
end
|
131
|
-
|
132
|
-
def published?
|
133
|
-
@published == true
|
134
|
-
end
|
135
|
-
|
136
|
-
def publish
|
137
|
-
return if published?
|
138
|
-
@published = true
|
139
|
-
freeze!
|
77
|
+
self.body[key.to_s]
|
140
78
|
end
|
141
79
|
|
142
80
|
def method_missing(meth, *args, &blk)
|
143
|
-
if body.
|
81
|
+
if body.has_key?(meth.to_s)
|
144
82
|
return body[meth]
|
83
|
+
elsif key = meth.to_s.match(/(.+)(?:\?$)/).to_a.last
|
84
|
+
body.has_key?(key.to_s)
|
145
85
|
else
|
146
86
|
super
|
147
87
|
end
|
@@ -0,0 +1,155 @@
|
|
1
|
+
module Cloudist
|
2
|
+
class Payload
|
3
|
+
include Utils
|
4
|
+
|
5
|
+
attr_reader :body, :publish_opts, :headers, :timestamp
|
6
|
+
|
7
|
+
def initialize(body, headers = {}, publish_opts = {})
|
8
|
+
@publish_opts, @headers = publish_opts, Hashie::Mash.new(headers)
|
9
|
+
@published = false
|
10
|
+
|
11
|
+
body = parse_message(body) if body.is_a?(String)
|
12
|
+
|
13
|
+
# raise Cloudist::BadPayload, "Expected Hash for payload" unless body.is_a?(Hash)
|
14
|
+
|
15
|
+
@timestamp = Time.now.to_f
|
16
|
+
|
17
|
+
@body = body
|
18
|
+
# Hashie::Mash.new(body)
|
19
|
+
|
20
|
+
update_headers
|
21
|
+
end
|
22
|
+
|
23
|
+
# Return message formatted as JSON and headers ready for transport in array
|
24
|
+
def formatted
|
25
|
+
update_headers
|
26
|
+
|
27
|
+
[encode_message(body), publish_opts]
|
28
|
+
end
|
29
|
+
|
30
|
+
def id
|
31
|
+
@id ||= event_hash.to_s
|
32
|
+
end
|
33
|
+
|
34
|
+
def id=(new_id)
|
35
|
+
@id = new_id.to_s
|
36
|
+
update_headers
|
37
|
+
end
|
38
|
+
|
39
|
+
def frozen?
|
40
|
+
headers.frozen?
|
41
|
+
end
|
42
|
+
|
43
|
+
def freeze!
|
44
|
+
headers.freeze
|
45
|
+
body.freeze
|
46
|
+
end
|
47
|
+
|
48
|
+
def update_headers
|
49
|
+
headers = extract_custom_headers
|
50
|
+
(publish_opts[:headers] ||= {}).merge!(headers)
|
51
|
+
end
|
52
|
+
|
53
|
+
def extract_custom_headers
|
54
|
+
raise StaleHeadersError, "Headers cannot be changed because payload has already been published" if published?
|
55
|
+
headers[:published_on] ||= body.is_a?(Hash) && body.delete(:published_on) || Time.now.utc.to_i
|
56
|
+
headers[:ttl] ||= body.is_a?(Hash) && body.delete('ttl') || Cloudist::DEFAULT_TTL
|
57
|
+
headers[:timestamp] = timestamp
|
58
|
+
# this is the event hash that gets transferred through various publish/reply actions
|
59
|
+
headers[:event_hash] ||= id
|
60
|
+
|
61
|
+
# this value should be unique for each published/received message pair
|
62
|
+
headers[:message_id] ||= id
|
63
|
+
|
64
|
+
# We use JSON for message transport exclusively
|
65
|
+
# headers[:content_type] ||= 'application/json'
|
66
|
+
|
67
|
+
# headers[:headers][:message_type] = 'event'
|
68
|
+
# ||= body.delete('message_type') || 'reply'
|
69
|
+
|
70
|
+
# headers[:headers] = custom_headers
|
71
|
+
|
72
|
+
# some strange behavior with integers makes it better to
|
73
|
+
# convert all amqp headers to strings to avoid any problems
|
74
|
+
headers.each { |k,v| headers[k] = v.to_s }
|
75
|
+
|
76
|
+
headers
|
77
|
+
end
|
78
|
+
|
79
|
+
def parse_custom_headers
|
80
|
+
return { } unless headers
|
81
|
+
|
82
|
+
h = headers.dup
|
83
|
+
|
84
|
+
h[:published_on] = h[:published_on].to_i
|
85
|
+
|
86
|
+
h[:ttl] = h[:ttl].to_i rescue -1
|
87
|
+
h[:ttl] = -1 if h[:ttl] == 0
|
88
|
+
|
89
|
+
h
|
90
|
+
end
|
91
|
+
|
92
|
+
def set_reply_to(queue_name)
|
93
|
+
headers["reply_to"] = reply_name(queue_name)
|
94
|
+
set_master_queue_name(queue_name)
|
95
|
+
end
|
96
|
+
|
97
|
+
def set_master_queue_name(queue_name)
|
98
|
+
headers[:master_queue] = queue_name
|
99
|
+
end
|
100
|
+
|
101
|
+
def reply_name(queue_name)
|
102
|
+
# "#{queue_name}.#{id}"
|
103
|
+
Utils.reply_prefix(queue_name)
|
104
|
+
end
|
105
|
+
|
106
|
+
def reply_to
|
107
|
+
headers["reply_to"]
|
108
|
+
end
|
109
|
+
|
110
|
+
def message_type
|
111
|
+
headers["message_type"]
|
112
|
+
end
|
113
|
+
|
114
|
+
def event_hash
|
115
|
+
@event_hash ||= headers["event_hash"] || create_event_hash
|
116
|
+
end
|
117
|
+
|
118
|
+
def create_event_hash
|
119
|
+
# s = Time.now.to_s + object_id.to_s + rand(100).to_s
|
120
|
+
# Digest::MD5.hexdigest(s)
|
121
|
+
UUID.generate
|
122
|
+
end
|
123
|
+
|
124
|
+
def parse_message(raw)
|
125
|
+
# return { } unless raw
|
126
|
+
# decode_json(raw)
|
127
|
+
decode_message(raw)
|
128
|
+
end
|
129
|
+
|
130
|
+
def [](key)
|
131
|
+
body[key]
|
132
|
+
end
|
133
|
+
|
134
|
+
def published?
|
135
|
+
@published == true
|
136
|
+
end
|
137
|
+
|
138
|
+
def publish
|
139
|
+
return if published?
|
140
|
+
@published = true
|
141
|
+
freeze!
|
142
|
+
end
|
143
|
+
|
144
|
+
def method_missing(meth, *args, &blk)
|
145
|
+
if body.is_a?(Hash) && body.has_key?(meth)
|
146
|
+
return body[meth]
|
147
|
+
elsif key = meth.to_s.match(/(.+)(?:\?$)/).to_a.last
|
148
|
+
body.has_key?(key.to_sym)
|
149
|
+
else
|
150
|
+
super
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
end
|
155
|
+
end
|