cloudist 0.2.1 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +15 -11
- data/Gemfile.lock +20 -7
- data/README.md +61 -39
- data/VERSION +1 -1
- data/cloudist.gemspec +50 -16
- data/examples/amqp/Gemfile +3 -0
- data/examples/amqp/Gemfile.lock +12 -0
- data/examples/amqp/amqp_consumer.rb +56 -0
- data/examples/amqp/amqp_publisher.rb +50 -0
- data/examples/queue_message.rb +7 -7
- data/examples/sandwich_client_with_custom_listener.rb +77 -0
- data/examples/sandwich_worker_with_class.rb +22 -7
- data/lib/cloudist.rb +113 -56
- data/lib/cloudist/application.rb +60 -0
- data/lib/cloudist/core_ext/class.rb +139 -0
- data/lib/cloudist/core_ext/kernel.rb +13 -0
- data/lib/cloudist/core_ext/module.rb +11 -0
- data/lib/cloudist/encoding.rb +13 -0
- data/lib/cloudist/errors.rb +2 -0
- data/lib/cloudist/job.rb +21 -18
- data/lib/cloudist/listener.rb +108 -54
- data/lib/cloudist/message.rb +97 -0
- data/lib/cloudist/messaging.rb +29 -0
- data/lib/cloudist/payload.rb +45 -105
- data/lib/cloudist/payload_old.rb +155 -0
- data/lib/cloudist/publisher.rb +7 -2
- data/lib/cloudist/queue.rb +152 -0
- data/lib/cloudist/queues/basic_queue.rb +83 -53
- data/lib/cloudist/queues/job_queue.rb +13 -24
- data/lib/cloudist/queues/reply_queue.rb +13 -21
- data/lib/cloudist/request.rb +33 -7
- data/lib/cloudist/worker.rb +9 -2
- data/lib/cloudist_old.rb +300 -0
- data/lib/em/em_timer_utils.rb +55 -0
- data/lib/em/iterator.rb +27 -0
- data/spec/cloudist/message_spec.rb +91 -0
- data/spec/cloudist/messaging_spec.rb +19 -0
- data/spec/cloudist/payload_spec.rb +10 -4
- data/spec/cloudist/payload_spec_2_spec.rb +78 -0
- data/spec/cloudist/queue_spec.rb +16 -0
- data/spec/cloudist_spec.rb +49 -45
- data/spec/spec_helper.rb +0 -1
- data/spec/support/amqp.rb +16 -0
- metadata +112 -102
- data/examples/extending_values.rb +0 -44
- data/examples/sandwich_client.rb +0 -57
- data/lib/cloudist/callback.rb +0 -16
- data/lib/cloudist/callback_methods.rb +0 -19
- data/lib/cloudist/callbacks/error_callback.rb +0 -14
@@ -0,0 +1,97 @@
|
|
1
|
+
module Cloudist
|
2
|
+
class Message
|
3
|
+
include Cloudist::Encoding
|
4
|
+
|
5
|
+
attr_reader :body, :headers, :id, :timestamp
|
6
|
+
|
7
|
+
# Expects body to be decoded
|
8
|
+
def initialize(body, headers = {})
|
9
|
+
@body = Hashie::Mash.new(body.dup)
|
10
|
+
|
11
|
+
@id ||= headers[:message_id] || headers[:id] && headers.delete(:id) || UUID.generate
|
12
|
+
@headers = Hashie::Mash.new(headers.dup)
|
13
|
+
|
14
|
+
@timestamp = Time.now.utc.to_f
|
15
|
+
|
16
|
+
update_headers(headers)
|
17
|
+
end
|
18
|
+
|
19
|
+
alias_method :data, :body
|
20
|
+
|
21
|
+
def update_headers(new_headers = {})
|
22
|
+
update_headers!
|
23
|
+
headers.merge!(new_headers)
|
24
|
+
end
|
25
|
+
|
26
|
+
def update_headers!
|
27
|
+
headers[:ttl] ||= Cloudist::DEFAULT_TTL
|
28
|
+
headers[:timestamp] = timestamp
|
29
|
+
headers[:message_id] ||= id
|
30
|
+
headers[:message_type] = 'message'
|
31
|
+
headers[:queue_name] ||= 'test'
|
32
|
+
|
33
|
+
headers.each { |k,v| headers[k] = v.to_s }
|
34
|
+
end
|
35
|
+
|
36
|
+
# Convenience method for replying
|
37
|
+
# Constructs a reply message and publishes it
|
38
|
+
def reply(body, reply_headers = {})
|
39
|
+
raise RuntimeError, "Cannot reply to an unpublished message" unless published?
|
40
|
+
|
41
|
+
msg = Message.new(body, reply_headers)
|
42
|
+
msg.set_reply_header
|
43
|
+
reply_q = Cloudist::ReplyQueue.new(headers[:queue_name])
|
44
|
+
msg.publish(reply_q)
|
45
|
+
end
|
46
|
+
|
47
|
+
# Publishes this message to the exchange or queue
|
48
|
+
# Queue should be a Cloudist::Queue object responding to #publish
|
49
|
+
def publish(queue)
|
50
|
+
raise ArgumentError, "Publish expects a Cloudist::Queue instance" unless queue.is_a?(Cloudist::Queue)
|
51
|
+
set_queue_name_header(queue)
|
52
|
+
update_published_date!
|
53
|
+
update_headers!
|
54
|
+
queue.publish(self)
|
55
|
+
end
|
56
|
+
|
57
|
+
def update_published_date!
|
58
|
+
headers[:published_on] = Time.now.utc.to_f
|
59
|
+
end
|
60
|
+
|
61
|
+
# This is so we can reply back to the sender
|
62
|
+
def set_queue_name_header(queue)
|
63
|
+
update_headers(:queue_name => queue.name)
|
64
|
+
end
|
65
|
+
|
66
|
+
def published?
|
67
|
+
@published ||= !!@headers.published_on
|
68
|
+
end
|
69
|
+
|
70
|
+
def created_at
|
71
|
+
headers.timestamp ? Time.at(headers.timestamp.to_f) : timestamp
|
72
|
+
end
|
73
|
+
|
74
|
+
def published_at
|
75
|
+
headers[:published_on] ? Time.at(headers[:published_on].to_f) : timestamp
|
76
|
+
end
|
77
|
+
|
78
|
+
def latency
|
79
|
+
(published_at.to_f - created_at.to_f)
|
80
|
+
end
|
81
|
+
|
82
|
+
def encoded
|
83
|
+
[encode(body), {:headers => headers}]
|
84
|
+
end
|
85
|
+
|
86
|
+
def inspect
|
87
|
+
"<#{self.class.name} id=#{id}>"
|
88
|
+
end
|
89
|
+
|
90
|
+
private
|
91
|
+
|
92
|
+
def set_reply_header
|
93
|
+
headers[:message_type] = 'reply'
|
94
|
+
end
|
95
|
+
|
96
|
+
end
|
97
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module Cloudist
|
2
|
+
autoload :Singleton, 'singleton'
|
3
|
+
|
4
|
+
class Messaging
|
5
|
+
include Singleton
|
6
|
+
|
7
|
+
class << self
|
8
|
+
|
9
|
+
def active_queues
|
10
|
+
instance.active_queues
|
11
|
+
end
|
12
|
+
|
13
|
+
def add_queue(queue)
|
14
|
+
(instance.active_queues ||= {}).merge!({queue.name.to_s => queue})
|
15
|
+
instance.active_queues
|
16
|
+
end
|
17
|
+
|
18
|
+
def remove_queue(queue_name)
|
19
|
+
(instance.active_queues ||= {}).delete(queue_name.to_s)
|
20
|
+
instance.active_queues
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
attr_accessor :active_queues
|
25
|
+
|
26
|
+
|
27
|
+
|
28
|
+
end
|
29
|
+
end
|
data/lib/cloudist/payload.rb
CHANGED
@@ -1,147 +1,87 @@
|
|
1
1
|
module Cloudist
|
2
|
-
DEFAULT_TTL = 300
|
3
|
-
|
4
2
|
class Payload
|
5
3
|
include Utils
|
4
|
+
include Encoding
|
6
5
|
|
7
|
-
attr_reader :body, :
|
8
|
-
|
9
|
-
def initialize(body, headers = {}
|
10
|
-
@publish_opts, @headers = publish_opts, headers
|
6
|
+
attr_reader :body, :headers, :amqp_headers, :timestamp
|
7
|
+
|
8
|
+
def initialize(body, headers = {})
|
11
9
|
@published = false
|
10
|
+
@timestamp = Time.now.to_f
|
12
11
|
|
13
|
-
body =
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
# HashWithIndifferentAccess.new(body)
|
19
|
-
update_headers
|
20
|
-
end
|
21
|
-
|
22
|
-
# Return message formatted as JSON and headers ready for transport in array
|
23
|
-
def formatted
|
24
|
-
update_headers
|
12
|
+
body = decode(body) if body.is_a?(String)
|
13
|
+
@body = Hashie::Mash.new(decode(body))
|
14
|
+
@headers = Hashie::Mash.new(headers)
|
15
|
+
@amqp_headers = {}
|
16
|
+
# puts "Initialised Payload: #{id}"
|
25
17
|
|
26
|
-
|
27
|
-
end
|
28
|
-
|
29
|
-
def id
|
30
|
-
@id ||= event_hash.to_s
|
31
|
-
end
|
32
|
-
|
33
|
-
def id=(new_id)
|
34
|
-
@id = new_id.to_s
|
35
|
-
update_headers
|
18
|
+
parse_headers!
|
36
19
|
end
|
37
20
|
|
38
|
-
def
|
39
|
-
headers.
|
21
|
+
def find_or_create_id
|
22
|
+
if headers["message_id"].present?
|
23
|
+
headers.message_id
|
24
|
+
else
|
25
|
+
UUID.generate
|
26
|
+
end
|
40
27
|
end
|
41
28
|
|
42
|
-
def
|
43
|
-
|
44
|
-
body.freeze
|
29
|
+
def id
|
30
|
+
find_or_create_id
|
45
31
|
end
|
46
32
|
|
47
|
-
def
|
48
|
-
headers
|
49
|
-
(publish_opts[:headers] ||= {}).merge!(headers)
|
33
|
+
def to_a
|
34
|
+
[encode(body), {:headers => encoded_headers}]
|
50
35
|
end
|
51
36
|
|
52
|
-
def
|
53
|
-
|
54
|
-
headers[:
|
55
|
-
headers[:ttl] ||= body.is_a?(Hash) && body.delete('ttl') || Cloudist::DEFAULT_TTL
|
56
|
-
|
57
|
-
# this is the event hash that gets transferred through various publish/reply actions
|
58
|
-
headers[:event_hash] ||= id
|
59
|
-
|
60
|
-
# this value should be unique for each published/received message pair
|
61
|
-
headers[:message_id] ||= id
|
37
|
+
def parse_headers!
|
38
|
+
headers[:published_on] ||= body.delete("timestamp") || timestamp
|
39
|
+
headers[:message_type] ||= body.delete("message_type") || 'reply'
|
62
40
|
|
63
|
-
|
64
|
-
|
41
|
+
headers[:ttl] ||= Cloudist::DEFAULT_TTL
|
42
|
+
headers[:message_id] = id
|
65
43
|
|
66
|
-
|
67
|
-
# ||= body.delete('message_type') || 'reply'
|
44
|
+
headers[:published_on] = headers[:published_on].to_f
|
68
45
|
|
69
|
-
|
46
|
+
headers[:ttl] = headers[:ttl].to_i rescue -1
|
47
|
+
headers[:ttl] = -1 if headers[:ttl] == 0
|
70
48
|
|
71
|
-
#
|
72
|
-
#
|
73
|
-
|
49
|
+
# If this payload was received with a timestamp,
|
50
|
+
# we don't want to override it on #timestamp
|
51
|
+
if timestamp > headers[:published_on]
|
52
|
+
@timestamp = headers[:published_on]
|
53
|
+
end
|
74
54
|
|
75
55
|
headers
|
76
56
|
end
|
77
|
-
|
78
|
-
def
|
79
|
-
return { } unless headers
|
80
|
-
|
57
|
+
|
58
|
+
def encoded_headers
|
81
59
|
h = headers.dup
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
h[:ttl] = h[:ttl].to_i rescue -1
|
86
|
-
h[:ttl] = -1 if h[:ttl] == 0
|
87
|
-
|
88
|
-
h
|
60
|
+
h.each { |k,v| h[k] = v.to_s }
|
61
|
+
return h
|
89
62
|
end
|
90
63
|
|
91
64
|
def set_reply_to(queue_name)
|
92
|
-
headers[:reply_to] =
|
93
|
-
set_master_queue_name(queue_name)
|
94
|
-
end
|
95
|
-
|
96
|
-
def set_master_queue_name(queue_name)
|
97
|
-
headers[:master_queue] = queue_name
|
98
|
-
end
|
99
|
-
|
100
|
-
def reply_name(queue_name)
|
101
|
-
# "#{queue_name}.#{id}"
|
102
|
-
Utils.reply_prefix(queue_name)
|
65
|
+
headers[:reply_to] = reply_prefix(queue_name)
|
103
66
|
end
|
104
67
|
|
105
68
|
def reply_to
|
106
|
-
headers
|
69
|
+
headers.reply_to
|
107
70
|
end
|
108
71
|
|
109
72
|
def message_type
|
110
|
-
headers
|
111
|
-
end
|
112
|
-
|
113
|
-
def event_hash
|
114
|
-
@event_hash ||= headers[:event_hash] || body.is_a?(Hash) && body.delete(:event_hash) || create_event_hash
|
115
|
-
end
|
116
|
-
|
117
|
-
def create_event_hash
|
118
|
-
s = Time.now.to_s + object_id.to_s + rand(100).to_s
|
119
|
-
Digest::MD5.hexdigest(s)
|
120
|
-
end
|
121
|
-
|
122
|
-
def parse_message(raw)
|
123
|
-
# return { } unless raw
|
124
|
-
# decode_json(raw)
|
125
|
-
decode_message(raw)
|
73
|
+
headers.message_type
|
126
74
|
end
|
127
75
|
|
128
76
|
def [](key)
|
129
|
-
body[key]
|
130
|
-
end
|
131
|
-
|
132
|
-
def published?
|
133
|
-
@published == true
|
134
|
-
end
|
135
|
-
|
136
|
-
def publish
|
137
|
-
return if published?
|
138
|
-
@published = true
|
139
|
-
freeze!
|
77
|
+
self.body[key.to_s]
|
140
78
|
end
|
141
79
|
|
142
80
|
def method_missing(meth, *args, &blk)
|
143
|
-
if body.
|
81
|
+
if body.has_key?(meth.to_s)
|
144
82
|
return body[meth]
|
83
|
+
elsif key = meth.to_s.match(/(.+)(?:\?$)/).to_a.last
|
84
|
+
body.has_key?(key.to_s)
|
145
85
|
else
|
146
86
|
super
|
147
87
|
end
|
@@ -0,0 +1,155 @@
|
|
1
|
+
module Cloudist
|
2
|
+
class Payload
|
3
|
+
include Utils
|
4
|
+
|
5
|
+
attr_reader :body, :publish_opts, :headers, :timestamp
|
6
|
+
|
7
|
+
def initialize(body, headers = {}, publish_opts = {})
|
8
|
+
@publish_opts, @headers = publish_opts, Hashie::Mash.new(headers)
|
9
|
+
@published = false
|
10
|
+
|
11
|
+
body = parse_message(body) if body.is_a?(String)
|
12
|
+
|
13
|
+
# raise Cloudist::BadPayload, "Expected Hash for payload" unless body.is_a?(Hash)
|
14
|
+
|
15
|
+
@timestamp = Time.now.to_f
|
16
|
+
|
17
|
+
@body = body
|
18
|
+
# Hashie::Mash.new(body)
|
19
|
+
|
20
|
+
update_headers
|
21
|
+
end
|
22
|
+
|
23
|
+
# Return message formatted as JSON and headers ready for transport in array
|
24
|
+
def formatted
|
25
|
+
update_headers
|
26
|
+
|
27
|
+
[encode_message(body), publish_opts]
|
28
|
+
end
|
29
|
+
|
30
|
+
def id
|
31
|
+
@id ||= event_hash.to_s
|
32
|
+
end
|
33
|
+
|
34
|
+
def id=(new_id)
|
35
|
+
@id = new_id.to_s
|
36
|
+
update_headers
|
37
|
+
end
|
38
|
+
|
39
|
+
def frozen?
|
40
|
+
headers.frozen?
|
41
|
+
end
|
42
|
+
|
43
|
+
def freeze!
|
44
|
+
headers.freeze
|
45
|
+
body.freeze
|
46
|
+
end
|
47
|
+
|
48
|
+
def update_headers
|
49
|
+
headers = extract_custom_headers
|
50
|
+
(publish_opts[:headers] ||= {}).merge!(headers)
|
51
|
+
end
|
52
|
+
|
53
|
+
def extract_custom_headers
|
54
|
+
raise StaleHeadersError, "Headers cannot be changed because payload has already been published" if published?
|
55
|
+
headers[:published_on] ||= body.is_a?(Hash) && body.delete(:published_on) || Time.now.utc.to_i
|
56
|
+
headers[:ttl] ||= body.is_a?(Hash) && body.delete('ttl') || Cloudist::DEFAULT_TTL
|
57
|
+
headers[:timestamp] = timestamp
|
58
|
+
# this is the event hash that gets transferred through various publish/reply actions
|
59
|
+
headers[:event_hash] ||= id
|
60
|
+
|
61
|
+
# this value should be unique for each published/received message pair
|
62
|
+
headers[:message_id] ||= id
|
63
|
+
|
64
|
+
# We use JSON for message transport exclusively
|
65
|
+
# headers[:content_type] ||= 'application/json'
|
66
|
+
|
67
|
+
# headers[:headers][:message_type] = 'event'
|
68
|
+
# ||= body.delete('message_type') || 'reply'
|
69
|
+
|
70
|
+
# headers[:headers] = custom_headers
|
71
|
+
|
72
|
+
# some strange behavior with integers makes it better to
|
73
|
+
# convert all amqp headers to strings to avoid any problems
|
74
|
+
headers.each { |k,v| headers[k] = v.to_s }
|
75
|
+
|
76
|
+
headers
|
77
|
+
end
|
78
|
+
|
79
|
+
def parse_custom_headers
|
80
|
+
return { } unless headers
|
81
|
+
|
82
|
+
h = headers.dup
|
83
|
+
|
84
|
+
h[:published_on] = h[:published_on].to_i
|
85
|
+
|
86
|
+
h[:ttl] = h[:ttl].to_i rescue -1
|
87
|
+
h[:ttl] = -1 if h[:ttl] == 0
|
88
|
+
|
89
|
+
h
|
90
|
+
end
|
91
|
+
|
92
|
+
def set_reply_to(queue_name)
|
93
|
+
headers["reply_to"] = reply_name(queue_name)
|
94
|
+
set_master_queue_name(queue_name)
|
95
|
+
end
|
96
|
+
|
97
|
+
def set_master_queue_name(queue_name)
|
98
|
+
headers[:master_queue] = queue_name
|
99
|
+
end
|
100
|
+
|
101
|
+
def reply_name(queue_name)
|
102
|
+
# "#{queue_name}.#{id}"
|
103
|
+
Utils.reply_prefix(queue_name)
|
104
|
+
end
|
105
|
+
|
106
|
+
def reply_to
|
107
|
+
headers["reply_to"]
|
108
|
+
end
|
109
|
+
|
110
|
+
def message_type
|
111
|
+
headers["message_type"]
|
112
|
+
end
|
113
|
+
|
114
|
+
def event_hash
|
115
|
+
@event_hash ||= headers["event_hash"] || create_event_hash
|
116
|
+
end
|
117
|
+
|
118
|
+
def create_event_hash
|
119
|
+
# s = Time.now.to_s + object_id.to_s + rand(100).to_s
|
120
|
+
# Digest::MD5.hexdigest(s)
|
121
|
+
UUID.generate
|
122
|
+
end
|
123
|
+
|
124
|
+
def parse_message(raw)
|
125
|
+
# return { } unless raw
|
126
|
+
# decode_json(raw)
|
127
|
+
decode_message(raw)
|
128
|
+
end
|
129
|
+
|
130
|
+
def [](key)
|
131
|
+
body[key]
|
132
|
+
end
|
133
|
+
|
134
|
+
def published?
|
135
|
+
@published == true
|
136
|
+
end
|
137
|
+
|
138
|
+
def publish
|
139
|
+
return if published?
|
140
|
+
@published = true
|
141
|
+
freeze!
|
142
|
+
end
|
143
|
+
|
144
|
+
def method_missing(meth, *args, &blk)
|
145
|
+
if body.is_a?(Hash) && body.has_key?(meth)
|
146
|
+
return body[meth]
|
147
|
+
elsif key = meth.to_s.match(/(.+)(?:\?$)/).to_a.last
|
148
|
+
body.has_key?(key.to_sym)
|
149
|
+
else
|
150
|
+
super
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
end
|
155
|
+
end
|