cloudist 0.2.1 → 0.4.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (49) hide show
  1. data/Gemfile +15 -11
  2. data/Gemfile.lock +20 -7
  3. data/README.md +61 -39
  4. data/VERSION +1 -1
  5. data/cloudist.gemspec +50 -16
  6. data/examples/amqp/Gemfile +3 -0
  7. data/examples/amqp/Gemfile.lock +12 -0
  8. data/examples/amqp/amqp_consumer.rb +56 -0
  9. data/examples/amqp/amqp_publisher.rb +50 -0
  10. data/examples/queue_message.rb +7 -7
  11. data/examples/sandwich_client_with_custom_listener.rb +77 -0
  12. data/examples/sandwich_worker_with_class.rb +22 -7
  13. data/lib/cloudist.rb +113 -56
  14. data/lib/cloudist/application.rb +60 -0
  15. data/lib/cloudist/core_ext/class.rb +139 -0
  16. data/lib/cloudist/core_ext/kernel.rb +13 -0
  17. data/lib/cloudist/core_ext/module.rb +11 -0
  18. data/lib/cloudist/encoding.rb +13 -0
  19. data/lib/cloudist/errors.rb +2 -0
  20. data/lib/cloudist/job.rb +21 -18
  21. data/lib/cloudist/listener.rb +108 -54
  22. data/lib/cloudist/message.rb +97 -0
  23. data/lib/cloudist/messaging.rb +29 -0
  24. data/lib/cloudist/payload.rb +45 -105
  25. data/lib/cloudist/payload_old.rb +155 -0
  26. data/lib/cloudist/publisher.rb +7 -2
  27. data/lib/cloudist/queue.rb +152 -0
  28. data/lib/cloudist/queues/basic_queue.rb +83 -53
  29. data/lib/cloudist/queues/job_queue.rb +13 -24
  30. data/lib/cloudist/queues/reply_queue.rb +13 -21
  31. data/lib/cloudist/request.rb +33 -7
  32. data/lib/cloudist/worker.rb +9 -2
  33. data/lib/cloudist_old.rb +300 -0
  34. data/lib/em/em_timer_utils.rb +55 -0
  35. data/lib/em/iterator.rb +27 -0
  36. data/spec/cloudist/message_spec.rb +91 -0
  37. data/spec/cloudist/messaging_spec.rb +19 -0
  38. data/spec/cloudist/payload_spec.rb +10 -4
  39. data/spec/cloudist/payload_spec_2_spec.rb +78 -0
  40. data/spec/cloudist/queue_spec.rb +16 -0
  41. data/spec/cloudist_spec.rb +49 -45
  42. data/spec/spec_helper.rb +0 -1
  43. data/spec/support/amqp.rb +16 -0
  44. metadata +112 -102
  45. data/examples/extending_values.rb +0 -44
  46. data/examples/sandwich_client.rb +0 -57
  47. data/lib/cloudist/callback.rb +0 -16
  48. data/lib/cloudist/callback_methods.rb +0 -19
  49. data/lib/cloudist/callbacks/error_callback.rb +0 -14
@@ -0,0 +1,97 @@
1
+ module Cloudist
2
+ class Message
3
+ include Cloudist::Encoding
4
+
5
+ attr_reader :body, :headers, :id, :timestamp
6
+
7
+ # Expects body to be decoded
8
+ def initialize(body, headers = {})
9
+ @body = Hashie::Mash.new(body.dup)
10
+
11
+ @id ||= headers[:message_id] || headers[:id] && headers.delete(:id) || UUID.generate
12
+ @headers = Hashie::Mash.new(headers.dup)
13
+
14
+ @timestamp = Time.now.utc.to_f
15
+
16
+ update_headers(headers)
17
+ end
18
+
19
+ alias_method :data, :body
20
+
21
+ def update_headers(new_headers = {})
22
+ update_headers!
23
+ headers.merge!(new_headers)
24
+ end
25
+
26
+ def update_headers!
27
+ headers[:ttl] ||= Cloudist::DEFAULT_TTL
28
+ headers[:timestamp] = timestamp
29
+ headers[:message_id] ||= id
30
+ headers[:message_type] = 'message'
31
+ headers[:queue_name] ||= 'test'
32
+
33
+ headers.each { |k,v| headers[k] = v.to_s }
34
+ end
35
+
36
+ # Convenience method for replying
37
+ # Constructs a reply message and publishes it
38
+ def reply(body, reply_headers = {})
39
+ raise RuntimeError, "Cannot reply to an unpublished message" unless published?
40
+
41
+ msg = Message.new(body, reply_headers)
42
+ msg.set_reply_header
43
+ reply_q = Cloudist::ReplyQueue.new(headers[:queue_name])
44
+ msg.publish(reply_q)
45
+ end
46
+
47
+ # Publishes this message to the exchange or queue
48
+ # Queue should be a Cloudist::Queue object responding to #publish
49
+ def publish(queue)
50
+ raise ArgumentError, "Publish expects a Cloudist::Queue instance" unless queue.is_a?(Cloudist::Queue)
51
+ set_queue_name_header(queue)
52
+ update_published_date!
53
+ update_headers!
54
+ queue.publish(self)
55
+ end
56
+
57
+ def update_published_date!
58
+ headers[:published_on] = Time.now.utc.to_f
59
+ end
60
+
61
+ # This is so we can reply back to the sender
62
+ def set_queue_name_header(queue)
63
+ update_headers(:queue_name => queue.name)
64
+ end
65
+
66
+ def published?
67
+ @published ||= !!@headers.published_on
68
+ end
69
+
70
+ def created_at
71
+ headers.timestamp ? Time.at(headers.timestamp.to_f) : timestamp
72
+ end
73
+
74
+ def published_at
75
+ headers[:published_on] ? Time.at(headers[:published_on].to_f) : timestamp
76
+ end
77
+
78
+ def latency
79
+ (published_at.to_f - created_at.to_f)
80
+ end
81
+
82
+ def encoded
83
+ [encode(body), {:headers => headers}]
84
+ end
85
+
86
+ def inspect
87
+ "<#{self.class.name} id=#{id}>"
88
+ end
89
+
90
+ private
91
+
92
+ def set_reply_header
93
+ headers[:message_type] = 'reply'
94
+ end
95
+
96
+ end
97
+ end
@@ -0,0 +1,29 @@
1
+ module Cloudist
2
+ autoload :Singleton, 'singleton'
3
+
4
+ class Messaging
5
+ include Singleton
6
+
7
+ class << self
8
+
9
+ def active_queues
10
+ instance.active_queues
11
+ end
12
+
13
+ def add_queue(queue)
14
+ (instance.active_queues ||= {}).merge!({queue.name.to_s => queue})
15
+ instance.active_queues
16
+ end
17
+
18
+ def remove_queue(queue_name)
19
+ (instance.active_queues ||= {}).delete(queue_name.to_s)
20
+ instance.active_queues
21
+ end
22
+ end
23
+
24
+ attr_accessor :active_queues
25
+
26
+
27
+
28
+ end
29
+ end
@@ -1,147 +1,87 @@
1
1
  module Cloudist
2
- DEFAULT_TTL = 300
3
-
4
2
  class Payload
5
3
  include Utils
4
+ include Encoding
6
5
 
7
- attr_reader :body, :publish_opts, :headers
8
-
9
- def initialize(body, headers = {}, publish_opts = {})
10
- @publish_opts, @headers = publish_opts, headers
6
+ attr_reader :body, :headers, :amqp_headers, :timestamp
7
+
8
+ def initialize(body, headers = {})
11
9
  @published = false
10
+ @timestamp = Time.now.to_f
12
11
 
13
- body = parse_message(body) if body.is_a?(String)
14
-
15
- # raise Cloudist::BadPayload, "Expected Hash for payload" unless body.is_a?(Hash)
16
-
17
- @body = body
18
- # HashWithIndifferentAccess.new(body)
19
- update_headers
20
- end
21
-
22
- # Return message formatted as JSON and headers ready for transport in array
23
- def formatted
24
- update_headers
12
+ body = decode(body) if body.is_a?(String)
13
+ @body = Hashie::Mash.new(decode(body))
14
+ @headers = Hashie::Mash.new(headers)
15
+ @amqp_headers = {}
16
+ # puts "Initialised Payload: #{id}"
25
17
 
26
- [encode_message(body), publish_opts]
27
- end
28
-
29
- def id
30
- @id ||= event_hash.to_s
31
- end
32
-
33
- def id=(new_id)
34
- @id = new_id.to_s
35
- update_headers
18
+ parse_headers!
36
19
  end
37
20
 
38
- def frozen?
39
- headers.frozen?
21
+ def find_or_create_id
22
+ if headers["message_id"].present?
23
+ headers.message_id
24
+ else
25
+ UUID.generate
26
+ end
40
27
  end
41
28
 
42
- def freeze!
43
- headers.freeze
44
- body.freeze
29
+ def id
30
+ find_or_create_id
45
31
  end
46
32
 
47
- def update_headers
48
- headers = extract_custom_headers
49
- (publish_opts[:headers] ||= {}).merge!(headers)
33
+ def to_a
34
+ [encode(body), {:headers => encoded_headers}]
50
35
  end
51
36
 
52
- def extract_custom_headers
53
- raise StaleHeadersError, "Headers cannot be changed because payload has already been published" if published?
54
- headers[:published_on] ||= body.is_a?(Hash) && body.delete(:published_on) || Time.now.utc.to_i
55
- headers[:ttl] ||= body.is_a?(Hash) && body.delete('ttl') || Cloudist::DEFAULT_TTL
56
-
57
- # this is the event hash that gets transferred through various publish/reply actions
58
- headers[:event_hash] ||= id
59
-
60
- # this value should be unique for each published/received message pair
61
- headers[:message_id] ||= id
37
+ def parse_headers!
38
+ headers[:published_on] ||= body.delete("timestamp") || timestamp
39
+ headers[:message_type] ||= body.delete("message_type") || 'reply'
62
40
 
63
- # We use JSON for message transport exclusively
64
- # headers[:content_type] ||= 'application/json'
41
+ headers[:ttl] ||= Cloudist::DEFAULT_TTL
42
+ headers[:message_id] = id
65
43
 
66
- # headers[:headers][:message_type] = 'event'
67
- # ||= body.delete('message_type') || 'reply'
44
+ headers[:published_on] = headers[:published_on].to_f
68
45
 
69
- # headers[:headers] = custom_headers
46
+ headers[:ttl] = headers[:ttl].to_i rescue -1
47
+ headers[:ttl] = -1 if headers[:ttl] == 0
70
48
 
71
- # some strange behavior with integers makes it better to
72
- # convert all amqp headers to strings to avoid any problems
73
- headers.each { |k,v| headers[k] = v.to_s }
49
+ # If this payload was received with a timestamp,
50
+ # we don't want to override it on #timestamp
51
+ if timestamp > headers[:published_on]
52
+ @timestamp = headers[:published_on]
53
+ end
74
54
 
75
55
  headers
76
56
  end
77
-
78
- def parse_custom_headers
79
- return { } unless headers
80
-
57
+
58
+ def encoded_headers
81
59
  h = headers.dup
82
-
83
- h[:published_on] = h[:published_on].to_i
84
-
85
- h[:ttl] = h[:ttl].to_i rescue -1
86
- h[:ttl] = -1 if h[:ttl] == 0
87
-
88
- h
60
+ h.each { |k,v| h[k] = v.to_s }
61
+ return h
89
62
  end
90
63
 
91
64
  def set_reply_to(queue_name)
92
- headers[:reply_to] = reply_name(queue_name)
93
- set_master_queue_name(queue_name)
94
- end
95
-
96
- def set_master_queue_name(queue_name)
97
- headers[:master_queue] = queue_name
98
- end
99
-
100
- def reply_name(queue_name)
101
- # "#{queue_name}.#{id}"
102
- Utils.reply_prefix(queue_name)
65
+ headers[:reply_to] = reply_prefix(queue_name)
103
66
  end
104
67
 
105
68
  def reply_to
106
- headers[:reply_to]
69
+ headers.reply_to
107
70
  end
108
71
 
109
72
  def message_type
110
- headers[:message_type]
111
- end
112
-
113
- def event_hash
114
- @event_hash ||= headers[:event_hash] || body.is_a?(Hash) && body.delete(:event_hash) || create_event_hash
115
- end
116
-
117
- def create_event_hash
118
- s = Time.now.to_s + object_id.to_s + rand(100).to_s
119
- Digest::MD5.hexdigest(s)
120
- end
121
-
122
- def parse_message(raw)
123
- # return { } unless raw
124
- # decode_json(raw)
125
- decode_message(raw)
73
+ headers.message_type
126
74
  end
127
75
 
128
76
  def [](key)
129
- body[key]
130
- end
131
-
132
- def published?
133
- @published == true
134
- end
135
-
136
- def publish
137
- return if published?
138
- @published = true
139
- freeze!
77
+ self.body[key.to_s]
140
78
  end
141
79
 
142
80
  def method_missing(meth, *args, &blk)
143
- if body.is_a?(Hash) && body.has_key?(meth)
81
+ if body.has_key?(meth.to_s)
144
82
  return body[meth]
83
+ elsif key = meth.to_s.match(/(.+)(?:\?$)/).to_a.last
84
+ body.has_key?(key.to_s)
145
85
  else
146
86
  super
147
87
  end
@@ -0,0 +1,155 @@
1
+ module Cloudist
2
+ class Payload
3
+ include Utils
4
+
5
+ attr_reader :body, :publish_opts, :headers, :timestamp
6
+
7
+ def initialize(body, headers = {}, publish_opts = {})
8
+ @publish_opts, @headers = publish_opts, Hashie::Mash.new(headers)
9
+ @published = false
10
+
11
+ body = parse_message(body) if body.is_a?(String)
12
+
13
+ # raise Cloudist::BadPayload, "Expected Hash for payload" unless body.is_a?(Hash)
14
+
15
+ @timestamp = Time.now.to_f
16
+
17
+ @body = body
18
+ # Hashie::Mash.new(body)
19
+
20
+ update_headers
21
+ end
22
+
23
+ # Return message formatted as JSON and headers ready for transport in array
24
+ def formatted
25
+ update_headers
26
+
27
+ [encode_message(body), publish_opts]
28
+ end
29
+
30
+ def id
31
+ @id ||= event_hash.to_s
32
+ end
33
+
34
+ def id=(new_id)
35
+ @id = new_id.to_s
36
+ update_headers
37
+ end
38
+
39
+ def frozen?
40
+ headers.frozen?
41
+ end
42
+
43
+ def freeze!
44
+ headers.freeze
45
+ body.freeze
46
+ end
47
+
48
+ def update_headers
49
+ headers = extract_custom_headers
50
+ (publish_opts[:headers] ||= {}).merge!(headers)
51
+ end
52
+
53
+ def extract_custom_headers
54
+ raise StaleHeadersError, "Headers cannot be changed because payload has already been published" if published?
55
+ headers[:published_on] ||= body.is_a?(Hash) && body.delete(:published_on) || Time.now.utc.to_i
56
+ headers[:ttl] ||= body.is_a?(Hash) && body.delete('ttl') || Cloudist::DEFAULT_TTL
57
+ headers[:timestamp] = timestamp
58
+ # this is the event hash that gets transferred through various publish/reply actions
59
+ headers[:event_hash] ||= id
60
+
61
+ # this value should be unique for each published/received message pair
62
+ headers[:message_id] ||= id
63
+
64
+ # We use JSON for message transport exclusively
65
+ # headers[:content_type] ||= 'application/json'
66
+
67
+ # headers[:headers][:message_type] = 'event'
68
+ # ||= body.delete('message_type') || 'reply'
69
+
70
+ # headers[:headers] = custom_headers
71
+
72
+ # some strange behavior with integers makes it better to
73
+ # convert all amqp headers to strings to avoid any problems
74
+ headers.each { |k,v| headers[k] = v.to_s }
75
+
76
+ headers
77
+ end
78
+
79
+ def parse_custom_headers
80
+ return { } unless headers
81
+
82
+ h = headers.dup
83
+
84
+ h[:published_on] = h[:published_on].to_i
85
+
86
+ h[:ttl] = h[:ttl].to_i rescue -1
87
+ h[:ttl] = -1 if h[:ttl] == 0
88
+
89
+ h
90
+ end
91
+
92
+ def set_reply_to(queue_name)
93
+ headers["reply_to"] = reply_name(queue_name)
94
+ set_master_queue_name(queue_name)
95
+ end
96
+
97
+ def set_master_queue_name(queue_name)
98
+ headers[:master_queue] = queue_name
99
+ end
100
+
101
+ def reply_name(queue_name)
102
+ # "#{queue_name}.#{id}"
103
+ Utils.reply_prefix(queue_name)
104
+ end
105
+
106
+ def reply_to
107
+ headers["reply_to"]
108
+ end
109
+
110
+ def message_type
111
+ headers["message_type"]
112
+ end
113
+
114
+ def event_hash
115
+ @event_hash ||= headers["event_hash"] || create_event_hash
116
+ end
117
+
118
+ def create_event_hash
119
+ # s = Time.now.to_s + object_id.to_s + rand(100).to_s
120
+ # Digest::MD5.hexdigest(s)
121
+ UUID.generate
122
+ end
123
+
124
+ def parse_message(raw)
125
+ # return { } unless raw
126
+ # decode_json(raw)
127
+ decode_message(raw)
128
+ end
129
+
130
+ def [](key)
131
+ body[key]
132
+ end
133
+
134
+ def published?
135
+ @published == true
136
+ end
137
+
138
+ def publish
139
+ return if published?
140
+ @published = true
141
+ freeze!
142
+ end
143
+
144
+ def method_missing(meth, *args, &blk)
145
+ if body.is_a?(Hash) && body.has_key?(meth)
146
+ return body[meth]
147
+ elsif key = meth.to_s.match(/(.+)(?:\?$)/).to_a.last
148
+ body.has_key?(key.to_sym)
149
+ else
150
+ super
151
+ end
152
+ end
153
+
154
+ end
155
+ end