cloudist 0.2.1 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. data/Gemfile +15 -11
  2. data/Gemfile.lock +20 -7
  3. data/README.md +61 -39
  4. data/VERSION +1 -1
  5. data/cloudist.gemspec +50 -16
  6. data/examples/amqp/Gemfile +3 -0
  7. data/examples/amqp/Gemfile.lock +12 -0
  8. data/examples/amqp/amqp_consumer.rb +56 -0
  9. data/examples/amqp/amqp_publisher.rb +50 -0
  10. data/examples/queue_message.rb +7 -7
  11. data/examples/sandwich_client_with_custom_listener.rb +77 -0
  12. data/examples/sandwich_worker_with_class.rb +22 -7
  13. data/lib/cloudist.rb +113 -56
  14. data/lib/cloudist/application.rb +60 -0
  15. data/lib/cloudist/core_ext/class.rb +139 -0
  16. data/lib/cloudist/core_ext/kernel.rb +13 -0
  17. data/lib/cloudist/core_ext/module.rb +11 -0
  18. data/lib/cloudist/encoding.rb +13 -0
  19. data/lib/cloudist/errors.rb +2 -0
  20. data/lib/cloudist/job.rb +21 -18
  21. data/lib/cloudist/listener.rb +108 -54
  22. data/lib/cloudist/message.rb +97 -0
  23. data/lib/cloudist/messaging.rb +29 -0
  24. data/lib/cloudist/payload.rb +45 -105
  25. data/lib/cloudist/payload_old.rb +155 -0
  26. data/lib/cloudist/publisher.rb +7 -2
  27. data/lib/cloudist/queue.rb +152 -0
  28. data/lib/cloudist/queues/basic_queue.rb +83 -53
  29. data/lib/cloudist/queues/job_queue.rb +13 -24
  30. data/lib/cloudist/queues/reply_queue.rb +13 -21
  31. data/lib/cloudist/request.rb +33 -7
  32. data/lib/cloudist/worker.rb +9 -2
  33. data/lib/cloudist_old.rb +300 -0
  34. data/lib/em/em_timer_utils.rb +55 -0
  35. data/lib/em/iterator.rb +27 -0
  36. data/spec/cloudist/message_spec.rb +91 -0
  37. data/spec/cloudist/messaging_spec.rb +19 -0
  38. data/spec/cloudist/payload_spec.rb +10 -4
  39. data/spec/cloudist/payload_spec_2_spec.rb +78 -0
  40. data/spec/cloudist/queue_spec.rb +16 -0
  41. data/spec/cloudist_spec.rb +49 -45
  42. data/spec/spec_helper.rb +0 -1
  43. data/spec/support/amqp.rb +16 -0
  44. metadata +112 -102
  45. data/examples/extending_values.rb +0 -44
  46. data/examples/sandwich_client.rb +0 -57
  47. data/lib/cloudist/callback.rb +0 -16
  48. data/lib/cloudist/callback_methods.rb +0 -19
  49. data/lib/cloudist/callbacks/error_callback.rb +0 -14
@@ -0,0 +1,97 @@
1
+ module Cloudist
2
+ class Message
3
+ include Cloudist::Encoding
4
+
5
+ attr_reader :body, :headers, :id, :timestamp
6
+
7
+ # Expects body to be decoded
8
+ def initialize(body, headers = {})
9
+ @body = Hashie::Mash.new(body.dup)
10
+
11
+ @id ||= headers[:message_id] || headers[:id] && headers.delete(:id) || UUID.generate
12
+ @headers = Hashie::Mash.new(headers.dup)
13
+
14
+ @timestamp = Time.now.utc.to_f
15
+
16
+ update_headers(headers)
17
+ end
18
+
19
+ alias_method :data, :body
20
+
21
+ def update_headers(new_headers = {})
22
+ update_headers!
23
+ headers.merge!(new_headers)
24
+ end
25
+
26
+ def update_headers!
27
+ headers[:ttl] ||= Cloudist::DEFAULT_TTL
28
+ headers[:timestamp] = timestamp
29
+ headers[:message_id] ||= id
30
+ headers[:message_type] = 'message'
31
+ headers[:queue_name] ||= 'test'
32
+
33
+ headers.each { |k,v| headers[k] = v.to_s }
34
+ end
35
+
36
+ # Convenience method for replying
37
+ # Constructs a reply message and publishes it
38
+ def reply(body, reply_headers = {})
39
+ raise RuntimeError, "Cannot reply to an unpublished message" unless published?
40
+
41
+ msg = Message.new(body, reply_headers)
42
+ msg.set_reply_header
43
+ reply_q = Cloudist::ReplyQueue.new(headers[:queue_name])
44
+ msg.publish(reply_q)
45
+ end
46
+
47
+ # Publishes this message to the exchange or queue
48
+ # Queue should be a Cloudist::Queue object responding to #publish
49
+ def publish(queue)
50
+ raise ArgumentError, "Publish expects a Cloudist::Queue instance" unless queue.is_a?(Cloudist::Queue)
51
+ set_queue_name_header(queue)
52
+ update_published_date!
53
+ update_headers!
54
+ queue.publish(self)
55
+ end
56
+
57
+ def update_published_date!
58
+ headers[:published_on] = Time.now.utc.to_f
59
+ end
60
+
61
+ # This is so we can reply back to the sender
62
+ def set_queue_name_header(queue)
63
+ update_headers(:queue_name => queue.name)
64
+ end
65
+
66
+ def published?
67
+ @published ||= !!@headers.published_on
68
+ end
69
+
70
+ def created_at
71
+ headers.timestamp ? Time.at(headers.timestamp.to_f) : timestamp
72
+ end
73
+
74
+ def published_at
75
+ headers[:published_on] ? Time.at(headers[:published_on].to_f) : timestamp
76
+ end
77
+
78
+ def latency
79
+ (published_at.to_f - created_at.to_f)
80
+ end
81
+
82
+ def encoded
83
+ [encode(body), {:headers => headers}]
84
+ end
85
+
86
+ def inspect
87
+ "<#{self.class.name} id=#{id}>"
88
+ end
89
+
90
+ private
91
+
92
+ def set_reply_header
93
+ headers[:message_type] = 'reply'
94
+ end
95
+
96
+ end
97
+ end
@@ -0,0 +1,29 @@
1
+ module Cloudist
2
+ autoload :Singleton, 'singleton'
3
+
4
+ class Messaging
5
+ include Singleton
6
+
7
+ class << self
8
+
9
+ def active_queues
10
+ instance.active_queues
11
+ end
12
+
13
+ def add_queue(queue)
14
+ (instance.active_queues ||= {}).merge!({queue.name.to_s => queue})
15
+ instance.active_queues
16
+ end
17
+
18
+ def remove_queue(queue_name)
19
+ (instance.active_queues ||= {}).delete(queue_name.to_s)
20
+ instance.active_queues
21
+ end
22
+ end
23
+
24
+ attr_accessor :active_queues
25
+
26
+
27
+
28
+ end
29
+ end
@@ -1,147 +1,87 @@
1
1
  module Cloudist
2
- DEFAULT_TTL = 300
3
-
4
2
  class Payload
5
3
  include Utils
4
+ include Encoding
6
5
 
7
- attr_reader :body, :publish_opts, :headers
8
-
9
- def initialize(body, headers = {}, publish_opts = {})
10
- @publish_opts, @headers = publish_opts, headers
6
+ attr_reader :body, :headers, :amqp_headers, :timestamp
7
+
8
+ def initialize(body, headers = {})
11
9
  @published = false
10
+ @timestamp = Time.now.to_f
12
11
 
13
- body = parse_message(body) if body.is_a?(String)
14
-
15
- # raise Cloudist::BadPayload, "Expected Hash for payload" unless body.is_a?(Hash)
16
-
17
- @body = body
18
- # HashWithIndifferentAccess.new(body)
19
- update_headers
20
- end
21
-
22
- # Return message formatted as JSON and headers ready for transport in array
23
- def formatted
24
- update_headers
12
+ body = decode(body) if body.is_a?(String)
13
+ @body = Hashie::Mash.new(decode(body))
14
+ @headers = Hashie::Mash.new(headers)
15
+ @amqp_headers = {}
16
+ # puts "Initialised Payload: #{id}"
25
17
 
26
- [encode_message(body), publish_opts]
27
- end
28
-
29
- def id
30
- @id ||= event_hash.to_s
31
- end
32
-
33
- def id=(new_id)
34
- @id = new_id.to_s
35
- update_headers
18
+ parse_headers!
36
19
  end
37
20
 
38
- def frozen?
39
- headers.frozen?
21
+ def find_or_create_id
22
+ if headers["message_id"].present?
23
+ headers.message_id
24
+ else
25
+ UUID.generate
26
+ end
40
27
  end
41
28
 
42
- def freeze!
43
- headers.freeze
44
- body.freeze
29
+ def id
30
+ find_or_create_id
45
31
  end
46
32
 
47
- def update_headers
48
- headers = extract_custom_headers
49
- (publish_opts[:headers] ||= {}).merge!(headers)
33
+ def to_a
34
+ [encode(body), {:headers => encoded_headers}]
50
35
  end
51
36
 
52
- def extract_custom_headers
53
- raise StaleHeadersError, "Headers cannot be changed because payload has already been published" if published?
54
- headers[:published_on] ||= body.is_a?(Hash) && body.delete(:published_on) || Time.now.utc.to_i
55
- headers[:ttl] ||= body.is_a?(Hash) && body.delete('ttl') || Cloudist::DEFAULT_TTL
56
-
57
- # this is the event hash that gets transferred through various publish/reply actions
58
- headers[:event_hash] ||= id
59
-
60
- # this value should be unique for each published/received message pair
61
- headers[:message_id] ||= id
37
+ def parse_headers!
38
+ headers[:published_on] ||= body.delete("timestamp") || timestamp
39
+ headers[:message_type] ||= body.delete("message_type") || 'reply'
62
40
 
63
- # We use JSON for message transport exclusively
64
- # headers[:content_type] ||= 'application/json'
41
+ headers[:ttl] ||= Cloudist::DEFAULT_TTL
42
+ headers[:message_id] = id
65
43
 
66
- # headers[:headers][:message_type] = 'event'
67
- # ||= body.delete('message_type') || 'reply'
44
+ headers[:published_on] = headers[:published_on].to_f
68
45
 
69
- # headers[:headers] = custom_headers
46
+ headers[:ttl] = headers[:ttl].to_i rescue -1
47
+ headers[:ttl] = -1 if headers[:ttl] == 0
70
48
 
71
- # some strange behavior with integers makes it better to
72
- # convert all amqp headers to strings to avoid any problems
73
- headers.each { |k,v| headers[k] = v.to_s }
49
+ # If this payload was received with a timestamp,
50
+ # we don't want to override it on #timestamp
51
+ if timestamp > headers[:published_on]
52
+ @timestamp = headers[:published_on]
53
+ end
74
54
 
75
55
  headers
76
56
  end
77
-
78
- def parse_custom_headers
79
- return { } unless headers
80
-
57
+
58
+ def encoded_headers
81
59
  h = headers.dup
82
-
83
- h[:published_on] = h[:published_on].to_i
84
-
85
- h[:ttl] = h[:ttl].to_i rescue -1
86
- h[:ttl] = -1 if h[:ttl] == 0
87
-
88
- h
60
+ h.each { |k,v| h[k] = v.to_s }
61
+ return h
89
62
  end
90
63
 
91
64
  def set_reply_to(queue_name)
92
- headers[:reply_to] = reply_name(queue_name)
93
- set_master_queue_name(queue_name)
94
- end
95
-
96
- def set_master_queue_name(queue_name)
97
- headers[:master_queue] = queue_name
98
- end
99
-
100
- def reply_name(queue_name)
101
- # "#{queue_name}.#{id}"
102
- Utils.reply_prefix(queue_name)
65
+ headers[:reply_to] = reply_prefix(queue_name)
103
66
  end
104
67
 
105
68
  def reply_to
106
- headers[:reply_to]
69
+ headers.reply_to
107
70
  end
108
71
 
109
72
  def message_type
110
- headers[:message_type]
111
- end
112
-
113
- def event_hash
114
- @event_hash ||= headers[:event_hash] || body.is_a?(Hash) && body.delete(:event_hash) || create_event_hash
115
- end
116
-
117
- def create_event_hash
118
- s = Time.now.to_s + object_id.to_s + rand(100).to_s
119
- Digest::MD5.hexdigest(s)
120
- end
121
-
122
- def parse_message(raw)
123
- # return { } unless raw
124
- # decode_json(raw)
125
- decode_message(raw)
73
+ headers.message_type
126
74
  end
127
75
 
128
76
  def [](key)
129
- body[key]
130
- end
131
-
132
- def published?
133
- @published == true
134
- end
135
-
136
- def publish
137
- return if published?
138
- @published = true
139
- freeze!
77
+ self.body[key.to_s]
140
78
  end
141
79
 
142
80
  def method_missing(meth, *args, &blk)
143
- if body.is_a?(Hash) && body.has_key?(meth)
81
+ if body.has_key?(meth.to_s)
144
82
  return body[meth]
83
+ elsif key = meth.to_s.match(/(.+)(?:\?$)/).to_a.last
84
+ body.has_key?(key.to_s)
145
85
  else
146
86
  super
147
87
  end
@@ -0,0 +1,155 @@
1
+ module Cloudist
2
+ class Payload
3
+ include Utils
4
+
5
+ attr_reader :body, :publish_opts, :headers, :timestamp
6
+
7
+ def initialize(body, headers = {}, publish_opts = {})
8
+ @publish_opts, @headers = publish_opts, Hashie::Mash.new(headers)
9
+ @published = false
10
+
11
+ body = parse_message(body) if body.is_a?(String)
12
+
13
+ # raise Cloudist::BadPayload, "Expected Hash for payload" unless body.is_a?(Hash)
14
+
15
+ @timestamp = Time.now.to_f
16
+
17
+ @body = body
18
+ # Hashie::Mash.new(body)
19
+
20
+ update_headers
21
+ end
22
+
23
+ # Return message formatted as JSON and headers ready for transport in array
24
+ def formatted
25
+ update_headers
26
+
27
+ [encode_message(body), publish_opts]
28
+ end
29
+
30
+ def id
31
+ @id ||= event_hash.to_s
32
+ end
33
+
34
+ def id=(new_id)
35
+ @id = new_id.to_s
36
+ update_headers
37
+ end
38
+
39
+ def frozen?
40
+ headers.frozen?
41
+ end
42
+
43
+ def freeze!
44
+ headers.freeze
45
+ body.freeze
46
+ end
47
+
48
+ def update_headers
49
+ headers = extract_custom_headers
50
+ (publish_opts[:headers] ||= {}).merge!(headers)
51
+ end
52
+
53
+ def extract_custom_headers
54
+ raise StaleHeadersError, "Headers cannot be changed because payload has already been published" if published?
55
+ headers[:published_on] ||= body.is_a?(Hash) && body.delete(:published_on) || Time.now.utc.to_i
56
+ headers[:ttl] ||= body.is_a?(Hash) && body.delete('ttl') || Cloudist::DEFAULT_TTL
57
+ headers[:timestamp] = timestamp
58
+ # this is the event hash that gets transferred through various publish/reply actions
59
+ headers[:event_hash] ||= id
60
+
61
+ # this value should be unique for each published/received message pair
62
+ headers[:message_id] ||= id
63
+
64
+ # We use JSON for message transport exclusively
65
+ # headers[:content_type] ||= 'application/json'
66
+
67
+ # headers[:headers][:message_type] = 'event'
68
+ # ||= body.delete('message_type') || 'reply'
69
+
70
+ # headers[:headers] = custom_headers
71
+
72
+ # some strange behavior with integers makes it better to
73
+ # convert all amqp headers to strings to avoid any problems
74
+ headers.each { |k,v| headers[k] = v.to_s }
75
+
76
+ headers
77
+ end
78
+
79
+ def parse_custom_headers
80
+ return { } unless headers
81
+
82
+ h = headers.dup
83
+
84
+ h[:published_on] = h[:published_on].to_i
85
+
86
+ h[:ttl] = h[:ttl].to_i rescue -1
87
+ h[:ttl] = -1 if h[:ttl] == 0
88
+
89
+ h
90
+ end
91
+
92
+ def set_reply_to(queue_name)
93
+ headers["reply_to"] = reply_name(queue_name)
94
+ set_master_queue_name(queue_name)
95
+ end
96
+
97
+ def set_master_queue_name(queue_name)
98
+ headers[:master_queue] = queue_name
99
+ end
100
+
101
+ def reply_name(queue_name)
102
+ # "#{queue_name}.#{id}"
103
+ Utils.reply_prefix(queue_name)
104
+ end
105
+
106
+ def reply_to
107
+ headers["reply_to"]
108
+ end
109
+
110
+ def message_type
111
+ headers["message_type"]
112
+ end
113
+
114
+ def event_hash
115
+ @event_hash ||= headers["event_hash"] || create_event_hash
116
+ end
117
+
118
+ def create_event_hash
119
+ # s = Time.now.to_s + object_id.to_s + rand(100).to_s
120
+ # Digest::MD5.hexdigest(s)
121
+ UUID.generate
122
+ end
123
+
124
+ def parse_message(raw)
125
+ # return { } unless raw
126
+ # decode_json(raw)
127
+ decode_message(raw)
128
+ end
129
+
130
+ def [](key)
131
+ body[key]
132
+ end
133
+
134
+ def published?
135
+ @published == true
136
+ end
137
+
138
+ def publish
139
+ return if published?
140
+ @published = true
141
+ freeze!
142
+ end
143
+
144
+ def method_missing(meth, *args, &blk)
145
+ if body.is_a?(Hash) && body.has_key?(meth)
146
+ return body[meth]
147
+ elsif key = meth.to_s.match(/(.+)(?:\?$)/).to_a.last
148
+ body.has_key?(key.to_sym)
149
+ else
150
+ super
151
+ end
152
+ end
153
+
154
+ end
155
+ end