cloudist 0.2.1 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. data/Gemfile +15 -11
  2. data/Gemfile.lock +20 -7
  3. data/README.md +61 -39
  4. data/VERSION +1 -1
  5. data/cloudist.gemspec +50 -16
  6. data/examples/amqp/Gemfile +3 -0
  7. data/examples/amqp/Gemfile.lock +12 -0
  8. data/examples/amqp/amqp_consumer.rb +56 -0
  9. data/examples/amqp/amqp_publisher.rb +50 -0
  10. data/examples/queue_message.rb +7 -7
  11. data/examples/sandwich_client_with_custom_listener.rb +77 -0
  12. data/examples/sandwich_worker_with_class.rb +22 -7
  13. data/lib/cloudist.rb +113 -56
  14. data/lib/cloudist/application.rb +60 -0
  15. data/lib/cloudist/core_ext/class.rb +139 -0
  16. data/lib/cloudist/core_ext/kernel.rb +13 -0
  17. data/lib/cloudist/core_ext/module.rb +11 -0
  18. data/lib/cloudist/encoding.rb +13 -0
  19. data/lib/cloudist/errors.rb +2 -0
  20. data/lib/cloudist/job.rb +21 -18
  21. data/lib/cloudist/listener.rb +108 -54
  22. data/lib/cloudist/message.rb +97 -0
  23. data/lib/cloudist/messaging.rb +29 -0
  24. data/lib/cloudist/payload.rb +45 -105
  25. data/lib/cloudist/payload_old.rb +155 -0
  26. data/lib/cloudist/publisher.rb +7 -2
  27. data/lib/cloudist/queue.rb +152 -0
  28. data/lib/cloudist/queues/basic_queue.rb +83 -53
  29. data/lib/cloudist/queues/job_queue.rb +13 -24
  30. data/lib/cloudist/queues/reply_queue.rb +13 -21
  31. data/lib/cloudist/request.rb +33 -7
  32. data/lib/cloudist/worker.rb +9 -2
  33. data/lib/cloudist_old.rb +300 -0
  34. data/lib/em/em_timer_utils.rb +55 -0
  35. data/lib/em/iterator.rb +27 -0
  36. data/spec/cloudist/message_spec.rb +91 -0
  37. data/spec/cloudist/messaging_spec.rb +19 -0
  38. data/spec/cloudist/payload_spec.rb +10 -4
  39. data/spec/cloudist/payload_spec_2_spec.rb +78 -0
  40. data/spec/cloudist/queue_spec.rb +16 -0
  41. data/spec/cloudist_spec.rb +49 -45
  42. data/spec/spec_helper.rb +0 -1
  43. data/spec/support/amqp.rb +16 -0
  44. metadata +112 -102
  45. data/examples/extending_values.rb +0 -44
  46. data/examples/sandwich_client.rb +0 -57
  47. data/lib/cloudist/callback.rb +0 -16
  48. data/lib/cloudist/callback_methods.rb +0 -19
  49. data/lib/cloudist/callbacks/error_callback.rb +0 -14
@@ -1,10 +1,13 @@
1
1
  module Cloudist
2
2
  class Worker
3
3
 
4
- attr_reader :job, :queue
4
+ attr_reader :job, :queue, :payload
5
5
 
6
6
  def initialize(job, queue)
7
- @job, @queue = job, queue
7
+ @job, @queue, @payload = job, queue, job.payload
8
+
9
+ # Do custom initialization
10
+ self.setup if self.respond_to?(:setup)
8
11
  end
9
12
 
10
13
  def data
@@ -15,6 +18,10 @@ module Cloudist
15
18
  job.headers
16
19
  end
17
20
 
21
+ def id
22
+ job.id
23
+ end
24
+
18
25
  def process
19
26
  raise NotImplementedError, "Your worker class must subclass this method"
20
27
  end
@@ -0,0 +1,300 @@
1
+ require 'uri'
2
+ require 'json' unless defined? ActiveSupport::JSON
3
+
4
+ require "amqp"
5
+
6
+ require "logger"
7
+ require "digest/md5"
8
+
9
+ $:.unshift File.dirname(__FILE__)
10
+ # require "em/iterator"
11
+ require "cloudist/core_ext/string"
12
+ require "cloudist/core_ext/object"
13
+ require "cloudist/core_ext/class"
14
+ require "cloudist/errors"
15
+ require "cloudist/utils"
16
+ require "cloudist/queues/basic_queue"
17
+ require "cloudist/queues/sync_queue"
18
+ require "cloudist/queues/job_queue"
19
+ require "cloudist/queues/sync_job_queue"
20
+ require "cloudist/queues/reply_queue"
21
+ require "cloudist/queues/sync_reply_queue"
22
+ require "cloudist/queues/log_queue"
23
+ require "cloudist/publisher"
24
+ require "cloudist/payload"
25
+ require "cloudist/request"
26
+ require "cloudist/callback_methods"
27
+ require "cloudist/listener"
28
+ require "cloudist/callback"
29
+ require "cloudist/callbacks/error_callback"
30
+ require "cloudist/job"
31
+ require "cloudist/worker"
32
+
33
+ module Cloudist
34
+ class << self
35
+
36
+ @@workers = {}
37
+
38
+ # Start the Cloudist loop
39
+ #
40
+ # Cloudist.start {
41
+ # # Do stuff in here
42
+ # }
43
+ #
44
+ # == Options
45
+ # * :user => 'name'
46
+ # * :pass => 'secret'
47
+ # * :host => 'localhost'
48
+ # * :port => 5672
49
+ # * :vhost => /
50
+ # * :heartbeat => 5
51
+ # * :logging => false
52
+ #
53
+ # Refer to default config below for how to set these as defaults
54
+ #
55
+ def start(options = {}, &block)
56
+ config = settings.update(options)
57
+ AMQP.start(config) do
58
+ AMQP.conn.connection_status do |status|
59
+ log.debug("AMQP connection status changed: #{status}")
60
+ if status == :disconnected
61
+ AMQP.conn.reconnect(true)
62
+ end
63
+ end
64
+
65
+ self.instance_eval(&block) if block_given?
66
+ end
67
+ end
68
+
69
+ # Define a worker. Must be called inside start loop
70
+ #
71
+ # worker {
72
+ # job('make.sandwich') {}
73
+ # }
74
+ #
75
+ # REMOVED
76
+ #
77
+ def worker(&block)
78
+ raise NotImplementedError, "This DSL format has been removed. Please use job('make.sandwich') {} instead."
79
+ end
80
+
81
+ # Defines a job handler (GenericWorker)
82
+ #
83
+ # job('make.sandwich') {
84
+ # job.started!
85
+ # # Work hard
86
+ # sleep(5)
87
+ # job.finished!
88
+ # }
89
+ #
90
+ # Refer to sandwich_worker.rb example
91
+ #
92
+ def job(queue_name)
93
+ if block_given?
94
+ block = Proc.new
95
+ register_worker(queue_name, &block)
96
+ else
97
+ raise ArgumentError, "You must supply a block as the last argument"
98
+ end
99
+ end
100
+
101
+ # Registers a worker class to handle a specific queue
102
+ #
103
+ # Cloudist.handle('make.sandwich', 'eat.sandwich').with(MyWorker)
104
+ #
105
+ # A standard worker would look like this:
106
+ #
107
+ # class MyWorker < Cloudist::Worker
108
+ # def process
109
+ # log.debug(data.inspect)
110
+ # end
111
+ # end
112
+ #
113
+ # A new instance of this worker will be created everytime a job arrives
114
+ #
115
+ # Refer to examples.
116
+ def handle(*queue_names)
117
+ class << queue_names
118
+ def with(handler)
119
+ self.each do |queue_name|
120
+ Cloudist.register_worker(queue_name.to_s, handler)
121
+ end
122
+ end
123
+ end
124
+ queue_names
125
+ end
126
+
127
+ def register_worker(queue_name, klass = nil, &block)
128
+ job_queue = JobQueue.new(queue_name)
129
+ job_queue.subscribe do |request|
130
+ j = Job.new(request.payload.dup)
131
+ # EM.defer do
132
+ begin
133
+ if block_given?
134
+ worker_instance = GenericWorker.new(j, job_queue.q)
135
+ worker_instance.process(&block)
136
+ elsif klass
137
+ worker_instance = klass.new(j, job_queue.q)
138
+ worker_instance.process
139
+ else
140
+ raise RuntimeError, "Failed to register worker, I need either a handler class or block."
141
+ end
142
+ rescue Exception => e
143
+ j.handle_error(e)
144
+ ensure
145
+ finished = Time.now.utc.to_f
146
+ log.debug("Finished Job in #{finished - request.start} seconds")
147
+ j.reply({:runtime => (finished - request.start)}, {:message_type => 'runtime'})
148
+ j.cleanup
149
+ end
150
+ # end
151
+ end
152
+
153
+ ((@@workers[queue_name.to_s] ||= []) << job_queue).uniq!
154
+ end
155
+
156
+ # Accepts either a queue name or a job instance returned from enqueue.
157
+ # This method operates in two modes, when given a queue name, it
158
+ # will return all responses regardless of job id so you can use the job
159
+ # id to lookup a database record to update etc.
160
+ # When given a job instance it will only return messages from that job.
161
+ #
162
+ # DEPRECATED
163
+ #
164
+ def listen(*queue_names, &block)
165
+ raise NotImplementedError, "This DSL method has been removed. Please use add_listener"
166
+
167
+ # @@listeners ||= []
168
+ # queue_names.each do |job_or_queue_name|
169
+ # _listener = Cloudist::Listener.new(job_or_queue_name)
170
+ # _listener.subscribe(&block)
171
+ # @@listeners << _listener
172
+ # end
173
+ # return @@listeners
174
+ end
175
+
176
+ # Adds a listener class
177
+ def add_listener(klass)
178
+ @@listeners ||= []
179
+
180
+ raise ArgumentError, "Your listener must extend Cloudist::Listener" unless klass.superclass == Cloudist::Listener
181
+ raise ArgumentError, "Your listener must declare at least one queue to listen to. Use listen_to 'queue.name'" if klass.job_queue_names.nil?
182
+
183
+ klass.job_queue_names.each do |queue_name|
184
+ klass.subscribe(queue_name)
185
+ end
186
+
187
+ @@listeners << klass
188
+
189
+ return @@listeners
190
+ end
191
+
192
+ # Enqueues a job.
193
+ # Takes a queue name and data hash to be sent to the worker.
194
+ # Returns Job instance
195
+ # Use Job#id to reference job later on.
196
+ def enqueue(job_queue_name, data = nil)
197
+ raise EnqueueError, "Incorrect arguments, you must include data when enqueuing job" if data.nil?
198
+ # TODO: Detect if inside loop, if not use bunny sync
199
+ Cloudist::Publisher.enqueue(job_queue_name, data)
200
+ end
201
+
202
+ # Send a reply synchronously
203
+ # This uses bunny instead of AMQP and as such can be run outside
204
+ # of EventMachine and the Cloudist start loop.
205
+ #
206
+ # Usage: Cloudist.reply('make.sandwich', {:sandwhich_id => 12345})
207
+ #
208
+ def reply(queue_name, job_id, data, options = {})
209
+ headers = {
210
+ :message_id => job_id,
211
+ :message_type => "reply",
212
+ # :event => 'working',
213
+ :message_type => 'reply'
214
+ }.update(options)
215
+
216
+ payload = Cloudist::Payload.new(data, headers)
217
+
218
+ queue = Cloudist::SyncReplyQueue.new(queue_name)
219
+
220
+ queue.setup
221
+ queue.publish_to_q(payload)
222
+ end
223
+
224
+ # Call this at anytime inside the loop to exit the app.
225
+ def stop_safely
226
+ if EM.reactor_running?
227
+ ::EM.add_timer(0.2) {
228
+ ::AMQP.stop {
229
+ ::EM.stop
230
+ }
231
+ }
232
+ end
233
+ end
234
+
235
+ alias :stop :stop_safely
236
+
237
+ def closing?
238
+ ::AMQP.closing?
239
+ end
240
+
241
+ def log
242
+ @@log ||= Logger.new($stdout)
243
+ end
244
+
245
+ def log=(log)
246
+ @@log = log
247
+ end
248
+
249
+ def handle_error(e)
250
+ log.error "#{e.class}: #{e.message}"#, :exception => e
251
+ log.error e.backtrace.join("\n")
252
+ end
253
+
254
+ def version
255
+ @@version ||= File.read(File.dirname(__FILE__) + '/../VERSION').strip
256
+ end
257
+
258
+ # EM beta
259
+
260
+ def default_settings
261
+ uri = URI.parse(ENV["AMQP_URL"] || 'amqp://guest:guest@localhost:5672/')
262
+ {
263
+ :vhost => uri.path,
264
+ :host => uri.host,
265
+ :user => uri.user,
266
+ :port => uri.port || 5672,
267
+ :pass => uri.password,
268
+ :heartbeat => 5,
269
+ :logging => false
270
+ }
271
+ rescue Object => e
272
+ raise "invalid AMQP_URL: (#{uri.inspect}) #{e.class} -> #{e.message}"
273
+ end
274
+
275
+ def settings
276
+ @@settings ||= default_settings
277
+ end
278
+
279
+ def settings=(settings_hash)
280
+ @@settings = default_settings.update(settings_hash)
281
+ end
282
+
283
+ def signal_trap!
284
+ ::Signal.trap('INT') { Cloudist.stop }
285
+ ::Signal.trap('TERM'){ Cloudist.stop }
286
+ end
287
+
288
+ alias :install_signal_trap :signal_trap!
289
+
290
+ def workers
291
+ @@workers
292
+ end
293
+
294
+ def remove_workers
295
+ @@workers = {}
296
+ end
297
+
298
+ end
299
+
300
+ end
@@ -0,0 +1,55 @@
1
+ module Cloudist
2
+ module EMTimerUtils
3
+ def self.included(base)
4
+ base.extend(ClassMethods)
5
+ end
6
+
7
+ module ClassMethods
8
+ # Trap exceptions leaving the block and log them. Do not re-raise
9
+ def trap_exceptions
10
+ yield
11
+ rescue => e
12
+ em_exception(e)
13
+ end
14
+
15
+ def em_exception(e)
16
+ msg = format_em_exception(e)
17
+ log.error "[EM.timer] #{msg}", :exception => e
18
+ end
19
+
20
+ def format_em_exception(e)
21
+ # avoid backtrace in /usr or vendor if possible
22
+ system, app = e.backtrace.partition { |b| b =~ /(^\/usr\/|vendor)/ }
23
+ reordered_backtrace = app + system
24
+
25
+ # avoid "/" as the method name (we want the controller action)
26
+ row = 0
27
+ row = 1 if reordered_backtrace[row].match(/in `\/'$/)
28
+
29
+ # get file and method name
30
+ begin
31
+ file, method = reordered_backtrace[row].match(/(.*):in `(.*)'$/)[1..2]
32
+ file.gsub!(/.*\//, '')
33
+ "#{e.class} in #{file} #{method}: #{e.message}"
34
+ rescue
35
+ "#{e.class} in #{e.backtrace.first}: #{e.message}"
36
+ end
37
+ end
38
+
39
+ # One-shot timer
40
+ def timer(duration, &blk)
41
+ EM.add_timer(duration) { trap_exceptions(&blk) }
42
+ end
43
+
44
+ # Add a periodic timer. If the now argument is true, run the block
45
+ # immediately in addition to scheduling the periodic timer.
46
+ def periodic_timer(duration, now=false, &blk)
47
+ timer(1, &blk) if now
48
+ EM.add_periodic_timer(duration) { trap_exceptions(&blk) }
49
+ end
50
+ end
51
+
52
+ def timer(*args, &blk); self.class.timer(*args, &blk); end
53
+ def periodic_timer(*args, &blk); self.class.periodic_timer(*args, &blk); end
54
+ end
55
+ end
@@ -0,0 +1,27 @@
1
+ module EventMachine
2
+ class Iterator
3
+
4
+ def initialize(container)
5
+ @container = container
6
+ end
7
+
8
+ def each(work, done=proc{})
9
+ do_work = proc {
10
+ if @container && !@container.empty?
11
+ work.call(@container.shift)
12
+ EM.next_tick(&do_work)
13
+ else
14
+ done.call
15
+ end
16
+ }
17
+ EM.next_tick(&do_work)
18
+ end
19
+
20
+ def map(work, done=proc{})
21
+ mapped = []
22
+ map_work = proc { |n| mapped << work.call(n) }
23
+ map_done = proc { done.call(mapped) }
24
+ each(map_work, map_done)
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,91 @@
1
+ require File.expand_path('../../spec_helper', __FILE__)
2
+
3
+ describe Cloudist::Message do
4
+ before(:each) do
5
+ stub_amqp!
6
+ @queue = Cloudist::Queue.new("test.queue")
7
+ @queue.stubs(:publish)
8
+ @headers = {}
9
+ end
10
+
11
+ it "should have a unique id when new" do
12
+ msg = Cloudist::Message.new({:hello => "world"}, @headers)
13
+ msg.id.size.should == "57b474f0-496c-012e-6f57-34159e11a916".size
14
+ end
15
+
16
+ it "should not update id when existing message" do
17
+ msg = Cloudist::Message.new({:hello => "world"}, {:id => "not-an-id"})
18
+ msg.update_headers
19
+ msg.id.should == "not-an-id"
20
+ end
21
+
22
+ it "should remove id from headers and update with message_id" do
23
+ msg = Cloudist::Message.new({:hello => "world"}, {:id => "not-an-id"})
24
+ msg.update_headers
25
+ msg.id.should == "not-an-id"
26
+ msg.headers.id.should == nil
27
+
28
+ msg = Cloudist::Message.new({:hello => "world"}, {:message_id => "not-an-id"})
29
+ msg.update_headers
30
+ msg.id.should == "not-an-id"
31
+ msg.headers.id.should == nil
32
+ end
33
+
34
+ it "should update headers" do
35
+ msg = Cloudist::Message.new({:hello => "world"}, {:id => "not-an-id"})
36
+ msg.update_headers
37
+
38
+ msg.headers.keys.should include *["ttl", "timestamp", "message_id"]
39
+ end
40
+
41
+ it "should allow custom header when updating" do
42
+ msg = Cloudist::Message.new({:hello => "world"}, {:id => "not-an-id"})
43
+ msg.update_headers(:message_type => "reply")
44
+
45
+ msg.headers.keys.should include *["ttl", "timestamp", "message_id", "message_type"]
46
+ end
47
+
48
+ it "should not be published if timestamp is not in headers" do
49
+ msg = Cloudist::Message.new({:hello => "world"}, {:id => "not-an-id"})
50
+ msg.published?.should be_false
51
+ end
52
+
53
+ it "should be published if timestamp is in headers" do
54
+ msg = Cloudist::Message.new({:hello => "world"}, {:id => "not-an-id"})
55
+ msg.publish(@queue)
56
+ msg.published?.should be_true
57
+ end
58
+
59
+ it "should include ttl in headers" do
60
+ msg = Cloudist::Message.new({:hello => "world"})
61
+ # msg.publish(@queue)
62
+ msg.headers[:ttl].should == "300"
63
+ end
64
+
65
+ it "should get created_at date from header" do
66
+ time = Time.now.to_f
67
+ msg = Cloudist::Message.new({:hello => "world"}, {:timestamp => time})
68
+ msg.created_at.to_f.should == time
69
+ end
70
+
71
+ it "should set published_at when publishing" do
72
+ time = Time.now.to_f
73
+ msg = Cloudist::Message.new({:hello => "world"}, {:timestamp => time})
74
+ msg.publish(@queue)
75
+ msg.published_at.to_f.should > time
76
+ end
77
+
78
+ it "should have latency" do
79
+ time = (Time.now).to_f
80
+ msg = Cloudist::Message.new({:hello => "world"}, {:timestamp => time})
81
+ sleep(0.1)
82
+ msg.publish(@queue)
83
+ msg.latency.should be_within(0.001).of(0.1)
84
+ end
85
+
86
+ it "should reply to sender" do
87
+ msg = Cloudist::Message.new({:hello => "world"}, {:id => "not-an-id"})
88
+ msg.reply(:success => true)
89
+ end
90
+
91
+ end