cloudist 0.1.2 → 0.2.0

data/Gemfile

@@ -8,7 +8,7 @@ gem "activesupport", "~> 3.0.3"
 # Include everything needed to run rake, tests, features, etc.
 group :development do
   gem "rspec", "~> 2.3.0"
-  gem "moqueue", :git => "git://github.com/customink/moqueue.git"
+  gem "moqueue", :git => "git://github.com/ivanvanderbyl/moqueue.git"
   gem "mocha"
   gem "bundler", "~> 1.0.0"
   gem "jeweler", "~> 1.5.2"

data/Gemfile.lock

@@ -1,6 +1,6 @@
 GIT
-  remote: git://github.com/customink/moqueue.git
-  revision: 091a8f57e5c79b0b25e152b4d5230e4031797d62
+  remote: git://github.com/ivanvanderbyl/moqueue.git
+  revision: cf2108e3bee7730a6a57e0a95a4c17082c2b2603
   specs:
     moqueue (0.1.4)
       amqp

data/VERSION

@@ -1 +1 @@
-0.1.2
+0.2.0

data/cloudist.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{cloudist}
-  s.version = "0.1.2"
+  s.version = "0.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Ivan Vanderbyl"]
-  s.date = %q{2011-01-18}
+  s.date = %q{2011-01-20}
   s.description = %q{Cloudist is a simple, highly scalable job queue for Ruby applications, it can run within Rails, DaemonKit or your own custom application. Refer to github page for examples}
   s.email = %q{ivanvanderbyl@me.com}
   s.extra_rdoc_files = [
@@ -26,21 +26,25 @@ Gem::Specification.new do |s|
     "Rakefile",
     "VERSION",
     "cloudist.gemspec",
+    "examples/extending_values.rb",
     "examples/queue_message.rb",
     "examples/sandwich_client.rb",
     "examples/sandwich_worker.rb",
+    "examples/sandwich_worker_with_class.rb",
     "lib/cloudist.rb",
-    "lib/cloudist/basic_queue.rb",
     "lib/cloudist/callback.rb",
     "lib/cloudist/callback_methods.rb",
+    "lib/cloudist/callbacks/error_callback.rb",
+    "lib/cloudist/core_ext/object.rb",
     "lib/cloudist/core_ext/string.rb",
     "lib/cloudist/errors.rb",
     "lib/cloudist/job.rb",
-    "lib/cloudist/job_queue.rb",
     "lib/cloudist/listener.rb",
     "lib/cloudist/payload.rb",
     "lib/cloudist/publisher.rb",
-    "lib/cloudist/reply_queue.rb",
+    "lib/cloudist/queues/basic_queue.rb",
+    "lib/cloudist/queues/job_queue.rb",
+    "lib/cloudist/queues/reply_queue.rb",
     "lib/cloudist/request.rb",
     "lib/cloudist/utils.rb",
     "lib/cloudist/worker.rb",
@@ -48,6 +52,7 @@ Gem::Specification.new do |s|
     "spec/cloudist/job_spec.rb",
     "spec/cloudist/payload_spec.rb",
     "spec/cloudist/request_spec.rb",
+    "spec/cloudist/utils_spec.rb",
     "spec/cloudist_spec.rb",
     "spec/core_ext/string_spec.rb",
     "spec/spec_helper.rb"
@@ -58,13 +63,16 @@ Gem::Specification.new do |s|
   s.rubygems_version = %q{1.3.7}
   s.summary = %q{Super fast job queue using AMQP}
   s.test_files = [
+    "examples/extending_values.rb",
     "examples/queue_message.rb",
     "examples/sandwich_client.rb",
     "examples/sandwich_worker.rb",
+    "examples/sandwich_worker_with_class.rb",
     "spec/cloudist/basic_queue_spec.rb",
     "spec/cloudist/job_spec.rb",
     "spec/cloudist/payload_spec.rb",
     "spec/cloudist/request_spec.rb",
+    "spec/cloudist/utils_spec.rb",
     "spec/cloudist_spec.rb",
     "spec/core_ext/string_spec.rb",
     "spec/spec_helper.rb"

data/examples/extending_values.rb

@@ -0,0 +1,44 @@
+class SandwichMaker
+  
+end
+
+class SandwichEater
+  
+end
+
+module Cloudist
+  class << self
+    @@workers = {}
+    
+    def handle(*queue_names)
+      class << queue_names
+        def with(handler)
+          self.each do |queue_name|
+            ((@@workers[queue_name.to_s] ||= []) << handler).uniq!
+          end
+        end
+      end
+      queue_names
+    end
+    
+    def use(handler)
+      proxy = handler.new
+      class << proxy
+        def to(queue_name)
+          ((@@workers[queue_name.to_s] ||= []) << self.class).uniq!
+        end
+      end
+      proxy
+    end
+    
+    def workers
+      @@workers
+    end
+  end
+end
+
+Cloudist.handle('make.sandwich', 'eat').with(SandwichMaker)
+Cloudist.use(SandwichEater).to('eat.sandwich')
+
+p Cloudist.workers
+# >> {"eat"=>[SandwichMaker], "make.sandwich"=>[SandwichMaker], "eat.sandwich"=>[SandwichEater]}

data/examples/sandwich_client.rb

@@ -18,13 +18,26 @@ Cloudist.signal_trap!
 Cloudist.start {
   
   log.info("Dispatching sandwich making job...")
-  enqueue('make.sandwich', {:bread => 'white'})
-  # enqueue('make.sandwich', {:bread => 'brown'})
   
+  unless ARGV.empty?
+    job_count = ARGV.pop.to_i
+    job_count.times { |i| enqueue('make.sandwich', {:bread => 'white', :sandwich_number => i})}
+  end
+  
+  
+  # enqueue('eat.sandwich', {:sandwich => job.id})
+  # enqueue('make.sandwich', {:bread => 'brown'})
+    
   # Listen to all sandwich jobs
-  listen('make.sandwich') {
+  listen('make.sandwich', 'eat.sandwich') {
     everything {
-      Cloudist.log.info("Job ID: #{job_id}")
+      Cloudist.log.info("#{headers[:message_type]} - Job ID: #{job_id}")
+    }
+    
+    error { |e|
+      Cloudist.log.error(e.inspect)
+      Cloudist.log.error(e.backtrace.inspect)
+      Cloudist.stop
     }
     
     progress {
@@ -37,6 +50,7 @@ Cloudist.start {
     
     event('finished'){
       Cloudist.log.info("Finished making sandwich at #{Time.now.to_s}")
+      Cloudist.stop
     }
   }
   

data/examples/sandwich_worker.rb

@@ -19,24 +19,18 @@ Cloudist.signal_trap!
 Cloudist.start {
   log.info("Started Worker")
   
-  worker {
-    job('make.sandwich') {
-      # Fire the started event
+  job('make.sandwich') {
+    log.info("JOB (#{id}) Make sandwich with #{data[:bread]} bread")
+    
+    job.started!
+    
+    (1..20).each do |i|
+      job.progress(i * 5)
+      sleep(1)
       
-      log.info("JOB (#{id}) Make sandwich with #{data[:bread]} bread")
-      log.debug(data.inspect)
-      
-      EM.defer {
-        progress(0)
-        started!
-        progress(10)
-        sleep(1)
-        progress(20)
-        sleep(5)
-        progress(90)
-        finished!
-        progress(100)
-      }
-    }    
+      raise ArgumentError, "NOT GOOD!" if i == 4
+    end
+    job.finished!
   }
+  
 }

data/examples/sandwich_worker_with_class.rb

@@ -0,0 +1,37 @@
+# Cloudst Example: Sandwich Worker
+# 
+# This example demonstrates receiving a job and sending back events to the client to let it know we've started and finsihed
+# making a sandwich. From here you could dispatch an eat.sandwich event.
+# 
+# Be sure to update the Cloudist connection settings if they differ from defaults:
+# user: guest
+# pass: guest
+# port: 5672
+# host: localhost
+# vhost: /
+# 
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require "rubygems"
+require "cloudist"
+
+class SandwichWorker < Cloudist::Worker
+  def process
+    log.info("Processing queue: #{queue.name}")
+    log.info(data.inspect)
+    
+    job.started!
+    (1..20).each do |i|
+      job.progress(i * 5)
+      sleep(1)
+      
+      raise ArgumentError, "NOT GOOD!" if i == 4
+    end
+    job.finished!
+  end
+end
+
+Cloudist.signal_trap!
+
+Cloudist.start {
+  Cloudist.handle('make.sandwich', 'eat.sandwich').with(SandwichWorker)
+}

data/lib/cloudist.rb

@@ -8,22 +8,27 @@ require "digest/md5"
 
 $:.unshift File.dirname(__FILE__)
 require "cloudist/core_ext/string"
+require "cloudist/core_ext/object"
 require "cloudist/errors"
 require "cloudist/utils"
-require "cloudist/basic_queue"
-require "cloudist/job_queue"
-require "cloudist/reply_queue"
+require "cloudist/queues/basic_queue"
+require "cloudist/queues/job_queue"
+require "cloudist/queues/reply_queue"
 require "cloudist/publisher"
 require "cloudist/payload"
 require "cloudist/request"
-require "cloudist/worker"
 require "cloudist/callback_methods"
 require "cloudist/listener"
 require "cloudist/callback"
+require "cloudist/callbacks/error_callback"
 require "cloudist/job"
+require "cloudist/worker"
 
 module Cloudist
   class << self
+    
+    @@workers = {}
+    
     # Start the Cloudist loop
     # 
     #   Cloudist.start {
@@ -48,15 +53,82 @@ module Cloudist
 
     # Define a worker. Must be called inside start loop
     # 
-    # worker {
-    #   job('make.sandwich') {}
-    # }
+    #   worker {
+    #     job('make.sandwich') {}
+    #   }
+    # 
+    # REMOVED
+    def worker(&block)
+      raise NotImplementedError, "This DSL format has been removed. Please use job('make.sandwich') {} instead."
+    end
+    
+    # Defines a job handler (GenericWorker)
+    # 
+    #   job('make.sandwich') {
+    #     job.started!
+    #     # Work hard
+    #     sleep(5)
+    #     job.finished!
+    #   }
+    # 
+    # Refer to sandwich_worker.rb example
+    # 
+    def job(queue_name, &block)
+      register_worker(queue_name, &block)
+    end
+    
+    # Registers a worker class to handle a specific queue
+    # 
+    #   Cloudist.handle('make.sandwich', 'eat.sandwich').with(MyWorker)
+    # 
+    # A standard worker would look like this:
+    #   
+    #   class MyWorker < Cloudist::Worker
+    #     def process
+    #       log.debug(data.inspect)
+    #     end
+    #   end
+    # 
+    # A new instance of this worker will be created everytime a job arrives
     # 
     # Refer to examples.
-    def worker(options = {}, &block)
-      _worker = Cloudist::Worker.new(options)
-      _worker.instance_eval(&block)
-      return _worker
+    def handle(*queue_names)
+      class << queue_names
+        def with(handler)
+          self.each do |queue_name|
+            Cloudist.register_worker(queue_name.to_s, handler)
+          end
+        end
+      end
+      queue_names
+    end
+    
+    def register_worker(queue_name, klass = nil, &block)
+      job_queue = JobQueue.new(queue_name)
+      job_queue.subscribe do |request|
+        j = Job.new(request.payload.dup)
+        EM.defer do
+          begin
+            if block_given?
+              worker_instance = GenericWorker.new(j, job_queue.q)
+              worker_instance.process(&block)
+            elsif klass
+              worker_instance = klass.new(j, job_queue.q)
+              worker_instance.process
+            else
+              raise RuntimeError, "Failed to register worker, I need either a handler class or block."
+            end
+            finished = Time.now.utc.to_i
+            log.debug("Finished Job in #{finished - request.start} seconds")
+            
+          rescue Exception => e
+            j.handle_error(e)
+          end
+        end
+        j.cleanup
+      end
+      
+      ((@@workers[queue_name.to_s] ||= []) << job_queue).uniq!
     end
     
     # Accepts either a queue name or a job instance returned from enqueue.
@@ -64,10 +136,14 @@ module Cloudist
     # will return all responses regardless of job id so you can use the job
     # id to lookup a database record to update etc.
     # When given a job instance it will only return messages from that job.
-    def listen(job_or_queue_name, &block)
-      _listener = Cloudist::Listener.new(job_or_queue_name)
-      _listener.subscribe(&block)
-      return _listener
+    def listen(*queue_names, &block)
+      @@listeners ||= []
+      queue_names.each do |job_or_queue_name|
+        _listener = Cloudist::Listener.new(job_or_queue_name)
+        _listener.subscribe(&block)
+        @@listeners << _listener
+      end
+      return @@listeners
     end
     
     # Enqueues a job.
@@ -82,11 +158,11 @@ module Cloudist
 
     # Call this at anytime inside the loop to exit the app.
     def stop_safely
-      ::EM.add_timer(0.2) { 
+      # ::EM.add_timer(0.2) { 
         ::AMQP.stop { 
           ::EM.stop
         }
-      }
+      # }
     end
     
     alias :stop :stop_safely
@@ -138,6 +214,14 @@ module Cloudist
       ::Signal.trap('TERM'){ Cloudist.stop }
     end
     
+    def workers
+      @@workers
+    end
+    
+    def remove_workers
+      @@workers = {}
+    end
+    
   end
   
 end

data/lib/cloudist/callbacks/error_callback.rb

@@ -0,0 +1,14 @@
+module Cloudist
+  class ErrorCallback < Callback  
+    def call(payload)
+      @payload = payload
+      
+      case source.arity
+      when 0
+        instance_exec(&source)
+      when 1
+        instance_exec(payload.exception, &source)
+      end
+    end
+  end
+end

data/lib/cloudist/core_ext/object.rb

@@ -0,0 +1,81 @@
+# Taken from Rails ActiveSupport
+class Object
+  def remove_subclasses_of(*superclasses) #:nodoc:
+    Class.remove_class(*subclasses_of(*superclasses))
+  end
+
+  begin
+    ObjectSpace.each_object(Class.new) {}
+
+    # Exclude this class unless it's a subclass of our supers and is defined.
+    # We check defined? in case we find a removed class that has yet to be
+    # garbage collected. This also fails for anonymous classes -- please
+    # submit a patch if you have a workaround.
+    def subclasses_of(*superclasses) #:nodoc:
+      subclasses = []
+
+      superclasses.each do |sup|
+        ObjectSpace.each_object(class << sup; self; end) do |k|
+          if k != sup && (k.name.blank? || eval("defined?(::#{k}) && ::#{k}.object_id == k.object_id"))
+            subclasses << k
+          end
+        end
+      end
+
+      subclasses
+    end
+  rescue RuntimeError
+    # JRuby and any implementations which cannot handle the objectspace traversal
+    # above fall back to this implementation
+    def subclasses_of(*superclasses) #:nodoc:
+      subclasses = []
+
+      superclasses.each do |sup|
+        ObjectSpace.each_object(Class) do |k|
+          if superclasses.any? { |superclass| k < superclass } &&
+            (k.name.blank? || eval("defined?(::#{k}) && ::#{k}.object_id == k.object_id"))
+            subclasses << k
+          end
+        end
+        subclasses.uniq!
+      end
+      subclasses
+    end
+  end
+
+  def extended_by #:nodoc:
+    ancestors = class << self; ancestors end
+    ancestors.select { |mod| mod.class == Module } - [ Object, Kernel ]
+  end
+
+  def extend_with_included_modules_from(object) #:nodoc:
+    object.extended_by.each { |mod| extend mod }
+  end
+
+  unless defined? instance_exec # 1.9
+    module InstanceExecMethods #:nodoc:
+    end
+    include InstanceExecMethods
+
+    # Evaluate the block with the given arguments within the context of
+    # this object, so self is set to the method receiver.
+    #
+    # From Mauricio's http://eigenclass.org/hiki/bounded+space+instance_exec
+    def instance_exec(*args, &block)
+      begin
+        old_critical, Thread.critical = Thread.critical, true
+        n = 0
+        n += 1 while respond_to?(method_name = "__instance_exec#{n}")
+        InstanceExecMethods.module_eval { define_method(method_name, &block) }
+      ensure
+        Thread.critical = old_critical
+      end
+
+      begin
+        send(method_name, *args)
+      ensure
+        InstanceExecMethods.module_eval { remove_method(method_name) } rescue nil
+      end
+    end
+  end
+end