cloudist 0.2.1 → 0.4.1

data/examples/amqp/amqp_publisher.rb

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require "rubygems"
+require 'amqp'
+
+def amqp_settings
+  uri = URI.parse(ENV["AMQP_URL"] || 'amqp://guest:guest@localhost:5672/')
+  {
+    :vhost => uri.path,
+    :host => uri.host,
+    :user => uri.user,
+    :port => uri.port || 5672,
+    :pass => uri.password,
+    :heartbeat => 120,
+    :logging => false
+  }
+rescue Object => e
+  raise "invalid AMQP_URL: (#{uri.inspect}) #{e.class} -> #{e.message}"
+end
+
+p amqp_settings
+
+def log(*args)
+  puts args.inspect
+end
+
+EM.run do
+  puts "Running..."
+  AMQP.start(amqp_settings) do |connection|
+    log "Connected to AMQP broker"
+
+    channel  = AMQP::Channel.new(connection)
+    channel.prefetch(1)
+    queue    = channel.queue("test.hello.world")
+    exchange = channel.direct
+    queue.bind(exchange)
+
+    EM.defer do
+      10000.times { |i|
+        log "Publishing message #{i+1}"
+        if i % 1000 == 0
+          puts "Sleeping..."
+          sleep(1)
+        end
+        exchange.publish "Hello, world! - #{i+1}"#, :routing_key => queue.name
+      }
+    end
+  end  
+end

data/examples/queue_message.rb

@@ -2,16 +2,16 @@ $:.unshift File.dirname(__FILE__) + '/../lib'
 require "rubygems"
 require "cloudist"
 
-::Signal.trap('INT') { Cloudist.stop }
-::Signal.trap('TERM'){ Cloudist.stop }
-
+Cloudist.signal_trap!
+# 
+# This demonstrates how to send a message to a listener
+# 
 Cloudist.start {
   
-  payload = Cloudist::Payload.new({:event => "started"})
+  payload = Cloudist::Payload.new(:event => :started, :message_type => 'event')
   
   q = Cloudist::ReplyQueue.new('temp.reply.make.sandwich')
-  q.setup
-  q.publish_to_q(payload)
+  q.publish(payload)
   
   stop
-}
+}

data/examples/sandwich_client_with_custom_listener.rb

@@ -0,0 +1,77 @@
+# Cloudst Example: Sandwich Client with custom listener class
+# 
+# This example demonstrates dispatching a job to the worker and receiving event callbacks.
+# 
+# Be sure to update the Cloudist connection settings if they differ from defaults:
+# user: guest
+# pass: guest
+# port: 5672
+# host: localhost
+# vhost: /
+#
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require "rubygems"
+require "cloudist"
+
+$total_jobs = 0
+
+class SandwichListener < Cloudist::Listener
+  listen_to "make.sandwich"
+  
+  before :find_job
+  
+  def find_job
+    puts "--- #{payload.id}"
+  end
+  
+  def progress(i)
+    puts "Progress: %1d%" % i
+  end
+  
+  def runtime(seconds)
+    puts "#{id} Finished job in #{seconds} seconds"
+    $total_jobs -= 1
+    puts "--- #{$total_jobs} jobs remaining"
+  end
+  
+  # def started
+  #   puts "Started"
+  # end
+
+  def event(type)
+    puts "Event: #{type}"
+  end
+  
+  def finished
+    puts "*** Finished ***"
+    
+    if $total_jobs == 0
+      puts "Completed all jobs"
+      Cloudist.stop
+    end
+  end
+  
+  def error(e)
+    puts "#{e.exception}: #{e.message} (#{e.backtrace.first})"
+  end
+end
+
+
+Cloudist.signal_trap!
+
+Cloudist.start(:logging => true) {
+  puts AMQP.settings.inspect
+  
+  unless ARGV.empty?
+    puts "*** Please ensure you have a worker running ***"
+    
+    job_count = ARGV.pop.to_i
+    $total_jobs = job_count
+    job_count.times { |i| 
+      log.info("Dispatching sandwich making job...")
+      puts "Queued job: " + enqueue('make.sandwich', {:bread => 'white', :sandwich_number => i}).id
+    }
+  end
+  
+  add_listener(SandwichListener)  
+}

data/examples/sandwich_worker_with_class.rb

@@ -16,22 +16,37 @@ require "cloudist"
 
 class SandwichWorker < Cloudist::Worker
   def process
-    log.info("Processing queue: #{queue.name}")
-    log.info(data.inspect)
+    log.info("Processing #{queue.name} job: #{id}")
     
+    # This will trigger the start event
+    # Appending ! to the end of a method will trigger an
+    # event reply with its name
+    # 
+    # e.g. job.working!
+    # 
     job.started!
-    (1..20).each do |i|
-      job.progress(i * 5)
+    
+    (1..5).each do |i|
+      # This sends a progress reply, you could use this to
+      # update a progress bar in your UI
+      # 
+      # usage: #progress([INTEGER 0 - 100])
+      job.progress(i * 20)
+      
+      # Work hard!
       sleep(1)
       
-      raise ArgumentError, "NOT GOOD!" if i == 4
+      # Uncomment this to test error handling in Listener
+      # raise ArgumentError, "NOT GOOD!" if i == 4
     end
+    
+    # Trigger finished event
     job.finished!
   end
 end
 
 Cloudist.signal_trap!
 
-Cloudist.start {
-  Cloudist.handle('make.sandwich', 'eat.sandwich').with(SandwichWorker)
+Cloudist.start(:logging => false) {
+  Cloudist.handle('make.sandwich').with(SandwichWorker)
 }

data/lib/cloudist.rb

@@ -1,33 +1,38 @@
 require 'uri'
 require 'json' unless defined? ActiveSupport::JSON
-require "active_support/hash_with_indifferent_access"
 require "amqp"
-require "mq"
+require "hashie"
 require "logger"
 require "digest/md5"
+require "uuid"
 
 $:.unshift File.dirname(__FILE__)
+
+require "em/em_timer_utils"
 require "cloudist/core_ext/string"
 require "cloudist/core_ext/object"
+require "cloudist/core_ext/class"
 require "cloudist/errors"
 require "cloudist/utils"
+require "cloudist/encoding"
 require "cloudist/queues/basic_queue"
 require "cloudist/queues/job_queue"
 require "cloudist/queues/reply_queue"
 require "cloudist/publisher"
 require "cloudist/payload"
 require "cloudist/request"
-require "cloudist/callback_methods"
 require "cloudist/listener"
-require "cloudist/callback"
-require "cloudist/callbacks/error_callback"
 require "cloudist/job"
 require "cloudist/worker"
 
 module Cloudist
+  DEFAULT_TTL = 300
+  
   class << self
-    
-    @@workers = {}
+    thread_local_accessor :channels, :default => {}
+    thread_local_accessor :workers, :default => {}
+    thread_local_accessor :listeners, :default => []
+    thread_local_accessor :listener_instances, :default => {}
     
     # Start the Cloudist loop
     # 
@@ -41,15 +46,25 @@ module Cloudist
     # * :host => 'localhost'
     # * :port => 5672
     # * :vhost => /
+    # * :heartbeat => 0
+    # * :logging => false
     # 
     # Refer to default config below for how to set these as defaults
     # 
     def start(options = {}, &block)
       config = settings.update(options)
       AMQP.start(config) do
-        self.instance_eval(&block)
+        self.instance_eval(&block) if block_given?
       end
     end
+    
+    def connection
+      AMQP.connection
+    end
+    
+    def connection=(conn)
+      AMQP.connection = conn
+    end
 
     # Define a worker. Must be called inside start loop
     # 
@@ -58,6 +73,7 @@ module Cloudist
     #   }
     # 
     # REMOVED
+    # 
     def worker(&block)
       raise NotImplementedError, "This DSL format has been removed. Please use job('make.sandwich') {} instead."
     end
@@ -73,8 +89,13 @@ module Cloudist
     # 
     # Refer to sandwich_worker.rb example
     # 
-    def job(queue_name, &block)
-      register_worker(queue_name, &block)
+    def job(queue_name)
+      if block_given?
+        block = Proc.new
+        register_worker(queue_name, &block)
+      else
+        raise ArgumentError, "You must supply a block as the last argument"
+      end
     end
     
     # Registers a worker class to handle a specific queue
@@ -107,28 +128,27 @@ module Cloudist
       job_queue = JobQueue.new(queue_name)
       job_queue.subscribe do |request|
         j = Job.new(request.payload.dup)
-        EM.defer do
-          begin
-            if block_given?
-              worker_instance = GenericWorker.new(j, job_queue.q)
-              worker_instance.process(&block)
-            elsif klass
-              worker_instance = klass.new(j, job_queue.q)
-              worker_instance.process
-            else
-              raise RuntimeError, "Failed to register worker, I need either a handler class or block."
-            end
-            finished = Time.now.utc.to_i
-            log.debug("Finished Job in #{finished - request.start} seconds")
-            
-          rescue Exception => e
-            j.handle_error(e)
+        begin
+          if block_given?
+            worker_instance = GenericWorker.new(j, job_queue.q)
+            worker_instance.process(&block)
+          elsif klass
+            worker_instance = klass.new(j, job_queue.q)
+            worker_instance.process
+          else
+            raise RuntimeError, "Failed to register worker, I need either a handler class or block."
           end
+        rescue Exception => e
+          j.handle_error(e)
+        ensure
+          finished = Time.now.utc.to_f
+          log.debug("Finished Job in #{finished - request.start} seconds")
+          j.reply({:runtime => (finished - request.start)}, {:message_type => 'runtime'})
+          j.cleanup
         end
-        j.cleanup
       end
       
-      ((@@workers[queue_name.to_s] ||= []) << job_queue).uniq!
+      ((self.workers[queue_name.to_s] ||= []) << job_queue).uniq!
     end
     
     # Accepts either a queue name or a job instance returned from enqueue.
@@ -136,14 +156,25 @@ module Cloudist
     # will return all responses regardless of job id so you can use the job
     # id to lookup a database record to update etc.
     # When given a job instance it will only return messages from that job.
+    # 
+    # DEPRECATED
+    # 
     def listen(*queue_names, &block)
-      @@listeners ||= []
-      queue_names.each do |job_or_queue_name|
-        _listener = Cloudist::Listener.new(job_or_queue_name)
-        _listener.subscribe(&block)
-        @@listeners << _listener
+      raise NotImplementedError, "This DSL method has been removed. Please use add_listener"
+    end
+    
+    # Adds a listener class
+    def add_listener(klass)
+      raise ArgumentError, "Your listener must extend Cloudist::Listener" unless klass.superclass == Cloudist::Listener
+      raise ArgumentError, "Your listener must declare at least one queue to listen to. Use listen_to 'queue.name'" if klass.job_queue_names.nil?
+      
+      klass.job_queue_names.each do |queue_name|
+        klass.subscribe(queue_name)
       end
-      return @@listeners
+      
+      self.listeners << klass
+      
+      return self.listeners
     end
     
     # Enqueues a job.
@@ -151,37 +182,52 @@ module Cloudist
     # Returns Job instance
     # Use Job#id to reference job later on.
     def enqueue(job_queue_name, data = nil)
-      raise EnqueueError, "Incorrect arguments, you must include data when enquing job" if data.nil?
+      raise EnqueueError, "Incorrect arguments, you must include data when enqueuing job" if data.nil?
       # TODO: Detect if inside loop, if not use bunny sync
       Cloudist::Publisher.enqueue(job_queue_name, data)
     end
+    
+    # Send a reply synchronously
+    # This uses bunny instead of AMQP and as such can be run outside
+    # of EventMachine and the Cloudist start loop.
+    # 
+    # Usage: Cloudist.reply('make.sandwich', {:sandwhich_id => 12345})
+    # 
+    # def reply(queue_name, job_id, data, options = {})
+    #   headers = {
+    #     :message_id => job_id,
+    #     :message_type => "reply",
+    #     # :event => 'working',
+    #     :message_type => 'reply'
+    #   }.update(options)
+    # 
+    #   payload = Cloudist::Payload.new(data, headers)
+    # 
+    #   queue = Cloudist::SyncReplyQueue.new(queue_name)
+    # 
+    #   queue.setup
+    #   queue.publish_to_q(payload)
+    # end
 
     # Call this at anytime inside the loop to exit the app.
     def stop_safely
-      # ::EM.add_timer(0.2) { 
-        ::AMQP.stop { 
-          ::EM.stop
+      if EM.reactor_running?
+        ::EM.add_timer(0.2) { 
+          ::AMQP.stop { 
+            ::EM.stop
+            puts "\n"
+          }
         }
-      # }
+      end
     end
     
     alias :stop :stop_safely
-
-    def closing?
-      ::AMQP.closing?
-    end
-
-    def log
-      @@log ||= Logger.new($stdout)
-    end
-    
-    def log=(log)
-      @@log = log
-    end
     
     def handle_error(e)
       log.error "#{e.class}: #{e.message}"#, :exception => e
-      log.error e.backtrace.join("\n")
+      e.backtrace.each do |line|
+        log.error line
+      end
     end
     
     def version
@@ -195,7 +241,9 @@ module Cloudist
         :host => uri.host,
         :user => uri.user,
         :port => uri.port || 5672,
-        :pass => uri.password
+        :pass => uri.password,
+        :heartbeat => 0,
+        :logging => false
       }
     rescue Object => e
       raise "invalid AMQP_URL: (#{uri.inspect}) #{e.class} -> #{e.message}"
@@ -214,14 +262,23 @@ module Cloudist
       ::Signal.trap('TERM'){ Cloudist.stop }
     end
     
-    def workers
-      @@workers
+    def log
+      @@log ||= Logger.new($stdout)
+    end
+
+    def log=(log)
+      @@log = log
     end
     
+    alias :install_signal_trap :signal_trap!
+    
     def remove_workers
-      @@workers = {}
+      self.workers.keys.each do |worker|
+        self.workers.delete(worker)
+      end
     end
     
   end
   
+  include Cloudist::EMTimerUtils
 end