cloudist 0.1.2 → 0.2.0

data/lib/cloudist/errors.rb

@@ -3,4 +3,4 @@ module Cloudist
   class BadPayload < Error; end
   class EnqueueError < Error; end
   class StaleHeadersError < BadPayload; end
-end
+end

data/lib/cloudist/job.rb

@@ -21,7 +21,7 @@ module Cloudist
       
     end
     
-    def reply(data, headers = {}, options = {})
+    def reply(body, headers = {}, options = {})
       options = {
         :echo => false
       }.update(options)
@@ -32,15 +32,15 @@ module Cloudist
       }.update(headers)
       
       # Echo the payload back
-      data.merge!(payload.body) if options[:echo] == true
+      # body.merge!(payload.body) if options[:echo] == true
       
-      reply_payload = Payload.new(data, headers)
+      reply_payload = Payload.new(body, headers)
       
       reply_queue = ReplyQueue.new(payload.reply_to)
       reply_queue.setup
       published_headers = reply_queue.publish_to_q(reply_payload)
       
-      log.debug("Replying: #{data.inspect} - Headers: #{published_headers.inspect}")
+      log.debug("Replying: #{body.inspect} HEADERS: #{headers.inspect}")
     end
     
     # Sends a progress update
@@ -55,6 +55,21 @@ module Cloudist
       reply(event_data, {:event => event_name, :message_type => 'event'}, options)
     end
     
+    def safely(&blk)
+      # begin
+      yield
+    rescue Exception => e
+      handle_error(e)
+      # end
+      # result
+    end
+    
+    # This will transfer the Exception object to the client
+    def handle_error(e)
+      # reply({:exception_class => e.class.name, :message => e.message, :backtrace => e.backtrace}, {:message_type => 'error'})
+      reply({:exception => e}, {:message_type => 'error'})
+    end
+    
     def method_missing(meth, *args, &blk)
       if meth.to_s.ends_with?("!")
         event(meth.to_s.gsub(/(!)$/, ''), args.shift)

data/lib/cloudist/listener.rb

@@ -4,7 +4,7 @@ module Cloudist
     
     attr_reader :job_queue_name, :job_id, :callbacks
     
-    @@valid_callbacks = ["event", "progress", "reply", "update"]
+    @@valid_callbacks = ["event", "progress", "reply", "update", "error"]
     
     def initialize(job_or_queue_name)
       @callbacks = {}
@@ -38,6 +38,13 @@ module Cloudist
           end
         end
         
+        if callbacks.has_key?('error')
+          callbacks['error'].each do |c|
+            # c.call(payload)
+            
+          end
+        end
+        
         if callbacks.has_key?(key)
           callbacks_to_call = callbacks[key]
           callbacks_to_call.each do |c|
@@ -57,7 +64,12 @@ module Cloudist
         # callback should in format of "event:started" or "progress"
         key = [meth.to_s, args.shift].compact.join(':')
         
-        (@callbacks[key] ||= []) << Callback.new(blk)
+        case meth.to_sym
+        when :error
+          (@callbacks[key] ||= []) << ErrorCallback.new(blk)
+        else
+          (@callbacks[key] ||= []) << Callback.new(blk)
+        end
       else
         super
       end

data/lib/cloudist/payload.rb

@@ -6,15 +6,16 @@ module Cloudist
     
     attr_reader :body, :publish_opts, :headers
 
-    def initialize(data_hash_or_json, headers = {}, publish_opts = {})
+    def initialize(body, headers = {}, publish_opts = {})
       @publish_opts, @headers = publish_opts, headers
       @published = false
       
-      data_hash_or_json = parse_message(data_hash_or_json) if data_hash_or_json.is_a?(String)
+      body = parse_message(body) if body.is_a?(String)
       
-      raise Cloudist::BadPayload, "Expected Hash for payload" unless data_hash_or_json.is_a?(Hash)
+      # raise Cloudist::BadPayload, "Expected Hash for payload" unless body.is_a?(Hash)
 
-      @body = HashWithIndifferentAccess.new(data_hash_or_json)
+      @body = body
+      # HashWithIndifferentAccess.new(body)
       update_headers
     end
 
@@ -22,7 +23,7 @@ module Cloudist
     def formatted
       update_headers
       
-      [body.to_json, publish_opts]
+      [encode_message(body), publish_opts]
     end
     
     def id
@@ -50,8 +51,8 @@ module Cloudist
     
     def extract_custom_headers
       raise StaleHeadersError, "Headers cannot be changed because payload has already been published" if published?
-      headers[:published_on] ||= body.delete('published_on') || Time.now.utc.to_i
-      headers[:ttl] ||= body.delete('ttl') || Cloudist::DEFAULT_TTL
+      headers[:published_on] ||= body.is_a?(Hash) && body.delete(:published_on) || Time.now.utc.to_i
+      headers[:ttl] ||= body.is_a?(Hash) && body.delete('ttl') || Cloudist::DEFAULT_TTL
 
       # this is the event hash that gets transferred through various publish/reply actions
       headers[:event_hash] ||= id
@@ -60,7 +61,7 @@ module Cloudist
       headers[:message_id] ||= id
       
       # We use JSON for message transport exclusively
-      headers[:content_type] ||= 'application/json'
+      # headers[:content_type] ||= 'application/json'
       
       # headers[:headers][:message_type] = 'event'
        # ||= body.delete('message_type') || 'reply'
@@ -110,7 +111,7 @@ module Cloudist
     end
     
     def event_hash
-      @event_hash ||= headers[:event_hash] || body.delete('event_hash') || create_event_hash
+      @event_hash ||= headers[:event_hash] || body.is_a?(Hash) && body.delete(:event_hash) || create_event_hash
     end
     
     def create_event_hash
@@ -119,8 +120,9 @@ module Cloudist
     end
     
     def parse_message(raw)
-      return { } unless raw
-      decode_json(raw)
+      # return { } unless raw
+      # decode_json(raw)
+      decode_message(raw)
     end
     
     def [](key)
@@ -137,5 +139,13 @@ module Cloudist
       freeze!
     end
     
+    def method_missing(meth, *args, &blk)
+      if body.is_a?(Hash) && body.has_key?(meth)
+        return body[meth]
+      else
+        super
+      end
+    end
+    
   end
 end

data/lib/cloudist/queues/basic_queue.rb

@@ -0,0 +1,101 @@
+module Cloudist
+  class UnknownReplyTo < RuntimeError; end
+  class ExpiredMessage < RuntimeError; end
+  
+  module Queues
+    class BasicQueue
+      attr_reader :queue_name, :opts
+      attr_reader :q, :ex, :mq
+
+      def initialize(queue_name, opts = {})
+        opts = {
+          :auto_delete => true,
+          :durable => false,
+          :prefetch => 1
+        }.update(opts)
+
+        @queue_name, @opts = queue_name, opts
+      end
+
+      def setup
+        return if @setup == true
+
+        @mq = MQ.new
+        @q = @mq.queue(queue_name, opts)
+        # if we don't specify an exchange name it defaults to the queue_name
+        @ex = @mq.direct(opts[:exchange_name] || queue_name)
+
+        q.bind(ex) if ex
+
+        @setup = true
+      end
+
+      def log
+        Cloudist.log
+      end
+
+      def tag
+        s = "queue=#{q.name}"
+        s += " exchange=#{ex.name}" if ex
+        s
+      end
+
+      def subscribe(amqp_opts={}, opts={})
+        setup
+        print_status
+        q.subscribe(amqp_opts) do |queue_header, json_encoded_message|
+          next if Cloudist.closing?
+
+          request = Cloudist::Request.new(self, json_encoded_message, queue_header)
+
+          begin
+            raise Cloudist::ExpiredMessage if request.expired?
+            yield request if block_given?
+            # finished = Time.now.utc.to_i
+            # log.debug("Finished Job in #{finished - request.start} seconds")
+
+          rescue Cloudist::ExpiredMessage
+            log.error "AMQP Message Timeout: #{tag} ttl=#{request.ttl} age=#{request.age}"
+            request.ack if amqp_opts[:ack]
+
+          rescue => e
+            request.ack if amqp_opts[:ack]
+            Cloudist.handle_error(e)
+          end
+        end
+        log.info "AMQP Subscribed: #{tag}"
+        self
+      end
+
+      def print_status
+        q.status{ |num_messages, num_consumers|
+          log.info("STATUS: #{q.name}: JOBS: #{num_messages} WORKERS: #{num_consumers+1}")
+        }
+      end
+
+      def publish(payload)
+        payload.set_reply_to(queue_name)
+        body, headers = payload.formatted
+        ex.publish(body, headers)
+        payload.publish
+      end
+
+      def publish_to_q(payload)
+        body, headers = payload.formatted
+        q.publish(body, headers)
+        payload.publish
+        return headers
+      end
+
+      def teardown
+        @q.unsubscribe
+        @mq.close
+        log.debug "AMQP Unsubscribed: #{tag}"
+      end
+
+      def destroy
+        teardown
+      end
+    end
+  end
+end

data/lib/cloudist/{job_queue.rb → queues/job_queue.rb}

@@ -1,5 +1,5 @@
 module Cloudist
-  class JobQueue < BasicQueue
+  class JobQueue < Cloudist::Queues::BasicQueue
     attr_reader :prefetch
 
     def initialize(queue_name, opts={})
@@ -20,7 +20,7 @@ module Cloudist
         begin
           yield request if block_given?
         ensure
-          request.ack unless amqp_opts[:auto_ack] == false
+          request.ack unless amqp_opts[:auto_ack] == false || Cloudist.closing?
         end
       end
     end

data/lib/cloudist/{reply_queue.rb → queues/reply_queue.rb}

@@ -1,5 +1,5 @@
 module Cloudist
-  class ReplyQueue < Cloudist::BasicQueue
+  class ReplyQueue < Cloudist::Queues::BasicQueue
     def initialize(queue_name, opts={})
       opts[:auto_delete] = true
       opts[:nowait] = false

data/lib/cloudist/request.rb

@@ -2,10 +2,10 @@ module Cloudist
   class Request
     attr_reader :queue_header, :qobj, :payload, :start, :headers
 
-    def initialize(queue, json_encoded_message, queue_header)
+    def initialize(queue, encoded_body, queue_header)
       @qobj, @queue_header = queue, queue_header
 
-      @payload = Cloudist::Payload.new(json_encoded_message.dup, queue_header.headers.dup)
+      @payload = Cloudist::Payload.new(encoded_body, queue_header.headers.dup)
       @headers = @payload.parse_custom_headers
 
       @start = Time.now.utc.to_i

data/lib/cloudist/utils.rb

@@ -6,6 +6,14 @@ module Cloudist
       "temp.reply.#{name}"
     end
 
+    def log_prefix(name)
+      "temp.log.#{name}"
+    end
+
+    def stats_prefix(name)
+      "temp.stats.#{name}"
+    end
+
     def generate_queue(exchange_name, second_name=nil)
       second_name ||= $$
       "#{generate_name_for_instance(exchange_name)}.#{second_name}"
@@ -33,6 +41,14 @@ module Cloudist
       "%04x%04x%04x%04x%04x%06x%06x" % values
     end
     
+    def encode_message(object)
+      Marshal.dump(object).to_s
+    end
+    
+    def decode_message(string)
+      Marshal.load(string)
+    end
+    
     def decode_json(string)
       if defined? ActiveSupport::JSON
         ActiveSupport::JSON.decode string

data/lib/cloudist/worker.rb

@@ -1,24 +1,33 @@
 module Cloudist
   class Worker
     
-    attr_reader :options
+    attr_reader :job, :queue
     
-    def initialize(options)
-      @options = options
+    def initialize(job, queue)
+      @job, @queue = job, queue
+    end
+    
+    def data
+      job.data
+    end
+    
+    def headers
+      job.headers
+    end
+    
+    def process
+      raise NotImplementedError, "Your worker class must subclass this method"
     end
     
     def log
       Cloudist.log
     end
     
-    def job(queue_name, &block)
-      q = JobQueue.new(queue_name)
-      q.subscribe do |request|
-        j = Job.new(request.payload.dup)
-        j.instance_eval(&block)
-        j.cleanup
-      end
+  end
+  
+  class GenericWorker < Worker
+    def process(&block)
+      instance_eval(&block)
     end
-    
   end
 end

data/spec/cloudist/basic_queue_spec.rb

@@ -1,7 +1,7 @@
 require File.expand_path(File.dirname(__FILE__) + '../../spec_helper')
 
 describe "Cloudist" do
-  describe "BasicQueue" do
+  describe "Cloudist::Queues::BasicQueue" do
     before(:each) do
       overload_amqp
       reset_broker
@@ -16,7 +16,7 @@ describe "Cloudist" do
       @queue.expects(:bind).with(@exchange)
       # @mq.expects(:queue).with("make.sandwich")
       
-      bq = Cloudist::BasicQueue.new("make.sandwich")
+      bq = Cloudist::Queues::BasicQueue.new("make.sandwich")
       bq.stubs(:q).returns(@queue)
       bq.stubs(:mq).returns(@mq)
       bq.stubs(:ex).returns(@exchange)

data/spec/cloudist/payload_spec.rb

@@ -1,11 +1,11 @@
 require File.expand_path(File.dirname(__FILE__) + '../../spec_helper')
 
 describe Cloudist::Payload do
-  it "should raise bad payload error unless data is a hash" do
-    lambda {
-      Cloudist::Payload.new([1,2,3])
-    }.should raise_error(Cloudist::BadPayload)
-  end
+  # it "should raise bad payload error unless data is a hash" do
+  #   lambda {
+  #     Cloudist::Payload.new([1,2,3])
+  #   }.should raise_error(Cloudist::BadPayload)
+  # end
   
   it "should accept a hash for data" do
     lambda {
@@ -15,10 +15,10 @@ describe Cloudist::Payload do
   
   it "should prepare headers" do
     payload = Cloudist::Payload.new({:bread => 'white'})
-    payload.body.should == {"bread"=>"white"}
+    payload.body.should == {:bread => "white"}
     payload.headers.has_key?(:ttl).should be_true
-    payload.headers.has_key?(:content_type).should be_true
-    payload.headers[:content_type].should == "application/json"
+    # payload.headers.has_key?(:content_type).should be_true
+    # payload.headers[:content_type].should == "application/json"
     payload.headers.has_key?(:published_on).should be_true
     payload.headers.has_key?(:event_hash).should be_true
     payload.headers.has_key?(:message_id).should be_true
@@ -26,24 +26,24 @@ describe Cloudist::Payload do
   
   it "should extract published_on from data" do
     payload = Cloudist::Payload.new({:bread => 'white', :published_on => 12345678})
-    payload.body.should == {"bread"=>"white"}
+    payload.body.should == {:bread=>"white"}
     payload.headers[:published_on].should == "12345678"
   end
   
   it "should extract custom event hash from data" do
     payload = Cloudist::Payload.new({:bread => 'white', :event_hash => 'foo'})
-    payload.body.should == {"bread"=>"white"}
+    payload.body.should == {:bread=>"white"}
     payload.headers[:event_hash].should == "foo"
   end
   
   it "should parse JSON message" do
-    payload = Cloudist::Payload.new({:bread => 'white', :event_hash => 'foo'}.to_json)
-    payload.body.should == {"bread"=>"white"}
+    payload = Cloudist::Payload.new(Marshal.dump({:bread => 'white', :event_hash => 'foo'}))
+    payload.body.should == {:bread => "white"}
   end
   
   it "should parse custom headers" do
-    payload = Cloudist::Payload.new({:bread => 'white', :event_hash => 'foo'}.to_json, {:published_on => 12345})
-    payload.parse_custom_headers.should == {:published_on=>12345, :event_hash=>"foo", :content_type=>"application/json", :message_id=>"foo", :ttl=>300}
+    payload = Cloudist::Payload.new(Marshal.dump({:bread => 'white', :event_hash => 'foo'}), {:published_on => 12345})
+    payload.parse_custom_headers.should == {:published_on=>12345, :event_hash=>"foo", :message_id=>"foo", :ttl=>300}
   end
   
   it "should create a unique event hash" do
@@ -65,10 +65,10 @@ describe Cloudist::Payload do
   
   it "should format payload for sending" do
     payload = Cloudist::Payload.new({:bread => 'white'}, {:event_hash => 'foo', :message_type => 'reply'})
-    json, popts = payload.formatted
+    body, popts = payload.formatted
     headers = popts[:headers]
     
-    json.should == "{\"bread\":\"white\"}"
+    body.should == Marshal.dump({:bread => 'white'})
     headers[:ttl].should == "300"
     headers[:message_type].should == 'reply'
   end
@@ -119,8 +119,13 @@ describe Cloudist::Payload do
   end
   
   it "should allow custom headers to be set" do
-    payload = Cloudist::Payload.new({:bread => 'white'}, {:reply_type => 'event'})
-    payload.headers[:reply_type].should == 'event'
+    payload = Cloudist::Payload.new({:bread => 'white'}, {:message_type => 'event'})
+    payload.headers[:message_type].should == 'event'
+  end
+  
+  it "should be able to transport an error" do
+    e = ArgumentError.new("FAILED")
+    payload = Cloudist::Payload.new(e, {:message_type => 'error'})
   end
   
 end