cloudist 0.2.1 → 0.4.1

data/lib/cloudist/publisher.rb

@@ -4,10 +4,15 @@ module Cloudist
     class << self
       def enqueue(queue_name, data)
         payload = Cloudist::Payload.new(data)
-
+        
         queue = Cloudist::JobQueue.new(queue_name)
+        
         queue.setup
-        queue.publish(payload)
+        
+        # send_message = proc {
+          queue.publish(payload)
+        # }
+        # EM.next_tick(&send_message)
         
         return Job.new(payload)
       end

data/lib/cloudist/queue.rb

@@ -0,0 +1,152 @@
+module Cloudist
+  class Queue
+    
+    attr_reader :options, :name, :channel, :q, :ex
+    
+    class_attribute :cached_queues
+    
+    def initialize(name, options = {})
+      self.class.cached_queues ||= {}
+      
+      options = {
+        :auto_delete => false,
+        :durable => true
+      }.update(options)
+      
+      @name, @options = name, options
+      
+      setup
+      p self.cached_queues.keys
+      
+      log.debug(tag)
+      purge
+    end
+    
+    def purge
+      q.purge
+    end
+    
+    def inspect
+      "<#{self.class.name} name=#{name} exchange=#{ex ? ex.name : 'nil'}>"
+    end
+    
+    def log
+      Cloudist.log
+    end
+
+    def tag
+      [].tap { |a|
+        a << "queue=#{q.name}" if q
+        a << "exchange=#{ex.name}" if ex
+      }.join(' ')
+    end
+    
+    def publish(msg)
+      raise ArgumentError, "Publish expects a Cloudist::Message object" unless msg.is_a?(Cloudist::Message)
+      
+      body, headers = msg.encoded
+      # EM.defer {
+        publish_to_ex(body, headers)
+      # }
+      
+      p msg.body.to_hash
+    end
+    
+    # def channel
+    #   self.class.channel
+    # end
+    # 
+    # def q
+    #   self.class.q
+    # end
+    # 
+    # def ex
+    #   self.class.ex
+    # end
+    
+    def publish_to_ex(body, headers = {})
+      ex.publish(body, headers)
+    end
+    
+    def publish_to_q(body, headers = {})
+      q.publish(body, headers)
+    end
+    
+    def teardown
+      q.unsubscribe
+      channel.close
+      log.debug "AMQP Unsubscribed: #{tag}"
+    end
+
+    def destroy
+      teardown
+    end
+    
+    def subscribe(options = {}, &block)
+      options[:ack] = true
+      q.subscribe(options) do |queue_header, encoded_message|
+        request = Cloudist::Request.new(self, encoded_message, queue_header)
+        
+        msg = Cloudist::Message.new(*request.for_message)
+        
+        EM.defer do
+          begin
+            raise Cloudist::ExpiredMessage if request.expired?
+            yield msg
+            
+          rescue Cloudist::ExpiredMessage
+            log.error "AMQP Message Timeout: #{tag} ttl=#{request.ttl} age=#{request.age}"
+            
+          rescue Exception => e
+            Cloudist.handle_error(e)
+            
+          ensure
+            request.ack unless Cloudist.closing?
+          end
+        end
+      end
+    end
+    
+    private
+    
+    def setup
+      if self.class.cached_queues.keys.include?(name.to_sym)
+        @q = self.class.cached_queues[name.to_sym][:q]
+        @ex = self.class.cached_queues[name.to_sym][:ex]
+        @channel = self.class.cached_queues[name.to_sym][:channel]
+        setup_binding
+      else
+        puts "Setup"
+
+        setup_channel
+        setup_queue
+        setup_exchange
+
+        self.class.cached_queues[name.to_sym] = {:q => q, :ex => ex, :channel => channel}
+      end
+      setup_binding
+    end
+    
+    def setup_channel
+      @channel = ::AMQP::Channel.new
+      
+      # Set up QOS. If you do not do this then the subscribe in receive_message
+      # will get overwelmd and the whole thing will collapse in on itself.
+      channel.prefetch(1)
+    end
+    
+    def setup_queue
+      @q = channel.queue(name, options)
+    end
+    
+    def setup_exchange
+      @ex = channel.direct(name)
+      # setup_binding
+    end
+    
+    def setup_binding
+      q.bind(ex)
+    end
+    
+  end
+end

data/lib/cloudist/queues/basic_queue.rb

@@ -4,92 +4,122 @@ module Cloudist
   
   module Queues
     class BasicQueue
-      attr_reader :queue_name, :opts
-      attr_reader :q, :ex, :mq
-
-      def initialize(queue_name, opts = {})
-        opts = {
+      attr_reader :queue_name, :options
+      attr_reader :queue, :exchange, :channel, :prefetch
+      
+      alias :q :queue
+      alias :ex :exchange
+      alias :mq :channel
+      
+      def initialize(queue_name, options = {})
+        @prefetch ||= options.delete(:prefetch) || 1
+        
+        options = {
           :auto_delete => true,
           :durable => false,
-          :prefetch => 1
-        }.update(opts)
+          :nowait => true
+        }.update(options)
 
-        @queue_name, @opts = queue_name, opts
+        @queue_name, @options = queue_name, options
+        
+        setup
+      end
+      
+      def inspect
+        "<#{self.class.name} queue_name=#{queue_name}>"
       end
 
       def setup
-        return if @setup == true
-
-        @mq = MQ.new
-        @q = @mq.queue(queue_name, opts)
-        # if we don't specify an exchange name it defaults to the queue_name
-        @ex = @mq.direct(opts[:exchange_name] || queue_name)
-
-        q.bind(ex) if ex
-
+        return if @setup.eql?(true)
+        
+        @channel ||= AMQP::Channel.new(Cloudist.connection) do
+          channel.prefetch(self.prefetch, false) if self.prefetch.present?
+        end
+        
+        @queue = @channel.queue(queue_name, options)
+        
+        setup_exchange
+        
         @setup = true
       end
+      
+      def setup_exchange
+        @exchange = channel.direct("")
+      end
+
+      # def setup_exchange
+      #   @exchange = channel.direct(queue_name)
+      #   queue.bind(exchange)
+      # end
 
       def log
         Cloudist.log
       end
 
       def tag
-        s = "queue=#{q.name}"
-        s += " exchange=#{ex.name}" if ex
+        s = "queue=#{queue.name}"
+        s += " exchange=#{exchange.name}" if exchange
         s
       end
-
-      def subscribe(amqp_opts={}, opts={})
-        setup
-        print_status
-        q.subscribe(amqp_opts) do |queue_header, json_encoded_message|
-          next if Cloudist.closing?
-
-          request = Cloudist::Request.new(self, json_encoded_message, queue_header)
-
-          begin
-            raise Cloudist::ExpiredMessage if request.expired?
-            yield request if block_given?
-            # finished = Time.now.utc.to_i
-            # log.debug("Finished Job in #{finished - request.start} seconds")
-
-          rescue Cloudist::ExpiredMessage
-            log.error "AMQP Message Timeout: #{tag} ttl=#{request.ttl} age=#{request.age}"
-            request.ack if amqp_opts[:ack]
-
-          rescue => e
-            request.ack if amqp_opts[:ack]
-            Cloudist.handle_error(e)
-          end
+      
+      def subscribe(&block)
+        queue.subscribe(:ack => true) do |queue_header, encoded_message|
+          # next if Cloudist.closing?
+
+          request = Cloudist::Request.new(self, encoded_message, queue_header)
+          
+          handle_request = proc {
+            begin
+              raise Cloudist::ExpiredMessage if request.expired?
+              # yield request if block_given?
+              block.call(request)
+
+            rescue Cloudist::ExpiredMessage
+              log.error "AMQP Message Timeout: #{tag} ttl=#{request.ttl} age=#{request.age}"
+
+            rescue => e
+              Cloudist.handle_error(e)
+            ensure
+              request.ack
+              # unless Cloudist.closing?
+              # finished = Time.now.utc.to_i
+              # log.debug("Finished Job in #{finished - request.start} seconds")
+            end
+          }
+          
+          handle_ack = proc {
+            request.ack
+          }
+          
+          EM.defer(handle_request, handle_ack)
         end
         log.info "AMQP Subscribed: #{tag}"
         self
       end
 
       def print_status
-        q.status{ |num_messages, num_consumers|
-          log.info("STATUS: #{q.name}: JOBS: #{num_messages} WORKERS: #{num_consumers+1}")
-        }
+        # queue.status{ |num_messages, num_consumers|
+        #   log.info("STATUS: #{queue.name}: JOBS: #{num_messages} WORKERS: #{num_consumers+1}")
+        # }
       end
 
       def publish(payload)
         payload.set_reply_to(queue_name)
-        body, headers = payload.formatted
-        ex.publish(body, headers)
-        payload.publish
+        body, headers = payload.to_a
+        headers.merge!(:routing_key => queue.name)
+        exchange.publish(body, headers)
       end
 
       def publish_to_q(payload)
-        body, headers = payload.formatted
-        q.publish(body, headers)
-        payload.publish
+        body, headers = payload.to_a
+        # headers.merge!(:routing_key => queue.name)
+        queue.publish(body, headers)
         return headers
       end
 
       def teardown
-        @q.unsubscribe
-        @mq.close
+        @queue.unsubscribe
+        @channel.close
         log.debug "AMQP Unsubscribed: #{tag}"
       end
 

data/lib/cloudist/queues/job_queue.rb

@@ -1,28 +1,17 @@
 module Cloudist
   class JobQueue < Cloudist::Queues::BasicQueue
-    attr_reader :prefetch
-
-    def initialize(queue_name, opts={})
-      @prefetch = opts.delete(:prefetch) || 1
-      opts[:auto_delete] = false
-
-      super(queue_name, opts)
-    end
-
-    def setup
-      super
-      @mq.prefetch(self.prefetch)
-    end
-
-    def subscribe(amqp_opts={}, opts={})
-      amqp_opts[:ack] ||= true
-      super(amqp_opts, opts) do |request|
-        begin
-          yield request if block_given?
-        ensure
-          request.ack unless amqp_opts[:auto_ack] == false || Cloudist.closing?
-        end
-      end
-    end
+    
+    # def initialize(queue_name, options={})
+    #   @prefetch = 1
+    #   # opts[:auto_delete] = false
+    #   
+    #   super(queue_name, options)
+    # end
+    
+    # def setup_exchange
+    #   @exchange = channel.direct(queue_name)
+    #   queue.bind(exchange)
+    # end
+    
   end
 end

data/lib/cloudist/queues/reply_queue.rb

@@ -1,31 +1,23 @@
 module Cloudist
   class ReplyQueue < Cloudist::Queues::BasicQueue
-    def initialize(queue_name, opts={})
-      opts[:auto_delete] = true
-      opts[:nowait] = false
-      super
+    def initialize(queue_name, options={})
+      options[:auto_delete] = true
+      options[:nowait] = true
+      
+      @prefetch = 2
+      
+      super(queue_name, options)
     end
-
-    def setup(key = nil)
-      @mq = MQ.new
-      @q = @mq.queue(queue_name, opts)
-      @ex = @mq.direct
-      if key
-        @q.bind(@ex, :key => key)
-      else
-        @q.bind(@ex)
-      end
-    end
-
-    # def subscribe(amqp_opts={}, opts={})
-    #   super(amqp_opts, opts) do |request|
+    
+    # def subscribe(&block)
+    #   super do |request|
     #     yield request if block_given?
-    #     self.destroy
+    #     teardown
     #   end
     # end
-    # 
+    
     # def teardown
-    #   @q.delete
+    #   queue.delete
     #   super
     # end
     

data/lib/cloudist/request.rb

@@ -1,31 +1,55 @@
 module Cloudist
   class Request
-    attr_reader :queue_header, :qobj, :payload, :start, :headers
+    include Cloudist::Encoding
+    
+    attr_reader :queue_header, :qobj, :payload, :start, :headers, :body
 
     def initialize(queue, encoded_body, queue_header)
       @qobj, @queue_header = queue, queue_header
 
+      @body = decode(encoded_body)
+      @headers = parse_custom_headers(queue_header)
+      
       @payload = Cloudist::Payload.new(encoded_body, queue_header.headers.dup)
-      @headers = @payload.parse_custom_headers
+      @headers = @payload.headers
 
-      @start = Time.now.utc.to_i
+      @start = Time.now.utc.to_f
+    end
+    
+    def parse_custom_headers(amqp_headers)
+      h = amqp_headers.headers.dup
+
+      h[:published_on] = h[:published_on].to_i
+
+      h[:ttl] = h[:ttl].to_i rescue -1
+      h[:ttl] = -1 if h[:ttl] == 0
+
+      h
+    end
+    
+    def for_message
+      [body.dup, queue_header.headers.dup]
     end
 
     def q
-      qobj.q
+      qobj.queue
     end
 
     def ex
-      qobj.ex
+      qobj.exchange
     end
 
     def mq
-      qobj.mq
+      qobj.channel
+    end
+    
+    def channel
+      mq
     end
 
     def age
       return -1 unless headers[:published_on]
-      start - headers[:published_on]
+      start - headers[:published_on].to_f
     end
 
     def ttl
@@ -45,6 +69,8 @@ module Cloudist
       return if acked?
       queue_header.ack
       @acked = true
+    rescue AMQP::ChannelClosedError => e
+      Cloudist.handle_error(e)
     end
 
   end