cloudist 0.0.2

data/lib/cloudist/basic_queue.rb

@@ -0,0 +1,91 @@
+module Cloudist
+  class UnknownReplyTo < RuntimeError; end
+  class ExpiredMessage < RuntimeError; end
+  
+  class BasicQueue
+    attr_reader :queue_name, :opts
+    attr_reader :q, :ex, :mq
+    
+    def initialize(queue_name, opts = {})
+      opts = {
+        :auto_delete => true,
+        :durable => false,
+        :prefetch => 1
+      }.update(opts)
+      
+      @queue_name, @opts = queue_name, opts
+    end
+    
+    def setup
+      return if @setup == true
+      
+      @mq = MQ.new
+      @q = @mq.queue(queue_name, opts)
+      # if we don't specify an exchange name it defaults to the queue_name
+      @ex = @mq.direct(opts[:exchange_name] || queue_name)
+
+      q.bind(ex) if ex
+      
+      @setup = true
+    end
+    
+    def log
+      Cloudist.log
+    end
+    
+    def tag
+      s = "queue=#{q.name}"
+      s += " exchange=#{ex.name}" if ex
+      s
+    end
+    
+    def subscribe(amqp_opts={}, opts={})
+      setup
+      
+      q.subscribe(amqp_opts) do |queue_header, json_encoded_message|
+        return if Cloudist.closing?
+        
+        request = Cloudist::Request.new(self, json_encoded_message, queue_header)
+        
+        begin
+          raise Cloudist::ExpiredMessage if request.expired?
+          yield request if block_given?
+          finished = Time.now.utc.to_i
+
+        rescue Cloudist::ExpiredMessage
+          log.info "amqp_message action=timeout #{tag} ttl=#{request.ttl} age=#{request.age} #{request.inspect}"
+          request.ack if amqp_opts[:ack]
+
+        rescue => e
+          request.ack if amqp_opts[:ack]
+          Cloudist.handle_error(e)
+        end
+      end
+      log.info "amqp_subscribe #{tag}"
+      self
+    end
+    
+    def publish(payload)
+      payload.set_reply_to(queue_name)
+      body, headers = payload.formatted
+      ex.publish(body, headers)
+      payload.publish
+    end
+    
+    def publish_to_q(payload)
+      body, headers = payload.formatted
+      q.publish(body, headers)
+      payload.publish
+    end
+    
+    def teardown
+      @q.unsubscribe
+      @mq.close
+      log.debug "amqp_unsubscribe #{tag}"
+    end
+
+    def destroy
+      teardown
+    end
+  end
+end

data/lib/cloudist/core_ext/string.rb

@@ -0,0 +1,11 @@
+class String
+  # Returns true iff +other+ appears exactly at the start of +self+.
+  def starts_with? other
+    self[0, other.length] == other
+  end
+
+  # Returns true iff +other+ appears exactly at the end of +self+.
+  def ends_with? other
+    self[-other.length, other.length] == other
+  end
+end

data/lib/cloudist/errors.rb

@@ -0,0 +1,6 @@
+module Cloudist
+  class Error < RuntimeError; end
+  class BadPayload < Error; end
+  class EnqueueError < Error; end
+  class StaleHeadersError < BadPayload; end
+end

data/lib/cloudist/job.rb

@@ -0,0 +1,54 @@
+module Cloudist
+  class Job
+    attr_reader :payload
+    def initialize(payload)
+      @payload = payload
+    end
+    
+    def id
+      payload.id
+    end
+    
+    def data
+      payload.body
+    end
+    
+    def log
+      Cloudist.log
+    end
+    
+    def cleanup
+      
+    end
+    
+    def reply(data, headers = {})
+      # headers.update(:message_id => payload.headers[:message_id])
+      headers = {
+        :message_id => payload.headers[:message_id],
+        :reply_type => "reply"
+      }.update(headers)
+      
+      reply_payload = Payload.new(data, headers)
+      
+      reply_queue = ReplyQueue.new(payload.reply_to)
+      reply_queue.setup
+      reply_queue.publish_to_q(reply_payload)
+      
+      # log.debug("Replying: #{data.inspect} - Payload: #{reply_payload.inspect}")
+    end
+    
+    def event(event_name, data = {})
+      data = {} unless data
+      reply({:event => event_name}.merge(payload.body), {:reply_type => "event"})
+    end
+    
+    def method_missing(meth, *args, &blk)
+      if meth.to_s.ends_with?("!")
+        event(meth.to_s.gsub(/(!)$/, ''), args.shift)
+      else
+        super
+      end
+    end
+    
+  end
+end

data/lib/cloudist/job_queue.rb

@@ -0,0 +1,28 @@
+module Cloudist
+  class JobQueue < BasicQueue
+    attr_reader :prefetch
+
+    def initialize(queue_name, opts={})
+      @prefetch = opts.delete(:prefetch) || 1
+      opts[:auto_delete] = false
+
+      super(queue_name, opts)
+    end
+
+    def setup
+      super
+      @mq.prefetch(self.prefetch)
+    end
+
+    def subscribe(amqp_opts={}, opts={})
+      amqp_opts[:ack] = true
+      super(amqp_opts, opts) do |request|
+        begin
+          yield request if block_given?
+        ensure
+          request.ack unless amqp_opts[:auto_ack] == false
+        end
+      end
+    end
+  end
+end

data/lib/cloudist/listener.rb

@@ -0,0 +1,30 @@
+module Cloudist
+  class Listener
+    
+    attr_reader :job_queue_name, :job_id
+    
+    def initialize(job_or_queue_name)
+      if job_or_queue_name.is_a?(Cloudist::Job)
+        @job_queue_name = Utils.reply_prefix(job_or_queue_name.payload.headers[:master_queue])
+        @job_id = job_or_queue_name.id
+      elsif job_or_queue_name.is_a?(String)
+        @job_queue_name = Utils.reply_prefix(job_or_queue_name)
+        @job_id = nil
+      else
+        raise ArgumentError, "Invalid listener type, accepts job queue name or Cloudist::Job instance"
+      end
+    end
+    
+    def subscribe(&block)
+      reply_queue = Cloudist::ReplyQueue.new(job_queue_name)
+      reply_queue.setup(job_id) if job_id
+      
+      reply_queue.subscribe do |request|
+        job = Job.new(request.payload)
+        
+        job.instance_eval(&block)
+      end
+    end
+    
+  end
+end

data/lib/cloudist/payload.rb

@@ -0,0 +1,128 @@
+module Cloudist
+  DEFAULT_TTL = 300
+
+  class Payload
+    include Utils
+    
+    attr_accessor :body, :headers
+
+    def initialize(data_hash_or_json, headers = {})
+      data_hash_or_json = parse_message(data_hash_or_json) if data_hash_or_json.is_a?(String)
+      
+      raise Cloudist::BadPayload, "Expected Hash for payload" unless data_hash_or_json.is_a?(Hash)
+
+      @body, @headers = HashWithIndifferentAccess.new(data_hash_or_json), headers
+      update_headers
+    end
+
+    def formatted
+      body, headers = apply_custom_headers
+
+      # Return message formatted as JSON and headers ready for transport
+      [body.to_json, headers]
+    end
+    
+    def id
+      @id ||= event_hash.to_s
+    end
+    
+    def id=(new_id)
+      @id = new_id.to_s
+      update_headers
+    end
+    
+    def frozen?
+      headers.frozen?
+    end
+    
+    def freeze!
+      headers.freeze
+      body.freeze
+    end
+    
+    def update_headers
+      raise StaleHeadersError, "Headers cannot be changed because payload has already been published" if published?
+      
+      headers[:published_on] ||= body.delete('published_on') || Time.now.utc.to_i
+      headers[:ttl] ||= body.delete('ttl') || Cloudist::DEFAULT_TTL
+
+      # this is the event hash that gets transferred through various publish/reply actions
+      headers[:event_hash] ||= id
+
+      # this value should be unique for each published/received message pair
+      headers[:message_id] ||= id
+      
+      # We use JSON for message transport exclusively
+      headers[:content_type] ||= 'application/json'
+
+      # some strange behavior with integers makes it better to
+      # convert all amqp headers to strings to avoid any problems
+      headers.each { |k,v| headers[k] = v.to_s }
+    end
+    
+    def apply_custom_headers
+      update_headers
+      [body, headers]
+    end
+
+    def parse_custom_headers
+      return { } unless headers
+
+      h = headers.dup
+
+      h[:published_on] = h[:published_on].to_i
+
+      h[:ttl] = h[:ttl].to_i rescue -1
+      h[:ttl] = -1 if h[:ttl] == 0
+
+      h
+    end
+    
+    def set_reply_to(queue_name)
+      headers[:reply_to] = reply_name(queue_name)
+      set_master_queue_name(queue_name)
+    end
+    
+    def set_master_queue_name(queue_name)
+      headers[:master_queue] = queue_name 
+    end
+    
+    def reply_name(queue_name)
+      # "#{queue_name}.#{id}"
+      Utils.reply_prefix(queue_name)
+    end
+    
+    def reply_to
+      headers[:reply_to]
+    end
+    
+    def event_hash
+      @event_hash ||= headers[:event_hash] || body.delete('event_hash') || create_event_hash
+    end
+    
+    def create_event_hash
+      s = Time.now.to_s + object_id.to_s + rand(100).to_s
+      Digest::MD5.hexdigest(s)
+    end
+    
+    def parse_message(raw)
+      return { } unless raw
+      JSON.parse(raw)
+    end
+    
+    def [](key)
+      body[key]
+    end
+    
+    def published?
+      @published == true
+    end
+    
+    def publish
+      return if published?
+      @published = true
+      freeze!
+    end
+    
+  end
+end

data/lib/cloudist/publisher.rb

@@ -0,0 +1,17 @@
+module Cloudist
+  class Publisher
+    
+    class << self
+      def enqueue(queue_name, data)
+        payload = Cloudist::Payload.new(data)
+
+        queue = Cloudist::JobQueue.new(queue_name)
+        queue.setup
+        queue.publish(payload)
+        
+        return Job.new(payload)
+      end
+    end
+    
+  end
+end

data/lib/cloudist/reply_queue.rb

@@ -0,0 +1,32 @@
+module Cloudist
+  class ReplyQueue < Cloudist::BasicQueue
+    def initialize(queue_name, opts={})
+      opts[:auto_delete] = true
+      super
+    end
+
+    def setup(key = nil)
+      @mq = MQ.new
+      @q = @mq.queue(queue_name, opts)
+      @ex = @mq.direct
+      if key
+        @q.bind(@ex, :key => key)
+      else
+        @q.bind(@ex)
+      end
+    end
+
+    # def subscribe(amqp_opts={}, opts={})
+    #   super(amqp_opts, opts) do |request|
+    #     yield request if block_given?
+    #     self.destroy
+    #   end
+    # end
+    # 
+    # def teardown
+    #   @q.delete
+    #   super
+    # end
+    
+  end
+end

data/lib/cloudist/request.rb

@@ -0,0 +1,51 @@
+module Cloudist
+  class Request
+    attr_reader :queue_header, :qobj, :payload, :start, :headers
+
+    def initialize(queue, json_encoded_message, queue_header)
+      @qobj, @queue_header = queue, queue_header
+
+      @payload = Cloudist::Payload.new(json_encoded_message.dup, queue_header.properties.dup)
+      @headers = @payload.parse_custom_headers
+      
+      @start = Time.now.utc.to_i
+    end
+
+    def q
+      qobj.q
+    end
+
+    def ex
+      qobj.ex
+    end
+
+    def mq
+      qobj.mq
+    end
+
+    def age
+      return -1 unless headers[:published_on]
+      start - headers[:published_on]
+    end
+
+    def ttl
+      headers[:ttl] || -1
+    end
+
+    def expired?
+      return false if ttl == -1
+      age > ttl
+    end
+
+    def acked?
+      @acked == true
+    end
+
+    def ack
+      return if acked?
+      queue_header.ack
+      @acked = true
+    end
+
+  end
+end