seanohalpin-smqueue 0.1.0

data/lib/smqueue.rb

@@ -0,0 +1,227 @@
+# SMQueue
+# Simple Message Queue client
+# Sean O'Halpin, 2008
+
+# The high-level client API is:
+# - SMQueue(config).put msg, headers = {}
+# - msg = SMQueue(config).get(headers = {})
+# - SMQueue(config).get(headers = {}) do |msg|
+#   end
+# todo - [X] add :durable option (and fail if no client_id specified)
+# todo - [ ] gemify - use Mr Bones
+# todo - [ ] change to class (so can be subclassed) - so you're working with an SMQueue instance
+# todo - [ ] write protocol (open, close, put, get) in SMQueue (so don't have to do it in adaptors)
+# todo - [ ] simplify StompAdapter (get rid of sent_messages stuff)
+# todo - [ ] simplify adapter interface
+# todo - [ ] sort out libpath
+
+require 'rubygems'
+require 'doodle'
+require 'doodle/version'
+require 'yaml'
+
+class Doodle
+  if Doodle::VERSION::STRING < '0.1.9'
+    def to_hash
+      doodle.attributes.inject({}) {|hash, (name, attribute)| hash[name] = send(name); hash}
+    end
+    class DoodleAttribute
+      has :doc, :kind => String
+    end
+  end
+end
+
+#class SMQueue < Doodle
+module SMQueue
+
+  # Mr Bones project skeleton boilerplate
+  # :stopdoc:
+  VERSION = '0.1.0'
+  LIBPATH = ::File.expand_path(::File.dirname(__FILE__)) + ::File::SEPARATOR
+  PATH = ::File.dirname(LIBPATH) + ::File::SEPARATOR
+  # :startdoc:
+
+  # Returns the version string for the library.
+  #
+  def self.version
+    VERSION
+  end
+
+  # Returns the library path for the module. If any arguments are given,
+  # they will be joined to the end of the libray path using
+  # <tt>File.join</tt>.
+  #
+  def self.libpath( *args )
+    args.empty? ? LIBPATH : ::File.join(LIBPATH, args.flatten)
+  end
+
+  # Returns the lpath for the module. If any arguments are given,
+  # they will be joined to the end of the path using
+  # <tt>File.join</tt>.
+  #
+  def self.path( *args )
+    args.empty? ? PATH : ::File.join(PATH, args.flatten)
+  end
+
+  # Utility method used to rquire all files ending in .rb that lie in the
+  # directory below this file that has the same name as the filename passed
+  # in. Optionally, a specific _directory_ name can be passed in such that
+  # the _filename_ does not have to be equivalent to the directory.
+  #
+  def self.require_all_libs_relative_to( fname, dir = nil )
+    dir ||= ::File.basename(fname, '.*')
+    search_me = ::File.expand_path(
+        ::File.join(::File.dirname(fname), dir, '*', '*.rb'))
+
+    Dir.glob(search_me).sort.each {|rb| require rb}
+  end
+
+  # end Bones boilerplate
+  
+  class << self
+    def dbg(*args, &block)
+      if $DEBUG
+        if args.size > 0
+          STDERR.print "SMQUEUE.DBG: "
+          STDERR.puts(*args)
+        end
+        if block_given?
+          STDERR.print "SMQUEUE.DBG: "
+          STDERR.puts(block.call)
+        end
+      end
+    end
+
+    # JMS expiry time in milliseconds from now
+    def calc_expiry_time(seconds = 86400 * 7) # one week
+      ((Time.now.utc + seconds).to_f * 1000).to_i
+    end
+    
+    # resolve a string representing a classname
+    def const_resolve(constant)
+      constant.to_s.split(/::/).reject{|x| x.empty?}.inject(self) { |prev, this| prev.const_get(this) }
+    end
+  end
+
+  class AdapterConfiguration < Doodle
+    has :logger, :default => nil
+
+    # need to use custom to_yaml because YAML won't serialize classes
+    def to_hash
+      doodle.attributes.inject({}) {|hash, (name, attribute)| hash[name] = send(name); hash}
+    end
+    def to_yaml(*opts)
+      to_hash.to_yaml(*opts)
+    end
+    def initialize(*args, &block)
+      #p [self.class, :initialize, args, caller]
+      super
+    end
+    has :adapter_class, :kind => Class do
+      from String, Symbol do |s|
+        SMQueue.const_resolve(s.to_s)
+      end
+      # Note: use closure so this is not evaluated until after NullAdapter class has been defined
+      default { NullAdapter }
+    end
+    has :configuration_class, :kind => Class do
+      init { adapter_class::Configuration }
+      from String do |s|
+        #Doodle::Utils.const_resolve(s)
+        SMQueue.const_resolve(s.to_s)
+      end
+    end
+  end
+
+  class Adapter < Doodle
+    has :configuration, :kind => AdapterConfiguration, :abstract => true do
+      from Hash do |h|
+        #p [:Adapter, :configuration_from_hash]
+        Doodle.context.last.class::Configuration.new(h)
+      end
+      from Object do |h|
+        #p [:Adapter, :configuration_from_object, h.inspect, h.class]
+        h
+      end
+    end
+    # these are not called anywhere...
+    def open(*args, &block)
+    end
+    def close(*args, &block)
+    end
+    # these are the core methods
+    def get(*args, &block)
+    end
+    def put(*args, &block)
+    end
+    def self.create(configuration)
+      # FIXME: dup config, otherwise can use it only once - prob. better way to do this
+      configuration = configuration.dup
+      adapter = configuration.delete(:adapter)
+      #p [:adapter, adapter]
+      ac = AdapterConfiguration.new(:adapter_class => adapter)
+      #p [:ac, ac]
+      klass = ac.adapter_class
+      #p [:class, klass]
+      #puts [:configuration, configuration].pretty_inspect
+#      klass.new(:configuration => configuration)
+      klass.new(:configuration => configuration)
+    end
+  end
+
+  class NullAdapter < Adapter
+    class Configuration < AdapterConfiguration
+    end
+  end
+
+  class Message < Doodle
+    has :headers, :default => { }
+    has :body
+  end
+  
+  class << self
+    def new(*args, &block)
+      a = args.first
+      if a.kind_of?(Hash) && a.key?(:configuration)
+        args = [a[:configuration]]
+      end
+      Adapter.create(*args, &block)
+    end
+  end
+
+end
+def SMQueue(*args, &block)
+ SMQueue.new(*args, &block)
+end
+
+# SMQueue.require_all_libs_relative_to(__FILE__)
+
+# require adapters relative to invocation path first, then from lib
+[$0, __FILE__].each do |path|
+  base_path = File.expand_path(File.dirname(path))
+  adapter_path = File.join(base_path, 'smqueue', 'adapters', '*.rb')
+  Dir[adapter_path].each do |file|
+    require file
+  end
+end
+
+if __FILE__ == $0
+  yaml = %[
+:adapter: :StompAdapter
+:host: localhost
+:port: 61613
+:name: /topic/smput.test
+:reliable: true
+:reconnect_delay: 5
+:subscription_name: test_stomp
+:client_id: hello_from_stomp_adapter
+:durable: false
+]
+
+  adapter = SMQueue(:configuration => YAML.load(yaml))
+  adapter.get do |msg|
+    puts msg.body
+  end
+
+end
+

data/lib/smqueue/adapters/spread.rb

@@ -0,0 +1,101 @@
+require 'spread'
+
+module SMQueue
+  class SpreadAdapter < Adapter
+    class Configuration < AdapterConfiguration
+      has :channel do
+        rx_hostname = /[a-z_\.]+/
+        rx_ip = /\d+(\.\d+){3}/ # dotted quad
+        must 'be a name in the form "port", "port@hostname", or "port@ip"' do |s|
+          s =~ /\d+(@(#{rx_hostname})|(#{rx_ip}))?/
+        end
+      end
+      has :group do
+        doc "a group name or array of group names"
+        must "be either a String group name or an array of group names" do |s|
+          s.kind_of?(String) || (s.kind_of?(Array) && s.all?{ |x| x.kind_of?(String)})
+        end
+      end
+      has :private_name, :default => '' do
+        doc <<EDOC
+private_name is the name of this connection. It must be unique among
+all the connections to a given Spread daemon. If not specified, Spread
+will assign a randomly-generated unique private name.
+EDOC
+      end
+      has :all_messages, :default => false do
+        doc <<EDOC
+all_messages indicates whether this connection should receive all
+Spread messages, or just data messages.
+EDOC
+      end
+      has :service_type, :default => Spread::AGREED_MESS do
+        service_type_map = {
+          :unreliable => Spread::UNRELIABLE_MESS,
+          :reliable => Spread::RELIABLE_MESS,
+          :fifo => Spread::FIFO_MESS,
+          :causal => Spread::CAUSAL_MESS,
+          :agreed => Spread::AGREED_MESS,
+          :safe => Spread::SAFE_MESS,
+          :regular => Spread::REGULAR_MESS,
+        }
+        from Symbol, String do |s|
+          s = s.to_s.to_sym
+          if service_type_map.key?(s)
+            service_type_map[s]
+          else
+            raise Doodle::ConversionError, "Did not recognize service_type #{s.inspect} - should be one of #{service_type_map.keys.inspect}"
+          end
+        end
+      end
+    end
+
+    has :connection, :default => nil
+    has :connected, :default => false
+
+    def connect
+      @connection = Spread::Connection.new(configuration.channel, configuration.private_name, configuration.all_messages )
+      connection.join(configuration.group)
+      configuration.private_name = connection.private_group
+      connected true
+    end
+    def disconnect
+      connection.leave group
+      connection.disconnect
+      connected false
+    end
+    def get(&block)
+      m = nil
+      connect if !connected
+      loop do
+        msg = connection.receive
+        if msg.data?
+          m = SMQueue::Message.new(
+                               :headers => {
+                                 :private_name => configuration.private_name,
+                                 :sender => msg.sender,
+                                 :type => msg.msg_type,
+                                 :groups => msg.groups,
+                                 :reliable => msg.reliable?,
+                                 :safe => msg.safe?,
+                                 :agreed => msg.agreed?,
+                                 :causal => msg.causal?,
+                                 :fifo => msg.fifo?,
+                               },
+                               :body => msg.message
+                               )
+          if block_given?
+            yield(m)
+          else
+            break
+          end
+        end
+      end
+      m
+    end
+    def put(msg)
+      connect if !connected
+      connection.multicast(msg, configuration.group, configuration.service_type, msg_type = 0, self_discard = true)
+    end
+  end
+end

data/lib/smqueue/adapters/stdio.rb

@@ -0,0 +1,59 @@
+# set of utility adapters for SMQueue
+
+module SMQueue
+  class StdioLineAdapter < Adapter
+    doc "reads STDIN input, creates new Message for each line of input"
+    class Configuration < AdapterConfiguration
+    end
+    def put(*args, &block)
+      STDOUT.puts *args
+    end
+    def get(*args, &block)
+      while input = STDIN.gets
+        msg = SMQueue::Message.new(:body => input)
+        if block_given?
+          yield(msg)
+        end
+      end
+    end
+  end
+end
+
+module SMQueue
+  class StdioAdapter < StdioLineAdapter
+    doc "reads complete STDIN input, creates one shot Message with :body => input"
+    def get(*args, &block)
+      input = STDIN.read
+      msg = SMQueue::Message.new(:body => input)
+      if block_given?
+        yield(msg)
+      end
+    end
+  end
+end
+
+module SMQueue
+  class YamlAdapter < StdioAdapter
+    doc "outputs message as YAML"
+    def put(*args)
+      STDOUT.puts args.to_yaml
+    end
+  end
+end
+
+require 'readline'
+module SMQueue
+  class ReadlineAdapter < StdioLineAdapter
+    doc "uses readline to read input from prompt, creates new Message for each line of input"
+    has :prompt, :default => "> "
+    has :history, :default => true
+    def get(*args, &block)
+      while input = Readline.readline(prompt, history)
+        msg = Message.new(:body => input)
+        if block_given?
+          yield(msg)
+        end
+      end
+    end
+  end
+end

data/lib/smqueue/adapters/stomp.rb

@@ -0,0 +1,291 @@
+require 'rstomp'
+module SMQueue
+  class StompAdapter < Adapter
+    class Configuration < AdapterConfiguration
+      has :host, :kind => String, :default => ""
+      has :port, :kind => Integer, :default => 61613  # Stomp
+      has :secondary_host, :kind => String, :default => ""
+      has :secondary_port, :kind => Integer, :default => 61613
+      has :name, :kind => String, :default => ""
+      has :user, :kind => String, :default => ""
+      has :password, :kind => String, :default => ""
+      has :reliable, :default => false
+      has :persistent, :default => true
+      has :reconnect_delay, :default => 5
+      has :client_id, :default => nil, :kind => String
+      has :logfile, :default => STDERR
+      has :logger, :default => nil
+      has :subscription_name, :default => nil
+      has :home, :default => File.dirname(File.expand_path(__FILE__))
+      has :single_delivery, :default => false do
+        doc <<-EDOC
+          Note: we can't guarantee single delivery - only best effort.
+          Use this when receiving a message more than once is very
+          costly. However, be aware that you ~will~ sometimes receive
+          the same message more than once (so it's your responsibility
+          to make sure that you guard against any consequences).
+        EDOC
+      end
+      has :seen_messages_file do
+        init { File.join(home, "seen_messages.#{subscription_name}.#{client_id}.yml") }
+      end
+      has :expires, :default => 86400 * 7 do
+        doc <<-EDOC
+          Time to live in milliseconds, i.e. a relative offset not an
+          absolute time (as it would be in JMS).
+
+          The default time to live is one week.
+        EDOC
+      end
+      # to get a durable subscription, you must specify
+      #   :durable => true
+      # and a :client_id (and optionally a :subscription_name)
+      has :durable, :default => false do
+        must "be true or false" do |b|
+          [true, false].include?(b)
+        end
+        doc <<-EDOC
+          Specify whether you want a durable or non-durable subscription.
+      
+          Note: durable queues are ~not~ the default as this could be
+          v. expensive in disk usage when used thoughtlessly.
+        EDOC
+      end
+      must "specify client_id if durable is true" do
+        #pp [:durable_test, client_id, durable, !client_id.to_s.strip.empty?]
+        !(client_id.to_s.strip.empty? and durable)
+      end
+    end
+    has :connection, :default => nil
+
+    # seen_messages is used to skip over messages that have already
+    # been seen - only activated when :single_delivery is specified
+    has :seen_messages, :init => []
+    has :seen_message_count, :init => 10
+    has :seen_messages_file do
+      init { configuration.seen_messages_file }
+    end
+
+    def initialize(*args, &block)
+      super
+      restore_remembered_messages
+      SMQueue.dbg { [:seen_messages, seen_messages].inspect }
+    end
+
+    # handle an error
+    def handle_error(exception_class, error_message, caller)
+      #configuration.logger.warn error_message
+      raise exception_class, error_message, caller
+    end
+
+    # connect to message broker
+    def connect(*args, &block)
+      self.connection = RStomp::Connection.open(configuration.to_hash)   
+      # If the connection has swapped hosts, then swap out primary and secondary
+      if connection.current_host != configuration.host
+        configuration.secondary_host = configuration.host
+        configuration.host = connection.current_host
+      end
+
+      # If the connection has swapped ports, then swap out primary and secondary
+      if connection.current_port != configuration.port
+        configuration.secondary_port = configuration.port
+        configuration.port = connection.current_port
+      end
+    end
+
+    # normalize hash keys (top level only)
+    # - normalizes keys to strings by default
+    # - optionally pass in name of method to use (e.g. :to_sym) to normalize keys
+    def normalize_keys(hash, method = :to_s)
+      hash = hash.dup
+      hash.keys.each do |k|
+        normalized_key = k.respond_to?(method) ? k.send(method) : k
+        hash[normalized_key] = hash.delete(k)
+      end
+      hash
+    end
+
+    # true if the message with this message_id has already been seen
+    def message_seen?(message_id)
+      self.seen_messages.include?(message_id)
+    end
+
+    # remember the message_id
+    def message_seen(message_id)
+      message_id = message_id.to_s.strip
+      if message_id != ""
+        self.seen_messages << message_id
+        SMQueue.dbg { [:smqueue, :ack, :message_seen, message_id].inspect }
+        if self.seen_messages.size > self.seen_message_count
+          self.seen_messages.shift
+        end
+        store_remembered_messages
+      else
+        SMQueue.dbg { [:smqueue, :ack, :message_seen, message_id].inspect }
+      end
+    end
+    
+    # store the remembered message ids in a yaml file
+    def store_remembered_messages
+      if configuration.single_delivery
+        File.open(seen_messages_file, 'w') do |file|
+          file.write seen_messages.to_yaml
+        end
+      end
+    end
+
+    # reload remembered message ids from a yaml file
+    def restore_remembered_messages
+      if configuration.single_delivery
+        yaml = default_yaml = "--- []"
+        begin
+          File.open(seen_messages_file, 'r') do |file|
+            yaml = file.read
+          end
+        rescue Object
+          yaml = default_yaml
+        end
+        buffer = []
+        begin
+          buffer = YAML.load(yaml)
+          if !buffer.kind_of?(Array) or !buffer.all?{ |x| x.kind_of?(String)}
+            raise Exception, "Invalid seen_messages.yml file"
+          end
+        rescue Object
+          buffer = []
+        end
+        self.seen_messages = buffer
+      end
+    end
+    
+    # acknowledge message (if headers["ack"] == "client")
+    def ack(subscription_headers, message)
+      #p [:ack, message.headers["message-id"]]
+      if message.headers["message-id"].to_s.strip != "" && subscription_headers["ack"].to_s == "client"
+        SMQueue.dbg { [:smqueue, :ack, :message, message].inspect }
+        connection.ack message.headers["message-id"], { }
+      else
+        SMQueue.dbg { [:smqueue, :ack, :not_acknowledging, message].inspect }
+      end
+      if ENV['PAUSE_SMQUEUE']
+        $stderr.print "press enter to continue> "
+        $stderr.flush
+        $stdin.gets
+      end
+    end
+
+    # get message from queue
+    # - if block supplied, loop forever and yield(message) for each
+    #   message received
+    # default headers are:
+    #   :ack               => "client"
+    #   :client_id         => configuration.client_id
+    #   :subscription_name => configuration.subscription_name
+    #   
+    def get(headers = {}, &block)
+      self.connect
+      SMQueue.dbg { [:smqueue, :get, headers].inspect }
+      subscription_headers = {"ack" => "client", "activemq.prefetchSize" => 1 }
+      if client_id = configuration.client_id
+        subscription_headers["client_id"] = client_id
+      end
+      if sub_name = configuration.subscription_name
+        subscription_headers["subscription_name"] = sub_name
+      end
+      # if a client_id is supplied, then user wants a durable subscription
+      # N.B. client_id must be unique for broker
+      subscription_headers.update(headers)
+      #p [:subscription_headers_before, subscription_headers]
+      subscription_headers = normalize_keys(subscription_headers)
+      if configuration.durable and client_id = configuration.client_id || subscription_headers["client_id"]
+        subscription_name = configuration.subscription_name || subscription_headers["subscription_name"] || client_id
+        # activemq only
+        subscription_headers["activemq.subscriptionName"] = subscription_name
+        # JMS
+        subscription_headers["durable-subscriber-name"] = subscription_name
+      end
+      #p [:subscription_headers_after, subscription_headers]
+      
+      destination = configuration.name
+      SMQueue.dbg { [:smqueue, :get, :subscribing, destination, :subscription_headers, subscription_headers].inspect }
+      connection.subscribe destination, subscription_headers
+      message = nil
+      SMQueue.dbg { [:smqueue, :get, :subscription_headers, subscription_headers].inspect }
+      begin
+        # TODO: refactor this
+        if block_given?
+          SMQueue.dbg { [:smqueue, :get, :block_given].inspect }
+          # todo: change to @running - (and set to false from exception handler)
+          # also should check to see if anything left to receive on connection before bailing out
+          while true
+            SMQueue.dbg { [:smqueue, :get, :receive].inspect }
+            # block until message ready
+            message = connection.receive
+            SMQueue.dbg { [:smqueue, :get, :received, message].inspect }
+            case message.command
+            when "ERROR"
+              SMQueue.dbg { [:smqueue, :get, :ERROR, message].inspect }
+            when "RECEIPT"
+              SMQueue.dbg { [:smqueue, :get, :RECEIPT, message].inspect }
+            else
+              SMQueue.dbg { [:smqueue, :get, :yielding].inspect }
+              if !message_seen?(message.headers["message-id"])
+                yield(message)
+              end
+              SMQueue.dbg { [:smqueue, :get, :message_seen, message.headers["message-id"]].inspect }
+              message_seen message.headers["message-id"]
+              SMQueue.dbg { [:smqueue, :get, :returned_from_yield_now_calling_ack].inspect }
+              ack(subscription_headers, message)
+              SMQueue.dbg { [:smqueue, :get, :returned_from_ack].inspect }
+            end
+          end
+        else
+          SMQueue.dbg { [:smqueue, :get, :single_shot].inspect }
+          message = connection.receive
+          SMQueue.dbg { [:smqueue, :get, :received, message].inspect }
+          if !(message.command == "ERROR" or message.command == "RECEIPT")
+            SMQueue.dbg { [:smqueue, :get, :message_seen, message.headers["message-id"]].inspect }
+            message_seen message.headers["message-id"]
+            SMQueue.dbg { [:smqueue, :get, :ack, message].inspect }
+            ack(subscription_headers, message)
+            SMQueue.dbg { [:smqueue, :get, :returned_from_ack].inspect }
+          end
+        end
+      rescue Object => e
+        SMQueue.dbg { [:smqueue, :get, :exception, e].inspect }
+        handle_error e, "Exception in SMQueue#get: #{e.message}", caller
+      ensure
+        SMQueue.dbg { [:smqueue, :get, :ensure].inspect }
+        SMQueue.dbg { [:smqueue, :unsubscribe, destination, subscription_headers].inspect }
+        connection.unsubscribe destination, subscription_headers
+        SMQueue.dbg { [:smqueue, :disconnect].inspect }
+        connection.disconnect
+      end
+      SMQueue.dbg { [:smqueue, :get, :return].inspect }
+      message
+    end
+
+    # put a message on the queue
+    # default headers are:
+    #   :persistent => true
+    #   :ack        => "auto"
+    #   :expires    => configuration.expires
+    def put(body, headers = { })
+      SMQueue.dbg { [:smqueue, :put, body, headers].inspect }
+      begin
+        self.connect
+        headers = {:persistent => true, :ack => "auto", :expires => SMQueue.calc_expiry_time(configuration.expires) }.merge(headers)
+        destination = configuration.name
+        SMQueue.dbg { [:smqueue, :send, body, headers].inspect }
+        connection.send destination, body, headers
+      rescue Exception => e
+        SMQueue.dbg { [:smqueue, :exception, e].inspect }
+        handle_error e, "Exception in SMQueue#put - #{e.message}", caller
+        #connection.disconnect
+      ensure
+        connection.disconnect
+      end
+    end
+  end
+end