stompserver_ng 1.0.6

data/lib/stomp_server_ng/qmonitor.rb

@@ -0,0 +1,58 @@
+#
+#
+#
+module StompServer
+#
+# = Queue Monitor
+#
+class QueueMonitor
+  #
+  # Initialize the queue monitor.
+  #
+  def initialize(qstore,queues)
+    @qstore = qstore
+    @queues = queues
+    @stompid = StompServer::StompId.new
+    #
+    @@log = Logger.new(STDOUT)
+    @@log.level = StompServer::LogHelper.get_loglevel()
+    @@log.debug("QueueMonitor initialize comletes")
+    #
+  end
+  #
+  # Start monitor timer.
+  #
+  def start
+    count =0
+    EventMachine::add_periodic_timer 5, proc {count+=1; monitor(count) }
+  end
+  #
+  # Respond to calls from the timer.  Do nothing if no clients are connected
+  # to the '/queue/monitor' destination.
+  #
+  def monitor(count)
+    return unless (@qstore.methods.include?(:monitor) | @qstore.methods.include?('monitor'))
+    users = @queues['/queue/monitor']
+    return if users.size == 0
+    stats = @qstore.monitor
+    return if stats.size == 0
+    body = ''
+    #
+    stats.each do |queue,qstats|
+      body << "Queue: #{queue}\n"
+      qstats.each {|stat,value| body << "#{stat}: #{value}\n"}
+      body << "\n"
+    end
+    #
+    headers = {
+      'message-id' => @stompid[count],
+      'destination' => '/queue/monitor',
+      'content-length' => body.size.to_s
+    }
+    #
+    frame = StompServer::StompFrame.new('MESSAGE', headers, body)
+    users.each {|user| user.connection.stomp_send_data(frame)}
+  end
+end # of class QueueMonitor
+end # of module StompServer
+

data/lib/stomp_server_ng/queue.rb

@@ -0,0 +1,248 @@
+#
+#
+#
+module StompServer
+  #
+  # == Queue
+  #
+  class Queue
+    # the check point interval
+    attr_accessor :checkpoint_interval
+
+    # initiialize
+    def initialize(directory='.stompserver', delete_empty=true)
+
+      @@log = Logger.new(STDOUT)
+      @@log.level = StompServer::LogHelper.get_loglevel()
+      @@log.debug("Q #{self} initialization starts")
+
+      @stompid = StompServer::StompId.new
+      @delete_empty = delete_empty
+      @directory = directory
+      Dir.mkdir(@directory) unless File.directory?(@directory)
+      if File.exists?("#{@directory}/qinfo")
+        qinfo = Hash.new
+        File.open("#{@directory}/qinfo", "rb") { |f| qinfo = Marshal.load(f.read)}
+        @queues = qinfo[:queues]
+        @frames = qinfo[:frames]
+      else
+        @queues = Hash.new
+        @frames = Hash.new
+      end
+
+      @queues.keys.each do |dest|
+        @@log.debug "Q  #{self} dest=#{dest} size=#{@queues[dest][:size]} enqueued=#{@queues[dest][:enqueued]} dequeued=#{@queues[dest][:dequeued]}"
+      end
+
+      @@log.debug("Q #{self} initialized in #{@directory}")
+
+      #
+      # Cleanup dead queues and save the state of the queues every so often.  
+      # Alternatively we could save the queue state every X number
+      # of frames that are put in the queue.  
+      # Should probably also read it after saving it to confirm integrity.
+      #
+      # Removed: this badly corrupts the queue when stopping with messages
+      #
+      # EventMachine::add_periodic_timer 1800, proc {@queues.keys.each 
+      # {|dest| close_queue(dest)};save_queue_state }
+      #
+    end
+
+    # stop
+    def stop(session_id)
+      @@log.debug "#{session_id} Shutting down Queues, queue count: #{@queues.size}"
+      #
+      @queues.keys.each do |dest|
+        @@log.debug "#{session_id}: Queue #{dest}: size=#{@queues[dest][:size]} enqueued=#{@queues[dest][:enqueued]} dequeued=#{@queues[dest][:dequeued]}"
+        close_queue(dest, session_id)
+      end
+      save_queue_state(session_id)
+    end
+
+    # save_queue_state
+    def save_queue_state(session_id)
+      @@log.debug "#{session_id} save_queue_state"
+      now=Time.now
+      @next_save ||=now
+      if now >= @next_save
+        @@log.debug "#{session_id} saving state"
+        qinfo = {:queues => @queues, :frames => @frames}
+        # write then rename to make sure this is atomic
+        File.open("#{@directory}/qinfo.new", "wb") { |f| f.write Marshal.dump(qinfo)}
+        File.rename("#{@directory}/qinfo.new","#{@directory}/qinfo")
+        @next_save=now+checkpoint_interval
+      end
+    end
+
+    # monitor
+    def monitor
+      @@log.debug "#{self} monitor"
+      stats = Hash.new
+      @queues.keys.each do |dest|
+        stats[dest] = { 
+          'size'        => @queues[dest][:size], 
+          'enqueued'    => @queues[dest][:enqueued], 
+          'dequeued'    => @queues[dest][:dequeued],
+          'exceptions'  => @queues[dest][:exceptions],
+        }
+      end
+      stats
+    end
+
+    # close_queue
+    def close_queue(dest, session_id)
+      @@log.debug "#{session_id} close_queue"
+      if @queues[dest][:size] == 0 and @queues[dest][:frames].size == 0 and @delete_empty
+        _close_queue(dest)
+        @queues.delete(dest)
+        @frames.delete(dest)
+        @@log.debug "#{session_id} Queue #{dest} removed."
+      end
+    end
+
+    # open_queue
+    def open_queue(dest, session_id)
+      @@log.debug "#{session_id} open_queue"
+      # New queue
+      @queues[dest] = Hash.new
+      # New frames for this queue
+      @frames[dest] = Hash.new
+      # Update queues
+      # :size, :frames, :msgid, :enqueued, :dequeued, :exceptions
+      @queues[dest][:size] = 0
+      @queues[dest][:frames] = Array.new
+      @queues[dest][:msgid] = 1
+      @queues[dest][:enqueued] = 0
+      @queues[dest][:dequeued] = 0
+      @queues[dest][:exceptions] = 0
+      _open_queue(dest)
+      @@log.debug "Created queue #{dest}"
+    end
+
+    # requeue
+    def requeue(dest,frame)
+      @@log.debug "#{frame.headers['session']} requeue, for #{dest}, frame: #{frame.inspect}"
+      open_queue(dest, frame.headers['session']) unless @queues.has_key?(dest)
+      msgid = frame.headers['message-id']
+      #
+      # Note: frame.headers['max-exceptions'] is currently _never_ set any where!
+      #
+      if frame.headers['max-exceptions'] and @frames[dest][msgid][:exceptions] >= frame.headers['max-exceptions'].to_i
+        enqueue("/queue/deadletter",frame)
+        return
+      end
+      #
+      writeframe(dest,frame,msgid)
+
+      # update queues (queues[dest])
+      # :size, :frames, :msgid, :enqueued, :dequeued, :exceptions
+      @queues[dest][:size] += 1
+      @queues[dest][:frames].unshift(msgid)
+      # no :msgid here
+      # no :enqueued here
+      # no :dequeued here
+      @queues[dest][:exceptions] += 1
+
+      # update frames
+      #
+      # Is this _always_ the case in this method ?????
+      unless @frames[dest][msgid]
+        new_frames_entry(dest, frame, msgid)
+      end
+      #
+      @frames[dest][msgid][:exceptions] += 1
+      @frames[dest][msgid][:requeued] += 1
+      save_queue_state(frame.headers['session'])
+      return true
+    end
+
+    # enqueue
+    def enqueue(dest,frame)
+      @@log.debug "#{frame.headers['session']} enqueue"
+      open_queue(dest, frame.headers['session']) unless @queues.has_key?(dest)
+      msgid = assign_id(frame, dest)
+      @@log.debug("#{frame.headers['session']} Enqueue for message: #{msgid} Client: #{frame.headers['client-id'] if frame.headers['client-id']}")
+      writeframe(dest,frame,msgid)
+
+      # update queues (queues[dest])
+      # :size, :frames, :msgid, :enqueued, :dequeued, :exceptions
+      @queues[dest][:size] += 1
+      @queues[dest][:frames].push(msgid)
+      @queues[dest][:msgid] += 1
+      @queues[dest][:enqueued] += 1
+      # no :dequeue here
+      # no :exceptions here
+
+      # Update frames
+      # Initialize frames entry for this: dest, frame, and msgid
+      new_frames_entry(dest, frame, msgid)
+      save_queue_state(frame.headers['session'])
+      return true
+    end
+
+    # dequeue
+    def dequeue(dest, session_id)
+      @@log.debug "#{session_id} dequeue, dest: #{dest}"
+      return false unless message_for?(dest, session_id)
+      # update queues ... dest .... :frames here
+      msgid = @queues[dest][:frames].shift
+      frame = readframe(dest,msgid,session_id)
+      @@log.debug("#{frame.headers['session']} Dequeue for message: #{msgid} Client: #{frame.headers['client-id'] if frame.headers['client-id']}")
+
+      # update queues (queues[dest])
+      # :size, :frames, :msgid, :enqueued, :dequeued, :exceptions
+      @queues[dest][:size] -= 1
+      # :frames - see above
+      @queues[dest][:msgid] -= 1
+      # :enqueued - no change
+      @queues[dest][:dequeued] += 1
+      # :exceptions - no change
+
+      @queues[dest].delete(msgid)
+
+      close_queue(dest, frame.headers['session'])
+      save_queue_state(frame.headers['session'])
+      return frame
+    end
+
+    # messsage_for?
+    def message_for?(dest, session_id)
+      retval = (@queues.has_key?(dest) and (!@queues[dest][:frames].empty?))
+      @@log.debug "#{session_id} message_for?, dest: #{dest}, #{retval}"
+      return retval
+    end
+
+    # writeframe
+    def writeframe(dest,frame,msgid)
+      @@log.debug "#{frame.headers['session']} writeframe, dest: #{dest}, frame: #{frame}, msgid: #{msgid}"
+      _writeframe(dest,frame,msgid)
+    end
+
+    # readframe
+    def readframe(dest,msgid, session_id)
+      @@log.debug "#{session_id} readframe, dest: #{dest}, msgid: #{msgid}"
+      _readframe(dest,msgid)
+    end
+
+    # assign_id
+    def assign_id(frame, dest)
+      @@log.debug "#{frame.headers['session']} assign_id, frame: #{frame}, dest: #{dest}"
+      msg_id = @queues[dest].nil? ? 1 : @queues[dest][:msgid] 
+      frame.headers['message-id'] = @stompid[msg_id] 
+    end
+
+    private
+    # new_frames_entry
+    def new_frames_entry(dest, frame, msgid)
+      @frames[dest][msgid] = Hash.new
+      @frames[dest][msgid][:exceptions] = 0
+      @frames[dest][msgid][:requeued] = 0
+      @frames[dest][msgid][:client_id] = frame.headers['client-id'] if frame.headers['client-id']
+      @frames[dest][msgid][:expires] = frame.headers['expires'] if frame.headers['expires']
+    end
+  #
+  end # class Queue
+#
+end # module StompServer
+

data/lib/stomp_server_ng/queue/activerecord_queue.rb

@@ -0,0 +1,118 @@
+## Queue implementation using ActiveRecord
+##
+## all messages are stored in a single table
+## they are indexed by 'stomp_id' which is the stomp 'message-id' header
+## which must be unique accross all queues
+##
+require 'stomp_server_ng/queue/ar_message'
+require 'yaml'
+
+module StompServer
+class ActiveRecordQueue
+  attr_accessor :checkpoint_interval
+
+  def initialize(configdir, storagedir, db_ymlfile)
+    # Default configuration, use SQLite for simplicity
+    db_params = {
+      'adapter' => 'sqlite3',
+      'database' => "#{configdir}/stompserver_development"
+    }
+    @@log = Logger::new(STDOUT)
+    @@log.level = StompServer::LogHelper.get_loglevel()
+    # Load DB configuration
+    @@log.debug "trying to read from #{db_ymlfile}"
+    if File.exists? db_ymlfile
+      @@log.debug("File #{db_ymlfile} exists.")
+      db_params.merge! YAML::load(File.open(db_ymlfile))
+    else
+      @@log.warn("File #{db_ymlfile} not found, using sqlite3 default.")
+    end
+    @@log.debug("using DB params: #{db_params.inspect}")
+    # Setup activerecord
+    ActiveRecord::Base.establish_connection(db_params)
+    @@log.debug("connection complete")
+
+    # AR Logger
+    ActiveRecord::Base.logger = Logger.new(STDOUT)
+    ActiveRecord::Base.logger.level = StompServer::LogHelper.get_loglevel()
+
+    # we need the connection, it can't be done earlier
+    ArMessage.reset_column_information
+    reload_queues
+    @stompid = StompServer::StompId.new
+  end
+
+  # Add a frame to the queue
+  def enqueue(queue_name, frame)
+    unless @frames[queue_name]
+      @frames[queue_name] = {
+        :last_index => 0,
+        :frames => [],
+      }
+    end
+    affect_msgid_and_store(frame, queue_name)
+    @frames[queue_name][:frames] << frame
+  end
+
+  # Get and remove a frame from the queue
+  def dequeue(queue_name, session_id)
+    return nil unless @frames[queue_name] && !@frames[queue_name][:frames].empty?
+    frame = @frames[queue_name][:frames].shift
+    remove_from_store(frame.headers['message-id'])
+    return frame
+  end
+
+  # Requeue the frame previously pending
+  def requeue(queue_name, frame)
+    @frames[queue_name][:frames] << frame
+    ArMessage.create!(:stomp_id => frame.headers['message-id'],
+                      :frame => frame)
+  end
+
+  # remove a frame from the store
+  def remove_from_store(message_id)
+    ArMessage.find_by_stomp_id(message_id).destroy
+  end
+
+  # store a frame (assigning it a message-id)
+  def affect_msgid_and_store(frame, queue_name)
+    msgid = assign_id(frame, queue_name)
+    ArMessage.create!(:stomp_id => msgid, :frame => frame)
+  end
+
+  def message_for?(queue_name, session_id)
+    @frames[queue_name] && !@frames[queue_name][:frames].empty?
+  end
+
+  def assign_id(frame, queue_name)
+
+    unless @frames[queue_name]
+      @frames[queue_name] = {
+        :last_index => 0,
+        :frames => [],
+      }
+    end
+
+    msgid = @stompid[@frames[queue_name][:last_index] += 1]
+    frame.headers['message-id'] = msgid
+  end
+
+  private
+  def reload_queues
+    @frames = Hash.new
+    ArMessage.find(:all).each { |message|
+      frame = message.frame
+      destination = frame.dest
+      msgid = message.stomp_id
+      @frames[destination] ||= Hash.new
+      @frames[destination][:frames] ||= Array.new
+      @frames[destination][:frames] << frame
+    }
+    # compute base index for each destination
+    @frames.each_pair { |destination,hash|
+      hash[:last_index] = hash[:frames].map{|f|
+        f.headers['message-id'].match(/(\d+)\Z/)[0].to_i}.max
+    }
+  end
+end
+end

data/lib/stomp_server_ng/queue/ar_message.rb

@@ -0,0 +1,21 @@
+gem 'activerecord'
+require 'active_record'
+require 'stomp_server_ng/queue/ar_reconnect'
+#
+#
+#
+class Logger
+  private
+  # Rails overrides this method so that it can customize the format
+  # of it's logs.  A consequence it that the date, time, etc. disappear
+  # from log output.  This little hack overrides the override :-).
+  def format_message(*args)
+    old_format_message(*args)
+  end
+end
+#
+#
+#
+class ArMessage < ActiveRecord::Base
+  serialize :frame
+end

data/lib/stomp_server_ng/queue/ar_reconnect.rb

@@ -0,0 +1,18 @@
+require 'active_record/connection_adapters/mysql_adapter'
+module ActiveRecord::ConnectionAdapters
+  class MysqlAdapter
+    alias_method :execute_without_retry, :execute
+    def execute(*args)
+      execute_without_retry(*args)
+    rescue ActiveRecord::StatementInvalid
+      if $!.message =~ /server has gone away/i
+        warn "Server timed out, retrying"
+        reconnect!
+        retry
+      end
+
+      raise
+    end
+  end
+end
+