reliable-msg 1.0.1 → 1.1.0

data/lib/reliable-msg/message-store.rb

@@ -2,26 +2,24 @@
 # = message-store.rb - Queue manager storage adapters
 #
 # Author:: Assaf Arkin  assaf@labnotes.org
-# Documentation:: http://trac.labnotes.org/cgi-bin/trac.cgi/wiki/RubyReliableMessaging
+# Documentation:: http://trac.labnotes.org/cgi-bin/trac.cgi/wiki/Ruby/ReliableMessaging
 # Copyright:: Copyright (c) 2005 Assaf Arkin
 # License:: MIT and/or Creative Commons Attribution-ShareAlike
 #
 #--
-# Changes:
-# 11/11/05
-#   Fixed: Messages retrieved in order after queue manager recovers when using MySQL.
 #++
 
 require 'thread'
-require 'uuid'
-require 'reliable-msg/queue'
+require 'reliable-msg/client'
 
 module ReliableMsg
 
+
     module MessageStore
 
         ERROR_INVALID_MESSAGE_STORE = "No message store '%s' available (note: case is not important)" #:nodoc:
 
+
         # Base class for message store.
         class Base
 
@@ -29,10 +27,12 @@ module ReliableMsg
 
             @@stores = {} #:nodoc:
 
+
             def initialize logger
                 @logger = logger
             end
 
+
             # Returns the message store type name.
             #
             # :call-seq:
@@ -42,6 +42,7 @@ module ReliableMsg
                 raise RuntimeException, "Not implemented"
             end
 
+
             # Set up the message store. Create files, database tables, etc.
             #
             # :call-seq:
@@ -50,6 +51,7 @@ module ReliableMsg
             def setup
             end
 
+
             # Returns the message store configuration as a hash.
             #
             # :call-seq:
@@ -59,6 +61,7 @@ module ReliableMsg
                 raise RuntimeException, "Not implemented"
             end
 
+
             # Activates the message store. Call this method before using the
             # message store.
             #
@@ -68,10 +71,12 @@ module ReliableMsg
             def activate
                 @mutex = Mutex.new
                 @queues = {Queue::DLQ=>[]}
+                @topics = {}
                 @cache = {}
                 # TODO: add recovery logic
             end
 
+
             # Deactivates the message store. Call this method when done using
             # the message store.
             #
@@ -79,9 +84,10 @@ module ReliableMsg
             #   store.deactivate
             #
             def deactivate
-                @mutex = @queues = @cache = nil
+                @mutex = @queues = @topics = @cache = nil
             end
 
+
             def transaction &block
                 result = block.call inserts = [], deletes = [], dlqs= []
                 begin
@@ -92,26 +98,44 @@ module ReliableMsg
                     # Empty the cache and reload the queue, before raising the error.
                     @cache = {}
                     @queues = {Queue::DLQ=>[]}
+                    @topics = {}
                     load_index
                     raise error
                 end
                 result
             end
 
-            def select queue, &block
-                queue = @queues[queue]
-                return nil unless queue
-                queue.each do |headers|
-                    selected = block.call(headers)
-                    if selected
+
+            def get_message queue, &block
+                messages = @queues[queue]
+                return nil unless messages
+                messages.each do |headers|
+                    if block.call(headers)
                         id = headers[:id]
-                        message = @cache[id] || load(id, queue)
+                        message = @cache[id] || load(id, :queue, queue)
                         return {:id=>id, :headers=>headers, :message=>message}
                     end
                 end
                 return nil
             end
 
+
+            def get_headers queue
+                return @queues[queue] || []
+            end
+
+
+            def get_last topic, seen, &block
+                headers = @topics[topic]
+                return nil if headers.nil? || headers[:id] == seen
+                if block.call(headers)
+                    id = headers[:id]
+                    message = @cache[id] || load(id, :topic, topic)
+                    {:id=>id, :headers=>headers, :message=>message}
+                end
+            end
+
+
             # Returns a message store from the specified configuration (previously
             # created with configure).
             #
@@ -125,31 +149,40 @@ module ReliableMsg
                 cls.new config, logger
             end
 
+
         protected
 
             def update inserts, deletes, dlqs
                 @mutex.synchronize do
                     inserts.each do |insert|
-                        queue = @queues[insert[:queue]] ||= []
-                        headers = insert[:headers]
-                        # Add element based on priority, higher priority comes first.
-                        priority = headers[:priority]
-                        added = false
-                        queue.each_index do |idx|
-                            if queue[idx][:priority] < priority
-                                queue[idx, 0] = headers
-                                added = true
-                                break
+                        if insert[:queue]
+                            queue = @queues[insert[:queue]] ||= []
+                            headers = insert[:headers]
+                            # Add element based on priority, higher priority comes first.
+                            priority = headers[:priority]
+                            added = false
+                            queue.each_index do |idx|
+                                if queue[idx][:priority] < priority
+                                    queue[idx, 0] = headers
+                                    added = true
+                                    break
+                                end
                             end
+                            queue << headers unless added
+                            @cache[insert[:id]] = insert[:message]
+                        elsif insert[:topic]
+                            @topics[insert[:topic]] = insert[:headers]
                         end
-                        queue << headers unless added
-                        @cache[insert[:id]] = insert[:message]
                     end
                     deletes.each do |delete|
-                        queue = @queues[delete[:queue]]
-                        id = delete[:id]
-                        queue.delete_if { |headers| headers[:id] == id }
-                        @cache.delete id
+                        if delete[:queue]
+                            queue = @queues[delete[:queue]]
+                            id = delete[:id]
+                            queue.delete_if { |headers| headers[:id] == id }
+                            @cache.delete id
+                        elsif delete[:topic]
+                            @topics.delete delete[:topic]
+                        end
                     end
                     dlqs.each do |dlq|
                         queue = @queues[dlq[:queue]]
@@ -185,6 +218,11 @@ module ReliableMsg
                 "path"=>DEFAULT_PATH
             }
 
+            ERROR_PATH_NOT_DIR = "The path %s is not a directory" #:nodoc:
+
+            ERROR_FAILED_WRITE_MASTER = "Cannot write to master index file %s" #:nodoc:
+
+
             def initialize config, logger
                 super logger
                 @fsync = config['fsync']
@@ -197,13 +235,15 @@ module ReliableMsg
                 @path = File.expand_path(config['path'] || DEFAULT_PATH)
             end
 
+
             def type
                 TYPE
             end
 
+
             def setup
                 if File.exist?(@path)
-                    raise RuntimeError, "The path '#{@path}' is not a directory" unless File.directory?(@path)
+                    raise RuntimeError, format(ERROR_PATH_NOT_DIR, @path) unless File.directory?(@path)
                     false
                 else
                     Dir.mkdir @path
@@ -211,17 +251,19 @@ module ReliableMsg
                 end
             end
 
+
             def configuration
                 { "type"=>TYPE, "path"=>@path }
             end
 
+
             def activate
                 super
                 Dir.mkdir @path unless File.exist?(@path)
-                raise RuntimeError, "The path '#{@path}' is not a directory" unless File.directory?(@path)
+                raise RuntimeError, format(ERROR_PATH_NOT_DIR, @path) unless File.directory?(@path)
                 index = "#{@path}/master.idx"
                 if File.exist? index
-                    raise RuntimeError, "Cannot write to master index file '#{index}'" unless File.writable?(index)
+                    raise RuntimeError, format(ERROR_FAILED_WRITE_MASTER, index) unless File.writable?(index)
                     @file = File.open index, "r+"
                     @file.flock File::LOCK_EX
                     @file.binmode # Things break if you forget binmode on Windows.
@@ -237,7 +279,9 @@ module ReliableMsg
                 end
             end
 
+
             def deactivate
+                @file.flock File::LOCK_UN
                 @file.close
                 @file_map.each_pair do |id, map|
                     map[1].close if map[1]
@@ -249,6 +293,7 @@ module ReliableMsg
                 super
             end
 
+
         protected
 
             def update inserts, deletes, dlqs
@@ -271,7 +316,8 @@ module ReliableMsg
                     # (message and file have different IDs).
                     file.sysseek 0, IO::SEEK_SET
                     file.syswrite insert[:message]
-                    file.truncate file.pos
+                    file.flush
+                    file.truncate insert[:message].length
                     file.flush
                     @mutex.synchronize do
                         @file_map[insert[:id]] = [name, file]
@@ -301,7 +347,7 @@ module ReliableMsg
                         file_map = {}
                         @file_map.each_pair { |id, name_file| file_map[id] = name_file[0] }
                         file_free = @file_free.collect { |name_file| name_file[0] }
-                        image = Marshal::dump({:queues=>@queues, :file_map=>file_map, :file_free=>file_free})
+                        image = Marshal::dump({:queues=>@queues, :topics=>@topics, :file_map=>file_map, :file_free=>file_free})
                         length = image.length
                         # Determine if we can store the new image before the last one (must have
                         # enough space from header), or append it to the end of the last one.
@@ -323,6 +369,7 @@ module ReliableMsg
                 end
             end
 
+
             def load_index
                 @file.sysseek 0, IO::SEEK_SET
                 last_block = @file.sysread(8).hex
@@ -332,17 +379,19 @@ module ReliableMsg
                     # master index.
                     @file.sysseek last_block, IO::SEEK_SET
                     length = @file.sysread(8).hex
-                    # Load the index image and create the queues, file_free and
-                    # file_map structures.
+                    # Load the index image and create the queues, topics, file_free
+                    # and file_map structures.
                     image = Marshal::load @file.sysread(length)
                     @queues = image[:queues]
+                    @topics = image[:topics]
                     image[:file_free].each { |name| @file_free << [name, nil] }
                     image[:file_map].each_pair { |id, name| @file_map[id] = [name, nil] }
                     @last_block, @last_block_end = last_block, last_block + length + 8
                 end
             end
 
-            def load id, queue
+
+            def load id, type, queue
                 # Find the file from the message/file mapping.
                 map = @file_map[id]
                 return nil unless map # TODO: Error?
@@ -371,6 +420,7 @@ module ReliableMsg
                 require 'active_record/vendor/mysql'
             end
 
+
             class MySQL < Base #:nodoc:
 
                 TYPE = self.name.split('::').last.downcase
@@ -383,29 +433,31 @@ module ReliableMsg
                 # Reference to an open MySQL connection held in the current thread.
                 THREAD_CURRENT_MYSQL = :reliable_msg_mysql #:nodoc:
 
+
                 def initialize config, logger
                     super logger
                     @config = { :host=>config['host'], :username=>config['username'], :password=>config['password'],
                         :database=>config['database'], :port=>config['port'], :socket=>config['socket'] }
                     @prefix = config['prefix'] || DEFAULT_PREFIX
                     @queues_table = "#{@prefix}queues"
+                    @topics_table = "#{@prefix}topics"
                 end
 
+
                 def type
                     TYPE
                 end
 
+
                 def setup
                     mysql = connection
-                    exists = false
+                    requires = 2 # Number of tables used by reliable-msg.
                     mysql.query "SHOW TABLES" do |result|
                         while row = result.fetch_row
-                            exists = true if row[0] == @queues_table
+                            requires -= 1 if row[0] == @queues_table || row[0] == @topics_table
                         end
                     end
-                    if exists
-                        false
-                    else
+                    if requires > 0
                         sql = File.open File.join(File.dirname(__FILE__), "mysql.sql"), "r" do |input|
                             input.readlines.join
                         end
@@ -415,6 +467,7 @@ module ReliableMsg
                     end
                 end
 
+
                 def configuration
                     config = { "type"=>TYPE, "host"=>@config[:host], "username"=>@config[:username],
                         "password"=>@config[:password], "database"=>@config[:database] }
@@ -424,11 +477,13 @@ module ReliableMsg
                     config
                 end
 
+
                 def activate
                     super
                     load_index
                 end
 
+
                 def deactivate
                     Thread.list.each do |thread|
                         if conn = thread[THREAD_CURRENT_MYSQL]
@@ -439,6 +494,7 @@ module ReliableMsg
                     super
                 end
 
+
             protected
 
                 def update inserts, deletes, dlqs
@@ -446,9 +502,15 @@ module ReliableMsg
                     mysql.query "BEGIN"
                     begin
                         inserts.each do |insert|
-                            mysql.query "INSERT INTO `#{@queues_table}` (id,queue,headers,object) VALUES('#{connection.quote  insert[:id]}','#{connection.quote insert[:queue]}',BINARY '#{connection.quote Marshal::dump(insert[:headers])}',BINARY '#{connection.quote insert[:message]}')"
+                            if insert[:queue]
+                                mysql.query "INSERT INTO `#{@queues_table}` (id,queue,headers,object) VALUES('#{connection.quote  insert[:id]}','#{connection.quote insert[:queue]}',BINARY '#{connection.quote Marshal::dump(insert[:headers])}',BINARY '#{connection.quote insert[:message]}')"
+                            else
+                                mysql.query "REPLACE `#{@topics_table}` (topic,headers,object) VALUES('#{connection.quote insert[:topic]}',BINARY '#{connection.quote Marshal::dump(insert[:headers])}',BINARY '#{connection.quote insert[:message]}')"
+                            end
+                        end
+                        ids = deletes.inject([]) do |array, delete|
+                            delete[:queue] ? array << "'#{delete[:id]}'" : array
                         end
-                        ids = deletes.collect {|delete| "'#{delete[:id]}'" }
                         if !ids.empty?
                             mysql.query "DELETE FROM `#{@queues_table}` WHERE id IN (#{ids.join ','})"
                         end
@@ -463,6 +525,7 @@ module ReliableMsg
                     super
                 end
 
+
                 def load_index
                     connection.query "SELECT id,queue,headers FROM `#{@queues_table}`" do |result|
                         while row = result.fetch_row
@@ -481,18 +544,33 @@ module ReliableMsg
                             queue << headers unless added
                         end
                     end
+                    connection.query "SELECT topic,headers FROM `#{@topics_table}`" do |result|
+                        while row = result.fetch_row
+                            @topics[row[0]] = Marshal::load row[1]
+                        end
+                    end
                 end
 
-                def load id, queue
+
+                def load id, type, queue_or_topic
                     message = nil
-                    connection.query "SELECT object FROM `#{@queues_table}` WHERE id='#{id}'" do |result|
-                        message = if row = result.fetch_row
-                            row[0]
+                    if type == :queue
+                        connection.query "SELECT object FROM `#{@queues_table}` WHERE id='#{id}'" do |result|
+                            message = if row = result.fetch_row
+                                row[0]
+                            end
+                        end
+                    else
+                        connection.query "SELECT object FROM `#{@topics_table}` WHERE topic='#{queue_or_topic}'" do |result|
+                            message = if row = result.fetch_row
+                                row[0]
+                            end
                         end
                     end
                     message
                 end
 
+
                 def connection
                     Thread.current[THREAD_CURRENT_MYSQL] ||= Mysql.new @config[:host], @config[:username], @config[:password],
                         @config[:database], @config[:port], @config[:socket]
@@ -501,6 +579,7 @@ module ReliableMsg
             end
 
         rescue LoadError
+            # do nothing
         end
 
     end