fmq 0.1.0

data/default-server/config.yml

@@ -0,0 +1,66 @@
+server:
+  # start the server on every nic
+  interface: 0.0.0.0
+  # this option set the port for the FMQ
+  port: 5884
+  # here you can setup the log level to see what is going wrong if needed
+  log-level: info
+  
+queue-manager:
+  # if someone pushes to a queue that don't exists
+  # the queue manager will create one for you. This
+  # is a good option if you are in development
+  # the queue will be a LoadBalancedQueue by default
+  auto-create-queues: true
+  # if you want some queues right from startup
+  # define there url and constraints here
+  defined-queues:
+    # queue names are not relevant use them to 
+    # organize your queues
+    test-queue-1:
+      # the path to the queue e.g. /app1/myframe/test1
+      # means http://localhost:5884/app1/myframe/test1
+      # this parameter is not optional
+      path: /fmq_test/test1
+      # this defines the maximum count of messages that 
+      # can be in the queue, if the queue is full every
+      # new message will be rejected with a http error
+      # this parameter is optional if you don't specify
+      # a max value the queue size depends on your system
+      max-messages: 1000000
+      # this optional to and specifys the max content size
+      # for all data of a queue
+      # valid extensions are kb, mb, gb
+      max-size: 10kb
+    test-queue-2:
+      path: /fmq_test/test2
+      # if you want you can specify the class of the queue
+      # this is interessting if you write your own queues
+      class: FreeMessageQueue::LoadBalancedQueue
+    test-queue-3:
+      path: /fmq_test/test3
+      class: MyTestQueue
+    # the admin queue is a special queue, that is used to
+    # administrate the queue_manager
+    admin-page-backend:
+      path: /admin/queue
+      class: FreeMessageQueue::AdminQueue
+      filter: /admin
+    # The following 3 queues are specail queues to, they 
+    # are the admin interface to the application and 
+    # make use of the file queue
+    admin-page-index:
+      path: /admin/index
+      class: FreeMessageQueue::FileQueue
+      file: admin-interface/index.html
+      content-type: text/html
+    admin-page-logo:
+      path: /admin/images/logo.png
+      class: FreeMessageQueue::FileQueue
+      file: admin-interface/images/logo.png
+      content-type: image/png
+    admin-page-prototype:
+      path: /admin/prototype.js
+      class: FreeMessageQueue::FileQueue
+      file: admin-interface/prototype.js
+      content-type: text/javascript

data/default-server/queues/my_test.rb

@@ -0,0 +1,20 @@
+require "ostruct"
+
+class MyTestQueue
+  attr_accessor :manager
+  attr_reader :bytes, :size
+  
+  def initialize
+    @bytes = @size = 1
+  end
+  
+  def put(data)
+    puts "NEW MESSAGE"
+  end
+  
+  def poll
+    item = OpenStruct.new
+    item.data = "Hello World"
+    item
+  end
+end

data/lib/fmq/boot.rb

@@ -0,0 +1,99 @@
+#
+# Copyright (c) 2008 Vincent Landgraf
+#
+# This file is part of the Free Message Queue.
+# 
+# Foobar is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# Foobar is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with Foobar.  If not, see <http://www.gnu.org/licenses/>.
+#
+require 'yaml' 
+require 'logger'
+require 'fileutils'
+
+require File.dirname(__FILE__) + '/version'
+require File.dirname(__FILE__) + '/queue_manager'
+require File.dirname(__FILE__) + '/mongrel_server'
+
+module FreeMessageQueue
+  SERVER_HEADER = "FMQ/#{FreeMessageQueue::VERSION::STRING} (#{RUBY_PLATFORM}) Ruby/#{RUBY_VERSION}"
+  
+  def self.logger
+    $FMQ_GLOBAL_LOGGER
+  end
+  
+  def self.create_logger(log_to = STDOUT)
+    $FMQ_GLOBAL_LOGGER ||= Logger.new(log_to)
+  end
+  
+  def self.set_log_level(level)
+    case level
+      when /fatal/i
+        FreeMessageQueue.logger.level = Logger::FATAL
+      when /error/i
+        FreeMessageQueue.logger.level = Logger::ERROR
+      when /warn/i
+        FreeMessageQueue.logger.level = Logger::WARN
+      when /info/i
+        FreeMessageQueue.logger.level = Logger::INFO
+      when /debug/i
+        FreeMessageQueue.logger.level = Logger::DEBUG
+    end
+    FreeMessageQueue.logger.debug "[Logger] set log level to #{level}"
+  end
+  
+  class Configuration
+    def initialize(file_path)
+      # open and read file
+      f = open(file_path, "r")
+      data = f.read
+      f.close
+      @config = YAML.load( data )
+      
+      # set log level
+      FreeMessageQueue.set_log_level(server["log-level"])
+      
+      # debug the configuration
+      FreeMessageQueue.logger.debug("[Configuration] Server: " + YAML.dump(server))
+      FreeMessageQueue.logger.debug("[Configuration] QueueManager: " + YAML.dump(queue_manager))
+    end
+    
+    def server
+      @config["server"]
+    end
+    
+    def queue_manager
+      @config["queue-manager"]
+    end
+  end
+  
+  class Boot
+    def self.start_server
+      # create logger and setup log level
+      logger = FreeMessageQueue.create_logger
+      conf = FreeMessageQueue::Configuration.new("config.yml")
+      
+      # create queue manager
+      queue_manager = FreeMessageQueue::QueueManager.new(conf.queue_manager)
+      
+      # setup and run server
+      logger.debug "[MongrelHandler] startup at #{conf.server["interface"]}:#{conf.server["port"]}"
+      servsock = Mongrel::HttpServer.new(conf.server["interface"], conf.server["port"])
+      servsock.register("/", FreeMessageQueue::MongrelHandler.new(queue_manager))
+      servsock.run.join
+    end
+    
+    def self.create_project(project_name)
+      FileUtils.cp_r(File.dirname(__FILE__) + '/../../default-server', project_name)
+    end
+  end
+end

data/lib/fmq/client.rb

@@ -0,0 +1,43 @@
+#
+# Copyright (c) 2008 Vincent Landgraf
+#
+# This file is part of the Free Message Queue.
+# 
+# Foobar is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# Foobar is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with Foobar.  If not, see <http://www.gnu.org/licenses/>.
+#
+require 'net/http'
+
+module FreeMessageQueue
+  class ClientQueue
+    def initialize(url)
+      @url = url
+    end
+    
+    def get()
+      url = URI.parse(@url)
+      req = Net::HTTP::Get.new(url.path)
+      res = Net::HTTP.start(url.host, url.port) do |http|
+        http.request(req)
+      end
+      res.body
+    end
+    
+    def put(data)
+      url = URI.parse(@url)
+      res = Net::HTTP.start(url.host, url.port) do |http|
+        http.post(url.path, data)
+      end
+    end
+  end
+end

data/lib/fmq/mongrel_server.rb

@@ -0,0 +1,142 @@
+#!/usr/bin/env ruby -wKU
+#
+# Copyright (c) 2008 Vincent Landgraf
+#
+# This file is part of the Free Message Queue.
+# 
+# Foobar is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# Foobar is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with Foobar.  If not, see <http://www.gnu.org/licenses/>.
+#
+begin
+  require "mongrel"
+rescue LoadError
+  require "rubygems"
+  require "mongrel"
+end
+
+module FreeMessageQueue
+  class MongrelHandler < Mongrel::HttpHandler
+    def initialize(queue_manager)
+      @queue_manager = queue_manager
+      @log = FreeMessageQueue.logger
+    end
+  
+    def process(request, response)
+      queue_path = request.params["REQUEST_PATH"]
+      method = request.params["REQUEST_METHOD"]
+      @log.debug("[MongrelHandler] Incomming request for #{queue_path} [#{method}] (#{request.params["REMOTE_ADDR"]})")
+      @log.debug("[MongrelHandler] Request data: #{YAML.dump(request.params)})")
+      
+      begin
+        # process supported features
+        if method.match(/^(GET|POST|HEAD|DELETE)$/) then
+          self.send("process_" + method.downcase, request, response, queue_path)
+        else
+          raise QueueManagerException.new("[MongrelHandler] Method is not supported '#{method}'", caller)
+        end
+      rescue QueueManagerException => ex
+        client_exception(request, response, queue_path, ex)
+      rescue => ex
+        server_exception(request, response, queue_path, ex)
+      end
+    end
+  
+  private
+  
+    # returns an item from queue and sends it to the client
+    # if there is no item to fetch send an 204 (NoContent) and same as HEAD
+    def process_get(request, response, queue_path)
+      queue_item = @queue_manager.poll(queue_path)
+      
+      if queue_item then
+        response.start(200) do |head,out|
+          @log.debug("[MongrelHandler] Response to GET (200)")
+          head["Content-Type"] = (queue_item.respond_to?(:content_type)) ? queue_item.content_type : "text/plain"
+          head["Server"] = SERVER_HEADER
+          head["Queue-Size"] = @queue_manager.queue_size(queue_path)
+          if !queue_item.data.nil? && queue_item.data.size > 0
+            @log.debug("[MongrelHandler] Response data: #{queue_item.data}")
+            out.write(queue_item.data)
+          end
+        end
+      else
+        response.start(204) do |head,out|
+          @log.debug("[MongrelHandler] Response to GET (204)")
+          head["Server"] = SERVER_HEADER
+          head["Queue-Size"] = 0
+        end
+      end
+    end
+    
+    # put new item to the queue and and return sam e as head action
+    def process_post(request, response, queue_path)
+      @log.debug("[MongrelHandler] Response to POST (200)")
+      data = request.body.read
+      @log.debug("[MongrelHandler] DATA: #{data}")
+      @queue_manager.put(queue_path, data)
+      
+      response.start(200) do |head,out|
+        head["Server"] = SERVER_HEADER
+        head["Queue-Size"] = @queue_manager.queue_size(queue_path)
+      end
+    end
+    
+    # just return server header and queue size
+    def process_head(request, response, queue_path)
+      @log.debug("[MongrelHandler] Response to HEAD (200)")
+      
+      response.start(200) do |head,out|
+        head["Server"] = SERVER_HEADER
+        head["Queue-Size"] = @queue_manager.queue_size(queue_path)
+      end
+    end
+    
+    # delete the queue and return server header
+    def process_delete(request, response, queue_path)
+      @log.debug("[MongrelHandler] Response to DELETE (200)")
+      @queue_manager.delete_queue(queue_path)
+
+      response.start(200) do |head,out|
+        head["Server"] = SERVER_HEADER
+      end
+    end
+    
+    # inform the client that he did something wrong
+    def client_exception(request, response, queue_path, ex)
+      @log.warn("[MongrelHandler] Client error: #{ex}")
+      response.start(400) do |head,out|
+        head["Server"] = SERVER_HEADER
+        head["Error"] = ex.message
+      end
+    end
+    
+    # report server error
+    def server_exception(request, response, queue_path, ex)
+      @log.fatal("[MongrelHandler] System error: #{ex}")
+      for line in ex.backtrace
+        @log.error line
+      end
+      
+      response.start(500) do |head,out|
+        head["Content-Type"] = "text/plain"
+        head["Server"] = SERVER_HEADER
+        head["Error"] = ex.message
+        
+        out.write(ex.message + "\r\n\r\n")
+        for line in ex.backtrace
+          out.write(line + "\r\n")
+        end
+      end
+    end
+  end
+end

data/lib/fmq/queue_manager.rb

@@ -0,0 +1,206 @@
+#
+# Copyright (c) 2008 Vincent Landgraf
+#
+# This file is part of the Free Message Queue.
+# 
+# Foobar is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# Foobar is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with Foobar.  If not, see <http://www.gnu.org/licenses/>.
+# 
+module FreeMessageQueue
+  class QueueManagerException < RuntimeError
+    attr_accessor :message, :backtrace
+  
+    def initialize(message, callstack)
+      @message = message
+      @backtrace = callstack
+    end
+    
+    def to_s
+      @message
+    end
+  end
+
+  class QueueManager
+    INFINITE = -1
+  
+    def initialize(config)
+      @queue = {}
+      @config = config
+      @queue_constraints = {}
+      @log = FreeMessageQueue.logger
+      setup_queue_manager()
+    end
+    
+    def auto_create_queues?
+      @config["auto-create-queues"]
+    end
+   
+    def create_queue(name, max_messages = INFINITE, max_size = INFINITE, default_class = FreeMessageQueue::SyncronizedQueue)
+      # path must begin with /
+      raise QueueManagerException.new("[QueueManager] Leading / in path '#{name}' missing", caller) if name[0..0] != "/"
+      
+      # path must have a minimus lenght of 3 character
+      raise QueueManagerException.new("[QueueManager] The queue path '#{name}' is to short 3 character is minimum", caller) if name.size - 1 < 3
+      
+      # don't create a queue twice
+      raise QueueManagerException.new("[QueueManager] The queue '#{name}' allready exists", caller) if queue_exists? name
+    
+      @log.info("[QueueManager] Create queue '#{name}' {type: #{default_class}, max_messages: #{max_messages}, max_size: #{max_size}}")
+
+      @queue[name] = default_class.new
+      @queue[name].manager = self
+      @queue_constraints[name] = {
+        :max_messages => max_messages,
+        :max_size => max_size
+      }
+      
+      @queue[name]
+    end
+   
+    def delete_queue(name)
+      if @queue[name]
+        @log.info("[QueueManager] Delete queue '#{name}' with #{@queue[name].size} messages")
+        @queue[name].clear
+        @queue.delete name
+        true
+      else
+        raise QueueManagerException.new("[QueueManager] There is no queue '#{name}'", caller)
+      end
+    end
+   
+    def poll(name)
+      if @queue[name]
+        @log.debug("[QueueManager] Poll from queue '#{name}' with #{@queue[name].size} messages")
+        queue_item = @queue[name].poll
+      else
+        raise QueueManagerException.new("[QueueManager] There is no queue '#{name}'", caller)
+      end
+    end
+   
+    def put(name, data)
+      unless @queue[name]
+        # only auto create queues if it is configured
+        if auto_create_queues?
+            create_queue(name) 
+        else
+          raise QueueManagerException.new("[QueueManager] There is no queue '#{name}'", caller)
+        end
+      end
+      
+      # check max size constraints
+      if @queue_constraints[name][:max_size] != INFINITE &&
+        @queue_constraints[name][:max_size] < queue(name).bytes + data.size
+        raise QueueManagerException.new("[QueueManager] The queue '#{name}' is full, max amount of space (#{@queue_constraints[name][:max_size]}) is exceeded", caller)
+      end
+      
+      # check max messages constraints
+      if @queue_constraints[name][:max_messages] != INFINITE &&
+        @queue_constraints[name][:max_messages] < queue(name).size + 1
+        raise QueueManagerException.new("[QueueManager] The queue '#{name}' is full, max amount of messages (#{@queue_constraints[name][:max_messages]}) is exceeded", caller)
+      end
+   
+      @log.debug("[QueueManager] put message to queue '#{name}' with #{@queue[name].size} messages")
+      @queue[name].put(data)
+    end
+   
+    def queues
+      @queue.keys
+    end
+   
+    def queue_size(name)
+      @queue[name].size
+    end
+    
+    def queue_constraints(name)
+      @queue_constraints[name]
+    end
+   
+    def queue(name)
+      @queue[name]
+    end
+    
+    def queue_exists?(name)
+      !queue(name).nil?
+    end
+    
+    # create a queue from a configuration hash
+    # <em>queue_name</em> this name is just for debugging and organizing the queue
+    # <em>queue_config</em> here you will pass the parameter:
+    # * path:
+    # * [optional] max-size: the maximum size e.g. 10mb
+    # * [optional] max-messages: the maximim messages that can be in the queue e.g. 9999999
+    # * [optional] class: the class that implements this queue e.g. FreeMessageQueue::SystemQueue
+    def create_queue_from_config(queue_name, queue_config)
+      @log.debug("[QueueManager] setup queue from config '#{queue_name}'")
+
+      # path need to be specified
+      raise QueueManagerException.new("[QueueManager] There is now path specified for queue '#{queue_name}'", caller) if queue_config["path"].nil?
+      path = queue_config["path"]
+      queue_config.delete "path"
+
+      # set max size parameter -- this parameter is optional
+      max_size = str_bytes(queue_config["max-size"])
+      max_size = INFINITE if max_size.nil? || max_size <= 0
+      queue_config.delete "max-size"
+
+      # set max messages parameter -- this parameter is optional
+      max_messages = queue_config["max-messages"].to_i
+      max_messages = INFINITE if max_messages.nil? || max_messages <= 0
+      queue_config.delete "max-messages"
+
+      # set class parameter -- this parameter is optional
+      default_class = queue_config["class"]
+      default_class = eval(default_class) unless default_class.nil?
+      queue_config.delete "class"
+
+      if default_class.nil?
+        queue = create_queue(path, max_messages, max_size)
+      else
+        queue = create_queue(path, max_messages, max_size, default_class)
+      end
+
+      if queue_config.size > 0
+        @log.debug("[QueueManager] Configure addional parameters for queue '#{queue_name}'; parameter: #{queue_config.inspect}")
+        for parameter in queue_config.keys
+          method_name = parameter.gsub("-", "_")
+          queue.send(method_name + "=", queue_config[parameter])
+        end
+      end
+    end
+    
+  private
+    # retuns count of bytes to a expression with kb, mb or gb
+    # e.g 10kb will return 10240
+    def str_bytes(str)
+      case str
+        when /([0-9]+)kb/i
+          bs = $1.to_i * 1024 
+        when /([0-9]+)mb/i
+          bs = $1.to_i * 1024 * 1024 
+        when /([0-9]+)gb/i
+          bs = $1.to_i * 1024 * 1024 * 1024
+        else
+          bs = INFINITE
+      end
+      bs
+    end
+    
+    # create the queues that are defined in the configuration
+    def setup_queue_manager
+      @log.info("[QueueManager] Create defined queues (#{@config["defined-queues"].size})")
+      for defined_queue in @config["defined-queues"]
+        create_queue_from_config(defined_queue[0], defined_queue[1])
+      end
+    end
+  end
+end

data/lib/fmq/queues/admin.rb

@@ -0,0 +1,88 @@
+#
+# Copyright (c) 2008 Vincent Landgraf
+#
+# This file is part of the Free Message Queue.
+# 
+# Foobar is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# Foobar is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with Foobar.  If not, see <http://www.gnu.org/licenses/>.
+# 
+require 'ostruct'
+
+module FreeMessageQueue
+  class AdminQueue
+    attr_accessor :manager
+    
+    def initialize()
+      super
+      @filter_queues = []
+    end
+  
+    def poll()
+      item = OpenStruct.new
+
+      queues_code = []
+      manager.queues.each do |queue_name|
+        # skip if it is filterd
+        next if filtered? queue_name
+        
+        # add a new entry to the array
+        queues_code << queue_to_json(queue_name)
+      end
+      
+      item.data = "[%s]" % queues_code.join(",")
+      item
+    end
+    
+    def put(data)
+      if data.match(/_method=delete&path=(.*)/)
+        # delete queue
+        manager.delete_queue($1)
+      elsif data.match(/_method=create&data=(.*)/)
+        # create queue
+        conf = eval($1.gsub(":", "=>").gsub("null", "-1"))
+        manager.create_queue_from_config("dynamic-created-queue", conf)
+      end
+    end
+    
+    def size
+      1 # there is always a message in the queue
+    end
+    
+    def filter=(str)
+      @filter_queues = str.split " "
+    end
+    
+  private
+  
+    # check if the system queue should filter the queue
+    def filtered? (name)
+      skip = false
+      for filter in @filter_queues
+        skip = true if name[0...filter.size] == filter
+      end
+      skip
+    end
+  
+    def queue_to_json(queue_name)
+      constraints = manager.queue_constraints(queue_name)
+    
+      "[\"%s\", %d, %d, %d, %d]" % [
+        queue_name,
+        manager.queue(queue_name).bytes,
+        constraints[:max_size],
+        manager.queue(queue_name).size,
+        constraints[:max_messages],
+      ]
+    end
+  end
+end