tinyq 0.5.0

data/bin/tinyq

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+$: << File.dirname($0) + "/../lib"
+
+require 'rubygems'
+require 'tinyq.rb'
+
+TinyQ::Server.new.main(ARGV)

data/lib/tinyq.rb

@@ -0,0 +1,37 @@
+# Install
+# gem install eventmachine json uuid
+# 
+#
+# TODO
+#   capped - for bucket and funnel
+#       - overflow or round-robin
+#
+#   permanent - for bucket and funnel
+#   stream - for GetMessages 
+#   (basically if count = 0 then send 1 message at a time, but never remove subscriber
+
+require 'rubygems'
+require 'eventmachine'
+require 'json'
+require 'logger'
+require 'time'
+require 'uuid'
+require 'getopt/std'
+require 'singleton'
+require 'daemons'
+
+require 'tinyq/subscriber'
+require 'tinyq/funnel'
+require 'tinyq/bucket'
+require 'tinyq/server'
+require 'tinyq/connection'
+require 'tinyq/permanent'
+require 'tinyq/client'
+require 'tinyq/version'
+require 'tinyq/defs'
+require 'tinyq/journal'
+
+
+$LOG = Logger.new(STDOUT)
+$LOG.level = Logger::INFO
+

data/lib/tinyq/bucket.rb

@@ -0,0 +1,138 @@
+module TinyQ
+    #
+    # Bucket, messages get dropped into the bucket and forwarded
+    # to the different funnels connected to that bucket
+    #
+    class Bucket
+        attr_accessor :name
+
+        attr_reader :messages
+        attr_reader :message_ids
+        attr_reader :references
+        attr_reader :pendings
+
+        attr_accessor :funnels
+
+        attr_accessor :permanent
+
+        def initialize(n, p = false)
+            @name = n
+            @permanent = p
+            @messages = {}
+            @message_ids = []
+            @references = {}
+            @pendings = {}
+            @funnels = {}
+            @uuid = UUID.new
+        end
+
+        def put_message(message)
+            message_id = @uuid.generate
+            #message[:__id] = message_id
+            #message[:__sent] = Time.now.iso8601
+
+            # If permantent bucket, then store
+            Permanent.store message,"#{message_id}.dat", { :gzip => true } unless !@permanent
+
+            @messages[message_id] = message
+            @message_ids.push(message_id)
+            @references[message_id] = @funnels.keys
+            @pendings[message_id] = []
+
+            if !@funnels.empty?
+                # Put message in each funnel
+                @funnels.each do |n,funnel|
+                    $LOG.debug("Bucket #{@name} - Notifying funnel #{funnel.name}")
+                    funnel.notify(self)
+                end
+            end
+        end
+
+        def dequeue(funnel)
+            if !@messages.empty?
+                message_id = nil
+
+                @message_ids.each do |mid|
+                    $LOG.debug("Bucket #{@name} - #{mid} references: #{@references[mid]}")
+                    if @references[mid].count(funnel.name) > 0
+                        # That message was not de-referenced yet
+                        $LOG.debug("Bucket #{@name} - #{mid} -> #{funnel.name}")
+                        message_id = mid
+                        break
+                    end
+                end
+
+                if nil != message_id
+                    $LOG.debug("Bucket #{@name} - Sending #{message_id} to funnel #{funnel.name}")
+                    message = @messages[message_id]
+
+                    # Remove the given funnel from a reference
+                    @references[message_id].delete(funnel.name)
+                    # Add the given funnel to the pending list for that message
+                    @pendings[message_id].push(funnel.name)
+
+                    [message, message_id]
+                else
+                    $LOG.debug("Bucket #{@name} - No more messages for funnel #{funnel.name}")
+                    [nil, nil]
+                end
+            else
+                [nil, nil]
+            end
+        end
+
+        def message_sent(funnel, message_id)
+            $LOG.debug("Bucket #{@name} - Message #{message_id} sent on #{funnel.name}")
+            @pendings[message_id].delete(funnel.name)
+
+            # If no funnels are either pending or referenced, then message can be removed
+            if @pendings[message_id].empty? && @references[message_id].empty?
+                $LOG.debug("Bucket #{@name} - Purge message #{message_id}")
+                # No more references, message can be deleted
+                Permanent.remove "#{message_id}.dat" unless !@permanent
+                @messages.delete(message_id)
+                @message_ids.delete(message_id)
+                @references.delete(message_id)
+                @pendings.delete(message_id)
+                true
+            end
+
+            false
+        end
+
+        def funnel(name, broadcaster = false)
+            funnel = @funnels[name]
+            if nil == funnel
+                $LOG.info("Bucket #{@name} - Creating funnel #{name}")
+                funnel = Funnel.new(name, broadcaster)
+                feed_funnel(funnel)
+            end
+            # Update potential settings
+            funnel.broadcaster = broadcaster
+
+            funnel
+        end
+
+        def feed_funnel(funnel)
+            @funnels[funnel.name] = funnel
+            # Add ourself to the list of buckets for that funnel
+            if funnel.buckets.count self == 0
+                funnel.buckets.push(self)
+            end
+
+            # If we have cached messages, notify new funnel
+            if !@messages.empty?
+                # Add the current funnels as a reference
+                @message_ids.each do |mid|
+                    @references[mid].push(funnel.name)
+                end
+                $LOG.debug("Bucket #{@name} - Notifying funnel #{funnel.name}")
+                funnel.notify(self)
+            end
+
+        end
+
+    end
+end
+
+

data/lib/tinyq/client.rb

@@ -0,0 +1,59 @@
+module TinyQ
+    class ClientConnection < EventMachine::Connection
+        include EventMachine::Protocols::LineText2
+
+        attr_accessor :client
+
+        def initialize
+            set_delimiter TinyQ::DELIMITER
+        end
+
+        def receive_line message
+            begin
+                reply = JSON.parse(message)
+                @client.callbacks.each { |c| c.call(reply) }
+            rescue Exception => e
+                puts e
+            end
+        end
+
+    end
+
+    class Client 
+        attr_accessor :callbacks
+
+        def initialize
+            @callbacks = []
+        end
+
+        def connect(host, port)
+            @host, @port = host, port
+            @connection = EM.connect @host, @port, TinyQ::ClientConnection
+            @connection.client = self
+        end
+
+        # 
+        # Public API
+        #
+        def put_message(bucket, message)
+            command = {:Command => "PutMessages", :Bucket => bucket, :Message => message}
+            @connection.send_data("#{command.to_json}#{TinyQ::DELIMITER}")
+        end
+
+        def get_message(funnel)
+            command = {:Command => "GetMessages", :Funnel => funnel}
+            @connection.send_data("#{command.to_json}#{TinyQ::DELIMITER}")
+        end
+
+        def onreply(&block)
+            @callbacks << block
+        end
+
+
+        def feed_funnel(bucket, funnel)
+            command = {:Command => "FeedFunnel", :Bucket => bucket, :Funnel => funnel}
+            @connection.send_data("#{command.to_json}#{TinyQ::DELIMITER}")
+        end
+
+    end
+end

data/lib/tinyq/connection.rb

@@ -0,0 +1,160 @@
+module TinyQ
+    class Connection < EventMachine::Connection
+        attr_accessor :server
+        attr_reader :ip
+        attr_reader :port
+
+        include EventMachine::Protocols::LineText2
+
+
+        def initialize
+            set_delimiter TinyQ::DELIMITER
+        end
+
+        def post_init
+            @port, *ip_parts = get_peername[2,6].unpack "nC4"
+            @ip = ip_parts.join('.')
+            
+            $LOG.info("#{@ip}:#{@port} connected")
+        end
+
+        def receive_line line
+            process_message line
+        end
+
+        def reply rsp
+            send_data("#{rsp.to_json}#{TinyQ::DELIMITER}")
+        end
+
+        def ok
+            response = {:Status => "OK"}
+            reply(response)
+        end
+
+        def failed errmsg
+            response = {:Status => "Failed", :Message => errmsg}
+            reply(response)
+        end
+
+
+        def process_message data
+            begin
+                request = JSON.parse(data)
+                $LOG.debug("Request: #{request}")
+                if request.has_key? 'Command'
+                    $LOG.debug("Command - #{request['Command']}")
+
+                    case request['Command']
+                    when "PutMessages"
+                        # Put message in a bucket
+                        if request.has_key? 'Bucket'
+                            bucket = @server.bucket(request['Bucket'])
+
+                            if request['Message'].kind_of?(Array)
+                                request['Message'].each do |message|
+                                    Journal.instance.event({:Event=>"PutMessage"})
+                                    bucket.put_message(message)
+                                end
+                            else
+                                Journal.instance.event({:Event=>"PutMessage"})
+                                bucket.put_message(request['Message'])
+                            end
+
+                            ok()
+                        else
+                            failed("Parameter Bucket missing")
+                        end
+                    when "FeedFunnel"
+                        # Feed Bucket to Funnel
+                        if request.has_key? 'Bucket' and request.has_key? 'Funnel'
+                            # Hook up a funnel to a bucket
+                            bucket = @server.bucket(request['Bucket'])
+                            funnel = @server.funnels[request['Funnel']]
+                            if nil != funnel
+                                bucket.feed_funnel(funnel)
+                            else
+                                if request.has_key? 'Broadcaster'
+                                    funnel = bucket.funnel(request['Funnel'], request['Broadcaster'])
+                                else
+                                    funnel = bucket.funnel(request['Funnel'])
+                                end
+                                @server.funnels[request['Funnel']] = funnel
+                            end
+
+                            ok()
+                        else
+                            failed("Parameter Bucket or Funnel missing")
+                        end
+                    when "GetMessages"
+                        # Get Messages from Funnel
+                        if request.has_key? 'Funnel'
+                            funnel = @server.funnels[request['Funnel']]
+                            if nil == funnel
+                                $LOG.warn("Funnel was never fed...")
+                                failed("Funnel was never fed")
+                            else
+                                if request.has_key? 'Ack'
+                                    ok()
+                                end
+
+                                if request.has_key? 'Count'
+                                    funnel.add_connection(self, request['Count'])
+                                else
+                                    funnel.add_connection(self)
+                                end
+                            end
+                        else
+                            failed("Parameter Funnel missing")
+                        end
+                    when "GetInfo"
+                        if request.has_key? 'Bucket'
+                            bucket = @server.bucket(request['Bucket'])
+
+                            response = {
+                                :Status => 'OK',
+                                :Bucket => {
+                                :Name => bucket.name,
+                                :MessageCount => bucket.messages.size,
+                                :Funnels => [],
+                            }
+                            }
+
+                            bucket.funnels.each do |n,funnel|
+                                response[:Bucket][:Funnels].push({
+                                    :Name => funnel.name,
+                                    :MessageCount => funnel.messages.size,
+                                    :SubscriberCount => funnel.subscribers.size,
+                                })
+                            end
+
+                            reply(response)
+
+                        elsif request.has_key? 'Funnel'
+                        else
+                            failed("Parameter Funnel or Bucket missing")
+                        end
+                    else
+                        # Unsupported command
+                        failed("Unsupported command")
+                    end
+
+                else
+                    # Did not understand
+                    failed("Parameter Command missing")
+                end
+            rescue Exception => e
+                $LOG.error("Failed #{e}")
+                failed("Internal error #{e}")
+            end
+        end
+
+        def unbind
+            $LOG.info("#{@ip}:#{@port} disconnected")
+ 
+            @server.remove_connection(self)
+        end
+
+    end
+end
+
+

data/lib/tinyq/defs.rb

@@ -0,0 +1,4 @@
+module TinyQ
+    DELIMITER = "\0"
+    #DELIMITER = "\n"
+end

data/lib/tinyq/funnel.rb

@@ -0,0 +1,91 @@
+module TinyQ
+    #
+    # Funnel will receive messages on the internal queue and will dispatch
+    # to subscribers
+    #
+    class Funnel
+        attr_accessor :name
+        attr_reader :queue
+        attr_accessor :buckets
+        attr_accessor :subscribers
+        attr_accessor :broadcaster
+
+        def initialize(n, b = false)
+            @name = n
+            @broadcaster = b
+            @queue = EventMachine::Queue.new
+            @subscribers = {}
+            @buckets = []
+
+            cb = Proc.new do |event|
+                $LOG.debug("Funnel #{@name} - Callback")
+                if !@subscribers.empty?
+                    # OK we can dequeue from bucket since we have somewhere to send it
+                    bucket = event[:Bucket]
+                    message,message_id = bucket.dequeue(self)
+
+                    $LOG.debug("Funnel #{@name} - Callback got '#{message_id}'")
+
+                    if message != nil
+                        if @broadcaster
+                            $LOG.debug("Funnel #{@name} - Broadcasting #{message_id}")
+                            @subscribers.each do |c,subscriber|
+                                if subscriber.put_message(bucket, self, message, message_id)
+                                    # Subscriber is done removing
+                                    $LOG.debug("Funnel #{@name} - Subscriber received requested count")
+                                    self.remove_connection(c)
+                                end
+                            end
+                        else
+                            $LOG.debug("Funnel #{@name} - Unicasting #{message_id}")
+                            c = @subscribers.keys[0]
+                            subscriber = @subscribers[c]
+                            $LOG.debug("Funnel #{@name} - Unicasting to #{subscriber.connection.ip}:#{subscriber.connection.port}")
+                            if subscriber.put_message(bucket, self, message, message_id)
+                                # Subscriber is done removing
+                                $LOG.debug("Funnel #{@name} - Subscriber #{subscriber.connection.ip}:#{subscriber.connection.port} received requested count")
+                                self.remove_connection(c)
+                            end
+                        end
+                    else
+                        $LOG.debug("Funnel #{@name} - Callback noop")
+                    end
+                end
+
+                # Wait for next event
+                @queue.pop &cb
+            end
+
+            @queue.pop &cb
+        end
+
+        # Method called by bucket when messages are available
+        def notify(bucket)
+            @queue.push({:Event => "New Message", :Bucket => bucket})
+        end
+
+        def add_connection(c,n = 1)
+            subscriber = @subscribers[c]
+            if nil == subscriber
+                subscriber = Subscriber.new(c,n)
+                @subscribers[c] = subscriber
+            end
+
+            # At this point, we need to see if any buckets we are connected to
+            # have pending messages
+            @buckets.each do |bucket|
+                if !bucket.messages.empty?
+                    self.notify(bucket)
+                end
+            end
+        end
+
+        def remove_connection(c)
+            $LOG.debug("Funnel #{@name} - Removing subscriber #{c.ip}:#{c.port}")
+            @subscribers.delete(c)
+        end
+
+    end
+end
+
+

data/lib/tinyq/journal.rb

@@ -0,0 +1,21 @@
+module TinyQ
+    class Journal
+        include Singleton
+
+        attr_accessor :enabled
+
+        def initialize
+            @journal = File.open("journal.txt", "a")
+            @enabled = false
+        end
+
+        def event(event)
+            if @enabled
+                now=Time.now.iso8601
+                @journal.puts("#{now},#{event[:Event]}")
+                @journal.fsync
+            end
+        end
+
+    end
+end

data/lib/tinyq/permanent.rb

@@ -0,0 +1,33 @@
+require 'zlib'
+
+module TinyQ
+    class Permanent
+
+
+        def self.store obj, filename, options = {}
+            dump = Marshal.dump(obj)
+            file = File.new(filename, 'w')
+            file = Zlib::GzipWriter.new(file) unless options[:gzip] == false
+            file.write dump
+            file.close
+            return obj
+        end
+
+        def self.load filename
+            begin
+                file = Zlib::GzipReader.open(filename)
+            rescue Zlib::GzipFile::Error
+                file = File.open(filename, 'r')
+            ensure
+                obj = Marshal.load file.read
+                file.close
+                return obj
+            end
+        end
+
+        def self.remove filename
+            File.delete(file)
+        end
+
+    end
+end

data/lib/tinyq/server.rb

@@ -0,0 +1,106 @@
+module TinyQ
+    class Server
+        attr_accessor :connections
+
+        attr_accessor :buckets
+        attr_accessor :funnels
+
+        def initialize
+            @connections = []
+            @buckets = {}
+            @funnels = {}
+        end
+
+        def usage
+            puts "#{$0}"
+            puts "version: #{TinyQ::VERSION}"
+            puts " -b go into background/daemonize"
+            puts " -p <port> port number to listen to (default: 64321)"
+            puts " -i <ip> ip to bind to (default: 127.0.0.1)"
+            puts " -d turn on debug"
+            puts " -h help, this message"
+        end
+
+        def main(args)
+            opt = Getopt::Std.getopts("bp:i:dh")
+
+            if opt['h']
+                usage
+                exit
+            end
+
+            @ip = opt['i'] || '127.0.0.1'
+            @port = opt['p'] || 64321
+
+            if opt['d']
+                $LOG.level = Logger::DEBUG
+            end
+
+            if opt['b']
+                puts "Going into background"
+                Daemons.daemonize
+            end
+            
+            start
+        end
+
+        def self.start
+            new().start
+        end
+
+        def start
+            EventMachine::run {
+                trap("TERM") { stop }
+                trap("INT") { stop }
+                EventMachine.epoll
+                @signature = EventMachine.start_server(@ip, @port, Connection) do |con|
+                    con.server = self
+                    # We actually do not want to wait on clients
+                    #@connections.push(con)
+                end
+                $LOG.info("TinyQ listening on 0.0.0.0:64321")
+            }
+        end
+
+
+        def stop
+            $LOG.info("TinyQ Exiting")
+            EventMachine.stop_server(@signature)
+
+            unless wait_for_connections_and_stop
+                EventMachine.add_periodic_timer(1) { wait_for_connections_and_stop }
+            end
+        end
+
+        def wait_for_connections_and_stop
+            if @connections.empty?
+                EventMachine.stop
+                true
+            else
+                $LOG.info("Waiting for #{@connections.size} connection(s) to finish...")
+                false
+            end
+        end
+
+        def bucket(name)
+            bucket = @buckets[name]
+            if nil == bucket
+                $LOG.info("Server - Creating bucket #{name}")
+                bucket = Bucket.new(name)
+                @buckets[name] = bucket
+            end
+
+            bucket
+        end
+
+        def remove_connection(c)
+            @connections.delete(c)
+            @funnels.each do |name,funnel|
+                funnel.remove_connection(c)
+            end
+        end
+
+    end
+end
+
+

data/lib/tinyq/subscriber.rb

@@ -0,0 +1,63 @@
+module TinyQ
+    class Subscriber
+        attr_accessor :connection
+        attr_reader :queue
+        attr_reader :requested
+        attr_accessor :count
+        attr_reader :messages
+        attr_reader :message_ids
+
+        def initialize(c,n = 1)
+            @connection = c
+            @requested = @count = n
+            @queue = EventMachine::Queue.new
+            @messages = []
+            @message_ids = {}
+
+            cb = Proc.new do |event|
+                $LOG.debug("Subscriber #{@connection.ip}:#{@connection.port} - Queue callback")
+
+                response = { :Messages => @messages }
+
+                connection.reply(response)
+                
+                # Only at that point should the message be removed from the bucket!
+                @message_ids.each do |message_id,info|
+                    bucket = info[:Bucket]
+                    funnel = info[:Funnel]
+
+                    bucket.message_sent(funnel, message_id)
+                end
+
+
+                @queue.pop &cb
+            end
+
+            @queue.pop &cb
+        end
+
+        def put_message(bucket, funnel, message, message_id)
+            @messages.push(message)
+            @message_ids[message_id] = {
+                :Funnel => funnel,
+                :Bucket => bucket
+            }
+
+            if @messages.size == @requested
+                # We have everything now, so we can send all messages to subscriber
+                event = {
+                    :Event => "New Message",
+                    :Funnel => funnel,
+                    :Bucket => bucket,
+                    :MessageID => message_id
+                }
+                @queue.push(event)
+                true
+            else
+                false
+            end
+        end
+    end
+end
+
+

data/lib/tinyq/version.rb

@@ -0,0 +1,3 @@
+module TinyQ
+    VERSION = "0.5.0"
+end

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification
+name: tinyq
+version: !ruby/object:Gem::Version
+  version: 0.5.0
+  prerelease: 
+platform: ruby
+authors:
+- Jerome Poichet
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-10-15 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: &70364366145240 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70364366145240
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: &70364366144820 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70364366144820
+- !ruby/object:Gem::Dependency
+  name: uuid
+  requirement: &70364366144400 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70364366144400
+- !ruby/object:Gem::Dependency
+  name: getopt
+  requirement: &70364366172140 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70364366172140
+- !ruby/object:Gem::Dependency
+  name: daemons
+  requirement: &70364366171720 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70364366171720
+description: TinyQ is a message broker with a simple JSON based protocol
+email:
+- poitch@gmail.com
+executables:
+- tinyq
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/tinyq.rb
+- lib/tinyq/bucket.rb
+- lib/tinyq/client.rb
+- lib/tinyq/connection.rb
+- lib/tinyq/defs.rb
+- lib/tinyq/funnel.rb
+- lib/tinyq/journal.rb
+- lib/tinyq/permanent.rb
+- lib/tinyq/server.rb
+- lib/tinyq/subscriber.rb
+- lib/tinyq/version.rb
+- bin/tinyq
+homepage: https://github.com/poitch/TinyQ
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.6
+signing_key: 
+specification_version: 3
+summary: Ruby/TinyQ
+test_files: []