mqrpc 0.0.1 → 0.0.3

data/lib/mqrpc/agent.rb

@@ -3,6 +3,7 @@ require 'amqp'
 require 'mq'
 require 'mqrpc/logger'
 require 'mqrpc/operation'
+require 'mqrpc/sizedhash'
 require 'thread'
 require 'uuid'
 
@@ -27,6 +28,7 @@ module MQRPC
   # TODO: document this class
   class Agent
     MAXBUF = 20
+    MAXMESSAGEWAIT = MAXBUF * 20
 
     def initialize(config)
       Thread::abort_on_exception = true
@@ -35,11 +37,13 @@ module MQRPC
       @id = UUID::generate
       @outbuffer = Hash.new { |h,k| h[k] = [] }
       @queues = []
-      @receive_queue = Queue.new
       @topics = []
-      @want_queues = []
-      @want_topics = []
-      #@slidingwindow = LogStash::SlidingWindowSet.new
+      @receive_queue = Queue.new
+      @want_subscriptions = Queue.new
+      @slidingwindow = Hash.new do |h,k| 
+        MQRPC::logger.info "New sliding window for #{k}"
+        h[k] = SizedThreadSafeHash.new(MAXMESSAGEWAIT) 
+      end
 
       @mq = nil
       @message_operations = Hash.new
@@ -49,9 +53,13 @@ module MQRPC
       @amqp_ready = false
 
       start_amqp
+
+      # Wait for our AMQP thread to get going. Mainly, it needs to set
+      # @mq, so we'll block until it's available.
       @startup_mutex.synchronize do
         MQRPC::logger.debug "Waiting for @mq ..."
         @startup_condvar.wait(@startup_mutex) if !@amqp_ready
+        MQRPC::logger.debug "Got it, continuing with #{self.class} init..."
       end
 
       start_receiver
@@ -78,10 +86,11 @@ module MQRPC
           MQRPC::logger.info "Subscribing to main queue #{@id}"
           mq_q = @mq.queue(@id, :auto_delete => true)
           mq_q.subscribe(:ack =>true) { |hdr, msg| @receive_queue << [hdr, msg] }
-          handle_new_subscriptions
+          #handle_new_subscriptions
           
           # TODO(sissel): make this a deferred thread that reads from a Queue
-          EM.add_periodic_timer(5) { handle_new_subscriptions }
+          #EM.add_periodic_timer(5) { handle_new_subscriptions }
+          EM.defer { handle_subscriptions }
 
           EM.add_periodic_timer(1) do
             # TODO(sissel): add locking
@@ -102,11 +111,11 @@ module MQRPC
     end # def start_receiver
 
     def subscribe(name)
-      @want_queues << name 
+      @want_subscriptions << [:queue, name]
     end # def subscribe
 
     def subscribe_topic(name)
-      @want_topics << name 
+      @want_subscriptions << [:topic, name]
     end # def subscribe_topic
 
     def handle_message(hdr, msg_body)
@@ -115,12 +124,20 @@ module MQRPC
         obj = [obj]
       end
 
+      queue = hdr.routing_key
       obj.each do |item|
         message = Message.new_from_data(item)
-        #if @slidingwindow.include?(message.id)
-          #puts "Removing ack for #{message.id}"
-          #@slidingwindow.delete(message.id)
-        #end
+        slidingwindow = @slidingwindow[queue]
+        if message.respond_to?(:from_queue)
+          slidingwindow = @slidingwindow[message.from_queue]
+        end
+        MQRPC::logger.debug "#{Thread.current} Got message #{message.class}##{message.id} on queue #{queue}"
+        MQRPC::logger.debug "Received message: #{message.inspect}"
+        if (message.respond_to?(:in_reply_to) and 
+            slidingwindow.include?(message.in_reply_to))
+          MQRPC::logger.debug "Got response to #{message.in_reply_to}"
+          slidingwindow.delete(message.in_reply_to)
+        end
         name = message.class.name.split(":")[-1]
         func = "#{name}Handler"
 
@@ -129,11 +146,12 @@ module MQRPC
         if (message.respond_to?(:in_reply_to) and
             @message_operations.has_key?(message.in_reply_to))
           operation = @message_operations[message.in_reply_to]
-          operation.call message
+          operation.call(message)
         elsif @handler.respond_to?(func) 
           @handler.send(func, message) do |response|
-            reply = message.reply_to
-            sendmsg(reply, response)
+            reply_destination = message.reply_to
+            response.from_queue = queue
+            sendmsg(reply_destination, response)
           end
 
           # We should allow the message handler to defer acking if they want
@@ -150,6 +168,28 @@ module MQRPC
       @amqpthread.join
     end # run
 
+    def handle_subscriptions
+      while true do
+        type, name = @want_subscriptions.pop
+        case type
+        when :queue
+          next if @queues.include?(name)
+          MQRPC::logger.info "Subscribing to queue #{name}"
+          mq_q = @mq.queue(name, :durable => true)
+          mq_q.subscribe(:ack => true) { |hdr, msg| @receive_queue << [hdr, msg] }
+          @queues << name
+        when :topic
+          MQRPC::logger.info "Subscribing to topic #{name}"
+          exchange = @mq.topic(@config.mqexchange)
+          mq_q = @mq.queue("#{@id}-#{name}",
+                           :exclusive => true,
+                           :auto_delete => true).bind(exchange, :key => name)
+          mq_q.subscribe { |hdr, msg| @receive_queue << [hdr, msg] }
+          @topics << name
+        end
+      end
+    end
+
     def handle_new_subscriptions
       todo = @want_queues - @queues
       todo.each do |queue|
@@ -162,7 +202,7 @@ module MQRPC
       todo = @want_topics - @topics
       todo.each do |topic|
         MQRPC::logger.info "Subscribing to topic #{topic}"
-        exchange = @mq.topic("amq.topic")
+        exchange = @mq.topic(@config.mqexchange)
         mq_q = @mq.queue("#{@id}-#{topic}",
                          :exclusive => true,
                          :auto_delete => true).bind(exchange, :key => topic)
@@ -172,8 +212,6 @@ module MQRPC
     end # handle_new_subscriptions
 
     def flushout(destination)
-      return unless @mq    # wait until we are connected
-
       msgs = @outbuffer[destination]
       return if msgs.length == 0
       data = msgs.to_json
@@ -182,28 +220,26 @@ module MQRPC
     end
 
     def sendmsg_topic(key, msg)
-      return unless @mq    # wait until we are connected
       if (msg.is_a?(RequestMessage) and msg.id == nil)
         msg.generate_id!
       end
       msg.timestamp = Time.now.to_f
 
       data = msg.to_json
-      @mq.topic("amq.topic").publish(data, :key => key)
+      @mq.topic(@config.mqexchange).publish(data, :key => key)
     end
 
     def sendmsg(destination, msg, &callback)
-      return unless @mq    # wait until we are connected
       if (msg.is_a?(RequestMessage) and msg.id == nil)
         msg.generate_id!
       end
       msg.timestamp = Time.now.to_f
       msg.reply_to = @id
 
-      #if (msg.is_a?(RequestMessage) and !msg.is_a?(ResponseMessage))
-        #MQRPC::logger.info "Tracking #{msg.class.name}##{msg.id}"
-        #@slidingwindow << msg.id
-      #end
+      if msg.is_a?(RequestMessage)
+        MQRPC::logger.debug "Tracking #{msg.class.name}##{msg.id} to #{destination}"
+        @slidingwindow[destination][msg.id] = true
+      end
 
       if msg.buffer?
         @outbuffer[destination] << msg
@@ -211,6 +247,7 @@ module MQRPC
           flushout(destination)
         end
       else
+        MQRPC::logger.debug "#{Thread.current} Sending to #{destination}: #{msg.inspect}"
         @mq.queue(destination, :durable => true).publish([msg].to_json, :persistent => true)
       end
 

data/lib/mqrpc/config.rb

@@ -5,6 +5,7 @@ module MQRPC
     attr_reader :mquser
     attr_reader :mqpass
     attr_reader :mqvhost
+    attr_reader :mqexchange
 
     def initialize(options = {})
       @mqhost = options["mqhost"] || "localhost"
@@ -12,6 +13,7 @@ module MQRPC
       @mquser = options["mquser"] || "guest"
       @mqpass = options["mqpass"] || "guest"
       @mqvhost = options["mqvhost"] || "/"
+      @mqexchange = options["mqexchange"] || "mqrpc.topic"
     end # def initialize
   end # class Config
 end # module MQRPC

data/lib/mqrpc/message.rb

@@ -26,6 +26,7 @@ module BindToHash
     )
   end
 
+  private
   def __genhashpath(key)
     # TODO(sissel): enforce 'key' needs to be a string or symbol?
     path = key.split("/").select { |x| x.length > 0 }\
@@ -48,14 +49,43 @@ module MQRPC
     header :message_class
     header :reply_to
     header :timestamp
+    header :args
+    def self.inherited(subclass)
+      MQRPC::logger.debug "Message '#{subclass.name}' subclasses #{self.name}"
+      @@knowntypes[subclass.name] = subclass
+
+      # Call the class initializer if it has one.
+      if subclass.respond_to?(:class_initialize)
+        subclass.class_initialize
+      end
+    end # def self.inherited
+
+    def self.new_from_data(data)
+      obj = nil
+      name = data["message_class"]
+      if @@knowntypes.has_key?(name)
+        obj = @@knowntypes[name].new
+      else
+        $stderr.puts "No known message class: #{name}, #{data.inspect}"
+        obj = Message.new
+      end
+      obj.data = data
+      return obj
+    end
 
     def initialize
+      @data = Hash.new
+      want_buffer(false)
+
       generate_id!
+      self.message_class = self.class.name
+      self.args = Hash.new
     end
 
     def generate_id!
       @@idlock.synchronize do
         self.id = @@idseq
+        #puts "Generating id. #{self.class}.id == #{self.id}"
         @@idseq += 1
       end
     end
@@ -72,35 +102,6 @@ module MQRPC
       @buffer = want_buffer
     end
 
-    def self.inherited(subclass)
-      MQRPC::logger.debug "Message '#{subclass.name}' subclasses #{self.name}"
-      @@knowntypes[subclass.name] = subclass
-
-      # Call the class initializer if it has one.
-      if subclass.respond_to?(:class_initialize)
-        subclass.class_initialize
-      end
-    end # def self.inherited
-
-    def initialize
-      @data = Hash.new
-      want_buffer(false)
-      self.message_class = self.class.name
-    end
-
-    def self.new_from_data(data)
-      obj = nil
-      name = data["message_class"]
-      if @@knowntypes.has_key?(name)
-        obj = @@knowntypes[name].new
-      else
-        $stderr.puts "No known message class: #{name}, #{data.inspect}"
-        obj = Message.new
-      end
-      obj.data = data
-      return obj
-    end
-
     def to_json(*args)
       return @data.to_json(*args)
     end
@@ -110,18 +111,12 @@ module MQRPC
   end # class Message
 
   class RequestMessage < Message
-    header :args
-
-    def initialize
-      super
-      self.args = Hash.new
-    end
-
+    # Nothing.
   end # class RequestMessage
 
   class ResponseMessage < Message
     header :in_reply_to
-    header :args
+    header :from_queue
 
     def initialize(source_request=nil)
       super()

data/lib/mqrpc/messages/ping.rb

@@ -1,5 +1,5 @@
 require 'rubygems'
-require 'lib/mqrpc'
+require 'mqrpc'
 
 module MQRPC::Messages
   class PingRequest < MQRPC::RequestMessage

data/lib/mqrpc/operation.rb

@@ -16,17 +16,16 @@ module MQRPC
     end # def initialize
 
     def call(*args)
-      # TODO(sissel): allow the callback to simply invoke 'finished' on this
-      # operation rather than requiring it to emit ':finished'
+      # TODO(sissel): Come up with a better way for the callback to declare
+      # that it is not done than simply returning ':continue'
       @mutex.synchronize do
         ret = @callback.call(*args)
-        if ret == :finished
-          MQRPC::logger.debug "operation #{self} finished"
+        if ret != :continue
+          #MQRPC::logger.debug "operation #{self} finished"
           @finished = true
           @cv.signal
-        else
-          return ret
         end
+        return ret
       end
     end # def call
 

data/lib/mqrpc/sizedhash.rb

@@ -0,0 +1,97 @@
+require 'thread'
+require 'mqrpc'
+
+# Thread-safe sized hash similar to SizedQueue.
+# The only time we will block execution is in setting new items.
+# That is, SizedThreadSafeHash#[]=
+class SizedThreadSafeHash
+  def initialize(size)
+    @lock = Mutex.new
+    @size = size
+    @condvar = ConditionVariable.new
+    @data = Hash.new
+  end # def initialize
+
+  # set a key and value
+  def []=(key, value)
+    @lock.synchronize do
+      # If adding a new item, wait if the hash is full
+      if !@data.has_key?(key) and _withlock_full?
+        MQRPC::logger.info "#{self}: Waiting to add key #{key.inspect}, hash is full"
+        #MQRPC::logger.info "#{self}: Keys: #{@data.keys.inspect}"
+        #MQRPC::logger.info "#{self} current thread is #{Thread.current}"
+        #pp @data
+        @condvar.wait(@lock)
+      end
+      @data[key] = value
+    end
+  end # def []=
+
+  # get an value by key
+  def [](key)
+    @lock.synchronize do
+      return @data[key]
+    end
+  end # def []
+
+  # boolean, does the hash have a given key?
+  def has_key?(key)
+    @lock.synchronize do
+      return @data.has_key?(key)
+    end
+  end # def has_key?
+
+  alias :include? :has_key?
+
+  # delete a key
+  def delete(key)
+    @lock.synchronize do
+      was_full = _withlock_full?
+      @data.delete(key)
+      if was_full
+        MQRPC::logger.info "#{self}: signalling non-fullness"
+        @condvar.signal
+      end
+    end
+  end # def delete
+
+  # boolean, indicates true when the hash is full (has size == initialized size)
+  def full?
+    @lock.synchronize do
+      return _withlock_full?
+    end
+  end # def full?
+
+  # return the size (total number of entries) in this hash.
+  def size
+    @lock.synchronize do
+      return @data.size
+    end
+  end
+
+  # Return an array of keys for this hash.
+  def keys
+    @lock.synchronize do
+      return @data.keys
+    end
+  end
+  alias :length :size
+
+  private
+  def _withlock_full?
+    return @data.size >= @size
+  end
+end # class SizedThreadSafeHash
+
+#sh = SizedThreadSafeHash.new(10)
+#Thread.new do
+  #0.upto(100) do |i|
+    #sh[i] = true
+  #end
+#end
+
+#0.upto(100) do |i|
+  #puts sh[i]
+  #sh.delete(i)
+  #sleep 1
+#end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: mqrpc
 version: !ruby/object:Gem::Version 
-  version: 0.0.1
+  version: 0.0.3
 platform: ruby
 authors: 
 - Jordan Sissel, Pete Fritchman
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-11-05 00:00:00 -08:00
+date: 2009-11-07 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -58,6 +58,7 @@ files:
 - lib/mqrpc/message.rb
 - lib/mqrpc/operation.rb
 - lib/mqrpc/messages/ping.rb
+- lib/mqrpc/sizedhash.rb
 - lib/mqrpc.rb
 has_rdoc: true
 homepage: http://code.google.com/p/logstash/wiki/MQRPC