mqrpc 0.0.6 → 0.0.7

data/lib/mqrpc/agent.rb

@@ -78,11 +78,7 @@ module MQRPC
             case state
             when :blocked
               MQRPC::logger.info("Queue '#{k}' is full, unsubscribing from #{source}")
-              exchange = @mq.topic(@config.mqexchange, :durable => true)
-              mq_q = @mq.queue(source, :durable => true)
-              mq_q.bind(exchange, :key => "*")
-              mq_q.unsubscribe
-              @queues.delete(source)
+              unsubscribe(source)
             when :ready
               MQRPC::logger.info("Queue '#{k}' is ready, resubscribing to #{source}")
               subscribe(source)
@@ -113,6 +109,7 @@ module MQRPC
 
     def start_amqp
       @amqpthread = Thread.new do 
+        Thread.current[:name] = "AMQP"
         # Create connection to AMQP, and in turn, the main EventMachine loop.
         amqp_config = {:host => @config.mqhost,
                        :port => @config.mqport,
@@ -133,11 +130,9 @@ module MQRPC
           subscribe(@id)
           
           # TODO(sissel): make this a deferred thread that reads from a Queue
-          #EM.add_periodic_timer(5) { handle_new_subscriptions }
           EM.defer { handle_subscriptions }
 
           EM.add_periodic_timer(1) do
-            # TODO(sissel): add locking
             @outbuffer.each_key { |dest| flushout(dest) }
             @outbuffer.clear
           end
@@ -147,6 +142,7 @@ module MQRPC
 
     def start_receiver
       Thread.new do 
+        Thread.current[:name] = "receiver"
         while true
           header, message = @receive_queue.pop
           handle_message(header, message)
@@ -155,9 +151,27 @@ module MQRPC
     end # def start_receiver
 
     def subscribe(name)
+      MQRPC::logger.info "Wanting to subscribe to queue #{name}"
       @want_subscriptions << [:queue, name]
     end # def subscribe
 
+    def unsubscribe(name)
+      exchange = @mq.topic(@config.mqexchange, :durable => true)
+      mq_q = @mq.queue(name, :durable => true)
+      mq_q.bind(exchange, :key => "*")
+
+      op = Operation.new
+      mq_q.unsubscribe { op.finished }
+      op.wait_until_finished
+      @queues.delete(name)
+
+      #mq_q.unsubscribe { @queues.delete(name) }
+      ## wait for unsubscribe to finish; it's async
+      #while @queues.member?(name)
+        #sleep(0.1)
+      #end
+    end # def unsubscribe
+
     def subscribe_topic(name)
       @want_subscriptions << [:topic, name]
     end # def subscribe_topic
@@ -177,8 +191,13 @@ module MQRPC
         return
       end
 
-      # TODO(sissel): handle any exceptions we might get from here.
-      obj = JSON::load(msg_body)
+      begin
+        obj = JSON::load(msg_body)
+      rescue JSON::ParserError
+        MQRPC::logger.warn("Skipping non-JSON message: #{msg_body}")
+        hdr.ack
+        return
+      end
       if !obj.is_a?(Array)
         obj = [obj]
       end
@@ -222,24 +241,52 @@ module MQRPC
     end # def handle_message
 
     def run
+      Thread.current[:name] ||= "#{self.class.name}#run"
       @amqpthread.join
     end # run
 
     def can_receive?(message_class)
+      if self.class.message_handlers == nil
+        self.class.message_handlers = []
+      end
+
       return self.class.message_handlers.include?(message_class)
     end
 
     def handle_subscriptions
+      Thread.current[:name] = "subscriptionhandler"
       while true do
-        type, name = @want_subscriptions.pop
-        case type
+        queuetype, name = @want_subscriptions.pop
+
+        case queuetype
         when :queue
-          next if @queues.include?(name)
+          if @queues.include?(name) 
+            MQRPC::logger.info "Ignoring subscription request to queue "\
+                               "#{name}, already subscribed."
+            next
+          end
           MQRPC::logger.info "Subscribing to queue #{name}"
+          # Send a dummy message to queue #{name} so there's at least
+          # one message to receive, and wake up our Operation.
+          sendmsg(name, DummyMessage.new)
           exchange = @mq.topic(@config.mqexchange, :durable => true)
           mq_q = @mq.queue(name, :durable => true)
           mq_q.bind(exchange, :key => "*")
-          mq_q.subscribe(:ack => true) { |hdr, msg| @receive_queue << [hdr, msg] }
+          op = Operation.new
+          mq_q.subscribe(:ack => true) do |hdr, msg| 
+            op.finished
+            return if msg.is_a?(DummyMessage)
+            queue = hdr.routing_key
+            MQRPC::logger.info("received message on #{queue}")
+            @receive_queue << [hdr, msg]
+            MQRPC::logger.info("finished receiving message on #{queue}") 
+            MQRPC::logger.info("msg: #{msg}")
+            MQRPC::logger.info("#{queue} queue size: #{@receive_queue.length}") 
+          end
+          # Wait until we receive our first message (might be DummyMessage,
+          # doesn't matter) -- this confirms we are subscribed. subscribe
+          # is async...
+          op.wait_until_finished
           @queues << name
         when :topic
           MQRPC::logger.info "Subscribing to topic #{name}"
@@ -249,30 +296,9 @@ module MQRPC
                            :auto_delete => true).bind(exchange, :key => name)
           mq_q.subscribe { |hdr, msg| @receive_queue << [hdr, msg] }
           @topics << name
-        end
-      end
-    end
-
-    def handle_new_subscriptions
-      todo = @want_queues - @queues
-      todo.each do |queue|
-        MQRPC::logger.info "Subscribing to queue #{queue}"
-        mq_q = @mq.queue(queue, :durable => true)
-        mq_q.subscribe(:ack => true) { |hdr, msg| @receive_queue << [hdr, msg] }
-        @queues << queue
-      end # todo.each
-
-      todo = @want_topics - @topics
-      todo.each do |topic|
-        MQRPC::logger.info "Subscribing to topic #{topic}"
-        exchange = @mq.topic(@config.mqexchange)
-        mq_q = @mq.queue("#{@id}-#{topic}",
-                         :exclusive => true,
-                         :auto_delete => true).bind(exchange, :key => topic)
-        mq_q.subscribe { |hdr, msg| @receive_queue << [hdr, msg] }
-        @topics << topic
-      end # todo.each
-    end # handle_new_subscriptions
+        end # case queuetype
+      end # while true
+    end # def handle_subscriptions
 
     def flushout(destination)
       msgs = @outbuffer[destination]
@@ -315,7 +341,8 @@ module MQRPC
       end
 
       if block_given?
-        op = Operation.new(callback)
+        op = Operation.new &callback
+        MQRPC::logger.debug "New operation for #{msg.id}"
         @message_operations[msg.id] = op
         return op
       end

data/lib/mqrpc/functions/ping.rb

@@ -1,7 +1,9 @@
 require 'rubygems'
 require 'mqrpc/messages/ping'
 
-module MQRPC; module Functions; module Ping
+module MQRPC; module Functions; class Ping
+  #handle MQRPC::Messages::PingRequest, :PingRequestHandler
+
   def PingRequestHandler(request)
     MQRPC::logger.debug "received PingRequest (#{request.pingdata})"
     response = MQRPC::Messages::PingResponse.new

data/lib/mqrpc/logger.rb

@@ -13,3 +13,16 @@ module MQRPC
     @logger = logger
   end
 end
+
+# Make logger include the thread id.
+class FormatterWithThread < Logger::Formatter
+  Format = "%s, [%s#%d/%s] %5s -- %s: %s\n"
+
+  def call(severity, time, progname, msg)
+    thread_id = (Thread.current[:name] or Thread.current)
+    Format % [severity[0..0], format_datetime(time), $$, thread_id,
+              severity, progname, msg2str(msg)]
+  end
+end
+
+MQRPC::logger.formatter = FormatterWithThread.new

data/lib/mqrpc/message.rb

@@ -108,6 +108,10 @@ module MQRPC
     # Nothing.
   end # class RequestMessage
 
+  class DummyMessage < Message
+    # Nothing.
+  end # class DummyMessage
+
   class ResponseMessage < Message
     header :in_reply_to
     header :from_queue

data/lib/mqrpc/message.rb.orig

@@ -0,0 +1,132 @@
+require 'json'
+require 'thread'
+require 'mqrpc/logger'
+
+module BindToHash
+  def header(method, key=nil)
+    key = method.to_s if key == nil
+    hashbind(method, "/#{key}")
+  end
+
+  def argument(method, key=nil)
+    key = method.to_s if key == nil
+    hashbind(method, "/args/#{key}")
+  end
+
+  private
+  def hashbind(method, key)
+    hashpath = __genhashpath(key)
+    self.class_eval %(
+      def #{method}
+        return #{hashpath}
+      end
+      def #{method}=(val)
+        #{hashpath} = val
+      end
+    )
+  end
+
+  private
+  def __genhashpath(key)
+    # TODO(sissel): enforce 'key' needs to be a string or symbol?
+    path = key.split("/").select { |x| x.length > 0 }\
+               .map { |x| "[#{x.inspect}]" }
+    return "@data#{path.join("")}"
+  end
+end # modules BindToHash
+
+module MQRPC
+  class Message
+    extend BindToHash
+    @@idseq = 0
+    @@idlock = Mutex.new
+    @@knowntypes = Hash.new
+    attr_accessor :data
+
+    # Message attributes
+    header :id
+    header :message_class
+    header :delayable
+    header :reply_to
+    header :timestamp
+    header :args
+
+    def self.inherited(subclass)
+      MQRPC::logger.debug "Message '#{subclass.name}' subclasses #{self.name}"
+      @@knowntypes[subclass.name] = subclass
+
+      # Call the class initializer if it has one.
+      if subclass.respond_to?(:class_initialize)
+        subclass.class_initialize
+      end
+    end # def self.inherited
+
+    def self.new_from_data(data)
+      obj = nil
+      name = data["message_class"]
+      if @@knowntypes.has_key?(name)
+        obj = @@knowntypes[name].new
+      else
+        $stderr.puts "No known message class: #{name}, #{data.inspect}"
+        obj = Message.new
+      end
+      obj.data = data
+      return obj
+    end
+
+    def initialize
+      @data = Hash.new
+      # Don't delay messages by defualt
+      self.delayable = false
+
+      generate_id!
+      self.message_class = self.class.name
+      self.args = Hash.new
+    end
+
+    def generate_id!
+      @@idlock.synchronize do
+        self.id = @@idseq
+        #puts "Generating id. #{self.class}.id == #{self.id}"
+        @@idseq += 1
+      end
+    end
+
+    def age
+      return Time.now.to_f - timestamp
+    end
+
+    def to_json(*args)
+      return @data.to_json(*args)
+    end
+
+    protected
+    attr :data
+  end # class Message
+
+  class RequestMessage < Message
+    # Nothing.
+  end # class RequestMessage
+
+  class ResponseMessage < Message
+    header :in_reply_to
+    header :from_queue
+
+    def initialize(source_request=nil)
+      super()
+
+      # Copy the request id if we are given a source_request
+      if source_request.is_a?(RequestMessage)
+        self.in_reply_to = source_request.id
+        #self.delayable = source_request.delayable
+      end
+      self.args = Hash.new
+    end
+
+    # Report the success of the request this response is for.
+    # Should be implemented by subclasses.
+    def success?
+      raise NotImplementedError
+    end
+  end # class ResponseMessage
+end # module MQRPC

data/lib/mqrpc/message.rb.rej

@@ -0,0 +1,19 @@
+***************
+*** 108,113 ****
+     # Nothing.
+   end # class RequestMessage
+  
+   class ResponseMessage < Message
+     header :in_reply_to
+     header :from_queue
+--- 108,117 ----
+     # Nothing.
+   end # class RequestMessage
+  
++   class DummyMessage < Message
++     # Nothing.
++   end # class DummyMessage
++ 
+   class ResponseMessage < Message
+     header :in_reply_to
+     header :from_queue

data/lib/mqrpc/operation.rb

@@ -6,9 +6,14 @@ module MQRPC
   # A single message operation
   # * Takes a callback to call when a message is received
   # * Allows you to wait for the operation to complete.
-  # * An operation is 'complete' when the callback returns :finished
+  # * An operation is 'complete' when the callback returns.
+  #
+  # If your callback returns :continue, then we will not call finished.
+  # This allows you to have an operation that is invoked multiple times,
+  # such as for streaming blocks of data, and only finish when you know
+  # you are done.
   class Operation
-    def initialize(callback)
+    def initialize(&callback)
       @mutex = Mutex.new
       @callback = callback
       @cv = ConditionVariable.new
@@ -21,9 +26,7 @@ module MQRPC
       @mutex.synchronize do
         ret = @callback.call(*args)
         if ret != :continue
-          #MQRPC::logger.debug "operation #{self} finished"
-          @finished = true
-          @cv.signal
+          _withlock_finished
         end
         return ret
       end
@@ -34,15 +37,35 @@ module MQRPC
     # immediately.
     def wait_until_finished
       @mutex.synchronize do
-        if !finished?
+        if !_withlock_finished?
           @cv.wait(@mutex)
         end
       end
     end # def wait_until_finished
 
-    protected
+    # Is the operation finished yet?
     def finished?
+      @mutex.synchronize do
+        return _withlock_finished?
+      end
+    end # def finished?
+
+    # Declare that the operation is finished.
+    def finished
+      @mutex.synchronize do
+        return _withlock_finished
+      end
+    end # def finished
+
+    protected
+    def _withlock_finished?
       return @finished
     end # def finished?
+
+    def _withlock_finished
+      @finished = true
+      @cv.signal
+    end
+
   end # class Operation
 end # module MQRPC

data/lib/mqrpc/sizedhash.rb

@@ -1,11 +1,12 @@
 require 'thread'
 require 'mqrpc'
 
+# A mutex which logs every enter/exit on synchronize
 class TrackingMutex < Mutex
   def synchronize(&blk)
-    #puts "Enter synchronize #{self} @ #{Thread.current} + #{caller[0]}"
+    MQRPC::logger.debug "Enter synchronize #{self} @ #{caller[0]}"
     super { blk.call }
-    #puts "Exit synchronize #{self} @ #{Thread.current} + #{caller[0]}"
+    MQRPC::logger.debug "Exit synchronize #{self} @ #{caller[0]}"
   end # def synchronize
 end # clas TrackingMutex < Mutex
 
@@ -17,7 +18,7 @@ class SizedThreadSafeHash
   attr_reader :size
 
   def initialize(size, &callback)
-    @lock = TrackingMutex.new
+    @lock = Mutex.new
     @size = size
     @condvar = ConditionVariable.new
     @data = Hash.new
@@ -30,7 +31,7 @@ class SizedThreadSafeHash
     @lock.synchronize do
       # If adding a new item, wait if the hash is full
       if !@data.has_key?(key) and _withlock_full?
-        MQRPC::logger.info "#{self}: Waiting to add key #{key.inspect}, hash is full (thraed #{Thread.current})"
+        MQRPC::logger.info "#{self}: Waiting to add key #{key.inspect}, hash is full (thread #{Thread.current})"
         if @state != :blocked
           @state = :blocked
           @callback.call(@state) if @callback

data/lib/mqrpc/test.diff

@@ -0,0 +1,32 @@
+Index: lib/mqrpc/agent.rb
+===================================================================
+--- agent.rb  (revision 318)
++++ agent.rb  (working copy)
+@@ -267,10 +267,16 @@
+          end
+          MQRPC::logger.info "Subscribing to queue #{name}"
+          @queues << name
++          # Send a dummy message to queue #{name} so there's at least
++          # one message to receive, and wake up our Operation.
++          sendmsg(name, DummyMessage.new)
+          exchange = @mq.topic(@config.mqexchange, :durable => true)
+          mq_q = @mq.queue(name, :durable => true)
+          mq_q.bind(exchange, :key => "*")
++          op = Operation.new
+          mq_q.subscribe(:ack => true) do |hdr, msg|
++            op.finished
++            return if msg.is_a?(DummyMessage)
+            queue = hdr.routing_key
+            MQRPC::logger.info("received message on #{queue}")
+            @receive_queue << [hdr, msg]
+@@ -278,6 +284,10 @@
+            MQRPC::logger.info("msg: #{msg}")
+            MQRPC::logger.info("#{queue} queue size: #{@receive_queue.length}")
+          end
++          # Wait until we receive our first message (might be DummyMessage,
++          # doesn't matter) -- this confirms we are subscribed. subscribe
++          # is async...
++          op.wait_until_finished
+        when :topic
+          MQRPC::logger.info "Subscribing to topic #{name}"
+          exchange = @mq.topic(@config.mqexchange, :durable => true)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: mqrpc
 version: !ruby/object:Gem::Version 
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors: 
 - Jordan Sissel, Pete Fritchman
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-11-14 00:00:00 -08:00
+date: 2009-11-20 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -53,12 +53,15 @@ extra_rdoc_files: []
 files: 
 - lib/mqrpc/agent.rb
 - lib/mqrpc/config.rb
+- lib/mqrpc/test.diff
 - lib/mqrpc/logger.rb
 - lib/mqrpc/functions/ping.rb
 - lib/mqrpc/message.rb
+- lib/mqrpc/message.rb.orig
 - lib/mqrpc/operation.rb
 - lib/mqrpc/messages/ping.rb
 - lib/mqrpc/sizedhash.rb
+- lib/mqrpc/message.rb.rej
 - lib/mqrpc.rb
 has_rdoc: true
 homepage: http://code.google.com/p/logstash/wiki/MQRPC