mqrpc 0.0.4 → 0.0.6

data/lib/mqrpc/agent.rb

@@ -6,6 +6,7 @@ require 'mqrpc/operation'
 require 'mqrpc/sizedhash'
 require 'thread'
 require 'uuid'
+require 'set'
 
 # http://github.com/tmm1/amqp/issues/#issue/3
 # This is our (lame) hack to at least notify the user that something is
@@ -14,7 +15,7 @@ module AMQP
   module Client
     alias :original_reconnect :reconnect 
     def reconnect(*args)
-      $logger.warn "reconnecting to broker (bad MQ settings?)"
+      MQRPC::logger.warn "reconnecting to broker (bad MQ settings?)"
 
       # some rate limiting
       sleep(5)
@@ -30,19 +31,64 @@ module MQRPC
     MAXBUF = 20
     MAXMESSAGEWAIT = MAXBUF * 20
 
+    class << self
+      attr_accessor :message_handlers
+      attr_accessor :pipelines
+    end
+
+    # Subclasses use this to declare their support of
+    # any given message
+    def self.handle(messageclass, method)
+      if self.message_handlers == nil
+        self.message_handlers = Hash.new
+      end
+      self.message_handlers[messageclass] = method
+    end
+
+    def self.pipeline(source, destination)
+      if self.pipelines == nil
+        self.pipelines = Hash.new
+      end
+
+      self.pipelines[destination] = source
+    end
+
     def initialize(config)
       Thread::abort_on_exception = true
       @config = config
       @handler = self
       @id = UUID::generate
       @outbuffer = Hash.new { |h,k| h[k] = [] }
-      @queues = []
-      @topics = []
+      @queues = Set.new
+      @topics = Set.new
       @receive_queue = Queue.new
       @want_subscriptions = Queue.new
+
+      # figure out how to really do this correctly, see also def self.pipeline
+      if self.class.pipelines == nil
+        self.class.pipelines = Hash.new
+      end
+
       @slidingwindow = Hash.new do |h,k| 
-        MQRPC::logger.info "New sliding window for #{k}"
-        h[k] = SizedThreadSafeHash.new(MAXMESSAGEWAIT) 
+        MQRPC::logger.debug "New sliding window for #{k}"
+        h[k] = SizedThreadSafeHash.new(MAXMESSAGEWAIT) do |state|
+          if self.class.pipelines[k]
+            source = self.class.pipelines[k]
+            MQRPC::logger.debug "Got sizedhash callback for #{k}: #{state}"
+            case state
+            when :blocked
+              MQRPC::logger.info("Queue '#{k}' is full, unsubscribing from #{source}")
+              exchange = @mq.topic(@config.mqexchange, :durable => true)
+              mq_q = @mq.queue(source, :durable => true)
+              mq_q.bind(exchange, :key => "*")
+              mq_q.unsubscribe
+              @queues.delete(source)
+            when :ready
+              MQRPC::logger.info("Queue '#{k}' is ready, resubscribing to #{source}")
+              subscribe(source)
+            end
+          end
+        end
       end
 
       @mq = nil
@@ -84,9 +130,7 @@ module MQRPC
           end
 
           MQRPC::logger.info "Subscribing to main queue #{@id}"
-          mq_q = @mq.queue(@id, :auto_delete => true)
-          mq_q.subscribe(:ack =>true) { |hdr, msg| @receive_queue << [hdr, msg] }
-          #handle_new_subscriptions
+          subscribe(@id)
           
           # TODO(sissel): make this a deferred thread that reads from a Queue
           #EM.add_periodic_timer(5) { handle_new_subscriptions }
@@ -119,27 +163,39 @@ module MQRPC
     end # def subscribe_topic
 
     def handle_message(hdr, msg_body)
+      queue = hdr.routing_key
+
+      # If we just unsubscribed from a queue, we may still have some
+      # messages buffered, so reject the message.
+      # Currently RabbitMQ doesn't support message rejection, so let's
+      # ack the message then push it back into the queue, unmodified.
+      if !@queues.include?(queue)
+        MQRPC::logger.warn("Got message on queue #{queue} that we are not "\
+                           "subscribed to; rejecting")
+        hdr.ack
+        @mq.queue(queue, :durable => true).publish(msg_body, :persistent => true)
+        return
+      end
+
+      # TODO(sissel): handle any exceptions we might get from here.
       obj = JSON::load(msg_body)
       if !obj.is_a?(Array)
         obj = [obj]
       end
 
-      queue = hdr.routing_key
       obj.each do |item|
         message = Message.new_from_data(item)
         slidingwindow = @slidingwindow[queue]
         if message.respond_to?(:from_queue)
           slidingwindow = @slidingwindow[message.from_queue]
         end
-        MQRPC::logger.debug "#{Thread.current} Got message #{message.class}##{message.id} on queue #{queue}"
-        MQRPC::logger.debug "Received message: #{message.inspect}"
+        MQRPC::logger.debug "Got message #{message.class}##{message.id} on queue #{queue}"
+        #MQRPC::logger.debug "Received message: #{message.inspect}"
         if (message.respond_to?(:in_reply_to) and 
             slidingwindow.include?(message.in_reply_to))
           MQRPC::logger.debug "Got response to #{message.in_reply_to}"
           slidingwindow.delete(message.in_reply_to)
         end
-        name = message.class.name.split(":")[-1]
-        func = "#{name}Handler"
 
         # Check if we have a specific operation looking for this
         # message.
@@ -147,19 +203,20 @@ module MQRPC
             @message_operations.has_key?(message.in_reply_to))
           operation = @message_operations[message.in_reply_to]
           operation.call(message)
-        elsif @handler.respond_to?(func) 
-          @handler.send(func, message) do |response|
+        elsif can_receive?(message.class)
+          func = self.class.message_handlers[message.class]
+          self.send(func, message) do |response|
             reply_destination = message.reply_to
             response.from_queue = queue
             sendmsg(reply_destination, response)
           end
 
-          # We should allow the message handler to defer acking if they want
-          # For instance, if we want to index things, but only want to ack
-          # things once we actually flush to disk.
+          # TODO(sissel): We should allow the message handler to defer acking
+          # if they want For instance, if we want to index things, but only
+          # want to ack things once we actually flush to disk.
         else
-          $stderr.puts "#{@handler.class.name} does not support #{func}"
-        end # if @handler.respond_to?(func)
+          $stderr.puts "#{@handler.class.name} does not support #{message.class}"
+        end 
       end
       hdr.ack
     end # def handle_message
@@ -168,6 +225,10 @@ module MQRPC
       @amqpthread.join
     end # run
 
+    def can_receive?(message_class)
+      return self.class.message_handlers.include?(message_class)
+    end
+
     def handle_subscriptions
       while true do
         type, name = @want_subscriptions.pop
@@ -175,12 +236,14 @@ module MQRPC
         when :queue
           next if @queues.include?(name)
           MQRPC::logger.info "Subscribing to queue #{name}"
+          exchange = @mq.topic(@config.mqexchange, :durable => true)
           mq_q = @mq.queue(name, :durable => true)
+          mq_q.bind(exchange, :key => "*")
           mq_q.subscribe(:ack => true) { |hdr, msg| @receive_queue << [hdr, msg] }
           @queues << name
         when :topic
           MQRPC::logger.info "Subscribing to topic #{name}"
-          exchange = @mq.topic(@config.mqexchange)
+          exchange = @mq.topic(@config.mqexchange, :durable => true)
           mq_q = @mq.queue("#{@id}-#{name}",
                            :exclusive => true,
                            :auto_delete => true).bind(exchange, :key => name)
@@ -247,7 +310,7 @@ module MQRPC
           flushout(destination)
         end
       else
-        MQRPC::logger.debug "#{Thread.current} Sending to #{destination}: #{msg.inspect}"
+        MQRPC::logger.debug "Sending to #{destination}: #{msg.inspect}"
         @mq.queue(destination, :durable => true).publish([msg].to_json, :persistent => true)
       end
 

data/lib/mqrpc/sizedhash.rb

@@ -1,15 +1,28 @@
 require 'thread'
 require 'mqrpc'
 
+class TrackingMutex < Mutex
+  def synchronize(&blk)
+    #puts "Enter synchronize #{self} @ #{Thread.current} + #{caller[0]}"
+    super { blk.call }
+    #puts "Exit synchronize #{self} @ #{Thread.current} + #{caller[0]}"
+  end # def synchronize
+end # clas TrackingMutex < Mutex
+
 # Thread-safe sized hash similar to SizedQueue.
 # The only time we will block execution is in setting new items.
 # That is, SizedThreadSafeHash#[]=
 class SizedThreadSafeHash
-  def initialize(size)
-    @lock = Mutex.new
+  attr_reader :callback
+  attr_reader :size
+
+  def initialize(size, &callback)
+    @lock = TrackingMutex.new
     @size = size
     @condvar = ConditionVariable.new
     @data = Hash.new
+    @callback = callback
+    @state = nil
   end # def initialize
 
   # set a key and value
@@ -17,11 +30,23 @@ class SizedThreadSafeHash
     @lock.synchronize do
       # If adding a new item, wait if the hash is full
       if !@data.has_key?(key) and _withlock_full?
-        MQRPC::logger.info "#{self}: Waiting to add key #{key.inspect}, hash is full"
-        #MQRPC::logger.info "#{self}: Keys: #{@data.keys.inspect}"
-        #MQRPC::logger.info "#{self} current thread is #{Thread.current}"
-        #pp @data
+        MQRPC::logger.info "#{self}: Waiting to add key #{key.inspect}, hash is full (thraed #{Thread.current})"
+        if @state != :blocked
+          @state = :blocked
+          @callback.call(@state) if @callback
+        else
+          puts "State is already #{@state} (want :blocked), skipping event call"
+        end
+
         @condvar.wait(@lock)
+
+        if @state != :ready
+          @state = :ready
+          MQRPC::logger.info "#{self}: state => :ready"
+          @callback.call(@state) if @callback
+        else
+          puts "State is already #{@state} (want :ready), skipping event call"
+        end
       end
       @data[key] = value
     end
@@ -82,16 +107,3 @@ class SizedThreadSafeHash
     return @data.size >= @size
   end
 end # class SizedThreadSafeHash
-
-#sh = SizedThreadSafeHash.new(10)
-#Thread.new do
-  #0.upto(100) do |i|
-    #sh[i] = true
-  #end
-#end
-
-#0.upto(100) do |i|
-  #puts sh[i]
-  #sh.delete(i)
-  #sleep 1
-#end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: mqrpc
 version: !ruby/object:Gem::Version 
-  version: 0.0.4
+  version: 0.0.6
 platform: ruby
 authors: 
 - Jordan Sissel, Pete Fritchman
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-11-07 00:00:00 -08:00
+date: 2009-11-14 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency