qup 1.2.0 → 1.4.0

data/lib/qup/adapter/kestrel/queue.rb

@@ -13,8 +13,8 @@ class Qup::Adapter::Kestrel
     # name    - the String name of the Topic
     #
     # Returns a new Queue
-    def initialize( address, name )
-      super(address, name)
+    def initialize( client, name )
+      super( client, name )
       @open_messages = {}
     end
 
@@ -25,7 +25,7 @@ class Qup::Adapter::Kestrel
     #
     # Returns nothing.
     def flush
-      @admin_client.flush(@name)
+      @client.flush(@name)
     end
 
 
@@ -33,8 +33,7 @@ class Qup::Adapter::Kestrel
     #
     # Returns an integer of the Queue depth
     def depth
-      stats = @admin_client.stat( @name )
-      return stats['items']
+      @client.stats['queues'][@name]['items']
     end
 
 
@@ -42,9 +41,6 @@ class Qup::Adapter::Kestrel
     #
     # message - the data to put onto the queue.
     #
-    # The 'message' that is passed in is wrapped in a Qup::Message before being
-    # stored.
-    #
     # A user of the Qup API should use a Producer instance to put items onto the
     # queue.
     #
@@ -62,16 +58,16 @@ class Qup::Adapter::Kestrel
     # A user of the Qup API should use a Consumer instance to retrieve items
     # from the Queue.
     #
-    # Returns a Message
+    # Returns a Message or nil if no message was on the queue
     def consume(&block)
-      data = @client.get( @name )
-      q_message = ::Qup::Message.new( data.object_id, data )
-      @open_messages[q_message.key] = q_message
+      q_item = @client.reserve( @name )
+      return nil unless q_item
+      q_message = ::Qup::Message.new( q_item.object_id, unmarshal_if_marshalled( q_item ))
+      @open_messages[q_message.key] = q_item
       if block_given? then
         yield_message( q_message, &block )
-      else
-        return q_message
       end
+      return q_message
     end
 
 
@@ -85,13 +81,22 @@ class Qup::Adapter::Kestrel
     def acknowledge( message )
       open_msg = @open_messages.delete( message.key )
       raise Qup::Error, "Message #{message.key} is not currently being consumed" unless open_msg
-      @client.close_last_transaction
+      @client.close( @name )
     end
 
     #######
     private
     #######
 
+    def unmarshal_if_marshalled( data )
+      if data[0].ord == 4 and data[1].ord == 8 then
+        Marshal::load( data )
+      else
+        data
+      end
+    end
+
+
     def yield_message( message, &block )
       yield message
     ensure

data/lib/qup/adapter/kestrel/topic.rb

@@ -1,3 +1,5 @@
+require 'json'
+require 'set'
 require 'qup/adapter/kestrel/destination'
 class Qup::Adapter::Kestrel
   #
@@ -27,20 +29,29 @@ class Qup::Adapter::Kestrel
     #
     # Returns a Subscriber
     def subscriber( name )
-      ::Qup::Subscriber.new( self, subscriber_queue( name ) )
+      ::Qup::Subscriber.new( self, subscriber_queue( name )  )
     end
 
 
     # Internal: Return the number of Subscribers to this Topic
     #
+    # We want the sub portion of the json document that is in the 'counters'
+    # section. The keys in the 'counters' section that represent queue counters
+    # are all prefixed with 'q/<queue_name>/<stat>'. To count the number of
+    # subscribers to this topic, we just count the uniqe <queue_name> elements
+    # that start with this queue's name and followed by a '+'
+    #
     # Returns integer
     def subscriber_count
-      c = 0
-      @client.stats['queues'].keys.each do |k|
-        next if k =~ /errors$/
-        c += 1 if k =~ /^#{@name}\+/
+      c = Set.new
+
+      stats['queues'].keys.each do |k|
+        next unless k =~ %r{\A#{@name}\+}
+        parts = k.split("+")
+        c << parts[1]
       end
-      return c
+
+      return c.size
     end
 
     # Internal: Publish a Message to all the Subscribers
@@ -49,7 +60,7 @@ class Qup::Adapter::Kestrel
     #
     # Returns nothing
     def publish( message )
-      @client.set( @name, message )
+      @client.set( @name,  message ) # do not expire the message
     end
 
     #######
@@ -58,11 +69,15 @@ class Qup::Adapter::Kestrel
 
     def subscriber_queue( sub_name )
       sname = subscriber_queue_name( sub_name )
-      ::Qup::Adapter::Kestrel::Queue.new( @address, sname )
+      ::Qup::Adapter::Kestrel::Queue.new( @client, sname )
     end
 
     def subscriber_queue_name( sub_name )
       "#{@name}+#{sub_name}"
     end
+
+    def stats
+      @client.stats
+    end
   end
 end

data/lib/qup/adapter/redis/queue.rb

@@ -76,7 +76,8 @@ class Qup::Adapter::Redis
     #
     # Returns a Message
     def consume(&block)
-      queue_name, data = @client.brpop name, 0 # blocking pop
+      data = @client.rpop( name )
+      return if data.nil?
       message = ::Qup::Message.new( data.object_id, data )
       @open_messages[message.key] = message
       if block_given? then

data/lib/qup/backoff_sleeper.rb

@@ -0,0 +1,52 @@
+# Used to sleep for exponentially increasing amounts of time between
+# successive unsuccessful attempts to do something like poll for queue data or
+# reconnect to a client. Maxes out at 1 second between attempts.
+#
+# Call #tick in every iteration of the task you are trying to accomplish. When
+# a successful iteration is completed (for example the queue had data, or
+# connecting was successful) call #reset to let the sleeper know that it can
+# reset the amount of time between attempts back to 0. The sleeper will not
+# call Kernel#sleep if it #reset was just called, so it is safe to call #tick
+# on every iteration, provided that you call #reset on success before calling
+# #tick.
+#
+# Examples:
+#
+#   sleeper = BackoffSleeper.new
+#
+#   loop do
+#     message = check_for_message
+#     sleeper.reset unless message.nil?
+#     sleeper.tick
+#   end
+module Qup
+  class BackoffSleeper
+    attr_reader :count
+
+    MULTIPLIERS = [0, 0.01, 0.1, 1]
+
+    def initialize
+      @count = 0
+    end
+
+    # Register an iteration. Sleeps if neccesary. Does not sleep if #reset has
+    # just been called.
+    def tick
+      Kernel.sleep(length) if length > 0
+      @count += 1
+    end
+
+    # Reset the backoff sequence to 0.
+    def reset
+      @count = 0
+    end
+
+    def length
+      (multiplier + (multiplier * Kernel.rand)) / 2
+    end
+
+    def multiplier
+      MULTIPLIERS[@count] || MULTIPLIERS.last
+    end
+  end
+end

data/lib/qup/batch_consumer.rb

@@ -0,0 +1,132 @@
+# BatchConsumer makes it easy to implement a batch-oriented consumer pattern
+# by calling the appropriate hooks on the provided "client" class. Clients
+# have the following callbacks:
+#
+# setup - Optional. Called before the client begins receiving messages
+#   (immediately after BatchConsumer#run is called).
+# process - Required. Called with an instance of Qup::Message when a
+#   message is removed from the queue. After process finishes the message is
+#   acknowledged from the queue. process is not wrapped in any exception
+#   handling, so exceptions raised in process will propagate through the
+#   BatchConsumer and the message will not be acknowledged.
+# teardown - Optional. Called after the client has received :max_size
+#   messages OR :max_age has been exceeded.
+#
+# BatchConsumers should be only be run once. If you need to run again, create
+# a new instance.
+#
+# See #initialize for the configuration options.
+#
+# Examples:
+#
+#   class FileDumper
+#     include Qup::BatchConsumerAPI
+#
+#     def setup
+#       @file = File.open("/my/file.txt")
+#     end
+#
+#     def process(message)
+#       @file.puts(message.data)
+#     end
+#
+#     def teardown
+#       @file.close
+#     end
+#   end
+#
+#   batch_consumer = Qup::BatchConsumer.new({
+#     :max_size   => 5000,
+#     :max_age    => 600,
+#     :client     => FileDumper.new,
+#     :queue_uri  => "maildir:///tmp/test-queue",
+#     :queue_name => "my-queue"
+#   })
+#
+#   batch_consumer.run # This blocks until there have been 5000 message OR 600 seconds have passed.
+module Qup
+  module BatchConsumerAPI
+    def setup
+    end
+
+    def process(message)
+      raise NotImplementedError
+    end
+
+    def teardown
+    end
+  end
+
+  class BatchConsumer
+    # options - A hash of configuration options.
+    #          :client - The object upon which the callbacks are fired. It's class
+    #                    should include Qup::BatchConsumerAPI for declarative
+    #                    documentation purposes. Technically, the only constraint
+    #                    is that this object implement #setup, #process and
+    #                    #teardown (including Qup::BatchConsumerAPI includes noop
+    #                    definitions for #setup and #teardown). Required.
+    #          :queue_uri - The Qup format queue URI. Required.
+    #          :queue_name - The name of the queue that messages will be consumed
+    #                        from. Required.
+    #          :max_size - The maximum number of messages to process. Optional.
+    #          :max_age - The maximum number of seconds (from when #run is called)
+    #                     before finishing. Note that :max_size and :max_age are
+    #                     ORed, meaning that the BatchConsumer will finish when
+    #                     the first constraint is met. If neither constraint is
+    #                     provided, the BatchConsumer will never finish (i.e. #run
+    #                     will block indefinitely). Optional.
+    def initialize(options = {})
+      @message_count = 0
+      @options       = options
+    end
+
+    def run
+      @start = Time.now
+      client.setup
+
+      while live?
+        sleeper.tick
+        consumer.consume do |message|
+          client.process(message)
+          @message_count += 1
+          sleeper.reset
+        end
+      end
+
+      client.teardown
+    end
+
+    def session
+      @session ||= Qup::Session.new(@options[:queue_uri], @options[:session_options] || {})
+    end
+
+    private
+
+    def client
+      @options[:client]
+    end
+
+    def sleeper
+      @sleeper ||= BackoffSleeper.new
+    end
+
+    def consumer
+      @consumer ||= begin
+        session.queue(@options[:queue_name]).consumer
+      end
+    end
+
+    def live?
+      !too_big? && !too_old?
+    end
+
+    def too_big?
+      @message_count >= @options[:max_size] if @options[:max_size]
+    end
+
+    def too_old?
+      (Time.now - @start) > @options[:max_age] if @options[:max_age]
+    end
+
+  end
+end

data/lib/qup/consumer.rb

@@ -38,5 +38,12 @@ module Qup
     def acknowledge( message )
       @queue.acknowledge( message )
     end
+
+    # Public: Return how many messages are on the queue for this consumer
+    #
+    # Returns an integer
+    def depth
+      @queue.depth
+    end
   end
 end

data/lib/qup/session.rb

@@ -20,6 +20,9 @@ module Qup
     # Public: The URI of this Session
     attr_reader :uri
 
+    # Internal: access the adapter specific options.
+    attr_reader :options
+
     # Public: Create a new Session
     #
     # uri     - The connection String used to connect to appropriate provider
@@ -47,9 +50,10 @@ module Qup
     def initialize( uri, options = {} )
       @uri       = URI.parse( uri )
       @root_path = Pathname.new( @uri.path )
+      @options   = options
 
       adapter_klass = Qup::Adapters[@uri.scheme]
-      @adapter      = adapter_klass.new( @uri, options )
+      @adapter      = adapter_klass.new( @uri, @options )
 
       @queues  = Hash.new
       @topics  = Hash.new

data/spec/qup/adapter/kestrel_spec.rb

@@ -5,4 +5,5 @@ require 'qup/adapter/kestrel_context'
 describe 'Qup::Adapter::Kestrel', :kestrel => true do
   include_context "Qup::Adapter::Kestrel"
   it_behaves_like Qup::Adapter
+
 end

data/spec/qup/adapter/redis/queue_spec.rb

@@ -23,9 +23,9 @@ describe 'Qup::Adapter::Redis::Queue', :redis => true do
 
     describe "#destroy" do
       it "removes its name from the parent topic's subscriber set" do
-        redis.smembers("parent").should == ["test"]
+        redis.smembers("parent").should be == ["test"]
         queue.destroy
-        redis.smembers("parent").should == []
+        redis.smembers("parent").should be == []
       end
     end
 

data/spec/qup/backoff_sleeper_sleeper_spec.rb

@@ -0,0 +1,73 @@
+require "spec_helper"
+
+module Qup
+  describe BackoffSleeper do
+    before { Kernel.stub(:sleep) }
+
+    describe "#length" do
+      it "it returns the multiplier averaged with the multiplier multiplied by rand" do
+        Kernel.stub(:rand).with().and_return(0.5)
+
+        sleeper = BackoffSleeper.new
+        sleeper.stub(:multiplier => 1)
+
+        sleeper.length.should be == ((1 + (1 * 0.5)) / 2)
+      end
+    end
+
+    describe "#tick" do
+      it "starts count at 0" do
+        subject.count.should be == 0
+      end
+
+      it "increments count by 1 everytime it's called" do
+        subject.tick
+        subject.count.should be == 1
+
+        subject.tick
+        subject.count.should be == 2
+      end
+
+      it "sleeps for #length if length is > 0" do
+        Kernel.should_receive(:sleep).with(0.123)
+
+        subject.stub(:length => 0.123)
+        subject.tick
+      end
+
+      it "doesn't call sleep if the length is 0" do
+        Kernel.should_not_receive(:sleep).with(0)
+
+        subject.stub(:length => 0)
+        subject.tick
+      end
+    end
+
+    describe "#reset" do
+      it "sets count back to 0" do
+        subject.tick
+        expect { subject.reset }.to change { subject.count }.to(0)
+      end
+    end
+
+    describe "#multiplier" do
+      it "starts at 0" do
+        subject.multiplier.should be == 0
+      end
+
+      it "backs off exponentially" do
+        multipliers = (0..2).map do
+          subject.tick
+          subject.multiplier
+        end
+
+        multipliers.should be == [0.01, 0.1, 1]
+      end
+
+      it "maxes out at the last value of MULTIPLIERS" do
+        100.times { subject.tick }
+        subject.multiplier.should == BackoffSleeper::MULTIPLIERS.last
+      end
+    end
+  end
+end