simple_queues 1.1.1 → 1.2.0

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format doc

data/README.textile

@@ -1,24 +1,30 @@
 h1. SimpleQueues
 
-In the Gang of Four books, one of the first few lines is "Program to an interface, not an implementation." When you need a queue, the only operations you need are enqueue and dequeue. It doesn't matter that Redis (a nice and simple queue server when you need it) has all kinds of operations available.
+In the Gang of Four books, one of the first few lines is "Program to an interface, not an implementation." When you need a queue, the only operations you need are enqueue and dequeue. It doesn't matter that Redis (a nice and simple queue server when you need it) has a ton of extra features which we aren't going to use.
 
-This library was written and spec'd on Ruby 1.9.2. It's so small it should work anywhere the Redis gem works.
+This library was written and spec'd on Ruby 1.9.2. It is also in use, in production, on JRuby in 1.8 mode.
 
 h2. Exceptions
 
-All underlying exceptions the Redis gem raises are let through: no extra exceptions are raised. This means you'll see <code>Errno::ECONNREFUSED</code>, <code>Errno::EAGAIN</code> and friends.
+All underlying exceptions the Redis gem raises are let through. This means you'll see <code>Errno::ECONNREFUSED</code>, <code>Errno::EAGAIN</code> and friends. Of course, you may also receive ArgumentError if you do something bad.
 
 h2. Usage
 
 <pre><code>require "simple_queues"
+# Sane defaults:
+# * Defaults to a Redis instance at 127.0.0.1:6379, database 0
+# * Defaults to the JSON encoder
 Queues = SimpleQueues::Redis.new
 
-Queues.enqueue :pages_to_crawl, :url => "http://blog.teksol.info"
-Queues.enqueue :pages_to_crawl, :url => "http://techcrunch.com"
+Queues.enqueue :pages_to_crawl, :url => "http://blog.teksol.info/"
+Queues.enqueue :pages_to_crawl, :url => "http://techcrunch.com/"
 
 # In another process
 
+Queues = SimpleQueues::Redis.new(Redis.new("192.168.1.9", 9281), :encoder => :json)
+
 loop do
+  # This blocks until a message is dequeued.
   message = Queues.dequeue_blocking :pages_to_crawl
   process(message)
 end
@@ -30,6 +36,7 @@ loop do
     process(message)
   else
     # Timed out
+    break
   end
 end
 <code></pre>

data/lib/simple_queues.rb

@@ -11,4 +11,8 @@
 # * +dequeue_blocking+
 module SimpleQueues
   autoload :Redis, "simple_queues/redis"
+
+  autoload :JsonEncoder, "simple_queues/encoders/json"
+  autoload :MessagePackEncoder, "simple_queues/encoders/message_pack"
+  autoload :IdentityEncoder, "simple_queues/encoders/identity"
 end

data/lib/simple_queues/encoders/identity.rb

@@ -0,0 +1,11 @@
+module SimpleQueues
+  class IdentityEncoder
+    def encode(message)
+      message
+    end
+
+    def decode(message)
+      message
+    end
+  end
+end

data/lib/simple_queues/encoders/json.rb

@@ -0,0 +1,13 @@
+require "json"
+
+module SimpleQueues
+  class JsonEncoder
+    def encode(message)
+      JSON.generate(message)
+    end
+
+    def decode(message)
+      JSON.parse(message)
+    end
+  end
+end

data/lib/simple_queues/encoders/message_pack.rb

@@ -0,0 +1,13 @@
+require "msgpack"
+
+module SimpleQueues
+  class MessagePackEncoder
+    def encode(message)
+      MessagePack.pack(message)
+    end
+
+    def decode(message)
+      MessagePack.unpack(message)
+    end
+  end
+end

data/lib/simple_queues/redis.rb

@@ -6,7 +6,25 @@ module SimpleQueues
   #
   # Messages are enqueued to the right, dequeued from the left - thus the most recent messages are at the end of the list.
   class Redis
-    def initialize(redis = ::Redis.new)
+    attr_reader :encoder
+
+    # @param redis A Redis instance, or something that looks like Redis.
+    # @param options [Hash] A set of options.
+    #
+    # @option :encoder [#decode, #encode, Symbol] Either one of the named encoders, or an object that responds to both +#encode+ and +#decode+. Defaults to :json.
+    def initialize(redis = ::Redis.new, options={})
+      @encoder = case options[:encoder]
+                 when :messagepack, :msgpack
+                   SimpleQueues::MessagePackEncoder.new
+                 when :json, nil
+                   SimpleQueues::JsonEncoder.new
+                 when :identity
+                   SimpleQueues::IdentityEncoder.new
+                 else # Use whatever was provided
+                   options[:encoder]
+                 end
+      raise ArgumentError, "Provided encoder #{@encoder.inspect} does not handle #encode and #decode" unless @encoder.respond_to?(:encode) && @encoder.respond_to?(:decode)
+
       @redis  = redis
       @queues = Hash.new
     end
@@ -15,14 +33,13 @@ module SimpleQueues
       @queues[q_name(queue_name)] = block
     end
 
-    def serialize(message)
+    def encode(message)
       raise ArgumentError, "message must be non-nil" if message.nil?
-      raise ArgumentError, "message must be respond to #to_json" unless message.respond_to?(:to_json)
-      message.to_json
+      encoder.encode(message)
     end
 
-    def deserialize(message)
-      JSON.parse(message) if message
+    def decode(message)
+      encoder.decode(message) if message
     end
 
     # Enqueues a new message to the Redis backend.
@@ -32,7 +49,9 @@ module SimpleQueues
     # @return No useful value.
     # @raise ArgumentError Whenever the queue name or the message are nil, or the queue name is empty.
     def enqueue(queue_name, message)
-      msg = serialize(message)
+      raise ArgumentError, "Only hashes are accepted as messages" unless message.is_a?(Hash)
+
+      msg = encode(message)
       @redis.rpush(q_name(queue_name), msg)
       msg
     end
@@ -67,12 +86,12 @@ module SimpleQueues
         raise ArgumentError, "Timeout must not be nil" if timeout.nil? || timeout.to_s.empty?
 
         queue, result = @redis.blpop(*[@queues.keys, timeout.to_i].flatten)
-        @queues.fetch(queue).call(deserialize(result)) if queue
+        @queues.fetch(queue).call(decode(result)) if queue
         queue
       when 2
         queue_name, timeout = args.shift, args.shift
         _, result = @redis.blpop(q_name(queue_name), timeout.to_i)
-        deserialize(result)
+        decode(result)
       else
         raise "NOT DONE"
       end

data/lib/simple_queues/version.rb

@@ -1,3 +1,3 @@
 module SimpleQueues
-  VERSION = "1.1.1"
+  VERSION = "1.2.0"
 end

data/simple_queues.gemspec

@@ -18,9 +18,10 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   s.add_dependency 'redis'
-  s.add_dependency 'json'
   s.add_development_dependency 'rspec'
   s.add_development_dependency 'yard'
   s.add_development_dependency 'RedCloth'
   s.add_development_dependency 'ruby-debug19'
+  s.add_development_dependency 'msgpack'
+  s.add_development_dependency 'json'
 end

data/spec/dequeueing_spec.rb

@@ -9,7 +9,7 @@ describe SimpleQueues::Redis, "dequeue_blocking" do
     SimpleQueues::Redis.new(redis)
   end
 
-  it "requests an infinite timeout from Redis #blpop" do
+  it "request indefinitely at the Redis layer" do
     redis.should_receive(:blpop).with("pages_to_crawl", 0)
     queue.dequeue_blocking("pages_to_crawl")
   end
@@ -30,8 +30,12 @@ describe SimpleQueues::Redis, "dequeue_with_timeout" do
     double("redis")
   end
 
+  let :encoder do
+    double("encoder").as_null_object
+  end
+
   let :queue do
-    SimpleQueues::Redis.new(redis)
+    SimpleQueues::Redis.new(redis, :encoder => encoder)
   end
 
   it "should call Redis' BLPOP with the requested timeout" do
@@ -46,11 +50,15 @@ describe SimpleQueues::Redis, "dequeue_with_timeout" do
 
   it "raises an ArgumentError when the queue name is nil or empty" do
     lambda { queue.dequeue_with_timeout(nil) }.should raise_error(ArgumentError)
-    lambda { queue.dequeue_with_timeout("") }.should raise_error(ArgumentError)
+    lambda { queue.dequeue_with_timeout("") }.should  raise_error(ArgumentError)
   end
-  
-  it "returns the unserialized object" do
-    redis.should_receive(:blpop).with("test", 5).and_return(["test", "{\"hello\":\"world\",\"x\":42}"] )
-    queue.dequeue_with_timeout(:test, 5).should == {"hello" => "world", "x" => 42}
+
+  it "should decode the message using the selected encoder" do
+    encoded_message = '{"hello":"world","x":42}'
+
+    redis.should_receive(:blpop).with("test", 5).and_return(["test", encoded_message])
+    encoder.should_receive(:decode).with(encoded_message).and_return("hello" => "world", "x" => 42)
+
+    queue.dequeue_with_timeout(:test, 5)
   end
 end

data/spec/enqueueing_spec.rb

@@ -5,48 +5,60 @@ describe SimpleQueues::Redis, "enqueue" do
     double("redis")
   end
 
-  let :queue do
-    SimpleQueues::Redis.new(redis)
+  let :encoder do
+    double("encoder").as_null_object
   end
 
-  it "should enqueue to the end of the list" do
-    redis.should_receive(:rpush).with("q", '"message"')
-    queue.enqueue("q", "message")
+  let :queue do
+    SimpleQueues::Redis.new(redis, :encoder => encoder)
   end
 
-  it "should return the serialized message" do
-    redis.should_receive(:rpush).twice
-    queue.enqueue("q", "whatever").should == queue.serialize("whatever")
-    queue.enqueue("q", {:message => "whatever"}).should == queue.serialize(:message => "whatever")
+  it "should enqueue using the passed-in queue name" do
+    redis.should_receive(:rpush).with("the_queue", anything)
+    queue.enqueue "the_queue", {}
   end
 
-  it "should reject invalid queue names" do
-    lambda { queue.enqueue(nil, "") }.should raise_error(ArgumentError)
-    lambda { queue.enqueue("", "") }.should raise_error(ArgumentError)
+  it "should accept symbols as queue names, translating to a string" do
+    redis.should_receive(:rpush).with("pages_to_crawl", anything)
+    queue.enqueue :pages_to_crawl, :url => "http://blog.teksol.info/"
   end
 
-  it "should reject nil messages" do
-    lambda { queue.enqueue(:a, nil) }.should raise_error(ArgumentError)
+  it "should enqueue the encoded message" do
+    encoded_message = "jfd9jdf"
+
+    encoder.should_receive(:encode).and_return(encoded_message)
+    redis.should_receive(:rpush).with(anything, encoded_message)
+
+    queue.enqueue("q", :key => "value")
   end
 
-  it "should allow blank messages (although does this make sense?)" do
+  it "should return the encoded message" do
+    encoded_message = "barfly123"
+    encoder.should_receive(:encode).and_return(encoded_message)
     redis.should_receive(:rpush)
-    lambda { queue.enqueue(:a, "") }.should_not raise_error
+
+    queue.enqueue("q", {:message => "whatever"}).should == encoded_message
   end
 
-  it "accepts symbols as queue names, translating to a string" do
-    redis.should_receive(:rpush).with("pages_to_crawl", '"http://blog.teksol.info/"')
-    queue.enqueue :pages_to_crawl, "http://blog.teksol.info/"
+  it "should raise an ArgumentError when the message isn't a Hash" do
+    lambda { queue.enqueue(:q, 42)      }.should raise_error(ArgumentError, "Only hashes are accepted as messages")
+    lambda { queue.enqueue(:q, nil)     }.should raise_error(ArgumentError, "Only hashes are accepted as messages")
+    lambda { queue.enqueue(:q, "")      }.should raise_error(ArgumentError, "Only hashes are accepted as messages")
+    lambda { queue.enqueue(:q, [1,2,3]) }.should raise_error(ArgumentError, "Only hashes are accepted as messages")
   end
 
-  it "translates the message using JSON before enqueueing" do
-    redis.should_receive(:rpush).with("ops", '"shutdown_and_destroy"')
-    queue.enqueue :ops, :shutdown_and_destroy
+  it "should reject invalid queue names" do
+    lambda { queue.enqueue(nil, "") }.should raise_error(ArgumentError)
+    lambda { queue.enqueue("", "") }.should raise_error(ArgumentError)
+  end
+
+  it "should encode the message using the provided encoder" do
+    message = {:a => "b"}
+    encoded_message = 'hfd0hs'
 
-    redis.should_receive(:rpush).with("ops", '[1,2,3]')
-    queue.enqueue :ops, [1, 2, 3]
+    encoder.should_receive(:encode).with(message).and_return(encoded_message)
+    redis.should_receive(:rpush)
 
-    redis.should_receive(:rpush).with("ops", "{\"hello\":\"world\",\"x\":42}")
-    queue.enqueue :ops, {:hello => "world", :x => 42}
+    queue.enqueue :q, message
   end
 end

data/spec/identity_encoder_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+
+describe SimpleQueues::IdentityEncoder do
+  it { should encode(a:1, b:2, c:"3").as(a:1, b:2, c:"3") }
+  it { should decode(a:1, b:2, c:"3").as(a:1, b:2, c:"3") }
+
+  it "should not stringify symbol keys" do
+    message = {:a => "b"}
+    encoded_message = subject.encode(message)
+    decoded_message = subject.decode(encoded_message)
+    decoded_message.should == {:a => "b"}
+  end
+end

data/spec/json_encoder_spec.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+require 'json'
+
+describe SimpleQueues::JsonEncoder do
+  it { should encode("a" => "b").as(JSON("a" => "b"))     }
+  it { should decode({"a" => "b"}.to_json).as("a" => "b") }
+
+  it "stringifies Symbol keys" do
+    message = {:a => "b"}
+    encoded_message = subject.encode(message)
+    decoded_message = subject.decode(encoded_message)
+    decoded_message.should == {"a" => "b"}
+  end
+end

data/spec/message_pack_encoder_spec.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+require 'msgpack'
+
+describe SimpleQueues::MessagePackEncoder do
+  it { should encode("a" => "b").as(MessagePack.pack("a" => "b")) }
+  it { should decode({"a" => "b"}.to_msgpack).as("a" => "b") }
+
+  it "stringifies Symbol keys" do
+    message = {:a => "b"}
+    encoded_message = subject.encode(message)
+    decoded_message = subject.decode(encoded_message)
+    decoded_message.should == {"a" => "b"}
+  end
+end

data/spec/redis_integration_spec.rb

@@ -1,32 +1,62 @@
 require "spec_helper"
+require "redis"
+
+describe SimpleQueues::Redis do
+  let :queue_name do
+    :test
+  end
+
+  let :redis do
+    ::Redis.new
+  end
 
-describe SimpleQueues::Redis, "enqueue" do
   let :queue do
-    SimpleQueues::Redis.new
+    SimpleQueues::Redis.new(redis)
   end
 
   before(:each) do
-    queue.clear(:test)
+    redis.flushdb
   end
 
-  it "communicates with Redis as expected" do
-    obj = {"hello" => 42}
-    queue.enqueue(:test, obj)
-    queue.dequeue_with_timeout(:test, 1).should == obj
-    queue.dequeue_with_timeout(:test, 1).should == nil
-  end
+  context "given an empty queue" do
+    context "#size" do
+      it { queue.size(queue_name).should == 0 }
+    end
+
+    context "#clear" do
+      it { lambda { queue.clear(queue_name) }.should_not raise_error }
+    end
 
-  it "clears the queue" do
-    queue.enqueue(:test, :yada => "whatever")
-    queue.clear(:test)
-    queue.size(:test).should be_zero
+    context "#dequeue_with_timeout" do
+      it { queue.dequeue_with_timeout(queue_name, 1).should == nil }
+    end
   end
 
-  it "returns the queue size" do
-    queue.enqueue(:test, 42)
-    queue.enqueue(:test, 42)
-    queue.enqueue(:test, 42)
-    queue.enqueue(:test, 42)
-    queue.size(:test).should == 4
+  context "given a queue with one message" do
+    let :message do
+      { "value" => Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S") }
+    end
+
+    before(:each) do
+      queue.enqueue(queue_name, message)
+    end
+
+    context "#size" do
+      it { queue.size(queue_name).should == 1 }
+    end
+
+    context "#dequeue_with_timeout" do
+      it { queue.dequeue_with_timeout(queue_name, 1).should == message }
+    end
+
+    context "#clear" do
+      before(:each) do
+        queue.clear(queue_name)
+      end
+
+      it "should remove all waiting messages" do
+        queue.size(queue_name).should == 0
+      end
+    end
   end
 end

data/spec/redis_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+
+describe SimpleQueues::Redis, "#initialize" do
+  it "should default to the JSON encoder" do
+    subject.encoder.should be_a(SimpleQueues::JsonEncoder)
+  end
+
+  it "should use the MessagePack encoder when passing :encoder => :messagepack" do
+    SimpleQueues::Redis.new(double("redis"), :encoder => :msgpack).encoder.should be_a(SimpleQueues::MessagePackEncoder)
+  end
+
+  it "should use the MessagePack encoder when passing :encoder => :messagepack" do
+    SimpleQueues::Redis.new(double("redis"), :encoder => :messagepack).encoder.should be_a(SimpleQueues::MessagePackEncoder)
+  end
+
+  it "should use the Identity encoder when passing :encoder => :identity" do
+    SimpleQueues::Redis.new(double("redis"), :encoder => :identity).encoder.should be_a(SimpleQueues::IdentityEncoder)
+  end
+
+  it "should use the JSON encoder when passing :encoder => :json" do
+    SimpleQueues::Redis.new(double("redis"), :encoder => :json).encoder.should be_a(SimpleQueues::JsonEncoder)
+  end
+
+  it "should use whatever encoder is sent" do
+    encoder = double("encoder", :encode => nil, :decode => nil)
+    SimpleQueues::Redis.new(double("redis"), :encoder => encoder).encoder.should be(encoder)
+  end
+end

data/spec/spec_helper.rb

@@ -1 +1,57 @@
 require "simple_queues"
+
+class DecoderMatcher
+  def initialize(message)
+    @message = message
+  end
+
+  def as(expected)
+    @expected = expected
+    self
+  end
+
+  def matches?(decoder)
+    @actual = decoder.decode(@message)
+    @expected == @actual
+  end
+
+  def failure_message_for_should
+    "expected decoded message <#{@actual.inspect} to equal <#{@expected.inspect}>"
+  end
+
+  def description
+    "decode #{@message.inspect}"
+  end
+end
+
+def decode(message)
+  DecoderMatcher.new(message)
+end
+
+class EncoderMatcher
+  def initialize(message)
+    @message = message
+  end
+
+  def as(expected)
+    @expected = expected
+    self
+  end
+
+  def matches?(encoder)
+    @actual = encoder.encode(@message)
+    @expected == @actual
+  end
+
+  def failure_message_for_should
+    "expected encoded message <#{@actual.inspect} to equal <#{@expected.inspect}>"
+  end
+
+  def description
+    "encode #{@message.inspect}"
+  end
+end
+
+def encode(message)
+  EncoderMatcher.new(message)
+end

metadata

@@ -2,7 +2,7 @@
 name: simple_queues
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 1.1.1
+  version: 1.2.0
 platform: ruby
 authors: 
 - "Fran\xC3\xA7ois Beausoleil"
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-03-24 00:00:00 -04:00
+date: 2011-03-25 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -25,7 +25,7 @@ dependencies:
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: json
+  name: rspec
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
@@ -33,10 +33,10 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version 
         version: "0"
-  type: :runtime
+  type: :development
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: rspec
+  name: yard
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
@@ -47,7 +47,7 @@ dependencies:
   type: :development
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: yard
+  name: RedCloth
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
@@ -58,7 +58,7 @@ dependencies:
   type: :development
   version_requirements: *id004
 - !ruby/object:Gem::Dependency 
-  name: RedCloth
+  name: ruby-debug19
   prerelease: false
   requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
@@ -69,7 +69,7 @@ dependencies:
   type: :development
   version_requirements: *id005
 - !ruby/object:Gem::Dependency 
-  name: ruby-debug19
+  name: msgpack
   prerelease: false
   requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
@@ -79,6 +79,17 @@ dependencies:
         version: "0"
   type: :development
   version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  version_requirements: *id007
 description: Program to an interface, not an implementation - hides Redis (used as a queue) behind a simple interface
 email: 
 - francois@teksol.info
@@ -90,6 +101,7 @@ extra_rdoc_files: []
 
 files: 
 - .gitignore
+- .rspec
 - .rvmrc
 - Gemfile
 - LICENSE
@@ -97,13 +109,20 @@ files:
 - Rakefile
 - autotest/discover.rb
 - lib/simple_queues.rb
+- lib/simple_queues/encoders/identity.rb
+- lib/simple_queues/encoders/json.rb
+- lib/simple_queues/encoders/message_pack.rb
 - lib/simple_queues/redis.rb
 - lib/simple_queues/version.rb
 - simple_queues.gemspec
 - spec/dequeueing_spec.rb
 - spec/enqueueing_spec.rb
+- spec/identity_encoder_spec.rb
+- spec/json_encoder_spec.rb
+- spec/message_pack_encoder_spec.rb
 - spec/multi_dequeue_spec.rb
 - spec/redis_integration_spec.rb
+- spec/redis_spec.rb
 - spec/spec_helper.rb
 has_rdoc: true
 homepage: https://github.com/francois/simple_queues
@@ -136,6 +155,10 @@ summary: Simple enqueue/dequeue API for working with queues
 test_files: 
 - spec/dequeueing_spec.rb
 - spec/enqueueing_spec.rb
+- spec/identity_encoder_spec.rb
+- spec/json_encoder_spec.rb
+- spec/message_pack_encoder_spec.rb
 - spec/multi_dequeue_spec.rb
 - spec/redis_integration_spec.rb
+- spec/redis_spec.rb
 - spec/spec_helper.rb