timberline 0.1.2 → 0.2.0

data/Rakefile

@@ -5,5 +5,4 @@ Rake::TestTask.new do |t|
   t.libs << "test"
   t.test_files = FileList['test/unit/test_*.rb']
   t.verbose = true
-  t.warning = true
 end

data/lib/timberline.rb

@@ -1,22 +1,18 @@
-require 'forwardable'
 require 'json'
 require 'logger'
 require 'yaml'
 
 require 'redis'
 require 'redis-namespace'
+require 'redis-expiring-set/monkeypatch'
 
 require_relative "timberline/version"
 require_relative "timberline/config"
 require_relative "timberline/queue"
 require_relative "timberline/envelope"
-require_relative "timberline/queue_manager"
 
 class Timberline
   class << self
-    extend Forwardable
-
-    def_delegators :@queue_manager, :error_job, :retry_job, :watch, :push
 
     attr_reader :config
 
@@ -38,9 +34,54 @@ class Timberline
       if @redis.nil?
         self.redis = Redis.new(@config.redis_config)
       end
+
       @redis
     end
 
+    def all_queues
+      Timberline.redis.smembers("timberline_queue_names").map { |name| queue(name) }
+    end
+
+    def error_queue
+      @error_queue ||= Queue.new("timberline_errors")
+    end
+
+    def queue(queue_name)
+      if @queue_list[queue_name].nil?
+        @queue_list[queue_name] = Queue.new(queue_name)
+      end
+      @queue_list[queue_name]
+    end
+
+    def push(queue_name, data, metadata={})
+      queue(queue_name).push(data, metadata)
+    end
+
+    def retry_item(item)
+      if (item.retries < max_retries)
+        item.retries += 1
+        item.last_tried_at = Time.now.to_f
+        queue(item.origin_queue).push(item)
+        raise ItemRetried
+      else
+        error_item(item)
+      end
+    end
+
+    def error_item(item)
+      item.fatal_error_at = Time.now.to_f
+      error_queue.push(item)
+      raise ItemErrored
+    end
+
+    def pause(queue_name)
+      queue(queue_name).pause
+    end
+
+    def unpause(queue_name)
+      queue(queue_name).unpause
+    end
+
     def config(&block)
       initialize_if_necessary
       yield @config
@@ -51,6 +92,36 @@ class Timberline
       @config.max_retries
     end
 
+    def stat_timeout
+      initialize_if_necessary
+      @config.stat_timeout
+    end
+
+    def stat_timeout_seconds
+      initialize_if_necessary
+      @config.stat_timeout * 60
+    end
+
+    def watch(queue_name, &block)
+      queue = queue(queue_name)
+      while(true)
+        item = queue.pop
+        fix_binding(block)
+        item.started_processing_at = Time.now.to_f
+
+        begin
+          block.call(item, self)
+        rescue ItemRetried
+          queue.add_retry_stat(item)
+        rescue ItemErrored
+          queue.add_error_stat(item)
+        else
+          item.finished_processing_at = Time.now.to_f
+          queue.add_success_stat(item)
+        end
+      end
+    end
+
     private
     # Don't know if I like doing this, but we want the configuration to be
     # lazy-loaded so as to be sure and give users a chance to set up their
@@ -58,8 +129,32 @@ class Timberline
     def initialize_if_necessary
       @config ||= Config.new
     end
+
+    # Hacky-hacky. I like the idea of calling retry_item(item) and
+    # error_item(item)
+    # directly from the watch block, but this seems ugly. There may be a better
+    # way to do this.
+    def fix_binding(block)
+      binding = block.binding
+      binding.eval <<-HERE
+        def retry_item(item)
+          Timberline.retry_item(item)
+          raise Timberline::ItemRetried
+        end
+
+        def error_item(item)
+          Timberline.error_item(item)
+          raise Timberline::ItemErrored
+        end
+      HERE
+    end
+  end
+
+  class ItemRetried < Exception
   end
 
+  class ItemErrored < Exception
+  end
 end
 
-Timberline.instance_variable_set("@queue_manager", Timberline::QueueManager.new)
+Timberline.instance_variable_set("@queue_list", {})

data/lib/timberline/config.rb

@@ -1,6 +1,6 @@
 class Timberline
   class Config
-    attr_accessor :database, :host, :port, :timeout, :password, :logger, :namespace, :max_retries
+    attr_accessor :database, :host, :port, :timeout, :password, :logger, :namespace, :max_retries, :stat_timeout
 
     def initialize
       if defined? TIMBERLINE_YAML
@@ -15,14 +15,11 @@ class Timberline
           load_from_yaml(config_file)
         end
       end
-    end
-
-    def namespace
-      @namespace ||= 'timberline'
-    end
 
-    def max_retries
+      # load defaults
+      @namespace  ||= 'timberline'
       @max_retries ||= 5
+      @stat_timeout ||= 60
     end
 
     def redis_config

data/lib/timberline/queue.rb

@@ -6,6 +6,15 @@ class Timberline
       @queue_name = queue_name
       @read_timeout = read_timeout
       @redis = Timberline.redis
+      @redis.sadd "timberline_queue_names", queue_name
+    end
+
+    def delete!
+      @redis.del @queue_name
+      @redis.keys("#{@queue_name}:*").each do |key|
+        @redis.del key
+      end
+      @redis.srem "timberline_queue_names", @queue_name
     end
 
     def length
@@ -13,6 +22,8 @@ class Timberline
     end
 
     def pop
+      block_while_paused
+
       br_tuple = @redis.brpop(@queue_name, read_timeout)
       envelope_string = br_tuple.nil? ? nil : br_tuple[1]
       if envelope_string.nil?
@@ -22,21 +33,91 @@ class Timberline
       end
     end
 
-    def push(item)
-      case item
+    def push(contents, metadata = {})
+      case contents
       when Envelope
-        @redis.lpush @queue_name, item
+        @redis.lpush @queue_name, contents
       else
-        @redis.lpush @queue_name, wrap(item)
+        @redis.lpush @queue_name, wrap(contents, metadata)
       end
     end
 
+    def pause
+      @redis[attr("paused")] = "true"
+    end
+
+    def unpause
+      @redis[attr("paused")] = "false"
+    end
+
+    def paused?
+      @redis[attr("paused")] == "true"
+    end
+
+    def attr(key)
+      "#{@queue_name}:#{key}"
+    end
+
+    def number_errors
+      Timberline.redis.xcard attr("error_stats")
+    end
+
+    def number_retries
+      Timberline.redis.xcard attr("retry_stats")
+    end
+
+    def number_successes
+      Timberline.redis.xcard attr("success_stats")
+    end
+
+    def average_execution_time
+      successes = Timberline.redis.xmembers(attr("success_stats")).map { |item| Envelope.from_json(item)}
+      times = successes.map { |item| item.finished_processing_at.to_f - item.started_processing_at.to_f }
+      times.inject(0, :+) / times.size.to_f
+    end
+
+    def add_retry_stat(item)
+      add_stat_for_key(attr("retry_stats"), item)
+    end
+
+    def add_error_stat(item)
+      add_stat_for_key(attr("success_stats"), item)
+    end
+
+    def add_success_stat(item)
+      add_stat_for_key(attr("success_stats"), item)
+    end
+
     private
 
-    def wrap(item)
+    def add_stat_for_key(key, item)
+      Timberline.redis.xadd key, item, Time.now + Timberline.stat_timeout_seconds
+    end
+
+    def next_id
+      Timberline.redis.incr attr("id_seq")
+    end
+
+    def wrap(contents, metadata)
       envelope = Envelope.new
-      envelope.contents = item
+      envelope.contents = contents
+      metadata.each do |key, value|
+        envelope.send("#{key.to_s}=", value)
+      end
+
+      # default metadata
+      envelope.item_id = next_id
+      envelope.retries = 0
+      envelope.submitted_at = Time.now.to_f
+      envelope.origin_queue = @queue_name
+
       envelope
     end
+
+    def block_while_paused
+      while(self.paused?)
+        sleep(1)
+      end
+    end
   end
 end

data/lib/timberline/version.rb

@@ -1,3 +1,3 @@
 class Timberline
-  VERSION = "0.1.2"
+  VERSION = "0.2.0"
 end

data/test/test_helper.rb

@@ -1,10 +1,18 @@
 require 'test/unit'
 
-require 'partial_minispec'
+require 'shoulda'
 
 # include the gem
 require 'timberline'
 
+def reset_timberline
+  Timberline.redis = nil
+  Timberline.instance_variable_set("@config", nil)
+  clear_test_db
+  Timberline.redis = nil
+  Timberline.instance_variable_set("@queue_list", {})
+end
+
 # Use database 15 for testing, so we don't risk overwriting any data that's
 # actually useful
 def clear_test_db
@@ -13,3 +21,4 @@ def clear_test_db
   end
   Timberline.redis.flushdb
 end
+

data/test/unit/test_config.rb

@@ -1,12 +1,12 @@
 require 'test_helper'
 
-describe Timberline::Config do
-  describe "Without any preset YAML configs" do
-    before do
+class ConfigTest < Test::Unit::TestCase
+  context "Without any preset YAML configs" do
+    setup do
       @config = Timberline::Config.new
     end
 
-    it "builds a proper config hash for Redis" do
+    should "build a proper config hash for Redis" do
       @logger = Logger.new STDERR
 
       @config.host = "localhost"
@@ -27,7 +27,7 @@ describe Timberline::Config do
 
     end
 
-    it "reads configuration from a YAML config file" do
+    should "reads configuration from a YAML config file" do
       base_dir = File.dirname(File.path(__FILE__))
       yaml_file = File.join(base_dir, "..", "test_config.yaml")
       @config.load_from_yaml(yaml_file)
@@ -40,34 +40,39 @@ describe Timberline::Config do
     end
   end
 
-  describe "when in a Rails app without a config file" do
-    before do
-      RAILS_ROOT = File.join(File.dirname(File.path(__FILE__)), "..", "gibberish")
+  context "when in a Rails app without a config file" do
+    setup do
+      Object::RAILS_ROOT = File.join(File.dirname(File.path(__FILE__)), "..", "gibberish")
       @config = Timberline::Config.new
     end
 
-    after do
+    teardown do
       Object.send(:remove_const, :RAILS_ROOT)
     end
 
-    it "should load successfully without any configs." do
-      ["database","host","port","timeout","password","logger","namespace"].each do |setting|
+    should "load successfully without any configs." do
+      ["database","host","port","timeout","password","logger"].each do |setting|
         assert_equal nil, @config.instance_variable_get("@#{setting}")
       end
+
+      # check defaults
+      assert_equal "timberline", @config.namespace
+      assert_equal 5, @config.max_retries
+      assert_equal 60, @config.stat_timeout
     end
   end
 
-  describe "when in a Rails app with a config file" do
-    before do
-      RAILS_ROOT = File.join(File.dirname(File.path(__FILE__)), "..", "fake_rails")
+  context "when in a Rails app with a config file" do
+    setup do
+      Object::RAILS_ROOT = File.join(File.dirname(File.path(__FILE__)), "..", "fake_rails")
       @config = Timberline::Config.new
     end
 
-    after do
+    teardown do
       Object.send(:remove_const, :RAILS_ROOT)
     end
 
-    it "should load the config/timberline.yaml file" do
+    should "load the config/timberline.yaml file" do
       assert_equal "localhost", @config.host
       assert_equal 12345, @config.port
       assert_equal 10, @config.timeout
@@ -77,17 +82,17 @@ describe Timberline::Config do
     end
   end
 
-  describe "when TIMBERLINE_YAML is defined" do
-    before do
-      TIMBERLINE_YAML = File.join(File.dirname(File.path(__FILE__)), "..", "test_config.yaml")
+  context "when TIMBERLINE_YAML is defined" do
+    setup do
+      Object::TIMBERLINE_YAML = File.join(File.dirname(File.path(__FILE__)), "..", "test_config.yaml")
       @config = Timberline::Config.new
     end
 
-    after do
+    teardown do
       Object.send(:remove_const, :TIMBERLINE_YAML)
     end
 
-    it "should load the specified yaml file" do
+    should "load the specified yaml file" do
       assert_equal "localhost", @config.host
       assert_equal 12345, @config.port
       assert_equal 10, @config.timeout
@@ -97,16 +102,16 @@ describe Timberline::Config do
     end
   end
 
-  describe "when TIMBERLINE_YAML is defined, but doesn't exist" do
-    before do
-      TIMBERLINE_YAML = File.join(File.dirname(File.path(__FILE__)), "..", "fake_config.yaml")
+  context "when TIMBERLINE_YAML is defined, but doesn't exist" do
+    setup do
+      Object::TIMBERLINE_YAML = File.join(File.dirname(File.path(__FILE__)), "..", "fake_config.yaml")
     end
 
-    after do
+    teardown do
       Object.send(:remove_const, :TIMBERLINE_YAML)
     end
 
-    it "should raise an exception" do
+    should "raise an exception" do
       assert_raises RuntimeError do 
         @config = Timberline::Config.new
       end

data/test/unit/test_envelope.rb

@@ -1,51 +1,53 @@
 require 'test_helper'
 require 'date'
 
-describe Timberline::Envelope do
-  before do
-    @envelope = Timberline::Envelope.new
-  end
+class EnvelopeTest < Test::Unit::TestCase
+
+  context "newly instantiated" do
+    setup do
+      @envelope = Timberline::Envelope.new
+    end
 
-  describe "newly instantiated" do
-    it "raises a MissingContentException when to_s is called because the contents are nil" do
+    should "raises a MissingContentException when to_s is called because the contents are nil" do
       assert_raises Timberline::MissingContentException do
         @envelope.to_s
       end
     end
 
-    it "has an empty hash for metadata" do
+    should "has an empty hash for metadata" do
       assert_equal({}, @envelope.metadata)
     end
 
-    it "allows for the reading of attributes via method_missing magic" do
+    should "allows for the reading of attributes via method_missing magic" do
       @envelope.metadata["original_queue"] = "test_queue"
       assert_equal "test_queue", @envelope.original_queue
     end
 
-    it "allows for the setting of attributes via method_missing magic" do
+    should "allows for the setting of attributes via method_missing magic" do
       @envelope.original_queue = "test_queue"
       assert_equal "test_queue", @envelope.metadata["original_queue"]
     end
   end
 
-  describe "with contents" do
-    before do
+  context "with contents" do
+    setup do
+      @envelope = Timberline::Envelope.new
       @envelope.contents = "Test data"
     end
 
-    it "returns a JSON string when to_s is called" do
+    should "returns a JSON string when to_s is called" do
       json_string = @envelope.to_s
       json_data = JSON.parse(json_string)
       assert_equal "Test data", json_data["contents"]
     end
 
-    it "only includes a 'contents' parameter by default" do
+    should "only includes a 'contents' parameter by default" do
       json_string = @envelope.to_s
       json_data = JSON.parse(json_string)
       assert_equal 1, json_data.keys.size
     end
 
-    it "also includes metadata, if provided" do
+    should "also includes metadata, if provided" do
       time = DateTime.now
       time_s = DateTime.now.to_s
       @envelope.first_posted = time
@@ -57,7 +59,7 @@ describe Timberline::Envelope do
       assert_equal time_s, json_data["first_posted"]
     end
 
-    it "parses itself back correctly using from_json" do
+    should "parses itself back correctly using from_json" do
       json_string = @envelope.to_s
       new_envelope = Timberline::Envelope.from_json(json_string)
       assert_equal @envelope.contents, new_envelope.contents

data/test/unit/test_queue.rb

@@ -1,36 +1,51 @@
 require 'test_helper'
 
-describe Timberline::Queue do
-  describe "newly instantiated" do
-    before do
+class QueueTest < Test::Unit::TestCase
+  context "newly instantiated" do
+    setup do
       clear_test_db
       @queue = Timberline::Queue.new("test_queue")
     end
 
-    it "saves the passed-in string as its queue name" do
+    should "saves the passed-in string as its queue name" do
       assert_equal "test_queue", @queue.queue_name
     end
 
-    it "has a length of 0" do
+    should "saves its existence in timberline_queue_names" do
+      assert_equal true, Timberline.redis.sismember("timberline_queue_names", "test_queue")
+    end
+
+    should "has a length of 0" do
       assert_equal 0, @queue.length
     end
 
-    it "has a default read_timeout of 0" do
+    should "starts in an unpaused state" do
+      assert_equal false, @queue.paused?
+    end
+
+    should "allows itself to be paused and unpaused" do
+      @queue.pause
+      assert_equal true, @queue.paused?
+      @queue.unpause
+      assert_equal false, @queue.paused?
+    end
+
+    should "has a default read_timeout of 0" do
       assert_equal 0, @queue.read_timeout
     end
 
-    it "responds nil to a pop request after the read_timeout occurs" do
+    should "responds nil to a pop request after the read_timeout occurs" do
       # Let's set the read_timeout to 1 in order for this test to return
       @queue.instance_variable_set("@read_timeout", 1)
       assert_equal nil, @queue.pop
     end
 
-    it "puts an item on the queue when that item is pushed" do
+    should "puts an item on the queue when that item is pushed" do
       test_item = "Test Queue Item"
       assert_equal 1, @queue.push(test_item)
     end
 
-    it "wraps an item in an envelope when that item is pushed" do
+    should "wraps an item in an envelope when that item is pushed" do
       test_item = "Test Queue Item"
       assert_equal 1, @queue.push(test_item)
       data = @queue.pop
@@ -38,7 +53,7 @@ describe Timberline::Queue do
       assert_equal test_item, data.contents
     end
 
-    it "doesn't wrap an envelope that gets pushed in another envelope" do
+    should "doesn't wrap an envelope that gets pushed in another envelope" do
       test_item = "Test Queue Item"
       env = Timberline::Envelope.new
       env.contents = test_item
@@ -47,25 +62,49 @@ describe Timberline::Queue do
       assert_kind_of Timberline::Envelope, data
       assert_equal test_item, data.contents
     end
+
+    should "removes everything associated with the queue when delete! is called" do
+      test_item = "Test Queue Item"
+      assert_equal 1, @queue.push(test_item)
+      Timberline.redis[@queue.attr("test")] = "test"
+      Timberline.redis[@queue.attr("foo")] = "foo"
+      Timberline.redis[@queue.attr("bar")] = "bar"
+
+      @queue.delete!
+      assert_equal nil, Timberline.redis[@queue.attr("test")]
+      assert_equal nil, Timberline.redis[@queue.attr("test")]
+      assert_equal nil, Timberline.redis[@queue.attr("test")]
+      assert_equal nil, Timberline.redis[@queue.queue_name]
+      assert_equal false, Timberline.redis.sismember("timberline_queue_names","test_queue")
+    end
+
+    should "uses any passed-in metadata to push when building the envelope" do
+      @queue.push("Howdy kids.", { :special_notes => "Super-awesome."})
+      assert_equal 1, @queue.length
+      data = @queue.pop
+      assert_equal "Howdy kids.", data.contents
+      assert_equal "Super-awesome.", data.special_notes
+    end
+
   end
 
-  describe "with one item" do
-    before do
+  context "with one item" do
+    setup do
       clear_test_db
       @test_item = "Test Queue Item"
       @queue = Timberline::Queue.new("test_queue")
       @queue.push(@test_item)
     end
 
-    it "has a length of 1" do
+    should "has a length of 1" do
       assert_equal 1, @queue.length
     end
 
-    it "responds to pop with the one item" do
+    should "responds to pop with the one item" do
       assert_equal @test_item, @queue.pop.contents
     end
 
-    it "responds nil to a second pop" do
+    should "responds nil to a second pop" do
       # Let's set the read_timeout to 1 in order for this test to return
       @queue.instance_variable_set("@read_timeout", 1)
       assert_equal @test_item, @queue.pop.contents

data/test/unit/test_timberline.rb

@@ -1,65 +1,143 @@
 require 'test_helper'
 
-describe Timberline do
-  before do
-    # Reset the singleton
-    Timberline.redis = nil
-    Timberline.instance_variable_set("@config", nil)
-  end
+class TimberlineTest < Test::Unit::TestCase
+  context "Freshly set up" do
+    setup do
+      reset_timberline
+    end
 
-  it "saves a passed-in redis namespace" do
-    redis = Redis.new
-    redisns = Redis::Namespace.new("timberline", redis)
-    Timberline.redis = redisns
-    assert_equal redisns, Timberline.redis
-  end
+    should "creates a new queue when asked if it doesn't exist" do
+      queue = Timberline.queue("test_queue")
+      assert_kind_of Timberline::Queue, queue
+      assert_equal queue, Timberline.instance_variable_get("@queue_list")["test_queue"]
+    end
 
-  it "Converts a standard redis server into a namespace" do
-    redis = Redis.new
-    Timberline.redis = redis
-    assert_equal Redis::Namespace, Timberline.redis.class
-    assert_equal redis, Timberline.redis.instance_variable_get("@redis")
-  end
+    should "doesn't create a new queue if a queue by the same name already exists" do
+      queue = Timberline.queue("test_queue")
+      new_queue = Timberline.queue("test_queue")
+      assert_equal queue, new_queue
+    end
 
-  it "generates a redis namespace on request if one isn't present" do
-    assert_equal Redis::Namespace, Timberline.redis.class
-  end
+    should "creates a new queue as necessary when 'push' is called and pushes the item" do
+      Timberline.push("test_queue", "Howdy kids.")
+      queue = Timberline.queue("test_queue")
+      assert_equal 1, queue.length
+      assert_equal "Howdy kids.", queue.pop.contents
+    end
 
-  it "uses a default namespace of 'timberline'" do
-    assert_equal "timberline", Timberline.redis.namespace
-  end
+    should "logs the existence of the queue so that other managers can see it" do
+      queue = Timberline.queue("test_queue")
+      assert_equal 1, Timberline.all_queues.size
+      assert_equal "test_queue", Timberline.all_queues.first.queue_name
+    end
 
-  it "can be configured" do
-    Timberline.config do |c|
-      c.database = 3 
+    should "saves a passed-in redis namespace" do
+      redis = Redis.new
+      redisns = Redis::Namespace.new("timberline", redis)
+      Timberline.redis = redisns
+      assert_equal redisns, Timberline.redis
     end
 
-    assert_equal 3, Timberline.instance_variable_get("@config").database
-  end
+    should "Converts a standard redis server into a namespace" do
+      redis = Redis.new
+      Timberline.redis = redis
+      assert_equal Redis::Namespace, Timberline.redis.class
+      assert_equal redis, Timberline.redis.instance_variable_get("@redis")
+    end
 
-  it "uses a pre-defined namespace name if configured" do
-    Timberline.config do |c|
-      c.namespace = "skyline"
+    should "generates a redis namespace on request if one isn't present" do
+      assert_equal Redis::Namespace, Timberline.redis.class
     end
 
-    assert_equal "skyline", Timberline.redis.namespace
-  end
+    should "uses a default namespace of 'timberline'" do
+      assert_equal "timberline", Timberline.redis.namespace
+    end
+
+    should "can be configured" do
+      Timberline.config do |c|
+        c.database = 3 
+      end
+
+      assert_equal 3, Timberline.instance_variable_get("@config").database
+    end
 
-  it "properly configures Redis" do
-    @logger = Logger.new STDERR
-    Timberline.config do |c|
-      c.host = "localhost"
-      c.timeout = 10
-      c.database = 3 
-      c.logger = @logger
+    should "uses a pre-defined namespace name if configured" do
+      Timberline.config do |c|
+        c.namespace = "skyline"
+      end
+
+      assert_equal "skyline", Timberline.redis.namespace
     end
 
-    redis = Timberline.redis
+    should "properly configures Redis" do
+      @logger = Logger.new STDERR
+      Timberline.config do |c|
+        c.host = "localhost"
+        c.timeout = 10
+        c.database = 3 
+        c.logger = @logger
+      end
+
+      redis = Timberline.redis
+
+      assert_equal "localhost", redis.client.host
+      assert_equal 10, redis.client.timeout
+      assert_equal 3, redis.client.db
+      assert_equal @logger, redis.client.logger
+
+    end
+
+    should "allows you to retry a job that has failed" do
+      Timberline.push("test_queue", "Howdy kids.")
+      queue = Timberline.queue("test_queue")
+      data = queue.pop
+
+      assert_equal 0, queue.length
+
+      assert_raises Timberline::ItemRetried do
+        Timberline.retry_item(data)
+      end
+
+      assert_equal 1, queue.length
 
-    assert_equal "localhost", redis.client.host
-    assert_equal 10, redis.client.timeout
-    assert_equal 3, redis.client.db
-    assert_equal @logger, redis.client.logger
+      data = queue.pop
+      assert_equal 1, data.retries
+      assert_kind_of Time, Time.at(data.last_tried_at)
+    end
+
+    should "will continue retrying until we pass the max retries (defaults to 5)" do
+      Timberline.push("test_queue", "Howdy kids.")
+      queue = Timberline.queue("test_queue")
+      data = queue.pop
+
+      5.times do |i|
+        assert_raises Timberline::ItemRetried do
+          Timberline.retry_item(data)
+        end
+        assert_equal 1, queue.length
+        data = queue.pop
+        assert_equal i + 1, data.retries
+      end
+
+      assert_raises Timberline::ItemErrored do
+        Timberline.retry_item(data)
+      end
+      assert_equal 0, queue.length
+      assert_equal 1, Timberline.error_queue.length
+    end
+
+    should "will allow you to directly error out a job" do
+      Timberline.push("test_queue", "Howdy kids.")
+      queue = Timberline.queue("test_queue")
+      data = queue.pop
+
+      assert_raises Timberline::ItemErrored do
+        Timberline.error_item(data)
+      end
+      assert_equal 1, Timberline.error_queue.length
+      data = Timberline.error_queue.pop
+      assert_kind_of Time, Time.at(data.fatal_error_at)
+    end
 
   end
 end

data/timberline.gemspec

@@ -23,6 +23,9 @@ Gem::Specification.new do |s|
   # s.add_runtime_dependency "rest-client"
   s.add_runtime_dependency "redis"
   s.add_runtime_dependency "redis-namespace"
+  s.add_runtime_dependency "redis-expiring-set"
   s.add_runtime_dependency "trollop"
   s.add_runtime_dependency "daemons"
+
+  s.add_development_dependency "shoulda"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: timberline
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-09 00:00:00.000000000 Z
+date: 2012-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70136237495580 !ruby/object:Gem::Requirement
+  requirement: &70354056089500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70136237495580
+  version_requirements: *70354056089500
 - !ruby/object:Gem::Dependency
   name: redis-namespace
-  requirement: &70136237494860 !ruby/object:Gem::Requirement
+  requirement: &70354056077200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,21 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70136237494860
+  version_requirements: *70354056077200
+- !ruby/object:Gem::Dependency
+  name: redis-expiring-set
+  requirement: &70354056075820 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70354056075820
 - !ruby/object:Gem::Dependency
   name: trollop
-  requirement: &70136237493980 !ruby/object:Gem::Requirement
+  requirement: &70354056073880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70136237493980
+  version_requirements: *70354056073880
 - !ruby/object:Gem::Dependency
   name: daemons
-  requirement: &70136237492680 !ruby/object:Gem::Requirement
+  requirement: &70354056071600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,7 +65,18 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70136237492680
+  version_requirements: *70354056071600
+- !ruby/object:Gem::Dependency
+  name: shoulda
+  requirement: &70354056070520 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70354056070520
 description: Timberline is a simple and extensible queuing system built in Ruby and
   backed by Redis. It makes as few assumptions as possible about how you want to interact
   with your queues while also allowing for some functionality that should be universally
@@ -75,7 +97,6 @@ files:
 - lib/timberline/config.rb
 - lib/timberline/envelope.rb
 - lib/timberline/queue.rb
-- lib/timberline/queue_manager.rb
 - lib/timberline/version.rb
 - test/fake_rails/config/timberline.yaml
 - test/partial_minispec.rb
@@ -84,7 +105,6 @@ files:
 - test/unit/test_config.rb
 - test/unit/test_envelope.rb
 - test/unit/test_queue.rb
-- test/unit/test_queue_manager.rb
 - test/unit/test_timberline.rb
 - timberline.gemspec
 homepage: http://github.com/duwanis/timberline
@@ -120,5 +140,4 @@ test_files:
 - test/unit/test_config.rb
 - test/unit/test_envelope.rb
 - test/unit/test_queue.rb
-- test/unit/test_queue_manager.rb
 - test/unit/test_timberline.rb

data/lib/timberline/queue_manager.rb

@@ -1,84 +0,0 @@
-class Timberline
-  class QueueManager
-    attr_reader :queue_list
-
-    def initialize
-      @queue_list = {}
-    end
-
-    def error_queue
-      @error_queue ||= Queue.new("timberline_errors")
-    end
-
-    def queue(queue_name)
-      if @queue_list[queue_name].nil?
-        @queue_list[queue_name] = Queue.new(queue_name)
-      end
-      @queue_list[queue_name]
-    end
-
-    def push(queue_name, data, metadata={})
-      data = wrap(data, metadata.merge({:origin_queue => queue_name}))
-      queue(queue_name).push(data)
-    end
-
-    def retry_job(job)
-      if (job.retries < Timberline.max_retries)
-        job.retries += 1
-        job.last_tried_at = DateTime.now
-        queue(job.origin_queue).push(job)
-      else
-        error_job(job)
-      end
-    end
-
-    def error_job(job)
-      job.fatal_error_at = DateTime.now
-      error_queue.push(job)
-    end
-
-    def watch(queue_name, &block)
-      queue = queue(queue_name)
-      while(true)
-        job = queue.pop
-        fix_binding(block)
-        block.call(job, self)
-      end
-    end
-
-    private
-    def wrap(contents, metadata)
-      env = Envelope.new
-      env.contents = contents
-      metadata.each do |key, value|
-        env.send("#{key.to_s}=", value)
-      end
-
-      env.job_id = next_id_for_queue(metadata[:origin_queue])
-      env.retries = 0
-      env.submitted_at = DateTime.now
-
-      env
-    end
-
-    def next_id_for_queue(queue_name)
-      Timberline.redis.incr "#{queue_name}_id_seq"
-    end
-
-    # Hacky-hacky. I like the idea of calling retry_job(job) and error_job(job)
-    # directly from the watch block, but this seems ugly. There may be a better
-    # way to do this.
-    def fix_binding(block)
-      binding = block.binding
-      binding.eval <<-HERE
-        def retry_job(job)
-          Timberline.retry_job(job)
-        end
-
-        def error_job(job)
-          Timberline.error_job(job)
-        end
-      HERE
-    end
-  end
-end

data/test/unit/test_queue_manager.rb

@@ -1,154 +0,0 @@
-require 'test_helper'
-
-describe Timberline::QueueManager do
-  describe "newly instantiated" do
-    before do
-      clear_test_db
-      @qm = Timberline::QueueManager.new
-    end
-
-    it "has an empty queue list" do
-      assert_equal 0, @qm.queue_list.size
-    end
-
-    it "instantiates an error_queue" do
-      assert_kind_of Timberline::Queue, @qm.error_queue
-      assert_equal "timberline_errors", @qm.error_queue.queue_name
-    end
-
-    it "creates a new queue when asked if it doesn't exist" do
-      queue = @qm.queue("test_queue")
-      assert_kind_of Timberline::Queue, queue
-      assert_equal 1, @qm.queue_list.size
-      assert_equal queue, @qm.queue_list["test_queue"]
-    end
-
-    it "doesn't create a new queue if a queue by the same name already exists" do
-      queue = @qm.queue("test_queue")
-      new_queue = @qm.queue("test_queue")
-      assert_equal queue, new_queue
-      assert_equal 1, @qm.queue_list.size
-    end
-
-    it "creates a new queue as necessary when 'push' is called and pushes the item" do
-      @qm.push("test_queue", "Howdy kids.")
-      assert_equal 1, @qm.queue_list.size
-      queue = @qm.queue("test_queue")
-      assert_equal 1, queue.length
-      assert_equal "Howdy kids.", queue.pop.contents
-    end
-
-    it "uses any passed-in metadata to push when building the envelope" do
-      @qm.push("test_queue", "Howdy kids.", { :special_notes => "Super-awesome."})
-      queue = @qm.queue("test_queue")
-      assert_equal 1, queue.length
-      data = queue.pop
-      assert_equal "Howdy kids.", data.contents
-      assert_equal "Super-awesome.", data.special_notes
-    end
-
-    it "includes some default information in the metadata" do
-      @qm.push("test_queue", "Howdy kids.")
-      queue = @qm.queue("test_queue")
-      data = queue.pop
-      assert_equal "Howdy kids.", data.contents
-      assert_kind_of DateTime, DateTime.parse(data.submitted_at)
-      assert_equal "test_queue", data.origin_queue
-      assert_equal 0, data.retries
-      assert_equal 1, data.job_id
-    end
-
-    it "allows you to retry a job that has failed" do
-      @qm.push("test_queue", "Howdy kids.")
-      queue = @qm.queue("test_queue")
-      data = queue.pop
-
-      assert_equal 0, queue.length
-      
-      @qm.retry_job(data)
-
-      assert_equal 1, queue.length
-
-      data = queue.pop
-      assert_equal 1, data.retries
-      assert_kind_of DateTime, DateTime.parse(data.last_tried_at)
-    end
-
-    it "will continue retrying until we pass the max retries (defaults to 5)" do
-      @qm.push("test_queue", "Howdy kids.")
-      queue = @qm.queue("test_queue")
-      data = queue.pop
-
-      5.times do |i|
-        @qm.retry_job(data)
-        assert_equal 1, queue.length
-        data = queue.pop
-        assert_equal i + 1, data.retries
-      end
-
-      @qm.retry_job(data)
-      assert_equal 0, queue.length
-      assert_equal 1, @qm.error_queue.length
-    end
-
-    it "will allow you to directly error out a job" do
-      @qm.push("test_queue", "Howdy kids.")
-      queue = @qm.queue("test_queue")
-      data = queue.pop
-
-      @qm.error_job(data)
-      assert_equal 1, @qm.error_queue.length
-      data = @qm.error_queue.pop
-      assert_kind_of DateTime, DateTime.parse(data.fatal_error_at)
-    end
-
-    it "will allow you to watch a queue" do
-      @qm.push("test_queue", "Howdy kids.")
-      
-      assert_raises RuntimeError do
-        @qm.watch "test_queue" do |job|
-          if "Howdy kids." == job.contents
-            raise "This works."
-          end
-        end
-      end
-
-      assert_equal 0, @qm.queue("test_queue").length
-    end
-
-    it "will allow you to retry from a queue watcher" do
-      @qm.push("test_queue", "Howdy kids.")
-
-      assert_raises RuntimeError do
-        @qm.watch "test_queue" do |job|
-          if "Howdy kids." == job.contents
-            retry_job job
-            raise "Job retried."
-          end
-        end
-      end
-
-      assert_equal 1, @qm.queue("test_queue").length
-      data = @qm.queue("test_queue").pop
-      assert_equal 1, data.retries
-    end
-
-    it "will allow you to error from a queue watcher" do
-      @qm.push("test_queue", "Howdy kids.")
-
-      assert_raises RuntimeError do
-        @qm.watch "test_queue" do |job|
-          if "Howdy kids." == job.contents
-            error_job job
-            raise "Job errored."
-          end
-        end
-      end
-
-      assert_equal 1, @qm.error_queue.length
-      data = @qm.error_queue.pop
-      assert_kind_of DateTime, DateTime.parse(data.fatal_error_at)
-    end
-  end
-
-end