timberline 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9b3f445273cef8a33fd4137ad6cffdb91c5af8f5
-  data.tar.gz: 049bab1a641c24643642554ee8f1c89e4fd9c78c
+  metadata.gz: 7aa514d129ba4ac74886845982675b82975051bb
+  data.tar.gz: d668a59889b0f78d60e04321b2a0224c1264712e
 SHA512:
-  metadata.gz: cb803ba88627fffc8e1af3ec9253733b584601397fd08d3701e02ace865a0887c9ac0219da754ff4539d0798f923f9c653db8d28e1711b94371f0ffb437b6bdc
-  data.tar.gz: 9d17830ab88a54d2bdde1fe6f0ea02fad125570bf36a9b03bb7aebde17c75a23e21b371df967cb010912a5769eaa66524cd405e22b20b938e8a1ba91799dd6b0
+  metadata.gz: 3f44d9c9e5f008d8b01e0ee8e008a6e4adeb22ca4db90dfffde5b5e3a2aa263976cc0d89f4a82217a639a2dbfafcefb7bcd9864f0df86c5178fe65452490a36b
+  data.tar.gz: 5ff02917829e79ddf5d7f8e0d9e99a35bd35df784893a54773dcde8878f351633bad12de48587e8a5dbac62fd9421c3a5fb1701ddfde36e5e029b8ebd6771216

data/CHANGELOG

@@ -0,0 +1,24 @@
+0.5.0
+  - Each queue now has its own error queue, rather than having one error_queue for everything
+  - Queues can now be created with a 'hidden' option that removes them from the Timberline listing
+0.4.1
+  - Fix a bug with calculating average_execution_time statistics
+0.4.0
+  - Test suite moved over to RSpec
+  - CI added via Travis
+  - Queue#delete! renamed to Queue#delete in accordance with ruby convention
+  - Timberline.config renamed to Timberline.configure to avoid collision with the attr_accessor
+0.3.1
+  - Fix a bug with handling errors while logging success stats
+0.3.0
+  - Rails compatability updated, support for Rails 2 removed
+0.2.1
+  - Tests switched over to nutrasuite
+  - Rails 3 compatability added
+0.2.0
+  - Added ability to pause queues
+  - Added statistics logging for successes/errors/retries
+0.1.2
+  - Rename Treeline to Timberline due to gem name conflict
+0.1.1
+  - First public release of Treeline

data/README.markdown

@@ -139,12 +139,11 @@ to note:
   retry the job in the event of a transient error, or put it on the error queue
   for processing if a more fatal error occurs.
 
-### The error queue
+### Error queues
 
-If you want to interact with the error queue directly, it's accessible via
-`Timberline#error_queue`. You can pop items directly off of the queue to operate
-on them if you want, or you could write a queue processor that reads off of that
-queue (its queue name should always be "Timberline\_errors").
+Each queue has its own error queue, accessible through `Queue#error_queue`.
+You can pop items directly off of the queue to operate on them if you want, or
+you could write a queue processor that reads off of that queue.
 
 ### Using the binary
 
@@ -172,19 +171,17 @@ There are some options to the Timberline binary that you may find helpful -
 
 Still to be done:
 
-- A simple Sinatra interface for monitoring the statuses of queues and
+- **Monitor** - A simple Sinatra interface for monitoring the statuses of queues and
   observing/resubmitting errored-out jobs.
-- Binary updates - the binary should probably fork processes for each job 
-  that it tries to process so that it's more robust.
-- DSL improvements - the DSL-ish setup for Timberline could probably use some
-  updates to be both more obvious and easier to use.
-- Documentation - need to get YARD docs added so that the API is more completely
+- **Documentation** - need to get Tomdoc added so that the API is more completely
   documented. For the time being, though, there are some fairly comprehensive
   test suites.
-- Timing - it would be crazy useful to be able to automatically log per-queue
-  statistics about how long jobs are taking. Definitely something like an "over
-  the last 5 minutes/past 1000 jobs" stat would be useful, but we may also be
-  interested in some kind of lifetime average.
+- **Refactor** - the singleton model made sense at some point for Timberline but now it's
+  cumbersome. Need to rewrite some of the basic stuff to be more OO-appropriate.
+- **Forking** - Timberline should support forking in its watcher model so that jobs can
+  be run in parallel and independently of each other.
+- **Relocate Rails** - Create a `timberline-rails` gem that includes all of our Rails-specific
+  functionality and configuration rather than have that in Timberline proper.
 
 ## Future
 

data/lib/timberline/queue.rb

@@ -2,14 +2,18 @@ class Timberline
   class Queue
     attr_reader :queue_name, :read_timeout
 
-    def initialize(queue_name, read_timeout= 0)
+    def initialize(queue_name, opts = {})
+      read_timeout = opts.fetch(:read_timeout, 0)
+      hidden = opts.fetch(:hidden, false)
       if queue_name.nil?
         raise ArgumentError.new("Queue name must be provided.")
       end
       @queue_name = queue_name
       @read_timeout = read_timeout
       @redis = Timberline.redis
-      @redis.sadd "timberline_queue_names", queue_name
+      unless hidden
+        @redis.sadd "timberline_queue_names", queue_name
+      end
     end
 
     def delete
@@ -92,12 +96,29 @@ class Timberline
       end
     end
 
+    def retry_item(item)
+      if (item.retries < Timberline.max_retries)
+        item.retries += 1
+        item.last_tried_at = Time.now.to_f
+        add_retry_stat(item)
+        push(item)
+      else
+        error_item(item)
+      end
+    end
+
+    def error_item(item)
+      item.fatal_error_at = Time.now.to_f
+      add_error_stat(item)
+      self.error_queue.push(item)
+    end
+
     def add_retry_stat(item)
       add_stat_for_key(attr("retry_stats"), item)
     end
 
     def add_error_stat(item)
-      add_stat_for_key(attr("success_stats"), item)
+      add_stat_for_key(attr("error_stats"), item)
     end
 
     def add_success_stat(item)
@@ -106,6 +127,10 @@ class Timberline
       $stderr.puts "Success Stat Error: #{e.inspect}, Item: #{item.inspect}"
     end
 
+    def error_queue
+      @error_queue ||= Timberline.queue(attr("errors"), hidden: true)
+    end
+
     private
 
     def add_stat_for_key(key, item)

data/lib/timberline/version.rb

@@ -1,3 +1,3 @@
 class Timberline
-  VERSION = "0.4.1"
+  VERSION = "0.5.0"
 end

data/lib/timberline.rb

@@ -43,12 +43,8 @@ class Timberline
     Timberline.redis.smembers("timberline_queue_names").map { |name| queue(name) }
   end
 
-  def self.error_queue
-    @error_queue ||= Queue.new("timberline_errors")
-  end
-
-  def self.queue(queue_name)
-    Queue.new(queue_name)
+  def self.queue(queue_name, opts = {})
+    Queue.new(queue_name, opts)
   end
 
   def self.push(queue_name, data, metadata={})
@@ -56,18 +52,13 @@ class Timberline
   end
 
   def self.retry_item(item)
-    if (item.retries < max_retries)
-      item.retries += 1
-      item.last_tried_at = Time.now.to_f
-      queue(item.origin_queue).push(item)
-    else
-      error_item(item)
-    end
+    origin_queue = queue(item.origin_queue)
+    origin_queue.retry_item(item)
   end
 
   def self.error_item(item)
-    item.fatal_error_at = Time.now.to_f
-    error_queue.push(item)
+    origin_queue = queue(item.origin_queue)
+    origin_queue.error_item(item)
   end
 
   def self.pause(queue_name)

data/spec/lib/timberline/queue_spec.rb

@@ -11,7 +11,17 @@ describe Timberline::Queue do
     end
 
     it "allows you to override the default read_timeout" do
-      expect(Timberline::Queue.new("fritters", 50).read_timeout).to eq(50)
+      expect(Timberline::Queue.new("fritters", read_timeout: 50).read_timeout).to eq(50)
+    end
+
+    context "if the queue is marked as hidden" do
+      before do
+        Timberline::Queue.new("fritters", hidden: true)
+      end
+
+      it "Doesn't add it to the queue listing in Timberline" do
+        expect(Timberline.all_queues.map { |q| q.queue_name }).not_to include("fritters")
+      end
     end
 
     it "adds the queue's name to the Timberline queue listing" do
@@ -97,4 +107,78 @@ describe Timberline::Queue do
     end
 
   end
+
+  describe "#error_item" do
+    subject    { Timberline::Queue.new("fritters") }
+    let(:item) { subject.push("apple"); subject.pop }
+
+    before do
+      subject.error_item(item)
+    end
+
+    it "updates the fatal_error_at timestamp for the item" do
+      expect(item.fatal_error_at).not_to be_nil
+    end
+
+    it "puts the item on the queue's error queue" do
+      expect(subject.error_queue.length).to eq(1)
+    end
+
+    it "adds an error statistic" do
+      expect(subject.number_errors).to eq(1)
+    end
+  end
+
+  describe "#retry_item" do
+    subject    { Timberline::Queue.new("fritters") }
+    let(:item) { subject.push("apple"); subject.pop }
+
+    context "when the item hasn't been retried before" do
+      before do
+        subject.retry_item(item)
+      end
+
+      it "adds a retry statistic" do
+        expect(subject.number_retries).to eq(1)
+      end
+
+      it "puts the item back on the queue" do
+        expect(subject.length).to eq(1)
+      end
+
+      it "updates the last_tried_at property on the item" do
+        expect(item.last_tried_at).not_to be_nil
+      end
+    end
+
+    context "when the item has been retried before" do
+      before do
+        item.retries = 3
+        subject.retry_item(item)
+      end
+
+      it "adds a retry statistic" do
+        expect(subject.number_retries).to eq(1)
+      end
+
+      it "puts the item back on the queue" do
+        expect(subject.length).to eq(1)
+      end
+
+      it "updates the last_tried_at property on the item" do
+        expect(item.last_tried_at).not_to be_nil
+      end
+    end
+
+    context "when the item has been retried the maximum number of times" do
+      before do
+        item.retries = 5
+      end
+
+      it "defers to #error_item" do
+        expect(subject).to receive(:error_item).with(item)
+        subject.retry_item(item)
+      end
+    end
+  end
 end

data/spec/lib/timberline_spec.rb

@@ -91,16 +91,6 @@ describe Timberline do
     end
   end
 
-  describe ".error_queue" do
-    it "returns a Timberline::Queue" do
-      expect(Timberline.error_queue).to be_a Timberline::Queue
-    end
-
-    it "returns the timberline_errors queue" do
-      expect(Timberline.error_queue.queue_name).to eq("timberline_errors")
-    end
-  end
-
   describe ".queue" do
     it "returns a Timberline::Queue" do
       expect(Timberline.queue("fritters")).to be_a Timberline::Queue
@@ -166,8 +156,8 @@ describe Timberline do
         item.retries = 5
       end
 
-      it "passes the item on to the error handling" do
-        expect(Timberline).to receive(:error_item).with(item)
+      it "passes the item on to the origin queue for retry" do
+        expect_any_instance_of(Timberline::Queue).to receive(:retry_item).with(item)
         Timberline.retry_item(item)
       end
     end
@@ -175,17 +165,12 @@ describe Timberline do
 
   describe ".error_item" do
     let(:queue) { Timberline.queue("test_queue") }
-    let(:error_queue) { Timberline.error_queue }
+    let(:error_queue) { queue.error_queue }
     let(:item) { Timberline.push(queue.queue_name, "Howdy kids."); queue.pop }
 
-    before { Timberline.error_item(item) }
-
-    it "puts the item on the error_queue" do
-      expect(error_queue.length).to eq(1) 
-    end
-
-    it "updates the fatal_error_at property on the item" do
-      expect(item.fatal_error_at).not_to be_nil
+    it "passes the item on to the queue for error handling" do
+      expect_any_instance_of(Timberline::Queue).to receive(:error_item).with(item)
+      Timberline.error_item(item)
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: timberline
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Tommy Morgan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-02 00:00:00.000000000 Z
+date: 2014-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -137,6 +137,7 @@ files:
 - .rspec
 - .ruby-gemset
 - .travis.yml
+- CHANGELOG
 - Gemfile
 - README.markdown
 - Rakefile