resqutils 0.0.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a63b6baf7b6534578500e949ccb71c6db6494794
-  data.tar.gz: 1a59736bd38007a380c5935e089125cbec529840
+  metadata.gz: 63c56aed110c280ee955278e9d007f242482e15e
+  data.tar.gz: 07eac3dc61ff36df991829d3002d805da88bd3d3
 SHA512:
-  metadata.gz: 16ca3aff7ae9da98e8a61295742e344fbbc76c139eeb309a13e0fd3a66c47644a7f6a4ec2396c242a10d460a13e6de91efbbe503f464ba82990174f1d8daa941
-  data.tar.gz: 627e1e5b9a6f9e60854a41ee261270c625392dfc446b5dfe20ee68510b37a85df47deb1c82a15fb6ac57b2c6da8e6b269a088943e8acb82843e264eb304c07aa
+  metadata.gz: 91fc61241a2cd1c0b623f9473fab38ecfcc19f1d7b83c92a1c5c91f72e62c8378ce614950e530f7b535987dec8f3029f62cef0c1e573d139f6d3e8dc885481c2
+  data.tar.gz: 2f5f870492b27271bd29ebc34e4b9d6f24c0117dcf443a479175ca545a514cb881352168e12095a807ce54670f8af685a6388bca85cdf18bb07c6d86fa7b90d7

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    resqutils (0.0.1)
+    resqutils (1.0.0)
       resque
 
 GEM

data/README.md

@@ -4,12 +4,14 @@ This is a small library of useful modules and functions that can help dealing wi
 
 Currently:
 
+* Job that kills stale workers
+* Means to identify stale workers
 * Spec helper `:some_queue.should have_job_queued(class: FooJob)`
 * Methods to introspect queues, including the delayed queue, in your specs
 * Simple `resque:work` task wrapper to better handle exceptions in the worker
 * Marker interface to document jobs which should not be retried
 
-Maybe will have more stuff1
+Maybe will have more stuff someday.
 
 ## To use
 
@@ -19,6 +21,30 @@ Add to your `Gemfile`:
 gem 'resqutils'
 ```
 
+## Stale Workers
+
+It's possible (and, on Heroku, highly likely) that your jobs will appear to be running for "too long".  Usually, this happens
+when a worker exits without cleaning up after itself.  Since Resque stores all state in Redis, and is process-based, it's
+actually fairly easy to create this situation.
+
+The good news is that, if your jobs are idempotent, you can just unregister the "stale" workers, which will kick-off the
+failed handling (which is hopefully to restart your jobs).
+
+You need a means of identifying these workers, and then killing them.
+
+```ruby
+Resqutils::StaleWorkers.new.each do |worker|
+  # this worker is still considered running but has started over an hour ago
+  Resque.enqueue(WorkerKillerJob,worker.id)
+end
+```
+
+`Resqutils::StaleWorkers`'s default of an hour can be overridden either in the constructor or by setting the
+`RESQUTILS_SECONDS_TO_BE_CONSIDERED_STALE` environment variable.
+
+The queue that `WorkerKillerJob` will queue to is `worker_killer_job` by default, but can either be set during the `enqueue`
+call or by setting the `RESQUTILS_WORKER_KILLER_JOB_QUEUE` environment variable.
+
 ## Spec Helpers
 
 ```ruby
@@ -35,6 +61,19 @@ end
 
 `require`ing the `resqutils/spec` will also set up the `have_job_queued` matcher, which is likely what you'll want to use.
 
+### Clearing Jobs
+
+The most important part of using Resque in tests as making sure the queue has what you
+think it has in it.  To that end, you'll likely need `clear_queue` in a `setup` or
+`before` block.
+
+```ruby
+before do
+  clear_queue(MyImportantJob) # clears whatever queue this job is configured to use
+  clear_queue(:foobar)        # clear the "foobar" queue
+end
+```
+
 ### Checking that Jobs Were Queued
 
 ```ruby
@@ -48,6 +87,12 @@ end
 
 # foo_service_spec.rb
 describe FooService do
+  include Resqutils::Spec::ResqueHelpers
+  
+  before do
+    clear_queue(FooJob) # Looks at what queue FooJob uses and clears before each test
+  end
+  
   it "queues a job" do
     result = FooService.new.doit("blah")
 
@@ -71,6 +116,11 @@ end
 # foo_service_spec.rb
 
 describe FooService do
+  include Resqutils::Spec::ResqueHelpers
+  
+  before do
+    clear_queue(:delayed) # Clears all delayed/scheduled queues
+  end
   it "queues a job" do
     result = FooService.new.doit("blah")
 

data/Rakefile

@@ -5,9 +5,12 @@ rescue LoadError
 end
 require 'rubygems/package_task'
 require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
 
 require 'rdoc/task'
 
+include Rake::DSL
+
 RDoc::Task.new(:rdoc) do |rdoc|
   rdoc.rdoc_dir = 'rdoc'
   rdoc.title    = 'ExtraExtra'

data/lib/resqutils.rb

@@ -2,4 +2,6 @@ module Resqutils
 end
 
 require 'resqutils/do_not_auto_retry'
+require 'resqutils/worker_killer_job'
+require 'resqutils/stale_workers'
 require 'resqutils/version'

data/lib/resqutils/stale_workers.rb

@@ -0,0 +1,59 @@
+module Resqutils
+  # Vends stale workers that have been running "too long"
+  class StaleWorkers
+    include Enumerable
+
+    # Create a StaleWorkers instance.
+    #
+    # seconds_to_be_considered_stale:: if present, this is the number of seconds a worker will have to have been
+    #                                  running to be considered stale
+    def initialize(seconds_to_be_considered_stale = 3600)
+      @seconds_to_be_considered_stale = seconds_to_be_considered_stale_from_env! || seconds_to_be_considered_stale
+    end
+
+    # Yield all currently stale workers.  The yielded objects are Resque's representation, which is not
+    # well documented, however you can reasonably assume it will respond to #id, #queue, and #run_at
+    def each(&block)
+      if block.nil?
+        stale_workers.to_enum
+      else
+        stale_workers.each(&block)
+      end
+    end
+
+  private
+
+    def stale_workers
+      Resque.workers.map(&self.method(:worker_with_start_time)).select(&:stale?).map(&:worker)
+    end
+
+    def worker_with_start_time(worker)
+      WorkerWithStartTime.new(worker,@seconds_to_be_considered_stale)
+    end
+
+    def seconds_to_be_considered_stale_from_env!
+      seconds_to_be_considered_stale = String(ENV["RESQUTILS_SECONDS_TO_BE_CONSIDERED_STALE"])
+      if seconds_to_be_considered_stale.strip.length == 0
+        nil
+      elsif seconds_to_be_considered_stale.to_i == 0
+        raise "You set a stale value of 0 seconds, making all jobs stale; probably not what you want"
+      else
+        seconds_to_be_considered_stale.to_i
+      end
+    end
+
+    class WorkerWithStartTime
+      attr_reader :worker
+      def initialize(worker,seconds_to_be_considered_stale)
+        @worker = worker
+        @seconds_to_be_considered_stale = seconds_to_be_considered_stale
+        @start_time = Time.parse(worker.job["run_at"]) rescue nil
+      end
+
+      def stale?
+        return false if @start_time.nil?
+        @start_time <= (Time.now - @seconds_to_be_considered_stale)
+      end
+    end
+  end
+end

data/lib/resqutils/version.rb

@@ -1,3 +1,3 @@
 module Resqutils
-  VERSION='0.0.1'
+  VERSION='1.0.0'
 end

data/lib/resqutils/worker_killer_job.rb

@@ -0,0 +1,16 @@
+module Resqutils
+  # Can be queued to kill a worker.  By default, will be queued to the 'worker_killer_job' queue, however
+  # you can specify RESQUTILS_WORKER_KILLER_JOB_QUEUE in the environment to set an override.  Of course, you can always
+  # forcibly enqueue it as needed.
+  class WorkerKillerJob
+    def self.queue
+      @queue ||= begin
+                   queue = String(ENV["RESQUTILS_WORKER_KILLER_JOB_QUEUE"]).strip
+                   queue.length == 0 ? :worker_killer_job : queue
+                 end
+    end
+    def self.perform(worker_id)
+      Resque.workers.detect { |_| _.id == worker_id }.unregister_worker
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -1,3 +1,6 @@
+require 'resque'
+require 'resqutils'
+
 GEM_ROOT = File.expand_path(File.join(File.dirname(__FILE__),'..'))
 Dir["#{GEM_ROOT}/spec/support/**/*.rb"].sort.each {|f| require f}
 

data/spec/stale_workers_spec.rb

@@ -0,0 +1,148 @@
+require 'spec_helper'
+
+describe Resqutils::StaleWorkers do
+  describe "each" do
+    let(:workers) {
+      [
+        worker,
+        worker(Time.now - 3601),
+        worker(Time.now - 5401),
+        worker,
+        worker(Time.now - 7201),
+      ]
+    }
+
+    before do
+      ENV.delete("RESQUTILS_SECONDS_TO_BE_CONSIDERED_STALE")
+      allow(Resque).to receive(:workers).and_return(workers)
+    end
+
+    context "default stale seconds" do
+
+      context "properly formed workers" do
+        subject(:stale_workers) { described_class.new }
+
+        context "with block" do
+          it "yields each stale worker" do
+            stale = []
+            stale_workers.each do |stale_worker|
+              stale << stale_worker
+            end
+            expect(stale.size).to eq(3)
+            expect(stale).to include(workers[1])
+            expect(stale).to include(workers[2])
+            expect(stale).to include(workers[4])
+          end
+        end
+        context "without block" do
+          it "returns an enumerator we can use" do
+            stale_worker_ids = stale_workers.each.map { |stale_worker|
+              stale_worker.id
+            }
+            expect(stale_worker_ids.size).to eq(3)
+            expect(stale_worker_ids).to include(workers[1].id)
+            expect(stale_worker_ids).to include(workers[2].id)
+            expect(stale_worker_ids).to include(workers[4].id)
+          end
+        end
+      end
+      context "some mangled workers" do
+
+        let(:workers) {
+          [
+            worker,
+            worker(Time.now - 3601),
+            worker("mangled time"),
+            "blah",
+            Object.new,
+          ]
+        }
+
+        subject(:stale_workers) { described_class.new }
+
+        it "yields each umangled stale worker" do
+          stale = []
+          stale_workers.each do |stale_worker|
+            stale << stale_worker
+          end
+          expect(stale.size).to eq(1)
+          expect(stale).to include(workers[1])
+        end
+      end
+    end
+    context "customized stale seconds" do
+      subject(:stale_workers) { described_class.new(7100) }
+
+      context "with block" do
+        it "yields each stale worker" do
+          stale = []
+          stale_workers.each do |stale_worker|
+            stale << stale_worker
+          end
+          expect(stale.size).to eq(1)
+          expect(stale).to include(workers[4])
+        end
+      end
+      context "without block" do
+        it "returns an enumerator we can use" do
+          stale_worker_ids = stale_workers.each.map { |stale_worker|
+            stale_worker.id
+          }
+          expect(stale_worker_ids.size).to eq(1)
+          expect(stale_worker_ids).to include(workers[4].id)
+        end
+      end
+    end
+    context "using the environment" do
+      context "with a sane value" do
+        before do
+          ENV["RESQUTILS_SECONDS_TO_BE_CONSIDERED_STALE"] = "5400"
+        end
+        subject(:stale_workers) { described_class.new(7100) }
+
+        context "with block" do
+          it "yields each stale worker" do
+            stale = []
+            stale_workers.each do |stale_worker|
+              stale << stale_worker
+            end
+            expect(stale.size).to eq(2)
+            expect(stale).to include(workers[2])
+            expect(stale).to include(workers[4])
+          end
+        end
+        context "without block" do
+          it "returns an enumerator we can use" do
+            stale_worker_ids = stale_workers.each.map { |stale_worker|
+              stale_worker.id
+            }
+            expect(stale_worker_ids.size).to eq(2)
+            expect(stale_worker_ids).to include(workers[2].id)
+            expect(stale_worker_ids).to include(workers[4].id)
+          end
+        end
+      end
+
+      context "erroneous values" do
+        it "does not like 0" do
+          ENV["RESQUTILS_SECONDS_TO_BE_CONSIDERED_STALE"] = "0"
+          expect {
+            described_class.new
+          }.to raise_error(/you set a stale value of 0 seconds, making all jobs stale.*probably not what you want/i)
+        end
+        it "does not like floats that are 0" do
+          ENV["RESQUTILS_SECONDS_TO_BE_CONSIDERED_STALE"] = "0.0000001"
+          expect {
+            described_class.new
+          }.to raise_error(/you set a stale value of 0 seconds, making all jobs stale.*probably not what you want/i)
+        end
+      end
+    end
+  end
+
+  def worker(run_at = Time.now)
+    double("resque worker", id: SecureRandom.uuid, job: { "queue" => "whatever",
+                                                          "run_at" => run_at.to_s,
+                                                          "payload" => { "class" => "Foo", "args" => [] } })
+  end
+end

data/spec/worker_killer_job_spec.rb

@@ -0,0 +1,62 @@
+require 'spec_helper'
+require 'securerandom'
+
+describe Resqutils::WorkerKillerJob do
+  describe "queue" do
+    before do
+      ENV.delete("RESQUTILS_WORKER_KILLER_JOB_QUEUE")
+      # since we are memoizing it on the class
+      described_class.instance_variable_set("@queue",nil)
+    end
+
+    after do
+      ENV.delete("RESQUTILS_WORKER_KILLER_JOB_QUEUE")
+    end
+
+    it "uses the worker_killer_job queue by default" do
+      expect(Resque.queue_from_class(described_class)).to eq(:worker_killer_job)
+    end
+
+    it "can use a different queue if specified by the environment" do
+      ENV["RESQUTILS_WORKER_KILLER_JOB_QUEUE"] = "foobar"
+      expect(Resque.queue_from_class(described_class)).to eq("foobar")
+    end
+  end
+  describe "::perform" do
+    let(:workers) {
+      [
+        double(id: worker_id),
+        double(id: worker_id),
+        double(id: worker_id),
+        double(id: worker_id),
+        double(id: worker_id),
+        double(id: worker_id),
+      ]
+    }
+
+    let(:worker_to_kill) { workers[2] }
+
+    before do
+      allow(Resque).to receive(:workers).and_return(workers)
+      workers.each do |worker|
+        allow(worker).to receive(:unregister_worker)
+      end
+      described_class.perform(worker_to_kill.id)
+    end
+
+    it "unregisters the worker we want to kill" do
+      expect(worker_to_kill).to have_received(:unregister_worker)
+    end
+
+    it "doesn't touch the other workers" do
+      workers.reject { |_| _ == worker_to_kill }.each do |worker|
+        expect(worker).not_to have_received(:unregister_worker)
+      end
+    end
+
+  end
+
+  def worker_id
+    "#{SecureRandom.uuid}:#{rand(100)}:some_event"
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: resqutils
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 1.0.0
 platform: ruby
 authors:
 - Stitch Fix Engineering
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-19 00:00:00.000000000 Z
+date: 2015-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
@@ -72,10 +72,14 @@ files:
 - lib/resqutils/spec.rb
 - lib/resqutils/spec/resque_helpers.rb
 - lib/resqutils/spec/resque_matchers.rb
+- lib/resqutils/stale_workers.rb
 - lib/resqutils/version.rb
+- lib/resqutils/worker_killer_job.rb
 - lib/resqutils/worker_task.rb
 - resqutils.gemspec
 - spec/spec_helper.rb
+- spec/stale_workers_spec.rb
+- spec/worker_killer_job_spec.rb
 homepage: http://tech.stitchfix.com
 licenses: []
 metadata: {}
@@ -95,9 +99,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: resqutils
-rubygems_version: 2.2.0.rc.1
+rubygems_version: 2.4.2
 signing_key: 
 specification_version: 4
 summary: Utilities for using Resque in a Rails app
 test_files:
 - spec/spec_helper.rb
+- spec/stale_workers_spec.rb
+- spec/worker_killer_job_spec.rb