resqutils 0.0.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.md +51 -1
- data/Rakefile +3 -0
- data/lib/resqutils.rb +2 -0
- data/lib/resqutils/stale_workers.rb +59 -0
- data/lib/resqutils/version.rb +1 -1
- data/lib/resqutils/worker_killer_job.rb +16 -0
- data/spec/spec_helper.rb +3 -0
- data/spec/stale_workers_spec.rb +148 -0
- data/spec/worker_killer_job_spec.rb +62 -0
- metadata +9 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 63c56aed110c280ee955278e9d007f242482e15e
|
4
|
+
data.tar.gz: 07eac3dc61ff36df991829d3002d805da88bd3d3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 91fc61241a2cd1c0b623f9473fab38ecfcc19f1d7b83c92a1c5c91f72e62c8378ce614950e530f7b535987dec8f3029f62cef0c1e573d139f6d3e8dc885481c2
|
7
|
+
data.tar.gz: 2f5f870492b27271bd29ebc34e4b9d6f24c0117dcf443a479175ca545a514cb881352168e12095a807ce54670f8af685a6388bca85cdf18bb07c6d86fa7b90d7
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -4,12 +4,14 @@ This is a small library of useful modules and functions that can help dealing wi
|
|
4
4
|
|
5
5
|
Currently:
|
6
6
|
|
7
|
+
* Job that kills stale workers
|
8
|
+
* Means to identify stale workers
|
7
9
|
* Spec helper `:some_queue.should have_job_queued(class: FooJob)`
|
8
10
|
* Methods to introspect queues, including the delayed queue, in your specs
|
9
11
|
* Simple `resque:work` task wrapper to better handle exceptions in the worker
|
10
12
|
* Marker interface to document jobs which should not be retried
|
11
13
|
|
12
|
-
Maybe will have more
|
14
|
+
Maybe will have more stuff someday.
|
13
15
|
|
14
16
|
## To use
|
15
17
|
|
@@ -19,6 +21,30 @@ Add to your `Gemfile`:
|
|
19
21
|
gem 'resqutils'
|
20
22
|
```
|
21
23
|
|
24
|
+
## Stale Workers
|
25
|
+
|
26
|
+
It's possible (and, on Heroku, highly likely) that your jobs will appear to be running for "too long". Usually, this happens
|
27
|
+
when a worker exits without cleaning up after itself. Since Resque stores all state in Redis, and is process-based, it's
|
28
|
+
actually fairly easy to create this situation.
|
29
|
+
|
30
|
+
The good news is that, if your jobs are idempotent, you can just unregister the "stale" workers, which will kick-off the
|
31
|
+
failed handling (which is hopefully to restart your jobs).
|
32
|
+
|
33
|
+
You need a means of identifying these workers, and then killing them.
|
34
|
+
|
35
|
+
```ruby
|
36
|
+
Resqutils::StaleWorkers.new.each do |worker|
|
37
|
+
# this worker is still considered running but has started over an hour ago
|
38
|
+
Resque.enqueue(WorkerKillerJob,worker.id)
|
39
|
+
end
|
40
|
+
```
|
41
|
+
|
42
|
+
`Resqutils::StaleWorkers`'s default of an hour can be overridden either in the constructor or by setting the
|
43
|
+
`RESQUTILS_SECONDS_TO_BE_CONSIDERED_STALE` environment variable.
|
44
|
+
|
45
|
+
The queue that `WorkerKillerJob` will queue to is `worker_killer_job` by default, but can either be set during the `enqueue`
|
46
|
+
call or by setting the `RESQUTILS_WORKER_KILLER_JOB_QUEUE` environment variable.
|
47
|
+
|
22
48
|
## Spec Helpers
|
23
49
|
|
24
50
|
```ruby
|
@@ -35,6 +61,19 @@ end
|
|
35
61
|
|
36
62
|
`require`ing the `resqutils/spec` will also set up the `have_job_queued` matcher, which is likely what you'll want to use.
|
37
63
|
|
64
|
+
### Clearing Jobs
|
65
|
+
|
66
|
+
The most important part of using Resque in tests as making sure the queue has what you
|
67
|
+
think it has in it. To that end, you'll likely need `clear_queue` in a `setup` or
|
68
|
+
`before` block.
|
69
|
+
|
70
|
+
```ruby
|
71
|
+
before do
|
72
|
+
clear_queue(MyImportantJob) # clears whatever queue this job is configured to use
|
73
|
+
clear_queue(:foobar) # clear the "foobar" queue
|
74
|
+
end
|
75
|
+
```
|
76
|
+
|
38
77
|
### Checking that Jobs Were Queued
|
39
78
|
|
40
79
|
```ruby
|
@@ -48,6 +87,12 @@ end
|
|
48
87
|
|
49
88
|
# foo_service_spec.rb
|
50
89
|
describe FooService do
|
90
|
+
include Resqutils::Spec::ResqueHelpers
|
91
|
+
|
92
|
+
before do
|
93
|
+
clear_queue(FooJob) # Looks at what queue FooJob uses and clears before each test
|
94
|
+
end
|
95
|
+
|
51
96
|
it "queues a job" do
|
52
97
|
result = FooService.new.doit("blah")
|
53
98
|
|
@@ -71,6 +116,11 @@ end
|
|
71
116
|
# foo_service_spec.rb
|
72
117
|
|
73
118
|
describe FooService do
|
119
|
+
include Resqutils::Spec::ResqueHelpers
|
120
|
+
|
121
|
+
before do
|
122
|
+
clear_queue(:delayed) # Clears all delayed/scheduled queues
|
123
|
+
end
|
74
124
|
it "queues a job" do
|
75
125
|
result = FooService.new.doit("blah")
|
76
126
|
|
data/Rakefile
CHANGED
@@ -5,9 +5,12 @@ rescue LoadError
|
|
5
5
|
end
|
6
6
|
require 'rubygems/package_task'
|
7
7
|
require 'rspec/core/rake_task'
|
8
|
+
RSpec::Core::RakeTask.new(:spec)
|
8
9
|
|
9
10
|
require 'rdoc/task'
|
10
11
|
|
12
|
+
include Rake::DSL
|
13
|
+
|
11
14
|
RDoc::Task.new(:rdoc) do |rdoc|
|
12
15
|
rdoc.rdoc_dir = 'rdoc'
|
13
16
|
rdoc.title = 'ExtraExtra'
|
data/lib/resqutils.rb
CHANGED
@@ -0,0 +1,59 @@
|
|
1
|
+
module Resqutils
|
2
|
+
# Vends stale workers that have been running "too long"
|
3
|
+
class StaleWorkers
|
4
|
+
include Enumerable
|
5
|
+
|
6
|
+
# Create a StaleWorkers instance.
|
7
|
+
#
|
8
|
+
# seconds_to_be_considered_stale:: if present, this is the number of seconds a worker will have to have been
|
9
|
+
# running to be considered stale
|
10
|
+
def initialize(seconds_to_be_considered_stale = 3600)
|
11
|
+
@seconds_to_be_considered_stale = seconds_to_be_considered_stale_from_env! || seconds_to_be_considered_stale
|
12
|
+
end
|
13
|
+
|
14
|
+
# Yield all currently stale workers. The yielded objects are Resque's representation, which is not
|
15
|
+
# well documented, however you can reasonably assume it will respond to #id, #queue, and #run_at
|
16
|
+
def each(&block)
|
17
|
+
if block.nil?
|
18
|
+
stale_workers.to_enum
|
19
|
+
else
|
20
|
+
stale_workers.each(&block)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def stale_workers
|
27
|
+
Resque.workers.map(&self.method(:worker_with_start_time)).select(&:stale?).map(&:worker)
|
28
|
+
end
|
29
|
+
|
30
|
+
def worker_with_start_time(worker)
|
31
|
+
WorkerWithStartTime.new(worker,@seconds_to_be_considered_stale)
|
32
|
+
end
|
33
|
+
|
34
|
+
def seconds_to_be_considered_stale_from_env!
|
35
|
+
seconds_to_be_considered_stale = String(ENV["RESQUTILS_SECONDS_TO_BE_CONSIDERED_STALE"])
|
36
|
+
if seconds_to_be_considered_stale.strip.length == 0
|
37
|
+
nil
|
38
|
+
elsif seconds_to_be_considered_stale.to_i == 0
|
39
|
+
raise "You set a stale value of 0 seconds, making all jobs stale; probably not what you want"
|
40
|
+
else
|
41
|
+
seconds_to_be_considered_stale.to_i
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
class WorkerWithStartTime
|
46
|
+
attr_reader :worker
|
47
|
+
def initialize(worker,seconds_to_be_considered_stale)
|
48
|
+
@worker = worker
|
49
|
+
@seconds_to_be_considered_stale = seconds_to_be_considered_stale
|
50
|
+
@start_time = Time.parse(worker.job["run_at"]) rescue nil
|
51
|
+
end
|
52
|
+
|
53
|
+
def stale?
|
54
|
+
return false if @start_time.nil?
|
55
|
+
@start_time <= (Time.now - @seconds_to_be_considered_stale)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
data/lib/resqutils/version.rb
CHANGED
@@ -0,0 +1,16 @@
|
|
1
|
+
module Resqutils
|
2
|
+
# Can be queued to kill a worker. By default, will be queued to the 'worker_killer_job' queue, however
|
3
|
+
# you can specify RESQUTILS_WORKER_KILLER_JOB_QUEUE in the environment to set an override. Of course, you can always
|
4
|
+
# forcibly enqueue it as needed.
|
5
|
+
class WorkerKillerJob
|
6
|
+
def self.queue
|
7
|
+
@queue ||= begin
|
8
|
+
queue = String(ENV["RESQUTILS_WORKER_KILLER_JOB_QUEUE"]).strip
|
9
|
+
queue.length == 0 ? :worker_killer_job : queue
|
10
|
+
end
|
11
|
+
end
|
12
|
+
def self.perform(worker_id)
|
13
|
+
Resque.workers.detect { |_| _.id == worker_id }.unregister_worker
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
data/spec/spec_helper.rb
CHANGED
@@ -0,0 +1,148 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Resqutils::StaleWorkers do
|
4
|
+
describe "each" do
|
5
|
+
let(:workers) {
|
6
|
+
[
|
7
|
+
worker,
|
8
|
+
worker(Time.now - 3601),
|
9
|
+
worker(Time.now - 5401),
|
10
|
+
worker,
|
11
|
+
worker(Time.now - 7201),
|
12
|
+
]
|
13
|
+
}
|
14
|
+
|
15
|
+
before do
|
16
|
+
ENV.delete("RESQUTILS_SECONDS_TO_BE_CONSIDERED_STALE")
|
17
|
+
allow(Resque).to receive(:workers).and_return(workers)
|
18
|
+
end
|
19
|
+
|
20
|
+
context "default stale seconds" do
|
21
|
+
|
22
|
+
context "properly formed workers" do
|
23
|
+
subject(:stale_workers) { described_class.new }
|
24
|
+
|
25
|
+
context "with block" do
|
26
|
+
it "yields each stale worker" do
|
27
|
+
stale = []
|
28
|
+
stale_workers.each do |stale_worker|
|
29
|
+
stale << stale_worker
|
30
|
+
end
|
31
|
+
expect(stale.size).to eq(3)
|
32
|
+
expect(stale).to include(workers[1])
|
33
|
+
expect(stale).to include(workers[2])
|
34
|
+
expect(stale).to include(workers[4])
|
35
|
+
end
|
36
|
+
end
|
37
|
+
context "without block" do
|
38
|
+
it "returns an enumerator we can use" do
|
39
|
+
stale_worker_ids = stale_workers.each.map { |stale_worker|
|
40
|
+
stale_worker.id
|
41
|
+
}
|
42
|
+
expect(stale_worker_ids.size).to eq(3)
|
43
|
+
expect(stale_worker_ids).to include(workers[1].id)
|
44
|
+
expect(stale_worker_ids).to include(workers[2].id)
|
45
|
+
expect(stale_worker_ids).to include(workers[4].id)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
context "some mangled workers" do
|
50
|
+
|
51
|
+
let(:workers) {
|
52
|
+
[
|
53
|
+
worker,
|
54
|
+
worker(Time.now - 3601),
|
55
|
+
worker("mangled time"),
|
56
|
+
"blah",
|
57
|
+
Object.new,
|
58
|
+
]
|
59
|
+
}
|
60
|
+
|
61
|
+
subject(:stale_workers) { described_class.new }
|
62
|
+
|
63
|
+
it "yields each umangled stale worker" do
|
64
|
+
stale = []
|
65
|
+
stale_workers.each do |stale_worker|
|
66
|
+
stale << stale_worker
|
67
|
+
end
|
68
|
+
expect(stale.size).to eq(1)
|
69
|
+
expect(stale).to include(workers[1])
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
context "customized stale seconds" do
|
74
|
+
subject(:stale_workers) { described_class.new(7100) }
|
75
|
+
|
76
|
+
context "with block" do
|
77
|
+
it "yields each stale worker" do
|
78
|
+
stale = []
|
79
|
+
stale_workers.each do |stale_worker|
|
80
|
+
stale << stale_worker
|
81
|
+
end
|
82
|
+
expect(stale.size).to eq(1)
|
83
|
+
expect(stale).to include(workers[4])
|
84
|
+
end
|
85
|
+
end
|
86
|
+
context "without block" do
|
87
|
+
it "returns an enumerator we can use" do
|
88
|
+
stale_worker_ids = stale_workers.each.map { |stale_worker|
|
89
|
+
stale_worker.id
|
90
|
+
}
|
91
|
+
expect(stale_worker_ids.size).to eq(1)
|
92
|
+
expect(stale_worker_ids).to include(workers[4].id)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
context "using the environment" do
|
97
|
+
context "with a sane value" do
|
98
|
+
before do
|
99
|
+
ENV["RESQUTILS_SECONDS_TO_BE_CONSIDERED_STALE"] = "5400"
|
100
|
+
end
|
101
|
+
subject(:stale_workers) { described_class.new(7100) }
|
102
|
+
|
103
|
+
context "with block" do
|
104
|
+
it "yields each stale worker" do
|
105
|
+
stale = []
|
106
|
+
stale_workers.each do |stale_worker|
|
107
|
+
stale << stale_worker
|
108
|
+
end
|
109
|
+
expect(stale.size).to eq(2)
|
110
|
+
expect(stale).to include(workers[2])
|
111
|
+
expect(stale).to include(workers[4])
|
112
|
+
end
|
113
|
+
end
|
114
|
+
context "without block" do
|
115
|
+
it "returns an enumerator we can use" do
|
116
|
+
stale_worker_ids = stale_workers.each.map { |stale_worker|
|
117
|
+
stale_worker.id
|
118
|
+
}
|
119
|
+
expect(stale_worker_ids.size).to eq(2)
|
120
|
+
expect(stale_worker_ids).to include(workers[2].id)
|
121
|
+
expect(stale_worker_ids).to include(workers[4].id)
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
context "erroneous values" do
|
127
|
+
it "does not like 0" do
|
128
|
+
ENV["RESQUTILS_SECONDS_TO_BE_CONSIDERED_STALE"] = "0"
|
129
|
+
expect {
|
130
|
+
described_class.new
|
131
|
+
}.to raise_error(/you set a stale value of 0 seconds, making all jobs stale.*probably not what you want/i)
|
132
|
+
end
|
133
|
+
it "does not like floats that are 0" do
|
134
|
+
ENV["RESQUTILS_SECONDS_TO_BE_CONSIDERED_STALE"] = "0.0000001"
|
135
|
+
expect {
|
136
|
+
described_class.new
|
137
|
+
}.to raise_error(/you set a stale value of 0 seconds, making all jobs stale.*probably not what you want/i)
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
def worker(run_at = Time.now)
|
144
|
+
double("resque worker", id: SecureRandom.uuid, job: { "queue" => "whatever",
|
145
|
+
"run_at" => run_at.to_s,
|
146
|
+
"payload" => { "class" => "Foo", "args" => [] } })
|
147
|
+
end
|
148
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'securerandom'
|
3
|
+
|
4
|
+
describe Resqutils::WorkerKillerJob do
|
5
|
+
describe "queue" do
|
6
|
+
before do
|
7
|
+
ENV.delete("RESQUTILS_WORKER_KILLER_JOB_QUEUE")
|
8
|
+
# since we are memoizing it on the class
|
9
|
+
described_class.instance_variable_set("@queue",nil)
|
10
|
+
end
|
11
|
+
|
12
|
+
after do
|
13
|
+
ENV.delete("RESQUTILS_WORKER_KILLER_JOB_QUEUE")
|
14
|
+
end
|
15
|
+
|
16
|
+
it "uses the worker_killer_job queue by default" do
|
17
|
+
expect(Resque.queue_from_class(described_class)).to eq(:worker_killer_job)
|
18
|
+
end
|
19
|
+
|
20
|
+
it "can use a different queue if specified by the environment" do
|
21
|
+
ENV["RESQUTILS_WORKER_KILLER_JOB_QUEUE"] = "foobar"
|
22
|
+
expect(Resque.queue_from_class(described_class)).to eq("foobar")
|
23
|
+
end
|
24
|
+
end
|
25
|
+
describe "::perform" do
|
26
|
+
let(:workers) {
|
27
|
+
[
|
28
|
+
double(id: worker_id),
|
29
|
+
double(id: worker_id),
|
30
|
+
double(id: worker_id),
|
31
|
+
double(id: worker_id),
|
32
|
+
double(id: worker_id),
|
33
|
+
double(id: worker_id),
|
34
|
+
]
|
35
|
+
}
|
36
|
+
|
37
|
+
let(:worker_to_kill) { workers[2] }
|
38
|
+
|
39
|
+
before do
|
40
|
+
allow(Resque).to receive(:workers).and_return(workers)
|
41
|
+
workers.each do |worker|
|
42
|
+
allow(worker).to receive(:unregister_worker)
|
43
|
+
end
|
44
|
+
described_class.perform(worker_to_kill.id)
|
45
|
+
end
|
46
|
+
|
47
|
+
it "unregisters the worker we want to kill" do
|
48
|
+
expect(worker_to_kill).to have_received(:unregister_worker)
|
49
|
+
end
|
50
|
+
|
51
|
+
it "doesn't touch the other workers" do
|
52
|
+
workers.reject { |_| _ == worker_to_kill }.each do |worker|
|
53
|
+
expect(worker).not_to have_received(:unregister_worker)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
end
|
58
|
+
|
59
|
+
def worker_id
|
60
|
+
"#{SecureRandom.uuid}:#{rand(100)}:some_event"
|
61
|
+
end
|
62
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: resqutils
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Stitch Fix Engineering
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-02-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: resque
|
@@ -72,10 +72,14 @@ files:
|
|
72
72
|
- lib/resqutils/spec.rb
|
73
73
|
- lib/resqutils/spec/resque_helpers.rb
|
74
74
|
- lib/resqutils/spec/resque_matchers.rb
|
75
|
+
- lib/resqutils/stale_workers.rb
|
75
76
|
- lib/resqutils/version.rb
|
77
|
+
- lib/resqutils/worker_killer_job.rb
|
76
78
|
- lib/resqutils/worker_task.rb
|
77
79
|
- resqutils.gemspec
|
78
80
|
- spec/spec_helper.rb
|
81
|
+
- spec/stale_workers_spec.rb
|
82
|
+
- spec/worker_killer_job_spec.rb
|
79
83
|
homepage: http://tech.stitchfix.com
|
80
84
|
licenses: []
|
81
85
|
metadata: {}
|
@@ -95,9 +99,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
95
99
|
version: '0'
|
96
100
|
requirements: []
|
97
101
|
rubyforge_project: resqutils
|
98
|
-
rubygems_version: 2.2
|
102
|
+
rubygems_version: 2.4.2
|
99
103
|
signing_key:
|
100
104
|
specification_version: 4
|
101
105
|
summary: Utilities for using Resque in a Rails app
|
102
106
|
test_files:
|
103
107
|
- spec/spec_helper.rb
|
108
|
+
- spec/stale_workers_spec.rb
|
109
|
+
- spec/worker_killer_job_spec.rb
|