timberline 0.4.1 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG +24 -0
- data/README.markdown +12 -15
- data/lib/timberline/queue.rb +28 -3
- data/lib/timberline/version.rb +1 -1
- data/lib/timberline.rb +6 -15
- data/spec/lib/timberline/queue_spec.rb +85 -1
- data/spec/lib/timberline_spec.rb +6 -21
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7aa514d129ba4ac74886845982675b82975051bb
|
4
|
+
data.tar.gz: d668a59889b0f78d60e04321b2a0224c1264712e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3f44d9c9e5f008d8b01e0ee8e008a6e4adeb22ca4db90dfffde5b5e3a2aa263976cc0d89f4a82217a639a2dbfafcefb7bcd9864f0df86c5178fe65452490a36b
|
7
|
+
data.tar.gz: 5ff02917829e79ddf5d7f8e0d9e99a35bd35df784893a54773dcde8878f351633bad12de48587e8a5dbac62fd9421c3a5fb1701ddfde36e5e029b8ebd6771216
|
data/CHANGELOG
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
0.5.0
|
2
|
+
- Each queue now has its own error queue, rather than having one error_queue for everything
|
3
|
+
- Queues can now be created with a 'hidden' option that removes them from the Timberline listing
|
4
|
+
0.4.1
|
5
|
+
- Fix a bug with calculating average_execution_time statistics
|
6
|
+
0.4.0
|
7
|
+
- Test suite moved over to RSpec
|
8
|
+
- CI added via Travis
|
9
|
+
- Queue#delete! renamed to Queue#delete in accordance with ruby convention
|
10
|
+
- Timberline.config renamed to Timberline.configure to avoid collision with the attr_accessor
|
11
|
+
0.3.1
|
12
|
+
- Fix a bug with handling errors while logging success stats
|
13
|
+
0.3.0
|
14
|
+
- Rails compatability updated, support for Rails 2 removed
|
15
|
+
0.2.1
|
16
|
+
- Tests switched over to nutrasuite
|
17
|
+
- Rails 3 compatability added
|
18
|
+
0.2.0
|
19
|
+
- Added ability to pause queues
|
20
|
+
- Added statistics logging for successes/errors/retries
|
21
|
+
0.1.2
|
22
|
+
- Rename Treeline to Timberline due to gem name conflict
|
23
|
+
0.1.1
|
24
|
+
- First public release of Treeline
|
data/README.markdown
CHANGED
@@ -139,12 +139,11 @@ to note:
|
|
139
139
|
retry the job in the event of a transient error, or put it on the error queue
|
140
140
|
for processing if a more fatal error occurs.
|
141
141
|
|
142
|
-
###
|
142
|
+
### Error queues
|
143
143
|
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
queue (its queue name should always be "Timberline\_errors").
|
144
|
+
Each queue has its own error queue, accessible through `Queue#error_queue`.
|
145
|
+
You can pop items directly off of the queue to operate on them if you want, or
|
146
|
+
you could write a queue processor that reads off of that queue.
|
148
147
|
|
149
148
|
### Using the binary
|
150
149
|
|
@@ -172,19 +171,17 @@ There are some options to the Timberline binary that you may find helpful -
|
|
172
171
|
|
173
172
|
Still to be done:
|
174
173
|
|
175
|
-
- A simple Sinatra interface for monitoring the statuses of queues and
|
174
|
+
- **Monitor** - A simple Sinatra interface for monitoring the statuses of queues and
|
176
175
|
observing/resubmitting errored-out jobs.
|
177
|
-
-
|
178
|
-
that it tries to process so that it's more robust.
|
179
|
-
- DSL improvements - the DSL-ish setup for Timberline could probably use some
|
180
|
-
updates to be both more obvious and easier to use.
|
181
|
-
- Documentation - need to get YARD docs added so that the API is more completely
|
176
|
+
- **Documentation** - need to get Tomdoc added so that the API is more completely
|
182
177
|
documented. For the time being, though, there are some fairly comprehensive
|
183
178
|
test suites.
|
184
|
-
-
|
185
|
-
|
186
|
-
|
187
|
-
|
179
|
+
- **Refactor** - the singleton model made sense at some point for Timberline but now it's
|
180
|
+
cumbersome. Need to rewrite some of the basic stuff to be more OO-appropriate.
|
181
|
+
- **Forking** - Timberline should support forking in its watcher model so that jobs can
|
182
|
+
be run in parallel and independently of each other.
|
183
|
+
- **Relocate Rails** - Create a `timberline-rails` gem that includes all of our Rails-specific
|
184
|
+
functionality and configuration rather than have that in Timberline proper.
|
188
185
|
|
189
186
|
## Future
|
190
187
|
|
data/lib/timberline/queue.rb
CHANGED
@@ -2,14 +2,18 @@ class Timberline
|
|
2
2
|
class Queue
|
3
3
|
attr_reader :queue_name, :read_timeout
|
4
4
|
|
5
|
-
def initialize(queue_name,
|
5
|
+
def initialize(queue_name, opts = {})
|
6
|
+
read_timeout = opts.fetch(:read_timeout, 0)
|
7
|
+
hidden = opts.fetch(:hidden, false)
|
6
8
|
if queue_name.nil?
|
7
9
|
raise ArgumentError.new("Queue name must be provided.")
|
8
10
|
end
|
9
11
|
@queue_name = queue_name
|
10
12
|
@read_timeout = read_timeout
|
11
13
|
@redis = Timberline.redis
|
12
|
-
|
14
|
+
unless hidden
|
15
|
+
@redis.sadd "timberline_queue_names", queue_name
|
16
|
+
end
|
13
17
|
end
|
14
18
|
|
15
19
|
def delete
|
@@ -92,12 +96,29 @@ class Timberline
|
|
92
96
|
end
|
93
97
|
end
|
94
98
|
|
99
|
+
def retry_item(item)
|
100
|
+
if (item.retries < Timberline.max_retries)
|
101
|
+
item.retries += 1
|
102
|
+
item.last_tried_at = Time.now.to_f
|
103
|
+
add_retry_stat(item)
|
104
|
+
push(item)
|
105
|
+
else
|
106
|
+
error_item(item)
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
def error_item(item)
|
111
|
+
item.fatal_error_at = Time.now.to_f
|
112
|
+
add_error_stat(item)
|
113
|
+
self.error_queue.push(item)
|
114
|
+
end
|
115
|
+
|
95
116
|
def add_retry_stat(item)
|
96
117
|
add_stat_for_key(attr("retry_stats"), item)
|
97
118
|
end
|
98
119
|
|
99
120
|
def add_error_stat(item)
|
100
|
-
add_stat_for_key(attr("
|
121
|
+
add_stat_for_key(attr("error_stats"), item)
|
101
122
|
end
|
102
123
|
|
103
124
|
def add_success_stat(item)
|
@@ -106,6 +127,10 @@ class Timberline
|
|
106
127
|
$stderr.puts "Success Stat Error: #{e.inspect}, Item: #{item.inspect}"
|
107
128
|
end
|
108
129
|
|
130
|
+
def error_queue
|
131
|
+
@error_queue ||= Timberline.queue(attr("errors"), hidden: true)
|
132
|
+
end
|
133
|
+
|
109
134
|
private
|
110
135
|
|
111
136
|
def add_stat_for_key(key, item)
|
data/lib/timberline/version.rb
CHANGED
data/lib/timberline.rb
CHANGED
@@ -43,12 +43,8 @@ class Timberline
|
|
43
43
|
Timberline.redis.smembers("timberline_queue_names").map { |name| queue(name) }
|
44
44
|
end
|
45
45
|
|
46
|
-
def self.
|
47
|
-
|
48
|
-
end
|
49
|
-
|
50
|
-
def self.queue(queue_name)
|
51
|
-
Queue.new(queue_name)
|
46
|
+
def self.queue(queue_name, opts = {})
|
47
|
+
Queue.new(queue_name, opts)
|
52
48
|
end
|
53
49
|
|
54
50
|
def self.push(queue_name, data, metadata={})
|
@@ -56,18 +52,13 @@ class Timberline
|
|
56
52
|
end
|
57
53
|
|
58
54
|
def self.retry_item(item)
|
59
|
-
|
60
|
-
|
61
|
-
item.last_tried_at = Time.now.to_f
|
62
|
-
queue(item.origin_queue).push(item)
|
63
|
-
else
|
64
|
-
error_item(item)
|
65
|
-
end
|
55
|
+
origin_queue = queue(item.origin_queue)
|
56
|
+
origin_queue.retry_item(item)
|
66
57
|
end
|
67
58
|
|
68
59
|
def self.error_item(item)
|
69
|
-
|
70
|
-
|
60
|
+
origin_queue = queue(item.origin_queue)
|
61
|
+
origin_queue.error_item(item)
|
71
62
|
end
|
72
63
|
|
73
64
|
def self.pause(queue_name)
|
@@ -11,7 +11,17 @@ describe Timberline::Queue do
|
|
11
11
|
end
|
12
12
|
|
13
13
|
it "allows you to override the default read_timeout" do
|
14
|
-
expect(Timberline::Queue.new("fritters", 50).read_timeout).to eq(50)
|
14
|
+
expect(Timberline::Queue.new("fritters", read_timeout: 50).read_timeout).to eq(50)
|
15
|
+
end
|
16
|
+
|
17
|
+
context "if the queue is marked as hidden" do
|
18
|
+
before do
|
19
|
+
Timberline::Queue.new("fritters", hidden: true)
|
20
|
+
end
|
21
|
+
|
22
|
+
it "Doesn't add it to the queue listing in Timberline" do
|
23
|
+
expect(Timberline.all_queues.map { |q| q.queue_name }).not_to include("fritters")
|
24
|
+
end
|
15
25
|
end
|
16
26
|
|
17
27
|
it "adds the queue's name to the Timberline queue listing" do
|
@@ -97,4 +107,78 @@ describe Timberline::Queue do
|
|
97
107
|
end
|
98
108
|
|
99
109
|
end
|
110
|
+
|
111
|
+
describe "#error_item" do
|
112
|
+
subject { Timberline::Queue.new("fritters") }
|
113
|
+
let(:item) { subject.push("apple"); subject.pop }
|
114
|
+
|
115
|
+
before do
|
116
|
+
subject.error_item(item)
|
117
|
+
end
|
118
|
+
|
119
|
+
it "updates the fatal_error_at timestamp for the item" do
|
120
|
+
expect(item.fatal_error_at).not_to be_nil
|
121
|
+
end
|
122
|
+
|
123
|
+
it "puts the item on the queue's error queue" do
|
124
|
+
expect(subject.error_queue.length).to eq(1)
|
125
|
+
end
|
126
|
+
|
127
|
+
it "adds an error statistic" do
|
128
|
+
expect(subject.number_errors).to eq(1)
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
describe "#retry_item" do
|
133
|
+
subject { Timberline::Queue.new("fritters") }
|
134
|
+
let(:item) { subject.push("apple"); subject.pop }
|
135
|
+
|
136
|
+
context "when the item hasn't been retried before" do
|
137
|
+
before do
|
138
|
+
subject.retry_item(item)
|
139
|
+
end
|
140
|
+
|
141
|
+
it "adds a retry statistic" do
|
142
|
+
expect(subject.number_retries).to eq(1)
|
143
|
+
end
|
144
|
+
|
145
|
+
it "puts the item back on the queue" do
|
146
|
+
expect(subject.length).to eq(1)
|
147
|
+
end
|
148
|
+
|
149
|
+
it "updates the last_tried_at property on the item" do
|
150
|
+
expect(item.last_tried_at).not_to be_nil
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
context "when the item has been retried before" do
|
155
|
+
before do
|
156
|
+
item.retries = 3
|
157
|
+
subject.retry_item(item)
|
158
|
+
end
|
159
|
+
|
160
|
+
it "adds a retry statistic" do
|
161
|
+
expect(subject.number_retries).to eq(1)
|
162
|
+
end
|
163
|
+
|
164
|
+
it "puts the item back on the queue" do
|
165
|
+
expect(subject.length).to eq(1)
|
166
|
+
end
|
167
|
+
|
168
|
+
it "updates the last_tried_at property on the item" do
|
169
|
+
expect(item.last_tried_at).not_to be_nil
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
context "when the item has been retried the maximum number of times" do
|
174
|
+
before do
|
175
|
+
item.retries = 5
|
176
|
+
end
|
177
|
+
|
178
|
+
it "defers to #error_item" do
|
179
|
+
expect(subject).to receive(:error_item).with(item)
|
180
|
+
subject.retry_item(item)
|
181
|
+
end
|
182
|
+
end
|
183
|
+
end
|
100
184
|
end
|
data/spec/lib/timberline_spec.rb
CHANGED
@@ -91,16 +91,6 @@ describe Timberline do
|
|
91
91
|
end
|
92
92
|
end
|
93
93
|
|
94
|
-
describe ".error_queue" do
|
95
|
-
it "returns a Timberline::Queue" do
|
96
|
-
expect(Timberline.error_queue).to be_a Timberline::Queue
|
97
|
-
end
|
98
|
-
|
99
|
-
it "returns the timberline_errors queue" do
|
100
|
-
expect(Timberline.error_queue.queue_name).to eq("timberline_errors")
|
101
|
-
end
|
102
|
-
end
|
103
|
-
|
104
94
|
describe ".queue" do
|
105
95
|
it "returns a Timberline::Queue" do
|
106
96
|
expect(Timberline.queue("fritters")).to be_a Timberline::Queue
|
@@ -166,8 +156,8 @@ describe Timberline do
|
|
166
156
|
item.retries = 5
|
167
157
|
end
|
168
158
|
|
169
|
-
it "passes the item on to the
|
170
|
-
|
159
|
+
it "passes the item on to the origin queue for retry" do
|
160
|
+
expect_any_instance_of(Timberline::Queue).to receive(:retry_item).with(item)
|
171
161
|
Timberline.retry_item(item)
|
172
162
|
end
|
173
163
|
end
|
@@ -175,17 +165,12 @@ describe Timberline do
|
|
175
165
|
|
176
166
|
describe ".error_item" do
|
177
167
|
let(:queue) { Timberline.queue("test_queue") }
|
178
|
-
let(:error_queue) {
|
168
|
+
let(:error_queue) { queue.error_queue }
|
179
169
|
let(:item) { Timberline.push(queue.queue_name, "Howdy kids."); queue.pop }
|
180
170
|
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
expect(error_queue.length).to eq(1)
|
185
|
-
end
|
186
|
-
|
187
|
-
it "updates the fatal_error_at property on the item" do
|
188
|
-
expect(item.fatal_error_at).not_to be_nil
|
171
|
+
it "passes the item on to the queue for error handling" do
|
172
|
+
expect_any_instance_of(Timberline::Queue).to receive(:error_item).with(item)
|
173
|
+
Timberline.error_item(item)
|
189
174
|
end
|
190
175
|
end
|
191
176
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: timberline
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tommy Morgan
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-06-
|
11
|
+
date: 2014-06-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: redis
|
@@ -137,6 +137,7 @@ files:
|
|
137
137
|
- .rspec
|
138
138
|
- .ruby-gemset
|
139
139
|
- .travis.yml
|
140
|
+
- CHANGELOG
|
140
141
|
- Gemfile
|
141
142
|
- README.markdown
|
142
143
|
- Rakefile
|