jiggler 0.1.0.rc2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +6 -0
- data/LICENSE +4 -0
- data/README.md +423 -0
- data/bin/jiggler +31 -0
- data/lib/jiggler/cleaner.rb +130 -0
- data/lib/jiggler/cli.rb +263 -0
- data/lib/jiggler/config.rb +165 -0
- data/lib/jiggler/core.rb +22 -0
- data/lib/jiggler/errors.rb +5 -0
- data/lib/jiggler/job.rb +116 -0
- data/lib/jiggler/launcher.rb +69 -0
- data/lib/jiggler/manager.rb +73 -0
- data/lib/jiggler/redis_store.rb +55 -0
- data/lib/jiggler/retrier.rb +122 -0
- data/lib/jiggler/scheduled/enqueuer.rb +78 -0
- data/lib/jiggler/scheduled/poller.rb +97 -0
- data/lib/jiggler/stats/collection.rb +26 -0
- data/lib/jiggler/stats/monitor.rb +103 -0
- data/lib/jiggler/summary.rb +101 -0
- data/lib/jiggler/support/helper.rb +35 -0
- data/lib/jiggler/version.rb +5 -0
- data/lib/jiggler/web/assets/stylesheets/application.css +64 -0
- data/lib/jiggler/web/views/application.erb +329 -0
- data/lib/jiggler/web.rb +80 -0
- data/lib/jiggler/worker.rb +179 -0
- data/lib/jiggler.rb +10 -0
- data/spec/examples.txt +79 -0
- data/spec/fixtures/config/jiggler.yml +4 -0
- data/spec/fixtures/jobs.rb +5 -0
- data/spec/fixtures/my_failed_job.rb +10 -0
- data/spec/fixtures/my_job.rb +9 -0
- data/spec/fixtures/my_job_with_args.rb +18 -0
- data/spec/jiggler/cleaner_spec.rb +171 -0
- data/spec/jiggler/cli_spec.rb +87 -0
- data/spec/jiggler/config_spec.rb +56 -0
- data/spec/jiggler/core_spec.rb +34 -0
- data/spec/jiggler/job_spec.rb +99 -0
- data/spec/jiggler/launcher_spec.rb +66 -0
- data/spec/jiggler/manager_spec.rb +52 -0
- data/spec/jiggler/redis_store_spec.rb +20 -0
- data/spec/jiggler/retrier_spec.rb +55 -0
- data/spec/jiggler/scheduled/enqueuer_spec.rb +81 -0
- data/spec/jiggler/scheduled/poller_spec.rb +40 -0
- data/spec/jiggler/stats/monitor_spec.rb +40 -0
- data/spec/jiggler/summary_spec.rb +168 -0
- data/spec/jiggler/web_spec.rb +37 -0
- data/spec/jiggler/worker_spec.rb +110 -0
- data/spec/spec_helper.rb +54 -0
- metadata +230 -0
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html>
|
|
3
|
+
<head>
|
|
4
|
+
<title>Jiggler</title>
|
|
5
|
+
<style>
|
|
6
|
+
<%= styles %>
|
|
7
|
+
</style>
|
|
8
|
+
</head>
|
|
9
|
+
<body>
|
|
10
|
+
<h1>Jiggler</h1>
|
|
11
|
+
<div id='content'>
|
|
12
|
+
|
|
13
|
+
<div class='stats-block'>
|
|
14
|
+
<div class='stats-entry' title='Jobs to be retried'>
|
|
15
|
+
<span><%= "To Retry: #{@summary['retry_jobs_count']}" %></span>
|
|
16
|
+
</div>
|
|
17
|
+
<div class='stats-entry' title='Jobs with delayed enqueuing'>
|
|
18
|
+
<span><%= "Scheduled: #{@summary['scheduled_jobs_count']}" %></span>
|
|
19
|
+
</div>
|
|
20
|
+
<div class='stats-entry' title='Completely failed jobs'>
|
|
21
|
+
<span><%= "Dead: #{@summary['dead_jobs_count']}" %></span>
|
|
22
|
+
</div>
|
|
23
|
+
<div class='stats-entry' title='Unsuccessful perform attempts'>
|
|
24
|
+
<span><%= "Failures: #{@summary['failures_count']}" %></span>
|
|
25
|
+
</div>
|
|
26
|
+
<div class='stats-entry' title='Successfully completed jobs'>
|
|
27
|
+
<span><%= "Processed: #{@summary['processed_count']}" %></span>
|
|
28
|
+
</div>
|
|
29
|
+
</div>
|
|
30
|
+
|
|
31
|
+
<table class='main-table processes'>
|
|
32
|
+
<thead>
|
|
33
|
+
<tr>
|
|
34
|
+
<th colspan='5' title='Process'>
|
|
35
|
+
<span>Process</span>
|
|
36
|
+
</th>
|
|
37
|
+
<th colspan='1' title='PID'>
|
|
38
|
+
<span>PID</span>
|
|
39
|
+
</th>
|
|
40
|
+
<th colspan='2' title='Concurrency'>
|
|
41
|
+
<span>Concurrency</span>
|
|
42
|
+
</th>
|
|
43
|
+
<th colspan='2' title='RSS'>
|
|
44
|
+
<span>RSS</span>
|
|
45
|
+
</th>
|
|
46
|
+
<th colspan='1' title='Last Heartbeat'>
|
|
47
|
+
<span>Heartbeat</span>
|
|
48
|
+
</th>
|
|
49
|
+
<th colspan='1' title='Started'>
|
|
50
|
+
<span>Started</span>
|
|
51
|
+
</th>
|
|
52
|
+
</tr>
|
|
53
|
+
</thead>
|
|
54
|
+
<tbody>
|
|
55
|
+
<% @summary['processes'].each do |uuid, data| %>
|
|
56
|
+
<tr class=<%= heartbeat_class(data['heartbeat']) %>>
|
|
57
|
+
<td colspan='5' title='Process'>
|
|
58
|
+
<span><%= "#{data['name']} - #{data['hostname']}" %></span>
|
|
59
|
+
<%= poller_badge(data['poller_enabled']) %>
|
|
60
|
+
</br>
|
|
61
|
+
<span><i><%= data['queues'] %></i></span>
|
|
62
|
+
</td>
|
|
63
|
+
<td colspan='1' title='PID' class='right'>
|
|
64
|
+
<span><%= data['pid'] %></span>
|
|
65
|
+
</td>
|
|
66
|
+
<td colspan='2' title='Concurrency' class='right'>
|
|
67
|
+
<span><%= data['concurrency'] %></span>
|
|
68
|
+
</td>
|
|
69
|
+
<td colspan='2' title=<%= data['rss'] %> class='right'>
|
|
70
|
+
<span><%= format_memory(data['rss']) %></span>
|
|
71
|
+
</td>
|
|
72
|
+
<td colspan='1' title=<%= format_datetime(data['heartbeat']) %> class='right'>
|
|
73
|
+
<span><%= time_ago_in_words(data['heartbeat']) %></span>
|
|
74
|
+
</td>
|
|
75
|
+
<td colspan='1' title=<%= format_datetime(data['started_at']) %> class='right'>
|
|
76
|
+
<span><%= time_ago_in_words(data['started_at']) %></span>
|
|
77
|
+
</td>
|
|
78
|
+
</tr>
|
|
79
|
+
<% end %>
|
|
80
|
+
</tbody>
|
|
81
|
+
</table>
|
|
82
|
+
|
|
83
|
+
<table class='main-table queues'>
|
|
84
|
+
<thead>
|
|
85
|
+
<tr>
|
|
86
|
+
<th colspan='6' title='Queue'>
|
|
87
|
+
<span>Queues</span>
|
|
88
|
+
</th>
|
|
89
|
+
<th colspan='6' title='Size'>
|
|
90
|
+
<span>Size</span>
|
|
91
|
+
</th>
|
|
92
|
+
</tr>
|
|
93
|
+
</thead>
|
|
94
|
+
<tbody>
|
|
95
|
+
<% @summary['queues'].each do |queue| %>
|
|
96
|
+
<tr>
|
|
97
|
+
<td colspan='6' title='Queue'>
|
|
98
|
+
<span><i><%= queue[0] %></i></span>
|
|
99
|
+
</td>
|
|
100
|
+
<td colspan='6' title='Size' class='right'>
|
|
101
|
+
<span><%= queue[1] %></span>
|
|
102
|
+
</td>
|
|
103
|
+
</tr>
|
|
104
|
+
<% end %>
|
|
105
|
+
</tbody>
|
|
106
|
+
</table>
|
|
107
|
+
|
|
108
|
+
<h3>Busy Jobs</h3>
|
|
109
|
+
<table class='main-table jobs'>
|
|
110
|
+
<thead>
|
|
111
|
+
<tr>
|
|
112
|
+
<th colspan='2' title='Process'>
|
|
113
|
+
<span>Process</span>
|
|
114
|
+
</th>
|
|
115
|
+
<th colspan='2' title='Job'>
|
|
116
|
+
<span>Job</span>
|
|
117
|
+
</th>
|
|
118
|
+
<th colspan='1' title='JID'>
|
|
119
|
+
<span>JID</span>
|
|
120
|
+
</th>
|
|
121
|
+
<th colspan='1' title='TID'>
|
|
122
|
+
<span>TID</span>
|
|
123
|
+
</th>
|
|
124
|
+
<th colspan='2' title='Queue'>
|
|
125
|
+
<span>Queue</span>
|
|
126
|
+
</th>
|
|
127
|
+
<th colspan='2' title='Args'>
|
|
128
|
+
<span>Args</span>
|
|
129
|
+
</th>
|
|
130
|
+
<th colspan='2' title='Started'>
|
|
131
|
+
<span>Started</span>
|
|
132
|
+
</th>
|
|
133
|
+
</tr>
|
|
134
|
+
</thead>
|
|
135
|
+
<tbody>
|
|
136
|
+
<% @summary['processes'].each do |uuid, process_data| %>
|
|
137
|
+
<% process_data['current_jobs'].each do |tid, job_data| %>
|
|
138
|
+
<tr>
|
|
139
|
+
<td colspan='2' title='Process'>
|
|
140
|
+
<span><%= uuid %></span></br>
|
|
141
|
+
</td>
|
|
142
|
+
<td colspan='2' title='Job'>
|
|
143
|
+
<span><%= job_data['job_args']['name'] %></span>
|
|
144
|
+
</td>
|
|
145
|
+
<td colspan='1' title='JID'>
|
|
146
|
+
<span><%= job_data['jid'] %></span>
|
|
147
|
+
</td>
|
|
148
|
+
<td colspan='1' title='TID'>
|
|
149
|
+
<span><%= tid %></span>
|
|
150
|
+
</td>
|
|
151
|
+
<td colspan='2' title='Queue'>
|
|
152
|
+
<span><%= job_data['queue'] %></span>
|
|
153
|
+
</td>
|
|
154
|
+
<td colspan='2' title='Arguments'>
|
|
155
|
+
<span><%= format_args(job_data['job_args']['args']) %></span>
|
|
156
|
+
</td>
|
|
157
|
+
<td colspan='2' title=<%= format_datetime(data['started_at']) %>>
|
|
158
|
+
<span><%= time_ago_in_words(data['started_at']) %></span>
|
|
159
|
+
</td>
|
|
160
|
+
</tr>
|
|
161
|
+
<% end %>
|
|
162
|
+
<% end %>
|
|
163
|
+
</tbody>
|
|
164
|
+
</table>
|
|
165
|
+
|
|
166
|
+
<% if @summary['retry_jobs_count'] > 0 %>
|
|
167
|
+
<h3>Last 5 Jobs to Retry</h3>
|
|
168
|
+
<table class='main-table retry_jobs'>
|
|
169
|
+
<thead>
|
|
170
|
+
<tr>
|
|
171
|
+
<th colspan='1' title='Job'>
|
|
172
|
+
<span>Job</span>
|
|
173
|
+
</th>
|
|
174
|
+
<th colspan='1' title='JID'>
|
|
175
|
+
<span>JID</span>
|
|
176
|
+
</th>
|
|
177
|
+
<th colspan='1' title='Queue'>
|
|
178
|
+
<span>Queue</span>
|
|
179
|
+
</th>
|
|
180
|
+
<th colspan='1' title='Error'>
|
|
181
|
+
<span>Error</span>
|
|
182
|
+
</th>
|
|
183
|
+
<th colspan='2' title='Args'>
|
|
184
|
+
<span>Args</span>
|
|
185
|
+
</th>
|
|
186
|
+
<th colspan='1' title='Attempt'>
|
|
187
|
+
<span>Attempt</span>
|
|
188
|
+
</th>
|
|
189
|
+
<th colspan='1' title='Started'>
|
|
190
|
+
<span>Started</span>
|
|
191
|
+
</th>
|
|
192
|
+
<th colspan='1' title='Retried'>
|
|
193
|
+
<span>Retried</span>
|
|
194
|
+
</th>
|
|
195
|
+
<th colspan='1' title='To Retry At'>
|
|
196
|
+
<span>To Retry At</span>
|
|
197
|
+
</th>
|
|
198
|
+
</tr>
|
|
199
|
+
</thead>
|
|
200
|
+
<tbody>
|
|
201
|
+
<% last_5_retry_jobs.each do |data| %>
|
|
202
|
+
<tr>
|
|
203
|
+
<td colspan='1' title='Job'>
|
|
204
|
+
<span><%= data['name'] %></span>
|
|
205
|
+
</td>
|
|
206
|
+
<td colspan='1' title='JID'>
|
|
207
|
+
<span><%= data['jid'] %></span>
|
|
208
|
+
</td>
|
|
209
|
+
<td colspan='1' title='Queue'>
|
|
210
|
+
<span><%= data['queue'] %></span>
|
|
211
|
+
</td>
|
|
212
|
+
<td colspan='1' title='Error'>
|
|
213
|
+
<span><%= "#{data['error_class']}: #{data['error_message']}" %></span>
|
|
214
|
+
</td>
|
|
215
|
+
<td colspan='2' title='Args'>
|
|
216
|
+
<span><%= data['args'] %></span>
|
|
217
|
+
</td>
|
|
218
|
+
<td colspan='1' title='Attempt' class='right'>
|
|
219
|
+
<span><%= data['attempt'] %></span>
|
|
220
|
+
</td>
|
|
221
|
+
<td colspan='1' title=<%= format_datetime(data['started_at']) %> class='right'>
|
|
222
|
+
<span><%= time_ago_in_words(data['started_at']) %></span>
|
|
223
|
+
</td>
|
|
224
|
+
<td colspan='1' title=<%= format_datetime(data['retried_at']) %> class='right'>
|
|
225
|
+
<span><%= time_ago_in_words(data['retried_at']) %></span>
|
|
226
|
+
</td>
|
|
227
|
+
<td colspan='1' title=<%= format_datetime(data['retry_at']) %> class='right'>
|
|
228
|
+
<span><%= format_datetime(data['retry_at']) %></span>
|
|
229
|
+
</td>
|
|
230
|
+
</tr>
|
|
231
|
+
<% end %>
|
|
232
|
+
</tbody>
|
|
233
|
+
</table>
|
|
234
|
+
<% end %>
|
|
235
|
+
|
|
236
|
+
<% if @summary['scheduled_jobs_count'] > 0 %>
|
|
237
|
+
<h3>Last 5 Scheduled Jobs</h3>
|
|
238
|
+
<table class='main-table retry_jobs'>
|
|
239
|
+
<thead>
|
|
240
|
+
<tr>
|
|
241
|
+
<th colspan='4' title='Job'>
|
|
242
|
+
<span>Job</span>
|
|
243
|
+
</th>
|
|
244
|
+
<th colspan='6' title='Args'>
|
|
245
|
+
<span>Args</span>
|
|
246
|
+
</th>
|
|
247
|
+
<th colspan='2' title='Scheduled At'>
|
|
248
|
+
<span>To be enqueued at</span>
|
|
249
|
+
</th>
|
|
250
|
+
</tr>
|
|
251
|
+
</thead>
|
|
252
|
+
<tbody>
|
|
253
|
+
<% last_5_scheduled_jobs.each do |data| %>
|
|
254
|
+
<tr>
|
|
255
|
+
<td colspan='4' title='Job'>
|
|
256
|
+
<span><%= data['name'] %></span>
|
|
257
|
+
</td>
|
|
258
|
+
<td colspan='6' title='Args'>
|
|
259
|
+
<span><%= format_args(data['args']) %></span>
|
|
260
|
+
</td>
|
|
261
|
+
<td colspan='2' title=<%= format_datetime(data['scheduled_at']) %> class='right'>
|
|
262
|
+
<span><%= format_datetime(data['scheduled_at']) %></span>
|
|
263
|
+
</td>
|
|
264
|
+
</tr>
|
|
265
|
+
<% end %>
|
|
266
|
+
</tbody>
|
|
267
|
+
</table>
|
|
268
|
+
<% end %>
|
|
269
|
+
|
|
270
|
+
<% if @summary['dead_jobs_count'] > 0 %>
|
|
271
|
+
<h3>Last 5 Dead Jobs</h3>
|
|
272
|
+
<table class='main-table dead_jobs'>
|
|
273
|
+
<thead>
|
|
274
|
+
<tr>
|
|
275
|
+
<th colspan='2' title='Job'>
|
|
276
|
+
<span>Job</span>
|
|
277
|
+
</th>
|
|
278
|
+
<th colspan='1' title='JID'>
|
|
279
|
+
<span>JID</span>
|
|
280
|
+
</th>
|
|
281
|
+
<th colspan='1' title='Queue'>
|
|
282
|
+
<span>Queue</span>
|
|
283
|
+
</th>
|
|
284
|
+
<th colspan='2' title='Error'>
|
|
285
|
+
<span>Error</span>
|
|
286
|
+
</th>
|
|
287
|
+
<th colspan='2' title='Args'>
|
|
288
|
+
<span>Args</span>
|
|
289
|
+
</th>
|
|
290
|
+
<th colspan='1' title='Started'>
|
|
291
|
+
<span>Started</span>
|
|
292
|
+
</th>
|
|
293
|
+
<th colspan='1' title='Retried'>
|
|
294
|
+
<span>Retried</span>
|
|
295
|
+
</th>
|
|
296
|
+
</tr>
|
|
297
|
+
</thead>
|
|
298
|
+
<tbody>
|
|
299
|
+
<% last_5_dead_jobs.each do |data| %>
|
|
300
|
+
<tr>
|
|
301
|
+
<td colspan='2' title='Job'>
|
|
302
|
+
<span><%= data['name'] %></span>
|
|
303
|
+
</td>
|
|
304
|
+
<td colspan='1' title='JID'>
|
|
305
|
+
<span><%= data['jid'] %></span>
|
|
306
|
+
</td>
|
|
307
|
+
<td colspan='1' title='Queue'>
|
|
308
|
+
<span><%= data['queue'] %></span>
|
|
309
|
+
</td>
|
|
310
|
+
<td colspan='2' title='Error'>
|
|
311
|
+
<span><%= "#{data['error_class']}: #{data['error_message']}" %></span>
|
|
312
|
+
</td>
|
|
313
|
+
<td colspan='2' title='Args'>
|
|
314
|
+
<span><%= format_args(data['args']) %></span>
|
|
315
|
+
</td>
|
|
316
|
+
<td colspan='1' title=<%= format_datetime(data['started_at']) %> class='right'>
|
|
317
|
+
<span><%= time_ago_in_words(data['started_at']) %></span>
|
|
318
|
+
</td>
|
|
319
|
+
<td colspan='1' title=<%= format_datetime(data['retried_at']) %> class='right'>
|
|
320
|
+
<span><%= time_ago_in_words(data['retried_at']) %></span>
|
|
321
|
+
</td>
|
|
322
|
+
</tr>
|
|
323
|
+
<% end %>
|
|
324
|
+
</tbody>
|
|
325
|
+
</table>
|
|
326
|
+
<% end %>
|
|
327
|
+
</div>
|
|
328
|
+
</body>
|
|
329
|
+
</html>
|
data/lib/jiggler/web.rb
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'erb'
|
|
4
|
+
|
|
5
|
+
module Jiggler
|
|
6
|
+
class Web
|
|
7
|
+
WEB_PATH = File.expand_path("#{File.dirname(__FILE__)}/web")
|
|
8
|
+
LAYOUT = "#{WEB_PATH}/views/application.erb"
|
|
9
|
+
STYLESHEET = "#{WEB_PATH}/assets/stylesheets/application.css"
|
|
10
|
+
|
|
11
|
+
def call(env)
|
|
12
|
+
@summary_instance = Jiggler::Summary.new(Jiggler.config)
|
|
13
|
+
@summary = @summary_instance.all
|
|
14
|
+
compiled_template = ERB.new(File.read(LAYOUT)).result(binding)
|
|
15
|
+
[200, {}, [compiled_template]]
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def last_5_dead_jobs
|
|
19
|
+
@summary_instance.last_dead_jobs(5)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def last_5_retry_jobs
|
|
23
|
+
@summary_instance.last_retry_jobs(5)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def last_5_scheduled_jobs
|
|
27
|
+
@summary_instance.last_scheduled_jobs(5)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def format_datetime(timestamp)
|
|
31
|
+
return if timestamp.nil?
|
|
32
|
+
Time.at(timestamp.to_f).to_datetime
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def format_memory(kb)
|
|
36
|
+
return '?' if kb.nil?
|
|
37
|
+
"#{(kb/1024.0).round(2)} MB"
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def time_ago_in_words(timestamp)
|
|
41
|
+
return if timestamp.nil?
|
|
42
|
+
seconds = Time.now.to_i - timestamp.to_i
|
|
43
|
+
case seconds
|
|
44
|
+
when 0..59
|
|
45
|
+
"#{seconds} seconds ago"
|
|
46
|
+
when 60..3599
|
|
47
|
+
"#{(seconds/60).round} minutes ago"
|
|
48
|
+
when 3600..86399
|
|
49
|
+
"#{(seconds/3600).round} hours ago"
|
|
50
|
+
when 86400..604799
|
|
51
|
+
"#{(seconds/86400).round} days ago"
|
|
52
|
+
else
|
|
53
|
+
"#{(seconds/604800).round} weeks ago"
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def heartbeat_class(timestamp)
|
|
58
|
+
return 'outdated' if outdated_heartbeat?(timestamp)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def outdated_heartbeat?(timestamp)
|
|
62
|
+
return true if timestamp.nil?
|
|
63
|
+
seconds = Time.now.to_i - timestamp.to_i
|
|
64
|
+
seconds > Jiggler.config[:stats_interval] * 2
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def format_args(args)
|
|
68
|
+
return if args.nil?
|
|
69
|
+
args.map { |arg| Oj.dump(arg, mode: :compat) }.join(', ')
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def poller_badge(poller_enabled)
|
|
73
|
+
poller_enabled ? '<span class=\'badge badge-success\'>Polling</span>' : '<span class=\'badge\'>Polling Disabled</span>'
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def styles
|
|
77
|
+
@styles ||= File.read(STYLESHEET)
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Jiggler
|
|
4
|
+
class Worker
|
|
5
|
+
include Support::Helper
|
|
6
|
+
TIMEOUT = 2 # timeout for brpop
|
|
7
|
+
|
|
8
|
+
CurrentJob = Struct.new(:queue, :args, keyword_init: true)
|
|
9
|
+
|
|
10
|
+
attr_reader :current_job, :config, :done, :collection
|
|
11
|
+
|
|
12
|
+
def initialize(config, collection, &callback)
|
|
13
|
+
@done = false
|
|
14
|
+
@current_job = nil
|
|
15
|
+
@callback = callback
|
|
16
|
+
@config = config
|
|
17
|
+
@collection = collection
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def run
|
|
21
|
+
reason = nil
|
|
22
|
+
@runner = safe_async('Worker') do
|
|
23
|
+
@tid = tid
|
|
24
|
+
loop do
|
|
25
|
+
if @done
|
|
26
|
+
@runner = nil
|
|
27
|
+
break @callback.call(self)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
process_job
|
|
31
|
+
|
|
32
|
+
# pass control to other fibers
|
|
33
|
+
Async::Task.current.yield
|
|
34
|
+
rescue Async::Stop
|
|
35
|
+
@runner = nil
|
|
36
|
+
break @callback.call(self)
|
|
37
|
+
rescue => err
|
|
38
|
+
collection.incr_failures
|
|
39
|
+
@runner = nil
|
|
40
|
+
break @callback.call(self, err)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def terminate
|
|
46
|
+
@done = true
|
|
47
|
+
@runner&.stop
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def suspend
|
|
51
|
+
@done = true
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def wait
|
|
55
|
+
@runner&.wait
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
private
|
|
59
|
+
|
|
60
|
+
def process_job
|
|
61
|
+
@current_job = fetch_one
|
|
62
|
+
return if current_job.nil? # timed out brpop or done
|
|
63
|
+
execute_job
|
|
64
|
+
@current_job = nil
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def fetch_one
|
|
68
|
+
queue, args = config.with_sync_redis { |conn| conn.blocking_call(false, 'BRPOP', *queues, TIMEOUT) }
|
|
69
|
+
return nil unless queue
|
|
70
|
+
|
|
71
|
+
if @done
|
|
72
|
+
requeue(queue, args)
|
|
73
|
+
nil
|
|
74
|
+
else
|
|
75
|
+
CurrentJob.new(queue: queue, args: args)
|
|
76
|
+
end
|
|
77
|
+
rescue Async::Stop => err
|
|
78
|
+
raise err
|
|
79
|
+
rescue => err
|
|
80
|
+
# error_message='undefined method `zero?' for nil:NilClass' context='Fetch error' tid=1tpj
|
|
81
|
+
# sometimes happens in async-pool-0.3.12/lib/async/pool/controller.rb:213:in `reuse'
|
|
82
|
+
handle_fetch_error(err)
|
|
83
|
+
nil
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def execute_job
|
|
87
|
+
parsed_args = Oj.load(current_job.args, mode: :compat)
|
|
88
|
+
execute(parsed_args, current_job.queue)
|
|
89
|
+
rescue Async::Stop => err
|
|
90
|
+
raise err
|
|
91
|
+
rescue UnknownJobError => err
|
|
92
|
+
collection.incr_failures
|
|
93
|
+
log_error_short(
|
|
94
|
+
err,
|
|
95
|
+
{
|
|
96
|
+
error_class: err.class.name,
|
|
97
|
+
job: parsed_args,
|
|
98
|
+
tid: @tid
|
|
99
|
+
}
|
|
100
|
+
)
|
|
101
|
+
rescue JSON::ParserError => err
|
|
102
|
+
collection.incr_failures
|
|
103
|
+
logger.error('Worker') { "Failed to parse job: #{current_job.args}" }
|
|
104
|
+
rescue Exception => ex
|
|
105
|
+
log_error(
|
|
106
|
+
ex,
|
|
107
|
+
{
|
|
108
|
+
context: '\'Internal exception\'',
|
|
109
|
+
tid: @tid,
|
|
110
|
+
jid: parsed_args['jid']
|
|
111
|
+
}
|
|
112
|
+
)
|
|
113
|
+
# raise ex
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def execute(parsed_job, queue)
|
|
117
|
+
instance = constantize(parsed_job['name']).new
|
|
118
|
+
add_current_job_to_collection(parsed_job, parsed_job['queue'])
|
|
119
|
+
|
|
120
|
+
retrier.wrapped(instance, parsed_job, queue) do
|
|
121
|
+
instance.perform(*parsed_job['args'])
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
collection.incr_processed
|
|
125
|
+
ensure
|
|
126
|
+
remove_current_job_from_collection
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def retrier
|
|
130
|
+
@retrier ||= Jiggler::Retrier.new(config, collection)
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def requeue(queue, args)
|
|
134
|
+
config.with_async_redis do |conn|
|
|
135
|
+
conn.call('RPUSH', queue, args)
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def handle_fetch_error(ex)
|
|
140
|
+
log_error_short(
|
|
141
|
+
ex,
|
|
142
|
+
{
|
|
143
|
+
context: '\'Fetch error\'',
|
|
144
|
+
tid: @tid
|
|
145
|
+
}
|
|
146
|
+
)
|
|
147
|
+
sleep(TIMEOUT + rand(5) * config[:concurrency]) # sleep for a while before retrying
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def add_current_job_to_collection(parsed_job, queue)
|
|
151
|
+
collection.data[:current_jobs][@tid] = {
|
|
152
|
+
job_args: parsed_job,
|
|
153
|
+
queue: queue,
|
|
154
|
+
started_at: Time.now.to_f
|
|
155
|
+
}
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def remove_current_job_from_collection
|
|
159
|
+
collection.data[:current_jobs].delete(@tid)
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def queues
|
|
163
|
+
@queues ||= config.prefixed_queues
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def constantize(str)
|
|
167
|
+
return Object.const_get(str) unless str.include?('::')
|
|
168
|
+
|
|
169
|
+
names = str.split('::')
|
|
170
|
+
names.shift if names.empty? || names.first.empty?
|
|
171
|
+
|
|
172
|
+
names.inject(Object) do |constant, name|
|
|
173
|
+
constant.const_get(name, false)
|
|
174
|
+
end
|
|
175
|
+
rescue => err
|
|
176
|
+
raise UnknownJobError, 'Cannot initialize job'
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
end
|
data/lib/jiggler.rb
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# require only classes which are going to be used on the client side
|
|
4
|
+
require 'jiggler/redis_store'
|
|
5
|
+
require 'jiggler/config'
|
|
6
|
+
require 'jiggler/cleaner'
|
|
7
|
+
require 'jiggler/summary'
|
|
8
|
+
require 'jiggler/version'
|
|
9
|
+
require 'jiggler/core'
|
|
10
|
+
require 'jiggler/job'
|
data/spec/examples.txt
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
example_id | status | run_time |
|
|
2
|
+
------------------------------------------------ | ------- | --------------- |
|
|
3
|
+
./spec/jiggler/cleaner_spec.rb[1:1:1] | passed | 0.00079 seconds |
|
|
4
|
+
./spec/jiggler/cleaner_spec.rb[1:2:1] | passed | 0.00104 seconds |
|
|
5
|
+
./spec/jiggler/cleaner_spec.rb[1:3:1] | passed | 0.00427 seconds |
|
|
6
|
+
./spec/jiggler/cleaner_spec.rb[1:4:1] | passed | 0.00071 seconds |
|
|
7
|
+
./spec/jiggler/cleaner_spec.rb[1:5:1] | passed | 0.00079 seconds |
|
|
8
|
+
./spec/jiggler/cleaner_spec.rb[1:6:1] | passed | 0.00071 seconds |
|
|
9
|
+
./spec/jiggler/cleaner_spec.rb[1:7:1] | passed | 0.00063 seconds |
|
|
10
|
+
./spec/jiggler/cleaner_spec.rb[1:8:1] | passed | 0.003 seconds |
|
|
11
|
+
./spec/jiggler/cleaner_spec.rb[1:9:1] | passed | 0.00077 seconds |
|
|
12
|
+
./spec/jiggler/cleaner_spec.rb[1:10:1] | passed | 0.00122 seconds |
|
|
13
|
+
./spec/jiggler/cleaner_spec.rb[1:11:1] | passed | 0.00286 seconds |
|
|
14
|
+
./spec/jiggler/cli_spec.rb[1:1:1:1] | passed | 0.00403 seconds |
|
|
15
|
+
./spec/jiggler/cli_spec.rb[1:1:2:1] | passed | 0.76231 seconds |
|
|
16
|
+
./spec/jiggler/cli_spec.rb[1:1:2:2] | passed | 0.00644 seconds |
|
|
17
|
+
./spec/jiggler/cli_spec.rb[1:1:2:3] | passed | 0.00535 seconds |
|
|
18
|
+
./spec/jiggler/cli_spec.rb[1:1:2:4] | passed | 0.0202 seconds |
|
|
19
|
+
./spec/jiggler/cli_spec.rb[1:1:2:5] | passed | 0.004 seconds |
|
|
20
|
+
./spec/jiggler/cli_spec.rb[1:1:2:6] | passed | 0.00955 seconds |
|
|
21
|
+
./spec/jiggler/cli_spec.rb[1:1:2:7] | passed | 0.02139 seconds |
|
|
22
|
+
./spec/jiggler/cli_spec.rb[1:1:2:8] | passed | 0.00429 seconds |
|
|
23
|
+
./spec/jiggler/cli_spec.rb[1:1:2:9] | passed | 0.00746 seconds |
|
|
24
|
+
./spec/jiggler/cli_spec.rb[1:1:3:1] | passed | 0.00336 seconds |
|
|
25
|
+
./spec/jiggler/cli_spec.rb[1:1:3:2] | passed | 0.00497 seconds |
|
|
26
|
+
./spec/jiggler/cli_spec.rb[1:1:3:3] | passed | 0.00312 seconds |
|
|
27
|
+
./spec/jiggler/cli_spec.rb[1:1:3:4] | passed | 0.0086 seconds |
|
|
28
|
+
./spec/jiggler/cli_spec.rb[1:1:3:5] | unknown | |
|
|
29
|
+
./spec/jiggler/config_spec.rb[1:1:1] | passed | 0.00055 seconds |
|
|
30
|
+
./spec/jiggler/config_spec.rb[1:1:2] | passed | 0.00057 seconds |
|
|
31
|
+
./spec/jiggler/config_spec.rb[1:1:3] | passed | 0.00004 seconds |
|
|
32
|
+
./spec/jiggler/config_spec.rb[1:1:4] | passed | 0.00004 seconds |
|
|
33
|
+
./spec/jiggler/core_spec.rb[1:1:1] | passed | 0.00103 seconds |
|
|
34
|
+
./spec/jiggler/job_spec.rb[1:1:1:1] | passed | 0.00007 seconds |
|
|
35
|
+
./spec/jiggler/job_spec.rb[1:1:2:1] | passed | 0.0009 seconds |
|
|
36
|
+
./spec/jiggler/job_spec.rb[1:2:1] | passed | 0.00064 seconds |
|
|
37
|
+
./spec/jiggler/job_spec.rb[1:3:1] | passed | 0.00341 seconds |
|
|
38
|
+
./spec/jiggler/job_spec.rb[1:4:1] | passed | 0.00051 seconds |
|
|
39
|
+
./spec/jiggler/job_spec.rb[1:4:2] | passed | 0.00073 seconds |
|
|
40
|
+
./spec/jiggler/job_spec.rb[1:5:1] | passed | 0.00051 seconds |
|
|
41
|
+
./spec/jiggler/launcher_spec.rb[1:1:1] | passed | 0.00598 seconds |
|
|
42
|
+
./spec/jiggler/launcher_spec.rb[1:2:1] | passed | 2.02 seconds |
|
|
43
|
+
./spec/jiggler/launcher_spec.rb[1:3:1] | passed | 2.1 seconds |
|
|
44
|
+
./spec/jiggler/launcher_spec.rb[1:4:1] | passed | 2.02 seconds |
|
|
45
|
+
./spec/jiggler/manager_spec.rb[1:1] | passed | 0.00094 seconds |
|
|
46
|
+
./spec/jiggler/manager_spec.rb[1:2:1] | passed | 1.53 seconds |
|
|
47
|
+
./spec/jiggler/manager_spec.rb[1:3:1] | passed | 2.14 seconds |
|
|
48
|
+
./spec/jiggler/manager_spec.rb[1:4:1] | passed | 3.56 seconds |
|
|
49
|
+
./spec/jiggler/redis_store_spec.rb[1:1:1] | passed | 0.00073 seconds |
|
|
50
|
+
./spec/jiggler/redis_store_spec.rb[1:1:2:1] | passed | 0.00071 seconds |
|
|
51
|
+
./spec/jiggler/retrier_spec.rb[1:1:1:1] | passed | 0.02616 seconds |
|
|
52
|
+
./spec/jiggler/retrier_spec.rb[1:1:1:2] | passed | 0.0143 seconds |
|
|
53
|
+
./spec/jiggler/retrier_spec.rb[1:1:2:1] | passed | 0.01566 seconds |
|
|
54
|
+
./spec/jiggler/scheduled/enqueuer_spec.rb[1:1:1] | passed | 0.01203 seconds |
|
|
55
|
+
./spec/jiggler/scheduled/enqueuer_spec.rb[1:1:2] | passed | 0.02395 seconds |
|
|
56
|
+
./spec/jiggler/scheduled/enqueuer_spec.rb[1:1:3] | passed | 0.0107 seconds |
|
|
57
|
+
./spec/jiggler/scheduled/enqueuer_spec.rb[1:2:1] | passed | 0.03692 seconds |
|
|
58
|
+
./spec/jiggler/scheduled/poller_spec.rb[1:1:1] | passed | 1.02 seconds |
|
|
59
|
+
./spec/jiggler/scheduled/poller_spec.rb[1:2:1] | passed | 1.02 seconds |
|
|
60
|
+
./spec/jiggler/stats/monitor_spec.rb[1:1:1] | passed | 2.02 seconds |
|
|
61
|
+
./spec/jiggler/stats/monitor_spec.rb[1:2:1] | passed | 1.03 seconds |
|
|
62
|
+
./spec/jiggler/summary_spec.rb[1:1:1] | passed | 0.00208 seconds |
|
|
63
|
+
./spec/jiggler/summary_spec.rb[1:1:2] | passed | 3.04 seconds |
|
|
64
|
+
./spec/jiggler/summary_spec.rb[1:1:3:1] | passed | 0.0076 seconds |
|
|
65
|
+
./spec/jiggler/summary_spec.rb[1:2:1] | passed | 1.01 seconds |
|
|
66
|
+
./spec/jiggler/summary_spec.rb[1:3:1] | passed | 1.01 seconds |
|
|
67
|
+
./spec/jiggler/summary_spec.rb[1:4:1] | passed | 0.00461 seconds |
|
|
68
|
+
./spec/jiggler/web_spec.rb[1:1:1] | passed | 0.00004 seconds |
|
|
69
|
+
./spec/jiggler/web_spec.rb[1:1:2] | passed | 0.0005 seconds |
|
|
70
|
+
./spec/jiggler/web_spec.rb[1:2:1] | passed | 0.00005 seconds |
|
|
71
|
+
./spec/jiggler/web_spec.rb[1:2:2] | passed | 0.00044 seconds |
|
|
72
|
+
./spec/jiggler/web_spec.rb[1:3:1] | passed | 0.00037 seconds |
|
|
73
|
+
./spec/jiggler/web_spec.rb[1:3:2] | passed | 0.00004 seconds |
|
|
74
|
+
./spec/jiggler/worker_spec.rb[1:1:1] | passed | 1.01 seconds |
|
|
75
|
+
./spec/jiggler/worker_spec.rb[1:1:2] | passed | 1.01 seconds |
|
|
76
|
+
./spec/jiggler/worker_spec.rb[1:1:3] | passed | 1.01 seconds |
|
|
77
|
+
./spec/jiggler/worker_spec.rb[1:2:1] | passed | 0.00527 seconds |
|
|
78
|
+
./spec/jiggler/worker_spec.rb[1:3:1:1] | passed | 2.05 seconds |
|
|
79
|
+
./spec/jiggler/worker_spec.rb[1:3:2:1] | passed | 0.00371 seconds |
|