wurk 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +43 -0
- data/CONTRIBUTING.md +73 -0
- data/LICENSE +21 -0
- data/README.md +137 -0
- data/SECURITY.md +39 -0
- data/app/controllers/wurk/api/pagination.rb +67 -0
- data/app/controllers/wurk/api/serializers.rb +131 -0
- data/app/controllers/wurk/api_controller.rb +248 -0
- data/app/controllers/wurk/application_controller.rb +7 -0
- data/app/controllers/wurk/dashboard_controller.rb +48 -0
- data/config/locales/en.yml +15 -0
- data/config/routes.rb +34 -0
- data/exe/wurk +22 -0
- data/lib/active_job/queue_adapters/wurk_adapter.rb +96 -0
- data/lib/generators/wurk/install/install_generator.rb +22 -0
- data/lib/generators/wurk/install/templates/wurk.rb +16 -0
- data/lib/wurk/active_job/wrapper.rb +32 -0
- data/lib/wurk/api/fast.rb +78 -0
- data/lib/wurk/batch/buffer.rb +26 -0
- data/lib/wurk/batch/callback_job.rb +37 -0
- data/lib/wurk/batch/callbacks.rb +176 -0
- data/lib/wurk/batch/client_middleware.rb +27 -0
- data/lib/wurk/batch/death_handler.rb +39 -0
- data/lib/wurk/batch/empty.rb +21 -0
- data/lib/wurk/batch/server_middleware.rb +62 -0
- data/lib/wurk/batch/status.rb +140 -0
- data/lib/wurk/batch.rb +351 -0
- data/lib/wurk/batch_set.rb +67 -0
- data/lib/wurk/capsule.rb +176 -0
- data/lib/wurk/cli.rb +349 -0
- data/lib/wurk/client/buffered.rb +372 -0
- data/lib/wurk/client.rb +330 -0
- data/lib/wurk/compat.rb +136 -0
- data/lib/wurk/component.rb +136 -0
- data/lib/wurk/configuration.rb +373 -0
- data/lib/wurk/context.rb +35 -0
- data/lib/wurk/cron.rb +636 -0
- data/lib/wurk/dashboard_manifest.rb +39 -0
- data/lib/wurk/dead_set.rb +78 -0
- data/lib/wurk/deploy.rb +91 -0
- data/lib/wurk/embedded.rb +94 -0
- data/lib/wurk/encryption.rb +276 -0
- data/lib/wurk/engine.rb +81 -0
- data/lib/wurk/fetcher/reaper.rb +264 -0
- data/lib/wurk/fetcher/reliable.rb +138 -0
- data/lib/wurk/fetcher.rb +11 -0
- data/lib/wurk/health.rb +193 -0
- data/lib/wurk/heartbeat.rb +211 -0
- data/lib/wurk/iterable_job.rb +292 -0
- data/lib/wurk/job/options.rb +70 -0
- data/lib/wurk/job.rb +33 -0
- data/lib/wurk/job_logger.rb +68 -0
- data/lib/wurk/job_record.rb +156 -0
- data/lib/wurk/job_retry.rb +320 -0
- data/lib/wurk/job_set.rb +212 -0
- data/lib/wurk/job_util.rb +162 -0
- data/lib/wurk/keys.rb +52 -0
- data/lib/wurk/launcher.rb +289 -0
- data/lib/wurk/leader.rb +221 -0
- data/lib/wurk/limiter/base.rb +138 -0
- data/lib/wurk/limiter/bucket.rb +80 -0
- data/lib/wurk/limiter/concurrent.rb +132 -0
- data/lib/wurk/limiter/leaky.rb +91 -0
- data/lib/wurk/limiter/points.rb +89 -0
- data/lib/wurk/limiter/server_middleware.rb +77 -0
- data/lib/wurk/limiter/unlimited.rb +48 -0
- data/lib/wurk/limiter/window.rb +80 -0
- data/lib/wurk/limiter.rb +255 -0
- data/lib/wurk/logger.rb +81 -0
- data/lib/wurk/lua/loader.rb +53 -0
- data/lib/wurk/lua.rb +187 -0
- data/lib/wurk/manager.rb +132 -0
- data/lib/wurk/metrics/history.rb +151 -0
- data/lib/wurk/metrics/query.rb +173 -0
- data/lib/wurk/metrics/rollup.rb +169 -0
- data/lib/wurk/metrics/statsd.rb +197 -0
- data/lib/wurk/metrics.rb +7 -0
- data/lib/wurk/middleware/chain.rb +128 -0
- data/lib/wurk/middleware/current_attributes.rb +87 -0
- data/lib/wurk/middleware/expiry.rb +50 -0
- data/lib/wurk/middleware/i18n.rb +63 -0
- data/lib/wurk/middleware/interrupt_handler.rb +45 -0
- data/lib/wurk/middleware/poison_pill.rb +149 -0
- data/lib/wurk/middleware.rb +34 -0
- data/lib/wurk/process_set.rb +243 -0
- data/lib/wurk/processor.rb +247 -0
- data/lib/wurk/queue.rb +108 -0
- data/lib/wurk/queues.rb +80 -0
- data/lib/wurk/rails.rb +9 -0
- data/lib/wurk/railtie.rb +28 -0
- data/lib/wurk/redis_pool.rb +79 -0
- data/lib/wurk/retry_set.rb +17 -0
- data/lib/wurk/scheduled.rb +189 -0
- data/lib/wurk/scheduled_set.rb +18 -0
- data/lib/wurk/sorted_entry.rb +95 -0
- data/lib/wurk/stats.rb +190 -0
- data/lib/wurk/swarm/child_boot.rb +105 -0
- data/lib/wurk/swarm.rb +260 -0
- data/lib/wurk/testing.rb +102 -0
- data/lib/wurk/topology.rb +74 -0
- data/lib/wurk/unique.rb +240 -0
- data/lib/wurk/version.rb +5 -0
- data/lib/wurk/web/config.rb +180 -0
- data/lib/wurk/web/enterprise.rb +138 -0
- data/lib/wurk/web/search.rb +139 -0
- data/lib/wurk/web.rb +25 -0
- data/lib/wurk/work_set.rb +116 -0
- data/lib/wurk/worker/setter.rb +93 -0
- data/lib/wurk/worker.rb +216 -0
- data/lib/wurk.rb +238 -0
- data/vendor/assets/dashboard/assets/index-8P3N_m1X.js +152 -0
- data/vendor/assets/dashboard/assets/index-Bqz4_SOQ.css +1 -0
- data/vendor/assets/dashboard/index.html +13 -0
- data/vendor/assets/dashboard/wurk-manifest.json +4 -0
- metadata +232 -0
data/lib/wurk/cron.rb
ADDED
|
@@ -0,0 +1,636 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'digest'
|
|
5
|
+
require 'time'
|
|
6
|
+
require_relative 'component'
|
|
7
|
+
require_relative 'client'
|
|
8
|
+
require_relative 'leader'
|
|
9
|
+
|
|
10
|
+
module Wurk
|
|
11
|
+
# Sidekiq Enterprise periodic jobs. Pure leader-driven cron — only the
|
|
12
|
+
# elected leader enqueues per tick; followers run nothing. No backfill
|
|
13
|
+
# on restart. DST-aware via per-loop timezone. In-tree crontab parser
|
|
14
|
+
# (no fugit dependency) supporting 5-field expressions plus the standard
|
|
15
|
+
# `@hourly` / `@daily` / `@weekly` / `@monthly` / `@yearly` aliases.
|
|
16
|
+
#
|
|
17
|
+
# Spec: docs/target/sidekiq-ent.md §2.
|
|
18
|
+
#
|
|
19
|
+
# Layout:
|
|
20
|
+
# * `Cron::Parser` — crontab → wall-clock match + `next_fire_at`. Walks
|
|
21
|
+
# forward minute-by-minute in the loop's TZ; DST gaps are skipped
|
|
22
|
+
# naturally because the wall-clock components advance past them.
|
|
23
|
+
# * `Cron::Loop` — one registered job. Identity = SHA1(schedule+klass+opts)
|
|
24
|
+
# so a re-registration of the same loop is idempotent.
|
|
25
|
+
# * `Cron::Manager` — registration DSL. `mgr.register(cron, klass, **opts)`
|
|
26
|
+
# with `tz=` mass-setter. Writes to Redis (`periodic` SET + `loops:{lid}`
|
|
27
|
+
# HASH).
|
|
28
|
+
# * `Cron::LoopSet` — Enumerable view (`each`/`size`/`fetch(lid)`).
|
|
29
|
+
# * `Cron::ConfigTester` — boot-time validator. Verifies cron syntax and
|
|
30
|
+
# that every worker class constant resolves.
|
|
31
|
+
# * `Cron::Poller` — once-per-minute tick loop. Only the cluster leader
|
|
32
|
+
# (`Component#leader?` / `dear-leader`) enqueues; non-leaders return
|
|
33
|
+
# early without iterating loops.
|
|
34
|
+
#
|
|
35
|
+
# Wire-compat: `periodic`, `loops:{lid}`, `loop-history:{lid}` per
|
|
36
|
+
# docs/target/sidekiq-ent.md §2.7. Periodic enqueue is gated by the single
|
|
37
|
+
# cluster leader (§6, `Component#leader?`) rather than a separate
|
|
38
|
+
# `cron-leader` lock — see the §2.7 divergence note.
|
|
39
|
+
module Cron
|
|
40
|
+
PERIODIC_KEY = 'periodic'
|
|
41
|
+
LOOP_PREFIX = 'loops:'
|
|
42
|
+
HISTORY_PREFIX = 'loop-history:'
|
|
43
|
+
|
|
44
|
+
HISTORY_CAP = 25
|
|
45
|
+
DEFAULT_TICK_SECONDS = 60
|
|
46
|
+
MISSED_TICK_THRESHOLD = 90
|
|
47
|
+
|
|
48
|
+
# 5-field crontab + `@aliases`. No seconds field, no DOW name aliases
|
|
49
|
+
# (sidekiq-ent §2.2 uses numeric DOW only).
|
|
50
|
+
class Parser
|
|
51
|
+
FIELDS = [
|
|
52
|
+
[0, 59],
|
|
53
|
+
[0, 23],
|
|
54
|
+
[1, 31],
|
|
55
|
+
[1, 12],
|
|
56
|
+
[0, 7]
|
|
57
|
+
].freeze
|
|
58
|
+
|
|
59
|
+
ALIASES = {
|
|
60
|
+
'@hourly' => '0 * * * *',
|
|
61
|
+
'@daily' => '0 0 * * *',
|
|
62
|
+
'@midnight' => '0 0 * * *',
|
|
63
|
+
'@weekly' => '0 0 * * 0',
|
|
64
|
+
'@monthly' => '0 0 1 * *',
|
|
65
|
+
'@yearly' => '0 0 1 1 *',
|
|
66
|
+
'@annually' => '0 0 1 1 *'
|
|
67
|
+
}.freeze
|
|
68
|
+
|
|
69
|
+
MAX_LOOKAHEAD_MINUTES = 366 * 24 * 60 * 4
|
|
70
|
+
|
|
71
|
+
attr_reader :expression, :fields
|
|
72
|
+
|
|
73
|
+
def initialize(expression)
|
|
74
|
+
parts = normalize_expression(expression)
|
|
75
|
+
@expression = parts.join(' ')
|
|
76
|
+
@fields = parts.each_with_index.map { |part, i| parse_field(part, *FIELDS[i]) }
|
|
77
|
+
@dom_restricted = parts[2] != '*'
|
|
78
|
+
@dow_restricted = parts[4] != '*'
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Smallest UTC epoch strictly greater than `from_epoch` whose wall-clock
|
|
82
|
+
# components in `tz` match every field. Returns nil if no match within
|
|
83
|
+
# ~4 years (a malformed loop like Feb 29 in a non-leap span).
|
|
84
|
+
def next_fire_at(from_epoch, tz = nil)
|
|
85
|
+
t = ((from_epoch.to_i / 60) + 1) * 60
|
|
86
|
+
MAX_LOOKAHEAD_MINUTES.times do
|
|
87
|
+
wc = wall_clock(t, tz)
|
|
88
|
+
return t if match_components?(wc)
|
|
89
|
+
|
|
90
|
+
t += 60
|
|
91
|
+
end
|
|
92
|
+
nil
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def match?(time, tz = nil)
|
|
96
|
+
match_components?(wall_clock(time.to_i, tz))
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Local wall-clock [min, hour, day, mon, dow] for `epoch` in `tz`. Public
|
|
100
|
+
# so the Loop/Poller can detect a DST fall-back fold — two distinct UTC
|
|
101
|
+
# instants that share the same local minute — and avoid a double-fire.
|
|
102
|
+
def local_components(epoch, tz = nil)
|
|
103
|
+
wall_clock(epoch, tz)
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
private
|
|
107
|
+
|
|
108
|
+
def match_components?(components)
|
|
109
|
+
min, hour, dom, mon, dow = components
|
|
110
|
+
return false unless @fields[0].include?(min)
|
|
111
|
+
return false unless @fields[1].include?(hour)
|
|
112
|
+
return false unless @fields[3].include?(mon)
|
|
113
|
+
|
|
114
|
+
match_day?(dom, dow)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# Cron quirk (§2.6): if both dom and dow are restricted, OR them so
|
|
118
|
+
# `0 0 13 * 5` matches "Friday the 13th". If only one is set, that
|
|
119
|
+
# one must match. Both wild → always match the day half.
|
|
120
|
+
def match_day?(dom, dow)
|
|
121
|
+
dom_ok = @fields[2].include?(dom)
|
|
122
|
+
dow_ok = @fields[4].include?(dow)
|
|
123
|
+
return dom_ok || dow_ok if @dom_restricted && @dow_restricted
|
|
124
|
+
return dom_ok if @dom_restricted
|
|
125
|
+
return dow_ok if @dow_restricted
|
|
126
|
+
|
|
127
|
+
true
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def normalize_expression(expression)
|
|
131
|
+
raise ArgumentError, 'cron expression must be a String' unless expression.is_a?(String)
|
|
132
|
+
|
|
133
|
+
normalized = (ALIASES[expression.strip] || expression).strip
|
|
134
|
+
parts = normalized.split(/\s+/)
|
|
135
|
+
return parts if parts.size == 5
|
|
136
|
+
|
|
137
|
+
raise ArgumentError, "cron expression must have 5 fields (got #{parts.size}): #{expression.inspect}"
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def parse_field(part, min, max)
|
|
141
|
+
raw = part == '*' ? (min..max).to_a : part.split(',').flat_map { |c| parse_chunk(c, min, max) }
|
|
142
|
+
normalized = raw.map { |v| max == 7 && v == 7 ? 0 : v }
|
|
143
|
+
Set.new(normalized)
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def parse_chunk(chunk, min, max)
|
|
147
|
+
if chunk.include?('/')
|
|
148
|
+
parse_step(chunk, min, max)
|
|
149
|
+
elsif chunk.include?('-')
|
|
150
|
+
parse_range(chunk, min, max)
|
|
151
|
+
else
|
|
152
|
+
v = Integer(chunk)
|
|
153
|
+
validate_value!(v, min, max)
|
|
154
|
+
[v]
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def parse_step(chunk, min, max)
|
|
159
|
+
base, step = chunk.split('/', 2)
|
|
160
|
+
step_i = Integer(step)
|
|
161
|
+
raise ArgumentError, "cron step must be >= 1 (got #{step_i})" if step_i < 1
|
|
162
|
+
|
|
163
|
+
base_values = base == '*' ? (min..max).to_a : parse_chunk(base, min, max)
|
|
164
|
+
start = base_values.first
|
|
165
|
+
base_values.select { |v| ((v - start) % step_i).zero? }
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
def parse_range(chunk, min, max)
|
|
169
|
+
a, b = chunk.split('-', 2).map { |x| Integer(x) }
|
|
170
|
+
raise ArgumentError, "cron range start > end (#{a} > #{b})" if a > b
|
|
171
|
+
|
|
172
|
+
validate_value!(a, min, max)
|
|
173
|
+
validate_value!(b, min, max)
|
|
174
|
+
(a..b).to_a
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def validate_value!(v, min, max)
|
|
178
|
+
return if v.between?(min, max)
|
|
179
|
+
|
|
180
|
+
raise ArgumentError, "cron value #{v} out of range #{min}..#{max}"
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# epoch → [min, hour, dom, mon, dow] in `tz`. Accepts:
|
|
184
|
+
# * nil → UTC
|
|
185
|
+
# * AS::TimeZone → responds to #at
|
|
186
|
+
# * TZInfo::Tz → responds to #utc_to_local
|
|
187
|
+
# * IANA String → parsed via ENV TZ override (POSIX `tzset(3)`)
|
|
188
|
+
def wall_clock(epoch, tz)
|
|
189
|
+
t = case tz
|
|
190
|
+
when nil then ::Time.at(epoch).utc
|
|
191
|
+
else local_time(epoch, tz)
|
|
192
|
+
end
|
|
193
|
+
dow = t.wday
|
|
194
|
+
[t.min, t.hour, t.day, t.mon, dow]
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def local_time(epoch, tz)
|
|
198
|
+
return tz.at(epoch) if tz.respond_to?(:at) && !tz.is_a?(String)
|
|
199
|
+
return tz.utc_to_local(::Time.at(epoch).utc) if tz.respond_to?(:utc_to_local)
|
|
200
|
+
|
|
201
|
+
with_tz_env(tz.to_s) { ::Time.at(epoch) }
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def with_tz_env(name)
|
|
205
|
+
old = ENV.fetch('TZ', nil)
|
|
206
|
+
ENV['TZ'] = name
|
|
207
|
+
yield
|
|
208
|
+
ensure
|
|
209
|
+
ENV['TZ'] = old
|
|
210
|
+
end
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# One registered loop. Carries identity, schedule, options, and the
|
|
214
|
+
# cached parser. Immutable after `register!` — re-registering the same
|
|
215
|
+
# (schedule, klass, options) triple no-ops.
|
|
216
|
+
class Loop
|
|
217
|
+
attr_reader :lid, :schedule, :klass, :options, :tz
|
|
218
|
+
|
|
219
|
+
def initialize(schedule:, klass:, options: {}, tz: nil, lid: nil)
|
|
220
|
+
raise ArgumentError, 'klass must be a String' unless klass.is_a?(String) && !klass.empty?
|
|
221
|
+
|
|
222
|
+
@schedule = schedule
|
|
223
|
+
@klass = klass
|
|
224
|
+
@options = stringify_options(options)
|
|
225
|
+
@tz = tz
|
|
226
|
+
@parser = Parser.new(schedule)
|
|
227
|
+
@lid = lid || Cron.lid(schedule, klass, @options)
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
def parser
|
|
231
|
+
@parser ||= Parser.new(@schedule)
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
def paused?
|
|
235
|
+
@options['paused'].to_s == '1' || @options['paused'] == true
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
def queue
|
|
239
|
+
@options['queue'] || 'default'
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
def args
|
|
243
|
+
Array(@options['args'])
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
def retry_value
|
|
247
|
+
@options.fetch('retry', true)
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
def history
|
|
251
|
+
Wurk.redis do |c|
|
|
252
|
+
entries = c.call('LRANGE', "#{HISTORY_PREFIX}#{@lid}", 0, -1)
|
|
253
|
+
entries.map { |e| JSON.parse(e) }
|
|
254
|
+
end
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
# Epoch of the most recent fire, or nil if this loop has never run. The
|
|
258
|
+
# poller LPUSHes `[fired_at, jid]` tuples so index 0 is newest; we read
|
|
259
|
+
# only that one entry instead of the whole history list.
|
|
260
|
+
def last_fired_at
|
|
261
|
+
raw = Wurk.redis { |c| c.call('LINDEX', "#{HISTORY_PREFIX}#{@lid}", 0) }
|
|
262
|
+
return nil if raw.nil?
|
|
263
|
+
|
|
264
|
+
entry = JSON.parse(raw)
|
|
265
|
+
return nil unless entry.is_a?(Array)
|
|
266
|
+
|
|
267
|
+
fired_at = entry[0]
|
|
268
|
+
fired_at.is_a?(Numeric) ? fired_at.to_i : nil
|
|
269
|
+
rescue JSON::ParserError
|
|
270
|
+
nil
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
def to_redis_hash
|
|
274
|
+
{
|
|
275
|
+
'schedule' => @schedule,
|
|
276
|
+
'klass' => @klass,
|
|
277
|
+
'options' => Wurk.dump_json(@options),
|
|
278
|
+
'tz' => tz_name.to_s,
|
|
279
|
+
'paused' => paused? ? '1' : '0'
|
|
280
|
+
}
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
def next_fire_at(from_epoch = ::Time.now.to_i)
|
|
284
|
+
parser.next_fire_at(from_epoch, @tz)
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
# Next scheduled fire after firing at `fired_slot`. `advance_from` (the
|
|
288
|
+
# poller passes `now`) is the search origin so missed ticks are not
|
|
289
|
+
# backfilled. The DST guard: on a fall-back the clock rolls back, so the
|
|
290
|
+
# same wall-clock minute recurs at a later UTC instant. That equality
|
|
291
|
+
# alone isn't enough to skip — it would also drop a legitimate hourly run
|
|
292
|
+
# whose repeated hour is real. So we only suppress the duplicate for a
|
|
293
|
+
# FIXED daily slot; an hourly schedule (wildcard hour) keeps both fold
|
|
294
|
+
# hours. A sub-hourly schedule (e.g. `*/30`) lands on a different minute,
|
|
295
|
+
# so the equality never even triggers. Spec: no DST double-fire without
|
|
296
|
+
# skipping legitimate hourly runs (sidekiq-ent.md §2.6).
|
|
297
|
+
def next_fire_after(fired_slot, advance_from = fired_slot)
|
|
298
|
+
nxt = next_fire_at(advance_from)
|
|
299
|
+
return nxt if nxt.nil? || local_components(nxt) != local_components(fired_slot)
|
|
300
|
+
return nxt if hourly? # repeated fall-back hour is a genuine second run
|
|
301
|
+
|
|
302
|
+
next_fire_at(nxt) # fixed daily slot: drop the fold duplicate
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
def local_components(epoch)
|
|
306
|
+
parser.local_components(epoch, @tz)
|
|
307
|
+
end
|
|
308
|
+
|
|
309
|
+
# True when the schedule fires every hour (hour field is `*` / all 24).
|
|
310
|
+
# Only such a schedule legitimately runs in both repeated fall-back hours;
|
|
311
|
+
# a fixed-hour slot (even a multi-hour one like `0 1,2 * * *`) must dedup.
|
|
312
|
+
def hourly?
|
|
313
|
+
parser.fields[1].size == 24
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
def tz_name
|
|
317
|
+
return nil if @tz.nil?
|
|
318
|
+
return @tz.name if @tz.respond_to?(:name)
|
|
319
|
+
return @tz.identifier if @tz.respond_to?(:identifier)
|
|
320
|
+
|
|
321
|
+
@tz.to_s
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
def self.from_redis(lid, hash)
|
|
325
|
+
h = hash.is_a?(Array) ? hash.each_slice(2).to_h : hash
|
|
326
|
+
opts = h['options'] ? JSON.parse(h['options']) : {}
|
|
327
|
+
opts['paused'] = '1' if h['paused'] == '1'
|
|
328
|
+
tz = h['tz'].to_s.empty? ? nil : h['tz']
|
|
329
|
+
new(lid: lid, schedule: h['schedule'], klass: h['klass'], options: opts, tz: tz)
|
|
330
|
+
end
|
|
331
|
+
|
|
332
|
+
private
|
|
333
|
+
|
|
334
|
+
def stringify_options(opts)
|
|
335
|
+
opts.transform_keys(&:to_s)
|
|
336
|
+
end
|
|
337
|
+
end
|
|
338
|
+
|
|
339
|
+
# Registration DSL. One Manager per `config.periodic` block; multiple
|
|
340
|
+
# blocks accumulate. `mgr.tz=` sets the default tz for subsequent
|
|
341
|
+
# `register` calls; per-call `tz:` overrides.
|
|
342
|
+
class Manager
|
|
343
|
+
attr_accessor :tz
|
|
344
|
+
attr_reader :loops
|
|
345
|
+
|
|
346
|
+
def initialize(config = nil)
|
|
347
|
+
@config = config
|
|
348
|
+
@loops = []
|
|
349
|
+
@tz = nil
|
|
350
|
+
end
|
|
351
|
+
|
|
352
|
+
def register(cron, klass, **opts)
|
|
353
|
+
klass_name = klass.is_a?(String) ? klass : klass.to_s
|
|
354
|
+
tz = opts.delete(:tz) || @tz
|
|
355
|
+
normalized = opts.transform_keys(&:to_s)
|
|
356
|
+
loop_obj = Loop.new(schedule: cron, klass: klass_name, options: normalized, tz: tz)
|
|
357
|
+
@loops << loop_obj
|
|
358
|
+
Cron.persist(loop_obj)
|
|
359
|
+
loop_obj
|
|
360
|
+
end
|
|
361
|
+
end
|
|
362
|
+
|
|
363
|
+
# Enumerable view of every registered loop. Reads `periodic` SET +
|
|
364
|
+
# `loops:{lid}` HASH on each iteration — cheap because the dashboard's
|
|
365
|
+
# list view is the only hot caller.
|
|
366
|
+
class LoopSet
|
|
367
|
+
include ::Enumerable
|
|
368
|
+
|
|
369
|
+
# `config` scopes Redis to that config's per-fork pool — the leader poller
|
|
370
|
+
# runs inside a swarm child and must not reach for the parent-inherited
|
|
371
|
+
# global socket. Dashboard/CLI callers pass nothing and get Wurk.redis.
|
|
372
|
+
def initialize(config = nil)
|
|
373
|
+
@config = config
|
|
374
|
+
end
|
|
375
|
+
|
|
376
|
+
def each
|
|
377
|
+
return enum_for(:each) unless block_given?
|
|
378
|
+
|
|
379
|
+
redis do |c|
|
|
380
|
+
lids = c.call('SMEMBERS', PERIODIC_KEY)
|
|
381
|
+
lids.each do |lid|
|
|
382
|
+
h = c.call('HGETALL', "#{LOOP_PREFIX}#{lid}")
|
|
383
|
+
next if h.nil? || h.empty?
|
|
384
|
+
|
|
385
|
+
yield Loop.from_redis(lid, h)
|
|
386
|
+
end
|
|
387
|
+
end
|
|
388
|
+
end
|
|
389
|
+
|
|
390
|
+
def size
|
|
391
|
+
redis { |c| c.call('SCARD', PERIODIC_KEY).to_i }
|
|
392
|
+
end
|
|
393
|
+
|
|
394
|
+
def fetch(lid)
|
|
395
|
+
h = redis { |c| c.call('HGETALL', "#{LOOP_PREFIX}#{lid}") }
|
|
396
|
+
return nil if h.nil? || h.empty?
|
|
397
|
+
|
|
398
|
+
h = h.each_slice(2).to_h if h.is_a?(Array)
|
|
399
|
+
return nil if h.empty?
|
|
400
|
+
|
|
401
|
+
Loop.from_redis(lid, h)
|
|
402
|
+
end
|
|
403
|
+
|
|
404
|
+
private
|
|
405
|
+
|
|
406
|
+
def redis(&)
|
|
407
|
+
@config ? @config.redis(&) : Wurk.redis(&)
|
|
408
|
+
end
|
|
409
|
+
end
|
|
410
|
+
|
|
411
|
+
# Boot-time validator. Runs the user's periodic block against a
|
|
412
|
+
# disposable Manager whose register call also resolves the klass
|
|
413
|
+
# constant — a typo'd class name surfaces here instead of on the
|
|
414
|
+
# first tick in production.
|
|
415
|
+
class ConfigTester
|
|
416
|
+
def verify(&block)
|
|
417
|
+
raise ArgumentError, 'block required' unless block
|
|
418
|
+
|
|
419
|
+
mgr = Manager.new
|
|
420
|
+
block.call(mgr)
|
|
421
|
+
mgr.loops.each { |lp| resolve_klass!(lp.klass) }
|
|
422
|
+
mgr.loops
|
|
423
|
+
rescue StandardError
|
|
424
|
+
# register persists each loop immediately, so a validation failure
|
|
425
|
+
# would otherwise leave partially-applied loops in the live LoopSet —
|
|
426
|
+
# they'd fire on the next poll. Roll the whole batch back, then re-raise.
|
|
427
|
+
mgr&.loops&.each { |lp| Cron.unregister(lp.lid) }
|
|
428
|
+
raise
|
|
429
|
+
end
|
|
430
|
+
|
|
431
|
+
private
|
|
432
|
+
|
|
433
|
+
def resolve_klass!(klass)
|
|
434
|
+
klass.split('::').inject(Object) { |mod, name| mod.const_get(name) }
|
|
435
|
+
rescue NameError => e
|
|
436
|
+
raise ArgumentError, "Cron worker class #{klass.inspect} could not be resolved: #{e.message}"
|
|
437
|
+
end
|
|
438
|
+
end
|
|
439
|
+
|
|
440
|
+
# Once-per-minute tick driver. Only the cluster leader iterates the LoopSet
|
|
441
|
+
# and enqueues; non-leaders return early. Missed-tick warning when
|
|
442
|
+
# wall-clock has drifted more than `MISSED_TICK_THRESHOLD` seconds past the
|
|
443
|
+
# expected fire.
|
|
444
|
+
class Poller
|
|
445
|
+
include Component
|
|
446
|
+
|
|
447
|
+
def initialize(config)
|
|
448
|
+
@config = config
|
|
449
|
+
@done = false
|
|
450
|
+
@mutex = ::Mutex.new
|
|
451
|
+
@sleeper = ::ConditionVariable.new
|
|
452
|
+
@client = Client.new(config: config)
|
|
453
|
+
@thread = nil
|
|
454
|
+
# Operators never need to touch this; integration tests shrink it so a
|
|
455
|
+
# due loop fires within the test window instead of waiting a full minute.
|
|
456
|
+
@tick_interval = config[:cron_tick_interval] || DEFAULT_TICK_SECONDS
|
|
457
|
+
end
|
|
458
|
+
|
|
459
|
+
def start
|
|
460
|
+
@poller_thread ||= safe_thread('cron-poller') do # rubocop:disable Naming/MemoizedInstanceVariableName
|
|
461
|
+
# Wait one interval before the first tick: don't fire a catch-up burst
|
|
462
|
+
# the instant we boot (the leader is barely settled), and let a
|
|
463
|
+
# short-lived process exit without ticking at all.
|
|
464
|
+
wait
|
|
465
|
+
until @done
|
|
466
|
+
tick
|
|
467
|
+
wait
|
|
468
|
+
end
|
|
469
|
+
end
|
|
470
|
+
end
|
|
471
|
+
|
|
472
|
+
def terminate
|
|
473
|
+
@mutex.synchronize do
|
|
474
|
+
@done = true
|
|
475
|
+
@sleeper.signal
|
|
476
|
+
end
|
|
477
|
+
end
|
|
478
|
+
|
|
479
|
+
# Leader-gated by the single cluster lock (Component#leader? reads
|
|
480
|
+
# `dear-leader`): non-leaders return early and never iterate the LoopSet.
|
|
481
|
+
# The Launcher owns the lock's renewal — the poller no longer runs (or
|
|
482
|
+
# expires) its own.
|
|
483
|
+
def tick
|
|
484
|
+
return unless leader?
|
|
485
|
+
|
|
486
|
+
LoopSet.new(@config).each { |lp| enqueue_if_due(lp) }
|
|
487
|
+
rescue StandardError => e
|
|
488
|
+
handle_exception(e, { context: 'cron-poller' })
|
|
489
|
+
end
|
|
490
|
+
|
|
491
|
+
def enqueue_if_due(loop_obj)
|
|
492
|
+
return if loop_obj.paused?
|
|
493
|
+
|
|
494
|
+
now = ::Time.now.to_i
|
|
495
|
+
prev_fire, next_fire = read_fire_marks(loop_obj.lid)
|
|
496
|
+
next_fire ||= loop_obj.next_fire_at(prev_fire || (now - @tick_interval))
|
|
497
|
+
return if next_fire.nil? || next_fire > now
|
|
498
|
+
|
|
499
|
+
warn_missed_tick(loop_obj, next_fire, now)
|
|
500
|
+
jid = enqueue!(loop_obj)
|
|
501
|
+
future = loop_obj.next_fire_after(next_fire, now)
|
|
502
|
+
record_fire(loop_obj, jid, now, future)
|
|
503
|
+
jid
|
|
504
|
+
end
|
|
505
|
+
|
|
506
|
+
# Fire one loop right now, bypassing both the leader gate and the
|
|
507
|
+
# schedule due-check, recording history + advancing the fire marks
|
|
508
|
+
# exactly like a real tick. Powers Cron.fire! (deterministic specs /
|
|
509
|
+
# manual "run now"); the scheduled, leader-gated path stays #tick.
|
|
510
|
+
def fire(loop_obj)
|
|
511
|
+
now = ::Time.now.to_i
|
|
512
|
+
jid = enqueue!(loop_obj)
|
|
513
|
+
record_fire(loop_obj, jid, now, loop_obj.next_fire_at(now))
|
|
514
|
+
jid
|
|
515
|
+
end
|
|
516
|
+
|
|
517
|
+
private
|
|
518
|
+
|
|
519
|
+
def wait
|
|
520
|
+
@mutex.synchronize do
|
|
521
|
+
@sleeper.wait(@mutex, @tick_interval) unless @done
|
|
522
|
+
end
|
|
523
|
+
end
|
|
524
|
+
|
|
525
|
+
def warn_missed_tick(loop_obj, expected, now)
|
|
526
|
+
return if now - expected <= MISSED_TICK_THRESHOLD
|
|
527
|
+
|
|
528
|
+
logger.warn(
|
|
529
|
+
"[cron] missed tick lid=#{loop_obj.lid} klass=#{loop_obj.klass} " \
|
|
530
|
+
"expected_at=#{expected} fired_at=#{now} drift=#{now - expected}s"
|
|
531
|
+
)
|
|
532
|
+
end
|
|
533
|
+
|
|
534
|
+
def enqueue!(loop_obj)
|
|
535
|
+
@client.push(
|
|
536
|
+
'class' => loop_obj.klass,
|
|
537
|
+
'args' => loop_obj.args,
|
|
538
|
+
'queue' => loop_obj.queue,
|
|
539
|
+
'retry' => loop_obj.retry_value
|
|
540
|
+
)
|
|
541
|
+
end
|
|
542
|
+
|
|
543
|
+
def read_fire_marks(lid)
|
|
544
|
+
@config.redis do |c|
|
|
545
|
+
vals = c.call('HMGET', "#{LOOP_PREFIX}#{lid}", 'lf', 'nf')
|
|
546
|
+
next_fire = vals[1]
|
|
547
|
+
# Preserve nil: treat missing or empty 'nf' as nil, not 0.
|
|
548
|
+
[vals[0]&.to_i, next_fire.nil? || next_fire.empty? ? nil : next_fire.to_i]
|
|
549
|
+
end
|
|
550
|
+
end
|
|
551
|
+
|
|
552
|
+
def record_fire(loop_obj, jid, fired_at, future)
|
|
553
|
+
history_entry = Wurk.dump_json([fired_at, jid])
|
|
554
|
+
@config.redis do |c|
|
|
555
|
+
if future.nil?
|
|
556
|
+
c.call('HSET', "#{LOOP_PREFIX}#{loop_obj.lid}", 'lf', fired_at.to_s)
|
|
557
|
+
c.call('HDEL', "#{LOOP_PREFIX}#{loop_obj.lid}", 'nf')
|
|
558
|
+
else
|
|
559
|
+
c.call('HSET', "#{LOOP_PREFIX}#{loop_obj.lid}", 'lf', fired_at.to_s, 'nf', future.to_s)
|
|
560
|
+
end
|
|
561
|
+
c.call('LPUSH', "#{HISTORY_PREFIX}#{loop_obj.lid}", history_entry)
|
|
562
|
+
c.call('LTRIM', "#{HISTORY_PREFIX}#{loop_obj.lid}", 0, HISTORY_CAP - 1)
|
|
563
|
+
end
|
|
564
|
+
end
|
|
565
|
+
end
|
|
566
|
+
|
|
567
|
+
class << self
|
|
568
|
+
# Stable 16-hex lid from (schedule, klass, options). Re-registering
|
|
569
|
+
# the same triple no-ops because the Redis writes overwrite under
|
|
570
|
+
# the same key.
|
|
571
|
+
def lid(schedule, klass, options)
|
|
572
|
+
opts = options.is_a?(Hash) ? options : {}
|
|
573
|
+
::Digest::SHA1.hexdigest("#{schedule}|#{klass}|#{JSON.dump(opts.sort.to_h)}")[0, 16]
|
|
574
|
+
end
|
|
575
|
+
|
|
576
|
+
# Task-stated convenience signature. `name` is treated as a label;
|
|
577
|
+
# the lid is still derived from (schedule, klass, opts) so the
|
|
578
|
+
# call is idempotent. Callers that want the Sidekiq DSL should use
|
|
579
|
+
# `Manager#register` via `config.periodic { |mgr| ... }`.
|
|
580
|
+
def register(name, cron, worker_class, args = [], **opts)
|
|
581
|
+
merged = opts.merge(args: args)
|
|
582
|
+
merged[:label] = name if name
|
|
583
|
+
Loop.new(schedule: cron, klass: worker_class.to_s, options: merged).tap { |lp| persist(lp) }
|
|
584
|
+
end
|
|
585
|
+
|
|
586
|
+
def persist(loop_obj)
|
|
587
|
+
Wurk.redis do |c|
|
|
588
|
+
c.call('SADD', PERIODIC_KEY, loop_obj.lid)
|
|
589
|
+
c.call('HSET', "#{LOOP_PREFIX}#{loop_obj.lid}", *loop_obj.to_redis_hash.flatten)
|
|
590
|
+
end
|
|
591
|
+
loop_obj
|
|
592
|
+
end
|
|
593
|
+
|
|
594
|
+
# Drop a loop entirely. Used by the Web UI delete action and by the
|
|
595
|
+
# config-reload path so a removed `register(...)` line vanishes from
|
|
596
|
+
# Redis on next boot.
|
|
597
|
+
def unregister(lid)
|
|
598
|
+
Wurk.redis do |c|
|
|
599
|
+
c.call('SREM', PERIODIC_KEY, lid)
|
|
600
|
+
c.call('DEL', "#{LOOP_PREFIX}#{lid}", "#{HISTORY_PREFIX}#{lid}")
|
|
601
|
+
end
|
|
602
|
+
end
|
|
603
|
+
|
|
604
|
+
# Test helper: wipe every Cron Redis key. Production code must not
|
|
605
|
+
# call this — it removes every registered loop in the cluster.
|
|
606
|
+
def reset!
|
|
607
|
+
Wurk.redis do |c|
|
|
608
|
+
lids = c.call('SMEMBERS', PERIODIC_KEY)
|
|
609
|
+
lids.each do |lid|
|
|
610
|
+
c.call('DEL', "#{LOOP_PREFIX}#{lid}", "#{HISTORY_PREFIX}#{lid}")
|
|
611
|
+
end
|
|
612
|
+
c.call('DEL', PERIODIC_KEY)
|
|
613
|
+
end
|
|
614
|
+
end
|
|
615
|
+
|
|
616
|
+
def jobs
|
|
617
|
+
LoopSet.new
|
|
618
|
+
end
|
|
619
|
+
|
|
620
|
+
# Test/ops helper: fire one registered loop immediately, ignoring the
|
|
621
|
+
# leader gate and the schedule due-check. Records history + advances the
|
|
622
|
+
# fire marks just like a leader tick, so specs can assert on the enqueue
|
|
623
|
+
# and history deterministically without waiting on wall-clock or
|
|
624
|
+
# stubbing leadership. Returns the enqueued jid, or nil for an unknown
|
|
625
|
+
# lid. Aliased as `Sidekiq::Periodic.fire!`.
|
|
626
|
+
def fire!(lid)
|
|
627
|
+
loop_obj = LoopSet.new.fetch(lid)
|
|
628
|
+
return nil if loop_obj.nil?
|
|
629
|
+
|
|
630
|
+
Poller.new(Wurk.configuration).fire(loop_obj)
|
|
631
|
+
end
|
|
632
|
+
end
|
|
633
|
+
end
|
|
634
|
+
|
|
635
|
+
Periodic = Cron unless const_defined?(:Periodic)
|
|
636
|
+
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'pathname'
|
|
5
|
+
|
|
6
|
+
module Wurk
|
|
7
|
+
# Validates that the precompiled dashboard bundle in vendor/assets matches
|
|
8
|
+
# this gem version. Run from the engine initializer in production to catch
|
|
9
|
+
# packaging mistakes (forgotten `frontend:build`, stale vendor/assets,
|
|
10
|
+
# mismatched release) at boot rather than as a runtime 500.
|
|
11
|
+
module DashboardManifest
|
|
12
|
+
class MissingError < ::StandardError; end
|
|
13
|
+
class VersionMismatch < ::StandardError; end
|
|
14
|
+
|
|
15
|
+
MANIFEST_NAME = 'wurk-manifest.json'
|
|
16
|
+
|
|
17
|
+
class << self
|
|
18
|
+
def path(root)
|
|
19
|
+
::Pathname.new(root).join('vendor', 'assets', 'dashboard', MANIFEST_NAME)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def check!(root: ::Wurk::Engine.root, expected: ::Wurk::VERSION)
|
|
23
|
+
file = path(root)
|
|
24
|
+
unless file.exist?
|
|
25
|
+
raise MissingError,
|
|
26
|
+
"Wurk dashboard manifest missing at #{file}. " \
|
|
27
|
+
'Did `bin/rake frontend:build` run before `gem build`?'
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
data = ::JSON.parse(file.read)
|
|
31
|
+
actual = data['version']
|
|
32
|
+
return true if actual == expected
|
|
33
|
+
|
|
34
|
+
raise VersionMismatch,
|
|
35
|
+
"Wurk dashboard manifest version #{actual.inspect} != gem #{expected.inspect}"
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|