wurk 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +43 -0
- data/CONTRIBUTING.md +73 -0
- data/LICENSE +21 -0
- data/README.md +137 -0
- data/SECURITY.md +39 -0
- data/app/controllers/wurk/api/pagination.rb +67 -0
- data/app/controllers/wurk/api/serializers.rb +131 -0
- data/app/controllers/wurk/api_controller.rb +248 -0
- data/app/controllers/wurk/application_controller.rb +7 -0
- data/app/controllers/wurk/dashboard_controller.rb +48 -0
- data/config/locales/en.yml +15 -0
- data/config/routes.rb +34 -0
- data/exe/wurk +22 -0
- data/lib/active_job/queue_adapters/wurk_adapter.rb +96 -0
- data/lib/generators/wurk/install/install_generator.rb +22 -0
- data/lib/generators/wurk/install/templates/wurk.rb +16 -0
- data/lib/wurk/active_job/wrapper.rb +32 -0
- data/lib/wurk/api/fast.rb +78 -0
- data/lib/wurk/batch/buffer.rb +26 -0
- data/lib/wurk/batch/callback_job.rb +37 -0
- data/lib/wurk/batch/callbacks.rb +176 -0
- data/lib/wurk/batch/client_middleware.rb +27 -0
- data/lib/wurk/batch/death_handler.rb +39 -0
- data/lib/wurk/batch/empty.rb +21 -0
- data/lib/wurk/batch/server_middleware.rb +62 -0
- data/lib/wurk/batch/status.rb +140 -0
- data/lib/wurk/batch.rb +351 -0
- data/lib/wurk/batch_set.rb +67 -0
- data/lib/wurk/capsule.rb +176 -0
- data/lib/wurk/cli.rb +349 -0
- data/lib/wurk/client/buffered.rb +372 -0
- data/lib/wurk/client.rb +330 -0
- data/lib/wurk/compat.rb +136 -0
- data/lib/wurk/component.rb +136 -0
- data/lib/wurk/configuration.rb +373 -0
- data/lib/wurk/context.rb +35 -0
- data/lib/wurk/cron.rb +636 -0
- data/lib/wurk/dashboard_manifest.rb +39 -0
- data/lib/wurk/dead_set.rb +78 -0
- data/lib/wurk/deploy.rb +91 -0
- data/lib/wurk/embedded.rb +94 -0
- data/lib/wurk/encryption.rb +276 -0
- data/lib/wurk/engine.rb +81 -0
- data/lib/wurk/fetcher/reaper.rb +264 -0
- data/lib/wurk/fetcher/reliable.rb +138 -0
- data/lib/wurk/fetcher.rb +11 -0
- data/lib/wurk/health.rb +193 -0
- data/lib/wurk/heartbeat.rb +211 -0
- data/lib/wurk/iterable_job.rb +292 -0
- data/lib/wurk/job/options.rb +70 -0
- data/lib/wurk/job.rb +33 -0
- data/lib/wurk/job_logger.rb +68 -0
- data/lib/wurk/job_record.rb +156 -0
- data/lib/wurk/job_retry.rb +320 -0
- data/lib/wurk/job_set.rb +212 -0
- data/lib/wurk/job_util.rb +162 -0
- data/lib/wurk/keys.rb +52 -0
- data/lib/wurk/launcher.rb +289 -0
- data/lib/wurk/leader.rb +221 -0
- data/lib/wurk/limiter/base.rb +138 -0
- data/lib/wurk/limiter/bucket.rb +80 -0
- data/lib/wurk/limiter/concurrent.rb +132 -0
- data/lib/wurk/limiter/leaky.rb +91 -0
- data/lib/wurk/limiter/points.rb +89 -0
- data/lib/wurk/limiter/server_middleware.rb +77 -0
- data/lib/wurk/limiter/unlimited.rb +48 -0
- data/lib/wurk/limiter/window.rb +80 -0
- data/lib/wurk/limiter.rb +255 -0
- data/lib/wurk/logger.rb +81 -0
- data/lib/wurk/lua/loader.rb +53 -0
- data/lib/wurk/lua.rb +187 -0
- data/lib/wurk/manager.rb +132 -0
- data/lib/wurk/metrics/history.rb +151 -0
- data/lib/wurk/metrics/query.rb +173 -0
- data/lib/wurk/metrics/rollup.rb +169 -0
- data/lib/wurk/metrics/statsd.rb +197 -0
- data/lib/wurk/metrics.rb +7 -0
- data/lib/wurk/middleware/chain.rb +128 -0
- data/lib/wurk/middleware/current_attributes.rb +87 -0
- data/lib/wurk/middleware/expiry.rb +50 -0
- data/lib/wurk/middleware/i18n.rb +63 -0
- data/lib/wurk/middleware/interrupt_handler.rb +45 -0
- data/lib/wurk/middleware/poison_pill.rb +149 -0
- data/lib/wurk/middleware.rb +34 -0
- data/lib/wurk/process_set.rb +243 -0
- data/lib/wurk/processor.rb +247 -0
- data/lib/wurk/queue.rb +108 -0
- data/lib/wurk/queues.rb +80 -0
- data/lib/wurk/rails.rb +9 -0
- data/lib/wurk/railtie.rb +28 -0
- data/lib/wurk/redis_pool.rb +79 -0
- data/lib/wurk/retry_set.rb +17 -0
- data/lib/wurk/scheduled.rb +189 -0
- data/lib/wurk/scheduled_set.rb +18 -0
- data/lib/wurk/sorted_entry.rb +95 -0
- data/lib/wurk/stats.rb +190 -0
- data/lib/wurk/swarm/child_boot.rb +105 -0
- data/lib/wurk/swarm.rb +260 -0
- data/lib/wurk/testing.rb +102 -0
- data/lib/wurk/topology.rb +74 -0
- data/lib/wurk/unique.rb +240 -0
- data/lib/wurk/version.rb +5 -0
- data/lib/wurk/web/config.rb +180 -0
- data/lib/wurk/web/enterprise.rb +138 -0
- data/lib/wurk/web/search.rb +139 -0
- data/lib/wurk/web.rb +25 -0
- data/lib/wurk/work_set.rb +116 -0
- data/lib/wurk/worker/setter.rb +93 -0
- data/lib/wurk/worker.rb +216 -0
- data/lib/wurk.rb +238 -0
- data/vendor/assets/dashboard/assets/index-8P3N_m1X.js +152 -0
- data/vendor/assets/dashboard/assets/index-Bqz4_SOQ.css +1 -0
- data/vendor/assets/dashboard/index.html +13 -0
- data/vendor/assets/dashboard/wurk-manifest.json +4 -0
- metadata +232 -0
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'base64'
|
|
4
|
+
require 'zlib'
|
|
5
|
+
|
|
6
|
+
module Wurk
|
|
7
|
+
# One job payload viewed from the data API (Queue#each / JobSet#each).
|
|
8
|
+
# Wraps the raw JSON string from Redis; parses lazily so an O(n) scan
|
|
9
|
+
# over a large queue doesn't pay JSON cost for jobs that go unused.
|
|
10
|
+
#
|
|
11
|
+
# The `value` (raw JSON string) is what Redis stores; `LREM` matches
|
|
12
|
+
# exact bytes, so `delete` must use it rather than re-serialize.
|
|
13
|
+
#
|
|
14
|
+
# Spec: docs/target/sidekiq-free.md §19.3.
|
|
15
|
+
class JobRecord
|
|
16
|
+
# Pre-compiled. ActiveJob's wrapper class varies per Rails minor
|
|
17
|
+
# (`ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper`,
|
|
18
|
+
# `ActiveJob::QueueAdapters::WurkAdapter::JobWrapper`, etc.).
|
|
19
|
+
ACTIVE_JOB_WRAPPER = /\AActiveJob::QueueAdapters::.+::JobWrapper\z/
|
|
20
|
+
ACTION_MAILER_JOBS = %w[
|
|
21
|
+
ActionMailer::DeliveryJob
|
|
22
|
+
ActionMailer::Parameterized::DeliveryJob
|
|
23
|
+
ActionMailer::MailDeliveryJob
|
|
24
|
+
].freeze
|
|
25
|
+
|
|
26
|
+
attr_reader :queue
|
|
27
|
+
|
|
28
|
+
# @param item [String, Hash] raw JSON payload or pre-parsed hash.
|
|
29
|
+
# @param queue_name [String, nil] queue this record came from.
|
|
30
|
+
def initialize(item, queue_name = nil)
|
|
31
|
+
if item.is_a?(String)
|
|
32
|
+
@value = item
|
|
33
|
+
@item = nil
|
|
34
|
+
else
|
|
35
|
+
@item = item
|
|
36
|
+
@value = nil
|
|
37
|
+
end
|
|
38
|
+
@queue = queue_name || (@item && @item['queue'])
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Lazily parsed payload. Memoized; never re-parses.
|
|
42
|
+
def item
|
|
43
|
+
@item ||= Wurk.load_json(@value)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Lazily serialized payload. When constructed from a Hash, we
|
|
47
|
+
# generate JSON on first call so `delete` (LREM) has exact bytes.
|
|
48
|
+
def value
|
|
49
|
+
@value ||= Wurk.dump_json(@item)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def klass = item['class']
|
|
53
|
+
def args = item['args']
|
|
54
|
+
def jid = item['jid']
|
|
55
|
+
def bid = item['bid']
|
|
56
|
+
def tags = item['tags'] || []
|
|
57
|
+
def enqueued_at = parse_time(item['enqueued_at'])
|
|
58
|
+
def created_at = parse_time(item['created_at'])
|
|
59
|
+
def failed_at = parse_time(item['failed_at'])
|
|
60
|
+
def retried_at = parse_time(item['retried_at'])
|
|
61
|
+
|
|
62
|
+
# Hash-like reader for arbitrary payload fields. Spec §19.3.
|
|
63
|
+
def [](name) = item[name]
|
|
64
|
+
|
|
65
|
+
# Sidekiq compresses the bt as base64(zlib.deflate(JSON.dump(bt))).
|
|
66
|
+
# Returns nil when no error has been recorded.
|
|
67
|
+
def error_backtrace
|
|
68
|
+
compressed = item['error_backtrace']
|
|
69
|
+
return nil unless compressed
|
|
70
|
+
|
|
71
|
+
Wurk.load_json(Zlib.inflate(Base64.decode64(compressed)))
|
|
72
|
+
rescue Zlib::DataError, ArgumentError, ::JSON::ParserError
|
|
73
|
+
nil
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Seconds since enqueued_at. Handles legacy float-seconds and
|
|
77
|
+
# current integer-ms `enqueued_at` shapes. Returns 0.0 when missing
|
|
78
|
+
# or somehow in the future (clock skew).
|
|
79
|
+
def latency
|
|
80
|
+
JobRecord.latency_from(item['enqueued_at'])
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Removes exactly this payload's bytes from the queue list. Returns
|
|
84
|
+
# true when LREM removed ≥1 entry. Idempotent. Method name is
|
|
85
|
+
# Sidekiq wire-compat — renaming would break `JobRecord#delete`.
|
|
86
|
+
def delete # rubocop:disable Naming/PredicateMethod
|
|
87
|
+
removed = Wurk.redis { |c| c.call('LREM', Keys.queue(@queue), 1, value) }
|
|
88
|
+
removed.to_i.positive?
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# ActiveJob/ActionMailer unwrappers — UI-facing only. For plain Wurk
|
|
92
|
+
# workers, returns the raw class name.
|
|
93
|
+
def display_class
|
|
94
|
+
return @display_class if defined?(@display_class)
|
|
95
|
+
|
|
96
|
+
@display_class = active_job_wrapper? ? unwrap_class : klass
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def display_args
|
|
100
|
+
return @display_args if defined?(@display_args)
|
|
101
|
+
|
|
102
|
+
@display_args = active_job_wrapper? ? unwrap_args : args
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# @api internal
|
|
106
|
+
# Shared latency math: ms ints (>= 10^10) and float secs (< 10^10)
|
|
107
|
+
# both stored in `enqueued_at` historically. See spec §31.5.
|
|
108
|
+
def self.latency_from(enqueued_at, now_ms = nil)
|
|
109
|
+
return 0.0 if enqueued_at.nil?
|
|
110
|
+
|
|
111
|
+
now_ms ||= ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
|
|
112
|
+
enq_ms = enqueued_at < 10_000_000_000 ? enqueued_at * 1_000 : enqueued_at
|
|
113
|
+
diff = (now_ms - enq_ms) / 1_000.0
|
|
114
|
+
diff.negative? ? 0.0 : diff
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
private
|
|
118
|
+
|
|
119
|
+
def active_job_wrapper?
|
|
120
|
+
klass.is_a?(String) && (ACTIVE_JOB_WRAPPER.match?(klass) || klass == 'Sidekiq::ActiveJob::Wrapper')
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def unwrap_class
|
|
124
|
+
job_class = item['wrapped'] || args.dig(0, 'job_class')
|
|
125
|
+
return klass unless job_class
|
|
126
|
+
|
|
127
|
+
mailer_display(job_class) || job_class
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def mailer_display(job_class)
|
|
131
|
+
return nil unless ACTION_MAILER_JOBS.include?(job_class)
|
|
132
|
+
|
|
133
|
+
mailer_args = args.dig(0, 'arguments') || []
|
|
134
|
+
return nil if mailer_args.size < 2
|
|
135
|
+
|
|
136
|
+
"#{mailer_args[0]}##{mailer_args[1]}"
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def unwrap_args
|
|
140
|
+
job_args = args.dig(0, 'arguments') || []
|
|
141
|
+
wrapped = item['wrapped']
|
|
142
|
+
return job_args unless ACTION_MAILER_JOBS.include?(wrapped)
|
|
143
|
+
|
|
144
|
+
# ActionMailer payloads start with [mailer, method, "deliver_now", *real_args]
|
|
145
|
+
job_args.drop(3)
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# Parse Sidekiq's mixed time formats (Float secs, Integer ms) into Time.
|
|
149
|
+
def parse_time(value)
|
|
150
|
+
return nil if value.nil?
|
|
151
|
+
|
|
152
|
+
ms = value < 10_000_000_000 ? value * 1_000 : value
|
|
153
|
+
::Time.at(ms / 1_000.0)
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
end
|
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'zlib'
|
|
4
|
+
require_relative 'component'
|
|
5
|
+
|
|
6
|
+
module Wurk
|
|
7
|
+
# Owns the retry pipeline. When perform raises, JobRetry decides whether to
|
|
8
|
+
# reschedule (`retry` ZSET, exponential backoff + jitter), drop, kill, or
|
|
9
|
+
# send to the morgue. Wire-compat sacred: error_message / error_class /
|
|
10
|
+
# retry_count / failed_at / retried_at / error_backtrace field names and
|
|
11
|
+
# encodings (base64 of zlib of JSON for the backtrace) match Sidekiq byte
|
|
12
|
+
# for byte — third-party gems and the dashboard read them directly.
|
|
13
|
+
#
|
|
14
|
+
# Two entry points wrap the dispatch onion in Processor#dispatch:
|
|
15
|
+
#
|
|
16
|
+
# * `global(jobstr, queue)` — outermost, no job instance required.
|
|
17
|
+
# Rescues `Exception` so pre-instantiation failures (const_get, reloader)
|
|
18
|
+
# still get a retry recorded. Re-raises `Handled` so the processor
|
|
19
|
+
# skips ACK logging.
|
|
20
|
+
# * `local(jobinst, jobstr, queue)` — inner, runs after the worker is
|
|
21
|
+
# instantiated. Honors per-class `sidekiq_retry_in_block` /
|
|
22
|
+
# `sidekiq_retries_exhausted_block` (and the wrapped-class variants).
|
|
23
|
+
# Raises `Handled` after booking the retry so `global` does not double-
|
|
24
|
+
# process the failure.
|
|
25
|
+
#
|
|
26
|
+
# Spec: docs/target/sidekiq-free.md §17.
|
|
27
|
+
class JobRetry # rubocop:disable Metrics/ClassLength
|
|
28
|
+
include Component
|
|
29
|
+
|
|
30
|
+
DEFAULT_MAX_RETRY_ATTEMPTS = 25
|
|
31
|
+
|
|
32
|
+
# Raised after process_retry has dealt with the failure. The processor
|
|
33
|
+
# rescues `Handled` and exits the job cleanly (acked, no re-raise).
|
|
34
|
+
class Handled < ::RuntimeError; end
|
|
35
|
+
|
|
36
|
+
# Subclass of Handled with the same semantics — used when middleware
|
|
37
|
+
# short-circuits processing (e.g. interrupt-handler re-pushed the job).
|
|
38
|
+
class Skip < Handled; end
|
|
39
|
+
|
|
40
|
+
def initialize(capsule)
|
|
41
|
+
@capsule = capsule
|
|
42
|
+
@config = capsule
|
|
43
|
+
@max_retries = inner_config_get(:max_retries) || DEFAULT_MAX_RETRY_ATTEMPTS
|
|
44
|
+
@backtrace_cleaner = inner_config_get(:backtrace_cleaner)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Outermost retry guard. Rescues `Exception` so const_get / reloader
|
|
48
|
+
# failures still get a retry. `Handled` is re-raised intact; `Shutdown`
|
|
49
|
+
# bubbles up so the swarm can drain.
|
|
50
|
+
def global(jobstr, queue)
|
|
51
|
+
yield
|
|
52
|
+
rescue Handled, Wurk::Shutdown
|
|
53
|
+
raise
|
|
54
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
55
|
+
raise Wurk::Shutdown if exception_caused_by_shutdown?(e)
|
|
56
|
+
|
|
57
|
+
msg = Wurk.load_json(jobstr)
|
|
58
|
+
if msg['retry']
|
|
59
|
+
process_retry(nil, msg, queue, e)
|
|
60
|
+
else
|
|
61
|
+
run_death_handlers(msg, e)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
raise Handled
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Per-job retry guard. Same rescue semantics as `global` but the worker
|
|
68
|
+
# instance is in hand, so per-class `sidekiq_retry_in_block` and
|
|
69
|
+
# `sidekiq_retries_exhausted_block` can run. Raises `Handled` to short-
|
|
70
|
+
# circuit `global`'s rescue.
|
|
71
|
+
def local(jobinst, jobstr, queue)
|
|
72
|
+
yield
|
|
73
|
+
rescue Handled, Wurk::Shutdown
|
|
74
|
+
raise
|
|
75
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
76
|
+
raise Wurk::Shutdown if exception_caused_by_shutdown?(e)
|
|
77
|
+
|
|
78
|
+
msg = Wurk.load_json(jobstr)
|
|
79
|
+
msg['retry'] = jobinst.class.get_sidekiq_options['retry'] if msg['retry'].nil?
|
|
80
|
+
|
|
81
|
+
raise e unless msg['retry']
|
|
82
|
+
|
|
83
|
+
process_retry(jobinst, msg, queue, e)
|
|
84
|
+
raise Handled
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Component's `handle_exception` delegates to `config.handle_exception`.
|
|
88
|
+
# When initialized with a Capsule, that's not defined directly; route
|
|
89
|
+
# through the underlying Configuration.
|
|
90
|
+
def handle_exception(ex, ctx = {})
|
|
91
|
+
inner_config.handle_exception(ex, ctx)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
private
|
|
95
|
+
|
|
96
|
+
def now_ms
|
|
97
|
+
::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Bumps retry counters, stamps the error payload, decides next action:
|
|
101
|
+
# * `retry_for` exceeded → retries_exhausted
|
|
102
|
+
# * count >= attempts → retries_exhausted
|
|
103
|
+
# * sidekiq_retry_in returned :discard → drop + death_handlers
|
|
104
|
+
# * sidekiq_retry_in returned :kill → retries_exhausted (morgue)
|
|
105
|
+
# * otherwise → ZADD into `retry` at now + delay + jitter
|
|
106
|
+
def process_retry(jobinst, msg, queue, exception)
|
|
107
|
+
max_attempts = retry_attempts_from(msg['retry'], @max_retries)
|
|
108
|
+
|
|
109
|
+
msg['queue'] = msg['retry_queue'] || queue
|
|
110
|
+
|
|
111
|
+
stamp_error(msg, exception)
|
|
112
|
+
count = bump_retry_count(msg)
|
|
113
|
+
stamp_backtrace(msg, exception)
|
|
114
|
+
|
|
115
|
+
return if exhausted?(jobinst, msg, count, max_attempts, exception)
|
|
116
|
+
|
|
117
|
+
strategy, delay = delay_for(jobinst, count, exception, msg)
|
|
118
|
+
case strategy
|
|
119
|
+
when :discard
|
|
120
|
+
msg['discarded_at'] = now_ms
|
|
121
|
+
return run_death_handlers(msg, exception)
|
|
122
|
+
when :kill
|
|
123
|
+
return retries_exhausted(jobinst, msg, exception)
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
schedule_retry(msg, count, delay)
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def exhausted?(jobinst, msg, count, max_attempts, exception)
|
|
130
|
+
rf = msg['retry_for']
|
|
131
|
+
if rf
|
|
132
|
+
return false unless retry_for_exceeded?(msg['failed_at'], rf)
|
|
133
|
+
elsif count < max_attempts
|
|
134
|
+
return false
|
|
135
|
+
end
|
|
136
|
+
retries_exhausted(jobinst, msg, exception)
|
|
137
|
+
true
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def stamp_error(msg, exception)
|
|
141
|
+
m = exception_message(exception)
|
|
142
|
+
if m.respond_to?(:scrub!)
|
|
143
|
+
m.force_encoding(::Encoding::UTF_8)
|
|
144
|
+
m.scrub!
|
|
145
|
+
end
|
|
146
|
+
msg['error_message'] = m
|
|
147
|
+
msg['error_class'] = exception.class.name
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# Returns the resulting count. First failure: `retry_count=0`, `failed_at`
|
|
151
|
+
# set; subsequent failures bump `retry_count` and set `retried_at`.
|
|
152
|
+
def bump_retry_count(msg)
|
|
153
|
+
if msg['retry_count']
|
|
154
|
+
msg['retried_at'] = now_ms
|
|
155
|
+
msg['retry_count'] += 1
|
|
156
|
+
else
|
|
157
|
+
msg['failed_at'] = now_ms
|
|
158
|
+
msg['retry_count'] = 0
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
# `backtrace: true` → keep all; `backtrace: N` → keep first N.
|
|
163
|
+
# Stored as base64(zlib(JSON(lines))) — wire-compat with Sidekiq, keeps
|
|
164
|
+
# Redis payload size bounded for jobs with deep stacks.
|
|
165
|
+
def stamp_backtrace(msg, exception)
|
|
166
|
+
return unless msg['backtrace']
|
|
167
|
+
return if exception.backtrace.nil?
|
|
168
|
+
|
|
169
|
+
cleaned = (@backtrace_cleaner || ->(bt) { bt }).call(exception.backtrace)
|
|
170
|
+
lines = msg['backtrace'] == true ? cleaned : cleaned[0...msg['backtrace'].to_i]
|
|
171
|
+
msg['error_backtrace'] = compress_backtrace(lines)
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def retry_for_exceeded?(failed_at, retry_for)
|
|
175
|
+
return false unless failed_at
|
|
176
|
+
|
|
177
|
+
time_for(failed_at) + retry_for < ::Time.now
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def schedule_retry(msg, count, delay)
|
|
181
|
+
jitter = rand(10 * (count + 1))
|
|
182
|
+
retry_at = ::Time.now.to_f + delay + jitter
|
|
183
|
+
payload = Wurk.dump_json(msg)
|
|
184
|
+
redis do |conn|
|
|
185
|
+
conn.call('ZADD', Keys::RETRY, retry_at.to_s, payload)
|
|
186
|
+
end
|
|
187
|
+
Wurk::Metrics::Statsd.increment(
|
|
188
|
+
'jobs.retried',
|
|
189
|
+
tags: ["worker:#{msg['class']}", "queue:#{msg['queue']}"]
|
|
190
|
+
)
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def time_for(item)
|
|
194
|
+
if item.is_a?(::Float)
|
|
195
|
+
::Time.at(item)
|
|
196
|
+
else
|
|
197
|
+
::Time.at(item / 1000, item % 1000)
|
|
198
|
+
end
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
# Returns `[strategy, seconds]`. Strategy ∈ {:default, :discard, :kill}.
|
|
202
|
+
# Caller branches on strategy; seconds is meaningful only for :default.
|
|
203
|
+
def delay_for(jobinst, count, exception, msg)
|
|
204
|
+
rv = run_retry_in_block(jobinst, count, exception, msg)
|
|
205
|
+
rv = rv.to_i if rv.is_a?(::Float)
|
|
206
|
+
default_delay = (count**4) + 15
|
|
207
|
+
|
|
208
|
+
case rv
|
|
209
|
+
when ::Integer
|
|
210
|
+
return [:default, rv] if rv.positive?
|
|
211
|
+
when :discard
|
|
212
|
+
return [:discard, nil]
|
|
213
|
+
when :kill
|
|
214
|
+
return [:kill, nil]
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
[:default, default_delay]
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
def run_retry_in_block(jobinst, count, exception, msg) # rubocop:disable Metrics/CyclomaticComplexity
|
|
221
|
+
block = jobinst&.class&.sidekiq_retry_in_block
|
|
222
|
+
block = wrapped_block(msg, :sidekiq_retry_in_block) || block if msg['wrapped']
|
|
223
|
+
block&.call(count, exception, msg)
|
|
224
|
+
rescue ::Exception => e # rubocop:disable Lint/RescueException
|
|
225
|
+
handle_exception(e, context: "Failure scheduling retry via `sidekiq_retry_in` on #{jobinst&.class&.name}")
|
|
226
|
+
nil
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# Runs `sidekiq_retries_exhausted` (or the wrapped-class variant). Then
|
|
230
|
+
# `:discard` / `msg["dead"] == false` skips the morgue; otherwise the
|
|
231
|
+
# raw JSON is ZADD'd into `dead` and trimmed. Death handlers always fire.
|
|
232
|
+
def retries_exhausted(jobinst, msg, exception)
|
|
233
|
+
rv = run_exhausted_block(jobinst, msg, exception)
|
|
234
|
+
discarded = msg['dead'] == false || rv == :discard
|
|
235
|
+
|
|
236
|
+
if discarded
|
|
237
|
+
msg['discarded_at'] = now_ms
|
|
238
|
+
else
|
|
239
|
+
send_to_morgue(msg)
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
run_death_handlers(msg, exception)
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
def run_exhausted_block(jobinst, msg, exception)
|
|
246
|
+
block = jobinst&.class&.sidekiq_retries_exhausted_block
|
|
247
|
+
block = wrapped_block(msg, :sidekiq_retries_exhausted_block) || block if msg['wrapped']
|
|
248
|
+
block&.call(msg, exception)
|
|
249
|
+
rescue ::Exception => e # rubocop:disable Lint/RescueException
|
|
250
|
+
handle_exception(e, context: 'Error calling retries_exhausted', job: msg)
|
|
251
|
+
nil
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
# Wrappers (ActiveJob, custom) expose retry blocks on the wrapped class
|
|
255
|
+
# via `msg["wrapped"]`. We look up the constant and prefer its block.
|
|
256
|
+
def wrapped_block(msg, attr)
|
|
257
|
+
wrapped = ::Object.const_get(msg['wrapped'])
|
|
258
|
+
wrapped.respond_to?(attr) ? wrapped.public_send(attr) : nil
|
|
259
|
+
rescue ::NameError
|
|
260
|
+
nil
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
def send_to_morgue(msg)
|
|
264
|
+
logger.info { "Adding dead #{msg['class']} job #{msg['jid']}" }
|
|
265
|
+
payload = Wurk.dump_json(msg)
|
|
266
|
+
now = ::Time.now.to_f
|
|
267
|
+
redis { |conn| conn.call('ZADD', Keys::DEAD, now.to_s, payload) }
|
|
268
|
+
DeadSet.new.trim
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
def run_death_handlers(job, exception)
|
|
272
|
+
inner_config.death_handlers.each do |handler|
|
|
273
|
+
handler.call(job, exception)
|
|
274
|
+
rescue ::StandardError => e
|
|
275
|
+
handle_exception(e, context: 'Error calling death handler', job: job)
|
|
276
|
+
end
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
def retry_attempts_from(msg_retry, default)
|
|
280
|
+
msg_retry.is_a?(::Integer) ? msg_retry : default
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
# Walks `e.cause` chain looking for a `Wurk::Shutdown`. Prevents user
|
|
284
|
+
# `rescue => e` blocks (that should have re-raised Shutdown) from being
|
|
285
|
+
# treated as a normal failure that triggers retry recording.
|
|
286
|
+
def exception_caused_by_shutdown?(exception, checked = [])
|
|
287
|
+
return false unless exception.cause
|
|
288
|
+
|
|
289
|
+
checked << exception.object_id
|
|
290
|
+
return false if checked.include?(exception.cause.object_id)
|
|
291
|
+
|
|
292
|
+
exception.cause.instance_of?(Wurk::Shutdown) ||
|
|
293
|
+
exception_caused_by_shutdown?(exception.cause, checked)
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
def exception_message(exception)
|
|
297
|
+
exception.message.to_s[0, 10_000]
|
|
298
|
+
rescue ::StandardError
|
|
299
|
+
+'!!! ERROR MESSAGE THREW AN ERROR !!!'
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
def compress_backtrace(backtrace)
|
|
303
|
+
serialized = Wurk.dump_json(backtrace)
|
|
304
|
+
compressed = ::Zlib::Deflate.deflate(serialized)
|
|
305
|
+
[compressed].pack('m0')
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
# Returns the Configuration even when @capsule is a Capsule. Capsule
|
|
309
|
+
# exposes config via `#config`; if a bare Configuration was passed
|
|
310
|
+
# (tests, embedded mode), it's already the config.
|
|
311
|
+
def inner_config
|
|
312
|
+
@capsule.respond_to?(:config) ? @capsule.config : @capsule
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
def inner_config_get(key)
|
|
316
|
+
cfg = inner_config
|
|
317
|
+
cfg.respond_to?(:[]) ? cfg[key] : nil
|
|
318
|
+
end
|
|
319
|
+
end
|
|
320
|
+
end
|
data/lib/wurk/job_set.rb
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'sorted_entry'
|
|
4
|
+
|
|
5
|
+
module Wurk
|
|
6
|
+
# Base class for the three Sidekiq-compatible sorted-set views over Redis
|
|
7
|
+
# (`schedule`, `retry`, `dead`). Splits responsibilities: `SortedSet` owns
|
|
8
|
+
# the generic ZSET reads/clear; `JobSet` owns the job-aware mutations
|
|
9
|
+
# (schedule/retry_all/kill_all and the JobRecord-yielding iteration).
|
|
10
|
+
#
|
|
11
|
+
# Subclasses pick the key by passing it to `super` in `initialize`:
|
|
12
|
+
# class RetrySet < JobSet ; def initialize ; super('retry') ; end ; end
|
|
13
|
+
#
|
|
14
|
+
# Wire-compat: every Redis call below matches Sidekiq OSS exactly.
|
|
15
|
+
# Spec: docs/target/sidekiq-free.md §19.5.
|
|
16
|
+
class SortedSet
|
|
17
|
+
include Enumerable
|
|
18
|
+
|
|
19
|
+
# Page size for paged ZRANGE — matches upstream so dashboards observing
|
|
20
|
+
# Redis traffic see the same query pattern.
|
|
21
|
+
PAGE_SIZE = 50
|
|
22
|
+
|
|
23
|
+
attr_reader :name
|
|
24
|
+
|
|
25
|
+
def initialize(name)
|
|
26
|
+
@name = name.to_s
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# ZCARD. O(1) on Redis side.
|
|
30
|
+
def size
|
|
31
|
+
Wurk.redis { |conn| conn.call('ZCARD', @name) }
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Streams every (value, score) pair through ZSCAN. `match` is wrapped in
|
|
35
|
+
# `*` glob characters — callers pass a jid or class name fragment.
|
|
36
|
+
# @yield [String, Float] raw JSON payload and its score.
|
|
37
|
+
def scan(match, count = 100)
|
|
38
|
+
return enum_for(:scan, match, count) unless block_given?
|
|
39
|
+
|
|
40
|
+
cursor = '0'
|
|
41
|
+
pattern = "*#{match}*"
|
|
42
|
+
Wurk.redis do |conn|
|
|
43
|
+
loop do
|
|
44
|
+
cursor, pairs = conn.call('ZSCAN', @name, cursor, 'MATCH', pattern, 'COUNT', count)
|
|
45
|
+
pairs.each_slice(2) { |value, score| yield value, score.to_f }
|
|
46
|
+
break if cursor == '0'
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# UNLINK over the whole set. Idempotent. Method name is Sidekiq
|
|
52
|
+
# wire-compat — `clear?` would break the alias.
|
|
53
|
+
def clear # rubocop:disable Naming/PredicateMethod
|
|
54
|
+
Wurk.redis { |conn| conn.call('UNLINK', @name) }
|
|
55
|
+
true
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def as_json(_options = nil) = { name: @name }
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# ZSET-of-jobs view. Reverse-paged iteration so callers see newest-first
|
|
62
|
+
# (highest score, i.e. furthest-out retry/schedule). Mutations use
|
|
63
|
+
# ZREM-by-value when the exact bytes are known and a (score, jid) scan
|
|
64
|
+
# otherwise.
|
|
65
|
+
#
|
|
66
|
+
# Spec: docs/target/sidekiq-free.md §19.5.
|
|
67
|
+
class JobSet < SortedSet
|
|
68
|
+
# ZADD with NX so re-scheduling the same payload doesn't reset its score.
|
|
69
|
+
# Mirrors Sidekiq::JobSet#schedule exactly.
|
|
70
|
+
def schedule(timestamp, message)
|
|
71
|
+
Wurk.redis { |conn| conn.call('ZADD', @name, timestamp.to_f.to_s, Wurk.dump_json(message)) }
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Newest-first paged ZRANGE. Yields a SortedEntry per row.
|
|
75
|
+
def each
|
|
76
|
+
return enum_for(:each) unless block_given?
|
|
77
|
+
|
|
78
|
+
page = 0
|
|
79
|
+
added = 0
|
|
80
|
+
loop do
|
|
81
|
+
start = page * PAGE_SIZE
|
|
82
|
+
stop = start + PAGE_SIZE - 1
|
|
83
|
+
slice = Wurk.redis { |conn| conn.call('ZRANGE', @name, start, stop, 'REV', 'WITHSCORES') }
|
|
84
|
+
slice.each do |value, score|
|
|
85
|
+
yield SortedEntry.new(self, score, value)
|
|
86
|
+
added += 1
|
|
87
|
+
end
|
|
88
|
+
break if slice.size < PAGE_SIZE
|
|
89
|
+
|
|
90
|
+
page += 1
|
|
91
|
+
end
|
|
92
|
+
added
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# ZPOPMIN loop. Each iteration pops the single oldest member (lowest
|
|
96
|
+
# score, e.g. earliest scheduled-at) and yields the raw JSON + score.
|
|
97
|
+
# Used by the scheduled-poller: it enqueues each popped job through the
|
|
98
|
+
# client. Stops when the set is empty.
|
|
99
|
+
def pop_each
|
|
100
|
+
loop do
|
|
101
|
+
result = Wurk.redis { |conn| conn.call('ZPOPMIN', @name, 1) }
|
|
102
|
+
break if result.nil? || result.empty?
|
|
103
|
+
|
|
104
|
+
# Newer redis-client returns nested `[[value, score]]` even with COUNT 1;
|
|
105
|
+
# older `[value, score]`. Normalize both.
|
|
106
|
+
value, score = result.first.is_a?(Array) ? result.first : result
|
|
107
|
+
yield value, score.to_f
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Re-enqueues every job in this set via the client. Lossy on errors
|
|
112
|
+
# mid-iteration; callers expecting transactional behavior should
|
|
113
|
+
# batch the work themselves.
|
|
114
|
+
def retry_all
|
|
115
|
+
count = 0
|
|
116
|
+
until size.zero?
|
|
117
|
+
each do |entry|
|
|
118
|
+
entry.retry
|
|
119
|
+
count += 1
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
count
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Moves every job in this set to the dead set. `notify_failure: false`
|
|
126
|
+
# because this is a UI-initiated bulk action, not a retry-exhausted
|
|
127
|
+
# event. Returns the count of jobs moved.
|
|
128
|
+
def kill_all(notify_failure: false, ex: nil)
|
|
129
|
+
count = 0
|
|
130
|
+
dead = DeadSet.new
|
|
131
|
+
until size.zero?
|
|
132
|
+
each do |entry|
|
|
133
|
+
entry.send(:remove_job) do |message|
|
|
134
|
+
dead.kill(Wurk.dump_json(message), notify_failure: notify_failure, ex: ex)
|
|
135
|
+
end
|
|
136
|
+
count += 1
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
count
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# O(score) lookup. `score` accepts Time, Numeric, or a Range of either.
|
|
143
|
+
# Returns the matching entries (possibly multiple at the same exact
|
|
144
|
+
# score). When `jid` is set, narrows to the single (score, jid) pair.
|
|
145
|
+
def fetch(score, jid = nil)
|
|
146
|
+
results = Wurk.redis { |conn| conn.call('ZRANGEBYSCORE', @name, *range_args(score), 'WITHSCORES') }
|
|
147
|
+
entries = results.map { |value, sc| SortedEntry.new(self, sc, value) }
|
|
148
|
+
return entries unless jid
|
|
149
|
+
|
|
150
|
+
entries.select { |e| e.jid == jid }
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# ZSCAN-based search by jid substring. Returns the first matching entry
|
|
154
|
+
# or nil. O(n) on the ZSET — callers iterating many jids should switch
|
|
155
|
+
# to per-jid hashes or the score-based fetch.
|
|
156
|
+
def find_job(jid)
|
|
157
|
+
scan(jid) do |value, score|
|
|
158
|
+
entry = SortedEntry.new(self, score, value)
|
|
159
|
+
return entry if entry.jid == jid
|
|
160
|
+
end
|
|
161
|
+
nil
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# Removes the exact (score, jid)-matching member. Backs SortedEntry#delete
|
|
165
|
+
# when no cached value bytes are present.
|
|
166
|
+
def remove_job(entry)
|
|
167
|
+
delete_by_value(@name, entry.value) || delete_by_jid(entry.score, entry.jid)
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# ZREM by exact bytes. Returns true when ≥1 element was removed. Method
|
|
171
|
+
# name is Sidekiq wire-compat — `delete_by_value?` would break the alias.
|
|
172
|
+
def delete_by_value(name, value) # rubocop:disable Naming/PredicateMethod
|
|
173
|
+
removed = Wurk.redis { |conn| conn.call('ZREM', name, value) }
|
|
174
|
+
removed.to_i.positive?
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
# Scan the score bracket for a jid match, ZREM the exact bytes once found.
|
|
178
|
+
# Returns true on success. Aliased as `delete` for Sidekiq wire-compat.
|
|
179
|
+
# Per-row JSON rescue so a single malformed entry can't shadow a valid
|
|
180
|
+
# match at the same score.
|
|
181
|
+
def delete_by_jid(score, jid) # rubocop:disable Naming/PredicateMethod
|
|
182
|
+
Wurk.redis do |conn|
|
|
183
|
+
rows = conn.call('ZRANGEBYSCORE', @name, score.to_f, score.to_f)
|
|
184
|
+
rows.each do |raw|
|
|
185
|
+
parsed = begin
|
|
186
|
+
Wurk.load_json(raw)
|
|
187
|
+
rescue ::JSON::ParserError
|
|
188
|
+
nil
|
|
189
|
+
end
|
|
190
|
+
next unless parsed && parsed['jid'] == jid
|
|
191
|
+
|
|
192
|
+
return conn.call('ZREM', @name, raw).to_i.positive?
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
false
|
|
196
|
+
end
|
|
197
|
+
alias delete delete_by_jid
|
|
198
|
+
|
|
199
|
+
private
|
|
200
|
+
|
|
201
|
+
# Translates ZRANGEBYSCORE input shapes (Time, Numeric, Range) to the
|
|
202
|
+
# `min max` pair Redis expects.
|
|
203
|
+
def range_args(score)
|
|
204
|
+
case score
|
|
205
|
+
when Range then [score.begin.to_f, score.end.to_f]
|
|
206
|
+
when ::Time, Numeric then [score.to_f, score.to_f]
|
|
207
|
+
else
|
|
208
|
+
raise ArgumentError, "score must be Numeric, Time, or Range: #{score.inspect}"
|
|
209
|
+
end
|
|
210
|
+
end
|
|
211
|
+
end
|
|
212
|
+
end
|