wurk 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +43 -0
- data/CONTRIBUTING.md +73 -0
- data/LICENSE +21 -0
- data/README.md +137 -0
- data/SECURITY.md +39 -0
- data/app/controllers/wurk/api/pagination.rb +67 -0
- data/app/controllers/wurk/api/serializers.rb +131 -0
- data/app/controllers/wurk/api_controller.rb +248 -0
- data/app/controllers/wurk/application_controller.rb +7 -0
- data/app/controllers/wurk/dashboard_controller.rb +48 -0
- data/config/locales/en.yml +15 -0
- data/config/routes.rb +34 -0
- data/exe/wurk +22 -0
- data/lib/active_job/queue_adapters/wurk_adapter.rb +96 -0
- data/lib/generators/wurk/install/install_generator.rb +22 -0
- data/lib/generators/wurk/install/templates/wurk.rb +16 -0
- data/lib/wurk/active_job/wrapper.rb +32 -0
- data/lib/wurk/api/fast.rb +78 -0
- data/lib/wurk/batch/buffer.rb +26 -0
- data/lib/wurk/batch/callback_job.rb +37 -0
- data/lib/wurk/batch/callbacks.rb +176 -0
- data/lib/wurk/batch/client_middleware.rb +27 -0
- data/lib/wurk/batch/death_handler.rb +39 -0
- data/lib/wurk/batch/empty.rb +21 -0
- data/lib/wurk/batch/server_middleware.rb +62 -0
- data/lib/wurk/batch/status.rb +140 -0
- data/lib/wurk/batch.rb +351 -0
- data/lib/wurk/batch_set.rb +67 -0
- data/lib/wurk/capsule.rb +176 -0
- data/lib/wurk/cli.rb +349 -0
- data/lib/wurk/client/buffered.rb +372 -0
- data/lib/wurk/client.rb +330 -0
- data/lib/wurk/compat.rb +136 -0
- data/lib/wurk/component.rb +136 -0
- data/lib/wurk/configuration.rb +373 -0
- data/lib/wurk/context.rb +35 -0
- data/lib/wurk/cron.rb +636 -0
- data/lib/wurk/dashboard_manifest.rb +39 -0
- data/lib/wurk/dead_set.rb +78 -0
- data/lib/wurk/deploy.rb +91 -0
- data/lib/wurk/embedded.rb +94 -0
- data/lib/wurk/encryption.rb +276 -0
- data/lib/wurk/engine.rb +81 -0
- data/lib/wurk/fetcher/reaper.rb +264 -0
- data/lib/wurk/fetcher/reliable.rb +138 -0
- data/lib/wurk/fetcher.rb +11 -0
- data/lib/wurk/health.rb +193 -0
- data/lib/wurk/heartbeat.rb +211 -0
- data/lib/wurk/iterable_job.rb +292 -0
- data/lib/wurk/job/options.rb +70 -0
- data/lib/wurk/job.rb +33 -0
- data/lib/wurk/job_logger.rb +68 -0
- data/lib/wurk/job_record.rb +156 -0
- data/lib/wurk/job_retry.rb +320 -0
- data/lib/wurk/job_set.rb +212 -0
- data/lib/wurk/job_util.rb +162 -0
- data/lib/wurk/keys.rb +52 -0
- data/lib/wurk/launcher.rb +289 -0
- data/lib/wurk/leader.rb +221 -0
- data/lib/wurk/limiter/base.rb +138 -0
- data/lib/wurk/limiter/bucket.rb +80 -0
- data/lib/wurk/limiter/concurrent.rb +132 -0
- data/lib/wurk/limiter/leaky.rb +91 -0
- data/lib/wurk/limiter/points.rb +89 -0
- data/lib/wurk/limiter/server_middleware.rb +77 -0
- data/lib/wurk/limiter/unlimited.rb +48 -0
- data/lib/wurk/limiter/window.rb +80 -0
- data/lib/wurk/limiter.rb +255 -0
- data/lib/wurk/logger.rb +81 -0
- data/lib/wurk/lua/loader.rb +53 -0
- data/lib/wurk/lua.rb +187 -0
- data/lib/wurk/manager.rb +132 -0
- data/lib/wurk/metrics/history.rb +151 -0
- data/lib/wurk/metrics/query.rb +173 -0
- data/lib/wurk/metrics/rollup.rb +169 -0
- data/lib/wurk/metrics/statsd.rb +197 -0
- data/lib/wurk/metrics.rb +7 -0
- data/lib/wurk/middleware/chain.rb +128 -0
- data/lib/wurk/middleware/current_attributes.rb +87 -0
- data/lib/wurk/middleware/expiry.rb +50 -0
- data/lib/wurk/middleware/i18n.rb +63 -0
- data/lib/wurk/middleware/interrupt_handler.rb +45 -0
- data/lib/wurk/middleware/poison_pill.rb +149 -0
- data/lib/wurk/middleware.rb +34 -0
- data/lib/wurk/process_set.rb +243 -0
- data/lib/wurk/processor.rb +247 -0
- data/lib/wurk/queue.rb +108 -0
- data/lib/wurk/queues.rb +80 -0
- data/lib/wurk/rails.rb +9 -0
- data/lib/wurk/railtie.rb +28 -0
- data/lib/wurk/redis_pool.rb +79 -0
- data/lib/wurk/retry_set.rb +17 -0
- data/lib/wurk/scheduled.rb +189 -0
- data/lib/wurk/scheduled_set.rb +18 -0
- data/lib/wurk/sorted_entry.rb +95 -0
- data/lib/wurk/stats.rb +190 -0
- data/lib/wurk/swarm/child_boot.rb +105 -0
- data/lib/wurk/swarm.rb +260 -0
- data/lib/wurk/testing.rb +102 -0
- data/lib/wurk/topology.rb +74 -0
- data/lib/wurk/unique.rb +240 -0
- data/lib/wurk/version.rb +5 -0
- data/lib/wurk/web/config.rb +180 -0
- data/lib/wurk/web/enterprise.rb +138 -0
- data/lib/wurk/web/search.rb +139 -0
- data/lib/wurk/web.rb +25 -0
- data/lib/wurk/work_set.rb +116 -0
- data/lib/wurk/worker/setter.rb +93 -0
- data/lib/wurk/worker.rb +216 -0
- data/lib/wurk.rb +238 -0
- data/vendor/assets/dashboard/assets/index-8P3N_m1X.js +152 -0
- data/vendor/assets/dashboard/assets/index-Bqz4_SOQ.css +1 -0
- data/vendor/assets/dashboard/index.html +13 -0
- data/vendor/assets/dashboard/wurk-manifest.json +4 -0
- metadata +232 -0
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'job_set'
|
|
4
|
+
|
|
5
|
+
module Wurk
|
|
6
|
+
# Capped ZSET of jobs that exhausted retries (the "morgue"). Bounded by
|
|
7
|
+
# `dead_max_jobs` and `dead_timeout_in_seconds` config knobs — every
|
|
8
|
+
# `kill` trims both axes. Death handlers fire on retry-exhausted kills
|
|
9
|
+
# (notify_failure: true), not on user-initiated UI kills.
|
|
10
|
+
#
|
|
11
|
+
# Spec: docs/target/sidekiq-free.md §19.5, §17.2, §31.8.
|
|
12
|
+
class DeadSet < JobSet
|
|
13
|
+
# Optional `name` allows tests to operate on a namespaced ZSET; production
|
|
14
|
+
# callers always use the default `'dead'` key (wire-compat with Sidekiq).
|
|
15
|
+
def initialize(name = 'dead')
|
|
16
|
+
super
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# Two-axis trim: `ZREMRANGEBYSCORE` evicts entries older than
|
|
20
|
+
# `dead_timeout_in_seconds`, `ZREMRANGEBYRANK 0 -dead_max_jobs` keeps
|
|
21
|
+
# the count bounded. Pipelined — partial failure leaves at most one
|
|
22
|
+
# axis applied (acceptable; trim is non-critical, runs again next kill).
|
|
23
|
+
#
|
|
24
|
+
# `max_jobs:` / `timeout:` override the global config for this call.
|
|
25
|
+
# Lets parallel tests run trim with isolated limits without mutating
|
|
26
|
+
# `Wurk.configuration` (which is process-global and races across threads).
|
|
27
|
+
def trim(max_jobs: nil, timeout: nil) # rubocop:disable Naming/PredicateMethod
|
|
28
|
+
config = Wurk.configuration
|
|
29
|
+
max_jobs ||= config[:dead_max_jobs] || 10_000
|
|
30
|
+
timeout ||= config[:dead_timeout_in_seconds] || (180 * 24 * 60 * 60)
|
|
31
|
+
cutoff = ::Process.clock_gettime(::Process::CLOCK_REALTIME) - timeout
|
|
32
|
+
|
|
33
|
+
Wurk.redis do |conn|
|
|
34
|
+
conn.pipelined do |pipe|
|
|
35
|
+
pipe.call('ZREMRANGEBYSCORE', @name, '-inf', "(#{cutoff}")
|
|
36
|
+
pipe.call('ZREMRANGEBYRANK', @name, 0, -(max_jobs + 1))
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
true
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# ZADD the raw JSON payload, trim, fire death handlers. `notify_failure:
|
|
43
|
+
# true` (default) routes the kill through the death-handler chain;
|
|
44
|
+
# UI-initiated kills pass false. `ex` is the originating exception (or
|
|
45
|
+
# synthesized RuntimeError when callers don't have one) — death handlers
|
|
46
|
+
# receive `(job, ex)`. `max_jobs:` / `timeout:` propagate to the auto-trim;
|
|
47
|
+
# see `#trim` for the rationale.
|
|
48
|
+
def kill(message, opts = {}) # rubocop:disable Naming/PredicateMethod
|
|
49
|
+
notify = opts.fetch(:notify_failure, true)
|
|
50
|
+
do_trim = opts.fetch(:trim, true)
|
|
51
|
+
ex = opts[:ex] || RuntimeError.new('Job killed')
|
|
52
|
+
|
|
53
|
+
now = ::Process.clock_gettime(::Process::CLOCK_REALTIME)
|
|
54
|
+
Wurk.redis { |conn| conn.call('ZADD', @name, now.to_s, message) }
|
|
55
|
+
trim(max_jobs: opts[:max_jobs], timeout: opts[:timeout]) if do_trim
|
|
56
|
+
fire_death_handlers(message, ex) if notify
|
|
57
|
+
true
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
private
|
|
61
|
+
|
|
62
|
+
def fire_death_handlers(message, ex)
|
|
63
|
+
job = parse_message(message)
|
|
64
|
+
handlers = Wurk.configuration.death_handlers
|
|
65
|
+
handlers.each do |handler|
|
|
66
|
+
handler.call(job, ex)
|
|
67
|
+
rescue StandardError => e
|
|
68
|
+
Wurk.configuration.handle_exception(e, context: 'death handler')
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def parse_message(message)
|
|
73
|
+
message.is_a?(String) ? Wurk.load_json(message) : message
|
|
74
|
+
rescue ::JSON::ParserError
|
|
75
|
+
{ 'class' => 'Unknown', 'args' => [], '_raw' => message }
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
data/lib/wurk/deploy.rb
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Wurk
|
|
4
|
+
# Records deploy markers into Redis so the history pane can overlay
|
|
5
|
+
# "deployed at" lines onto job-throughput charts. Wire-compatible with
|
|
6
|
+
# Sidekiq's `Sidekiq::Deploy`:
|
|
7
|
+
#
|
|
8
|
+
# <YYYYMMDD>-marks HASH fields = iso8601 timestamp (rounded to minute),
|
|
9
|
+
# values = label string, TTL = 90 days
|
|
10
|
+
# deploylock-<label> STRING SET NX EX 60, dedupe lock for same-label marks
|
|
11
|
+
#
|
|
12
|
+
# The lock collapses multiple `mark!` calls for the same label inside a
|
|
13
|
+
# 60-second window into a single HSET — fleet-wide deploys where every
|
|
14
|
+
# process calls `Wurk::Deploy.mark!` at boot end up with one row, not N.
|
|
15
|
+
#
|
|
16
|
+
# Spec: docs/target/sidekiq-free.md §23.
|
|
17
|
+
class Deploy
|
|
18
|
+
MARK_TTL = 90 * 24 * 60 * 60 # 90 days
|
|
19
|
+
LOCK_TTL = 60 # 1 minute dedupe window
|
|
20
|
+
LOCK_PREFIX = 'deploylock-'
|
|
21
|
+
MARKS_SUFFIX = '-marks'
|
|
22
|
+
|
|
23
|
+
# Default label maker: short git SHA + commit subject. Same shape as
|
|
24
|
+
# Sidekiq Pro's LABEL_MAKER so existing deploy hooks keep working.
|
|
25
|
+
LABEL_MAKER = -> { `git log -1 --format="%h %s"`.strip }
|
|
26
|
+
|
|
27
|
+
# Class-level shorthand `Wurk::Deploy.mark!(label: "abc")` builds a
|
|
28
|
+
# one-shot Deploy and delegates. Matches Sidekiq's `Sidekiq::Deploy.mark!`.
|
|
29
|
+
def self.mark!(label: nil, at: ::Time.now)
|
|
30
|
+
new.mark!(label: label, at: at)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def initialize(pool: nil)
|
|
34
|
+
@pool = pool
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Writes a deploy marker. Returns the iso8601 timestamp written, or
|
|
38
|
+
# `nil` when the per-label lock was already held (a previous process
|
|
39
|
+
# within the last 60 seconds beat us to it for the same label).
|
|
40
|
+
def mark!(label: nil, at: ::Time.now)
|
|
41
|
+
label = resolve_label(label)
|
|
42
|
+
return nil if label.nil? || label.empty?
|
|
43
|
+
|
|
44
|
+
ts = round_to_minute(at)
|
|
45
|
+
iso = ts.iso8601
|
|
46
|
+
write_mark?(label, ts, iso) ? iso : nil
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Returns the deploy marks for `date` (Date or Time) as `{iso8601 => label}`.
|
|
50
|
+
def fetch(date = ::Time.now.utc)
|
|
51
|
+
date = date.utc if date.respond_to?(:utc)
|
|
52
|
+
key = "#{date.strftime('%Y%m%d')}#{MARKS_SUFFIX}"
|
|
53
|
+
with_redis { |c| c.call('HGETALL', key) } || {}
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
private
|
|
57
|
+
|
|
58
|
+
def round_to_minute(at)
|
|
59
|
+
utc = at.utc
|
|
60
|
+
::Time.utc(utc.year, utc.month, utc.day, utc.hour, utc.min)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def write_mark?(label, time, iso)
|
|
64
|
+
lock_key = "#{LOCK_PREFIX}#{label}"
|
|
65
|
+
marks_key = "#{time.strftime('%Y%m%d')}#{MARKS_SUFFIX}"
|
|
66
|
+
with_redis do |conn|
|
|
67
|
+
return false unless conn.call('SET', lock_key, iso, 'NX', 'EX', LOCK_TTL) == 'OK'
|
|
68
|
+
|
|
69
|
+
conn.call('HSET', marks_key, iso, label)
|
|
70
|
+
conn.call('EXPIRE', marks_key, MARK_TTL)
|
|
71
|
+
end
|
|
72
|
+
true
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def resolve_label(label)
|
|
76
|
+
return label unless label.nil? || label.empty?
|
|
77
|
+
|
|
78
|
+
LABEL_MAKER.call
|
|
79
|
+
rescue StandardError
|
|
80
|
+
nil
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def with_redis(&)
|
|
84
|
+
if @pool
|
|
85
|
+
@pool.with(&)
|
|
86
|
+
else
|
|
87
|
+
Wurk.redis(&)
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'component'
|
|
4
|
+
require_relative 'launcher'
|
|
5
|
+
|
|
6
|
+
module Wurk
|
|
7
|
+
# Embeds Wurk inside an arbitrary Ruby process (Puma, rake task, custom
|
|
8
|
+
# daemon) without forking a swarm. Same Launcher/Manager/Processor stack
|
|
9
|
+
# the CLI uses — only the parent-process supervision is skipped.
|
|
10
|
+
#
|
|
11
|
+
# Typical use:
|
|
12
|
+
# instance = Wurk.configure_embed { |c| c.queues = %w[critical default] }
|
|
13
|
+
# instance.run
|
|
14
|
+
# # ... host process serves traffic ...
|
|
15
|
+
# instance.stop
|
|
16
|
+
#
|
|
17
|
+
# Concurrency defaults to 2 (set in `Wurk.configure_embed`) because the GIL
|
|
18
|
+
# makes contention worse in a host process that already has its own thread
|
|
19
|
+
# pool. The heartbeat marks `embedded: true` so the dashboard distinguishes
|
|
20
|
+
# embedded workers from swarm-forked workers.
|
|
21
|
+
#
|
|
22
|
+
# Spec: docs/target/sidekiq-free.md §22 (Sidekiq::Embedded).
|
|
23
|
+
class Embedded
|
|
24
|
+
include Component
|
|
25
|
+
|
|
26
|
+
attr_reader :launcher
|
|
27
|
+
|
|
28
|
+
def initialize(config)
|
|
29
|
+
@config = config
|
|
30
|
+
@launcher = nil
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Validates Redis, fires :startup, boots the launcher, sleeps 0.2 so
|
|
34
|
+
# the worker threads have time to spin up before the host goes back to
|
|
35
|
+
# serving traffic. Mirrors Sidekiq::Embedded#run.
|
|
36
|
+
def run
|
|
37
|
+
housekeeping
|
|
38
|
+
fire_event(:startup, reverse: false, reraise: true)
|
|
39
|
+
@launcher = build_launcher
|
|
40
|
+
@launcher.run
|
|
41
|
+
sleep 0.2
|
|
42
|
+
logger.info { "Wurk running embedded, total process thread count: #{Thread.list.size}" }
|
|
43
|
+
logger.debug { Thread.list.map(&:name).to_s }
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Stop fetching new work; in-flight jobs continue.
|
|
47
|
+
def quiet
|
|
48
|
+
@launcher&.quiet
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Graceful drain inside config[:timeout]; cancels in-flight jobs that
|
|
52
|
+
# exceed the deadline.
|
|
53
|
+
def stop
|
|
54
|
+
@launcher&.stop
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
# Extracted so tests can swap in a fake launcher without monkey-patching
|
|
60
|
+
# Wurk::Launcher.new globally (which races other parallel tests).
|
|
61
|
+
def build_launcher
|
|
62
|
+
Launcher.new(@config, embedded: true)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Same checks Wurk::CLI performs before launch: tag default, Redis
|
|
66
|
+
# version floor, maxmemory-policy advisory. We intentionally do NOT
|
|
67
|
+
# run validate_pool_sizes! — the host owns its connection pools and
|
|
68
|
+
# may legitimately undersize for embedded workloads.
|
|
69
|
+
def housekeeping
|
|
70
|
+
@config[:tag] ||= default_tag
|
|
71
|
+
logger.info "Running in #{RUBY_DESCRIPTION}"
|
|
72
|
+
validate_redis!
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def validate_redis!
|
|
76
|
+
info = @config.redis_pool.info
|
|
77
|
+
ver = ::Gem::Version.new(info['redis_version'])
|
|
78
|
+
if ver < ::Gem::Version.new(CLI::MIN_REDIS_VERSION)
|
|
79
|
+
raise "You are connected to Redis #{ver}, Wurk requires Redis #{CLI::MIN_REDIS_VERSION} or greater"
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
policy = info['maxmemory_policy']
|
|
83
|
+
return if policy.nil? || policy.empty? || policy == 'noeviction'
|
|
84
|
+
|
|
85
|
+
logger.warn { <<~WARN }
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
WARNING: Your Redis instance will evict Wurk data under heavy load.
|
|
89
|
+
The 'noeviction' maxmemory policy is recommended (current policy: '#{policy}').
|
|
90
|
+
|
|
91
|
+
WARN
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'base64'
|
|
4
|
+
require 'json'
|
|
5
|
+
require 'openssl'
|
|
6
|
+
require_relative 'middleware'
|
|
7
|
+
require_relative 'dead_set'
|
|
8
|
+
require_relative 'job_retry'
|
|
9
|
+
require_relative 'metrics/statsd'
|
|
10
|
+
|
|
11
|
+
module Wurk
|
|
12
|
+
# Sidekiq Enterprise encryption. AES-256-GCM over the **last** positional
|
|
13
|
+
# argument of `perform`. Implemented as a client/server middleware pair —
|
|
14
|
+
# client envelopes the last arg into a `{v, iv, ct, tag}` Hash, server
|
|
15
|
+
# peels it back before invoking perform.
|
|
16
|
+
#
|
|
17
|
+
# Activation:
|
|
18
|
+
#
|
|
19
|
+
# Sidekiq::Enterprise::Crypto.enable(active_version: 1) do |version|
|
|
20
|
+
# File.read("config/crypto/secret.#{Rails.env}.#{version}.key", mode: 'rb')
|
|
21
|
+
# end
|
|
22
|
+
#
|
|
23
|
+
# The block is the **only** key source: file, ENV, KMS — anything that maps
|
|
24
|
+
# `Integer version → 32-byte binary key`. Wurk caches resolved keys per
|
|
25
|
+
# version in-process; rotate by writing a new key file, bumping
|
|
26
|
+
# `active_version`, and calling `enable` again.
|
|
27
|
+
#
|
|
28
|
+
# Per-worker opt-in:
|
|
29
|
+
#
|
|
30
|
+
# class PrivateJob
|
|
31
|
+
# include Sidekiq::Job
|
|
32
|
+
# sidekiq_options encrypt: true
|
|
33
|
+
# def perform(public_arg, secret_bag); end
|
|
34
|
+
# end
|
|
35
|
+
#
|
|
36
|
+
# Wire format (per docs/target/sidekiq-ent.md §4.4): the last arg becomes
|
|
37
|
+
# a plain JSON Hash `{"v"=>N, "iv"=>b64(iv), "ct"=>b64(ct), "tag"=>b64(tag)}`
|
|
38
|
+
# — *not* a base64 blob of a binary envelope — so the args array stays
|
|
39
|
+
# valid JSON for inspectors that don't know about encryption.
|
|
40
|
+
#
|
|
41
|
+
# Constraints:
|
|
42
|
+
# * `perform` must take ≥ 2 positional args. Pass `nil` first if no
|
|
43
|
+
# cleartext payload exists.
|
|
44
|
+
# * Only the last positional argument is encrypted. All earlier args
|
|
45
|
+
# remain plaintext.
|
|
46
|
+
# * Incompatible with Wurk::Unique (each ciphertext differs → digest
|
|
47
|
+
# defeats the lock). Documented invariant.
|
|
48
|
+
# * Web UI redacts the last arg when `encrypt: true` is set on the job.
|
|
49
|
+
#
|
|
50
|
+
# Spec: docs/target/sidekiq-ent.md §4.
|
|
51
|
+
module Encryption # rubocop:disable Metrics/ModuleLength
|
|
52
|
+
CIPHER_NAME = 'aes-256-gcm'
|
|
53
|
+
KEY_BYTES = 32
|
|
54
|
+
IV_BYTES = 12 # GCM standard: 96-bit IV.
|
|
55
|
+
TAG_BYTES = 16
|
|
56
|
+
ENVELOPE_MARKER = '__wurk_enc__'
|
|
57
|
+
|
|
58
|
+
# Reason tag stamped on the dead-set record when a job can't be
|
|
59
|
+
# decrypted. Surfaced as `error_class` (dashboard "Dead" column) and the
|
|
60
|
+
# `encryption_error:` prefix on `error_message`, plus the `jobs.encryption_error`
|
|
61
|
+
# statsd counter — so operators can alert on rotation gaps.
|
|
62
|
+
DEAD_REASON = 'encryption_error'
|
|
63
|
+
DECRYPTION_ERROR_CLASS = 'Wurk::Encryption::DecryptionError'
|
|
64
|
+
|
|
65
|
+
class Error < StandardError; end
|
|
66
|
+
class KeyMissingError < Error; end
|
|
67
|
+
|
|
68
|
+
# Marker for a terminal, non-retryable decryption failure (missing or
|
|
69
|
+
# rotated-away key, tampered ciphertext). The server middleware raises
|
|
70
|
+
# JobRetry::Skip after routing the job to the dead set, so this class
|
|
71
|
+
# is what callers see as the dead record's `error_class`.
|
|
72
|
+
class DecryptionError < Error; end
|
|
73
|
+
|
|
74
|
+
class << self
|
|
75
|
+
attr_reader :active_version
|
|
76
|
+
|
|
77
|
+
def enabled?
|
|
78
|
+
@enabled == true
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Install crypto with a key resolver. `active_version` is the version
|
|
82
|
+
# used to *encrypt* new pushes; the resolver block must still return
|
|
83
|
+
# keys for any older in-flight versions so they decrypt.
|
|
84
|
+
#
|
|
85
|
+
# Idempotent: re-calling rebinds the resolver and rebuilds the cache.
|
|
86
|
+
# Middleware is installed at most once per chain.
|
|
87
|
+
def enable(active_version:, &resolver) # rubocop:disable Naming/PredicateMethod
|
|
88
|
+
raise ArgumentError, 'active_version is required' unless active_version
|
|
89
|
+
raise ArgumentError, 'block returning the key bytes is required' unless resolver
|
|
90
|
+
|
|
91
|
+
@active_version = Integer(active_version)
|
|
92
|
+
@resolver = resolver
|
|
93
|
+
@key_cache = {}
|
|
94
|
+
@enabled = true
|
|
95
|
+
register_middleware!
|
|
96
|
+
true
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Test helper — not part of the public Sidekiq surface.
|
|
100
|
+
def disable!
|
|
101
|
+
@enabled = false
|
|
102
|
+
@active_version = nil
|
|
103
|
+
@resolver = nil
|
|
104
|
+
@key_cache = nil
|
|
105
|
+
nil
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Resolve and cache the 32-byte key for `version`. Re-raises with a
|
|
109
|
+
# Wurk-specific exception when the resolver returns nothing usable so
|
|
110
|
+
# callers don't have to detect "nil from block" themselves.
|
|
111
|
+
def key_for(version)
|
|
112
|
+
raise Error, 'Wurk::Encryption not enabled' unless enabled?
|
|
113
|
+
|
|
114
|
+
@key_cache ||= {}
|
|
115
|
+
@key_cache[version] ||= validate_key!(version, @resolver.call(version))
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Encrypt `value` (any JSON-serializable Ruby value) under
|
|
119
|
+
# `active_version`. Returns a Hash literal — JSON-friendly, so the
|
|
120
|
+
# job payload stays inspectable.
|
|
121
|
+
def encrypt(value)
|
|
122
|
+
version = @active_version
|
|
123
|
+
key = key_for(version)
|
|
124
|
+
iv = ::OpenSSL::Random.random_bytes(IV_BYTES)
|
|
125
|
+
cipher = ::OpenSSL::Cipher.new(CIPHER_NAME).encrypt
|
|
126
|
+
cipher.key = key
|
|
127
|
+
cipher.iv = iv
|
|
128
|
+
ct = cipher.update(::JSON.dump(value)) + cipher.final
|
|
129
|
+
tag = cipher.auth_tag(TAG_BYTES)
|
|
130
|
+
|
|
131
|
+
{
|
|
132
|
+
ENVELOPE_MARKER => true,
|
|
133
|
+
'v' => version,
|
|
134
|
+
'iv' => ::Base64.strict_encode64(iv),
|
|
135
|
+
'ct' => ::Base64.strict_encode64(ct),
|
|
136
|
+
'tag' => ::Base64.strict_encode64(tag)
|
|
137
|
+
}
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# Decrypt the envelope produced by `encrypt`. Raises
|
|
141
|
+
# `OpenSSL::Cipher::CipherError` on tag mismatch (bad key / tamper)
|
|
142
|
+
# — server middleware lets it bubble so the failure flows through
|
|
143
|
+
# the retry/dead pipeline per §4.6.
|
|
144
|
+
def decrypt(envelope)
|
|
145
|
+
version = Integer(envelope['v'])
|
|
146
|
+
cipher = build_decrypt_cipher(envelope, key_for(version))
|
|
147
|
+
plain = cipher.update(::Base64.strict_decode64(envelope['ct'])) + cipher.final
|
|
148
|
+
::JSON.parse(plain, quirks_mode: true)
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# @return [Boolean] true if `value` looks like a Wurk crypto envelope.
|
|
152
|
+
# Used by both server middleware and the Web UI redactor — single
|
|
153
|
+
# source of truth so the two cannot drift.
|
|
154
|
+
def envelope?(value)
|
|
155
|
+
value.is_a?(::Hash) && value[ENVELOPE_MARKER] == true &&
|
|
156
|
+
value.key?('v') && value.key?('iv') && value.key?('ct') && value.key?('tag')
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# Web UI display helper (§4.7). Given a job hash, returns the args
|
|
160
|
+
# array with the last element replaced by the literal `"<encrypted>"`
|
|
161
|
+
# when the job opted in. Cleartext preceding args are untouched so
|
|
162
|
+
# operators can still triage on user_id / object_id / etc.
|
|
163
|
+
def redact_args(job)
|
|
164
|
+
args = job['args'] || job[:args] || []
|
|
165
|
+
return args unless job['encrypt'] || job[:encrypt]
|
|
166
|
+
return args if args.empty?
|
|
167
|
+
|
|
168
|
+
args[0..-2] + ['<encrypted>']
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# A decryption failure means the key is gone (rotated away) or the
|
|
172
|
+
# ciphertext is bad — neither heals with time, so retrying 25× over
|
|
173
|
+
# ~21 days is a pointless crash loop. Instead the server middleware
|
|
174
|
+
# routes the job straight to the dead set, tagged `encryption_error`,
|
|
175
|
+
# and ACKs it (raises JobRetry::Skip). Done in <1s, death handlers fire
|
|
176
|
+
# so operators get paged. The still-encrypted envelope is kept on the
|
|
177
|
+
# record; earlier plaintext args stay visible for triage (§4.6).
|
|
178
|
+
def route_to_dead(job, cause)
|
|
179
|
+
record = job.merge(
|
|
180
|
+
'error_class' => DECRYPTION_ERROR_CLASS,
|
|
181
|
+
'error_message' => "#{DEAD_REASON}: #{cause.class}: #{cause.message}",
|
|
182
|
+
'failed_at' => ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
|
|
183
|
+
)
|
|
184
|
+
Wurk::Metrics::Statsd.increment('jobs.encryption_error', tags: ["worker:#{job['class']}"])
|
|
185
|
+
Wurk::DeadSet.new.kill(Wurk.dump_json(record), ex: cause)
|
|
186
|
+
nil
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
private
|
|
190
|
+
|
|
191
|
+
def build_decrypt_cipher(envelope, key)
|
|
192
|
+
cipher = ::OpenSSL::Cipher.new(CIPHER_NAME).decrypt
|
|
193
|
+
cipher.key = key
|
|
194
|
+
cipher.iv = ::Base64.strict_decode64(envelope['iv'])
|
|
195
|
+
cipher.auth_tag = ::Base64.strict_decode64(envelope['tag'])
|
|
196
|
+
cipher
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def validate_key!(version, bytes)
|
|
200
|
+
raise KeyMissingError, "key resolver returned nil for version #{version}" if bytes.nil?
|
|
201
|
+
|
|
202
|
+
bytes = bytes.dup.force_encoding(::Encoding::ASCII_8BIT)
|
|
203
|
+
unless bytes.bytesize == KEY_BYTES
|
|
204
|
+
raise Error, "key for version #{version} must be #{KEY_BYTES} bytes, got #{bytes.bytesize}"
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
bytes
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
def register_middleware!
|
|
211
|
+
Wurk.configuration.client_middleware.add(ClientMiddleware) \
|
|
212
|
+
unless Wurk.configuration.client_middleware.exists?(ClientMiddleware)
|
|
213
|
+
Wurk.configuration.server_middleware.add(ServerMiddleware) \
|
|
214
|
+
unless Wurk.configuration.server_middleware.exists?(ServerMiddleware)
|
|
215
|
+
end
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# Client middleware — envelopes the **last** positional argument when
|
|
219
|
+
# the job opts in via `sidekiq_options encrypt: true`. Skips silently
|
|
220
|
+
# when crypto is disabled, the job didn't opt in, or `args` is empty
|
|
221
|
+
# (per §4.3, an opt-in worker with empty args is a user bug, but we
|
|
222
|
+
# don't enqueue garbage — let perform raise ArgumentError if it cares).
|
|
223
|
+
class ClientMiddleware
|
|
224
|
+
include Wurk::Middleware::ClientMiddleware
|
|
225
|
+
|
|
226
|
+
def call(_worker, job, _queue, _redis_pool)
|
|
227
|
+
return yield unless Wurk::Encryption.enabled? && job['encrypt']
|
|
228
|
+
|
|
229
|
+
args = job['args']
|
|
230
|
+
if args.is_a?(::Array) && !args.empty? && !Wurk::Encryption.envelope?(args.last)
|
|
231
|
+
job['args'] = args[0..-2] + [Wurk::Encryption.encrypt(args.last)]
|
|
232
|
+
end
|
|
233
|
+
yield
|
|
234
|
+
end
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
# Server middleware — peels the envelope before perform runs. A decrypt
|
|
238
|
+
# failure (missing/rotated key, bad tag) is terminal and non-retryable,
|
|
239
|
+
# so rather than let it bubble into the 25× retry pipeline we route the
|
|
240
|
+
# job straight to the dead set tagged `encryption_error` and ACK it via
|
|
241
|
+
# JobRetry::Skip — see Wurk::Encryption.route_to_dead. Plaintext args
|
|
242
|
+
# remain visible for triage per §4.6.
|
|
243
|
+
class ServerMiddleware
|
|
244
|
+
include Wurk::Middleware::ServerMiddleware
|
|
245
|
+
|
|
246
|
+
def call(_worker, job, _queue)
|
|
247
|
+
return yield unless Wurk::Encryption.enabled? && job['encrypt']
|
|
248
|
+
|
|
249
|
+
decrypt_last_arg!(job)
|
|
250
|
+
yield
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
private
|
|
254
|
+
|
|
255
|
+
def decrypt_last_arg!(job)
|
|
256
|
+
args = job['args']
|
|
257
|
+
return unless args.is_a?(::Array) && !args.empty? && Wurk::Encryption.envelope?(args.last)
|
|
258
|
+
|
|
259
|
+
job['args'] = args[0..-2] + [Wurk::Encryption.decrypt(args.last)]
|
|
260
|
+
rescue Wurk::Encryption::Error,
|
|
261
|
+
::OpenSSL::Cipher::CipherError,
|
|
262
|
+
::ArgumentError,
|
|
263
|
+
::TypeError,
|
|
264
|
+
::JSON::ParserError => e
|
|
265
|
+
# Wurk::Encryption::Error covers KeyMissingError / DecryptionError.
|
|
266
|
+
# OpenSSL::Cipher::CipherError → tampered ciphertext / bad key (tag mismatch).
|
|
267
|
+
# ArgumentError → Base64.strict_decode64 / Integer() on a malformed envelope.
|
|
268
|
+
# TypeError → Integer(nil) on a missing 'v' field.
|
|
269
|
+
# JSON::ParserError → ciphertext decrypts to non-JSON (different key, but auth tag is gone via GCM → rare,
|
|
270
|
+
# but cheap to cover so malformed payloads never re-enter the 25× retry loop).
|
|
271
|
+
Wurk::Encryption.route_to_dead(job, e)
|
|
272
|
+
raise Wurk::JobRetry::Skip, "#{Wurk::Encryption::DEAD_REASON}: #{e.class}"
|
|
273
|
+
end
|
|
274
|
+
end
|
|
275
|
+
end
|
|
276
|
+
end
|
data/lib/wurk/engine.rb
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'rails/engine'
|
|
4
|
+
require_relative '../active_job/queue_adapters/wurk_adapter'
|
|
5
|
+
require_relative 'dashboard_manifest'
|
|
6
|
+
require_relative 'web'
|
|
7
|
+
|
|
8
|
+
module Wurk
|
|
9
|
+
# Rails mountable engine. Owns the dashboard mount, the asset path for
|
|
10
|
+
# the precompiled SPA, and (via the sibling railtie) the after_initialize
|
|
11
|
+
# hook that boots the swarm.
|
|
12
|
+
class Engine < ::Rails::Engine
|
|
13
|
+
isolate_namespace Wurk
|
|
14
|
+
|
|
15
|
+
config.generators do |g|
|
|
16
|
+
g.test_framework :minitest, fixtures: false
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# Rack::Files doesn't strip the URL prefix before file lookup, so
|
|
20
|
+
# `/wurk-assets/assets/foo.js` would resolve under `vendor/assets/dashboard/
|
|
21
|
+
# wurk-assets/assets/foo.js` — a path that doesn't exist. AssetMount
|
|
22
|
+
# rewrites PATH_INFO to drop the `/wurk-assets` prefix before delegating
|
|
23
|
+
# to Rack::Files, then falls through to the next middleware on 404 so
|
|
24
|
+
# host-app routes outside the mount keep working.
|
|
25
|
+
class AssetMount
|
|
26
|
+
PREFIX = '/wurk-assets'
|
|
27
|
+
|
|
28
|
+
def initialize(app, root:)
|
|
29
|
+
@app = app
|
|
30
|
+
@files = ::Rack::Files.new(root)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def call(env)
|
|
34
|
+
path = env[::Rack::PATH_INFO]
|
|
35
|
+
return @app.call(env) unless path == PREFIX || path.start_with?("#{PREFIX}/")
|
|
36
|
+
|
|
37
|
+
inner = env.dup
|
|
38
|
+
stripped = path.delete_prefix(PREFIX)
|
|
39
|
+
inner[::Rack::PATH_INFO] = stripped.empty? ? '/' : stripped
|
|
40
|
+
response = @files.call(inner)
|
|
41
|
+
response[0] == 404 ? @app.call(env) : response
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Precompiled SPA lives in vendor/assets/dashboard; the engine serves
|
|
46
|
+
# those files as static assets under the /wurk-assets mount point via
|
|
47
|
+
# AssetMount (above).
|
|
48
|
+
initializer 'wurk.assets' do |app|
|
|
49
|
+
assets_path = Wurk::Engine.root.join('vendor', 'assets', 'dashboard')
|
|
50
|
+
if assets_path.exist?
|
|
51
|
+
app.middleware.insert_before(
|
|
52
|
+
::ActionDispatch::Static,
|
|
53
|
+
::Wurk::Engine::AssetMount,
|
|
54
|
+
root: assets_path.to_s
|
|
55
|
+
)
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Fail boot in production if the precompiled bundle is missing or its
|
|
60
|
+
# version doesn't match the gem. Dev and test skip — contributors don't
|
|
61
|
+
# always have a fresh build, and Vite dev mode owns the shell directly.
|
|
62
|
+
initializer 'wurk.dashboard_manifest_check' do
|
|
63
|
+
next if ENV['WURK_VITE_DEV'] == '1'
|
|
64
|
+
next unless ::Rails.env.production?
|
|
65
|
+
|
|
66
|
+
::Wurk::DashboardManifest.check!
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Engine-scoped Rack middleware. Inserted into the engine — not the host —
|
|
70
|
+
# so the host's own controllers stay unaffected; only requests routed
|
|
71
|
+
# under the mount point pass through.
|
|
72
|
+
#
|
|
73
|
+
# * MiddlewareStack — host-app auth via `Wurk::Web.use` (#41). Outermost,
|
|
74
|
+
# so Devise/Warden/Sorcery run first and their `env` is visible to the
|
|
75
|
+
# authorization hook below.
|
|
76
|
+
# * Authorization — the `Wurk::Web.configure` authorization + read-only
|
|
77
|
+
# hook (sidekiq-ent §9.2), 403 on falsey.
|
|
78
|
+
middleware.use ::Wurk::Web::MiddlewareStack
|
|
79
|
+
middleware.use ::Wurk::Web::Authorization
|
|
80
|
+
end
|
|
81
|
+
end
|