wurk 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +43 -0
- data/CONTRIBUTING.md +73 -0
- data/LICENSE +21 -0
- data/README.md +137 -0
- data/SECURITY.md +39 -0
- data/app/controllers/wurk/api/pagination.rb +67 -0
- data/app/controllers/wurk/api/serializers.rb +131 -0
- data/app/controllers/wurk/api_controller.rb +248 -0
- data/app/controllers/wurk/application_controller.rb +7 -0
- data/app/controllers/wurk/dashboard_controller.rb +48 -0
- data/config/locales/en.yml +15 -0
- data/config/routes.rb +34 -0
- data/exe/wurk +22 -0
- data/lib/active_job/queue_adapters/wurk_adapter.rb +96 -0
- data/lib/generators/wurk/install/install_generator.rb +22 -0
- data/lib/generators/wurk/install/templates/wurk.rb +16 -0
- data/lib/wurk/active_job/wrapper.rb +32 -0
- data/lib/wurk/api/fast.rb +78 -0
- data/lib/wurk/batch/buffer.rb +26 -0
- data/lib/wurk/batch/callback_job.rb +37 -0
- data/lib/wurk/batch/callbacks.rb +176 -0
- data/lib/wurk/batch/client_middleware.rb +27 -0
- data/lib/wurk/batch/death_handler.rb +39 -0
- data/lib/wurk/batch/empty.rb +21 -0
- data/lib/wurk/batch/server_middleware.rb +62 -0
- data/lib/wurk/batch/status.rb +140 -0
- data/lib/wurk/batch.rb +351 -0
- data/lib/wurk/batch_set.rb +67 -0
- data/lib/wurk/capsule.rb +176 -0
- data/lib/wurk/cli.rb +349 -0
- data/lib/wurk/client/buffered.rb +372 -0
- data/lib/wurk/client.rb +330 -0
- data/lib/wurk/compat.rb +136 -0
- data/lib/wurk/component.rb +136 -0
- data/lib/wurk/configuration.rb +373 -0
- data/lib/wurk/context.rb +35 -0
- data/lib/wurk/cron.rb +636 -0
- data/lib/wurk/dashboard_manifest.rb +39 -0
- data/lib/wurk/dead_set.rb +78 -0
- data/lib/wurk/deploy.rb +91 -0
- data/lib/wurk/embedded.rb +94 -0
- data/lib/wurk/encryption.rb +276 -0
- data/lib/wurk/engine.rb +81 -0
- data/lib/wurk/fetcher/reaper.rb +264 -0
- data/lib/wurk/fetcher/reliable.rb +138 -0
- data/lib/wurk/fetcher.rb +11 -0
- data/lib/wurk/health.rb +193 -0
- data/lib/wurk/heartbeat.rb +211 -0
- data/lib/wurk/iterable_job.rb +292 -0
- data/lib/wurk/job/options.rb +70 -0
- data/lib/wurk/job.rb +33 -0
- data/lib/wurk/job_logger.rb +68 -0
- data/lib/wurk/job_record.rb +156 -0
- data/lib/wurk/job_retry.rb +320 -0
- data/lib/wurk/job_set.rb +212 -0
- data/lib/wurk/job_util.rb +162 -0
- data/lib/wurk/keys.rb +52 -0
- data/lib/wurk/launcher.rb +289 -0
- data/lib/wurk/leader.rb +221 -0
- data/lib/wurk/limiter/base.rb +138 -0
- data/lib/wurk/limiter/bucket.rb +80 -0
- data/lib/wurk/limiter/concurrent.rb +132 -0
- data/lib/wurk/limiter/leaky.rb +91 -0
- data/lib/wurk/limiter/points.rb +89 -0
- data/lib/wurk/limiter/server_middleware.rb +77 -0
- data/lib/wurk/limiter/unlimited.rb +48 -0
- data/lib/wurk/limiter/window.rb +80 -0
- data/lib/wurk/limiter.rb +255 -0
- data/lib/wurk/logger.rb +81 -0
- data/lib/wurk/lua/loader.rb +53 -0
- data/lib/wurk/lua.rb +187 -0
- data/lib/wurk/manager.rb +132 -0
- data/lib/wurk/metrics/history.rb +151 -0
- data/lib/wurk/metrics/query.rb +173 -0
- data/lib/wurk/metrics/rollup.rb +169 -0
- data/lib/wurk/metrics/statsd.rb +197 -0
- data/lib/wurk/metrics.rb +7 -0
- data/lib/wurk/middleware/chain.rb +128 -0
- data/lib/wurk/middleware/current_attributes.rb +87 -0
- data/lib/wurk/middleware/expiry.rb +50 -0
- data/lib/wurk/middleware/i18n.rb +63 -0
- data/lib/wurk/middleware/interrupt_handler.rb +45 -0
- data/lib/wurk/middleware/poison_pill.rb +149 -0
- data/lib/wurk/middleware.rb +34 -0
- data/lib/wurk/process_set.rb +243 -0
- data/lib/wurk/processor.rb +247 -0
- data/lib/wurk/queue.rb +108 -0
- data/lib/wurk/queues.rb +80 -0
- data/lib/wurk/rails.rb +9 -0
- data/lib/wurk/railtie.rb +28 -0
- data/lib/wurk/redis_pool.rb +79 -0
- data/lib/wurk/retry_set.rb +17 -0
- data/lib/wurk/scheduled.rb +189 -0
- data/lib/wurk/scheduled_set.rb +18 -0
- data/lib/wurk/sorted_entry.rb +95 -0
- data/lib/wurk/stats.rb +190 -0
- data/lib/wurk/swarm/child_boot.rb +105 -0
- data/lib/wurk/swarm.rb +260 -0
- data/lib/wurk/testing.rb +102 -0
- data/lib/wurk/topology.rb +74 -0
- data/lib/wurk/unique.rb +240 -0
- data/lib/wurk/version.rb +5 -0
- data/lib/wurk/web/config.rb +180 -0
- data/lib/wurk/web/enterprise.rb +138 -0
- data/lib/wurk/web/search.rb +139 -0
- data/lib/wurk/web.rb +25 -0
- data/lib/wurk/work_set.rb +116 -0
- data/lib/wurk/worker/setter.rb +93 -0
- data/lib/wurk/worker.rb +216 -0
- data/lib/wurk.rb +238 -0
- data/vendor/assets/dashboard/assets/index-8P3N_m1X.js +152 -0
- data/vendor/assets/dashboard/assets/index-Bqz4_SOQ.css +1 -0
- data/vendor/assets/dashboard/index.html +13 -0
- data/vendor/assets/dashboard/wurk-manifest.json +4 -0
- metadata +232 -0
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'base'
|
|
4
|
+
|
|
5
|
+
module Wurk
|
|
6
|
+
module Limiter
|
|
7
|
+
# Token-bucket with explicit `estimate:` per call. Refills at
|
|
8
|
+
# `refill_per_second` capped at `initial_points`. Failure mode is
|
|
9
|
+
# immediate (spec §1.4 — no sleep loop). The block is invoked with a
|
|
10
|
+
# `Handle` so user code may refund/over-charge via `handle.points_used`.
|
|
11
|
+
class Points < Base
|
|
12
|
+
class Handle
|
|
13
|
+
def initialize(limiter, estimate)
|
|
14
|
+
@limiter = limiter
|
|
15
|
+
@estimate = estimate
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Positive delta returns points to the bucket; negative records an
|
|
19
|
+
# under-estimate. Either way, clamped to [0, cap].
|
|
20
|
+
def points_used(actual)
|
|
21
|
+
delta = @estimate - actual
|
|
22
|
+
@limiter.send(:refund, delta)
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def type = :points
|
|
27
|
+
|
|
28
|
+
# Apply refill on read so the size matches what the *next* acquire
|
|
29
|
+
# would see. Stored balance only updates on acquire/refund; without
|
|
30
|
+
# this, a fully-refilled bucket reports stale low numbers.
|
|
31
|
+
def size
|
|
32
|
+
cap = @options[:initial].to_f
|
|
33
|
+
data = Wurk::Limiter.redis { |c| c.call('HMGET', state_key, 'points', 'last') }
|
|
34
|
+
stored = data[0]
|
|
35
|
+
return cap if stored.nil?
|
|
36
|
+
|
|
37
|
+
last = (data[1] || ::Time.now.to_f).to_f
|
|
38
|
+
elapsed = [::Time.now.to_f - last, 0.0].max
|
|
39
|
+
[cap, stored.to_f + (elapsed * @options[:refill].to_f)].min
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# used = points consumed (cap − available); limit = the cap; reset_at =
|
|
43
|
+
# when the bucket refills to full, or nil when already full (#16).
|
|
44
|
+
def status
|
|
45
|
+
cap = @options[:initial].to_f
|
|
46
|
+
available = size
|
|
47
|
+
used = cap - available
|
|
48
|
+
refill = @options[:refill].to_f
|
|
49
|
+
reset_at = available < cap && refill.positive? ? ::Time.now.to_f + ((cap - available) / refill) : nil
|
|
50
|
+
build_status(used: used, limit: cap, reset_at: reset_at)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def within_limit(estimate:, &block)
|
|
54
|
+
raise ArgumentError, 'block required' unless block
|
|
55
|
+
raise ArgumentError, 'estimate must be positive' if estimate <= 0
|
|
56
|
+
|
|
57
|
+
ok, _remaining = acquire(estimate)
|
|
58
|
+
raise OverLimit, self unless ok.to_i == 1
|
|
59
|
+
|
|
60
|
+
handle = Handle.new(self, estimate)
|
|
61
|
+
block.call(handle)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
protected
|
|
65
|
+
|
|
66
|
+
def state_keys
|
|
67
|
+
[state_key]
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
private
|
|
71
|
+
|
|
72
|
+
def state_key
|
|
73
|
+
"lmtr-p:#{@name}"
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def acquire(estimate)
|
|
77
|
+
lua(:limiter_points_acquire,
|
|
78
|
+
keys: [state_key],
|
|
79
|
+
argv: [@options[:initial], @options[:refill], estimate, ttl])
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def refund(delta)
|
|
83
|
+
lua(:limiter_points_refund,
|
|
84
|
+
keys: [state_key],
|
|
85
|
+
argv: [delta, @options[:initial], ttl]).to_f
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Wurk
|
|
4
|
+
module Limiter
|
|
5
|
+
# Catches OverLimit (and any class registered in `Limiter.config.errors`),
|
|
6
|
+
# bumps `job['overrated']`, and decides what to do next:
|
|
7
|
+
#
|
|
8
|
+
# * reschedule disabled (`reschedule: 0`) → re-raise so the normal
|
|
9
|
+
# retry/dead pipeline handles it (spec §1.2/§1.4 behaviour).
|
|
10
|
+
# * still under the cap → reschedule onto the same queue at
|
|
11
|
+
# `Time.now + backoff` via `Client.push`.
|
|
12
|
+
# * cap reached (`overrated >= reschedule`, default 20) → **poison
|
|
13
|
+
# brake** (#16): a job that's still rate-limited after N reschedules
|
|
14
|
+
# is saturating the limiter, so instead of dumping it into another
|
|
15
|
+
# 25× retry loop we route it straight to the dead set tagged
|
|
16
|
+
# `rate_limited`, bumping `jobs.rate_limited` and firing death
|
|
17
|
+
# handlers. Bounded: termination at exactly `reschedule` attempts.
|
|
18
|
+
class ServerMiddleware
|
|
19
|
+
include Wurk::Middleware::ServerMiddleware
|
|
20
|
+
|
|
21
|
+
DEAD_REASON = 'rate_limited'
|
|
22
|
+
|
|
23
|
+
def call(_worker, job, _queue)
|
|
24
|
+
yield
|
|
25
|
+
rescue StandardError => e
|
|
26
|
+
raise unless over_limit?(e)
|
|
27
|
+
|
|
28
|
+
handle_over_limit(job, e)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def over_limit?(exc)
|
|
34
|
+
Wurk::Limiter.config.errors.any? { |k| exc.is_a?(k) }
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def handle_over_limit(job, exc)
|
|
38
|
+
job['overrated'] = job.fetch('overrated', 0).to_i + 1
|
|
39
|
+
limiter = exc.respond_to?(:limiter) ? exc.limiter : nil
|
|
40
|
+
exc.job = job if exc.respond_to?(:job=)
|
|
41
|
+
cap = reschedule_cap(limiter)
|
|
42
|
+
|
|
43
|
+
raise exc if cap.zero? # rescheduling disabled → normal retry/dead pipeline
|
|
44
|
+
return route_to_dead(job, exc) if job['overrated'] >= cap
|
|
45
|
+
|
|
46
|
+
reschedule(job, exc, limiter)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# nil (concurrent/leaky/points never set it) → the default 20.
|
|
50
|
+
def reschedule_cap(limiter)
|
|
51
|
+
return DEFAULT_RESCHEDULE unless limiter
|
|
52
|
+
|
|
53
|
+
cap = limiter.options[:reschedule]
|
|
54
|
+
cap.nil? ? DEFAULT_RESCHEDULE : cap
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def reschedule(job, exc, limiter)
|
|
58
|
+
backoff_proc = (limiter && limiter.options[:backoff]) || Wurk::Limiter.config.backoff
|
|
59
|
+
delay = backoff_proc.call(limiter, job, exc).to_f
|
|
60
|
+
Wurk::Client.new.push(job.merge('at' => ::Time.now.to_f + delay))
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Poison brake: stamp a clear reason, drop the job in the dead set, and
|
|
64
|
+
# ACK by returning normally (no re-raise) so it isn't also retried.
|
|
65
|
+
def route_to_dead(job, exc)
|
|
66
|
+
record = job.merge(
|
|
67
|
+
'error_class' => exc.class.name,
|
|
68
|
+
'error_message' => "#{DEAD_REASON}: #{exc.message} (overrated=#{job['overrated']})",
|
|
69
|
+
'failed_at' => ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
|
|
70
|
+
)
|
|
71
|
+
Wurk::Metrics::Statsd.increment('jobs.rate_limited', tags: ["worker:#{job['class']}"])
|
|
72
|
+
Wurk::DeadSet.new.kill(Wurk.dump_json(record), ex: exc)
|
|
73
|
+
nil
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'points'
|
|
4
|
+
|
|
5
|
+
module Wurk
|
|
6
|
+
module Limiter
|
|
7
|
+
# No-op for the tests + bypass scenarios documented in §1.8. The
|
|
8
|
+
# within_limit block runs unconditionally and the introspection
|
|
9
|
+
# methods all return zeros so dashboards render predictably.
|
|
10
|
+
class Unlimited
|
|
11
|
+
def name = 'unlimited'
|
|
12
|
+
def type = :unlimited
|
|
13
|
+
def options = {}
|
|
14
|
+
def size = 0
|
|
15
|
+
def fingerprint = Digest::SHA256.hexdigest('unlimited')
|
|
16
|
+
def reset = nil
|
|
17
|
+
def delete = nil
|
|
18
|
+
|
|
19
|
+
# Uniform status shape (#16): no limit, always available.
|
|
20
|
+
def status
|
|
21
|
+
{ used: 0, limit: nil, reset_at: nil, available?: true }
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Accept all the kwargs the other limiters take so a worker can swap
|
|
25
|
+
# `limiter = Sidekiq::Limiter.unlimited` in tests without touching
|
|
26
|
+
# call sites. `points`-style callers pass an `estimate:` and expect a
|
|
27
|
+
# `|handle|` block param — we yield a zero-cost handle just in case.
|
|
28
|
+
def within_limit(**_kwargs, &block)
|
|
29
|
+
raise ArgumentError, 'block required' unless block
|
|
30
|
+
|
|
31
|
+
if block.arity.zero?
|
|
32
|
+
block.call
|
|
33
|
+
else
|
|
34
|
+
block.call(Points::Handle.new(self, 0))
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
# Points::Handle#points_used calls `refund` on the limiter; without this
|
|
41
|
+
# no-op an Unlimited swap-in would raise NoMethodError the moment user
|
|
42
|
+
# code records actual usage. Drop-in contract trumps purity.
|
|
43
|
+
def refund(_delta)
|
|
44
|
+
0.0
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'base'
|
|
4
|
+
|
|
5
|
+
module Wurk
|
|
6
|
+
module Limiter
|
|
7
|
+
# Sliding window via ZSET of timestamps. Accepts symbolic units or a
|
|
8
|
+
# raw Integer (in seconds). Used-units expand to N ZADDs in the Lua
|
|
9
|
+
# script so multi-charge calls remain atomic.
|
|
10
|
+
class Window < Base
|
|
11
|
+
WAIT_SLEEP = 0.5
|
|
12
|
+
|
|
13
|
+
def type = :window
|
|
14
|
+
|
|
15
|
+
def initialize(name, **options)
|
|
16
|
+
# Symbol or raw Integer (spec §1.2: window accepts both).
|
|
17
|
+
Limiter.interval_seconds(options[:interval], allow_integer: true)
|
|
18
|
+
super
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def size
|
|
22
|
+
cutoff = ::Time.now.to_f - interval_seconds
|
|
23
|
+
Wurk::Limiter.redis do |c|
|
|
24
|
+
c.call('ZREMRANGEBYSCORE', state_key, '-inf', "(#{cutoff}")
|
|
25
|
+
c.call('ZCARD', state_key).to_i
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# used = entries still inside the window; limit = count; reset_at =
|
|
30
|
+
# when the oldest entry slides out (freeing a slot), or nil when
|
|
31
|
+
# idle (#16).
|
|
32
|
+
def status
|
|
33
|
+
build_status(used: size, limit: @options[:count], reset_at: oldest_expiry)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def within_limit(used: 1, &block)
|
|
37
|
+
raise ArgumentError, 'block required' unless block
|
|
38
|
+
|
|
39
|
+
deadline = ::Time.now.to_f + @options[:wait_timeout]
|
|
40
|
+
loop do
|
|
41
|
+
ok, _current, _oldest = acquire(used)
|
|
42
|
+
return block.call if ok.to_i == 1
|
|
43
|
+
|
|
44
|
+
remaining = deadline - ::Time.now.to_f
|
|
45
|
+
raise OverLimit, self if remaining <= 0
|
|
46
|
+
|
|
47
|
+
sleep [remaining, WAIT_SLEEP].min
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
protected
|
|
52
|
+
|
|
53
|
+
def state_keys
|
|
54
|
+
[state_key]
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
def state_key
|
|
60
|
+
"lmtr-w:#{@name}"
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Oldest timestamp + interval = the moment it leaves the window.
|
|
64
|
+
def oldest_expiry
|
|
65
|
+
row = Wurk::Limiter.redis { |c| c.call('ZRANGE', state_key, 0, 0, 'WITHSCORES') }
|
|
66
|
+
row && !row.empty? ? row[1].to_f + interval_seconds : nil
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def interval_seconds
|
|
70
|
+
@interval_seconds ||= Limiter.interval_seconds(@options[:interval], allow_integer: true)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def acquire(used)
|
|
74
|
+
lua(:limiter_window_acquire,
|
|
75
|
+
keys: [state_key],
|
|
76
|
+
argv: [@options[:count], interval_seconds, used, ttl, random_id])
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
data/lib/wurk/limiter.rb
ADDED
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'digest'
|
|
5
|
+
require 'securerandom'
|
|
6
|
+
require_relative 'lua'
|
|
7
|
+
|
|
8
|
+
module Wurk
|
|
9
|
+
# Sidekiq Enterprise rate limiters: concurrent, bucket, window, leaky,
|
|
10
|
+
# points, unlimited. Lua-backed; all timing inside Lua is from TIME so
|
|
11
|
+
# clock skew across hosts doesn't matter inside one Redis. Spec:
|
|
12
|
+
# docs/target/sidekiq-ent.md §1.
|
|
13
|
+
#
|
|
14
|
+
# Layout (one file per type under `lib/wurk/limiter/`):
|
|
15
|
+
# * `Limiter::Base` owns the metadata write (lmtr:{name}) + the global
|
|
16
|
+
# `lmtr-list` registration so the Web UI can list every limiter, and
|
|
17
|
+
# the uniform `status` shape.
|
|
18
|
+
# * Per-type subclasses (Concurrent / Bucket / Window / Leaky / Points)
|
|
19
|
+
# own their acquire/wait loop. Each delegates the atomic step to a
|
|
20
|
+
# Lua script in `lib/wurk/lua/limiter_*.lua`.
|
|
21
|
+
# * `Unlimited` is a no-op stub for tests and the `unlimited(*)`
|
|
22
|
+
# constructor — same `within_limit` surface, never raises.
|
|
23
|
+
# * `ServerMiddleware` catches OverLimit, reschedules, and applies the
|
|
24
|
+
# poison brake.
|
|
25
|
+
#
|
|
26
|
+
# Wire-compat: every key uses the `lmtr-...:` prefix family from §1.7
|
|
27
|
+
# and the limiter is added to the shared `lmtr-list` SET.
|
|
28
|
+
module Limiter
|
|
29
|
+
DEFAULT_TTL = 90 * 24 * 3600
|
|
30
|
+
DEFAULT_WAIT_TIMEOUT = 5
|
|
31
|
+
DEFAULT_LOCK_TIMEOUT = 30
|
|
32
|
+
DEFAULT_RESCHEDULE = 20
|
|
33
|
+
DEFAULT_BACKOFF = lambda do |_limiter, job, _exc|
|
|
34
|
+
overrated = job.is_a?(Hash) ? job.fetch('overrated', 0).to_i : 0
|
|
35
|
+
(300 * overrated) + rand(300) + 1
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
NAME_PATTERN = /\A[\w\-:.\#@]+\z/
|
|
39
|
+
|
|
40
|
+
LIST_KEY = 'lmtr-list'
|
|
41
|
+
|
|
42
|
+
# Server middleware catches OverLimit, increments `job['overrated']`, and
|
|
43
|
+
# reschedules onto the same queue with `Time.now + backoff`. The
|
|
44
|
+
# `#limiter` attr lets the middleware reach the per-limiter backoff
|
|
45
|
+
# proc + reschedule cap. `#job` is set by the middleware just before the
|
|
46
|
+
# re-raise so error_handlers can see which job was in flight.
|
|
47
|
+
class OverLimit < StandardError
|
|
48
|
+
attr_reader :limiter
|
|
49
|
+
attr_accessor :job
|
|
50
|
+
|
|
51
|
+
def initialize(limiter, job = nil, msg = nil)
|
|
52
|
+
@limiter = limiter
|
|
53
|
+
@job = job
|
|
54
|
+
super(msg || "limit '#{limiter.name}' (#{limiter.type}) reached")
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Global config. Sidekiq Enterprise documents three knobs (§1.6):
|
|
59
|
+
# `backoff` (Proc), `redis` (a Hash that builds a dedicated pool), and
|
|
60
|
+
# `errors` (Array of exception classes the middleware also treats as
|
|
61
|
+
# OverLimit). All three are mutable and re-read on every push/perform.
|
|
62
|
+
class Config
|
|
63
|
+
attr_accessor :backoff, :errors
|
|
64
|
+
attr_reader :redis
|
|
65
|
+
|
|
66
|
+
def initialize
|
|
67
|
+
@backoff = DEFAULT_BACKOFF
|
|
68
|
+
@errors = [OverLimit]
|
|
69
|
+
@redis = nil
|
|
70
|
+
@redis_pool = nil
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Accept either a Hash (the documented Sidekiq Ent shape — `{ size:,
|
|
74
|
+
# url: }`) or an already-built `RedisPool`. The first redis read
|
|
75
|
+
# lazily materializes the pool; per-fork safety is the caller's
|
|
76
|
+
# responsibility (same contract as Wurk.redis_pool).
|
|
77
|
+
def redis=(value)
|
|
78
|
+
@redis = value
|
|
79
|
+
@redis_pool = nil
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def pool
|
|
83
|
+
return nil if @redis.nil?
|
|
84
|
+
|
|
85
|
+
@pool ||= case @redis
|
|
86
|
+
when Wurk::RedisPool then @redis
|
|
87
|
+
when Hash
|
|
88
|
+
Wurk::RedisPool.new(
|
|
89
|
+
size: @redis[:size] || 10,
|
|
90
|
+
url: @redis[:url] || Wurk::RedisPool::DEFAULT_URL,
|
|
91
|
+
timeout: @redis[:timeout] || Wurk::RedisPool::DEFAULT_TIMEOUT,
|
|
92
|
+
name: 'limiter'
|
|
93
|
+
)
|
|
94
|
+
else
|
|
95
|
+
raise ArgumentError, "Limiter.config.redis must be Hash or RedisPool, got #{@redis.class}"
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# `:second :minute :hour :day` symbols → seconds. Window also accepts
|
|
101
|
+
# a raw Integer; bucket does not (boundary semantics require a unit).
|
|
102
|
+
INTERVAL_UNITS = {
|
|
103
|
+
second: 1,
|
|
104
|
+
minute: 60,
|
|
105
|
+
hour: 3600,
|
|
106
|
+
day: 86_400
|
|
107
|
+
}.freeze
|
|
108
|
+
|
|
109
|
+
# Type string (as stored in the `lmtr:{name}` meta hash) → subclass.
|
|
110
|
+
# Drives `build` for dashboard introspection.
|
|
111
|
+
TYPE_CLASSES = {
|
|
112
|
+
'concurrent' => 'Concurrent',
|
|
113
|
+
'bucket' => 'Bucket',
|
|
114
|
+
'window' => 'Window',
|
|
115
|
+
'leaky' => 'Leaky',
|
|
116
|
+
'points' => 'Points'
|
|
117
|
+
}.freeze
|
|
118
|
+
|
|
119
|
+
class << self
|
|
120
|
+
def configure
|
|
121
|
+
yield config
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def config
|
|
125
|
+
@config ||= Config.new
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# Test helper: blow away config + cached pool so a test that mutates
|
|
129
|
+
# `config.backoff` doesn't leak into the next one. Not part of the
|
|
130
|
+
# public Sidekiq surface.
|
|
131
|
+
def reset_config!
|
|
132
|
+
@config = nil
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# Redis access: caller-supplied pool (Limiter.configure.redis = …) wins,
|
|
136
|
+
# else fall back to the default Wurk pool. This is the same hierarchy
|
|
137
|
+
# Sidekiq Ent documents — dedicated rate-limiter pool is opt-in.
|
|
138
|
+
def redis(&)
|
|
139
|
+
pool = config.pool || Wurk.redis_pool
|
|
140
|
+
pool.with(&)
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def concurrent(name, limit, wait_timeout: DEFAULT_WAIT_TIMEOUT, lock_timeout: DEFAULT_LOCK_TIMEOUT,
|
|
144
|
+
policy: :raise, backoff: nil, ttl: DEFAULT_TTL)
|
|
145
|
+
Concurrent.new(name,
|
|
146
|
+
limit: limit,
|
|
147
|
+
wait_timeout: wait_timeout,
|
|
148
|
+
lock_timeout: lock_timeout,
|
|
149
|
+
policy: policy,
|
|
150
|
+
backoff: backoff,
|
|
151
|
+
ttl: ttl)
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def bucket(name, count, interval, wait_timeout: DEFAULT_WAIT_TIMEOUT, backoff: nil,
|
|
155
|
+
ttl: DEFAULT_TTL, reschedule: DEFAULT_RESCHEDULE)
|
|
156
|
+
Bucket.new(name,
|
|
157
|
+
count: count,
|
|
158
|
+
interval: interval,
|
|
159
|
+
wait_timeout: wait_timeout,
|
|
160
|
+
backoff: backoff,
|
|
161
|
+
ttl: ttl,
|
|
162
|
+
reschedule: reschedule)
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def window(name, count, interval, wait_timeout: DEFAULT_WAIT_TIMEOUT, backoff: nil,
|
|
166
|
+
ttl: DEFAULT_TTL, reschedule: DEFAULT_RESCHEDULE)
|
|
167
|
+
Window.new(name,
|
|
168
|
+
count: count,
|
|
169
|
+
interval: interval,
|
|
170
|
+
wait_timeout: wait_timeout,
|
|
171
|
+
backoff: backoff,
|
|
172
|
+
ttl: ttl,
|
|
173
|
+
reschedule: reschedule)
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def leaky(name, bucket_size, drain, wait_timeout: DEFAULT_WAIT_TIMEOUT, backoff: nil, ttl: DEFAULT_TTL)
|
|
177
|
+
Leaky.new(name,
|
|
178
|
+
bucket_size: bucket_size,
|
|
179
|
+
drain: drain,
|
|
180
|
+
wait_timeout: wait_timeout,
|
|
181
|
+
backoff: backoff,
|
|
182
|
+
ttl: ttl)
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
def points(name, initial_points, refill_per_second, backoff: nil, ttl: DEFAULT_TTL)
|
|
186
|
+
Points.new(name,
|
|
187
|
+
initial: initial_points,
|
|
188
|
+
refill: refill_per_second,
|
|
189
|
+
backoff: backoff,
|
|
190
|
+
ttl: ttl)
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def unlimited(*_args, **_opts)
|
|
194
|
+
Unlimited.new
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# Reconstruct a limiter from its persisted metadata for read-only
|
|
198
|
+
# introspection (the dashboard `status` column). `register: false`
|
|
199
|
+
# keeps the GET side-effect-free. Returns nil for an unknown type.
|
|
200
|
+
def build(name, type, options, register: false)
|
|
201
|
+
return Unlimited.new if type.to_s == 'unlimited'
|
|
202
|
+
|
|
203
|
+
klass_name = TYPE_CLASSES[type.to_s]
|
|
204
|
+
return nil unless klass_name
|
|
205
|
+
|
|
206
|
+
const_get(klass_name).new(name, register: register, **coerce_build_options(options))
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def interval_seconds(interval, allow_integer:)
|
|
210
|
+
interval = interval.to_sym if interval.is_a?(String) && INTERVAL_UNITS.key?(interval.to_sym)
|
|
211
|
+
case interval
|
|
212
|
+
when Symbol
|
|
213
|
+
INTERVAL_UNITS.fetch(interval) do
|
|
214
|
+
raise ArgumentError, "interval must be one of #{INTERVAL_UNITS.keys.inspect} (got #{interval.inspect})"
|
|
215
|
+
end
|
|
216
|
+
when Integer
|
|
217
|
+
unless allow_integer
|
|
218
|
+
raise ArgumentError, "interval must be a Symbol (got Integer); use #{INTERVAL_UNITS.keys.inspect}"
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
interval
|
|
222
|
+
else
|
|
223
|
+
raise ArgumentError, "interval must be Symbol or Integer (got #{interval.class})"
|
|
224
|
+
end
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
private
|
|
228
|
+
|
|
229
|
+
# Stored options round-trip through JSON, so symbol keys arrive as
|
|
230
|
+
# strings and unit symbols (`:minute`) as `"minute"`. Restore both so a
|
|
231
|
+
# rebuilt limiter validates the same as a freshly-constructed one.
|
|
232
|
+
def coerce_build_options(options)
|
|
233
|
+
opts = options.transform_keys(&:to_sym)
|
|
234
|
+
%i[interval drain].each do |k|
|
|
235
|
+
v = opts[k]
|
|
236
|
+
opts[k] = v.to_sym if v.is_a?(String) && INTERVAL_UNITS.key?(v.to_sym)
|
|
237
|
+
end
|
|
238
|
+
opts
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
end
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
require_relative 'limiter/base'
|
|
245
|
+
require_relative 'limiter/concurrent'
|
|
246
|
+
require_relative 'limiter/bucket'
|
|
247
|
+
require_relative 'limiter/window'
|
|
248
|
+
require_relative 'limiter/leaky'
|
|
249
|
+
require_relative 'limiter/points'
|
|
250
|
+
require_relative 'limiter/unlimited'
|
|
251
|
+
require_relative 'middleware'
|
|
252
|
+
require_relative 'limiter/server_middleware'
|
|
253
|
+
# Server-middleware registration happens in wurk.rb after the Wurk
|
|
254
|
+
# `class << self` block defines `Wurk.configuration` — limiter.rb
|
|
255
|
+
# loads earlier than that, so trying to call it here would NoMethodError.
|
data/lib/wurk/logger.rb
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'logger'
|
|
4
|
+
require 'time'
|
|
5
|
+
require 'json'
|
|
6
|
+
|
|
7
|
+
module Wurk
|
|
8
|
+
# Wurk's stdlib-compatible ::Logger subclass. The point of subclassing
|
|
9
|
+
# (rather than configuring a vanilla ::Logger) is to ship default
|
|
10
|
+
# formatters that read the thread-local Wurk::Context so every line
|
|
11
|
+
# carries jid/bid/tags/elapsed without callers threading the hash through.
|
|
12
|
+
#
|
|
13
|
+
# Formatter selection at boot:
|
|
14
|
+
# - ENV["DYNO"] set → Formatters::WithoutTimestamp (Heroku already prefixes)
|
|
15
|
+
# - otherwise → Formatters::Pretty
|
|
16
|
+
# Switch to Formatters::JSON manually for log aggregators that want NDJSON.
|
|
17
|
+
#
|
|
18
|
+
# Spec: docs/target/sidekiq-free.md §29.
|
|
19
|
+
class Logger < ::Logger
|
|
20
|
+
module Formatters
|
|
21
|
+
class Base < ::Logger::Formatter
|
|
22
|
+
SEVERITY_COLORS = {
|
|
23
|
+
'DEBUG' => "\e[1;32mDEBUG\e[0m",
|
|
24
|
+
'INFO' => "\e[1;34mINFO \e[0m",
|
|
25
|
+
'WARN' => "\e[1;33mWARN \e[0m",
|
|
26
|
+
'ERROR' => "\e[1;31mERROR\e[0m",
|
|
27
|
+
'FATAL' => "\e[1;35mFATAL\e[0m"
|
|
28
|
+
}.freeze
|
|
29
|
+
|
|
30
|
+
def tid
|
|
31
|
+
Thread.current[:wurk_tid] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def format_context(ctxt = Wurk::Context.current)
|
|
35
|
+
return '' if ctxt.empty?
|
|
36
|
+
|
|
37
|
+
" #{ctxt.map { |k, v| v.is_a?(Array) ? "#{k}=#{v.join(',')}" : "#{k}=#{v}" }.join(' ')}"
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
class Pretty < Base
|
|
42
|
+
def call(severity, time, _program_name, message)
|
|
43
|
+
"#{SEVERITY_COLORS[severity]} #{time.utc.iso8601(3)} pid=#{::Process.pid} " \
|
|
44
|
+
"tid=#{tid}#{format_context}: #{message}\n"
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
class WithoutTimestamp < Pretty
|
|
49
|
+
def call(severity, _time, _program_name, message)
|
|
50
|
+
"#{SEVERITY_COLORS[severity]} pid=#{::Process.pid} tid=#{tid}#{format_context}: #{message}\n"
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
class JSON < Base
|
|
55
|
+
def call(severity, time, _program_name, message)
|
|
56
|
+
hash = {
|
|
57
|
+
ts: time.utc.iso8601(3),
|
|
58
|
+
pid: ::Process.pid,
|
|
59
|
+
tid: tid,
|
|
60
|
+
lvl: severity,
|
|
61
|
+
msg: message
|
|
62
|
+
}
|
|
63
|
+
ctx = Wurk::Context.current
|
|
64
|
+
hash[:ctx] = ctx unless ctx.empty?
|
|
65
|
+
"#{::JSON.generate(hash)}\n"
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def initialize(*, **)
|
|
71
|
+
super
|
|
72
|
+
self.formatter = default_formatter
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
private
|
|
76
|
+
|
|
77
|
+
def default_formatter
|
|
78
|
+
ENV['DYNO'] ? Formatters::WithoutTimestamp.new : Formatters::Pretty.new
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Wurk
|
|
4
|
+
module Lua
|
|
5
|
+
# EVALSHA wrapper with `NOSCRIPT` recovery. The SHA1 of each script
|
|
6
|
+
# source is precomputed in `Wurk::Lua::SHAS`, so the first call to
|
|
7
|
+
# `eval_cached` after a fork has a fast path: a single EVALSHA, no
|
|
8
|
+
# per-pool bookkeeping. If the script cache was flushed (manual
|
|
9
|
+
# `SCRIPT FLUSH`, replica failover, OOM eviction), `EVALSHA` returns
|
|
10
|
+
# `NOSCRIPT` — we then `SCRIPT LOAD` once and retry exactly once.
|
|
11
|
+
#
|
|
12
|
+
# Spec: docs/target/sidekiq-free.md §20 (Lua script caching).
|
|
13
|
+
class Loader
|
|
14
|
+
NOSCRIPT_PREFIX = 'NOSCRIPT'
|
|
15
|
+
|
|
16
|
+
class << self
|
|
17
|
+
# Eagerly upload every registered script to the given connection.
|
|
18
|
+
# Idempotent on the Redis side: `SCRIPT LOAD` of the same source
|
|
19
|
+
# returns the same SHA regardless of how often it's called.
|
|
20
|
+
# Manager calls this once per child after the post-fork reconnect.
|
|
21
|
+
def script_load_all(redis)
|
|
22
|
+
SCRIPTS.each_value { |src| redis.call('SCRIPT', 'LOAD', src) }
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# @param redis [RedisClient] a single connection (not a pool)
|
|
26
|
+
# @param name [Symbol] key into Wurk::Lua::SCRIPTS
|
|
27
|
+
# @param keys [Array<String>] EVALSHA KEYS
|
|
28
|
+
# @param argv [Array] EVALSHA ARGV (coerced to strings by Redis)
|
|
29
|
+
# @return Lua script return value
|
|
30
|
+
def eval_cached(redis, name, keys:, argv:)
|
|
31
|
+
src = SCRIPTS.fetch(name) { raise ArgumentError, "unknown Lua script: #{name.inspect}" }
|
|
32
|
+
sha = SHAS.fetch(name)
|
|
33
|
+
evalsha(redis, sha, keys, argv)
|
|
34
|
+
rescue RedisClient::CommandError => e
|
|
35
|
+
raise unless noscript?(e)
|
|
36
|
+
|
|
37
|
+
redis.call('SCRIPT', 'LOAD', src)
|
|
38
|
+
evalsha(redis, sha, keys, argv)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
def evalsha(redis, sha, keys, argv)
|
|
44
|
+
redis.call('EVALSHA', sha, keys.size, *keys, *argv)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def noscript?(err)
|
|
48
|
+
err.message.to_s.start_with?(NOSCRIPT_PREFIX)
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|