wurk 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +43 -0
  3. data/CONTRIBUTING.md +73 -0
  4. data/LICENSE +21 -0
  5. data/README.md +137 -0
  6. data/SECURITY.md +39 -0
  7. data/app/controllers/wurk/api/pagination.rb +67 -0
  8. data/app/controllers/wurk/api/serializers.rb +131 -0
  9. data/app/controllers/wurk/api_controller.rb +248 -0
  10. data/app/controllers/wurk/application_controller.rb +7 -0
  11. data/app/controllers/wurk/dashboard_controller.rb +48 -0
  12. data/config/locales/en.yml +15 -0
  13. data/config/routes.rb +34 -0
  14. data/exe/wurk +22 -0
  15. data/lib/active_job/queue_adapters/wurk_adapter.rb +96 -0
  16. data/lib/generators/wurk/install/install_generator.rb +22 -0
  17. data/lib/generators/wurk/install/templates/wurk.rb +16 -0
  18. data/lib/wurk/active_job/wrapper.rb +32 -0
  19. data/lib/wurk/api/fast.rb +78 -0
  20. data/lib/wurk/batch/buffer.rb +26 -0
  21. data/lib/wurk/batch/callback_job.rb +37 -0
  22. data/lib/wurk/batch/callbacks.rb +176 -0
  23. data/lib/wurk/batch/client_middleware.rb +27 -0
  24. data/lib/wurk/batch/death_handler.rb +39 -0
  25. data/lib/wurk/batch/empty.rb +21 -0
  26. data/lib/wurk/batch/server_middleware.rb +62 -0
  27. data/lib/wurk/batch/status.rb +140 -0
  28. data/lib/wurk/batch.rb +351 -0
  29. data/lib/wurk/batch_set.rb +67 -0
  30. data/lib/wurk/capsule.rb +176 -0
  31. data/lib/wurk/cli.rb +349 -0
  32. data/lib/wurk/client/buffered.rb +372 -0
  33. data/lib/wurk/client.rb +330 -0
  34. data/lib/wurk/compat.rb +136 -0
  35. data/lib/wurk/component.rb +136 -0
  36. data/lib/wurk/configuration.rb +373 -0
  37. data/lib/wurk/context.rb +35 -0
  38. data/lib/wurk/cron.rb +636 -0
  39. data/lib/wurk/dashboard_manifest.rb +39 -0
  40. data/lib/wurk/dead_set.rb +78 -0
  41. data/lib/wurk/deploy.rb +91 -0
  42. data/lib/wurk/embedded.rb +94 -0
  43. data/lib/wurk/encryption.rb +276 -0
  44. data/lib/wurk/engine.rb +81 -0
  45. data/lib/wurk/fetcher/reaper.rb +264 -0
  46. data/lib/wurk/fetcher/reliable.rb +138 -0
  47. data/lib/wurk/fetcher.rb +11 -0
  48. data/lib/wurk/health.rb +193 -0
  49. data/lib/wurk/heartbeat.rb +211 -0
  50. data/lib/wurk/iterable_job.rb +292 -0
  51. data/lib/wurk/job/options.rb +70 -0
  52. data/lib/wurk/job.rb +33 -0
  53. data/lib/wurk/job_logger.rb +68 -0
  54. data/lib/wurk/job_record.rb +156 -0
  55. data/lib/wurk/job_retry.rb +320 -0
  56. data/lib/wurk/job_set.rb +212 -0
  57. data/lib/wurk/job_util.rb +162 -0
  58. data/lib/wurk/keys.rb +52 -0
  59. data/lib/wurk/launcher.rb +289 -0
  60. data/lib/wurk/leader.rb +221 -0
  61. data/lib/wurk/limiter/base.rb +138 -0
  62. data/lib/wurk/limiter/bucket.rb +80 -0
  63. data/lib/wurk/limiter/concurrent.rb +132 -0
  64. data/lib/wurk/limiter/leaky.rb +91 -0
  65. data/lib/wurk/limiter/points.rb +89 -0
  66. data/lib/wurk/limiter/server_middleware.rb +77 -0
  67. data/lib/wurk/limiter/unlimited.rb +48 -0
  68. data/lib/wurk/limiter/window.rb +80 -0
  69. data/lib/wurk/limiter.rb +255 -0
  70. data/lib/wurk/logger.rb +81 -0
  71. data/lib/wurk/lua/loader.rb +53 -0
  72. data/lib/wurk/lua.rb +187 -0
  73. data/lib/wurk/manager.rb +132 -0
  74. data/lib/wurk/metrics/history.rb +151 -0
  75. data/lib/wurk/metrics/query.rb +173 -0
  76. data/lib/wurk/metrics/rollup.rb +169 -0
  77. data/lib/wurk/metrics/statsd.rb +197 -0
  78. data/lib/wurk/metrics.rb +7 -0
  79. data/lib/wurk/middleware/chain.rb +128 -0
  80. data/lib/wurk/middleware/current_attributes.rb +87 -0
  81. data/lib/wurk/middleware/expiry.rb +50 -0
  82. data/lib/wurk/middleware/i18n.rb +63 -0
  83. data/lib/wurk/middleware/interrupt_handler.rb +45 -0
  84. data/lib/wurk/middleware/poison_pill.rb +149 -0
  85. data/lib/wurk/middleware.rb +34 -0
  86. data/lib/wurk/process_set.rb +243 -0
  87. data/lib/wurk/processor.rb +247 -0
  88. data/lib/wurk/queue.rb +108 -0
  89. data/lib/wurk/queues.rb +80 -0
  90. data/lib/wurk/rails.rb +9 -0
  91. data/lib/wurk/railtie.rb +28 -0
  92. data/lib/wurk/redis_pool.rb +79 -0
  93. data/lib/wurk/retry_set.rb +17 -0
  94. data/lib/wurk/scheduled.rb +189 -0
  95. data/lib/wurk/scheduled_set.rb +18 -0
  96. data/lib/wurk/sorted_entry.rb +95 -0
  97. data/lib/wurk/stats.rb +190 -0
  98. data/lib/wurk/swarm/child_boot.rb +105 -0
  99. data/lib/wurk/swarm.rb +260 -0
  100. data/lib/wurk/testing.rb +102 -0
  101. data/lib/wurk/topology.rb +74 -0
  102. data/lib/wurk/unique.rb +240 -0
  103. data/lib/wurk/version.rb +5 -0
  104. data/lib/wurk/web/config.rb +180 -0
  105. data/lib/wurk/web/enterprise.rb +138 -0
  106. data/lib/wurk/web/search.rb +139 -0
  107. data/lib/wurk/web.rb +25 -0
  108. data/lib/wurk/work_set.rb +116 -0
  109. data/lib/wurk/worker/setter.rb +93 -0
  110. data/lib/wurk/worker.rb +216 -0
  111. data/lib/wurk.rb +238 -0
  112. data/vendor/assets/dashboard/assets/index-8P3N_m1X.js +152 -0
  113. data/vendor/assets/dashboard/assets/index-Bqz4_SOQ.css +1 -0
  114. data/vendor/assets/dashboard/index.html +13 -0
  115. data/vendor/assets/dashboard/wurk-manifest.json +4 -0
  116. metadata +232 -0
@@ -0,0 +1,89 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'base'
4
+
5
+ module Wurk
6
+ module Limiter
7
+ # Token-bucket with explicit `estimate:` per call. Refills at
8
+ # `refill_per_second` capped at `initial_points`. Failure mode is
9
+ # immediate (spec §1.4 — no sleep loop). The block is invoked with a
10
+ # `Handle` so user code may refund/over-charge via `handle.points_used`.
11
+ class Points < Base
12
+ class Handle
13
+ def initialize(limiter, estimate)
14
+ @limiter = limiter
15
+ @estimate = estimate
16
+ end
17
+
18
+ # Positive delta returns points to the bucket; negative records an
19
+ # under-estimate. Either way, clamped to [0, cap].
20
+ def points_used(actual)
21
+ delta = @estimate - actual
22
+ @limiter.send(:refund, delta)
23
+ end
24
+ end
25
+
26
+ def type = :points
27
+
28
+ # Apply refill on read so the size matches what the *next* acquire
29
+ # would see. Stored balance only updates on acquire/refund; without
30
+ # this, a fully-refilled bucket reports stale low numbers.
31
+ def size
32
+ cap = @options[:initial].to_f
33
+ data = Wurk::Limiter.redis { |c| c.call('HMGET', state_key, 'points', 'last') }
34
+ stored = data[0]
35
+ return cap if stored.nil?
36
+
37
+ last = (data[1] || ::Time.now.to_f).to_f
38
+ elapsed = [::Time.now.to_f - last, 0.0].max
39
+ [cap, stored.to_f + (elapsed * @options[:refill].to_f)].min
40
+ end
41
+
42
+ # used = points consumed (cap − available); limit = the cap; reset_at =
43
+ # when the bucket refills to full, or nil when already full (#16).
44
+ def status
45
+ cap = @options[:initial].to_f
46
+ available = size
47
+ used = cap - available
48
+ refill = @options[:refill].to_f
49
+ reset_at = available < cap && refill.positive? ? ::Time.now.to_f + ((cap - available) / refill) : nil
50
+ build_status(used: used, limit: cap, reset_at: reset_at)
51
+ end
52
+
53
+ def within_limit(estimate:, &block)
54
+ raise ArgumentError, 'block required' unless block
55
+ raise ArgumentError, 'estimate must be positive' if estimate <= 0
56
+
57
+ ok, _remaining = acquire(estimate)
58
+ raise OverLimit, self unless ok.to_i == 1
59
+
60
+ handle = Handle.new(self, estimate)
61
+ block.call(handle)
62
+ end
63
+
64
+ protected
65
+
66
+ def state_keys
67
+ [state_key]
68
+ end
69
+
70
+ private
71
+
72
+ def state_key
73
+ "lmtr-p:#{@name}"
74
+ end
75
+
76
+ def acquire(estimate)
77
+ lua(:limiter_points_acquire,
78
+ keys: [state_key],
79
+ argv: [@options[:initial], @options[:refill], estimate, ttl])
80
+ end
81
+
82
+ def refund(delta)
83
+ lua(:limiter_points_refund,
84
+ keys: [state_key],
85
+ argv: [delta, @options[:initial], ttl]).to_f
86
+ end
87
+ end
88
+ end
89
+ end
@@ -0,0 +1,77 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wurk
4
+ module Limiter
5
+ # Catches OverLimit (and any class registered in `Limiter.config.errors`),
6
+ # bumps `job['overrated']`, and decides what to do next:
7
+ #
8
+ # * reschedule disabled (`reschedule: 0`) → re-raise so the normal
9
+ # retry/dead pipeline handles it (spec §1.2/§1.4 behaviour).
10
+ # * still under the cap → reschedule onto the same queue at
11
+ # `Time.now + backoff` via `Client.push`.
12
+ # * cap reached (`overrated >= reschedule`, default 20) → **poison
13
+ # brake** (#16): a job that's still rate-limited after N reschedules
14
+ # is saturating the limiter, so instead of dumping it into another
15
+ # 25× retry loop we route it straight to the dead set tagged
16
+ # `rate_limited`, bumping `jobs.rate_limited` and firing death
17
+ # handlers. Bounded: termination at exactly `reschedule` attempts.
18
+ class ServerMiddleware
19
+ include Wurk::Middleware::ServerMiddleware
20
+
21
+ DEAD_REASON = 'rate_limited'
22
+
23
+ def call(_worker, job, _queue)
24
+ yield
25
+ rescue StandardError => e
26
+ raise unless over_limit?(e)
27
+
28
+ handle_over_limit(job, e)
29
+ end
30
+
31
+ private
32
+
33
+ def over_limit?(exc)
34
+ Wurk::Limiter.config.errors.any? { |k| exc.is_a?(k) }
35
+ end
36
+
37
+ def handle_over_limit(job, exc)
38
+ job['overrated'] = job.fetch('overrated', 0).to_i + 1
39
+ limiter = exc.respond_to?(:limiter) ? exc.limiter : nil
40
+ exc.job = job if exc.respond_to?(:job=)
41
+ cap = reschedule_cap(limiter)
42
+
43
+ raise exc if cap.zero? # rescheduling disabled → normal retry/dead pipeline
44
+ return route_to_dead(job, exc) if job['overrated'] >= cap
45
+
46
+ reschedule(job, exc, limiter)
47
+ end
48
+
49
+ # nil (concurrent/leaky/points never set it) → the default 20.
50
+ def reschedule_cap(limiter)
51
+ return DEFAULT_RESCHEDULE unless limiter
52
+
53
+ cap = limiter.options[:reschedule]
54
+ cap.nil? ? DEFAULT_RESCHEDULE : cap
55
+ end
56
+
57
+ def reschedule(job, exc, limiter)
58
+ backoff_proc = (limiter && limiter.options[:backoff]) || Wurk::Limiter.config.backoff
59
+ delay = backoff_proc.call(limiter, job, exc).to_f
60
+ Wurk::Client.new.push(job.merge('at' => ::Time.now.to_f + delay))
61
+ end
62
+
63
+ # Poison brake: stamp a clear reason, drop the job in the dead set, and
64
+ # ACK by returning normally (no re-raise) so it isn't also retried.
65
+ def route_to_dead(job, exc)
66
+ record = job.merge(
67
+ 'error_class' => exc.class.name,
68
+ 'error_message' => "#{DEAD_REASON}: #{exc.message} (overrated=#{job['overrated']})",
69
+ 'failed_at' => ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
70
+ )
71
+ Wurk::Metrics::Statsd.increment('jobs.rate_limited', tags: ["worker:#{job['class']}"])
72
+ Wurk::DeadSet.new.kill(Wurk.dump_json(record), ex: exc)
73
+ nil
74
+ end
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,48 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'points'
4
+
5
+ module Wurk
6
+ module Limiter
7
+ # No-op for the tests + bypass scenarios documented in §1.8. The
8
+ # within_limit block runs unconditionally and the introspection
9
+ # methods all return zeros so dashboards render predictably.
10
+ class Unlimited
11
+ def name = 'unlimited'
12
+ def type = :unlimited
13
+ def options = {}
14
+ def size = 0
15
+ def fingerprint = Digest::SHA256.hexdigest('unlimited')
16
+ def reset = nil
17
+ def delete = nil
18
+
19
+ # Uniform status shape (#16): no limit, always available.
20
+ def status
21
+ { used: 0, limit: nil, reset_at: nil, available?: true }
22
+ end
23
+
24
+ # Accept all the kwargs the other limiters take so a worker can swap
25
+ # `limiter = Sidekiq::Limiter.unlimited` in tests without touching
26
+ # call sites. `points`-style callers pass an `estimate:` and expect a
27
+ # `|handle|` block param — we yield a zero-cost handle just in case.
28
+ def within_limit(**_kwargs, &block)
29
+ raise ArgumentError, 'block required' unless block
30
+
31
+ if block.arity.zero?
32
+ block.call
33
+ else
34
+ block.call(Points::Handle.new(self, 0))
35
+ end
36
+ end
37
+
38
+ private
39
+
40
+ # Points::Handle#points_used calls `refund` on the limiter; without this
41
+ # no-op an Unlimited swap-in would raise NoMethodError the moment user
42
+ # code records actual usage. Drop-in contract trumps purity.
43
+ def refund(_delta)
44
+ 0.0
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,80 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'base'
4
+
5
+ module Wurk
6
+ module Limiter
7
+ # Sliding window via ZSET of timestamps. Accepts symbolic units or a
8
+ # raw Integer (in seconds). Used-units expand to N ZADDs in the Lua
9
+ # script so multi-charge calls remain atomic.
10
+ class Window < Base
11
+ WAIT_SLEEP = 0.5
12
+
13
+ def type = :window
14
+
15
+ def initialize(name, **options)
16
+ # Symbol or raw Integer (spec §1.2: window accepts both).
17
+ Limiter.interval_seconds(options[:interval], allow_integer: true)
18
+ super
19
+ end
20
+
21
+ def size
22
+ cutoff = ::Time.now.to_f - interval_seconds
23
+ Wurk::Limiter.redis do |c|
24
+ c.call('ZREMRANGEBYSCORE', state_key, '-inf', "(#{cutoff}")
25
+ c.call('ZCARD', state_key).to_i
26
+ end
27
+ end
28
+
29
+ # used = entries still inside the window; limit = count; reset_at =
30
+ # when the oldest entry slides out (freeing a slot), or nil when
31
+ # idle (#16).
32
+ def status
33
+ build_status(used: size, limit: @options[:count], reset_at: oldest_expiry)
34
+ end
35
+
36
+ def within_limit(used: 1, &block)
37
+ raise ArgumentError, 'block required' unless block
38
+
39
+ deadline = ::Time.now.to_f + @options[:wait_timeout]
40
+ loop do
41
+ ok, _current, _oldest = acquire(used)
42
+ return block.call if ok.to_i == 1
43
+
44
+ remaining = deadline - ::Time.now.to_f
45
+ raise OverLimit, self if remaining <= 0
46
+
47
+ sleep [remaining, WAIT_SLEEP].min
48
+ end
49
+ end
50
+
51
+ protected
52
+
53
+ def state_keys
54
+ [state_key]
55
+ end
56
+
57
+ private
58
+
59
+ def state_key
60
+ "lmtr-w:#{@name}"
61
+ end
62
+
63
+ # Oldest timestamp + interval = the moment it leaves the window.
64
+ def oldest_expiry
65
+ row = Wurk::Limiter.redis { |c| c.call('ZRANGE', state_key, 0, 0, 'WITHSCORES') }
66
+ row && !row.empty? ? row[1].to_f + interval_seconds : nil
67
+ end
68
+
69
+ def interval_seconds
70
+ @interval_seconds ||= Limiter.interval_seconds(@options[:interval], allow_integer: true)
71
+ end
72
+
73
+ def acquire(used)
74
+ lua(:limiter_window_acquire,
75
+ keys: [state_key],
76
+ argv: [@options[:count], interval_seconds, used, ttl, random_id])
77
+ end
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,255 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'json'
4
+ require 'digest'
5
+ require 'securerandom'
6
+ require_relative 'lua'
7
+
8
+ module Wurk
9
+ # Sidekiq Enterprise rate limiters: concurrent, bucket, window, leaky,
10
+ # points, unlimited. Lua-backed; all timing inside Lua is from TIME so
11
+ # clock skew across hosts doesn't matter inside one Redis. Spec:
12
+ # docs/target/sidekiq-ent.md §1.
13
+ #
14
+ # Layout (one file per type under `lib/wurk/limiter/`):
15
+ # * `Limiter::Base` owns the metadata write (lmtr:{name}) + the global
16
+ # `lmtr-list` registration so the Web UI can list every limiter, and
17
+ # the uniform `status` shape.
18
+ # * Per-type subclasses (Concurrent / Bucket / Window / Leaky / Points)
19
+ # own their acquire/wait loop. Each delegates the atomic step to a
20
+ # Lua script in `lib/wurk/lua/limiter_*.lua`.
21
+ # * `Unlimited` is a no-op stub for tests and the `unlimited(*)`
22
+ # constructor — same `within_limit` surface, never raises.
23
+ # * `ServerMiddleware` catches OverLimit, reschedules, and applies the
24
+ # poison brake.
25
+ #
26
+ # Wire-compat: every key uses the `lmtr-...:` prefix family from §1.7
27
+ # and the limiter is added to the shared `lmtr-list` SET.
28
+ module Limiter
29
+ DEFAULT_TTL = 90 * 24 * 3600
30
+ DEFAULT_WAIT_TIMEOUT = 5
31
+ DEFAULT_LOCK_TIMEOUT = 30
32
+ DEFAULT_RESCHEDULE = 20
33
+ DEFAULT_BACKOFF = lambda do |_limiter, job, _exc|
34
+ overrated = job.is_a?(Hash) ? job.fetch('overrated', 0).to_i : 0
35
+ (300 * overrated) + rand(300) + 1
36
+ end
37
+
38
+ NAME_PATTERN = /\A[\w\-:.\#@]+\z/
39
+
40
+ LIST_KEY = 'lmtr-list'
41
+
42
+ # Server middleware catches OverLimit, increments `job['overrated']`, and
43
+ # reschedules onto the same queue with `Time.now + backoff`. The
44
+ # `#limiter` attr lets the middleware reach the per-limiter backoff
45
+ # proc + reschedule cap. `#job` is set by the middleware just before the
46
+ # re-raise so error_handlers can see which job was in flight.
47
+ class OverLimit < StandardError
48
+ attr_reader :limiter
49
+ attr_accessor :job
50
+
51
+ def initialize(limiter, job = nil, msg = nil)
52
+ @limiter = limiter
53
+ @job = job
54
+ super(msg || "limit '#{limiter.name}' (#{limiter.type}) reached")
55
+ end
56
+ end
57
+
58
+ # Global config. Sidekiq Enterprise documents three knobs (§1.6):
59
+ # `backoff` (Proc), `redis` (a Hash that builds a dedicated pool), and
60
+ # `errors` (Array of exception classes the middleware also treats as
61
+ # OverLimit). All three are mutable and re-read on every push/perform.
62
+ class Config
63
+ attr_accessor :backoff, :errors
64
+ attr_reader :redis
65
+
66
+ def initialize
67
+ @backoff = DEFAULT_BACKOFF
68
+ @errors = [OverLimit]
69
+ @redis = nil
70
+ @redis_pool = nil
71
+ end
72
+
73
+ # Accept either a Hash (the documented Sidekiq Ent shape — `{ size:,
74
+ # url: }`) or an already-built `RedisPool`. The first redis read
75
+ # lazily materializes the pool; per-fork safety is the caller's
76
+ # responsibility (same contract as Wurk.redis_pool).
77
+ def redis=(value)
78
+ @redis = value
79
+ @redis_pool = nil
80
+ end
81
+
82
+ def pool
83
+ return nil if @redis.nil?
84
+
85
+ @pool ||= case @redis
86
+ when Wurk::RedisPool then @redis
87
+ when Hash
88
+ Wurk::RedisPool.new(
89
+ size: @redis[:size] || 10,
90
+ url: @redis[:url] || Wurk::RedisPool::DEFAULT_URL,
91
+ timeout: @redis[:timeout] || Wurk::RedisPool::DEFAULT_TIMEOUT,
92
+ name: 'limiter'
93
+ )
94
+ else
95
+ raise ArgumentError, "Limiter.config.redis must be Hash or RedisPool, got #{@redis.class}"
96
+ end
97
+ end
98
+ end
99
+
100
+ # `:second :minute :hour :day` symbols → seconds. Window also accepts
101
+ # a raw Integer; bucket does not (boundary semantics require a unit).
102
+ INTERVAL_UNITS = {
103
+ second: 1,
104
+ minute: 60,
105
+ hour: 3600,
106
+ day: 86_400
107
+ }.freeze
108
+
109
+ # Type string (as stored in the `lmtr:{name}` meta hash) → subclass.
110
+ # Drives `build` for dashboard introspection.
111
+ TYPE_CLASSES = {
112
+ 'concurrent' => 'Concurrent',
113
+ 'bucket' => 'Bucket',
114
+ 'window' => 'Window',
115
+ 'leaky' => 'Leaky',
116
+ 'points' => 'Points'
117
+ }.freeze
118
+
119
+ class << self
120
+ def configure
121
+ yield config
122
+ end
123
+
124
+ def config
125
+ @config ||= Config.new
126
+ end
127
+
128
+ # Test helper: blow away config + cached pool so a test that mutates
129
+ # `config.backoff` doesn't leak into the next one. Not part of the
130
+ # public Sidekiq surface.
131
+ def reset_config!
132
+ @config = nil
133
+ end
134
+
135
+ # Redis access: caller-supplied pool (Limiter.configure.redis = …) wins,
136
+ # else fall back to the default Wurk pool. This is the same hierarchy
137
+ # Sidekiq Ent documents — dedicated rate-limiter pool is opt-in.
138
+ def redis(&)
139
+ pool = config.pool || Wurk.redis_pool
140
+ pool.with(&)
141
+ end
142
+
143
+ def concurrent(name, limit, wait_timeout: DEFAULT_WAIT_TIMEOUT, lock_timeout: DEFAULT_LOCK_TIMEOUT,
144
+ policy: :raise, backoff: nil, ttl: DEFAULT_TTL)
145
+ Concurrent.new(name,
146
+ limit: limit,
147
+ wait_timeout: wait_timeout,
148
+ lock_timeout: lock_timeout,
149
+ policy: policy,
150
+ backoff: backoff,
151
+ ttl: ttl)
152
+ end
153
+
154
+ def bucket(name, count, interval, wait_timeout: DEFAULT_WAIT_TIMEOUT, backoff: nil,
155
+ ttl: DEFAULT_TTL, reschedule: DEFAULT_RESCHEDULE)
156
+ Bucket.new(name,
157
+ count: count,
158
+ interval: interval,
159
+ wait_timeout: wait_timeout,
160
+ backoff: backoff,
161
+ ttl: ttl,
162
+ reschedule: reschedule)
163
+ end
164
+
165
+ def window(name, count, interval, wait_timeout: DEFAULT_WAIT_TIMEOUT, backoff: nil,
166
+ ttl: DEFAULT_TTL, reschedule: DEFAULT_RESCHEDULE)
167
+ Window.new(name,
168
+ count: count,
169
+ interval: interval,
170
+ wait_timeout: wait_timeout,
171
+ backoff: backoff,
172
+ ttl: ttl,
173
+ reschedule: reschedule)
174
+ end
175
+
176
+ def leaky(name, bucket_size, drain, wait_timeout: DEFAULT_WAIT_TIMEOUT, backoff: nil, ttl: DEFAULT_TTL)
177
+ Leaky.new(name,
178
+ bucket_size: bucket_size,
179
+ drain: drain,
180
+ wait_timeout: wait_timeout,
181
+ backoff: backoff,
182
+ ttl: ttl)
183
+ end
184
+
185
+ def points(name, initial_points, refill_per_second, backoff: nil, ttl: DEFAULT_TTL)
186
+ Points.new(name,
187
+ initial: initial_points,
188
+ refill: refill_per_second,
189
+ backoff: backoff,
190
+ ttl: ttl)
191
+ end
192
+
193
+ def unlimited(*_args, **_opts)
194
+ Unlimited.new
195
+ end
196
+
197
+ # Reconstruct a limiter from its persisted metadata for read-only
198
+ # introspection (the dashboard `status` column). `register: false`
199
+ # keeps the GET side-effect-free. Returns nil for an unknown type.
200
+ def build(name, type, options, register: false)
201
+ return Unlimited.new if type.to_s == 'unlimited'
202
+
203
+ klass_name = TYPE_CLASSES[type.to_s]
204
+ return nil unless klass_name
205
+
206
+ const_get(klass_name).new(name, register: register, **coerce_build_options(options))
207
+ end
208
+
209
+ def interval_seconds(interval, allow_integer:)
210
+ interval = interval.to_sym if interval.is_a?(String) && INTERVAL_UNITS.key?(interval.to_sym)
211
+ case interval
212
+ when Symbol
213
+ INTERVAL_UNITS.fetch(interval) do
214
+ raise ArgumentError, "interval must be one of #{INTERVAL_UNITS.keys.inspect} (got #{interval.inspect})"
215
+ end
216
+ when Integer
217
+ unless allow_integer
218
+ raise ArgumentError, "interval must be a Symbol (got Integer); use #{INTERVAL_UNITS.keys.inspect}"
219
+ end
220
+
221
+ interval
222
+ else
223
+ raise ArgumentError, "interval must be Symbol or Integer (got #{interval.class})"
224
+ end
225
+ end
226
+
227
+ private
228
+
229
+ # Stored options round-trip through JSON, so symbol keys arrive as
230
+ # strings and unit symbols (`:minute`) as `"minute"`. Restore both so a
231
+ # rebuilt limiter validates the same as a freshly-constructed one.
232
+ def coerce_build_options(options)
233
+ opts = options.transform_keys(&:to_sym)
234
+ %i[interval drain].each do |k|
235
+ v = opts[k]
236
+ opts[k] = v.to_sym if v.is_a?(String) && INTERVAL_UNITS.key?(v.to_sym)
237
+ end
238
+ opts
239
+ end
240
+ end
241
+ end
242
+ end
243
+
244
+ require_relative 'limiter/base'
245
+ require_relative 'limiter/concurrent'
246
+ require_relative 'limiter/bucket'
247
+ require_relative 'limiter/window'
248
+ require_relative 'limiter/leaky'
249
+ require_relative 'limiter/points'
250
+ require_relative 'limiter/unlimited'
251
+ require_relative 'middleware'
252
+ require_relative 'limiter/server_middleware'
253
+ # Server-middleware registration happens in wurk.rb after the Wurk
254
+ # `class << self` block defines `Wurk.configuration` — limiter.rb
255
+ # loads earlier than that, so trying to call it here would NoMethodError.
@@ -0,0 +1,81 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'logger'
4
+ require 'time'
5
+ require 'json'
6
+
7
+ module Wurk
8
+ # Wurk's stdlib-compatible ::Logger subclass. The point of subclassing
9
+ # (rather than configuring a vanilla ::Logger) is to ship default
10
+ # formatters that read the thread-local Wurk::Context so every line
11
+ # carries jid/bid/tags/elapsed without callers threading the hash through.
12
+ #
13
+ # Formatter selection at boot:
14
+ # - ENV["DYNO"] set → Formatters::WithoutTimestamp (Heroku already prefixes)
15
+ # - otherwise → Formatters::Pretty
16
+ # Switch to Formatters::JSON manually for log aggregators that want NDJSON.
17
+ #
18
+ # Spec: docs/target/sidekiq-free.md §29.
19
+ class Logger < ::Logger
20
+ module Formatters
21
+ class Base < ::Logger::Formatter
22
+ SEVERITY_COLORS = {
23
+ 'DEBUG' => "\e[1;32mDEBUG\e[0m",
24
+ 'INFO' => "\e[1;34mINFO \e[0m",
25
+ 'WARN' => "\e[1;33mWARN \e[0m",
26
+ 'ERROR' => "\e[1;31mERROR\e[0m",
27
+ 'FATAL' => "\e[1;35mFATAL\e[0m"
28
+ }.freeze
29
+
30
+ def tid
31
+ Thread.current[:wurk_tid] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
32
+ end
33
+
34
+ def format_context(ctxt = Wurk::Context.current)
35
+ return '' if ctxt.empty?
36
+
37
+ " #{ctxt.map { |k, v| v.is_a?(Array) ? "#{k}=#{v.join(',')}" : "#{k}=#{v}" }.join(' ')}"
38
+ end
39
+ end
40
+
41
+ class Pretty < Base
42
+ def call(severity, time, _program_name, message)
43
+ "#{SEVERITY_COLORS[severity]} #{time.utc.iso8601(3)} pid=#{::Process.pid} " \
44
+ "tid=#{tid}#{format_context}: #{message}\n"
45
+ end
46
+ end
47
+
48
+ class WithoutTimestamp < Pretty
49
+ def call(severity, _time, _program_name, message)
50
+ "#{SEVERITY_COLORS[severity]} pid=#{::Process.pid} tid=#{tid}#{format_context}: #{message}\n"
51
+ end
52
+ end
53
+
54
+ class JSON < Base
55
+ def call(severity, time, _program_name, message)
56
+ hash = {
57
+ ts: time.utc.iso8601(3),
58
+ pid: ::Process.pid,
59
+ tid: tid,
60
+ lvl: severity,
61
+ msg: message
62
+ }
63
+ ctx = Wurk::Context.current
64
+ hash[:ctx] = ctx unless ctx.empty?
65
+ "#{::JSON.generate(hash)}\n"
66
+ end
67
+ end
68
+ end
69
+
70
+ def initialize(*, **)
71
+ super
72
+ self.formatter = default_formatter
73
+ end
74
+
75
+ private
76
+
77
+ def default_formatter
78
+ ENV['DYNO'] ? Formatters::WithoutTimestamp.new : Formatters::Pretty.new
79
+ end
80
+ end
81
+ end
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wurk
4
+ module Lua
5
+ # EVALSHA wrapper with `NOSCRIPT` recovery. The SHA1 of each script
6
+ # source is precomputed in `Wurk::Lua::SHAS`, so the first call to
7
+ # `eval_cached` after a fork has a fast path: a single EVALSHA, no
8
+ # per-pool bookkeeping. If the script cache was flushed (manual
9
+ # `SCRIPT FLUSH`, replica failover, OOM eviction), `EVALSHA` returns
10
+ # `NOSCRIPT` — we then `SCRIPT LOAD` once and retry exactly once.
11
+ #
12
+ # Spec: docs/target/sidekiq-free.md §20 (Lua script caching).
13
+ class Loader
14
+ NOSCRIPT_PREFIX = 'NOSCRIPT'
15
+
16
+ class << self
17
+ # Eagerly upload every registered script to the given connection.
18
+ # Idempotent on the Redis side: `SCRIPT LOAD` of the same source
19
+ # returns the same SHA regardless of how often it's called.
20
+ # Manager calls this once per child after the post-fork reconnect.
21
+ def script_load_all(redis)
22
+ SCRIPTS.each_value { |src| redis.call('SCRIPT', 'LOAD', src) }
23
+ end
24
+
25
+ # @param redis [RedisClient] a single connection (not a pool)
26
+ # @param name [Symbol] key into Wurk::Lua::SCRIPTS
27
+ # @param keys [Array<String>] EVALSHA KEYS
28
+ # @param argv [Array] EVALSHA ARGV (coerced to strings by Redis)
29
+ # @return Lua script return value
30
+ def eval_cached(redis, name, keys:, argv:)
31
+ src = SCRIPTS.fetch(name) { raise ArgumentError, "unknown Lua script: #{name.inspect}" }
32
+ sha = SHAS.fetch(name)
33
+ evalsha(redis, sha, keys, argv)
34
+ rescue RedisClient::CommandError => e
35
+ raise unless noscript?(e)
36
+
37
+ redis.call('SCRIPT', 'LOAD', src)
38
+ evalsha(redis, sha, keys, argv)
39
+ end
40
+
41
+ private
42
+
43
+ def evalsha(redis, sha, keys, argv)
44
+ redis.call('EVALSHA', sha, keys.size, *keys, *argv)
45
+ end
46
+
47
+ def noscript?(err)
48
+ err.message.to_s.start_with?(NOSCRIPT_PREFIX)
49
+ end
50
+ end
51
+ end
52
+ end
53
+ end