wurk 1.5.0 → 1.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (173) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +40 -2
  3. data/app/controllers/wurk/api/serializers.rb +51 -1
  4. data/app/controllers/wurk/api_controller.rb +42 -6
  5. data/app/controllers/wurk/dashboard_controller.rb +67 -10
  6. data/config/routes.rb +30 -3
  7. data/exe/wurk +6 -2
  8. data/lib/generators/wurk/install/install_generator.rb +3 -3
  9. data/lib/sidekiq/job_retry.rb +4 -0
  10. data/lib/sidekiq/manager.rb +4 -0
  11. data/lib/sidekiq/processor.rb +4 -0
  12. data/lib/wurk/api/app.rb +186 -0
  13. data/lib/wurk/api/auth.rb +167 -0
  14. data/lib/wurk/api/flows.rb +66 -0
  15. data/lib/wurk/api/idempotency.rb +168 -0
  16. data/lib/wurk/api/jobs.rb +180 -0
  17. data/lib/wurk/api/page.rb +98 -0
  18. data/lib/wurk/api/problem.rb +121 -0
  19. data/lib/wurk/api/queues.rb +126 -0
  20. data/lib/wurk/api/read_only.rb +68 -0
  21. data/lib/wurk/api/request.rb +46 -0
  22. data/lib/wurk/api/response.rb +28 -0
  23. data/lib/wurk/api/roll_up.rb +126 -0
  24. data/lib/wurk/api/router.rb +88 -0
  25. data/lib/wurk/api/serializers.rb +232 -0
  26. data/lib/wurk/api/swarm.rb +297 -0
  27. data/lib/wurk/api/throttle.rb +109 -0
  28. data/lib/wurk/api/validation.rb +306 -0
  29. data/lib/wurk/api.rb +96 -0
  30. data/lib/wurk/batch/server_middleware.rb +2 -2
  31. data/lib/wurk/batch.rb +12 -3
  32. data/lib/wurk/capsule.rb +8 -1
  33. data/lib/wurk/cli.rb +110 -8
  34. data/lib/wurk/client/buffered.rb +1 -1
  35. data/lib/wurk/client.rb +42 -3
  36. data/lib/wurk/collapse.rb +383 -0
  37. data/lib/wurk/command_builder.rb +49 -0
  38. data/lib/wurk/compat.rb +19 -0
  39. data/lib/wurk/component.rb +16 -3
  40. data/lib/wurk/configuration.rb +303 -4
  41. data/lib/wurk/debounce.rb +125 -0
  42. data/lib/wurk/encryption.rb +6 -1
  43. data/lib/wurk/fetcher/capped.rb +218 -0
  44. data/lib/wurk/fetcher/reliable.rb +22 -66
  45. data/lib/wurk/fetcher/unit_of_work.rb +107 -0
  46. data/lib/wurk/flow/builder.rb +271 -0
  47. data/lib/wurk/flow/chain.rb +42 -0
  48. data/lib/wurk/flow/completion.rb +114 -0
  49. data/lib/wurk/flow/creation.rb +254 -0
  50. data/lib/wurk/flow/node.rb +124 -0
  51. data/lib/wurk/flow/status.rb +206 -0
  52. data/lib/wurk/flow.rb +255 -0
  53. data/lib/wurk/flow_set.rb +53 -0
  54. data/lib/wurk/health.rb +8 -7
  55. data/lib/wurk/heartbeat.rb +21 -3
  56. data/lib/wurk/job/options.rb +11 -1
  57. data/lib/wurk/job.rb +19 -0
  58. data/lib/wurk/job_logger.rb +6 -2
  59. data/lib/wurk/job_retry.rb +6 -1
  60. data/lib/wurk/job_util.rb +110 -9
  61. data/lib/wurk/keys.rb +125 -0
  62. data/lib/wurk/launcher.rb +1 -1
  63. data/lib/wurk/leader.rb +1 -1
  64. data/lib/wurk/limiter/bucket.rb +1 -1
  65. data/lib/wurk/limiter/points.rb +1 -1
  66. data/lib/wurk/lua/debounce.lua +75 -0
  67. data/lib/wurk/lua/fetch_slot.lua +83 -0
  68. data/lib/wurk/lua/flow_abandon.lua +65 -0
  69. data/lib/wurk/lua/flow_advance.lua +177 -0
  70. data/lib/wurk/lua/flow_create.lua +141 -0
  71. data/lib/wurk/lua/flow_fail.lua +52 -0
  72. data/lib/wurk/lua/limiter_bucket_acquire.lua +26 -0
  73. data/lib/wurk/lua/limiter_concurrent_acquire.lua +33 -0
  74. data/lib/wurk/lua/limiter_concurrent_release.lua +7 -0
  75. data/lib/wurk/lua/limiter_leaky_acquire.lua +31 -0
  76. data/lib/wurk/lua/limiter_list_sweep.lua +30 -0
  77. data/lib/wurk/lua/limiter_points_acquire.lua +34 -0
  78. data/lib/wurk/lua/limiter_points_refund.lua +18 -0
  79. data/lib/wurk/lua/limiter_register.lua +27 -0
  80. data/lib/wurk/lua/limiter_window_acquire.lua +35 -0
  81. data/lib/wurk/lua/limiter_window_status.lua +22 -0
  82. data/lib/wurk/lua/loader.rb +23 -0
  83. data/lib/wurk/lua/queue_slot.lua +83 -0
  84. data/lib/wurk/lua/refresh_slots.lua +38 -0
  85. data/lib/wurk/lua/status_write.lua +29 -0
  86. data/lib/wurk/lua/throttle_slot.lua +71 -0
  87. data/lib/wurk/lua.rb +3 -3
  88. data/lib/wurk/manager.rb +12 -0
  89. data/lib/wurk/metrics/history.rb +34 -2
  90. data/lib/wurk/metrics/query.rb +1 -1
  91. data/lib/wurk/metrics/statsd.rb +14 -0
  92. data/lib/wurk/middleware/expiry.rb +71 -10
  93. data/lib/wurk/middleware/status.rb +274 -0
  94. data/lib/wurk/middleware/timeout.rb +137 -0
  95. data/lib/wurk/pid_cache.rb +43 -0
  96. data/lib/wurk/processor.rb +66 -22
  97. data/lib/wurk/profiler.rb +5 -3
  98. data/lib/wurk/queue_slot.rb +285 -0
  99. data/lib/wurk/rails.rb +3 -3
  100. data/lib/wurk/redis_pool.rb +5 -2
  101. data/lib/wurk/status/progress.rb +103 -0
  102. data/lib/wurk/status/record.rb +81 -0
  103. data/lib/wurk/status.rb +155 -0
  104. data/lib/wurk/telemetry/client_middleware.rb +59 -0
  105. data/lib/wurk/telemetry/server_middleware.rb +152 -0
  106. data/lib/wurk/telemetry.rb +137 -0
  107. data/lib/wurk/throttle.rb +142 -0
  108. data/lib/wurk/unique.rb +10 -2
  109. data/lib/wurk/version.rb +1 -1
  110. data/lib/wurk/watchdog.rb +188 -0
  111. data/lib/wurk/web/config.rb +77 -10
  112. data/lib/wurk/web/extension.rb +12 -4
  113. data/lib/wurk/web/locale_negotiator.rb +67 -0
  114. data/lib/wurk/web.rb +1 -0
  115. data/lib/wurk/worker.rb +39 -4
  116. data/lib/wurk.rb +42 -1
  117. data/vendor/assets/dashboard/assets/ArgsValue-D-x_ifLY.js +1 -0
  118. data/vendor/assets/dashboard/assets/BatchDetail-C39NJuew.js +1 -0
  119. data/vendor/assets/dashboard/assets/Batches-CSwo7Asa.js +1 -0
  120. data/vendor/assets/dashboard/assets/Busy-BOFMu-sq.js +1 -0
  121. data/vendor/assets/dashboard/assets/Cron-Dy8RQzDI.js +1 -0
  122. data/vendor/assets/dashboard/assets/Dashboard-BuTHI-O1.js +1 -0
  123. data/vendor/assets/dashboard/assets/Dead-B9KRvQ0N.js +1 -0
  124. data/vendor/assets/dashboard/assets/Extension-BnBVHfux.js +1 -0
  125. data/vendor/assets/dashboard/assets/FilterBox-DC24zite.js +1 -0
  126. data/vendor/assets/dashboard/assets/FlowDetail-DyLuzUvt.js +1 -0
  127. data/vendor/assets/dashboard/assets/FlowState-DAPKUahm.js +1 -0
  128. data/vendor/assets/dashboard/assets/Flows-Fr3rjZM_.js +1 -0
  129. data/vendor/assets/dashboard/assets/JobDetailModal-N6kiJXq3.js +2 -0
  130. data/vendor/assets/dashboard/assets/Limiters-kbFA7uS1.js +1 -0
  131. data/vendor/assets/dashboard/assets/Metrics-Dj2uoZ3o.js +1 -0
  132. data/vendor/assets/dashboard/assets/PageHeader-B_F94azl.js +1 -0
  133. data/vendor/assets/dashboard/assets/Profiles-D_DjEezN.js +1 -0
  134. data/vendor/assets/dashboard/assets/Queues-CO4V9hAz.js +1 -0
  135. data/vendor/assets/dashboard/assets/Retries-DCWnzeLa.js +1 -0
  136. data/vendor/assets/dashboard/assets/Scheduled-BebDUjLU.js +1 -0
  137. data/vendor/assets/dashboard/assets/Search-Cvr5fy4Y.js +1 -0
  138. data/vendor/assets/dashboard/assets/Skeleton-Bu3Ke6rV.js +1 -0
  139. data/vendor/assets/dashboard/assets/charts-BCs9bQKz.js +1 -0
  140. data/vendor/assets/dashboard/assets/index-BIwyOC5Q.js +141 -0
  141. data/vendor/assets/dashboard/assets/index-DBQN6Jk8.css +1 -0
  142. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Bzh-BJyL.js +1 -0
  143. data/vendor/assets/dashboard/assets/useSort-COA3fVJ5.js +1 -0
  144. data/vendor/assets/dashboard/assets/utils-BIrvZ1hi.js +1 -0
  145. data/vendor/assets/dashboard/index.html +42 -11
  146. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  147. metadata +102 -33
  148. data/vendor/assets/dashboard/assets/ArgsValue-CcR2ya6e.js +0 -1
  149. data/vendor/assets/dashboard/assets/BatchDetail-CUXJUQ3Q.js +0 -1
  150. data/vendor/assets/dashboard/assets/Batches-Cxan6Ngw.js +0 -1
  151. data/vendor/assets/dashboard/assets/Busy-DC5EGM0g.js +0 -1
  152. data/vendor/assets/dashboard/assets/Cron-Dlt8tXJA.js +0 -1
  153. data/vendor/assets/dashboard/assets/Dashboard-DNLu_WCg.js +0 -1
  154. data/vendor/assets/dashboard/assets/Dead-dZ7VGlKS.js +0 -1
  155. data/vendor/assets/dashboard/assets/Extension-DaFpEIJf.js +0 -1
  156. data/vendor/assets/dashboard/assets/FilterBox-CO3aYWIq.js +0 -1
  157. data/vendor/assets/dashboard/assets/JobDetailModal-DSWbT6G0.js +0 -2
  158. data/vendor/assets/dashboard/assets/Limiters-Cb4PKXNR.js +0 -1
  159. data/vendor/assets/dashboard/assets/Metrics-CCGzgCsT.js +0 -1
  160. data/vendor/assets/dashboard/assets/Modal-B86q6ruL.js +0 -1
  161. data/vendor/assets/dashboard/assets/PageHeader-fPrCcp_-.js +0 -1
  162. data/vendor/assets/dashboard/assets/Profiles-BnS82nR_.js +0 -1
  163. data/vendor/assets/dashboard/assets/Queues-CIyPevOy.js +0 -1
  164. data/vendor/assets/dashboard/assets/Retries-DopwXkXl.js +0 -1
  165. data/vendor/assets/dashboard/assets/Scheduled-1-Z7i1zE.js +0 -1
  166. data/vendor/assets/dashboard/assets/Search-ByA6eTma.js +0 -1
  167. data/vendor/assets/dashboard/assets/Skeleton-bC7HfQ9r.js +0 -1
  168. data/vendor/assets/dashboard/assets/charts-CLLzJ7vK.js +0 -1
  169. data/vendor/assets/dashboard/assets/index-B1N8hQUh.js +0 -141
  170. data/vendor/assets/dashboard/assets/index-BdiUEDXX.css +0 -1
  171. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-DpBjkf6_.js +0 -1
  172. data/vendor/assets/dashboard/assets/useSort-DvpwuNQE.js +0 -1
  173. data/vendor/assets/dashboard/assets/utils-DDJC7tJV.js +0 -1
@@ -0,0 +1,306 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'json'
4
+ require_relative 'problem'
5
+
6
+ module Wurk
7
+ module API
8
+ # Everything the API refuses at its boundary, before a Wurk object sees it.
9
+ #
10
+ # Client validates a job hash written by Ruby running in this process. This
11
+ # validates one typed by a stranger, and the two are different jobs.
12
+ # `JobUtil::TRANSIENT_ATTRIBUTES` (job_util.rb) is a *strip*-list: it names
13
+ # the two keys that must never reach the wire and passes everything else
14
+ # through, which is the right default for a producer that already runs your
15
+ # code and the wrong one for a producer that doesn't. So the allow-list
16
+ # JobUtil has no reason to own lives here, at the only door a stranger can
17
+ # knock on.
18
+ #
19
+ # Nothing in this file changes what a Ruby `perform_async` may push. The
20
+ # boundary is the HTTP request, not the payload format — wire-compat is
21
+ # untouched.
22
+ module Validation
23
+ # Carries the problem it should be rendered as, so a route rescues one
24
+ # class and renders whatever it was handed instead of mapping message
25
+ # text back onto a status code.
26
+ class Invalid < StandardError
27
+ attr_reader :type, :status, :extra
28
+
29
+ def initialize(detail, type: Problem::INVALID_REQUEST, status: 400, **extra)
30
+ super(detail)
31
+ @type = type
32
+ @status = status
33
+ @extra = extra
34
+ end
35
+ end
36
+
37
+ # A Ruby constant path and nothing else. `class` reaches the consumer as
38
+ # a constantize plus `new.perform`, so the string is a code selector, not
39
+ # data: any shape that is not a constant path is either an attack or a
40
+ # bug, and both are better answered here than by a NameError in a worker
41
+ # process two hosts away.
42
+ #
43
+ # Linear, not backtracking: `:` is outside the character class, so each
44
+ # `::` boundary is unambiguous.
45
+ CLASS_FORMAT = /\A[A-Z][A-Za-z0-9_]*(?:::[A-Z][A-Za-z0-9_]*)*\z/
46
+
47
+ # Length is checked separately from shape so a megabyte of `A`s is
48
+ # refused on a fact about the string rather than matched, stored in a
49
+ # payload and echoed back in an error.
50
+ MAX_CLASS_LENGTH = 255
51
+
52
+ # A jid reaches Redis as a ZSCAN glob (JobSet#find_job wraps it in `*`),
53
+ # so an unbounded or metacharacter-carrying jid off the network is a scan
54
+ # this API runs on the client's behalf. Sidekiq's own jids are
55
+ # `SecureRandom.hex(12)`; admit any URL-safe token of a sane length and
56
+ # refuse the rest at the door.
57
+ JID_FORMAT = /\A[A-Za-z0-9_-]{1,255}\z/
58
+
59
+ # A queue name arrives as one path segment (percent-decoded by the
60
+ # router, so a name containing '/' is reachable) and reaches Redis as
61
+ # `queue:<name>`. The Sidekiq schema bounds it nowhere, so this door
62
+ # does: whitespace and control characters are out because a name
63
+ # carrying them is one no operator can type back into the dashboard or a
64
+ # CLI flag, and the length cap exists for the same reason a jid's does —
65
+ # it is echoed back on every listing.
66
+ QUEUE_NAME_FORMAT = /\A[^[:space:][:cntrl:]]{1,255}\z/
67
+
68
+ # A process identity is `<hostname>:<pid>:<nonce>` and is itself a Redis
69
+ # key — the heartbeat HASH, `<identity>:work` and `<identity>-signals`
70
+ # are all built by interpolating it. The same shape as a queue name for
71
+ # the same reason, spelled separately because the two doors are
72
+ # independent: bounding what may be addressed as a queue says nothing
73
+ # about what may be addressed as a process.
74
+ IDENTITY_FORMAT = /\A[^[:space:][:cntrl:]]{1,255}\z/
75
+
76
+ # Same word the router uses for "any scope": here it turns the class
77
+ # allow-list off.
78
+ ANY = :any
79
+
80
+ # The top-level keys a job hash may carry in over HTTP.
81
+ #
82
+ # Anything absent is refused rather than dropped: a silently ignored
83
+ # `deadlin:` typo is a job that runs unbounded and a producer that never
84
+ # finds out. The server-written fields are absent on purpose —
85
+ # `created_at`/`enqueued_at` would let a client backdate its own queue
86
+ # latency, `retry_count`/`failed_at` would let it lie to the retrier,
87
+ # `expiry`/`deadline_at` are stamps derived from `expires_in`/`deadline`,
88
+ # `bid` would attach a job to a batch whose `pending` counter was never
89
+ # incremented for it, and `pool`/`client_class` name Ruby objects a JSON
90
+ # body cannot hold at all.
91
+ #
92
+ # Mutable for the same reason `TRANSIENT_ATTRIBUTES` is: a gem that adds
93
+ # a job option (sidekiq-unique-jobs' `lock:`) has to be able to append
94
+ # one without reopening this file. Baked into the literal rather than
95
+ # appended at load, so the parallel test runner never observes a
96
+ # half-built list.
97
+ # rubocop:disable Style/MutableConstant
98
+ JOB_KEYS = %w[
99
+ class args queue at retry jid backtrace tags dead
100
+ retry_for retry_queue expires_in track timeout deadline
101
+ encrypt unique_for unique_until collapse log_level locale wrapped
102
+ ]
103
+ # rubocop:enable Style/MutableConstant
104
+
105
+ # `push_bulk`'s envelope keys, on top of the job keys it shares.
106
+ BULK_KEYS = %w[batch_size spread_interval].freeze
107
+
108
+ # Echoing a rejected key back is the only way a client finds its typo,
109
+ # but the keys came off the network — bound how many and how long, for
110
+ # the same reason the JSON parser's own message is never forwarded.
111
+ MAX_REPORTED_KEYS = 10
112
+ MAX_REPORTED_KEY_LENGTH = 40
113
+
114
+ module_function
115
+
116
+ # Reads at most `limit + 1` bytes, so an oversized body is refused on the
117
+ # strength of what a cap-sized read already proved rather than by
118
+ # buffering the whole thing to measure it.
119
+ def body!(request, limit)
120
+ raw = request.read_body(limit + 1)
121
+ return raw unless raw.bytesize > limit
122
+
123
+ raise Invalid.new(
124
+ "The request body may not exceed #{limit} bytes.",
125
+ type: Problem::PAYLOAD_TOO_LARGE, status: 413, max_bytes: limit
126
+ )
127
+ end
128
+
129
+ # The body is the job hash with no envelope around it, so this only
130
+ # insists it is a JSON object at all — everything downstream indexes it
131
+ # by key.
132
+ def object!(raw)
133
+ parsed = ::JSON.parse(raw)
134
+ raise Invalid, 'The request body must be a JSON object.' unless parsed.is_a?(::Hash)
135
+
136
+ parsed
137
+ rescue ::JSON::ParserError
138
+ # Deliberately not the parser's own message: it quotes the unparsed
139
+ # remainder of the document, so a large malformed body would come back
140
+ # as a large error.
141
+ raise Invalid, 'The request body is not valid JSON.'
142
+ end
143
+
144
+ # One job hash, for `POST /jobs`.
145
+ def job!(payload, principal:, config:)
146
+ known_keys!(payload, JOB_KEYS)
147
+ jid!(payload['jid']) if payload.key?('jid')
148
+ allowed!(class!(payload), principal, config) if payload.key?('class')
149
+ args_size!(payload['args'], config.api_max_args_bytes)
150
+ payload
151
+ end
152
+
153
+ # The `push_bulk` envelope, for `POST /jobs/bulk`: the same job keys plus
154
+ # the two that describe the batch, and an `args` that is one array per
155
+ # job rather than one array of arguments.
156
+ def bulk!(payload, principal:, config:)
157
+ known_keys!(payload, JOB_KEYS + BULK_KEYS)
158
+ jid!(payload['jid']) if payload.key?('jid')
159
+ allowed!(class!(payload), principal, config) if payload.key?('class')
160
+ bulk_args_size!(payload['args'], config.api_max_args_bytes)
161
+ payload
162
+ end
163
+
164
+ # @raise [Invalid] unless `jid` is a URL-safe token of a sane length.
165
+ # @return [String] the jid.
166
+ def jid!(jid) = url_safe!(jid, 'jid')
167
+
168
+ # A bid indexes `b-<bid>` and its satellite keys, and Batch::Status
169
+ # builds every one of them by interpolation — the same key-fragment
170
+ # exposure a jid has, so the same door.
171
+ # @return [String] the bid.
172
+ def bid!(bid) = url_safe!(bid, 'batch id')
173
+
174
+ # A fid interpolates into `flow:<fid>` and, one segment further, into
175
+ # every node record under it — the same key-fragment exposure a bid has,
176
+ # so the same door.
177
+ # @return [String] the fid.
178
+ def fid!(fid) = url_safe!(fid, 'flow id')
179
+
180
+ def url_safe!(value, label)
181
+ return value if value.is_a?(::String) && JID_FORMAT.match?(value)
182
+
183
+ raise Invalid, "A #{label} is a URL-safe token of up to 255 characters."
184
+ end
185
+
186
+ # @raise [Invalid] unless `name` fits {QUEUE_NAME_FORMAT}.
187
+ # @return [String] the queue name.
188
+ def queue_name!(name)
189
+ return name if name.is_a?(::String) && QUEUE_NAME_FORMAT.match?(name)
190
+
191
+ raise Invalid, 'A queue name is up to 255 characters with no whitespace or control characters.'
192
+ end
193
+
194
+ # @raise [Invalid] unless `identity` fits {IDENTITY_FORMAT}.
195
+ # @return [String] the process identity.
196
+ def identity!(identity)
197
+ return identity if identity.is_a?(::String) && IDENTITY_FORMAT.match?(identity)
198
+
199
+ raise Invalid, 'A process identity is up to 255 characters with no whitespace or control characters.'
200
+ end
201
+
202
+ # Validates the allow-list where it is declared, so a typo'd class name
203
+ # raises in the initializer that wrote it instead of 403ing a producer in
204
+ # production. Accepts Classes as well as names — a host listing the
205
+ # constant it already has in scope is spelling the same thing.
206
+ #
207
+ # @return [Array<String>, :any, nil] normalized allow-list.
208
+ def enqueueable_classes!(classes)
209
+ return nil if classes.nil?
210
+ return ANY if classes == ANY
211
+
212
+ list = Array(classes).map(&:to_s)
213
+ raise ArgumentError, 'api_enqueue_classes must name at least one class, :any, or nil' if list.empty?
214
+
215
+ unknown = list.reject { |name| class_name?(name) }
216
+ raise ArgumentError, "api_enqueue_classes entries must be Ruby class names: #{unknown.inspect}" if unknown.any?
217
+
218
+ list.freeze
219
+ end
220
+
221
+ def class_name?(value)
222
+ value.is_a?(::String) && value.length <= MAX_CLASS_LENGTH && CLASS_FORMAT.match?(value)
223
+ end
224
+
225
+ def known_keys!(payload, allowed)
226
+ unknown = payload.keys - allowed
227
+ return if unknown.empty?
228
+
229
+ reported = reportable(unknown)
230
+ raise Invalid.new("Unknown job keys: #{reported.join(', ')}.", unknown_keys: reported)
231
+ end
232
+
233
+ def reportable(keys)
234
+ keys.sort.first(MAX_REPORTED_KEYS).map { |key| key.to_s[0, MAX_REPORTED_KEY_LENGTH] }
235
+ end
236
+
237
+ def class!(payload)
238
+ job_class = payload['class']
239
+ return job_class if class_name?(job_class)
240
+
241
+ raise Invalid, %(Job 'class' must be a Ruby class name, e.g. "HardWorker" or "Billing::Charge".)
242
+ end
243
+
244
+ # The allow-list JobUtil has no reason to own:
245
+ #
246
+ # nil → only an `admin` token may enqueue, and it may enqueue anything
247
+ # Array → exactly these names, for every token, `admin` included
248
+ # :any → the check is off
249
+ #
250
+ # Unset denies an enqueue-scoped producer everything, because `class`
251
+ # selects which of the host's jobs runs and a producer that can name any
252
+ # constant is choosing. `admin` is exempt only while no list exists: it
253
+ # already holds the destructive routes, so restricting which class it may
254
+ # enqueue would be theatre. Once a host writes a list, it means it.
255
+ def allowed!(job_class, principal, config)
256
+ return if enqueueable?(job_class, principal, config.api_enqueue_classes)
257
+
258
+ raise Invalid.new(
259
+ "#{job_class} may not be enqueued over HTTP; config.api_enqueue_classes names the classes that may.",
260
+ type: Problem::CLASS_NOT_ALLOWED, status: 403, job_class: job_class
261
+ )
262
+ end
263
+
264
+ def enqueueable?(job_class, principal, classes)
265
+ case classes
266
+ when nil then principal.permits?(:admin)
267
+ when ANY then true
268
+ else classes.include?(job_class)
269
+ end
270
+ end
271
+
272
+ # Only an Array is sizeable and only an Array is legal; Client names the
273
+ # other shapes, with the offending value in the message.
274
+ def args_size!(args, limit)
275
+ return unless args.is_a?(::Array)
276
+
277
+ size = ::JSON.generate(args).bytesize
278
+ return if size <= limit
279
+
280
+ raise Invalid.new(
281
+ "Job args may not exceed #{limit} bytes; these serialize to #{size}.",
282
+ type: Problem::PAYLOAD_TOO_LARGE, status: 413, max_bytes: limit
283
+ )
284
+ end
285
+
286
+ # The body cap bounds the request; this bounds each job the request turns
287
+ # into, which for a bulk push is the only one of the two that bounds what
288
+ # lands in Redis.
289
+ def bulk_args_size!(args, limit)
290
+ return unless args.is_a?(::Array)
291
+
292
+ args.each_with_index do |job_args, index|
293
+ next unless job_args.is_a?(::Array)
294
+
295
+ size = ::JSON.generate(job_args).bytesize
296
+ next if size <= limit
297
+
298
+ raise Invalid.new(
299
+ "Job #{index}'s args may not exceed #{limit} bytes; they serialize to #{size}.",
300
+ type: Problem::PAYLOAD_TOO_LARGE, status: 413, max_bytes: limit, job_index: index
301
+ )
302
+ end
303
+ end
304
+ end
305
+ end
306
+ end
data/lib/wurk/api.rb ADDED
@@ -0,0 +1,96 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wurk
4
+ # Namespace for two unrelated things that both answer to "the API": the Pro
5
+ # data-API Lua extensions (API::Fast, required by lib/wurk.rb at its
6
+ # load-order-sensitive point) and the machine-facing HTTP API (API::App).
7
+ module API
8
+ # The one prefix the machine plane answers under, and the version it pins.
9
+ # They live on the namespace rather than on App because the engine's mount
10
+ # constraint has to answer "is this path mine?" on every routing pass, and
11
+ # loading App to ask would undo the lazy load below.
12
+ API_VERSION = 'v1'
13
+ VERSION_PREFIX = "/#{API_VERSION}".freeze
14
+ NESTED_PREFIX = "#{VERSION_PREFIX}/".freeze
15
+ SUPPORTED_VERSIONS = [API_VERSION].freeze
16
+
17
+ # A path that names *some* version of this plane — `/v1`, `/v2/jobs`, not
18
+ # `/v1x` and not the dashboard's own `/stats`. Version-shaped rather than
19
+ # `v1`-only because the mount has to claim a version it does not serve in
20
+ # order for App to answer it `unsupported_api_version`, which is what the
21
+ # standalone and separately-mounted modes already do. Claiming only `/v1`
22
+ # would leave mode 1 alone in falling through to the host's router, and a
23
+ # client would learn "wrong version" from Rails' 404 in one deployment
24
+ # shape and from a problem document naming the supported versions in the
25
+ # other two.
26
+ VERSION_PATH = %r{\A/v\d+(?:/|\z)}
27
+
28
+ # Where mount mode 1 puts this plane inside the engine (config/routes.rb).
29
+ # Named here because two callers have to agree on it: the mount itself, and
30
+ # `Wurk::Web::Authorization`, which runs before routing and so sees the
31
+ # engine-relative path with this prefix still on it.
32
+ ENGINE_MOUNT = '/api'
33
+
34
+ # Rack env key the engine's Authorization middleware stamps when the
35
+ # dashboard is read-only. It is how mount mode 1 — and only mode 1 —
36
+ # inherits `WURK_WEB_READ_ONLY`: a separately mounted or standalone API is
37
+ # a different deployment and opts in on its own (`config.api_read_only`).
38
+ READ_ONLY_ENV = 'wurk.web.read_only'
39
+
40
+ class << self
41
+ # Class-level Rack entry, the same shape as Wurk::Web.call, so
42
+ # `mount Wurk::API => '/wurk-api'` and `run Wurk::API` both work.
43
+ #
44
+ # App and its dependencies (Rack::Request among them) load on the first
45
+ # request rather than at `require "wurk"`. The HTTP API is off unless a
46
+ # host mounts it, and eager-loading it cost every swarm child ~1.2 MB of
47
+ # pre-fork heap and measurably slower boot for a surface it never serves.
48
+ def call(env)
49
+ app.call(env)
50
+ end
51
+
52
+ # Whether the API should answer for `path` — the request path relative to
53
+ # wherever it was mounted. The engine's conditional mount (config/
54
+ # routes.rb) is a constraint over this, which settles two things a bare
55
+ # `mount` could not:
56
+ #
57
+ # * Off means absent. With no token registered the constraint fails,
58
+ # Rails falls through to the next route, and the surface does not
59
+ # exist — not a 401 advertising one that does (07 plan, step 1).
60
+ # Asked per request, not at draw time: routes load once at boot, and
61
+ # a host is free to register its token after that (a Puma-cluster web
62
+ # process never enters server mode, so its `configure_server` block
63
+ # has not run by then).
64
+ # * Nested in the engine, this plane shares the /api prefix with the
65
+ # dashboard's own JSON API. Without the version check a mistyped
66
+ # dashboard path would fall through to the machine plane and draw a
67
+ # bearer challenge for a route that was never part of this contract.
68
+ # A version-shaped path is never one of the dashboard's, so this
69
+ # claims every version and lets App refuse the ones it cannot serve.
70
+ def serves?(path, config = ::Wurk.configuration)
71
+ return false unless config.api_enabled?
72
+
73
+ VERSION_PATH.match?(path)
74
+ end
75
+
76
+ # The same question asked one prefix out, for callers that run before the
77
+ # engine's mount has stripped it — `Wurk::Web::Authorization` is the only
78
+ # one. It has to tell a machine-plane path from a dashboard one (both live
79
+ # under /api) without loading App, which is why this is here and not there.
80
+ def engine_serves?(path, config = ::Wurk.configuration)
81
+ path.start_with?(ENGINE_MOUNT) && serves?(path.delete_prefix(ENGINE_MOUNT), config)
82
+ end
83
+
84
+ private
85
+
86
+ # App is stateless, so every mount — engine-nested, separately mounted,
87
+ # standalone — can share one instance.
88
+ def app
89
+ @app ||= begin
90
+ require_relative 'api/app'
91
+ App.new
92
+ end
93
+ end
94
+ end
95
+ end
96
+ end
@@ -30,7 +30,7 @@ module Wurk
30
30
  class ServerMiddleware
31
31
  include Wurk::Middleware::ServerMiddleware
32
32
 
33
- def call(_worker, job, _queue)
33
+ def call(_worker, job, _queue, &)
34
34
  bid = job['bid']
35
35
  return yield unless bid
36
36
 
@@ -39,7 +39,7 @@ module Wurk
39
39
  return
40
40
  end
41
41
 
42
- run_and_ack(bid, job['jid']) { yield }
42
+ run_and_ack(bid, job['jid'], &)
43
43
  end
44
44
 
45
45
  private
data/lib/wurk/batch.rb CHANGED
@@ -36,7 +36,7 @@ module Wurk
36
36
  # Nested batches: a job opening its OWN batch (`batch.jobs { ... }`)
37
37
  # increments live counters on the existing batch. A callback opening its
38
38
  # PARENT batch links via parent_bid and adds child BID to b-<bid>-kids.
39
- class Batch
39
+ class Batch # rubocop:disable Metrics/ClassLength
40
40
  DEFAULT_EXPIRY_SECONDS = 30 * 24 * 60 * 60
41
41
  POST_SUCCESS_EXPIRY_SECONDS = 24 * 60 * 60
42
42
  CALLBACK_NOTIFY_TTL = 30 * 24 * 60 * 60
@@ -117,9 +117,18 @@ module Wurk
117
117
  # can drive the trim on isolated limits without mutating the process-global
118
118
  # `Wurk.configuration`.
119
119
  def self.trim_index(pipe, key, max: nil, timeout: nil)
120
+ score, rank = trim_bounds(max: max, timeout: timeout)
121
+ pipe.call('ZREMRANGEBYSCORE', key, '-inf', score)
122
+ pipe.call('ZREMRANGEBYRANK', key, 0, rank)
123
+ end
124
+
125
+ # The same two bounds as arguments, for a writer that cannot append to a
126
+ # pipeline: `Wurk::Flow`'s creation script has to trim from inside its own
127
+ # atomic write, and one policy read out of here beats a second copy of the
128
+ # cutoff arithmetic that drifts the day this one changes.
129
+ def self.trim_bounds(max: nil, timeout: nil)
120
130
  cutoff = ::Process.clock_gettime(::Process::CLOCK_REALTIME) - (timeout || DEFAULT_EXPIRY_SECONDS)
121
- pipe.call('ZREMRANGEBYSCORE', key, '-inf', "(#{cutoff}")
122
- pipe.call('ZREMRANGEBYRANK', key, 0, -(max || INDEX_MAX))
131
+ ["(#{cutoff}", -(max || INDEX_MAX)]
123
132
  end
124
133
 
125
134
  def initialize(bid = nil)
data/lib/wurk/capsule.rb CHANGED
@@ -4,6 +4,7 @@ require_relative 'redis_pool'
4
4
  require_relative 'pool_checkout'
5
5
  require_relative 'middleware/chain'
6
6
  require_relative 'fetcher/reliable'
7
+ require_relative 'watchdog'
7
8
 
8
9
  module Wurk
9
10
  # One processing unit: a set of threads + queues sharing a fetcher and a
@@ -14,7 +15,7 @@ module Wurk
14
15
  class Capsule
15
16
  MODES = %i[strict weighted random].freeze
16
17
 
17
- attr_reader :name, :queues, :mode, :weights, :config
18
+ attr_reader :name, :queues, :mode, :weights, :config, :watchdog
18
19
  attr_accessor :concurrency, :fetcher
19
20
 
20
21
  # Capsule-hosted components (Manager, Processor, Fetcher) hand their capsule
@@ -41,6 +42,7 @@ module Wurk
41
42
  @pools = {}
42
43
  @client_chain = nil
43
44
  @server_chain = nil
45
+ @watchdog = nil
44
46
  end
45
47
 
46
48
  def to_h
@@ -78,6 +80,11 @@ module Wurk
78
80
  def prepare!
79
81
  prepare_shared!
80
82
  @fetcher ||= build_fetcher
83
+ # Here rather than in the constructor so a client-only process — a web
84
+ # dyno building the same Configuration — allocates nothing, and so the
85
+ # scanner thread a bounded job spawns belongs to the child that runs it.
86
+ # The object itself holds no thread until something is bounded.
87
+ @watchdog ||= Watchdog.new(self)
81
88
  redis_pool
82
89
  fetch_redis_pool
83
90
  self
data/lib/wurk/cli.rb CHANGED
@@ -1,9 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'yaml'
4
- require 'optparse'
5
- require 'erb'
6
-
3
+ # yaml, optparse and erb are required inside the two methods that use them
4
+ # (#parse_config and #option_parser) rather than here. Together they cost ~45ms
5
+ # of the ~290ms `require "wurk"` spends, and only `exe/wurk` ever reaches them —
6
+ # a Rails app that enqueues and processes jobs pays that on every boot for a
7
+ # command-line parser it never runs. `require` is idempotent, so the lazy call
8
+ # is a $LOADED_FEATURES hash lookup on every invocation after the first.
7
9
  require_relative 'version'
8
10
  require_relative 'configuration'
9
11
  require_relative 'component'
@@ -24,6 +26,27 @@ module Wurk
24
26
  # format and Lua scripts rely on commands introduced in Redis 7.
25
27
  MIN_REDIS_VERSION = '7.0.0'
26
28
 
29
+ # Subcommands. Bare `wurk` runs the worker — the historical shape, and the
30
+ # only one the swarm binaries accept — so a subcommand is never anything
31
+ # but the first argument, and every argument after it is that command's.
32
+ COMMANDS = %w[api].freeze
33
+
34
+ # `wurk api` defaults. 0.0.0.0 because the point of the machine API is for
35
+ # another service to reach it; it is bearer-gated, and answers 404 to
36
+ # everything until a token exists. `--bind 127.0.0.1` narrows it.
37
+ DEFAULT_API_BIND = '0.0.0.0'
38
+ DEFAULT_API_PORT = 7433
39
+
40
+ # Serving the API with nothing registered would bind a port and 404 every
41
+ # request — the surface is off until a token exists, so say so at boot
42
+ # rather than leaving an operator to diagnose it over HTTP.
43
+ NO_API_TOKEN_MESSAGE = <<~MSG
44
+ No API token is registered, so every request would answer 404.
45
+ Register one where the app loaded by -r configures wurk:
46
+
47
+ Wurk.configuration.api_token(ENV.fetch('WURK_API_TOKEN'), scopes: %i[enqueue read])
48
+ MSG
49
+
27
50
  # Thread-backtrace dumper used by both TTIN and INFO. Same body — INFO is
28
51
  # the modern name, TTIN is kept for parity with older Sidekiq users.
29
52
  BACKTRACE_DUMPER = lambda do |cli|
@@ -68,8 +91,20 @@ module Wurk
68
91
  ['-C', '--config PATH', :config_file, 'path to YAML config file']
69
92
  ].freeze
70
93
 
94
+ # Same table shape, offered only when `api` is the subcommand: none of it
95
+ # means anything to a worker process, and `wurk --help` stays the worker's.
96
+ API_OPTION_FLAGS = [
97
+ ['-b', '--bind ADDRESS', :api_bind, "Address to bind (default #{DEFAULT_API_BIND})"],
98
+ ['-p', '--port PORT', :api_port, "Port to listen on (default #{DEFAULT_API_PORT})", :to_i],
99
+ ['-s', '--server NAME', :api_server, 'Rack handler to serve with (default: the first installed)']
100
+ ].freeze
101
+
71
102
  attr_accessor :launcher, :environment, :config
72
103
 
104
+ # The subcommand `parse` pulled off argv, or nil for the worker runner.
105
+ # The binary picks the mode from it — `exe/wurk` runs the API for `api`.
106
+ attr_reader :command
107
+
73
108
  def self.instance
74
109
  @instance ||= new
75
110
  end
@@ -84,12 +119,14 @@ module Wurk
84
119
  @launcher = nil
85
120
  @environment = nil
86
121
  @parser = nil
122
+ @command = nil
87
123
  end
88
124
 
89
125
  # `parse` is split from `run` so tests can drive option parsing without
90
126
  # touching Redis or booting the host app.
91
127
  def parse(args = ARGV.dup)
92
128
  @config ||= Wurk.default_configuration
129
+ @command = extract_command!(args)
93
130
  setup_options(args)
94
131
  initialize_logger
95
132
  validate!
@@ -137,6 +174,8 @@ module Wurk
137
174
  # `SIDEKIQ_PRELOAD_APP` whole-app eager-load) and boots `Process.warmup`
138
175
  # before the fork so children share warmed pages (copy-on-write). Spec §7.
139
176
  def run_swarm(boot_app: true, warmup: true)
177
+ raise ArgumentError, "the swarm runner takes no subcommand; run `wurk #{@command}` instead" if @command
178
+
140
179
  # Server mode before the app loads — see #run. The flag rides through the
141
180
  # fork into every child (the config object is copied), so configure_server
142
181
  # blocks registered in the parent take effect in the workers.
@@ -163,6 +202,32 @@ module Wurk
163
202
  end
164
203
  end
165
204
 
205
+ # `wurk api` — serve the machine-facing HTTP API and nothing else: no
206
+ # fetcher, no heartbeat, no job ever runs here. This is mount mode 3, the
207
+ # standalone one: the same Rack app the engine nests and a host can mount
208
+ # itself, served without Rails.
209
+ #
210
+ # Server mode is entered for the reason #run does it — a host that
211
+ # registers its token inside a `configure_server` block would otherwise
212
+ # boot an API with no credential and 404 every request. Nothing starts as a
213
+ # side effect: no lifecycle event fires and no Launcher is ever built.
214
+ #
215
+ # `handler:` is a test seam; production resolves one from the installed
216
+ # gems.
217
+ def run_api(boot_app: true, handler: nil)
218
+ enter_server_mode
219
+ boot_application if boot_app
220
+ validate_redis!
221
+ # Load the app here, not on the first request: a broken load should fail
222
+ # the boot an operator is watching, not the first client to call.
223
+ require_relative 'api/app'
224
+ raise ArgumentError, NO_API_TOKEN_MESSAGE unless @config.api_enabled?
225
+
226
+ handler ||= api_handler(@config[:api_server])
227
+ logger.info { "Wurk API listening on http://#{api_bind}:#{api_port}#{Wurk::API::VERSION_PREFIX}" }
228
+ handler.run(Wurk::API, Host: api_bind, Port: api_port)
229
+ end
230
+
166
231
  def handle_signal(sig)
167
232
  logger.debug { "Got #{sig} signal" }
168
233
  handler = SIGNAL_HANDLERS[sig]
@@ -188,6 +253,29 @@ module Wurk
188
253
  ::Process.warmup
189
254
  end
190
255
 
256
+ def api_bind = @config[:api_bind] || DEFAULT_API_BIND
257
+ def api_port = @config[:api_port] || DEFAULT_API_PORT
258
+
259
+ # Rack 3 moved the server handlers out to the `rackup` gem; rack 2 still
260
+ # ships them itself. Neither is a wurk dependency — the web server is the
261
+ # host's choice here exactly as it is for the app this sits next to — so
262
+ # resolve one at run time and name the fix when there is none.
263
+ def api_handler(name)
264
+ registry = handler_registry
265
+ name ? registry.get(name) : registry.default
266
+ rescue ::LoadError, ::NameError, ::ArgumentError => e
267
+ raise ArgumentError, "wurk api could not start a web server (#{e.message}). Add one to your Gemfile " \
268
+ '— puma, falcon, or rackup + webrick — or name an installed one with --server.'
269
+ end
270
+
271
+ def handler_registry
272
+ require 'rackup/handler'
273
+ ::Rackup::Handler
274
+ rescue ::LoadError
275
+ require 'rack/handler'
276
+ ::Rack::Handler
277
+ end
278
+
191
279
  def launch(self_read)
192
280
  @launcher = Wurk::Launcher.new(@config)
193
281
  begin
@@ -270,6 +358,14 @@ module Wurk
270
358
 
271
359
  # --- options + config-file --------------------------------------------
272
360
 
361
+ # Pulled off before OptionParser sees argv, so the banner and the flag
362
+ # table can depend on which command was named. A bare word OptionParser
363
+ # doesn't recognize would otherwise survive `parse!` and be silently
364
+ # ignored, which is how `wurk api` would look like it worked.
365
+ def extract_command!(args)
366
+ COMMANDS.include?(args.first) ? args.shift : nil
367
+ end
368
+
273
369
  def setup_options(args)
274
370
  opts = parse_options(args)
275
371
  set_environment(opts[:environment])
@@ -316,6 +412,9 @@ module Wurk
316
412
  def parse_config(path)
317
413
  raise ArgumentError, "No such file #{path}" unless ::File.exist?(path)
318
414
 
415
+ require 'erb'
416
+ require 'yaml'
417
+
319
418
  erb = ::ERB.new(::File.read(path), trim_mode: '-')
320
419
  erb.filename = ::File.expand_path(path)
321
420
  opts = ::YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
@@ -350,15 +449,18 @@ module Wurk
350
449
  end
351
450
 
352
451
  def option_parser(opts)
452
+ require 'optparse'
453
+
353
454
  ::OptionParser.new do |o|
354
- o.banner = 'wurk [options]'
355
- define_value_flags(o, opts)
455
+ o.banner = @command ? "wurk #{@command} [options]" : 'wurk [options]'
456
+ define_value_flags(o, opts, OPTION_FLAGS)
457
+ define_value_flags(o, opts, API_OPTION_FLAGS) if @command == 'api'
356
458
  define_meta_flags(o, opts)
357
459
  end
358
460
  end
359
461
 
360
- def define_value_flags(parser, opts)
361
- OPTION_FLAGS.each do |short, long, key, desc, transform|
462
+ def define_value_flags(parser, opts, flags)
463
+ flags.each do |short, long, key, desc, transform|
362
464
  parser.on(short, long, desc) { |arg| assign_flag(opts, key, arg, transform) }
363
465
  end
364
466
  end