wurk 1.4.0 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (190) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +41 -3
  3. data/app/controllers/concerns/wurk/stream_concurrency_guard.rb +22 -5
  4. data/app/controllers/wurk/api/serializers.rb +51 -1
  5. data/app/controllers/wurk/api_controller.rb +42 -6
  6. data/app/controllers/wurk/dashboard_controller.rb +66 -10
  7. data/config/routes.rb +30 -3
  8. data/exe/wurk +6 -2
  9. data/lib/generators/wurk/install/install_generator.rb +3 -3
  10. data/lib/sidekiq/job_retry.rb +4 -0
  11. data/lib/sidekiq/manager.rb +4 -0
  12. data/lib/sidekiq/processor.rb +4 -0
  13. data/lib/wurk/api/app.rb +186 -0
  14. data/lib/wurk/api/auth.rb +167 -0
  15. data/lib/wurk/api/flows.rb +66 -0
  16. data/lib/wurk/api/idempotency.rb +168 -0
  17. data/lib/wurk/api/jobs.rb +180 -0
  18. data/lib/wurk/api/page.rb +98 -0
  19. data/lib/wurk/api/problem.rb +121 -0
  20. data/lib/wurk/api/queues.rb +126 -0
  21. data/lib/wurk/api/read_only.rb +68 -0
  22. data/lib/wurk/api/request.rb +46 -0
  23. data/lib/wurk/api/response.rb +28 -0
  24. data/lib/wurk/api/roll_up.rb +126 -0
  25. data/lib/wurk/api/router.rb +88 -0
  26. data/lib/wurk/api/serializers.rb +232 -0
  27. data/lib/wurk/api/swarm.rb +297 -0
  28. data/lib/wurk/api/throttle.rb +109 -0
  29. data/lib/wurk/api/validation.rb +306 -0
  30. data/lib/wurk/api.rb +96 -0
  31. data/lib/wurk/batch/server_middleware.rb +2 -2
  32. data/lib/wurk/batch.rb +12 -3
  33. data/lib/wurk/capsule.rb +48 -13
  34. data/lib/wurk/cli.rb +99 -4
  35. data/lib/wurk/client/buffered.rb +8 -1
  36. data/lib/wurk/client.rb +144 -35
  37. data/lib/wurk/collapse.rb +383 -0
  38. data/lib/wurk/compat.rb +19 -0
  39. data/lib/wurk/component.rb +48 -4
  40. data/lib/wurk/configuration.rb +348 -10
  41. data/lib/wurk/context.rb +1 -1
  42. data/lib/wurk/debounce.rb +125 -0
  43. data/lib/wurk/encryption.rb +6 -1
  44. data/lib/wurk/engine.rb +41 -1
  45. data/lib/wurk/errors.rb +15 -0
  46. data/lib/wurk/fetcher/capped.rb +218 -0
  47. data/lib/wurk/fetcher/reaper.rb +12 -2
  48. data/lib/wurk/fetcher/reliable.rb +307 -90
  49. data/lib/wurk/fetcher/unit_of_work.rb +100 -0
  50. data/lib/wurk/fetcher.rb +6 -0
  51. data/lib/wurk/flow/builder.rb +271 -0
  52. data/lib/wurk/flow/chain.rb +42 -0
  53. data/lib/wurk/flow/completion.rb +114 -0
  54. data/lib/wurk/flow/creation.rb +254 -0
  55. data/lib/wurk/flow/node.rb +124 -0
  56. data/lib/wurk/flow/status.rb +206 -0
  57. data/lib/wurk/flow.rb +255 -0
  58. data/lib/wurk/flow_set.rb +53 -0
  59. data/lib/wurk/health.rb +8 -7
  60. data/lib/wurk/heartbeat.rb +23 -5
  61. data/lib/wurk/job/options.rb +11 -1
  62. data/lib/wurk/job.rb +19 -0
  63. data/lib/wurk/job_logger.rb +16 -7
  64. data/lib/wurk/job_retry.rb +6 -1
  65. data/lib/wurk/job_set.rb +3 -2
  66. data/lib/wurk/job_util.rb +153 -32
  67. data/lib/wurk/keys.rb +125 -0
  68. data/lib/wurk/launcher.rb +117 -73
  69. data/lib/wurk/leader.rb +37 -7
  70. data/lib/wurk/limiter/bucket.rb +1 -1
  71. data/lib/wurk/limiter/points.rb +1 -1
  72. data/lib/wurk/logger.rb +1 -1
  73. data/lib/wurk/lua/debounce.lua +75 -0
  74. data/lib/wurk/lua/fetch_slot.lua +83 -0
  75. data/lib/wurk/lua/flow_abandon.lua +65 -0
  76. data/lib/wurk/lua/flow_advance.lua +177 -0
  77. data/lib/wurk/lua/flow_create.lua +141 -0
  78. data/lib/wurk/lua/flow_fail.lua +52 -0
  79. data/lib/wurk/lua/limiter_bucket_acquire.lua +26 -0
  80. data/lib/wurk/lua/limiter_concurrent_acquire.lua +33 -0
  81. data/lib/wurk/lua/limiter_concurrent_release.lua +7 -0
  82. data/lib/wurk/lua/limiter_leaky_acquire.lua +31 -0
  83. data/lib/wurk/lua/limiter_list_sweep.lua +30 -0
  84. data/lib/wurk/lua/limiter_points_acquire.lua +34 -0
  85. data/lib/wurk/lua/limiter_points_refund.lua +18 -0
  86. data/lib/wurk/lua/limiter_register.lua +27 -0
  87. data/lib/wurk/lua/limiter_window_acquire.lua +35 -0
  88. data/lib/wurk/lua/limiter_window_status.lua +22 -0
  89. data/lib/wurk/lua/loader.rb +32 -3
  90. data/lib/wurk/lua/queue_slot.lua +83 -0
  91. data/lib/wurk/lua/refresh_slots.lua +38 -0
  92. data/lib/wurk/lua/status_write.lua +29 -0
  93. data/lib/wurk/lua/throttle_slot.lua +71 -0
  94. data/lib/wurk/lua.rb +3 -3
  95. data/lib/wurk/manager.rb +12 -0
  96. data/lib/wurk/metrics/accumulator.rb +95 -0
  97. data/lib/wurk/metrics/flusher.rb +70 -0
  98. data/lib/wurk/metrics/history.rb +132 -31
  99. data/lib/wurk/metrics/query.rb +1 -1
  100. data/lib/wurk/metrics/statsd.rb +46 -18
  101. data/lib/wurk/middleware/chain.rb +31 -14
  102. data/lib/wurk/middleware/expiry.rb +71 -10
  103. data/lib/wurk/middleware/poison_pill.rb +23 -2
  104. data/lib/wurk/middleware/status.rb +274 -0
  105. data/lib/wurk/middleware/timeout.rb +137 -0
  106. data/lib/wurk/pool_checkout.rb +10 -0
  107. data/lib/wurk/processor.rb +141 -29
  108. data/lib/wurk/profiler.rb +6 -4
  109. data/lib/wurk/queue.rb +8 -0
  110. data/lib/wurk/queue_slot.rb +285 -0
  111. data/lib/wurk/rails.rb +3 -3
  112. data/lib/wurk/redis_client_adapter.rb +1 -1
  113. data/lib/wurk/redis_pool.rb +2 -1
  114. data/lib/wurk/shutdown_gate.rb +79 -0
  115. data/lib/wurk/stats.rb +5 -1
  116. data/lib/wurk/status/progress.rb +103 -0
  117. data/lib/wurk/status/record.rb +81 -0
  118. data/lib/wurk/status.rb +155 -0
  119. data/lib/wurk/swarm/child_boot.rb +24 -4
  120. data/lib/wurk/swarm.rb +88 -14
  121. data/lib/wurk/telemetry/client_middleware.rb +59 -0
  122. data/lib/wurk/telemetry/server_middleware.rb +152 -0
  123. data/lib/wurk/telemetry.rb +137 -0
  124. data/lib/wurk/throttle.rb +142 -0
  125. data/lib/wurk/unique.rb +10 -2
  126. data/lib/wurk/version.rb +1 -1
  127. data/lib/wurk/watchdog.rb +188 -0
  128. data/lib/wurk/web/config.rb +88 -17
  129. data/lib/wurk/web/extension.rb +2 -2
  130. data/lib/wurk/web/locale_negotiator.rb +67 -0
  131. data/lib/wurk/web.rb +1 -0
  132. data/lib/wurk/worker.rb +39 -4
  133. data/lib/wurk.rb +42 -9
  134. data/vendor/assets/dashboard/assets/ArgsValue-D-x_ifLY.js +1 -0
  135. data/vendor/assets/dashboard/assets/BatchDetail-C39NJuew.js +1 -0
  136. data/vendor/assets/dashboard/assets/Batches-CSwo7Asa.js +1 -0
  137. data/vendor/assets/dashboard/assets/Busy-BOFMu-sq.js +1 -0
  138. data/vendor/assets/dashboard/assets/Cron-Dy8RQzDI.js +1 -0
  139. data/vendor/assets/dashboard/assets/Dashboard-BuTHI-O1.js +1 -0
  140. data/vendor/assets/dashboard/assets/Dead-B9KRvQ0N.js +1 -0
  141. data/vendor/assets/dashboard/assets/Extension-BnBVHfux.js +1 -0
  142. data/vendor/assets/dashboard/assets/FilterBox-DC24zite.js +1 -0
  143. data/vendor/assets/dashboard/assets/FlowDetail-DyLuzUvt.js +1 -0
  144. data/vendor/assets/dashboard/assets/FlowState-DAPKUahm.js +1 -0
  145. data/vendor/assets/dashboard/assets/Flows-Fr3rjZM_.js +1 -0
  146. data/vendor/assets/dashboard/assets/JobDetailModal-N6kiJXq3.js +2 -0
  147. data/vendor/assets/dashboard/assets/Limiters-kbFA7uS1.js +1 -0
  148. data/vendor/assets/dashboard/assets/Metrics-Dj2uoZ3o.js +1 -0
  149. data/vendor/assets/dashboard/assets/PageHeader-B_F94azl.js +1 -0
  150. data/vendor/assets/dashboard/assets/Profiles-D_DjEezN.js +1 -0
  151. data/vendor/assets/dashboard/assets/Queues-CO4V9hAz.js +1 -0
  152. data/vendor/assets/dashboard/assets/Retries-DCWnzeLa.js +1 -0
  153. data/vendor/assets/dashboard/assets/Scheduled-BebDUjLU.js +1 -0
  154. data/vendor/assets/dashboard/assets/Search-Cvr5fy4Y.js +1 -0
  155. data/vendor/assets/dashboard/assets/Skeleton-Bu3Ke6rV.js +1 -0
  156. data/vendor/assets/dashboard/assets/charts-BCs9bQKz.js +1 -0
  157. data/vendor/assets/dashboard/assets/index-BIwyOC5Q.js +141 -0
  158. data/vendor/assets/dashboard/assets/index-DBQN6Jk8.css +1 -0
  159. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Bzh-BJyL.js +1 -0
  160. data/vendor/assets/dashboard/assets/useSort-COA3fVJ5.js +1 -0
  161. data/vendor/assets/dashboard/assets/utils-BIrvZ1hi.js +1 -0
  162. data/vendor/assets/dashboard/index.html +42 -11
  163. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  164. metadata +105 -34
  165. data/vendor/assets/dashboard/assets/ArgsValue-D74zX0MI.js +0 -1
  166. data/vendor/assets/dashboard/assets/BatchDetail-OmC5NPgw.js +0 -1
  167. data/vendor/assets/dashboard/assets/Batches-CIpai7St.js +0 -1
  168. data/vendor/assets/dashboard/assets/Busy-A_kwSR6Q.js +0 -1
  169. data/vendor/assets/dashboard/assets/Cron-BG7HTqlp.js +0 -1
  170. data/vendor/assets/dashboard/assets/Dashboard-A_ToqHoo.js +0 -1
  171. data/vendor/assets/dashboard/assets/Dead-8J21jMyK.js +0 -1
  172. data/vendor/assets/dashboard/assets/Extension-B4Q9FIQu.js +0 -1
  173. data/vendor/assets/dashboard/assets/FilterBox-Fh_Ae7UW.js +0 -1
  174. data/vendor/assets/dashboard/assets/JobDetailModal-Ceng0PMB.js +0 -2
  175. data/vendor/assets/dashboard/assets/Limiters-CruDWvNZ.js +0 -1
  176. data/vendor/assets/dashboard/assets/Metrics-CIT7VCoN.js +0 -1
  177. data/vendor/assets/dashboard/assets/Modal-CN3rdKA_.js +0 -1
  178. data/vendor/assets/dashboard/assets/PageHeader-C44KNMGm.js +0 -1
  179. data/vendor/assets/dashboard/assets/Profiles-xEVTyS2N.js +0 -1
  180. data/vendor/assets/dashboard/assets/Queues-D86FYohJ.js +0 -1
  181. data/vendor/assets/dashboard/assets/Retries-Bz1O1D-i.js +0 -1
  182. data/vendor/assets/dashboard/assets/Scheduled-B6h2akTu.js +0 -1
  183. data/vendor/assets/dashboard/assets/Search-OOu22e5s.js +0 -1
  184. data/vendor/assets/dashboard/assets/Skeleton-DzR7XNxz.js +0 -1
  185. data/vendor/assets/dashboard/assets/charts-BVHHGof7.js +0 -1
  186. data/vendor/assets/dashboard/assets/index-BdiUEDXX.css +0 -1
  187. data/vendor/assets/dashboard/assets/index-D_lSDwKw.js +0 -141
  188. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-dVPGEWzn.js +0 -1
  189. data/vendor/assets/dashboard/assets/useSort-BeYbztkN.js +0 -1
  190. data/vendor/assets/dashboard/assets/utils-DDJC7tJV.js +0 -1
@@ -0,0 +1,180 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'idempotency'
4
+ require_relative 'problem'
5
+ require_relative 'response'
6
+ require_relative 'validation'
7
+
8
+ module Wurk
9
+ module API
10
+ # The produce plane: enqueue one job, enqueue many, ask what became of one,
11
+ # take a not-yet-run job back out.
12
+ #
13
+ # The request body *is* the Sidekiq job hash — `{"class", "args", "queue",
14
+ # "at", "retry"}` — and every write below hands it to {Wurk::Client}, the
15
+ # same object `perform_async` reaches. Nothing here builds, renames, or
16
+ # re-keys a payload. That is the drop-in guarantee written as code: an
17
+ # HTTP-enqueued job is the same bytes in the same Redis structures as a
18
+ # Ruby-enqueued one, so stock Sidekiq runs it. A payload assembled here
19
+ # would pass its own tests forever and be wrong the first time Client
20
+ # changed.
21
+ #
22
+ # Client is instantiated per request rather than memoized, and without a
23
+ # config, for the same reason: it must resolve the process's own middleware
24
+ # chain and pool at call time — exactly what a Ruby producer in this process
25
+ # gets — and the canonical inspectors this plane shares Redis with
26
+ # (ScheduledSet, RetrySet) read `Wurk.redis` unconditionally.
27
+ #
28
+ # What a stranger may send is {Validation}'s job, and whether a retry of it
29
+ # counts twice is {Idempotency}'s. This module owns only the three routes
30
+ # and the order those two run in.
31
+ module Jobs
32
+ module_function
33
+
34
+ def draw(router)
35
+ router.post('/jobs', scope: :enqueue) { |request| create(request) }
36
+ router.post('/jobs/bulk', scope: :enqueue) { |request| create_bulk(request) }
37
+ # :read. A producer that wants to poll what it pushed is granted
38
+ # `%i[enqueue read]` — the same pair the configuration docs show —
39
+ # rather than having every enqueue token able to read a jid it did not
40
+ # produce, which is what folding this into :enqueue would mean.
41
+ router.get('/jobs/:jid', scope: :read) { |request| show(request) }
42
+ # :admin, not :enqueue. A jid is not bound to the token that produced
43
+ # it, so an enqueue-scoped producer holding this route could walk the
44
+ # retry set one jid at a time. Widening a scope later is additive to
45
+ # every client; narrowing one breaks them.
46
+ router.delete('/jobs/:jid', scope: :admin) { |request| destroy(request) }
47
+ end
48
+
49
+ # 201 with the jid, or 200 with a null one when client middleware halted
50
+ # the push — a `collapse:`/`unique_for:` drop is the producer's own policy
51
+ # doing its job, not a failure to report as one.
52
+ def create(request)
53
+ produce(request) do |payload, config|
54
+ Validation.job!(payload, principal: request.principal, config: config)
55
+ jid = ::Wurk::Client.new.push(payload)
56
+ Response.json(jid ? 201 : 200, jid: jid)
57
+ end
58
+ end
59
+
60
+ # The `push_bulk` shape verbatim: one `class`, an array of arg arrays, and
61
+ # optionally `at`/`spread_interval`/`batch_size`. Nil entries in `jids`
62
+ # mark the jobs middleware halted, positionally.
63
+ def create_bulk(request)
64
+ produce(request) do |payload, config|
65
+ Validation.bulk!(payload, principal: request.principal, config: config)
66
+ jids = ::Wurk::Client.new.push_bulk(payload)
67
+ Response.json(jids.any? { |jid| !jid.nil? } ? 201 : 200, jids: jids)
68
+ end
69
+ end
70
+
71
+ # The shape both produce routes share.
72
+ #
73
+ # The body cap runs first, because it is the only check that can be made
74
+ # without holding the request. The `Idempotency-Key` claim wraps
75
+ # everything after it — parsing, validation and the push alike. That
76
+ # looks like it burns a client's key on a malformed body and does not:
77
+ # the claim releases on any answer that isn't a success, so the
78
+ # correction can be sent under the same key. What it buys is that the key
79
+ # is claimed *before* the push, the only ordering in which two concurrent
80
+ # retries of a dropped connection can't both enqueue.
81
+ def produce(request)
82
+ config = request.config
83
+ raw = Validation.body!(request, config.api_max_body_bytes)
84
+ Idempotency.around(request, raw, config) { rejectable(request) { yield(Validation.object!(raw), config) } }
85
+ rescue Validation::Invalid => e
86
+ # Only the two checks above the claim reach here — the body cap, and
87
+ # the shape of the Idempotency-Key itself. Everything below is answered
88
+ # inside the claim, where the status it decides on is visible.
89
+ problem(request, e)
90
+ end
91
+
92
+ # Renders a refusal *inside* the claim, so what the claim weighs is a
93
+ # response with a status on it rather than an exception it could only
94
+ # release on. Client owns what a valid job hash is past the boundary; the
95
+ # route surfaces its verdict instead of duplicating it.
96
+ def rejectable(request)
97
+ yield
98
+ rescue Validation::Invalid => e
99
+ problem(request, e)
100
+ rescue ::ArgumentError => e
101
+ invalid_request(request, e.message)
102
+ end
103
+
104
+ # The record {Wurk::Status} keeps for one jid: state, progress, result,
105
+ # error. Sidekiq keeps nothing at all about a job once it succeeds, so
106
+ # this route can only answer for a class that opted in with
107
+ # `track: true`; an untracked jid, an unknown one and a row whose TTL has
108
+ # lapsed are all the same answer, because Redis holds nothing that tells
109
+ # them apart.
110
+ def show(request)
111
+ jid = Validation.jid!(request.path_params[:jid].to_s)
112
+ record = ::Wurk::Status.get(jid)
113
+ return status_not_found(request, jid) unless record
114
+
115
+ Response.json(200, record.to_h)
116
+ rescue Validation::Invalid => e
117
+ problem(request, e)
118
+ end
119
+
120
+ # Removes a job that has not run yet from `schedule` or `retry`. Not a
121
+ # cancel: a job already handed to a processor keeps running, and one that
122
+ # has already died stays in `dead`, where deleting it is an operator
123
+ # action against a different set.
124
+ def destroy(request)
125
+ jid = Validation.jid!(request.path_params[:jid].to_s)
126
+ set = cancellable_sets.find { |candidate| remove(candidate, jid) }
127
+ return job_not_found(request, jid) unless set
128
+
129
+ Response.json(200, jid: jid, set: set.name)
130
+ rescue Validation::Invalid => e
131
+ problem(request, e)
132
+ end
133
+
134
+ # `schedule` first: a job merely waiting for its time is the one a
135
+ # producer usually means to call off, and a jid in both sets at once
136
+ # would mean something else is already broken.
137
+ def cancellable_sets = [::Wurk::ScheduledSet.new, ::Wurk::RetrySet.new]
138
+
139
+ # Find-then-delete through the canonical inspectors rather than a second
140
+ # deletion path over the same ZSETs — the dashboard and every third-party
141
+ # tool use these objects, and two ways to remove a member is a place for
142
+ # them to disagree. A job promoted out of the set between the two steps
143
+ # removes nothing and reads as "already gone", the honest answer to a
144
+ # cancel that lost the race.
145
+ def remove(set, jid)
146
+ entry = set.find_job(jid)
147
+ entry ? entry.delete : false
148
+ end
149
+
150
+ # Validation decided which problem this is; rendering it is all that is
151
+ # left, so a new rejection never needs a new arm here.
152
+ def problem(request, error) = Problem.from(error, instance: request.path)
153
+
154
+ def invalid_request(request, detail)
155
+ Problem.render(Problem::INVALID_REQUEST, status: 400, detail: detail, instance: request.path)
156
+ end
157
+
158
+ def job_not_found(request, jid)
159
+ Problem.render(
160
+ Problem::JOB_NOT_FOUND,
161
+ status: 404,
162
+ detail: "No scheduled or retrying job has jid #{jid}.",
163
+ instance: request.path,
164
+ jid: jid
165
+ )
166
+ end
167
+
168
+ def status_not_found(request, jid)
169
+ Problem.render(
170
+ Problem::JOB_NOT_FOUND,
171
+ status: 404,
172
+ detail: "No status is recorded for jid #{jid}; the job is unknown, its class does not set " \
173
+ 'track: true, or its record has expired.',
174
+ instance: request.path,
175
+ jid: jid
176
+ )
177
+ end
178
+ end
179
+ end
180
+ end
@@ -0,0 +1,98 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rack'
4
+ require_relative 'validation'
5
+
6
+ module Wurk
7
+ module API
8
+ # Offset paging for the observe plane's listings.
9
+ #
10
+ # Deliberately not the dashboard's Wurk::Api::Pagination. That module is
11
+ # autoloaded out of app/ by the engine, and the API has to page in
12
+ # standalone mode where Rails is never loaded (CLAUDE.md); requiring an
13
+ # autoloadable file by hand is exactly what Zeitwerk forbids. The contracts
14
+ # differ too — the dashboard's may change whenever the SPA does, everything
15
+ # under /v1 may not — so one shared module would tie a frozen contract to a
16
+ # mutable one.
17
+ module Page
18
+ DEFAULT_COUNT = 25
19
+ MAX_COUNT = 200
20
+
21
+ # Offset paging reaches page N by walking N*count members through Ruby,
22
+ # and their Redis round trips with them, so an unclamped `page` lets one
23
+ # request drag an entire million-row set through this process. A
24
+ # thousand pages is past any real client's depth and still bounds the
25
+ # worst case.
26
+ MAX_PAGE = 1_000
27
+
28
+ Window = Data.define(:page, :count) do
29
+ def offset = page * count
30
+ end
31
+
32
+ module_function
33
+
34
+ # @raise [Validation::Invalid] when a paging parameter is present but is
35
+ # not an integer. Out of range is clamped instead of refused — a client
36
+ # asking for more rows than the API will give is not a client bug — and
37
+ # every listing echoes the effective values back, so the clamp is
38
+ # visible rather than silent.
39
+ def window!(request) = window(query!(request))
40
+
41
+ # The parsed-query half, for a route that reads a parameter of its own
42
+ # (`?filter=`) beside the paging ones: it parses once and hands the same
43
+ # Hash to both, rather than re-parsing the query string per parameter.
44
+ def window(query)
45
+ Window.new(
46
+ page: integer!(query['page'], 'page', 0).clamp(0, MAX_PAGE),
47
+ count: integer!(query['count'], 'count', DEFAULT_COUNT).clamp(1, MAX_COUNT)
48
+ )
49
+ end
50
+
51
+ # Rack::Request#params merges the POST body in, which the paged routes
52
+ # must never read. parse_query is also flat: `?page[]=1` arrives as the
53
+ # key it literally is instead of nesting into a Hash that reaches
54
+ # Integer().
55
+ #
56
+ # Rack's own refusals (InvalidParameterError on bad %-encoding,
57
+ # QueryLimitError on an oversized one) are ArgumentError and RangeError
58
+ # respectively; caught here so a malformed query string is the 400 it is
59
+ # rather than the 500 App's catch-all would make of it.
60
+ def query!(request)
61
+ ::Rack::Utils.parse_query(request.query_string)
62
+ rescue ::ArgumentError, ::TypeError, ::RangeError
63
+ raise Validation::Invalid, 'The query string could not be parsed.'
64
+ end
65
+
66
+ # A repeated parameter (`?page=1&page=2`) parses to an Array, which is
67
+ # the TypeError arm — an ambiguous request, answered rather than guessed.
68
+ def integer!(raw, name, default)
69
+ return default if raw.nil? || raw == ''
70
+
71
+ Integer(raw, 10)
72
+ rescue ::ArgumentError, ::TypeError
73
+ raise Validation::Invalid, "The '#{name}' parameter must be an integer."
74
+ end
75
+
76
+ # Walks `enumerable`, skips `window.offset` members, and hands at most
77
+ # `window.count` of them to the block, which returns the row to emit.
78
+ #
79
+ # Skipped members are never yielded, so nothing before the offset is
80
+ # serialized: on a queue page that is a JobRecord whose JSON is never
81
+ # parsed, which is most of the cost of a deep page.
82
+ def slice(enumerable, window)
83
+ rows = []
84
+ skipped = 0
85
+ enumerable.each do |member|
86
+ if skipped < window.offset
87
+ skipped += 1
88
+ next
89
+ end
90
+
91
+ rows << yield(member)
92
+ break if rows.size >= window.count
93
+ end
94
+ rows
95
+ end
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,121 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'json'
4
+
5
+ module Wurk
6
+ module API
7
+ # Error bodies for the HTTP API, shaped like RFC 9457 problem documents.
8
+ #
9
+ # One deliberate divergence from the RFC: `type` is a bare stable slug
10
+ # ('not_found'), not a URI. A URI would either hardcode a docs host that can
11
+ # move or emit a mount-relative path the client can't dereference, and the
12
+ # slug is the part a client actually branches on. `/v1` makes these slugs a
13
+ # contract: adding one is fine, renaming one is a breaking change.
14
+ module Problem
15
+ CONTENT_TYPE = 'application/problem+json'
16
+
17
+ NOT_FOUND = 'not_found'
18
+ METHOD_NOT_ALLOWED = 'method_not_allowed'
19
+ UNSUPPORTED_API_VERSION = 'unsupported_api_version'
20
+ INTERNAL_ERROR = 'internal_error'
21
+ UNAUTHORIZED = 'unauthorized'
22
+ INVALID_REQUEST = 'invalid_request'
23
+ # Distinct from `not_found`, which means the API has no such route. A
24
+ # client that asked to cancel a job needs to tell "you addressed nothing"
25
+ # apart from "that job already ran".
26
+ JOB_NOT_FOUND = 'job_not_found'
27
+ # The same distinction one addressable resource over: a well-formed bid
28
+ # the `batches` set has never held, as opposed to a mistyped path.
29
+ BATCH_NOT_FOUND = 'batch_not_found'
30
+ # And one relation further out: a well-formed fid the `flows` set has
31
+ # never held. Distinct from `batch_not_found` because a flow's nodes are
32
+ # batches — a client told "batch not found" for a fid would go looking
33
+ # for the wrong thing.
34
+ FLOW_NOT_FOUND = 'flow_not_found'
35
+ # A well-formed identity that no live heartbeat answers to — the process
36
+ # exited, or its beat lapsed and Redis reaped the row.
37
+ PROCESS_NOT_FOUND = 'process_not_found'
38
+ # The process is live and the caller may signal it, but this one cannot
39
+ # be signalled at all: an embedded process shares its host application's
40
+ # PID, so a TSTP or TERM aimed at it would hit the web server around it.
41
+ PROCESS_NOT_SIGNALABLE = 'process_not_signalable'
42
+ # Named for RFC 6750 §3.1 so the slug and the `error=` the 403 carries in
43
+ # WWW-Authenticate are the same word.
44
+ INSUFFICIENT_SCOPE = 'insufficient_scope'
45
+ # Separate from `invalid_request` because the client's fix is different:
46
+ # nothing about the request's shape is wrong, it is only too big, and the
47
+ # `max_bytes` extension member says by how much.
48
+ PAYLOAD_TOO_LARGE = 'payload_too_large'
49
+ # An authorization verdict, like `insufficient_scope` — the credential is
50
+ # valid and the body is well-formed, but this class is not one the HTTP
51
+ # API may enqueue.
52
+ CLASS_NOT_ALLOWED = 'class_not_allowed'
53
+ # The two ways an `Idempotency-Key` collides. Reused means the same key
54
+ # arrived with different bytes, which is a client bug it must fix by
55
+ # rotating the key; in-progress means the first request holding it has
56
+ # not answered yet, which it fixes by waiting.
57
+ IDEMPOTENCY_KEY_REUSED = 'idempotency_key_reused'
58
+ REQUEST_IN_PROGRESS = 'request_in_progress'
59
+ # Not `insufficient_scope`: the credential is granted everything it needs
60
+ # and would work against another deployment unchanged. Nothing the client
61
+ # can send fixes this one, which is why it gets its own slug — a producer
62
+ # that retried a 403 forever on a frozen deployment is the failure this
63
+ # avoids.
64
+ READ_ONLY = 'read_only'
65
+ # The one refusal that comes with a time attached. `retry_after` repeats
66
+ # the header as a number so a client that already parses this body does
67
+ # not have to reach back into the headers for it.
68
+ RATE_LIMITED = 'rate_limited'
69
+
70
+ # Every slug needs a human title; `fetch` below turns a missing one into a
71
+ # loud failure in the test suite rather than a half-formed error body.
72
+ TITLES = {
73
+ NOT_FOUND => 'Not Found',
74
+ METHOD_NOT_ALLOWED => 'Method Not Allowed',
75
+ UNSUPPORTED_API_VERSION => 'Unsupported API Version',
76
+ INTERNAL_ERROR => 'Internal Server Error',
77
+ UNAUTHORIZED => 'Unauthorized',
78
+ INSUFFICIENT_SCOPE => 'Insufficient Scope',
79
+ INVALID_REQUEST => 'Invalid Request',
80
+ JOB_NOT_FOUND => 'Job Not Found',
81
+ BATCH_NOT_FOUND => 'Batch Not Found',
82
+ FLOW_NOT_FOUND => 'Flow Not Found',
83
+ PROCESS_NOT_FOUND => 'Process Not Found',
84
+ PROCESS_NOT_SIGNALABLE => 'Process Not Signalable',
85
+ PAYLOAD_TOO_LARGE => 'Payload Too Large',
86
+ CLASS_NOT_ALLOWED => 'Class Not Allowed',
87
+ IDEMPOTENCY_KEY_REUSED => 'Idempotency Key Reused',
88
+ REQUEST_IN_PROGRESS => 'Request In Progress',
89
+ READ_ONLY => 'Read-Only Mode',
90
+ RATE_LIMITED => 'Too Many Requests'
91
+ }.freeze
92
+
93
+ module_function
94
+
95
+ # `extra` keywords become extension members (RFC 9457 §3.2), e.g.
96
+ # `supported_versions:`. Returns a Rack response triple.
97
+ def render(type, status:, detail:, instance:, headers: nil, **extra)
98
+ body = {
99
+ type: type, title: TITLES.fetch(type), status: status, detail: detail, instance: instance
100
+ }
101
+ body.merge!(extra)
102
+ [status, response_headers(headers), [::JSON.generate(body)]]
103
+ end
104
+
105
+ # Renders a refusal that already knows which problem it is — the shape
106
+ # Validation::Invalid carries. The mapping from a rejection to a status
107
+ # code stays with the check that made it, so a new rejection never needs
108
+ # a new arm in the route that surfaces it.
109
+ def from(error, instance:)
110
+ render(error.type, status: error.status, detail: error.message, instance: instance, **error.extra)
111
+ end
112
+
113
+ # nosniff so a browser pointed at an error can never be talked into
114
+ # rendering the reflected request path as anything but data.
115
+ def response_headers(extra)
116
+ headers = { 'content-type' => CONTENT_TYPE, 'x-content-type-options' => 'nosniff' }
117
+ extra ? headers.merge(extra) : headers
118
+ end
119
+ end
120
+ end
121
+ end
@@ -0,0 +1,126 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'page'
4
+ require_relative 'problem'
5
+ require_relative 'response'
6
+ require_relative 'serializers'
7
+ require_relative 'validation'
8
+
9
+ module Wurk
10
+ module API
11
+ # The observe plane: how deep the queues are, what is waiting to retry,
12
+ # what died, and the counters over the lot.
13
+ #
14
+ # Every route reads through a canonical inspector — {Wurk::Stats},
15
+ # {Wurk::Queue}, {Wurk::RetrySet}, {Wurk::ScheduledSet}, {Wurk::DeadSet},
16
+ # {Wurk::Batch::Status} — the same objects the dashboard reads. Not to save
17
+ # code: a second reader over the same Redis keys is a second place for the
18
+ # size-descending queue order, the latency math, the `paused` set and the
19
+ # ActiveJob unwrapping to drift, and a dashboard and an API that disagree
20
+ # about how deep a queue is are worse than either on its own.
21
+ #
22
+ # Pause and unpause are the only writes, and they go through
23
+ # {Wurk::Queue#pause!} / {Wurk::Queue#unpause!}, which expire this
24
+ # process's fetcher cache of the `paused` set. A bare SADD here would leave
25
+ # the swarm fetching from a queue the API had just reported paused.
26
+ module Queues
27
+ module_function
28
+
29
+ # Verb, pattern, the scope a caller must hold, handler. A table rather
30
+ # than nine `router.get` calls because the scope column is the part that
31
+ # has to be read at a glance: the two writes take :admin, not :read.
32
+ # Pausing `default` stops the fleet from working without enqueueing or
33
+ # deleting anything, so it belongs with the destructive routes rather
34
+ # than with the listings it sits beside.
35
+ TABLE = [
36
+ [:get, '/stats', :read, :stats],
37
+ [:get, '/queues', :read, :index],
38
+ [:get, '/queues/:name', :read, :show],
39
+ [:get, '/retries', :read, :retries],
40
+ [:get, '/scheduled', :read, :scheduled],
41
+ [:get, '/dead', :read, :dead],
42
+ [:get, '/batches/:bid', :read, :batch],
43
+ [:post, '/queues/:name/pause', :admin, :pause],
44
+ [:post, '/queues/:name/unpause', :admin, :unpause]
45
+ ].freeze
46
+
47
+ # The refusal rendering wraps every handler here rather than sitting in
48
+ # each one: they all validate something that came off the network, and a
49
+ # rescue repeated per route is one a new route gets written without.
50
+ def draw(router)
51
+ TABLE.each do |verb, pattern, scope, handler|
52
+ router.public_send(verb, pattern, scope: scope) do |request|
53
+ public_send(handler, request)
54
+ rescue Validation::Invalid => e
55
+ Problem.from(e, instance: request.path)
56
+ end
57
+ end
58
+ end
59
+
60
+ def stats(_request)
61
+ Response.json(200, Serializers.stats(::Wurk::Stats.new))
62
+ end
63
+
64
+ # Stats#queue_summaries rather than Queue.all: one pipeline instead of an
65
+ # LLEN round trip per queue, and it comes back in the size-descending
66
+ # order every other Wurk surface lists queues in.
67
+ def index(_request)
68
+ Response.json(200, queues: ::Wurk::Stats.new.queue_summaries.map { |q| Serializers.queue_summary(q) })
69
+ end
70
+
71
+ # No 404 for a name nothing was ever enqueued under. A queue is not an
72
+ # entity in the Sidekiq schema — it is a LIST that exists only while it
73
+ # has members — so "never used" and "drained" are the same state in
74
+ # Redis, and `size: 0` is the honest reading of both. Pausing one before
75
+ # its first job is a legitimate pre-deploy move for the same reason.
76
+ def show(request)
77
+ window = Page.window!(request)
78
+ queue = ::Wurk::Queue.new(Validation.queue_name!(request.path_params[:name]))
79
+ jobs = Page.slice(queue, window) { |record| Serializers.job_record(record) }
80
+ Response.json(200, Serializers.queue_gauges(queue).merge(page: window.page, count: window.count, jobs: jobs))
81
+ end
82
+
83
+ def retries(request) = sorted_set(request, ::Wurk::RetrySet.new)
84
+ def scheduled(request) = sorted_set(request, ::Wurk::ScheduledSet.new)
85
+ def dead(request) = sorted_set(request, ::Wurk::DeadSet.new)
86
+
87
+ def pause(request) = toggle(request, paused: true)
88
+ def unpause(request) = toggle(request, paused: false)
89
+
90
+ # Idempotent both ways (SADD/SREM), and the resulting state comes back so
91
+ # a client does not need a second request to confirm the toggle.
92
+ def toggle(request, paused:)
93
+ queue = ::Wurk::Queue.new(Validation.queue_name!(request.path_params[:name]))
94
+ paused ? queue.pause! : queue.unpause!
95
+ Response.json(200, name: queue.name, paused: paused)
96
+ end
97
+
98
+ # `total` is read before the page is walked, so it describes the set the
99
+ # page was taken from rather than the one left after a poller drained it.
100
+ def sorted_set(request, set)
101
+ window = Page.window!(request)
102
+ total = set.size
103
+ jobs = Page.slice(set, window) { |entry| Serializers.sorted_entry(entry) }
104
+ Response.json(200, name: set.name, total: total, page: window.page, count: window.count, jobs: jobs)
105
+ end
106
+
107
+ def batch(request)
108
+ bid = Validation.bid!(request.path_params[:bid])
109
+ status = ::Wurk::Batch::Status.new(bid)
110
+ return batch_not_found(request, bid) unless status.exists?
111
+
112
+ Response.json(200, status.data)
113
+ end
114
+
115
+ def batch_not_found(request, bid)
116
+ Problem.render(
117
+ Problem::BATCH_NOT_FOUND,
118
+ status: 404,
119
+ detail: "No batch has bid #{bid}; it was never created, or it has expired.",
120
+ instance: request.path,
121
+ bid: bid
122
+ )
123
+ end
124
+ end
125
+ end
126
+ end
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../api'
4
+ require_relative 'problem'
5
+
6
+ module Wurk
7
+ module API
8
+ # Whether this deployment answers anything but reads.
9
+ #
10
+ # The dashboard has had a read-only mode since the beginning
11
+ # (`Wurk::Web.config.read_only`, `WURK_WEB_READ_ONLY=1`): a viewer-only
12
+ # deploy — the public demo is one — where every non-safe verb 403s. The
13
+ # machine plane keeps exactly that rule, verb for verb, so "read-only"
14
+ # means one thing across Wurk instead of two. That freezes `admin`'s
15
+ # destructive routes and `enqueue` alike: a deployment an operator may not
16
+ # retry a job on is not one a stranger should be able to start a thousand
17
+ # on, and the alternative is a word whose meaning depends on which plane
18
+ # you are standing in.
19
+ #
20
+ # Only the *mount* differs, and it has to — see `Configuration#api_read_only`
21
+ # for the three states. Nested in the engine, this API is part of the
22
+ # dashboard's deployment and inherits its flag through the Rack env
23
+ # (`Wurk::Web::Authorization` stamps {Wurk::API::READ_ONLY_ENV}). Mounted on
24
+ # its own path or run standalone it is a different deployment, has no engine
25
+ # in front of it to stamp anything, and says so itself or not at all.
26
+ module ReadOnly
27
+ # RFC 9110 §9.2.1 — the verbs with no side effects the client asked for.
28
+ # The same three the dashboard's Authorization middleware allows; the API
29
+ # registers no OPTIONS route, but a read-only deploy refusing a preflight
30
+ # would be a lie about why.
31
+ SAFE_METHODS = %w[GET HEAD OPTIONS].freeze
32
+
33
+ module_function
34
+
35
+ # An explicit setting wins both ways — `false` is how a host keeps its
36
+ # producer live under a read-only dashboard, so it has to outrank what the
37
+ # engine stamped, not merely be OR'd into it.
38
+ #
39
+ # @return [Boolean] whether writes are frozen for this request's mount.
40
+ def enabled?(request)
41
+ explicit = request.config.api_read_only
42
+ return explicit unless explicit.nil?
43
+
44
+ inherited?(request) || request.config.api_read_only?
45
+ end
46
+
47
+ # @return [Array, nil] a 403 problem triple, or nil to let the request on.
48
+ def refuse(request)
49
+ return nil if SAFE_METHODS.include?(request.request_method)
50
+ return nil unless enabled?(request)
51
+
52
+ Problem.render(
53
+ Problem::READ_ONLY,
54
+ status: 403,
55
+ detail: 'This Wurk deployment is read-only; it answers GET and HEAD only.',
56
+ instance: request.path
57
+ )
58
+ end
59
+
60
+ # Checked before routing, so a write to a path that does not exist is
61
+ # refused for the reason that applies to every write here rather than
62
+ # 404ing and leaving a client to discover the freeze one route later.
63
+ def inherited?(request)
64
+ !!request.get_header(::Wurk::API::READ_ONLY_ENV)
65
+ end
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rack'
4
+
5
+ module Wurk
6
+ module API
7
+ # Rack::Request plus the three things every API handler needs: the segment
8
+ # captures the router bound ('/jobs/:jid' → `path_params[:jid]`), the
9
+ # authenticated principal, and mount-agnostic URL building.
10
+ class Request < ::Rack::Request
11
+ # Set by App once Auth has accepted the credential, so a handler can ask
12
+ # what the caller was granted without re-reading the header. Never nil by
13
+ # the time a handler runs — an unauthenticated request never reaches one.
14
+ attr_accessor :principal
15
+
16
+ # The configuration this app is answering from, stamped by App before
17
+ # anything reads it. Injectable there for tests, so a handler that wants
18
+ # a boundary cap or a token's grants must ask the request rather than
19
+ # reaching for the process-wide singleton behind the app's back.
20
+ attr_accessor :config
21
+
22
+ attr_writer :path_params
23
+
24
+ def path_params
25
+ @path_params ||= {}
26
+ end
27
+
28
+ # Root-relative on purpose. The API answers under three different
29
+ # prefixes (engine-nested, separately mounted, standalone) and usually
30
+ # sits behind a proxy, so SCRIPT_NAME is the only prefix it can trust —
31
+ # rebuilding an absolute URL out of Host/X-Forwarded-* would hand clients
32
+ # links to an origin they never asked for.
33
+ def url_for(path)
34
+ "#{script_name.to_s.chomp('/')}#{path}"
35
+ end
36
+
37
+ # At most `limit` bytes off the request body, so a caller can refuse an
38
+ # oversized one without ever holding it. Rack 3.1 made `rack.input`
39
+ # optional and a stream already at EOF answers nil, so both come back as
40
+ # an empty string: a bodyless request is a request, not a crash.
41
+ def read_body(limit)
42
+ body&.read(limit).to_s
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'json'
4
+
5
+ module Wurk
6
+ module API
7
+ # Success bodies for the HTTP API; Problem is the error half. One owner per
8
+ # content type the API emits, so a route table never spells out a header for
9
+ # itself and the two halves can't drift on the ones they share.
10
+ module Response
11
+ CONTENT_TYPE = 'application/json'
12
+
13
+ # nosniff for Problem's reason: a body that echoes what the client sent
14
+ # must never be talked into rendering as anything but data.
15
+ HEADERS = { 'content-type' => CONTENT_TYPE, 'x-content-type-options' => 'nosniff' }.freeze
16
+
17
+ module_function
18
+
19
+ # Dups the headers because a Rack middleware downstream is entitled to
20
+ # add to them, and the frozen literal is shared by every response.
21
+ #
22
+ # @return [Array(Integer, Hash, Array<String>)] Rack response triple.
23
+ def json(status, payload)
24
+ [status, HEADERS.dup, [::JSON.generate(payload)]]
25
+ end
26
+ end
27
+ end
28
+ end