wurk 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +38 -2
- data/app/controllers/wurk/api/serializers.rb +51 -1
- data/app/controllers/wurk/api_controller.rb +42 -6
- data/app/controllers/wurk/dashboard_controller.rb +66 -10
- data/config/routes.rb +30 -3
- data/exe/wurk +6 -2
- data/lib/generators/wurk/install/install_generator.rb +3 -3
- data/lib/sidekiq/job_retry.rb +4 -0
- data/lib/sidekiq/manager.rb +4 -0
- data/lib/sidekiq/processor.rb +4 -0
- data/lib/wurk/api/app.rb +186 -0
- data/lib/wurk/api/auth.rb +167 -0
- data/lib/wurk/api/flows.rb +66 -0
- data/lib/wurk/api/idempotency.rb +168 -0
- data/lib/wurk/api/jobs.rb +180 -0
- data/lib/wurk/api/page.rb +98 -0
- data/lib/wurk/api/problem.rb +121 -0
- data/lib/wurk/api/queues.rb +126 -0
- data/lib/wurk/api/read_only.rb +68 -0
- data/lib/wurk/api/request.rb +46 -0
- data/lib/wurk/api/response.rb +28 -0
- data/lib/wurk/api/roll_up.rb +126 -0
- data/lib/wurk/api/router.rb +88 -0
- data/lib/wurk/api/serializers.rb +232 -0
- data/lib/wurk/api/swarm.rb +297 -0
- data/lib/wurk/api/throttle.rb +109 -0
- data/lib/wurk/api/validation.rb +306 -0
- data/lib/wurk/api.rb +96 -0
- data/lib/wurk/batch/server_middleware.rb +2 -2
- data/lib/wurk/batch.rb +12 -3
- data/lib/wurk/capsule.rb +8 -1
- data/lib/wurk/cli.rb +99 -4
- data/lib/wurk/client/buffered.rb +1 -1
- data/lib/wurk/client.rb +42 -3
- data/lib/wurk/collapse.rb +383 -0
- data/lib/wurk/compat.rb +19 -0
- data/lib/wurk/component.rb +14 -2
- data/lib/wurk/configuration.rb +303 -4
- data/lib/wurk/debounce.rb +125 -0
- data/lib/wurk/encryption.rb +6 -1
- data/lib/wurk/fetcher/capped.rb +218 -0
- data/lib/wurk/fetcher/reliable.rb +20 -65
- data/lib/wurk/fetcher/unit_of_work.rb +100 -0
- data/lib/wurk/flow/builder.rb +271 -0
- data/lib/wurk/flow/chain.rb +42 -0
- data/lib/wurk/flow/completion.rb +114 -0
- data/lib/wurk/flow/creation.rb +254 -0
- data/lib/wurk/flow/node.rb +124 -0
- data/lib/wurk/flow/status.rb +206 -0
- data/lib/wurk/flow.rb +255 -0
- data/lib/wurk/flow_set.rb +53 -0
- data/lib/wurk/health.rb +8 -7
- data/lib/wurk/heartbeat.rb +21 -3
- data/lib/wurk/job/options.rb +11 -1
- data/lib/wurk/job.rb +19 -0
- data/lib/wurk/job_retry.rb +6 -1
- data/lib/wurk/job_util.rb +110 -9
- data/lib/wurk/keys.rb +125 -0
- data/lib/wurk/launcher.rb +1 -1
- data/lib/wurk/leader.rb +1 -1
- data/lib/wurk/limiter/bucket.rb +1 -1
- data/lib/wurk/limiter/points.rb +1 -1
- data/lib/wurk/lua/debounce.lua +75 -0
- data/lib/wurk/lua/fetch_slot.lua +83 -0
- data/lib/wurk/lua/flow_abandon.lua +65 -0
- data/lib/wurk/lua/flow_advance.lua +177 -0
- data/lib/wurk/lua/flow_create.lua +141 -0
- data/lib/wurk/lua/flow_fail.lua +52 -0
- data/lib/wurk/lua/limiter_bucket_acquire.lua +26 -0
- data/lib/wurk/lua/limiter_concurrent_acquire.lua +33 -0
- data/lib/wurk/lua/limiter_concurrent_release.lua +7 -0
- data/lib/wurk/lua/limiter_leaky_acquire.lua +31 -0
- data/lib/wurk/lua/limiter_list_sweep.lua +30 -0
- data/lib/wurk/lua/limiter_points_acquire.lua +34 -0
- data/lib/wurk/lua/limiter_points_refund.lua +18 -0
- data/lib/wurk/lua/limiter_register.lua +27 -0
- data/lib/wurk/lua/limiter_window_acquire.lua +35 -0
- data/lib/wurk/lua/limiter_window_status.lua +22 -0
- data/lib/wurk/lua/loader.rb +23 -0
- data/lib/wurk/lua/queue_slot.lua +83 -0
- data/lib/wurk/lua/refresh_slots.lua +38 -0
- data/lib/wurk/lua/status_write.lua +29 -0
- data/lib/wurk/lua/throttle_slot.lua +71 -0
- data/lib/wurk/lua.rb +3 -3
- data/lib/wurk/manager.rb +12 -0
- data/lib/wurk/metrics/history.rb +34 -2
- data/lib/wurk/metrics/query.rb +1 -1
- data/lib/wurk/metrics/statsd.rb +14 -0
- data/lib/wurk/middleware/expiry.rb +71 -10
- data/lib/wurk/middleware/status.rb +274 -0
- data/lib/wurk/middleware/timeout.rb +137 -0
- data/lib/wurk/processor.rb +66 -22
- data/lib/wurk/profiler.rb +0 -2
- data/lib/wurk/queue_slot.rb +285 -0
- data/lib/wurk/rails.rb +3 -3
- data/lib/wurk/status/progress.rb +103 -0
- data/lib/wurk/status/record.rb +81 -0
- data/lib/wurk/status.rb +155 -0
- data/lib/wurk/telemetry/client_middleware.rb +59 -0
- data/lib/wurk/telemetry/server_middleware.rb +152 -0
- data/lib/wurk/telemetry.rb +137 -0
- data/lib/wurk/throttle.rb +142 -0
- data/lib/wurk/unique.rb +10 -2
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/watchdog.rb +188 -0
- data/lib/wurk/web/config.rb +77 -10
- data/lib/wurk/web/extension.rb +2 -2
- data/lib/wurk/web/locale_negotiator.rb +67 -0
- data/lib/wurk/web.rb +1 -0
- data/lib/wurk/worker.rb +39 -4
- data/lib/wurk.rb +40 -1
- data/vendor/assets/dashboard/assets/ArgsValue-D-x_ifLY.js +1 -0
- data/vendor/assets/dashboard/assets/BatchDetail-C39NJuew.js +1 -0
- data/vendor/assets/dashboard/assets/Batches-CSwo7Asa.js +1 -0
- data/vendor/assets/dashboard/assets/Busy-BOFMu-sq.js +1 -0
- data/vendor/assets/dashboard/assets/Cron-Dy8RQzDI.js +1 -0
- data/vendor/assets/dashboard/assets/Dashboard-BuTHI-O1.js +1 -0
- data/vendor/assets/dashboard/assets/Dead-B9KRvQ0N.js +1 -0
- data/vendor/assets/dashboard/assets/Extension-BnBVHfux.js +1 -0
- data/vendor/assets/dashboard/assets/FilterBox-DC24zite.js +1 -0
- data/vendor/assets/dashboard/assets/FlowDetail-DyLuzUvt.js +1 -0
- data/vendor/assets/dashboard/assets/FlowState-DAPKUahm.js +1 -0
- data/vendor/assets/dashboard/assets/Flows-Fr3rjZM_.js +1 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-N6kiJXq3.js +2 -0
- data/vendor/assets/dashboard/assets/Limiters-kbFA7uS1.js +1 -0
- data/vendor/assets/dashboard/assets/Metrics-Dj2uoZ3o.js +1 -0
- data/vendor/assets/dashboard/assets/PageHeader-B_F94azl.js +1 -0
- data/vendor/assets/dashboard/assets/Profiles-D_DjEezN.js +1 -0
- data/vendor/assets/dashboard/assets/Queues-CO4V9hAz.js +1 -0
- data/vendor/assets/dashboard/assets/Retries-DCWnzeLa.js +1 -0
- data/vendor/assets/dashboard/assets/Scheduled-BebDUjLU.js +1 -0
- data/vendor/assets/dashboard/assets/Search-Cvr5fy4Y.js +1 -0
- data/vendor/assets/dashboard/assets/Skeleton-Bu3Ke6rV.js +1 -0
- data/vendor/assets/dashboard/assets/charts-BCs9bQKz.js +1 -0
- data/vendor/assets/dashboard/assets/index-BIwyOC5Q.js +141 -0
- data/vendor/assets/dashboard/assets/index-DBQN6Jk8.css +1 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Bzh-BJyL.js +1 -0
- data/vendor/assets/dashboard/assets/useSort-COA3fVJ5.js +1 -0
- data/vendor/assets/dashboard/assets/utils-BIrvZ1hi.js +1 -0
- data/vendor/assets/dashboard/index.html +42 -11
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +100 -33
- data/vendor/assets/dashboard/assets/ArgsValue-CcR2ya6e.js +0 -1
- data/vendor/assets/dashboard/assets/BatchDetail-CUXJUQ3Q.js +0 -1
- data/vendor/assets/dashboard/assets/Batches-Cxan6Ngw.js +0 -1
- data/vendor/assets/dashboard/assets/Busy-DC5EGM0g.js +0 -1
- data/vendor/assets/dashboard/assets/Cron-Dlt8tXJA.js +0 -1
- data/vendor/assets/dashboard/assets/Dashboard-DNLu_WCg.js +0 -1
- data/vendor/assets/dashboard/assets/Dead-dZ7VGlKS.js +0 -1
- data/vendor/assets/dashboard/assets/Extension-DaFpEIJf.js +0 -1
- data/vendor/assets/dashboard/assets/FilterBox-CO3aYWIq.js +0 -1
- data/vendor/assets/dashboard/assets/JobDetailModal-DSWbT6G0.js +0 -2
- data/vendor/assets/dashboard/assets/Limiters-Cb4PKXNR.js +0 -1
- data/vendor/assets/dashboard/assets/Metrics-CCGzgCsT.js +0 -1
- data/vendor/assets/dashboard/assets/Modal-B86q6ruL.js +0 -1
- data/vendor/assets/dashboard/assets/PageHeader-fPrCcp_-.js +0 -1
- data/vendor/assets/dashboard/assets/Profiles-BnS82nR_.js +0 -1
- data/vendor/assets/dashboard/assets/Queues-CIyPevOy.js +0 -1
- data/vendor/assets/dashboard/assets/Retries-DopwXkXl.js +0 -1
- data/vendor/assets/dashboard/assets/Scheduled-1-Z7i1zE.js +0 -1
- data/vendor/assets/dashboard/assets/Search-ByA6eTma.js +0 -1
- data/vendor/assets/dashboard/assets/Skeleton-bC7HfQ9r.js +0 -1
- data/vendor/assets/dashboard/assets/charts-CLLzJ7vK.js +0 -1
- data/vendor/assets/dashboard/assets/index-B1N8hQUh.js +0 -141
- data/vendor/assets/dashboard/assets/index-BdiUEDXX.css +0 -1
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-DpBjkf6_.js +0 -1
- data/vendor/assets/dashboard/assets/useSort-DvpwuNQE.js +0 -1
- data/vendor/assets/dashboard/assets/utils-DDJC7tJV.js +0 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: cbdba1bc48d5cc6029c22b78376e6b6a0b93e4f519f6c717fd3f2ce84b5900d2
|
|
4
|
+
data.tar.gz: 0d0c5e36c75c898fa05d1b448e4730391cbcf883d39402db49ac879d3168aa91
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 29c1cb70a235bffed95cfa399328be393c4f653df388eff3f06a36e605f7f941ec21f1dc21b6fa832876f09635991c69087ff18dec9dc3a08cedb50f85945338
|
|
7
|
+
data.tar.gz: ca4aa23ed8658e6fc98cbe14501a1789cc38cc43a6a539c6a41fd3aebdcba97bb91fef58c84ae26f0d36cc42518b8279b348336efa93a3b4f0aaf37bf886c625
|
data/README.md
CHANGED
|
@@ -21,6 +21,8 @@
|
|
|
21
21
|
|
|
22
22
|
Wurk is wire-compatible with Sidekiq — same Redis keys, same job JSON, same Ruby DSL. Swap one line in your `Gemfile` and your existing jobs, batches, limiters, cron entries, and live Redis data keep working untouched. The Pro and Enterprise feature sets ship in the same free gem, with no license check and no tiers.
|
|
23
23
|
|
|
24
|
+
**On Sidekiq:** Sidekiq is the reason Ruby background processing works as well as it does. Mike Perham and Contributed Systems have maintained it for well over a decade and funded that work through Pro and Enterprise — a model that kept a critical piece of the ecosystem healthy, documented, and supported, and one the whole community has benefited from. Wurk implements Sidekiq's API because it is a genuinely good API, and exists to make a different bet about what maintenance costs now — see [Why Wurk exists](#why-wurk-exists).
|
|
25
|
+
|
|
24
26
|
**On speed:** Wurk is not currently faster than stock Sidekiq — it runs at roughly 0.87×–1.02× depending on workload shape, with parity on CPU and I/O but still behind on framework overhead (noop) and boot time. Numbers, method, and the reproduction command are in [docs/benchmarks.md](docs/benchmarks.md); run them yourself with `rake bench:vs_sidekiq`.
|
|
25
27
|
|
|
26
28
|
## Install
|
|
@@ -42,7 +44,7 @@ gem "wurk"
|
|
|
42
44
|
|
|
43
45
|
## Feature matrix
|
|
44
46
|
|
|
45
|
-
Everything below is in the one free gem. The "Sidekiq tier" column
|
|
47
|
+
Everything below is in the one free gem. The "Sidekiq tier" column maps each area onto Sidekiq's own lineup, so you can see at a glance what a migration covers.
|
|
46
48
|
|
|
47
49
|
| Area | What you get | Sidekiq tier |
|
|
48
50
|
|---|---|---|
|
|
@@ -53,7 +55,26 @@ Everything below is in the one free gem. The "Sidekiq tier" column is only there
|
|
|
53
55
|
| **Encryption** | Transparent AES-256-GCM job-argument encryption with zero-downtime key rotation | Enterprise |
|
|
54
56
|
| **Dashboard** | Mountable Rails engine, precompiled SolidJS SPA (no Node needed), live SSE, charts, host-app auth hook | OSS + Pro/Ent |
|
|
55
57
|
|
|
56
|
-
Plus Wurk extras: a worker topology DSL, a Kubernetes liveness/readiness listener,
|
|
58
|
+
Plus the [Wurk extras](#wurk-extras) below: a worker topology DSL, a Kubernetes liveness/readiness listener, OpenTelemetry tracing, job status/progress/results, an HTTP producer+observe API, per-job timeouts/deadlines, debounce/throttle/collapse, global per-queue concurrency caps, and DAG flows.
|
|
59
|
+
|
|
60
|
+
## Wurk extras
|
|
61
|
+
|
|
62
|
+
Sidekiq has no equivalent for any of these — they aren't parity, they're new surface. Each is documented as **Wurk-only**: using it ties that code to Wurk, so migrating back to plain Sidekiq means removing or reimplementing it. Everything that touches the job path is **opt-in and free when unused** — no extra Redis round trip on the hot path until you turn it on. The dashboard's theme, locale and timezone are the exception: they're active whenever the dashboard is, and cost the job path nothing either way.
|
|
63
|
+
|
|
64
|
+
| Extra | What it does | Give up if you migrate back to Sidekiq |
|
|
65
|
+
|---|---|---|
|
|
66
|
+
| **[Job status, progress & results](docs/job-status.md)** | Opt-in `sidekiq_options track: true` persists a `status:<jid>` row — state, coalesced progress writes, the return value (size-capped, withheld under encryption) | `Wurk::Status` reads/writes and the dashboard's per-job progress bar |
|
|
67
|
+
| **[HTTP producer + observe API](docs/api-http.md)** | A bearer-token-scoped `/v1` JSON API — enqueue, bulk-enqueue, inspect queues/jobs/swarm — mountable standalone, nested in the engine, or via the `wurk api` CLI | The whole `/v1` surface; non-Ruby producers lose their enqueue/inspect path |
|
|
68
|
+
| **[OpenTelemetry tracing](docs/telemetry.md)** | W3C `traceparent`/`tracestate` propagated client → server, one span per attempt, linked (not force-parented) across long delays | Distributed traces across your job graph |
|
|
69
|
+
| **[Flows — DAG-on-batches](docs/flows.md)** | `Wurk::Flow` chains and fans batches out/in with dependency edges, piped results between nodes, cycle/depth/width limits | The DAG builder, `pipe:` result-passing, `Flow.abandon` |
|
|
70
|
+
| **[Debounce, throttle-to-slot & collapse](docs/unique-jobs.md)** | `collapse: { policy: :debounce }` coalesces bursts into one job (last payload wins); `collapse: { policy: :throttle }` admits one job per fixed time slot | Burst coalescing — every enqueue in the window runs standalone again |
|
|
71
|
+
| **[Per-job timeouts & deadlines](docs/retries.md)** | `timeout:` bounds one attempt, `deadline:` bounds the whole job from enqueue, enforced by a lightweight per-capsule watchdog thread (no thread-per-job) | Runaway/stuck jobs run unbounded except for `shutdown_timeout` |
|
|
72
|
+
| **[Global per-queue concurrency caps](docs/rate-limiting.md)** | `config.global_concurrency = { critical: 20 }` caps in-flight jobs for a queue across the whole cluster, folded into the fetch pipeline | The cluster-wide cap; only per-key `Limiter`s remain |
|
|
73
|
+
| **Worker topology DSL** | Declare which queues/classes a given fleet role runs, in code instead of ad hoc `-q` flags | The declarative topology; fall back to CLI queue flags |
|
|
74
|
+
| **[Kubernetes probes](#kubernetes-probes)** | `config.health_check` opens a thin `/live`/`/ready` HTTP listener, self-electing across a swarm's children | The built-in probe listener; roll your own liveness check |
|
|
75
|
+
| **Dashboard theme, locale & timezone** | Light/dark/system theme, per-visitor locale override, and a 400-zone timezone picker for every timestamp in the SPA | Nothing server-side — this is dashboard-only |
|
|
76
|
+
|
|
77
|
+
AI dashboard panes — anomaly detection, natural-language queries, error triage, and capacity forecasting — are **planned, not shipped**: they're [roadmap M5](docs/idea/13-roadmap.md#m5--ai-dashboard), after the M4.5 extras above.
|
|
57
78
|
|
|
58
79
|
## Documentation
|
|
59
80
|
|
|
@@ -182,6 +203,21 @@ end
|
|
|
182
203
|
|
|
183
204
|
Knobs: `health_check(port:, bind: "0.0.0.0", ready_window: 30)`. In swarm mode one child owns the port; the others poll every 5s and take it over if the owner dies, so probes survive a child restart.
|
|
184
205
|
|
|
206
|
+
## Why Wurk exists
|
|
207
|
+
|
|
208
|
+
Sidekiq's split into OSS, Pro, and Enterprise is how a decade of serious maintenance got funded, and it worked. Ruby got a background-job library that stayed maintained, documented, and answerable to its users for longer than most infrastructure gems survive at all — and the API in this README is the one that came out of it. Wurk is standing on that work.
|
|
209
|
+
|
|
210
|
+
What Wurk bets on is that the economics underneath changed. Wurk is built and maintained **AI-first** — implementation, parity suite, docs, and benchmarks are written and kept current by AI agents working under human review. That is what makes it practical to:
|
|
211
|
+
|
|
212
|
+
- ship the entire Pro + Enterprise surface with no tier, no flag gate, and no license check;
|
|
213
|
+
- keep parity honest mechanically rather than by hand — Sidekiq's own tests run as an oracle suite, and third-party gems (sidekiq-cron, sidekiq-unique-jobs, sidekiq-scheduler, sidekiq-status, sidekiq-failures, sidekiq-throttled) run their upstream suites against Wurk on every push;
|
|
214
|
+
- keep adding surface Sidekiq doesn't have — the [Wurk extras](#wurk-extras) above landed as one release;
|
|
215
|
+
- sustain that over the long run, because the marginal cost of a fix, a doc update, or a version bump is no longer somebody's week.
|
|
216
|
+
|
|
217
|
+
It also means Wurk holds itself to published numbers instead of adjectives: the benchmark suite runs against stock Sidekiq every release and the results ship [as measured](docs/benchmarks.md), including the unflattering ones.
|
|
218
|
+
|
|
219
|
+
None of this makes Wurk the right call for everyone. Sidekiq Pro and Enterprise come with a commercial support contract, a decade of production track record, and a human on the other end of an email. If that is what your risk profile needs, buy it — it is worth the money, and it is the reason the API Wurk implements exists in the first place.
|
|
220
|
+
|
|
185
221
|
## Migrating from Sidekiq
|
|
186
222
|
|
|
187
223
|
```diff
|
|
@@ -66,7 +66,7 @@ module Wurk
|
|
|
66
66
|
# `leader_identity` is the cluster's `dear-leader` value (ProcessSet#leader,
|
|
67
67
|
# memoized to one Redis GET per request) — comparing here avoids the N+1
|
|
68
68
|
# `Process#leader?` GET-per-row that a per-process lookup would cost.
|
|
69
|
-
def process_row(process, leader_identity: nil)
|
|
69
|
+
def process_row(process, leader_identity: nil) # rubocop:disable Metrics/AbcSize
|
|
70
70
|
{
|
|
71
71
|
identity: process.identity,
|
|
72
72
|
hostname: process['hostname'],
|
|
@@ -139,6 +139,56 @@ module Wurk
|
|
|
139
139
|
}
|
|
140
140
|
end
|
|
141
141
|
|
|
142
|
+
# One row of the flow listing. Header fields only: {Wurk::Flow::Status}
|
|
143
|
+
# reads its node records lazily, and a page of 25 flows that each fetched
|
|
144
|
+
# a thousand of them would be 25,000 HMGETs to render a progress bar.
|
|
145
|
+
def flow_row(status)
|
|
146
|
+
{
|
|
147
|
+
fid: status.fid,
|
|
148
|
+
state: status.state,
|
|
149
|
+
total: status.total,
|
|
150
|
+
pending: status.pending,
|
|
151
|
+
succeeded: status.succeeded_count,
|
|
152
|
+
depth: status.depth,
|
|
153
|
+
width: status.width,
|
|
154
|
+
created_at: status.created_at,
|
|
155
|
+
finished_at: status.finished_at,
|
|
156
|
+
failed_at: status.failed_at,
|
|
157
|
+
abandoned_at: status.abandoned_at
|
|
158
|
+
}
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# The graph. Nodes ride inside the same document rather than behind a
|
|
162
|
+
# second request: the SPA lays the DAG out from `depends_on`, and a
|
|
163
|
+
# header fetched separately from its edges can describe a different
|
|
164
|
+
# revision of the flow than the one being drawn.
|
|
165
|
+
def flow_detail(status)
|
|
166
|
+
flow_row(status).merge(
|
|
167
|
+
dead_nodes: status.dead_indexes,
|
|
168
|
+
nodes: status.nodes.map { |node| flow_node(node) }
|
|
169
|
+
)
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# `error` carries a broken pipe's reason (slice 11 decision 2) — the one
|
|
173
|
+
# node state whose cause is not discoverable from the job itself, because
|
|
174
|
+
# no job ever ran.
|
|
175
|
+
def flow_node(node)
|
|
176
|
+
{
|
|
177
|
+
index: node.index,
|
|
178
|
+
name: node.name,
|
|
179
|
+
klass: node.klass,
|
|
180
|
+
queue: node.queue,
|
|
181
|
+
jid: node.jid,
|
|
182
|
+
bid: node.bid,
|
|
183
|
+
state: node.state,
|
|
184
|
+
depends_on: node.dependencies,
|
|
185
|
+
dependents: node.dependents,
|
|
186
|
+
remaining: node.remaining,
|
|
187
|
+
piped: node.piped?,
|
|
188
|
+
error: node.error
|
|
189
|
+
}
|
|
190
|
+
end
|
|
191
|
+
|
|
142
192
|
def metric_row(klass, totals)
|
|
143
193
|
{ klass: klass, processed: totals[:p], failed: totals[:f], runtime_ms: totals[:ms] }
|
|
144
194
|
end
|
|
@@ -13,7 +13,7 @@ module Wurk
|
|
|
13
13
|
# objects (Stats, Queue, RetrySet, ScheduledSet, DeadSet, ProcessSet,
|
|
14
14
|
# BatchSet, Cron::LoopSet) so dashboards stay aligned with the Redis schema
|
|
15
15
|
# in `docs/target/sidekiq-{free,pro,ent}.md`.
|
|
16
|
-
class ApiController < ApplicationController
|
|
16
|
+
class ApiController < ApplicationController # rubocop:disable Metrics/ClassLength
|
|
17
17
|
include ActionController::Live
|
|
18
18
|
# The SPA is a token-less JSON client; SameOriginGuard supplies Sidekiq's
|
|
19
19
|
# same-origin CSRF defense (spec §25.1) so every mutating endpoint is
|
|
@@ -182,6 +182,31 @@ module Wurk
|
|
|
182
182
|
render json: { error: 'unknown batch' }, status: :not_found
|
|
183
183
|
end
|
|
184
184
|
|
|
185
|
+
def flows
|
|
186
|
+
set = ::Wurk::FlowSet.new
|
|
187
|
+
page = ::Wurk::Api::Pagination.window(params)
|
|
188
|
+
rows = ::Wurk::Api::Pagination.slice(set, page) { |status| ::Wurk::Api::Serializers.flow_row(status) }
|
|
189
|
+
render json: { total: set.size, page: page[:page], count: page[:count], flows: rows }
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def flow
|
|
193
|
+
status = flow_status
|
|
194
|
+
return render(json: { error: 'unknown flow' }, status: :not_found) unless status&.exists?
|
|
195
|
+
|
|
196
|
+
render json: ::Wurk::Api::Serializers.flow_detail(status)
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
# The kill switch (slice 11 decision 4). Read-only mode 403s it via the
|
|
200
|
+
# Authorization middleware, like every other non-GET here. `abandoned`
|
|
201
|
+
# comes back false for a flow that had already finished or been abandoned —
|
|
202
|
+
# neither is stuck, and neither is touched.
|
|
203
|
+
def abandon_flow
|
|
204
|
+
status = flow_status
|
|
205
|
+
return render(json: { error: 'unknown flow' }, status: :not_found) unless status&.exists?
|
|
206
|
+
|
|
207
|
+
render json: { ok: true, abandoned: ::Wurk::Flow.abandon(status.fid) }
|
|
208
|
+
end
|
|
209
|
+
|
|
185
210
|
def limiters
|
|
186
211
|
names = ::Wurk::Web::Enterprise::Limits.list(filter: params[:substr])
|
|
187
212
|
page = ::Wurk::Api::Pagination.window(params)
|
|
@@ -256,7 +281,7 @@ module Wurk
|
|
|
256
281
|
# `:bucket` is 1m/5m/1h; `?window=24h` (s/m/h/d) is clamped to the bucket's
|
|
257
282
|
# retention; optional `?queue=<name>` narrows to one queue. Each queue's
|
|
258
283
|
# `points` are chart-ready.
|
|
259
|
-
def queue_history
|
|
284
|
+
def queue_history # rubocop:disable Metrics/AbcSize
|
|
260
285
|
window = parse_window(params[:window])
|
|
261
286
|
queues = params[:queue].present? ? [params[:queue].to_s] : nil
|
|
262
287
|
series = ::Wurk::Web::Enterprise::Historical.queue_history(params[:bucket].to_s, window: window, queues: queues)
|
|
@@ -304,8 +329,6 @@ module Wurk
|
|
|
304
329
|
end
|
|
305
330
|
end
|
|
306
331
|
|
|
307
|
-
private
|
|
308
|
-
|
|
309
332
|
# Engine-relative path probed as a representative mutation. Must be a real
|
|
310
333
|
# mutating route (POST /api/retries — bulk retry/delete/kill) so that a
|
|
311
334
|
# path-sensitive hook resolves it the same way the Authorization middleware
|
|
@@ -313,6 +336,9 @@ module Wurk
|
|
|
313
336
|
# is the GET /api/meta path) would let such a hook allow the probe while
|
|
314
337
|
# still 403ing real mutations, reviving the "button shows, then 403s" gap.
|
|
315
338
|
MUTATION_PROBE_PATH = '/api/retries'
|
|
339
|
+
private_constant :MUTATION_PROBE_PATH
|
|
340
|
+
|
|
341
|
+
private
|
|
316
342
|
|
|
317
343
|
# Per-request read-only signal for the SPA. When a registered authorization
|
|
318
344
|
# hook would reject a *mutating* request for this user (e.g. a viewer role
|
|
@@ -327,6 +353,16 @@ module Wurk
|
|
|
327
353
|
config.authorized?(request.env, 'POST', MUTATION_PROBE_PATH)
|
|
328
354
|
end
|
|
329
355
|
|
|
356
|
+
# nil for an empty fid, which Flow::Status refuses rather than reading
|
|
357
|
+
# `flow:` — the key prefix itself, which is not a flow. Both callers turn
|
|
358
|
+
# that into the same 404 a fid nothing was created under gets, because from
|
|
359
|
+
# outside there is no difference worth telling apart.
|
|
360
|
+
def flow_status
|
|
361
|
+
::Wurk::Flow::Status.new(params[:fid].to_s)
|
|
362
|
+
rescue ::ArgumentError
|
|
363
|
+
nil
|
|
364
|
+
end
|
|
365
|
+
|
|
330
366
|
# Resolves a single entry by "<score>|<jid>" key and applies a whitelisted
|
|
331
367
|
# action. 400 on an unknown action, 404 when the key matches nothing (e.g.
|
|
332
368
|
# the entry was already retried/deleted from another tab).
|
|
@@ -342,7 +378,7 @@ module Wurk
|
|
|
342
378
|
end
|
|
343
379
|
|
|
344
380
|
# Bulk variant: `keys[]` + a single `cmd` applied to every resolved entry.
|
|
345
|
-
def bulk_entry_action(set, actions)
|
|
381
|
+
def bulk_entry_action(set, actions) # rubocop:disable Metrics/AbcSize
|
|
346
382
|
method = actions[params[:cmd].to_s]
|
|
347
383
|
return render(json: { error: 'unknown action' }, status: :bad_request) unless method
|
|
348
384
|
|
|
@@ -364,7 +400,7 @@ module Wurk
|
|
|
364
400
|
# live process when identity is blank/"all". Embedded processes are skipped
|
|
365
401
|
# (they raise on quiet!/stop! — there's no separate process to signal). 404s
|
|
366
402
|
# when a named identity isn't in the live set.
|
|
367
|
-
def signal_processes(method)
|
|
403
|
+
def signal_processes(method) # rubocop:disable Metrics/AbcSize
|
|
368
404
|
identity = params[:identity].to_s
|
|
369
405
|
if identity.empty? || identity == 'all'
|
|
370
406
|
count = ::Wurk::ProcessSet.new.reject(&:embedded?).each { |p| p.public_send(method) }.size
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require 'json'
|
|
4
4
|
require 'net/http'
|
|
5
5
|
require 'uri'
|
|
6
|
+
require 'wurk/web'
|
|
6
7
|
|
|
7
8
|
module Wurk
|
|
8
9
|
# Serves the SPA shell. Everything else is JSON from ApiController.
|
|
@@ -25,36 +26,91 @@ module Wurk
|
|
|
25
26
|
VITE_ASSET_BASE = "#{::Wurk::Engine::AssetMount::PREFIX}/".freeze # "/wurk-assets/"
|
|
26
27
|
VITE_DEV_URL = "#{VITE_DEV_HOST}#{VITE_ASSET_BASE}".freeze
|
|
27
28
|
INDEX_REL_PATH = ['vendor', 'assets', 'dashboard', 'index.html'].freeze
|
|
29
|
+
# The SPA mount point, matched without its closing `>` so the locale hint
|
|
30
|
+
# appends to whatever attributes the shell already carries.
|
|
31
|
+
ROOT_DIV = '<div id="wurk-root"'
|
|
32
|
+
# The theme hint's anchor. It is the one payload that has to land at the
|
|
33
|
+
# *top* of <head> rather than before </head> with the others: the shell's
|
|
34
|
+
# pre-paint script reads it, and that script runs before the stylesheet so
|
|
35
|
+
# a host default never flashes the other palette.
|
|
36
|
+
HEAD_OPEN = '<head>'
|
|
28
37
|
|
|
29
38
|
def index
|
|
30
|
-
render layout: false, html:
|
|
39
|
+
render layout: false, html: decorate_shell(spa_html).html_safe
|
|
31
40
|
end
|
|
32
41
|
|
|
33
42
|
private
|
|
34
43
|
|
|
44
|
+
def decorate_shell(html)
|
|
45
|
+
inject_theme_default(inject_locale_hint(inject_head(html)))
|
|
46
|
+
end
|
|
47
|
+
|
|
35
48
|
def spa_html
|
|
36
49
|
ENV['WURK_VITE_DEV'] == '1' ? fetch_vite_dev_shell : read_built_index
|
|
37
50
|
end
|
|
38
51
|
|
|
52
|
+
# Block form deliberately: a string replacement would interpret `\0`/`\&`
|
|
53
|
+
# backreferences inside the host-configured JSON these scripts carry.
|
|
54
|
+
def inject_head(html)
|
|
55
|
+
html.sub('</head>') { "#{mount_base_script}#{host_translations_script}</head>" }
|
|
56
|
+
end
|
|
57
|
+
|
|
39
58
|
# The SPA is mount-agnostic: it reads window.__WURK_BASE__ to build every API
|
|
40
59
|
# URL and its client-router base, so the same precompiled bundle works whether
|
|
41
60
|
# the host mounts the engine at /wurk, /sidekiq, or /admin/jobs. script_name
|
|
42
61
|
# is the mount prefix ("/sidekiq"), or "" at the app root. Injected into every
|
|
43
62
|
# served shell (built and Vite-dev) so a non-/wurk mount needs no rebuild.
|
|
44
|
-
def inject_mount_base(html)
|
|
45
|
-
html.sub('</head>', "#{mount_base_script}</head>")
|
|
46
|
-
end
|
|
47
|
-
|
|
48
63
|
def mount_base_script
|
|
49
64
|
base = request.script_name.to_s.chomp('/')
|
|
50
|
-
%(<script>window.__WURK_BASE__ = #{
|
|
65
|
+
%(<script>window.__WURK_BASE__ = #{json_literal(base)};</script>\n )
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Host copy overrides (Wurk::Web.config.translations), read once at SPA boot
|
|
69
|
+
# by loadHostOverrides() in frontend/src/i18n/index.ts and deep-merged over
|
|
70
|
+
# the shipped bundle for the locale in use.
|
|
71
|
+
def host_translations_script
|
|
72
|
+
overrides = ::Wurk::Web.config.translations
|
|
73
|
+
return '' unless overrides.is_a?(::Hash) && !overrides.empty?
|
|
74
|
+
|
|
75
|
+
%(<script type="application/json" id="wurk-i18n">#{json_literal(overrides)}</script>\n )
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# First-paint language hint, third in the SPA's precedence chain (`?locale=`
|
|
79
|
+
# and a stored pick both outrank it), so it only decides the render for a
|
|
80
|
+
# visitor who has never chosen. The negotiator returns a tag from the host's
|
|
81
|
+
# offered list or nothing — request text never reaches the attribute — and
|
|
82
|
+
# no hint at all is the right answer for an unrecognized language: the SPA
|
|
83
|
+
# then falls through to navigator.languages.
|
|
84
|
+
def inject_locale_hint(html)
|
|
85
|
+
locale = negotiated_locale
|
|
86
|
+
return html unless locale
|
|
87
|
+
|
|
88
|
+
html.sub(ROOT_DIV) { %(#{ROOT_DIV} data-locale="#{::ERB::Util.html_escape(locale)}") }
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# First-paint theme default (Wurk::Web.config.default_theme), last in the
|
|
92
|
+
# SPA's precedence chain behind a stored pick — so it only decides the
|
|
93
|
+
# palette for a visitor who has never chosen one. No config emits nothing
|
|
94
|
+
# and leaves the SPA on 'system', which follows the visitor's OS.
|
|
95
|
+
def inject_theme_default(html)
|
|
96
|
+
theme = ::Wurk::Web.config.default_theme
|
|
97
|
+
return html unless theme
|
|
98
|
+
|
|
99
|
+
script = %(<script>window.__WURK_THEME__ = #{json_literal(theme)};</script>)
|
|
100
|
+
html.sub(HEAD_OPEN) { "#{HEAD_OPEN}\n #{script}" }
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def negotiated_locale
|
|
104
|
+
config = ::Wurk::Web.config
|
|
105
|
+
header = request.get_header('HTTP_ACCEPT_LANGUAGE')
|
|
106
|
+
::Wurk::Web::LocaleNegotiator.call(header, offered: config.offered_locales) || config.default_locale
|
|
51
107
|
end
|
|
52
108
|
|
|
53
|
-
# JSON-
|
|
54
|
-
#
|
|
109
|
+
# JSON-encode a value as a literal inside <script>, escaping the sequences
|
|
110
|
+
# that could otherwise break out of the tag. Both payloads are
|
|
55
111
|
# host-configured, not request input, but the injection stays safe regardless.
|
|
56
|
-
def
|
|
57
|
-
|
|
112
|
+
def json_literal(value)
|
|
113
|
+
value.to_json.gsub(/[<>&]/) { |c| format('\u%04x', c.ord) }
|
|
58
114
|
end
|
|
59
115
|
|
|
60
116
|
def fetch_vite_dev_shell
|
data/config/routes.rb
CHANGED
|
@@ -9,7 +9,7 @@ Wurk::Engine.routes.draw do
|
|
|
9
9
|
get 'stats', to: 'api#stats'
|
|
10
10
|
get 'queues', to: 'api#queues'
|
|
11
11
|
get 'queues/:name', to: 'api#queue', as: :api_queue, constraints: { name: %r{[^/]+} }
|
|
12
|
-
post 'queues/:name/clear', to: 'api#clear_queue',
|
|
12
|
+
post 'queues/:name/clear', to: 'api#clear_queue', as: :api_clear_queue, constraints: { name: %r{[^/]+} }
|
|
13
13
|
post 'queues/:name/delete', to: 'api#delete_queue_job', as: :api_delete_queue_job, constraints: { name: %r{[^/]+} }
|
|
14
14
|
# Per-queue Pause/Unpause (Pro §6, §10.1): toggles membership of the `paused`
|
|
15
15
|
# SET that fetchers consult. Read-only mode 403s these via Authorization.
|
|
@@ -28,7 +28,7 @@ Wurk::Engine.routes.draw do
|
|
|
28
28
|
get 'retries', to: 'api#retries'
|
|
29
29
|
post 'retries', to: 'api#retries_bulk', as: :api_retries_bulk
|
|
30
30
|
post 'retries/all/:cmd', to: 'api#retries_all', as: :api_retries_all
|
|
31
|
-
post 'retries/:key', to: 'api#retry_job', as: :api_retry_job,
|
|
31
|
+
post 'retries/:key', to: 'api#retry_job', as: :api_retry_job, constraints: { key: %r{[^/]+} }
|
|
32
32
|
get 'scheduled', to: 'api#scheduled'
|
|
33
33
|
post 'scheduled', to: 'api#scheduled_bulk', as: :api_scheduled_bulk
|
|
34
34
|
post 'scheduled/all/:cmd', to: 'api#scheduled_all', as: :api_scheduled_all
|
|
@@ -45,6 +45,14 @@ Wurk::Engine.routes.draw do
|
|
|
45
45
|
# middleware. Spec: docs/target/sidekiq-free.md §25.4 (POST /busy).
|
|
46
46
|
post 'busy/quiet', to: 'api#quiet_process', as: :api_quiet_process
|
|
47
47
|
post 'busy/stop', to: 'api#stop_process', as: :api_stop_process
|
|
48
|
+
# Flows (Wurk::Flow) read the way the batches below them do — a paged index
|
|
49
|
+
# plus one document per id — because a flow *is* the parent relation between
|
|
50
|
+
# batches. `abandon` is the kill switch (slice 11 decision 4): a non-GET, so
|
|
51
|
+
# read-only mode 403s it via the Authorization middleware, no guard needed
|
|
52
|
+
# here.
|
|
53
|
+
get 'flows', to: 'api#flows'
|
|
54
|
+
get 'flows/:fid', to: 'api#flow', as: :api_flow
|
|
55
|
+
post 'flows/:fid/abandon', to: 'api#abandon_flow', as: :api_abandon_flow
|
|
48
56
|
get 'batches', to: 'api#batches'
|
|
49
57
|
get 'batches/:bid', to: 'api#batch', as: :api_batch
|
|
50
58
|
get 'limiters', to: 'api#limiters'
|
|
@@ -57,13 +65,32 @@ Wurk::Engine.routes.draw do
|
|
|
57
65
|
get 'metrics', to: 'api#metrics'
|
|
58
66
|
get 'metrics/:klass', to: 'api#metrics_for_job', as: :api_metrics_for_job, constraints: { klass: %r{[^/]+} }
|
|
59
67
|
get 'history/snapshots', to: 'api#history_snapshots', as: :api_history_snapshots
|
|
60
|
-
get 'history/:bucket',
|
|
68
|
+
get 'history/:bucket', to: 'api#history', as: :api_history
|
|
61
69
|
get 'queue-history/:bucket', to: 'api#queue_history', as: :api_queue_history
|
|
62
70
|
get 'search', to: 'api#search'
|
|
63
71
|
get 'profiles', to: 'api#profiles'
|
|
64
72
|
get 'stream', to: 'api#stream' # SSE
|
|
65
73
|
end
|
|
66
74
|
|
|
75
|
+
# The machine-facing HTTP API (lib/wurk/api/) — mount mode 1 of three. An app
|
|
76
|
+
# that already mounts the engine gets it at `<mount>/api/v1` for free once a
|
|
77
|
+
# token exists; `Wurk::API.serves?` decides per request, so with none the
|
|
78
|
+
# constraint fails, Rails falls through, and there is no surface to find. It
|
|
79
|
+
# shares the /api prefix with the dashboard's own JSON API above; the
|
|
80
|
+
# constraint — not the declaration order — is what keeps those matching. It
|
|
81
|
+
# claims every version-shaped path rather than `/v1` alone, so an unknown
|
|
82
|
+
# version reaches the app and comes back as the same `unsupported_api_version`
|
|
83
|
+
# problem document the other two mounts answer with.
|
|
84
|
+
#
|
|
85
|
+
# Nested here it also inherits the engine's middleware — the host's
|
|
86
|
+
# `Wurk::Web.use` chain and the read-only gate — so a dashboard behind Devise
|
|
87
|
+
# keeps gating this path too. A machine client that cannot pass a browser
|
|
88
|
+
# login wants mode 2 instead: `mount Wurk::API => '/wurk-api'` in the host's
|
|
89
|
+
# own routes, declared BEFORE the engine when its path extends the engine's —
|
|
90
|
+
# Rails matches a mounted app by bare string prefix, so `mount Wurk::Engine =>
|
|
91
|
+
# '/wurk'` declared first swallows every `/wurk-api/...` request.
|
|
92
|
+
mount Wurk::API => Wurk::API::ENGINE_MOUNT, constraints: ->(request) { Wurk::API.serves?(request.path_info) }
|
|
93
|
+
|
|
67
94
|
# Profiles (v8.0+) — not under /api: `:key/data` streams the gzipped gecko
|
|
68
95
|
# blob with a gzip Content-Encoding, and `:key` POST-uploads the profile to
|
|
69
96
|
# the Firefox profiler then 302s to its public view. `:key` is "<token>-<jid>".
|
data/exe/wurk
CHANGED
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
4
|
# Standalone runner. Does NOT load the Rails engine.
|
|
5
|
-
# Usage: `exe/wurk -C config/wurk.yml`
|
|
5
|
+
# Usage: `exe/wurk -C config/wurk.yml` — worker: one process, thread pool
|
|
6
|
+
# `exe/wurk api --port 7433` — machine HTTP API, runs no jobs
|
|
6
7
|
|
|
7
8
|
$stdout.sync = true
|
|
8
9
|
$LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
|
|
@@ -12,7 +13,10 @@ require 'wurk'
|
|
|
12
13
|
begin
|
|
13
14
|
cli = Wurk::CLI.instance
|
|
14
15
|
cli.parse
|
|
15
|
-
cli.
|
|
16
|
+
case cli.command
|
|
17
|
+
when 'api' then cli.run_api
|
|
18
|
+
else cli.run
|
|
19
|
+
end
|
|
16
20
|
rescue StandardError => e
|
|
17
21
|
raise e if $DEBUG
|
|
18
22
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require
|
|
3
|
+
require 'rails/generators'
|
|
4
4
|
|
|
5
5
|
module Wurk
|
|
6
6
|
module Generators
|
|
@@ -8,10 +8,10 @@ module Wurk
|
|
|
8
8
|
# Writes a config initializer and adds a commented-out mount line to routes.
|
|
9
9
|
# The host app picks the mount path; the generator suggests /wurk.
|
|
10
10
|
class InstallGenerator < ::Rails::Generators::Base
|
|
11
|
-
source_root File.expand_path(
|
|
11
|
+
source_root File.expand_path('templates', __dir__)
|
|
12
12
|
|
|
13
13
|
def copy_initializer
|
|
14
|
-
template
|
|
14
|
+
template 'wurk.rb', 'config/initializers/wurk.rb'
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
def insert_mount_line
|