wurk 1.9.0 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +9 -9
- data/app/controllers/wurk/api_controller.rb +4 -1
- data/app/controllers/wurk/dashboard_controller.rb +11 -6
- data/exe/wurk +7 -2
- data/exe/wurkswarm +1 -2
- data/lib/generators/wurk/install/install_generator.rb +12 -0
- data/lib/sidekiq/web.rb +5 -0
- data/lib/wurk/boot_checks.rb +93 -0
- data/lib/wurk/capsule.rb +33 -8
- data/lib/wurk/cli.rb +204 -30
- data/lib/wurk/client/buffered.rb +240 -26
- data/lib/wurk/client.rb +16 -5
- data/lib/wurk/compat.rb +15 -0
- data/lib/wurk/configuration.rb +114 -16
- data/lib/wurk/cron.rb +141 -16
- data/lib/wurk/debounce.rb +2 -2
- data/lib/wurk/embedded.rb +8 -2
- data/lib/wurk/encryption.rb +20 -5
- data/lib/wurk/error_coalescer.rb +35 -0
- data/lib/wurk/fetcher/at_maxmemory.rb +79 -0
- data/lib/wurk/fetcher/capped.rb +4 -0
- data/lib/wurk/fetcher/many_queues.rb +17 -1
- data/lib/wurk/fetcher/oom_drain.rb +102 -0
- data/lib/wurk/fetcher/orphan_grace.rb +60 -44
- data/lib/wurk/fetcher/private_list_key.rb +1 -16
- data/lib/wurk/fetcher/reaper.rb +169 -127
- data/lib/wurk/fetcher/reliable.rb +26 -4
- data/lib/wurk/fetcher/self_reconcile.rb +175 -0
- data/lib/wurk/fetcher/unit_of_work.rb +32 -8
- data/lib/wurk/health.rb +23 -4
- data/lib/wurk/health_probe.rb +64 -0
- data/lib/wurk/heartbeat.rb +55 -3
- data/lib/wurk/job/options.rb +9 -1
- data/lib/wurk/job.rb +2 -24
- data/lib/wurk/job_record.rb +17 -9
- data/lib/wurk/job_retry.rb +38 -10
- data/lib/wurk/job_set.rb +13 -5
- data/lib/wurk/launcher.rb +134 -52
- data/lib/wurk/leader.rb +71 -11
- data/lib/wurk/limiter/base.rb +17 -1
- data/lib/wurk/lua/debounce.lua +11 -6
- data/lib/wurk/lua/loader.rb +13 -1
- data/lib/wurk/lua.rb +71 -8
- data/lib/wurk/manager.rb +1 -1
- data/lib/wurk/metrics/process_counters.rb +27 -0
- data/lib/wurk/metrics/prometheus.rb +35 -14
- data/lib/wurk/middleware/current_attributes.rb +26 -10
- data/lib/wurk/middleware/poison_pill.rb +1 -2
- data/lib/wurk/preflight/checks.rb +203 -0
- data/lib/wurk/preflight/command.rb +135 -0
- data/lib/wurk/preflight/keyspace.rb +109 -0
- data/lib/wurk/preflight/owners.rb +74 -0
- data/lib/wurk/preflight/payload_sample.rb +108 -0
- data/lib/wurk/preflight/reader.rb +60 -0
- data/lib/wurk/preflight/report.rb +82 -0
- data/lib/wurk/preflight/verify.rb +161 -0
- data/lib/wurk/preflight.rb +40 -0
- data/lib/wurk/process_set.rb +3 -1
- data/lib/wurk/processor.rb +82 -18
- data/lib/wurk/queue_consumers.rb +134 -0
- data/lib/wurk/rails_boot.rb +40 -2
- data/lib/wurk/rails_reloader.rb +51 -0
- data/lib/wurk/railtie.rb +20 -1
- data/lib/wurk/redact.rb +12 -0
- data/lib/wurk/redis_pool.rb +34 -9
- data/lib/wurk/scheduled.rb +142 -67
- data/lib/wurk/sentry/death_handler.rb +29 -0
- data/lib/wurk/sentry/error_handler.rb +52 -10
- data/lib/wurk/sentry/middleware.rb +8 -1
- data/lib/wurk/sentry/retry_policy.rb +12 -3
- data/lib/wurk/sentry.rb +42 -5
- data/lib/wurk/sorted_entry.rb +38 -13
- data/lib/wurk/swarm/child_boot.rb +44 -31
- data/lib/wurk/swarm/crash_watch.rb +65 -0
- data/lib/wurk/swarm/test_mode_guard.rb +32 -0
- data/lib/wurk/swarm.rb +79 -11
- data/lib/wurk/testing.rb +74 -0
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/config.rb +8 -4
- data/lib/wurk/web/extension.rb +41 -62
- data/lib/wurk/web/helpers.rb +133 -0
- data/lib/wurk/web/paginator.rb +49 -0
- data/lib/wurk/web/rack_app.rb +30 -6
- data/lib/wurk/work_set.rb +6 -0
- data/lib/wurk/worker/setter.rb +11 -6
- data/lib/wurk/worker.rb +17 -1
- data/lib/wurk.rb +1 -1
- data/vendor/assets/dashboard/assets/{ArgsValue-DDqX6XWH.js → ArgsValue-CmaXArv8.js} +1 -1
- data/vendor/assets/dashboard/assets/{BatchDetail-lIytcn2C.js → BatchDetail-nc5yePZW.js} +1 -1
- data/vendor/assets/dashboard/assets/{Batches-BEumt-1O.js → Batches-DgEcyOZ_.js} +1 -1
- data/vendor/assets/dashboard/assets/{Busy-Vk9HmZcN.js → Busy-CUZncvCy.js} +1 -1
- data/vendor/assets/dashboard/assets/{Cron-CWcM-Cfr.js → Cron-CIZbaaZs.js} +1 -1
- data/vendor/assets/dashboard/assets/{Dashboard-Bgv1kusv.js → Dashboard-BWmfAook.js} +1 -1
- data/vendor/assets/dashboard/assets/{Dead-Di5Rct2H.js → Dead-CGKsTZHq.js} +1 -1
- data/vendor/assets/dashboard/assets/{Extension-LZY5-imc.js → Extension-Doy4bJoA.js} +1 -1
- data/vendor/assets/dashboard/assets/{FilterBox-BDo-7IAy.js → FilterBox-C4bO26_d.js} +1 -1
- data/vendor/assets/dashboard/assets/{FlowDetail-CehwRLTt.js → FlowDetail-Dh3SJdFh.js} +1 -1
- data/vendor/assets/dashboard/assets/{FlowState-glK2Tnf8.js → FlowState-yb7virqR.js} +1 -1
- data/vendor/assets/dashboard/assets/{Flows-cgy-Fsbp.js → Flows-C5SMRuhq.js} +1 -1
- data/vendor/assets/dashboard/assets/{JobDetailModal-Cuk3jruU.js → JobDetailModal-Cg6sAsJB.js} +1 -1
- data/vendor/assets/dashboard/assets/{Limiters-Ci8G2yZG.js → Limiters-BIxwDOJM.js} +1 -1
- data/vendor/assets/dashboard/assets/{Metrics-CQFUeZTR.js → Metrics-Qp9q0nF8.js} +1 -1
- data/vendor/assets/dashboard/assets/{NotFound-CYjhuDsI.js → NotFound-DwjPxFjU.js} +1 -1
- data/vendor/assets/dashboard/assets/{PageHeader-DjIAFTyX.js → PageHeader-vWeBbSPF.js} +1 -1
- data/vendor/assets/dashboard/assets/{Profiles-DTowqIRg.js → Profiles-7dabYVe3.js} +1 -1
- data/vendor/assets/dashboard/assets/Queues-CMg-gyP8.js +1 -0
- data/vendor/assets/dashboard/assets/{Retries-SJ3BaKC0.js → Retries-CNX8ZtTH.js} +1 -1
- data/vendor/assets/dashboard/assets/{Scheduled-DxSRpmFZ.js → Scheduled-ClVBj0nc.js} +1 -1
- data/vendor/assets/dashboard/assets/{Search-bBbi5JNK.js → Search-BAsVE2qw.js} +1 -1
- data/vendor/assets/dashboard/assets/{charts-Bg2YBcaD.js → charts-DZs138BV.js} +1 -1
- data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +1 -0
- data/vendor/assets/dashboard/assets/index--xvvNlYx.css +1 -0
- data/vendor/assets/dashboard/assets/{index-BOTVSP9M.js → index-CMzPFxNE.js} +3 -3
- data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-BRPNyV2M.js → useResetPageOnEmpty-DxkLZCIm.js} +1 -1
- data/vendor/assets/dashboard/assets/{useSort-B-BLJik7.js → useSort-jBYVwXvg.js} +1 -1
- data/vendor/assets/dashboard/assets/{utils-CGyrKpTh.js → utils-B0kQFNE8.js} +1 -1
- data/vendor/assets/dashboard/index.html +3 -3
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +53 -30
- data/vendor/assets/dashboard/assets/Queues-DUGRPjME.js +0 -1
- data/vendor/assets/dashboard/assets/i18n-BZClP62Q.js +0 -1
- data/vendor/assets/dashboard/assets/index-Rewk4eSE.css +0 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d9594aab6027f299c217e81215d9d10d5764a5f5533b3e40cde2cd1feaae8a86
|
|
4
|
+
data.tar.gz: 26161ff0d0ea58e60acbabd550e212ca354824f31434f0d085fabbc9401b8490
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3f2c0b2115f2c7132ab9104e7850f85cf34ac694d4d67b89f615cac765112582506197ae139753489de05f7e6217abb7770129aefbd3125e6dd2be17c04a50ac
|
|
7
|
+
data.tar.gz: 0f08e99addbeeebefc503916d3a033aa88dd9ccc3ad1f557ce4aabfd43e7809b07392589e0137deff11b358304b435ed798766d89f3d3f205c3a105484a65875
|
data/README.md
CHANGED
|
@@ -22,11 +22,11 @@
|
|
|
22
22
|
|
|
23
23
|
Wurk is wire-compatible with Sidekiq — same Redis keys, same job JSON, same Ruby DSL. Swap one line in your `Gemfile` and your existing jobs, batches, limiters, cron entries, and live Redis data keep working untouched. The Pro and Enterprise feature sets ship in the same free gem, with no license check and no tiers.
|
|
24
24
|
|
|
25
|
-
**In production:** Wurk runs
|
|
25
|
+
**In production:** Wurk runs background work at partner deployments, across many servers, on the fork-based swarm described below. It is not a preview. Throughput numbers from those deployments are not published yet; what has been measured is in [docs/benchmarks.md](docs/benchmarks.md).
|
|
26
26
|
|
|
27
27
|
**At scale:** Wurk is built for fleets, not just for one box. Kubernetes `/live` + `/ready` probes are a config line, not a sidecar; a bearer-scoped [HTTP API](docs/api-http.md) lets non-Ruby services enqueue and inspect; [OpenTelemetry](docs/telemetry.md) traces propagate client → server; per-queue [global concurrency caps](docs/rate-limiting.md) hold cluster-wide limits; and monitoring is the dashboard you already mount — live SSE, charts, no separate stack to run. See [Wurk extras](#wurk-extras).
|
|
28
28
|
|
|
29
|
-
**AI-managed:** Wurk is a 100% AI-managed project. Every author's issues and pull requests, org members' included, and dependency updates, are triaged, reviewed, merged or closed by the [developerz.ai](https://developerz.ai) maintainer agent, which asks a human only for security reports
|
|
29
|
+
**AI-managed:** Wurk is a 100% AI-managed project. Every author's issues and pull requests, org members' included, and dependency updates, are triaged, reviewed, merged or closed by the [developerz.ai](https://developerz.ai) maintainer agent, which asks a human only for security reports, license questions and breaking changes, and judges weekly whether a release is due. New features are still built by us, humans and AI together — see [CONTRIBUTING.md](CONTRIBUTING.md) and [AI-maintained](#ai-maintained). The agent always identifies itself as a bot: it works publicly as `developerz-ai[bot]` and marks every comment it posts as bot-made. Its work and its rules are public too — browse the [repository](https://github.com/developerz-ai/wurk) and read the committed policy in [.dz/maintainer/maintainer.yml](.dz/maintainer/maintainer.yml).
|
|
30
30
|
|
|
31
31
|
**On Sidekiq:** Wurk implements Sidekiq's API because it is a genuinely good API. Sidekiq is human-maintained and funds that work through its paid tiers; Wurk is AI-maintained, which is what lets the same surface be free software. Wurk is independent and not affiliated with or endorsed by Sidekiq or its maintainers — see [Why Wurk exists](#why-wurk-exists).
|
|
32
32
|
|
|
@@ -151,7 +151,7 @@ Method, per-invocation records, workload definitions, and the separate `rake ben
|
|
|
151
151
|
- **[Batches](https://github.com/developerz-ai/wurk/blob/main/docs/batches.md)** · **[Rate limiting](https://github.com/developerz-ai/wurk/blob/main/docs/rate-limiting.md)** · **[Periodic (cron) jobs](https://github.com/developerz-ai/wurk/blob/main/docs/periodic-jobs.md)** · **[Unique jobs](https://github.com/developerz-ai/wurk/blob/main/docs/unique-jobs.md)** — the Pro/Ent features, free.
|
|
152
152
|
- **[Iterable jobs](https://github.com/developerz-ai/wurk/blob/main/docs/iterable-jobs.md)** · **[Middleware](https://github.com/developerz-ai/wurk/blob/main/docs/middleware.md)** · **[Encryption](https://github.com/developerz-ai/wurk/blob/main/docs/encryption.md)** · **[Profiling](https://github.com/developerz-ai/wurk/blob/main/docs/profiling.md)**
|
|
153
153
|
- **[Data API](https://github.com/developerz-ai/wurk/blob/main/docs/api.md)** · **[Metrics](https://github.com/developerz-ai/wurk/blob/main/docs/metrics.md)** — inspect queues and jobs from Ruby; job metrics, Statsd/DogStatsD, custom history.
|
|
154
|
-
- **[Sentry](https://github.com/developerz-ai/wurk/blob/main/docs/sentry.md)** — built-in error reporting (`sentry-sidekiq`
|
|
154
|
+
- **[Sentry](https://github.com/developerz-ai/wurk/blob/main/docs/sentry.md)** — built-in error reporting (`sentry-sidekiq` needs the `sidekiq` shim gem; prefer this); terminal-failure-only reports, per-job scope, no job args.
|
|
155
155
|
- **API reference (parity specs):** [Sidekiq OSS](https://github.com/developerz-ai/wurk/blob/main/docs/target/sidekiq-free.md) · [Pro](https://github.com/developerz-ai/wurk/blob/main/docs/target/sidekiq-pro.md) · [Enterprise](https://github.com/developerz-ai/wurk/blob/main/docs/target/sidekiq-ent.md) — the authoritative surface Wurk matches exactly.
|
|
156
156
|
- **[Authentication & authorization](https://github.com/developerz-ai/wurk/blob/main/docs/authentication.md)** — gate the dashboard behind Devise/Warden, Sorcery, Basic auth, or a token; role-based read/write; CSRF.
|
|
157
157
|
- **[Securing the dashboard](https://github.com/developerz-ai/wurk/blob/main/docs/dashboard.md)** · **[Metrics history](https://github.com/developerz-ai/wurk/blob/main/docs/metrics-history.md)**
|
|
@@ -167,7 +167,7 @@ Method, per-invocation records, workload definitions, and the separate `rake ben
|
|
|
167
167
|
|
|
168
168
|
The full test suite also runs as a non-required `compat` workflow job across Ruby 3.2.x and the newest Ruby on the current Rails series, the oldest supported Rails series on the newest Ruby, and Valkey 8 as a drop-in store alongside Redis 7.x.
|
|
169
169
|
|
|
170
|
-
JRuby, TruffleRuby,
|
|
170
|
+
Without `fork` (JRuby, TruffleRuby, Windows; untested in CI) only the single-process `wurk` runner and `config.wurk.embed_in_web` apply: `wurkswarm` and the railtie's swarm have no fallback.
|
|
171
171
|
|
|
172
172
|
## Running the workers
|
|
173
173
|
|
|
@@ -284,13 +284,13 @@ Infrastructure this basic should be free software. A Rails app shouldn't need a
|
|
|
284
284
|
What has made that hard is maintenance: someone has to be paid to do it. Sidekiq funds a decade of *human* maintenance through its paid tiers, which is an honest trade. Wurk makes a different one — it is maintained **by AI**: every author's issues and pull requests, the team's own included, and dependency updates, are handled end to end by a maintainer agent, and the implementation, parity suite, docs, and benchmarks are written and kept current by AI agents working with the team, which still builds the features and cuts the releases. A fix, a doc update, or a version bump is no longer somebody's week, which is what makes it practical to:
|
|
285
285
|
|
|
286
286
|
- ship the entire Pro + Enterprise surface with no tier, no flag gate, and no license check;
|
|
287
|
-
- keep parity honest mechanically rather than by hand — an independently written parity oracle suite, pinned to a documented Sidekiq revision, plus sidekiq-cron
|
|
287
|
+
- keep parity honest mechanically rather than by hand — an independently written parity oracle suite, pinned to a documented Sidekiq revision, plus the upstream suites of sidekiq-cron, sidekiq-unique-jobs, sidekiq-failures and sentry-sidekiq running against Wurk on every push ([docs/idea/14-ecosystem-compat.md](https://github.com/developerz-ai/wurk/blob/main/docs/idea/14-ecosystem-compat.md) tracks the rest);
|
|
288
288
|
- keep adding surface Sidekiq doesn't have — the [Wurk extras](#wurk-extras) above landed as one release;
|
|
289
289
|
- hold ourselves to published numbers instead of adjectives — the measured comparison against stock Sidekiq is published [as measured](docs/benchmarks.md), including the unflattering ones, and `rake bench:vs_sidekiq` reproduces it. It is a published measurement, not a per-release CI run.
|
|
290
290
|
|
|
291
291
|
DHH makes the broader version of this argument in [Let the agents democratize open source](https://world.hey.com/dhh/let-the-agents-democratize-open-source-9fd630a9): open source fought for everyone's right to change the software they run, and refusing agent-written code re-erects the gate it spent decades tearing down — "all programmers are equal, but some programmers are more equal than others." His subject is contributions; ours is maintenance, which is the same economics from the other end. The reason a licence key guards batches and cron is not that the code is precious, it's that somebody had to be paid to keep it working. Drop that cost and the tier stops being necessary.
|
|
292
292
|
|
|
293
|
-
What makes it work in practice is that the agents run inside machinery built to check them. The parity oracle above is a merge-blocking gate, not decoration;
|
|
293
|
+
What makes it work in practice is that the agents run inside machinery built to check them. The parity oracle above is a merge-blocking gate, not decoration; four gems' upstream suites run against Wurk in CI and report on PRs, but they are not required checks and never gate a merge; the `rake bench` PR regression check flags deltas past a noise-adjusted threshold as an advisory comment; and the published benchmark numbers are one-off historical measurements, reproduced on demand rather than enforced in CI. The release gate has never once let an unverified gem reach RubyGems. Agents supply the pace, the gates supply the certainty — and when a gate does catch something, the fix is to make that class of mistake structurally impossible rather than to slow the agents down. The release lane derives its own tag from `Wurk::VERSION` so the two can't drift apart; [RELEASE.md](https://github.com/developerz-ai/wurk/blob/main/RELEASE.md) walks through it.
|
|
294
294
|
|
|
295
295
|
Wurk is MIT and stays that way. If what you need is a commercial support contract and a human on the other end of an email, buying that is a perfectly good answer.
|
|
296
296
|
|
|
@@ -303,9 +303,9 @@ Wurk is MIT and stays that way. If what you need is a commercial support contrac
|
|
|
303
303
|
+ gem "wurk"
|
|
304
304
|
```
|
|
305
305
|
|
|
306
|
-
`bundle install && restart`. Wurk reads and writes the same Redis schema, so there is no data migration: Wurk picks up the queues, schedules, retries, batches and periodic loops Sidekiq left behind. Running Sidekiq and Wurk workers against one Redis *at the same time* is not tested yet, so a live production move is a **drain cutover** (stop Sidekiq's workers, confirm nothing is in flight, start Wurk), with a matching rollback; the [production cutover](https://github.com/developerz-ai/wurk/blob/main/docs/migrate-from-sidekiq.md#9-production-cutover) section has every step and command. sidekiq-cron
|
|
306
|
+
`bundle install && restart`. Wurk reads and writes the same Redis schema, so there is no data migration: Wurk picks up the queues, schedules, retries, batches and periodic loops Sidekiq left behind. Running Sidekiq and Wurk workers against one Redis *at the same time* is not tested yet, so a live production move is a **drain cutover** (stop Sidekiq's workers, confirm nothing is in flight, start Wurk), with a matching rollback; the [production cutover](https://github.com/developerz-ai/wurk/blob/main/docs/migrate-from-sidekiq.md#9-production-cutover) section has every step and command. The upstream suites of sidekiq-cron, sidekiq-unique-jobs, sidekiq-failures and sentry-sidekiq run against Wurk in the [`ecosystem` CI job](https://github.com/developerz-ai/wurk/blob/main/.github/workflows/ecosystem.yml) (see [`test/ecosystem/`](https://github.com/developerz-ai/wurk/tree/main/test/ecosystem)). sidekiq-scheduler, sidekiq-status and sidekiq-throttled are target additions tracked in [docs/idea/14-ecosystem-compat.md](https://github.com/developerz-ai/wurk/blob/main/docs/idea/14-ecosystem-compat.md).
|
|
307
307
|
|
|
308
|
-
**Keeping a `sidekiq-*` add-on gem?** Those gems declare `add_dependency "sidekiq"`, which would reinstall real Sidekiq next to Wurk. Satisfy it with Wurk's git-only [`sidekiq` shim gem](https://github.com/developerz-ai/wurk/tree/main/ecosystem/sidekiq-shim) instead: `gem "sidekiq", github: "developerz-ai/wurk", glob: "ecosystem/sidekiq-shim/*.gemspec"`. Only sidekiq-cron
|
|
308
|
+
**Keeping a `sidekiq-*` add-on gem?** Those gems declare `add_dependency "sidekiq"`, which would reinstall real Sidekiq next to Wurk. Satisfy it with Wurk's git-only [`sidekiq` shim gem](https://github.com/developerz-ai/wurk/tree/main/ecosystem/sidekiq-shim) instead: `gem "sidekiq", github: "developerz-ai/wurk", glob: "ecosystem/sidekiq-shim/*.gemspec"`. Only sidekiq-cron, sidekiq-unique-jobs, sidekiq-failures and sentry-sidekiq are exercised that way, by the `ecosystem` workflow; treat the others as untested.
|
|
309
309
|
|
|
310
310
|
Full walkthrough — config side-by-side, the Redis key/`sidekiq_options` mapping, known incompatibilities, moving sidekiq-cron schedules over (`rake wurk:import:cron`), and the production cutover and rollback playbook: **[docs/migrate-from-sidekiq.md](https://github.com/developerz-ai/wurk/blob/main/docs/migrate-from-sidekiq.md)**. On the night, keep the **[incident runbook](https://github.com/developerz-ai/wurk/blob/main/docs/runbook.md)** open.
|
|
311
311
|
|
|
@@ -332,7 +332,7 @@ TypeScript end to end, MIT, and as measured about its own claims as this file is
|
|
|
332
332
|
| Dependabot | driven to green and merged |
|
|
333
333
|
| an issue or pull request by anyone with write access (org members, collaborators) | handled like anyone's: triaged, reviewed, driven to green, merged |
|
|
334
334
|
| an outside change to CI, release or publish automation | declined: members maintain the pipeline that publishes the gem |
|
|
335
|
-
| a breaking change |
|
|
335
|
+
| a breaking change | a human decides; a change that breaks the drop-in Sidekiq contract is declined, since a revert cannot take that promise back |
|
|
336
336
|
|
|
337
337
|
Releases follow the work, not a calendar. A weekday checkup asks whether a shipping fix has merged and nobody has released it. When one has, the bot files a release request, its coding fleet opens the `Wurk::VERSION` bump PR, and merging that PR publishes the gem ([RELEASE.md](RELEASE.md)). The bot's policy is committed in [`.dz/maintainer/maintainer.yml`](.dz/maintainer/maintainer.yml).
|
|
338
338
|
|
|
@@ -62,7 +62,10 @@ module Wurk
|
|
|
62
62
|
end
|
|
63
63
|
|
|
64
64
|
def queues
|
|
65
|
-
|
|
65
|
+
consumers = ::Wurk.redis(idempotent: true) { |conn| ::Wurk::QueueConsumers.counts(conn) }
|
|
66
|
+
render json: ::Wurk::Stats.new.queue_summaries.map { |q|
|
|
67
|
+
::Wurk::Api::Serializers.queue_summary(q).merge(consumers: consumers.fetch(q.name, 0))
|
|
68
|
+
}
|
|
66
69
|
end
|
|
67
70
|
|
|
68
71
|
def queue
|
|
@@ -18,14 +18,18 @@ module Wurk
|
|
|
18
18
|
# Vite dev server (default :5173) so contributors get HMR without
|
|
19
19
|
# rebuilding the bundle on every change.
|
|
20
20
|
class DashboardController < ApplicationController
|
|
21
|
+
# bin/dev exports VITE_PORT and vite.config.ts binds it, with strictPort, so this
|
|
22
|
+
# has to read the same variable or an overridden port answers every page with a 500.
|
|
23
|
+
def self.vite_dev_url
|
|
24
|
+
"http://localhost:#{ENV.fetch('VITE_PORT', '5173')}#{VITE_ASSET_BASE}"
|
|
25
|
+
end
|
|
26
|
+
|
|
21
27
|
# Vite serves the dev shell under its `base` (vite.config.ts), which is the
|
|
22
28
|
# same path the engine mounts assets at — so derive these from
|
|
23
29
|
# AssetMount::PREFIX to keep the Ruby side in lockstep. Fetching the server
|
|
24
30
|
# root instead returns Vite's "did you mean /wurk-assets/" hint page, not the
|
|
25
31
|
# shell (issue #181).
|
|
26
|
-
VITE_DEV_HOST = 'http://localhost:5173'
|
|
27
32
|
VITE_ASSET_BASE = "#{::Wurk::Engine::AssetMount::PREFIX}/".freeze # "/wurk-assets/"
|
|
28
|
-
VITE_DEV_URL = "#{VITE_DEV_HOST}#{VITE_ASSET_BASE}".freeze
|
|
29
33
|
INDEX_REL_PATH = ['vendor', 'assets', 'dashboard', 'index.html'].freeze
|
|
30
34
|
# The SPA mount point, matched without its closing `>` so the locale hint
|
|
31
35
|
# appends to whatever attributes the shell already carries.
|
|
@@ -115,11 +119,12 @@ module Wurk
|
|
|
115
119
|
end
|
|
116
120
|
|
|
117
121
|
def fetch_vite_dev_shell
|
|
118
|
-
uri = ::URI.parse(
|
|
122
|
+
uri = ::URI.parse(self.class.vite_dev_url)
|
|
119
123
|
rewrite_dev_asset_urls(::Net::HTTP.get(uri))
|
|
120
124
|
rescue ::StandardError => e
|
|
121
|
-
raise "Wurk dashboard: cannot reach Vite dev server at #{
|
|
122
|
-
"(#{e.class}: #{e.message}). Run `bin/rake frontend:dev` from the gem root
|
|
125
|
+
raise "Wurk dashboard: cannot reach Vite dev server at #{self.class.vite_dev_url} " \
|
|
126
|
+
"(#{e.class}: #{e.message}). Run `bin/rake frontend:dev` from the gem root, or `bin/dev`, " \
|
|
127
|
+
'with the same VITE_PORT.'
|
|
123
128
|
end
|
|
124
129
|
|
|
125
130
|
# The shell's entry URLs (`src="/wurk-assets/@vite/client"`,
|
|
@@ -130,7 +135,7 @@ module Wurk
|
|
|
130
135
|
# `server.origin` covers runtime-generated asset URLs but not these entry
|
|
131
136
|
# tags, so rewrite them here.
|
|
132
137
|
def rewrite_dev_asset_urls(html)
|
|
133
|
-
html.gsub(/(["'])#{::Regexp.escape(VITE_ASSET_BASE)}/, "\\1#{
|
|
138
|
+
html.gsub(/(["'])#{::Regexp.escape(VITE_ASSET_BASE)}/, "\\1#{self.class.vite_dev_url}")
|
|
134
139
|
end
|
|
135
140
|
|
|
136
141
|
def read_built_index
|
data/exe/wurk
CHANGED
|
@@ -4,12 +4,18 @@
|
|
|
4
4
|
# Standalone runner. Does NOT load the Rails engine.
|
|
5
5
|
# Usage: `exe/wurk -C config/wurk.yml` — worker: one process, thread pool
|
|
6
6
|
# `exe/wurk api --port 7433` — machine HTTP API, runs no jobs
|
|
7
|
+
# `exe/wurk preflight` / `verify` — read-only cutover checks, run no jobs
|
|
7
8
|
|
|
8
9
|
$stdout.sync = true
|
|
9
10
|
$LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
|
|
10
11
|
|
|
11
12
|
require 'wurk'
|
|
12
13
|
|
|
14
|
+
if %w[preflight verify].include?(ARGV.first)
|
|
15
|
+
require 'wurk/preflight/command'
|
|
16
|
+
exit Wurk::Preflight::Command.run(ARGV.dup)
|
|
17
|
+
end
|
|
18
|
+
|
|
13
19
|
begin
|
|
14
20
|
cli = Wurk::CLI.instance
|
|
15
21
|
cli.parse
|
|
@@ -20,7 +26,6 @@ begin
|
|
|
20
26
|
rescue StandardError => e
|
|
21
27
|
raise e if $DEBUG
|
|
22
28
|
|
|
23
|
-
|
|
24
|
-
warn e.backtrace.join("\n")
|
|
29
|
+
Wurk::CLI.report_failure('wurk', e)
|
|
25
30
|
exit 1
|
|
26
31
|
end
|
data/exe/wurkswarm
CHANGED
|
@@ -17,6 +17,18 @@ module Wurk
|
|
|
17
17
|
def insert_mount_line
|
|
18
18
|
route(%(# mount Wurk::Engine => "/wurk" # uncomment to expose the Wurk dashboard))
|
|
19
19
|
end
|
|
20
|
+
|
|
21
|
+
def show_next_steps
|
|
22
|
+
say <<~STEPS
|
|
23
|
+
|
|
24
|
+
Next steps:
|
|
25
|
+
1. Set the queue adapter if you use Active Job: config.active_job.queue_adapter = :wurk
|
|
26
|
+
2. Start workers: bundle exec wurkswarm (or bundle exec wurk for one process)
|
|
27
|
+
3. To use the dashboard, uncomment the mount line in config/routes.rb.
|
|
28
|
+
It has no login of its own, so put it behind authentication first:
|
|
29
|
+
docs/authentication.md in the wurk gem's repository
|
|
30
|
+
STEPS
|
|
31
|
+
end
|
|
20
32
|
end
|
|
21
33
|
end
|
|
22
34
|
end
|
data/lib/sidekiq/web.rb
CHANGED
|
@@ -2,3 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
# Drop-in require path (see lib/sidekiq.rb). Wurk loads this surface whole.
|
|
4
4
|
require 'wurk'
|
|
5
|
+
|
|
6
|
+
# Extension renderer dependencies, deferred on the worker path (web/extension.rb).
|
|
7
|
+
# Loaded here for a helper called outside a request, as unique-jobs' specs do.
|
|
8
|
+
require 'cgi'
|
|
9
|
+
require 'erb'
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'keys'
|
|
4
|
+
require_relative 'queue_consumers'
|
|
5
|
+
|
|
6
|
+
module Wurk
|
|
7
|
+
# What a standalone runner says once at boot, after the app loaded. Log only,
|
|
8
|
+
# and never allowed to stop a boot: none of this changes behaviour by backend,
|
|
9
|
+
# version or peers.
|
|
10
|
+
module BootChecks
|
|
11
|
+
# Oldest version each non-Redis backend has been run against.
|
|
12
|
+
TESTED_FLOORS = { 'dragonfly' => '1.34.0', 'valkey' => '8.0.0' }.freeze
|
|
13
|
+
MAX_LISTED = 20
|
|
14
|
+
SUBCLASS = ::Module.instance_method(:<)
|
|
15
|
+
NAME = ::Module.instance_method(:name)
|
|
16
|
+
|
|
17
|
+
module_function
|
|
18
|
+
|
|
19
|
+
def run(config, info)
|
|
20
|
+
log_backend(config.logger, info)
|
|
21
|
+
warn_on_foreign_processes(config)
|
|
22
|
+
warn_on_ignored_locks(config.logger)
|
|
23
|
+
rescue StandardError => e
|
|
24
|
+
config.logger.warn { "wurk boot checks skipped: #{e.class}: #{e.message}" }
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def backend(info)
|
|
28
|
+
return ['dragonfly', info['dragonfly_version'].to_s.delete_prefix('df-v')] if info['dragonfly_version']
|
|
29
|
+
return ['valkey', info['valkey_version'].to_s] if info['valkey_version']
|
|
30
|
+
|
|
31
|
+
['redis', info['redis_version'].to_s]
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def log_backend(logger, info)
|
|
35
|
+
name, version = backend(info)
|
|
36
|
+
logger.info { "Wurk #{Wurk::VERSION} on #{name} #{version}" }
|
|
37
|
+
floor = TESTED_FLOORS[name]
|
|
38
|
+
return unless floor && ::Gem::Version.correct?(version) && ::Gem::Version.new(version) < ::Gem::Version.new(floor)
|
|
39
|
+
|
|
40
|
+
logger.warn { "#{name} #{version} is older than the oldest version Wurk is tested against (#{floor})." }
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Both runtimes share `processes` and the same queues, so a mix is safe only
|
|
44
|
+
# if it is deliberate. Wurk's heartbeat always carries `cores`; stock does not.
|
|
45
|
+
def warn_on_foreign_processes(config)
|
|
46
|
+
foreign = process_infos(config).reject { |_, info| info.nil? || ::JSON.parse(info).key?('cores') }.map(&:first)
|
|
47
|
+
return if foreign.empty?
|
|
48
|
+
|
|
49
|
+
config.logger.warn do
|
|
50
|
+
"Live processes that are not Wurk share this Redis: #{listing(foreign)}. " \
|
|
51
|
+
'They fetch from the same queues. If this is not a deliberate rolling cutover, stop them.'
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def process_infos(config)
|
|
56
|
+
config.redis(idempotent: true) do |conn|
|
|
57
|
+
ids = conn.call('SMEMBERS', Keys::PROCESSES)
|
|
58
|
+
ids.empty? ? [] : ids.zip(conn.pipelined { |pipe| ids.each { |id| pipe.call('HGET', id, 'info') } })
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Without the gem's middleware the option does nothing: the "unique" job
|
|
63
|
+
# simply enqueues twice. Only classes the app has loaded by now are seen.
|
|
64
|
+
def warn_on_ignored_locks(logger)
|
|
65
|
+
return if defined?(::SidekiqUniqueJobs)
|
|
66
|
+
|
|
67
|
+
classes = locked_job_classes
|
|
68
|
+
return if classes.empty?
|
|
69
|
+
|
|
70
|
+
logger.warn do
|
|
71
|
+
'These jobs set `lock:` but sidekiq-unique-jobs is not loaded, so the option does nothing and ' \
|
|
72
|
+
"duplicates will run: #{listing(classes)}. Keep the gem, or move them to `unique_for:`."
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# A walk over every class meets classes that redefine these with other arguments
|
|
77
|
+
# (Faker::Travel::Airport.name takes required keywords), and one such class must
|
|
78
|
+
# not cost the app this check.
|
|
79
|
+
def locked_job_classes
|
|
80
|
+
::ObjectSpace.each_object(::Class).filter_map do |klass|
|
|
81
|
+
next unless SUBCLASS.bind_call(klass, Wurk::Worker)
|
|
82
|
+
|
|
83
|
+
name = NAME.bind_call(klass)
|
|
84
|
+
name if name && klass.get_sidekiq_options.key?('lock')
|
|
85
|
+
end.sort
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def listing(names)
|
|
89
|
+
extra = names.size - MAX_LISTED
|
|
90
|
+
names.first(MAX_LISTED).join(', ') + (extra.positive? ? " and #{extra} more" : '')
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
data/lib/wurk/capsule.rb
CHANGED
|
@@ -53,6 +53,8 @@ module Wurk
|
|
|
53
53
|
# %w[high default low] → mode :strict, weights all 0
|
|
54
54
|
# %w[high,3 default,2 low,1] → mode :weighted, weights {q=>w}
|
|
55
55
|
# %w[a,1 b,1 c,1] → mode :random, weights all 1
|
|
56
|
+
# %w[a,2 b] → mode :weighted, unweighted b fetched as if 1
|
|
57
|
+
# %w[a,0 b,0] → mode :strict, 0 being the same as no weight
|
|
56
58
|
# `@queues` is expanded by weight so a uniform shuffle gives weighted
|
|
57
59
|
# fairness (e.g. ["high","high","high","default","default","low"]).
|
|
58
60
|
def queues=(val)
|
|
@@ -188,6 +190,16 @@ module Wurk
|
|
|
188
190
|
@pools.clear
|
|
189
191
|
end
|
|
190
192
|
|
|
193
|
+
# The fork-path form: closes idle sockets but keeps every pool usable, for a
|
|
194
|
+
# parent whose own threads may be mid-checkout.
|
|
195
|
+
def reload_redis_pools!
|
|
196
|
+
@pools.each_value(&:reload!)
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def redis_pools_built?
|
|
200
|
+
!@pools.empty?
|
|
201
|
+
end
|
|
202
|
+
|
|
191
203
|
def redis(idempotent: false, &)
|
|
192
204
|
PoolCheckout.with(redis_pool, idempotent, &)
|
|
193
205
|
end
|
|
@@ -229,19 +241,32 @@ module Wurk
|
|
|
229
241
|
copy = server_middleware
|
|
230
242
|
return copy if frozen?
|
|
231
243
|
|
|
232
|
-
@config.server_middleware.entries
|
|
244
|
+
global = @config.server_middleware.entries
|
|
245
|
+
global.each_with_index do |entry, index|
|
|
233
246
|
next if @server_middleware_seen.include?(entry.klass)
|
|
234
247
|
|
|
235
248
|
@server_middleware_seen << entry.klass
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
rebound = entry.dup
|
|
239
|
-
rebound.config = self
|
|
240
|
-
copy.entries << rebound
|
|
249
|
+
adopt_server_middleware(copy, entry, global.drop(index + 1)) unless copy.exists?(entry.klass)
|
|
241
250
|
end
|
|
242
251
|
copy
|
|
243
252
|
end
|
|
244
253
|
|
|
254
|
+
def adopt_server_middleware(copy, entry, followers)
|
|
255
|
+
rebound = entry.dup
|
|
256
|
+
rebound.config = self
|
|
257
|
+
copy.entries.insert(refresh_position(copy, followers), rebound)
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
# Ahead of the first entry that follows it in the global chain, so a late
|
|
261
|
+
# `prepend` still wraps what it was prepended to. -1 appends.
|
|
262
|
+
def refresh_position(copy, followers)
|
|
263
|
+
followers.each do |follower|
|
|
264
|
+
position = copy.entries.index { |entry| entry.klass == follower.klass }
|
|
265
|
+
return position if position
|
|
266
|
+
end
|
|
267
|
+
-1
|
|
268
|
+
end
|
|
269
|
+
|
|
245
270
|
# Drop-in fetch pluggability (spec §5 / §4.1): instantiate
|
|
246
271
|
# `config[:fetch_class]` when the host set one — a custom or Pro fetcher —
|
|
247
272
|
# else the default reliable BLMOVE fetcher (`Sidekiq::BasicFetch`). A
|
|
@@ -276,7 +301,7 @@ module Wurk
|
|
|
276
301
|
return [qname, 0] if weight.nil?
|
|
277
302
|
|
|
278
303
|
weight = Integer(weight)
|
|
279
|
-
raise ArgumentError, "queue weight must be
|
|
304
|
+
raise ArgumentError, "queue weight must not be negative: `#{entry}`" if weight.negative?
|
|
280
305
|
|
|
281
306
|
[qname, weight]
|
|
282
307
|
end
|
|
@@ -295,7 +320,7 @@ module Wurk
|
|
|
295
320
|
def expand_by_weight(parsed, mode)
|
|
296
321
|
return parsed.map(&:first) if %i[strict random].include?(mode)
|
|
297
322
|
|
|
298
|
-
parsed.flat_map { |q, w| [q] * w }
|
|
323
|
+
parsed.flat_map { |q, w| [q] * [w, 1].max }
|
|
299
324
|
end
|
|
300
325
|
|
|
301
326
|
# `config.redis = { size: N }` pins the main pool at N; otherwise
|