wurk 1.9.0 → 1.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +9 -9
  3. data/app/controllers/wurk/api_controller.rb +4 -1
  4. data/app/controllers/wurk/dashboard_controller.rb +11 -6
  5. data/exe/wurk +7 -2
  6. data/exe/wurkswarm +1 -2
  7. data/lib/generators/wurk/install/install_generator.rb +12 -0
  8. data/lib/sidekiq/web.rb +5 -0
  9. data/lib/wurk/boot_checks.rb +93 -0
  10. data/lib/wurk/capsule.rb +33 -8
  11. data/lib/wurk/cli.rb +204 -30
  12. data/lib/wurk/client/buffered.rb +240 -26
  13. data/lib/wurk/client.rb +16 -5
  14. data/lib/wurk/compat.rb +15 -0
  15. data/lib/wurk/configuration.rb +114 -16
  16. data/lib/wurk/cron.rb +141 -16
  17. data/lib/wurk/debounce.rb +2 -2
  18. data/lib/wurk/embedded.rb +8 -2
  19. data/lib/wurk/encryption.rb +20 -5
  20. data/lib/wurk/error_coalescer.rb +35 -0
  21. data/lib/wurk/fetcher/at_maxmemory.rb +79 -0
  22. data/lib/wurk/fetcher/capped.rb +4 -0
  23. data/lib/wurk/fetcher/many_queues.rb +17 -1
  24. data/lib/wurk/fetcher/oom_drain.rb +102 -0
  25. data/lib/wurk/fetcher/orphan_grace.rb +60 -44
  26. data/lib/wurk/fetcher/private_list_key.rb +1 -16
  27. data/lib/wurk/fetcher/reaper.rb +169 -127
  28. data/lib/wurk/fetcher/reliable.rb +26 -4
  29. data/lib/wurk/fetcher/self_reconcile.rb +175 -0
  30. data/lib/wurk/fetcher/unit_of_work.rb +32 -8
  31. data/lib/wurk/health.rb +23 -4
  32. data/lib/wurk/health_probe.rb +64 -0
  33. data/lib/wurk/heartbeat.rb +55 -3
  34. data/lib/wurk/job/options.rb +9 -1
  35. data/lib/wurk/job.rb +2 -24
  36. data/lib/wurk/job_record.rb +17 -9
  37. data/lib/wurk/job_retry.rb +38 -10
  38. data/lib/wurk/job_set.rb +13 -5
  39. data/lib/wurk/launcher.rb +134 -52
  40. data/lib/wurk/leader.rb +71 -11
  41. data/lib/wurk/limiter/base.rb +17 -1
  42. data/lib/wurk/lua/debounce.lua +11 -6
  43. data/lib/wurk/lua/loader.rb +13 -1
  44. data/lib/wurk/lua.rb +71 -8
  45. data/lib/wurk/manager.rb +1 -1
  46. data/lib/wurk/metrics/process_counters.rb +27 -0
  47. data/lib/wurk/metrics/prometheus.rb +35 -14
  48. data/lib/wurk/middleware/current_attributes.rb +26 -10
  49. data/lib/wurk/middleware/poison_pill.rb +1 -2
  50. data/lib/wurk/preflight/checks.rb +203 -0
  51. data/lib/wurk/preflight/command.rb +135 -0
  52. data/lib/wurk/preflight/keyspace.rb +109 -0
  53. data/lib/wurk/preflight/owners.rb +74 -0
  54. data/lib/wurk/preflight/payload_sample.rb +108 -0
  55. data/lib/wurk/preflight/reader.rb +60 -0
  56. data/lib/wurk/preflight/report.rb +82 -0
  57. data/lib/wurk/preflight/verify.rb +161 -0
  58. data/lib/wurk/preflight.rb +40 -0
  59. data/lib/wurk/process_set.rb +3 -1
  60. data/lib/wurk/processor.rb +82 -18
  61. data/lib/wurk/queue_consumers.rb +134 -0
  62. data/lib/wurk/rails_boot.rb +40 -2
  63. data/lib/wurk/rails_reloader.rb +51 -0
  64. data/lib/wurk/railtie.rb +20 -1
  65. data/lib/wurk/redact.rb +12 -0
  66. data/lib/wurk/redis_pool.rb +34 -9
  67. data/lib/wurk/scheduled.rb +142 -67
  68. data/lib/wurk/sentry/death_handler.rb +29 -0
  69. data/lib/wurk/sentry/error_handler.rb +52 -10
  70. data/lib/wurk/sentry/middleware.rb +8 -1
  71. data/lib/wurk/sentry/retry_policy.rb +12 -3
  72. data/lib/wurk/sentry.rb +42 -5
  73. data/lib/wurk/sorted_entry.rb +38 -13
  74. data/lib/wurk/swarm/child_boot.rb +44 -31
  75. data/lib/wurk/swarm/crash_watch.rb +65 -0
  76. data/lib/wurk/swarm/test_mode_guard.rb +32 -0
  77. data/lib/wurk/swarm.rb +79 -11
  78. data/lib/wurk/testing.rb +74 -0
  79. data/lib/wurk/version.rb +1 -1
  80. data/lib/wurk/web/config.rb +8 -4
  81. data/lib/wurk/web/extension.rb +41 -62
  82. data/lib/wurk/web/helpers.rb +133 -0
  83. data/lib/wurk/web/paginator.rb +49 -0
  84. data/lib/wurk/web/rack_app.rb +30 -6
  85. data/lib/wurk/work_set.rb +6 -0
  86. data/lib/wurk/worker/setter.rb +11 -6
  87. data/lib/wurk/worker.rb +17 -1
  88. data/lib/wurk.rb +1 -1
  89. data/vendor/assets/dashboard/assets/{ArgsValue-DDqX6XWH.js → ArgsValue-CmaXArv8.js} +1 -1
  90. data/vendor/assets/dashboard/assets/{BatchDetail-lIytcn2C.js → BatchDetail-nc5yePZW.js} +1 -1
  91. data/vendor/assets/dashboard/assets/{Batches-BEumt-1O.js → Batches-DgEcyOZ_.js} +1 -1
  92. data/vendor/assets/dashboard/assets/{Busy-Vk9HmZcN.js → Busy-CUZncvCy.js} +1 -1
  93. data/vendor/assets/dashboard/assets/{Cron-CWcM-Cfr.js → Cron-CIZbaaZs.js} +1 -1
  94. data/vendor/assets/dashboard/assets/{Dashboard-Bgv1kusv.js → Dashboard-BWmfAook.js} +1 -1
  95. data/vendor/assets/dashboard/assets/{Dead-Di5Rct2H.js → Dead-CGKsTZHq.js} +1 -1
  96. data/vendor/assets/dashboard/assets/{Extension-LZY5-imc.js → Extension-Doy4bJoA.js} +1 -1
  97. data/vendor/assets/dashboard/assets/{FilterBox-BDo-7IAy.js → FilterBox-C4bO26_d.js} +1 -1
  98. data/vendor/assets/dashboard/assets/{FlowDetail-CehwRLTt.js → FlowDetail-Dh3SJdFh.js} +1 -1
  99. data/vendor/assets/dashboard/assets/{FlowState-glK2Tnf8.js → FlowState-yb7virqR.js} +1 -1
  100. data/vendor/assets/dashboard/assets/{Flows-cgy-Fsbp.js → Flows-C5SMRuhq.js} +1 -1
  101. data/vendor/assets/dashboard/assets/{JobDetailModal-Cuk3jruU.js → JobDetailModal-Cg6sAsJB.js} +1 -1
  102. data/vendor/assets/dashboard/assets/{Limiters-Ci8G2yZG.js → Limiters-BIxwDOJM.js} +1 -1
  103. data/vendor/assets/dashboard/assets/{Metrics-CQFUeZTR.js → Metrics-Qp9q0nF8.js} +1 -1
  104. data/vendor/assets/dashboard/assets/{NotFound-CYjhuDsI.js → NotFound-DwjPxFjU.js} +1 -1
  105. data/vendor/assets/dashboard/assets/{PageHeader-DjIAFTyX.js → PageHeader-vWeBbSPF.js} +1 -1
  106. data/vendor/assets/dashboard/assets/{Profiles-DTowqIRg.js → Profiles-7dabYVe3.js} +1 -1
  107. data/vendor/assets/dashboard/assets/Queues-CMg-gyP8.js +1 -0
  108. data/vendor/assets/dashboard/assets/{Retries-SJ3BaKC0.js → Retries-CNX8ZtTH.js} +1 -1
  109. data/vendor/assets/dashboard/assets/{Scheduled-DxSRpmFZ.js → Scheduled-ClVBj0nc.js} +1 -1
  110. data/vendor/assets/dashboard/assets/{Search-bBbi5JNK.js → Search-BAsVE2qw.js} +1 -1
  111. data/vendor/assets/dashboard/assets/{charts-Bg2YBcaD.js → charts-DZs138BV.js} +1 -1
  112. data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +1 -0
  113. data/vendor/assets/dashboard/assets/index--xvvNlYx.css +1 -0
  114. data/vendor/assets/dashboard/assets/{index-BOTVSP9M.js → index-CMzPFxNE.js} +3 -3
  115. data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-BRPNyV2M.js → useResetPageOnEmpty-DxkLZCIm.js} +1 -1
  116. data/vendor/assets/dashboard/assets/{useSort-B-BLJik7.js → useSort-jBYVwXvg.js} +1 -1
  117. data/vendor/assets/dashboard/assets/{utils-CGyrKpTh.js → utils-B0kQFNE8.js} +1 -1
  118. data/vendor/assets/dashboard/index.html +3 -3
  119. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  120. metadata +53 -30
  121. data/vendor/assets/dashboard/assets/Queues-DUGRPjME.js +0 -1
  122. data/vendor/assets/dashboard/assets/i18n-BZClP62Q.js +0 -1
  123. data/vendor/assets/dashboard/assets/index-Rewk4eSE.css +0 -1
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e5540591682172d88b9972b1cfc33cfca88c9c88d823c0b50fe80cb3556e8c08
4
- data.tar.gz: a19e9d8e68fd2f9f93131cfad4f845595bc95f45aacb65047da452da52916cdd
3
+ metadata.gz: d9594aab6027f299c217e81215d9d10d5764a5f5533b3e40cde2cd1feaae8a86
4
+ data.tar.gz: 26161ff0d0ea58e60acbabd550e212ca354824f31434f0d085fabbc9401b8490
5
5
  SHA512:
6
- metadata.gz: 0d264b8039051e56d2f6cec5aaebb4ce1f99d62ad498a247bece1b6464c316685f43562a76c87ec46f31e846ff6cb6d536f51a11b6d9d06348c32c999d4f2be5
7
- data.tar.gz: 9f695bfc7da4e5ad9e24e6874431a115b80bf3dab52f175a11b8f383d0586694b2d611f3137b6698eb896c1c2107ef18e97d33b7877a1a4fd3af60f380f0f47a
6
+ metadata.gz: 3f2c0b2115f2c7132ab9104e7850f85cf34ac694d4d67b89f615cac765112582506197ae139753489de05f7e6217abb7770129aefbd3125e6dd2be17c04a50ac
7
+ data.tar.gz: 0f08e99addbeeebefc503916d3a033aa88dd9ccc3ad1f557ce4aabfd43e7809b07392589e0137deff11b358304b435ed798766d89f3d3f205c3a105484a65875
data/README.md CHANGED
@@ -22,11 +22,11 @@
22
22
 
23
23
  Wurk is wire-compatible with Sidekiq — same Redis keys, same job JSON, same Ruby DSL. Swap one line in your `Gemfile` and your existing jobs, batches, limiters, cron entries, and live Redis data keep working untouched. The Pro and Enterprise feature sets ship in the same free gem, with no license check and no tiers.
24
24
 
25
- **In production:** Wurk runs the background work at [developerz.ai](https://developerz.ai) and at partner deployments, across many servers, on the fork-based swarm described below. It is not a preview. Throughput numbers from those deployments are not published yet; what has been measured is in [docs/benchmarks.md](docs/benchmarks.md).
25
+ **In production:** Wurk runs background work at partner deployments, across many servers, on the fork-based swarm described below. It is not a preview. Throughput numbers from those deployments are not published yet; what has been measured is in [docs/benchmarks.md](docs/benchmarks.md).
26
26
 
27
27
  **At scale:** Wurk is built for fleets, not just for one box. Kubernetes `/live` + `/ready` probes are a config line, not a sidecar; a bearer-scoped [HTTP API](docs/api-http.md) lets non-Ruby services enqueue and inspect; [OpenTelemetry](docs/telemetry.md) traces propagate client → server; per-queue [global concurrency caps](docs/rate-limiting.md) hold cluster-wide limits; and monitoring is the dashboard you already mount — live SSE, charts, no separate stack to run. See [Wurk extras](#wurk-extras).
28
28
 
29
- **AI-managed:** Wurk is a 100% AI-managed project. Every author's issues and pull requests, org members' included, and dependency updates, are triaged, reviewed, merged or closed by the [developerz.ai](https://developerz.ai) maintainer agent, which asks a human only for security reports and license questions, and cuts a release when a shipped fix has waited for one. New features are still built by us, humans and AI together — see [CONTRIBUTING.md](CONTRIBUTING.md) and [AI-maintained](#ai-maintained). The agent always identifies itself as a bot: it works publicly as `developerz-ai[bot]` and marks every comment it posts as bot-made. Its work and its rules are public too — browse the [repository](https://github.com/developerz-ai/wurk) and read the committed policy in [.dz/maintainer/maintainer.yml](.dz/maintainer/maintainer.yml).
29
+ **AI-managed:** Wurk is a 100% AI-managed project. Every author's issues and pull requests, org members' included, and dependency updates, are triaged, reviewed, merged or closed by the [developerz.ai](https://developerz.ai) maintainer agent, which asks a human only for security reports, license questions and breaking changes, and judges weekly whether a release is due. New features are still built by us, humans and AI together — see [CONTRIBUTING.md](CONTRIBUTING.md) and [AI-maintained](#ai-maintained). The agent always identifies itself as a bot: it works publicly as `developerz-ai[bot]` and marks every comment it posts as bot-made. Its work and its rules are public too — browse the [repository](https://github.com/developerz-ai/wurk) and read the committed policy in [.dz/maintainer/maintainer.yml](.dz/maintainer/maintainer.yml).
30
30
 
31
31
  **On Sidekiq:** Wurk implements Sidekiq's API because it is a genuinely good API. Sidekiq is human-maintained and funds that work through its paid tiers; Wurk is AI-maintained, which is what lets the same surface be free software. Wurk is independent and not affiliated with or endorsed by Sidekiq or its maintainers — see [Why Wurk exists](#why-wurk-exists).
32
32
 
@@ -151,7 +151,7 @@ Method, per-invocation records, workload definitions, and the separate `rake ben
151
151
  - **[Batches](https://github.com/developerz-ai/wurk/blob/main/docs/batches.md)** · **[Rate limiting](https://github.com/developerz-ai/wurk/blob/main/docs/rate-limiting.md)** · **[Periodic (cron) jobs](https://github.com/developerz-ai/wurk/blob/main/docs/periodic-jobs.md)** · **[Unique jobs](https://github.com/developerz-ai/wurk/blob/main/docs/unique-jobs.md)** — the Pro/Ent features, free.
152
152
  - **[Iterable jobs](https://github.com/developerz-ai/wurk/blob/main/docs/iterable-jobs.md)** · **[Middleware](https://github.com/developerz-ai/wurk/blob/main/docs/middleware.md)** · **[Encryption](https://github.com/developerz-ai/wurk/blob/main/docs/encryption.md)** · **[Profiling](https://github.com/developerz-ai/wurk/blob/main/docs/profiling.md)**
153
153
  - **[Data API](https://github.com/developerz-ai/wurk/blob/main/docs/api.md)** · **[Metrics](https://github.com/developerz-ai/wurk/blob/main/docs/metrics.md)** — inspect queues and jobs from Ruby; job metrics, Statsd/DogStatsD, custom history.
154
- - **[Sentry](https://github.com/developerz-ai/wurk/blob/main/docs/sentry.md)** — built-in error reporting (`sentry-sidekiq` can't be installed alongside Wurk); terminal-failure-only reports, per-job scope, no job args.
154
+ - **[Sentry](https://github.com/developerz-ai/wurk/blob/main/docs/sentry.md)** — built-in error reporting (`sentry-sidekiq` needs the `sidekiq` shim gem; prefer this); terminal-failure-only reports, per-job scope, no job args.
155
155
  - **API reference (parity specs):** [Sidekiq OSS](https://github.com/developerz-ai/wurk/blob/main/docs/target/sidekiq-free.md) · [Pro](https://github.com/developerz-ai/wurk/blob/main/docs/target/sidekiq-pro.md) · [Enterprise](https://github.com/developerz-ai/wurk/blob/main/docs/target/sidekiq-ent.md) — the authoritative surface Wurk matches exactly.
156
156
  - **[Authentication & authorization](https://github.com/developerz-ai/wurk/blob/main/docs/authentication.md)** — gate the dashboard behind Devise/Warden, Sorcery, Basic auth, or a token; role-based read/write; CSRF.
157
157
  - **[Securing the dashboard](https://github.com/developerz-ai/wurk/blob/main/docs/dashboard.md)** · **[Metrics history](https://github.com/developerz-ai/wurk/blob/main/docs/metrics-history.md)**
@@ -167,7 +167,7 @@ Method, per-invocation records, workload definitions, and the separate `rake ben
167
167
 
168
168
  The full test suite also runs as a non-required `compat` workflow job across Ruby 3.2.x and the newest Ruby on the current Rails series, the oldest supported Rails series on the newest Ruby, and Valkey 8 as a drop-in store alongside Redis 7.x.
169
169
 
170
- JRuby, TruffleRuby, and Windows fall back to threads-only mode (no fork) — behaviorally equivalent to stock Sidekiq.
170
+ Without `fork` (JRuby, TruffleRuby, Windows; untested in CI) only the single-process `wurk` runner and `config.wurk.embed_in_web` apply: `wurkswarm` and the railtie's swarm have no fallback.
171
171
 
172
172
  ## Running the workers
173
173
 
@@ -284,13 +284,13 @@ Infrastructure this basic should be free software. A Rails app shouldn't need a
284
284
  What has made that hard is maintenance: someone has to be paid to do it. Sidekiq funds a decade of *human* maintenance through its paid tiers, which is an honest trade. Wurk makes a different one — it is maintained **by AI**: every author's issues and pull requests, the team's own included, and dependency updates, are handled end to end by a maintainer agent, and the implementation, parity suite, docs, and benchmarks are written and kept current by AI agents working with the team, which still builds the features and cuts the releases. A fix, a doc update, or a version bump is no longer somebody's week, which is what makes it practical to:
285
285
 
286
286
  - ship the entire Pro + Enterprise surface with no tier, no flag gate, and no license check;
287
- - keep parity honest mechanically rather than by hand — an independently written parity oracle suite, pinned to a documented Sidekiq revision, plus sidekiq-cron's upstream suite running against Wurk on every push ([docs/idea/14-ecosystem-compat.md](https://github.com/developerz-ai/wurk/blob/main/docs/idea/14-ecosystem-compat.md) tracks the rest);
287
+ - keep parity honest mechanically rather than by hand — an independently written parity oracle suite, pinned to a documented Sidekiq revision, plus the upstream suites of sidekiq-cron, sidekiq-unique-jobs, sidekiq-failures and sentry-sidekiq running against Wurk on every push ([docs/idea/14-ecosystem-compat.md](https://github.com/developerz-ai/wurk/blob/main/docs/idea/14-ecosystem-compat.md) tracks the rest);
288
288
  - keep adding surface Sidekiq doesn't have — the [Wurk extras](#wurk-extras) above landed as one release;
289
289
  - hold ourselves to published numbers instead of adjectives — the measured comparison against stock Sidekiq is published [as measured](docs/benchmarks.md), including the unflattering ones, and `rake bench:vs_sidekiq` reproduces it. It is a published measurement, not a per-release CI run.
290
290
 
291
291
  DHH makes the broader version of this argument in [Let the agents democratize open source](https://world.hey.com/dhh/let-the-agents-democratize-open-source-9fd630a9): open source fought for everyone's right to change the software they run, and refusing agent-written code re-erects the gate it spent decades tearing down — "all programmers are equal, but some programmers are more equal than others." His subject is contributions; ours is maintenance, which is the same economics from the other end. The reason a licence key guards batches and cron is not that the code is precious, it's that somebody had to be paid to keep it working. Drop that cost and the tier stops being necessary.
292
292
 
293
- What makes it work in practice is that the agents run inside machinery built to check them. The parity oracle above is a merge-blocking gate, not decoration; sidekiq-cron's upstream suite runs against Wurk in CI and reports on PRs, but it is not a required check and never gates a merge; the `rake bench` PR regression check flags deltas past a noise-adjusted threshold as an advisory comment; and the published benchmark numbers are one-off historical measurements, reproduced on demand rather than enforced in CI. The release gate has never once let an unverified gem reach RubyGems. Agents supply the pace, the gates supply the certainty — and when a gate does catch something, the fix is to make that class of mistake structurally impossible rather than to slow the agents down. The release lane derives its own tag from `Wurk::VERSION` so the two can't drift apart; [RELEASE.md](https://github.com/developerz-ai/wurk/blob/main/RELEASE.md) walks through it.
293
+ What makes it work in practice is that the agents run inside machinery built to check them. The parity oracle above is a merge-blocking gate, not decoration; four gems' upstream suites run against Wurk in CI and report on PRs, but they are not required checks and never gate a merge; the `rake bench` PR regression check flags deltas past a noise-adjusted threshold as an advisory comment; and the published benchmark numbers are one-off historical measurements, reproduced on demand rather than enforced in CI. The release gate has never once let an unverified gem reach RubyGems. Agents supply the pace, the gates supply the certainty — and when a gate does catch something, the fix is to make that class of mistake structurally impossible rather than to slow the agents down. The release lane derives its own tag from `Wurk::VERSION` so the two can't drift apart; [RELEASE.md](https://github.com/developerz-ai/wurk/blob/main/RELEASE.md) walks through it.
294
294
 
295
295
  Wurk is MIT and stays that way. If what you need is a commercial support contract and a human on the other end of an email, buying that is a perfectly good answer.
296
296
 
@@ -303,9 +303,9 @@ Wurk is MIT and stays that way. If what you need is a commercial support contrac
303
303
  + gem "wurk"
304
304
  ```
305
305
 
306
- `bundle install && restart`. Wurk reads and writes the same Redis schema, so there is no data migration: Wurk picks up the queues, schedules, retries, batches and periodic loops Sidekiq left behind. Running Sidekiq and Wurk workers against one Redis *at the same time* is not tested yet, so a live production move is a **drain cutover** (stop Sidekiq's workers, confirm nothing is in flight, start Wurk), with a matching rollback; the [production cutover](https://github.com/developerz-ai/wurk/blob/main/docs/migrate-from-sidekiq.md#9-production-cutover) section has every step and command. sidekiq-cron's upstream suite runs against Wurk in the [`ecosystem` CI job](https://github.com/developerz-ai/wurk/blob/main/.github/workflows/ecosystem.yml) (see [`test/ecosystem/`](https://github.com/developerz-ai/wurk/tree/main/test/ecosystem)). sidekiq-unique-jobs, sidekiq-scheduler, sidekiq-status, sidekiq-failures and sidekiq-throttled are target additions tracked in [docs/idea/14-ecosystem-compat.md](https://github.com/developerz-ai/wurk/blob/main/docs/idea/14-ecosystem-compat.md).
306
+ `bundle install && restart`. Wurk reads and writes the same Redis schema, so there is no data migration: Wurk picks up the queues, schedules, retries, batches and periodic loops Sidekiq left behind. Running Sidekiq and Wurk workers against one Redis *at the same time* is not tested yet, so a live production move is a **drain cutover** (stop Sidekiq's workers, confirm nothing is in flight, start Wurk), with a matching rollback; the [production cutover](https://github.com/developerz-ai/wurk/blob/main/docs/migrate-from-sidekiq.md#9-production-cutover) section has every step and command. The upstream suites of sidekiq-cron, sidekiq-unique-jobs, sidekiq-failures and sentry-sidekiq run against Wurk in the [`ecosystem` CI job](https://github.com/developerz-ai/wurk/blob/main/.github/workflows/ecosystem.yml) (see [`test/ecosystem/`](https://github.com/developerz-ai/wurk/tree/main/test/ecosystem)). sidekiq-scheduler, sidekiq-status and sidekiq-throttled are target additions tracked in [docs/idea/14-ecosystem-compat.md](https://github.com/developerz-ai/wurk/blob/main/docs/idea/14-ecosystem-compat.md).
307
307
 
308
- **Keeping a `sidekiq-*` add-on gem?** Those gems declare `add_dependency "sidekiq"`, which would reinstall real Sidekiq next to Wurk. Satisfy it with Wurk's git-only [`sidekiq` shim gem](https://github.com/developerz-ai/wurk/tree/main/ecosystem/sidekiq-shim) instead: `gem "sidekiq", github: "developerz-ai/wurk", glob: "ecosystem/sidekiq-shim/*.gemspec"`. Only sidekiq-cron is exercised that way on every PR; treat the others as untested.
308
+ **Keeping a `sidekiq-*` add-on gem?** Those gems declare `add_dependency "sidekiq"`, which would reinstall real Sidekiq next to Wurk. Satisfy it with Wurk's git-only [`sidekiq` shim gem](https://github.com/developerz-ai/wurk/tree/main/ecosystem/sidekiq-shim) instead: `gem "sidekiq", github: "developerz-ai/wurk", glob: "ecosystem/sidekiq-shim/*.gemspec"`. Only sidekiq-cron, sidekiq-unique-jobs, sidekiq-failures and sentry-sidekiq are exercised that way, by the `ecosystem` workflow; treat the others as untested.
309
309
 
310
310
  Full walkthrough — config side-by-side, the Redis key/`sidekiq_options` mapping, known incompatibilities, moving sidekiq-cron schedules over (`rake wurk:import:cron`), and the production cutover and rollback playbook: **[docs/migrate-from-sidekiq.md](https://github.com/developerz-ai/wurk/blob/main/docs/migrate-from-sidekiq.md)**. On the night, keep the **[incident runbook](https://github.com/developerz-ai/wurk/blob/main/docs/runbook.md)** open.
311
311
 
@@ -332,7 +332,7 @@ TypeScript end to end, MIT, and as measured about its own claims as this file is
332
332
  | Dependabot | driven to green and merged |
333
333
  | an issue or pull request by anyone with write access (org members, collaborators) | handled like anyone's: triaged, reviewed, driven to green, merged |
334
334
  | an outside change to CI, release or publish automation | declined: members maintain the pipeline that publishes the gem |
335
- | a breaking change | the bot's call, not a human's; a change that breaks the drop-in Sidekiq contract is declined, since a revert cannot take that promise back |
335
+ | a breaking change | a human decides; a change that breaks the drop-in Sidekiq contract is declined, since a revert cannot take that promise back |
336
336
 
337
337
  Releases follow the work, not a calendar. A weekday checkup asks whether a shipping fix has merged and nobody has released it. When one has, the bot files a release request, its coding fleet opens the `Wurk::VERSION` bump PR, and merging that PR publishes the gem ([RELEASE.md](RELEASE.md)). The bot's policy is committed in [`.dz/maintainer/maintainer.yml`](.dz/maintainer/maintainer.yml).
338
338
 
@@ -62,7 +62,10 @@ module Wurk
62
62
  end
63
63
 
64
64
  def queues
65
- render json: ::Wurk::Stats.new.queue_summaries.map { |q| ::Wurk::Api::Serializers.queue_summary(q) }
65
+ consumers = ::Wurk.redis(idempotent: true) { |conn| ::Wurk::QueueConsumers.counts(conn) }
66
+ render json: ::Wurk::Stats.new.queue_summaries.map { |q|
67
+ ::Wurk::Api::Serializers.queue_summary(q).merge(consumers: consumers.fetch(q.name, 0))
68
+ }
66
69
  end
67
70
 
68
71
  def queue
@@ -18,14 +18,18 @@ module Wurk
18
18
  # Vite dev server (default :5173) so contributors get HMR without
19
19
  # rebuilding the bundle on every change.
20
20
  class DashboardController < ApplicationController
21
+ # bin/dev exports VITE_PORT and vite.config.ts binds it, with strictPort, so this
22
+ # has to read the same variable or an overridden port answers every page with a 500.
23
+ def self.vite_dev_url
24
+ "http://localhost:#{ENV.fetch('VITE_PORT', '5173')}#{VITE_ASSET_BASE}"
25
+ end
26
+
21
27
  # Vite serves the dev shell under its `base` (vite.config.ts), which is the
22
28
  # same path the engine mounts assets at — so derive these from
23
29
  # AssetMount::PREFIX to keep the Ruby side in lockstep. Fetching the server
24
30
  # root instead returns Vite's "did you mean /wurk-assets/" hint page, not the
25
31
  # shell (issue #181).
26
- VITE_DEV_HOST = 'http://localhost:5173'
27
32
  VITE_ASSET_BASE = "#{::Wurk::Engine::AssetMount::PREFIX}/".freeze # "/wurk-assets/"
28
- VITE_DEV_URL = "#{VITE_DEV_HOST}#{VITE_ASSET_BASE}".freeze
29
33
  INDEX_REL_PATH = ['vendor', 'assets', 'dashboard', 'index.html'].freeze
30
34
  # The SPA mount point, matched without its closing `>` so the locale hint
31
35
  # appends to whatever attributes the shell already carries.
@@ -115,11 +119,12 @@ module Wurk
115
119
  end
116
120
 
117
121
  def fetch_vite_dev_shell
118
- uri = ::URI.parse(VITE_DEV_URL)
122
+ uri = ::URI.parse(self.class.vite_dev_url)
119
123
  rewrite_dev_asset_urls(::Net::HTTP.get(uri))
120
124
  rescue ::StandardError => e
121
- raise "Wurk dashboard: cannot reach Vite dev server at #{VITE_DEV_URL} " \
122
- "(#{e.class}: #{e.message}). Run `bin/rake frontend:dev` from the gem root."
125
+ raise "Wurk dashboard: cannot reach Vite dev server at #{self.class.vite_dev_url} " \
126
+ "(#{e.class}: #{e.message}). Run `bin/rake frontend:dev` from the gem root, or `bin/dev`, " \
127
+ 'with the same VITE_PORT.'
123
128
  end
124
129
 
125
130
  # The shell's entry URLs (`src="/wurk-assets/@vite/client"`,
@@ -130,7 +135,7 @@ module Wurk
130
135
  # `server.origin` covers runtime-generated asset URLs but not these entry
131
136
  # tags, so rewrite them here.
132
137
  def rewrite_dev_asset_urls(html)
133
- html.gsub(/(["'])#{::Regexp.escape(VITE_ASSET_BASE)}/, "\\1#{VITE_DEV_URL}")
138
+ html.gsub(/(["'])#{::Regexp.escape(VITE_ASSET_BASE)}/, "\\1#{self.class.vite_dev_url}")
134
139
  end
135
140
 
136
141
  def read_built_index
data/exe/wurk CHANGED
@@ -4,12 +4,18 @@
4
4
  # Standalone runner. Does NOT load the Rails engine.
5
5
  # Usage: `exe/wurk -C config/wurk.yml` — worker: one process, thread pool
6
6
  # `exe/wurk api --port 7433` — machine HTTP API, runs no jobs
7
+ # `exe/wurk preflight` / `verify` — read-only cutover checks, run no jobs
7
8
 
8
9
  $stdout.sync = true
9
10
  $LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
10
11
 
11
12
  require 'wurk'
12
13
 
14
+ if %w[preflight verify].include?(ARGV.first)
15
+ require 'wurk/preflight/command'
16
+ exit Wurk::Preflight::Command.run(ARGV.dup)
17
+ end
18
+
13
19
  begin
14
20
  cli = Wurk::CLI.instance
15
21
  cli.parse
@@ -20,7 +26,6 @@ begin
20
26
  rescue StandardError => e
21
27
  raise e if $DEBUG
22
28
 
23
- warn "wurk: #{e.message}"
24
- warn e.backtrace.join("\n")
29
+ Wurk::CLI.report_failure('wurk', e)
25
30
  exit 1
26
31
  end
data/exe/wurkswarm CHANGED
@@ -17,7 +17,6 @@ begin
17
17
  rescue StandardError => e
18
18
  raise e if $DEBUG
19
19
 
20
- warn "wurkswarm: #{e.message}"
21
- warn e.backtrace.join("\n")
20
+ Wurk::CLI.report_failure('wurkswarm', e)
22
21
  exit 1
23
22
  end
@@ -17,6 +17,18 @@ module Wurk
17
17
  def insert_mount_line
18
18
  route(%(# mount Wurk::Engine => "/wurk" # uncomment to expose the Wurk dashboard))
19
19
  end
20
+
21
+ def show_next_steps
22
+ say <<~STEPS
23
+
24
+ Next steps:
25
+ 1. Set the queue adapter if you use Active Job: config.active_job.queue_adapter = :wurk
26
+ 2. Start workers: bundle exec wurkswarm (or bundle exec wurk for one process)
27
+ 3. To use the dashboard, uncomment the mount line in config/routes.rb.
28
+ It has no login of its own, so put it behind authentication first:
29
+ docs/authentication.md in the wurk gem's repository
30
+ STEPS
31
+ end
20
32
  end
21
33
  end
22
34
  end
data/lib/sidekiq/web.rb CHANGED
@@ -2,3 +2,8 @@
2
2
 
3
3
  # Drop-in require path (see lib/sidekiq.rb). Wurk loads this surface whole.
4
4
  require 'wurk'
5
+
6
+ # Extension renderer dependencies, deferred on the worker path (web/extension.rb).
7
+ # Loaded here for a helper called outside a request, as unique-jobs' specs do.
8
+ require 'cgi'
9
+ require 'erb'
@@ -0,0 +1,93 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'keys'
4
+ require_relative 'queue_consumers'
5
+
6
+ module Wurk
7
+ # What a standalone runner says once at boot, after the app loaded. Log only,
8
+ # and never allowed to stop a boot: none of this changes behaviour by backend,
9
+ # version or peers.
10
+ module BootChecks
11
+ # Oldest version each non-Redis backend has been run against.
12
+ TESTED_FLOORS = { 'dragonfly' => '1.34.0', 'valkey' => '8.0.0' }.freeze
13
+ MAX_LISTED = 20
14
+ SUBCLASS = ::Module.instance_method(:<)
15
+ NAME = ::Module.instance_method(:name)
16
+
17
+ module_function
18
+
19
+ def run(config, info)
20
+ log_backend(config.logger, info)
21
+ warn_on_foreign_processes(config)
22
+ warn_on_ignored_locks(config.logger)
23
+ rescue StandardError => e
24
+ config.logger.warn { "wurk boot checks skipped: #{e.class}: #{e.message}" }
25
+ end
26
+
27
+ def backend(info)
28
+ return ['dragonfly', info['dragonfly_version'].to_s.delete_prefix('df-v')] if info['dragonfly_version']
29
+ return ['valkey', info['valkey_version'].to_s] if info['valkey_version']
30
+
31
+ ['redis', info['redis_version'].to_s]
32
+ end
33
+
34
+ def log_backend(logger, info)
35
+ name, version = backend(info)
36
+ logger.info { "Wurk #{Wurk::VERSION} on #{name} #{version}" }
37
+ floor = TESTED_FLOORS[name]
38
+ return unless floor && ::Gem::Version.correct?(version) && ::Gem::Version.new(version) < ::Gem::Version.new(floor)
39
+
40
+ logger.warn { "#{name} #{version} is older than the oldest version Wurk is tested against (#{floor})." }
41
+ end
42
+
43
+ # Both runtimes share `processes` and the same queues, so a mix is safe only
44
+ # if it is deliberate. Wurk's heartbeat always carries `cores`; stock does not.
45
+ def warn_on_foreign_processes(config)
46
+ foreign = process_infos(config).reject { |_, info| info.nil? || ::JSON.parse(info).key?('cores') }.map(&:first)
47
+ return if foreign.empty?
48
+
49
+ config.logger.warn do
50
+ "Live processes that are not Wurk share this Redis: #{listing(foreign)}. " \
51
+ 'They fetch from the same queues. If this is not a deliberate rolling cutover, stop them.'
52
+ end
53
+ end
54
+
55
+ def process_infos(config)
56
+ config.redis(idempotent: true) do |conn|
57
+ ids = conn.call('SMEMBERS', Keys::PROCESSES)
58
+ ids.empty? ? [] : ids.zip(conn.pipelined { |pipe| ids.each { |id| pipe.call('HGET', id, 'info') } })
59
+ end
60
+ end
61
+
62
+ # Without the gem's middleware the option does nothing: the "unique" job
63
+ # simply enqueues twice. Only classes the app has loaded by now are seen.
64
+ def warn_on_ignored_locks(logger)
65
+ return if defined?(::SidekiqUniqueJobs)
66
+
67
+ classes = locked_job_classes
68
+ return if classes.empty?
69
+
70
+ logger.warn do
71
+ 'These jobs set `lock:` but sidekiq-unique-jobs is not loaded, so the option does nothing and ' \
72
+ "duplicates will run: #{listing(classes)}. Keep the gem, or move them to `unique_for:`."
73
+ end
74
+ end
75
+
76
+ # A walk over every class meets classes that redefine these with other arguments
77
+ # (Faker::Travel::Airport.name takes required keywords), and one such class must
78
+ # not cost the app this check.
79
+ def locked_job_classes
80
+ ::ObjectSpace.each_object(::Class).filter_map do |klass|
81
+ next unless SUBCLASS.bind_call(klass, Wurk::Worker)
82
+
83
+ name = NAME.bind_call(klass)
84
+ name if name && klass.get_sidekiq_options.key?('lock')
85
+ end.sort
86
+ end
87
+
88
+ def listing(names)
89
+ extra = names.size - MAX_LISTED
90
+ names.first(MAX_LISTED).join(', ') + (extra.positive? ? " and #{extra} more" : '')
91
+ end
92
+ end
93
+ end
data/lib/wurk/capsule.rb CHANGED
@@ -53,6 +53,8 @@ module Wurk
53
53
  # %w[high default low] → mode :strict, weights all 0
54
54
  # %w[high,3 default,2 low,1] → mode :weighted, weights {q=>w}
55
55
  # %w[a,1 b,1 c,1] → mode :random, weights all 1
56
+ # %w[a,2 b] → mode :weighted, unweighted b fetched as if 1
57
+ # %w[a,0 b,0] → mode :strict, 0 being the same as no weight
56
58
  # `@queues` is expanded by weight so a uniform shuffle gives weighted
57
59
  # fairness (e.g. ["high","high","high","default","default","low"]).
58
60
  def queues=(val)
@@ -188,6 +190,16 @@ module Wurk
188
190
  @pools.clear
189
191
  end
190
192
 
193
+ # The fork-path form: closes idle sockets but keeps every pool usable, for a
194
+ # parent whose own threads may be mid-checkout.
195
+ def reload_redis_pools!
196
+ @pools.each_value(&:reload!)
197
+ end
198
+
199
+ def redis_pools_built?
200
+ !@pools.empty?
201
+ end
202
+
191
203
  def redis(idempotent: false, &)
192
204
  PoolCheckout.with(redis_pool, idempotent, &)
193
205
  end
@@ -229,19 +241,32 @@ module Wurk
229
241
  copy = server_middleware
230
242
  return copy if frozen?
231
243
 
232
- @config.server_middleware.entries.each do |entry|
244
+ global = @config.server_middleware.entries
245
+ global.each_with_index do |entry, index|
233
246
  next if @server_middleware_seen.include?(entry.klass)
234
247
 
235
248
  @server_middleware_seen << entry.klass
236
- next if copy.exists?(entry.klass)
237
-
238
- rebound = entry.dup
239
- rebound.config = self
240
- copy.entries << rebound
249
+ adopt_server_middleware(copy, entry, global.drop(index + 1)) unless copy.exists?(entry.klass)
241
250
  end
242
251
  copy
243
252
  end
244
253
 
254
+ def adopt_server_middleware(copy, entry, followers)
255
+ rebound = entry.dup
256
+ rebound.config = self
257
+ copy.entries.insert(refresh_position(copy, followers), rebound)
258
+ end
259
+
260
+ # Ahead of the first entry that follows it in the global chain, so a late
261
+ # `prepend` still wraps what it was prepended to. -1 appends.
262
+ def refresh_position(copy, followers)
263
+ followers.each do |follower|
264
+ position = copy.entries.index { |entry| entry.klass == follower.klass }
265
+ return position if position
266
+ end
267
+ -1
268
+ end
269
+
245
270
  # Drop-in fetch pluggability (spec §5 / §4.1): instantiate
246
271
  # `config[:fetch_class]` when the host set one — a custom or Pro fetcher —
247
272
  # else the default reliable BLMOVE fetcher (`Sidekiq::BasicFetch`). A
@@ -276,7 +301,7 @@ module Wurk
276
301
  return [qname, 0] if weight.nil?
277
302
 
278
303
  weight = Integer(weight)
279
- raise ArgumentError, "queue weight must be > 0: `#{entry}`" if weight <= 0
304
+ raise ArgumentError, "queue weight must not be negative: `#{entry}`" if weight.negative?
280
305
 
281
306
  [qname, weight]
282
307
  end
@@ -295,7 +320,7 @@ module Wurk
295
320
  def expand_by_weight(parsed, mode)
296
321
  return parsed.map(&:first) if %i[strict random].include?(mode)
297
322
 
298
- parsed.flat_map { |q, w| [q] * w }
323
+ parsed.flat_map { |q, w| [q] * [w, 1].max }
299
324
  end
300
325
 
301
326
  # `config.redis = { size: N }` pins the main pool at N; otherwise