wurk 1.13.0 → 1.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/app/controllers/concerns/wurk/same_origin_guard.rb +30 -8
- data/app/controllers/concerns/wurk/sse_streaming.rb +36 -1
- data/app/controllers/wurk/api/pagination.rb +74 -55
- data/app/controllers/wurk/api/serializers.rb +8 -2
- data/app/controllers/wurk/api_controller.rb +138 -36
- data/app/controllers/wurk/application_controller.rb +0 -2
- data/app/controllers/wurk/dashboard_controller.rb +26 -3
- data/app/controllers/wurk/profiles_controller.rb +28 -23
- data/config/routes.rb +13 -7
- data/lib/sidekiq/cli.rb +4 -3
- data/lib/sidekiq/deploy.rb +2 -1
- data/lib/sidekiq/embedded.rb +2 -1
- data/lib/sidekiq/iterable_job.rb +2 -1
- data/lib/sidekiq/job/iterable/active_record_enumerator.rb +2 -1
- data/lib/sidekiq/job/iterable/csv_enumerator.rb +2 -1
- data/lib/sidekiq/job/iterable.rb +2 -1
- data/lib/sidekiq/launcher.rb +2 -1
- data/lib/sidekiq/manager.rb +2 -1
- data/lib/sidekiq/metrics/query.rb +2 -1
- data/lib/sidekiq/pro/web.rb +2 -1
- data/lib/sidekiq/scheduled.rb +2 -1
- data/lib/sidekiq/web.rb +2 -1
- data/lib/sidekiq.rb +3 -2
- data/lib/wurk/api/admin.rb +84 -0
- data/lib/wurk/api/app.rb +7 -2
- data/lib/wurk/api/fast.rb +4 -18
- data/lib/wurk/api/jobs.rb +11 -3
- data/lib/wurk/api/queues.rb +22 -10
- data/lib/wurk/api/router.rb +6 -0
- data/lib/wurk/api/serializers.rb +6 -7
- data/lib/wurk/api/swarm.rb +0 -1
- data/lib/wurk/api/validation.rb +17 -0
- data/lib/wurk/batch/callbacks.rb +9 -2
- data/lib/wurk/batch/server_middleware.rb +4 -0
- data/lib/wurk/batch/status.rb +45 -22
- data/lib/wurk/batch.rb +15 -4
- data/lib/wurk/batch_set.rb +56 -0
- data/lib/wurk/cli.rb +21 -21
- data/lib/wurk/client/buffered.rb +198 -71
- data/lib/wurk/client.rb +21 -1
- data/lib/wurk/compat/deploy.rb +7 -0
- data/lib/wurk/compat/encryption.rb +7 -0
- data/lib/wurk/compat/flow.rb +8 -0
- data/lib/wurk/compat/iterable_job.rb +8 -0
- data/lib/wurk/compat/profile_set.rb +8 -0
- data/lib/wurk/compat/server.rb +15 -0
- data/lib/wurk/compat/web.rb +20 -0
- data/lib/wurk/compat.rb +34 -40
- data/lib/wurk/configuration.rb +74 -9
- data/lib/wurk/cron.rb +146 -47
- data/lib/wurk/dead_set.rb +10 -6
- data/lib/wurk/embedded.rb +0 -2
- data/lib/wurk/engine.rb +46 -4
- data/lib/wurk/error_coalescer.rb +21 -1
- data/lib/wurk/fetcher/many_queues.rb +18 -6
- data/lib/wurk/fetcher/poison_pill.rb +3 -1
- data/lib/wurk/fetcher/reaper.rb +3 -5
- data/lib/wurk/fetcher/reliable.rb +64 -31
- data/lib/wurk/fetcher/self_reconcile.rb +0 -1
- data/lib/wurk/fetcher/unit_of_work.rb +16 -1
- data/lib/wurk/flow/status.rb +10 -4
- data/lib/wurk/flow_set.rb +24 -2
- data/lib/wurk/health.rb +6 -2
- data/lib/wurk/health_probe.rb +45 -7
- data/lib/wurk/heartbeat.rb +42 -13
- data/lib/wurk/import/sidekiq_cron.rb +0 -1
- data/lib/wurk/job_record.rb +2 -1
- data/lib/wurk/job_retry.rb +1 -1
- data/lib/wurk/job_set.rb +157 -9
- data/lib/wurk/launcher.rb +0 -12
- data/lib/wurk/lazy_loader.rb +60 -0
- data/lib/wurk/leader.rb +2 -0
- data/lib/wurk/limiter/bucket.rb +0 -2
- data/lib/wurk/limiter/concurrent.rb +19 -36
- data/lib/wurk/limiter/leaky.rb +0 -2
- data/lib/wurk/limiter/points.rb +0 -2
- data/lib/wurk/limiter/server_middleware.rb +4 -1
- data/lib/wurk/limiter/unlimited.rb +0 -2
- data/lib/wurk/limiter/window.rb +0 -2
- data/lib/wurk/limiter.rb +0 -7
- data/lib/wurk/logger.rb +44 -5
- data/lib/wurk/lua/fast_delete.rb +58 -0
- data/lib/wurk/lua/fast_delete_window.lua +46 -0
- data/lib/wurk/lua/limiter_concurrent_acquire.lua +10 -3
- data/lib/wurk/lua/limiter_concurrent_release.lua +13 -4
- data/lib/wurk/lua.rb +1 -41
- data/lib/wurk/metrics/accumulator.rb +27 -2
- data/lib/wurk/metrics/dashboard_series.rb +38 -18
- data/lib/wurk/metrics/exposition.rb +45 -0
- data/lib/wurk/metrics/history.rb +30 -12
- data/lib/wurk/metrics/pool_registry.rb +27 -0
- data/lib/wurk/metrics/process_counters.rb +16 -0
- data/lib/wurk/metrics/process_families.rb +115 -0
- data/lib/wurk/metrics/prometheus.rb +84 -52
- data/lib/wurk/metrics/query.rb +0 -2
- data/lib/wurk/metrics/queue_rollup.rb +33 -11
- data/lib/wurk/metrics/statsd.rb +4 -0
- data/lib/wurk/middleware/chain.rb +25 -3
- data/lib/wurk/middleware/expiry.rb +4 -0
- data/lib/wurk/middleware/interrupt_handler.rb +4 -1
- data/lib/wurk/middleware/status.rb +4 -0
- data/lib/wurk/middleware/timeout.rb +4 -0
- data/lib/wurk/preflight/owners.rb +0 -1
- data/lib/wurk/preflight/payload_sample.rb +0 -1
- data/lib/wurk/preflight/verify.rb +0 -2
- data/lib/wurk/process_set.rb +6 -1
- data/lib/wurk/profiler.rb +2 -1
- data/lib/wurk/queue.rb +33 -6
- data/lib/wurk/queue_consumers.rb +10 -1
- data/lib/wurk/rails_boot.rb +2 -2
- data/lib/wurk/railtie.rb +3 -4
- data/lib/wurk/redis_pool.rb +6 -1
- data/lib/wurk/scheduled.rb +7 -0
- data/lib/wurk/sentry/error_handler.rb +5 -0
- data/lib/wurk/sentry/middleware.rb +4 -0
- data/lib/wurk/sorted_entry.rb +16 -12
- data/lib/wurk/stats.rb +85 -1
- data/lib/wurk/swarm/child_boot.rb +6 -3
- data/lib/wurk/swarm/crash_watch.rb +5 -0
- data/lib/wurk/swarm/liveness.rb +0 -2
- data/lib/wurk/swarm/memory_watch.rb +110 -0
- data/lib/wurk/swarm/prefork.rb +54 -0
- data/lib/wurk/swarm/restart.rb +14 -5
- data/lib/wurk/swarm/restart_counters.rb +79 -0
- data/lib/wurk/swarm.rb +46 -47
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/enterprise.rb +63 -7
- data/lib/wurk.rb +34 -29
- data/vendor/assets/dashboard/assets/ArgsValue-D10BT0xi.js +1 -0
- data/vendor/assets/dashboard/assets/ArgsValue-D10BT0xi.js.br +0 -0
- data/vendor/assets/dashboard/assets/ArgsValue-D10BT0xi.js.gz +0 -0
- data/vendor/assets/dashboard/assets/BatchDetail-ByPRHv86.js +1 -0
- data/vendor/assets/dashboard/assets/BatchDetail-ByPRHv86.js.br +0 -0
- data/vendor/assets/dashboard/assets/BatchDetail-ByPRHv86.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Batches-CVVOJFhz.js +1 -0
- data/vendor/assets/dashboard/assets/Batches-CVVOJFhz.js.br +0 -0
- data/vendor/assets/dashboard/assets/Batches-CVVOJFhz.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Busy-Dp26yz1A.js +1 -0
- data/vendor/assets/dashboard/assets/Busy-Dp26yz1A.js.br +0 -0
- data/vendor/assets/dashboard/assets/Busy-Dp26yz1A.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Cron-BiRV6eoU.js +1 -0
- data/vendor/assets/dashboard/assets/Cron-BiRV6eoU.js.br +0 -0
- data/vendor/assets/dashboard/assets/Cron-BiRV6eoU.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Dashboard-CnpCQ8Of.js +1 -0
- data/vendor/assets/dashboard/assets/Dashboard-CnpCQ8Of.js.br +0 -0
- data/vendor/assets/dashboard/assets/Dashboard-CnpCQ8Of.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Dead-ykYbsK8n.js +1 -0
- data/vendor/assets/dashboard/assets/Dead-ykYbsK8n.js.br +0 -0
- data/vendor/assets/dashboard/assets/Dead-ykYbsK8n.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Extension-BUsciXhN.js +1 -0
- data/vendor/assets/dashboard/assets/Extension-BUsciXhN.js.br +0 -0
- data/vendor/assets/dashboard/assets/Extension-BUsciXhN.js.gz +0 -0
- data/vendor/assets/dashboard/assets/FilterBox-DBIZXNtY.js +1 -0
- data/vendor/assets/dashboard/assets/FilterBox-DBIZXNtY.js.br +0 -0
- data/vendor/assets/dashboard/assets/FilterBox-DBIZXNtY.js.gz +0 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CIGu7pRb.js +1 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CIGu7pRb.js.br +0 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CIGu7pRb.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{FlowState-yb7virqR.js → FlowState-B04ZLDe-.js} +1 -1
- data/vendor/assets/dashboard/assets/FlowState-B04ZLDe-.js.br +0 -0
- data/vendor/assets/dashboard/assets/FlowState-B04ZLDe-.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Flows-Ce9Qoq0c.js +1 -0
- data/vendor/assets/dashboard/assets/Flows-Ce9Qoq0c.js.br +0 -0
- data/vendor/assets/dashboard/assets/Flows-Ce9Qoq0c.js.gz +0 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-DWp0OMoV.js +2 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-DWp0OMoV.js.br +0 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-DWp0OMoV.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Limiters-DiWCiVay.js +1 -0
- data/vendor/assets/dashboard/assets/Limiters-DiWCiVay.js.br +0 -0
- data/vendor/assets/dashboard/assets/Limiters-DiWCiVay.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Metrics-DmMS2wo_.js +1 -0
- data/vendor/assets/dashboard/assets/Metrics-DmMS2wo_.js.br +0 -0
- data/vendor/assets/dashboard/assets/Metrics-DmMS2wo_.js.gz +0 -0
- data/vendor/assets/dashboard/assets/NotFound-CM1j8-Fv.js +1 -0
- data/vendor/assets/dashboard/assets/NotFound-CM1j8-Fv.js.br +0 -0
- data/vendor/assets/dashboard/assets/NotFound-CM1j8-Fv.js.gz +0 -0
- data/vendor/assets/dashboard/assets/PageHeader-BcHVxRbX.js +1 -0
- data/vendor/assets/dashboard/assets/PageHeader-BcHVxRbX.js.br +0 -0
- data/vendor/assets/dashboard/assets/PageHeader-BcHVxRbX.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Profiles-B70MlVYQ.js +1 -0
- data/vendor/assets/dashboard/assets/Profiles-B70MlVYQ.js.br +0 -0
- data/vendor/assets/dashboard/assets/Profiles-B70MlVYQ.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Queues-D3JZtWBv.js +1 -0
- data/vendor/assets/dashboard/assets/Queues-D3JZtWBv.js.br +0 -0
- data/vendor/assets/dashboard/assets/Queues-D3JZtWBv.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Retries-BTWoV6so.js +1 -0
- data/vendor/assets/dashboard/assets/Retries-BTWoV6so.js.br +0 -0
- data/vendor/assets/dashboard/assets/Retries-BTWoV6so.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Scheduled-D3PyiFqr.js +1 -0
- data/vendor/assets/dashboard/assets/Scheduled-D3PyiFqr.js.br +0 -0
- data/vendor/assets/dashboard/assets/Scheduled-D3PyiFqr.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Search-bLwIhsaR.js +1 -0
- data/vendor/assets/dashboard/assets/Search-bLwIhsaR.js.br +0 -0
- data/vendor/assets/dashboard/assets/Search-bLwIhsaR.js.gz +0 -0
- data/vendor/assets/dashboard/assets/ar-CugpXGea.js +1 -0
- data/vendor/assets/dashboard/assets/ar-CugpXGea.js.br +0 -0
- data/vendor/assets/dashboard/assets/ar-CugpXGea.js.gz +0 -0
- data/vendor/assets/dashboard/assets/charts-DZyiygHX.js +1 -0
- data/vendor/assets/dashboard/assets/charts-DZyiygHX.js.br +0 -0
- data/vendor/assets/dashboard/assets/charts-DZyiygHX.js.gz +0 -0
- data/vendor/assets/dashboard/assets/de-Bl5Jn_0l.js +1 -0
- data/vendor/assets/dashboard/assets/de-Bl5Jn_0l.js.br +0 -0
- data/vendor/assets/dashboard/assets/de-Bl5Jn_0l.js.gz +0 -0
- data/vendor/assets/dashboard/assets/es-lqQrmjYI.js +1 -0
- data/vendor/assets/dashboard/assets/es-lqQrmjYI.js.br +0 -0
- data/vendor/assets/dashboard/assets/es-lqQrmjYI.js.gz +0 -0
- data/vendor/assets/dashboard/assets/fr-D70VBoh3.js +1 -0
- data/vendor/assets/dashboard/assets/fr-D70VBoh3.js.br +0 -0
- data/vendor/assets/dashboard/assets/fr-D70VBoh3.js.gz +0 -0
- data/vendor/assets/dashboard/assets/i18n-DdPClaCG.js +1 -0
- data/vendor/assets/dashboard/assets/i18n-DdPClaCG.js.br +0 -0
- data/vendor/assets/dashboard/assets/i18n-DdPClaCG.js.gz +0 -0
- data/vendor/assets/dashboard/assets/index-CtK2Dy_G.css +1 -0
- data/vendor/assets/dashboard/assets/index-CtK2Dy_G.css.br +0 -0
- data/vendor/assets/dashboard/assets/index-CtK2Dy_G.css.gz +0 -0
- data/vendor/assets/dashboard/assets/index-ygQ-XC-L.js +141 -0
- data/vendor/assets/dashboard/assets/index-ygQ-XC-L.js.br +0 -0
- data/vendor/assets/dashboard/assets/index-ygQ-XC-L.js.gz +0 -0
- data/vendor/assets/dashboard/assets/ja-BYMqnHjg.js +1 -0
- data/vendor/assets/dashboard/assets/ja-BYMqnHjg.js.br +0 -0
- data/vendor/assets/dashboard/assets/ja-BYMqnHjg.js.gz +0 -0
- data/vendor/assets/dashboard/assets/pt-BR-4k5M2MYH.js +1 -0
- data/vendor/assets/dashboard/assets/pt-BR-4k5M2MYH.js.br +0 -0
- data/vendor/assets/dashboard/assets/pt-BR-4k5M2MYH.js.gz +0 -0
- data/vendor/assets/dashboard/assets/useJobSetList-D-uafkUL.js +2 -0
- data/vendor/assets/dashboard/assets/useJobSetList-D-uafkUL.js.br +0 -0
- data/vendor/assets/dashboard/assets/useJobSetList-D-uafkUL.js.gz +0 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-DUA0JTgs.js +1 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-DUA0JTgs.js.br +0 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-DUA0JTgs.js.gz +0 -0
- data/vendor/assets/dashboard/assets/useSort-BwoC7g__.js +1 -0
- data/vendor/assets/dashboard/assets/useSort-BwoC7g__.js.br +0 -0
- data/vendor/assets/dashboard/assets/useSort-BwoC7g__.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{utils-CnWZzW2L.js → utils-CQj4rqng.js} +1 -1
- data/vendor/assets/dashboard/assets/utils-CQj4rqng.js.br +0 -0
- data/vendor/assets/dashboard/assets/utils-CQj4rqng.js.gz +0 -0
- data/vendor/assets/dashboard/assets/wurk-logo-ydeSB9B8.png +0 -0
- data/vendor/assets/dashboard/assets/zh-CN-B9E1MvsM.js +1 -0
- data/vendor/assets/dashboard/assets/zh-CN-B9E1MvsM.js.br +0 -0
- data/vendor/assets/dashboard/assets/zh-CN-B9E1MvsM.js.gz +0 -0
- data/vendor/assets/dashboard/index.html +8 -28
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +130 -37
- data/vendor/assets/dashboard/assets/ArgsValue-BJol9S0s.js +0 -1
- data/vendor/assets/dashboard/assets/BatchDetail-D_zGQ9Sd.js +0 -1
- data/vendor/assets/dashboard/assets/Batches-Dt_xC660.js +0 -1
- data/vendor/assets/dashboard/assets/Busy-CWRthsmc.js +0 -1
- data/vendor/assets/dashboard/assets/Cron-qeGr2YAm.js +0 -1
- data/vendor/assets/dashboard/assets/Dashboard-DQWaAwPA.js +0 -1
- data/vendor/assets/dashboard/assets/Dead-D9gOPHtK.js +0 -1
- data/vendor/assets/dashboard/assets/Extension-DONXVLdA.js +0 -1
- data/vendor/assets/dashboard/assets/FilterBox-BbB6gXHp.js +0 -2
- data/vendor/assets/dashboard/assets/FlowDetail-CrmnVRp0.js +0 -1
- data/vendor/assets/dashboard/assets/Flows-g4GC8KmE.js +0 -1
- data/vendor/assets/dashboard/assets/JobDetailModal-C86k_1wj.js +0 -2
- data/vendor/assets/dashboard/assets/Limiters-zyO-iMlK.js +0 -1
- data/vendor/assets/dashboard/assets/Metrics-Ckq5fNBv.js +0 -1
- data/vendor/assets/dashboard/assets/NotFound-C8NnyjlY.js +0 -1
- data/vendor/assets/dashboard/assets/PageHeader-vWeBbSPF.js +0 -1
- data/vendor/assets/dashboard/assets/Profiles-DRl-FXPA.js +0 -1
- data/vendor/assets/dashboard/assets/Queues-DkXWrihA.js +0 -1
- data/vendor/assets/dashboard/assets/Retries-CmlojYIy.js +0 -1
- data/vendor/assets/dashboard/assets/Scheduled-Cbbpp3xw.js +0 -1
- data/vendor/assets/dashboard/assets/Search-LaEpj2fM.js +0 -1
- data/vendor/assets/dashboard/assets/charts-D_PQLoyc.js +0 -1
- data/vendor/assets/dashboard/assets/fa-brands-400-Bs6tcqqs.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-regular-400-DRN8N0d1.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-solid-900-IAB4Droh.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-v4compatibility-CErXDOsT.woff2 +0 -0
- data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +0 -1
- data/vendor/assets/dashboard/assets/index--xvvNlYx.css +0 -1
- data/vendor/assets/dashboard/assets/index-FBKGtysy.js +0 -141
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-BiwhHqgh.js +0 -1
- data/vendor/assets/dashboard/assets/useSort-jBYVwXvg.js +0 -1
- data/vendor/assets/dashboard/assets/wurk-logo-Vy3xW4K0.png +0 -0
- data/vendor/assets/dashboard/og-image.png +0 -0
- data/vendor/assets/dashboard/wurk-logo.webp +0 -0
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 96bee6b140f21dd1b2fb9d8cf382f6942600a91896dcae3bde52bbcb615fd7ff
|
|
4
|
+
data.tar.gz: 0f0a74b458b2bc9de92759ee0c20a3ffa8ba9ca3fa40dcd891abe07afcb8a3db
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c46917c8d37f35ae24aec5c0f6305a05d1c2d85af050da7dfda1eff8f73f0537898dcc3957849bede6ad3128836fbfd3db1782397771e0d8af36096098948838
|
|
7
|
+
data.tar.gz: 59fd02cb8bbbf4e66b4ffda4e5260c6052ea1b0d7708f069afcdc7d22d88408b0fd72ff65f36958629d4b75cbf85cb2a95ce37a6e76358ff85013e7cbae5e818
|
data/README.md
CHANGED
|
@@ -305,7 +305,7 @@ Wurk is MIT and stays that way. If what you need is a commercial support contrac
|
|
|
305
305
|
|
|
306
306
|
`bundle install && restart`. The worker command stays as it was: Wurk ships `sidekiq` and `sidekiqswarm` executables, so `bundle exec sidekiq -C config/sidekiq.yml` in a Procfile or manifest keeps working. Wurk reads and writes the same Redis schema, so there is no data migration: Wurk picks up the queues, schedules, retries, batches and periodic loops Sidekiq left behind. Running Sidekiq and Wurk workers against one Redis *at the same time* is not tested yet, so a live production move is a **drain cutover** (stop Sidekiq's workers, confirm nothing is in flight, start Wurk), with a matching rollback; the [production cutover](https://github.com/developerz-ai/wurk/blob/main/docs/migrate-from-sidekiq.md#9-production-cutover) section has every step and command. The upstream suites of sidekiq-cron, sidekiq-unique-jobs, sidekiq-failures and sentry-sidekiq run against Wurk in the [`ecosystem` CI job](https://github.com/developerz-ai/wurk/blob/main/.github/workflows/ecosystem.yml) (see [`test/ecosystem/`](https://github.com/developerz-ai/wurk/tree/main/test/ecosystem)). sidekiq-scheduler, sidekiq-status and sidekiq-throttled are target additions tracked in [docs/idea/14-ecosystem-compat.md](https://github.com/developerz-ai/wurk/blob/main/docs/idea/14-ecosystem-compat.md).
|
|
307
307
|
|
|
308
|
-
**Keeping a `sidekiq-*` add-on gem?** Those gems declare `add_dependency "sidekiq"`, which would reinstall real Sidekiq next to Wurk. Satisfy it with Wurk's git-only [`sidekiq` shim gem](https://github.com/developerz-ai/wurk/tree/main/ecosystem/sidekiq-shim) instead, pinned to the tag of the Wurk release you run: `gem "sidekiq", github: "developerz-ai/wurk", tag: "v1.
|
|
308
|
+
**Keeping a `sidekiq-*` add-on gem?** Those gems declare `add_dependency "sidekiq"`, which would reinstall real Sidekiq next to Wurk. Satisfy it with Wurk's git-only [`sidekiq` shim gem](https://github.com/developerz-ai/wurk/tree/main/ecosystem/sidekiq-shim) instead, pinned to the tag of the Wurk release you run: `gem "sidekiq", github: "developerz-ai/wurk", tag: "v1.15.0", glob: "ecosystem/sidekiq-shim/*.gemspec"`. Only sidekiq-cron, sidekiq-unique-jobs, sidekiq-failures and sentry-sidekiq are exercised that way, by the `ecosystem` workflow; treat the others as untested.
|
|
309
309
|
|
|
310
310
|
Full walkthrough — config side-by-side, the Redis key/`sidekiq_options` mapping, known incompatibilities, moving sidekiq-cron schedules over (`rake wurk:import:cron`), and the production cutover and rollback playbook: **[docs/migrate-from-sidekiq.md](https://github.com/developerz-ai/wurk/blob/main/docs/migrate-from-sidekiq.md)**. On the night, keep the **[incident runbook](https://github.com/developerz-ai/wurk/blob/main/docs/runbook.md)** open.
|
|
311
311
|
|
|
@@ -6,12 +6,18 @@ module Wurk
|
|
|
6
6
|
# extension forms send no Rails authenticity token, so `protect_from_forgery`
|
|
7
7
|
# can't apply — guard unsafe methods with a same-origin check instead. A
|
|
8
8
|
# cross-site page can drive a victim's browser but cannot forge
|
|
9
|
-
# `Sec-Fetch-Site: same-origin` (the browser sets it)
|
|
10
|
-
# denied too, since only a non-browser client omits it and it carries no
|
|
11
|
-
# ambient session cookie to abuse.
|
|
9
|
+
# `Sec-Fetch-Site: same-origin` (the browser sets it).
|
|
12
10
|
#
|
|
13
|
-
#
|
|
14
|
-
#
|
|
11
|
+
# Browsers send Fetch Metadata only to potentially-trustworthy origins
|
|
12
|
+
# (HTTPS, localhost), so a dashboard over plain HTTP on a LAN host never sees
|
|
13
|
+
# it. Then the request's `Origin` — or, when absent, its `Referer` — must be
|
|
14
|
+
# exactly this request's scheme, host and port (owner decision U9, #101 W3).
|
|
15
|
+
# Neither header present is denied: only a non-browser client omits both,
|
|
16
|
+
# and it carries no ambient session cookie to abuse.
|
|
17
|
+
#
|
|
18
|
+
# The standalone Rack surface (Wurk::Web#safe_request?) applies the same
|
|
19
|
+
# Sec-Fetch-Site rule without this fallback: it stays strict, so a
|
|
20
|
+
# plain-HTTP standalone mount refuses unsafe requests (parity-divergences).
|
|
15
21
|
module SameOriginGuard
|
|
16
22
|
extend ActiveSupport::Concern
|
|
17
23
|
|
|
@@ -28,13 +34,29 @@ module Wurk
|
|
|
28
34
|
|
|
29
35
|
private
|
|
30
36
|
|
|
37
|
+
# The SPA's toast branches on this body to name the cause instead of
|
|
38
|
+
# reading the 403 as read-only mode.
|
|
31
39
|
def verify_same_origin!
|
|
32
|
-
|
|
40
|
+
render json: { error: 'cross_origin' }, status: :forbidden unless safe_request?
|
|
33
41
|
end
|
|
34
42
|
|
|
35
43
|
def safe_request?
|
|
36
|
-
SAFE_METHODS.include?(request.request_method)
|
|
37
|
-
|
|
44
|
+
return true if SAFE_METHODS.include?(request.request_method)
|
|
45
|
+
|
|
46
|
+
site = request.headers['Sec-Fetch-Site']
|
|
47
|
+
site ? site == 'same-origin' : same_origin_source?
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def same_origin_source?
|
|
51
|
+
source = request.headers['Origin'].presence || request.referer.presence
|
|
52
|
+
return false unless source
|
|
53
|
+
|
|
54
|
+
uri = ::URI.parse(source)
|
|
55
|
+
[uri.scheme, uri.host, uri.port] == own_origin
|
|
56
|
+
rescue ::URI::InvalidURIError
|
|
57
|
+
false
|
|
38
58
|
end
|
|
59
|
+
|
|
60
|
+
def own_origin = [request.scheme, request.host, request.port]
|
|
39
61
|
end
|
|
40
62
|
end
|
|
@@ -8,6 +8,40 @@ module Wurk
|
|
|
8
8
|
module SseStreaming
|
|
9
9
|
extend ActiveSupport::Concern
|
|
10
10
|
|
|
11
|
+
# One stats payload per process per TTL, shared by every open stream: ten
|
|
12
|
+
# dashboard tabs ticking every 2s cost one Redis read, not ten. Streams
|
|
13
|
+
# that tick together wait on the lock and take the payload the first one
|
|
14
|
+
# read. Keyed on the pool so a payload read from one Redis never answers
|
|
15
|
+
# for another; a raising read caches nothing.
|
|
16
|
+
class SharedTick
|
|
17
|
+
def initialize(ttl)
|
|
18
|
+
@ttl = ttl
|
|
19
|
+
@lock = ::Mutex.new
|
|
20
|
+
clear!
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def fetch
|
|
24
|
+
pool = ::Wurk.redis_pool
|
|
25
|
+
@lock.synchronize do
|
|
26
|
+
now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
27
|
+
return @payload if @payload && @pool.equal?(pool) && now - @read_at < @ttl
|
|
28
|
+
|
|
29
|
+
payload = yield.freeze
|
|
30
|
+
@payload = payload
|
|
31
|
+
@pool = pool
|
|
32
|
+
@read_at = now
|
|
33
|
+
payload
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def clear!
|
|
38
|
+
@lock.synchronize { @payload = @pool = @read_at = nil }
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Under the 2s tick, so one stream never reads its own previous payload.
|
|
43
|
+
SHARED_TICK = SharedTick.new(1.0)
|
|
44
|
+
|
|
11
45
|
private
|
|
12
46
|
|
|
13
47
|
def stream_headers!
|
|
@@ -43,7 +77,8 @@ module Wurk
|
|
|
43
77
|
end
|
|
44
78
|
|
|
45
79
|
def stream_tick_payload
|
|
46
|
-
::Wurk::Api::Serializers.stats_payload(::Wurk::Stats.new)
|
|
80
|
+
SHARED_TICK.fetch { ::Wurk::Api::Serializers.stats_payload(::Wurk::Stats::Snapshot.new) }
|
|
81
|
+
.merge(at: ::Time.now.to_f)
|
|
47
82
|
end
|
|
48
83
|
end
|
|
49
84
|
end
|
|
@@ -5,7 +5,8 @@ module Wurk
|
|
|
5
5
|
# Pagination helpers shared by every listing endpoint. The contract:
|
|
6
6
|
# * `?count=` page size (default 25, clamped to 1..200)
|
|
7
7
|
# * `?page=` 0-indexed page number
|
|
8
|
-
# * `?substr=` case-insensitive
|
|
8
|
+
# * `?substr=` case-insensitive filter on a row's klass, jid, error
|
|
9
|
+
# class, error message or args
|
|
9
10
|
#
|
|
10
11
|
# Helpers expect an Enumerable that yields whatever JSON-shaped Hash the
|
|
11
12
|
# caller built; substr filtering and slicing happen after serialization so
|
|
@@ -13,16 +14,26 @@ module Wurk
|
|
|
13
14
|
module Pagination
|
|
14
15
|
DEFAULT_PAGE_SIZE = 25
|
|
15
16
|
MAX_PAGE_SIZE = 200
|
|
16
|
-
#
|
|
17
|
-
#
|
|
18
|
-
#
|
|
19
|
-
# depth while bounding
|
|
17
|
+
# An unfiltered listing reads its page at the offset (the set's
|
|
18
|
+
# #snapshot), whatever the depth; the walks that remain (a filtered
|
|
19
|
+
# #slice, the Limits names) still pay per skipped row, and 1000 pages is
|
|
20
|
+
# far beyond any real UI depth while bounding them.
|
|
20
21
|
MAX_PAGE = 1_000
|
|
21
22
|
# A substr that matches nothing would otherwise stream the whole backing
|
|
22
|
-
# set through Ruby
|
|
23
|
-
#
|
|
24
|
-
#
|
|
25
|
-
FILTER_SCAN_LIMIT =
|
|
23
|
+
# set through Ruby, so a filtered request examines at most Web::Search's
|
|
24
|
+
# per-request budget of raw rows; results within it are exact, beyond it
|
|
25
|
+
# the page just comes back short.
|
|
26
|
+
FILTER_SCAN_LIMIT = ::Wurk::Web::Search::SCAN_BUDGET
|
|
27
|
+
# A filter walk reads the set through its #snapshot, starting at an
|
|
28
|
+
# ordinary listing page and doubling to FILTER_PAGE: a match near the
|
|
29
|
+
# top costs what it did, and the full budget costs ~25 round trips
|
|
30
|
+
# rather than one per 50 rows.
|
|
31
|
+
FIRST_FILTER_PAGE = 50
|
|
32
|
+
FILTER_PAGE = 1_000
|
|
33
|
+
# Entries one filtered `<set>/all/:cmd` request acts on, the cap the
|
|
34
|
+
# bulk `keys[]` action already has.
|
|
35
|
+
SWEEP_LIMIT = 1_000
|
|
36
|
+
MATCHED_FIELDS = %i[klass jid error_class error_message].freeze
|
|
26
37
|
|
|
27
38
|
module_function
|
|
28
39
|
|
|
@@ -46,6 +57,9 @@ module Wurk
|
|
|
46
57
|
fields.merge(filtered_total: page[:matched], filtered_total_exact: page[:exhausted])
|
|
47
58
|
end
|
|
48
59
|
|
|
60
|
+
# Rows before the requested page: where an unfiltered listing reads from.
|
|
61
|
+
def offset(page) = page[:page] * page[:count]
|
|
62
|
+
|
|
49
63
|
def clamp_int(value, min, max, default)
|
|
50
64
|
Integer(value, 10).clamp(min, max)
|
|
51
65
|
rescue ::ArgumentError, ::TypeError
|
|
@@ -58,50 +72,15 @@ module Wurk
|
|
|
58
72
|
default
|
|
59
73
|
end
|
|
60
74
|
|
|
61
|
-
#
|
|
62
|
-
#
|
|
63
|
-
#
|
|
64
|
-
#
|
|
65
|
-
#
|
|
66
|
-
#
|
|
67
|
-
#
|
|
68
|
-
#
|
|
69
|
-
|
|
70
|
-
def slice(enumerable, page, &)
|
|
71
|
-
offset = page[:page] * page[:count]
|
|
72
|
-
if page[:substr].empty?
|
|
73
|
-
slice_raw(enumerable, offset, page[:count], &)
|
|
74
|
-
else
|
|
75
|
-
slice_filtered(enumerable, offset, page, &)
|
|
76
|
-
end
|
|
77
|
-
end
|
|
78
|
-
|
|
79
|
-
# Fast path: skip serialization entirely for rows before the offset.
|
|
80
|
-
def slice_raw(enumerable, offset, count)
|
|
81
|
-
results = []
|
|
82
|
-
seen = 0
|
|
83
|
-
enumerable.each do |member|
|
|
84
|
-
if seen < offset
|
|
85
|
-
seen += 1
|
|
86
|
-
next
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
payload = yield(member)
|
|
90
|
-
next unless payload
|
|
91
|
-
|
|
92
|
-
results << payload
|
|
93
|
-
break if results.size >= count
|
|
94
|
-
end
|
|
95
|
-
results
|
|
96
|
-
end
|
|
97
|
-
|
|
98
|
-
# Filtered path: the offset counts MATCHING rows, so page boundaries
|
|
99
|
-
# stay stable (raw-index offsets re-emit or skip matches across pages).
|
|
100
|
-
# Records the matches seen in `page[:matched]` and whether the set was
|
|
101
|
-
# walked to its end in `page[:exhausted]`: stopping once the page fills
|
|
102
|
-
# is what keeps a filtered request cheap, so the count is only exact
|
|
103
|
-
# when nothing was left unread.
|
|
104
|
-
def slice_filtered(enumerable, offset, page)
|
|
75
|
+
# One page of a filtered listing: up to `count` payloads (the block maps
|
|
76
|
+
# each member to one; nil skips it) from the matches of `substr`, in
|
|
77
|
+
# `enumerable`'s order. The offset counts MATCHING rows, so page
|
|
78
|
+
# boundaries stay stable (raw-index offsets re-emit or skip matches
|
|
79
|
+
# across pages). Records the matches seen in `page[:matched]` and whether
|
|
80
|
+
# the set was walked to its end in `page[:exhausted]`: stopping once the
|
|
81
|
+
# page fills is what keeps a filtered request cheap, so the count is only
|
|
82
|
+
# exact when nothing was left unread.
|
|
83
|
+
def slice(enumerable, page)
|
|
105
84
|
results = []
|
|
106
85
|
matched = 0
|
|
107
86
|
examined = 0
|
|
@@ -111,7 +90,7 @@ module Wurk
|
|
|
111
90
|
payload = yield(member)
|
|
112
91
|
next unless payload && match?(payload, page[:substr])
|
|
113
92
|
|
|
114
|
-
results << payload if (matched += 1) > offset
|
|
93
|
+
results << payload if (matched += 1) > offset(page)
|
|
115
94
|
break if results.size >= page[:count]
|
|
116
95
|
end
|
|
117
96
|
record_match_count(page, matched, examined, results.size)
|
|
@@ -123,11 +102,51 @@ module Wurk
|
|
|
123
102
|
page[:exhausted] = examined <= FILTER_SCAN_LIMIT && served < page[:count]
|
|
124
103
|
end
|
|
125
104
|
|
|
105
|
+
# Up to SWEEP_LIMIT of `rows`' members whose row (the block maps a member
|
|
106
|
+
# to it) matches `substr`, within FILTER_SCAN_LIMIT examined:
|
|
107
|
+
# [members, rows examined, whether `rows` ran out].
|
|
108
|
+
def matching(rows, substr)
|
|
109
|
+
members = []
|
|
110
|
+
examined = 0
|
|
111
|
+
rows.each do |member|
|
|
112
|
+
return [members, examined, false] if examined >= FILTER_SCAN_LIMIT || members.size >= SWEEP_LIMIT
|
|
113
|
+
|
|
114
|
+
examined += 1
|
|
115
|
+
members << member if match?(yield(member), substr)
|
|
116
|
+
end
|
|
117
|
+
[members, examined, true]
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# Every row from raw offset `from` on, read page by page through the
|
|
121
|
+
# block, which answers `count` rows from `offset` in listing order.
|
|
122
|
+
# Stops after the first short page.
|
|
123
|
+
def walk(from = 0, &read)
|
|
124
|
+
::Enumerator.new do |out|
|
|
125
|
+
offset = from
|
|
126
|
+
size = FIRST_FILTER_PAGE
|
|
127
|
+
loop do
|
|
128
|
+
rows = read.call(offset, size)
|
|
129
|
+
rows.each { |row| out << row }
|
|
130
|
+
break if rows.size < size
|
|
131
|
+
|
|
132
|
+
offset += size
|
|
133
|
+
size = [size * 2, FILTER_PAGE].min
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
|
|
126
138
|
def match?(payload, substr)
|
|
127
139
|
return true if substr.nil? || substr.empty?
|
|
128
140
|
|
|
129
141
|
needle = substr.downcase
|
|
130
|
-
|
|
142
|
+
MATCHED_FIELDS.any? { |field| payload[field].to_s.downcase.include?(needle) } ||
|
|
143
|
+
args_match?(payload[:args], needle)
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# Against the args as the SPA shows them: JSON, so `"a@b.c"` or `42]`
|
|
147
|
+
# match as typed.
|
|
148
|
+
def args_match?(args, needle)
|
|
149
|
+
!args.nil? && ::Wurk.dump_json(args).downcase.include?(needle)
|
|
131
150
|
end
|
|
132
151
|
end
|
|
133
152
|
end
|
|
@@ -52,17 +52,23 @@ module Wurk
|
|
|
52
52
|
rows.empty? ? base : base.merge(custom_rows: rows)
|
|
53
53
|
end
|
|
54
54
|
|
|
55
|
+
# A listing row. No `error_backtrace`: it is stored compressed, and
|
|
56
|
+
# inflating it for every row of a page served a field only the detail
|
|
57
|
+
# view (#sorted_entry_detail) shows.
|
|
55
58
|
def sorted_entry(entry)
|
|
56
59
|
job_record(entry).merge(
|
|
57
60
|
score: entry.score,
|
|
58
61
|
at: entry.at.to_f,
|
|
59
62
|
error_class: entry['error_class'].to_s,
|
|
60
63
|
error_message: entry['error_message'].to_s,
|
|
61
|
-
retry_count: entry['retry_count']
|
|
62
|
-
error_backtrace: entry.error_backtrace
|
|
64
|
+
retry_count: entry['retry_count']
|
|
63
65
|
)
|
|
64
66
|
end
|
|
65
67
|
|
|
68
|
+
def sorted_entry_detail(entry)
|
|
69
|
+
sorted_entry(entry).merge(error_backtrace: entry.error_backtrace)
|
|
70
|
+
end
|
|
71
|
+
|
|
66
72
|
# `leader_identity` is the cluster's `dear-leader` value (ProcessSet#leader,
|
|
67
73
|
# memoized to one Redis GET per request) — comparing here avoids the N+1
|
|
68
74
|
# `Process#leader?` GET-per-row that a per-process lookup would cost.
|
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative 'api/serializers'
|
|
4
4
|
require_relative 'api/pagination'
|
|
5
|
-
require 'wurk/web'
|
|
6
5
|
|
|
7
6
|
module Wurk
|
|
8
7
|
# JSON APIs consumed by the SolidJS SPA. Action methods stay thin; mapping to
|
|
@@ -36,6 +35,10 @@ module Wurk
|
|
|
36
35
|
|
|
37
36
|
HISTORY_WINDOW_UNITS = { 's' => 1, 'm' => 60, 'h' => 3600, 'd' => 86_400 }.freeze
|
|
38
37
|
DEFAULT_HISTORY_WINDOW = 24 * 3600
|
|
38
|
+
# Sidekiq Web's dashboard `?days=N` range (free §25.4).
|
|
39
|
+
MAX_HISTORY_DAYS = 180
|
|
40
|
+
# Far past any real set; bounds a `?cursor=` before it reaches LRANGE.
|
|
41
|
+
MAX_CURSOR = 1 << 40
|
|
39
42
|
|
|
40
43
|
# Per-set action whitelists. Maps the SPA's action name to the
|
|
41
44
|
# SortedEntry/JobSet method. Anything not listed 400s — keeps the bulk/
|
|
@@ -58,22 +61,24 @@ module Wurk
|
|
|
58
61
|
end
|
|
59
62
|
|
|
60
63
|
def stats
|
|
61
|
-
render json: ::Wurk::Api::Serializers.stats_payload(::Wurk::Stats.new)
|
|
64
|
+
render json: ::Wurk::Api::Serializers.stats_payload(::Wurk::Stats::Snapshot.new)
|
|
62
65
|
end
|
|
63
66
|
|
|
64
67
|
def queues
|
|
65
|
-
|
|
66
|
-
render json:
|
|
67
|
-
::Wurk::Api::Serializers.queue_summary(q).merge(consumers: consumers.fetch(q.name, 0))
|
|
68
|
+
snapshot = ::Wurk::Stats::Snapshot.new(consumers: true)
|
|
69
|
+
render json: snapshot.queue_summaries.map { |q|
|
|
70
|
+
::Wurk::Api::Serializers.queue_summary(q).merge(consumers: snapshot.consumers.fetch(q.name, 0))
|
|
68
71
|
}
|
|
69
72
|
end
|
|
70
73
|
|
|
74
|
+
# `?direction=asc` lists oldest first, the job the next fetch takes at the
|
|
75
|
+
# top, as Sidekiq Web's queue page does (free §25.4).
|
|
71
76
|
def queue
|
|
72
77
|
q = ::Wurk::Queue.new(params[:name].to_s)
|
|
73
78
|
page = ::Wurk::Api::Pagination.window(params)
|
|
74
|
-
jobs =
|
|
79
|
+
gauges, jobs = queue_page(q, page, params[:direction].to_s == 'asc')
|
|
75
80
|
render json: {
|
|
76
|
-
name: q.name, size:
|
|
81
|
+
name: q.name, size: gauges.size, latency: gauges.latency, paused: gauges.paused?,
|
|
77
82
|
**::Wurk::Api::Pagination.meta(page), jobs: jobs
|
|
78
83
|
}
|
|
79
84
|
end
|
|
@@ -109,12 +114,21 @@ module Wurk
|
|
|
109
114
|
def scheduled = render_sorted_set(::Wurk::ScheduledSet.new)
|
|
110
115
|
def dead = render_sorted_set(::Wurk::DeadSet.new)
|
|
111
116
|
|
|
117
|
+
# One entry with its backtrace, which listing rows leave out: inflating
|
|
118
|
+
# it costs a base64 decode, an inflate and a JSON parse per row, for a
|
|
119
|
+
# field only the detail view shows. `:key` is "<score>|<jid>".
|
|
120
|
+
def retry_entry = render_sorted_entry(::Wurk::RetrySet.new)
|
|
121
|
+
def scheduled_entry = render_sorted_entry(::Wurk::ScheduledSet.new)
|
|
122
|
+
def dead_entry = render_sorted_entry(::Wurk::DeadSet.new)
|
|
123
|
+
|
|
112
124
|
# --- Retry set mutations -------------------------------------------------
|
|
113
125
|
def retry_job = single_entry_action(::Wurk::RetrySet.new, RETRY_ACTIONS, params[:cmd])
|
|
114
126
|
def retries_bulk = bulk_entry_action(::Wurk::RetrySet.new, RETRY_ACTIONS)
|
|
115
127
|
|
|
116
128
|
def retries_all
|
|
117
129
|
set = ::Wurk::RetrySet.new
|
|
130
|
+
return sweep_matching(set, RETRY_ACTIONS) if filtered_sweep?
|
|
131
|
+
|
|
118
132
|
count = case params[:cmd].to_s
|
|
119
133
|
when 'retry' then set.retry_all
|
|
120
134
|
when 'kill' then set.kill_all
|
|
@@ -130,6 +144,8 @@ module Wurk
|
|
|
130
144
|
|
|
131
145
|
def scheduled_all
|
|
132
146
|
set = ::Wurk::ScheduledSet.new
|
|
147
|
+
return sweep_matching(set, SCHEDULED_ACTIONS) if filtered_sweep?
|
|
148
|
+
|
|
133
149
|
case params[:cmd].to_s
|
|
134
150
|
when 'delete' then render json: { ok: true, count: clear_set(set) }
|
|
135
151
|
when 'add_to_queue' then drain_set(set, :add_to_queue)
|
|
@@ -143,6 +159,8 @@ module Wurk
|
|
|
143
159
|
|
|
144
160
|
def dead_all
|
|
145
161
|
set = ::Wurk::DeadSet.new
|
|
162
|
+
return sweep_matching(set, DEAD_ACTIONS) if filtered_sweep?
|
|
163
|
+
|
|
146
164
|
count = case params[:cmd].to_s
|
|
147
165
|
when 'retry' then set.retry_all
|
|
148
166
|
when 'delete' then clear_set(set)
|
|
@@ -153,8 +171,9 @@ module Wurk
|
|
|
153
171
|
|
|
154
172
|
def processes
|
|
155
173
|
set = ::Wurk::ProcessSet.new
|
|
174
|
+
live = set.to_a # before #leader: the walk reads `dear-leader` in its pipeline
|
|
156
175
|
leader_identity = set.leader
|
|
157
|
-
render json:
|
|
176
|
+
render json: live.map { |p| ::Wurk::Api::Serializers.process_row(p, leader_identity: leader_identity) }
|
|
158
177
|
end
|
|
159
178
|
|
|
160
179
|
# Currently-executing jobs across the cluster (WorkSet), oldest first.
|
|
@@ -170,11 +189,16 @@ module Wurk
|
|
|
170
189
|
def quiet_process = signal_processes(:quiet!)
|
|
171
190
|
def stop_process = signal_processes(:stop!)
|
|
172
191
|
|
|
192
|
+
# Rows are Batch::Status#summary: #data with the failed and dead jid sets
|
|
193
|
+
# counted rather than listed. GET /api/batches/:bid carries the jids.
|
|
194
|
+
# `?dead=1` lists `dead-batches` instead (Pro's Dead batches tab);
|
|
195
|
+
# `?tag=` narrows either to the batches carrying that tag, and takes the
|
|
196
|
+
# place of `?substr=` (BatchSet#tagged_snapshot).
|
|
173
197
|
def batches
|
|
174
|
-
set = ::Wurk::BatchSet.new
|
|
175
198
|
page = ::Wurk::Api::Pagination.window(params)
|
|
176
|
-
|
|
177
|
-
|
|
199
|
+
set = %w[1 true].include?(params[:dead].to_s) ? ::Wurk::Batch::DeadSet.new : ::Wurk::BatchSet.new
|
|
200
|
+
total, rows = batch_rows(set, page, params[:tag].to_s)
|
|
201
|
+
render json: { total: total, **::Wurk::Api::Pagination.meta(page), batches: rows }
|
|
178
202
|
end
|
|
179
203
|
|
|
180
204
|
def batch
|
|
@@ -187,10 +211,9 @@ module Wurk
|
|
|
187
211
|
end
|
|
188
212
|
|
|
189
213
|
def flows
|
|
190
|
-
set = ::Wurk::FlowSet.new
|
|
191
214
|
page = ::Wurk::Api::Pagination.window(params)
|
|
192
|
-
rows = ::Wurk::
|
|
193
|
-
render json: { total:
|
|
215
|
+
total, rows = page_rows(::Wurk::FlowSet.new, page) { |status| ::Wurk::Api::Serializers.flow_row(status) }
|
|
216
|
+
render json: { total: total, **::Wurk::Api::Pagination.meta(page), flows: rows }
|
|
194
217
|
end
|
|
195
218
|
|
|
196
219
|
def flow
|
|
@@ -212,9 +235,12 @@ module Wurk
|
|
|
212
235
|
end
|
|
213
236
|
|
|
214
237
|
def limiters
|
|
215
|
-
names = ::Wurk::Web::Enterprise::Limits.list(filter: params[:substr])
|
|
216
238
|
page = ::Wurk::Api::Pagination.window(params)
|
|
217
|
-
|
|
239
|
+
total, rows = ::Wurk::Web::Enterprise::Limits.page(
|
|
240
|
+
filter: params[:substr], offset: ::Wurk::Api::Pagination.offset(page), count: page[:count]
|
|
241
|
+
)
|
|
242
|
+
render json: { total: total, **::Wurk::Api::Pagination.meta(page),
|
|
243
|
+
limiters: rows.map { |name, meta| ::Wurk::Api::Serializers.limiter_row(name, meta) } }
|
|
218
244
|
end
|
|
219
245
|
|
|
220
246
|
def reset_limiter
|
|
@@ -272,10 +298,11 @@ module Wurk
|
|
|
272
298
|
end
|
|
273
299
|
|
|
274
300
|
# Cluster-total throughput/failures time-series for the dashboard charts.
|
|
275
|
-
# `:bucket` is 1m/5m/1h; `?window=24h` (s/m/h/d suffix)
|
|
276
|
-
# bucket's retention.
|
|
301
|
+
# `:bucket` is 1m/5m/1h; `?window=24h` (s/m/h/d suffix) or `?days=N`
|
|
302
|
+
# (1..180, wins over window) is clamped to the bucket's retention.
|
|
303
|
+
# Chart-ready array under `series`.
|
|
277
304
|
def history
|
|
278
|
-
window =
|
|
305
|
+
window = history_window(params)
|
|
279
306
|
series = ::Wurk::Web::Enterprise::Historical.history(params[:bucket].to_s, window: window)
|
|
280
307
|
render json: { bucket: params[:bucket].to_s, window: window, series: series.map { |row| ::Wurk::Api::Serializers.history_point(row) } }
|
|
281
308
|
rescue ::ArgumentError => e
|
|
@@ -295,10 +322,10 @@ module Wurk
|
|
|
295
322
|
|
|
296
323
|
# Per-queue size/latency gauge time-series for the Metrics/Historical tab.
|
|
297
324
|
# `:bucket` is 1m/5m/1h; `?window=24h` (s/m/h/d) is clamped to the bucket's
|
|
298
|
-
# retention; optional `?queue=<name>` narrows
|
|
299
|
-
# `points` are chart-ready.
|
|
325
|
+
# retention (`?days=N` as on #history); optional `?queue=<name>` narrows
|
|
326
|
+
# to one queue. Each queue's `points` are chart-ready.
|
|
300
327
|
def queue_history # rubocop:disable Metrics/AbcSize
|
|
301
|
-
window =
|
|
328
|
+
window = history_window(params)
|
|
302
329
|
queues = params[:queue].present? ? [params[:queue].to_s] : nil
|
|
303
330
|
series = ::Wurk::Web::Enterprise::Historical.queue_history(params[:bucket].to_s, window: window, queues: queues)
|
|
304
331
|
render json: {
|
|
@@ -401,7 +428,7 @@ module Wurk
|
|
|
401
428
|
return render(json: { error: 'unknown action' }, status: :bad_request) unless method
|
|
402
429
|
|
|
403
430
|
keys = Array(params[:keys]).map(&:to_s).uniq
|
|
404
|
-
# Each key costs a ZRANGEBYSCORE plus per-entry mutations; an uncapped
|
|
431
|
+
# Each key costs a pipelined ZRANGEBYSCORE plus per-entry mutations; an uncapped
|
|
405
432
|
# list lets one POST issue tens of thousands of round-trips. The UI's
|
|
406
433
|
# per-row selections accumulate across pages (toggleAll works against
|
|
407
434
|
# the visible page's keys only) and the bulk POST sends the whole Set,
|
|
@@ -431,8 +458,8 @@ module Wurk
|
|
|
431
458
|
|
|
432
459
|
# 422 when any entry failed, so the SPA's error toast fires; `count` is
|
|
433
460
|
# what was applied either way.
|
|
434
|
-
def render_applied(total, failures)
|
|
435
|
-
body = { ok: failures.empty?, count: total - failures.size }
|
|
461
|
+
def render_applied(total, failures, extra = {})
|
|
462
|
+
body = { ok: failures.empty?, count: total - failures.size, **extra }
|
|
436
463
|
return render(json: body) if failures.empty?
|
|
437
464
|
|
|
438
465
|
render json: body.merge(failed: failures), status: 422
|
|
@@ -459,13 +486,50 @@ module Wurk
|
|
|
459
486
|
# Maps "<score>|<jid>" keys to live SortedEntry objects via score-bracketed
|
|
460
487
|
# fetch (exact float match, narrowed by jid) — avoids depending on float→
|
|
461
488
|
# string round-tripping between JS and Ruby. Skips malformed/empty keys.
|
|
489
|
+
# One pipelined round trip for every key (JobSet#fetch_pairs).
|
|
462
490
|
def entries_for(set, keys)
|
|
463
|
-
keys.
|
|
491
|
+
pairs = keys.filter_map do |key|
|
|
464
492
|
score, jid = key.to_s.split('|', 2)
|
|
465
|
-
|
|
493
|
+
[score.to_f, jid] unless jid.nil? || jid.empty?
|
|
494
|
+
end
|
|
495
|
+
set.fetch_pairs(pairs)
|
|
496
|
+
end
|
|
497
|
+
|
|
498
|
+
def batch_rows(set, page, tag)
|
|
499
|
+
symbolize = ->(summary) { summary.transform_keys(&:to_sym) }
|
|
500
|
+
return page_rows(set, page, &symbolize) if tag.empty?
|
|
466
501
|
|
|
467
|
-
|
|
502
|
+
total, summaries = set.tagged_snapshot(tag, ::Wurk::Api::Pagination.offset(page), page[:count])
|
|
503
|
+
[total, summaries.map(&symbolize)]
|
|
504
|
+
end
|
|
505
|
+
|
|
506
|
+
def filtered_sweep? = !params[:substr].to_s.empty?
|
|
507
|
+
|
|
508
|
+
# `<set>/all/:cmd?substr=`: applies `cmd` to the entries the same filtered
|
|
509
|
+
# listing shows, newest first, at most SWEEP_LIMIT of them within the
|
|
510
|
+
# filter's scan budget per request. The answer carries `done` (the walk
|
|
511
|
+
# reached the end of the set) and `cursor`, the raw offset to send back as
|
|
512
|
+
# `?cursor=` to carry on: the rows examined minus those this request moved
|
|
513
|
+
# out of the set. Entries added or removed meanwhile above that offset
|
|
514
|
+
# shift it, so a resumed sweep can re-examine or skip a few rows.
|
|
515
|
+
def sweep_matching(set, actions)
|
|
516
|
+
method = actions[params[:cmd].to_s]
|
|
517
|
+
return render(json: { error: 'unknown action' }, status: :bad_request) unless method
|
|
518
|
+
|
|
519
|
+
entries, scanned_to, done = matching_entries(set)
|
|
520
|
+
failures = apply_each(entries, method)
|
|
521
|
+
cursor = scanned_to - (entries.size - failures.size)
|
|
522
|
+
render_applied(entries.size, failures, matched: entries.size, done: done, cursor: cursor)
|
|
523
|
+
end
|
|
524
|
+
|
|
525
|
+
# [the matches from `?cursor=`, the raw offset the walk stopped at, done].
|
|
526
|
+
def matching_entries(set)
|
|
527
|
+
from = ::Wurk::Api::Pagination.clamp_int(params[:cursor], 0, MAX_CURSOR, 0)
|
|
528
|
+
rows = ::Wurk::Api::Pagination.walk(from) { |offset, count| set.snapshot(offset, count).last }
|
|
529
|
+
entries, examined, done = ::Wurk::Api::Pagination.matching(rows, params[:substr].to_s) do |entry|
|
|
530
|
+
::Wurk::Api::Serializers.sorted_entry(entry)
|
|
468
531
|
end
|
|
532
|
+
[entries, from + examined, done]
|
|
469
533
|
end
|
|
470
534
|
|
|
471
535
|
# UNLINKs the whole set, returning the count removed (read before clearing
|
|
@@ -489,18 +553,49 @@ module Wurk
|
|
|
489
553
|
|
|
490
554
|
def render_sorted_set(set)
|
|
491
555
|
page = ::Wurk::Api::Pagination.window(params)
|
|
492
|
-
total = set.
|
|
493
|
-
entries = ::Wurk::Api::Pagination.slice(set, page) { |entry| ::Wurk::Api::Serializers.sorted_entry(entry) }
|
|
556
|
+
total, entries = page_rows(set, page) { |entry| ::Wurk::Api::Serializers.sorted_entry(entry) }
|
|
494
557
|
render json: { total: total, **::Wurk::Api::Pagination.meta(page), entries: entries }
|
|
495
558
|
end
|
|
496
559
|
|
|
497
|
-
#
|
|
498
|
-
#
|
|
499
|
-
#
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
560
|
+
# [total, rows] of one listing page. Unfiltered, the page is read straight
|
|
561
|
+
# off Redis at its offset (the set's #snapshot), so page 999 costs what
|
|
562
|
+
# page 0 does; a filter counts matching rows, so it walks the set through
|
|
563
|
+
# the same #snapshot, bounded by FILTER_SCAN_LIMIT.
|
|
564
|
+
def page_rows(set, page, &)
|
|
565
|
+
if page[:substr].empty?
|
|
566
|
+
total, members = set.snapshot(::Wurk::Api::Pagination.offset(page), page[:count])
|
|
567
|
+
return [total, members.map(&)]
|
|
503
568
|
end
|
|
569
|
+
|
|
570
|
+
rows = ::Wurk::Api::Pagination.walk { |offset, count| set.snapshot(offset, count).last }
|
|
571
|
+
[set.size, ::Wurk::Api::Pagination.slice(rows, page, &)]
|
|
572
|
+
end
|
|
573
|
+
|
|
574
|
+
# The gauges (anything answering size/latency/paused?) and the page's rows.
|
|
575
|
+
def queue_page(queue, page, oldest_first)
|
|
576
|
+
serialize = ->(rec) { ::Wurk::Api::Serializers.job_record(rec) }
|
|
577
|
+
unless page[:substr].empty?
|
|
578
|
+
rows = ::Wurk::Api::Pagination.walk { |offset, count| queue_records(queue, offset, count, oldest_first).last }
|
|
579
|
+
return [queue, ::Wurk::Api::Pagination.slice(rows, page, &serialize)]
|
|
580
|
+
end
|
|
581
|
+
|
|
582
|
+
snapshot, records = queue_records(queue, ::Wurk::Api::Pagination.offset(page), page[:count], oldest_first)
|
|
583
|
+
[snapshot, records.map(&serialize)]
|
|
584
|
+
end
|
|
585
|
+
|
|
586
|
+
# [snapshot, its records in listing order]. Oldest first reads the same
|
|
587
|
+
# LRANGE from the tail: negative indexes `-(offset + count)..-(offset + 1)`,
|
|
588
|
+
# which Redis clamps at the head, then reversed.
|
|
589
|
+
def queue_records(queue, offset, count, oldest_first)
|
|
590
|
+
snapshot = queue.snapshot(oldest_first ? -(offset + count) : offset, count)
|
|
591
|
+
[snapshot, oldest_first ? snapshot.records.reverse : snapshot.records]
|
|
592
|
+
end
|
|
593
|
+
|
|
594
|
+
def render_sorted_entry(set)
|
|
595
|
+
entry = entries_for(set, [params[:key]]).first
|
|
596
|
+
return render(json: { error: 'unknown job' }, status: :not_found) unless entry
|
|
597
|
+
|
|
598
|
+
render json: ::Wurk::Api::Serializers.sorted_entry_detail(entry)
|
|
504
599
|
end
|
|
505
600
|
|
|
506
601
|
def render_cron_action(success)
|
|
@@ -519,6 +614,13 @@ module Wurk
|
|
|
519
614
|
)
|
|
520
615
|
end
|
|
521
616
|
|
|
617
|
+
# `?days=N` (1..MAX_HISTORY_DAYS) in seconds when it parses, else
|
|
618
|
+
# `?window=`.
|
|
619
|
+
def history_window(params)
|
|
620
|
+
days = ::Wurk::Api::Pagination.clamp_int(params[:days], 1, MAX_HISTORY_DAYS, nil)
|
|
621
|
+
days ? days * HISTORY_WINDOW_UNITS.fetch('d') : parse_window(params[:window])
|
|
622
|
+
end
|
|
623
|
+
|
|
522
624
|
# Resolves the per-class metrics window. `minutes:` wins when present;
|
|
523
625
|
# `hours:` is used otherwise; default falls back to 60 minutes so callers
|
|
524
626
|
# that pass neither still get a useful series.
|