wurk 1.13.0 → 1.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (279) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -1
  3. data/app/controllers/concerns/wurk/same_origin_guard.rb +30 -8
  4. data/app/controllers/concerns/wurk/sse_streaming.rb +36 -1
  5. data/app/controllers/wurk/api/pagination.rb +74 -55
  6. data/app/controllers/wurk/api/serializers.rb +8 -2
  7. data/app/controllers/wurk/api_controller.rb +138 -36
  8. data/app/controllers/wurk/application_controller.rb +0 -2
  9. data/app/controllers/wurk/dashboard_controller.rb +26 -3
  10. data/app/controllers/wurk/profiles_controller.rb +28 -23
  11. data/config/routes.rb +13 -7
  12. data/lib/sidekiq/cli.rb +4 -3
  13. data/lib/sidekiq/deploy.rb +2 -1
  14. data/lib/sidekiq/embedded.rb +2 -1
  15. data/lib/sidekiq/iterable_job.rb +2 -1
  16. data/lib/sidekiq/job/iterable/active_record_enumerator.rb +2 -1
  17. data/lib/sidekiq/job/iterable/csv_enumerator.rb +2 -1
  18. data/lib/sidekiq/job/iterable.rb +2 -1
  19. data/lib/sidekiq/launcher.rb +2 -1
  20. data/lib/sidekiq/manager.rb +2 -1
  21. data/lib/sidekiq/metrics/query.rb +2 -1
  22. data/lib/sidekiq/pro/web.rb +2 -1
  23. data/lib/sidekiq/scheduled.rb +2 -1
  24. data/lib/sidekiq/web.rb +2 -1
  25. data/lib/sidekiq.rb +3 -2
  26. data/lib/wurk/api/admin.rb +84 -0
  27. data/lib/wurk/api/app.rb +7 -2
  28. data/lib/wurk/api/fast.rb +4 -18
  29. data/lib/wurk/api/jobs.rb +11 -3
  30. data/lib/wurk/api/queues.rb +22 -10
  31. data/lib/wurk/api/router.rb +6 -0
  32. data/lib/wurk/api/serializers.rb +6 -7
  33. data/lib/wurk/api/swarm.rb +0 -1
  34. data/lib/wurk/api/validation.rb +17 -0
  35. data/lib/wurk/batch/callbacks.rb +9 -2
  36. data/lib/wurk/batch/server_middleware.rb +4 -0
  37. data/lib/wurk/batch/status.rb +45 -22
  38. data/lib/wurk/batch.rb +15 -4
  39. data/lib/wurk/batch_set.rb +56 -0
  40. data/lib/wurk/cli.rb +21 -21
  41. data/lib/wurk/client/buffered.rb +198 -71
  42. data/lib/wurk/client.rb +21 -1
  43. data/lib/wurk/compat/deploy.rb +7 -0
  44. data/lib/wurk/compat/encryption.rb +7 -0
  45. data/lib/wurk/compat/flow.rb +8 -0
  46. data/lib/wurk/compat/iterable_job.rb +8 -0
  47. data/lib/wurk/compat/profile_set.rb +8 -0
  48. data/lib/wurk/compat/server.rb +15 -0
  49. data/lib/wurk/compat/web.rb +20 -0
  50. data/lib/wurk/compat.rb +34 -40
  51. data/lib/wurk/configuration.rb +74 -9
  52. data/lib/wurk/cron.rb +146 -47
  53. data/lib/wurk/dead_set.rb +10 -6
  54. data/lib/wurk/embedded.rb +0 -2
  55. data/lib/wurk/engine.rb +46 -4
  56. data/lib/wurk/error_coalescer.rb +21 -1
  57. data/lib/wurk/fetcher/many_queues.rb +18 -6
  58. data/lib/wurk/fetcher/poison_pill.rb +3 -1
  59. data/lib/wurk/fetcher/reaper.rb +3 -5
  60. data/lib/wurk/fetcher/reliable.rb +64 -31
  61. data/lib/wurk/fetcher/self_reconcile.rb +0 -1
  62. data/lib/wurk/fetcher/unit_of_work.rb +16 -1
  63. data/lib/wurk/flow/status.rb +10 -4
  64. data/lib/wurk/flow_set.rb +24 -2
  65. data/lib/wurk/health.rb +6 -2
  66. data/lib/wurk/health_probe.rb +45 -7
  67. data/lib/wurk/heartbeat.rb +42 -13
  68. data/lib/wurk/import/sidekiq_cron.rb +0 -1
  69. data/lib/wurk/job_record.rb +2 -1
  70. data/lib/wurk/job_retry.rb +1 -1
  71. data/lib/wurk/job_set.rb +157 -9
  72. data/lib/wurk/launcher.rb +0 -12
  73. data/lib/wurk/lazy_loader.rb +60 -0
  74. data/lib/wurk/leader.rb +2 -0
  75. data/lib/wurk/limiter/bucket.rb +0 -2
  76. data/lib/wurk/limiter/concurrent.rb +19 -36
  77. data/lib/wurk/limiter/leaky.rb +0 -2
  78. data/lib/wurk/limiter/points.rb +0 -2
  79. data/lib/wurk/limiter/server_middleware.rb +4 -1
  80. data/lib/wurk/limiter/unlimited.rb +0 -2
  81. data/lib/wurk/limiter/window.rb +0 -2
  82. data/lib/wurk/limiter.rb +0 -7
  83. data/lib/wurk/logger.rb +44 -5
  84. data/lib/wurk/lua/fast_delete.rb +58 -0
  85. data/lib/wurk/lua/fast_delete_window.lua +46 -0
  86. data/lib/wurk/lua/limiter_concurrent_acquire.lua +10 -3
  87. data/lib/wurk/lua/limiter_concurrent_release.lua +13 -4
  88. data/lib/wurk/lua.rb +1 -41
  89. data/lib/wurk/metrics/accumulator.rb +27 -2
  90. data/lib/wurk/metrics/dashboard_series.rb +38 -18
  91. data/lib/wurk/metrics/exposition.rb +45 -0
  92. data/lib/wurk/metrics/history.rb +30 -12
  93. data/lib/wurk/metrics/pool_registry.rb +27 -0
  94. data/lib/wurk/metrics/process_counters.rb +16 -0
  95. data/lib/wurk/metrics/process_families.rb +115 -0
  96. data/lib/wurk/metrics/prometheus.rb +84 -52
  97. data/lib/wurk/metrics/query.rb +0 -2
  98. data/lib/wurk/metrics/queue_rollup.rb +33 -11
  99. data/lib/wurk/metrics/statsd.rb +4 -0
  100. data/lib/wurk/middleware/chain.rb +25 -3
  101. data/lib/wurk/middleware/expiry.rb +4 -0
  102. data/lib/wurk/middleware/interrupt_handler.rb +4 -1
  103. data/lib/wurk/middleware/status.rb +4 -0
  104. data/lib/wurk/middleware/timeout.rb +4 -0
  105. data/lib/wurk/preflight/owners.rb +0 -1
  106. data/lib/wurk/preflight/payload_sample.rb +0 -1
  107. data/lib/wurk/preflight/verify.rb +0 -2
  108. data/lib/wurk/process_set.rb +6 -1
  109. data/lib/wurk/profiler.rb +2 -1
  110. data/lib/wurk/queue.rb +33 -6
  111. data/lib/wurk/queue_consumers.rb +10 -1
  112. data/lib/wurk/rails_boot.rb +2 -2
  113. data/lib/wurk/railtie.rb +3 -4
  114. data/lib/wurk/redis_pool.rb +6 -1
  115. data/lib/wurk/scheduled.rb +7 -0
  116. data/lib/wurk/sentry/error_handler.rb +5 -0
  117. data/lib/wurk/sentry/middleware.rb +4 -0
  118. data/lib/wurk/sorted_entry.rb +16 -12
  119. data/lib/wurk/stats.rb +85 -1
  120. data/lib/wurk/swarm/child_boot.rb +6 -3
  121. data/lib/wurk/swarm/crash_watch.rb +5 -0
  122. data/lib/wurk/swarm/liveness.rb +0 -2
  123. data/lib/wurk/swarm/memory_watch.rb +110 -0
  124. data/lib/wurk/swarm/prefork.rb +54 -0
  125. data/lib/wurk/swarm/restart.rb +14 -5
  126. data/lib/wurk/swarm/restart_counters.rb +79 -0
  127. data/lib/wurk/swarm.rb +46 -47
  128. data/lib/wurk/version.rb +1 -1
  129. data/lib/wurk/web/enterprise.rb +63 -7
  130. data/lib/wurk.rb +34 -29
  131. data/vendor/assets/dashboard/assets/ArgsValue-D10BT0xi.js +1 -0
  132. data/vendor/assets/dashboard/assets/ArgsValue-D10BT0xi.js.br +0 -0
  133. data/vendor/assets/dashboard/assets/ArgsValue-D10BT0xi.js.gz +0 -0
  134. data/vendor/assets/dashboard/assets/BatchDetail-ByPRHv86.js +1 -0
  135. data/vendor/assets/dashboard/assets/BatchDetail-ByPRHv86.js.br +0 -0
  136. data/vendor/assets/dashboard/assets/BatchDetail-ByPRHv86.js.gz +0 -0
  137. data/vendor/assets/dashboard/assets/Batches-CVVOJFhz.js +1 -0
  138. data/vendor/assets/dashboard/assets/Batches-CVVOJFhz.js.br +0 -0
  139. data/vendor/assets/dashboard/assets/Batches-CVVOJFhz.js.gz +0 -0
  140. data/vendor/assets/dashboard/assets/Busy-Dp26yz1A.js +1 -0
  141. data/vendor/assets/dashboard/assets/Busy-Dp26yz1A.js.br +0 -0
  142. data/vendor/assets/dashboard/assets/Busy-Dp26yz1A.js.gz +0 -0
  143. data/vendor/assets/dashboard/assets/Cron-BiRV6eoU.js +1 -0
  144. data/vendor/assets/dashboard/assets/Cron-BiRV6eoU.js.br +0 -0
  145. data/vendor/assets/dashboard/assets/Cron-BiRV6eoU.js.gz +0 -0
  146. data/vendor/assets/dashboard/assets/Dashboard-CnpCQ8Of.js +1 -0
  147. data/vendor/assets/dashboard/assets/Dashboard-CnpCQ8Of.js.br +0 -0
  148. data/vendor/assets/dashboard/assets/Dashboard-CnpCQ8Of.js.gz +0 -0
  149. data/vendor/assets/dashboard/assets/Dead-ykYbsK8n.js +1 -0
  150. data/vendor/assets/dashboard/assets/Dead-ykYbsK8n.js.br +0 -0
  151. data/vendor/assets/dashboard/assets/Dead-ykYbsK8n.js.gz +0 -0
  152. data/vendor/assets/dashboard/assets/Extension-BUsciXhN.js +1 -0
  153. data/vendor/assets/dashboard/assets/Extension-BUsciXhN.js.br +0 -0
  154. data/vendor/assets/dashboard/assets/Extension-BUsciXhN.js.gz +0 -0
  155. data/vendor/assets/dashboard/assets/FilterBox-DBIZXNtY.js +1 -0
  156. data/vendor/assets/dashboard/assets/FilterBox-DBIZXNtY.js.br +0 -0
  157. data/vendor/assets/dashboard/assets/FilterBox-DBIZXNtY.js.gz +0 -0
  158. data/vendor/assets/dashboard/assets/FlowDetail-CIGu7pRb.js +1 -0
  159. data/vendor/assets/dashboard/assets/FlowDetail-CIGu7pRb.js.br +0 -0
  160. data/vendor/assets/dashboard/assets/FlowDetail-CIGu7pRb.js.gz +0 -0
  161. data/vendor/assets/dashboard/assets/{FlowState-yb7virqR.js → FlowState-B04ZLDe-.js} +1 -1
  162. data/vendor/assets/dashboard/assets/FlowState-B04ZLDe-.js.br +0 -0
  163. data/vendor/assets/dashboard/assets/FlowState-B04ZLDe-.js.gz +0 -0
  164. data/vendor/assets/dashboard/assets/Flows-Ce9Qoq0c.js +1 -0
  165. data/vendor/assets/dashboard/assets/Flows-Ce9Qoq0c.js.br +0 -0
  166. data/vendor/assets/dashboard/assets/Flows-Ce9Qoq0c.js.gz +0 -0
  167. data/vendor/assets/dashboard/assets/JobDetailModal-DWp0OMoV.js +2 -0
  168. data/vendor/assets/dashboard/assets/JobDetailModal-DWp0OMoV.js.br +0 -0
  169. data/vendor/assets/dashboard/assets/JobDetailModal-DWp0OMoV.js.gz +0 -0
  170. data/vendor/assets/dashboard/assets/Limiters-DiWCiVay.js +1 -0
  171. data/vendor/assets/dashboard/assets/Limiters-DiWCiVay.js.br +0 -0
  172. data/vendor/assets/dashboard/assets/Limiters-DiWCiVay.js.gz +0 -0
  173. data/vendor/assets/dashboard/assets/Metrics-DmMS2wo_.js +1 -0
  174. data/vendor/assets/dashboard/assets/Metrics-DmMS2wo_.js.br +0 -0
  175. data/vendor/assets/dashboard/assets/Metrics-DmMS2wo_.js.gz +0 -0
  176. data/vendor/assets/dashboard/assets/NotFound-CM1j8-Fv.js +1 -0
  177. data/vendor/assets/dashboard/assets/NotFound-CM1j8-Fv.js.br +0 -0
  178. data/vendor/assets/dashboard/assets/NotFound-CM1j8-Fv.js.gz +0 -0
  179. data/vendor/assets/dashboard/assets/PageHeader-BcHVxRbX.js +1 -0
  180. data/vendor/assets/dashboard/assets/PageHeader-BcHVxRbX.js.br +0 -0
  181. data/vendor/assets/dashboard/assets/PageHeader-BcHVxRbX.js.gz +0 -0
  182. data/vendor/assets/dashboard/assets/Profiles-B70MlVYQ.js +1 -0
  183. data/vendor/assets/dashboard/assets/Profiles-B70MlVYQ.js.br +0 -0
  184. data/vendor/assets/dashboard/assets/Profiles-B70MlVYQ.js.gz +0 -0
  185. data/vendor/assets/dashboard/assets/Queues-D3JZtWBv.js +1 -0
  186. data/vendor/assets/dashboard/assets/Queues-D3JZtWBv.js.br +0 -0
  187. data/vendor/assets/dashboard/assets/Queues-D3JZtWBv.js.gz +0 -0
  188. data/vendor/assets/dashboard/assets/Retries-BTWoV6so.js +1 -0
  189. data/vendor/assets/dashboard/assets/Retries-BTWoV6so.js.br +0 -0
  190. data/vendor/assets/dashboard/assets/Retries-BTWoV6so.js.gz +0 -0
  191. data/vendor/assets/dashboard/assets/Scheduled-D3PyiFqr.js +1 -0
  192. data/vendor/assets/dashboard/assets/Scheduled-D3PyiFqr.js.br +0 -0
  193. data/vendor/assets/dashboard/assets/Scheduled-D3PyiFqr.js.gz +0 -0
  194. data/vendor/assets/dashboard/assets/Search-bLwIhsaR.js +1 -0
  195. data/vendor/assets/dashboard/assets/Search-bLwIhsaR.js.br +0 -0
  196. data/vendor/assets/dashboard/assets/Search-bLwIhsaR.js.gz +0 -0
  197. data/vendor/assets/dashboard/assets/ar-CugpXGea.js +1 -0
  198. data/vendor/assets/dashboard/assets/ar-CugpXGea.js.br +0 -0
  199. data/vendor/assets/dashboard/assets/ar-CugpXGea.js.gz +0 -0
  200. data/vendor/assets/dashboard/assets/charts-DZyiygHX.js +1 -0
  201. data/vendor/assets/dashboard/assets/charts-DZyiygHX.js.br +0 -0
  202. data/vendor/assets/dashboard/assets/charts-DZyiygHX.js.gz +0 -0
  203. data/vendor/assets/dashboard/assets/de-Bl5Jn_0l.js +1 -0
  204. data/vendor/assets/dashboard/assets/de-Bl5Jn_0l.js.br +0 -0
  205. data/vendor/assets/dashboard/assets/de-Bl5Jn_0l.js.gz +0 -0
  206. data/vendor/assets/dashboard/assets/es-lqQrmjYI.js +1 -0
  207. data/vendor/assets/dashboard/assets/es-lqQrmjYI.js.br +0 -0
  208. data/vendor/assets/dashboard/assets/es-lqQrmjYI.js.gz +0 -0
  209. data/vendor/assets/dashboard/assets/fr-D70VBoh3.js +1 -0
  210. data/vendor/assets/dashboard/assets/fr-D70VBoh3.js.br +0 -0
  211. data/vendor/assets/dashboard/assets/fr-D70VBoh3.js.gz +0 -0
  212. data/vendor/assets/dashboard/assets/i18n-DdPClaCG.js +1 -0
  213. data/vendor/assets/dashboard/assets/i18n-DdPClaCG.js.br +0 -0
  214. data/vendor/assets/dashboard/assets/i18n-DdPClaCG.js.gz +0 -0
  215. data/vendor/assets/dashboard/assets/index-CtK2Dy_G.css +1 -0
  216. data/vendor/assets/dashboard/assets/index-CtK2Dy_G.css.br +0 -0
  217. data/vendor/assets/dashboard/assets/index-CtK2Dy_G.css.gz +0 -0
  218. data/vendor/assets/dashboard/assets/index-ygQ-XC-L.js +141 -0
  219. data/vendor/assets/dashboard/assets/index-ygQ-XC-L.js.br +0 -0
  220. data/vendor/assets/dashboard/assets/index-ygQ-XC-L.js.gz +0 -0
  221. data/vendor/assets/dashboard/assets/ja-BYMqnHjg.js +1 -0
  222. data/vendor/assets/dashboard/assets/ja-BYMqnHjg.js.br +0 -0
  223. data/vendor/assets/dashboard/assets/ja-BYMqnHjg.js.gz +0 -0
  224. data/vendor/assets/dashboard/assets/pt-BR-4k5M2MYH.js +1 -0
  225. data/vendor/assets/dashboard/assets/pt-BR-4k5M2MYH.js.br +0 -0
  226. data/vendor/assets/dashboard/assets/pt-BR-4k5M2MYH.js.gz +0 -0
  227. data/vendor/assets/dashboard/assets/useJobSetList-D-uafkUL.js +2 -0
  228. data/vendor/assets/dashboard/assets/useJobSetList-D-uafkUL.js.br +0 -0
  229. data/vendor/assets/dashboard/assets/useJobSetList-D-uafkUL.js.gz +0 -0
  230. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-DUA0JTgs.js +1 -0
  231. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-DUA0JTgs.js.br +0 -0
  232. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-DUA0JTgs.js.gz +0 -0
  233. data/vendor/assets/dashboard/assets/useSort-BwoC7g__.js +1 -0
  234. data/vendor/assets/dashboard/assets/useSort-BwoC7g__.js.br +0 -0
  235. data/vendor/assets/dashboard/assets/useSort-BwoC7g__.js.gz +0 -0
  236. data/vendor/assets/dashboard/assets/{utils-CnWZzW2L.js → utils-CQj4rqng.js} +1 -1
  237. data/vendor/assets/dashboard/assets/utils-CQj4rqng.js.br +0 -0
  238. data/vendor/assets/dashboard/assets/utils-CQj4rqng.js.gz +0 -0
  239. data/vendor/assets/dashboard/assets/wurk-logo-ydeSB9B8.png +0 -0
  240. data/vendor/assets/dashboard/assets/zh-CN-B9E1MvsM.js +1 -0
  241. data/vendor/assets/dashboard/assets/zh-CN-B9E1MvsM.js.br +0 -0
  242. data/vendor/assets/dashboard/assets/zh-CN-B9E1MvsM.js.gz +0 -0
  243. data/vendor/assets/dashboard/index.html +8 -28
  244. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  245. metadata +130 -37
  246. data/vendor/assets/dashboard/assets/ArgsValue-BJol9S0s.js +0 -1
  247. data/vendor/assets/dashboard/assets/BatchDetail-D_zGQ9Sd.js +0 -1
  248. data/vendor/assets/dashboard/assets/Batches-Dt_xC660.js +0 -1
  249. data/vendor/assets/dashboard/assets/Busy-CWRthsmc.js +0 -1
  250. data/vendor/assets/dashboard/assets/Cron-qeGr2YAm.js +0 -1
  251. data/vendor/assets/dashboard/assets/Dashboard-DQWaAwPA.js +0 -1
  252. data/vendor/assets/dashboard/assets/Dead-D9gOPHtK.js +0 -1
  253. data/vendor/assets/dashboard/assets/Extension-DONXVLdA.js +0 -1
  254. data/vendor/assets/dashboard/assets/FilterBox-BbB6gXHp.js +0 -2
  255. data/vendor/assets/dashboard/assets/FlowDetail-CrmnVRp0.js +0 -1
  256. data/vendor/assets/dashboard/assets/Flows-g4GC8KmE.js +0 -1
  257. data/vendor/assets/dashboard/assets/JobDetailModal-C86k_1wj.js +0 -2
  258. data/vendor/assets/dashboard/assets/Limiters-zyO-iMlK.js +0 -1
  259. data/vendor/assets/dashboard/assets/Metrics-Ckq5fNBv.js +0 -1
  260. data/vendor/assets/dashboard/assets/NotFound-C8NnyjlY.js +0 -1
  261. data/vendor/assets/dashboard/assets/PageHeader-vWeBbSPF.js +0 -1
  262. data/vendor/assets/dashboard/assets/Profiles-DRl-FXPA.js +0 -1
  263. data/vendor/assets/dashboard/assets/Queues-DkXWrihA.js +0 -1
  264. data/vendor/assets/dashboard/assets/Retries-CmlojYIy.js +0 -1
  265. data/vendor/assets/dashboard/assets/Scheduled-Cbbpp3xw.js +0 -1
  266. data/vendor/assets/dashboard/assets/Search-LaEpj2fM.js +0 -1
  267. data/vendor/assets/dashboard/assets/charts-D_PQLoyc.js +0 -1
  268. data/vendor/assets/dashboard/assets/fa-brands-400-Bs6tcqqs.woff2 +0 -0
  269. data/vendor/assets/dashboard/assets/fa-regular-400-DRN8N0d1.woff2 +0 -0
  270. data/vendor/assets/dashboard/assets/fa-solid-900-IAB4Droh.woff2 +0 -0
  271. data/vendor/assets/dashboard/assets/fa-v4compatibility-CErXDOsT.woff2 +0 -0
  272. data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +0 -1
  273. data/vendor/assets/dashboard/assets/index--xvvNlYx.css +0 -1
  274. data/vendor/assets/dashboard/assets/index-FBKGtysy.js +0 -141
  275. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-BiwhHqgh.js +0 -1
  276. data/vendor/assets/dashboard/assets/useSort-jBYVwXvg.js +0 -1
  277. data/vendor/assets/dashboard/assets/wurk-logo-Vy3xW4K0.png +0 -0
  278. data/vendor/assets/dashboard/og-image.png +0 -0
  279. data/vendor/assets/dashboard/wurk-logo.webp +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b0817941e1b8f3ff6a7195567e31ac9d22c5039c338d708f9c329ff87f0bbcec
4
- data.tar.gz: 966125b576e3a775f868ad39224a70ffe6baddf6cbd35eb8b76fc81d63e6b762
3
+ metadata.gz: 96bee6b140f21dd1b2fb9d8cf382f6942600a91896dcae3bde52bbcb615fd7ff
4
+ data.tar.gz: 0f0a74b458b2bc9de92759ee0c20a3ffa8ba9ca3fa40dcd891abe07afcb8a3db
5
5
  SHA512:
6
- metadata.gz: c265be3c09774f76f6a9d560a59434a6800d56c06eadd471c6391a1cad589eb709ba51fdfbf672c14c5ef39c482d2e40a24a9a3a9ad8861d917a2bb5daa01d14
7
- data.tar.gz: 998f43d63f710cd96b6e413fe20b6c4783149113c2bc1128d74afa3c33a225e75189b65d2515b4cd8351b8d14f02bfd4bfc390a164d9c01914147161e1532559
6
+ metadata.gz: c46917c8d37f35ae24aec5c0f6305a05d1c2d85af050da7dfda1eff8f73f0537898dcc3957849bede6ad3128836fbfd3db1782397771e0d8af36096098948838
7
+ data.tar.gz: 59fd02cb8bbbf4e66b4ffda4e5260c6052ea1b0d7708f069afcdc7d22d88408b0fd72ff65f36958629d4b75cbf85cb2a95ce37a6e76358ff85013e7cbae5e818
data/README.md CHANGED
@@ -305,7 +305,7 @@ Wurk is MIT and stays that way. If what you need is a commercial support contrac
305
305
 
306
306
  `bundle install && restart`. The worker command stays as it was: Wurk ships `sidekiq` and `sidekiqswarm` executables, so `bundle exec sidekiq -C config/sidekiq.yml` in a Procfile or manifest keeps working. Wurk reads and writes the same Redis schema, so there is no data migration: Wurk picks up the queues, schedules, retries, batches and periodic loops Sidekiq left behind. Running Sidekiq and Wurk workers against one Redis *at the same time* is not tested yet, so a live production move is a **drain cutover** (stop Sidekiq's workers, confirm nothing is in flight, start Wurk), with a matching rollback; the [production cutover](https://github.com/developerz-ai/wurk/blob/main/docs/migrate-from-sidekiq.md#9-production-cutover) section has every step and command. The upstream suites of sidekiq-cron, sidekiq-unique-jobs, sidekiq-failures and sentry-sidekiq run against Wurk in the [`ecosystem` CI job](https://github.com/developerz-ai/wurk/blob/main/.github/workflows/ecosystem.yml) (see [`test/ecosystem/`](https://github.com/developerz-ai/wurk/tree/main/test/ecosystem)). sidekiq-scheduler, sidekiq-status and sidekiq-throttled are target additions tracked in [docs/idea/14-ecosystem-compat.md](https://github.com/developerz-ai/wurk/blob/main/docs/idea/14-ecosystem-compat.md).
307
307
 
308
- **Keeping a `sidekiq-*` add-on gem?** Those gems declare `add_dependency "sidekiq"`, which would reinstall real Sidekiq next to Wurk. Satisfy it with Wurk's git-only [`sidekiq` shim gem](https://github.com/developerz-ai/wurk/tree/main/ecosystem/sidekiq-shim) instead, pinned to the tag of the Wurk release you run: `gem "sidekiq", github: "developerz-ai/wurk", tag: "v1.13.0", glob: "ecosystem/sidekiq-shim/*.gemspec"`. Only sidekiq-cron, sidekiq-unique-jobs, sidekiq-failures and sentry-sidekiq are exercised that way, by the `ecosystem` workflow; treat the others as untested.
308
+ **Keeping a `sidekiq-*` add-on gem?** Those gems declare `add_dependency "sidekiq"`, which would reinstall real Sidekiq next to Wurk. Satisfy it with Wurk's git-only [`sidekiq` shim gem](https://github.com/developerz-ai/wurk/tree/main/ecosystem/sidekiq-shim) instead, pinned to the tag of the Wurk release you run: `gem "sidekiq", github: "developerz-ai/wurk", tag: "v1.15.0", glob: "ecosystem/sidekiq-shim/*.gemspec"`. Only sidekiq-cron, sidekiq-unique-jobs, sidekiq-failures and sentry-sidekiq are exercised that way, by the `ecosystem` workflow; treat the others as untested.
309
309
 
310
310
  Full walkthrough — config side-by-side, the Redis key/`sidekiq_options` mapping, known incompatibilities, moving sidekiq-cron schedules over (`rake wurk:import:cron`), and the production cutover and rollback playbook: **[docs/migrate-from-sidekiq.md](https://github.com/developerz-ai/wurk/blob/main/docs/migrate-from-sidekiq.md)**. On the night, keep the **[incident runbook](https://github.com/developerz-ai/wurk/blob/main/docs/runbook.md)** open.
311
311
 
@@ -6,12 +6,18 @@ module Wurk
6
6
  # extension forms send no Rails authenticity token, so `protect_from_forgery`
7
7
  # can't apply — guard unsafe methods with a same-origin check instead. A
8
8
  # cross-site page can drive a victim's browser but cannot forge
9
- # `Sec-Fetch-Site: same-origin` (the browser sets it); a missing header is
10
- # denied too, since only a non-browser client omits it and it carries no
11
- # ambient session cookie to abuse.
9
+ # `Sec-Fetch-Site: same-origin` (the browser sets it).
12
10
  #
13
- # Mirrors the standalone Rack surface (Wurk::Web#safe_request?) so every
14
- # mutating dashboard entry point enforces one rule.
11
+ # Browsers send Fetch Metadata only to potentially-trustworthy origins
12
+ # (HTTPS, localhost), so a dashboard over plain HTTP on a LAN host never sees
13
+ # it. Then the request's `Origin` — or, when absent, its `Referer` — must be
14
+ # exactly this request's scheme, host and port (owner decision U9, #101 W3).
15
+ # Neither header present is denied: only a non-browser client omits both,
16
+ # and it carries no ambient session cookie to abuse.
17
+ #
18
+ # The standalone Rack surface (Wurk::Web#safe_request?) applies the same
19
+ # Sec-Fetch-Site rule without this fallback: it stays strict, so a
20
+ # plain-HTTP standalone mount refuses unsafe requests (parity-divergences).
15
21
  module SameOriginGuard
16
22
  extend ActiveSupport::Concern
17
23
 
@@ -28,13 +34,29 @@ module Wurk
28
34
 
29
35
  private
30
36
 
37
+ # The SPA's toast branches on this body to name the cause instead of
38
+ # reading the 403 as read-only mode.
31
39
  def verify_same_origin!
32
- head :forbidden unless safe_request?
40
+ render json: { error: 'cross_origin' }, status: :forbidden unless safe_request?
33
41
  end
34
42
 
35
43
  def safe_request?
36
- SAFE_METHODS.include?(request.request_method) ||
37
- request.headers['Sec-Fetch-Site'] == 'same-origin'
44
+ return true if SAFE_METHODS.include?(request.request_method)
45
+
46
+ site = request.headers['Sec-Fetch-Site']
47
+ site ? site == 'same-origin' : same_origin_source?
48
+ end
49
+
50
+ def same_origin_source?
51
+ source = request.headers['Origin'].presence || request.referer.presence
52
+ return false unless source
53
+
54
+ uri = ::URI.parse(source)
55
+ [uri.scheme, uri.host, uri.port] == own_origin
56
+ rescue ::URI::InvalidURIError
57
+ false
38
58
  end
59
+
60
+ def own_origin = [request.scheme, request.host, request.port]
39
61
  end
40
62
  end
@@ -8,6 +8,40 @@ module Wurk
8
8
  module SseStreaming
9
9
  extend ActiveSupport::Concern
10
10
 
11
+ # One stats payload per process per TTL, shared by every open stream: ten
12
+ # dashboard tabs ticking every 2s cost one Redis read, not ten. Streams
13
+ # that tick together wait on the lock and take the payload the first one
14
+ # read. Keyed on the pool so a payload read from one Redis never answers
15
+ # for another; a raising read caches nothing.
16
+ class SharedTick
17
+ def initialize(ttl)
18
+ @ttl = ttl
19
+ @lock = ::Mutex.new
20
+ clear!
21
+ end
22
+
23
+ def fetch
24
+ pool = ::Wurk.redis_pool
25
+ @lock.synchronize do
26
+ now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
27
+ return @payload if @payload && @pool.equal?(pool) && now - @read_at < @ttl
28
+
29
+ payload = yield.freeze
30
+ @payload = payload
31
+ @pool = pool
32
+ @read_at = now
33
+ payload
34
+ end
35
+ end
36
+
37
+ def clear!
38
+ @lock.synchronize { @payload = @pool = @read_at = nil }
39
+ end
40
+ end
41
+
42
+ # Under the 2s tick, so one stream never reads its own previous payload.
43
+ SHARED_TICK = SharedTick.new(1.0)
44
+
11
45
  private
12
46
 
13
47
  def stream_headers!
@@ -43,7 +77,8 @@ module Wurk
43
77
  end
44
78
 
45
79
  def stream_tick_payload
46
- ::Wurk::Api::Serializers.stats_payload(::Wurk::Stats.new).merge(at: ::Time.now.to_f)
80
+ SHARED_TICK.fetch { ::Wurk::Api::Serializers.stats_payload(::Wurk::Stats::Snapshot.new) }
81
+ .merge(at: ::Time.now.to_f)
47
82
  end
48
83
  end
49
84
  end
@@ -5,7 +5,8 @@ module Wurk
5
5
  # Pagination helpers shared by every listing endpoint. The contract:
6
6
  # * `?count=` page size (default 25, clamped to 1..200)
7
7
  # * `?page=` 0-indexed page number
8
- # * `?substr=` case-insensitive klass/jid filter on the page
8
+ # * `?substr=` case-insensitive filter on a row's klass, jid, error
9
+ # class, error message or args
9
10
  #
10
11
  # Helpers expect an Enumerable that yields whatever JSON-shaped Hash the
11
12
  # caller built; substr filtering and slicing happen after serialization so
@@ -13,16 +14,26 @@ module Wurk
13
14
  module Pagination
14
15
  DEFAULT_PAGE_SIZE = 25
15
16
  MAX_PAGE_SIZE = 200
16
- # Offset pagination reaches page N by walking N*count rows through Ruby
17
- # (and their Redis round-trips). An unclamped ?page= lets one request
18
- # walk an entire million-row set; 1000 pages is far beyond any real UI
19
- # depth while bounding the worst-case walk.
17
+ # An unfiltered listing reads its page at the offset (the set's
18
+ # #snapshot), whatever the depth; the walks that remain (a filtered
19
+ # #slice, the Limits names) still pay per skipped row, and 1000 pages is
20
+ # far beyond any real UI depth while bounding them.
20
21
  MAX_PAGE = 1_000
21
22
  # A substr that matches nothing would otherwise stream the whole backing
22
- # set through Ruby (same DoS shape Web::Search bounds with SCAN_BUDGET).
23
- # Raw rows examined per filtered request; results within the budget are
24
- # exact, beyond it the page just comes back short.
25
- FILTER_SCAN_LIMIT = 20_000
23
+ # set through Ruby, so a filtered request examines at most Web::Search's
24
+ # per-request budget of raw rows; results within it are exact, beyond it
25
+ # the page just comes back short.
26
+ FILTER_SCAN_LIMIT = ::Wurk::Web::Search::SCAN_BUDGET
27
+ # A filter walk reads the set through its #snapshot, starting at an
28
+ # ordinary listing page and doubling to FILTER_PAGE: a match near the
29
+ # top costs what it did, and the full budget costs ~25 round trips
30
+ # rather than one per 50 rows.
31
+ FIRST_FILTER_PAGE = 50
32
+ FILTER_PAGE = 1_000
33
+ # Entries one filtered `<set>/all/:cmd` request acts on, the cap the
34
+ # bulk `keys[]` action already has.
35
+ SWEEP_LIMIT = 1_000
36
+ MATCHED_FIELDS = %i[klass jid error_class error_message].freeze
26
37
 
27
38
  module_function
28
39
 
@@ -46,6 +57,9 @@ module Wurk
46
57
  fields.merge(filtered_total: page[:matched], filtered_total_exact: page[:exhausted])
47
58
  end
48
59
 
60
+ # Rows before the requested page: where an unfiltered listing reads from.
61
+ def offset(page) = page[:page] * page[:count]
62
+
49
63
  def clamp_int(value, min, max, default)
50
64
  Integer(value, 10).clamp(min, max)
51
65
  rescue ::ArgumentError, ::TypeError
@@ -58,50 +72,15 @@ module Wurk
58
72
  default
59
73
  end
60
74
 
61
- # Iterates `enumerable` and collects up to `count` payloads from the
62
- # requested page after substr filtering. `block` maps each member to a
63
- # JSON Hash; nil from the block skips the member entirely.
64
- #
65
- # With a substr filter the offset must count *matching* rows, not raw
66
- # rows — otherwise page N restarts at raw index N*count and re-emits
67
- # (or skips) matches already assigned to earlier pages. Without a
68
- # filter we keep the fast path that skips serialization before the
69
- # offset.
70
- def slice(enumerable, page, &)
71
- offset = page[:page] * page[:count]
72
- if page[:substr].empty?
73
- slice_raw(enumerable, offset, page[:count], &)
74
- else
75
- slice_filtered(enumerable, offset, page, &)
76
- end
77
- end
78
-
79
- # Fast path: skip serialization entirely for rows before the offset.
80
- def slice_raw(enumerable, offset, count)
81
- results = []
82
- seen = 0
83
- enumerable.each do |member|
84
- if seen < offset
85
- seen += 1
86
- next
87
- end
88
-
89
- payload = yield(member)
90
- next unless payload
91
-
92
- results << payload
93
- break if results.size >= count
94
- end
95
- results
96
- end
97
-
98
- # Filtered path: the offset counts MATCHING rows, so page boundaries
99
- # stay stable (raw-index offsets re-emit or skip matches across pages).
100
- # Records the matches seen in `page[:matched]` and whether the set was
101
- # walked to its end in `page[:exhausted]`: stopping once the page fills
102
- # is what keeps a filtered request cheap, so the count is only exact
103
- # when nothing was left unread.
104
- def slice_filtered(enumerable, offset, page)
75
+ # One page of a filtered listing: up to `count` payloads (the block maps
76
+ # each member to one; nil skips it) from the matches of `substr`, in
77
+ # `enumerable`'s order. The offset counts MATCHING rows, so page
78
+ # boundaries stay stable (raw-index offsets re-emit or skip matches
79
+ # across pages). Records the matches seen in `page[:matched]` and whether
80
+ # the set was walked to its end in `page[:exhausted]`: stopping once the
81
+ # page fills is what keeps a filtered request cheap, so the count is only
82
+ # exact when nothing was left unread.
83
+ def slice(enumerable, page)
105
84
  results = []
106
85
  matched = 0
107
86
  examined = 0
@@ -111,7 +90,7 @@ module Wurk
111
90
  payload = yield(member)
112
91
  next unless payload && match?(payload, page[:substr])
113
92
 
114
- results << payload if (matched += 1) > offset
93
+ results << payload if (matched += 1) > offset(page)
115
94
  break if results.size >= page[:count]
116
95
  end
117
96
  record_match_count(page, matched, examined, results.size)
@@ -123,11 +102,51 @@ module Wurk
123
102
  page[:exhausted] = examined <= FILTER_SCAN_LIMIT && served < page[:count]
124
103
  end
125
104
 
105
+ # Up to SWEEP_LIMIT of `rows`' members whose row (the block maps a member
106
+ # to it) matches `substr`, within FILTER_SCAN_LIMIT examined:
107
+ # [members, rows examined, whether `rows` ran out].
108
+ def matching(rows, substr)
109
+ members = []
110
+ examined = 0
111
+ rows.each do |member|
112
+ return [members, examined, false] if examined >= FILTER_SCAN_LIMIT || members.size >= SWEEP_LIMIT
113
+
114
+ examined += 1
115
+ members << member if match?(yield(member), substr)
116
+ end
117
+ [members, examined, true]
118
+ end
119
+
120
+ # Every row from raw offset `from` on, read page by page through the
121
+ # block, which answers `count` rows from `offset` in listing order.
122
+ # Stops after the first short page.
123
+ def walk(from = 0, &read)
124
+ ::Enumerator.new do |out|
125
+ offset = from
126
+ size = FIRST_FILTER_PAGE
127
+ loop do
128
+ rows = read.call(offset, size)
129
+ rows.each { |row| out << row }
130
+ break if rows.size < size
131
+
132
+ offset += size
133
+ size = [size * 2, FILTER_PAGE].min
134
+ end
135
+ end
136
+ end
137
+
126
138
  def match?(payload, substr)
127
139
  return true if substr.nil? || substr.empty?
128
140
 
129
141
  needle = substr.downcase
130
- payload[:klass].to_s.downcase.include?(needle) || payload[:jid].to_s.downcase.include?(needle)
142
+ MATCHED_FIELDS.any? { |field| payload[field].to_s.downcase.include?(needle) } ||
143
+ args_match?(payload[:args], needle)
144
+ end
145
+
146
+ # Against the args as the SPA shows them: JSON, so `"a@b.c"` or `42]`
147
+ # match as typed.
148
+ def args_match?(args, needle)
149
+ !args.nil? && ::Wurk.dump_json(args).downcase.include?(needle)
131
150
  end
132
151
  end
133
152
  end
@@ -52,17 +52,23 @@ module Wurk
52
52
  rows.empty? ? base : base.merge(custom_rows: rows)
53
53
  end
54
54
 
55
+ # A listing row. No `error_backtrace`: it is stored compressed, and
56
+ # inflating it for every row of a page served a field only the detail
57
+ # view (#sorted_entry_detail) shows.
55
58
  def sorted_entry(entry)
56
59
  job_record(entry).merge(
57
60
  score: entry.score,
58
61
  at: entry.at.to_f,
59
62
  error_class: entry['error_class'].to_s,
60
63
  error_message: entry['error_message'].to_s,
61
- retry_count: entry['retry_count'],
62
- error_backtrace: entry.error_backtrace
64
+ retry_count: entry['retry_count']
63
65
  )
64
66
  end
65
67
 
68
+ def sorted_entry_detail(entry)
69
+ sorted_entry(entry).merge(error_backtrace: entry.error_backtrace)
70
+ end
71
+
66
72
  # `leader_identity` is the cluster's `dear-leader` value (ProcessSet#leader,
67
73
  # memoized to one Redis GET per request) — comparing here avoids the N+1
68
74
  # `Process#leader?` GET-per-row that a per-process lookup would cost.
@@ -2,7 +2,6 @@
2
2
 
3
3
  require_relative 'api/serializers'
4
4
  require_relative 'api/pagination'
5
- require 'wurk/web'
6
5
 
7
6
  module Wurk
8
7
  # JSON APIs consumed by the SolidJS SPA. Action methods stay thin; mapping to
@@ -36,6 +35,10 @@ module Wurk
36
35
 
37
36
  HISTORY_WINDOW_UNITS = { 's' => 1, 'm' => 60, 'h' => 3600, 'd' => 86_400 }.freeze
38
37
  DEFAULT_HISTORY_WINDOW = 24 * 3600
38
+ # Sidekiq Web's dashboard `?days=N` range (free §25.4).
39
+ MAX_HISTORY_DAYS = 180
40
+ # Far past any real set; bounds a `?cursor=` before it reaches LRANGE.
41
+ MAX_CURSOR = 1 << 40
39
42
 
40
43
  # Per-set action whitelists. Maps the SPA's action name to the
41
44
  # SortedEntry/JobSet method. Anything not listed 400s — keeps the bulk/
@@ -58,22 +61,24 @@ module Wurk
58
61
  end
59
62
 
60
63
  def stats
61
- render json: ::Wurk::Api::Serializers.stats_payload(::Wurk::Stats.new)
64
+ render json: ::Wurk::Api::Serializers.stats_payload(::Wurk::Stats::Snapshot.new)
62
65
  end
63
66
 
64
67
  def queues
65
- consumers = ::Wurk.redis(idempotent: true) { |conn| ::Wurk::QueueConsumers.counts(conn) }
66
- render json: ::Wurk::Stats.new.queue_summaries.map { |q|
67
- ::Wurk::Api::Serializers.queue_summary(q).merge(consumers: consumers.fetch(q.name, 0))
68
+ snapshot = ::Wurk::Stats::Snapshot.new(consumers: true)
69
+ render json: snapshot.queue_summaries.map { |q|
70
+ ::Wurk::Api::Serializers.queue_summary(q).merge(consumers: snapshot.consumers.fetch(q.name, 0))
68
71
  }
69
72
  end
70
73
 
74
+ # `?direction=asc` lists oldest first, the job the next fetch takes at the
75
+ # top, as Sidekiq Web's queue page does (free §25.4).
71
76
  def queue
72
77
  q = ::Wurk::Queue.new(params[:name].to_s)
73
78
  page = ::Wurk::Api::Pagination.window(params)
74
- jobs = ::Wurk::Api::Pagination.slice(q, page) { |rec| ::Wurk::Api::Serializers.job_record(rec) }
79
+ gauges, jobs = queue_page(q, page, params[:direction].to_s == 'asc')
75
80
  render json: {
76
- name: q.name, size: q.size, latency: q.latency, paused: q.paused?,
81
+ name: q.name, size: gauges.size, latency: gauges.latency, paused: gauges.paused?,
77
82
  **::Wurk::Api::Pagination.meta(page), jobs: jobs
78
83
  }
79
84
  end
@@ -109,12 +114,21 @@ module Wurk
109
114
  def scheduled = render_sorted_set(::Wurk::ScheduledSet.new)
110
115
  def dead = render_sorted_set(::Wurk::DeadSet.new)
111
116
 
117
+ # One entry with its backtrace, which listing rows leave out: inflating
118
+ # it costs a base64 decode, an inflate and a JSON parse per row, for a
119
+ # field only the detail view shows. `:key` is "<score>|<jid>".
120
+ def retry_entry = render_sorted_entry(::Wurk::RetrySet.new)
121
+ def scheduled_entry = render_sorted_entry(::Wurk::ScheduledSet.new)
122
+ def dead_entry = render_sorted_entry(::Wurk::DeadSet.new)
123
+
112
124
  # --- Retry set mutations -------------------------------------------------
113
125
  def retry_job = single_entry_action(::Wurk::RetrySet.new, RETRY_ACTIONS, params[:cmd])
114
126
  def retries_bulk = bulk_entry_action(::Wurk::RetrySet.new, RETRY_ACTIONS)
115
127
 
116
128
  def retries_all
117
129
  set = ::Wurk::RetrySet.new
130
+ return sweep_matching(set, RETRY_ACTIONS) if filtered_sweep?
131
+
118
132
  count = case params[:cmd].to_s
119
133
  when 'retry' then set.retry_all
120
134
  when 'kill' then set.kill_all
@@ -130,6 +144,8 @@ module Wurk
130
144
 
131
145
  def scheduled_all
132
146
  set = ::Wurk::ScheduledSet.new
147
+ return sweep_matching(set, SCHEDULED_ACTIONS) if filtered_sweep?
148
+
133
149
  case params[:cmd].to_s
134
150
  when 'delete' then render json: { ok: true, count: clear_set(set) }
135
151
  when 'add_to_queue' then drain_set(set, :add_to_queue)
@@ -143,6 +159,8 @@ module Wurk
143
159
 
144
160
  def dead_all
145
161
  set = ::Wurk::DeadSet.new
162
+ return sweep_matching(set, DEAD_ACTIONS) if filtered_sweep?
163
+
146
164
  count = case params[:cmd].to_s
147
165
  when 'retry' then set.retry_all
148
166
  when 'delete' then clear_set(set)
@@ -153,8 +171,9 @@ module Wurk
153
171
 
154
172
  def processes
155
173
  set = ::Wurk::ProcessSet.new
174
+ live = set.to_a # before #leader: the walk reads `dear-leader` in its pipeline
156
175
  leader_identity = set.leader
157
- render json: set.map { |p| ::Wurk::Api::Serializers.process_row(p, leader_identity: leader_identity) }
176
+ render json: live.map { |p| ::Wurk::Api::Serializers.process_row(p, leader_identity: leader_identity) }
158
177
  end
159
178
 
160
179
  # Currently-executing jobs across the cluster (WorkSet), oldest first.
@@ -170,11 +189,16 @@ module Wurk
170
189
  def quiet_process = signal_processes(:quiet!)
171
190
  def stop_process = signal_processes(:stop!)
172
191
 
192
+ # Rows are Batch::Status#summary: #data with the failed and dead jid sets
193
+ # counted rather than listed. GET /api/batches/:bid carries the jids.
194
+ # `?dead=1` lists `dead-batches` instead (Pro's Dead batches tab);
195
+ # `?tag=` narrows either to the batches carrying that tag, and takes the
196
+ # place of `?substr=` (BatchSet#tagged_snapshot).
173
197
  def batches
174
- set = ::Wurk::BatchSet.new
175
198
  page = ::Wurk::Api::Pagination.window(params)
176
- rows = ::Wurk::Api::Pagination.slice(set, page) { |status| status.data.transform_keys(&:to_sym) }
177
- render json: { total: set.size, **::Wurk::Api::Pagination.meta(page), batches: rows }
199
+ set = %w[1 true].include?(params[:dead].to_s) ? ::Wurk::Batch::DeadSet.new : ::Wurk::BatchSet.new
200
+ total, rows = batch_rows(set, page, params[:tag].to_s)
201
+ render json: { total: total, **::Wurk::Api::Pagination.meta(page), batches: rows }
178
202
  end
179
203
 
180
204
  def batch
@@ -187,10 +211,9 @@ module Wurk
187
211
  end
188
212
 
189
213
  def flows
190
- set = ::Wurk::FlowSet.new
191
214
  page = ::Wurk::Api::Pagination.window(params)
192
- rows = ::Wurk::Api::Pagination.slice(set, page) { |status| ::Wurk::Api::Serializers.flow_row(status) }
193
- render json: { total: set.size, **::Wurk::Api::Pagination.meta(page), flows: rows }
215
+ total, rows = page_rows(::Wurk::FlowSet.new, page) { |status| ::Wurk::Api::Serializers.flow_row(status) }
216
+ render json: { total: total, **::Wurk::Api::Pagination.meta(page), flows: rows }
194
217
  end
195
218
 
196
219
  def flow
@@ -212,9 +235,12 @@ module Wurk
212
235
  end
213
236
 
214
237
  def limiters
215
- names = ::Wurk::Web::Enterprise::Limits.list(filter: params[:substr])
216
238
  page = ::Wurk::Api::Pagination.window(params)
217
- render json: { total: names.size, **::Wurk::Api::Pagination.meta(page), limiters: limiter_rows(names, page) }
239
+ total, rows = ::Wurk::Web::Enterprise::Limits.page(
240
+ filter: params[:substr], offset: ::Wurk::Api::Pagination.offset(page), count: page[:count]
241
+ )
242
+ render json: { total: total, **::Wurk::Api::Pagination.meta(page),
243
+ limiters: rows.map { |name, meta| ::Wurk::Api::Serializers.limiter_row(name, meta) } }
218
244
  end
219
245
 
220
246
  def reset_limiter
@@ -272,10 +298,11 @@ module Wurk
272
298
  end
273
299
 
274
300
  # Cluster-total throughput/failures time-series for the dashboard charts.
275
- # `:bucket` is 1m/5m/1h; `?window=24h` (s/m/h/d suffix) is clamped to the
276
- # bucket's retention. Chart-ready array under `series`.
301
+ # `:bucket` is 1m/5m/1h; `?window=24h` (s/m/h/d suffix) or `?days=N`
302
+ # (1..180, wins over window) is clamped to the bucket's retention.
303
+ # Chart-ready array under `series`.
277
304
  def history
278
- window = parse_window(params[:window])
305
+ window = history_window(params)
279
306
  series = ::Wurk::Web::Enterprise::Historical.history(params[:bucket].to_s, window: window)
280
307
  render json: { bucket: params[:bucket].to_s, window: window, series: series.map { |row| ::Wurk::Api::Serializers.history_point(row) } }
281
308
  rescue ::ArgumentError => e
@@ -295,10 +322,10 @@ module Wurk
295
322
 
296
323
  # Per-queue size/latency gauge time-series for the Metrics/Historical tab.
297
324
  # `:bucket` is 1m/5m/1h; `?window=24h` (s/m/h/d) is clamped to the bucket's
298
- # retention; optional `?queue=<name>` narrows to one queue. Each queue's
299
- # `points` are chart-ready.
325
+ # retention (`?days=N` as on #history); optional `?queue=<name>` narrows
326
+ # to one queue. Each queue's `points` are chart-ready.
300
327
  def queue_history # rubocop:disable Metrics/AbcSize
301
- window = parse_window(params[:window])
328
+ window = history_window(params)
302
329
  queues = params[:queue].present? ? [params[:queue].to_s] : nil
303
330
  series = ::Wurk::Web::Enterprise::Historical.queue_history(params[:bucket].to_s, window: window, queues: queues)
304
331
  render json: {
@@ -401,7 +428,7 @@ module Wurk
401
428
  return render(json: { error: 'unknown action' }, status: :bad_request) unless method
402
429
 
403
430
  keys = Array(params[:keys]).map(&:to_s).uniq
404
- # Each key costs a ZRANGEBYSCORE plus per-entry mutations; an uncapped
431
+ # Each key costs a pipelined ZRANGEBYSCORE plus per-entry mutations; an uncapped
405
432
  # list lets one POST issue tens of thousands of round-trips. The UI's
406
433
  # per-row selections accumulate across pages (toggleAll works against
407
434
  # the visible page's keys only) and the bulk POST sends the whole Set,
@@ -431,8 +458,8 @@ module Wurk
431
458
 
432
459
  # 422 when any entry failed, so the SPA's error toast fires; `count` is
433
460
  # what was applied either way.
434
- def render_applied(total, failures)
435
- body = { ok: failures.empty?, count: total - failures.size }
461
+ def render_applied(total, failures, extra = {})
462
+ body = { ok: failures.empty?, count: total - failures.size, **extra }
436
463
  return render(json: body) if failures.empty?
437
464
 
438
465
  render json: body.merge(failed: failures), status: 422
@@ -459,13 +486,50 @@ module Wurk
459
486
  # Maps "<score>|<jid>" keys to live SortedEntry objects via score-bracketed
460
487
  # fetch (exact float match, narrowed by jid) — avoids depending on float→
461
488
  # string round-tripping between JS and Ruby. Skips malformed/empty keys.
489
+ # One pipelined round trip for every key (JobSet#fetch_pairs).
462
490
  def entries_for(set, keys)
463
- keys.flat_map do |key|
491
+ pairs = keys.filter_map do |key|
464
492
  score, jid = key.to_s.split('|', 2)
465
- next [] if jid.nil? || jid.empty?
493
+ [score.to_f, jid] unless jid.nil? || jid.empty?
494
+ end
495
+ set.fetch_pairs(pairs)
496
+ end
497
+
498
+ def batch_rows(set, page, tag)
499
+ symbolize = ->(summary) { summary.transform_keys(&:to_sym) }
500
+ return page_rows(set, page, &symbolize) if tag.empty?
466
501
 
467
- set.fetch(score.to_f, jid)
502
+ total, summaries = set.tagged_snapshot(tag, ::Wurk::Api::Pagination.offset(page), page[:count])
503
+ [total, summaries.map(&symbolize)]
504
+ end
505
+
506
+ def filtered_sweep? = !params[:substr].to_s.empty?
507
+
508
+ # `<set>/all/:cmd?substr=`: applies `cmd` to the entries the same filtered
509
+ # listing shows, newest first, at most SWEEP_LIMIT of them within the
510
+ # filter's scan budget per request. The answer carries `done` (the walk
511
+ # reached the end of the set) and `cursor`, the raw offset to send back as
512
+ # `?cursor=` to carry on: the rows examined minus those this request moved
513
+ # out of the set. Entries added or removed meanwhile above that offset
514
+ # shift it, so a resumed sweep can re-examine or skip a few rows.
515
+ def sweep_matching(set, actions)
516
+ method = actions[params[:cmd].to_s]
517
+ return render(json: { error: 'unknown action' }, status: :bad_request) unless method
518
+
519
+ entries, scanned_to, done = matching_entries(set)
520
+ failures = apply_each(entries, method)
521
+ cursor = scanned_to - (entries.size - failures.size)
522
+ render_applied(entries.size, failures, matched: entries.size, done: done, cursor: cursor)
523
+ end
524
+
525
+ # [the matches from `?cursor=`, the raw offset the walk stopped at, done].
526
+ def matching_entries(set)
527
+ from = ::Wurk::Api::Pagination.clamp_int(params[:cursor], 0, MAX_CURSOR, 0)
528
+ rows = ::Wurk::Api::Pagination.walk(from) { |offset, count| set.snapshot(offset, count).last }
529
+ entries, examined, done = ::Wurk::Api::Pagination.matching(rows, params[:substr].to_s) do |entry|
530
+ ::Wurk::Api::Serializers.sorted_entry(entry)
468
531
  end
532
+ [entries, from + examined, done]
469
533
  end
470
534
 
471
535
  # UNLINKs the whole set, returning the count removed (read before clearing
@@ -489,18 +553,49 @@ module Wurk
489
553
 
490
554
  def render_sorted_set(set)
491
555
  page = ::Wurk::Api::Pagination.window(params)
492
- total = set.size
493
- entries = ::Wurk::Api::Pagination.slice(set, page) { |entry| ::Wurk::Api::Serializers.sorted_entry(entry) }
556
+ total, entries = page_rows(set, page) { |entry| ::Wurk::Api::Serializers.sorted_entry(entry) }
494
557
  render json: { total: total, **::Wurk::Api::Pagination.meta(page), entries: entries }
495
558
  end
496
559
 
497
- # substr is already applied by Limits.list (matches on name), so slice the
498
- # filtered names directly — only the page's rows pay the per-limiter Redis
499
- # reads in limiter_row (which folds in live status).
500
- def limiter_rows(names, page)
501
- (names.slice(page[:page] * page[:count], page[:count]) || []).map do |name|
502
- ::Wurk::Api::Serializers.limiter_row(name, ::Wurk::Web::Enterprise::Limits.metadata(name))
560
+ # [total, rows] of one listing page. Unfiltered, the page is read straight
561
+ # off Redis at its offset (the set's #snapshot), so page 999 costs what
562
+ # page 0 does; a filter counts matching rows, so it walks the set through
563
+ # the same #snapshot, bounded by FILTER_SCAN_LIMIT.
564
+ def page_rows(set, page, &)
565
+ if page[:substr].empty?
566
+ total, members = set.snapshot(::Wurk::Api::Pagination.offset(page), page[:count])
567
+ return [total, members.map(&)]
503
568
  end
569
+
570
+ rows = ::Wurk::Api::Pagination.walk { |offset, count| set.snapshot(offset, count).last }
571
+ [set.size, ::Wurk::Api::Pagination.slice(rows, page, &)]
572
+ end
573
+
574
+ # The gauges (anything answering size/latency/paused?) and the page's rows.
575
+ def queue_page(queue, page, oldest_first)
576
+ serialize = ->(rec) { ::Wurk::Api::Serializers.job_record(rec) }
577
+ unless page[:substr].empty?
578
+ rows = ::Wurk::Api::Pagination.walk { |offset, count| queue_records(queue, offset, count, oldest_first).last }
579
+ return [queue, ::Wurk::Api::Pagination.slice(rows, page, &serialize)]
580
+ end
581
+
582
+ snapshot, records = queue_records(queue, ::Wurk::Api::Pagination.offset(page), page[:count], oldest_first)
583
+ [snapshot, records.map(&serialize)]
584
+ end
585
+
586
+ # [snapshot, its records in listing order]. Oldest first reads the same
587
+ # LRANGE from the tail: negative indexes `-(offset + count)..-(offset + 1)`,
588
+ # which Redis clamps at the head, then reversed.
589
+ def queue_records(queue, offset, count, oldest_first)
590
+ snapshot = queue.snapshot(oldest_first ? -(offset + count) : offset, count)
591
+ [snapshot, oldest_first ? snapshot.records.reverse : snapshot.records]
592
+ end
593
+
594
+ def render_sorted_entry(set)
595
+ entry = entries_for(set, [params[:key]]).first
596
+ return render(json: { error: 'unknown job' }, status: :not_found) unless entry
597
+
598
+ render json: ::Wurk::Api::Serializers.sorted_entry_detail(entry)
504
599
  end
505
600
 
506
601
  def render_cron_action(success)
@@ -519,6 +614,13 @@ module Wurk
519
614
  )
520
615
  end
521
616
 
617
+ # `?days=N` (1..MAX_HISTORY_DAYS) in seconds when it parses, else
618
+ # `?window=`.
619
+ def history_window(params)
620
+ days = ::Wurk::Api::Pagination.clamp_int(params[:days], 1, MAX_HISTORY_DAYS, nil)
621
+ days ? days * HISTORY_WINDOW_UNITS.fetch('d') : parse_window(params[:window])
622
+ end
623
+
522
624
  # Resolves the per-class metrics window. `minutes:` wins when present;
523
625
  # `hours:` is used otherwise; default falls back to 60 minutes so callers
524
626
  # that pass neither still get a useful series.