wurk 1.14.0 → 1.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (248) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -1
  3. data/app/controllers/concerns/wurk/same_origin_guard.rb +30 -8
  4. data/app/controllers/wurk/api/pagination.rb +70 -53
  5. data/app/controllers/wurk/api_controller.rb +96 -27
  6. data/app/controllers/wurk/dashboard_controller.rb +26 -2
  7. data/app/controllers/wurk/profiles_controller.rb +28 -23
  8. data/config/routes.rb +8 -6
  9. data/lib/wurk/api/admin.rb +84 -0
  10. data/lib/wurk/api/app.rb +7 -2
  11. data/lib/wurk/api/jobs.rb +11 -3
  12. data/lib/wurk/api/queues.rb +10 -0
  13. data/lib/wurk/api/router.rb +6 -0
  14. data/lib/wurk/api/validation.rb +17 -0
  15. data/lib/wurk/batch/callbacks.rb +9 -2
  16. data/lib/wurk/batch.rb +15 -4
  17. data/lib/wurk/batch_set.rb +22 -0
  18. data/lib/wurk/cli.rb +4 -2
  19. data/lib/wurk/client/buffered.rb +104 -48
  20. data/lib/wurk/configuration.rb +28 -0
  21. data/lib/wurk/fetcher/many_queues.rb +18 -8
  22. data/lib/wurk/fetcher/poison_pill.rb +2 -0
  23. data/lib/wurk/fetcher/reaper.rb +3 -0
  24. data/lib/wurk/fetcher/reliable.rb +26 -12
  25. data/lib/wurk/health.rb +6 -1
  26. data/lib/wurk/health_probe.rb +45 -7
  27. data/lib/wurk/heartbeat.rb +29 -6
  28. data/lib/wurk/job_set.rb +25 -3
  29. data/lib/wurk/leader.rb +2 -0
  30. data/lib/wurk/metrics/accumulator.rb +27 -2
  31. data/lib/wurk/metrics/exposition.rb +45 -0
  32. data/lib/wurk/metrics/history.rb +10 -5
  33. data/lib/wurk/metrics/pool_registry.rb +27 -0
  34. data/lib/wurk/metrics/process_counters.rb +16 -0
  35. data/lib/wurk/metrics/process_families.rb +115 -0
  36. data/lib/wurk/metrics/prometheus.rb +84 -51
  37. data/lib/wurk/redis_pool.rb +6 -1
  38. data/lib/wurk/scheduled.rb +7 -0
  39. data/lib/wurk/swarm/child_boot.rb +6 -1
  40. data/lib/wurk/swarm/crash_watch.rb +5 -0
  41. data/lib/wurk/swarm/memory_watch.rb +1 -7
  42. data/lib/wurk/swarm/restart.rb +14 -5
  43. data/lib/wurk/swarm/restart_counters.rb +79 -0
  44. data/lib/wurk/swarm.rb +14 -6
  45. data/lib/wurk/version.rb +1 -1
  46. data/lib/wurk.rb +2 -1
  47. data/vendor/assets/dashboard/assets/{ArgsValue-BSTICTw4.js → ArgsValue-D10BT0xi.js} +1 -1
  48. data/vendor/assets/dashboard/assets/ArgsValue-D10BT0xi.js.br +0 -0
  49. data/vendor/assets/dashboard/assets/ArgsValue-D10BT0xi.js.gz +0 -0
  50. data/vendor/assets/dashboard/assets/{BatchDetail-BBMiNTok.js → BatchDetail-ByPRHv86.js} +1 -1
  51. data/vendor/assets/dashboard/assets/BatchDetail-ByPRHv86.js.br +0 -0
  52. data/vendor/assets/dashboard/assets/BatchDetail-ByPRHv86.js.gz +0 -0
  53. data/vendor/assets/dashboard/assets/Batches-CVVOJFhz.js +1 -0
  54. data/vendor/assets/dashboard/assets/Batches-CVVOJFhz.js.br +0 -0
  55. data/vendor/assets/dashboard/assets/Batches-CVVOJFhz.js.gz +0 -0
  56. data/vendor/assets/dashboard/assets/{Busy-CcSvIQFY.js → Busy-Dp26yz1A.js} +1 -1
  57. data/vendor/assets/dashboard/assets/Busy-Dp26yz1A.js.br +0 -0
  58. data/vendor/assets/dashboard/assets/Busy-Dp26yz1A.js.gz +0 -0
  59. data/vendor/assets/dashboard/assets/{Cron-ByRxtItH.js → Cron-BiRV6eoU.js} +1 -1
  60. data/vendor/assets/dashboard/assets/Cron-BiRV6eoU.js.br +0 -0
  61. data/vendor/assets/dashboard/assets/Cron-BiRV6eoU.js.gz +0 -0
  62. data/vendor/assets/dashboard/assets/Dashboard-CnpCQ8Of.js +1 -0
  63. data/vendor/assets/dashboard/assets/Dashboard-CnpCQ8Of.js.br +0 -0
  64. data/vendor/assets/dashboard/assets/Dashboard-CnpCQ8Of.js.gz +0 -0
  65. data/vendor/assets/dashboard/assets/Dead-ykYbsK8n.js +1 -0
  66. data/vendor/assets/dashboard/assets/Dead-ykYbsK8n.js.br +0 -0
  67. data/vendor/assets/dashboard/assets/Dead-ykYbsK8n.js.gz +0 -0
  68. data/vendor/assets/dashboard/assets/{Extension--fRIfdcW.js → Extension-BUsciXhN.js} +1 -1
  69. data/vendor/assets/dashboard/assets/Extension-BUsciXhN.js.br +0 -0
  70. data/vendor/assets/dashboard/assets/Extension-BUsciXhN.js.gz +0 -0
  71. data/vendor/assets/dashboard/assets/FilterBox-DBIZXNtY.js +1 -0
  72. data/vendor/assets/dashboard/assets/FilterBox-DBIZXNtY.js.br +0 -0
  73. data/vendor/assets/dashboard/assets/FilterBox-DBIZXNtY.js.gz +0 -0
  74. data/vendor/assets/dashboard/assets/{FlowDetail-CxmWxNpF.js → FlowDetail-CIGu7pRb.js} +1 -1
  75. data/vendor/assets/dashboard/assets/FlowDetail-CIGu7pRb.js.br +0 -0
  76. data/vendor/assets/dashboard/assets/FlowDetail-CIGu7pRb.js.gz +0 -0
  77. data/vendor/assets/dashboard/assets/{FlowState-ClzInw0-.js → FlowState-B04ZLDe-.js} +1 -1
  78. data/vendor/assets/dashboard/assets/FlowState-B04ZLDe-.js.br +0 -0
  79. data/vendor/assets/dashboard/assets/FlowState-B04ZLDe-.js.gz +0 -0
  80. data/vendor/assets/dashboard/assets/{Flows-CCnTVfkN.js → Flows-Ce9Qoq0c.js} +1 -1
  81. data/vendor/assets/dashboard/assets/Flows-Ce9Qoq0c.js.br +0 -0
  82. data/vendor/assets/dashboard/assets/Flows-Ce9Qoq0c.js.gz +0 -0
  83. data/vendor/assets/dashboard/assets/{JobDetailModal-Btzt9HOQ.js → JobDetailModal-DWp0OMoV.js} +1 -1
  84. data/vendor/assets/dashboard/assets/JobDetailModal-DWp0OMoV.js.br +0 -0
  85. data/vendor/assets/dashboard/assets/JobDetailModal-DWp0OMoV.js.gz +0 -0
  86. data/vendor/assets/dashboard/assets/{Limiters-DYxSN0sy.js → Limiters-DiWCiVay.js} +1 -1
  87. data/vendor/assets/dashboard/assets/Limiters-DiWCiVay.js.br +0 -0
  88. data/vendor/assets/dashboard/assets/Limiters-DiWCiVay.js.gz +0 -0
  89. data/vendor/assets/dashboard/assets/{Metrics-BfSRkQ2U.js → Metrics-DmMS2wo_.js} +1 -1
  90. data/vendor/assets/dashboard/assets/Metrics-DmMS2wo_.js.br +0 -0
  91. data/vendor/assets/dashboard/assets/Metrics-DmMS2wo_.js.gz +0 -0
  92. data/vendor/assets/dashboard/assets/{NotFound-DrAE8U92.js → NotFound-CM1j8-Fv.js} +1 -1
  93. data/vendor/assets/dashboard/assets/NotFound-CM1j8-Fv.js.br +0 -0
  94. data/vendor/assets/dashboard/assets/NotFound-CM1j8-Fv.js.gz +0 -0
  95. data/vendor/assets/dashboard/assets/{PageHeader-w3hA_hMm.js → PageHeader-BcHVxRbX.js} +1 -1
  96. data/vendor/assets/dashboard/assets/PageHeader-BcHVxRbX.js.br +0 -0
  97. data/vendor/assets/dashboard/assets/PageHeader-BcHVxRbX.js.gz +0 -0
  98. data/vendor/assets/dashboard/assets/Profiles-B70MlVYQ.js +1 -0
  99. data/vendor/assets/dashboard/assets/Profiles-B70MlVYQ.js.br +0 -0
  100. data/vendor/assets/dashboard/assets/Profiles-B70MlVYQ.js.gz +0 -0
  101. data/vendor/assets/dashboard/assets/Queues-D3JZtWBv.js +1 -0
  102. data/vendor/assets/dashboard/assets/Queues-D3JZtWBv.js.br +0 -0
  103. data/vendor/assets/dashboard/assets/Queues-D3JZtWBv.js.gz +0 -0
  104. data/vendor/assets/dashboard/assets/Retries-BTWoV6so.js +1 -0
  105. data/vendor/assets/dashboard/assets/Retries-BTWoV6so.js.br +0 -0
  106. data/vendor/assets/dashboard/assets/Retries-BTWoV6so.js.gz +0 -0
  107. data/vendor/assets/dashboard/assets/Scheduled-D3PyiFqr.js +1 -0
  108. data/vendor/assets/dashboard/assets/Scheduled-D3PyiFqr.js.br +0 -0
  109. data/vendor/assets/dashboard/assets/Scheduled-D3PyiFqr.js.gz +0 -0
  110. data/vendor/assets/dashboard/assets/{Search-D42Va7lB.js → Search-bLwIhsaR.js} +1 -1
  111. data/vendor/assets/dashboard/assets/Search-bLwIhsaR.js.br +0 -0
  112. data/vendor/assets/dashboard/assets/Search-bLwIhsaR.js.gz +0 -0
  113. data/vendor/assets/dashboard/assets/ar-CugpXGea.js +1 -0
  114. data/vendor/assets/dashboard/assets/ar-CugpXGea.js.br +0 -0
  115. data/vendor/assets/dashboard/assets/ar-CugpXGea.js.gz +0 -0
  116. data/vendor/assets/dashboard/assets/{charts-C1xMUYFp.js → charts-DZyiygHX.js} +1 -1
  117. data/vendor/assets/dashboard/assets/charts-DZyiygHX.js.br +0 -0
  118. data/vendor/assets/dashboard/assets/charts-DZyiygHX.js.gz +0 -0
  119. data/vendor/assets/dashboard/assets/de-Bl5Jn_0l.js +1 -0
  120. data/vendor/assets/dashboard/assets/de-Bl5Jn_0l.js.br +0 -0
  121. data/vendor/assets/dashboard/assets/de-Bl5Jn_0l.js.gz +0 -0
  122. data/vendor/assets/dashboard/assets/es-lqQrmjYI.js +1 -0
  123. data/vendor/assets/dashboard/assets/es-lqQrmjYI.js.br +0 -0
  124. data/vendor/assets/dashboard/assets/es-lqQrmjYI.js.gz +0 -0
  125. data/vendor/assets/dashboard/assets/fr-D70VBoh3.js +1 -0
  126. data/vendor/assets/dashboard/assets/fr-D70VBoh3.js.br +0 -0
  127. data/vendor/assets/dashboard/assets/fr-D70VBoh3.js.gz +0 -0
  128. data/vendor/assets/dashboard/assets/{i18n-BXp3XejR.js → i18n-DdPClaCG.js} +1 -1
  129. data/vendor/assets/dashboard/assets/i18n-DdPClaCG.js.br +0 -0
  130. data/vendor/assets/dashboard/assets/i18n-DdPClaCG.js.gz +0 -0
  131. data/vendor/assets/dashboard/assets/index-CtK2Dy_G.css +1 -0
  132. data/vendor/assets/dashboard/assets/index-CtK2Dy_G.css.br +0 -0
  133. data/vendor/assets/dashboard/assets/index-CtK2Dy_G.css.gz +0 -0
  134. data/vendor/assets/dashboard/assets/{index-CiLU1zz8.js → index-ygQ-XC-L.js} +3 -3
  135. data/vendor/assets/dashboard/assets/index-ygQ-XC-L.js.br +0 -0
  136. data/vendor/assets/dashboard/assets/index-ygQ-XC-L.js.gz +0 -0
  137. data/vendor/assets/dashboard/assets/ja-BYMqnHjg.js +1 -0
  138. data/vendor/assets/dashboard/assets/ja-BYMqnHjg.js.br +0 -0
  139. data/vendor/assets/dashboard/assets/ja-BYMqnHjg.js.gz +0 -0
  140. data/vendor/assets/dashboard/assets/pt-BR-4k5M2MYH.js +1 -0
  141. data/vendor/assets/dashboard/assets/pt-BR-4k5M2MYH.js.br +0 -0
  142. data/vendor/assets/dashboard/assets/pt-BR-4k5M2MYH.js.gz +0 -0
  143. data/vendor/assets/dashboard/assets/useJobSetList-D-uafkUL.js +2 -0
  144. data/vendor/assets/dashboard/assets/useJobSetList-D-uafkUL.js.br +0 -0
  145. data/vendor/assets/dashboard/assets/useJobSetList-D-uafkUL.js.gz +0 -0
  146. data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-Cuv4BHkC.js → useResetPageOnEmpty-DUA0JTgs.js} +1 -1
  147. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-DUA0JTgs.js.br +0 -0
  148. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-DUA0JTgs.js.gz +0 -0
  149. data/vendor/assets/dashboard/assets/{useSort-V9yUrByY.js → useSort-BwoC7g__.js} +1 -1
  150. data/vendor/assets/dashboard/assets/useSort-BwoC7g__.js.br +0 -0
  151. data/vendor/assets/dashboard/assets/useSort-BwoC7g__.js.gz +0 -0
  152. data/vendor/assets/dashboard/assets/{utils-69XSoGbQ.js → utils-CQj4rqng.js} +1 -1
  153. data/vendor/assets/dashboard/assets/utils-CQj4rqng.js.br +0 -0
  154. data/vendor/assets/dashboard/assets/utils-CQj4rqng.js.gz +0 -0
  155. data/vendor/assets/dashboard/assets/zh-CN-B9E1MvsM.js +1 -0
  156. data/vendor/assets/dashboard/assets/zh-CN-B9E1MvsM.js.br +0 -0
  157. data/vendor/assets/dashboard/assets/zh-CN-B9E1MvsM.js.gz +0 -0
  158. data/vendor/assets/dashboard/index.html +8 -8
  159. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  160. metadata +117 -109
  161. data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.br +0 -0
  162. data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.gz +0 -0
  163. data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.br +0 -0
  164. data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.gz +0 -0
  165. data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js +0 -1
  166. data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.br +0 -0
  167. data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.gz +0 -0
  168. data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.br +0 -0
  169. data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.gz +0 -0
  170. data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.br +0 -0
  171. data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.gz +0 -0
  172. data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js +0 -1
  173. data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.br +0 -0
  174. data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.gz +0 -0
  175. data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js +0 -1
  176. data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.br +0 -0
  177. data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.gz +0 -0
  178. data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.br +0 -0
  179. data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.gz +0 -0
  180. data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js +0 -2
  181. data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.br +0 -0
  182. data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.gz +0 -0
  183. data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.br +0 -0
  184. data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.gz +0 -0
  185. data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.br +0 -0
  186. data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.gz +0 -0
  187. data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.br +0 -0
  188. data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.gz +0 -0
  189. data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.br +0 -0
  190. data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.gz +0 -0
  191. data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.br +0 -0
  192. data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.gz +0 -0
  193. data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.br +0 -0
  194. data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.gz +0 -0
  195. data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.br +0 -0
  196. data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.gz +0 -0
  197. data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.br +0 -0
  198. data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.gz +0 -0
  199. data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js +0 -1
  200. data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.br +0 -0
  201. data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.gz +0 -0
  202. data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js +0 -1
  203. data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.br +0 -0
  204. data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.gz +0 -0
  205. data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js +0 -1
  206. data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.br +0 -0
  207. data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.gz +0 -0
  208. data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js +0 -1
  209. data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.br +0 -0
  210. data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.gz +0 -0
  211. data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.br +0 -0
  212. data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.gz +0 -0
  213. data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js +0 -1
  214. data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.br +0 -0
  215. data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.gz +0 -0
  216. data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.br +0 -0
  217. data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.gz +0 -0
  218. data/vendor/assets/dashboard/assets/de-B2NCIuzo.js +0 -1
  219. data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.br +0 -0
  220. data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.gz +0 -0
  221. data/vendor/assets/dashboard/assets/es-BGeu_JNB.js +0 -1
  222. data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.br +0 -0
  223. data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.gz +0 -0
  224. data/vendor/assets/dashboard/assets/fr-DQVBWFId.js +0 -1
  225. data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.br +0 -0
  226. data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.gz +0 -0
  227. data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.br +0 -0
  228. data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.gz +0 -0
  229. data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.br +0 -0
  230. data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.gz +0 -0
  231. data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css +0 -1
  232. data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.br +0 -0
  233. data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.gz +0 -0
  234. data/vendor/assets/dashboard/assets/ja-D06IBqzL.js +0 -1
  235. data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.br +0 -0
  236. data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.gz +0 -0
  237. data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js +0 -1
  238. data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.br +0 -0
  239. data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.gz +0 -0
  240. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.br +0 -0
  241. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.gz +0 -0
  242. data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.br +0 -0
  243. data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.gz +0 -0
  244. data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.br +0 -0
  245. data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.gz +0 -0
  246. data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js +0 -1
  247. data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.br +0 -0
  248. data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.gz +0 -0
@@ -0,0 +1,84 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'idempotency'
4
+ require_relative 'jobs'
5
+ require_relative 'page'
6
+ require_relative 'problem'
7
+ require_relative 'response'
8
+ require_relative 'validation'
9
+
10
+ module Wurk
11
+ module API
12
+ # The operator actions the dashboard already offers, for a client that has
13
+ # no screen: put one retrying or dead job back on its queue, empty a queue.
14
+ #
15
+ # Each goes through the object the dashboard's action calls —
16
+ # SortedEntry#retry, Queue#clear — so the two surfaces cannot disagree about
17
+ # what "retry" does to `retry_count` or what "clear" removes.
18
+ #
19
+ # Neither is safe to send twice blind. A second retry after a lost reply
20
+ # finds nothing and reads as a 404 the job did not earn, and a second clear
21
+ # empties whatever was enqueued in between. So every route here takes
22
+ # `Idempotency-Key` exactly as the produce plane does.
23
+ module Admin
24
+ module_function
25
+
26
+ TABLE = [
27
+ [:post, '/retries/:jid/retry', :admin, :retry_retrying],
28
+ [:post, '/dead/:jid/retry', :admin, :retry_dead],
29
+ [:post, '/queues/:name/clear', :admin, :clear]
30
+ ].freeze
31
+
32
+ def draw(router)
33
+ TABLE.each do |verb, pattern, scope, handler|
34
+ router.public_send(verb, pattern, scope: scope) do |request|
35
+ claimed(request) { public_send(handler, request) }
36
+ end
37
+ end
38
+ end
39
+
40
+ def retry_retrying(request) = retry_entry(request, ::Wurk::RetrySet.new)
41
+ def retry_dead(request) = retry_entry(request, ::Wurk::DeadSet.new)
42
+
43
+ # A job the poller promoted, or another client retried, between the
44
+ # lookup and the removal is not retried twice: SortedEntry#retry answers
45
+ # nil when its ZREM finds nothing, and that is the same 404 a jid that
46
+ # was never there gets.
47
+ def retry_entry(request, set)
48
+ jid = Validation.jid!(request.path_params[:jid].to_s)
49
+ score = Validation.score!(Page.query!(request))
50
+ entry = Jobs.find_entry(set, jid, score)
51
+ return job_not_found(request, set, jid) unless entry&.retry
52
+
53
+ Response.json(200, jid: jid, set: set.name, queue: entry.queue)
54
+ end
55
+
56
+ def clear(request)
57
+ queue = ::Wurk::Queue.new(Validation.queue_name!(request.path_params[:name]))
58
+ queue.clear
59
+ Response.json(200, name: queue.name, cleared: true)
60
+ end
61
+
62
+ # The body cap and the shape of the key are checked before the claim, the
63
+ # rest inside it, in the order Jobs#produce uses and for its reason: a
64
+ # refusal releases the key, a success is what a resend replays.
65
+ def claimed(request, &action)
66
+ config = request.config
67
+ raw = Validation.body!(request, config.api_max_body_bytes)
68
+ Idempotency.around(request, raw, config) { Jobs.rejectable(request, &action) }
69
+ rescue Validation::Invalid => e
70
+ Problem.from(e, instance: request.path)
71
+ end
72
+
73
+ def job_not_found(request, set, jid)
74
+ Problem.render(
75
+ Problem::JOB_NOT_FOUND,
76
+ status: 404,
77
+ detail: "No job in #{set.name} has jid #{jid}.",
78
+ instance: request.path,
79
+ jid: jid
80
+ )
81
+ end
82
+ end
83
+ end
84
+ end
data/lib/wurk/api/app.rb CHANGED
@@ -3,6 +3,7 @@
3
3
  require 'rack'
4
4
  require_relative '../api'
5
5
  require_relative '../version'
6
+ require_relative 'admin'
6
7
  require_relative 'auth'
7
8
  require_relative 'flows'
8
9
  require_relative 'jobs'
@@ -36,6 +37,10 @@ module Wurk
36
37
  draw(@router)
37
38
  end
38
39
 
40
+ # Every route this app answers, for test/unit/api_openapi_parity_test.rb
41
+ # to hold docs/api/openapi.yaml to.
42
+ def routes = @router.routes
43
+
39
44
  def call(env)
40
45
  request = Request.new(env)
41
46
  # Stamped once, before anything reads it, so every handler sees the
@@ -52,14 +57,14 @@ module Wurk
52
57
  @config || ::Wurk.configuration
53
58
  end
54
59
 
55
- # One table per plane, each owning its own routes and handlers. Queues,
56
- # Flows and Swarm join Jobs here.
60
+ # One table per plane, each owning its own routes and handlers.
57
61
  def draw(router)
58
62
  router.get('/', scope: Auth::ANY) { |request| root(request) }
59
63
  Jobs.draw(router)
60
64
  Queues.draw(router)
61
65
  Flows.draw(router)
62
66
  Swarm.draw(router)
67
+ Admin.draw(router)
63
68
  end
64
69
 
65
70
  # Authentication runs before the version gate and before routing, so an
data/lib/wurk/api/jobs.rb CHANGED
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative 'idempotency'
4
+ require_relative 'page'
4
5
  require_relative 'problem'
5
6
  require_relative 'response'
6
7
  require_relative 'validation'
@@ -123,7 +124,8 @@ module Wurk
123
124
  # action against a different set.
124
125
  def destroy(request)
125
126
  jid = Validation.jid!(request.path_params[:jid].to_s)
126
- set = cancellable_sets.find { |candidate| remove(candidate, jid) }
127
+ score = Validation.score!(Page.query!(request))
128
+ set = cancellable_sets.find { |candidate| remove(candidate, jid, score) }
127
129
  return job_not_found(request, jid) unless set
128
130
 
129
131
  Response.json(200, jid: jid, set: set.name)
@@ -142,11 +144,17 @@ module Wurk
142
144
  # them to disagree. A job promoted out of the set between the two steps
143
145
  # removes nothing and reads as "already gone", the honest answer to a
144
146
  # cancel that lost the race.
145
- def remove(set, jid)
146
- entry = set.find_job(jid)
147
+ def remove(set, jid, score)
148
+ entry = find_entry(set, jid, score)
147
149
  entry ? entry.delete : false
148
150
  end
149
151
 
152
+ # Without a score this is JobSet#find_job's ZSCAN, which walks the whole
153
+ # set on a miss; with one it is a single ZRANGEBYSCORE.
154
+ def find_entry(set, jid, score)
155
+ score ? set.fetch(score, jid).first : set.find_job(jid)
156
+ end
157
+
150
158
  # Validation decided which problem this is; rendering it is all that is
151
159
  # left, so a new rejection never needs a new arm here.
152
160
  def problem(request, error) = Problem.from(error, instance: request.path)
@@ -39,6 +39,7 @@ module Wurk
39
39
  [:get, '/retries', :read, :retries],
40
40
  [:get, '/scheduled', :read, :scheduled],
41
41
  [:get, '/dead', :read, :dead],
42
+ [:get, '/batches', :read, :batches],
42
43
  [:get, '/batches/:bid', :read, :batch],
43
44
  [:post, '/queues/:name/pause', :admin, :pause],
44
45
  [:post, '/queues/:name/unpause', :admin, :unpause]
@@ -106,6 +107,15 @@ module Wurk
106
107
  Response.json(200, name: set.name, total: total, page: window.page, count: window.count, jobs: jobs)
107
108
  end
108
109
 
110
+ # Newest first, each row Batch::Status#summary: the failed and dead jid
111
+ # sets counted rather than listed, so a page of 200 batches with 10k
112
+ # failures each is not two million jids. `/batches/:bid` carries them.
113
+ def batches(request)
114
+ window = Page.window!(request)
115
+ total, rows = ::Wurk::BatchSet.new.snapshot(window.offset, window.count)
116
+ Response.json(200, total: total, page: window.page, count: window.count, batches: rows)
117
+ end
118
+
109
119
  def batch(request)
110
120
  bid = Validation.bid!(request.path_params[:bid])
111
121
  status = ::Wurk::Batch::Status.new(bid)
@@ -31,6 +31,12 @@ module Wurk
31
31
  @routes = []
32
32
  end
33
33
 
34
+ # Pattern strings rebuilt from segments, so '/jobs/:jid' reads back as
35
+ # it was drawn.
36
+ def routes
37
+ @routes.map { |route| [route.verb, "/#{route.segments.join('/')}", route.scope] }
38
+ end
39
+
34
40
  def get(pattern, scope:, &handler) = add('GET', pattern, scope, handler)
35
41
  def post(pattern, scope:, &handler) = add('POST', pattern, scope, handler)
36
42
  def delete(pattern, scope:, &handler) = add('DELETE', pattern, scope, handler)
@@ -176,6 +176,23 @@ module Wurk
176
176
  # @return [String] the fid.
177
177
  def fid!(fid) = url_safe!(fid, 'flow id')
178
178
 
179
+ # `?score=` is a member's ZSET score, the `at` a listing row carries.
180
+ # With it a lookup is one ZRANGEBYSCORE at that score instead of a ZSCAN
181
+ # over the whole set; nil when absent, so the caller falls back to the scan.
182
+ # @return [Float, nil]
183
+ def score!(query)
184
+ raw = query['score']
185
+ return nil if raw.nil? || raw == ''
186
+
187
+ score = Float(raw)
188
+ # Float accepts "nan" and "inf"; Redis refuses NaN as a range bound.
189
+ raise Invalid, "The 'score' parameter must be a number." unless score.finite?
190
+
191
+ score
192
+ rescue ::ArgumentError, ::TypeError
193
+ raise Invalid, "The 'score' parameter must be a number."
194
+ end
195
+
179
196
  def url_safe!(value, label)
180
197
  return value if value.is_a?(::String) && JID_FORMAT.match?(value)
181
198
 
@@ -283,9 +283,16 @@ module Wurk
283
283
  Wurk.redis { |conn| conn.call('SCARD', "b-#{bid}-died") }.to_i.positive?
284
284
  end
285
285
 
286
+ # Two round trips however many kids: the death flags are read in one
287
+ # pipeline rather than one HGET per kid.
286
288
  def any_child_dead?(bid)
287
- kids = Wurk.redis { |conn| conn.call('SMEMBERS', "b-#{bid}-kids") }
288
- kids.any? { |kid| death_fired?(kid) }
289
+ Wurk.redis do |conn|
290
+ kids = conn.call('SMEMBERS', "b-#{bid}-kids")
291
+ next false if kids.empty?
292
+
293
+ flags = conn.pipelined { |pipe| kids.each { |kid| pipe.call('HGET', "b-#{kid}", 'death') } }
294
+ flags.include?('1')
295
+ end
289
296
  end
290
297
 
291
298
  # Per-callback rescue: one bad spec or a transient enqueue failure must
data/lib/wurk/batch.rb CHANGED
@@ -546,11 +546,22 @@ module Wurk
546
546
  end
547
547
  end
548
548
 
549
+ # One pipeline per generation of the batch tree rather than two round
550
+ # trips per batch: each flags its batches and reads their kids, the next
551
+ # flags those. Replay-safe for pipelined_eval: the flag converges and the
552
+ # reads are reads.
549
553
  def cascade_invalidate(bid)
550
- Wurk.redis do |conn|
551
- Wurk::Lua::Loader.eval_cached(conn, :batch_invalidate, keys: ["b-#{bid}"], argv: [])
552
- kids = conn.call('SMEMBERS', "b-#{bid}-kids") || []
553
- kids.each { |child| cascade_invalidate(child) }
554
+ generation = [bid]
555
+ until generation.empty?
556
+ replies = Wurk.redis do |conn|
557
+ Wurk::Lua::Loader.pipelined_eval(conn) do |pipe, eval_method|
558
+ generation.each do |member|
559
+ Wurk::Lua::Loader.public_send(eval_method, pipe, :batch_invalidate, keys: ["b-#{member}"], argv: [])
560
+ pipe.call('SMEMBERS', "b-#{member}-kids")
561
+ end
562
+ end
563
+ end
564
+ generation = replies.each_slice(2).flat_map { |_, kids| kids || [] }
554
565
  end
555
566
  end
556
567
 
@@ -11,6 +11,7 @@ module Wurk
11
11
  include Enumerable
12
12
 
13
13
  PAGE_SIZE = 100
14
+ TAG_SCAN_LIMIT = 20_000
14
15
 
15
16
  def initialize(key: 'batches')
16
17
  @key = key
@@ -52,6 +53,21 @@ module Wurk
52
53
  end
53
54
  end
54
55
 
56
+ # #snapshot narrowed to the batches tagged `tag` and still indexed in
57
+ # this set: [how many, the page of summaries]. The tag index is a SET, so
58
+ # ordering it newest-first reads every member's score (one ZMSCORE) —
59
+ # bounded at TAG_SCAN_LIMIT members, past which the rest are not listed.
60
+ # Internal reader for the dashboard.
61
+ #
62
+ # @return [Array(Integer, Array<Hash>)]
63
+ def tagged_snapshot(tag, offset, count)
64
+ bids = scan_tags(tag).first(TAG_SCAN_LIMIT)
65
+ Wurk.redis do |conn|
66
+ ranked = newest_first(bids, bids.empty? ? [] : conn.call('ZMSCORE', @key, *bids))
67
+ [ranked.size, summaries(conn, ranked[offset, count] || [])]
68
+ end
69
+ end
70
+
55
71
  # Yields each BID indexed under the given tag (set at `tags:<tag>`).
56
72
  # No JSON parsing; cheap iteration. Caller wraps in `Status.new(bid)`
57
73
  # if it needs full state.
@@ -78,6 +94,12 @@ module Wurk
78
94
  end
79
95
  end
80
96
 
97
+ # The bids `scores` ranks, in ZRANGE REV order (score, then member,
98
+ # descending); an unscored bid is not in this set.
99
+ def newest_first(bids, scores)
100
+ bids.zip(scores).select(&:last).sort { |(a, sa), (b, sb)| [sb.to_f, b] <=> [sa.to_f, a] }.map(&:first)
101
+ end
102
+
81
103
  def summaries(conn, bids)
82
104
  return [] if bids.empty?
83
105
 
data/lib/wurk/cli.rb CHANGED
@@ -119,7 +119,8 @@ module Wurk
119
119
  ['-w', '--window SECONDS', :health_window, 'A heartbeat older than this does not count (default 60)', :to_i],
120
120
  ['-m', '--min COUNT', :health_min, 'Processes on this host that must be beating (default 1)', :to_i],
121
121
  ['-u', '--redis-url URL', :health_redis_url, 'Redis to read (default REDIS_URL)'],
122
- ['-s', '--strict', :health_strict, 'Fail when Redis cannot be read, rather than passing']
122
+ ['-s', '--strict', :health_strict, 'Fail when Redis cannot be read, rather than passing'],
123
+ ['-a', '--max-job-age SECONDS', :health_max_job_age, 'Also fail when a job here has run this long (off)', :to_i]
123
124
  ].freeze
124
125
 
125
126
  HELP_COMMANDS = "\nCommands:\n " \
@@ -298,7 +299,8 @@ module Wurk
298
299
  opts = @health_options
299
300
  @config.redis = { url: opts[:health_redis_url] } if opts[:health_redis_url]
300
301
  HealthProbe.new(@config, window: opts[:health_window] || HealthProbe::DEFAULT_WINDOW,
301
- min: opts[:health_min] || 1, strict: opts[:health_strict] == true).call
302
+ min: opts[:health_min] || 1, strict: opts[:health_strict] == true,
303
+ max_job_age: opts[:health_max_job_age]).call
302
304
  end
303
305
 
304
306
  # `wurk api` — serve the machine-facing HTTP API and nothing else: no
@@ -24,6 +24,12 @@ module Wurk
24
24
  DEFAULT_BUFFER_CAP = 1_000
25
25
  DRAINING_KEY = :wurk_reliable_push_draining
26
26
  UNBUFFERED_KEY = :wurk_reliable_push_unbuffered
27
+ # What a failed replay's raw_push had already landed, handed from the
28
+ # prepended raw_push to Buffered.attempt_replay.
29
+ REPLAY_ACKED_KEY = :wurk_reliable_push_replay_acked
30
+ # Payloads per replayed raw_push. A backlog reaches Redis in about
31
+ # N / REPLAY_SLICE calls, each one pipeline per queue.
32
+ REPLAY_SLICE = 100
27
33
 
28
34
  # A clean exit tries to deliver what is still in memory, but never holds the
29
35
  # process up for longer than this: a hung Redis would otherwise stall shutdown.
@@ -528,15 +534,17 @@ module Wurk
528
534
  end
529
535
 
530
536
  # Leaves the circuit open on the first failed replay and closed once the
531
- # backlog for this Redis has run out.
537
+ # backlog for this Redis has run out. Jobs buffered while a pass is in
538
+ # flight are taken by the next pass of the loop.
532
539
  def replay_all(client, target)
533
540
  drained = 0
534
- while (entry = take_next(target))
535
- unless replay(client, entry)
536
- trip!(target)
537
- return drained
538
- end
539
- drained += 1
541
+ until (entries = take_all(target)).empty?
542
+ landed = replay_entries(client, entries)
543
+ drained += landed
544
+ next if landed == entries.size
545
+
546
+ trip!(target)
547
+ return drained
540
548
  end
541
549
  circuit.close(target)
542
550
  drained
@@ -555,72 +563,113 @@ module Wurk
555
563
  end
556
564
  end
557
565
 
558
- # The oldest entry bound for `target`, removed from the buffer.
559
- def take_next(target)
566
+ # Every entry bound for `target`, oldest first, removed from the buffer
567
+ # in one pass.
568
+ def take_all(target)
560
569
  buffer_mutex.synchronize do
561
- index = buffer.index { |entry| entry.origin.resolve.equal?(target) }
562
- next unless index
570
+ taken, kept = buffer.partition { |entry| entry.origin.resolve.equal?(target) }
571
+ next taken if taken.empty?
563
572
 
564
- @replaying += 1
565
- remove_at(index)
573
+ buffer.replace(kept)
574
+ @buffer_bytes -= taken.sum(&:bytes)
575
+ @replaying += taken.size
576
+ taken
566
577
  end
567
578
  end
568
579
 
569
- def replay_settled
570
- buffer_mutex.synchronize { @replaying = [@replaying - 1, 0].max }
580
+ def replay_settled(count)
581
+ buffer_mutex.synchronize { @replaying = [@replaying - count, 0].max }
571
582
  end
572
583
 
573
584
  def buffered_pools
574
585
  buffer_mutex.synchronize { buffer.map { |entry| entry.origin.resolve } }.uniq(&:object_id)
575
586
  end
576
587
 
577
- # The entry once it is in Redis, nil when it went back to the buffer.
578
- # Same-origin order is all that matters (other entries go to another
579
- # server), so a payload that could not be replayed goes back at the
580
- # absolute head: ahead of every later entry for its own Redis.
581
- def replay(client, entry)
582
- begin
583
- replayed = attempt_replay(client, entry.payload)
584
- rescue StandardError
585
- # Non-connection failures (OOM, LOADING, READONLY…) must not drop
586
- # the taken payload — restore it before propagating, or a
587
- # recovering-but-not-ready Redis silently eats one buffered job per
588
- # drain tick. Redis answered, so the circuit closes and pushes go back
589
- # to surfacing the refusal to their callers.
590
- restore(entry)
591
- circuit.close(client.send(:pool))
592
- raise
593
- end
594
- unless replayed
595
- restore(entry)
596
- return
588
+ # Replays `entries` slice by slice and returns how many reached Redis.
589
+ # On the first slice that does not fully land, what did not land and
590
+ # every later slice go back at the absolute head of the buffer, in
591
+ # order: same-origin order is all that matters (other entries go to
592
+ # another server).
593
+ #
594
+ # A slice is a run of immediate or of scheduled payloads, because
595
+ # Client#atomic_push routes a whole call by its first payload.
596
+ def replay_entries(client, entries)
597
+ landed = 0
598
+ remaining = entries
599
+ replay_slices(entries).each do |slice|
600
+ remaining = remaining.drop(slice.size)
601
+ undelivered, error = replay(client, slice)
602
+ landed += slice.size - undelivered.size
603
+ next if undelivered.empty? && error.nil?
604
+
605
+ return give_back(client, undelivered + remaining, error, landed)
597
606
  end
598
-
599
- client.send(:emit_enqueued, [entry.payload])
600
- Wurk::Metrics::Statsd.increment('jobs.recovered.push')
601
- entry
607
+ landed
602
608
  ensure
603
- replay_settled
609
+ replay_settled(entries.size)
610
+ end
611
+
612
+ def replay_slices(entries)
613
+ entries.chunk_while { |a, b| a.payload.key?('at') == b.payload.key?('at') }
614
+ .flat_map { |run| run.each_slice(REPLAY_SLICE).to_a }
604
615
  end
605
616
 
606
- def restore(entry)
617
+ # Puts `entries` back and returns `landed`. Non-connection failures
618
+ # (OOM, LOADING, READONLY…) must not drop the taken payloads either, or
619
+ # a recovering-but-not-ready Redis silently eats buffered jobs; they
620
+ # are restored before the error is raised. Redis answered,
621
+ # so the circuit closes and pushes go back to surfacing the refusal to
622
+ # their callers.
623
+ def give_back(client, entries, error, landed)
624
+ restore(entries)
625
+ return landed unless error
626
+
627
+ circuit.close(client.send(:pool))
628
+ raise error
629
+ end
630
+
631
+ # [entries that did not reach Redis, the non-connection error or nil].
632
+ def replay(client, slice)
633
+ acknowledged, error = attempt_replay(client, slice.map(&:payload))
634
+ landed, undelivered = acknowledged ? split_acknowledged(slice, acknowledged) : [slice, []]
635
+ payloads = landed.map(&:payload)
636
+ client.send(:emit_enqueued, payloads)
637
+ payloads.each { Wurk::Metrics::Statsd.increment('jobs.recovered.push') }
638
+ [undelivered, error]
639
+ end
640
+
641
+ def split_acknowledged(slice, acknowledged)
642
+ seen = {}.compare_by_identity
643
+ acknowledged.each { |payload| seen[payload] = true }
644
+ slice.partition { |entry| seen.key?(entry.payload) }
645
+ end
646
+
647
+ def restore(entries)
648
+ return if entries.empty?
649
+
607
650
  buffer_mutex.synchronize do
608
- @buffer_bytes += entry.bytes
609
- buffer.unshift(entry)
651
+ @buffer_bytes += entries.sum(&:bytes)
652
+ buffer.unshift(*entries)
610
653
  end
611
654
  end
612
655
 
613
656
  # Drain marks the thread so our prepended raw_push re-raises the
614
657
  # transient error back here instead of swallowing it into the buffer
615
- # (which would spin forever).
616
- def attempt_replay(client, payload)
658
+ # (which would spin forever). Returns [nil, nil] when the call landed,
659
+ # else [the payloads Redis acknowledged, the non-connection error or
660
+ # nil]: one call is a pipeline per queue, so it can fail after some
661
+ # of them landed.
662
+ def attempt_replay(client, payloads)
617
663
  Thread.current[DRAINING_KEY] = true
618
- client.send(:raw_push, [payload])
619
- true
664
+ client.send(:raw_push, payloads)
665
+ [nil, nil]
620
666
  rescue RedisClient::ConnectionError, ConnectionPool::TimeoutError
621
- false
667
+ [Thread.current[REPLAY_ACKED_KEY] || [], nil]
668
+ rescue StandardError => e
669
+ [Thread.current[REPLAY_ACKED_KEY] || [], e]
622
670
  ensure
623
671
  Thread.current[DRAINING_KEY] = false
672
+ Thread.current[REPLAY_ACKED_KEY] = nil
624
673
  end
625
674
  end
626
675
 
@@ -672,6 +721,7 @@ module Wurk
672
721
  # replace the push's own result (its return, or the Overflow it is
673
722
  # raising), so the metric is strictly best-effort.
674
723
  def count_drop
724
+ Wurk::Metrics::ProcessCounters.increment(:client_buffer_drops)
675
725
  Wurk::Metrics::Statsd.increment('jobs.dropped.push')
676
726
  rescue StandardError
677
727
  nil
@@ -820,6 +870,12 @@ module Wurk
820
870
 
821
871
  buffer_undelivered(payloads)
822
872
  ensure
873
+ close_ledger
874
+ end
875
+
876
+ def close_ledger
877
+ ledger = Thread.current[DELIVERED_KEY]
878
+ Thread.current[Buffered::REPLAY_ACKED_KEY] = ledger if Thread.current[Buffered::DRAINING_KEY]
823
879
  Thread.current[DELIVERED_KEY] = nil
824
880
  end
825
881
 
@@ -60,6 +60,9 @@ module Wurk
60
60
  # own behavior, where a job that succeeds leaves nothing behind.
61
61
  status_ttl: Keys::STATUS_TTL,
62
62
  status_retention: nil,
63
+ # TTL in seconds of the 10-minute `j|` metrics buckets (Metrics::History);
64
+ # nil keeps MID_TERM, 30 days. See #metrics_retention=.
65
+ metrics_retention: nil,
63
66
  # Distributed tracing (Wurk::Telemetry). Seeded here, not written on
64
67
  # demand, so the read side answers on a config that `freeze!` has already
65
68
  # closed — and so a gem inspecting @options sees the same key set whether
@@ -186,6 +189,7 @@ module Wurk
186
189
  def []=(key, val)
187
190
  guard_frozen!
188
191
  RedisPool.validate_idle_timeout(val) if key == :redis_idle_timeout
192
+ val = parse_metrics_retention(val) if key == :metrics_retention && !val.nil?
189
193
  @options[key] = val
190
194
  end
191
195
 
@@ -195,6 +199,7 @@ module Wurk
195
199
  def merge!(other)
196
200
  guard_frozen!
197
201
  RedisPool.validate_idle_timeout(other[:redis_idle_timeout]) if other.key?(:redis_idle_timeout)
202
+ parse_metrics_retention(other[:metrics_retention]) unless other[:metrics_retention].nil?
198
203
  @options.merge!(other)
199
204
  end
200
205
 
@@ -973,6 +978,28 @@ module Wurk
973
978
  # WURK_MAXMEM_METRIC sets it, an explicit value wins.
974
979
  MEMORY_LIMIT_METRICS = %i[rss pss uss].freeze
975
980
 
981
+ # Upstream's own TTL for the 10-minute buckets; `hours:` windows read up to
982
+ # 72 hours of them, so nothing shorter is accepted.
983
+ METRICS_RETENTION_FLOOR = 3 * 24 * 60 * 60
984
+
985
+ # How long the 10-minute `j|` metrics buckets live. A TTL only: the keys,
986
+ # fields and scores stay Sidekiq's. Each live job class costs 3 fields per
987
+ # bucket, one bucket per 10 minutes, so retention scales those keys
988
+ # linearly (docs/metrics.md, "Redis footprint and retention"). nil restores
989
+ # the 30-day default.
990
+ def metrics_retention=(seconds)
991
+ guard_frozen!
992
+ @options[:metrics_retention] = seconds.nil? ? nil : parse_metrics_retention(seconds)
993
+ end
994
+
995
+ def parse_metrics_retention(seconds)
996
+ value = seconds.respond_to?(:to_i) ? seconds.to_i : 0
997
+ return value if value >= METRICS_RETENTION_FLOOR
998
+
999
+ raise ArgumentError, "metrics_retention must be >= #{METRICS_RETENTION_FLOOR}s (3 days), got #{seconds.inspect}"
1000
+ end
1001
+ private :parse_metrics_retention
1002
+
976
1003
  def memory_limit_metric
977
1004
  @memory_limit_metric || env_memory_limit_metric || :rss
978
1005
  end
@@ -1101,6 +1128,7 @@ module Wurk
1101
1128
  def validate_constructor_options!
1102
1129
  @options[:global_concurrency] = normalize_global_concurrency(@options[:global_concurrency])
1103
1130
  RedisPool.validate_idle_timeout(@options[:redis_idle_timeout])
1131
+ parse_metrics_retention(@options[:metrics_retention]) unless @options[:metrics_retention].nil?
1104
1132
  end
1105
1133
 
1106
1134
  # One flat fork running the default capsule's queues + concurrency. The
@@ -62,22 +62,32 @@ module Wurk
62
62
  end
63
63
 
64
64
  # `[public, private, public, private, ...]` for queues 2..n, fetch_first's
65
- # KEYS, built as one Array rather than a pair per queue flattened. Strict
66
- # mode with nothing paused hands #walk the same frozen array every pass,
67
- # so the steady state reuses one build; a shuffled or filtered list is a
68
- # fresh array per pass and rebuilds. Keyed on the pid too, for the reason
69
- # #queue_keys is. Copy-on-write: processor threads share this fetcher.
65
+ # KEYS. A list #queues_cmd reuses across passes comes frozen (strict, with
66
+ # or without a pause), and its KEYS are built once and kept, keyed on the
67
+ # list's identity and the pid (for the reason #queue_keys is).
68
+ # Copy-on-write: processor threads share this fetcher. A shuffled list is
69
+ # new every pass, so its KEYS go into this thread's buffer instead: the
70
+ # script call copies them out before the buffer is written again.
70
71
  def claim_keys(queues)
71
72
  pid = PidCache.pid
72
73
  cached = @claim_keys
73
74
  return cached[2] if cached && cached[0].equal?(queues) && cached[1] == pid
75
+ return fill_claim_keys(claim_keys_buffer, queues) unless queues.frozen?
74
76
 
75
- keys = []
76
- queues.each_with_index { |public_q, i| keys << public_q << queue_keys(public_q).first unless i.zero? }
77
- keys.freeze
77
+ keys = fill_claim_keys([], queues).freeze
78
78
  @claim_keys = [queues, pid, keys].freeze
79
79
  keys
80
80
  end
81
+
82
+ def fill_claim_keys(keys, queues)
83
+ keys.clear
84
+ queues.each_with_index { |public_q, i| keys << public_q << queue_keys(public_q).first unless i.zero? }
85
+ keys
86
+ end
87
+
88
+ def claim_keys_buffer
89
+ ::Thread.current[:wurk_claim_keys] ||= []
90
+ end
81
91
  end
82
92
  end
83
93
  end