wurk 1.12.0 → 1.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (273) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -2
  3. data/app/controllers/concerns/wurk/sse_streaming.rb +36 -1
  4. data/app/controllers/wurk/api/pagination.rb +9 -7
  5. data/app/controllers/wurk/api/serializers.rb +8 -2
  6. data/app/controllers/wurk/api_controller.rb +57 -24
  7. data/app/controllers/wurk/application_controller.rb +0 -2
  8. data/app/controllers/wurk/dashboard_controller.rb +0 -1
  9. data/config/routes.rb +13 -1
  10. data/exe/sidekiq +8 -0
  11. data/exe/wurk +1 -1
  12. data/lib/generators/wurk/install/install_generator.rb +56 -0
  13. data/lib/sidekiq/cli.rb +4 -3
  14. data/lib/sidekiq/deploy.rb +2 -1
  15. data/lib/sidekiq/embedded.rb +2 -1
  16. data/lib/sidekiq/iterable_job.rb +2 -1
  17. data/lib/sidekiq/job/iterable/active_record_enumerator.rb +2 -1
  18. data/lib/sidekiq/job/iterable/csv_enumerator.rb +2 -1
  19. data/lib/sidekiq/job/iterable.rb +2 -1
  20. data/lib/sidekiq/launcher.rb +2 -1
  21. data/lib/sidekiq/manager.rb +2 -1
  22. data/lib/sidekiq/metrics/query.rb +2 -1
  23. data/lib/sidekiq/pro/web.rb +2 -1
  24. data/lib/sidekiq/scheduled.rb +2 -1
  25. data/lib/sidekiq/web.rb +2 -1
  26. data/lib/sidekiq.rb +3 -2
  27. data/lib/wurk/api/fast.rb +4 -18
  28. data/lib/wurk/api/queues.rb +12 -10
  29. data/lib/wurk/api/serializers.rb +6 -7
  30. data/lib/wurk/api/swarm.rb +0 -1
  31. data/lib/wurk/batch/server_middleware.rb +4 -0
  32. data/lib/wurk/batch/status.rb +45 -22
  33. data/lib/wurk/batch_set.rb +34 -0
  34. data/lib/wurk/boot_checks.rb +80 -0
  35. data/lib/wurk/cli.rb +72 -36
  36. data/lib/wurk/client/buffered.rb +97 -26
  37. data/lib/wurk/client.rb +21 -1
  38. data/lib/wurk/compat/deploy.rb +7 -0
  39. data/lib/wurk/compat/encryption.rb +7 -0
  40. data/lib/wurk/compat/flow.rb +8 -0
  41. data/lib/wurk/compat/iterable_job.rb +8 -0
  42. data/lib/wurk/compat/profile_set.rb +8 -0
  43. data/lib/wurk/compat/server.rb +15 -0
  44. data/lib/wurk/compat/web.rb +20 -0
  45. data/lib/wurk/compat.rb +34 -40
  46. data/lib/wurk/configuration.rb +73 -15
  47. data/lib/wurk/cron.rb +146 -47
  48. data/lib/wurk/dead_set.rb +10 -6
  49. data/lib/wurk/embedded.rb +0 -2
  50. data/lib/wurk/engine.rb +46 -4
  51. data/lib/wurk/error_coalescer.rb +21 -1
  52. data/lib/wurk/fetcher/many_queues.rb +8 -6
  53. data/lib/wurk/fetcher/orphan_reclaim.rb +48 -11
  54. data/lib/wurk/{middleware → fetcher}/poison_pill.rb +4 -2
  55. data/lib/wurk/fetcher/reaper.rb +28 -14
  56. data/lib/wurk/fetcher/reliable.rb +50 -32
  57. data/lib/wurk/fetcher/self_reconcile.rb +0 -1
  58. data/lib/wurk/fetcher/unit_of_work.rb +17 -2
  59. data/lib/wurk/flow/status.rb +10 -4
  60. data/lib/wurk/flow_set.rb +24 -2
  61. data/lib/wurk/health.rb +0 -1
  62. data/lib/wurk/heartbeat.rb +13 -7
  63. data/lib/wurk/import/sidekiq_cron.rb +0 -1
  64. data/lib/wurk/job_logger.rb +4 -2
  65. data/lib/wurk/job_record.rb +2 -1
  66. data/lib/wurk/job_retry.rb +1 -1
  67. data/lib/wurk/job_set.rb +132 -6
  68. data/lib/wurk/job_util.rb +14 -0
  69. data/lib/wurk/launcher.rb +30 -15
  70. data/lib/wurk/lazy_loader.rb +60 -0
  71. data/lib/wurk/leader.rb +1 -8
  72. data/lib/wurk/limiter/bucket.rb +0 -2
  73. data/lib/wurk/limiter/concurrent.rb +19 -36
  74. data/lib/wurk/limiter/leaky.rb +0 -2
  75. data/lib/wurk/limiter/points.rb +0 -2
  76. data/lib/wurk/limiter/server_middleware.rb +4 -1
  77. data/lib/wurk/limiter/unlimited.rb +0 -2
  78. data/lib/wurk/limiter/window.rb +0 -2
  79. data/lib/wurk/limiter.rb +0 -7
  80. data/lib/wurk/logger.rb +44 -5
  81. data/lib/wurk/lua/fast_delete.rb +58 -0
  82. data/lib/wurk/lua/fast_delete_window.lua +46 -0
  83. data/lib/wurk/lua/limiter_concurrent_acquire.lua +10 -3
  84. data/lib/wurk/lua/limiter_concurrent_release.lua +13 -4
  85. data/lib/wurk/lua.rb +1 -41
  86. data/lib/wurk/manager.rb +27 -1
  87. data/lib/wurk/metrics/dashboard_series.rb +38 -18
  88. data/lib/wurk/metrics/history.rb +24 -11
  89. data/lib/wurk/metrics/prometheus.rb +0 -1
  90. data/lib/wurk/metrics/query.rb +0 -2
  91. data/lib/wurk/metrics/queue_rollup.rb +33 -11
  92. data/lib/wurk/metrics/statsd.rb +4 -0
  93. data/lib/wurk/middleware/chain.rb +25 -3
  94. data/lib/wurk/middleware/expiry.rb +4 -0
  95. data/lib/wurk/middleware/interrupt_handler.rb +4 -1
  96. data/lib/wurk/middleware/status.rb +4 -0
  97. data/lib/wurk/middleware/timeout.rb +10 -6
  98. data/lib/wurk/preflight/app_scan.rb +182 -0
  99. data/lib/wurk/preflight/command.rb +21 -5
  100. data/lib/wurk/preflight/owners.rb +0 -1
  101. data/lib/wurk/preflight/payload_sample.rb +0 -1
  102. data/lib/wurk/preflight/redis_guard.rb +98 -0
  103. data/lib/wurk/preflight/verify.rb +0 -2
  104. data/lib/wurk/preflight.rb +9 -5
  105. data/lib/wurk/process_set.rb +6 -1
  106. data/lib/wurk/profiler.rb +2 -1
  107. data/lib/wurk/queue.rb +33 -6
  108. data/lib/wurk/queue_consumers.rb +10 -1
  109. data/lib/wurk/rails_boot.rb +9 -4
  110. data/lib/wurk/railtie.rb +3 -4
  111. data/lib/wurk/redis_options.rb +5 -0
  112. data/lib/wurk/redis_pool.rb +24 -1
  113. data/lib/wurk/sentry/error_handler.rb +5 -0
  114. data/lib/wurk/sentry/middleware.rb +4 -0
  115. data/lib/wurk/sentry.rb +11 -0
  116. data/lib/wurk/sorted_entry.rb +16 -12
  117. data/lib/wurk/stats.rb +85 -1
  118. data/lib/wurk/swarm/child_boot.rb +0 -2
  119. data/lib/wurk/swarm/liveness.rb +0 -2
  120. data/lib/wurk/swarm/memory_watch.rb +116 -0
  121. data/lib/wurk/swarm/prefork.rb +54 -0
  122. data/lib/wurk/swarm.rb +95 -55
  123. data/lib/wurk/version.rb +1 -1
  124. data/lib/wurk/web/enterprise.rb +63 -7
  125. data/lib/wurk/web/rack_app.rb +8 -6
  126. data/lib/wurk/web/stats_routes.rb +93 -0
  127. data/lib/wurk.rb +34 -30
  128. data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js +1 -0
  129. data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.br +0 -0
  130. data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.gz +0 -0
  131. data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js +1 -0
  132. data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.br +0 -0
  133. data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.gz +0 -0
  134. data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js +1 -0
  135. data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.br +0 -0
  136. data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.gz +0 -0
  137. data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js +1 -0
  138. data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.br +0 -0
  139. data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.gz +0 -0
  140. data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js +1 -0
  141. data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.br +0 -0
  142. data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.gz +0 -0
  143. data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js +1 -0
  144. data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.br +0 -0
  145. data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.gz +0 -0
  146. data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js +1 -0
  147. data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.br +0 -0
  148. data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.gz +0 -0
  149. data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js +1 -0
  150. data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.br +0 -0
  151. data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.gz +0 -0
  152. data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js +2 -0
  153. data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.br +0 -0
  154. data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.gz +0 -0
  155. data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js +1 -0
  156. data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.br +0 -0
  157. data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.gz +0 -0
  158. data/vendor/assets/dashboard/assets/{FlowState-yb7virqR.js → FlowState-ClzInw0-.js} +1 -1
  159. data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.br +0 -0
  160. data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.gz +0 -0
  161. data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js +1 -0
  162. data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.br +0 -0
  163. data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.gz +0 -0
  164. data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js +2 -0
  165. data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.br +0 -0
  166. data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.gz +0 -0
  167. data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js +1 -0
  168. data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.br +0 -0
  169. data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.gz +0 -0
  170. data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js +1 -0
  171. data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.br +0 -0
  172. data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.gz +0 -0
  173. data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js +1 -0
  174. data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.br +0 -0
  175. data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.gz +0 -0
  176. data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js +1 -0
  177. data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.br +0 -0
  178. data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.gz +0 -0
  179. data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js +1 -0
  180. data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.br +0 -0
  181. data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.gz +0 -0
  182. data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js +1 -0
  183. data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.br +0 -0
  184. data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.gz +0 -0
  185. data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js +1 -0
  186. data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.br +0 -0
  187. data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.gz +0 -0
  188. data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js +1 -0
  189. data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.br +0 -0
  190. data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.gz +0 -0
  191. data/vendor/assets/dashboard/assets/Search-D42Va7lB.js +1 -0
  192. data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.br +0 -0
  193. data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.gz +0 -0
  194. data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js +1 -0
  195. data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.br +0 -0
  196. data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.gz +0 -0
  197. data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js +1 -0
  198. data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.br +0 -0
  199. data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.gz +0 -0
  200. data/vendor/assets/dashboard/assets/de-B2NCIuzo.js +1 -0
  201. data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.br +0 -0
  202. data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.gz +0 -0
  203. data/vendor/assets/dashboard/assets/es-BGeu_JNB.js +1 -0
  204. data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.br +0 -0
  205. data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.gz +0 -0
  206. data/vendor/assets/dashboard/assets/fr-DQVBWFId.js +1 -0
  207. data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.br +0 -0
  208. data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.gz +0 -0
  209. data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js +1 -0
  210. data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.br +0 -0
  211. data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.gz +0 -0
  212. data/vendor/assets/dashboard/assets/index-CiLU1zz8.js +141 -0
  213. data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.br +0 -0
  214. data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.gz +0 -0
  215. data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css +1 -0
  216. data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.br +0 -0
  217. data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.gz +0 -0
  218. data/vendor/assets/dashboard/assets/ja-D06IBqzL.js +1 -0
  219. data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.br +0 -0
  220. data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.gz +0 -0
  221. data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js +1 -0
  222. data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.br +0 -0
  223. data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.gz +0 -0
  224. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js +1 -0
  225. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.br +0 -0
  226. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.gz +0 -0
  227. data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js +1 -0
  228. data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.br +0 -0
  229. data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.gz +0 -0
  230. data/vendor/assets/dashboard/assets/{utils-CnWZzW2L.js → utils-69XSoGbQ.js} +1 -1
  231. data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.br +0 -0
  232. data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.gz +0 -0
  233. data/vendor/assets/dashboard/assets/wurk-logo-ydeSB9B8.png +0 -0
  234. data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js +1 -0
  235. data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.br +0 -0
  236. data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.gz +0 -0
  237. data/vendor/assets/dashboard/index.html +3 -23
  238. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  239. metadata +132 -38
  240. data/vendor/assets/dashboard/assets/ArgsValue-BJol9S0s.js +0 -1
  241. data/vendor/assets/dashboard/assets/BatchDetail-D_zGQ9Sd.js +0 -1
  242. data/vendor/assets/dashboard/assets/Batches-Dt_xC660.js +0 -1
  243. data/vendor/assets/dashboard/assets/Busy-CWRthsmc.js +0 -1
  244. data/vendor/assets/dashboard/assets/Cron-qeGr2YAm.js +0 -1
  245. data/vendor/assets/dashboard/assets/Dashboard-DQWaAwPA.js +0 -1
  246. data/vendor/assets/dashboard/assets/Dead-D9gOPHtK.js +0 -1
  247. data/vendor/assets/dashboard/assets/Extension-DONXVLdA.js +0 -1
  248. data/vendor/assets/dashboard/assets/FilterBox-BbB6gXHp.js +0 -2
  249. data/vendor/assets/dashboard/assets/FlowDetail-CrmnVRp0.js +0 -1
  250. data/vendor/assets/dashboard/assets/Flows-g4GC8KmE.js +0 -1
  251. data/vendor/assets/dashboard/assets/JobDetailModal-C86k_1wj.js +0 -2
  252. data/vendor/assets/dashboard/assets/Limiters-zyO-iMlK.js +0 -1
  253. data/vendor/assets/dashboard/assets/Metrics-Ckq5fNBv.js +0 -1
  254. data/vendor/assets/dashboard/assets/NotFound-C8NnyjlY.js +0 -1
  255. data/vendor/assets/dashboard/assets/PageHeader-vWeBbSPF.js +0 -1
  256. data/vendor/assets/dashboard/assets/Profiles-DRl-FXPA.js +0 -1
  257. data/vendor/assets/dashboard/assets/Queues-DkXWrihA.js +0 -1
  258. data/vendor/assets/dashboard/assets/Retries-CmlojYIy.js +0 -1
  259. data/vendor/assets/dashboard/assets/Scheduled-Cbbpp3xw.js +0 -1
  260. data/vendor/assets/dashboard/assets/Search-LaEpj2fM.js +0 -1
  261. data/vendor/assets/dashboard/assets/charts-D_PQLoyc.js +0 -1
  262. data/vendor/assets/dashboard/assets/fa-brands-400-Bs6tcqqs.woff2 +0 -0
  263. data/vendor/assets/dashboard/assets/fa-regular-400-DRN8N0d1.woff2 +0 -0
  264. data/vendor/assets/dashboard/assets/fa-solid-900-IAB4Droh.woff2 +0 -0
  265. data/vendor/assets/dashboard/assets/fa-v4compatibility-CErXDOsT.woff2 +0 -0
  266. data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +0 -1
  267. data/vendor/assets/dashboard/assets/index--xvvNlYx.css +0 -1
  268. data/vendor/assets/dashboard/assets/index-FBKGtysy.js +0 -141
  269. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-BiwhHqgh.js +0 -1
  270. data/vendor/assets/dashboard/assets/useSort-jBYVwXvg.js +0 -1
  271. data/vendor/assets/dashboard/assets/wurk-logo-Vy3xW4K0.png +0 -0
  272. data/vendor/assets/dashboard/og-image.png +0 -0
  273. data/vendor/assets/dashboard/wurk-logo.webp +0 -0
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wurk
4
+ module Lua
5
+ # Pro Fast API (§11) deletes, `Queue#delete_job` and `#delete_by_class`: a
6
+ # cursor walk over the queue list from the head, one `fast_delete_window`
7
+ # call per WINDOW entries. The scripts this replaced copied the whole list
8
+ # into the Lua heap and LREMed per match, holding Redis for the entire
9
+ # queue (~300ms per call at 100k jobs).
10
+ #
11
+ # Each call is atomic and removes only payloads it matched, so nothing that
12
+ # did not match is ever deleted. Between calls the list moves:
13
+ # * pops from the tail (every fetcher: LMOVE/BLMOVE RIGHT, BRPOP) shift no
14
+ # head index, so they cannot make the walk skip an entry;
15
+ # * pushes at the head (LPUSH, every enqueue) shift entries already
16
+ # visited forward, into a window still to come: they are read again,
17
+ # which re-checks a non-match and costs nothing else;
18
+ # * the walk's own removals are subtracted from the cursor.
19
+ # What can make it miss a match is another removal ahead of the cursor
20
+ # between calls (a concurrent delete on the same queue, an LREM or LPOP
21
+ # by another client): the entries shift back past the cursor. Such a
22
+ # match stays in the queue and a second call finds it; nothing extra is
23
+ # ever removed.
24
+ module FastDelete
25
+ WINDOW = 1000
26
+ # Matches one call rewrites before it hands the cursor on: each is an
27
+ # LSET that walks the list to its index (see fast_delete_window.lua).
28
+ MAX_REMOVED_PER_CALL = 100
29
+
30
+ # No JSON payload starts with a NUL byte.
31
+ TOMBSTONE = "\0wurk-fast-delete\0"
32
+
33
+ module_function
34
+
35
+ # [payloads removed, first removed payload or nil].
36
+ def call(conn, key, needle, window: WINDOW, max_removed: MAX_REMOVED_PER_CALL)
37
+ cursor = 0
38
+ removed = 0
39
+ first = nil
40
+ loop do
41
+ argv = [needle, cursor, window, TOMBSTONE, max_removed]
42
+ cursor, count, payload = Loader.eval_cached(conn, :fast_delete_window, keys: [key], argv: argv)
43
+ removed += count
44
+ first ||= payload
45
+ return [removed, first] if cursor.negative?
46
+ end
47
+ end
48
+
49
+ def delete_job(conn, key, jid)
50
+ call(conn, key, %("jid":"#{jid}")).last
51
+ end
52
+
53
+ def delete_by_class(conn, key, klass)
54
+ call(conn, key, %("class":"#{klass}")).first
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,46 @@
1
+ -- Pro Fast API (§11) delete, one window of a queue list at a time.
2
+ -- Lua::FastDelete walks the list from the head with a cursor and calls this per
3
+ -- window, so Redis is held for one window, never for a copy of the whole queue.
4
+ -- KEYS[1] = queue:<name>
5
+ -- ARGV[1] = needle: `"jid":"<jid>"` or `"class":"<klass>"`, a plain substring
6
+ -- ARGV[2] = cursor: index of the window's first element
7
+ -- ARGV[3] = window size
8
+ -- ARGV[4] = tombstone, a value no payload can equal
9
+ -- ARGV[5] = most matches one call rewrites
10
+ -- Returns {next cursor or -1 when the list is done, removed, first removed
11
+ -- payload or false}.
12
+ --
13
+ -- Every match in the window is LSET to the tombstone at the index LRANGE just
14
+ -- read it from, in this same atomic call, then one LREM removes exactly those:
15
+ -- only a payload that matched is ever removed, whatever runs between calls.
16
+ -- The LREM scans from the end nearer the window.
17
+ --
18
+ -- Each LSET walks the list to its index, so a window dense with matches deep
19
+ -- in a long list costs one such walk per match. The call therefore stops after
20
+ -- ARGV[5] matches and hands the rest of the window to the next call. The LREM
21
+ -- still scans to the window (no remove-by-index), which bounds a removing call
22
+ -- deep in a long list: measured on a 1M-entry queue, 1,000 matches halfway in
23
+ -- held Redis 17ms in one call uncapped, ~10ms per call capped at 100.
24
+ local key = KEYS[1]
25
+ local start = tonumber(ARGV[2])
26
+ local window = tonumber(ARGV[3])
27
+ local max_removed = tonumber(ARGV[5])
28
+ local items = redis.call('lrange', key, start, start + window - 1)
29
+ local removed, first, scanned = 0, false, #items
30
+ for i = 1, #items do
31
+ if string.find(items[i], ARGV[1], 1, true) then
32
+ redis.call('lset', key, start + i - 1, ARGV[4])
33
+ removed = removed + 1
34
+ if not first then first = items[i] end
35
+ if removed >= max_removed then
36
+ scanned = i
37
+ break
38
+ end
39
+ end
40
+ end
41
+ if removed > 0 then
42
+ local len = redis.call('llen', key)
43
+ redis.call('lrem', key, (start * 2 > len) and -removed or removed, ARGV[4])
44
+ end
45
+ if scanned == #items and #items < window then return {-1, removed, first} end
46
+ return {start + scanned - removed, removed, first}
@@ -5,11 +5,12 @@
5
5
  -- ARGV[2] = lock_timeout seconds
6
6
  -- ARGV[3] = slot_id (caller-generated; SHA-fingerprinted on the Ruby side)
7
7
  -- ARGV[4] = ttl seconds
8
+ -- ARGV[5] = '' when this attempt follows no wait, else the whole seconds waited
8
9
  -- Returns {acquired, held, reclaimed}. acquired: 1 if slot recorded, 0 if full.
9
10
  -- held: live ZCARD gauge after this pass. reclaimed: expired slots evicted here.
10
- -- The lifetime `held` counter in the stats hash is bumped here, not on the Ruby
11
- -- side, so it stays atomic with the ZADD (a process dying mid-acquire can't
12
- -- undercount) and costs no extra round trip on the acquire path.
11
+ -- The `held` counter and the `immediate` / `waited` + `wait_time` split are
12
+ -- bumped here, not on the Ruby side, so they stay atomic with the ZADD (a
13
+ -- process dying mid-acquire can't undercount) and cost no extra round trip.
13
14
  local cs_key = KEYS[1]
14
15
  local st_key = KEYS[2]
15
16
  local t = redis.call('TIME')
@@ -26,6 +27,12 @@ local held = redis.call('ZCARD', cs_key)
26
27
  if held < limit then
27
28
  redis.call('ZADD', cs_key, now + lock_to, slot_id)
28
29
  redis.call('HINCRBY', st_key, 'held', 1)
30
+ if ARGV[5] == '' then
31
+ redis.call('HINCRBY', st_key, 'immediate', 1)
32
+ else
33
+ redis.call('HINCRBY', st_key, 'waited', 1)
34
+ redis.call('HINCRBY', st_key, 'wait_time', ARGV[5])
35
+ end
29
36
  redis.call('EXPIRE', cs_key, ttl)
30
37
  redis.call('EXPIRE', st_key, ttl)
31
38
  return {1, held + 1, reclaimed}
@@ -1,7 +1,16 @@
1
- -- Concurrent limiter: release a held slot. ZREM returns 0 when the slot
2
- -- was already reclaimed by the acquire path's ZREMRANGEBYSCORE — caller
3
- -- treats that as an "overage" and bumps the metric on the Ruby side.
1
+ -- Concurrent limiter: release a held slot and book its metrics.
2
+ -- ZREM returns 0 when the slot was already reclaimed by the acquire path's
3
+ -- ZREMRANGEBYSCORE: the holder outran `lock_timeout`, counted as an overage.
4
4
  -- KEYS[1] = lmtr-cs:<name>
5
+ -- KEYS[2] = lmtr-stats:<name>
5
6
  -- ARGV[1] = slot_id
7
+ -- ARGV[2] = whole seconds the slot was held
8
+ -- ARGV[3] = ttl seconds
6
9
  -- Returns 1 if removed, 0 if already reclaimed.
7
- return redis.call('ZREM', KEYS[1], ARGV[1])
10
+ local removed = redis.call('ZREM', KEYS[1], ARGV[1])
11
+ if removed == 0 then
12
+ redis.call('HINCRBY', KEYS[2], 'overages', 1)
13
+ end
14
+ redis.call('HINCRBY', KEYS[2], 'held_time', ARGV[2])
15
+ redis.call('EXPIRE', KEYS[2], ARGV[3])
16
+ return removed
data/lib/wurk/lua.rb CHANGED
@@ -172,45 +172,6 @@ module Wurk
172
172
  return 0
173
173
  LUA
174
174
 
175
- # Pro Fast API (§11): server-side LRANGE+LREM to delete a single job by
176
- # jid from a queue list. Pure-Ruby Queue#find_job + JobRecord#delete is
177
- # O(N) round-trips; this is O(1) round-trip with O(N) Lua work.
178
- # KEYS = [queue:<name>]
179
- # ARGV = [jid]
180
- # Returns the first payload removed, or nil (Lua false) when none matched —
181
- # Pro's `delete_job` return value. Every match is removed, so a
182
- # duplicate-jid corruption is cleaned up in the same pass.
183
- FAST_DELETE_JOB = <<~LUA
184
- local items = redis.call("lrange", KEYS[1], 0, -1)
185
- local deleted = false
186
- for i = 1, #items do
187
- if string.find(items[i], '"jid":"' .. ARGV[1] .. '"', 1, true) then
188
- if redis.call("lrem", KEYS[1], 1, items[i]) > 0 and not deleted then
189
- deleted = items[i]
190
- end
191
- end
192
- end
193
- return deleted
194
- LUA
195
-
196
- # Pro Fast API (§11): server-side LRANGE+LREM removing every payload whose
197
- # `"class":"<klass>"` field matches. Plain-text scan (no JSON parse) so
198
- # it tolerates partial corruption — caller drops only well-formed matches.
199
- # KEYS = [queue:<name>]
200
- # ARGV = [klass]
201
- # Returns the number of payloads removed.
202
- FAST_DELETE_BY_CLASS = <<~LUA
203
- local items = redis.call("lrange", KEYS[1], 0, -1)
204
- local removed = 0
205
- local needle = '"class":"' .. ARGV[1] .. '"'
206
- for i = 1, #items do
207
- if string.find(items[i], needle, 1, true) then
208
- removed = removed + redis.call("lrem", KEYS[1], 1, items[i])
209
- end
210
- end
211
- return removed
212
- LUA
213
-
214
175
  # Ent Periodic (§2): compare-and-swap the fire marks of `loops:<lid>`.
215
176
  # The cron poller's leader gate is a cached read (`Component#leader?`,
216
177
  # LEADER_CACHE_TTL_MS), so for a few seconds after a handover two
@@ -262,8 +223,6 @@ module Wurk
262
223
  scheduled_claim: SCHEDULED_CLAIM,
263
224
  reliable_requeue: RELIABLE_REQUEUE,
264
225
  requeue_unit: REQUEUE_UNIT,
265
- fast_delete_job: FAST_DELETE_JOB,
266
- fast_delete_by_class: FAST_DELETE_BY_CLASS,
267
226
  release_if_owner: RELEASE_IF_OWNER,
268
227
  cron_claim_fire: CRON_CLAIM_FIRE
269
228
  }.merge(FILE_SCRIPTS).freeze
@@ -275,3 +234,4 @@ module Wurk
275
234
  end
276
235
 
277
236
  require_relative 'lua/loader'
237
+ require_relative 'lua/fast_delete'
data/lib/wurk/manager.rb CHANGED
@@ -167,12 +167,38 @@ module Wurk
167
167
 
168
168
  jobs = cleanup.map(&:job).compact
169
169
  logger.warn { "Terminating #{cleanup.size} busy threads" }
170
- logger.debug { "Jobs still in progress #{jobs.inspect}" }
170
+ log_interrupted(jobs)
171
171
  # `&.` like #quiet: a TERM in the pre-prepare! window (traps install
172
172
  # before launcher.run) reaches here with no fetcher built yet.
173
173
  capsule.fetcher&.bulk_requeue(jobs)
174
174
  end
175
175
 
176
+ # One WARN per job a deploy cut short, because each one runs again from the
177
+ # top (docs/idempotency.md) and the operator chasing a double side effect
178
+ # needs its jid. Capped: a capsule can hold hundreds of threads.
179
+ REQUEUE_LOG_LIMIT = 20
180
+ private_constant :REQUEUE_LOG_LIMIT
181
+
182
+ def log_interrupted(jobs)
183
+ jobs.first(REQUEUE_LOG_LIMIT).each do |uow|
184
+ logger.warn { "Interrupting #{describe_job(uow)}: past the shutdown deadline, it is requeued to run again" }
185
+ end
186
+ extra = jobs.size - REQUEUE_LOG_LIMIT
187
+ logger.warn { "Interrupting #{extra} more in-flight jobs, which run again too" } if extra.positive?
188
+ end
189
+
190
+ # The payload is parsed here, off the hot path, because the fetcher never
191
+ # parses. A plug-in fetcher's unit of work may carry something that is not
192
+ # job JSON; it is named by its queue alone.
193
+ def describe_job(uow)
194
+ job = Wurk.load_json(uow.job)
195
+ job = {} unless job.is_a?(::Hash)
196
+ klass = job['wrapped'] || job['class']
197
+ "#{klass} jid=#{job['jid']} queue=#{uow.queue_name}"
198
+ rescue ::JSON::ParserError, ::TypeError
199
+ "job queue=#{uow.queue_name}"
200
+ end
201
+
176
202
  # The @workers Set is mutated from Processor threads (processor_result)
177
203
  # while the lifecycle methods read/iterate it from the manager thread.
178
204
  # Snapshot under @plock, then act on the copy outside the lock.
@@ -1,8 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative '../keys'
4
- require_relative 'rollup'
5
- require_relative 'queue_rollup'
6
4
 
7
5
  module Wurk
8
6
  module Metrics
@@ -12,6 +10,13 @@ module Wurk
12
10
  # the API can answer 400. The Sidekiq-shaped API is Query's instance side.
13
11
  module DashboardSeries
14
12
  TOTAL_FIELDS = %w[p f ms].freeze
13
+ TOTAL_KINDS = { 'p' => :p, 'f' => :f, 'ms' => :ms }.freeze
14
+ NO_FIELD = [nil, nil].freeze
15
+
16
+ # Buckets per pipelined read in #top_jobs. A 30-day window is 4320
17
+ # buckets of every class's fields; reading and folding a day at a time
18
+ # keeps one day's replies in memory instead of the whole month's.
19
+ TOP_JOBS_SLICE = 144
15
20
 
16
21
  # Dashboard top-N: `[[klass, {p:, f:, ms:}], …]` over the window,
17
22
  # busiest (`p`, which counts failures) first. `class_filter` is a
@@ -19,7 +24,7 @@ module Wurk
19
24
  # Query#top_jobs, whose per-bucket series a table never shows.
20
25
  def top_jobs(class_filter: nil, minutes: 60, hours: nil, now: ::Time.now)
21
26
  _stamps, keys = window_buckets(minutes, hours, now)
22
- rows = sum_totals(pipeline_hgetall(keys)).to_a
27
+ rows = sum_totals(keys).to_a
23
28
  rows.select! { |(klass, _)| klass.start_with?(class_filter) } if class_filter && !class_filter.empty?
24
29
  rows.sort_by { |(_k, sums)| -sums[:p] }
25
30
  end
@@ -55,8 +60,8 @@ module Wurk
55
60
  names = queue_names(queues)
56
61
  return [] if names.empty?
57
62
 
58
- hashes = queue_bucket_hashes(bucket, starts)
59
- names.map { |name| { name: name, points: queue_points(name, starts, hashes) } }
63
+ rows = queue_bucket_rows(bucket, starts, names)
64
+ names.each_with_index.map { |name, i| { name: name, points: queue_points(i, starts, rows) } }
60
65
  end
61
66
 
62
67
  private
@@ -89,17 +94,30 @@ module Wurk
89
94
  [nil, (checked / 10.0).ceil]
90
95
  end
91
96
 
92
- def sum_totals(hashes)
97
+ # Every bucket repeats the same few hundred `<klass>|<kind>` fields, so
98
+ # each distinct field is split once per call, not once per bucket.
99
+ def sum_totals(keys)
93
100
  totals = ::Hash.new { |h, k| h[k] = { p: 0, f: 0, ms: 0 } }
94
- hashes.each do |raw|
95
- hash_reply(raw).each do |field, value|
96
- klass, kind = field.split('|', 2)
97
- totals[klass][kind.to_sym] += value.to_i if kind && TOTAL_FIELDS.include?(kind)
98
- end
101
+ parsed = ::Hash.new { |h, field| h[field] = parse_total_field(field) }
102
+ keys.each_slice(TOP_JOBS_SLICE) do |slice|
103
+ pipeline_hgetall(slice).each { |raw| fold_totals(totals, parsed, hash_reply(raw)) }
99
104
  end
100
105
  totals
101
106
  end
102
107
 
108
+ def fold_totals(totals, parsed, hash)
109
+ hash.each do |field, value|
110
+ klass, kind = parsed[field]
111
+ totals[klass][kind] += value.to_i if kind
112
+ end
113
+ end
114
+
115
+ def parse_total_field(field)
116
+ klass, kind = field.split('|', 2)
117
+ kind = TOTAL_KINDS[kind]
118
+ kind ? [klass, kind] : NO_FIELD
119
+ end
120
+
103
121
  def hash_reply(raw)
104
122
  raw.is_a?(::Array) ? raw.each_slice(2).to_h : (raw || {})
105
123
  end
@@ -108,8 +126,12 @@ module Wurk
108
126
  stamps.zip(rows).map { |at, (p, f, ms)| { at: at, p: p.to_i, f: f.to_i, ms: ms.to_i } }
109
127
  end
110
128
 
111
- def queue_bucket_hashes(bucket, starts)
112
- pipeline_hgetall(starts.map { |s| QueueRollup.bucket_key(bucket, s) }).map { |h| hash_reply(h) }
129
+ # Only the requested queues' two fields per bucket, not every queue's: a
130
+ # bucket holds a pair per live queue, and a series names at most
131
+ # MAX_QUEUE_SERIES of them.
132
+ def queue_bucket_rows(bucket, starts, names)
133
+ fields = names.flat_map { |name| ["#{name}|#{QueueRollup::SIZE_KIND}", "#{name}|#{QueueRollup::LAT_KIND}"] }
134
+ pipeline_hmget(starts.map { |s| QueueRollup.bucket_key(bucket, s) }, fields)
113
135
  end
114
136
 
115
137
  def queue_names(queues)
@@ -117,11 +139,9 @@ module Wurk
117
139
  names.sort.first(Query::MAX_QUEUE_SERIES)
118
140
  end
119
141
 
120
- def queue_points(name, starts, hashes)
121
- size_field = "#{name}|#{QueueRollup::SIZE_KIND}"
122
- lat_field = "#{name}|#{QueueRollup::LAT_KIND}"
123
- starts.zip(hashes).map do |at, hash|
124
- { at: at, size: hash[size_field].to_i, latency: (hash[lat_field] || 0).to_f }
142
+ def queue_points(index, starts, rows)
143
+ starts.zip(rows).map do |at, row|
144
+ { at: at, size: row[index * 2].to_i, latency: (row[(index * 2) + 1] || 0).to_f }
125
145
  end
126
146
  end
127
147
 
@@ -24,7 +24,8 @@ module Wurk
24
24
  # (`14:05`), the 10-minute bucket always one (`14:0`). `<klass>` is the
25
25
  # `wrapped` class for ActiveJob payloads, like upstream.
26
26
  #
27
- # Every bucket TTL is set on every write (not NX), as upstream does.
27
+ # Every bucket TTL is set on every flush (not NX), once per bucket key after
28
+ # its HINCRBYs, as upstream's ExecutionTracker#flush does.
28
29
  #
29
30
  # The middleware is hot-path — every job pays for it — so it does not touch
30
31
  # Redis at all. Each execution folds into a process-wide Accumulator and
@@ -32,6 +33,10 @@ module Wurk
32
33
  class History
33
34
  include Wurk::Middleware::ServerMiddleware
34
35
 
36
+ # One instance per chain entry, shared by every job (Middleware::Chain::Entry):
37
+ # #call keeps its state in locals and writes no ivar.
38
+ def self.shared_instance? = true
39
+
35
40
  # Upstream's ExecutionTracker constants, same names. MID_TERM diverges:
36
41
  # upstream TTLs the 10-minute buckets at 3 days, but Wurk's dashboard
37
42
  # serves 7d/30d tabs off them (Query::MAX_HOURS = 720), so they must
@@ -178,24 +183,33 @@ module Wurk
178
183
  at.to_i / 60
179
184
  end
180
185
 
186
+ # The `j|` keys are shared by every class, so each gets its EXPIRE once,
187
+ # after the last HINCRBY into it; a 10-minute key two minutes share gets
188
+ # one too. A histogram key is per class and minute, so it keeps its own.
181
189
  def write(pool, minutes)
182
190
  with_pool(pool) do |conn|
183
191
  conn.pipelined do |pipe|
184
- minutes.each do |minute, classes|
185
- at = ::Time.at(minute * 60).utc
186
- short = minute_key(at)
187
- keys = [short, short.chop]
188
- classes.each { |klass, counts| write_class(pipe, keys, at, klass, counts) }
189
- end
192
+ ttls = minutes.each_with_object({}) { |(minute, classes), acc| write_minute(pipe, minute, classes, acc) }
193
+ ttls.each { |key, ttl| pipe.call('EXPIRE', key, ttl) }
190
194
  end
191
195
  end
192
196
  end
193
197
 
198
+ # Writes one minute's classes and notes its two `j|` keys' TTLs in `ttls`.
199
+ def write_minute(pipe, minute, classes, ttls)
200
+ at = ::Time.at(minute * 60).utc
201
+ short = minute_key(at)
202
+ keys = [short, short.chop]
203
+ classes.each { |klass, counts| write_class(pipe, keys, at, klass, counts) }
204
+ ttls[keys[1]] = MID_TERM
205
+ ttls[keys[0]] = SHORT_TERM
206
+ end
207
+
194
208
  # `keys` is [minute key, 10-minute key].
195
209
  def write_class(pipe, keys, at, klass, counts)
196
210
  fields = ["#{klass}|p", "#{klass}|f", "#{klass}|ms"]
197
- incr_bucket(pipe, keys[1], fields, counts, MID_TERM)
198
- incr_bucket(pipe, keys[0], fields, counts, SHORT_TERM)
211
+ incr_bucket(pipe, keys[1], fields, counts)
212
+ incr_bucket(pipe, keys[0], fields, counts)
199
213
  return unless counts[3]
200
214
 
201
215
  pipe.call(*Histogram.incr_command(klass, at, counts[3]))
@@ -206,12 +220,11 @@ module Wurk
206
220
  # `f` when something failed, `ms` when something succeeded (by zero
207
221
  # included). `HINCRBY <field> 0` beyond that would materialize a
208
222
  # counter Sidekiq never writes.
209
- def incr_bucket(pipe, key, fields, counts, ttl)
223
+ def incr_bucket(pipe, key, fields, counts)
210
224
  processed, failed, ms = counts
211
225
  pipe.call('HINCRBY', key, fields[0], processed)
212
226
  pipe.call('HINCRBY', key, fields[1], failed) if failed.positive?
213
227
  pipe.call('HINCRBY', key, fields[2], ms) if processed > failed
214
- pipe.call('EXPIRE', key, ttl)
215
228
  end
216
229
  end
217
230
 
@@ -3,7 +3,6 @@
3
3
  require 'json'
4
4
  require_relative '../component'
5
5
  require_relative '../keys'
6
- require_relative '../leader'
7
6
  require_relative '../version'
8
7
  require_relative '../queue_consumers'
9
8
  require_relative 'process_counters'
@@ -1,11 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'time'
4
- require_relative '../deploy'
5
4
  require_relative '../pool_checkout'
6
5
  require_relative 'history'
7
6
  require_relative 'histogram'
8
- require_relative 'dashboard_series'
9
7
 
10
8
  module Wurk
11
9
  module Metrics
@@ -4,7 +4,6 @@ require_relative '../component'
4
4
  require_relative '../keys'
5
5
  require_relative '../job_record'
6
6
  require_relative '../timer_loop'
7
- require_relative 'rollup'
8
7
 
9
8
  module Wurk
10
9
  module Metrics
@@ -82,25 +81,48 @@ module Wurk
82
81
 
83
82
  # One sampling pass, bypassing the leader gate and the sleep loop. Public
84
83
  # so deterministic specs and a manual "sample now" can drive it directly.
84
+ #
85
+ # A zero gauge is not stored: readers default a missing field to 0, and
86
+ # an idle fleet's queues are mostly empty, so storing them cost two
87
+ # fields per queue in every bucket for nothing. It is HDEL'd instead of
88
+ # skipped, so a queue that drained inside a coarse bucket reads 0 there
89
+ # rather than keeping the earlier sample. All three buckets go in one
90
+ # pipeline.
85
91
  def sample(now = ::Time.now)
86
92
  gauges = queue_gauges
87
93
  return if gauges.empty?
88
94
 
89
- fields = gauges.flat_map do |name, (size, lat)|
90
- ["#{name}|#{SIZE_KIND}", size, "#{name}|#{LAT_KIND}", lat]
91
- end
92
- BUCKETS.each do |bucket, (step, ttl)|
93
- key = self.class.bucket_key(bucket, (now.to_i / step) * step)
94
- redis do |c|
95
- c.call('HSET', key, *fields)
96
- c.call('EXPIRE', key, ttl)
97
- end
98
- end
95
+ set, zero = gauge_fields(gauges)
96
+ keys = bucket_keys(now)
97
+ redis { |c| c.pipelined { |p| keys.each { |key, ttl| write_bucket(p, key, ttl, set, zero) } } }
99
98
  nil
100
99
  end
101
100
 
102
101
  private
103
102
 
103
+ # [[bucket key, ttl], ...] for the bucket `now` falls in at each resolution.
104
+ def bucket_keys(now)
105
+ BUCKETS.map { |bucket, (step, ttl)| [self.class.bucket_key(bucket, (now.to_i / step) * step), ttl] }
106
+ end
107
+
108
+ def write_bucket(pipe, key, ttl, set, zero)
109
+ pipe.call('HSET', key, *set) unless set.empty?
110
+ pipe.call('HDEL', key, *zero) unless zero.empty?
111
+ pipe.call('EXPIRE', key, ttl)
112
+ end
113
+
114
+ # [flat field/value pairs to HSET, fields to HDEL], zero gauges in the
115
+ # second list.
116
+ def gauge_fields(gauges)
117
+ set = []
118
+ zero = []
119
+ gauges.each do |name, (size, lat)|
120
+ size.zero? ? zero << "#{name}|#{SIZE_KIND}" : set.push("#{name}|#{SIZE_KIND}", size)
121
+ lat.zero? ? zero << "#{name}|#{LAT_KIND}" : set.push("#{name}|#{LAT_KIND}", lat)
122
+ end
123
+ [set, zero]
124
+ end
125
+
104
126
  # [[queue_name, [size, latency_seconds]], ...] for every live queue. The
105
127
  # latency is the head-of-line wait — the tail of the LIST is the oldest
106
128
  # job — in seconds, rounded for compact storage.
@@ -34,6 +34,10 @@ module Wurk
34
34
  class Statsd
35
35
  include Wurk::Middleware::ServerMiddleware
36
36
 
37
+ # One instance per chain entry, shared by every job (Middleware::Chain::Entry):
38
+ # #call keeps its state in locals and writes no ivar.
39
+ def self.shared_instance? = true
40
+
37
41
  METRIC_PREFIX = 'sidekiq.'
38
42
  DEFAULT_SAMPLE_RATE = 1.0
39
43
 
@@ -154,7 +154,7 @@ module Wurk
154
154
  def traverse(entries, index, args, &block)
155
155
  return yield if index == entries.size
156
156
 
157
- entries[index].make_new.call(*args) do
157
+ entries[index].instance.call(*args) do
158
158
  traverse(entries, index + 1, args, &block)
159
159
  end
160
160
  end
@@ -188,6 +188,15 @@ module Wurk
188
188
  # instantiation until `retrieve`/`invoke` runs so each job gets a fresh
189
189
  # object.
190
190
  #
191
+ # One opt-in exception, for Wurk's own built-ins: a class that defines
192
+ # `self.shared_instance?` as true on itself is built once per entry, when
193
+ # the entry is bound to its config, and `invoke` hands every job that one
194
+ # instance. Only for a class whose #call writes no instance state — it
195
+ # runs on every processor thread of the capsule at once. Asked of the
196
+ # class itself, never inherited: a host's subclass of a built-in may keep
197
+ # state, and gets Sidekiq's fresh instance per job like any other
198
+ # middleware. `make_new`, and so `retrieve`, always builds a fresh one.
199
+ #
191
200
  # Whether the class declares `config=` is resolved once, at registration,
192
201
  # so the common case (the setter comes from `include
193
202
  # Wurk::Middleware::ServerMiddleware`) skips a `respond_to?` on every job
@@ -197,13 +206,26 @@ module Wurk
197
206
  # `method_missing`, or one added to the class after it was registered — so
198
207
  # a class that does not declare it still gets asked.
199
208
  class Entry
200
- attr_reader :klass
201
- attr_accessor :config
209
+ attr_reader :klass, :config
202
210
 
203
211
  def initialize(klass, *args)
204
212
  @klass = klass
205
213
  @args = args
206
214
  @takes_config = klass.public_method_defined?(:config=)
215
+ @shared = klass.singleton_class.method_defined?(:shared_instance?, false) && klass.shared_instance?
216
+ @instance = nil
217
+ end
218
+
219
+ # Rebuilds the shared instance too: Chain#copy_for rebinds a duplicate
220
+ # entry to a capsule, and the instance must answer to that capsule.
221
+ def config=(value)
222
+ @config = value
223
+ @instance = make_new if @shared
224
+ end
225
+
226
+ # What `invoke` runs: the shared instance, or a fresh one.
227
+ def instance
228
+ @instance || make_new
207
229
  end
208
230
 
209
231
  def make_new
@@ -43,6 +43,10 @@ module Wurk
43
43
  class Expiry
44
44
  include Wurk::Middleware::ServerMiddleware
45
45
 
46
+ # One instance per chain entry, shared by every job (Middleware::Chain::Entry):
47
+ # #call and its helpers keep their state in locals and write no ivar.
48
+ def self.shared_instance? = true
49
+
46
50
  # The rescue is scoped to the guarded call rather than the method body:
47
51
  # only a job that carries a cutoff can be cut by one, and a job that
48
52
  # raises `DeadlineExceeded` from its own code without carrying one has to
@@ -3,7 +3,6 @@
3
3
  require_relative '../middleware'
4
4
  require_relative '../job'
5
5
  require_relative '../job_retry'
6
- require_relative '../encryption'
7
6
 
8
7
  module Wurk
9
8
  module Middleware
@@ -30,6 +29,10 @@ module Wurk
30
29
  class InterruptHandler
31
30
  include Wurk::Middleware::ServerMiddleware
32
31
 
32
+ # One instance per chain entry, shared by every job (Middleware::Chain::Entry):
33
+ # #call and its helpers keep their state in locals and write no ivar.
34
+ def self.shared_instance? = true
35
+
33
36
  def call(_job_instance, job, queue)
34
37
  yield
35
38
  rescue Wurk::Job::Interrupted
@@ -32,6 +32,10 @@ module Wurk
32
32
  class Status
33
33
  include Wurk::Middleware::ServerMiddleware
34
34
 
35
+ # One instance per chain entry, shared by every job (Middleware::Chain::Entry):
36
+ # #call and its helpers keep their state in locals and write no ivar.
37
+ def self.shared_instance? = true
38
+
35
39
  # Hard cap on a stored result, in bytes. A job that returns an
36
40
  # ActiveRecord relation must cost Redis 8 KB, not a table dump — past the
37
41
  # cap the head is kept and `result_truncated` says the rest was dropped.