wurk 1.13.0 → 1.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (255) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -1
  3. data/app/controllers/concerns/wurk/sse_streaming.rb +36 -1
  4. data/app/controllers/wurk/api/pagination.rb +9 -7
  5. data/app/controllers/wurk/api/serializers.rb +8 -2
  6. data/app/controllers/wurk/api_controller.rb +57 -24
  7. data/app/controllers/wurk/application_controller.rb +0 -2
  8. data/app/controllers/wurk/dashboard_controller.rb +0 -1
  9. data/config/routes.rb +5 -1
  10. data/lib/sidekiq/cli.rb +4 -3
  11. data/lib/sidekiq/deploy.rb +2 -1
  12. data/lib/sidekiq/embedded.rb +2 -1
  13. data/lib/sidekiq/iterable_job.rb +2 -1
  14. data/lib/sidekiq/job/iterable/active_record_enumerator.rb +2 -1
  15. data/lib/sidekiq/job/iterable/csv_enumerator.rb +2 -1
  16. data/lib/sidekiq/job/iterable.rb +2 -1
  17. data/lib/sidekiq/launcher.rb +2 -1
  18. data/lib/sidekiq/manager.rb +2 -1
  19. data/lib/sidekiq/metrics/query.rb +2 -1
  20. data/lib/sidekiq/pro/web.rb +2 -1
  21. data/lib/sidekiq/scheduled.rb +2 -1
  22. data/lib/sidekiq/web.rb +2 -1
  23. data/lib/sidekiq.rb +3 -2
  24. data/lib/wurk/api/fast.rb +4 -18
  25. data/lib/wurk/api/queues.rb +12 -10
  26. data/lib/wurk/api/serializers.rb +6 -7
  27. data/lib/wurk/api/swarm.rb +0 -1
  28. data/lib/wurk/batch/server_middleware.rb +4 -0
  29. data/lib/wurk/batch/status.rb +45 -22
  30. data/lib/wurk/batch_set.rb +34 -0
  31. data/lib/wurk/cli.rb +17 -19
  32. data/lib/wurk/client/buffered.rb +97 -26
  33. data/lib/wurk/client.rb +21 -1
  34. data/lib/wurk/compat/deploy.rb +7 -0
  35. data/lib/wurk/compat/encryption.rb +7 -0
  36. data/lib/wurk/compat/flow.rb +8 -0
  37. data/lib/wurk/compat/iterable_job.rb +8 -0
  38. data/lib/wurk/compat/profile_set.rb +8 -0
  39. data/lib/wurk/compat/server.rb +15 -0
  40. data/lib/wurk/compat/web.rb +20 -0
  41. data/lib/wurk/compat.rb +34 -40
  42. data/lib/wurk/configuration.rb +46 -9
  43. data/lib/wurk/cron.rb +146 -47
  44. data/lib/wurk/dead_set.rb +10 -6
  45. data/lib/wurk/embedded.rb +0 -2
  46. data/lib/wurk/engine.rb +46 -4
  47. data/lib/wurk/error_coalescer.rb +21 -1
  48. data/lib/wurk/fetcher/many_queues.rb +8 -6
  49. data/lib/wurk/fetcher/poison_pill.rb +1 -1
  50. data/lib/wurk/fetcher/reaper.rb +0 -5
  51. data/lib/wurk/fetcher/reliable.rb +46 -27
  52. data/lib/wurk/fetcher/self_reconcile.rb +0 -1
  53. data/lib/wurk/fetcher/unit_of_work.rb +16 -1
  54. data/lib/wurk/flow/status.rb +10 -4
  55. data/lib/wurk/flow_set.rb +24 -2
  56. data/lib/wurk/health.rb +0 -1
  57. data/lib/wurk/heartbeat.rb +13 -7
  58. data/lib/wurk/import/sidekiq_cron.rb +0 -1
  59. data/lib/wurk/job_record.rb +2 -1
  60. data/lib/wurk/job_retry.rb +1 -1
  61. data/lib/wurk/job_set.rb +132 -6
  62. data/lib/wurk/launcher.rb +0 -12
  63. data/lib/wurk/lazy_loader.rb +60 -0
  64. data/lib/wurk/limiter/bucket.rb +0 -2
  65. data/lib/wurk/limiter/concurrent.rb +19 -36
  66. data/lib/wurk/limiter/leaky.rb +0 -2
  67. data/lib/wurk/limiter/points.rb +0 -2
  68. data/lib/wurk/limiter/server_middleware.rb +4 -1
  69. data/lib/wurk/limiter/unlimited.rb +0 -2
  70. data/lib/wurk/limiter/window.rb +0 -2
  71. data/lib/wurk/limiter.rb +0 -7
  72. data/lib/wurk/logger.rb +44 -5
  73. data/lib/wurk/lua/fast_delete.rb +58 -0
  74. data/lib/wurk/lua/fast_delete_window.lua +46 -0
  75. data/lib/wurk/lua/limiter_concurrent_acquire.lua +10 -3
  76. data/lib/wurk/lua/limiter_concurrent_release.lua +13 -4
  77. data/lib/wurk/lua.rb +1 -41
  78. data/lib/wurk/metrics/dashboard_series.rb +38 -18
  79. data/lib/wurk/metrics/history.rb +24 -11
  80. data/lib/wurk/metrics/prometheus.rb +0 -1
  81. data/lib/wurk/metrics/query.rb +0 -2
  82. data/lib/wurk/metrics/queue_rollup.rb +33 -11
  83. data/lib/wurk/metrics/statsd.rb +4 -0
  84. data/lib/wurk/middleware/chain.rb +25 -3
  85. data/lib/wurk/middleware/expiry.rb +4 -0
  86. data/lib/wurk/middleware/interrupt_handler.rb +4 -1
  87. data/lib/wurk/middleware/status.rb +4 -0
  88. data/lib/wurk/middleware/timeout.rb +4 -0
  89. data/lib/wurk/preflight/owners.rb +0 -1
  90. data/lib/wurk/preflight/payload_sample.rb +0 -1
  91. data/lib/wurk/preflight/verify.rb +0 -2
  92. data/lib/wurk/process_set.rb +6 -1
  93. data/lib/wurk/profiler.rb +2 -1
  94. data/lib/wurk/queue.rb +33 -6
  95. data/lib/wurk/queue_consumers.rb +10 -1
  96. data/lib/wurk/rails_boot.rb +2 -2
  97. data/lib/wurk/railtie.rb +3 -4
  98. data/lib/wurk/sentry/error_handler.rb +5 -0
  99. data/lib/wurk/sentry/middleware.rb +4 -0
  100. data/lib/wurk/sorted_entry.rb +16 -12
  101. data/lib/wurk/stats.rb +85 -1
  102. data/lib/wurk/swarm/child_boot.rb +0 -2
  103. data/lib/wurk/swarm/liveness.rb +0 -2
  104. data/lib/wurk/swarm/memory_watch.rb +116 -0
  105. data/lib/wurk/swarm/prefork.rb +54 -0
  106. data/lib/wurk/swarm.rb +33 -42
  107. data/lib/wurk/version.rb +1 -1
  108. data/lib/wurk/web/enterprise.rb +63 -7
  109. data/lib/wurk.rb +33 -29
  110. data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js +1 -0
  111. data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.br +0 -0
  112. data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.gz +0 -0
  113. data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js +1 -0
  114. data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.br +0 -0
  115. data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.gz +0 -0
  116. data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js +1 -0
  117. data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.br +0 -0
  118. data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.gz +0 -0
  119. data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js +1 -0
  120. data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.br +0 -0
  121. data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.gz +0 -0
  122. data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js +1 -0
  123. data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.br +0 -0
  124. data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.gz +0 -0
  125. data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js +1 -0
  126. data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.br +0 -0
  127. data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.gz +0 -0
  128. data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js +1 -0
  129. data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.br +0 -0
  130. data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.gz +0 -0
  131. data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js +1 -0
  132. data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.br +0 -0
  133. data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.gz +0 -0
  134. data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js +2 -0
  135. data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.br +0 -0
  136. data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.gz +0 -0
  137. data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js +1 -0
  138. data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.br +0 -0
  139. data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.gz +0 -0
  140. data/vendor/assets/dashboard/assets/{FlowState-yb7virqR.js → FlowState-ClzInw0-.js} +1 -1
  141. data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.br +0 -0
  142. data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.gz +0 -0
  143. data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js +1 -0
  144. data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.br +0 -0
  145. data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.gz +0 -0
  146. data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js +2 -0
  147. data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.br +0 -0
  148. data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.gz +0 -0
  149. data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js +1 -0
  150. data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.br +0 -0
  151. data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.gz +0 -0
  152. data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js +1 -0
  153. data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.br +0 -0
  154. data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.gz +0 -0
  155. data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js +1 -0
  156. data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.br +0 -0
  157. data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.gz +0 -0
  158. data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js +1 -0
  159. data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.br +0 -0
  160. data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.gz +0 -0
  161. data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js +1 -0
  162. data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.br +0 -0
  163. data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.gz +0 -0
  164. data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js +1 -0
  165. data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.br +0 -0
  166. data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.gz +0 -0
  167. data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js +1 -0
  168. data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.br +0 -0
  169. data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.gz +0 -0
  170. data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js +1 -0
  171. data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.br +0 -0
  172. data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.gz +0 -0
  173. data/vendor/assets/dashboard/assets/Search-D42Va7lB.js +1 -0
  174. data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.br +0 -0
  175. data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.gz +0 -0
  176. data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js +1 -0
  177. data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.br +0 -0
  178. data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.gz +0 -0
  179. data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js +1 -0
  180. data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.br +0 -0
  181. data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.gz +0 -0
  182. data/vendor/assets/dashboard/assets/de-B2NCIuzo.js +1 -0
  183. data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.br +0 -0
  184. data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.gz +0 -0
  185. data/vendor/assets/dashboard/assets/es-BGeu_JNB.js +1 -0
  186. data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.br +0 -0
  187. data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.gz +0 -0
  188. data/vendor/assets/dashboard/assets/fr-DQVBWFId.js +1 -0
  189. data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.br +0 -0
  190. data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.gz +0 -0
  191. data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js +1 -0
  192. data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.br +0 -0
  193. data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.gz +0 -0
  194. data/vendor/assets/dashboard/assets/index-CiLU1zz8.js +141 -0
  195. data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.br +0 -0
  196. data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.gz +0 -0
  197. data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css +1 -0
  198. data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.br +0 -0
  199. data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.gz +0 -0
  200. data/vendor/assets/dashboard/assets/ja-D06IBqzL.js +1 -0
  201. data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.br +0 -0
  202. data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.gz +0 -0
  203. data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js +1 -0
  204. data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.br +0 -0
  205. data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.gz +0 -0
  206. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js +1 -0
  207. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.br +0 -0
  208. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.gz +0 -0
  209. data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js +1 -0
  210. data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.br +0 -0
  211. data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.gz +0 -0
  212. data/vendor/assets/dashboard/assets/{utils-CnWZzW2L.js → utils-69XSoGbQ.js} +1 -1
  213. data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.br +0 -0
  214. data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.gz +0 -0
  215. data/vendor/assets/dashboard/assets/wurk-logo-ydeSB9B8.png +0 -0
  216. data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js +1 -0
  217. data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.br +0 -0
  218. data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.gz +0 -0
  219. data/vendor/assets/dashboard/index.html +3 -23
  220. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  221. metadata +122 -37
  222. data/vendor/assets/dashboard/assets/ArgsValue-BJol9S0s.js +0 -1
  223. data/vendor/assets/dashboard/assets/BatchDetail-D_zGQ9Sd.js +0 -1
  224. data/vendor/assets/dashboard/assets/Batches-Dt_xC660.js +0 -1
  225. data/vendor/assets/dashboard/assets/Busy-CWRthsmc.js +0 -1
  226. data/vendor/assets/dashboard/assets/Cron-qeGr2YAm.js +0 -1
  227. data/vendor/assets/dashboard/assets/Dashboard-DQWaAwPA.js +0 -1
  228. data/vendor/assets/dashboard/assets/Dead-D9gOPHtK.js +0 -1
  229. data/vendor/assets/dashboard/assets/Extension-DONXVLdA.js +0 -1
  230. data/vendor/assets/dashboard/assets/FilterBox-BbB6gXHp.js +0 -2
  231. data/vendor/assets/dashboard/assets/FlowDetail-CrmnVRp0.js +0 -1
  232. data/vendor/assets/dashboard/assets/Flows-g4GC8KmE.js +0 -1
  233. data/vendor/assets/dashboard/assets/JobDetailModal-C86k_1wj.js +0 -2
  234. data/vendor/assets/dashboard/assets/Limiters-zyO-iMlK.js +0 -1
  235. data/vendor/assets/dashboard/assets/Metrics-Ckq5fNBv.js +0 -1
  236. data/vendor/assets/dashboard/assets/NotFound-C8NnyjlY.js +0 -1
  237. data/vendor/assets/dashboard/assets/PageHeader-vWeBbSPF.js +0 -1
  238. data/vendor/assets/dashboard/assets/Profiles-DRl-FXPA.js +0 -1
  239. data/vendor/assets/dashboard/assets/Queues-DkXWrihA.js +0 -1
  240. data/vendor/assets/dashboard/assets/Retries-CmlojYIy.js +0 -1
  241. data/vendor/assets/dashboard/assets/Scheduled-Cbbpp3xw.js +0 -1
  242. data/vendor/assets/dashboard/assets/Search-LaEpj2fM.js +0 -1
  243. data/vendor/assets/dashboard/assets/charts-D_PQLoyc.js +0 -1
  244. data/vendor/assets/dashboard/assets/fa-brands-400-Bs6tcqqs.woff2 +0 -0
  245. data/vendor/assets/dashboard/assets/fa-regular-400-DRN8N0d1.woff2 +0 -0
  246. data/vendor/assets/dashboard/assets/fa-solid-900-IAB4Droh.woff2 +0 -0
  247. data/vendor/assets/dashboard/assets/fa-v4compatibility-CErXDOsT.woff2 +0 -0
  248. data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +0 -1
  249. data/vendor/assets/dashboard/assets/index--xvvNlYx.css +0 -1
  250. data/vendor/assets/dashboard/assets/index-FBKGtysy.js +0 -141
  251. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-BiwhHqgh.js +0 -1
  252. data/vendor/assets/dashboard/assets/useSort-jBYVwXvg.js +0 -1
  253. data/vendor/assets/dashboard/assets/wurk-logo-Vy3xW4K0.png +0 -0
  254. data/vendor/assets/dashboard/og-image.png +0 -0
  255. data/vendor/assets/dashboard/wurk-logo.webp +0 -0
data/lib/wurk/railtie.rb CHANGED
@@ -2,7 +2,6 @@
2
2
 
3
3
  require 'rails/railtie'
4
4
  require 'active_support/ordered_options'
5
- require_relative 'rails_boot'
6
5
 
7
6
  module Wurk
8
7
  # Rails integration surface only: register the host-facing `config.wurk`
@@ -24,9 +23,9 @@ module Wurk
24
23
 
25
24
  # Enter server mode BEFORE config/initializers load — otherwise the app's
26
25
  # `Sidekiq.configure_server` blocks gate on `config.server?` (still false)
27
- # and are dropped, each skip logging one WARN naming the gate
28
- # (`configure_server` / `server? is false`). RailsBoot decides whether this
29
- # process serves.
26
+ # and are deferred, running only when server mode is entered — after the
27
+ # rest of the app's initializers instead of in their place. RailsBoot
28
+ # decides whether this process serves.
30
29
  initializer 'wurk.server_mode', before: :load_config_initializers do |app|
31
30
  Wurk::RailsBoot.boot_scheduled = true
32
31
  Wurk::RailsBoot.enter_server_mode_if_serving(app)
@@ -42,6 +42,11 @@ module Wurk
42
42
  Thread.current[REPORTED_ON_THREAD].equal?(exception)
43
43
  end
44
44
 
45
+ def self.forget_thread!
46
+ Thread.current[SEEN_BY_MIDDLEWARE] = nil
47
+ Thread.current[REPORTED_ON_THREAD] = nil
48
+ end
49
+
45
50
  # Transport blips the pool already retried before re-raising. Wurk's
46
51
  # default handler logs these at WARN precisely because they are
47
52
  # self-healing, and the fetch loop runs them in a tight `sleep(1)` cycle:
@@ -28,6 +28,10 @@ module Wurk
28
28
  def call(instance, job, queue, &block)
29
29
  return yield unless Wurk::Sentry.enabled?
30
30
 
31
+ # The previous job's exception (with its backtrace and anything it
32
+ # references) would otherwise stay pinned in this thread's slots until
33
+ # the next failure here replaced it. Both handshakes are within one job.
34
+ ErrorHandler.forget_thread!
31
35
  # Each Processor owns a thread, and Sentry's hub is thread-local.
32
36
  # Without this the worker thread starts from an empty hub and the
33
37
  # scope set below is invisible to the capture.
@@ -91,6 +91,22 @@ module Wurk
91
91
 
92
92
  def error? = !item['error_class'].nil?
93
93
 
94
+ # Puts the original bytes back at the original score, after this entry was
95
+ # removed and its push failed. Internal: #remove_job and JobSet's batched
96
+ # retry_all, which removes a page before pushing it.
97
+ #
98
+ # Past maxmemory the push is refused, and so is a plain ZADD putting the
99
+ # entry back. Dragonfly refuses both forms: all that is left is the log.
100
+ def restore_after_failed_push
101
+ Wurk.redis do |conn|
102
+ conn.call('ZADD', @parent.name, @score, value)
103
+ rescue RedisClient::OutOfMemoryError => e
104
+ Fetcher::OomDrain.zadd(conn, e, @parent.name, @score, value)
105
+ end
106
+ rescue StandardError => e
107
+ Wurk.logger.error("#{@parent.name} entry lost, ZADD refused (#{e.class}): #{loggable_payload}")
108
+ end
109
+
94
110
  private
95
111
 
96
112
  # Pulls the message out of Redis, yields its raw JSON for the caller's
@@ -116,18 +132,6 @@ module Wurk
116
132
  item
117
133
  end
118
134
 
119
- # Past maxmemory the push is refused, and so is a plain ZADD putting the
120
- # entry back. Dragonfly refuses both forms: all that is left is the log.
121
- def restore_after_failed_push
122
- Wurk.redis do |conn|
123
- conn.call('ZADD', @parent.name, @score, value)
124
- rescue RedisClient::OutOfMemoryError => e
125
- Fetcher::OomDrain.zadd(conn, e, @parent.name, @score, value)
126
- end
127
- rescue StandardError => e
128
- Wurk.logger.error("#{@parent.name} entry lost, ZADD refused (#{e.class}): #{loggable_payload}")
129
- end
130
-
131
135
  # The last copy of the job, so it is logged whole, short of what the host's
132
136
  # redact_args hides.
133
137
  def loggable_payload
data/lib/wurk/stats.rb CHANGED
@@ -2,6 +2,7 @@
2
2
 
3
3
  require 'date'
4
4
  require_relative 'pool_checkout'
5
+ require_relative 'queue_consumers'
5
6
 
6
7
  module Wurk
7
8
  # Read-only inspector for cluster state in Redis. The cheap counters are
@@ -76,7 +77,7 @@ module Wurk
76
77
  names = conn.call('SMEMBERS', Keys::QUEUES_SET)
77
78
  next [] if names.empty?
78
79
 
79
- paused_set = conn.call('SMEMBERS', 'paused')
80
+ paused_set = conn.call('SMEMBERS', 'paused').to_set
80
81
  results = conn.pipelined do |pipe|
81
82
  names.each do |q|
82
83
  pipe.call('LLEN', Keys.queue(q))
@@ -166,6 +167,89 @@ module Wurk
166
167
  0.0
167
168
  end
168
169
 
170
+ # Every Stats reading taken at one moment, in two round trips: the
171
+ # counters with the `queues`, `paused` and `processes` members, then each
172
+ # queue's LLEN + tail and each process's `busy`. A plain Stats re-reads the
173
+ # queue set per method and pays seven round trips for what the dashboard's
174
+ # landing payload asks of it. Internal reader for the dashboard and the
175
+ # HTTP API: Stats keeps Sidekiq's per-method reads (§19.1).
176
+ #
177
+ # `consumers: true` also reads each process's `info` and answers
178
+ # #consumers, the per-queue subscriber count of QueueConsumers.counts.
179
+ class Snapshot < Stats
180
+ attr_reader :queue_summaries, :workers_size, :default_queue_latency
181
+
182
+ def initialize(consumers: false) # rubocop:disable Lint/MissingSuper
183
+ @with_consumers = consumers
184
+ Wurk.redis(idempotent: true) do |conn|
185
+ names, paused, identities = read_members(conn)
186
+ read_gauges(conn, names, paused.to_set, identities)
187
+ end
188
+ end
189
+
190
+ def enqueued = @queue_summaries.sum(&:size)
191
+ def queues = @queue_summaries.to_h { |summary| [summary.name, summary.size] }
192
+
193
+ # @return [Hash{String=>Integer}] queue → live subscribers.
194
+ def consumers
195
+ raise ArgumentError, 'Snapshot read without consumers: true' unless @with_consumers
196
+
197
+ @consumers
198
+ end
199
+
200
+ private
201
+
202
+ def read_members(conn)
203
+ replies = conn.pipelined do |pipe|
204
+ FAST_QUERIES.each { |args| pipe.call(*args) }
205
+ pipe.call('SMEMBERS', Keys::QUEUES_SET)
206
+ pipe.call('SMEMBERS', 'paused')
207
+ pipe.call('SMEMBERS', Keys::PROCESSES)
208
+ end
209
+ @stats = FAST_KEYS.zip(replies.first(FAST_KEYS.size).map(&:to_i)).to_h
210
+ replies.last(3)
211
+ end
212
+
213
+ # Queue replies first (LLEN, tail per queue), then the `default` tail when
214
+ # that queue is not listed, then one reply per process.
215
+ def read_gauges(conn, names, paused, identities)
216
+ default_listed = names.include?('default')
217
+ replies = conn.pipelined do |pipe|
218
+ queue_commands(pipe, names, default_listed)
219
+ identities.each { |id| process_command(pipe, id) }
220
+ end
221
+ queue_replies = replies.shift(names.size * 2)
222
+ default_tail = default_listed ? queue_replies[(names.index('default') * 2) + 1] : replies.shift
223
+ build_snapshot(names, queue_replies, paused, default_tail, replies)
224
+ end
225
+
226
+ def queue_commands(pipe, names, default_listed)
227
+ names.each do |q|
228
+ pipe.call('LLEN', Keys.queue(q))
229
+ pipe.call('LRANGE', Keys.queue(q), -1, -1)
230
+ end
231
+ pipe.call('LRANGE', Keys.queue('default'), -1, -1) unless default_listed
232
+ end
233
+
234
+ def process_command(pipe, identity)
235
+ return pipe.call('HGET', identity, 'busy') unless @with_consumers
236
+
237
+ pipe.call('HMGET', identity, 'busy', 'info')
238
+ end
239
+
240
+ def build_snapshot(names, queue_replies, paused, default_tail, process_replies)
241
+ @queue_summaries = build_summaries(names, queue_replies, paused).sort_by { |qd| -qd.size }
242
+ now_ms = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
243
+ @default_queue_latency = compute_latency(default_tail.first, now_ms)
244
+ if @with_consumers
245
+ @workers_size = process_replies.sum { |busy, _| busy.to_i }
246
+ @consumers = QueueConsumers.tally(process_replies.map(&:last))
247
+ else
248
+ @workers_size = process_replies.sum(&:to_i)
249
+ end
250
+ end
251
+ end
252
+
169
253
  # Per-day historical processed/failed/expired counts. Reads
170
254
  # `stat:processed:YYYY-MM-DD`, `stat:failed:YYYY-MM-DD`, and
171
255
  # `stat:expired:YYYY-MM-DD` strings; missing days return 0. Range
@@ -1,11 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative '../component'
4
- require_relative '../launcher'
5
4
  require_relative '../client/buffered'
6
5
  require_relative '../fetcher/reliable'
7
6
  require_relative '../lua'
8
- require_relative '../health'
9
7
  require_relative 'orphan_guard'
10
8
  require_relative 'test_mode_guard'
11
9
 
@@ -1,7 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require_relative '../heartbeat'
4
-
5
3
  module Wurk
6
4
  class Swarm
7
5
  # Parent-side child liveness. A child can stay alive as a pid while doing
@@ -0,0 +1,116 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wurk
4
+ class Swarm
5
+ # MAXMEM recycling (Ent §7.5): every CHECK_INTERVAL the supervisor reads
6
+ # each child's memory and hands one over the limit to the restart machine
7
+ # (`recycle`), which replaces it gracefully.
8
+ #
9
+ # A child already over the limit while still FRESH is never recycled: its
10
+ # replacement would boot to the same size and be recycled in turn, a fork
11
+ # loop that replaces a slot every few seconds and drains nothing useful.
12
+ # Such a child is marked for the rest of its life and the limit reported
13
+ # once per slot index at ERROR — the setting is below what a booted child
14
+ # needs. A child that crosses the limit only after FRESH seconds is recycled
15
+ # as before; if its replacement is over within its own fresh window, the
16
+ # replacement is the one marked, so a slot recycles at most once per cause.
17
+ #
18
+ # The metric is `memory_limit_metric`: `:rss` from /proc/<pid>/statm (Ent's
19
+ # meaning; counts pages still shared with the parent), `:pss`/`:uss` from
20
+ # /proc/<pid>/smaps_rollup. Without /proc nothing is read and nothing is
21
+ # recycled; `warn_if_unreadable` says so at boot.
22
+ class MemoryWatch
23
+ CHECK_INTERVAL = 10
24
+ FRESH = Backoff::RESET_AFTER
25
+ SMAPS_FIELDS = { pss: %w[Pss], uss: %w[Private_Clean Private_Dirty] }.freeze
26
+
27
+ def initialize(limit_kb:, metric:, logger:, recycle:, interval: CHECK_INTERVAL, fresh: FRESH,
28
+ proc_root: '/proc', now: -> { ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) })
29
+ @limit_kb = limit_kb
30
+ @metric = metric
31
+ @logger = logger
32
+ @recycle = recycle
33
+ @interval = interval
34
+ @fresh = fresh
35
+ @now = now
36
+ @proc_root = proc_root
37
+ @last_check = nil
38
+ @marked = {}
39
+ @reported_slots = {}
40
+ end
41
+
42
+ # `children` is `{ pid => [seconds since fork, slot index] }`.
43
+ def tick(children)
44
+ now = @now.call
45
+ return if @last_check && now - @last_check < @interval
46
+
47
+ @last_check = now
48
+ children.each { |pid, (age, slot)| check(pid, age, slot) }
49
+ end
50
+
51
+ def forget(pid)
52
+ @marked.delete(pid)
53
+ end
54
+
55
+ def warn_if_unreadable
56
+ return if read_kb(::Process.pid)
57
+
58
+ @logger.warn do
59
+ "swarm: memory_limit_mb is set but this host has no /proc #{source} to read #{@metric} from; " \
60
+ 'children will not be recycled'
61
+ end
62
+ end
63
+
64
+ private
65
+
66
+ def check(pid, age, slot)
67
+ return if @marked.key?(pid)
68
+
69
+ size_kb = read_kb(pid)
70
+ return if size_kb.nil? || size_kb < @limit_kb
71
+ return mark(pid, size_kb, age, slot) if age < @fresh
72
+
73
+ @logger.warn do
74
+ "swarm: child #{pid} (slot #{slot}) #{@metric.upcase} #{size_kb}KB >= #{@limit_kb}KB " \
75
+ "#{age.round}s after boot; recycling"
76
+ end
77
+ @recycle.call(pid)
78
+ end
79
+
80
+ def mark(pid, size_kb, age, slot)
81
+ @marked[pid] = true
82
+ return if @reported_slots.key?(slot)
83
+
84
+ @reported_slots[slot] = true
85
+ @logger.error do
86
+ "swarm: child #{pid} (slot #{slot}) is #{size_kb}KB #{@metric.upcase} #{age.round}s after boot, " \
87
+ "already over memory_limit_mb (#{@limit_kb / 1024} MB). Recycling it would only fork another " \
88
+ 'child as large, so it is not recycled. Raise the limit above what a booted child uses, or ' \
89
+ 'compare against memory_limit_metric :pss. Reported once per slot.'
90
+ end
91
+ end
92
+
93
+ def source = @metric == :rss ? 'statm' : 'smaps_rollup'
94
+
95
+ def read_kb(pid)
96
+ @metric == :rss ? statm_kb(pid) : smaps_kb(pid)
97
+ rescue ::SystemCallError, ::IOError
98
+ nil
99
+ end
100
+
101
+ def statm_kb(pid)
102
+ Heartbeat.statm_rss_kb(::File.read("#{@proc_root}/#{pid}/statm"))
103
+ end
104
+
105
+ def smaps_kb(pid)
106
+ fields = SMAPS_FIELDS.fetch(@metric)
107
+ total = nil
108
+ ::File.foreach("#{@proc_root}/#{pid}/smaps_rollup") do |line|
109
+ name, value = line.split(':', 2)
110
+ total = total.to_i + value.to_i if value && fields.include?(name)
111
+ end
112
+ total
113
+ end
114
+ end
115
+ end
116
+ end
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wurk
4
+ class Swarm
5
+ # The parent's last memory step before its first fork, shared by every
6
+ # swarm entry point (`wurkswarm` and the railtie's co-hosted swarm): load
7
+ # what each child would otherwise load for itself (Wurk's autoloads
8
+ # included: Wurk.eager_load!), then settle the heap, so
9
+ # the children start on pages they share copy-on-write instead of N private
10
+ # copies. Opens no socket, so it may sit between steps 3 and 4. Its effect
11
+ # is the t=0 shared ratio in bench/rss_soak.rb (`RSS_JOBS=0`).
12
+ module Prefork
13
+ # Features a child requires after the fork. `fiddle` arms PR_SET_PDEATHSIG
14
+ # (OrphanGuard#arm_pdeathsig) in every Linux child.
15
+ CHILD_FEATURES = %w[fiddle].freeze
16
+
17
+ module_function
18
+
19
+ # Once, before the first fork only: a respawn forks from a parent that may
20
+ # be a live web process, where a full compaction is a pause its requests
21
+ # would feel. RUBY_DISABLE_WARMUP=1 skips it, as it does `Process.warmup`.
22
+ def run(env = ENV, periodic: nil)
23
+ Wurk.eager_load!
24
+ require_child_features
25
+ Wurk::Cron.prefork(periodic) if periodic
26
+ warmup unless env['RUBY_DISABLE_WARMUP'] == '1'
27
+ end
28
+
29
+ def require_child_features
30
+ CHILD_FEATURES.each { |feature| try_require(feature) }
31
+ # Only a host that loaded Vernier can profile a job (Profiler).
32
+ try_require('tempfile') if defined?(::Vernier)
33
+ end
34
+
35
+ # `Process.warmup` (Ruby >= 3.3) is a full GC, compaction, promotion of
36
+ # survivors to the old generation and a malloc_trim. The 3.2 floor gets
37
+ # the part it has: full GC, then compaction where the platform supports it.
38
+ def warmup(process = ::Process, collector = ::GC)
39
+ return process.warmup if process.respond_to?(:warmup)
40
+
41
+ collector.start(full_mark: true, immediate_sweep: true)
42
+ collector.compact if collector.respond_to?(:compact)
43
+ rescue ::NotImplementedError
44
+ nil
45
+ end
46
+
47
+ def try_require(feature)
48
+ require feature
49
+ rescue ::LoadError
50
+ nil
51
+ end
52
+ end
53
+ end
54
+ end
data/lib/wurk/swarm.rb CHANGED
@@ -3,10 +3,7 @@
3
3
  require 'monitor'
4
4
 
5
5
  require_relative 'component'
6
- require_relative 'heartbeat'
7
- require_relative 'launcher'
8
6
  require_relative 'fetcher/reliable'
9
- require_relative 'fetcher/reaper'
10
7
  require_relative 'capsule'
11
8
  require_relative 'keys'
12
9
  require_relative 'swarm/child_boot'
@@ -15,6 +12,8 @@ require_relative 'swarm/crash_watch'
15
12
  require_relative 'swarm/restart'
16
13
  require_relative 'swarm/orphan_guard'
17
14
  require_relative 'swarm/liveness'
15
+ require_relative 'swarm/memory_watch'
16
+ require_relative 'swarm/prefork'
18
17
 
19
18
  module Wurk
20
19
  # Parent supervisor. Forks N children per the worker topology, monitors
@@ -48,7 +47,6 @@ module Wurk
48
47
  SUPERVISE_TICK = 0.2
49
48
  RESPAWN_BACKOFF = 1.0
50
49
  HEARTBEAT_WAIT = 30
51
- MEMORY_CHECK_INTERVAL = 10
52
50
  DEFAULT_SHUTDOWN_TIMEOUT = 25
53
51
 
54
52
  # The supervisor's own Redis pool: one connection, disjoint from every
@@ -132,7 +130,6 @@ module Wurk
132
130
  crash_exit_after: config.swarm_crash_exit_after, exit_on_crash_loop: false)
133
131
  @topology = topology
134
132
  @config = config
135
- @memory_limit = memory_limit
136
133
  @shutdown_timeout = shutdown_timeout
137
134
  # One reentrant lock covers the child table AND the restart machine: they
138
135
  # are mutually recursive (Restart#advance calls back into `describe` and
@@ -149,7 +146,6 @@ module Wurk
149
146
  @shutdown_requested = false
150
147
  @quieted = false
151
148
  @requiet_at = 0
152
- @last_memory_check = 0
153
149
  @respawn_backoff = Backoff.new(base: RESPAWN_BACKOFF)
154
150
  @crash_watch = CrashWatch.new(exit_after: crash_exit_after, young_within: Backoff::RESET_AFTER)
155
151
  @exit_on_crash_loop = exit_on_crash_loop
@@ -157,6 +153,7 @@ module Wurk
157
153
  @title_program = nil
158
154
  @restart = build_restart
159
155
  @liveness = build_liveness(heartbeat_timeout)
156
+ @memory_watch = build_memory_watch(memory_limit)
160
157
  @orphan_reaper = Fetcher::Reaper.new(config)
161
158
  init_fork_unsafe_handles
162
159
  end
@@ -171,12 +168,14 @@ module Wurk
171
168
  # children. Installed first, the trap queues the TERM and the supervise
172
169
  # loop drains it (relaying to children) even if it arrives mid-boot.
173
170
  #
174
- # `prepare_for_fork!` is the last thing before the fork and stays after
175
- # `close_parent_sockets`: it materializes the slot-independent config so
176
- # every child inherits it copy-on-write, and none of what it touches opens
177
- # a socket — anything that did would land back on the wrong side of step 3.
171
+ # `prepare_for_fork!` stays after `close_parent_sockets`: it materializes
172
+ # the slot-independent config so every child inherits it copy-on-write, and
173
+ # none of what it touches opens a socket — anything that did would land
174
+ # back on the wrong side of step 3. `settle_memory_before_fork` comes last,
175
+ # after every parent allocation, so the compaction covers them; it opens no
176
+ # socket either.
178
177
  #
179
- # Nothing else belongs between steps 3 and 4. #101 boot-audit: a pre-fork
178
+ # Nothing that talks to Redis belongs between steps 3 and 4. #101 boot-audit: a pre-fork
180
179
  # `SCRIPT LOAD` was tried here (the cache is server-global, so one upload
181
180
  # could serve the whole fleet) and MEASURED SLOWER — `bench:swarm_boot`
182
181
  # 152 -> 95 i/s, a 36.7% regression. Step 3 has just closed every parent
@@ -196,6 +195,7 @@ module Wurk
196
195
  install_signal_handlers if install_signals
197
196
  close_parent_sockets
198
197
  @config.prepare_for_fork!
198
+ settle_memory_before_fork
199
199
  fork_children
200
200
  update_proctitle
201
201
  child_pids
@@ -300,7 +300,7 @@ module Wurk
300
300
  check_liveness unless @stopping
301
301
  check_crash_loop unless @stopping
302
302
  requiet_children
303
- check_memory_pressure
303
+ @memory_watch&.tick(child_ages)
304
304
  update_proctitle
305
305
  rescue StandardError => e
306
306
  handle_exception(e, { context: 'swarm-supervise' })
@@ -351,6 +351,19 @@ module Wurk
351
351
  ))
352
352
  end
353
353
 
354
+ def build_memory_watch(limit_kb)
355
+ return nil unless limit_kb
356
+
357
+ MemoryWatch.new(limit_kb: limit_kb, metric: @config.memory_limit_metric, logger: logger,
358
+ recycle: ->(pid) { @lock.synchronize { @restart.enqueue([pid]) } })
359
+ end
360
+
361
+ # `{ pid => [seconds since fork, slot index] }` for every child in the table.
362
+ def child_ages
363
+ now = monotonic
364
+ @lock.synchronize { @children.transform_values { |meta| [now - meta[:spawned_at], meta[:index]] } }
365
+ end
366
+
354
367
  def build_liveness(configured)
355
368
  timeout = Liveness.timeout_from(configured)
356
369
  return nil unless timeout
@@ -376,6 +389,13 @@ module Wurk
376
389
  @liveness.tick(child_pids, paused: paused)
377
390
  end
378
391
 
392
+ # Between steps 3 and 4, once per swarm (a respawn forks from a parent that
393
+ # may be serving requests, where a full compaction is a pause they feel).
394
+ def settle_memory_before_fork
395
+ Prefork.run(periodic: @config.periodic_manager)
396
+ @memory_watch&.warn_if_unreadable
397
+ end
398
+
379
399
  # Step 3.
380
400
  def close_parent_sockets
381
401
  @config.reset_redis_pools!
@@ -573,6 +593,7 @@ module Wurk
573
593
  return unless meta
574
594
 
575
595
  @liveness&.forget(pid)
596
+ @memory_watch&.forget(pid)
576
597
  return meta if @restart.claim_exit(pid)
577
598
 
578
599
  if @stopping
@@ -681,36 +702,6 @@ module Wurk
681
702
  relay_signal('TSTP')
682
703
  end
683
704
 
684
- def check_memory_pressure
685
- return unless @memory_limit
686
-
687
- now = monotonic
688
- return if now - @last_memory_check < MEMORY_CHECK_INTERVAL
689
-
690
- @last_memory_check = now
691
- child_pids.each { |pid| recycle_if_bloated(pid) }
692
- end
693
-
694
- # Route a bloated child through the restart state machine (same path as a
695
- # rolling restart) so recycle is graceful — a healthy replacement takes over
696
- # before the old child is TERMed — and can't overlap a restart already in
697
- # flight on the slot.
698
- def recycle_if_bloated(pid)
699
- rss = pid_rss_kb(pid)
700
- return if rss.nil? || rss < @memory_limit
701
-
702
- logger.warn { "swarm: child #{pid} RSS #{rss}KB >= #{@memory_limit}KB; recycling" }
703
- @lock.synchronize { @restart.enqueue([pid]) }
704
- end
705
-
706
- def pid_rss_kb(pid)
707
- return nil unless ::File.exist?("/proc/#{pid}/statm")
708
-
709
- Heartbeat.statm_rss_kb(::File.read("/proc/#{pid}/statm"))
710
- rescue StandardError
711
- nil
712
- end
713
-
714
705
  def relay_signal(sig)
715
706
  child_pids.each { |pid| safe_kill(pid, sig) }
716
707
  end
data/lib/wurk/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Wurk
4
- VERSION = '1.13.0'
4
+ VERSION = '1.14.0'
5
5
  end
@@ -1,9 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative '../limiter'
4
- require_relative '../cron'
5
4
  require_relative '../client'
6
- require_relative '../metrics/query'
7
5
 
8
6
  module Wurk
9
7
  class Web
@@ -45,11 +43,7 @@ module Wurk
45
43
  # what it returns but leaves the SET alone: GET must not write.
46
44
  def list(filter: nil, sweep: !Wurk::Web.config.read_only?)
47
45
  members = Wurk.redis(idempotent: true) { |c| c.call('SMEMBERS', Wurk::Limiter::LIST_KEY) }
48
- names = (sweep ? sweep(members) : live(members)).sort
49
- return names if filter.nil? || filter.to_s.empty?
50
-
51
- needle = filter.to_s.downcase
52
- names.select { |n| n.downcase.include?(needle) }
46
+ filtered((sweep ? sweep(members) : live(members)).sort, filter)
53
47
  end
54
48
 
55
49
  # Drop the names whose metadata is gone, return the ones still live.
@@ -92,6 +86,68 @@ module Wurk
92
86
  raw.is_a?(Array) ? raw.each_slice(2).to_h : raw
93
87
  end
94
88
 
89
+ # Liveness probes per process, at most one per this many seconds.
90
+ PROBE_INTERVAL = 60.0
91
+ @probe_lock = ::Mutex.new
92
+ @probe = nil
93
+
94
+ # The dashboard's listing: `count` live limiters from `offset` of the
95
+ # sorted, filtered names, each `[name, metadata]`, and how many names
96
+ # there are. #list probes every member's metadata (an EXISTS each) on
97
+ # every call; here that probe — and, with `sweep`, the removal of the
98
+ # dead — runs at most once per PROBE_INTERVAL per process, and pages in
99
+ # between pay one SMEMBERS and one pipelined HGETALL for their rows.
100
+ # The cost of that: a limiter whose metadata expired since the last
101
+ # probe counts in `total` and is left off its page, and a name the
102
+ # probe found dead and that registers again stays hidden until the
103
+ # next probe.
104
+ #
105
+ # @return [Array(Integer, Array<Array(String, Hash)>)]
106
+ def page(filter:, offset:, count:, sweep: !Wurk::Web.config.read_only?)
107
+ members = Wurk.redis(idempotent: true) { |c| c.call('SMEMBERS', Wurk::Limiter::LIST_KEY) }
108
+ names = filtered((members - known_dead(members, sweep)).sort, filter)
109
+ [names.size, metadata_rows(names[offset, count] || [])]
110
+ end
111
+
112
+ def filtered(names, filter)
113
+ return names if filter.nil? || filter.to_s.empty?
114
+
115
+ needle = filter.to_s.downcase
116
+ names.select { |n| n.downcase.include?(needle) }
117
+ end
118
+
119
+ # Names the last probe of this pool found dead, probing (and sweeping)
120
+ # again once PROBE_INTERVAL has passed.
121
+ def known_dead(members, sweep)
122
+ pool = Wurk.redis_pool
123
+ now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
124
+ @probe_lock.synchronize do
125
+ probe = @probe
126
+ return probe[:dead] if probe && probe[:pool].equal?(pool) && now - probe[:at] < PROBE_INTERVAL
127
+
128
+ dead = members - (sweep ? sweep(members) : live(members))
129
+ @probe = { pool: pool, at: now, dead: dead }
130
+ dead
131
+ end
132
+ end
133
+
134
+ def metadata_rows(names)
135
+ return [] if names.empty?
136
+
137
+ raws = Wurk.redis(idempotent: true) do |c|
138
+ c.pipelined { |pipe| names.each { |n| pipe.call('HGETALL', meta_key(n)) } }
139
+ end
140
+ names.zip(raws).filter_map do |name, raw|
141
+ meta = raw.is_a?(Array) ? raw.each_slice(2).to_h : raw
142
+ [name, meta] unless meta.empty?
143
+ end
144
+ end
145
+
146
+ # Forgets the last probe, so the next #page probes. For tests.
147
+ def reset_probe!
148
+ @probe_lock.synchronize { @probe = nil }
149
+ end
150
+
95
151
  def meta_key(name)
96
152
  "lmtr:#{name}"
97
153
  end