wurk 1.13.0 → 1.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (255) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -1
  3. data/app/controllers/concerns/wurk/sse_streaming.rb +36 -1
  4. data/app/controllers/wurk/api/pagination.rb +9 -7
  5. data/app/controllers/wurk/api/serializers.rb +8 -2
  6. data/app/controllers/wurk/api_controller.rb +57 -24
  7. data/app/controllers/wurk/application_controller.rb +0 -2
  8. data/app/controllers/wurk/dashboard_controller.rb +0 -1
  9. data/config/routes.rb +5 -1
  10. data/lib/sidekiq/cli.rb +4 -3
  11. data/lib/sidekiq/deploy.rb +2 -1
  12. data/lib/sidekiq/embedded.rb +2 -1
  13. data/lib/sidekiq/iterable_job.rb +2 -1
  14. data/lib/sidekiq/job/iterable/active_record_enumerator.rb +2 -1
  15. data/lib/sidekiq/job/iterable/csv_enumerator.rb +2 -1
  16. data/lib/sidekiq/job/iterable.rb +2 -1
  17. data/lib/sidekiq/launcher.rb +2 -1
  18. data/lib/sidekiq/manager.rb +2 -1
  19. data/lib/sidekiq/metrics/query.rb +2 -1
  20. data/lib/sidekiq/pro/web.rb +2 -1
  21. data/lib/sidekiq/scheduled.rb +2 -1
  22. data/lib/sidekiq/web.rb +2 -1
  23. data/lib/sidekiq.rb +3 -2
  24. data/lib/wurk/api/fast.rb +4 -18
  25. data/lib/wurk/api/queues.rb +12 -10
  26. data/lib/wurk/api/serializers.rb +6 -7
  27. data/lib/wurk/api/swarm.rb +0 -1
  28. data/lib/wurk/batch/server_middleware.rb +4 -0
  29. data/lib/wurk/batch/status.rb +45 -22
  30. data/lib/wurk/batch_set.rb +34 -0
  31. data/lib/wurk/cli.rb +17 -19
  32. data/lib/wurk/client/buffered.rb +97 -26
  33. data/lib/wurk/client.rb +21 -1
  34. data/lib/wurk/compat/deploy.rb +7 -0
  35. data/lib/wurk/compat/encryption.rb +7 -0
  36. data/lib/wurk/compat/flow.rb +8 -0
  37. data/lib/wurk/compat/iterable_job.rb +8 -0
  38. data/lib/wurk/compat/profile_set.rb +8 -0
  39. data/lib/wurk/compat/server.rb +15 -0
  40. data/lib/wurk/compat/web.rb +20 -0
  41. data/lib/wurk/compat.rb +34 -40
  42. data/lib/wurk/configuration.rb +46 -9
  43. data/lib/wurk/cron.rb +146 -47
  44. data/lib/wurk/dead_set.rb +10 -6
  45. data/lib/wurk/embedded.rb +0 -2
  46. data/lib/wurk/engine.rb +46 -4
  47. data/lib/wurk/error_coalescer.rb +21 -1
  48. data/lib/wurk/fetcher/many_queues.rb +8 -6
  49. data/lib/wurk/fetcher/poison_pill.rb +1 -1
  50. data/lib/wurk/fetcher/reaper.rb +0 -5
  51. data/lib/wurk/fetcher/reliable.rb +46 -27
  52. data/lib/wurk/fetcher/self_reconcile.rb +0 -1
  53. data/lib/wurk/fetcher/unit_of_work.rb +16 -1
  54. data/lib/wurk/flow/status.rb +10 -4
  55. data/lib/wurk/flow_set.rb +24 -2
  56. data/lib/wurk/health.rb +0 -1
  57. data/lib/wurk/heartbeat.rb +13 -7
  58. data/lib/wurk/import/sidekiq_cron.rb +0 -1
  59. data/lib/wurk/job_record.rb +2 -1
  60. data/lib/wurk/job_retry.rb +1 -1
  61. data/lib/wurk/job_set.rb +132 -6
  62. data/lib/wurk/launcher.rb +0 -12
  63. data/lib/wurk/lazy_loader.rb +60 -0
  64. data/lib/wurk/limiter/bucket.rb +0 -2
  65. data/lib/wurk/limiter/concurrent.rb +19 -36
  66. data/lib/wurk/limiter/leaky.rb +0 -2
  67. data/lib/wurk/limiter/points.rb +0 -2
  68. data/lib/wurk/limiter/server_middleware.rb +4 -1
  69. data/lib/wurk/limiter/unlimited.rb +0 -2
  70. data/lib/wurk/limiter/window.rb +0 -2
  71. data/lib/wurk/limiter.rb +0 -7
  72. data/lib/wurk/logger.rb +44 -5
  73. data/lib/wurk/lua/fast_delete.rb +58 -0
  74. data/lib/wurk/lua/fast_delete_window.lua +46 -0
  75. data/lib/wurk/lua/limiter_concurrent_acquire.lua +10 -3
  76. data/lib/wurk/lua/limiter_concurrent_release.lua +13 -4
  77. data/lib/wurk/lua.rb +1 -41
  78. data/lib/wurk/metrics/dashboard_series.rb +38 -18
  79. data/lib/wurk/metrics/history.rb +24 -11
  80. data/lib/wurk/metrics/prometheus.rb +0 -1
  81. data/lib/wurk/metrics/query.rb +0 -2
  82. data/lib/wurk/metrics/queue_rollup.rb +33 -11
  83. data/lib/wurk/metrics/statsd.rb +4 -0
  84. data/lib/wurk/middleware/chain.rb +25 -3
  85. data/lib/wurk/middleware/expiry.rb +4 -0
  86. data/lib/wurk/middleware/interrupt_handler.rb +4 -1
  87. data/lib/wurk/middleware/status.rb +4 -0
  88. data/lib/wurk/middleware/timeout.rb +4 -0
  89. data/lib/wurk/preflight/owners.rb +0 -1
  90. data/lib/wurk/preflight/payload_sample.rb +0 -1
  91. data/lib/wurk/preflight/verify.rb +0 -2
  92. data/lib/wurk/process_set.rb +6 -1
  93. data/lib/wurk/profiler.rb +2 -1
  94. data/lib/wurk/queue.rb +33 -6
  95. data/lib/wurk/queue_consumers.rb +10 -1
  96. data/lib/wurk/rails_boot.rb +2 -2
  97. data/lib/wurk/railtie.rb +3 -4
  98. data/lib/wurk/sentry/error_handler.rb +5 -0
  99. data/lib/wurk/sentry/middleware.rb +4 -0
  100. data/lib/wurk/sorted_entry.rb +16 -12
  101. data/lib/wurk/stats.rb +85 -1
  102. data/lib/wurk/swarm/child_boot.rb +0 -2
  103. data/lib/wurk/swarm/liveness.rb +0 -2
  104. data/lib/wurk/swarm/memory_watch.rb +116 -0
  105. data/lib/wurk/swarm/prefork.rb +54 -0
  106. data/lib/wurk/swarm.rb +33 -42
  107. data/lib/wurk/version.rb +1 -1
  108. data/lib/wurk/web/enterprise.rb +63 -7
  109. data/lib/wurk.rb +33 -29
  110. data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js +1 -0
  111. data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.br +0 -0
  112. data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.gz +0 -0
  113. data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js +1 -0
  114. data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.br +0 -0
  115. data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.gz +0 -0
  116. data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js +1 -0
  117. data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.br +0 -0
  118. data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.gz +0 -0
  119. data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js +1 -0
  120. data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.br +0 -0
  121. data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.gz +0 -0
  122. data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js +1 -0
  123. data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.br +0 -0
  124. data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.gz +0 -0
  125. data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js +1 -0
  126. data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.br +0 -0
  127. data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.gz +0 -0
  128. data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js +1 -0
  129. data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.br +0 -0
  130. data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.gz +0 -0
  131. data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js +1 -0
  132. data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.br +0 -0
  133. data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.gz +0 -0
  134. data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js +2 -0
  135. data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.br +0 -0
  136. data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.gz +0 -0
  137. data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js +1 -0
  138. data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.br +0 -0
  139. data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.gz +0 -0
  140. data/vendor/assets/dashboard/assets/{FlowState-yb7virqR.js → FlowState-ClzInw0-.js} +1 -1
  141. data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.br +0 -0
  142. data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.gz +0 -0
  143. data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js +1 -0
  144. data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.br +0 -0
  145. data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.gz +0 -0
  146. data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js +2 -0
  147. data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.br +0 -0
  148. data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.gz +0 -0
  149. data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js +1 -0
  150. data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.br +0 -0
  151. data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.gz +0 -0
  152. data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js +1 -0
  153. data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.br +0 -0
  154. data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.gz +0 -0
  155. data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js +1 -0
  156. data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.br +0 -0
  157. data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.gz +0 -0
  158. data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js +1 -0
  159. data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.br +0 -0
  160. data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.gz +0 -0
  161. data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js +1 -0
  162. data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.br +0 -0
  163. data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.gz +0 -0
  164. data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js +1 -0
  165. data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.br +0 -0
  166. data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.gz +0 -0
  167. data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js +1 -0
  168. data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.br +0 -0
  169. data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.gz +0 -0
  170. data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js +1 -0
  171. data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.br +0 -0
  172. data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.gz +0 -0
  173. data/vendor/assets/dashboard/assets/Search-D42Va7lB.js +1 -0
  174. data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.br +0 -0
  175. data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.gz +0 -0
  176. data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js +1 -0
  177. data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.br +0 -0
  178. data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.gz +0 -0
  179. data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js +1 -0
  180. data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.br +0 -0
  181. data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.gz +0 -0
  182. data/vendor/assets/dashboard/assets/de-B2NCIuzo.js +1 -0
  183. data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.br +0 -0
  184. data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.gz +0 -0
  185. data/vendor/assets/dashboard/assets/es-BGeu_JNB.js +1 -0
  186. data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.br +0 -0
  187. data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.gz +0 -0
  188. data/vendor/assets/dashboard/assets/fr-DQVBWFId.js +1 -0
  189. data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.br +0 -0
  190. data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.gz +0 -0
  191. data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js +1 -0
  192. data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.br +0 -0
  193. data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.gz +0 -0
  194. data/vendor/assets/dashboard/assets/index-CiLU1zz8.js +141 -0
  195. data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.br +0 -0
  196. data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.gz +0 -0
  197. data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css +1 -0
  198. data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.br +0 -0
  199. data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.gz +0 -0
  200. data/vendor/assets/dashboard/assets/ja-D06IBqzL.js +1 -0
  201. data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.br +0 -0
  202. data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.gz +0 -0
  203. data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js +1 -0
  204. data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.br +0 -0
  205. data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.gz +0 -0
  206. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js +1 -0
  207. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.br +0 -0
  208. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.gz +0 -0
  209. data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js +1 -0
  210. data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.br +0 -0
  211. data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.gz +0 -0
  212. data/vendor/assets/dashboard/assets/{utils-CnWZzW2L.js → utils-69XSoGbQ.js} +1 -1
  213. data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.br +0 -0
  214. data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.gz +0 -0
  215. data/vendor/assets/dashboard/assets/wurk-logo-ydeSB9B8.png +0 -0
  216. data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js +1 -0
  217. data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.br +0 -0
  218. data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.gz +0 -0
  219. data/vendor/assets/dashboard/index.html +3 -23
  220. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  221. metadata +122 -37
  222. data/vendor/assets/dashboard/assets/ArgsValue-BJol9S0s.js +0 -1
  223. data/vendor/assets/dashboard/assets/BatchDetail-D_zGQ9Sd.js +0 -1
  224. data/vendor/assets/dashboard/assets/Batches-Dt_xC660.js +0 -1
  225. data/vendor/assets/dashboard/assets/Busy-CWRthsmc.js +0 -1
  226. data/vendor/assets/dashboard/assets/Cron-qeGr2YAm.js +0 -1
  227. data/vendor/assets/dashboard/assets/Dashboard-DQWaAwPA.js +0 -1
  228. data/vendor/assets/dashboard/assets/Dead-D9gOPHtK.js +0 -1
  229. data/vendor/assets/dashboard/assets/Extension-DONXVLdA.js +0 -1
  230. data/vendor/assets/dashboard/assets/FilterBox-BbB6gXHp.js +0 -2
  231. data/vendor/assets/dashboard/assets/FlowDetail-CrmnVRp0.js +0 -1
  232. data/vendor/assets/dashboard/assets/Flows-g4GC8KmE.js +0 -1
  233. data/vendor/assets/dashboard/assets/JobDetailModal-C86k_1wj.js +0 -2
  234. data/vendor/assets/dashboard/assets/Limiters-zyO-iMlK.js +0 -1
  235. data/vendor/assets/dashboard/assets/Metrics-Ckq5fNBv.js +0 -1
  236. data/vendor/assets/dashboard/assets/NotFound-C8NnyjlY.js +0 -1
  237. data/vendor/assets/dashboard/assets/PageHeader-vWeBbSPF.js +0 -1
  238. data/vendor/assets/dashboard/assets/Profiles-DRl-FXPA.js +0 -1
  239. data/vendor/assets/dashboard/assets/Queues-DkXWrihA.js +0 -1
  240. data/vendor/assets/dashboard/assets/Retries-CmlojYIy.js +0 -1
  241. data/vendor/assets/dashboard/assets/Scheduled-Cbbpp3xw.js +0 -1
  242. data/vendor/assets/dashboard/assets/Search-LaEpj2fM.js +0 -1
  243. data/vendor/assets/dashboard/assets/charts-D_PQLoyc.js +0 -1
  244. data/vendor/assets/dashboard/assets/fa-brands-400-Bs6tcqqs.woff2 +0 -0
  245. data/vendor/assets/dashboard/assets/fa-regular-400-DRN8N0d1.woff2 +0 -0
  246. data/vendor/assets/dashboard/assets/fa-solid-900-IAB4Droh.woff2 +0 -0
  247. data/vendor/assets/dashboard/assets/fa-v4compatibility-CErXDOsT.woff2 +0 -0
  248. data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +0 -1
  249. data/vendor/assets/dashboard/assets/index--xvvNlYx.css +0 -1
  250. data/vendor/assets/dashboard/assets/index-FBKGtysy.js +0 -141
  251. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-BiwhHqgh.js +0 -1
  252. data/vendor/assets/dashboard/assets/useSort-jBYVwXvg.js +0 -1
  253. data/vendor/assets/dashboard/assets/wurk-logo-Vy3xW4K0.png +0 -0
  254. data/vendor/assets/dashboard/og-image.png +0 -0
  255. data/vendor/assets/dashboard/wurk-logo.webp +0 -0
data/lib/wurk/compat.rb CHANGED
@@ -50,16 +50,6 @@ module Sidekiq
50
50
  # classes under them — but `Sidekiq.pro?` / `Sidekiq.ent?` still return
51
51
  # `false` per docs/target/sidekiq-free.md §32 (Wurk advertises as free OSS).
52
52
  module Pro
53
- # Sidekiq Pro mounts the dashboard at `Sidekiq::Pro::Web`; it's the same
54
- # board as `Sidekiq::Web` here, so a Pro app's `mount Sidekiq::Pro::Web`
55
- # drops in unchanged. `Wurk::Web` is already required by the time this file
56
- # loads (lib/wurk.rb requires wurk/web before wurk/compat).
57
- Web = Wurk::Web
58
-
59
- # `use Sidekiq::Pro::BatchStatus` — the polling Rack middleware that serves
60
- # GET /batch_status/<bid>.json. Spec: docs/target/sidekiq-pro.md §10.3.
61
- BatchStatus = Wurk::Web::BatchStatus
62
-
63
53
  # Pro module-level statsd accessor (spec §1): `Sidekiq::Pro.dogstatsd = ...`.
64
54
  # Delegates to the canonical `config.dogstatsd=` (spec §9.1), so either form
65
55
  # feeds the same Wurk::Metrics::Statsd client.
@@ -119,38 +109,14 @@ module Sidekiq
119
109
  private_constant :RailsNamespace
120
110
 
121
111
  BasicFetch = Wurk::Fetcher::Reliable
122
- WebHelpers = Wurk::Web::Extension::Helpers
123
- Paginator = Wurk::Web::Paginator
124
112
  Batch = Wurk::Batch
125
113
  BatchSet = Wurk::BatchSet
126
114
  Capsule = Wurk::Capsule
127
- CLI = Wurk::CLI
128
115
  Client = Wurk::Client
129
116
  Component = Wurk::Component
130
117
  Config = Wurk::Configuration
131
118
  Context = Wurk::Context
132
- # No `Sidekiq::Cron` alias on purpose (#204): real Sidekiq never defines it
133
- # — that namespace belongs to the third-party sidekiq-cron gem, and squatting
134
- # it made the gem's own classes (Poller, Job) collide at load. Wurk's native
135
- # periodic surface is `Sidekiq::Periodic` (Ent parity) / `Wurk::Cron`.
136
- CronParser = Wurk::Cron::Parser
137
- Periodic = Wurk::Cron
138
119
  DeadSet = Wurk::DeadSet
139
- Deploy = Wurk::Deploy
140
- Embedded = Wurk::Embedded
141
- Encryption = Wurk::Encryption
142
- # A Wurk extra, not parity — aliased because nothing else claims the name.
143
- # Checked before adding it, the way `Sidekiq::Cron` (#204) and
144
- # `Sidekiq::Status` taught us to: upstream Sidekiq defines no `Flow` anywhere
145
- # in `lib/` at the pinned parity SHA, RubyGems has no `sidekiq-flow` or
146
- # `sidekiq_flow` gem, and the only `Sidekiq::Flow` in public Ruby is a
147
- # `FlowIdMiddleware` nested under a gem's *own* `Sidekiq` module, which this
148
- # constant cannot reach.
149
- Flow = Wurk::Flow
150
- FlowSet = Wurk::FlowSet
151
- History = Wurk::History
152
- IterableJob = Wurk::IterableJob
153
- IterableJobQuery = Wurk::IterableJobQuery
154
120
  Job = Wurk::Job
155
121
  JobLogger = Wurk::JobLogger
156
122
  JobRecord = Wurk::JobRecord
@@ -161,11 +127,9 @@ module Sidekiq
161
127
  JobSet = Wurk::JobSet
162
128
  JobUtil = Wurk::JobUtil
163
129
  Keys = Wurk::Keys
164
- Launcher = Wurk::Launcher
165
130
  Limiter = Wurk::Limiter
166
131
  Loader = Wurk::Loader
167
132
  Logger = Wurk::Logger
168
- Manager = Wurk::Manager
169
133
  Metrics = Wurk::Metrics
170
134
  Middleware = Wurk::Middleware
171
135
  ServerMiddleware = Wurk::Middleware::ServerMiddleware
@@ -174,13 +138,10 @@ module Sidekiq
174
138
  ProcessSet = Wurk::ProcessSet
175
139
  Processor = Wurk::Processor
176
140
  Profiler = Wurk::Profiler
177
- ProfileSet = Wurk::ProfileSet
178
- ProfileRecord = Wurk::ProfileRecord
179
141
  Queue = Wurk::Queue
180
142
  RedisClientAdapter = Wurk::RedisClientAdapter
181
143
  RedisConnection = Wurk::RedisConnection
182
144
  RetrySet = Wurk::RetrySet
183
- Scheduled = Wurk::Scheduled
184
145
  ScheduledSet = Wurk::ScheduledSet
185
146
  Shutdown = Wurk::Shutdown
186
147
  SortedEntry = Wurk::SortedEntry
@@ -200,12 +161,45 @@ module Sidekiq
200
161
  TransactionAwareClient = Wurk::TransactionAwareClient
201
162
  Queues = Wurk::Queues
202
163
  EmptyQueueError = Wurk::Testing::EmptyQueueError
203
- Web = Wurk::Web
204
164
  Work = Wurk::Work
205
165
  Worker = Wurk::Worker
206
166
  Workers = Wurk::Workers
207
167
  WorkSet = Wurk::WorkSet
208
168
 
169
+ # Aliases of lazy Wurk constants (Wurk::LazyLoader) are autoloads too, so
170
+ # `defined?(Sidekiq::Web)` holds before the load and the alias is the very
171
+ # object its target is. Each points at a file under lib/wurk/compat/ that
172
+ # loads the target and assigns the alias: the target's own file defines only
173
+ # the Wurk name, so an autoload onto it would leave `Sidekiq::X` undefined.
174
+ # Those files name their targets rather than require them: web.rb, swarm.rb
175
+ # and iterable_job.rb require subfiles that reopen the class before defining
176
+ # it, which a direct require would turn into a circular one.
177
+ #
178
+ # No `Sidekiq::Cron` alias on purpose (#204): real Sidekiq never defines it
179
+ # — that namespace belongs to the third-party sidekiq-cron gem, and squatting
180
+ # it made the gem's own classes (Poller, Job) collide at load. Wurk's native
181
+ # periodic surface is `Sidekiq::Periodic` (Ent parity) / `Wurk::Cron`.
182
+ #
183
+ # `Flow` is a Wurk extra, not parity — aliased because nothing else claims
184
+ # the name. Checked before adding it, the way `Sidekiq::Cron` (#204) and
185
+ # `Sidekiq::Status` taught us to: upstream Sidekiq defines no `Flow` anywhere
186
+ # in `lib/` at the pinned parity SHA, RubyGems has no `sidekiq-flow` or
187
+ # `sidekiq_flow` gem, and the only `Sidekiq::Flow` in public Ruby is a
188
+ # `FlowIdMiddleware` nested under a gem's *own* `Sidekiq` module, which this
189
+ # constant cannot reach.
190
+ Wurk::LazyLoader.register(
191
+ self,
192
+ CLI: 'compat/server', Embedded: 'compat/server', Launcher: 'compat/server',
193
+ Manager: 'compat/server', Scheduled: 'compat/server',
194
+ History: 'compat/server', Periodic: 'compat/server', CronParser: 'compat/server',
195
+ Web: 'compat/web', WebHelpers: 'compat/web', Paginator: 'compat/web',
196
+ ProfileSet: 'compat/profile_set', ProfileRecord: 'compat/profile_set',
197
+ Flow: 'compat/flow', FlowSet: 'compat/flow',
198
+ IterableJob: 'compat/iterable_job', IterableJobQuery: 'compat/iterable_job',
199
+ Deploy: 'compat/deploy', Encryption: 'compat/encryption'
200
+ )
201
+ Wurk::LazyLoader.register(Pro, Web: 'compat/web', BatchStatus: 'compat/web')
202
+
209
203
  # Top-level Sidekiq.configure_server / configure_client / redis / logger
210
204
  # delegate to Wurk's class methods. Third-party gems treat these as the
211
205
  # canonical entry points.
@@ -5,7 +5,6 @@ require_relative 'middleware/chain'
5
5
  require_relative 'capsule'
6
6
  require_relative 'pool_checkout'
7
7
  require_relative 'context'
8
- require_relative 'topology'
9
8
  require_relative 'cpu_count'
10
9
  require_relative 'redis_options'
11
10
  require_relative 'keys'
@@ -488,12 +487,15 @@ module Wurk
488
487
  #
489
488
  # Spec: docs/target/sidekiq-ent.md §2.
490
489
  def periodic
491
- require_relative 'cron'
492
490
  @periodic_manager ||= Wurk::Cron::Manager.new(self, deferred: true)
493
491
  yield @periodic_manager if block_given?
494
492
  @periodic_manager
495
493
  end
496
494
 
495
+ # The periodic manager if `config.periodic` was ever called, else nil:
496
+ # unlike #periodic, never creates one (Swarm::Prefork reads it).
497
+ attr_reader :periodic_manager
498
+
497
499
  # Writes the `config.periodic` registrations to Redis. Launcher#boot calls
498
500
  # it, so the loops are registered by a booting server, against the Redis
499
501
  # settings the app ended up with.
@@ -732,12 +734,12 @@ module Wurk
732
734
 
733
735
  # Web UI configuration: the authorization hook and read-only mode. Returns
734
736
  # the process-wide `Wurk::Web.config` singleton so `config.web.read_only =
735
- # true` and the engine middleware share one source of truth. Lazy-requires
736
- # the web layer to keep standalone boot lean.
737
+ # true` and the engine middleware share one source of truth. The web layer
738
+ # is an autoload (Wurk::LazyLoader), so standalone boot stays lean until this
739
+ # is called.
737
740
  #
738
741
  # Spec: docs/target/sidekiq-ent.md §9.2.
739
742
  def web
740
- require_relative 'web/config'
741
743
  Wurk::Web.config
742
744
  end
743
745
 
@@ -875,6 +877,11 @@ module Wurk
875
877
  # time, and under the railtie that is Bundler.require, before the
876
878
  # `wurk.server_mode` initializer: dropping it left sidekiq-cron, sentry-sidekiq
877
879
  # and the like silently absent from a process that does run workers.
880
+ #
881
+ # So the skip is DEBUG, as in stock Sidekiq, which skips it silently: every
882
+ # worker entry point enters server mode and runs the kept blocks, and a
883
+ # process that never does (a test run, `rails runner`, a web process) is
884
+ # meant to skip them. A WARN here was noise in each of those.
878
885
  def configure_server(&block)
879
886
  return unless block
880
887
  return run_server_block(block) if server?
@@ -883,7 +890,7 @@ module Wurk
883
890
  return if @configure_server_skip_warned
884
891
 
885
892
  @configure_server_skip_warned = true
886
- logger.warn { 'configure_server block skipped: server? is false' }
893
+ logger.debug { 'configure_server block skipped: server? is false' }
887
894
  nil
888
895
  end
889
896
 
@@ -952,13 +959,29 @@ module Wurk
952
959
  @memory_limit_mb = value.nil? ? nil : Integer(value)
953
960
  end
954
961
 
955
- # Threshold in KB, the unit the swarm compares against /proc/<pid>/statm
956
- # (pages × 4KB). nil when recycling is disabled.
962
+ # Threshold in KB, the unit the swarm reads child memory in. nil when
963
+ # recycling is disabled.
957
964
  def memory_limit_kb
958
965
  mb = memory_limit_mb
959
966
  mb&.positive? ? mb * 1024 : nil
960
967
  end
961
968
 
969
+ # What `memory_limit_mb` is compared against. `:rss` (the default, and
970
+ # Enterprise's) counts pages still shared copy-on-write with the parent, so
971
+ # every child carries the whole preloaded app; `:pss` splits shared pages
972
+ # across their sharers and `:uss` counts only the child's own. Wurk-only;
973
+ # WURK_MAXMEM_METRIC sets it, an explicit value wins.
974
+ MEMORY_LIMIT_METRICS = %i[rss pss uss].freeze
975
+
976
+ def memory_limit_metric
977
+ @memory_limit_metric || env_memory_limit_metric || :rss
978
+ end
979
+
980
+ def memory_limit_metric=(value)
981
+ guard_frozen!
982
+ @memory_limit_metric = value.nil? ? nil : parse_memory_limit_metric(value)
983
+ end
984
+
962
985
  # The pre-fork half of `freeze!`, and the only half a forking parent can
963
986
  # run: capsules stay writable until each child has applied its slot
964
987
  # (ChildBoot#apply_slot_to_config) and opened its own Redis pools. What is
@@ -1174,6 +1197,21 @@ module Wurk
1174
1197
  nil
1175
1198
  end
1176
1199
 
1200
+ def env_memory_limit_metric
1201
+ raw = ENV.fetch('WURK_MAXMEM_METRIC', nil)
1202
+ return nil if raw.nil? || raw.strip.empty?
1203
+
1204
+ parse_memory_limit_metric(raw.strip.downcase)
1205
+ end
1206
+
1207
+ def parse_memory_limit_metric(value)
1208
+ metric = value.to_s.to_sym
1209
+ return metric if MEMORY_LIMIT_METRICS.include?(metric)
1210
+
1211
+ raise ArgumentError,
1212
+ "memory_limit_metric must be one of #{MEMORY_LIMIT_METRICS.join(', ')} (got #{value.inspect})"
1213
+ end
1214
+
1177
1215
  def guard_frozen!
1178
1216
  raise FrozenError, 'Wurk::Configuration is frozen' if @frozen
1179
1217
  end
@@ -1185,7 +1223,6 @@ module Wurk
1185
1223
  def truthy_setting?(value)
1186
1224
  return !!value unless value.is_a?(String)
1187
1225
 
1188
- require_relative 'web/config'
1189
1226
  !Wurk::Web::Config::FALSEY_STRINGS.include?(value.strip.downcase)
1190
1227
  end
1191
1228
 
data/lib/wurk/cron.rb CHANGED
@@ -5,7 +5,6 @@ require 'digest'
5
5
  require 'time'
6
6
  require_relative 'component'
7
7
  require_relative 'client'
8
- require_relative 'leader'
9
8
  require_relative 'timer_loop'
10
9
 
11
10
  module Wurk
@@ -74,8 +73,27 @@ module Wurk
74
73
 
75
74
  MAX_LOOKAHEAD_MINUTES = 366 * 24 * 60 * 4
76
75
 
76
+ # Distinct schedules a process keeps parsed. A loop is rebuilt from its
77
+ # hash on every tick, and a fleet has far fewer schedules than loops.
78
+ CACHE_LIMIT = 1024
79
+
77
80
  attr_reader :expression, :fields
78
81
 
82
+ @cache = {}
83
+ @cache_mutex = ::Mutex.new
84
+
85
+ # A shared, already-parsed Parser for `expression`. Raises ArgumentError
86
+ # like `new` on a bad one, which is never cached. Parsers are not mutated
87
+ # after construction, so sharing one across loops and threads is safe.
88
+ def self.cached(expression)
89
+ @cache_mutex.synchronize do
90
+ @cache.fetch(expression) do
91
+ @cache.clear if @cache.size >= CACHE_LIMIT
92
+ @cache[expression] = new(expression)
93
+ end
94
+ end
95
+ end
96
+
79
97
  def initialize(expression)
80
98
  parts = normalize_expression(expression)
81
99
  @expression = parts.join(' ')
@@ -287,13 +305,18 @@ module Wurk
287
305
  @klass = klass
288
306
  @options = stringify_options(options)
289
307
  @tz = tz
290
- @parser = Parser.new(schedule)
308
+ @parser = Parser.cached(schedule)
291
309
  @lid = lid || Cron.lid(schedule, klass, @options)
292
310
  @paused = paused.nil? ? [true, '1', 1].include?(@options['paused']) : paused
311
+ @fire_marks = nil
293
312
  end
294
313
 
314
+ # The raw `lf` / `nf` strings of the hash this loop was read from, so a
315
+ # tick decides on the read it already made; nil for a loop built in code.
316
+ attr_accessor :fire_marks
317
+
295
318
  def parser
296
- @parser ||= Parser.new(@schedule)
319
+ @parser ||= Parser.cached(@schedule)
297
320
  end
298
321
 
299
322
  def paused?
@@ -492,7 +515,8 @@ module Wurk
492
515
  return self unless @deferred
493
516
 
494
517
  @deferred = false
495
- @loops.each { |loop_obj| persist(loop_obj) }
518
+ Cron.persist_all(@loops)
519
+ prune_superseded(@loops.map(&:klass).uniq)
496
520
  prune_unregistered if @authoritative
497
521
  self
498
522
  end
@@ -530,7 +554,7 @@ module Wurk
530
554
 
531
555
  def persist(loop_obj)
532
556
  Cron.persist(loop_obj)
533
- prune_superseded(loop_obj.klass)
557
+ prune_superseded([loop_obj.klass])
534
558
  end
535
559
 
536
560
  def prune_unregistered
@@ -559,12 +583,14 @@ module Wurk
559
583
  # an edited `register` line yields a new loop and orphans the old one).
560
584
  # Registering the same class twice in one block is fine — the second
561
585
  # register re-persists its loop right after the first one's prune.
562
- def prune_superseded(klass_name)
586
+ def prune_superseded(klass_names)
563
587
  return unless @prune
564
- return if @pruned_klasses[klass_name]
565
588
 
566
- @pruned_klasses[klass_name] = true
567
- Cron.prune_superseded(klass_name, @loops.map(&:lid))
589
+ fresh = klass_names.reject { |name| @pruned_klasses[name] }
590
+ return if fresh.empty?
591
+
592
+ fresh.each { |name| @pruned_klasses[name] = true }
593
+ Cron.prune_superseded(fresh, @loops.map(&:lid))
568
594
  end
569
595
  end
570
596
 
@@ -588,13 +614,7 @@ module Wurk
588
614
  def each
589
615
  return enum_for(:each) unless block_given?
590
616
 
591
- rows = redis do |c|
592
- c.call('SMEMBERS', PERIODIC_KEY).filter_map do |lid|
593
- h = c.call('HGETALL', "#{LOOP_PREFIX}#{lid}")
594
- [lid, h] unless h.nil? || h.empty?
595
- end
596
- end
597
- rows.each do |lid, h|
617
+ read_rows.each do |lid, h|
598
618
  loop_obj = parse(lid, h)
599
619
  yield loop_obj if loop_obj
600
620
  end
@@ -616,8 +636,23 @@ module Wurk
616
636
 
617
637
  private
618
638
 
639
+ # [lid, hash] for every registered loop that still has its hash: one
640
+ # SMEMBERS, then every HGETALL in one pipeline.
641
+ def read_rows
642
+ redis(idempotent: true) do |c|
643
+ lids = c.call('SMEMBERS', PERIODIC_KEY)
644
+ next [] if lids.empty?
645
+
646
+ hashes = c.pipelined { |p| lids.each { |lid| p.call('HGETALL', "#{LOOP_PREFIX}#{lid}") } }
647
+ lids.zip(hashes).filter_map do |lid, h|
648
+ h = h.each_slice(2).to_h if h.is_a?(Array)
649
+ [lid, h] unless h.nil? || h.empty?
650
+ end
651
+ end
652
+ end
653
+
619
654
  def parse(lid, hash)
620
- Loop.from_redis(lid, hash)
655
+ Loop.from_redis(lid, hash).tap { |lp| lp.fire_marks = hash.values_at('lf', 'nf') }
621
656
  rescue JSON::ParserError, ArgumentError => e
622
657
  Wurk.logger&.warn("[cron] skipping unreadable loop lid=#{lid}: #{e.class}: #{e.message}")
623
658
  nil
@@ -679,6 +714,7 @@ module Wurk
679
714
  @sleeper = ::ConditionVariable.new
680
715
  @on_leader = method(:wake_for_leadership)
681
716
  @unknown_reported = ::Set.new
717
+ @derived = {}
682
718
  end
683
719
 
684
720
  # The loop waits one interval before its first tick unless this process
@@ -720,11 +756,14 @@ module Wurk
720
756
  def tick
721
757
  return unless leader?
722
758
 
759
+ seen = ::Set.new
723
760
  LoopSet.new(@config).each do |lp|
724
- enqueue_if_due(lp)
761
+ seen << derived_key(lp)
762
+ enqueue_if_due(lp, lp.fire_marks)
725
763
  rescue StandardError => e
726
764
  handle_exception(e, { context: 'cron-poller', lid: lp.lid, klass: lp.klass })
727
765
  end
766
+ @derived.select! { |key, _| seen.include?(key) }
728
767
  rescue StandardError => e
729
768
  handle_exception(e, { context: 'cron-poller' })
730
769
  end
@@ -741,11 +780,14 @@ module Wurk
741
780
  # One claim, one push at most, whichever slot it pushes for: the occurrence
742
781
  # due now stands in for a stale one under the stale one's mark, so a poller
743
782
  # that read the same mark loses the CAS for both.
744
- def enqueue_if_due(loop_obj)
783
+ #
784
+ # `marks` is the raw [`lf`, `nf`] pair the caller just read (a tick's
785
+ # LoopSet read); without it they are read here.
786
+ def enqueue_if_due(loop_obj, marks = nil)
745
787
  return if loop_obj.paused?
746
788
 
747
789
  now = ::Time.now.to_i
748
- slot, mark = due_slot(loop_obj, now)
790
+ slot, mark = due_slot(loop_obj, now, marks)
749
791
  return if slot.nil?
750
792
 
751
793
  firing = slot_to_fire(loop_obj, slot, now)
@@ -822,14 +864,32 @@ module Wurk
822
864
  # the loop has no resolvable occurrence or none is due yet. The token is
823
865
  # the stored `nf` when there is one, otherwise the slot we derived — see
824
866
  # the `cron_claim_fire` script for how the two cases differ.
825
- def due_slot(loop_obj, now)
826
- prev_fire, next_fire, mark = read_fire_marks(loop_obj.lid)
827
- next_fire ||= loop_obj.next_fire_at(prev_fire || (now - @tick_interval))
867
+ def due_slot(loop_obj, now, marks)
868
+ prev_fire, next_fire, mark = marks ? parse_fire_marks(marks) : read_fire_marks(loop_obj.lid)
869
+ next_fire ||= derived_next_fire(loop_obj, prev_fire || (now - @tick_interval))
828
870
  return if next_fire.nil? || next_fire > now
829
871
 
830
872
  [next_fire, mark || next_fire.to_s]
831
873
  end
832
874
 
875
+ # A loop with no `nf` (never fired, or an exhausted schedule) derives its
876
+ # next occurrence by walking Parser#next_fire_at minute by minute: a year
877
+ # of minutes for a yearly loop, on every tick. The answer for one origin
878
+ # still holds for any later origin it is past, so it is kept per loop
879
+ # until the origin overtakes it. Nothing is written: the stored hash stays
880
+ # what a first fire leaves.
881
+ def derived_next_fire(loop_obj, from)
882
+ key = derived_key(loop_obj)
883
+ origin, found = @derived[key]
884
+ return found if origin && from >= origin && (found ? found > from : from == origin)
885
+
886
+ loop_obj.next_fire_at(from).tap { |epoch| @derived[key] = [from, epoch] }
887
+ end
888
+
889
+ def derived_key(loop_obj)
890
+ [loop_obj.lid, loop_obj.schedule, loop_obj.tz_name]
891
+ end
892
+
833
893
  # Never tighter than a tick and a half: a raised tick interval must not
834
894
  # turn every on-time fire into a "missed" one.
835
895
  def missed_threshold
@@ -928,16 +988,17 @@ module Wurk
928
988
  job.provider_job_id if job.respond_to?(:provider_job_id)
929
989
  end
930
990
 
991
+ def read_fire_marks(lid)
992
+ parse_fire_marks(@config.redis { |c| c.call('HMGET', "#{LOOP_PREFIX}#{lid}", 'lf', 'nf') })
993
+ end
994
+
931
995
  # Third element is the raw `nf` string — the CAS token for #claim_fire.
932
996
  # Comparing the bytes Redis holds, not the parsed integer, keeps the
933
- # token exact regardless of how the value was written.
934
- def read_fire_marks(lid)
935
- @config.redis do |c|
936
- prev_fire, next_fire = c.call('HMGET', "#{LOOP_PREFIX}#{lid}", 'lf', 'nf')
937
- # Preserve nil: treat missing or empty 'nf' as nil, not 0.
938
- next_fire = nil if next_fire.nil? || next_fire.empty?
939
- [prev_fire&.to_i, next_fire&.to_i, next_fire]
940
- end
997
+ # token exact regardless of how the value was written. A missing or
998
+ # empty `nf` is nil, not 0.
999
+ def parse_fire_marks((prev_fire, next_fire))
1000
+ next_fire = nil if next_fire.nil? || next_fire.empty?
1001
+ [prev_fire&.to_i, next_fire&.to_i, next_fire]
941
1002
  end
942
1003
 
943
1004
  def record_fire(loop_obj, jid, fired_at, future)
@@ -992,37 +1053,62 @@ module Wurk
992
1053
  # the field belongs to the runtime (Web UI pause/unpause). Writing it
993
1054
  # unconditionally un-paused a dashboard-paused loop on the next boot.
994
1055
  def persist(loop_obj)
995
- fields = loop_obj.to_redis_hash
996
- paused = fields.delete('paused')
997
- key = "#{LOOP_PREFIX}#{loop_obj.lid}"
1056
+ persist_all([loop_obj])
1057
+ loop_obj
1058
+ end
1059
+
1060
+ # #persist for many loops in one pipelined round trip: a booting process
1061
+ # (each swarm child) writes its whole `config.periodic` block at once.
1062
+ # Nothing here reads, so a pipeline is the same writes in the same order.
1063
+ def persist_all(loops)
1064
+ return loops if loops.empty?
1065
+
998
1066
  Wurk.redis do |c|
999
- c.call('SADD', PERIODIC_KEY, loop_obj.lid)
1000
- c.call('HSET', key, *fields.flatten)
1001
- c.call('HSETNX', key, 'paused', paused)
1067
+ c.pipelined do |p|
1068
+ loops.each do |loop_obj|
1069
+ fields = loop_obj.to_redis_hash
1070
+ paused = fields.delete('paused')
1071
+ key = "#{LOOP_PREFIX}#{loop_obj.lid}"
1072
+ p.call('SADD', PERIODIC_KEY, loop_obj.lid)
1073
+ p.call('HSET', key, *fields.flatten)
1074
+ p.call('HSETNX', key, 'paused', paused)
1075
+ end
1076
+ end
1002
1077
  end
1003
- loop_obj
1078
+ loops
1004
1079
  end
1005
1080
 
1006
- # Drop every registered loop for `klass` except `keep_lids` — the loops a
1007
- # booting process just registered for that class. Editing a schedule or
1008
- # any option changes the lid, so without this the pre-edit loop stays in
1009
- # `periodic` and keeps firing next to its replacement, forever.
1081
+ # Drop every registered loop for `klass` (a name or a list of names)
1082
+ # except `keep_lids` — the loops a booting process just registered for
1083
+ # those classes. Editing a schedule or any option changes the lid, so
1084
+ # without this the pre-edit loop stays in `periodic` and keeps firing next
1085
+ # to its replacement, forever.
1010
1086
  #
1011
- # Scoped to one class deliberately: the registry is fleet-wide, so a
1087
+ # Scoped to the registered classes deliberately: the registry is fleet-wide, so a
1012
1088
  # process that registers a subset of the fleet's loops (or none at all,
1013
1089
  # e.g. a client-only app) must never delete loops for classes it doesn't
1014
1090
  # register. Idempotent and safe to race — two processes booting the same
1015
1091
  # code compute the same candidate set, and SREM/DEL repeat harmlessly.
1092
+ #
1093
+ # One SMEMBERS and one pipelined HGET per candidate, however many classes:
1094
+ # a boot registering C classes used to read every foreign loop C times.
1016
1095
  def prune_superseded(klass, keep_lids)
1096
+ klasses = Array(klass)
1017
1097
  keep = Array(keep_lids)
1018
- stale = Wurk.redis do |c|
1019
- candidates = c.call('SMEMBERS', PERIODIC_KEY) - keep
1020
- candidates.select { |lid| c.call('HGET', "#{LOOP_PREFIX}#{lid}", 'klass') == klass }
1021
- end
1098
+ stale = Wurk.redis { |c| superseded_lids(c, klasses, keep) }
1022
1099
  stale.each { |lid| unregister(lid) }
1023
1100
  stale
1024
1101
  end
1025
1102
 
1103
+ def superseded_lids(conn, klasses, keep)
1104
+ candidates = conn.call('SMEMBERS', PERIODIC_KEY) - keep
1105
+ return [] if candidates.empty?
1106
+
1107
+ owners = conn.pipelined { |p| candidates.each { |lid| p.call('HGET', "#{LOOP_PREFIX}#{lid}", 'klass') } }
1108
+ candidates.zip(owners).filter_map { |lid, owner| lid if klasses.include?(owner) }
1109
+ end
1110
+ private :superseded_lids
1111
+
1026
1112
  # Drop a loop entirely: registry membership, hash, and fire history.
1027
1113
  # Public API for retiring a loop whose `register(...)` line is gone (no
1028
1114
  # process registers its class any more, so `prune_superseded` cannot see
@@ -1050,6 +1136,19 @@ module Wurk
1050
1136
  LoopSet.new
1051
1137
  end
1052
1138
 
1139
+ # For the swarm parent's pre-fork step (Swarm::Prefork): resolve the IANA
1140
+ # zone of every loop `manager` registered, so tzinfo and the zone data
1141
+ # are loaded once and shared copy-on-write rather than loaded by the
1142
+ # leader child on its first tick. Opens no socket. `erb` and `yaml` need
1143
+ # no warming: Manager#load_file runs in the parent, at configuration.
1144
+ def prefork(manager)
1145
+ manager&.loops&.each do |lp|
1146
+ name = lp.tz_name
1147
+ Parser.resolve_zone(name) if lp.tz.is_a?(String) && !name.empty?
1148
+ end
1149
+ nil
1150
+ end
1151
+
1053
1152
  # Test/ops helper: fire one registered loop immediately, ignoring the
1054
1153
  # leader gate and the schedule due-check. Records history + advances the
1055
1154
  # fire marks just like a leader tick, so specs can assert on the enqueue
data/lib/wurk/dead_set.rb CHANGED
@@ -65,19 +65,23 @@ module Wurk
65
65
  # receive `(job, ex)`. `max_jobs:` / `timeout:` propagate to the auto-trim;
66
66
  # see `#trim` for the rationale.
67
67
  def kill(message, opts = {})
68
- notify = opts.fetch(:notify_failure, true)
69
- do_trim = opts.fetch(:trim, true)
70
- ex = opts[:ex] || RuntimeError.new(API_KILL_MESSAGE).tap { |e| e.set_backtrace(caller) }
71
-
72
68
  now = ::Process.clock_gettime(::Process::CLOCK_REALTIME)
73
69
  Wurk.redis { |conn| conn.call('ZADD', @name, now.to_s, message) }
74
- trim(max_jobs: opts[:max_jobs], timeout: opts[:timeout]) if do_trim
75
- fire_death_handlers(message, ex) if notify
70
+ trim(max_jobs: opts[:max_jobs], timeout: opts[:timeout]) if opts.fetch(:trim, true)
71
+ fire_death_handlers(message, opts[:ex] || api_kill_error) if opts.fetch(:notify_failure, true)
76
72
  true
77
73
  end
78
74
 
79
75
  private
80
76
 
77
+ # Only the death handlers see it, so a kill that notifies none (a UI kill,
78
+ # `kill_all` over a whole set) never pays for the caller's backtrace.
79
+ def api_kill_error
80
+ error = RuntimeError.new(API_KILL_MESSAGE)
81
+ error.set_backtrace(caller(2))
82
+ error
83
+ end
84
+
81
85
  def fire_death_handlers(message, ex)
82
86
  job = parse_message(message)
83
87
  handlers = Wurk.configuration.death_handlers
data/lib/wurk/embedded.rb CHANGED
@@ -1,8 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative 'component'
4
- require_relative 'launcher'
5
- require_relative 'swarm/test_mode_guard'
6
4
 
7
5
  module Wurk
8
6
  # Embeds Wurk inside an arbitrary Ruby process (Puma, rake task, custom