wurk 1.13.0 → 1.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (255) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -1
  3. data/app/controllers/concerns/wurk/sse_streaming.rb +36 -1
  4. data/app/controllers/wurk/api/pagination.rb +9 -7
  5. data/app/controllers/wurk/api/serializers.rb +8 -2
  6. data/app/controllers/wurk/api_controller.rb +57 -24
  7. data/app/controllers/wurk/application_controller.rb +0 -2
  8. data/app/controllers/wurk/dashboard_controller.rb +0 -1
  9. data/config/routes.rb +5 -1
  10. data/lib/sidekiq/cli.rb +4 -3
  11. data/lib/sidekiq/deploy.rb +2 -1
  12. data/lib/sidekiq/embedded.rb +2 -1
  13. data/lib/sidekiq/iterable_job.rb +2 -1
  14. data/lib/sidekiq/job/iterable/active_record_enumerator.rb +2 -1
  15. data/lib/sidekiq/job/iterable/csv_enumerator.rb +2 -1
  16. data/lib/sidekiq/job/iterable.rb +2 -1
  17. data/lib/sidekiq/launcher.rb +2 -1
  18. data/lib/sidekiq/manager.rb +2 -1
  19. data/lib/sidekiq/metrics/query.rb +2 -1
  20. data/lib/sidekiq/pro/web.rb +2 -1
  21. data/lib/sidekiq/scheduled.rb +2 -1
  22. data/lib/sidekiq/web.rb +2 -1
  23. data/lib/sidekiq.rb +3 -2
  24. data/lib/wurk/api/fast.rb +4 -18
  25. data/lib/wurk/api/queues.rb +12 -10
  26. data/lib/wurk/api/serializers.rb +6 -7
  27. data/lib/wurk/api/swarm.rb +0 -1
  28. data/lib/wurk/batch/server_middleware.rb +4 -0
  29. data/lib/wurk/batch/status.rb +45 -22
  30. data/lib/wurk/batch_set.rb +34 -0
  31. data/lib/wurk/cli.rb +17 -19
  32. data/lib/wurk/client/buffered.rb +97 -26
  33. data/lib/wurk/client.rb +21 -1
  34. data/lib/wurk/compat/deploy.rb +7 -0
  35. data/lib/wurk/compat/encryption.rb +7 -0
  36. data/lib/wurk/compat/flow.rb +8 -0
  37. data/lib/wurk/compat/iterable_job.rb +8 -0
  38. data/lib/wurk/compat/profile_set.rb +8 -0
  39. data/lib/wurk/compat/server.rb +15 -0
  40. data/lib/wurk/compat/web.rb +20 -0
  41. data/lib/wurk/compat.rb +34 -40
  42. data/lib/wurk/configuration.rb +46 -9
  43. data/lib/wurk/cron.rb +146 -47
  44. data/lib/wurk/dead_set.rb +10 -6
  45. data/lib/wurk/embedded.rb +0 -2
  46. data/lib/wurk/engine.rb +46 -4
  47. data/lib/wurk/error_coalescer.rb +21 -1
  48. data/lib/wurk/fetcher/many_queues.rb +8 -6
  49. data/lib/wurk/fetcher/poison_pill.rb +1 -1
  50. data/lib/wurk/fetcher/reaper.rb +0 -5
  51. data/lib/wurk/fetcher/reliable.rb +46 -27
  52. data/lib/wurk/fetcher/self_reconcile.rb +0 -1
  53. data/lib/wurk/fetcher/unit_of_work.rb +16 -1
  54. data/lib/wurk/flow/status.rb +10 -4
  55. data/lib/wurk/flow_set.rb +24 -2
  56. data/lib/wurk/health.rb +0 -1
  57. data/lib/wurk/heartbeat.rb +13 -7
  58. data/lib/wurk/import/sidekiq_cron.rb +0 -1
  59. data/lib/wurk/job_record.rb +2 -1
  60. data/lib/wurk/job_retry.rb +1 -1
  61. data/lib/wurk/job_set.rb +132 -6
  62. data/lib/wurk/launcher.rb +0 -12
  63. data/lib/wurk/lazy_loader.rb +60 -0
  64. data/lib/wurk/limiter/bucket.rb +0 -2
  65. data/lib/wurk/limiter/concurrent.rb +19 -36
  66. data/lib/wurk/limiter/leaky.rb +0 -2
  67. data/lib/wurk/limiter/points.rb +0 -2
  68. data/lib/wurk/limiter/server_middleware.rb +4 -1
  69. data/lib/wurk/limiter/unlimited.rb +0 -2
  70. data/lib/wurk/limiter/window.rb +0 -2
  71. data/lib/wurk/limiter.rb +0 -7
  72. data/lib/wurk/logger.rb +44 -5
  73. data/lib/wurk/lua/fast_delete.rb +58 -0
  74. data/lib/wurk/lua/fast_delete_window.lua +46 -0
  75. data/lib/wurk/lua/limiter_concurrent_acquire.lua +10 -3
  76. data/lib/wurk/lua/limiter_concurrent_release.lua +13 -4
  77. data/lib/wurk/lua.rb +1 -41
  78. data/lib/wurk/metrics/dashboard_series.rb +38 -18
  79. data/lib/wurk/metrics/history.rb +24 -11
  80. data/lib/wurk/metrics/prometheus.rb +0 -1
  81. data/lib/wurk/metrics/query.rb +0 -2
  82. data/lib/wurk/metrics/queue_rollup.rb +33 -11
  83. data/lib/wurk/metrics/statsd.rb +4 -0
  84. data/lib/wurk/middleware/chain.rb +25 -3
  85. data/lib/wurk/middleware/expiry.rb +4 -0
  86. data/lib/wurk/middleware/interrupt_handler.rb +4 -1
  87. data/lib/wurk/middleware/status.rb +4 -0
  88. data/lib/wurk/middleware/timeout.rb +4 -0
  89. data/lib/wurk/preflight/owners.rb +0 -1
  90. data/lib/wurk/preflight/payload_sample.rb +0 -1
  91. data/lib/wurk/preflight/verify.rb +0 -2
  92. data/lib/wurk/process_set.rb +6 -1
  93. data/lib/wurk/profiler.rb +2 -1
  94. data/lib/wurk/queue.rb +33 -6
  95. data/lib/wurk/queue_consumers.rb +10 -1
  96. data/lib/wurk/rails_boot.rb +2 -2
  97. data/lib/wurk/railtie.rb +3 -4
  98. data/lib/wurk/sentry/error_handler.rb +5 -0
  99. data/lib/wurk/sentry/middleware.rb +4 -0
  100. data/lib/wurk/sorted_entry.rb +16 -12
  101. data/lib/wurk/stats.rb +85 -1
  102. data/lib/wurk/swarm/child_boot.rb +0 -2
  103. data/lib/wurk/swarm/liveness.rb +0 -2
  104. data/lib/wurk/swarm/memory_watch.rb +116 -0
  105. data/lib/wurk/swarm/prefork.rb +54 -0
  106. data/lib/wurk/swarm.rb +33 -42
  107. data/lib/wurk/version.rb +1 -1
  108. data/lib/wurk/web/enterprise.rb +63 -7
  109. data/lib/wurk.rb +33 -29
  110. data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js +1 -0
  111. data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.br +0 -0
  112. data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.gz +0 -0
  113. data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js +1 -0
  114. data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.br +0 -0
  115. data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.gz +0 -0
  116. data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js +1 -0
  117. data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.br +0 -0
  118. data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.gz +0 -0
  119. data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js +1 -0
  120. data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.br +0 -0
  121. data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.gz +0 -0
  122. data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js +1 -0
  123. data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.br +0 -0
  124. data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.gz +0 -0
  125. data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js +1 -0
  126. data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.br +0 -0
  127. data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.gz +0 -0
  128. data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js +1 -0
  129. data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.br +0 -0
  130. data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.gz +0 -0
  131. data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js +1 -0
  132. data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.br +0 -0
  133. data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.gz +0 -0
  134. data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js +2 -0
  135. data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.br +0 -0
  136. data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.gz +0 -0
  137. data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js +1 -0
  138. data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.br +0 -0
  139. data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.gz +0 -0
  140. data/vendor/assets/dashboard/assets/{FlowState-yb7virqR.js → FlowState-ClzInw0-.js} +1 -1
  141. data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.br +0 -0
  142. data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.gz +0 -0
  143. data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js +1 -0
  144. data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.br +0 -0
  145. data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.gz +0 -0
  146. data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js +2 -0
  147. data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.br +0 -0
  148. data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.gz +0 -0
  149. data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js +1 -0
  150. data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.br +0 -0
  151. data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.gz +0 -0
  152. data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js +1 -0
  153. data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.br +0 -0
  154. data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.gz +0 -0
  155. data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js +1 -0
  156. data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.br +0 -0
  157. data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.gz +0 -0
  158. data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js +1 -0
  159. data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.br +0 -0
  160. data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.gz +0 -0
  161. data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js +1 -0
  162. data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.br +0 -0
  163. data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.gz +0 -0
  164. data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js +1 -0
  165. data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.br +0 -0
  166. data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.gz +0 -0
  167. data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js +1 -0
  168. data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.br +0 -0
  169. data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.gz +0 -0
  170. data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js +1 -0
  171. data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.br +0 -0
  172. data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.gz +0 -0
  173. data/vendor/assets/dashboard/assets/Search-D42Va7lB.js +1 -0
  174. data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.br +0 -0
  175. data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.gz +0 -0
  176. data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js +1 -0
  177. data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.br +0 -0
  178. data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.gz +0 -0
  179. data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js +1 -0
  180. data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.br +0 -0
  181. data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.gz +0 -0
  182. data/vendor/assets/dashboard/assets/de-B2NCIuzo.js +1 -0
  183. data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.br +0 -0
  184. data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.gz +0 -0
  185. data/vendor/assets/dashboard/assets/es-BGeu_JNB.js +1 -0
  186. data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.br +0 -0
  187. data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.gz +0 -0
  188. data/vendor/assets/dashboard/assets/fr-DQVBWFId.js +1 -0
  189. data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.br +0 -0
  190. data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.gz +0 -0
  191. data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js +1 -0
  192. data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.br +0 -0
  193. data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.gz +0 -0
  194. data/vendor/assets/dashboard/assets/index-CiLU1zz8.js +141 -0
  195. data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.br +0 -0
  196. data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.gz +0 -0
  197. data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css +1 -0
  198. data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.br +0 -0
  199. data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.gz +0 -0
  200. data/vendor/assets/dashboard/assets/ja-D06IBqzL.js +1 -0
  201. data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.br +0 -0
  202. data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.gz +0 -0
  203. data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js +1 -0
  204. data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.br +0 -0
  205. data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.gz +0 -0
  206. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js +1 -0
  207. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.br +0 -0
  208. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.gz +0 -0
  209. data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js +1 -0
  210. data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.br +0 -0
  211. data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.gz +0 -0
  212. data/vendor/assets/dashboard/assets/{utils-CnWZzW2L.js → utils-69XSoGbQ.js} +1 -1
  213. data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.br +0 -0
  214. data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.gz +0 -0
  215. data/vendor/assets/dashboard/assets/wurk-logo-ydeSB9B8.png +0 -0
  216. data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js +1 -0
  217. data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.br +0 -0
  218. data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.gz +0 -0
  219. data/vendor/assets/dashboard/index.html +3 -23
  220. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  221. metadata +122 -37
  222. data/vendor/assets/dashboard/assets/ArgsValue-BJol9S0s.js +0 -1
  223. data/vendor/assets/dashboard/assets/BatchDetail-D_zGQ9Sd.js +0 -1
  224. data/vendor/assets/dashboard/assets/Batches-Dt_xC660.js +0 -1
  225. data/vendor/assets/dashboard/assets/Busy-CWRthsmc.js +0 -1
  226. data/vendor/assets/dashboard/assets/Cron-qeGr2YAm.js +0 -1
  227. data/vendor/assets/dashboard/assets/Dashboard-DQWaAwPA.js +0 -1
  228. data/vendor/assets/dashboard/assets/Dead-D9gOPHtK.js +0 -1
  229. data/vendor/assets/dashboard/assets/Extension-DONXVLdA.js +0 -1
  230. data/vendor/assets/dashboard/assets/FilterBox-BbB6gXHp.js +0 -2
  231. data/vendor/assets/dashboard/assets/FlowDetail-CrmnVRp0.js +0 -1
  232. data/vendor/assets/dashboard/assets/Flows-g4GC8KmE.js +0 -1
  233. data/vendor/assets/dashboard/assets/JobDetailModal-C86k_1wj.js +0 -2
  234. data/vendor/assets/dashboard/assets/Limiters-zyO-iMlK.js +0 -1
  235. data/vendor/assets/dashboard/assets/Metrics-Ckq5fNBv.js +0 -1
  236. data/vendor/assets/dashboard/assets/NotFound-C8NnyjlY.js +0 -1
  237. data/vendor/assets/dashboard/assets/PageHeader-vWeBbSPF.js +0 -1
  238. data/vendor/assets/dashboard/assets/Profiles-DRl-FXPA.js +0 -1
  239. data/vendor/assets/dashboard/assets/Queues-DkXWrihA.js +0 -1
  240. data/vendor/assets/dashboard/assets/Retries-CmlojYIy.js +0 -1
  241. data/vendor/assets/dashboard/assets/Scheduled-Cbbpp3xw.js +0 -1
  242. data/vendor/assets/dashboard/assets/Search-LaEpj2fM.js +0 -1
  243. data/vendor/assets/dashboard/assets/charts-D_PQLoyc.js +0 -1
  244. data/vendor/assets/dashboard/assets/fa-brands-400-Bs6tcqqs.woff2 +0 -0
  245. data/vendor/assets/dashboard/assets/fa-regular-400-DRN8N0d1.woff2 +0 -0
  246. data/vendor/assets/dashboard/assets/fa-solid-900-IAB4Droh.woff2 +0 -0
  247. data/vendor/assets/dashboard/assets/fa-v4compatibility-CErXDOsT.woff2 +0 -0
  248. data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +0 -1
  249. data/vendor/assets/dashboard/assets/index--xvvNlYx.css +0 -1
  250. data/vendor/assets/dashboard/assets/index-FBKGtysy.js +0 -141
  251. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-BiwhHqgh.js +0 -1
  252. data/vendor/assets/dashboard/assets/useSort-jBYVwXvg.js +0 -1
  253. data/vendor/assets/dashboard/assets/wurk-logo-Vy3xW4K0.png +0 -0
  254. data/vendor/assets/dashboard/og-image.png +0 -0
  255. data/vendor/assets/dashboard/wurk-logo.webp +0 -0
@@ -18,11 +18,19 @@ module Wurk
18
18
 
19
19
  attr_reader :bid
20
20
 
21
- def initialize(bid)
21
+ # `data:` is the `b-<bid>` hash already read, by a listing that fetched a
22
+ # page of them in one pipeline (BatchSet#snapshot); without it the hash
23
+ # is read here.
24
+ def initialize(bid, data: nil)
22
25
  raise ArgumentError, 'bid required' if bid.nil? || bid.to_s.empty?
23
26
 
24
27
  @bid = bid.to_s
25
- reload!
28
+ data ? self.data_hash = data : reload!
29
+ end
30
+
31
+ # The four set sizes #summary takes, as one pipeline's commands.
32
+ def self.summary_commands(bid)
33
+ %w[failed died kids jids].map { |set| ['SCARD', "b-#{bid}-#{set}"] }
26
34
  end
27
35
 
28
36
  # False when no `b-<bid>` hash exists — a well-formed bid that was never
@@ -87,24 +95,18 @@ module Wurk
87
95
  # set reads it needs ride one pipeline.
88
96
  def data
89
97
  failed, dead, kids, live = set_snapshot
90
- {
91
- 'bid' => @bid,
92
- 'total' => total,
93
- 'pending' => pending,
94
- 'failures' => failures,
95
- 'created_at' => created_at,
96
- 'complete_at' => complete_at,
97
- 'success_at' => success_at,
98
- 'death_at' => death_at,
99
- 'complete' => complete_with?(live.to_i),
100
- 'invalidated' => invalidated?,
101
- 'description' => description,
102
- 'parent_bid' => parent_bid,
103
- 'tags' => tags,
104
- 'failed_jids' => failed,
105
- 'dead_jids' => dead,
106
- 'child_count' => kids.to_i
107
- }
98
+ header(live).merge('failed_jids' => failed, 'dead_jids' => dead, 'child_count' => kids.to_i)
99
+ end
100
+
101
+ # #data for a listing row: the failed and dead jid sets by size rather
102
+ # than by member, so a page of batches with ten thousand failures each
103
+ # is 25 rows and not 250,000 jids. `sizes` are the replies to
104
+ # .summary_commands when the caller pipelined them for a whole page.
105
+ def summary(sizes = nil)
106
+ failed, dead, kids, live = sizes || Wurk.redis do |conn|
107
+ conn.pipelined { |pipe| Status.summary_commands(@bid).each { |cmd| pipe.call(*cmd) } }
108
+ end
109
+ header(live).merge('failed_count' => failed.to_i, 'dead_count' => dead.to_i, 'child_count' => kids.to_i)
108
110
  end
109
111
 
110
112
  # Blocks the current thread until `complete?` is true. Test/util only —
@@ -139,13 +141,34 @@ module Wurk
139
141
  end
140
142
 
141
143
  def reload!
142
- raw = Wurk.redis { |conn| conn.call('HGETALL', "b-#{@bid}") }
143
- @data = raw.is_a?(Hash) ? raw : raw.each_slice(2).to_h
144
+ self.data_hash = Wurk.redis { |conn| conn.call('HGETALL', "b-#{@bid}") }
144
145
  self
145
146
  end
146
147
 
147
148
  private
148
149
 
150
+ def header(live)
151
+ {
152
+ 'bid' => @bid,
153
+ 'total' => total,
154
+ 'pending' => pending,
155
+ 'failures' => failures,
156
+ 'created_at' => created_at,
157
+ 'complete_at' => complete_at,
158
+ 'success_at' => success_at,
159
+ 'death_at' => death_at,
160
+ 'complete' => complete_with?(live.to_i),
161
+ 'invalidated' => invalidated?,
162
+ 'description' => description,
163
+ 'parent_bid' => parent_bid,
164
+ 'tags' => tags
165
+ }
166
+ end
167
+
168
+ def data_hash=(raw)
169
+ @data = raw.is_a?(Hash) ? raw : raw.each_slice(2).to_h
170
+ end
171
+
149
172
  # [failed jids, dead jids, child count, live count] in one round trip.
150
173
  def set_snapshot
151
174
  Wurk.redis do |conn|
@@ -38,6 +38,20 @@ module Wurk
38
38
  end
39
39
  end
40
40
 
41
+ # The set's size and the #each-ordered page of `count` batches from
42
+ # `offset`, each as its Status#summary: the page's ids in one round trip
43
+ # and every row's hash and set sizes in a second, where #each pays a round
44
+ # trip per batch from the top of the set down. Internal reader for the
45
+ # dashboard.
46
+ #
47
+ # @return [Array(Integer, Array<Hash>)]
48
+ def snapshot(offset, count)
49
+ Wurk.redis do |conn|
50
+ size, bids = page_ids(conn, offset, count)
51
+ [size.to_i, summaries(conn, bids)]
52
+ end
53
+ end
54
+
41
55
  # Yields each BID indexed under the given tag (set at `tags:<tag>`).
42
56
  # No JSON parsing; cheap iteration. Caller wraps in `Status.new(bid)`
43
57
  # if it needs full state.
@@ -53,6 +67,26 @@ module Wurk
53
67
  end
54
68
  end
55
69
  end
70
+
71
+ private
72
+
73
+ # ZCARD and one newest-first page of ids, in one round trip.
74
+ def page_ids(conn, offset, count)
75
+ conn.pipelined do |pipe|
76
+ pipe.call('ZCARD', @key)
77
+ pipe.call('ZRANGE', @key, offset, offset + count - 1, 'REV')
78
+ end
79
+ end
80
+
81
+ def summaries(conn, bids)
82
+ return [] if bids.empty?
83
+
84
+ commands = bids.map { |bid| [['HGETALL', "b-#{bid}"], *Wurk::Batch::Status.summary_commands(bid)] }
85
+ replies = conn.pipelined { |pipe| commands.flatten(1).each { |cmd| pipe.call(*cmd) } }
86
+ bids.zip(replies.each_slice(commands.first.size)).map do |bid, (hash, *sizes)|
87
+ Wurk::Batch::Status.new(bid, data: hash).summary(sizes)
88
+ end
89
+ end
56
90
  end
57
91
 
58
92
  class Batch
data/lib/wurk/cli.rb CHANGED
@@ -11,8 +11,6 @@ require_relative 'version'
11
11
  require_relative 'configuration'
12
12
  require_relative 'component'
13
13
  require_relative 'capsule'
14
- require_relative 'launcher'
15
- require_relative 'rails_boot'
16
14
  require_relative 'boot_checks'
17
15
 
18
16
  module Wurk
@@ -204,19 +202,17 @@ module Wurk
204
202
  # Mark server mode BEFORE the app loads so `configure_server` blocks in
205
203
  # the required initializer actually fire (they gate on `config.server?`).
206
204
  # Matches Sidekiq, which sets `Sidekiq.server = true` before requiring
207
- # the app in its CLI. Skipping this drops server middleware, error
208
- # handlers, and lifecycle hooks registered via configure_server, and each
209
- # skip logs one WARN naming the gate (`configure_server` / `server? is
210
- # false`).
205
+ # the app in its CLI. Each configure_server block the app's load reaches
206
+ # before it would be kept and run only here, after the app's own code.
211
207
  enter_server_mode
212
208
  boot_application if boot_app
213
209
  Swarm::TestModeGuard.enforce!(@config.logger)
214
210
  self_read, self_write = ::IO.pipe
215
211
  begin
216
212
  trap_signals(self_write)
217
- validate_redis!
213
+ info = validate_redis!
218
214
  validate_pool_sizes!
219
- BootChecks.run(@config, redis_info)
215
+ BootChecks.run(@config, info)
220
216
  BootChecks.warn_on_ignored_topology(@config, 'wurk')
221
217
  BootChecks.log_process_role(@config, 'wurk')
222
218
  @config[:identity] = identity
@@ -243,9 +239,9 @@ module Wurk
243
239
  # This is the only way to get fork-based parallelism without Rails — the
244
240
  # railtie auto-boot path is Rails-only. Honors the swarm preload knobs
245
241
  # (`WURK_PRELOAD`/`SIDEKIQ_PRELOAD` Bundler groups, `WURK_PRELOAD_APP`/
246
- # `SIDEKIQ_PRELOAD_APP` whole-app eager-load) and boots `Process.warmup`
247
- # before the fork so children share warmed pages (copy-on-write). Spec §7.
248
- def run_swarm(boot_app: true, warmup: true) # rubocop:disable Metrics/AbcSize
242
+ # `SIDEKIQ_PRELOAD_APP` whole-app eager-load); Swarm#boot warms the heap
243
+ # before the fork so children share its pages (Swarm::Prefork). Spec §7.
244
+ def run_swarm(boot_app: true) # rubocop:disable Metrics/AbcSize
249
245
  raise StartupError, "the swarm runner takes no subcommand; run `wurk #{@command}` instead" if @command
250
246
 
251
247
  # Server mode before the app loads — see #run. The flag rides through the
@@ -256,12 +252,11 @@ module Wurk
256
252
  # crash-looping the fleet until the crash-loop exit.
257
253
  Health.options_from_env
258
254
  boot_swarm_application if boot_app
259
- validate_redis!
255
+ info = validate_redis!
260
256
  validate_pool_sizes!
261
- BootChecks.run(@config, redis_info)
257
+ BootChecks.run(@config, info)
262
258
  log_swarm_role
263
259
  @config[:identity] = identity
264
- warm_up_process if warmup
265
260
  @swarm = Wurk::Swarm.new(topology: @config.topology, config: @config, exit_on_crash_loop: true,
266
261
  shutdown_timeout: @config[:timeout] || Swarm::DEFAULT_SHUTDOWN_TIMEOUT)
267
262
  begin
@@ -352,9 +347,9 @@ module Wurk
352
347
  Wurk.enter_server_mode(@config)
353
348
  end
354
349
 
355
- # Pre-fault and compact the heap before workers start — in `run_swarm` that
356
- # happens before the fork, so children share the warmed pages copy-on-write.
357
- # RUBY_DISABLE_WARMUP=1 is the escape hatch for hosts that already warmed up.
350
+ # Pre-fault and compact the heap before workers start (the swarm does its
351
+ # own, before the fork: Swarm::Prefork). RUBY_DISABLE_WARMUP=1 is the escape
352
+ # hatch for hosts that already warmed up.
358
353
  def warm_up_process
359
354
  return unless ::Process.respond_to?(:warmup) && ENV['RUBY_DISABLE_WARMUP'] != '1'
360
355
 
@@ -436,6 +431,8 @@ module Wurk
436
431
  warn "wurk: USR2 log reopen failed, still writing to the previous log: #{e.class}: #{e.message}"
437
432
  end
438
433
 
434
+ # Returns the INFO it read, so a boot pays one round trip for the version
435
+ # check and BootChecks together.
439
436
  def validate_redis!
440
437
  info = redis_info
441
438
  ver = ::Gem::Version.new(info['redis_version'])
@@ -444,7 +441,7 @@ module Wurk
444
441
  end
445
442
 
446
443
  maxmemory_policy = info['maxmemory_policy']
447
- return if maxmemory_policy.nil? || maxmemory_policy.empty? || maxmemory_policy == 'noeviction'
444
+ return info if maxmemory_policy.nil? || maxmemory_policy.empty? || maxmemory_policy == 'noeviction'
448
445
 
449
446
  logger.warn { <<~WARN }
450
447
 
@@ -453,6 +450,7 @@ module Wurk
453
450
  The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
454
451
 
455
452
  WARN
453
+ info
456
454
  end
457
455
 
458
456
  def redis_info
@@ -496,6 +494,7 @@ module Wurk
496
494
 
497
495
  # What a worker loads only later: every constant, and the routes.
498
496
  def boot_check_reach
497
+ Wurk.eager_load!
499
498
  eager_load_rails
500
499
  draw_routes
501
500
  end
@@ -504,7 +503,6 @@ module Wurk
504
503
  return unless @config[:boot_check_redis]
505
504
 
506
505
  validate_redis!
507
- redis_info
508
506
  end
509
507
 
510
508
  def log_swarm_role
@@ -58,7 +58,9 @@ module Wurk
58
58
  # Returned by the append helpers when everything fit.
59
59
  NOTHING_UNDELIVERED = [].freeze
60
60
 
61
- Entry = Struct.new(:payload, :origin)
61
+ # `bytes` is the payload's JSON size, what `reliable_push_buffer_bytes`
62
+ # counts.
63
+ Entry = Struct.new(:payload, :origin, :bytes)
62
64
 
63
65
  # Where a buffered payload has to be replayed. What must NOT be captured
64
66
  # is a pool object the config hands out: `reset_redis_pools!` — every
@@ -159,6 +161,7 @@ module Wurk
159
161
  @install_mutex = Mutex.new
160
162
  @buffer_mutex = Mutex.new
161
163
  @replaying = 0
164
+ @buffer_bytes = 0
162
165
  @circuit = Circuit.new
163
166
 
164
167
  # Process that owns the state above; a mismatch means we're running in a
@@ -167,7 +170,7 @@ module Wurk
167
170
 
168
171
  class << self
169
172
  # Idempotent. Prepends the wrapper module into Wurk::Client so push /
170
- # push_bulk drain the buffer before each call and raw_push catches
173
+ # push_bulk / push_prepared drain the buffer before each call and raw_push catches
171
174
  # connection errors. Safe to call from multiple threads.
172
175
  def install!
173
176
  install_mutex.synchronize do
@@ -195,10 +198,30 @@ module Wurk
195
198
  @buffer_cap = value
196
199
  end
197
200
 
201
+ # Optional second cap, in bytes of payload JSON (Wurk-only; nil, the
202
+ # default, leaves only the entry cap). The entry cap alone lets a burst
203
+ # of large payloads hold `cap x payload size` in a web process for the
204
+ # whole outage. Over either cap the overflow mode applies; a payload
205
+ # larger than this cap on its own never fits, so it is the one dropped
206
+ # (or raised).
207
+ attr_reader :buffer_bytes_cap
208
+
209
+ def buffer_bytes_cap=(value)
210
+ unless value.nil? || (value.is_a?(Integer) && value.positive?)
211
+ raise ArgumentError, 'reliable_push_buffer_bytes must be nil or a positive Integer'
212
+ end
213
+
214
+ @buffer_bytes_cap = value
215
+ end
216
+
198
217
  def buffer_size
199
218
  buffer_mutex.synchronize { buffer.size }
200
219
  end
201
220
 
221
+ def buffer_bytes
222
+ buffer_mutex.synchronize { @buffer_bytes }
223
+ end
224
+
202
225
  def overflow_mode
203
226
  @overflow_mode ||= DEFAULT_OVERFLOW_MODE
204
227
  end
@@ -220,7 +243,9 @@ module Wurk
220
243
  def reset!
221
244
  buffer_mutex.synchronize do
222
245
  @buffer = []
246
+ @buffer_bytes = 0
223
247
  @buffer_cap = nil
248
+ @buffer_bytes_cap = nil
224
249
  @overflow_mode = nil
225
250
  @replaying = 0
226
251
  end
@@ -263,6 +288,7 @@ module Wurk
263
288
  @install_mutex = Mutex.new
264
289
  @buffer_mutex = Mutex.new
265
290
  @buffer = []
291
+ @buffer_bytes = 0
266
292
  @replaying = 0
267
293
  @circuit = Circuit.new
268
294
  @drops = DropLog.new
@@ -283,29 +309,34 @@ module Wurk
283
309
  # `client` is the one whose push failed; every payload is tagged with
284
310
  # the Redis that client was writing to (see Origin).
285
311
  def enbuffer(payloads, client:)
286
- origin = Origin.for(client)
287
- entries = payloads.map { |payload| Entry.new(payload, origin) }
288
-
289
- cap = buffer_cap
312
+ entries = entries_for(payloads, client)
313
+ caps = [buffer_cap, buffer_bytes_cap]
290
314
  mode = overflow_mode
291
315
  undelivered = buffer_mutex.synchronize do
292
- mode == :raise ? append_within_capacity(entries, cap) : append_dropping_oldest(entries, cap)
316
+ mode == :raise ? append_within_capacity(entries, *caps) : append_dropping_oldest(entries, *caps)
293
317
  end
294
318
 
295
319
  raise Overflow, undelivered.map(&:payload) unless undelivered.empty?
296
320
  ensure
297
- drops.report(cap)
321
+ drops.report(*caps)
298
322
  end
299
323
 
300
324
  private
301
325
 
326
+ def entries_for(payloads, client)
327
+ origin = Origin.for(client)
328
+ payloads.map { |payload| Entry.new(payload, origin, Wurk.dump_json(payload).bytesize) }
329
+ end
330
+
302
331
  # Both append helpers run with buffer_mutex held and return the entries
303
332
  # they could not take.
304
333
 
305
- def append_dropping_oldest(entries, cap)
334
+ def append_dropping_oldest(entries, cap, bytes_cap)
306
335
  entries.each do |entry|
307
- drops.record(buffer.shift.payload) if buffer.size >= cap
308
- buffer << entry
336
+ next drops.record(entry.payload) if bytes_cap && entry.bytes > bytes_cap
337
+
338
+ drops.record(take_oldest.payload) until fits?(entry, cap, bytes_cap)
339
+ push_entry(entry)
309
340
  end
310
341
  NOTHING_UNDELIVERED
311
342
  end
@@ -316,15 +347,38 @@ module Wurk
316
347
  # negative when the cap was lowered after the buffer filled — clamped,
317
348
  # so an over-full buffer rejects the whole call instead of raising on
318
349
  # `first`/`drop`.
319
- def append_within_capacity(entries, cap)
350
+ def append_within_capacity(entries, cap, bytes_cap)
320
351
  room = (cap - buffer.size).clamp(0, entries.size)
321
- if room == entries.size
322
- buffer.concat(entries)
323
- return NOTHING_UNDELIVERED
324
- end
352
+ room = bytes_room(entries.first(room), bytes_cap) if bytes_cap
353
+ entries.first(room).each { |entry| push_entry(entry) }
354
+ room == entries.size ? NOTHING_UNDELIVERED : entries.drop(room)
355
+ end
356
+
357
+ # How many of `entries`, in order, fit under the byte cap.
358
+ def bytes_room(entries, bytes_cap)
359
+ total = @buffer_bytes
360
+ entries.take_while { |entry| (total += entry.bytes) <= bytes_cap }.size
361
+ end
325
362
 
326
- buffer.concat(entries.first(room))
327
- entries.drop(room)
363
+ def fits?(entry, cap, bytes_cap)
364
+ buffer.size < cap && (bytes_cap.nil? || @buffer_bytes + entry.bytes <= bytes_cap)
365
+ end
366
+
367
+ # The three buffer mutations, each under buffer_mutex, each keeping the
368
+ # byte total (`@buffer_bytes`) in step.
369
+ def push_entry(entry)
370
+ @buffer_bytes += entry.bytes
371
+ buffer << entry
372
+ end
373
+
374
+ def take_oldest
375
+ remove_at(0)
376
+ end
377
+
378
+ def remove_at(index)
379
+ entry = buffer.delete_at(index)
380
+ @buffer_bytes -= entry.bytes
381
+ entry
328
382
  end
329
383
 
330
384
  public
@@ -508,7 +562,7 @@ module Wurk
508
562
  next unless index
509
563
 
510
564
  @replaying += 1
511
- buffer.delete_at(index)
565
+ remove_at(index)
512
566
  end
513
567
  end
514
568
 
@@ -550,7 +604,10 @@ module Wurk
550
604
  end
551
605
 
552
606
  def restore(entry)
553
- buffer_mutex.synchronize { buffer.unshift(entry) }
607
+ buffer_mutex.synchronize do
608
+ @buffer_bytes += entry.bytes
609
+ buffer.unshift(entry)
610
+ end
554
611
  end
555
612
 
556
613
  # Drain marks the thread so our prepended raw_push re-raises the
@@ -587,7 +644,7 @@ module Wurk
587
644
  @lock.synchronize { @pending << payload }
588
645
  end
589
646
 
590
- def report(cap)
647
+ def report(cap, bytes_cap = nil)
591
648
  dropped, announce = @lock.synchronize do
592
649
  next [nil, false] if @pending.empty?
593
650
 
@@ -599,7 +656,7 @@ module Wurk
599
656
  end
600
657
  return unless dropped
601
658
 
602
- log(dropped.first, cap) if announce
659
+ log(dropped.first, cap, bytes_cap) if announce
603
660
  dropped.size.times { count_drop }
604
661
  end
605
662
 
@@ -620,13 +677,14 @@ module Wurk
620
677
  nil
621
678
  end
622
679
 
623
- def log(first, cap)
680
+ def log(first, cap, bytes_cap)
624
681
  Wurk.configuration.logger.error do
625
- "reliable_push buffer full (cap=#{cap}): dropping the oldest buffered jobs to make room — " \
682
+ "reliable_push buffer full (cap=#{cap}#{", bytes=#{bytes_cap}" if bytes_cap}): " \
683
+ 'dropping the oldest buffered jobs to make room — ' \
626
684
  "#{first['class']} jid=#{first['jid']} is the first of this burst and will never run. " \
627
685
  'Further drops are counted in statsd jobs.dropped.push until Redis takes a replay. Raise ' \
628
- '`Wurk::Client.reliable_push_buffer`, or set `reliable_push_overflow = :raise` to handle ' \
629
- 'overflow yourself.'
686
+ '`Wurk::Client.reliable_push_buffer` (or `reliable_push_buffer_bytes`), or set ' \
687
+ '`reliable_push_overflow = :raise` to handle overflow yourself.'
630
688
  end
631
689
  rescue StandardError
632
690
  nil
@@ -742,6 +800,11 @@ module Wurk
742
800
  super
743
801
  end
744
802
 
803
+ def push_prepared(payloads)
804
+ Buffered.drain!(self) unless Buffered.unbuffered?
805
+ super
806
+ end
807
+
745
808
  private
746
809
 
747
810
  # Opens the delivery ledger Client writes into. It lives here rather
@@ -851,6 +914,14 @@ module Wurk
851
914
  Buffered.buffer_cap = value
852
915
  end
853
916
 
917
+ def reliable_push_buffer_bytes
918
+ Buffered.buffer_bytes_cap
919
+ end
920
+
921
+ def reliable_push_buffer_bytes=(value)
922
+ Buffered.buffer_bytes_cap = value
923
+ end
924
+
854
925
  def reliable_push_overflow
855
926
  Buffered.overflow_mode
856
927
  end
data/lib/wurk/client.rb CHANGED
@@ -1,6 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require_relative 'iterable_job'
4
3
  require_relative 'job_util'
5
4
  require_relative 'lua'
6
5
  require_relative 'status'
@@ -90,6 +89,27 @@ module Wurk
90
89
  payload['jid']
91
90
  end
92
91
 
92
+ # #push in two halves, for a caller that pushes many jobs in one write
93
+ # (JobSet#retry_all): #prepare_push runs one item through normalize →
94
+ # middleware → verify, #push_prepared writes what came back. Internal.
95
+ #
96
+ # @return [Hash, nil] the payload to write; nil when middleware halted it.
97
+ def prepare_push(item)
98
+ payload = invoke_chain(normalize_item(item))
99
+ verify_json(payload) if payload
100
+ payload
101
+ end
102
+
103
+ # One #raw_push for every payload, so they should share a queue, a batch
104
+ # or none, and a schedule or none: anything else is several writes, and one
105
+ # can land before another fails.
106
+ # @return [Array<String>] jids
107
+ def push_prepared(payloads)
108
+ buffered = raw_push(payloads)
109
+ emit_enqueued(payloads, buffered)
110
+ payloads.map { |payload| payload['jid'] }
111
+ end
112
+
93
113
  # @param items [Hash] String keys only: class, args (Array<Array>), at?, spread_interval?, batch_size?, jid?
94
114
  # @return [Array<String, nil>] jids in submission order; nil entries mark middleware-halted jobs.
95
115
  def push_bulk(items)
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Autoload target of `Sidekiq::Deploy` (compat.rb).
4
+
5
+ module Sidekiq
6
+ Deploy = Wurk::Deploy
7
+ end
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Autoload target of `Sidekiq::Encryption` (compat.rb).
4
+
5
+ module Sidekiq
6
+ Encryption = Wurk::Encryption
7
+ end
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Autoload target of Flow's Sidekiq aliases (compat.rb).
4
+
5
+ module Sidekiq
6
+ Flow = Wurk::Flow
7
+ FlowSet = Wurk::FlowSet
8
+ end
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Autoload target of IterableJob's Sidekiq aliases (compat.rb).
4
+
5
+ module Sidekiq
6
+ IterableJob = Wurk::IterableJob
7
+ IterableJobQuery = Wurk::IterableJobQuery
8
+ end
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Autoload target of the profile store's Sidekiq aliases (compat.rb).
4
+
5
+ module Sidekiq
6
+ ProfileSet = Wurk::ProfileSet
7
+ ProfileRecord = Wurk::ProfileRecord
8
+ end
@@ -0,0 +1,15 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Autoload target of the server tier's Sidekiq aliases (compat.rb). The Wurk
4
+ # files define only their Wurk names; this assigns the Sidekiq ones.
5
+
6
+ module Sidekiq
7
+ CLI = Wurk::CLI
8
+ Embedded = Wurk::Embedded
9
+ Launcher = Wurk::Launcher
10
+ Manager = Wurk::Manager
11
+ Scheduled = Wurk::Scheduled
12
+ History = Wurk::History
13
+ Periodic = Wurk::Cron
14
+ CronParser = Wurk::Cron::Parser
15
+ end
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Autoload target of the dashboard's Sidekiq aliases (compat.rb).
4
+
5
+ module Sidekiq
6
+ Web = Wurk::Web
7
+ WebHelpers = Wurk::Web::Extension::Helpers
8
+ Paginator = Wurk::Web::Paginator
9
+
10
+ module Pro
11
+ # Sidekiq Pro mounts the dashboard at `Sidekiq::Pro::Web`; it is the same
12
+ # board as `Sidekiq::Web` here, so a Pro app's `mount Sidekiq::Pro::Web`
13
+ # drops in unchanged.
14
+ Web = Wurk::Web
15
+
16
+ # `use Sidekiq::Pro::BatchStatus` — the polling Rack middleware that serves
17
+ # GET /batch_status/<bid>.json. Spec: docs/target/sidekiq-pro.md §10.3.
18
+ BatchStatus = Wurk::Web::BatchStatus
19
+ end
20
+ end