wurk 1.12.0 → 1.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (273) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -2
  3. data/app/controllers/concerns/wurk/sse_streaming.rb +36 -1
  4. data/app/controllers/wurk/api/pagination.rb +9 -7
  5. data/app/controllers/wurk/api/serializers.rb +8 -2
  6. data/app/controllers/wurk/api_controller.rb +57 -24
  7. data/app/controllers/wurk/application_controller.rb +0 -2
  8. data/app/controllers/wurk/dashboard_controller.rb +0 -1
  9. data/config/routes.rb +13 -1
  10. data/exe/sidekiq +8 -0
  11. data/exe/wurk +1 -1
  12. data/lib/generators/wurk/install/install_generator.rb +56 -0
  13. data/lib/sidekiq/cli.rb +4 -3
  14. data/lib/sidekiq/deploy.rb +2 -1
  15. data/lib/sidekiq/embedded.rb +2 -1
  16. data/lib/sidekiq/iterable_job.rb +2 -1
  17. data/lib/sidekiq/job/iterable/active_record_enumerator.rb +2 -1
  18. data/lib/sidekiq/job/iterable/csv_enumerator.rb +2 -1
  19. data/lib/sidekiq/job/iterable.rb +2 -1
  20. data/lib/sidekiq/launcher.rb +2 -1
  21. data/lib/sidekiq/manager.rb +2 -1
  22. data/lib/sidekiq/metrics/query.rb +2 -1
  23. data/lib/sidekiq/pro/web.rb +2 -1
  24. data/lib/sidekiq/scheduled.rb +2 -1
  25. data/lib/sidekiq/web.rb +2 -1
  26. data/lib/sidekiq.rb +3 -2
  27. data/lib/wurk/api/fast.rb +4 -18
  28. data/lib/wurk/api/queues.rb +12 -10
  29. data/lib/wurk/api/serializers.rb +6 -7
  30. data/lib/wurk/api/swarm.rb +0 -1
  31. data/lib/wurk/batch/server_middleware.rb +4 -0
  32. data/lib/wurk/batch/status.rb +45 -22
  33. data/lib/wurk/batch_set.rb +34 -0
  34. data/lib/wurk/boot_checks.rb +80 -0
  35. data/lib/wurk/cli.rb +72 -36
  36. data/lib/wurk/client/buffered.rb +97 -26
  37. data/lib/wurk/client.rb +21 -1
  38. data/lib/wurk/compat/deploy.rb +7 -0
  39. data/lib/wurk/compat/encryption.rb +7 -0
  40. data/lib/wurk/compat/flow.rb +8 -0
  41. data/lib/wurk/compat/iterable_job.rb +8 -0
  42. data/lib/wurk/compat/profile_set.rb +8 -0
  43. data/lib/wurk/compat/server.rb +15 -0
  44. data/lib/wurk/compat/web.rb +20 -0
  45. data/lib/wurk/compat.rb +34 -40
  46. data/lib/wurk/configuration.rb +73 -15
  47. data/lib/wurk/cron.rb +146 -47
  48. data/lib/wurk/dead_set.rb +10 -6
  49. data/lib/wurk/embedded.rb +0 -2
  50. data/lib/wurk/engine.rb +46 -4
  51. data/lib/wurk/error_coalescer.rb +21 -1
  52. data/lib/wurk/fetcher/many_queues.rb +8 -6
  53. data/lib/wurk/fetcher/orphan_reclaim.rb +48 -11
  54. data/lib/wurk/{middleware → fetcher}/poison_pill.rb +4 -2
  55. data/lib/wurk/fetcher/reaper.rb +28 -14
  56. data/lib/wurk/fetcher/reliable.rb +50 -32
  57. data/lib/wurk/fetcher/self_reconcile.rb +0 -1
  58. data/lib/wurk/fetcher/unit_of_work.rb +17 -2
  59. data/lib/wurk/flow/status.rb +10 -4
  60. data/lib/wurk/flow_set.rb +24 -2
  61. data/lib/wurk/health.rb +0 -1
  62. data/lib/wurk/heartbeat.rb +13 -7
  63. data/lib/wurk/import/sidekiq_cron.rb +0 -1
  64. data/lib/wurk/job_logger.rb +4 -2
  65. data/lib/wurk/job_record.rb +2 -1
  66. data/lib/wurk/job_retry.rb +1 -1
  67. data/lib/wurk/job_set.rb +132 -6
  68. data/lib/wurk/job_util.rb +14 -0
  69. data/lib/wurk/launcher.rb +30 -15
  70. data/lib/wurk/lazy_loader.rb +60 -0
  71. data/lib/wurk/leader.rb +1 -8
  72. data/lib/wurk/limiter/bucket.rb +0 -2
  73. data/lib/wurk/limiter/concurrent.rb +19 -36
  74. data/lib/wurk/limiter/leaky.rb +0 -2
  75. data/lib/wurk/limiter/points.rb +0 -2
  76. data/lib/wurk/limiter/server_middleware.rb +4 -1
  77. data/lib/wurk/limiter/unlimited.rb +0 -2
  78. data/lib/wurk/limiter/window.rb +0 -2
  79. data/lib/wurk/limiter.rb +0 -7
  80. data/lib/wurk/logger.rb +44 -5
  81. data/lib/wurk/lua/fast_delete.rb +58 -0
  82. data/lib/wurk/lua/fast_delete_window.lua +46 -0
  83. data/lib/wurk/lua/limiter_concurrent_acquire.lua +10 -3
  84. data/lib/wurk/lua/limiter_concurrent_release.lua +13 -4
  85. data/lib/wurk/lua.rb +1 -41
  86. data/lib/wurk/manager.rb +27 -1
  87. data/lib/wurk/metrics/dashboard_series.rb +38 -18
  88. data/lib/wurk/metrics/history.rb +24 -11
  89. data/lib/wurk/metrics/prometheus.rb +0 -1
  90. data/lib/wurk/metrics/query.rb +0 -2
  91. data/lib/wurk/metrics/queue_rollup.rb +33 -11
  92. data/lib/wurk/metrics/statsd.rb +4 -0
  93. data/lib/wurk/middleware/chain.rb +25 -3
  94. data/lib/wurk/middleware/expiry.rb +4 -0
  95. data/lib/wurk/middleware/interrupt_handler.rb +4 -1
  96. data/lib/wurk/middleware/status.rb +4 -0
  97. data/lib/wurk/middleware/timeout.rb +10 -6
  98. data/lib/wurk/preflight/app_scan.rb +182 -0
  99. data/lib/wurk/preflight/command.rb +21 -5
  100. data/lib/wurk/preflight/owners.rb +0 -1
  101. data/lib/wurk/preflight/payload_sample.rb +0 -1
  102. data/lib/wurk/preflight/redis_guard.rb +98 -0
  103. data/lib/wurk/preflight/verify.rb +0 -2
  104. data/lib/wurk/preflight.rb +9 -5
  105. data/lib/wurk/process_set.rb +6 -1
  106. data/lib/wurk/profiler.rb +2 -1
  107. data/lib/wurk/queue.rb +33 -6
  108. data/lib/wurk/queue_consumers.rb +10 -1
  109. data/lib/wurk/rails_boot.rb +9 -4
  110. data/lib/wurk/railtie.rb +3 -4
  111. data/lib/wurk/redis_options.rb +5 -0
  112. data/lib/wurk/redis_pool.rb +24 -1
  113. data/lib/wurk/sentry/error_handler.rb +5 -0
  114. data/lib/wurk/sentry/middleware.rb +4 -0
  115. data/lib/wurk/sentry.rb +11 -0
  116. data/lib/wurk/sorted_entry.rb +16 -12
  117. data/lib/wurk/stats.rb +85 -1
  118. data/lib/wurk/swarm/child_boot.rb +0 -2
  119. data/lib/wurk/swarm/liveness.rb +0 -2
  120. data/lib/wurk/swarm/memory_watch.rb +116 -0
  121. data/lib/wurk/swarm/prefork.rb +54 -0
  122. data/lib/wurk/swarm.rb +95 -55
  123. data/lib/wurk/version.rb +1 -1
  124. data/lib/wurk/web/enterprise.rb +63 -7
  125. data/lib/wurk/web/rack_app.rb +8 -6
  126. data/lib/wurk/web/stats_routes.rb +93 -0
  127. data/lib/wurk.rb +34 -30
  128. data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js +1 -0
  129. data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.br +0 -0
  130. data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.gz +0 -0
  131. data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js +1 -0
  132. data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.br +0 -0
  133. data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.gz +0 -0
  134. data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js +1 -0
  135. data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.br +0 -0
  136. data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.gz +0 -0
  137. data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js +1 -0
  138. data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.br +0 -0
  139. data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.gz +0 -0
  140. data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js +1 -0
  141. data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.br +0 -0
  142. data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.gz +0 -0
  143. data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js +1 -0
  144. data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.br +0 -0
  145. data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.gz +0 -0
  146. data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js +1 -0
  147. data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.br +0 -0
  148. data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.gz +0 -0
  149. data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js +1 -0
  150. data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.br +0 -0
  151. data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.gz +0 -0
  152. data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js +2 -0
  153. data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.br +0 -0
  154. data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.gz +0 -0
  155. data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js +1 -0
  156. data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.br +0 -0
  157. data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.gz +0 -0
  158. data/vendor/assets/dashboard/assets/{FlowState-yb7virqR.js → FlowState-ClzInw0-.js} +1 -1
  159. data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.br +0 -0
  160. data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.gz +0 -0
  161. data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js +1 -0
  162. data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.br +0 -0
  163. data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.gz +0 -0
  164. data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js +2 -0
  165. data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.br +0 -0
  166. data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.gz +0 -0
  167. data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js +1 -0
  168. data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.br +0 -0
  169. data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.gz +0 -0
  170. data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js +1 -0
  171. data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.br +0 -0
  172. data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.gz +0 -0
  173. data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js +1 -0
  174. data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.br +0 -0
  175. data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.gz +0 -0
  176. data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js +1 -0
  177. data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.br +0 -0
  178. data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.gz +0 -0
  179. data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js +1 -0
  180. data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.br +0 -0
  181. data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.gz +0 -0
  182. data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js +1 -0
  183. data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.br +0 -0
  184. data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.gz +0 -0
  185. data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js +1 -0
  186. data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.br +0 -0
  187. data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.gz +0 -0
  188. data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js +1 -0
  189. data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.br +0 -0
  190. data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.gz +0 -0
  191. data/vendor/assets/dashboard/assets/Search-D42Va7lB.js +1 -0
  192. data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.br +0 -0
  193. data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.gz +0 -0
  194. data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js +1 -0
  195. data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.br +0 -0
  196. data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.gz +0 -0
  197. data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js +1 -0
  198. data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.br +0 -0
  199. data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.gz +0 -0
  200. data/vendor/assets/dashboard/assets/de-B2NCIuzo.js +1 -0
  201. data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.br +0 -0
  202. data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.gz +0 -0
  203. data/vendor/assets/dashboard/assets/es-BGeu_JNB.js +1 -0
  204. data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.br +0 -0
  205. data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.gz +0 -0
  206. data/vendor/assets/dashboard/assets/fr-DQVBWFId.js +1 -0
  207. data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.br +0 -0
  208. data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.gz +0 -0
  209. data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js +1 -0
  210. data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.br +0 -0
  211. data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.gz +0 -0
  212. data/vendor/assets/dashboard/assets/index-CiLU1zz8.js +141 -0
  213. data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.br +0 -0
  214. data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.gz +0 -0
  215. data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css +1 -0
  216. data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.br +0 -0
  217. data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.gz +0 -0
  218. data/vendor/assets/dashboard/assets/ja-D06IBqzL.js +1 -0
  219. data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.br +0 -0
  220. data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.gz +0 -0
  221. data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js +1 -0
  222. data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.br +0 -0
  223. data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.gz +0 -0
  224. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js +1 -0
  225. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.br +0 -0
  226. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.gz +0 -0
  227. data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js +1 -0
  228. data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.br +0 -0
  229. data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.gz +0 -0
  230. data/vendor/assets/dashboard/assets/{utils-CnWZzW2L.js → utils-69XSoGbQ.js} +1 -1
  231. data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.br +0 -0
  232. data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.gz +0 -0
  233. data/vendor/assets/dashboard/assets/wurk-logo-ydeSB9B8.png +0 -0
  234. data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js +1 -0
  235. data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.br +0 -0
  236. data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.gz +0 -0
  237. data/vendor/assets/dashboard/index.html +3 -23
  238. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  239. metadata +132 -38
  240. data/vendor/assets/dashboard/assets/ArgsValue-BJol9S0s.js +0 -1
  241. data/vendor/assets/dashboard/assets/BatchDetail-D_zGQ9Sd.js +0 -1
  242. data/vendor/assets/dashboard/assets/Batches-Dt_xC660.js +0 -1
  243. data/vendor/assets/dashboard/assets/Busy-CWRthsmc.js +0 -1
  244. data/vendor/assets/dashboard/assets/Cron-qeGr2YAm.js +0 -1
  245. data/vendor/assets/dashboard/assets/Dashboard-DQWaAwPA.js +0 -1
  246. data/vendor/assets/dashboard/assets/Dead-D9gOPHtK.js +0 -1
  247. data/vendor/assets/dashboard/assets/Extension-DONXVLdA.js +0 -1
  248. data/vendor/assets/dashboard/assets/FilterBox-BbB6gXHp.js +0 -2
  249. data/vendor/assets/dashboard/assets/FlowDetail-CrmnVRp0.js +0 -1
  250. data/vendor/assets/dashboard/assets/Flows-g4GC8KmE.js +0 -1
  251. data/vendor/assets/dashboard/assets/JobDetailModal-C86k_1wj.js +0 -2
  252. data/vendor/assets/dashboard/assets/Limiters-zyO-iMlK.js +0 -1
  253. data/vendor/assets/dashboard/assets/Metrics-Ckq5fNBv.js +0 -1
  254. data/vendor/assets/dashboard/assets/NotFound-C8NnyjlY.js +0 -1
  255. data/vendor/assets/dashboard/assets/PageHeader-vWeBbSPF.js +0 -1
  256. data/vendor/assets/dashboard/assets/Profiles-DRl-FXPA.js +0 -1
  257. data/vendor/assets/dashboard/assets/Queues-DkXWrihA.js +0 -1
  258. data/vendor/assets/dashboard/assets/Retries-CmlojYIy.js +0 -1
  259. data/vendor/assets/dashboard/assets/Scheduled-Cbbpp3xw.js +0 -1
  260. data/vendor/assets/dashboard/assets/Search-LaEpj2fM.js +0 -1
  261. data/vendor/assets/dashboard/assets/charts-D_PQLoyc.js +0 -1
  262. data/vendor/assets/dashboard/assets/fa-brands-400-Bs6tcqqs.woff2 +0 -0
  263. data/vendor/assets/dashboard/assets/fa-regular-400-DRN8N0d1.woff2 +0 -0
  264. data/vendor/assets/dashboard/assets/fa-solid-900-IAB4Droh.woff2 +0 -0
  265. data/vendor/assets/dashboard/assets/fa-v4compatibility-CErXDOsT.woff2 +0 -0
  266. data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +0 -1
  267. data/vendor/assets/dashboard/assets/index--xvvNlYx.css +0 -1
  268. data/vendor/assets/dashboard/assets/index-FBKGtysy.js +0 -141
  269. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-BiwhHqgh.js +0 -1
  270. data/vendor/assets/dashboard/assets/useSort-jBYVwXvg.js +0 -1
  271. data/vendor/assets/dashboard/assets/wurk-logo-Vy3xW4K0.png +0 -0
  272. data/vendor/assets/dashboard/og-image.png +0 -0
  273. data/vendor/assets/dashboard/wurk-logo.webp +0 -0
@@ -58,7 +58,9 @@ module Wurk
58
58
  # Returned by the append helpers when everything fit.
59
59
  NOTHING_UNDELIVERED = [].freeze
60
60
 
61
- Entry = Struct.new(:payload, :origin)
61
+ # `bytes` is the payload's JSON size, what `reliable_push_buffer_bytes`
62
+ # counts.
63
+ Entry = Struct.new(:payload, :origin, :bytes)
62
64
 
63
65
  # Where a buffered payload has to be replayed. What must NOT be captured
64
66
  # is a pool object the config hands out: `reset_redis_pools!` — every
@@ -159,6 +161,7 @@ module Wurk
159
161
  @install_mutex = Mutex.new
160
162
  @buffer_mutex = Mutex.new
161
163
  @replaying = 0
164
+ @buffer_bytes = 0
162
165
  @circuit = Circuit.new
163
166
 
164
167
  # Process that owns the state above; a mismatch means we're running in a
@@ -167,7 +170,7 @@ module Wurk
167
170
 
168
171
  class << self
169
172
  # Idempotent. Prepends the wrapper module into Wurk::Client so push /
170
- # push_bulk drain the buffer before each call and raw_push catches
173
+ # push_bulk / push_prepared drain the buffer before each call and raw_push catches
171
174
  # connection errors. Safe to call from multiple threads.
172
175
  def install!
173
176
  install_mutex.synchronize do
@@ -195,10 +198,30 @@ module Wurk
195
198
  @buffer_cap = value
196
199
  end
197
200
 
201
+ # Optional second cap, in bytes of payload JSON (Wurk-only; nil, the
202
+ # default, leaves only the entry cap). The entry cap alone lets a burst
203
+ # of large payloads hold `cap x payload size` in a web process for the
204
+ # whole outage. Over either cap the overflow mode applies; a payload
205
+ # larger than this cap on its own never fits, so it is the one dropped
206
+ # (or raised).
207
+ attr_reader :buffer_bytes_cap
208
+
209
+ def buffer_bytes_cap=(value)
210
+ unless value.nil? || (value.is_a?(Integer) && value.positive?)
211
+ raise ArgumentError, 'reliable_push_buffer_bytes must be nil or a positive Integer'
212
+ end
213
+
214
+ @buffer_bytes_cap = value
215
+ end
216
+
198
217
  def buffer_size
199
218
  buffer_mutex.synchronize { buffer.size }
200
219
  end
201
220
 
221
+ def buffer_bytes
222
+ buffer_mutex.synchronize { @buffer_bytes }
223
+ end
224
+
202
225
  def overflow_mode
203
226
  @overflow_mode ||= DEFAULT_OVERFLOW_MODE
204
227
  end
@@ -220,7 +243,9 @@ module Wurk
220
243
  def reset!
221
244
  buffer_mutex.synchronize do
222
245
  @buffer = []
246
+ @buffer_bytes = 0
223
247
  @buffer_cap = nil
248
+ @buffer_bytes_cap = nil
224
249
  @overflow_mode = nil
225
250
  @replaying = 0
226
251
  end
@@ -263,6 +288,7 @@ module Wurk
263
288
  @install_mutex = Mutex.new
264
289
  @buffer_mutex = Mutex.new
265
290
  @buffer = []
291
+ @buffer_bytes = 0
266
292
  @replaying = 0
267
293
  @circuit = Circuit.new
268
294
  @drops = DropLog.new
@@ -283,29 +309,34 @@ module Wurk
283
309
  # `client` is the one whose push failed; every payload is tagged with
284
310
  # the Redis that client was writing to (see Origin).
285
311
  def enbuffer(payloads, client:)
286
- origin = Origin.for(client)
287
- entries = payloads.map { |payload| Entry.new(payload, origin) }
288
-
289
- cap = buffer_cap
312
+ entries = entries_for(payloads, client)
313
+ caps = [buffer_cap, buffer_bytes_cap]
290
314
  mode = overflow_mode
291
315
  undelivered = buffer_mutex.synchronize do
292
- mode == :raise ? append_within_capacity(entries, cap) : append_dropping_oldest(entries, cap)
316
+ mode == :raise ? append_within_capacity(entries, *caps) : append_dropping_oldest(entries, *caps)
293
317
  end
294
318
 
295
319
  raise Overflow, undelivered.map(&:payload) unless undelivered.empty?
296
320
  ensure
297
- drops.report(cap)
321
+ drops.report(*caps)
298
322
  end
299
323
 
300
324
  private
301
325
 
326
+ def entries_for(payloads, client)
327
+ origin = Origin.for(client)
328
+ payloads.map { |payload| Entry.new(payload, origin, Wurk.dump_json(payload).bytesize) }
329
+ end
330
+
302
331
  # Both append helpers run with buffer_mutex held and return the entries
303
332
  # they could not take.
304
333
 
305
- def append_dropping_oldest(entries, cap)
334
+ def append_dropping_oldest(entries, cap, bytes_cap)
306
335
  entries.each do |entry|
307
- drops.record(buffer.shift.payload) if buffer.size >= cap
308
- buffer << entry
336
+ next drops.record(entry.payload) if bytes_cap && entry.bytes > bytes_cap
337
+
338
+ drops.record(take_oldest.payload) until fits?(entry, cap, bytes_cap)
339
+ push_entry(entry)
309
340
  end
310
341
  NOTHING_UNDELIVERED
311
342
  end
@@ -316,15 +347,38 @@ module Wurk
316
347
  # negative when the cap was lowered after the buffer filled — clamped,
317
348
  # so an over-full buffer rejects the whole call instead of raising on
318
349
  # `first`/`drop`.
319
- def append_within_capacity(entries, cap)
350
+ def append_within_capacity(entries, cap, bytes_cap)
320
351
  room = (cap - buffer.size).clamp(0, entries.size)
321
- if room == entries.size
322
- buffer.concat(entries)
323
- return NOTHING_UNDELIVERED
324
- end
352
+ room = bytes_room(entries.first(room), bytes_cap) if bytes_cap
353
+ entries.first(room).each { |entry| push_entry(entry) }
354
+ room == entries.size ? NOTHING_UNDELIVERED : entries.drop(room)
355
+ end
356
+
357
+ # How many of `entries`, in order, fit under the byte cap.
358
+ def bytes_room(entries, bytes_cap)
359
+ total = @buffer_bytes
360
+ entries.take_while { |entry| (total += entry.bytes) <= bytes_cap }.size
361
+ end
325
362
 
326
- buffer.concat(entries.first(room))
327
- entries.drop(room)
363
+ def fits?(entry, cap, bytes_cap)
364
+ buffer.size < cap && (bytes_cap.nil? || @buffer_bytes + entry.bytes <= bytes_cap)
365
+ end
366
+
367
+ # The three buffer mutations, each under buffer_mutex, each keeping the
368
+ # byte total (`@buffer_bytes`) in step.
369
+ def push_entry(entry)
370
+ @buffer_bytes += entry.bytes
371
+ buffer << entry
372
+ end
373
+
374
+ def take_oldest
375
+ remove_at(0)
376
+ end
377
+
378
+ def remove_at(index)
379
+ entry = buffer.delete_at(index)
380
+ @buffer_bytes -= entry.bytes
381
+ entry
328
382
  end
329
383
 
330
384
  public
@@ -508,7 +562,7 @@ module Wurk
508
562
  next unless index
509
563
 
510
564
  @replaying += 1
511
- buffer.delete_at(index)
565
+ remove_at(index)
512
566
  end
513
567
  end
514
568
 
@@ -550,7 +604,10 @@ module Wurk
550
604
  end
551
605
 
552
606
  def restore(entry)
553
- buffer_mutex.synchronize { buffer.unshift(entry) }
607
+ buffer_mutex.synchronize do
608
+ @buffer_bytes += entry.bytes
609
+ buffer.unshift(entry)
610
+ end
554
611
  end
555
612
 
556
613
  # Drain marks the thread so our prepended raw_push re-raises the
@@ -587,7 +644,7 @@ module Wurk
587
644
  @lock.synchronize { @pending << payload }
588
645
  end
589
646
 
590
- def report(cap)
647
+ def report(cap, bytes_cap = nil)
591
648
  dropped, announce = @lock.synchronize do
592
649
  next [nil, false] if @pending.empty?
593
650
 
@@ -599,7 +656,7 @@ module Wurk
599
656
  end
600
657
  return unless dropped
601
658
 
602
- log(dropped.first, cap) if announce
659
+ log(dropped.first, cap, bytes_cap) if announce
603
660
  dropped.size.times { count_drop }
604
661
  end
605
662
 
@@ -620,13 +677,14 @@ module Wurk
620
677
  nil
621
678
  end
622
679
 
623
- def log(first, cap)
680
+ def log(first, cap, bytes_cap)
624
681
  Wurk.configuration.logger.error do
625
- "reliable_push buffer full (cap=#{cap}): dropping the oldest buffered jobs to make room — " \
682
+ "reliable_push buffer full (cap=#{cap}#{", bytes=#{bytes_cap}" if bytes_cap}): " \
683
+ 'dropping the oldest buffered jobs to make room — ' \
626
684
  "#{first['class']} jid=#{first['jid']} is the first of this burst and will never run. " \
627
685
  'Further drops are counted in statsd jobs.dropped.push until Redis takes a replay. Raise ' \
628
- '`Wurk::Client.reliable_push_buffer`, or set `reliable_push_overflow = :raise` to handle ' \
629
- 'overflow yourself.'
686
+ '`Wurk::Client.reliable_push_buffer` (or `reliable_push_buffer_bytes`), or set ' \
687
+ '`reliable_push_overflow = :raise` to handle overflow yourself.'
630
688
  end
631
689
  rescue StandardError
632
690
  nil
@@ -742,6 +800,11 @@ module Wurk
742
800
  super
743
801
  end
744
802
 
803
+ def push_prepared(payloads)
804
+ Buffered.drain!(self) unless Buffered.unbuffered?
805
+ super
806
+ end
807
+
745
808
  private
746
809
 
747
810
  # Opens the delivery ledger Client writes into. It lives here rather
@@ -851,6 +914,14 @@ module Wurk
851
914
  Buffered.buffer_cap = value
852
915
  end
853
916
 
917
+ def reliable_push_buffer_bytes
918
+ Buffered.buffer_bytes_cap
919
+ end
920
+
921
+ def reliable_push_buffer_bytes=(value)
922
+ Buffered.buffer_bytes_cap = value
923
+ end
924
+
854
925
  def reliable_push_overflow
855
926
  Buffered.overflow_mode
856
927
  end
data/lib/wurk/client.rb CHANGED
@@ -1,6 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require_relative 'iterable_job'
4
3
  require_relative 'job_util'
5
4
  require_relative 'lua'
6
5
  require_relative 'status'
@@ -90,6 +89,27 @@ module Wurk
90
89
  payload['jid']
91
90
  end
92
91
 
92
+ # #push in two halves, for a caller that pushes many jobs in one write
93
+ # (JobSet#retry_all): #prepare_push runs one item through normalize →
94
+ # middleware → verify, #push_prepared writes what came back. Internal.
95
+ #
96
+ # @return [Hash, nil] the payload to write; nil when middleware halted it.
97
+ def prepare_push(item)
98
+ payload = invoke_chain(normalize_item(item))
99
+ verify_json(payload) if payload
100
+ payload
101
+ end
102
+
103
+ # One #raw_push for every payload, so they should share a queue, a batch
104
+ # or none, and a schedule or none: anything else is several writes, and one
105
+ # can land before another fails.
106
+ # @return [Array<String>] jids
107
+ def push_prepared(payloads)
108
+ buffered = raw_push(payloads)
109
+ emit_enqueued(payloads, buffered)
110
+ payloads.map { |payload| payload['jid'] }
111
+ end
112
+
93
113
  # @param items [Hash] String keys only: class, args (Array<Array>), at?, spread_interval?, batch_size?, jid?
94
114
  # @return [Array<String, nil>] jids in submission order; nil entries mark middleware-halted jobs.
95
115
  def push_bulk(items)
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Autoload target of `Sidekiq::Deploy` (compat.rb).
4
+
5
+ module Sidekiq
6
+ Deploy = Wurk::Deploy
7
+ end
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Autoload target of `Sidekiq::Encryption` (compat.rb).
4
+
5
+ module Sidekiq
6
+ Encryption = Wurk::Encryption
7
+ end
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Autoload target of Flow's Sidekiq aliases (compat.rb).
4
+
5
+ module Sidekiq
6
+ Flow = Wurk::Flow
7
+ FlowSet = Wurk::FlowSet
8
+ end
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Autoload target of IterableJob's Sidekiq aliases (compat.rb).
4
+
5
+ module Sidekiq
6
+ IterableJob = Wurk::IterableJob
7
+ IterableJobQuery = Wurk::IterableJobQuery
8
+ end
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Autoload target of the profile store's Sidekiq aliases (compat.rb).
4
+
5
+ module Sidekiq
6
+ ProfileSet = Wurk::ProfileSet
7
+ ProfileRecord = Wurk::ProfileRecord
8
+ end
@@ -0,0 +1,15 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Autoload target of the server tier's Sidekiq aliases (compat.rb). The Wurk
4
+ # files define only their Wurk names; this assigns the Sidekiq ones.
5
+
6
+ module Sidekiq
7
+ CLI = Wurk::CLI
8
+ Embedded = Wurk::Embedded
9
+ Launcher = Wurk::Launcher
10
+ Manager = Wurk::Manager
11
+ Scheduled = Wurk::Scheduled
12
+ History = Wurk::History
13
+ Periodic = Wurk::Cron
14
+ CronParser = Wurk::Cron::Parser
15
+ end
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Autoload target of the dashboard's Sidekiq aliases (compat.rb).
4
+
5
+ module Sidekiq
6
+ Web = Wurk::Web
7
+ WebHelpers = Wurk::Web::Extension::Helpers
8
+ Paginator = Wurk::Web::Paginator
9
+
10
+ module Pro
11
+ # Sidekiq Pro mounts the dashboard at `Sidekiq::Pro::Web`; it is the same
12
+ # board as `Sidekiq::Web` here, so a Pro app's `mount Sidekiq::Pro::Web`
13
+ # drops in unchanged.
14
+ Web = Wurk::Web
15
+
16
+ # `use Sidekiq::Pro::BatchStatus` — the polling Rack middleware that serves
17
+ # GET /batch_status/<bid>.json. Spec: docs/target/sidekiq-pro.md §10.3.
18
+ BatchStatus = Wurk::Web::BatchStatus
19
+ end
20
+ end
data/lib/wurk/compat.rb CHANGED
@@ -50,16 +50,6 @@ module Sidekiq
50
50
  # classes under them — but `Sidekiq.pro?` / `Sidekiq.ent?` still return
51
51
  # `false` per docs/target/sidekiq-free.md §32 (Wurk advertises as free OSS).
52
52
  module Pro
53
- # Sidekiq Pro mounts the dashboard at `Sidekiq::Pro::Web`; it's the same
54
- # board as `Sidekiq::Web` here, so a Pro app's `mount Sidekiq::Pro::Web`
55
- # drops in unchanged. `Wurk::Web` is already required by the time this file
56
- # loads (lib/wurk.rb requires wurk/web before wurk/compat).
57
- Web = Wurk::Web
58
-
59
- # `use Sidekiq::Pro::BatchStatus` — the polling Rack middleware that serves
60
- # GET /batch_status/<bid>.json. Spec: docs/target/sidekiq-pro.md §10.3.
61
- BatchStatus = Wurk::Web::BatchStatus
62
-
63
53
  # Pro module-level statsd accessor (spec §1): `Sidekiq::Pro.dogstatsd = ...`.
64
54
  # Delegates to the canonical `config.dogstatsd=` (spec §9.1), so either form
65
55
  # feeds the same Wurk::Metrics::Statsd client.
@@ -119,38 +109,14 @@ module Sidekiq
119
109
  private_constant :RailsNamespace
120
110
 
121
111
  BasicFetch = Wurk::Fetcher::Reliable
122
- WebHelpers = Wurk::Web::Extension::Helpers
123
- Paginator = Wurk::Web::Paginator
124
112
  Batch = Wurk::Batch
125
113
  BatchSet = Wurk::BatchSet
126
114
  Capsule = Wurk::Capsule
127
- CLI = Wurk::CLI
128
115
  Client = Wurk::Client
129
116
  Component = Wurk::Component
130
117
  Config = Wurk::Configuration
131
118
  Context = Wurk::Context
132
- # No `Sidekiq::Cron` alias on purpose (#204): real Sidekiq never defines it
133
- # — that namespace belongs to the third-party sidekiq-cron gem, and squatting
134
- # it made the gem's own classes (Poller, Job) collide at load. Wurk's native
135
- # periodic surface is `Sidekiq::Periodic` (Ent parity) / `Wurk::Cron`.
136
- CronParser = Wurk::Cron::Parser
137
- Periodic = Wurk::Cron
138
119
  DeadSet = Wurk::DeadSet
139
- Deploy = Wurk::Deploy
140
- Embedded = Wurk::Embedded
141
- Encryption = Wurk::Encryption
142
- # A Wurk extra, not parity — aliased because nothing else claims the name.
143
- # Checked before adding it, the way `Sidekiq::Cron` (#204) and
144
- # `Sidekiq::Status` taught us to: upstream Sidekiq defines no `Flow` anywhere
145
- # in `lib/` at the pinned parity SHA, RubyGems has no `sidekiq-flow` or
146
- # `sidekiq_flow` gem, and the only `Sidekiq::Flow` in public Ruby is a
147
- # `FlowIdMiddleware` nested under a gem's *own* `Sidekiq` module, which this
148
- # constant cannot reach.
149
- Flow = Wurk::Flow
150
- FlowSet = Wurk::FlowSet
151
- History = Wurk::History
152
- IterableJob = Wurk::IterableJob
153
- IterableJobQuery = Wurk::IterableJobQuery
154
120
  Job = Wurk::Job
155
121
  JobLogger = Wurk::JobLogger
156
122
  JobRecord = Wurk::JobRecord
@@ -161,11 +127,9 @@ module Sidekiq
161
127
  JobSet = Wurk::JobSet
162
128
  JobUtil = Wurk::JobUtil
163
129
  Keys = Wurk::Keys
164
- Launcher = Wurk::Launcher
165
130
  Limiter = Wurk::Limiter
166
131
  Loader = Wurk::Loader
167
132
  Logger = Wurk::Logger
168
- Manager = Wurk::Manager
169
133
  Metrics = Wurk::Metrics
170
134
  Middleware = Wurk::Middleware
171
135
  ServerMiddleware = Wurk::Middleware::ServerMiddleware
@@ -174,13 +138,10 @@ module Sidekiq
174
138
  ProcessSet = Wurk::ProcessSet
175
139
  Processor = Wurk::Processor
176
140
  Profiler = Wurk::Profiler
177
- ProfileSet = Wurk::ProfileSet
178
- ProfileRecord = Wurk::ProfileRecord
179
141
  Queue = Wurk::Queue
180
142
  RedisClientAdapter = Wurk::RedisClientAdapter
181
143
  RedisConnection = Wurk::RedisConnection
182
144
  RetrySet = Wurk::RetrySet
183
- Scheduled = Wurk::Scheduled
184
145
  ScheduledSet = Wurk::ScheduledSet
185
146
  Shutdown = Wurk::Shutdown
186
147
  SortedEntry = Wurk::SortedEntry
@@ -200,12 +161,45 @@ module Sidekiq
200
161
  TransactionAwareClient = Wurk::TransactionAwareClient
201
162
  Queues = Wurk::Queues
202
163
  EmptyQueueError = Wurk::Testing::EmptyQueueError
203
- Web = Wurk::Web
204
164
  Work = Wurk::Work
205
165
  Worker = Wurk::Worker
206
166
  Workers = Wurk::Workers
207
167
  WorkSet = Wurk::WorkSet
208
168
 
169
+ # Aliases of lazy Wurk constants (Wurk::LazyLoader) are autoloads too, so
170
+ # `defined?(Sidekiq::Web)` holds before the load and the alias is the very
171
+ # object its target is. Each points at a file under lib/wurk/compat/ that
172
+ # loads the target and assigns the alias: the target's own file defines only
173
+ # the Wurk name, so an autoload onto it would leave `Sidekiq::X` undefined.
174
+ # Those files name their targets rather than require them: web.rb, swarm.rb
175
+ # and iterable_job.rb require subfiles that reopen the class before defining
176
+ # it, which a direct require would turn into a circular one.
177
+ #
178
+ # No `Sidekiq::Cron` alias on purpose (#204): real Sidekiq never defines it
179
+ # — that namespace belongs to the third-party sidekiq-cron gem, and squatting
180
+ # it made the gem's own classes (Poller, Job) collide at load. Wurk's native
181
+ # periodic surface is `Sidekiq::Periodic` (Ent parity) / `Wurk::Cron`.
182
+ #
183
+ # `Flow` is a Wurk extra, not parity — aliased because nothing else claims
184
+ # the name. Checked before adding it, the way `Sidekiq::Cron` (#204) and
185
+ # `Sidekiq::Status` taught us to: upstream Sidekiq defines no `Flow` anywhere
186
+ # in `lib/` at the pinned parity SHA, RubyGems has no `sidekiq-flow` or
187
+ # `sidekiq_flow` gem, and the only `Sidekiq::Flow` in public Ruby is a
188
+ # `FlowIdMiddleware` nested under a gem's *own* `Sidekiq` module, which this
189
+ # constant cannot reach.
190
+ Wurk::LazyLoader.register(
191
+ self,
192
+ CLI: 'compat/server', Embedded: 'compat/server', Launcher: 'compat/server',
193
+ Manager: 'compat/server', Scheduled: 'compat/server',
194
+ History: 'compat/server', Periodic: 'compat/server', CronParser: 'compat/server',
195
+ Web: 'compat/web', WebHelpers: 'compat/web', Paginator: 'compat/web',
196
+ ProfileSet: 'compat/profile_set', ProfileRecord: 'compat/profile_set',
197
+ Flow: 'compat/flow', FlowSet: 'compat/flow',
198
+ IterableJob: 'compat/iterable_job', IterableJobQuery: 'compat/iterable_job',
199
+ Deploy: 'compat/deploy', Encryption: 'compat/encryption'
200
+ )
201
+ Wurk::LazyLoader.register(Pro, Web: 'compat/web', BatchStatus: 'compat/web')
202
+
209
203
  # Top-level Sidekiq.configure_server / configure_client / redis / logger
210
204
  # delegate to Wurk's class methods. Third-party gems treat these as the
211
205
  # canonical entry points.