wurk 1.13.0 → 1.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (255) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -1
  3. data/app/controllers/concerns/wurk/sse_streaming.rb +36 -1
  4. data/app/controllers/wurk/api/pagination.rb +9 -7
  5. data/app/controllers/wurk/api/serializers.rb +8 -2
  6. data/app/controllers/wurk/api_controller.rb +57 -24
  7. data/app/controllers/wurk/application_controller.rb +0 -2
  8. data/app/controllers/wurk/dashboard_controller.rb +0 -1
  9. data/config/routes.rb +5 -1
  10. data/lib/sidekiq/cli.rb +4 -3
  11. data/lib/sidekiq/deploy.rb +2 -1
  12. data/lib/sidekiq/embedded.rb +2 -1
  13. data/lib/sidekiq/iterable_job.rb +2 -1
  14. data/lib/sidekiq/job/iterable/active_record_enumerator.rb +2 -1
  15. data/lib/sidekiq/job/iterable/csv_enumerator.rb +2 -1
  16. data/lib/sidekiq/job/iterable.rb +2 -1
  17. data/lib/sidekiq/launcher.rb +2 -1
  18. data/lib/sidekiq/manager.rb +2 -1
  19. data/lib/sidekiq/metrics/query.rb +2 -1
  20. data/lib/sidekiq/pro/web.rb +2 -1
  21. data/lib/sidekiq/scheduled.rb +2 -1
  22. data/lib/sidekiq/web.rb +2 -1
  23. data/lib/sidekiq.rb +3 -2
  24. data/lib/wurk/api/fast.rb +4 -18
  25. data/lib/wurk/api/queues.rb +12 -10
  26. data/lib/wurk/api/serializers.rb +6 -7
  27. data/lib/wurk/api/swarm.rb +0 -1
  28. data/lib/wurk/batch/server_middleware.rb +4 -0
  29. data/lib/wurk/batch/status.rb +45 -22
  30. data/lib/wurk/batch_set.rb +34 -0
  31. data/lib/wurk/cli.rb +17 -19
  32. data/lib/wurk/client/buffered.rb +97 -26
  33. data/lib/wurk/client.rb +21 -1
  34. data/lib/wurk/compat/deploy.rb +7 -0
  35. data/lib/wurk/compat/encryption.rb +7 -0
  36. data/lib/wurk/compat/flow.rb +8 -0
  37. data/lib/wurk/compat/iterable_job.rb +8 -0
  38. data/lib/wurk/compat/profile_set.rb +8 -0
  39. data/lib/wurk/compat/server.rb +15 -0
  40. data/lib/wurk/compat/web.rb +20 -0
  41. data/lib/wurk/compat.rb +34 -40
  42. data/lib/wurk/configuration.rb +46 -9
  43. data/lib/wurk/cron.rb +146 -47
  44. data/lib/wurk/dead_set.rb +10 -6
  45. data/lib/wurk/embedded.rb +0 -2
  46. data/lib/wurk/engine.rb +46 -4
  47. data/lib/wurk/error_coalescer.rb +21 -1
  48. data/lib/wurk/fetcher/many_queues.rb +8 -6
  49. data/lib/wurk/fetcher/poison_pill.rb +1 -1
  50. data/lib/wurk/fetcher/reaper.rb +0 -5
  51. data/lib/wurk/fetcher/reliable.rb +46 -27
  52. data/lib/wurk/fetcher/self_reconcile.rb +0 -1
  53. data/lib/wurk/fetcher/unit_of_work.rb +16 -1
  54. data/lib/wurk/flow/status.rb +10 -4
  55. data/lib/wurk/flow_set.rb +24 -2
  56. data/lib/wurk/health.rb +0 -1
  57. data/lib/wurk/heartbeat.rb +13 -7
  58. data/lib/wurk/import/sidekiq_cron.rb +0 -1
  59. data/lib/wurk/job_record.rb +2 -1
  60. data/lib/wurk/job_retry.rb +1 -1
  61. data/lib/wurk/job_set.rb +132 -6
  62. data/lib/wurk/launcher.rb +0 -12
  63. data/lib/wurk/lazy_loader.rb +60 -0
  64. data/lib/wurk/limiter/bucket.rb +0 -2
  65. data/lib/wurk/limiter/concurrent.rb +19 -36
  66. data/lib/wurk/limiter/leaky.rb +0 -2
  67. data/lib/wurk/limiter/points.rb +0 -2
  68. data/lib/wurk/limiter/server_middleware.rb +4 -1
  69. data/lib/wurk/limiter/unlimited.rb +0 -2
  70. data/lib/wurk/limiter/window.rb +0 -2
  71. data/lib/wurk/limiter.rb +0 -7
  72. data/lib/wurk/logger.rb +44 -5
  73. data/lib/wurk/lua/fast_delete.rb +58 -0
  74. data/lib/wurk/lua/fast_delete_window.lua +46 -0
  75. data/lib/wurk/lua/limiter_concurrent_acquire.lua +10 -3
  76. data/lib/wurk/lua/limiter_concurrent_release.lua +13 -4
  77. data/lib/wurk/lua.rb +1 -41
  78. data/lib/wurk/metrics/dashboard_series.rb +38 -18
  79. data/lib/wurk/metrics/history.rb +24 -11
  80. data/lib/wurk/metrics/prometheus.rb +0 -1
  81. data/lib/wurk/metrics/query.rb +0 -2
  82. data/lib/wurk/metrics/queue_rollup.rb +33 -11
  83. data/lib/wurk/metrics/statsd.rb +4 -0
  84. data/lib/wurk/middleware/chain.rb +25 -3
  85. data/lib/wurk/middleware/expiry.rb +4 -0
  86. data/lib/wurk/middleware/interrupt_handler.rb +4 -1
  87. data/lib/wurk/middleware/status.rb +4 -0
  88. data/lib/wurk/middleware/timeout.rb +4 -0
  89. data/lib/wurk/preflight/owners.rb +0 -1
  90. data/lib/wurk/preflight/payload_sample.rb +0 -1
  91. data/lib/wurk/preflight/verify.rb +0 -2
  92. data/lib/wurk/process_set.rb +6 -1
  93. data/lib/wurk/profiler.rb +2 -1
  94. data/lib/wurk/queue.rb +33 -6
  95. data/lib/wurk/queue_consumers.rb +10 -1
  96. data/lib/wurk/rails_boot.rb +2 -2
  97. data/lib/wurk/railtie.rb +3 -4
  98. data/lib/wurk/sentry/error_handler.rb +5 -0
  99. data/lib/wurk/sentry/middleware.rb +4 -0
  100. data/lib/wurk/sorted_entry.rb +16 -12
  101. data/lib/wurk/stats.rb +85 -1
  102. data/lib/wurk/swarm/child_boot.rb +0 -2
  103. data/lib/wurk/swarm/liveness.rb +0 -2
  104. data/lib/wurk/swarm/memory_watch.rb +116 -0
  105. data/lib/wurk/swarm/prefork.rb +54 -0
  106. data/lib/wurk/swarm.rb +33 -42
  107. data/lib/wurk/version.rb +1 -1
  108. data/lib/wurk/web/enterprise.rb +63 -7
  109. data/lib/wurk.rb +33 -29
  110. data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js +1 -0
  111. data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.br +0 -0
  112. data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.gz +0 -0
  113. data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js +1 -0
  114. data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.br +0 -0
  115. data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.gz +0 -0
  116. data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js +1 -0
  117. data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.br +0 -0
  118. data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.gz +0 -0
  119. data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js +1 -0
  120. data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.br +0 -0
  121. data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.gz +0 -0
  122. data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js +1 -0
  123. data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.br +0 -0
  124. data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.gz +0 -0
  125. data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js +1 -0
  126. data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.br +0 -0
  127. data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.gz +0 -0
  128. data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js +1 -0
  129. data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.br +0 -0
  130. data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.gz +0 -0
  131. data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js +1 -0
  132. data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.br +0 -0
  133. data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.gz +0 -0
  134. data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js +2 -0
  135. data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.br +0 -0
  136. data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.gz +0 -0
  137. data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js +1 -0
  138. data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.br +0 -0
  139. data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.gz +0 -0
  140. data/vendor/assets/dashboard/assets/{FlowState-yb7virqR.js → FlowState-ClzInw0-.js} +1 -1
  141. data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.br +0 -0
  142. data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.gz +0 -0
  143. data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js +1 -0
  144. data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.br +0 -0
  145. data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.gz +0 -0
  146. data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js +2 -0
  147. data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.br +0 -0
  148. data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.gz +0 -0
  149. data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js +1 -0
  150. data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.br +0 -0
  151. data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.gz +0 -0
  152. data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js +1 -0
  153. data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.br +0 -0
  154. data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.gz +0 -0
  155. data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js +1 -0
  156. data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.br +0 -0
  157. data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.gz +0 -0
  158. data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js +1 -0
  159. data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.br +0 -0
  160. data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.gz +0 -0
  161. data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js +1 -0
  162. data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.br +0 -0
  163. data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.gz +0 -0
  164. data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js +1 -0
  165. data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.br +0 -0
  166. data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.gz +0 -0
  167. data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js +1 -0
  168. data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.br +0 -0
  169. data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.gz +0 -0
  170. data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js +1 -0
  171. data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.br +0 -0
  172. data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.gz +0 -0
  173. data/vendor/assets/dashboard/assets/Search-D42Va7lB.js +1 -0
  174. data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.br +0 -0
  175. data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.gz +0 -0
  176. data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js +1 -0
  177. data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.br +0 -0
  178. data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.gz +0 -0
  179. data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js +1 -0
  180. data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.br +0 -0
  181. data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.gz +0 -0
  182. data/vendor/assets/dashboard/assets/de-B2NCIuzo.js +1 -0
  183. data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.br +0 -0
  184. data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.gz +0 -0
  185. data/vendor/assets/dashboard/assets/es-BGeu_JNB.js +1 -0
  186. data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.br +0 -0
  187. data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.gz +0 -0
  188. data/vendor/assets/dashboard/assets/fr-DQVBWFId.js +1 -0
  189. data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.br +0 -0
  190. data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.gz +0 -0
  191. data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js +1 -0
  192. data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.br +0 -0
  193. data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.gz +0 -0
  194. data/vendor/assets/dashboard/assets/index-CiLU1zz8.js +141 -0
  195. data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.br +0 -0
  196. data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.gz +0 -0
  197. data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css +1 -0
  198. data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.br +0 -0
  199. data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.gz +0 -0
  200. data/vendor/assets/dashboard/assets/ja-D06IBqzL.js +1 -0
  201. data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.br +0 -0
  202. data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.gz +0 -0
  203. data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js +1 -0
  204. data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.br +0 -0
  205. data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.gz +0 -0
  206. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js +1 -0
  207. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.br +0 -0
  208. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.gz +0 -0
  209. data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js +1 -0
  210. data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.br +0 -0
  211. data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.gz +0 -0
  212. data/vendor/assets/dashboard/assets/{utils-CnWZzW2L.js → utils-69XSoGbQ.js} +1 -1
  213. data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.br +0 -0
  214. data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.gz +0 -0
  215. data/vendor/assets/dashboard/assets/wurk-logo-ydeSB9B8.png +0 -0
  216. data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js +1 -0
  217. data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.br +0 -0
  218. data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.gz +0 -0
  219. data/vendor/assets/dashboard/index.html +3 -23
  220. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  221. metadata +122 -37
  222. data/vendor/assets/dashboard/assets/ArgsValue-BJol9S0s.js +0 -1
  223. data/vendor/assets/dashboard/assets/BatchDetail-D_zGQ9Sd.js +0 -1
  224. data/vendor/assets/dashboard/assets/Batches-Dt_xC660.js +0 -1
  225. data/vendor/assets/dashboard/assets/Busy-CWRthsmc.js +0 -1
  226. data/vendor/assets/dashboard/assets/Cron-qeGr2YAm.js +0 -1
  227. data/vendor/assets/dashboard/assets/Dashboard-DQWaAwPA.js +0 -1
  228. data/vendor/assets/dashboard/assets/Dead-D9gOPHtK.js +0 -1
  229. data/vendor/assets/dashboard/assets/Extension-DONXVLdA.js +0 -1
  230. data/vendor/assets/dashboard/assets/FilterBox-BbB6gXHp.js +0 -2
  231. data/vendor/assets/dashboard/assets/FlowDetail-CrmnVRp0.js +0 -1
  232. data/vendor/assets/dashboard/assets/Flows-g4GC8KmE.js +0 -1
  233. data/vendor/assets/dashboard/assets/JobDetailModal-C86k_1wj.js +0 -2
  234. data/vendor/assets/dashboard/assets/Limiters-zyO-iMlK.js +0 -1
  235. data/vendor/assets/dashboard/assets/Metrics-Ckq5fNBv.js +0 -1
  236. data/vendor/assets/dashboard/assets/NotFound-C8NnyjlY.js +0 -1
  237. data/vendor/assets/dashboard/assets/PageHeader-vWeBbSPF.js +0 -1
  238. data/vendor/assets/dashboard/assets/Profiles-DRl-FXPA.js +0 -1
  239. data/vendor/assets/dashboard/assets/Queues-DkXWrihA.js +0 -1
  240. data/vendor/assets/dashboard/assets/Retries-CmlojYIy.js +0 -1
  241. data/vendor/assets/dashboard/assets/Scheduled-Cbbpp3xw.js +0 -1
  242. data/vendor/assets/dashboard/assets/Search-LaEpj2fM.js +0 -1
  243. data/vendor/assets/dashboard/assets/charts-D_PQLoyc.js +0 -1
  244. data/vendor/assets/dashboard/assets/fa-brands-400-Bs6tcqqs.woff2 +0 -0
  245. data/vendor/assets/dashboard/assets/fa-regular-400-DRN8N0d1.woff2 +0 -0
  246. data/vendor/assets/dashboard/assets/fa-solid-900-IAB4Droh.woff2 +0 -0
  247. data/vendor/assets/dashboard/assets/fa-v4compatibility-CErXDOsT.woff2 +0 -0
  248. data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +0 -1
  249. data/vendor/assets/dashboard/assets/index--xvvNlYx.css +0 -1
  250. data/vendor/assets/dashboard/assets/index-FBKGtysy.js +0 -141
  251. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-BiwhHqgh.js +0 -1
  252. data/vendor/assets/dashboard/assets/useSort-jBYVwXvg.js +0 -1
  253. data/vendor/assets/dashboard/assets/wurk-logo-Vy3xW4K0.png +0 -0
  254. data/vendor/assets/dashboard/og-image.png +0 -0
  255. data/vendor/assets/dashboard/wurk-logo.webp +0 -0
data/lib/wurk/engine.rb CHANGED
@@ -3,7 +3,6 @@
3
3
  require 'rails/engine'
4
4
  require_relative '../active_job/queue_adapters/wurk_adapter'
5
5
  require_relative 'dashboard_manifest'
6
- require_relative 'web'
7
6
 
8
7
  module Wurk
9
8
  # Rails mountable engine. Owns the dashboard mount, the asset path for
@@ -44,6 +43,14 @@ module Wurk
44
43
  CACHEABLE_METHODS = %w[GET HEAD].freeze
45
44
  CACHEABLE_STATUSES = [200, 206, 304].freeze
46
45
 
46
+ # `rake frontend:build` writes a .br and a .gz next to each fingerprinted
47
+ # text asset (frontend/vite.config.ts, wurk-precompress). The mount sits
48
+ # ahead of any deflater the host runs, so without these the dashboard
49
+ # went out uncompressed. Preference order when the client's q-values tie.
50
+ ENCODINGS = { 'br' => '.br', 'gzip' => '.gz' }.freeze
51
+ OFFERED_ENCODINGS = [*ENCODINGS.keys, 'identity'].freeze
52
+ PRECOMPRESSED = /\.(?:js|css|svg|json)\z/
53
+
47
54
  def initialize(app, root:)
48
55
  @app = app
49
56
  @files = ::Rack::Files.new(root)
@@ -56,20 +63,55 @@ module Wurk
56
63
  inner = env.dup
57
64
  stripped = path.delete_prefix(PREFIX)
58
65
  inner[::Rack::PATH_INFO] = stripped.empty? ? '/' : stripped
59
- response = @files.call(inner)
66
+ response = precompressed(inner, env, stripped) || @files.call(inner)
60
67
  return @app.call(env) if response[0] == 404
61
68
 
62
- stamp_cache_control(response, env, stripped)
69
+ stamp_cache_headers(response, env, stripped)
63
70
  response
64
71
  end
65
72
 
66
73
  private
67
74
 
75
+ # The variant the client accepts and the build wrote, served with the
76
+ # original's content type; nil sends the identity file. A Range request
77
+ # gets identity so byte offsets mean what the client thinks they mean.
78
+ def precompressed(inner, env, stripped)
79
+ return unless CACHEABLE_METHODS.include?(env[::Rack::REQUEST_METHOD])
80
+ return unless precompressible?(stripped)
81
+ return if env['HTTP_RANGE']
82
+
83
+ encoding = negotiated_encoding(env)
84
+ return unless encoding
85
+
86
+ variant = @files.call(inner.merge(::Rack::PATH_INFO => "#{stripped}#{ENCODINGS[encoding]}"))
87
+ return unless variant[0] == 200
88
+
89
+ variant[1]['content-type'] = ::Rack::Mime.mime_type(::File.extname(stripped))
90
+ variant[1]['content-encoding'] = encoding
91
+ variant
92
+ end
93
+
94
+ # 'br', 'gzip', or nil for identity (no header, or every coding refused).
95
+ def negotiated_encoding(env)
96
+ accepted = ::Rack::Utils.q_values(env['HTTP_ACCEPT_ENCODING'].to_s)
97
+ encoding = ::Rack::Utils.select_best_encoding(OFFERED_ENCODINGS, accepted)
98
+ encoding if ENCODINGS.key?(encoding)
99
+ end
100
+
101
+ def precompressible?(stripped)
102
+ stripped.start_with?(FINGERPRINTED_PREFIX) && stripped.match?(PRECOMPRESSED)
103
+ end
104
+
68
105
  # This mount is inserted at index 0 of the HOST app's stack (see the
69
106
  # initializer below), so Rack::ETag and Rack::ConditionalGet never see
70
107
  # these responses — without this the dashboard's fingerprinted bundle
71
108
  # shipped with no cache directive at all and was refetched every load.
72
- def stamp_cache_control(response, env, stripped)
109
+ #
110
+ # Vary goes on every answer for a path that has variants, identity
111
+ # included, or a shared cache could hand gzip to a client that can't
112
+ # read it.
113
+ def stamp_cache_headers(response, env, stripped)
114
+ response[1]['vary'] = 'accept-encoding' if precompressible?(stripped)
73
115
  return unless CACHEABLE_METHODS.include?(env[::Rack::REQUEST_METHOD])
74
116
  return unless CACHEABLE_STATUSES.include?(response[0])
75
117
 
@@ -4,11 +4,19 @@ module Wurk
4
4
  # Lets the first report of a key through, then one per window, and says how
5
5
  # many it swallowed in between. Per process: every forked child speaks for
6
6
  # itself, so a fleet still reports at worst children x keys per window.
7
+ #
8
+ # Keys can come from data (an error message's first word), so at most
9
+ # MAX_KEYS are remembered: a new key first evicts the keys whose window has
10
+ # passed with nothing swallowed, then, if still full, the oldest. An evicted
11
+ # key's next report goes straight out.
7
12
  class ErrorCoalescer
8
13
  DEFAULT_WINDOW = 60
14
+ MAX_KEYS = 1024
9
15
 
10
- def initialize(window: DEFAULT_WINDOW, clock: -> { ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) })
16
+ def initialize(window: DEFAULT_WINDOW, max_keys: MAX_KEYS,
17
+ clock: -> { ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) })
11
18
  @window = window
19
+ @max_keys = max_keys
12
20
  @clock = clock
13
21
  @lock = ::Mutex.new
14
22
  @entries = {}
@@ -19,6 +27,7 @@ module Wurk
19
27
  def admit(key)
20
28
  @lock.synchronize do
21
29
  now = @clock.call
30
+ make_room(now) unless @entries.key?(key)
22
31
  entry = (@entries[key] ||= [nil, 0])
23
32
  if entry[0] && now - entry[0] < @window
24
33
  entry[1] += 1
@@ -31,5 +40,16 @@ module Wurk
31
40
  suppressed
32
41
  end
33
42
  end
43
+
44
+ def size = @lock.synchronize { @entries.size }
45
+
46
+ private
47
+
48
+ def make_room(now)
49
+ return if @entries.size < @max_keys
50
+
51
+ @entries.delete_if { |_, (at, swallowed)| swallowed.zero? && now - at >= @window }
52
+ @entries.shift while @entries.size >= @max_keys
53
+ end
34
54
  end
35
55
  end
@@ -62,17 +62,19 @@ module Wurk
62
62
  end
63
63
 
64
64
  # `[public, private, public, private, ...]` for queues 2..n, fetch_first's
65
- # KEYS. Strict mode with nothing paused hands #walk the same frozen array
66
- # every pass, so the steady state reuses one build; a shuffled or filtered
67
- # list is a fresh array per pass and rebuilds. Keyed on the pid too, for
68
- # the reason #queue_keys is. Copy-on-write: processor threads share this
69
- # fetcher.
65
+ # KEYS, built as one Array rather than a pair per queue flattened. Strict
66
+ # mode with nothing paused hands #walk the same frozen array every pass,
67
+ # so the steady state reuses one build; a shuffled or filtered list is a
68
+ # fresh array per pass and rebuilds. Keyed on the pid too, for the reason
69
+ # #queue_keys is. Copy-on-write: processor threads share this fetcher.
70
70
  def claim_keys(queues)
71
71
  pid = PidCache.pid
72
72
  cached = @claim_keys
73
73
  return cached[2] if cached && cached[0].equal?(queues) && cached[1] == pid
74
74
 
75
- keys = queues.drop(1).flat_map { |public_q| [public_q, queue_keys(public_q).first] }.freeze
75
+ keys = []
76
+ queues.each_with_index { |public_q, i| keys << public_q << queue_keys(public_q).first unless i.zero? }
77
+ keys.freeze
76
78
  @claim_keys = [queues, pid, keys].freeze
77
79
  keys
78
80
  end
@@ -119,7 +119,7 @@ module Wurk
119
119
  def clear_in(pipe, jid)
120
120
  return if jid.nil? || jid.to_s.empty?
121
121
 
122
- pipe.call('DEL', counter_key(jid))
122
+ pipe.call_v(['DEL', counter_key(jid)])
123
123
  end
124
124
 
125
125
  def counter_key(jid)
@@ -3,12 +3,7 @@
3
3
  require_relative '../component'
4
4
  require_relative '../keys'
5
5
  require_relative '../timer_loop'
6
- require_relative 'private_list_key'
7
- require_relative 'unparseable_keys'
8
- require_relative 'orphan_grace'
9
6
  require_relative 'oom_drain'
10
- require_relative 'orphan_reclaim'
11
- require_relative 'self_reconcile'
12
7
  require_relative 'reliable'
13
8
 
14
9
  module Wurk
@@ -36,7 +36,7 @@ module Wurk
36
36
  #
37
37
  # Spec: docs/target/sidekiq-pro.md §3 (super_fetch, §3.3 poll interval),
38
38
  # docs/target/sidekiq-free.md §15 (TIMEOUT=2).
39
- class Reliable < Fetcher
39
+ class Reliable < Fetcher # rubocop:disable Metrics/ClassLength
40
40
  include Component
41
41
  # Global per-queue concurrency, as a decorator over #walk. Prepended
42
42
  # rather than mixed in so there stays exactly one copy of the uncapped
@@ -59,6 +59,16 @@ module Wurk
59
59
  # docs/plans/2026/08/06/101-faster-than-sidekiq/00-semantics-signoff.md.
60
60
  PAUSED_TTL = 2
61
61
 
62
+ # The clock that cache is timed on, read once per fetch. A TTL in seconds
63
+ # does not need CLOCK_MONOTONIC's nanoseconds, and that clock is a real
64
+ # syscall wherever the kernel's clocksource has no vDSO read (hpet, some
65
+ # hypervisors), while the coarse one is a vDSO read everywhere Linux has
66
+ # it. It lags by up to its resolution, which the cache deadline gives back
67
+ # (PAUSED_CLOCK_SLACK), so the cache still never outlives PAUSED_TTL.
68
+ PAUSED_CLOCK =
69
+ defined?(::Process::CLOCK_MONOTONIC_COARSE) ? ::Process::CLOCK_MONOTONIC_COARSE : ::Process::CLOCK_MONOTONIC
70
+ PAUSED_CLOCK_SLACK = ::Process.clock_getres(PAUSED_CLOCK)
71
+
62
72
  # Backoff for the quieted short-circuit. Manager#quiet terminates the
63
73
  # shared fetcher before it terminates the processors, and Processor#run
64
74
  # loops on its *own* flag — so in that window every processor would spin
@@ -234,12 +244,17 @@ module Wurk
234
244
  # prebuilt array as-is — the steady-state fetch allocates nothing here,
235
245
  # matching Sidekiq's own strict path (fetch.rb:79-87). Random/weighted
236
246
  # shuffle each call — `@queues` is pre-expanded by weight in
237
- # Capsule#queues=, so uniform shuffle yields weighted fairness; .uniq
238
- # trims duplicates. Paused queues are filtered after shuffle so the
239
- # membership test runs on the smallest possible set.
247
+ # Capsule#queues=, so uniform shuffle yields weighted fairness, and
248
+ # duplicates are trimmed in place, only from a list that has any. Paused
249
+ # queues are filtered after shuffle so the membership test runs on the
250
+ # smallest possible set.
240
251
  def queues_cmd
241
252
  paused = paused_keys
242
- keys = config.mode == :strict ? prefixed_queues : prefixed_queues.shuffle.uniq
253
+ keys, distinct = prefixed_queues
254
+ unless config.mode == :strict
255
+ keys = keys.shuffle
256
+ keys.uniq! unless distinct
257
+ end
243
258
  return keys if paused.empty?
244
259
 
245
260
  keys.reject { |key| paused.include?(key) }
@@ -344,8 +359,8 @@ module Wurk
344
359
  # tests them against the `queue:<name>` keys it is about to LMOVE, and
345
360
  # prefixing here is once per TTL instead of once per queue per pass.
346
361
  #
347
- # Monotonic clock, so an NTP step can't pin the cache open. Per instance,
348
- # so it dies with the fetcher and can never cross a fork.
362
+ # Monotonic clock (PAUSED_CLOCK), so an NTP step can't pin the cache open.
363
+ # Per instance, so it dies with the fetcher and can never cross a fork.
349
364
  #
350
365
  # Processor threads share one fetcher and race here. The loser pays a
351
366
  # second SMEMBERS and overwrites with an equally fresh read; @paused is
@@ -353,35 +368,39 @@ module Wurk
353
368
  # can never see the value they don't belong to.
354
369
  def paused_keys
355
370
  generation = self.class.paused_generation
356
- now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
371
+ now = ::Process.clock_gettime(PAUSED_CLOCK)
357
372
  return @paused if @paused_generation == generation && now < @paused_expires_at
358
373
 
359
374
  members = config.redis(idempotent: true) { |conn| conn.call('SMEMBERS', Keys::PAUSED_SET) }
360
375
  @paused = members.to_set { |q| "#{Keys::QUEUE_PREFIX}#{q}" }
361
376
  @paused_generation = generation
362
- @paused_expires_at = now + PAUSED_TTL
377
+ @paused_expires_at = now + PAUSED_TTL - PAUSED_CLOCK_SLACK
363
378
  @paused
364
379
  end
365
380
 
366
- # `queue:<name>` for every queue this capsule serves. Rebuilt only when
367
- # Capsule#queues= swaps the list in — it always allocates a fresh array,
368
- # so identity is the whole check. Published before the source it came
369
- # from, so a reader that trusts the source can never see a list built
370
- # from a different one.
381
+ # `[queue:<name> for every queue this capsule serves, whether those are
382
+ # all distinct]` — a weighted list repeats each queue once per weight.
383
+ # Rebuilt only when Capsule#queues= swaps the list in — it always
384
+ # allocates a fresh array, so identity is the whole check. One frozen pair,
385
+ # published before the source it came from, so a reader that trusts the
386
+ # source can never see a list built from a different one.
371
387
  def prefixed_queues
372
388
  source = config.queues
373
389
  return @prefixed_queues if @prefixed_source.equal?(source)
374
390
 
375
- @prefixed_queues = source.map { |q| "#{Keys::QUEUE_PREFIX}#{q}".freeze }.freeze
391
+ queues = source.map { |q| "#{Keys::QUEUE_PREFIX}#{q}".freeze }.freeze
392
+ @prefixed_queues = [queues, queues.uniq.size == queues.size].freeze
376
393
  @prefixed_source = source
377
394
  @prefixed_queues
378
395
  end
379
396
 
380
- # The two strings a fetch derives from a public queue key: this process's
381
- # private list for it (a `gethostname` syscall plus a five-part
382
- # interpolation) and the unprefixed name the Processor tags the job with.
383
- # Both are pure functions of the queue and this process's identity, so
384
- # they cost one build per queue rather than two per job.
397
+ # What a fetch derives from a public queue key: this process's private
398
+ # list for it (a `gethostname` syscall plus a five-part interpolation), the
399
+ # unprefixed name the Processor tags the job with, and the LMOVE between
400
+ # the two. All pure functions of the queue and this process's identity, so
401
+ # they cost one build per queue rather than one per job. The LMOVE goes
402
+ # out by `call_v`: `call` re-collects its arguments into a fresh Array and
403
+ # a keyword Hash on every job, through two layers of client decorators.
385
404
  #
386
405
  # Rebuilt when the pid moves: a fetcher materialized before a fork would
387
406
  # otherwise keep claiming into a private list stamped with the parent's
@@ -402,8 +421,9 @@ module Wurk
402
421
  cached = cache[public_q]
403
422
  return cached if cached
404
423
 
405
- built = [self.class.private_queue_name(public_q).freeze,
406
- public_q.delete_prefix(Keys::QUEUE_PREFIX).freeze].freeze
424
+ priv = self.class.private_queue_name(public_q).freeze
425
+ built = [priv, public_q.delete_prefix(Keys::QUEUE_PREFIX).freeze,
426
+ ['LMOVE', public_q, priv, 'RIGHT', 'LEFT'].freeze].freeze
407
427
  @queue_keys = cache.merge(public_q => built)
408
428
  built
409
429
  end
@@ -418,15 +438,15 @@ module Wurk
418
438
  # A replayed ACK is a no-op (LREM removes nothing the second time, DEL is
419
439
  # already done), so folding it in costs the retry nothing.
420
440
  def lmove(public_q, ack = nil)
421
- priv, name = queue_keys(public_q)
441
+ priv, name, command = queue_keys(public_q)
422
442
  job = config.redis(idempotent: true) do |conn|
423
443
  if ack
424
444
  conn.pipelined do |pipe|
425
445
  ack.write_ack(pipe)
426
- pipe.call('LMOVE', public_q, priv, 'RIGHT', 'LEFT')
446
+ pipe.call_v(command)
427
447
  end.last
428
448
  else
429
- conn.call('LMOVE', public_q, priv, 'RIGHT', 'LEFT')
449
+ conn.call_v(command)
430
450
  end
431
451
  end
432
452
  job ? unit_of_work(public_q, priv, name, job) : nil
@@ -459,8 +479,7 @@ module Wurk
459
479
  # Both fetch paths hand the unit every string its Processor and its ACK
460
480
  # will need, so neither re-derives one per job.
461
481
  def unit_of_work(public_q, priv, name, job)
462
- uow = UnitOfWork.new(queue: public_q, queue_name: name, private_queue: priv,
463
- job: job, config: config, fetcher: self)
482
+ uow = UnitOfWork.claimed(public_q, name, priv, job, config, self)
464
483
  ::Thread.current.thread_variable_set(CLAIM_KEY, uow)
465
484
  uow
466
485
  end
@@ -1,7 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative '../keys'
4
- require_relative 'orphan_reclaim'
5
4
 
6
5
  module Wurk
7
6
  class Fetcher
@@ -54,6 +54,21 @@ module Wurk
54
54
  end
55
55
  end
56
56
 
57
+ # The fetch path's constructor. Positional, filling the members straight
58
+ # in: the keyword form costs the caller's Hash, a second one for the
59
+ # `**` forward, and Struct's keyword parsing — per job, for six fields
60
+ # whose names never change.
61
+ def self.claimed(queue, queue_name, private_queue, job, config, fetcher) # rubocop:disable Metrics/ParameterLists
62
+ uow = allocate
63
+ uow.queue = queue
64
+ uow.queue_name = queue_name
65
+ uow.private_queue = private_queue
66
+ uow.job = job
67
+ uow.config = config
68
+ uow.fetcher = fetcher
69
+ uow
70
+ end
71
+
57
72
  # Deferred, never skipped: the LREM goes back to the fetcher, which
58
73
  # pipelines it in front of the next fetch's LMOVE instead of spending a
59
74
  # round trip of its own. Ordering against the job is unchanged — the
@@ -75,7 +90,7 @@ module Wurk
75
90
  # wire-frozen, so the reaper cannot flag it). Reading the counter to
76
91
  # decide would spend the very round trip the DEL is riding for free.
77
92
  def write_ack(pipe)
78
- pipe.call('LREM', private_queue, LREM_COUNT, job)
93
+ pipe.call_v(['LREM', private_queue, LREM_COUNT, job])
79
94
  job_jid = jid.to_s
80
95
  Middleware::PoisonPill.clear_in(pipe, job_jid) unless job_jid.empty?
81
96
  release_slot_in(pipe) if slot_key
@@ -31,11 +31,14 @@ module Wurk
31
31
 
32
32
  attr_reader :fid
33
33
 
34
- def initialize(fid)
34
+ # `header:` is the `flow:<fid>` hash already read, by a listing that
35
+ # fetched a page of them in one pipeline (FlowSet#snapshot); without it
36
+ # the header is read here.
37
+ def initialize(fid, header: nil)
35
38
  raise ArgumentError, 'fid required' if fid.nil? || fid.to_s.empty?
36
39
 
37
40
  @fid = fid.to_s
38
- reload!
41
+ header ? self.header = header : reload!
39
42
  end
40
43
 
41
44
  # False when no `flow:<fid>` hash exists: a well-formed fid that was
@@ -98,8 +101,7 @@ module Wurk
98
101
  end
99
102
 
100
103
  def reload!
101
- raw = Wurk.redis { |conn| conn.call('HGETALL', Keys.flow(@fid)) }
102
- @header = raw.is_a?(Hash) ? raw : raw.each_slice(2).to_h
104
+ self.header = Wurk.redis { |conn| conn.call('HGETALL', Keys.flow(@fid)) }
103
105
  @nodes = nil
104
106
  @dead_indexes = nil
105
107
  self
@@ -191,6 +193,10 @@ module Wurk
191
193
  rows.filter_map { |row| Node.new(NODE_FIELDS.zip(row).to_h) if row.first }
192
194
  end
193
195
 
196
+ def header=(raw)
197
+ @header = raw.is_a?(Hash) ? raw : raw.each_slice(2).to_h
198
+ end
199
+
194
200
  # "No timestamp" and "a timestamp nothing can read" are the same answer to
195
201
  # a caller rendering a clock — and the same reason {#parse_indexes} gives
196
202
  # for not raising out of a read.
data/lib/wurk/flow_set.rb CHANGED
@@ -1,7 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require_relative 'flow'
4
-
5
3
  module Wurk
6
4
  # Discovery over the `flows` sorted set: `size`, and `each` newest-first
7
5
  # yielding {Wurk::Flow::Status}. The counterpart of {Wurk::BatchSet}, and a
@@ -49,5 +47,29 @@ module Wurk
49
47
  page += 1
50
48
  end
51
49
  end
50
+
51
+ # The set's size and the #each-ordered page of `count` flows from
52
+ # `offset`: the page's ids in one round trip and every header in a second,
53
+ # where #each pays a round trip per flow from the top of the set down.
54
+ # Internal reader for the dashboard.
55
+ #
56
+ # @return [Array(Integer, Array<Wurk::Flow::Status>)]
57
+ def snapshot(offset, count)
58
+ Wurk.redis do |conn|
59
+ size, fids = page_ids(conn, offset, count)
60
+ headers = fids.empty? ? [] : conn.pipelined { |pipe| fids.each { |fid| pipe.call('HGETALL', Keys.flow(fid)) } }
61
+ [size.to_i, fids.zip(headers).map { |fid, header| Wurk::Flow::Status.new(fid, header: header) }]
62
+ end
63
+ end
64
+
65
+ private
66
+
67
+ # ZCARD and one newest-first page of ids, in one round trip.
68
+ def page_ids(conn, offset, count)
69
+ conn.pipelined do |pipe|
70
+ pipe.call('ZCARD', @key)
71
+ pipe.call('ZRANGE', @key, offset, offset + count - 1, 'REV')
72
+ end
73
+ end
52
74
  end
53
75
  end
data/lib/wurk/health.rb CHANGED
@@ -2,7 +2,6 @@
2
2
 
3
3
  require 'socket'
4
4
  require 'json'
5
- require_relative 'metrics/prometheus'
6
5
 
7
6
  module Wurk
8
7
  # Thin HTTP listener for k8s liveness/readiness probes. Optional, off by
@@ -23,7 +23,7 @@ module Wurk
23
23
  # EXPIRE <identity>:work 60 (only if WORK_STATE non-empty)
24
24
  # EVALSHA refresh_slots <held slots> (only if QueueSlot::HELD non-empty)
25
25
  # then the signal drain:
26
- # RPOP <identity>-signals × BEAT_PAUSE
26
+ # RPOP <identity>-signals BEAT_PAUSE
27
27
  # Two rather than one because only the first is safe to replay — see
28
28
  # #pipelined_beat and #drain_signals.
29
29
  #
@@ -110,7 +110,7 @@ module Wurk
110
110
  end
111
111
  fire_event(:beat, oneshot: false)
112
112
  emit_statsd_gauges
113
- sigs.compact
113
+ sigs
114
114
  rescue StandardError => e
115
115
  @first_heartbeat = true
116
116
  handle_exception(e, { context: 'heartbeat' })
@@ -236,17 +236,23 @@ module Wurk
236
236
  # that reports us alive has landed.
237
237
  #
238
238
  # RPOP because ProcessSet::Process#signal LPUSHes: signals run in the
239
- # order they were sent (Sidekiq's launcher pops the same end). One entry
240
- # per second of cadence so a flood of queued signals can't stall the beat;
241
- # anything newer drains on the next beat.
239
+ # order they were sent (Sidekiq's launcher pops the same end), and a counted
240
+ # RPOP returns them in the order it pops them. One entry per second of
241
+ # cadence so a flood of queued signals can't stall the beat; anything newer
242
+ # drains on the next beat. One counted command, where it used to be
243
+ # BEAT_PAUSE single pops in a pipeline: the same round trip and the same
244
+ # entries, a tenth of the commands. Nil, not an empty Array, when the list
245
+ # is missing.
242
246
  def drain_signals
243
- redis { |conn| conn.pipelined { |pipe| BEAT_PAUSE.times { pipe.call('RPOP', "#{@identity}-signals") } } }
247
+ redis { |conn| conn.call('RPOP', "#{@identity}-signals", BEAT_PAUSE) } || []
244
248
  end
245
249
 
250
+ # The hostname is read once per Heartbeat, like the identity it is paired
251
+ # with, rather than a gethostname syscall every beat.
246
252
  def info_hash
247
253
  caps = @config.capsules.each_value
248
254
  {
249
- 'hostname' => hostname,
255
+ 'hostname' => (@hostname ||= hostname),
250
256
  'started_at' => @started_at,
251
257
  'pid' => ::Process.pid,
252
258
  'tag' => @config[:tag] || default_tag,
@@ -1,7 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'json'
4
- require_relative '../cron'
5
4
 
6
5
  module Wurk
7
6
  module Import
@@ -1,7 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'base64'
4
- require 'zlib'
5
4
  require_relative 'redact'
6
5
 
7
6
  module Wurk
@@ -90,6 +89,8 @@ module Wurk
90
89
  compressed = item['error_backtrace']
91
90
  return nil unless compressed
92
91
 
92
+ # Before anything that can raise: the rescue below names Zlib.
93
+ require 'zlib'
93
94
  Wurk.load_json(Zlib.inflate(Base64.decode64(compressed)))
94
95
  rescue Zlib::DataError, ArgumentError, ::JSON::ParserError
95
96
  nil
@@ -1,6 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'zlib'
4
3
  require_relative 'component'
5
4
  require_relative 'dead_set'
6
5
  require_relative 'fetcher/oom_drain'
@@ -337,6 +336,7 @@ module Wurk
337
336
  end
338
337
 
339
338
  def compress_backtrace(backtrace)
339
+ require 'zlib'
340
340
  serialized = Wurk.dump_json(backtrace)
341
341
  compressed = ::Zlib::Deflate.deflate(serialized)
342
342
  [compressed].pack('m0')