wurk 1.12.0 → 1.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -2
- data/app/controllers/concerns/wurk/sse_streaming.rb +36 -1
- data/app/controllers/wurk/api/pagination.rb +9 -7
- data/app/controllers/wurk/api/serializers.rb +8 -2
- data/app/controllers/wurk/api_controller.rb +57 -24
- data/app/controllers/wurk/application_controller.rb +0 -2
- data/app/controllers/wurk/dashboard_controller.rb +0 -1
- data/config/routes.rb +13 -1
- data/exe/sidekiq +8 -0
- data/exe/wurk +1 -1
- data/lib/generators/wurk/install/install_generator.rb +56 -0
- data/lib/sidekiq/cli.rb +4 -3
- data/lib/sidekiq/deploy.rb +2 -1
- data/lib/sidekiq/embedded.rb +2 -1
- data/lib/sidekiq/iterable_job.rb +2 -1
- data/lib/sidekiq/job/iterable/active_record_enumerator.rb +2 -1
- data/lib/sidekiq/job/iterable/csv_enumerator.rb +2 -1
- data/lib/sidekiq/job/iterable.rb +2 -1
- data/lib/sidekiq/launcher.rb +2 -1
- data/lib/sidekiq/manager.rb +2 -1
- data/lib/sidekiq/metrics/query.rb +2 -1
- data/lib/sidekiq/pro/web.rb +2 -1
- data/lib/sidekiq/scheduled.rb +2 -1
- data/lib/sidekiq/web.rb +2 -1
- data/lib/sidekiq.rb +3 -2
- data/lib/wurk/api/fast.rb +4 -18
- data/lib/wurk/api/queues.rb +12 -10
- data/lib/wurk/api/serializers.rb +6 -7
- data/lib/wurk/api/swarm.rb +0 -1
- data/lib/wurk/batch/server_middleware.rb +4 -0
- data/lib/wurk/batch/status.rb +45 -22
- data/lib/wurk/batch_set.rb +34 -0
- data/lib/wurk/boot_checks.rb +80 -0
- data/lib/wurk/cli.rb +72 -36
- data/lib/wurk/client/buffered.rb +97 -26
- data/lib/wurk/client.rb +21 -1
- data/lib/wurk/compat/deploy.rb +7 -0
- data/lib/wurk/compat/encryption.rb +7 -0
- data/lib/wurk/compat/flow.rb +8 -0
- data/lib/wurk/compat/iterable_job.rb +8 -0
- data/lib/wurk/compat/profile_set.rb +8 -0
- data/lib/wurk/compat/server.rb +15 -0
- data/lib/wurk/compat/web.rb +20 -0
- data/lib/wurk/compat.rb +34 -40
- data/lib/wurk/configuration.rb +73 -15
- data/lib/wurk/cron.rb +146 -47
- data/lib/wurk/dead_set.rb +10 -6
- data/lib/wurk/embedded.rb +0 -2
- data/lib/wurk/engine.rb +46 -4
- data/lib/wurk/error_coalescer.rb +21 -1
- data/lib/wurk/fetcher/many_queues.rb +8 -6
- data/lib/wurk/fetcher/orphan_reclaim.rb +48 -11
- data/lib/wurk/{middleware → fetcher}/poison_pill.rb +4 -2
- data/lib/wurk/fetcher/reaper.rb +28 -14
- data/lib/wurk/fetcher/reliable.rb +50 -32
- data/lib/wurk/fetcher/self_reconcile.rb +0 -1
- data/lib/wurk/fetcher/unit_of_work.rb +17 -2
- data/lib/wurk/flow/status.rb +10 -4
- data/lib/wurk/flow_set.rb +24 -2
- data/lib/wurk/health.rb +0 -1
- data/lib/wurk/heartbeat.rb +13 -7
- data/lib/wurk/import/sidekiq_cron.rb +0 -1
- data/lib/wurk/job_logger.rb +4 -2
- data/lib/wurk/job_record.rb +2 -1
- data/lib/wurk/job_retry.rb +1 -1
- data/lib/wurk/job_set.rb +132 -6
- data/lib/wurk/job_util.rb +14 -0
- data/lib/wurk/launcher.rb +30 -15
- data/lib/wurk/lazy_loader.rb +60 -0
- data/lib/wurk/leader.rb +1 -8
- data/lib/wurk/limiter/bucket.rb +0 -2
- data/lib/wurk/limiter/concurrent.rb +19 -36
- data/lib/wurk/limiter/leaky.rb +0 -2
- data/lib/wurk/limiter/points.rb +0 -2
- data/lib/wurk/limiter/server_middleware.rb +4 -1
- data/lib/wurk/limiter/unlimited.rb +0 -2
- data/lib/wurk/limiter/window.rb +0 -2
- data/lib/wurk/limiter.rb +0 -7
- data/lib/wurk/logger.rb +44 -5
- data/lib/wurk/lua/fast_delete.rb +58 -0
- data/lib/wurk/lua/fast_delete_window.lua +46 -0
- data/lib/wurk/lua/limiter_concurrent_acquire.lua +10 -3
- data/lib/wurk/lua/limiter_concurrent_release.lua +13 -4
- data/lib/wurk/lua.rb +1 -41
- data/lib/wurk/manager.rb +27 -1
- data/lib/wurk/metrics/dashboard_series.rb +38 -18
- data/lib/wurk/metrics/history.rb +24 -11
- data/lib/wurk/metrics/prometheus.rb +0 -1
- data/lib/wurk/metrics/query.rb +0 -2
- data/lib/wurk/metrics/queue_rollup.rb +33 -11
- data/lib/wurk/metrics/statsd.rb +4 -0
- data/lib/wurk/middleware/chain.rb +25 -3
- data/lib/wurk/middleware/expiry.rb +4 -0
- data/lib/wurk/middleware/interrupt_handler.rb +4 -1
- data/lib/wurk/middleware/status.rb +4 -0
- data/lib/wurk/middleware/timeout.rb +10 -6
- data/lib/wurk/preflight/app_scan.rb +182 -0
- data/lib/wurk/preflight/command.rb +21 -5
- data/lib/wurk/preflight/owners.rb +0 -1
- data/lib/wurk/preflight/payload_sample.rb +0 -1
- data/lib/wurk/preflight/redis_guard.rb +98 -0
- data/lib/wurk/preflight/verify.rb +0 -2
- data/lib/wurk/preflight.rb +9 -5
- data/lib/wurk/process_set.rb +6 -1
- data/lib/wurk/profiler.rb +2 -1
- data/lib/wurk/queue.rb +33 -6
- data/lib/wurk/queue_consumers.rb +10 -1
- data/lib/wurk/rails_boot.rb +9 -4
- data/lib/wurk/railtie.rb +3 -4
- data/lib/wurk/redis_options.rb +5 -0
- data/lib/wurk/redis_pool.rb +24 -1
- data/lib/wurk/sentry/error_handler.rb +5 -0
- data/lib/wurk/sentry/middleware.rb +4 -0
- data/lib/wurk/sentry.rb +11 -0
- data/lib/wurk/sorted_entry.rb +16 -12
- data/lib/wurk/stats.rb +85 -1
- data/lib/wurk/swarm/child_boot.rb +0 -2
- data/lib/wurk/swarm/liveness.rb +0 -2
- data/lib/wurk/swarm/memory_watch.rb +116 -0
- data/lib/wurk/swarm/prefork.rb +54 -0
- data/lib/wurk/swarm.rb +95 -55
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/enterprise.rb +63 -7
- data/lib/wurk/web/rack_app.rb +8 -6
- data/lib/wurk/web/stats_routes.rb +93 -0
- data/lib/wurk.rb +34 -30
- data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js +1 -0
- data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.br +0 -0
- data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.gz +0 -0
- data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js +1 -0
- data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.br +0 -0
- data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js +1 -0
- data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.br +0 -0
- data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js +1 -0
- data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.br +0 -0
- data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js +1 -0
- data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.br +0 -0
- data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js +1 -0
- data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.br +0 -0
- data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js +1 -0
- data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.br +0 -0
- data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js +1 -0
- data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.br +0 -0
- data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.gz +0 -0
- data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js +2 -0
- data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.br +0 -0
- data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.gz +0 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js +1 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.br +0 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{FlowState-yb7virqR.js → FlowState-ClzInw0-.js} +1 -1
- data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.br +0 -0
- data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js +1 -0
- data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.br +0 -0
- data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.gz +0 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js +2 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.br +0 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js +1 -0
- data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.br +0 -0
- data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js +1 -0
- data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.br +0 -0
- data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.gz +0 -0
- data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js +1 -0
- data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.br +0 -0
- data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.gz +0 -0
- data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js +1 -0
- data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.br +0 -0
- data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js +1 -0
- data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.br +0 -0
- data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js +1 -0
- data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.br +0 -0
- data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js +1 -0
- data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.br +0 -0
- data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js +1 -0
- data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.br +0 -0
- data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Search-D42Va7lB.js +1 -0
- data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.br +0 -0
- data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.gz +0 -0
- data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js +1 -0
- data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.br +0 -0
- data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.gz +0 -0
- data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js +1 -0
- data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.br +0 -0
- data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.gz +0 -0
- data/vendor/assets/dashboard/assets/de-B2NCIuzo.js +1 -0
- data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.br +0 -0
- data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.gz +0 -0
- data/vendor/assets/dashboard/assets/es-BGeu_JNB.js +1 -0
- data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.br +0 -0
- data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.gz +0 -0
- data/vendor/assets/dashboard/assets/fr-DQVBWFId.js +1 -0
- data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.br +0 -0
- data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.gz +0 -0
- data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js +1 -0
- data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.br +0 -0
- data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.gz +0 -0
- data/vendor/assets/dashboard/assets/index-CiLU1zz8.js +141 -0
- data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.br +0 -0
- data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.gz +0 -0
- data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css +1 -0
- data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.br +0 -0
- data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.gz +0 -0
- data/vendor/assets/dashboard/assets/ja-D06IBqzL.js +1 -0
- data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.br +0 -0
- data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.gz +0 -0
- data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js +1 -0
- data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.br +0 -0
- data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.gz +0 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js +1 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.br +0 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.gz +0 -0
- data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js +1 -0
- data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.br +0 -0
- data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{utils-CnWZzW2L.js → utils-69XSoGbQ.js} +1 -1
- data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.br +0 -0
- data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.gz +0 -0
- data/vendor/assets/dashboard/assets/wurk-logo-ydeSB9B8.png +0 -0
- data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js +1 -0
- data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.br +0 -0
- data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.gz +0 -0
- data/vendor/assets/dashboard/index.html +3 -23
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +132 -38
- data/vendor/assets/dashboard/assets/ArgsValue-BJol9S0s.js +0 -1
- data/vendor/assets/dashboard/assets/BatchDetail-D_zGQ9Sd.js +0 -1
- data/vendor/assets/dashboard/assets/Batches-Dt_xC660.js +0 -1
- data/vendor/assets/dashboard/assets/Busy-CWRthsmc.js +0 -1
- data/vendor/assets/dashboard/assets/Cron-qeGr2YAm.js +0 -1
- data/vendor/assets/dashboard/assets/Dashboard-DQWaAwPA.js +0 -1
- data/vendor/assets/dashboard/assets/Dead-D9gOPHtK.js +0 -1
- data/vendor/assets/dashboard/assets/Extension-DONXVLdA.js +0 -1
- data/vendor/assets/dashboard/assets/FilterBox-BbB6gXHp.js +0 -2
- data/vendor/assets/dashboard/assets/FlowDetail-CrmnVRp0.js +0 -1
- data/vendor/assets/dashboard/assets/Flows-g4GC8KmE.js +0 -1
- data/vendor/assets/dashboard/assets/JobDetailModal-C86k_1wj.js +0 -2
- data/vendor/assets/dashboard/assets/Limiters-zyO-iMlK.js +0 -1
- data/vendor/assets/dashboard/assets/Metrics-Ckq5fNBv.js +0 -1
- data/vendor/assets/dashboard/assets/NotFound-C8NnyjlY.js +0 -1
- data/vendor/assets/dashboard/assets/PageHeader-vWeBbSPF.js +0 -1
- data/vendor/assets/dashboard/assets/Profiles-DRl-FXPA.js +0 -1
- data/vendor/assets/dashboard/assets/Queues-DkXWrihA.js +0 -1
- data/vendor/assets/dashboard/assets/Retries-CmlojYIy.js +0 -1
- data/vendor/assets/dashboard/assets/Scheduled-Cbbpp3xw.js +0 -1
- data/vendor/assets/dashboard/assets/Search-LaEpj2fM.js +0 -1
- data/vendor/assets/dashboard/assets/charts-D_PQLoyc.js +0 -1
- data/vendor/assets/dashboard/assets/fa-brands-400-Bs6tcqqs.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-regular-400-DRN8N0d1.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-solid-900-IAB4Droh.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-v4compatibility-CErXDOsT.woff2 +0 -0
- data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +0 -1
- data/vendor/assets/dashboard/assets/index--xvvNlYx.css +0 -1
- data/vendor/assets/dashboard/assets/index-FBKGtysy.js +0 -141
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-BiwhHqgh.js +0 -1
- data/vendor/assets/dashboard/assets/useSort-jBYVwXvg.js +0 -1
- data/vendor/assets/dashboard/assets/wurk-logo-Vy3xW4K0.png +0 -0
- data/vendor/assets/dashboard/og-image.png +0 -0
- data/vendor/assets/dashboard/wurk-logo.webp +0 -0
data/lib/wurk/client/buffered.rb
CHANGED
|
@@ -58,7 +58,9 @@ module Wurk
|
|
|
58
58
|
# Returned by the append helpers when everything fit.
|
|
59
59
|
NOTHING_UNDELIVERED = [].freeze
|
|
60
60
|
|
|
61
|
-
|
|
61
|
+
# `bytes` is the payload's JSON size, what `reliable_push_buffer_bytes`
|
|
62
|
+
# counts.
|
|
63
|
+
Entry = Struct.new(:payload, :origin, :bytes)
|
|
62
64
|
|
|
63
65
|
# Where a buffered payload has to be replayed. What must NOT be captured
|
|
64
66
|
# is a pool object the config hands out: `reset_redis_pools!` — every
|
|
@@ -159,6 +161,7 @@ module Wurk
|
|
|
159
161
|
@install_mutex = Mutex.new
|
|
160
162
|
@buffer_mutex = Mutex.new
|
|
161
163
|
@replaying = 0
|
|
164
|
+
@buffer_bytes = 0
|
|
162
165
|
@circuit = Circuit.new
|
|
163
166
|
|
|
164
167
|
# Process that owns the state above; a mismatch means we're running in a
|
|
@@ -167,7 +170,7 @@ module Wurk
|
|
|
167
170
|
|
|
168
171
|
class << self
|
|
169
172
|
# Idempotent. Prepends the wrapper module into Wurk::Client so push /
|
|
170
|
-
# push_bulk drain the buffer before each call and raw_push catches
|
|
173
|
+
# push_bulk / push_prepared drain the buffer before each call and raw_push catches
|
|
171
174
|
# connection errors. Safe to call from multiple threads.
|
|
172
175
|
def install!
|
|
173
176
|
install_mutex.synchronize do
|
|
@@ -195,10 +198,30 @@ module Wurk
|
|
|
195
198
|
@buffer_cap = value
|
|
196
199
|
end
|
|
197
200
|
|
|
201
|
+
# Optional second cap, in bytes of payload JSON (Wurk-only; nil, the
|
|
202
|
+
# default, leaves only the entry cap). The entry cap alone lets a burst
|
|
203
|
+
# of large payloads hold `cap x payload size` in a web process for the
|
|
204
|
+
# whole outage. Over either cap the overflow mode applies; a payload
|
|
205
|
+
# larger than this cap on its own never fits, so it is the one dropped
|
|
206
|
+
# (or raised).
|
|
207
|
+
attr_reader :buffer_bytes_cap
|
|
208
|
+
|
|
209
|
+
def buffer_bytes_cap=(value)
|
|
210
|
+
unless value.nil? || (value.is_a?(Integer) && value.positive?)
|
|
211
|
+
raise ArgumentError, 'reliable_push_buffer_bytes must be nil or a positive Integer'
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
@buffer_bytes_cap = value
|
|
215
|
+
end
|
|
216
|
+
|
|
198
217
|
def buffer_size
|
|
199
218
|
buffer_mutex.synchronize { buffer.size }
|
|
200
219
|
end
|
|
201
220
|
|
|
221
|
+
def buffer_bytes
|
|
222
|
+
buffer_mutex.synchronize { @buffer_bytes }
|
|
223
|
+
end
|
|
224
|
+
|
|
202
225
|
def overflow_mode
|
|
203
226
|
@overflow_mode ||= DEFAULT_OVERFLOW_MODE
|
|
204
227
|
end
|
|
@@ -220,7 +243,9 @@ module Wurk
|
|
|
220
243
|
def reset!
|
|
221
244
|
buffer_mutex.synchronize do
|
|
222
245
|
@buffer = []
|
|
246
|
+
@buffer_bytes = 0
|
|
223
247
|
@buffer_cap = nil
|
|
248
|
+
@buffer_bytes_cap = nil
|
|
224
249
|
@overflow_mode = nil
|
|
225
250
|
@replaying = 0
|
|
226
251
|
end
|
|
@@ -263,6 +288,7 @@ module Wurk
|
|
|
263
288
|
@install_mutex = Mutex.new
|
|
264
289
|
@buffer_mutex = Mutex.new
|
|
265
290
|
@buffer = []
|
|
291
|
+
@buffer_bytes = 0
|
|
266
292
|
@replaying = 0
|
|
267
293
|
@circuit = Circuit.new
|
|
268
294
|
@drops = DropLog.new
|
|
@@ -283,29 +309,34 @@ module Wurk
|
|
|
283
309
|
# `client` is the one whose push failed; every payload is tagged with
|
|
284
310
|
# the Redis that client was writing to (see Origin).
|
|
285
311
|
def enbuffer(payloads, client:)
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
cap = buffer_cap
|
|
312
|
+
entries = entries_for(payloads, client)
|
|
313
|
+
caps = [buffer_cap, buffer_bytes_cap]
|
|
290
314
|
mode = overflow_mode
|
|
291
315
|
undelivered = buffer_mutex.synchronize do
|
|
292
|
-
mode == :raise ? append_within_capacity(entries,
|
|
316
|
+
mode == :raise ? append_within_capacity(entries, *caps) : append_dropping_oldest(entries, *caps)
|
|
293
317
|
end
|
|
294
318
|
|
|
295
319
|
raise Overflow, undelivered.map(&:payload) unless undelivered.empty?
|
|
296
320
|
ensure
|
|
297
|
-
drops.report(
|
|
321
|
+
drops.report(*caps)
|
|
298
322
|
end
|
|
299
323
|
|
|
300
324
|
private
|
|
301
325
|
|
|
326
|
+
def entries_for(payloads, client)
|
|
327
|
+
origin = Origin.for(client)
|
|
328
|
+
payloads.map { |payload| Entry.new(payload, origin, Wurk.dump_json(payload).bytesize) }
|
|
329
|
+
end
|
|
330
|
+
|
|
302
331
|
# Both append helpers run with buffer_mutex held and return the entries
|
|
303
332
|
# they could not take.
|
|
304
333
|
|
|
305
|
-
def append_dropping_oldest(entries, cap)
|
|
334
|
+
def append_dropping_oldest(entries, cap, bytes_cap)
|
|
306
335
|
entries.each do |entry|
|
|
307
|
-
drops.record(
|
|
308
|
-
|
|
336
|
+
next drops.record(entry.payload) if bytes_cap && entry.bytes > bytes_cap
|
|
337
|
+
|
|
338
|
+
drops.record(take_oldest.payload) until fits?(entry, cap, bytes_cap)
|
|
339
|
+
push_entry(entry)
|
|
309
340
|
end
|
|
310
341
|
NOTHING_UNDELIVERED
|
|
311
342
|
end
|
|
@@ -316,15 +347,38 @@ module Wurk
|
|
|
316
347
|
# negative when the cap was lowered after the buffer filled — clamped,
|
|
317
348
|
# so an over-full buffer rejects the whole call instead of raising on
|
|
318
349
|
# `first`/`drop`.
|
|
319
|
-
def append_within_capacity(entries, cap)
|
|
350
|
+
def append_within_capacity(entries, cap, bytes_cap)
|
|
320
351
|
room = (cap - buffer.size).clamp(0, entries.size)
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
352
|
+
room = bytes_room(entries.first(room), bytes_cap) if bytes_cap
|
|
353
|
+
entries.first(room).each { |entry| push_entry(entry) }
|
|
354
|
+
room == entries.size ? NOTHING_UNDELIVERED : entries.drop(room)
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
# How many of `entries`, in order, fit under the byte cap.
|
|
358
|
+
def bytes_room(entries, bytes_cap)
|
|
359
|
+
total = @buffer_bytes
|
|
360
|
+
entries.take_while { |entry| (total += entry.bytes) <= bytes_cap }.size
|
|
361
|
+
end
|
|
325
362
|
|
|
326
|
-
|
|
327
|
-
|
|
363
|
+
def fits?(entry, cap, bytes_cap)
|
|
364
|
+
buffer.size < cap && (bytes_cap.nil? || @buffer_bytes + entry.bytes <= bytes_cap)
|
|
365
|
+
end
|
|
366
|
+
|
|
367
|
+
# The three buffer mutations, each under buffer_mutex, each keeping the
|
|
368
|
+
# byte total (`@buffer_bytes`) in step.
|
|
369
|
+
def push_entry(entry)
|
|
370
|
+
@buffer_bytes += entry.bytes
|
|
371
|
+
buffer << entry
|
|
372
|
+
end
|
|
373
|
+
|
|
374
|
+
def take_oldest
|
|
375
|
+
remove_at(0)
|
|
376
|
+
end
|
|
377
|
+
|
|
378
|
+
def remove_at(index)
|
|
379
|
+
entry = buffer.delete_at(index)
|
|
380
|
+
@buffer_bytes -= entry.bytes
|
|
381
|
+
entry
|
|
328
382
|
end
|
|
329
383
|
|
|
330
384
|
public
|
|
@@ -508,7 +562,7 @@ module Wurk
|
|
|
508
562
|
next unless index
|
|
509
563
|
|
|
510
564
|
@replaying += 1
|
|
511
|
-
|
|
565
|
+
remove_at(index)
|
|
512
566
|
end
|
|
513
567
|
end
|
|
514
568
|
|
|
@@ -550,7 +604,10 @@ module Wurk
|
|
|
550
604
|
end
|
|
551
605
|
|
|
552
606
|
def restore(entry)
|
|
553
|
-
buffer_mutex.synchronize
|
|
607
|
+
buffer_mutex.synchronize do
|
|
608
|
+
@buffer_bytes += entry.bytes
|
|
609
|
+
buffer.unshift(entry)
|
|
610
|
+
end
|
|
554
611
|
end
|
|
555
612
|
|
|
556
613
|
# Drain marks the thread so our prepended raw_push re-raises the
|
|
@@ -587,7 +644,7 @@ module Wurk
|
|
|
587
644
|
@lock.synchronize { @pending << payload }
|
|
588
645
|
end
|
|
589
646
|
|
|
590
|
-
def report(cap)
|
|
647
|
+
def report(cap, bytes_cap = nil)
|
|
591
648
|
dropped, announce = @lock.synchronize do
|
|
592
649
|
next [nil, false] if @pending.empty?
|
|
593
650
|
|
|
@@ -599,7 +656,7 @@ module Wurk
|
|
|
599
656
|
end
|
|
600
657
|
return unless dropped
|
|
601
658
|
|
|
602
|
-
log(dropped.first, cap) if announce
|
|
659
|
+
log(dropped.first, cap, bytes_cap) if announce
|
|
603
660
|
dropped.size.times { count_drop }
|
|
604
661
|
end
|
|
605
662
|
|
|
@@ -620,13 +677,14 @@ module Wurk
|
|
|
620
677
|
nil
|
|
621
678
|
end
|
|
622
679
|
|
|
623
|
-
def log(first, cap)
|
|
680
|
+
def log(first, cap, bytes_cap)
|
|
624
681
|
Wurk.configuration.logger.error do
|
|
625
|
-
"reliable_push buffer full (cap=#{cap}
|
|
682
|
+
"reliable_push buffer full (cap=#{cap}#{", bytes=#{bytes_cap}" if bytes_cap}): " \
|
|
683
|
+
'dropping the oldest buffered jobs to make room — ' \
|
|
626
684
|
"#{first['class']} jid=#{first['jid']} is the first of this burst and will never run. " \
|
|
627
685
|
'Further drops are counted in statsd jobs.dropped.push until Redis takes a replay. Raise ' \
|
|
628
|
-
'`Wurk::Client.reliable_push_buffer
|
|
629
|
-
'overflow yourself.'
|
|
686
|
+
'`Wurk::Client.reliable_push_buffer` (or `reliable_push_buffer_bytes`), or set ' \
|
|
687
|
+
'`reliable_push_overflow = :raise` to handle overflow yourself.'
|
|
630
688
|
end
|
|
631
689
|
rescue StandardError
|
|
632
690
|
nil
|
|
@@ -742,6 +800,11 @@ module Wurk
|
|
|
742
800
|
super
|
|
743
801
|
end
|
|
744
802
|
|
|
803
|
+
def push_prepared(payloads)
|
|
804
|
+
Buffered.drain!(self) unless Buffered.unbuffered?
|
|
805
|
+
super
|
|
806
|
+
end
|
|
807
|
+
|
|
745
808
|
private
|
|
746
809
|
|
|
747
810
|
# Opens the delivery ledger Client writes into. It lives here rather
|
|
@@ -851,6 +914,14 @@ module Wurk
|
|
|
851
914
|
Buffered.buffer_cap = value
|
|
852
915
|
end
|
|
853
916
|
|
|
917
|
+
def reliable_push_buffer_bytes
|
|
918
|
+
Buffered.buffer_bytes_cap
|
|
919
|
+
end
|
|
920
|
+
|
|
921
|
+
def reliable_push_buffer_bytes=(value)
|
|
922
|
+
Buffered.buffer_bytes_cap = value
|
|
923
|
+
end
|
|
924
|
+
|
|
854
925
|
def reliable_push_overflow
|
|
855
926
|
Buffered.overflow_mode
|
|
856
927
|
end
|
data/lib/wurk/client.rb
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require_relative 'iterable_job'
|
|
4
3
|
require_relative 'job_util'
|
|
5
4
|
require_relative 'lua'
|
|
6
5
|
require_relative 'status'
|
|
@@ -90,6 +89,27 @@ module Wurk
|
|
|
90
89
|
payload['jid']
|
|
91
90
|
end
|
|
92
91
|
|
|
92
|
+
# #push in two halves, for a caller that pushes many jobs in one write
|
|
93
|
+
# (JobSet#retry_all): #prepare_push runs one item through normalize →
|
|
94
|
+
# middleware → verify, #push_prepared writes what came back. Internal.
|
|
95
|
+
#
|
|
96
|
+
# @return [Hash, nil] the payload to write; nil when middleware halted it.
|
|
97
|
+
def prepare_push(item)
|
|
98
|
+
payload = invoke_chain(normalize_item(item))
|
|
99
|
+
verify_json(payload) if payload
|
|
100
|
+
payload
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# One #raw_push for every payload, so they should share a queue, a batch
|
|
104
|
+
# or none, and a schedule or none: anything else is several writes, and one
|
|
105
|
+
# can land before another fails.
|
|
106
|
+
# @return [Array<String>] jids
|
|
107
|
+
def push_prepared(payloads)
|
|
108
|
+
buffered = raw_push(payloads)
|
|
109
|
+
emit_enqueued(payloads, buffered)
|
|
110
|
+
payloads.map { |payload| payload['jid'] }
|
|
111
|
+
end
|
|
112
|
+
|
|
93
113
|
# @param items [Hash] String keys only: class, args (Array<Array>), at?, spread_interval?, batch_size?, jid?
|
|
94
114
|
# @return [Array<String, nil>] jids in submission order; nil entries mark middleware-halted jobs.
|
|
95
115
|
def push_bulk(items)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Autoload target of the server tier's Sidekiq aliases (compat.rb). The Wurk
|
|
4
|
+
# files define only their Wurk names; this assigns the Sidekiq ones.
|
|
5
|
+
|
|
6
|
+
module Sidekiq
|
|
7
|
+
CLI = Wurk::CLI
|
|
8
|
+
Embedded = Wurk::Embedded
|
|
9
|
+
Launcher = Wurk::Launcher
|
|
10
|
+
Manager = Wurk::Manager
|
|
11
|
+
Scheduled = Wurk::Scheduled
|
|
12
|
+
History = Wurk::History
|
|
13
|
+
Periodic = Wurk::Cron
|
|
14
|
+
CronParser = Wurk::Cron::Parser
|
|
15
|
+
end
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Autoload target of the dashboard's Sidekiq aliases (compat.rb).
|
|
4
|
+
|
|
5
|
+
module Sidekiq
|
|
6
|
+
Web = Wurk::Web
|
|
7
|
+
WebHelpers = Wurk::Web::Extension::Helpers
|
|
8
|
+
Paginator = Wurk::Web::Paginator
|
|
9
|
+
|
|
10
|
+
module Pro
|
|
11
|
+
# Sidekiq Pro mounts the dashboard at `Sidekiq::Pro::Web`; it is the same
|
|
12
|
+
# board as `Sidekiq::Web` here, so a Pro app's `mount Sidekiq::Pro::Web`
|
|
13
|
+
# drops in unchanged.
|
|
14
|
+
Web = Wurk::Web
|
|
15
|
+
|
|
16
|
+
# `use Sidekiq::Pro::BatchStatus` — the polling Rack middleware that serves
|
|
17
|
+
# GET /batch_status/<bid>.json. Spec: docs/target/sidekiq-pro.md §10.3.
|
|
18
|
+
BatchStatus = Wurk::Web::BatchStatus
|
|
19
|
+
end
|
|
20
|
+
end
|
data/lib/wurk/compat.rb
CHANGED
|
@@ -50,16 +50,6 @@ module Sidekiq
|
|
|
50
50
|
# classes under them — but `Sidekiq.pro?` / `Sidekiq.ent?` still return
|
|
51
51
|
# `false` per docs/target/sidekiq-free.md §32 (Wurk advertises as free OSS).
|
|
52
52
|
module Pro
|
|
53
|
-
# Sidekiq Pro mounts the dashboard at `Sidekiq::Pro::Web`; it's the same
|
|
54
|
-
# board as `Sidekiq::Web` here, so a Pro app's `mount Sidekiq::Pro::Web`
|
|
55
|
-
# drops in unchanged. `Wurk::Web` is already required by the time this file
|
|
56
|
-
# loads (lib/wurk.rb requires wurk/web before wurk/compat).
|
|
57
|
-
Web = Wurk::Web
|
|
58
|
-
|
|
59
|
-
# `use Sidekiq::Pro::BatchStatus` — the polling Rack middleware that serves
|
|
60
|
-
# GET /batch_status/<bid>.json. Spec: docs/target/sidekiq-pro.md §10.3.
|
|
61
|
-
BatchStatus = Wurk::Web::BatchStatus
|
|
62
|
-
|
|
63
53
|
# Pro module-level statsd accessor (spec §1): `Sidekiq::Pro.dogstatsd = ...`.
|
|
64
54
|
# Delegates to the canonical `config.dogstatsd=` (spec §9.1), so either form
|
|
65
55
|
# feeds the same Wurk::Metrics::Statsd client.
|
|
@@ -119,38 +109,14 @@ module Sidekiq
|
|
|
119
109
|
private_constant :RailsNamespace
|
|
120
110
|
|
|
121
111
|
BasicFetch = Wurk::Fetcher::Reliable
|
|
122
|
-
WebHelpers = Wurk::Web::Extension::Helpers
|
|
123
|
-
Paginator = Wurk::Web::Paginator
|
|
124
112
|
Batch = Wurk::Batch
|
|
125
113
|
BatchSet = Wurk::BatchSet
|
|
126
114
|
Capsule = Wurk::Capsule
|
|
127
|
-
CLI = Wurk::CLI
|
|
128
115
|
Client = Wurk::Client
|
|
129
116
|
Component = Wurk::Component
|
|
130
117
|
Config = Wurk::Configuration
|
|
131
118
|
Context = Wurk::Context
|
|
132
|
-
# No `Sidekiq::Cron` alias on purpose (#204): real Sidekiq never defines it
|
|
133
|
-
# — that namespace belongs to the third-party sidekiq-cron gem, and squatting
|
|
134
|
-
# it made the gem's own classes (Poller, Job) collide at load. Wurk's native
|
|
135
|
-
# periodic surface is `Sidekiq::Periodic` (Ent parity) / `Wurk::Cron`.
|
|
136
|
-
CronParser = Wurk::Cron::Parser
|
|
137
|
-
Periodic = Wurk::Cron
|
|
138
119
|
DeadSet = Wurk::DeadSet
|
|
139
|
-
Deploy = Wurk::Deploy
|
|
140
|
-
Embedded = Wurk::Embedded
|
|
141
|
-
Encryption = Wurk::Encryption
|
|
142
|
-
# A Wurk extra, not parity — aliased because nothing else claims the name.
|
|
143
|
-
# Checked before adding it, the way `Sidekiq::Cron` (#204) and
|
|
144
|
-
# `Sidekiq::Status` taught us to: upstream Sidekiq defines no `Flow` anywhere
|
|
145
|
-
# in `lib/` at the pinned parity SHA, RubyGems has no `sidekiq-flow` or
|
|
146
|
-
# `sidekiq_flow` gem, and the only `Sidekiq::Flow` in public Ruby is a
|
|
147
|
-
# `FlowIdMiddleware` nested under a gem's *own* `Sidekiq` module, which this
|
|
148
|
-
# constant cannot reach.
|
|
149
|
-
Flow = Wurk::Flow
|
|
150
|
-
FlowSet = Wurk::FlowSet
|
|
151
|
-
History = Wurk::History
|
|
152
|
-
IterableJob = Wurk::IterableJob
|
|
153
|
-
IterableJobQuery = Wurk::IterableJobQuery
|
|
154
120
|
Job = Wurk::Job
|
|
155
121
|
JobLogger = Wurk::JobLogger
|
|
156
122
|
JobRecord = Wurk::JobRecord
|
|
@@ -161,11 +127,9 @@ module Sidekiq
|
|
|
161
127
|
JobSet = Wurk::JobSet
|
|
162
128
|
JobUtil = Wurk::JobUtil
|
|
163
129
|
Keys = Wurk::Keys
|
|
164
|
-
Launcher = Wurk::Launcher
|
|
165
130
|
Limiter = Wurk::Limiter
|
|
166
131
|
Loader = Wurk::Loader
|
|
167
132
|
Logger = Wurk::Logger
|
|
168
|
-
Manager = Wurk::Manager
|
|
169
133
|
Metrics = Wurk::Metrics
|
|
170
134
|
Middleware = Wurk::Middleware
|
|
171
135
|
ServerMiddleware = Wurk::Middleware::ServerMiddleware
|
|
@@ -174,13 +138,10 @@ module Sidekiq
|
|
|
174
138
|
ProcessSet = Wurk::ProcessSet
|
|
175
139
|
Processor = Wurk::Processor
|
|
176
140
|
Profiler = Wurk::Profiler
|
|
177
|
-
ProfileSet = Wurk::ProfileSet
|
|
178
|
-
ProfileRecord = Wurk::ProfileRecord
|
|
179
141
|
Queue = Wurk::Queue
|
|
180
142
|
RedisClientAdapter = Wurk::RedisClientAdapter
|
|
181
143
|
RedisConnection = Wurk::RedisConnection
|
|
182
144
|
RetrySet = Wurk::RetrySet
|
|
183
|
-
Scheduled = Wurk::Scheduled
|
|
184
145
|
ScheduledSet = Wurk::ScheduledSet
|
|
185
146
|
Shutdown = Wurk::Shutdown
|
|
186
147
|
SortedEntry = Wurk::SortedEntry
|
|
@@ -200,12 +161,45 @@ module Sidekiq
|
|
|
200
161
|
TransactionAwareClient = Wurk::TransactionAwareClient
|
|
201
162
|
Queues = Wurk::Queues
|
|
202
163
|
EmptyQueueError = Wurk::Testing::EmptyQueueError
|
|
203
|
-
Web = Wurk::Web
|
|
204
164
|
Work = Wurk::Work
|
|
205
165
|
Worker = Wurk::Worker
|
|
206
166
|
Workers = Wurk::Workers
|
|
207
167
|
WorkSet = Wurk::WorkSet
|
|
208
168
|
|
|
169
|
+
# Aliases of lazy Wurk constants (Wurk::LazyLoader) are autoloads too, so
|
|
170
|
+
# `defined?(Sidekiq::Web)` holds before the load and the alias is the very
|
|
171
|
+
# object its target is. Each points at a file under lib/wurk/compat/ that
|
|
172
|
+
# loads the target and assigns the alias: the target's own file defines only
|
|
173
|
+
# the Wurk name, so an autoload onto it would leave `Sidekiq::X` undefined.
|
|
174
|
+
# Those files name their targets rather than require them: web.rb, swarm.rb
|
|
175
|
+
# and iterable_job.rb require subfiles that reopen the class before defining
|
|
176
|
+
# it, which a direct require would turn into a circular one.
|
|
177
|
+
#
|
|
178
|
+
# No `Sidekiq::Cron` alias on purpose (#204): real Sidekiq never defines it
|
|
179
|
+
# — that namespace belongs to the third-party sidekiq-cron gem, and squatting
|
|
180
|
+
# it made the gem's own classes (Poller, Job) collide at load. Wurk's native
|
|
181
|
+
# periodic surface is `Sidekiq::Periodic` (Ent parity) / `Wurk::Cron`.
|
|
182
|
+
#
|
|
183
|
+
# `Flow` is a Wurk extra, not parity — aliased because nothing else claims
|
|
184
|
+
# the name. Checked before adding it, the way `Sidekiq::Cron` (#204) and
|
|
185
|
+
# `Sidekiq::Status` taught us to: upstream Sidekiq defines no `Flow` anywhere
|
|
186
|
+
# in `lib/` at the pinned parity SHA, RubyGems has no `sidekiq-flow` or
|
|
187
|
+
# `sidekiq_flow` gem, and the only `Sidekiq::Flow` in public Ruby is a
|
|
188
|
+
# `FlowIdMiddleware` nested under a gem's *own* `Sidekiq` module, which this
|
|
189
|
+
# constant cannot reach.
|
|
190
|
+
Wurk::LazyLoader.register(
|
|
191
|
+
self,
|
|
192
|
+
CLI: 'compat/server', Embedded: 'compat/server', Launcher: 'compat/server',
|
|
193
|
+
Manager: 'compat/server', Scheduled: 'compat/server',
|
|
194
|
+
History: 'compat/server', Periodic: 'compat/server', CronParser: 'compat/server',
|
|
195
|
+
Web: 'compat/web', WebHelpers: 'compat/web', Paginator: 'compat/web',
|
|
196
|
+
ProfileSet: 'compat/profile_set', ProfileRecord: 'compat/profile_set',
|
|
197
|
+
Flow: 'compat/flow', FlowSet: 'compat/flow',
|
|
198
|
+
IterableJob: 'compat/iterable_job', IterableJobQuery: 'compat/iterable_job',
|
|
199
|
+
Deploy: 'compat/deploy', Encryption: 'compat/encryption'
|
|
200
|
+
)
|
|
201
|
+
Wurk::LazyLoader.register(Pro, Web: 'compat/web', BatchStatus: 'compat/web')
|
|
202
|
+
|
|
209
203
|
# Top-level Sidekiq.configure_server / configure_client / redis / logger
|
|
210
204
|
# delegate to Wurk's class methods. Third-party gems treat these as the
|
|
211
205
|
# canonical entry points.
|