wurk 1.12.0 → 1.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -2
- data/app/controllers/concerns/wurk/sse_streaming.rb +36 -1
- data/app/controllers/wurk/api/pagination.rb +9 -7
- data/app/controllers/wurk/api/serializers.rb +8 -2
- data/app/controllers/wurk/api_controller.rb +57 -24
- data/app/controllers/wurk/application_controller.rb +0 -2
- data/app/controllers/wurk/dashboard_controller.rb +0 -1
- data/config/routes.rb +13 -1
- data/exe/sidekiq +8 -0
- data/exe/wurk +1 -1
- data/lib/generators/wurk/install/install_generator.rb +56 -0
- data/lib/sidekiq/cli.rb +4 -3
- data/lib/sidekiq/deploy.rb +2 -1
- data/lib/sidekiq/embedded.rb +2 -1
- data/lib/sidekiq/iterable_job.rb +2 -1
- data/lib/sidekiq/job/iterable/active_record_enumerator.rb +2 -1
- data/lib/sidekiq/job/iterable/csv_enumerator.rb +2 -1
- data/lib/sidekiq/job/iterable.rb +2 -1
- data/lib/sidekiq/launcher.rb +2 -1
- data/lib/sidekiq/manager.rb +2 -1
- data/lib/sidekiq/metrics/query.rb +2 -1
- data/lib/sidekiq/pro/web.rb +2 -1
- data/lib/sidekiq/scheduled.rb +2 -1
- data/lib/sidekiq/web.rb +2 -1
- data/lib/sidekiq.rb +3 -2
- data/lib/wurk/api/fast.rb +4 -18
- data/lib/wurk/api/queues.rb +12 -10
- data/lib/wurk/api/serializers.rb +6 -7
- data/lib/wurk/api/swarm.rb +0 -1
- data/lib/wurk/batch/server_middleware.rb +4 -0
- data/lib/wurk/batch/status.rb +45 -22
- data/lib/wurk/batch_set.rb +34 -0
- data/lib/wurk/boot_checks.rb +80 -0
- data/lib/wurk/cli.rb +72 -36
- data/lib/wurk/client/buffered.rb +97 -26
- data/lib/wurk/client.rb +21 -1
- data/lib/wurk/compat/deploy.rb +7 -0
- data/lib/wurk/compat/encryption.rb +7 -0
- data/lib/wurk/compat/flow.rb +8 -0
- data/lib/wurk/compat/iterable_job.rb +8 -0
- data/lib/wurk/compat/profile_set.rb +8 -0
- data/lib/wurk/compat/server.rb +15 -0
- data/lib/wurk/compat/web.rb +20 -0
- data/lib/wurk/compat.rb +34 -40
- data/lib/wurk/configuration.rb +73 -15
- data/lib/wurk/cron.rb +146 -47
- data/lib/wurk/dead_set.rb +10 -6
- data/lib/wurk/embedded.rb +0 -2
- data/lib/wurk/engine.rb +46 -4
- data/lib/wurk/error_coalescer.rb +21 -1
- data/lib/wurk/fetcher/many_queues.rb +8 -6
- data/lib/wurk/fetcher/orphan_reclaim.rb +48 -11
- data/lib/wurk/{middleware → fetcher}/poison_pill.rb +4 -2
- data/lib/wurk/fetcher/reaper.rb +28 -14
- data/lib/wurk/fetcher/reliable.rb +50 -32
- data/lib/wurk/fetcher/self_reconcile.rb +0 -1
- data/lib/wurk/fetcher/unit_of_work.rb +17 -2
- data/lib/wurk/flow/status.rb +10 -4
- data/lib/wurk/flow_set.rb +24 -2
- data/lib/wurk/health.rb +0 -1
- data/lib/wurk/heartbeat.rb +13 -7
- data/lib/wurk/import/sidekiq_cron.rb +0 -1
- data/lib/wurk/job_logger.rb +4 -2
- data/lib/wurk/job_record.rb +2 -1
- data/lib/wurk/job_retry.rb +1 -1
- data/lib/wurk/job_set.rb +132 -6
- data/lib/wurk/job_util.rb +14 -0
- data/lib/wurk/launcher.rb +30 -15
- data/lib/wurk/lazy_loader.rb +60 -0
- data/lib/wurk/leader.rb +1 -8
- data/lib/wurk/limiter/bucket.rb +0 -2
- data/lib/wurk/limiter/concurrent.rb +19 -36
- data/lib/wurk/limiter/leaky.rb +0 -2
- data/lib/wurk/limiter/points.rb +0 -2
- data/lib/wurk/limiter/server_middleware.rb +4 -1
- data/lib/wurk/limiter/unlimited.rb +0 -2
- data/lib/wurk/limiter/window.rb +0 -2
- data/lib/wurk/limiter.rb +0 -7
- data/lib/wurk/logger.rb +44 -5
- data/lib/wurk/lua/fast_delete.rb +58 -0
- data/lib/wurk/lua/fast_delete_window.lua +46 -0
- data/lib/wurk/lua/limiter_concurrent_acquire.lua +10 -3
- data/lib/wurk/lua/limiter_concurrent_release.lua +13 -4
- data/lib/wurk/lua.rb +1 -41
- data/lib/wurk/manager.rb +27 -1
- data/lib/wurk/metrics/dashboard_series.rb +38 -18
- data/lib/wurk/metrics/history.rb +24 -11
- data/lib/wurk/metrics/prometheus.rb +0 -1
- data/lib/wurk/metrics/query.rb +0 -2
- data/lib/wurk/metrics/queue_rollup.rb +33 -11
- data/lib/wurk/metrics/statsd.rb +4 -0
- data/lib/wurk/middleware/chain.rb +25 -3
- data/lib/wurk/middleware/expiry.rb +4 -0
- data/lib/wurk/middleware/interrupt_handler.rb +4 -1
- data/lib/wurk/middleware/status.rb +4 -0
- data/lib/wurk/middleware/timeout.rb +10 -6
- data/lib/wurk/preflight/app_scan.rb +182 -0
- data/lib/wurk/preflight/command.rb +21 -5
- data/lib/wurk/preflight/owners.rb +0 -1
- data/lib/wurk/preflight/payload_sample.rb +0 -1
- data/lib/wurk/preflight/redis_guard.rb +98 -0
- data/lib/wurk/preflight/verify.rb +0 -2
- data/lib/wurk/preflight.rb +9 -5
- data/lib/wurk/process_set.rb +6 -1
- data/lib/wurk/profiler.rb +2 -1
- data/lib/wurk/queue.rb +33 -6
- data/lib/wurk/queue_consumers.rb +10 -1
- data/lib/wurk/rails_boot.rb +9 -4
- data/lib/wurk/railtie.rb +3 -4
- data/lib/wurk/redis_options.rb +5 -0
- data/lib/wurk/redis_pool.rb +24 -1
- data/lib/wurk/sentry/error_handler.rb +5 -0
- data/lib/wurk/sentry/middleware.rb +4 -0
- data/lib/wurk/sentry.rb +11 -0
- data/lib/wurk/sorted_entry.rb +16 -12
- data/lib/wurk/stats.rb +85 -1
- data/lib/wurk/swarm/child_boot.rb +0 -2
- data/lib/wurk/swarm/liveness.rb +0 -2
- data/lib/wurk/swarm/memory_watch.rb +116 -0
- data/lib/wurk/swarm/prefork.rb +54 -0
- data/lib/wurk/swarm.rb +95 -55
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/enterprise.rb +63 -7
- data/lib/wurk/web/rack_app.rb +8 -6
- data/lib/wurk/web/stats_routes.rb +93 -0
- data/lib/wurk.rb +34 -30
- data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js +1 -0
- data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.br +0 -0
- data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.gz +0 -0
- data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js +1 -0
- data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.br +0 -0
- data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js +1 -0
- data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.br +0 -0
- data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js +1 -0
- data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.br +0 -0
- data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js +1 -0
- data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.br +0 -0
- data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js +1 -0
- data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.br +0 -0
- data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js +1 -0
- data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.br +0 -0
- data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js +1 -0
- data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.br +0 -0
- data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.gz +0 -0
- data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js +2 -0
- data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.br +0 -0
- data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.gz +0 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js +1 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.br +0 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{FlowState-yb7virqR.js → FlowState-ClzInw0-.js} +1 -1
- data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.br +0 -0
- data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js +1 -0
- data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.br +0 -0
- data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.gz +0 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js +2 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.br +0 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js +1 -0
- data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.br +0 -0
- data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js +1 -0
- data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.br +0 -0
- data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.gz +0 -0
- data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js +1 -0
- data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.br +0 -0
- data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.gz +0 -0
- data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js +1 -0
- data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.br +0 -0
- data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js +1 -0
- data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.br +0 -0
- data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js +1 -0
- data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.br +0 -0
- data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js +1 -0
- data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.br +0 -0
- data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js +1 -0
- data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.br +0 -0
- data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Search-D42Va7lB.js +1 -0
- data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.br +0 -0
- data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.gz +0 -0
- data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js +1 -0
- data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.br +0 -0
- data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.gz +0 -0
- data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js +1 -0
- data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.br +0 -0
- data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.gz +0 -0
- data/vendor/assets/dashboard/assets/de-B2NCIuzo.js +1 -0
- data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.br +0 -0
- data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.gz +0 -0
- data/vendor/assets/dashboard/assets/es-BGeu_JNB.js +1 -0
- data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.br +0 -0
- data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.gz +0 -0
- data/vendor/assets/dashboard/assets/fr-DQVBWFId.js +1 -0
- data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.br +0 -0
- data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.gz +0 -0
- data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js +1 -0
- data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.br +0 -0
- data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.gz +0 -0
- data/vendor/assets/dashboard/assets/index-CiLU1zz8.js +141 -0
- data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.br +0 -0
- data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.gz +0 -0
- data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css +1 -0
- data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.br +0 -0
- data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.gz +0 -0
- data/vendor/assets/dashboard/assets/ja-D06IBqzL.js +1 -0
- data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.br +0 -0
- data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.gz +0 -0
- data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js +1 -0
- data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.br +0 -0
- data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.gz +0 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js +1 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.br +0 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.gz +0 -0
- data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js +1 -0
- data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.br +0 -0
- data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{utils-CnWZzW2L.js → utils-69XSoGbQ.js} +1 -1
- data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.br +0 -0
- data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.gz +0 -0
- data/vendor/assets/dashboard/assets/wurk-logo-ydeSB9B8.png +0 -0
- data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js +1 -0
- data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.br +0 -0
- data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.gz +0 -0
- data/vendor/assets/dashboard/index.html +3 -23
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +132 -38
- data/vendor/assets/dashboard/assets/ArgsValue-BJol9S0s.js +0 -1
- data/vendor/assets/dashboard/assets/BatchDetail-D_zGQ9Sd.js +0 -1
- data/vendor/assets/dashboard/assets/Batches-Dt_xC660.js +0 -1
- data/vendor/assets/dashboard/assets/Busy-CWRthsmc.js +0 -1
- data/vendor/assets/dashboard/assets/Cron-qeGr2YAm.js +0 -1
- data/vendor/assets/dashboard/assets/Dashboard-DQWaAwPA.js +0 -1
- data/vendor/assets/dashboard/assets/Dead-D9gOPHtK.js +0 -1
- data/vendor/assets/dashboard/assets/Extension-DONXVLdA.js +0 -1
- data/vendor/assets/dashboard/assets/FilterBox-BbB6gXHp.js +0 -2
- data/vendor/assets/dashboard/assets/FlowDetail-CrmnVRp0.js +0 -1
- data/vendor/assets/dashboard/assets/Flows-g4GC8KmE.js +0 -1
- data/vendor/assets/dashboard/assets/JobDetailModal-C86k_1wj.js +0 -2
- data/vendor/assets/dashboard/assets/Limiters-zyO-iMlK.js +0 -1
- data/vendor/assets/dashboard/assets/Metrics-Ckq5fNBv.js +0 -1
- data/vendor/assets/dashboard/assets/NotFound-C8NnyjlY.js +0 -1
- data/vendor/assets/dashboard/assets/PageHeader-vWeBbSPF.js +0 -1
- data/vendor/assets/dashboard/assets/Profiles-DRl-FXPA.js +0 -1
- data/vendor/assets/dashboard/assets/Queues-DkXWrihA.js +0 -1
- data/vendor/assets/dashboard/assets/Retries-CmlojYIy.js +0 -1
- data/vendor/assets/dashboard/assets/Scheduled-Cbbpp3xw.js +0 -1
- data/vendor/assets/dashboard/assets/Search-LaEpj2fM.js +0 -1
- data/vendor/assets/dashboard/assets/charts-D_PQLoyc.js +0 -1
- data/vendor/assets/dashboard/assets/fa-brands-400-Bs6tcqqs.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-regular-400-DRN8N0d1.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-solid-900-IAB4Droh.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-v4compatibility-CErXDOsT.woff2 +0 -0
- data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +0 -1
- data/vendor/assets/dashboard/assets/index--xvvNlYx.css +0 -1
- data/vendor/assets/dashboard/assets/index-FBKGtysy.js +0 -141
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-BiwhHqgh.js +0 -1
- data/vendor/assets/dashboard/assets/useSort-jBYVwXvg.js +0 -1
- data/vendor/assets/dashboard/assets/wurk-logo-Vy3xW4K0.png +0 -0
- data/vendor/assets/dashboard/og-image.png +0 -0
- data/vendor/assets/dashboard/wurk-logo.webp +0 -0
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Wurk
|
|
4
|
+
module Lua
|
|
5
|
+
# Pro Fast API (§11) deletes, `Queue#delete_job` and `#delete_by_class`: a
|
|
6
|
+
# cursor walk over the queue list from the head, one `fast_delete_window`
|
|
7
|
+
# call per WINDOW entries. The scripts this replaced copied the whole list
|
|
8
|
+
# into the Lua heap and LREMed per match, holding Redis for the entire
|
|
9
|
+
# queue (~300ms per call at 100k jobs).
|
|
10
|
+
#
|
|
11
|
+
# Each call is atomic and removes only payloads it matched, so nothing that
|
|
12
|
+
# did not match is ever deleted. Between calls the list moves:
|
|
13
|
+
# * pops from the tail (every fetcher: LMOVE/BLMOVE RIGHT, BRPOP) shift no
|
|
14
|
+
# head index, so they cannot make the walk skip an entry;
|
|
15
|
+
# * pushes at the head (LPUSH, every enqueue) shift entries already
|
|
16
|
+
# visited forward, into a window still to come: they are read again,
|
|
17
|
+
# which re-checks a non-match and costs nothing else;
|
|
18
|
+
# * the walk's own removals are subtracted from the cursor.
|
|
19
|
+
# What can make it miss a match is another removal ahead of the cursor
|
|
20
|
+
# between calls (a concurrent delete on the same queue, an LREM or LPOP
|
|
21
|
+
# by another client): the entries shift back past the cursor. Such a
|
|
22
|
+
# match stays in the queue and a second call finds it; nothing extra is
|
|
23
|
+
# ever removed.
|
|
24
|
+
module FastDelete
|
|
25
|
+
WINDOW = 1000
|
|
26
|
+
# Matches one call rewrites before it hands the cursor on: each is an
|
|
27
|
+
# LSET that walks the list to its index (see fast_delete_window.lua).
|
|
28
|
+
MAX_REMOVED_PER_CALL = 100
|
|
29
|
+
|
|
30
|
+
# No JSON payload starts with a NUL byte.
|
|
31
|
+
TOMBSTONE = "\0wurk-fast-delete\0"
|
|
32
|
+
|
|
33
|
+
module_function
|
|
34
|
+
|
|
35
|
+
# [payloads removed, first removed payload or nil].
|
|
36
|
+
def call(conn, key, needle, window: WINDOW, max_removed: MAX_REMOVED_PER_CALL)
|
|
37
|
+
cursor = 0
|
|
38
|
+
removed = 0
|
|
39
|
+
first = nil
|
|
40
|
+
loop do
|
|
41
|
+
argv = [needle, cursor, window, TOMBSTONE, max_removed]
|
|
42
|
+
cursor, count, payload = Loader.eval_cached(conn, :fast_delete_window, keys: [key], argv: argv)
|
|
43
|
+
removed += count
|
|
44
|
+
first ||= payload
|
|
45
|
+
return [removed, first] if cursor.negative?
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def delete_job(conn, key, jid)
|
|
50
|
+
call(conn, key, %("jid":"#{jid}")).last
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def delete_by_class(conn, key, klass)
|
|
54
|
+
call(conn, key, %("class":"#{klass}")).first
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
-- Pro Fast API (§11) delete, one window of a queue list at a time.
|
|
2
|
+
-- Lua::FastDelete walks the list from the head with a cursor and calls this per
|
|
3
|
+
-- window, so Redis is held for one window, never for a copy of the whole queue.
|
|
4
|
+
-- KEYS[1] = queue:<name>
|
|
5
|
+
-- ARGV[1] = needle: `"jid":"<jid>"` or `"class":"<klass>"`, a plain substring
|
|
6
|
+
-- ARGV[2] = cursor: index of the window's first element
|
|
7
|
+
-- ARGV[3] = window size
|
|
8
|
+
-- ARGV[4] = tombstone, a value no payload can equal
|
|
9
|
+
-- ARGV[5] = most matches one call rewrites
|
|
10
|
+
-- Returns {next cursor or -1 when the list is done, removed, first removed
|
|
11
|
+
-- payload or false}.
|
|
12
|
+
--
|
|
13
|
+
-- Every match in the window is LSET to the tombstone at the index LRANGE just
|
|
14
|
+
-- read it from, in this same atomic call, then one LREM removes exactly those:
|
|
15
|
+
-- only a payload that matched is ever removed, whatever runs between calls.
|
|
16
|
+
-- The LREM scans from the end nearer the window.
|
|
17
|
+
--
|
|
18
|
+
-- Each LSET walks the list to its index, so a window dense with matches deep
|
|
19
|
+
-- in a long list costs one such walk per match. The call therefore stops after
|
|
20
|
+
-- ARGV[5] matches and hands the rest of the window to the next call. The LREM
|
|
21
|
+
-- still scans to the window (no remove-by-index), which bounds a removing call
|
|
22
|
+
-- deep in a long list: measured on a 1M-entry queue, 1,000 matches halfway in
|
|
23
|
+
-- held Redis 17ms in one call uncapped, ~10ms per call capped at 100.
|
|
24
|
+
local key = KEYS[1]
|
|
25
|
+
local start = tonumber(ARGV[2])
|
|
26
|
+
local window = tonumber(ARGV[3])
|
|
27
|
+
local max_removed = tonumber(ARGV[5])
|
|
28
|
+
local items = redis.call('lrange', key, start, start + window - 1)
|
|
29
|
+
local removed, first, scanned = 0, false, #items
|
|
30
|
+
for i = 1, #items do
|
|
31
|
+
if string.find(items[i], ARGV[1], 1, true) then
|
|
32
|
+
redis.call('lset', key, start + i - 1, ARGV[4])
|
|
33
|
+
removed = removed + 1
|
|
34
|
+
if not first then first = items[i] end
|
|
35
|
+
if removed >= max_removed then
|
|
36
|
+
scanned = i
|
|
37
|
+
break
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
if removed > 0 then
|
|
42
|
+
local len = redis.call('llen', key)
|
|
43
|
+
redis.call('lrem', key, (start * 2 > len) and -removed or removed, ARGV[4])
|
|
44
|
+
end
|
|
45
|
+
if scanned == #items and #items < window then return {-1, removed, first} end
|
|
46
|
+
return {start + scanned - removed, removed, first}
|
|
@@ -5,11 +5,12 @@
|
|
|
5
5
|
-- ARGV[2] = lock_timeout seconds
|
|
6
6
|
-- ARGV[3] = slot_id (caller-generated; SHA-fingerprinted on the Ruby side)
|
|
7
7
|
-- ARGV[4] = ttl seconds
|
|
8
|
+
-- ARGV[5] = '' when this attempt follows no wait, else the whole seconds waited
|
|
8
9
|
-- Returns {acquired, held, reclaimed}. acquired: 1 if slot recorded, 0 if full.
|
|
9
10
|
-- held: live ZCARD gauge after this pass. reclaimed: expired slots evicted here.
|
|
10
|
-
-- The
|
|
11
|
-
-- side, so
|
|
12
|
-
-- undercount) and
|
|
11
|
+
-- The `held` counter and the `immediate` / `waited` + `wait_time` split are
|
|
12
|
+
-- bumped here, not on the Ruby side, so they stay atomic with the ZADD (a
|
|
13
|
+
-- process dying mid-acquire can't undercount) and cost no extra round trip.
|
|
13
14
|
local cs_key = KEYS[1]
|
|
14
15
|
local st_key = KEYS[2]
|
|
15
16
|
local t = redis.call('TIME')
|
|
@@ -26,6 +27,12 @@ local held = redis.call('ZCARD', cs_key)
|
|
|
26
27
|
if held < limit then
|
|
27
28
|
redis.call('ZADD', cs_key, now + lock_to, slot_id)
|
|
28
29
|
redis.call('HINCRBY', st_key, 'held', 1)
|
|
30
|
+
if ARGV[5] == '' then
|
|
31
|
+
redis.call('HINCRBY', st_key, 'immediate', 1)
|
|
32
|
+
else
|
|
33
|
+
redis.call('HINCRBY', st_key, 'waited', 1)
|
|
34
|
+
redis.call('HINCRBY', st_key, 'wait_time', ARGV[5])
|
|
35
|
+
end
|
|
29
36
|
redis.call('EXPIRE', cs_key, ttl)
|
|
30
37
|
redis.call('EXPIRE', st_key, ttl)
|
|
31
38
|
return {1, held + 1, reclaimed}
|
|
@@ -1,7 +1,16 @@
|
|
|
1
|
-
-- Concurrent limiter: release a held slot
|
|
2
|
-
-- was already reclaimed by the acquire path's
|
|
3
|
-
--
|
|
1
|
+
-- Concurrent limiter: release a held slot and book its metrics.
|
|
2
|
+
-- ZREM returns 0 when the slot was already reclaimed by the acquire path's
|
|
3
|
+
-- ZREMRANGEBYSCORE: the holder outran `lock_timeout`, counted as an overage.
|
|
4
4
|
-- KEYS[1] = lmtr-cs:<name>
|
|
5
|
+
-- KEYS[2] = lmtr-stats:<name>
|
|
5
6
|
-- ARGV[1] = slot_id
|
|
7
|
+
-- ARGV[2] = whole seconds the slot was held
|
|
8
|
+
-- ARGV[3] = ttl seconds
|
|
6
9
|
-- Returns 1 if removed, 0 if already reclaimed.
|
|
7
|
-
|
|
10
|
+
local removed = redis.call('ZREM', KEYS[1], ARGV[1])
|
|
11
|
+
if removed == 0 then
|
|
12
|
+
redis.call('HINCRBY', KEYS[2], 'overages', 1)
|
|
13
|
+
end
|
|
14
|
+
redis.call('HINCRBY', KEYS[2], 'held_time', ARGV[2])
|
|
15
|
+
redis.call('EXPIRE', KEYS[2], ARGV[3])
|
|
16
|
+
return removed
|
data/lib/wurk/lua.rb
CHANGED
|
@@ -172,45 +172,6 @@ module Wurk
|
|
|
172
172
|
return 0
|
|
173
173
|
LUA
|
|
174
174
|
|
|
175
|
-
# Pro Fast API (§11): server-side LRANGE+LREM to delete a single job by
|
|
176
|
-
# jid from a queue list. Pure-Ruby Queue#find_job + JobRecord#delete is
|
|
177
|
-
# O(N) round-trips; this is O(1) round-trip with O(N) Lua work.
|
|
178
|
-
# KEYS = [queue:<name>]
|
|
179
|
-
# ARGV = [jid]
|
|
180
|
-
# Returns the first payload removed, or nil (Lua false) when none matched —
|
|
181
|
-
# Pro's `delete_job` return value. Every match is removed, so a
|
|
182
|
-
# duplicate-jid corruption is cleaned up in the same pass.
|
|
183
|
-
FAST_DELETE_JOB = <<~LUA
|
|
184
|
-
local items = redis.call("lrange", KEYS[1], 0, -1)
|
|
185
|
-
local deleted = false
|
|
186
|
-
for i = 1, #items do
|
|
187
|
-
if string.find(items[i], '"jid":"' .. ARGV[1] .. '"', 1, true) then
|
|
188
|
-
if redis.call("lrem", KEYS[1], 1, items[i]) > 0 and not deleted then
|
|
189
|
-
deleted = items[i]
|
|
190
|
-
end
|
|
191
|
-
end
|
|
192
|
-
end
|
|
193
|
-
return deleted
|
|
194
|
-
LUA
|
|
195
|
-
|
|
196
|
-
# Pro Fast API (§11): server-side LRANGE+LREM removing every payload whose
|
|
197
|
-
# `"class":"<klass>"` field matches. Plain-text scan (no JSON parse) so
|
|
198
|
-
# it tolerates partial corruption — caller drops only well-formed matches.
|
|
199
|
-
# KEYS = [queue:<name>]
|
|
200
|
-
# ARGV = [klass]
|
|
201
|
-
# Returns the number of payloads removed.
|
|
202
|
-
FAST_DELETE_BY_CLASS = <<~LUA
|
|
203
|
-
local items = redis.call("lrange", KEYS[1], 0, -1)
|
|
204
|
-
local removed = 0
|
|
205
|
-
local needle = '"class":"' .. ARGV[1] .. '"'
|
|
206
|
-
for i = 1, #items do
|
|
207
|
-
if string.find(items[i], needle, 1, true) then
|
|
208
|
-
removed = removed + redis.call("lrem", KEYS[1], 1, items[i])
|
|
209
|
-
end
|
|
210
|
-
end
|
|
211
|
-
return removed
|
|
212
|
-
LUA
|
|
213
|
-
|
|
214
175
|
# Ent Periodic (§2): compare-and-swap the fire marks of `loops:<lid>`.
|
|
215
176
|
# The cron poller's leader gate is a cached read (`Component#leader?`,
|
|
216
177
|
# LEADER_CACHE_TTL_MS), so for a few seconds after a handover two
|
|
@@ -262,8 +223,6 @@ module Wurk
|
|
|
262
223
|
scheduled_claim: SCHEDULED_CLAIM,
|
|
263
224
|
reliable_requeue: RELIABLE_REQUEUE,
|
|
264
225
|
requeue_unit: REQUEUE_UNIT,
|
|
265
|
-
fast_delete_job: FAST_DELETE_JOB,
|
|
266
|
-
fast_delete_by_class: FAST_DELETE_BY_CLASS,
|
|
267
226
|
release_if_owner: RELEASE_IF_OWNER,
|
|
268
227
|
cron_claim_fire: CRON_CLAIM_FIRE
|
|
269
228
|
}.merge(FILE_SCRIPTS).freeze
|
|
@@ -275,3 +234,4 @@ module Wurk
|
|
|
275
234
|
end
|
|
276
235
|
|
|
277
236
|
require_relative 'lua/loader'
|
|
237
|
+
require_relative 'lua/fast_delete'
|
data/lib/wurk/manager.rb
CHANGED
|
@@ -167,12 +167,38 @@ module Wurk
|
|
|
167
167
|
|
|
168
168
|
jobs = cleanup.map(&:job).compact
|
|
169
169
|
logger.warn { "Terminating #{cleanup.size} busy threads" }
|
|
170
|
-
|
|
170
|
+
log_interrupted(jobs)
|
|
171
171
|
# `&.` like #quiet: a TERM in the pre-prepare! window (traps install
|
|
172
172
|
# before launcher.run) reaches here with no fetcher built yet.
|
|
173
173
|
capsule.fetcher&.bulk_requeue(jobs)
|
|
174
174
|
end
|
|
175
175
|
|
|
176
|
+
# One WARN per job a deploy cut short, because each one runs again from the
|
|
177
|
+
# top (docs/idempotency.md) and the operator chasing a double side effect
|
|
178
|
+
# needs its jid. Capped: a capsule can hold hundreds of threads.
|
|
179
|
+
REQUEUE_LOG_LIMIT = 20
|
|
180
|
+
private_constant :REQUEUE_LOG_LIMIT
|
|
181
|
+
|
|
182
|
+
def log_interrupted(jobs)
|
|
183
|
+
jobs.first(REQUEUE_LOG_LIMIT).each do |uow|
|
|
184
|
+
logger.warn { "Interrupting #{describe_job(uow)}: past the shutdown deadline, it is requeued to run again" }
|
|
185
|
+
end
|
|
186
|
+
extra = jobs.size - REQUEUE_LOG_LIMIT
|
|
187
|
+
logger.warn { "Interrupting #{extra} more in-flight jobs, which run again too" } if extra.positive?
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
# The payload is parsed here, off the hot path, because the fetcher never
|
|
191
|
+
# parses. A plug-in fetcher's unit of work may carry something that is not
|
|
192
|
+
# job JSON; it is named by its queue alone.
|
|
193
|
+
def describe_job(uow)
|
|
194
|
+
job = Wurk.load_json(uow.job)
|
|
195
|
+
job = {} unless job.is_a?(::Hash)
|
|
196
|
+
klass = job['wrapped'] || job['class']
|
|
197
|
+
"#{klass} jid=#{job['jid']} queue=#{uow.queue_name}"
|
|
198
|
+
rescue ::JSON::ParserError, ::TypeError
|
|
199
|
+
"job queue=#{uow.queue_name}"
|
|
200
|
+
end
|
|
201
|
+
|
|
176
202
|
# The @workers Set is mutated from Processor threads (processor_result)
|
|
177
203
|
# while the lifecycle methods read/iterate it from the manager thread.
|
|
178
204
|
# Snapshot under @plock, then act on the copy outside the lock.
|
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative '../keys'
|
|
4
|
-
require_relative 'rollup'
|
|
5
|
-
require_relative 'queue_rollup'
|
|
6
4
|
|
|
7
5
|
module Wurk
|
|
8
6
|
module Metrics
|
|
@@ -12,6 +10,13 @@ module Wurk
|
|
|
12
10
|
# the API can answer 400. The Sidekiq-shaped API is Query's instance side.
|
|
13
11
|
module DashboardSeries
|
|
14
12
|
TOTAL_FIELDS = %w[p f ms].freeze
|
|
13
|
+
TOTAL_KINDS = { 'p' => :p, 'f' => :f, 'ms' => :ms }.freeze
|
|
14
|
+
NO_FIELD = [nil, nil].freeze
|
|
15
|
+
|
|
16
|
+
# Buckets per pipelined read in #top_jobs. A 30-day window is 4320
|
|
17
|
+
# buckets of every class's fields; reading and folding a day at a time
|
|
18
|
+
# keeps one day's replies in memory instead of the whole month's.
|
|
19
|
+
TOP_JOBS_SLICE = 144
|
|
15
20
|
|
|
16
21
|
# Dashboard top-N: `[[klass, {p:, f:, ms:}], …]` over the window,
|
|
17
22
|
# busiest (`p`, which counts failures) first. `class_filter` is a
|
|
@@ -19,7 +24,7 @@ module Wurk
|
|
|
19
24
|
# Query#top_jobs, whose per-bucket series a table never shows.
|
|
20
25
|
def top_jobs(class_filter: nil, minutes: 60, hours: nil, now: ::Time.now)
|
|
21
26
|
_stamps, keys = window_buckets(minutes, hours, now)
|
|
22
|
-
rows = sum_totals(
|
|
27
|
+
rows = sum_totals(keys).to_a
|
|
23
28
|
rows.select! { |(klass, _)| klass.start_with?(class_filter) } if class_filter && !class_filter.empty?
|
|
24
29
|
rows.sort_by { |(_k, sums)| -sums[:p] }
|
|
25
30
|
end
|
|
@@ -55,8 +60,8 @@ module Wurk
|
|
|
55
60
|
names = queue_names(queues)
|
|
56
61
|
return [] if names.empty?
|
|
57
62
|
|
|
58
|
-
|
|
59
|
-
names.map { |name| { name: name, points: queue_points(
|
|
63
|
+
rows = queue_bucket_rows(bucket, starts, names)
|
|
64
|
+
names.each_with_index.map { |name, i| { name: name, points: queue_points(i, starts, rows) } }
|
|
60
65
|
end
|
|
61
66
|
|
|
62
67
|
private
|
|
@@ -89,17 +94,30 @@ module Wurk
|
|
|
89
94
|
[nil, (checked / 10.0).ceil]
|
|
90
95
|
end
|
|
91
96
|
|
|
92
|
-
|
|
97
|
+
# Every bucket repeats the same few hundred `<klass>|<kind>` fields, so
|
|
98
|
+
# each distinct field is split once per call, not once per bucket.
|
|
99
|
+
def sum_totals(keys)
|
|
93
100
|
totals = ::Hash.new { |h, k| h[k] = { p: 0, f: 0, ms: 0 } }
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
totals[klass][kind.to_sym] += value.to_i if kind && TOTAL_FIELDS.include?(kind)
|
|
98
|
-
end
|
|
101
|
+
parsed = ::Hash.new { |h, field| h[field] = parse_total_field(field) }
|
|
102
|
+
keys.each_slice(TOP_JOBS_SLICE) do |slice|
|
|
103
|
+
pipeline_hgetall(slice).each { |raw| fold_totals(totals, parsed, hash_reply(raw)) }
|
|
99
104
|
end
|
|
100
105
|
totals
|
|
101
106
|
end
|
|
102
107
|
|
|
108
|
+
def fold_totals(totals, parsed, hash)
|
|
109
|
+
hash.each do |field, value|
|
|
110
|
+
klass, kind = parsed[field]
|
|
111
|
+
totals[klass][kind] += value.to_i if kind
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def parse_total_field(field)
|
|
116
|
+
klass, kind = field.split('|', 2)
|
|
117
|
+
kind = TOTAL_KINDS[kind]
|
|
118
|
+
kind ? [klass, kind] : NO_FIELD
|
|
119
|
+
end
|
|
120
|
+
|
|
103
121
|
def hash_reply(raw)
|
|
104
122
|
raw.is_a?(::Array) ? raw.each_slice(2).to_h : (raw || {})
|
|
105
123
|
end
|
|
@@ -108,8 +126,12 @@ module Wurk
|
|
|
108
126
|
stamps.zip(rows).map { |at, (p, f, ms)| { at: at, p: p.to_i, f: f.to_i, ms: ms.to_i } }
|
|
109
127
|
end
|
|
110
128
|
|
|
111
|
-
|
|
112
|
-
|
|
129
|
+
# Only the requested queues' two fields per bucket, not every queue's: a
|
|
130
|
+
# bucket holds a pair per live queue, and a series names at most
|
|
131
|
+
# MAX_QUEUE_SERIES of them.
|
|
132
|
+
def queue_bucket_rows(bucket, starts, names)
|
|
133
|
+
fields = names.flat_map { |name| ["#{name}|#{QueueRollup::SIZE_KIND}", "#{name}|#{QueueRollup::LAT_KIND}"] }
|
|
134
|
+
pipeline_hmget(starts.map { |s| QueueRollup.bucket_key(bucket, s) }, fields)
|
|
113
135
|
end
|
|
114
136
|
|
|
115
137
|
def queue_names(queues)
|
|
@@ -117,11 +139,9 @@ module Wurk
|
|
|
117
139
|
names.sort.first(Query::MAX_QUEUE_SERIES)
|
|
118
140
|
end
|
|
119
141
|
|
|
120
|
-
def queue_points(
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
starts.zip(hashes).map do |at, hash|
|
|
124
|
-
{ at: at, size: hash[size_field].to_i, latency: (hash[lat_field] || 0).to_f }
|
|
142
|
+
def queue_points(index, starts, rows)
|
|
143
|
+
starts.zip(rows).map do |at, row|
|
|
144
|
+
{ at: at, size: row[index * 2].to_i, latency: (row[(index * 2) + 1] || 0).to_f }
|
|
125
145
|
end
|
|
126
146
|
end
|
|
127
147
|
|
data/lib/wurk/metrics/history.rb
CHANGED
|
@@ -24,7 +24,8 @@ module Wurk
|
|
|
24
24
|
# (`14:05`), the 10-minute bucket always one (`14:0`). `<klass>` is the
|
|
25
25
|
# `wrapped` class for ActiveJob payloads, like upstream.
|
|
26
26
|
#
|
|
27
|
-
# Every bucket TTL is set on every
|
|
27
|
+
# Every bucket TTL is set on every flush (not NX), once per bucket key after
|
|
28
|
+
# its HINCRBYs, as upstream's ExecutionTracker#flush does.
|
|
28
29
|
#
|
|
29
30
|
# The middleware is hot-path — every job pays for it — so it does not touch
|
|
30
31
|
# Redis at all. Each execution folds into a process-wide Accumulator and
|
|
@@ -32,6 +33,10 @@ module Wurk
|
|
|
32
33
|
class History
|
|
33
34
|
include Wurk::Middleware::ServerMiddleware
|
|
34
35
|
|
|
36
|
+
# One instance per chain entry, shared by every job (Middleware::Chain::Entry):
|
|
37
|
+
# #call keeps its state in locals and writes no ivar.
|
|
38
|
+
def self.shared_instance? = true
|
|
39
|
+
|
|
35
40
|
# Upstream's ExecutionTracker constants, same names. MID_TERM diverges:
|
|
36
41
|
# upstream TTLs the 10-minute buckets at 3 days, but Wurk's dashboard
|
|
37
42
|
# serves 7d/30d tabs off them (Query::MAX_HOURS = 720), so they must
|
|
@@ -178,24 +183,33 @@ module Wurk
|
|
|
178
183
|
at.to_i / 60
|
|
179
184
|
end
|
|
180
185
|
|
|
186
|
+
# The `j|` keys are shared by every class, so each gets its EXPIRE once,
|
|
187
|
+
# after the last HINCRBY into it; a 10-minute key two minutes share gets
|
|
188
|
+
# one too. A histogram key is per class and minute, so it keeps its own.
|
|
181
189
|
def write(pool, minutes)
|
|
182
190
|
with_pool(pool) do |conn|
|
|
183
191
|
conn.pipelined do |pipe|
|
|
184
|
-
minutes.
|
|
185
|
-
|
|
186
|
-
short = minute_key(at)
|
|
187
|
-
keys = [short, short.chop]
|
|
188
|
-
classes.each { |klass, counts| write_class(pipe, keys, at, klass, counts) }
|
|
189
|
-
end
|
|
192
|
+
ttls = minutes.each_with_object({}) { |(minute, classes), acc| write_minute(pipe, minute, classes, acc) }
|
|
193
|
+
ttls.each { |key, ttl| pipe.call('EXPIRE', key, ttl) }
|
|
190
194
|
end
|
|
191
195
|
end
|
|
192
196
|
end
|
|
193
197
|
|
|
198
|
+
# Writes one minute's classes and notes its two `j|` keys' TTLs in `ttls`.
|
|
199
|
+
def write_minute(pipe, minute, classes, ttls)
|
|
200
|
+
at = ::Time.at(minute * 60).utc
|
|
201
|
+
short = minute_key(at)
|
|
202
|
+
keys = [short, short.chop]
|
|
203
|
+
classes.each { |klass, counts| write_class(pipe, keys, at, klass, counts) }
|
|
204
|
+
ttls[keys[1]] = MID_TERM
|
|
205
|
+
ttls[keys[0]] = SHORT_TERM
|
|
206
|
+
end
|
|
207
|
+
|
|
194
208
|
# `keys` is [minute key, 10-minute key].
|
|
195
209
|
def write_class(pipe, keys, at, klass, counts)
|
|
196
210
|
fields = ["#{klass}|p", "#{klass}|f", "#{klass}|ms"]
|
|
197
|
-
incr_bucket(pipe, keys[1], fields, counts
|
|
198
|
-
incr_bucket(pipe, keys[0], fields, counts
|
|
211
|
+
incr_bucket(pipe, keys[1], fields, counts)
|
|
212
|
+
incr_bucket(pipe, keys[0], fields, counts)
|
|
199
213
|
return unless counts[3]
|
|
200
214
|
|
|
201
215
|
pipe.call(*Histogram.incr_command(klass, at, counts[3]))
|
|
@@ -206,12 +220,11 @@ module Wurk
|
|
|
206
220
|
# `f` when something failed, `ms` when something succeeded (by zero
|
|
207
221
|
# included). `HINCRBY <field> 0` beyond that would materialize a
|
|
208
222
|
# counter Sidekiq never writes.
|
|
209
|
-
def incr_bucket(pipe, key, fields, counts
|
|
223
|
+
def incr_bucket(pipe, key, fields, counts)
|
|
210
224
|
processed, failed, ms = counts
|
|
211
225
|
pipe.call('HINCRBY', key, fields[0], processed)
|
|
212
226
|
pipe.call('HINCRBY', key, fields[1], failed) if failed.positive?
|
|
213
227
|
pipe.call('HINCRBY', key, fields[2], ms) if processed > failed
|
|
214
|
-
pipe.call('EXPIRE', key, ttl)
|
|
215
228
|
end
|
|
216
229
|
end
|
|
217
230
|
|
data/lib/wurk/metrics/query.rb
CHANGED
|
@@ -4,7 +4,6 @@ require_relative '../component'
|
|
|
4
4
|
require_relative '../keys'
|
|
5
5
|
require_relative '../job_record'
|
|
6
6
|
require_relative '../timer_loop'
|
|
7
|
-
require_relative 'rollup'
|
|
8
7
|
|
|
9
8
|
module Wurk
|
|
10
9
|
module Metrics
|
|
@@ -82,25 +81,48 @@ module Wurk
|
|
|
82
81
|
|
|
83
82
|
# One sampling pass, bypassing the leader gate and the sleep loop. Public
|
|
84
83
|
# so deterministic specs and a manual "sample now" can drive it directly.
|
|
84
|
+
#
|
|
85
|
+
# A zero gauge is not stored: readers default a missing field to 0, and
|
|
86
|
+
# an idle fleet's queues are mostly empty, so storing them cost two
|
|
87
|
+
# fields per queue in every bucket for nothing. It is HDEL'd instead of
|
|
88
|
+
# skipped, so a queue that drained inside a coarse bucket reads 0 there
|
|
89
|
+
# rather than keeping the earlier sample. All three buckets go in one
|
|
90
|
+
# pipeline.
|
|
85
91
|
def sample(now = ::Time.now)
|
|
86
92
|
gauges = queue_gauges
|
|
87
93
|
return if gauges.empty?
|
|
88
94
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
BUCKETS.each do |bucket, (step, ttl)|
|
|
93
|
-
key = self.class.bucket_key(bucket, (now.to_i / step) * step)
|
|
94
|
-
redis do |c|
|
|
95
|
-
c.call('HSET', key, *fields)
|
|
96
|
-
c.call('EXPIRE', key, ttl)
|
|
97
|
-
end
|
|
98
|
-
end
|
|
95
|
+
set, zero = gauge_fields(gauges)
|
|
96
|
+
keys = bucket_keys(now)
|
|
97
|
+
redis { |c| c.pipelined { |p| keys.each { |key, ttl| write_bucket(p, key, ttl, set, zero) } } }
|
|
99
98
|
nil
|
|
100
99
|
end
|
|
101
100
|
|
|
102
101
|
private
|
|
103
102
|
|
|
103
|
+
# [[bucket key, ttl], ...] for the bucket `now` falls in at each resolution.
|
|
104
|
+
def bucket_keys(now)
|
|
105
|
+
BUCKETS.map { |bucket, (step, ttl)| [self.class.bucket_key(bucket, (now.to_i / step) * step), ttl] }
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def write_bucket(pipe, key, ttl, set, zero)
|
|
109
|
+
pipe.call('HSET', key, *set) unless set.empty?
|
|
110
|
+
pipe.call('HDEL', key, *zero) unless zero.empty?
|
|
111
|
+
pipe.call('EXPIRE', key, ttl)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# [flat field/value pairs to HSET, fields to HDEL], zero gauges in the
|
|
115
|
+
# second list.
|
|
116
|
+
def gauge_fields(gauges)
|
|
117
|
+
set = []
|
|
118
|
+
zero = []
|
|
119
|
+
gauges.each do |name, (size, lat)|
|
|
120
|
+
size.zero? ? zero << "#{name}|#{SIZE_KIND}" : set.push("#{name}|#{SIZE_KIND}", size)
|
|
121
|
+
lat.zero? ? zero << "#{name}|#{LAT_KIND}" : set.push("#{name}|#{LAT_KIND}", lat)
|
|
122
|
+
end
|
|
123
|
+
[set, zero]
|
|
124
|
+
end
|
|
125
|
+
|
|
104
126
|
# [[queue_name, [size, latency_seconds]], ...] for every live queue. The
|
|
105
127
|
# latency is the head-of-line wait — the tail of the LIST is the oldest
|
|
106
128
|
# job — in seconds, rounded for compact storage.
|
data/lib/wurk/metrics/statsd.rb
CHANGED
|
@@ -34,6 +34,10 @@ module Wurk
|
|
|
34
34
|
class Statsd
|
|
35
35
|
include Wurk::Middleware::ServerMiddleware
|
|
36
36
|
|
|
37
|
+
# One instance per chain entry, shared by every job (Middleware::Chain::Entry):
|
|
38
|
+
# #call keeps its state in locals and writes no ivar.
|
|
39
|
+
def self.shared_instance? = true
|
|
40
|
+
|
|
37
41
|
METRIC_PREFIX = 'sidekiq.'
|
|
38
42
|
DEFAULT_SAMPLE_RATE = 1.0
|
|
39
43
|
|
|
@@ -154,7 +154,7 @@ module Wurk
|
|
|
154
154
|
def traverse(entries, index, args, &block)
|
|
155
155
|
return yield if index == entries.size
|
|
156
156
|
|
|
157
|
-
entries[index].
|
|
157
|
+
entries[index].instance.call(*args) do
|
|
158
158
|
traverse(entries, index + 1, args, &block)
|
|
159
159
|
end
|
|
160
160
|
end
|
|
@@ -188,6 +188,15 @@ module Wurk
|
|
|
188
188
|
# instantiation until `retrieve`/`invoke` runs so each job gets a fresh
|
|
189
189
|
# object.
|
|
190
190
|
#
|
|
191
|
+
# One opt-in exception, for Wurk's own built-ins: a class that defines
|
|
192
|
+
# `self.shared_instance?` as true on itself is built once per entry, when
|
|
193
|
+
# the entry is bound to its config, and `invoke` hands every job that one
|
|
194
|
+
# instance. Only for a class whose #call writes no instance state — it
|
|
195
|
+
# runs on every processor thread of the capsule at once. Asked of the
|
|
196
|
+
# class itself, never inherited: a host's subclass of a built-in may keep
|
|
197
|
+
# state, and gets Sidekiq's fresh instance per job like any other
|
|
198
|
+
# middleware. `make_new`, and so `retrieve`, always builds a fresh one.
|
|
199
|
+
#
|
|
191
200
|
# Whether the class declares `config=` is resolved once, at registration,
|
|
192
201
|
# so the common case (the setter comes from `include
|
|
193
202
|
# Wurk::Middleware::ServerMiddleware`) skips a `respond_to?` on every job
|
|
@@ -197,13 +206,26 @@ module Wurk
|
|
|
197
206
|
# `method_missing`, or one added to the class after it was registered — so
|
|
198
207
|
# a class that does not declare it still gets asked.
|
|
199
208
|
class Entry
|
|
200
|
-
attr_reader :klass
|
|
201
|
-
attr_accessor :config
|
|
209
|
+
attr_reader :klass, :config
|
|
202
210
|
|
|
203
211
|
def initialize(klass, *args)
|
|
204
212
|
@klass = klass
|
|
205
213
|
@args = args
|
|
206
214
|
@takes_config = klass.public_method_defined?(:config=)
|
|
215
|
+
@shared = klass.singleton_class.method_defined?(:shared_instance?, false) && klass.shared_instance?
|
|
216
|
+
@instance = nil
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# Rebuilds the shared instance too: Chain#copy_for rebinds a duplicate
|
|
220
|
+
# entry to a capsule, and the instance must answer to that capsule.
|
|
221
|
+
def config=(value)
|
|
222
|
+
@config = value
|
|
223
|
+
@instance = make_new if @shared
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
# What `invoke` runs: the shared instance, or a fresh one.
|
|
227
|
+
def instance
|
|
228
|
+
@instance || make_new
|
|
207
229
|
end
|
|
208
230
|
|
|
209
231
|
def make_new
|
|
@@ -43,6 +43,10 @@ module Wurk
|
|
|
43
43
|
class Expiry
|
|
44
44
|
include Wurk::Middleware::ServerMiddleware
|
|
45
45
|
|
|
46
|
+
# One instance per chain entry, shared by every job (Middleware::Chain::Entry):
|
|
47
|
+
# #call and its helpers keep their state in locals and write no ivar.
|
|
48
|
+
def self.shared_instance? = true
|
|
49
|
+
|
|
46
50
|
# The rescue is scoped to the guarded call rather than the method body:
|
|
47
51
|
# only a job that carries a cutoff can be cut by one, and a job that
|
|
48
52
|
# raises `DeadlineExceeded` from its own code without carrying one has to
|
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
require_relative '../middleware'
|
|
4
4
|
require_relative '../job'
|
|
5
5
|
require_relative '../job_retry'
|
|
6
|
-
require_relative '../encryption'
|
|
7
6
|
|
|
8
7
|
module Wurk
|
|
9
8
|
module Middleware
|
|
@@ -30,6 +29,10 @@ module Wurk
|
|
|
30
29
|
class InterruptHandler
|
|
31
30
|
include Wurk::Middleware::ServerMiddleware
|
|
32
31
|
|
|
32
|
+
# One instance per chain entry, shared by every job (Middleware::Chain::Entry):
|
|
33
|
+
# #call and its helpers keep their state in locals and write no ivar.
|
|
34
|
+
def self.shared_instance? = true
|
|
35
|
+
|
|
33
36
|
def call(_job_instance, job, queue)
|
|
34
37
|
yield
|
|
35
38
|
rescue Wurk::Job::Interrupted
|
|
@@ -32,6 +32,10 @@ module Wurk
|
|
|
32
32
|
class Status
|
|
33
33
|
include Wurk::Middleware::ServerMiddleware
|
|
34
34
|
|
|
35
|
+
# One instance per chain entry, shared by every job (Middleware::Chain::Entry):
|
|
36
|
+
# #call and its helpers keep their state in locals and write no ivar.
|
|
37
|
+
def self.shared_instance? = true
|
|
38
|
+
|
|
35
39
|
# Hard cap on a stored result, in bytes. A job that returns an
|
|
36
40
|
# ActiveRecord relation must cost Redis 8 KB, not a table dump — past the
|
|
37
41
|
# cap the head is kept and `result_truncated` says the rest was dropped.
|