wurk 1.13.0 → 1.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/app/controllers/concerns/wurk/sse_streaming.rb +36 -1
- data/app/controllers/wurk/api/pagination.rb +9 -7
- data/app/controllers/wurk/api/serializers.rb +8 -2
- data/app/controllers/wurk/api_controller.rb +57 -24
- data/app/controllers/wurk/application_controller.rb +0 -2
- data/app/controllers/wurk/dashboard_controller.rb +0 -1
- data/config/routes.rb +5 -1
- data/lib/sidekiq/cli.rb +4 -3
- data/lib/sidekiq/deploy.rb +2 -1
- data/lib/sidekiq/embedded.rb +2 -1
- data/lib/sidekiq/iterable_job.rb +2 -1
- data/lib/sidekiq/job/iterable/active_record_enumerator.rb +2 -1
- data/lib/sidekiq/job/iterable/csv_enumerator.rb +2 -1
- data/lib/sidekiq/job/iterable.rb +2 -1
- data/lib/sidekiq/launcher.rb +2 -1
- data/lib/sidekiq/manager.rb +2 -1
- data/lib/sidekiq/metrics/query.rb +2 -1
- data/lib/sidekiq/pro/web.rb +2 -1
- data/lib/sidekiq/scheduled.rb +2 -1
- data/lib/sidekiq/web.rb +2 -1
- data/lib/sidekiq.rb +3 -2
- data/lib/wurk/api/fast.rb +4 -18
- data/lib/wurk/api/queues.rb +12 -10
- data/lib/wurk/api/serializers.rb +6 -7
- data/lib/wurk/api/swarm.rb +0 -1
- data/lib/wurk/batch/server_middleware.rb +4 -0
- data/lib/wurk/batch/status.rb +45 -22
- data/lib/wurk/batch_set.rb +34 -0
- data/lib/wurk/cli.rb +17 -19
- data/lib/wurk/client/buffered.rb +97 -26
- data/lib/wurk/client.rb +21 -1
- data/lib/wurk/compat/deploy.rb +7 -0
- data/lib/wurk/compat/encryption.rb +7 -0
- data/lib/wurk/compat/flow.rb +8 -0
- data/lib/wurk/compat/iterable_job.rb +8 -0
- data/lib/wurk/compat/profile_set.rb +8 -0
- data/lib/wurk/compat/server.rb +15 -0
- data/lib/wurk/compat/web.rb +20 -0
- data/lib/wurk/compat.rb +34 -40
- data/lib/wurk/configuration.rb +46 -9
- data/lib/wurk/cron.rb +146 -47
- data/lib/wurk/dead_set.rb +10 -6
- data/lib/wurk/embedded.rb +0 -2
- data/lib/wurk/engine.rb +46 -4
- data/lib/wurk/error_coalescer.rb +21 -1
- data/lib/wurk/fetcher/many_queues.rb +8 -6
- data/lib/wurk/fetcher/poison_pill.rb +1 -1
- data/lib/wurk/fetcher/reaper.rb +0 -5
- data/lib/wurk/fetcher/reliable.rb +46 -27
- data/lib/wurk/fetcher/self_reconcile.rb +0 -1
- data/lib/wurk/fetcher/unit_of_work.rb +16 -1
- data/lib/wurk/flow/status.rb +10 -4
- data/lib/wurk/flow_set.rb +24 -2
- data/lib/wurk/health.rb +0 -1
- data/lib/wurk/heartbeat.rb +13 -7
- data/lib/wurk/import/sidekiq_cron.rb +0 -1
- data/lib/wurk/job_record.rb +2 -1
- data/lib/wurk/job_retry.rb +1 -1
- data/lib/wurk/job_set.rb +132 -6
- data/lib/wurk/launcher.rb +0 -12
- data/lib/wurk/lazy_loader.rb +60 -0
- data/lib/wurk/limiter/bucket.rb +0 -2
- data/lib/wurk/limiter/concurrent.rb +19 -36
- data/lib/wurk/limiter/leaky.rb +0 -2
- data/lib/wurk/limiter/points.rb +0 -2
- data/lib/wurk/limiter/server_middleware.rb +4 -1
- data/lib/wurk/limiter/unlimited.rb +0 -2
- data/lib/wurk/limiter/window.rb +0 -2
- data/lib/wurk/limiter.rb +0 -7
- data/lib/wurk/logger.rb +44 -5
- data/lib/wurk/lua/fast_delete.rb +58 -0
- data/lib/wurk/lua/fast_delete_window.lua +46 -0
- data/lib/wurk/lua/limiter_concurrent_acquire.lua +10 -3
- data/lib/wurk/lua/limiter_concurrent_release.lua +13 -4
- data/lib/wurk/lua.rb +1 -41
- data/lib/wurk/metrics/dashboard_series.rb +38 -18
- data/lib/wurk/metrics/history.rb +24 -11
- data/lib/wurk/metrics/prometheus.rb +0 -1
- data/lib/wurk/metrics/query.rb +0 -2
- data/lib/wurk/metrics/queue_rollup.rb +33 -11
- data/lib/wurk/metrics/statsd.rb +4 -0
- data/lib/wurk/middleware/chain.rb +25 -3
- data/lib/wurk/middleware/expiry.rb +4 -0
- data/lib/wurk/middleware/interrupt_handler.rb +4 -1
- data/lib/wurk/middleware/status.rb +4 -0
- data/lib/wurk/middleware/timeout.rb +4 -0
- data/lib/wurk/preflight/owners.rb +0 -1
- data/lib/wurk/preflight/payload_sample.rb +0 -1
- data/lib/wurk/preflight/verify.rb +0 -2
- data/lib/wurk/process_set.rb +6 -1
- data/lib/wurk/profiler.rb +2 -1
- data/lib/wurk/queue.rb +33 -6
- data/lib/wurk/queue_consumers.rb +10 -1
- data/lib/wurk/rails_boot.rb +2 -2
- data/lib/wurk/railtie.rb +3 -4
- data/lib/wurk/sentry/error_handler.rb +5 -0
- data/lib/wurk/sentry/middleware.rb +4 -0
- data/lib/wurk/sorted_entry.rb +16 -12
- data/lib/wurk/stats.rb +85 -1
- data/lib/wurk/swarm/child_boot.rb +0 -2
- data/lib/wurk/swarm/liveness.rb +0 -2
- data/lib/wurk/swarm/memory_watch.rb +116 -0
- data/lib/wurk/swarm/prefork.rb +54 -0
- data/lib/wurk/swarm.rb +33 -42
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/enterprise.rb +63 -7
- data/lib/wurk.rb +33 -29
- data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js +1 -0
- data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.br +0 -0
- data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.gz +0 -0
- data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js +1 -0
- data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.br +0 -0
- data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js +1 -0
- data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.br +0 -0
- data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js +1 -0
- data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.br +0 -0
- data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js +1 -0
- data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.br +0 -0
- data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js +1 -0
- data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.br +0 -0
- data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js +1 -0
- data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.br +0 -0
- data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js +1 -0
- data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.br +0 -0
- data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.gz +0 -0
- data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js +2 -0
- data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.br +0 -0
- data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.gz +0 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js +1 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.br +0 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{FlowState-yb7virqR.js → FlowState-ClzInw0-.js} +1 -1
- data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.br +0 -0
- data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js +1 -0
- data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.br +0 -0
- data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.gz +0 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js +2 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.br +0 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js +1 -0
- data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.br +0 -0
- data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js +1 -0
- data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.br +0 -0
- data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.gz +0 -0
- data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js +1 -0
- data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.br +0 -0
- data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.gz +0 -0
- data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js +1 -0
- data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.br +0 -0
- data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js +1 -0
- data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.br +0 -0
- data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js +1 -0
- data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.br +0 -0
- data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js +1 -0
- data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.br +0 -0
- data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js +1 -0
- data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.br +0 -0
- data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Search-D42Va7lB.js +1 -0
- data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.br +0 -0
- data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.gz +0 -0
- data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js +1 -0
- data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.br +0 -0
- data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.gz +0 -0
- data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js +1 -0
- data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.br +0 -0
- data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.gz +0 -0
- data/vendor/assets/dashboard/assets/de-B2NCIuzo.js +1 -0
- data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.br +0 -0
- data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.gz +0 -0
- data/vendor/assets/dashboard/assets/es-BGeu_JNB.js +1 -0
- data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.br +0 -0
- data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.gz +0 -0
- data/vendor/assets/dashboard/assets/fr-DQVBWFId.js +1 -0
- data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.br +0 -0
- data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.gz +0 -0
- data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js +1 -0
- data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.br +0 -0
- data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.gz +0 -0
- data/vendor/assets/dashboard/assets/index-CiLU1zz8.js +141 -0
- data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.br +0 -0
- data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.gz +0 -0
- data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css +1 -0
- data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.br +0 -0
- data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.gz +0 -0
- data/vendor/assets/dashboard/assets/ja-D06IBqzL.js +1 -0
- data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.br +0 -0
- data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.gz +0 -0
- data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js +1 -0
- data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.br +0 -0
- data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.gz +0 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js +1 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.br +0 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.gz +0 -0
- data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js +1 -0
- data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.br +0 -0
- data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{utils-CnWZzW2L.js → utils-69XSoGbQ.js} +1 -1
- data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.br +0 -0
- data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.gz +0 -0
- data/vendor/assets/dashboard/assets/wurk-logo-ydeSB9B8.png +0 -0
- data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js +1 -0
- data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.br +0 -0
- data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.gz +0 -0
- data/vendor/assets/dashboard/index.html +3 -23
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +122 -37
- data/vendor/assets/dashboard/assets/ArgsValue-BJol9S0s.js +0 -1
- data/vendor/assets/dashboard/assets/BatchDetail-D_zGQ9Sd.js +0 -1
- data/vendor/assets/dashboard/assets/Batches-Dt_xC660.js +0 -1
- data/vendor/assets/dashboard/assets/Busy-CWRthsmc.js +0 -1
- data/vendor/assets/dashboard/assets/Cron-qeGr2YAm.js +0 -1
- data/vendor/assets/dashboard/assets/Dashboard-DQWaAwPA.js +0 -1
- data/vendor/assets/dashboard/assets/Dead-D9gOPHtK.js +0 -1
- data/vendor/assets/dashboard/assets/Extension-DONXVLdA.js +0 -1
- data/vendor/assets/dashboard/assets/FilterBox-BbB6gXHp.js +0 -2
- data/vendor/assets/dashboard/assets/FlowDetail-CrmnVRp0.js +0 -1
- data/vendor/assets/dashboard/assets/Flows-g4GC8KmE.js +0 -1
- data/vendor/assets/dashboard/assets/JobDetailModal-C86k_1wj.js +0 -2
- data/vendor/assets/dashboard/assets/Limiters-zyO-iMlK.js +0 -1
- data/vendor/assets/dashboard/assets/Metrics-Ckq5fNBv.js +0 -1
- data/vendor/assets/dashboard/assets/NotFound-C8NnyjlY.js +0 -1
- data/vendor/assets/dashboard/assets/PageHeader-vWeBbSPF.js +0 -1
- data/vendor/assets/dashboard/assets/Profiles-DRl-FXPA.js +0 -1
- data/vendor/assets/dashboard/assets/Queues-DkXWrihA.js +0 -1
- data/vendor/assets/dashboard/assets/Retries-CmlojYIy.js +0 -1
- data/vendor/assets/dashboard/assets/Scheduled-Cbbpp3xw.js +0 -1
- data/vendor/assets/dashboard/assets/Search-LaEpj2fM.js +0 -1
- data/vendor/assets/dashboard/assets/charts-D_PQLoyc.js +0 -1
- data/vendor/assets/dashboard/assets/fa-brands-400-Bs6tcqqs.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-regular-400-DRN8N0d1.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-solid-900-IAB4Droh.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-v4compatibility-CErXDOsT.woff2 +0 -0
- data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +0 -1
- data/vendor/assets/dashboard/assets/index--xvvNlYx.css +0 -1
- data/vendor/assets/dashboard/assets/index-FBKGtysy.js +0 -141
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-BiwhHqgh.js +0 -1
- data/vendor/assets/dashboard/assets/useSort-jBYVwXvg.js +0 -1
- data/vendor/assets/dashboard/assets/wurk-logo-Vy3xW4K0.png +0 -0
- data/vendor/assets/dashboard/og-image.png +0 -0
- data/vendor/assets/dashboard/wurk-logo.webp +0 -0
data/lib/wurk/railtie.rb
CHANGED
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
require 'rails/railtie'
|
|
4
4
|
require 'active_support/ordered_options'
|
|
5
|
-
require_relative 'rails_boot'
|
|
6
5
|
|
|
7
6
|
module Wurk
|
|
8
7
|
# Rails integration surface only: register the host-facing `config.wurk`
|
|
@@ -24,9 +23,9 @@ module Wurk
|
|
|
24
23
|
|
|
25
24
|
# Enter server mode BEFORE config/initializers load — otherwise the app's
|
|
26
25
|
# `Sidekiq.configure_server` blocks gate on `config.server?` (still false)
|
|
27
|
-
# and are
|
|
28
|
-
#
|
|
29
|
-
# process serves.
|
|
26
|
+
# and are deferred, running only when server mode is entered — after the
|
|
27
|
+
# rest of the app's initializers instead of in their place. RailsBoot
|
|
28
|
+
# decides whether this process serves.
|
|
30
29
|
initializer 'wurk.server_mode', before: :load_config_initializers do |app|
|
|
31
30
|
Wurk::RailsBoot.boot_scheduled = true
|
|
32
31
|
Wurk::RailsBoot.enter_server_mode_if_serving(app)
|
|
@@ -42,6 +42,11 @@ module Wurk
|
|
|
42
42
|
Thread.current[REPORTED_ON_THREAD].equal?(exception)
|
|
43
43
|
end
|
|
44
44
|
|
|
45
|
+
def self.forget_thread!
|
|
46
|
+
Thread.current[SEEN_BY_MIDDLEWARE] = nil
|
|
47
|
+
Thread.current[REPORTED_ON_THREAD] = nil
|
|
48
|
+
end
|
|
49
|
+
|
|
45
50
|
# Transport blips the pool already retried before re-raising. Wurk's
|
|
46
51
|
# default handler logs these at WARN precisely because they are
|
|
47
52
|
# self-healing, and the fetch loop runs them in a tight `sleep(1)` cycle:
|
|
@@ -28,6 +28,10 @@ module Wurk
|
|
|
28
28
|
def call(instance, job, queue, &block)
|
|
29
29
|
return yield unless Wurk::Sentry.enabled?
|
|
30
30
|
|
|
31
|
+
# The previous job's exception (with its backtrace and anything it
|
|
32
|
+
# references) would otherwise stay pinned in this thread's slots until
|
|
33
|
+
# the next failure here replaced it. Both handshakes are within one job.
|
|
34
|
+
ErrorHandler.forget_thread!
|
|
31
35
|
# Each Processor owns a thread, and Sentry's hub is thread-local.
|
|
32
36
|
# Without this the worker thread starts from an empty hub and the
|
|
33
37
|
# scope set below is invisible to the capture.
|
data/lib/wurk/sorted_entry.rb
CHANGED
|
@@ -91,6 +91,22 @@ module Wurk
|
|
|
91
91
|
|
|
92
92
|
def error? = !item['error_class'].nil?
|
|
93
93
|
|
|
94
|
+
# Puts the original bytes back at the original score, after this entry was
|
|
95
|
+
# removed and its push failed. Internal: #remove_job and JobSet's batched
|
|
96
|
+
# retry_all, which removes a page before pushing it.
|
|
97
|
+
#
|
|
98
|
+
# Past maxmemory the push is refused, and so is a plain ZADD putting the
|
|
99
|
+
# entry back. Dragonfly refuses both forms: all that is left is the log.
|
|
100
|
+
def restore_after_failed_push
|
|
101
|
+
Wurk.redis do |conn|
|
|
102
|
+
conn.call('ZADD', @parent.name, @score, value)
|
|
103
|
+
rescue RedisClient::OutOfMemoryError => e
|
|
104
|
+
Fetcher::OomDrain.zadd(conn, e, @parent.name, @score, value)
|
|
105
|
+
end
|
|
106
|
+
rescue StandardError => e
|
|
107
|
+
Wurk.logger.error("#{@parent.name} entry lost, ZADD refused (#{e.class}): #{loggable_payload}")
|
|
108
|
+
end
|
|
109
|
+
|
|
94
110
|
private
|
|
95
111
|
|
|
96
112
|
# Pulls the message out of Redis, yields its raw JSON for the caller's
|
|
@@ -116,18 +132,6 @@ module Wurk
|
|
|
116
132
|
item
|
|
117
133
|
end
|
|
118
134
|
|
|
119
|
-
# Past maxmemory the push is refused, and so is a plain ZADD putting the
|
|
120
|
-
# entry back. Dragonfly refuses both forms: all that is left is the log.
|
|
121
|
-
def restore_after_failed_push
|
|
122
|
-
Wurk.redis do |conn|
|
|
123
|
-
conn.call('ZADD', @parent.name, @score, value)
|
|
124
|
-
rescue RedisClient::OutOfMemoryError => e
|
|
125
|
-
Fetcher::OomDrain.zadd(conn, e, @parent.name, @score, value)
|
|
126
|
-
end
|
|
127
|
-
rescue StandardError => e
|
|
128
|
-
Wurk.logger.error("#{@parent.name} entry lost, ZADD refused (#{e.class}): #{loggable_payload}")
|
|
129
|
-
end
|
|
130
|
-
|
|
131
135
|
# The last copy of the job, so it is logged whole, short of what the host's
|
|
132
136
|
# redact_args hides.
|
|
133
137
|
def loggable_payload
|
data/lib/wurk/stats.rb
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require 'date'
|
|
4
4
|
require_relative 'pool_checkout'
|
|
5
|
+
require_relative 'queue_consumers'
|
|
5
6
|
|
|
6
7
|
module Wurk
|
|
7
8
|
# Read-only inspector for cluster state in Redis. The cheap counters are
|
|
@@ -76,7 +77,7 @@ module Wurk
|
|
|
76
77
|
names = conn.call('SMEMBERS', Keys::QUEUES_SET)
|
|
77
78
|
next [] if names.empty?
|
|
78
79
|
|
|
79
|
-
paused_set = conn.call('SMEMBERS', 'paused')
|
|
80
|
+
paused_set = conn.call('SMEMBERS', 'paused').to_set
|
|
80
81
|
results = conn.pipelined do |pipe|
|
|
81
82
|
names.each do |q|
|
|
82
83
|
pipe.call('LLEN', Keys.queue(q))
|
|
@@ -166,6 +167,89 @@ module Wurk
|
|
|
166
167
|
0.0
|
|
167
168
|
end
|
|
168
169
|
|
|
170
|
+
# Every Stats reading taken at one moment, in two round trips: the
|
|
171
|
+
# counters with the `queues`, `paused` and `processes` members, then each
|
|
172
|
+
# queue's LLEN + tail and each process's `busy`. A plain Stats re-reads the
|
|
173
|
+
# queue set per method and pays seven round trips for what the dashboard's
|
|
174
|
+
# landing payload asks of it. Internal reader for the dashboard and the
|
|
175
|
+
# HTTP API: Stats keeps Sidekiq's per-method reads (§19.1).
|
|
176
|
+
#
|
|
177
|
+
# `consumers: true` also reads each process's `info` and answers
|
|
178
|
+
# #consumers, the per-queue subscriber count of QueueConsumers.counts.
|
|
179
|
+
class Snapshot < Stats
|
|
180
|
+
attr_reader :queue_summaries, :workers_size, :default_queue_latency
|
|
181
|
+
|
|
182
|
+
def initialize(consumers: false) # rubocop:disable Lint/MissingSuper
|
|
183
|
+
@with_consumers = consumers
|
|
184
|
+
Wurk.redis(idempotent: true) do |conn|
|
|
185
|
+
names, paused, identities = read_members(conn)
|
|
186
|
+
read_gauges(conn, names, paused.to_set, identities)
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def enqueued = @queue_summaries.sum(&:size)
|
|
191
|
+
def queues = @queue_summaries.to_h { |summary| [summary.name, summary.size] }
|
|
192
|
+
|
|
193
|
+
# @return [Hash{String=>Integer}] queue → live subscribers.
|
|
194
|
+
def consumers
|
|
195
|
+
raise ArgumentError, 'Snapshot read without consumers: true' unless @with_consumers
|
|
196
|
+
|
|
197
|
+
@consumers
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
private
|
|
201
|
+
|
|
202
|
+
def read_members(conn)
|
|
203
|
+
replies = conn.pipelined do |pipe|
|
|
204
|
+
FAST_QUERIES.each { |args| pipe.call(*args) }
|
|
205
|
+
pipe.call('SMEMBERS', Keys::QUEUES_SET)
|
|
206
|
+
pipe.call('SMEMBERS', 'paused')
|
|
207
|
+
pipe.call('SMEMBERS', Keys::PROCESSES)
|
|
208
|
+
end
|
|
209
|
+
@stats = FAST_KEYS.zip(replies.first(FAST_KEYS.size).map(&:to_i)).to_h
|
|
210
|
+
replies.last(3)
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# Queue replies first (LLEN, tail per queue), then the `default` tail when
|
|
214
|
+
# that queue is not listed, then one reply per process.
|
|
215
|
+
def read_gauges(conn, names, paused, identities)
|
|
216
|
+
default_listed = names.include?('default')
|
|
217
|
+
replies = conn.pipelined do |pipe|
|
|
218
|
+
queue_commands(pipe, names, default_listed)
|
|
219
|
+
identities.each { |id| process_command(pipe, id) }
|
|
220
|
+
end
|
|
221
|
+
queue_replies = replies.shift(names.size * 2)
|
|
222
|
+
default_tail = default_listed ? queue_replies[(names.index('default') * 2) + 1] : replies.shift
|
|
223
|
+
build_snapshot(names, queue_replies, paused, default_tail, replies)
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
def queue_commands(pipe, names, default_listed)
|
|
227
|
+
names.each do |q|
|
|
228
|
+
pipe.call('LLEN', Keys.queue(q))
|
|
229
|
+
pipe.call('LRANGE', Keys.queue(q), -1, -1)
|
|
230
|
+
end
|
|
231
|
+
pipe.call('LRANGE', Keys.queue('default'), -1, -1) unless default_listed
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
def process_command(pipe, identity)
|
|
235
|
+
return pipe.call('HGET', identity, 'busy') unless @with_consumers
|
|
236
|
+
|
|
237
|
+
pipe.call('HMGET', identity, 'busy', 'info')
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def build_snapshot(names, queue_replies, paused, default_tail, process_replies)
|
|
241
|
+
@queue_summaries = build_summaries(names, queue_replies, paused).sort_by { |qd| -qd.size }
|
|
242
|
+
now_ms = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
|
|
243
|
+
@default_queue_latency = compute_latency(default_tail.first, now_ms)
|
|
244
|
+
if @with_consumers
|
|
245
|
+
@workers_size = process_replies.sum { |busy, _| busy.to_i }
|
|
246
|
+
@consumers = QueueConsumers.tally(process_replies.map(&:last))
|
|
247
|
+
else
|
|
248
|
+
@workers_size = process_replies.sum(&:to_i)
|
|
249
|
+
end
|
|
250
|
+
end
|
|
251
|
+
end
|
|
252
|
+
|
|
169
253
|
# Per-day historical processed/failed/expired counts. Reads
|
|
170
254
|
# `stat:processed:YYYY-MM-DD`, `stat:failed:YYYY-MM-DD`, and
|
|
171
255
|
# `stat:expired:YYYY-MM-DD` strings; missing days return 0. Range
|
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative '../component'
|
|
4
|
-
require_relative '../launcher'
|
|
5
4
|
require_relative '../client/buffered'
|
|
6
5
|
require_relative '../fetcher/reliable'
|
|
7
6
|
require_relative '../lua'
|
|
8
|
-
require_relative '../health'
|
|
9
7
|
require_relative 'orphan_guard'
|
|
10
8
|
require_relative 'test_mode_guard'
|
|
11
9
|
|
data/lib/wurk/swarm/liveness.rb
CHANGED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Wurk
|
|
4
|
+
class Swarm
|
|
5
|
+
# MAXMEM recycling (Ent §7.5): every CHECK_INTERVAL the supervisor reads
|
|
6
|
+
# each child's memory and hands one over the limit to the restart machine
|
|
7
|
+
# (`recycle`), which replaces it gracefully.
|
|
8
|
+
#
|
|
9
|
+
# A child already over the limit while still FRESH is never recycled: its
|
|
10
|
+
# replacement would boot to the same size and be recycled in turn, a fork
|
|
11
|
+
# loop that replaces a slot every few seconds and drains nothing useful.
|
|
12
|
+
# Such a child is marked for the rest of its life and the limit reported
|
|
13
|
+
# once per slot index at ERROR — the setting is below what a booted child
|
|
14
|
+
# needs. A child that crosses the limit only after FRESH seconds is recycled
|
|
15
|
+
# as before; if its replacement is over within its own fresh window, the
|
|
16
|
+
# replacement is the one marked, so a slot recycles at most once per cause.
|
|
17
|
+
#
|
|
18
|
+
# The metric is `memory_limit_metric`: `:rss` from /proc/<pid>/statm (Ent's
|
|
19
|
+
# meaning; counts pages still shared with the parent), `:pss`/`:uss` from
|
|
20
|
+
# /proc/<pid>/smaps_rollup. Without /proc nothing is read and nothing is
|
|
21
|
+
# recycled; `warn_if_unreadable` says so at boot.
|
|
22
|
+
class MemoryWatch
|
|
23
|
+
CHECK_INTERVAL = 10
|
|
24
|
+
FRESH = Backoff::RESET_AFTER
|
|
25
|
+
SMAPS_FIELDS = { pss: %w[Pss], uss: %w[Private_Clean Private_Dirty] }.freeze
|
|
26
|
+
|
|
27
|
+
def initialize(limit_kb:, metric:, logger:, recycle:, interval: CHECK_INTERVAL, fresh: FRESH,
|
|
28
|
+
proc_root: '/proc', now: -> { ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) })
|
|
29
|
+
@limit_kb = limit_kb
|
|
30
|
+
@metric = metric
|
|
31
|
+
@logger = logger
|
|
32
|
+
@recycle = recycle
|
|
33
|
+
@interval = interval
|
|
34
|
+
@fresh = fresh
|
|
35
|
+
@now = now
|
|
36
|
+
@proc_root = proc_root
|
|
37
|
+
@last_check = nil
|
|
38
|
+
@marked = {}
|
|
39
|
+
@reported_slots = {}
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# `children` is `{ pid => [seconds since fork, slot index] }`.
|
|
43
|
+
def tick(children)
|
|
44
|
+
now = @now.call
|
|
45
|
+
return if @last_check && now - @last_check < @interval
|
|
46
|
+
|
|
47
|
+
@last_check = now
|
|
48
|
+
children.each { |pid, (age, slot)| check(pid, age, slot) }
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def forget(pid)
|
|
52
|
+
@marked.delete(pid)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def warn_if_unreadable
|
|
56
|
+
return if read_kb(::Process.pid)
|
|
57
|
+
|
|
58
|
+
@logger.warn do
|
|
59
|
+
"swarm: memory_limit_mb is set but this host has no /proc #{source} to read #{@metric} from; " \
|
|
60
|
+
'children will not be recycled'
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
private
|
|
65
|
+
|
|
66
|
+
def check(pid, age, slot)
|
|
67
|
+
return if @marked.key?(pid)
|
|
68
|
+
|
|
69
|
+
size_kb = read_kb(pid)
|
|
70
|
+
return if size_kb.nil? || size_kb < @limit_kb
|
|
71
|
+
return mark(pid, size_kb, age, slot) if age < @fresh
|
|
72
|
+
|
|
73
|
+
@logger.warn do
|
|
74
|
+
"swarm: child #{pid} (slot #{slot}) #{@metric.upcase} #{size_kb}KB >= #{@limit_kb}KB " \
|
|
75
|
+
"#{age.round}s after boot; recycling"
|
|
76
|
+
end
|
|
77
|
+
@recycle.call(pid)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def mark(pid, size_kb, age, slot)
|
|
81
|
+
@marked[pid] = true
|
|
82
|
+
return if @reported_slots.key?(slot)
|
|
83
|
+
|
|
84
|
+
@reported_slots[slot] = true
|
|
85
|
+
@logger.error do
|
|
86
|
+
"swarm: child #{pid} (slot #{slot}) is #{size_kb}KB #{@metric.upcase} #{age.round}s after boot, " \
|
|
87
|
+
"already over memory_limit_mb (#{@limit_kb / 1024} MB). Recycling it would only fork another " \
|
|
88
|
+
'child as large, so it is not recycled. Raise the limit above what a booted child uses, or ' \
|
|
89
|
+
'compare against memory_limit_metric :pss. Reported once per slot.'
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def source = @metric == :rss ? 'statm' : 'smaps_rollup'
|
|
94
|
+
|
|
95
|
+
def read_kb(pid)
|
|
96
|
+
@metric == :rss ? statm_kb(pid) : smaps_kb(pid)
|
|
97
|
+
rescue ::SystemCallError, ::IOError
|
|
98
|
+
nil
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def statm_kb(pid)
|
|
102
|
+
Heartbeat.statm_rss_kb(::File.read("#{@proc_root}/#{pid}/statm"))
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def smaps_kb(pid)
|
|
106
|
+
fields = SMAPS_FIELDS.fetch(@metric)
|
|
107
|
+
total = nil
|
|
108
|
+
::File.foreach("#{@proc_root}/#{pid}/smaps_rollup") do |line|
|
|
109
|
+
name, value = line.split(':', 2)
|
|
110
|
+
total = total.to_i + value.to_i if value && fields.include?(name)
|
|
111
|
+
end
|
|
112
|
+
total
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Wurk
|
|
4
|
+
class Swarm
|
|
5
|
+
# The parent's last memory step before its first fork, shared by every
|
|
6
|
+
# swarm entry point (`wurkswarm` and the railtie's co-hosted swarm): load
|
|
7
|
+
# what each child would otherwise load for itself (Wurk's autoloads
|
|
8
|
+
# included: Wurk.eager_load!), then settle the heap, so
|
|
9
|
+
# the children start on pages they share copy-on-write instead of N private
|
|
10
|
+
# copies. Opens no socket, so it may sit between steps 3 and 4. Its effect
|
|
11
|
+
# is the t=0 shared ratio in bench/rss_soak.rb (`RSS_JOBS=0`).
|
|
12
|
+
module Prefork
|
|
13
|
+
# Features a child requires after the fork. `fiddle` arms PR_SET_PDEATHSIG
|
|
14
|
+
# (OrphanGuard#arm_pdeathsig) in every Linux child.
|
|
15
|
+
CHILD_FEATURES = %w[fiddle].freeze
|
|
16
|
+
|
|
17
|
+
module_function
|
|
18
|
+
|
|
19
|
+
# Once, before the first fork only: a respawn forks from a parent that may
|
|
20
|
+
# be a live web process, where a full compaction is a pause its requests
|
|
21
|
+
# would feel. RUBY_DISABLE_WARMUP=1 skips it, as it does `Process.warmup`.
|
|
22
|
+
def run(env = ENV, periodic: nil)
|
|
23
|
+
Wurk.eager_load!
|
|
24
|
+
require_child_features
|
|
25
|
+
Wurk::Cron.prefork(periodic) if periodic
|
|
26
|
+
warmup unless env['RUBY_DISABLE_WARMUP'] == '1'
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def require_child_features
|
|
30
|
+
CHILD_FEATURES.each { |feature| try_require(feature) }
|
|
31
|
+
# Only a host that loaded Vernier can profile a job (Profiler).
|
|
32
|
+
try_require('tempfile') if defined?(::Vernier)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# `Process.warmup` (Ruby >= 3.3) is a full GC, compaction, promotion of
|
|
36
|
+
# survivors to the old generation and a malloc_trim. The 3.2 floor gets
|
|
37
|
+
# the part it has: full GC, then compaction where the platform supports it.
|
|
38
|
+
def warmup(process = ::Process, collector = ::GC)
|
|
39
|
+
return process.warmup if process.respond_to?(:warmup)
|
|
40
|
+
|
|
41
|
+
collector.start(full_mark: true, immediate_sweep: true)
|
|
42
|
+
collector.compact if collector.respond_to?(:compact)
|
|
43
|
+
rescue ::NotImplementedError
|
|
44
|
+
nil
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def try_require(feature)
|
|
48
|
+
require feature
|
|
49
|
+
rescue ::LoadError
|
|
50
|
+
nil
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
data/lib/wurk/swarm.rb
CHANGED
|
@@ -3,10 +3,7 @@
|
|
|
3
3
|
require 'monitor'
|
|
4
4
|
|
|
5
5
|
require_relative 'component'
|
|
6
|
-
require_relative 'heartbeat'
|
|
7
|
-
require_relative 'launcher'
|
|
8
6
|
require_relative 'fetcher/reliable'
|
|
9
|
-
require_relative 'fetcher/reaper'
|
|
10
7
|
require_relative 'capsule'
|
|
11
8
|
require_relative 'keys'
|
|
12
9
|
require_relative 'swarm/child_boot'
|
|
@@ -15,6 +12,8 @@ require_relative 'swarm/crash_watch'
|
|
|
15
12
|
require_relative 'swarm/restart'
|
|
16
13
|
require_relative 'swarm/orphan_guard'
|
|
17
14
|
require_relative 'swarm/liveness'
|
|
15
|
+
require_relative 'swarm/memory_watch'
|
|
16
|
+
require_relative 'swarm/prefork'
|
|
18
17
|
|
|
19
18
|
module Wurk
|
|
20
19
|
# Parent supervisor. Forks N children per the worker topology, monitors
|
|
@@ -48,7 +47,6 @@ module Wurk
|
|
|
48
47
|
SUPERVISE_TICK = 0.2
|
|
49
48
|
RESPAWN_BACKOFF = 1.0
|
|
50
49
|
HEARTBEAT_WAIT = 30
|
|
51
|
-
MEMORY_CHECK_INTERVAL = 10
|
|
52
50
|
DEFAULT_SHUTDOWN_TIMEOUT = 25
|
|
53
51
|
|
|
54
52
|
# The supervisor's own Redis pool: one connection, disjoint from every
|
|
@@ -132,7 +130,6 @@ module Wurk
|
|
|
132
130
|
crash_exit_after: config.swarm_crash_exit_after, exit_on_crash_loop: false)
|
|
133
131
|
@topology = topology
|
|
134
132
|
@config = config
|
|
135
|
-
@memory_limit = memory_limit
|
|
136
133
|
@shutdown_timeout = shutdown_timeout
|
|
137
134
|
# One reentrant lock covers the child table AND the restart machine: they
|
|
138
135
|
# are mutually recursive (Restart#advance calls back into `describe` and
|
|
@@ -149,7 +146,6 @@ module Wurk
|
|
|
149
146
|
@shutdown_requested = false
|
|
150
147
|
@quieted = false
|
|
151
148
|
@requiet_at = 0
|
|
152
|
-
@last_memory_check = 0
|
|
153
149
|
@respawn_backoff = Backoff.new(base: RESPAWN_BACKOFF)
|
|
154
150
|
@crash_watch = CrashWatch.new(exit_after: crash_exit_after, young_within: Backoff::RESET_AFTER)
|
|
155
151
|
@exit_on_crash_loop = exit_on_crash_loop
|
|
@@ -157,6 +153,7 @@ module Wurk
|
|
|
157
153
|
@title_program = nil
|
|
158
154
|
@restart = build_restart
|
|
159
155
|
@liveness = build_liveness(heartbeat_timeout)
|
|
156
|
+
@memory_watch = build_memory_watch(memory_limit)
|
|
160
157
|
@orphan_reaper = Fetcher::Reaper.new(config)
|
|
161
158
|
init_fork_unsafe_handles
|
|
162
159
|
end
|
|
@@ -171,12 +168,14 @@ module Wurk
|
|
|
171
168
|
# children. Installed first, the trap queues the TERM and the supervise
|
|
172
169
|
# loop drains it (relaying to children) even if it arrives mid-boot.
|
|
173
170
|
#
|
|
174
|
-
# `prepare_for_fork!`
|
|
175
|
-
#
|
|
176
|
-
#
|
|
177
|
-
#
|
|
171
|
+
# `prepare_for_fork!` stays after `close_parent_sockets`: it materializes
|
|
172
|
+
# the slot-independent config so every child inherits it copy-on-write, and
|
|
173
|
+
# none of what it touches opens a socket — anything that did would land
|
|
174
|
+
# back on the wrong side of step 3. `settle_memory_before_fork` comes last,
|
|
175
|
+
# after every parent allocation, so the compaction covers them; it opens no
|
|
176
|
+
# socket either.
|
|
178
177
|
#
|
|
179
|
-
# Nothing
|
|
178
|
+
# Nothing that talks to Redis belongs between steps 3 and 4. #101 boot-audit: a pre-fork
|
|
180
179
|
# `SCRIPT LOAD` was tried here (the cache is server-global, so one upload
|
|
181
180
|
# could serve the whole fleet) and MEASURED SLOWER — `bench:swarm_boot`
|
|
182
181
|
# 152 -> 95 i/s, a 36.7% regression. Step 3 has just closed every parent
|
|
@@ -196,6 +195,7 @@ module Wurk
|
|
|
196
195
|
install_signal_handlers if install_signals
|
|
197
196
|
close_parent_sockets
|
|
198
197
|
@config.prepare_for_fork!
|
|
198
|
+
settle_memory_before_fork
|
|
199
199
|
fork_children
|
|
200
200
|
update_proctitle
|
|
201
201
|
child_pids
|
|
@@ -300,7 +300,7 @@ module Wurk
|
|
|
300
300
|
check_liveness unless @stopping
|
|
301
301
|
check_crash_loop unless @stopping
|
|
302
302
|
requiet_children
|
|
303
|
-
|
|
303
|
+
@memory_watch&.tick(child_ages)
|
|
304
304
|
update_proctitle
|
|
305
305
|
rescue StandardError => e
|
|
306
306
|
handle_exception(e, { context: 'swarm-supervise' })
|
|
@@ -351,6 +351,19 @@ module Wurk
|
|
|
351
351
|
))
|
|
352
352
|
end
|
|
353
353
|
|
|
354
|
+
def build_memory_watch(limit_kb)
|
|
355
|
+
return nil unless limit_kb
|
|
356
|
+
|
|
357
|
+
MemoryWatch.new(limit_kb: limit_kb, metric: @config.memory_limit_metric, logger: logger,
|
|
358
|
+
recycle: ->(pid) { @lock.synchronize { @restart.enqueue([pid]) } })
|
|
359
|
+
end
|
|
360
|
+
|
|
361
|
+
# `{ pid => [seconds since fork, slot index] }` for every child in the table.
|
|
362
|
+
def child_ages
|
|
363
|
+
now = monotonic
|
|
364
|
+
@lock.synchronize { @children.transform_values { |meta| [now - meta[:spawned_at], meta[:index]] } }
|
|
365
|
+
end
|
|
366
|
+
|
|
354
367
|
def build_liveness(configured)
|
|
355
368
|
timeout = Liveness.timeout_from(configured)
|
|
356
369
|
return nil unless timeout
|
|
@@ -376,6 +389,13 @@ module Wurk
|
|
|
376
389
|
@liveness.tick(child_pids, paused: paused)
|
|
377
390
|
end
|
|
378
391
|
|
|
392
|
+
# Between steps 3 and 4, once per swarm (a respawn forks from a parent that
|
|
393
|
+
# may be serving requests, where a full compaction is a pause they feel).
|
|
394
|
+
def settle_memory_before_fork
|
|
395
|
+
Prefork.run(periodic: @config.periodic_manager)
|
|
396
|
+
@memory_watch&.warn_if_unreadable
|
|
397
|
+
end
|
|
398
|
+
|
|
379
399
|
# Step 3.
|
|
380
400
|
def close_parent_sockets
|
|
381
401
|
@config.reset_redis_pools!
|
|
@@ -573,6 +593,7 @@ module Wurk
|
|
|
573
593
|
return unless meta
|
|
574
594
|
|
|
575
595
|
@liveness&.forget(pid)
|
|
596
|
+
@memory_watch&.forget(pid)
|
|
576
597
|
return meta if @restart.claim_exit(pid)
|
|
577
598
|
|
|
578
599
|
if @stopping
|
|
@@ -681,36 +702,6 @@ module Wurk
|
|
|
681
702
|
relay_signal('TSTP')
|
|
682
703
|
end
|
|
683
704
|
|
|
684
|
-
def check_memory_pressure
|
|
685
|
-
return unless @memory_limit
|
|
686
|
-
|
|
687
|
-
now = monotonic
|
|
688
|
-
return if now - @last_memory_check < MEMORY_CHECK_INTERVAL
|
|
689
|
-
|
|
690
|
-
@last_memory_check = now
|
|
691
|
-
child_pids.each { |pid| recycle_if_bloated(pid) }
|
|
692
|
-
end
|
|
693
|
-
|
|
694
|
-
# Route a bloated child through the restart state machine (same path as a
|
|
695
|
-
# rolling restart) so recycle is graceful — a healthy replacement takes over
|
|
696
|
-
# before the old child is TERMed — and can't overlap a restart already in
|
|
697
|
-
# flight on the slot.
|
|
698
|
-
def recycle_if_bloated(pid)
|
|
699
|
-
rss = pid_rss_kb(pid)
|
|
700
|
-
return if rss.nil? || rss < @memory_limit
|
|
701
|
-
|
|
702
|
-
logger.warn { "swarm: child #{pid} RSS #{rss}KB >= #{@memory_limit}KB; recycling" }
|
|
703
|
-
@lock.synchronize { @restart.enqueue([pid]) }
|
|
704
|
-
end
|
|
705
|
-
|
|
706
|
-
def pid_rss_kb(pid)
|
|
707
|
-
return nil unless ::File.exist?("/proc/#{pid}/statm")
|
|
708
|
-
|
|
709
|
-
Heartbeat.statm_rss_kb(::File.read("/proc/#{pid}/statm"))
|
|
710
|
-
rescue StandardError
|
|
711
|
-
nil
|
|
712
|
-
end
|
|
713
|
-
|
|
714
705
|
def relay_signal(sig)
|
|
715
706
|
child_pids.each { |pid| safe_kill(pid, sig) }
|
|
716
707
|
end
|
data/lib/wurk/version.rb
CHANGED
data/lib/wurk/web/enterprise.rb
CHANGED
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative '../limiter'
|
|
4
|
-
require_relative '../cron'
|
|
5
4
|
require_relative '../client'
|
|
6
|
-
require_relative '../metrics/query'
|
|
7
5
|
|
|
8
6
|
module Wurk
|
|
9
7
|
class Web
|
|
@@ -45,11 +43,7 @@ module Wurk
|
|
|
45
43
|
# what it returns but leaves the SET alone: GET must not write.
|
|
46
44
|
def list(filter: nil, sweep: !Wurk::Web.config.read_only?)
|
|
47
45
|
members = Wurk.redis(idempotent: true) { |c| c.call('SMEMBERS', Wurk::Limiter::LIST_KEY) }
|
|
48
|
-
|
|
49
|
-
return names if filter.nil? || filter.to_s.empty?
|
|
50
|
-
|
|
51
|
-
needle = filter.to_s.downcase
|
|
52
|
-
names.select { |n| n.downcase.include?(needle) }
|
|
46
|
+
filtered((sweep ? sweep(members) : live(members)).sort, filter)
|
|
53
47
|
end
|
|
54
48
|
|
|
55
49
|
# Drop the names whose metadata is gone, return the ones still live.
|
|
@@ -92,6 +86,68 @@ module Wurk
|
|
|
92
86
|
raw.is_a?(Array) ? raw.each_slice(2).to_h : raw
|
|
93
87
|
end
|
|
94
88
|
|
|
89
|
+
# Liveness probes per process, at most one per this many seconds.
|
|
90
|
+
PROBE_INTERVAL = 60.0
|
|
91
|
+
@probe_lock = ::Mutex.new
|
|
92
|
+
@probe = nil
|
|
93
|
+
|
|
94
|
+
# The dashboard's listing: `count` live limiters from `offset` of the
|
|
95
|
+
# sorted, filtered names, each `[name, metadata]`, and how many names
|
|
96
|
+
# there are. #list probes every member's metadata (an EXISTS each) on
|
|
97
|
+
# every call; here that probe — and, with `sweep`, the removal of the
|
|
98
|
+
# dead — runs at most once per PROBE_INTERVAL per process, and pages in
|
|
99
|
+
# between pay one SMEMBERS and one pipelined HGETALL for their rows.
|
|
100
|
+
# The cost of that: a limiter whose metadata expired since the last
|
|
101
|
+
# probe counts in `total` and is left off its page, and a name the
|
|
102
|
+
# probe found dead and that registers again stays hidden until the
|
|
103
|
+
# next probe.
|
|
104
|
+
#
|
|
105
|
+
# @return [Array(Integer, Array<Array(String, Hash)>)]
|
|
106
|
+
def page(filter:, offset:, count:, sweep: !Wurk::Web.config.read_only?)
|
|
107
|
+
members = Wurk.redis(idempotent: true) { |c| c.call('SMEMBERS', Wurk::Limiter::LIST_KEY) }
|
|
108
|
+
names = filtered((members - known_dead(members, sweep)).sort, filter)
|
|
109
|
+
[names.size, metadata_rows(names[offset, count] || [])]
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def filtered(names, filter)
|
|
113
|
+
return names if filter.nil? || filter.to_s.empty?
|
|
114
|
+
|
|
115
|
+
needle = filter.to_s.downcase
|
|
116
|
+
names.select { |n| n.downcase.include?(needle) }
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Names the last probe of this pool found dead, probing (and sweeping)
|
|
120
|
+
# again once PROBE_INTERVAL has passed.
|
|
121
|
+
def known_dead(members, sweep)
|
|
122
|
+
pool = Wurk.redis_pool
|
|
123
|
+
now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
124
|
+
@probe_lock.synchronize do
|
|
125
|
+
probe = @probe
|
|
126
|
+
return probe[:dead] if probe && probe[:pool].equal?(pool) && now - probe[:at] < PROBE_INTERVAL
|
|
127
|
+
|
|
128
|
+
dead = members - (sweep ? sweep(members) : live(members))
|
|
129
|
+
@probe = { pool: pool, at: now, dead: dead }
|
|
130
|
+
dead
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def metadata_rows(names)
|
|
135
|
+
return [] if names.empty?
|
|
136
|
+
|
|
137
|
+
raws = Wurk.redis(idempotent: true) do |c|
|
|
138
|
+
c.pipelined { |pipe| names.each { |n| pipe.call('HGETALL', meta_key(n)) } }
|
|
139
|
+
end
|
|
140
|
+
names.zip(raws).filter_map do |name, raw|
|
|
141
|
+
meta = raw.is_a?(Array) ? raw.each_slice(2).to_h : raw
|
|
142
|
+
[name, meta] unless meta.empty?
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# Forgets the last probe, so the next #page probes. For tests.
|
|
147
|
+
def reset_probe!
|
|
148
|
+
@probe_lock.synchronize { @probe = nil }
|
|
149
|
+
end
|
|
150
|
+
|
|
95
151
|
def meta_key(name)
|
|
96
152
|
"lmtr:#{name}"
|
|
97
153
|
end
|