wurk 1.12.0 → 1.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -2
- data/app/controllers/concerns/wurk/sse_streaming.rb +36 -1
- data/app/controllers/wurk/api/pagination.rb +9 -7
- data/app/controllers/wurk/api/serializers.rb +8 -2
- data/app/controllers/wurk/api_controller.rb +57 -24
- data/app/controllers/wurk/application_controller.rb +0 -2
- data/app/controllers/wurk/dashboard_controller.rb +0 -1
- data/config/routes.rb +13 -1
- data/exe/sidekiq +8 -0
- data/exe/wurk +1 -1
- data/lib/generators/wurk/install/install_generator.rb +56 -0
- data/lib/sidekiq/cli.rb +4 -3
- data/lib/sidekiq/deploy.rb +2 -1
- data/lib/sidekiq/embedded.rb +2 -1
- data/lib/sidekiq/iterable_job.rb +2 -1
- data/lib/sidekiq/job/iterable/active_record_enumerator.rb +2 -1
- data/lib/sidekiq/job/iterable/csv_enumerator.rb +2 -1
- data/lib/sidekiq/job/iterable.rb +2 -1
- data/lib/sidekiq/launcher.rb +2 -1
- data/lib/sidekiq/manager.rb +2 -1
- data/lib/sidekiq/metrics/query.rb +2 -1
- data/lib/sidekiq/pro/web.rb +2 -1
- data/lib/sidekiq/scheduled.rb +2 -1
- data/lib/sidekiq/web.rb +2 -1
- data/lib/sidekiq.rb +3 -2
- data/lib/wurk/api/fast.rb +4 -18
- data/lib/wurk/api/queues.rb +12 -10
- data/lib/wurk/api/serializers.rb +6 -7
- data/lib/wurk/api/swarm.rb +0 -1
- data/lib/wurk/batch/server_middleware.rb +4 -0
- data/lib/wurk/batch/status.rb +45 -22
- data/lib/wurk/batch_set.rb +34 -0
- data/lib/wurk/boot_checks.rb +80 -0
- data/lib/wurk/cli.rb +72 -36
- data/lib/wurk/client/buffered.rb +97 -26
- data/lib/wurk/client.rb +21 -1
- data/lib/wurk/compat/deploy.rb +7 -0
- data/lib/wurk/compat/encryption.rb +7 -0
- data/lib/wurk/compat/flow.rb +8 -0
- data/lib/wurk/compat/iterable_job.rb +8 -0
- data/lib/wurk/compat/profile_set.rb +8 -0
- data/lib/wurk/compat/server.rb +15 -0
- data/lib/wurk/compat/web.rb +20 -0
- data/lib/wurk/compat.rb +34 -40
- data/lib/wurk/configuration.rb +73 -15
- data/lib/wurk/cron.rb +146 -47
- data/lib/wurk/dead_set.rb +10 -6
- data/lib/wurk/embedded.rb +0 -2
- data/lib/wurk/engine.rb +46 -4
- data/lib/wurk/error_coalescer.rb +21 -1
- data/lib/wurk/fetcher/many_queues.rb +8 -6
- data/lib/wurk/fetcher/orphan_reclaim.rb +48 -11
- data/lib/wurk/{middleware → fetcher}/poison_pill.rb +4 -2
- data/lib/wurk/fetcher/reaper.rb +28 -14
- data/lib/wurk/fetcher/reliable.rb +50 -32
- data/lib/wurk/fetcher/self_reconcile.rb +0 -1
- data/lib/wurk/fetcher/unit_of_work.rb +17 -2
- data/lib/wurk/flow/status.rb +10 -4
- data/lib/wurk/flow_set.rb +24 -2
- data/lib/wurk/health.rb +0 -1
- data/lib/wurk/heartbeat.rb +13 -7
- data/lib/wurk/import/sidekiq_cron.rb +0 -1
- data/lib/wurk/job_logger.rb +4 -2
- data/lib/wurk/job_record.rb +2 -1
- data/lib/wurk/job_retry.rb +1 -1
- data/lib/wurk/job_set.rb +132 -6
- data/lib/wurk/job_util.rb +14 -0
- data/lib/wurk/launcher.rb +30 -15
- data/lib/wurk/lazy_loader.rb +60 -0
- data/lib/wurk/leader.rb +1 -8
- data/lib/wurk/limiter/bucket.rb +0 -2
- data/lib/wurk/limiter/concurrent.rb +19 -36
- data/lib/wurk/limiter/leaky.rb +0 -2
- data/lib/wurk/limiter/points.rb +0 -2
- data/lib/wurk/limiter/server_middleware.rb +4 -1
- data/lib/wurk/limiter/unlimited.rb +0 -2
- data/lib/wurk/limiter/window.rb +0 -2
- data/lib/wurk/limiter.rb +0 -7
- data/lib/wurk/logger.rb +44 -5
- data/lib/wurk/lua/fast_delete.rb +58 -0
- data/lib/wurk/lua/fast_delete_window.lua +46 -0
- data/lib/wurk/lua/limiter_concurrent_acquire.lua +10 -3
- data/lib/wurk/lua/limiter_concurrent_release.lua +13 -4
- data/lib/wurk/lua.rb +1 -41
- data/lib/wurk/manager.rb +27 -1
- data/lib/wurk/metrics/dashboard_series.rb +38 -18
- data/lib/wurk/metrics/history.rb +24 -11
- data/lib/wurk/metrics/prometheus.rb +0 -1
- data/lib/wurk/metrics/query.rb +0 -2
- data/lib/wurk/metrics/queue_rollup.rb +33 -11
- data/lib/wurk/metrics/statsd.rb +4 -0
- data/lib/wurk/middleware/chain.rb +25 -3
- data/lib/wurk/middleware/expiry.rb +4 -0
- data/lib/wurk/middleware/interrupt_handler.rb +4 -1
- data/lib/wurk/middleware/status.rb +4 -0
- data/lib/wurk/middleware/timeout.rb +10 -6
- data/lib/wurk/preflight/app_scan.rb +182 -0
- data/lib/wurk/preflight/command.rb +21 -5
- data/lib/wurk/preflight/owners.rb +0 -1
- data/lib/wurk/preflight/payload_sample.rb +0 -1
- data/lib/wurk/preflight/redis_guard.rb +98 -0
- data/lib/wurk/preflight/verify.rb +0 -2
- data/lib/wurk/preflight.rb +9 -5
- data/lib/wurk/process_set.rb +6 -1
- data/lib/wurk/profiler.rb +2 -1
- data/lib/wurk/queue.rb +33 -6
- data/lib/wurk/queue_consumers.rb +10 -1
- data/lib/wurk/rails_boot.rb +9 -4
- data/lib/wurk/railtie.rb +3 -4
- data/lib/wurk/redis_options.rb +5 -0
- data/lib/wurk/redis_pool.rb +24 -1
- data/lib/wurk/sentry/error_handler.rb +5 -0
- data/lib/wurk/sentry/middleware.rb +4 -0
- data/lib/wurk/sentry.rb +11 -0
- data/lib/wurk/sorted_entry.rb +16 -12
- data/lib/wurk/stats.rb +85 -1
- data/lib/wurk/swarm/child_boot.rb +0 -2
- data/lib/wurk/swarm/liveness.rb +0 -2
- data/lib/wurk/swarm/memory_watch.rb +116 -0
- data/lib/wurk/swarm/prefork.rb +54 -0
- data/lib/wurk/swarm.rb +95 -55
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/enterprise.rb +63 -7
- data/lib/wurk/web/rack_app.rb +8 -6
- data/lib/wurk/web/stats_routes.rb +93 -0
- data/lib/wurk.rb +34 -30
- data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js +1 -0
- data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.br +0 -0
- data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.gz +0 -0
- data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js +1 -0
- data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.br +0 -0
- data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js +1 -0
- data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.br +0 -0
- data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js +1 -0
- data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.br +0 -0
- data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js +1 -0
- data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.br +0 -0
- data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js +1 -0
- data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.br +0 -0
- data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js +1 -0
- data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.br +0 -0
- data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js +1 -0
- data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.br +0 -0
- data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.gz +0 -0
- data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js +2 -0
- data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.br +0 -0
- data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.gz +0 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js +1 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.br +0 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{FlowState-yb7virqR.js → FlowState-ClzInw0-.js} +1 -1
- data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.br +0 -0
- data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js +1 -0
- data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.br +0 -0
- data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.gz +0 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js +2 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.br +0 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js +1 -0
- data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.br +0 -0
- data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js +1 -0
- data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.br +0 -0
- data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.gz +0 -0
- data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js +1 -0
- data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.br +0 -0
- data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.gz +0 -0
- data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js +1 -0
- data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.br +0 -0
- data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js +1 -0
- data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.br +0 -0
- data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js +1 -0
- data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.br +0 -0
- data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js +1 -0
- data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.br +0 -0
- data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js +1 -0
- data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.br +0 -0
- data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Search-D42Va7lB.js +1 -0
- data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.br +0 -0
- data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.gz +0 -0
- data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js +1 -0
- data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.br +0 -0
- data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.gz +0 -0
- data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js +1 -0
- data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.br +0 -0
- data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.gz +0 -0
- data/vendor/assets/dashboard/assets/de-B2NCIuzo.js +1 -0
- data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.br +0 -0
- data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.gz +0 -0
- data/vendor/assets/dashboard/assets/es-BGeu_JNB.js +1 -0
- data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.br +0 -0
- data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.gz +0 -0
- data/vendor/assets/dashboard/assets/fr-DQVBWFId.js +1 -0
- data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.br +0 -0
- data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.gz +0 -0
- data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js +1 -0
- data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.br +0 -0
- data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.gz +0 -0
- data/vendor/assets/dashboard/assets/index-CiLU1zz8.js +141 -0
- data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.br +0 -0
- data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.gz +0 -0
- data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css +1 -0
- data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.br +0 -0
- data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.gz +0 -0
- data/vendor/assets/dashboard/assets/ja-D06IBqzL.js +1 -0
- data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.br +0 -0
- data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.gz +0 -0
- data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js +1 -0
- data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.br +0 -0
- data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.gz +0 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js +1 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.br +0 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.gz +0 -0
- data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js +1 -0
- data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.br +0 -0
- data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{utils-CnWZzW2L.js → utils-69XSoGbQ.js} +1 -1
- data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.br +0 -0
- data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.gz +0 -0
- data/vendor/assets/dashboard/assets/wurk-logo-ydeSB9B8.png +0 -0
- data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js +1 -0
- data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.br +0 -0
- data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.gz +0 -0
- data/vendor/assets/dashboard/index.html +3 -23
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +132 -38
- data/vendor/assets/dashboard/assets/ArgsValue-BJol9S0s.js +0 -1
- data/vendor/assets/dashboard/assets/BatchDetail-D_zGQ9Sd.js +0 -1
- data/vendor/assets/dashboard/assets/Batches-Dt_xC660.js +0 -1
- data/vendor/assets/dashboard/assets/Busy-CWRthsmc.js +0 -1
- data/vendor/assets/dashboard/assets/Cron-qeGr2YAm.js +0 -1
- data/vendor/assets/dashboard/assets/Dashboard-DQWaAwPA.js +0 -1
- data/vendor/assets/dashboard/assets/Dead-D9gOPHtK.js +0 -1
- data/vendor/assets/dashboard/assets/Extension-DONXVLdA.js +0 -1
- data/vendor/assets/dashboard/assets/FilterBox-BbB6gXHp.js +0 -2
- data/vendor/assets/dashboard/assets/FlowDetail-CrmnVRp0.js +0 -1
- data/vendor/assets/dashboard/assets/Flows-g4GC8KmE.js +0 -1
- data/vendor/assets/dashboard/assets/JobDetailModal-C86k_1wj.js +0 -2
- data/vendor/assets/dashboard/assets/Limiters-zyO-iMlK.js +0 -1
- data/vendor/assets/dashboard/assets/Metrics-Ckq5fNBv.js +0 -1
- data/vendor/assets/dashboard/assets/NotFound-C8NnyjlY.js +0 -1
- data/vendor/assets/dashboard/assets/PageHeader-vWeBbSPF.js +0 -1
- data/vendor/assets/dashboard/assets/Profiles-DRl-FXPA.js +0 -1
- data/vendor/assets/dashboard/assets/Queues-DkXWrihA.js +0 -1
- data/vendor/assets/dashboard/assets/Retries-CmlojYIy.js +0 -1
- data/vendor/assets/dashboard/assets/Scheduled-Cbbpp3xw.js +0 -1
- data/vendor/assets/dashboard/assets/Search-LaEpj2fM.js +0 -1
- data/vendor/assets/dashboard/assets/charts-D_PQLoyc.js +0 -1
- data/vendor/assets/dashboard/assets/fa-brands-400-Bs6tcqqs.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-regular-400-DRN8N0d1.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-solid-900-IAB4Droh.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-v4compatibility-CErXDOsT.woff2 +0 -0
- data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +0 -1
- data/vendor/assets/dashboard/assets/index--xvvNlYx.css +0 -1
- data/vendor/assets/dashboard/assets/index-FBKGtysy.js +0 -141
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-BiwhHqgh.js +0 -1
- data/vendor/assets/dashboard/assets/useSort-jBYVwXvg.js +0 -1
- data/vendor/assets/dashboard/assets/wurk-logo-Vy3xW4K0.png +0 -0
- data/vendor/assets/dashboard/og-image.png +0 -0
- data/vendor/assets/dashboard/wurk-logo.webp +0 -0
data/lib/wurk/configuration.rb
CHANGED
|
@@ -5,7 +5,6 @@ require_relative 'middleware/chain'
|
|
|
5
5
|
require_relative 'capsule'
|
|
6
6
|
require_relative 'pool_checkout'
|
|
7
7
|
require_relative 'context'
|
|
8
|
-
require_relative 'topology'
|
|
9
8
|
require_relative 'cpu_count'
|
|
10
9
|
require_relative 'redis_options'
|
|
11
10
|
require_relative 'keys'
|
|
@@ -488,12 +487,15 @@ module Wurk
|
|
|
488
487
|
#
|
|
489
488
|
# Spec: docs/target/sidekiq-ent.md §2.
|
|
490
489
|
def periodic
|
|
491
|
-
require_relative 'cron'
|
|
492
490
|
@periodic_manager ||= Wurk::Cron::Manager.new(self, deferred: true)
|
|
493
491
|
yield @periodic_manager if block_given?
|
|
494
492
|
@periodic_manager
|
|
495
493
|
end
|
|
496
494
|
|
|
495
|
+
# The periodic manager if `config.periodic` was ever called, else nil:
|
|
496
|
+
# unlike #periodic, never creates one (Swarm::Prefork reads it).
|
|
497
|
+
attr_reader :periodic_manager
|
|
498
|
+
|
|
497
499
|
# Writes the `config.periodic` registrations to Redis. Launcher#boot calls
|
|
498
500
|
# it, so the loops are registered by a booting server, against the Redis
|
|
499
501
|
# settings the app ended up with.
|
|
@@ -732,12 +734,12 @@ module Wurk
|
|
|
732
734
|
|
|
733
735
|
# Web UI configuration: the authorization hook and read-only mode. Returns
|
|
734
736
|
# the process-wide `Wurk::Web.config` singleton so `config.web.read_only =
|
|
735
|
-
# true` and the engine middleware share one source of truth.
|
|
736
|
-
#
|
|
737
|
+
# true` and the engine middleware share one source of truth. The web layer
|
|
738
|
+
# is an autoload (Wurk::LazyLoader), so standalone boot stays lean until this
|
|
739
|
+
# is called.
|
|
737
740
|
#
|
|
738
741
|
# Spec: docs/target/sidekiq-ent.md §9.2.
|
|
739
742
|
def web
|
|
740
|
-
require_relative 'web/config'
|
|
741
743
|
Wurk::Web.config
|
|
742
744
|
end
|
|
743
745
|
|
|
@@ -875,37 +877,53 @@ module Wurk
|
|
|
875
877
|
# time, and under the railtie that is Bundler.require, before the
|
|
876
878
|
# `wurk.server_mode` initializer: dropping it left sidekiq-cron, sentry-sidekiq
|
|
877
879
|
# and the like silently absent from a process that does run workers.
|
|
880
|
+
#
|
|
881
|
+
# So the skip is DEBUG, as in stock Sidekiq, which skips it silently: every
|
|
882
|
+
# worker entry point enters server mode and runs the kept blocks, and a
|
|
883
|
+
# process that never does (a test run, `rails runner`, a web process) is
|
|
884
|
+
# meant to skip them. A WARN here was noise in each of those.
|
|
878
885
|
def configure_server(&block)
|
|
879
886
|
return unless block
|
|
880
|
-
return
|
|
887
|
+
return run_server_block(block) if server?
|
|
881
888
|
|
|
882
889
|
@deferred_server_blocks << block
|
|
883
890
|
return if @configure_server_skip_warned
|
|
884
891
|
|
|
885
892
|
@configure_server_skip_warned = true
|
|
886
|
-
logger.
|
|
893
|
+
logger.debug { 'configure_server block skipped: server? is false' }
|
|
887
894
|
nil
|
|
888
895
|
end
|
|
889
896
|
|
|
890
897
|
def enter_server_mode
|
|
891
898
|
self[:server] = true
|
|
892
|
-
@deferred_server_blocks.shift
|
|
899
|
+
run_server_block(@deferred_server_blocks.shift) until @deferred_server_blocks.empty?
|
|
893
900
|
end
|
|
894
901
|
|
|
895
902
|
# A web process that also boots the swarm (Wurk::RailsBoot) is flagged
|
|
896
903
|
# server, yet every enqueue it makes from a request is client work, so
|
|
897
|
-
# `client_in_server` lets its configure_client blocks run too.
|
|
898
|
-
#
|
|
904
|
+
# `client_in_server` lets its configure_client blocks run too.
|
|
905
|
+
#
|
|
906
|
+
# In a server, skipping the client block is the Sidekiq contract and the
|
|
907
|
+
# usual case: an app configures both roles. So the skip itself is DEBUG,
|
|
908
|
+
# and BootChecks warns at boot only for the case worth one, a server
|
|
909
|
+
# process in which no configure_server block ran at all (#816).
|
|
899
910
|
def configure_client
|
|
900
911
|
return unless block_given?
|
|
901
912
|
return yield self if !server? || @options[:client_in_server]
|
|
913
|
+
|
|
902
914
|
return if @configure_client_skip_warned
|
|
903
915
|
|
|
904
916
|
@configure_client_skip_warned = true
|
|
905
|
-
logger.
|
|
917
|
+
logger.debug { 'configure_client block skipped: server? is true and client_in_server is false or nil' }
|
|
906
918
|
nil
|
|
907
919
|
end
|
|
908
920
|
|
|
921
|
+
# True once a configure_client block was skipped and no configure_server
|
|
922
|
+
# block ran: the server got none of the settings the app made for a role.
|
|
923
|
+
def client_config_only?
|
|
924
|
+
@configure_client_skip_warned == true && !@configure_server_ran
|
|
925
|
+
end
|
|
926
|
+
|
|
909
927
|
def server?
|
|
910
928
|
@options[:server] == true
|
|
911
929
|
end
|
|
@@ -920,9 +938,14 @@ module Wurk
|
|
|
920
938
|
|
|
921
939
|
def topology=(value)
|
|
922
940
|
guard_frozen!
|
|
941
|
+
@topology_assigned = true
|
|
923
942
|
@topology = value
|
|
924
943
|
end
|
|
925
944
|
|
|
945
|
+
# Whether the app declared a topology, as opposed to the default one
|
|
946
|
+
# #topology builds. Only a swarm reads it.
|
|
947
|
+
def topology_assigned? = @topology_assigned == true
|
|
948
|
+
|
|
926
949
|
# Memory-based child recycling (Sidekiq Ent §7.5): the swarm parent TERMs
|
|
927
950
|
# (and respawns) any child whose RSS exceeds this many MB. Set in code or
|
|
928
951
|
# via SIDEKIQ_MAXMEM_MB (WURK_MAXMEM_MB is the native alias); an explicit
|
|
@@ -936,13 +959,29 @@ module Wurk
|
|
|
936
959
|
@memory_limit_mb = value.nil? ? nil : Integer(value)
|
|
937
960
|
end
|
|
938
961
|
|
|
939
|
-
# Threshold in KB, the unit the swarm
|
|
940
|
-
#
|
|
962
|
+
# Threshold in KB, the unit the swarm reads child memory in. nil when
|
|
963
|
+
# recycling is disabled.
|
|
941
964
|
def memory_limit_kb
|
|
942
965
|
mb = memory_limit_mb
|
|
943
966
|
mb&.positive? ? mb * 1024 : nil
|
|
944
967
|
end
|
|
945
968
|
|
|
969
|
+
# What `memory_limit_mb` is compared against. `:rss` (the default, and
|
|
970
|
+
# Enterprise's) counts pages still shared copy-on-write with the parent, so
|
|
971
|
+
# every child carries the whole preloaded app; `:pss` splits shared pages
|
|
972
|
+
# across their sharers and `:uss` counts only the child's own. Wurk-only;
|
|
973
|
+
# WURK_MAXMEM_METRIC sets it, an explicit value wins.
|
|
974
|
+
MEMORY_LIMIT_METRICS = %i[rss pss uss].freeze
|
|
975
|
+
|
|
976
|
+
def memory_limit_metric
|
|
977
|
+
@memory_limit_metric || env_memory_limit_metric || :rss
|
|
978
|
+
end
|
|
979
|
+
|
|
980
|
+
def memory_limit_metric=(value)
|
|
981
|
+
guard_frozen!
|
|
982
|
+
@memory_limit_metric = value.nil? ? nil : parse_memory_limit_metric(value)
|
|
983
|
+
end
|
|
984
|
+
|
|
946
985
|
# The pre-fork half of `freeze!`, and the only half a forking parent can
|
|
947
986
|
# run: capsules stay writable until each child has applied its slot
|
|
948
987
|
# (ChildBoot#apply_slot_to_config) and opened its own Redis pools. What is
|
|
@@ -1001,6 +1040,11 @@ module Wurk
|
|
|
1001
1040
|
|
|
1002
1041
|
private
|
|
1003
1042
|
|
|
1043
|
+
def run_server_block(block)
|
|
1044
|
+
@configure_server_ran = true
|
|
1045
|
+
block.call(self)
|
|
1046
|
+
end
|
|
1047
|
+
|
|
1004
1048
|
# Pools read the hash once, when built, so the later assignment silently does
|
|
1005
1049
|
# nothing to this process's connections until something rebuilds them.
|
|
1006
1050
|
def warn_on_late_redis_assignment
|
|
@@ -1079,7 +1123,7 @@ module Wurk
|
|
|
1079
1123
|
# as throttling or memory pressure.
|
|
1080
1124
|
def default_child_count
|
|
1081
1125
|
count, source = child_count_and_source
|
|
1082
|
-
logger.info { "Wurk swarm:
|
|
1126
|
+
logger.info { "Wurk swarm size: #{count} #{count == 1 ? 'child' : 'children'} (#{source})" }
|
|
1083
1127
|
count
|
|
1084
1128
|
end
|
|
1085
1129
|
|
|
@@ -1153,6 +1197,21 @@ module Wurk
|
|
|
1153
1197
|
nil
|
|
1154
1198
|
end
|
|
1155
1199
|
|
|
1200
|
+
def env_memory_limit_metric
|
|
1201
|
+
raw = ENV.fetch('WURK_MAXMEM_METRIC', nil)
|
|
1202
|
+
return nil if raw.nil? || raw.strip.empty?
|
|
1203
|
+
|
|
1204
|
+
parse_memory_limit_metric(raw.strip.downcase)
|
|
1205
|
+
end
|
|
1206
|
+
|
|
1207
|
+
def parse_memory_limit_metric(value)
|
|
1208
|
+
metric = value.to_s.to_sym
|
|
1209
|
+
return metric if MEMORY_LIMIT_METRICS.include?(metric)
|
|
1210
|
+
|
|
1211
|
+
raise ArgumentError,
|
|
1212
|
+
"memory_limit_metric must be one of #{MEMORY_LIMIT_METRICS.join(', ')} (got #{value.inspect})"
|
|
1213
|
+
end
|
|
1214
|
+
|
|
1156
1215
|
def guard_frozen!
|
|
1157
1216
|
raise FrozenError, 'Wurk::Configuration is frozen' if @frozen
|
|
1158
1217
|
end
|
|
@@ -1164,7 +1223,6 @@ module Wurk
|
|
|
1164
1223
|
def truthy_setting?(value)
|
|
1165
1224
|
return !!value unless value.is_a?(String)
|
|
1166
1225
|
|
|
1167
|
-
require_relative 'web/config'
|
|
1168
1226
|
!Wurk::Web::Config::FALSEY_STRINGS.include?(value.strip.downcase)
|
|
1169
1227
|
end
|
|
1170
1228
|
|
data/lib/wurk/cron.rb
CHANGED
|
@@ -5,7 +5,6 @@ require 'digest'
|
|
|
5
5
|
require 'time'
|
|
6
6
|
require_relative 'component'
|
|
7
7
|
require_relative 'client'
|
|
8
|
-
require_relative 'leader'
|
|
9
8
|
require_relative 'timer_loop'
|
|
10
9
|
|
|
11
10
|
module Wurk
|
|
@@ -74,8 +73,27 @@ module Wurk
|
|
|
74
73
|
|
|
75
74
|
MAX_LOOKAHEAD_MINUTES = 366 * 24 * 60 * 4
|
|
76
75
|
|
|
76
|
+
# Distinct schedules a process keeps parsed. A loop is rebuilt from its
|
|
77
|
+
# hash on every tick, and a fleet has far fewer schedules than loops.
|
|
78
|
+
CACHE_LIMIT = 1024
|
|
79
|
+
|
|
77
80
|
attr_reader :expression, :fields
|
|
78
81
|
|
|
82
|
+
@cache = {}
|
|
83
|
+
@cache_mutex = ::Mutex.new
|
|
84
|
+
|
|
85
|
+
# A shared, already-parsed Parser for `expression`. Raises ArgumentError
|
|
86
|
+
# like `new` on a bad one, which is never cached. Parsers are not mutated
|
|
87
|
+
# after construction, so sharing one across loops and threads is safe.
|
|
88
|
+
def self.cached(expression)
|
|
89
|
+
@cache_mutex.synchronize do
|
|
90
|
+
@cache.fetch(expression) do
|
|
91
|
+
@cache.clear if @cache.size >= CACHE_LIMIT
|
|
92
|
+
@cache[expression] = new(expression)
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
|
|
79
97
|
def initialize(expression)
|
|
80
98
|
parts = normalize_expression(expression)
|
|
81
99
|
@expression = parts.join(' ')
|
|
@@ -287,13 +305,18 @@ module Wurk
|
|
|
287
305
|
@klass = klass
|
|
288
306
|
@options = stringify_options(options)
|
|
289
307
|
@tz = tz
|
|
290
|
-
@parser = Parser.
|
|
308
|
+
@parser = Parser.cached(schedule)
|
|
291
309
|
@lid = lid || Cron.lid(schedule, klass, @options)
|
|
292
310
|
@paused = paused.nil? ? [true, '1', 1].include?(@options['paused']) : paused
|
|
311
|
+
@fire_marks = nil
|
|
293
312
|
end
|
|
294
313
|
|
|
314
|
+
# The raw `lf` / `nf` strings of the hash this loop was read from, so a
|
|
315
|
+
# tick decides on the read it already made; nil for a loop built in code.
|
|
316
|
+
attr_accessor :fire_marks
|
|
317
|
+
|
|
295
318
|
def parser
|
|
296
|
-
@parser ||= Parser.
|
|
319
|
+
@parser ||= Parser.cached(@schedule)
|
|
297
320
|
end
|
|
298
321
|
|
|
299
322
|
def paused?
|
|
@@ -492,7 +515,8 @@ module Wurk
|
|
|
492
515
|
return self unless @deferred
|
|
493
516
|
|
|
494
517
|
@deferred = false
|
|
495
|
-
@loops
|
|
518
|
+
Cron.persist_all(@loops)
|
|
519
|
+
prune_superseded(@loops.map(&:klass).uniq)
|
|
496
520
|
prune_unregistered if @authoritative
|
|
497
521
|
self
|
|
498
522
|
end
|
|
@@ -530,7 +554,7 @@ module Wurk
|
|
|
530
554
|
|
|
531
555
|
def persist(loop_obj)
|
|
532
556
|
Cron.persist(loop_obj)
|
|
533
|
-
prune_superseded(loop_obj.klass)
|
|
557
|
+
prune_superseded([loop_obj.klass])
|
|
534
558
|
end
|
|
535
559
|
|
|
536
560
|
def prune_unregistered
|
|
@@ -559,12 +583,14 @@ module Wurk
|
|
|
559
583
|
# an edited `register` line yields a new loop and orphans the old one).
|
|
560
584
|
# Registering the same class twice in one block is fine — the second
|
|
561
585
|
# register re-persists its loop right after the first one's prune.
|
|
562
|
-
def prune_superseded(
|
|
586
|
+
def prune_superseded(klass_names)
|
|
563
587
|
return unless @prune
|
|
564
|
-
return if @pruned_klasses[klass_name]
|
|
565
588
|
|
|
566
|
-
@pruned_klasses[
|
|
567
|
-
|
|
589
|
+
fresh = klass_names.reject { |name| @pruned_klasses[name] }
|
|
590
|
+
return if fresh.empty?
|
|
591
|
+
|
|
592
|
+
fresh.each { |name| @pruned_klasses[name] = true }
|
|
593
|
+
Cron.prune_superseded(fresh, @loops.map(&:lid))
|
|
568
594
|
end
|
|
569
595
|
end
|
|
570
596
|
|
|
@@ -588,13 +614,7 @@ module Wurk
|
|
|
588
614
|
def each
|
|
589
615
|
return enum_for(:each) unless block_given?
|
|
590
616
|
|
|
591
|
-
|
|
592
|
-
c.call('SMEMBERS', PERIODIC_KEY).filter_map do |lid|
|
|
593
|
-
h = c.call('HGETALL', "#{LOOP_PREFIX}#{lid}")
|
|
594
|
-
[lid, h] unless h.nil? || h.empty?
|
|
595
|
-
end
|
|
596
|
-
end
|
|
597
|
-
rows.each do |lid, h|
|
|
617
|
+
read_rows.each do |lid, h|
|
|
598
618
|
loop_obj = parse(lid, h)
|
|
599
619
|
yield loop_obj if loop_obj
|
|
600
620
|
end
|
|
@@ -616,8 +636,23 @@ module Wurk
|
|
|
616
636
|
|
|
617
637
|
private
|
|
618
638
|
|
|
639
|
+
# [lid, hash] for every registered loop that still has its hash: one
|
|
640
|
+
# SMEMBERS, then every HGETALL in one pipeline.
|
|
641
|
+
def read_rows
|
|
642
|
+
redis(idempotent: true) do |c|
|
|
643
|
+
lids = c.call('SMEMBERS', PERIODIC_KEY)
|
|
644
|
+
next [] if lids.empty?
|
|
645
|
+
|
|
646
|
+
hashes = c.pipelined { |p| lids.each { |lid| p.call('HGETALL', "#{LOOP_PREFIX}#{lid}") } }
|
|
647
|
+
lids.zip(hashes).filter_map do |lid, h|
|
|
648
|
+
h = h.each_slice(2).to_h if h.is_a?(Array)
|
|
649
|
+
[lid, h] unless h.nil? || h.empty?
|
|
650
|
+
end
|
|
651
|
+
end
|
|
652
|
+
end
|
|
653
|
+
|
|
619
654
|
def parse(lid, hash)
|
|
620
|
-
Loop.from_redis(lid, hash)
|
|
655
|
+
Loop.from_redis(lid, hash).tap { |lp| lp.fire_marks = hash.values_at('lf', 'nf') }
|
|
621
656
|
rescue JSON::ParserError, ArgumentError => e
|
|
622
657
|
Wurk.logger&.warn("[cron] skipping unreadable loop lid=#{lid}: #{e.class}: #{e.message}")
|
|
623
658
|
nil
|
|
@@ -679,6 +714,7 @@ module Wurk
|
|
|
679
714
|
@sleeper = ::ConditionVariable.new
|
|
680
715
|
@on_leader = method(:wake_for_leadership)
|
|
681
716
|
@unknown_reported = ::Set.new
|
|
717
|
+
@derived = {}
|
|
682
718
|
end
|
|
683
719
|
|
|
684
720
|
# The loop waits one interval before its first tick unless this process
|
|
@@ -720,11 +756,14 @@ module Wurk
|
|
|
720
756
|
def tick
|
|
721
757
|
return unless leader?
|
|
722
758
|
|
|
759
|
+
seen = ::Set.new
|
|
723
760
|
LoopSet.new(@config).each do |lp|
|
|
724
|
-
|
|
761
|
+
seen << derived_key(lp)
|
|
762
|
+
enqueue_if_due(lp, lp.fire_marks)
|
|
725
763
|
rescue StandardError => e
|
|
726
764
|
handle_exception(e, { context: 'cron-poller', lid: lp.lid, klass: lp.klass })
|
|
727
765
|
end
|
|
766
|
+
@derived.select! { |key, _| seen.include?(key) }
|
|
728
767
|
rescue StandardError => e
|
|
729
768
|
handle_exception(e, { context: 'cron-poller' })
|
|
730
769
|
end
|
|
@@ -741,11 +780,14 @@ module Wurk
|
|
|
741
780
|
# One claim, one push at most, whichever slot it pushes for: the occurrence
|
|
742
781
|
# due now stands in for a stale one under the stale one's mark, so a poller
|
|
743
782
|
# that read the same mark loses the CAS for both.
|
|
744
|
-
|
|
783
|
+
#
|
|
784
|
+
# `marks` is the raw [`lf`, `nf`] pair the caller just read (a tick's
|
|
785
|
+
# LoopSet read); without it they are read here.
|
|
786
|
+
def enqueue_if_due(loop_obj, marks = nil)
|
|
745
787
|
return if loop_obj.paused?
|
|
746
788
|
|
|
747
789
|
now = ::Time.now.to_i
|
|
748
|
-
slot, mark = due_slot(loop_obj, now)
|
|
790
|
+
slot, mark = due_slot(loop_obj, now, marks)
|
|
749
791
|
return if slot.nil?
|
|
750
792
|
|
|
751
793
|
firing = slot_to_fire(loop_obj, slot, now)
|
|
@@ -822,14 +864,32 @@ module Wurk
|
|
|
822
864
|
# the loop has no resolvable occurrence or none is due yet. The token is
|
|
823
865
|
# the stored `nf` when there is one, otherwise the slot we derived — see
|
|
824
866
|
# the `cron_claim_fire` script for how the two cases differ.
|
|
825
|
-
def due_slot(loop_obj, now)
|
|
826
|
-
prev_fire, next_fire, mark = read_fire_marks(loop_obj.lid)
|
|
827
|
-
next_fire ||= loop_obj
|
|
867
|
+
def due_slot(loop_obj, now, marks)
|
|
868
|
+
prev_fire, next_fire, mark = marks ? parse_fire_marks(marks) : read_fire_marks(loop_obj.lid)
|
|
869
|
+
next_fire ||= derived_next_fire(loop_obj, prev_fire || (now - @tick_interval))
|
|
828
870
|
return if next_fire.nil? || next_fire > now
|
|
829
871
|
|
|
830
872
|
[next_fire, mark || next_fire.to_s]
|
|
831
873
|
end
|
|
832
874
|
|
|
875
|
+
# A loop with no `nf` (never fired, or an exhausted schedule) derives its
|
|
876
|
+
# next occurrence by walking Parser#next_fire_at minute by minute: a year
|
|
877
|
+
# of minutes for a yearly loop, on every tick. The answer for one origin
|
|
878
|
+
# still holds for any later origin it is past, so it is kept per loop
|
|
879
|
+
# until the origin overtakes it. Nothing is written: the stored hash stays
|
|
880
|
+
# what a first fire leaves.
|
|
881
|
+
def derived_next_fire(loop_obj, from)
|
|
882
|
+
key = derived_key(loop_obj)
|
|
883
|
+
origin, found = @derived[key]
|
|
884
|
+
return found if origin && from >= origin && (found ? found > from : from == origin)
|
|
885
|
+
|
|
886
|
+
loop_obj.next_fire_at(from).tap { |epoch| @derived[key] = [from, epoch] }
|
|
887
|
+
end
|
|
888
|
+
|
|
889
|
+
def derived_key(loop_obj)
|
|
890
|
+
[loop_obj.lid, loop_obj.schedule, loop_obj.tz_name]
|
|
891
|
+
end
|
|
892
|
+
|
|
833
893
|
# Never tighter than a tick and a half: a raised tick interval must not
|
|
834
894
|
# turn every on-time fire into a "missed" one.
|
|
835
895
|
def missed_threshold
|
|
@@ -928,16 +988,17 @@ module Wurk
|
|
|
928
988
|
job.provider_job_id if job.respond_to?(:provider_job_id)
|
|
929
989
|
end
|
|
930
990
|
|
|
991
|
+
def read_fire_marks(lid)
|
|
992
|
+
parse_fire_marks(@config.redis { |c| c.call('HMGET', "#{LOOP_PREFIX}#{lid}", 'lf', 'nf') })
|
|
993
|
+
end
|
|
994
|
+
|
|
931
995
|
# Third element is the raw `nf` string — the CAS token for #claim_fire.
|
|
932
996
|
# Comparing the bytes Redis holds, not the parsed integer, keeps the
|
|
933
|
-
# token exact regardless of how the value was written.
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
next_fire = nil if next_fire.nil? || next_fire.empty?
|
|
939
|
-
[prev_fire&.to_i, next_fire&.to_i, next_fire]
|
|
940
|
-
end
|
|
997
|
+
# token exact regardless of how the value was written. A missing or
|
|
998
|
+
# empty `nf` is nil, not 0.
|
|
999
|
+
def parse_fire_marks((prev_fire, next_fire))
|
|
1000
|
+
next_fire = nil if next_fire.nil? || next_fire.empty?
|
|
1001
|
+
[prev_fire&.to_i, next_fire&.to_i, next_fire]
|
|
941
1002
|
end
|
|
942
1003
|
|
|
943
1004
|
def record_fire(loop_obj, jid, fired_at, future)
|
|
@@ -992,37 +1053,62 @@ module Wurk
|
|
|
992
1053
|
# the field belongs to the runtime (Web UI pause/unpause). Writing it
|
|
993
1054
|
# unconditionally un-paused a dashboard-paused loop on the next boot.
|
|
994
1055
|
def persist(loop_obj)
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
1056
|
+
persist_all([loop_obj])
|
|
1057
|
+
loop_obj
|
|
1058
|
+
end
|
|
1059
|
+
|
|
1060
|
+
# #persist for many loops in one pipelined round trip: a booting process
|
|
1061
|
+
# (each swarm child) writes its whole `config.periodic` block at once.
|
|
1062
|
+
# Nothing here reads, so a pipeline is the same writes in the same order.
|
|
1063
|
+
def persist_all(loops)
|
|
1064
|
+
return loops if loops.empty?
|
|
1065
|
+
|
|
998
1066
|
Wurk.redis do |c|
|
|
999
|
-
c.
|
|
1000
|
-
|
|
1001
|
-
|
|
1067
|
+
c.pipelined do |p|
|
|
1068
|
+
loops.each do |loop_obj|
|
|
1069
|
+
fields = loop_obj.to_redis_hash
|
|
1070
|
+
paused = fields.delete('paused')
|
|
1071
|
+
key = "#{LOOP_PREFIX}#{loop_obj.lid}"
|
|
1072
|
+
p.call('SADD', PERIODIC_KEY, loop_obj.lid)
|
|
1073
|
+
p.call('HSET', key, *fields.flatten)
|
|
1074
|
+
p.call('HSETNX', key, 'paused', paused)
|
|
1075
|
+
end
|
|
1076
|
+
end
|
|
1002
1077
|
end
|
|
1003
|
-
|
|
1078
|
+
loops
|
|
1004
1079
|
end
|
|
1005
1080
|
|
|
1006
|
-
# Drop every registered loop for `klass`
|
|
1007
|
-
#
|
|
1008
|
-
# any option changes the lid, so
|
|
1009
|
-
# `periodic` and keeps firing next
|
|
1081
|
+
# Drop every registered loop for `klass` (a name or a list of names)
|
|
1082
|
+
# except `keep_lids` — the loops a booting process just registered for
|
|
1083
|
+
# those classes. Editing a schedule or any option changes the lid, so
|
|
1084
|
+
# without this the pre-edit loop stays in `periodic` and keeps firing next
|
|
1085
|
+
# to its replacement, forever.
|
|
1010
1086
|
#
|
|
1011
|
-
# Scoped to
|
|
1087
|
+
# Scoped to the registered classes deliberately: the registry is fleet-wide, so a
|
|
1012
1088
|
# process that registers a subset of the fleet's loops (or none at all,
|
|
1013
1089
|
# e.g. a client-only app) must never delete loops for classes it doesn't
|
|
1014
1090
|
# register. Idempotent and safe to race — two processes booting the same
|
|
1015
1091
|
# code compute the same candidate set, and SREM/DEL repeat harmlessly.
|
|
1092
|
+
#
|
|
1093
|
+
# One SMEMBERS and one pipelined HGET per candidate, however many classes:
|
|
1094
|
+
# a boot registering C classes used to read every foreign loop C times.
|
|
1016
1095
|
def prune_superseded(klass, keep_lids)
|
|
1096
|
+
klasses = Array(klass)
|
|
1017
1097
|
keep = Array(keep_lids)
|
|
1018
|
-
stale = Wurk.redis
|
|
1019
|
-
candidates = c.call('SMEMBERS', PERIODIC_KEY) - keep
|
|
1020
|
-
candidates.select { |lid| c.call('HGET', "#{LOOP_PREFIX}#{lid}", 'klass') == klass }
|
|
1021
|
-
end
|
|
1098
|
+
stale = Wurk.redis { |c| superseded_lids(c, klasses, keep) }
|
|
1022
1099
|
stale.each { |lid| unregister(lid) }
|
|
1023
1100
|
stale
|
|
1024
1101
|
end
|
|
1025
1102
|
|
|
1103
|
+
def superseded_lids(conn, klasses, keep)
|
|
1104
|
+
candidates = conn.call('SMEMBERS', PERIODIC_KEY) - keep
|
|
1105
|
+
return [] if candidates.empty?
|
|
1106
|
+
|
|
1107
|
+
owners = conn.pipelined { |p| candidates.each { |lid| p.call('HGET', "#{LOOP_PREFIX}#{lid}", 'klass') } }
|
|
1108
|
+
candidates.zip(owners).filter_map { |lid, owner| lid if klasses.include?(owner) }
|
|
1109
|
+
end
|
|
1110
|
+
private :superseded_lids
|
|
1111
|
+
|
|
1026
1112
|
# Drop a loop entirely: registry membership, hash, and fire history.
|
|
1027
1113
|
# Public API for retiring a loop whose `register(...)` line is gone (no
|
|
1028
1114
|
# process registers its class any more, so `prune_superseded` cannot see
|
|
@@ -1050,6 +1136,19 @@ module Wurk
|
|
|
1050
1136
|
LoopSet.new
|
|
1051
1137
|
end
|
|
1052
1138
|
|
|
1139
|
+
# For the swarm parent's pre-fork step (Swarm::Prefork): resolve the IANA
|
|
1140
|
+
# zone of every loop `manager` registered, so tzinfo and the zone data
|
|
1141
|
+
# are loaded once and shared copy-on-write rather than loaded by the
|
|
1142
|
+
# leader child on its first tick. Opens no socket. `erb` and `yaml` need
|
|
1143
|
+
# no warming: Manager#load_file runs in the parent, at configuration.
|
|
1144
|
+
def prefork(manager)
|
|
1145
|
+
manager&.loops&.each do |lp|
|
|
1146
|
+
name = lp.tz_name
|
|
1147
|
+
Parser.resolve_zone(name) if lp.tz.is_a?(String) && !name.empty?
|
|
1148
|
+
end
|
|
1149
|
+
nil
|
|
1150
|
+
end
|
|
1151
|
+
|
|
1053
1152
|
# Test/ops helper: fire one registered loop immediately, ignoring the
|
|
1054
1153
|
# leader gate and the schedule due-check. Records history + advances the
|
|
1055
1154
|
# fire marks just like a leader tick, so specs can assert on the enqueue
|
data/lib/wurk/dead_set.rb
CHANGED
|
@@ -65,19 +65,23 @@ module Wurk
|
|
|
65
65
|
# receive `(job, ex)`. `max_jobs:` / `timeout:` propagate to the auto-trim;
|
|
66
66
|
# see `#trim` for the rationale.
|
|
67
67
|
def kill(message, opts = {})
|
|
68
|
-
notify = opts.fetch(:notify_failure, true)
|
|
69
|
-
do_trim = opts.fetch(:trim, true)
|
|
70
|
-
ex = opts[:ex] || RuntimeError.new(API_KILL_MESSAGE).tap { |e| e.set_backtrace(caller) }
|
|
71
|
-
|
|
72
68
|
now = ::Process.clock_gettime(::Process::CLOCK_REALTIME)
|
|
73
69
|
Wurk.redis { |conn| conn.call('ZADD', @name, now.to_s, message) }
|
|
74
|
-
trim(max_jobs: opts[:max_jobs], timeout: opts[:timeout]) if
|
|
75
|
-
fire_death_handlers(message, ex) if
|
|
70
|
+
trim(max_jobs: opts[:max_jobs], timeout: opts[:timeout]) if opts.fetch(:trim, true)
|
|
71
|
+
fire_death_handlers(message, opts[:ex] || api_kill_error) if opts.fetch(:notify_failure, true)
|
|
76
72
|
true
|
|
77
73
|
end
|
|
78
74
|
|
|
79
75
|
private
|
|
80
76
|
|
|
77
|
+
# Only the death handlers see it, so a kill that notifies none (a UI kill,
|
|
78
|
+
# `kill_all` over a whole set) never pays for the caller's backtrace.
|
|
79
|
+
def api_kill_error
|
|
80
|
+
error = RuntimeError.new(API_KILL_MESSAGE)
|
|
81
|
+
error.set_backtrace(caller(2))
|
|
82
|
+
error
|
|
83
|
+
end
|
|
84
|
+
|
|
81
85
|
def fire_death_handlers(message, ex)
|
|
82
86
|
job = parse_message(message)
|
|
83
87
|
handlers = Wurk.configuration.death_handlers
|
data/lib/wurk/embedded.rb
CHANGED
data/lib/wurk/engine.rb
CHANGED
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
require 'rails/engine'
|
|
4
4
|
require_relative '../active_job/queue_adapters/wurk_adapter'
|
|
5
5
|
require_relative 'dashboard_manifest'
|
|
6
|
-
require_relative 'web'
|
|
7
6
|
|
|
8
7
|
module Wurk
|
|
9
8
|
# Rails mountable engine. Owns the dashboard mount, the asset path for
|
|
@@ -44,6 +43,14 @@ module Wurk
|
|
|
44
43
|
CACHEABLE_METHODS = %w[GET HEAD].freeze
|
|
45
44
|
CACHEABLE_STATUSES = [200, 206, 304].freeze
|
|
46
45
|
|
|
46
|
+
# `rake frontend:build` writes a .br and a .gz next to each fingerprinted
|
|
47
|
+
# text asset (frontend/vite.config.ts, wurk-precompress). The mount sits
|
|
48
|
+
# ahead of any deflater the host runs, so without these the dashboard
|
|
49
|
+
# went out uncompressed. Preference order when the client's q-values tie.
|
|
50
|
+
ENCODINGS = { 'br' => '.br', 'gzip' => '.gz' }.freeze
|
|
51
|
+
OFFERED_ENCODINGS = [*ENCODINGS.keys, 'identity'].freeze
|
|
52
|
+
PRECOMPRESSED = /\.(?:js|css|svg|json)\z/
|
|
53
|
+
|
|
47
54
|
def initialize(app, root:)
|
|
48
55
|
@app = app
|
|
49
56
|
@files = ::Rack::Files.new(root)
|
|
@@ -56,20 +63,55 @@ module Wurk
|
|
|
56
63
|
inner = env.dup
|
|
57
64
|
stripped = path.delete_prefix(PREFIX)
|
|
58
65
|
inner[::Rack::PATH_INFO] = stripped.empty? ? '/' : stripped
|
|
59
|
-
response = @files.call(inner)
|
|
66
|
+
response = precompressed(inner, env, stripped) || @files.call(inner)
|
|
60
67
|
return @app.call(env) if response[0] == 404
|
|
61
68
|
|
|
62
|
-
|
|
69
|
+
stamp_cache_headers(response, env, stripped)
|
|
63
70
|
response
|
|
64
71
|
end
|
|
65
72
|
|
|
66
73
|
private
|
|
67
74
|
|
|
75
|
+
# The variant the client accepts and the build wrote, served with the
|
|
76
|
+
# original's content type; nil sends the identity file. A Range request
|
|
77
|
+
# gets identity so byte offsets mean what the client thinks they mean.
|
|
78
|
+
def precompressed(inner, env, stripped)
|
|
79
|
+
return unless CACHEABLE_METHODS.include?(env[::Rack::REQUEST_METHOD])
|
|
80
|
+
return unless precompressible?(stripped)
|
|
81
|
+
return if env['HTTP_RANGE']
|
|
82
|
+
|
|
83
|
+
encoding = negotiated_encoding(env)
|
|
84
|
+
return unless encoding
|
|
85
|
+
|
|
86
|
+
variant = @files.call(inner.merge(::Rack::PATH_INFO => "#{stripped}#{ENCODINGS[encoding]}"))
|
|
87
|
+
return unless variant[0] == 200
|
|
88
|
+
|
|
89
|
+
variant[1]['content-type'] = ::Rack::Mime.mime_type(::File.extname(stripped))
|
|
90
|
+
variant[1]['content-encoding'] = encoding
|
|
91
|
+
variant
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# 'br', 'gzip', or nil for identity (no header, or every coding refused).
|
|
95
|
+
def negotiated_encoding(env)
|
|
96
|
+
accepted = ::Rack::Utils.q_values(env['HTTP_ACCEPT_ENCODING'].to_s)
|
|
97
|
+
encoding = ::Rack::Utils.select_best_encoding(OFFERED_ENCODINGS, accepted)
|
|
98
|
+
encoding if ENCODINGS.key?(encoding)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def precompressible?(stripped)
|
|
102
|
+
stripped.start_with?(FINGERPRINTED_PREFIX) && stripped.match?(PRECOMPRESSED)
|
|
103
|
+
end
|
|
104
|
+
|
|
68
105
|
# This mount is inserted at index 0 of the HOST app's stack (see the
|
|
69
106
|
# initializer below), so Rack::ETag and Rack::ConditionalGet never see
|
|
70
107
|
# these responses — without this the dashboard's fingerprinted bundle
|
|
71
108
|
# shipped with no cache directive at all and was refetched every load.
|
|
72
|
-
|
|
109
|
+
#
|
|
110
|
+
# Vary goes on every answer for a path that has variants, identity
|
|
111
|
+
# included, or a shared cache could hand gzip to a client that can't
|
|
112
|
+
# read it.
|
|
113
|
+
def stamp_cache_headers(response, env, stripped)
|
|
114
|
+
response[1]['vary'] = 'accept-encoding' if precompressible?(stripped)
|
|
73
115
|
return unless CACHEABLE_METHODS.include?(env[::Rack::REQUEST_METHOD])
|
|
74
116
|
return unless CACHEABLE_STATUSES.include?(response[0])
|
|
75
117
|
|