wurk 1.13.0 → 1.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/app/controllers/concerns/wurk/sse_streaming.rb +36 -1
- data/app/controllers/wurk/api/pagination.rb +9 -7
- data/app/controllers/wurk/api/serializers.rb +8 -2
- data/app/controllers/wurk/api_controller.rb +57 -24
- data/app/controllers/wurk/application_controller.rb +0 -2
- data/app/controllers/wurk/dashboard_controller.rb +0 -1
- data/config/routes.rb +5 -1
- data/lib/sidekiq/cli.rb +4 -3
- data/lib/sidekiq/deploy.rb +2 -1
- data/lib/sidekiq/embedded.rb +2 -1
- data/lib/sidekiq/iterable_job.rb +2 -1
- data/lib/sidekiq/job/iterable/active_record_enumerator.rb +2 -1
- data/lib/sidekiq/job/iterable/csv_enumerator.rb +2 -1
- data/lib/sidekiq/job/iterable.rb +2 -1
- data/lib/sidekiq/launcher.rb +2 -1
- data/lib/sidekiq/manager.rb +2 -1
- data/lib/sidekiq/metrics/query.rb +2 -1
- data/lib/sidekiq/pro/web.rb +2 -1
- data/lib/sidekiq/scheduled.rb +2 -1
- data/lib/sidekiq/web.rb +2 -1
- data/lib/sidekiq.rb +3 -2
- data/lib/wurk/api/fast.rb +4 -18
- data/lib/wurk/api/queues.rb +12 -10
- data/lib/wurk/api/serializers.rb +6 -7
- data/lib/wurk/api/swarm.rb +0 -1
- data/lib/wurk/batch/server_middleware.rb +4 -0
- data/lib/wurk/batch/status.rb +45 -22
- data/lib/wurk/batch_set.rb +34 -0
- data/lib/wurk/cli.rb +17 -19
- data/lib/wurk/client/buffered.rb +97 -26
- data/lib/wurk/client.rb +21 -1
- data/lib/wurk/compat/deploy.rb +7 -0
- data/lib/wurk/compat/encryption.rb +7 -0
- data/lib/wurk/compat/flow.rb +8 -0
- data/lib/wurk/compat/iterable_job.rb +8 -0
- data/lib/wurk/compat/profile_set.rb +8 -0
- data/lib/wurk/compat/server.rb +15 -0
- data/lib/wurk/compat/web.rb +20 -0
- data/lib/wurk/compat.rb +34 -40
- data/lib/wurk/configuration.rb +46 -9
- data/lib/wurk/cron.rb +146 -47
- data/lib/wurk/dead_set.rb +10 -6
- data/lib/wurk/embedded.rb +0 -2
- data/lib/wurk/engine.rb +46 -4
- data/lib/wurk/error_coalescer.rb +21 -1
- data/lib/wurk/fetcher/many_queues.rb +8 -6
- data/lib/wurk/fetcher/poison_pill.rb +1 -1
- data/lib/wurk/fetcher/reaper.rb +0 -5
- data/lib/wurk/fetcher/reliable.rb +46 -27
- data/lib/wurk/fetcher/self_reconcile.rb +0 -1
- data/lib/wurk/fetcher/unit_of_work.rb +16 -1
- data/lib/wurk/flow/status.rb +10 -4
- data/lib/wurk/flow_set.rb +24 -2
- data/lib/wurk/health.rb +0 -1
- data/lib/wurk/heartbeat.rb +13 -7
- data/lib/wurk/import/sidekiq_cron.rb +0 -1
- data/lib/wurk/job_record.rb +2 -1
- data/lib/wurk/job_retry.rb +1 -1
- data/lib/wurk/job_set.rb +132 -6
- data/lib/wurk/launcher.rb +0 -12
- data/lib/wurk/lazy_loader.rb +60 -0
- data/lib/wurk/limiter/bucket.rb +0 -2
- data/lib/wurk/limiter/concurrent.rb +19 -36
- data/lib/wurk/limiter/leaky.rb +0 -2
- data/lib/wurk/limiter/points.rb +0 -2
- data/lib/wurk/limiter/server_middleware.rb +4 -1
- data/lib/wurk/limiter/unlimited.rb +0 -2
- data/lib/wurk/limiter/window.rb +0 -2
- data/lib/wurk/limiter.rb +0 -7
- data/lib/wurk/logger.rb +44 -5
- data/lib/wurk/lua/fast_delete.rb +58 -0
- data/lib/wurk/lua/fast_delete_window.lua +46 -0
- data/lib/wurk/lua/limiter_concurrent_acquire.lua +10 -3
- data/lib/wurk/lua/limiter_concurrent_release.lua +13 -4
- data/lib/wurk/lua.rb +1 -41
- data/lib/wurk/metrics/dashboard_series.rb +38 -18
- data/lib/wurk/metrics/history.rb +24 -11
- data/lib/wurk/metrics/prometheus.rb +0 -1
- data/lib/wurk/metrics/query.rb +0 -2
- data/lib/wurk/metrics/queue_rollup.rb +33 -11
- data/lib/wurk/metrics/statsd.rb +4 -0
- data/lib/wurk/middleware/chain.rb +25 -3
- data/lib/wurk/middleware/expiry.rb +4 -0
- data/lib/wurk/middleware/interrupt_handler.rb +4 -1
- data/lib/wurk/middleware/status.rb +4 -0
- data/lib/wurk/middleware/timeout.rb +4 -0
- data/lib/wurk/preflight/owners.rb +0 -1
- data/lib/wurk/preflight/payload_sample.rb +0 -1
- data/lib/wurk/preflight/verify.rb +0 -2
- data/lib/wurk/process_set.rb +6 -1
- data/lib/wurk/profiler.rb +2 -1
- data/lib/wurk/queue.rb +33 -6
- data/lib/wurk/queue_consumers.rb +10 -1
- data/lib/wurk/rails_boot.rb +2 -2
- data/lib/wurk/railtie.rb +3 -4
- data/lib/wurk/sentry/error_handler.rb +5 -0
- data/lib/wurk/sentry/middleware.rb +4 -0
- data/lib/wurk/sorted_entry.rb +16 -12
- data/lib/wurk/stats.rb +85 -1
- data/lib/wurk/swarm/child_boot.rb +0 -2
- data/lib/wurk/swarm/liveness.rb +0 -2
- data/lib/wurk/swarm/memory_watch.rb +116 -0
- data/lib/wurk/swarm/prefork.rb +54 -0
- data/lib/wurk/swarm.rb +33 -42
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/enterprise.rb +63 -7
- data/lib/wurk.rb +33 -29
- data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js +1 -0
- data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.br +0 -0
- data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.gz +0 -0
- data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js +1 -0
- data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.br +0 -0
- data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js +1 -0
- data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.br +0 -0
- data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js +1 -0
- data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.br +0 -0
- data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js +1 -0
- data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.br +0 -0
- data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js +1 -0
- data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.br +0 -0
- data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js +1 -0
- data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.br +0 -0
- data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js +1 -0
- data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.br +0 -0
- data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.gz +0 -0
- data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js +2 -0
- data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.br +0 -0
- data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.gz +0 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js +1 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.br +0 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{FlowState-yb7virqR.js → FlowState-ClzInw0-.js} +1 -1
- data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.br +0 -0
- data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js +1 -0
- data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.br +0 -0
- data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.gz +0 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js +2 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.br +0 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js +1 -0
- data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.br +0 -0
- data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js +1 -0
- data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.br +0 -0
- data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.gz +0 -0
- data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js +1 -0
- data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.br +0 -0
- data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.gz +0 -0
- data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js +1 -0
- data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.br +0 -0
- data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js +1 -0
- data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.br +0 -0
- data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js +1 -0
- data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.br +0 -0
- data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js +1 -0
- data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.br +0 -0
- data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js +1 -0
- data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.br +0 -0
- data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Search-D42Va7lB.js +1 -0
- data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.br +0 -0
- data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.gz +0 -0
- data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js +1 -0
- data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.br +0 -0
- data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.gz +0 -0
- data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js +1 -0
- data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.br +0 -0
- data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.gz +0 -0
- data/vendor/assets/dashboard/assets/de-B2NCIuzo.js +1 -0
- data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.br +0 -0
- data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.gz +0 -0
- data/vendor/assets/dashboard/assets/es-BGeu_JNB.js +1 -0
- data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.br +0 -0
- data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.gz +0 -0
- data/vendor/assets/dashboard/assets/fr-DQVBWFId.js +1 -0
- data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.br +0 -0
- data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.gz +0 -0
- data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js +1 -0
- data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.br +0 -0
- data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.gz +0 -0
- data/vendor/assets/dashboard/assets/index-CiLU1zz8.js +141 -0
- data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.br +0 -0
- data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.gz +0 -0
- data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css +1 -0
- data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.br +0 -0
- data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.gz +0 -0
- data/vendor/assets/dashboard/assets/ja-D06IBqzL.js +1 -0
- data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.br +0 -0
- data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.gz +0 -0
- data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js +1 -0
- data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.br +0 -0
- data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.gz +0 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js +1 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.br +0 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.gz +0 -0
- data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js +1 -0
- data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.br +0 -0
- data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{utils-CnWZzW2L.js → utils-69XSoGbQ.js} +1 -1
- data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.br +0 -0
- data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.gz +0 -0
- data/vendor/assets/dashboard/assets/wurk-logo-ydeSB9B8.png +0 -0
- data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js +1 -0
- data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.br +0 -0
- data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.gz +0 -0
- data/vendor/assets/dashboard/index.html +3 -23
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +122 -37
- data/vendor/assets/dashboard/assets/ArgsValue-BJol9S0s.js +0 -1
- data/vendor/assets/dashboard/assets/BatchDetail-D_zGQ9Sd.js +0 -1
- data/vendor/assets/dashboard/assets/Batches-Dt_xC660.js +0 -1
- data/vendor/assets/dashboard/assets/Busy-CWRthsmc.js +0 -1
- data/vendor/assets/dashboard/assets/Cron-qeGr2YAm.js +0 -1
- data/vendor/assets/dashboard/assets/Dashboard-DQWaAwPA.js +0 -1
- data/vendor/assets/dashboard/assets/Dead-D9gOPHtK.js +0 -1
- data/vendor/assets/dashboard/assets/Extension-DONXVLdA.js +0 -1
- data/vendor/assets/dashboard/assets/FilterBox-BbB6gXHp.js +0 -2
- data/vendor/assets/dashboard/assets/FlowDetail-CrmnVRp0.js +0 -1
- data/vendor/assets/dashboard/assets/Flows-g4GC8KmE.js +0 -1
- data/vendor/assets/dashboard/assets/JobDetailModal-C86k_1wj.js +0 -2
- data/vendor/assets/dashboard/assets/Limiters-zyO-iMlK.js +0 -1
- data/vendor/assets/dashboard/assets/Metrics-Ckq5fNBv.js +0 -1
- data/vendor/assets/dashboard/assets/NotFound-C8NnyjlY.js +0 -1
- data/vendor/assets/dashboard/assets/PageHeader-vWeBbSPF.js +0 -1
- data/vendor/assets/dashboard/assets/Profiles-DRl-FXPA.js +0 -1
- data/vendor/assets/dashboard/assets/Queues-DkXWrihA.js +0 -1
- data/vendor/assets/dashboard/assets/Retries-CmlojYIy.js +0 -1
- data/vendor/assets/dashboard/assets/Scheduled-Cbbpp3xw.js +0 -1
- data/vendor/assets/dashboard/assets/Search-LaEpj2fM.js +0 -1
- data/vendor/assets/dashboard/assets/charts-D_PQLoyc.js +0 -1
- data/vendor/assets/dashboard/assets/fa-brands-400-Bs6tcqqs.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-regular-400-DRN8N0d1.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-solid-900-IAB4Droh.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-v4compatibility-CErXDOsT.woff2 +0 -0
- data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +0 -1
- data/vendor/assets/dashboard/assets/index--xvvNlYx.css +0 -1
- data/vendor/assets/dashboard/assets/index-FBKGtysy.js +0 -141
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-BiwhHqgh.js +0 -1
- data/vendor/assets/dashboard/assets/useSort-jBYVwXvg.js +0 -1
- data/vendor/assets/dashboard/assets/wurk-logo-Vy3xW4K0.png +0 -0
- data/vendor/assets/dashboard/og-image.png +0 -0
- data/vendor/assets/dashboard/wurk-logo.webp +0 -0
data/lib/wurk/compat.rb
CHANGED
|
@@ -50,16 +50,6 @@ module Sidekiq
|
|
|
50
50
|
# classes under them — but `Sidekiq.pro?` / `Sidekiq.ent?` still return
|
|
51
51
|
# `false` per docs/target/sidekiq-free.md §32 (Wurk advertises as free OSS).
|
|
52
52
|
module Pro
|
|
53
|
-
# Sidekiq Pro mounts the dashboard at `Sidekiq::Pro::Web`; it's the same
|
|
54
|
-
# board as `Sidekiq::Web` here, so a Pro app's `mount Sidekiq::Pro::Web`
|
|
55
|
-
# drops in unchanged. `Wurk::Web` is already required by the time this file
|
|
56
|
-
# loads (lib/wurk.rb requires wurk/web before wurk/compat).
|
|
57
|
-
Web = Wurk::Web
|
|
58
|
-
|
|
59
|
-
# `use Sidekiq::Pro::BatchStatus` — the polling Rack middleware that serves
|
|
60
|
-
# GET /batch_status/<bid>.json. Spec: docs/target/sidekiq-pro.md §10.3.
|
|
61
|
-
BatchStatus = Wurk::Web::BatchStatus
|
|
62
|
-
|
|
63
53
|
# Pro module-level statsd accessor (spec §1): `Sidekiq::Pro.dogstatsd = ...`.
|
|
64
54
|
# Delegates to the canonical `config.dogstatsd=` (spec §9.1), so either form
|
|
65
55
|
# feeds the same Wurk::Metrics::Statsd client.
|
|
@@ -119,38 +109,14 @@ module Sidekiq
|
|
|
119
109
|
private_constant :RailsNamespace
|
|
120
110
|
|
|
121
111
|
BasicFetch = Wurk::Fetcher::Reliable
|
|
122
|
-
WebHelpers = Wurk::Web::Extension::Helpers
|
|
123
|
-
Paginator = Wurk::Web::Paginator
|
|
124
112
|
Batch = Wurk::Batch
|
|
125
113
|
BatchSet = Wurk::BatchSet
|
|
126
114
|
Capsule = Wurk::Capsule
|
|
127
|
-
CLI = Wurk::CLI
|
|
128
115
|
Client = Wurk::Client
|
|
129
116
|
Component = Wurk::Component
|
|
130
117
|
Config = Wurk::Configuration
|
|
131
118
|
Context = Wurk::Context
|
|
132
|
-
# No `Sidekiq::Cron` alias on purpose (#204): real Sidekiq never defines it
|
|
133
|
-
# — that namespace belongs to the third-party sidekiq-cron gem, and squatting
|
|
134
|
-
# it made the gem's own classes (Poller, Job) collide at load. Wurk's native
|
|
135
|
-
# periodic surface is `Sidekiq::Periodic` (Ent parity) / `Wurk::Cron`.
|
|
136
|
-
CronParser = Wurk::Cron::Parser
|
|
137
|
-
Periodic = Wurk::Cron
|
|
138
119
|
DeadSet = Wurk::DeadSet
|
|
139
|
-
Deploy = Wurk::Deploy
|
|
140
|
-
Embedded = Wurk::Embedded
|
|
141
|
-
Encryption = Wurk::Encryption
|
|
142
|
-
# A Wurk extra, not parity — aliased because nothing else claims the name.
|
|
143
|
-
# Checked before adding it, the way `Sidekiq::Cron` (#204) and
|
|
144
|
-
# `Sidekiq::Status` taught us to: upstream Sidekiq defines no `Flow` anywhere
|
|
145
|
-
# in `lib/` at the pinned parity SHA, RubyGems has no `sidekiq-flow` or
|
|
146
|
-
# `sidekiq_flow` gem, and the only `Sidekiq::Flow` in public Ruby is a
|
|
147
|
-
# `FlowIdMiddleware` nested under a gem's *own* `Sidekiq` module, which this
|
|
148
|
-
# constant cannot reach.
|
|
149
|
-
Flow = Wurk::Flow
|
|
150
|
-
FlowSet = Wurk::FlowSet
|
|
151
|
-
History = Wurk::History
|
|
152
|
-
IterableJob = Wurk::IterableJob
|
|
153
|
-
IterableJobQuery = Wurk::IterableJobQuery
|
|
154
120
|
Job = Wurk::Job
|
|
155
121
|
JobLogger = Wurk::JobLogger
|
|
156
122
|
JobRecord = Wurk::JobRecord
|
|
@@ -161,11 +127,9 @@ module Sidekiq
|
|
|
161
127
|
JobSet = Wurk::JobSet
|
|
162
128
|
JobUtil = Wurk::JobUtil
|
|
163
129
|
Keys = Wurk::Keys
|
|
164
|
-
Launcher = Wurk::Launcher
|
|
165
130
|
Limiter = Wurk::Limiter
|
|
166
131
|
Loader = Wurk::Loader
|
|
167
132
|
Logger = Wurk::Logger
|
|
168
|
-
Manager = Wurk::Manager
|
|
169
133
|
Metrics = Wurk::Metrics
|
|
170
134
|
Middleware = Wurk::Middleware
|
|
171
135
|
ServerMiddleware = Wurk::Middleware::ServerMiddleware
|
|
@@ -174,13 +138,10 @@ module Sidekiq
|
|
|
174
138
|
ProcessSet = Wurk::ProcessSet
|
|
175
139
|
Processor = Wurk::Processor
|
|
176
140
|
Profiler = Wurk::Profiler
|
|
177
|
-
ProfileSet = Wurk::ProfileSet
|
|
178
|
-
ProfileRecord = Wurk::ProfileRecord
|
|
179
141
|
Queue = Wurk::Queue
|
|
180
142
|
RedisClientAdapter = Wurk::RedisClientAdapter
|
|
181
143
|
RedisConnection = Wurk::RedisConnection
|
|
182
144
|
RetrySet = Wurk::RetrySet
|
|
183
|
-
Scheduled = Wurk::Scheduled
|
|
184
145
|
ScheduledSet = Wurk::ScheduledSet
|
|
185
146
|
Shutdown = Wurk::Shutdown
|
|
186
147
|
SortedEntry = Wurk::SortedEntry
|
|
@@ -200,12 +161,45 @@ module Sidekiq
|
|
|
200
161
|
TransactionAwareClient = Wurk::TransactionAwareClient
|
|
201
162
|
Queues = Wurk::Queues
|
|
202
163
|
EmptyQueueError = Wurk::Testing::EmptyQueueError
|
|
203
|
-
Web = Wurk::Web
|
|
204
164
|
Work = Wurk::Work
|
|
205
165
|
Worker = Wurk::Worker
|
|
206
166
|
Workers = Wurk::Workers
|
|
207
167
|
WorkSet = Wurk::WorkSet
|
|
208
168
|
|
|
169
|
+
# Aliases of lazy Wurk constants (Wurk::LazyLoader) are autoloads too, so
|
|
170
|
+
# `defined?(Sidekiq::Web)` holds before the load and the alias is the very
|
|
171
|
+
# object its target is. Each points at a file under lib/wurk/compat/ that
|
|
172
|
+
# loads the target and assigns the alias: the target's own file defines only
|
|
173
|
+
# the Wurk name, so an autoload onto it would leave `Sidekiq::X` undefined.
|
|
174
|
+
# Those files name their targets rather than require them: web.rb, swarm.rb
|
|
175
|
+
# and iterable_job.rb require subfiles that reopen the class before defining
|
|
176
|
+
# it, which a direct require would turn into a circular one.
|
|
177
|
+
#
|
|
178
|
+
# No `Sidekiq::Cron` alias on purpose (#204): real Sidekiq never defines it
|
|
179
|
+
# — that namespace belongs to the third-party sidekiq-cron gem, and squatting
|
|
180
|
+
# it made the gem's own classes (Poller, Job) collide at load. Wurk's native
|
|
181
|
+
# periodic surface is `Sidekiq::Periodic` (Ent parity) / `Wurk::Cron`.
|
|
182
|
+
#
|
|
183
|
+
# `Flow` is a Wurk extra, not parity — aliased because nothing else claims
|
|
184
|
+
# the name. Checked before adding it, the way `Sidekiq::Cron` (#204) and
|
|
185
|
+
# `Sidekiq::Status` taught us to: upstream Sidekiq defines no `Flow` anywhere
|
|
186
|
+
# in `lib/` at the pinned parity SHA, RubyGems has no `sidekiq-flow` or
|
|
187
|
+
# `sidekiq_flow` gem, and the only `Sidekiq::Flow` in public Ruby is a
|
|
188
|
+
# `FlowIdMiddleware` nested under a gem's *own* `Sidekiq` module, which this
|
|
189
|
+
# constant cannot reach.
|
|
190
|
+
Wurk::LazyLoader.register(
|
|
191
|
+
self,
|
|
192
|
+
CLI: 'compat/server', Embedded: 'compat/server', Launcher: 'compat/server',
|
|
193
|
+
Manager: 'compat/server', Scheduled: 'compat/server',
|
|
194
|
+
History: 'compat/server', Periodic: 'compat/server', CronParser: 'compat/server',
|
|
195
|
+
Web: 'compat/web', WebHelpers: 'compat/web', Paginator: 'compat/web',
|
|
196
|
+
ProfileSet: 'compat/profile_set', ProfileRecord: 'compat/profile_set',
|
|
197
|
+
Flow: 'compat/flow', FlowSet: 'compat/flow',
|
|
198
|
+
IterableJob: 'compat/iterable_job', IterableJobQuery: 'compat/iterable_job',
|
|
199
|
+
Deploy: 'compat/deploy', Encryption: 'compat/encryption'
|
|
200
|
+
)
|
|
201
|
+
Wurk::LazyLoader.register(Pro, Web: 'compat/web', BatchStatus: 'compat/web')
|
|
202
|
+
|
|
209
203
|
# Top-level Sidekiq.configure_server / configure_client / redis / logger
|
|
210
204
|
# delegate to Wurk's class methods. Third-party gems treat these as the
|
|
211
205
|
# canonical entry points.
|
data/lib/wurk/configuration.rb
CHANGED
|
@@ -5,7 +5,6 @@ require_relative 'middleware/chain'
|
|
|
5
5
|
require_relative 'capsule'
|
|
6
6
|
require_relative 'pool_checkout'
|
|
7
7
|
require_relative 'context'
|
|
8
|
-
require_relative 'topology'
|
|
9
8
|
require_relative 'cpu_count'
|
|
10
9
|
require_relative 'redis_options'
|
|
11
10
|
require_relative 'keys'
|
|
@@ -488,12 +487,15 @@ module Wurk
|
|
|
488
487
|
#
|
|
489
488
|
# Spec: docs/target/sidekiq-ent.md §2.
|
|
490
489
|
def periodic
|
|
491
|
-
require_relative 'cron'
|
|
492
490
|
@periodic_manager ||= Wurk::Cron::Manager.new(self, deferred: true)
|
|
493
491
|
yield @periodic_manager if block_given?
|
|
494
492
|
@periodic_manager
|
|
495
493
|
end
|
|
496
494
|
|
|
495
|
+
# The periodic manager if `config.periodic` was ever called, else nil:
|
|
496
|
+
# unlike #periodic, never creates one (Swarm::Prefork reads it).
|
|
497
|
+
attr_reader :periodic_manager
|
|
498
|
+
|
|
497
499
|
# Writes the `config.periodic` registrations to Redis. Launcher#boot calls
|
|
498
500
|
# it, so the loops are registered by a booting server, against the Redis
|
|
499
501
|
# settings the app ended up with.
|
|
@@ -732,12 +734,12 @@ module Wurk
|
|
|
732
734
|
|
|
733
735
|
# Web UI configuration: the authorization hook and read-only mode. Returns
|
|
734
736
|
# the process-wide `Wurk::Web.config` singleton so `config.web.read_only =
|
|
735
|
-
# true` and the engine middleware share one source of truth.
|
|
736
|
-
#
|
|
737
|
+
# true` and the engine middleware share one source of truth. The web layer
|
|
738
|
+
# is an autoload (Wurk::LazyLoader), so standalone boot stays lean until this
|
|
739
|
+
# is called.
|
|
737
740
|
#
|
|
738
741
|
# Spec: docs/target/sidekiq-ent.md §9.2.
|
|
739
742
|
def web
|
|
740
|
-
require_relative 'web/config'
|
|
741
743
|
Wurk::Web.config
|
|
742
744
|
end
|
|
743
745
|
|
|
@@ -875,6 +877,11 @@ module Wurk
|
|
|
875
877
|
# time, and under the railtie that is Bundler.require, before the
|
|
876
878
|
# `wurk.server_mode` initializer: dropping it left sidekiq-cron, sentry-sidekiq
|
|
877
879
|
# and the like silently absent from a process that does run workers.
|
|
880
|
+
#
|
|
881
|
+
# So the skip is DEBUG, as in stock Sidekiq, which skips it silently: every
|
|
882
|
+
# worker entry point enters server mode and runs the kept blocks, and a
|
|
883
|
+
# process that never does (a test run, `rails runner`, a web process) is
|
|
884
|
+
# meant to skip them. A WARN here was noise in each of those.
|
|
878
885
|
def configure_server(&block)
|
|
879
886
|
return unless block
|
|
880
887
|
return run_server_block(block) if server?
|
|
@@ -883,7 +890,7 @@ module Wurk
|
|
|
883
890
|
return if @configure_server_skip_warned
|
|
884
891
|
|
|
885
892
|
@configure_server_skip_warned = true
|
|
886
|
-
logger.
|
|
893
|
+
logger.debug { 'configure_server block skipped: server? is false' }
|
|
887
894
|
nil
|
|
888
895
|
end
|
|
889
896
|
|
|
@@ -952,13 +959,29 @@ module Wurk
|
|
|
952
959
|
@memory_limit_mb = value.nil? ? nil : Integer(value)
|
|
953
960
|
end
|
|
954
961
|
|
|
955
|
-
# Threshold in KB, the unit the swarm
|
|
956
|
-
#
|
|
962
|
+
# Threshold in KB, the unit the swarm reads child memory in. nil when
|
|
963
|
+
# recycling is disabled.
|
|
957
964
|
def memory_limit_kb
|
|
958
965
|
mb = memory_limit_mb
|
|
959
966
|
mb&.positive? ? mb * 1024 : nil
|
|
960
967
|
end
|
|
961
968
|
|
|
969
|
+
# What `memory_limit_mb` is compared against. `:rss` (the default, and
|
|
970
|
+
# Enterprise's) counts pages still shared copy-on-write with the parent, so
|
|
971
|
+
# every child carries the whole preloaded app; `:pss` splits shared pages
|
|
972
|
+
# across their sharers and `:uss` counts only the child's own. Wurk-only;
|
|
973
|
+
# WURK_MAXMEM_METRIC sets it, an explicit value wins.
|
|
974
|
+
MEMORY_LIMIT_METRICS = %i[rss pss uss].freeze
|
|
975
|
+
|
|
976
|
+
def memory_limit_metric
|
|
977
|
+
@memory_limit_metric || env_memory_limit_metric || :rss
|
|
978
|
+
end
|
|
979
|
+
|
|
980
|
+
def memory_limit_metric=(value)
|
|
981
|
+
guard_frozen!
|
|
982
|
+
@memory_limit_metric = value.nil? ? nil : parse_memory_limit_metric(value)
|
|
983
|
+
end
|
|
984
|
+
|
|
962
985
|
# The pre-fork half of `freeze!`, and the only half a forking parent can
|
|
963
986
|
# run: capsules stay writable until each child has applied its slot
|
|
964
987
|
# (ChildBoot#apply_slot_to_config) and opened its own Redis pools. What is
|
|
@@ -1174,6 +1197,21 @@ module Wurk
|
|
|
1174
1197
|
nil
|
|
1175
1198
|
end
|
|
1176
1199
|
|
|
1200
|
+
def env_memory_limit_metric
|
|
1201
|
+
raw = ENV.fetch('WURK_MAXMEM_METRIC', nil)
|
|
1202
|
+
return nil if raw.nil? || raw.strip.empty?
|
|
1203
|
+
|
|
1204
|
+
parse_memory_limit_metric(raw.strip.downcase)
|
|
1205
|
+
end
|
|
1206
|
+
|
|
1207
|
+
def parse_memory_limit_metric(value)
|
|
1208
|
+
metric = value.to_s.to_sym
|
|
1209
|
+
return metric if MEMORY_LIMIT_METRICS.include?(metric)
|
|
1210
|
+
|
|
1211
|
+
raise ArgumentError,
|
|
1212
|
+
"memory_limit_metric must be one of #{MEMORY_LIMIT_METRICS.join(', ')} (got #{value.inspect})"
|
|
1213
|
+
end
|
|
1214
|
+
|
|
1177
1215
|
def guard_frozen!
|
|
1178
1216
|
raise FrozenError, 'Wurk::Configuration is frozen' if @frozen
|
|
1179
1217
|
end
|
|
@@ -1185,7 +1223,6 @@ module Wurk
|
|
|
1185
1223
|
def truthy_setting?(value)
|
|
1186
1224
|
return !!value unless value.is_a?(String)
|
|
1187
1225
|
|
|
1188
|
-
require_relative 'web/config'
|
|
1189
1226
|
!Wurk::Web::Config::FALSEY_STRINGS.include?(value.strip.downcase)
|
|
1190
1227
|
end
|
|
1191
1228
|
|
data/lib/wurk/cron.rb
CHANGED
|
@@ -5,7 +5,6 @@ require 'digest'
|
|
|
5
5
|
require 'time'
|
|
6
6
|
require_relative 'component'
|
|
7
7
|
require_relative 'client'
|
|
8
|
-
require_relative 'leader'
|
|
9
8
|
require_relative 'timer_loop'
|
|
10
9
|
|
|
11
10
|
module Wurk
|
|
@@ -74,8 +73,27 @@ module Wurk
|
|
|
74
73
|
|
|
75
74
|
MAX_LOOKAHEAD_MINUTES = 366 * 24 * 60 * 4
|
|
76
75
|
|
|
76
|
+
# Distinct schedules a process keeps parsed. A loop is rebuilt from its
|
|
77
|
+
# hash on every tick, and a fleet has far fewer schedules than loops.
|
|
78
|
+
CACHE_LIMIT = 1024
|
|
79
|
+
|
|
77
80
|
attr_reader :expression, :fields
|
|
78
81
|
|
|
82
|
+
@cache = {}
|
|
83
|
+
@cache_mutex = ::Mutex.new
|
|
84
|
+
|
|
85
|
+
# A shared, already-parsed Parser for `expression`. Raises ArgumentError
|
|
86
|
+
# like `new` on a bad one, which is never cached. Parsers are not mutated
|
|
87
|
+
# after construction, so sharing one across loops and threads is safe.
|
|
88
|
+
def self.cached(expression)
|
|
89
|
+
@cache_mutex.synchronize do
|
|
90
|
+
@cache.fetch(expression) do
|
|
91
|
+
@cache.clear if @cache.size >= CACHE_LIMIT
|
|
92
|
+
@cache[expression] = new(expression)
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
|
|
79
97
|
def initialize(expression)
|
|
80
98
|
parts = normalize_expression(expression)
|
|
81
99
|
@expression = parts.join(' ')
|
|
@@ -287,13 +305,18 @@ module Wurk
|
|
|
287
305
|
@klass = klass
|
|
288
306
|
@options = stringify_options(options)
|
|
289
307
|
@tz = tz
|
|
290
|
-
@parser = Parser.
|
|
308
|
+
@parser = Parser.cached(schedule)
|
|
291
309
|
@lid = lid || Cron.lid(schedule, klass, @options)
|
|
292
310
|
@paused = paused.nil? ? [true, '1', 1].include?(@options['paused']) : paused
|
|
311
|
+
@fire_marks = nil
|
|
293
312
|
end
|
|
294
313
|
|
|
314
|
+
# The raw `lf` / `nf` strings of the hash this loop was read from, so a
|
|
315
|
+
# tick decides on the read it already made; nil for a loop built in code.
|
|
316
|
+
attr_accessor :fire_marks
|
|
317
|
+
|
|
295
318
|
def parser
|
|
296
|
-
@parser ||= Parser.
|
|
319
|
+
@parser ||= Parser.cached(@schedule)
|
|
297
320
|
end
|
|
298
321
|
|
|
299
322
|
def paused?
|
|
@@ -492,7 +515,8 @@ module Wurk
|
|
|
492
515
|
return self unless @deferred
|
|
493
516
|
|
|
494
517
|
@deferred = false
|
|
495
|
-
@loops
|
|
518
|
+
Cron.persist_all(@loops)
|
|
519
|
+
prune_superseded(@loops.map(&:klass).uniq)
|
|
496
520
|
prune_unregistered if @authoritative
|
|
497
521
|
self
|
|
498
522
|
end
|
|
@@ -530,7 +554,7 @@ module Wurk
|
|
|
530
554
|
|
|
531
555
|
def persist(loop_obj)
|
|
532
556
|
Cron.persist(loop_obj)
|
|
533
|
-
prune_superseded(loop_obj.klass)
|
|
557
|
+
prune_superseded([loop_obj.klass])
|
|
534
558
|
end
|
|
535
559
|
|
|
536
560
|
def prune_unregistered
|
|
@@ -559,12 +583,14 @@ module Wurk
|
|
|
559
583
|
# an edited `register` line yields a new loop and orphans the old one).
|
|
560
584
|
# Registering the same class twice in one block is fine — the second
|
|
561
585
|
# register re-persists its loop right after the first one's prune.
|
|
562
|
-
def prune_superseded(
|
|
586
|
+
def prune_superseded(klass_names)
|
|
563
587
|
return unless @prune
|
|
564
|
-
return if @pruned_klasses[klass_name]
|
|
565
588
|
|
|
566
|
-
@pruned_klasses[
|
|
567
|
-
|
|
589
|
+
fresh = klass_names.reject { |name| @pruned_klasses[name] }
|
|
590
|
+
return if fresh.empty?
|
|
591
|
+
|
|
592
|
+
fresh.each { |name| @pruned_klasses[name] = true }
|
|
593
|
+
Cron.prune_superseded(fresh, @loops.map(&:lid))
|
|
568
594
|
end
|
|
569
595
|
end
|
|
570
596
|
|
|
@@ -588,13 +614,7 @@ module Wurk
|
|
|
588
614
|
def each
|
|
589
615
|
return enum_for(:each) unless block_given?
|
|
590
616
|
|
|
591
|
-
|
|
592
|
-
c.call('SMEMBERS', PERIODIC_KEY).filter_map do |lid|
|
|
593
|
-
h = c.call('HGETALL', "#{LOOP_PREFIX}#{lid}")
|
|
594
|
-
[lid, h] unless h.nil? || h.empty?
|
|
595
|
-
end
|
|
596
|
-
end
|
|
597
|
-
rows.each do |lid, h|
|
|
617
|
+
read_rows.each do |lid, h|
|
|
598
618
|
loop_obj = parse(lid, h)
|
|
599
619
|
yield loop_obj if loop_obj
|
|
600
620
|
end
|
|
@@ -616,8 +636,23 @@ module Wurk
|
|
|
616
636
|
|
|
617
637
|
private
|
|
618
638
|
|
|
639
|
+
# [lid, hash] for every registered loop that still has its hash: one
|
|
640
|
+
# SMEMBERS, then every HGETALL in one pipeline.
|
|
641
|
+
def read_rows
|
|
642
|
+
redis(idempotent: true) do |c|
|
|
643
|
+
lids = c.call('SMEMBERS', PERIODIC_KEY)
|
|
644
|
+
next [] if lids.empty?
|
|
645
|
+
|
|
646
|
+
hashes = c.pipelined { |p| lids.each { |lid| p.call('HGETALL', "#{LOOP_PREFIX}#{lid}") } }
|
|
647
|
+
lids.zip(hashes).filter_map do |lid, h|
|
|
648
|
+
h = h.each_slice(2).to_h if h.is_a?(Array)
|
|
649
|
+
[lid, h] unless h.nil? || h.empty?
|
|
650
|
+
end
|
|
651
|
+
end
|
|
652
|
+
end
|
|
653
|
+
|
|
619
654
|
def parse(lid, hash)
|
|
620
|
-
Loop.from_redis(lid, hash)
|
|
655
|
+
Loop.from_redis(lid, hash).tap { |lp| lp.fire_marks = hash.values_at('lf', 'nf') }
|
|
621
656
|
rescue JSON::ParserError, ArgumentError => e
|
|
622
657
|
Wurk.logger&.warn("[cron] skipping unreadable loop lid=#{lid}: #{e.class}: #{e.message}")
|
|
623
658
|
nil
|
|
@@ -679,6 +714,7 @@ module Wurk
|
|
|
679
714
|
@sleeper = ::ConditionVariable.new
|
|
680
715
|
@on_leader = method(:wake_for_leadership)
|
|
681
716
|
@unknown_reported = ::Set.new
|
|
717
|
+
@derived = {}
|
|
682
718
|
end
|
|
683
719
|
|
|
684
720
|
# The loop waits one interval before its first tick unless this process
|
|
@@ -720,11 +756,14 @@ module Wurk
|
|
|
720
756
|
def tick
|
|
721
757
|
return unless leader?
|
|
722
758
|
|
|
759
|
+
seen = ::Set.new
|
|
723
760
|
LoopSet.new(@config).each do |lp|
|
|
724
|
-
|
|
761
|
+
seen << derived_key(lp)
|
|
762
|
+
enqueue_if_due(lp, lp.fire_marks)
|
|
725
763
|
rescue StandardError => e
|
|
726
764
|
handle_exception(e, { context: 'cron-poller', lid: lp.lid, klass: lp.klass })
|
|
727
765
|
end
|
|
766
|
+
@derived.select! { |key, _| seen.include?(key) }
|
|
728
767
|
rescue StandardError => e
|
|
729
768
|
handle_exception(e, { context: 'cron-poller' })
|
|
730
769
|
end
|
|
@@ -741,11 +780,14 @@ module Wurk
|
|
|
741
780
|
# One claim, one push at most, whichever slot it pushes for: the occurrence
|
|
742
781
|
# due now stands in for a stale one under the stale one's mark, so a poller
|
|
743
782
|
# that read the same mark loses the CAS for both.
|
|
744
|
-
|
|
783
|
+
#
|
|
784
|
+
# `marks` is the raw [`lf`, `nf`] pair the caller just read (a tick's
|
|
785
|
+
# LoopSet read); without it they are read here.
|
|
786
|
+
def enqueue_if_due(loop_obj, marks = nil)
|
|
745
787
|
return if loop_obj.paused?
|
|
746
788
|
|
|
747
789
|
now = ::Time.now.to_i
|
|
748
|
-
slot, mark = due_slot(loop_obj, now)
|
|
790
|
+
slot, mark = due_slot(loop_obj, now, marks)
|
|
749
791
|
return if slot.nil?
|
|
750
792
|
|
|
751
793
|
firing = slot_to_fire(loop_obj, slot, now)
|
|
@@ -822,14 +864,32 @@ module Wurk
|
|
|
822
864
|
# the loop has no resolvable occurrence or none is due yet. The token is
|
|
823
865
|
# the stored `nf` when there is one, otherwise the slot we derived — see
|
|
824
866
|
# the `cron_claim_fire` script for how the two cases differ.
|
|
825
|
-
def due_slot(loop_obj, now)
|
|
826
|
-
prev_fire, next_fire, mark = read_fire_marks(loop_obj.lid)
|
|
827
|
-
next_fire ||= loop_obj
|
|
867
|
+
def due_slot(loop_obj, now, marks)
|
|
868
|
+
prev_fire, next_fire, mark = marks ? parse_fire_marks(marks) : read_fire_marks(loop_obj.lid)
|
|
869
|
+
next_fire ||= derived_next_fire(loop_obj, prev_fire || (now - @tick_interval))
|
|
828
870
|
return if next_fire.nil? || next_fire > now
|
|
829
871
|
|
|
830
872
|
[next_fire, mark || next_fire.to_s]
|
|
831
873
|
end
|
|
832
874
|
|
|
875
|
+
# A loop with no `nf` (never fired, or an exhausted schedule) derives its
|
|
876
|
+
# next occurrence by walking Parser#next_fire_at minute by minute: a year
|
|
877
|
+
# of minutes for a yearly loop, on every tick. The answer for one origin
|
|
878
|
+
# still holds for any later origin it is past, so it is kept per loop
|
|
879
|
+
# until the origin overtakes it. Nothing is written: the stored hash stays
|
|
880
|
+
# what a first fire leaves.
|
|
881
|
+
def derived_next_fire(loop_obj, from)
|
|
882
|
+
key = derived_key(loop_obj)
|
|
883
|
+
origin, found = @derived[key]
|
|
884
|
+
return found if origin && from >= origin && (found ? found > from : from == origin)
|
|
885
|
+
|
|
886
|
+
loop_obj.next_fire_at(from).tap { |epoch| @derived[key] = [from, epoch] }
|
|
887
|
+
end
|
|
888
|
+
|
|
889
|
+
def derived_key(loop_obj)
|
|
890
|
+
[loop_obj.lid, loop_obj.schedule, loop_obj.tz_name]
|
|
891
|
+
end
|
|
892
|
+
|
|
833
893
|
# Never tighter than a tick and a half: a raised tick interval must not
|
|
834
894
|
# turn every on-time fire into a "missed" one.
|
|
835
895
|
def missed_threshold
|
|
@@ -928,16 +988,17 @@ module Wurk
|
|
|
928
988
|
job.provider_job_id if job.respond_to?(:provider_job_id)
|
|
929
989
|
end
|
|
930
990
|
|
|
991
|
+
def read_fire_marks(lid)
|
|
992
|
+
parse_fire_marks(@config.redis { |c| c.call('HMGET', "#{LOOP_PREFIX}#{lid}", 'lf', 'nf') })
|
|
993
|
+
end
|
|
994
|
+
|
|
931
995
|
# Third element is the raw `nf` string — the CAS token for #claim_fire.
|
|
932
996
|
# Comparing the bytes Redis holds, not the parsed integer, keeps the
|
|
933
|
-
# token exact regardless of how the value was written.
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
next_fire = nil if next_fire.nil? || next_fire.empty?
|
|
939
|
-
[prev_fire&.to_i, next_fire&.to_i, next_fire]
|
|
940
|
-
end
|
|
997
|
+
# token exact regardless of how the value was written. A missing or
|
|
998
|
+
# empty `nf` is nil, not 0.
|
|
999
|
+
def parse_fire_marks((prev_fire, next_fire))
|
|
1000
|
+
next_fire = nil if next_fire.nil? || next_fire.empty?
|
|
1001
|
+
[prev_fire&.to_i, next_fire&.to_i, next_fire]
|
|
941
1002
|
end
|
|
942
1003
|
|
|
943
1004
|
def record_fire(loop_obj, jid, fired_at, future)
|
|
@@ -992,37 +1053,62 @@ module Wurk
|
|
|
992
1053
|
# the field belongs to the runtime (Web UI pause/unpause). Writing it
|
|
993
1054
|
# unconditionally un-paused a dashboard-paused loop on the next boot.
|
|
994
1055
|
def persist(loop_obj)
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
1056
|
+
persist_all([loop_obj])
|
|
1057
|
+
loop_obj
|
|
1058
|
+
end
|
|
1059
|
+
|
|
1060
|
+
# #persist for many loops in one pipelined round trip: a booting process
|
|
1061
|
+
# (each swarm child) writes its whole `config.periodic` block at once.
|
|
1062
|
+
# Nothing here reads, so a pipeline is the same writes in the same order.
|
|
1063
|
+
def persist_all(loops)
|
|
1064
|
+
return loops if loops.empty?
|
|
1065
|
+
|
|
998
1066
|
Wurk.redis do |c|
|
|
999
|
-
c.
|
|
1000
|
-
|
|
1001
|
-
|
|
1067
|
+
c.pipelined do |p|
|
|
1068
|
+
loops.each do |loop_obj|
|
|
1069
|
+
fields = loop_obj.to_redis_hash
|
|
1070
|
+
paused = fields.delete('paused')
|
|
1071
|
+
key = "#{LOOP_PREFIX}#{loop_obj.lid}"
|
|
1072
|
+
p.call('SADD', PERIODIC_KEY, loop_obj.lid)
|
|
1073
|
+
p.call('HSET', key, *fields.flatten)
|
|
1074
|
+
p.call('HSETNX', key, 'paused', paused)
|
|
1075
|
+
end
|
|
1076
|
+
end
|
|
1002
1077
|
end
|
|
1003
|
-
|
|
1078
|
+
loops
|
|
1004
1079
|
end
|
|
1005
1080
|
|
|
1006
|
-
# Drop every registered loop for `klass`
|
|
1007
|
-
#
|
|
1008
|
-
# any option changes the lid, so
|
|
1009
|
-
# `periodic` and keeps firing next
|
|
1081
|
+
# Drop every registered loop for `klass` (a name or a list of names)
|
|
1082
|
+
# except `keep_lids` — the loops a booting process just registered for
|
|
1083
|
+
# those classes. Editing a schedule or any option changes the lid, so
|
|
1084
|
+
# without this the pre-edit loop stays in `periodic` and keeps firing next
|
|
1085
|
+
# to its replacement, forever.
|
|
1010
1086
|
#
|
|
1011
|
-
# Scoped to
|
|
1087
|
+
# Scoped to the registered classes deliberately: the registry is fleet-wide, so a
|
|
1012
1088
|
# process that registers a subset of the fleet's loops (or none at all,
|
|
1013
1089
|
# e.g. a client-only app) must never delete loops for classes it doesn't
|
|
1014
1090
|
# register. Idempotent and safe to race — two processes booting the same
|
|
1015
1091
|
# code compute the same candidate set, and SREM/DEL repeat harmlessly.
|
|
1092
|
+
#
|
|
1093
|
+
# One SMEMBERS and one pipelined HGET per candidate, however many classes:
|
|
1094
|
+
# a boot registering C classes used to read every foreign loop C times.
|
|
1016
1095
|
def prune_superseded(klass, keep_lids)
|
|
1096
|
+
klasses = Array(klass)
|
|
1017
1097
|
keep = Array(keep_lids)
|
|
1018
|
-
stale = Wurk.redis
|
|
1019
|
-
candidates = c.call('SMEMBERS', PERIODIC_KEY) - keep
|
|
1020
|
-
candidates.select { |lid| c.call('HGET', "#{LOOP_PREFIX}#{lid}", 'klass') == klass }
|
|
1021
|
-
end
|
|
1098
|
+
stale = Wurk.redis { |c| superseded_lids(c, klasses, keep) }
|
|
1022
1099
|
stale.each { |lid| unregister(lid) }
|
|
1023
1100
|
stale
|
|
1024
1101
|
end
|
|
1025
1102
|
|
|
1103
|
+
def superseded_lids(conn, klasses, keep)
|
|
1104
|
+
candidates = conn.call('SMEMBERS', PERIODIC_KEY) - keep
|
|
1105
|
+
return [] if candidates.empty?
|
|
1106
|
+
|
|
1107
|
+
owners = conn.pipelined { |p| candidates.each { |lid| p.call('HGET', "#{LOOP_PREFIX}#{lid}", 'klass') } }
|
|
1108
|
+
candidates.zip(owners).filter_map { |lid, owner| lid if klasses.include?(owner) }
|
|
1109
|
+
end
|
|
1110
|
+
private :superseded_lids
|
|
1111
|
+
|
|
1026
1112
|
# Drop a loop entirely: registry membership, hash, and fire history.
|
|
1027
1113
|
# Public API for retiring a loop whose `register(...)` line is gone (no
|
|
1028
1114
|
# process registers its class any more, so `prune_superseded` cannot see
|
|
@@ -1050,6 +1136,19 @@ module Wurk
|
|
|
1050
1136
|
LoopSet.new
|
|
1051
1137
|
end
|
|
1052
1138
|
|
|
1139
|
+
# For the swarm parent's pre-fork step (Swarm::Prefork): resolve the IANA
|
|
1140
|
+
# zone of every loop `manager` registered, so tzinfo and the zone data
|
|
1141
|
+
# are loaded once and shared copy-on-write rather than loaded by the
|
|
1142
|
+
# leader child on its first tick. Opens no socket. `erb` and `yaml` need
|
|
1143
|
+
# no warming: Manager#load_file runs in the parent, at configuration.
|
|
1144
|
+
def prefork(manager)
|
|
1145
|
+
manager&.loops&.each do |lp|
|
|
1146
|
+
name = lp.tz_name
|
|
1147
|
+
Parser.resolve_zone(name) if lp.tz.is_a?(String) && !name.empty?
|
|
1148
|
+
end
|
|
1149
|
+
nil
|
|
1150
|
+
end
|
|
1151
|
+
|
|
1053
1152
|
# Test/ops helper: fire one registered loop immediately, ignoring the
|
|
1054
1153
|
# leader gate and the schedule due-check. Records history + advances the
|
|
1055
1154
|
# fire marks just like a leader tick, so specs can assert on the enqueue
|
data/lib/wurk/dead_set.rb
CHANGED
|
@@ -65,19 +65,23 @@ module Wurk
|
|
|
65
65
|
# receive `(job, ex)`. `max_jobs:` / `timeout:` propagate to the auto-trim;
|
|
66
66
|
# see `#trim` for the rationale.
|
|
67
67
|
def kill(message, opts = {})
|
|
68
|
-
notify = opts.fetch(:notify_failure, true)
|
|
69
|
-
do_trim = opts.fetch(:trim, true)
|
|
70
|
-
ex = opts[:ex] || RuntimeError.new(API_KILL_MESSAGE).tap { |e| e.set_backtrace(caller) }
|
|
71
|
-
|
|
72
68
|
now = ::Process.clock_gettime(::Process::CLOCK_REALTIME)
|
|
73
69
|
Wurk.redis { |conn| conn.call('ZADD', @name, now.to_s, message) }
|
|
74
|
-
trim(max_jobs: opts[:max_jobs], timeout: opts[:timeout]) if
|
|
75
|
-
fire_death_handlers(message, ex) if
|
|
70
|
+
trim(max_jobs: opts[:max_jobs], timeout: opts[:timeout]) if opts.fetch(:trim, true)
|
|
71
|
+
fire_death_handlers(message, opts[:ex] || api_kill_error) if opts.fetch(:notify_failure, true)
|
|
76
72
|
true
|
|
77
73
|
end
|
|
78
74
|
|
|
79
75
|
private
|
|
80
76
|
|
|
77
|
+
# Only the death handlers see it, so a kill that notifies none (a UI kill,
|
|
78
|
+
# `kill_all` over a whole set) never pays for the caller's backtrace.
|
|
79
|
+
def api_kill_error
|
|
80
|
+
error = RuntimeError.new(API_KILL_MESSAGE)
|
|
81
|
+
error.set_backtrace(caller(2))
|
|
82
|
+
error
|
|
83
|
+
end
|
|
84
|
+
|
|
81
85
|
def fire_death_handlers(message, ex)
|
|
82
86
|
job = parse_message(message)
|
|
83
87
|
handlers = Wurk.configuration.death_handlers
|