rails_pod_kit 0.0.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +284 -25
- data/VERSION +1 -1
- data/lib/rails_pod_kit/config.rb +28 -2
- data/lib/rails_pod_kit/error_reporter.rb +34 -0
- data/lib/rails_pod_kit/exporter.rb +42 -0
- data/lib/rails_pod_kit/global_exporter.rb +27 -9
- data/lib/rails_pod_kit/global_scheduler.rb +156 -0
- data/lib/rails_pod_kit/shutdown.rb +18 -0
- data/lib/rails_pod_kit/sidekiq.rb +5 -12
- data/lib/rails_pod_kit/solid_queue/metrics.rb +186 -0
- data/lib/rails_pod_kit/solid_queue/scheduler_runner.rb +118 -0
- data/lib/rails_pod_kit/solid_queue.rb +108 -0
- data/lib/rails_pod_kit/supervisor.rb +90 -0
- data/lib/rails_pod_kit.rb +8 -5
- metadata +32 -7
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'rails_pod_kit/config'
|
|
4
|
+
require 'rails_pod_kit/supervisor'
|
|
5
|
+
|
|
6
|
+
module RailsPodKit
|
|
7
|
+
# Runs the sidekiq-cron poller in a process that is *not* a Sidekiq server, so
|
|
8
|
+
# a Sidekiq deployment can be autoscaled to zero without losing its schedule.
|
|
9
|
+
#
|
|
10
|
+
# sidekiq-cron installs its poller from inside `Sidekiq.configure_server`, so
|
|
11
|
+
# the schedule exists only while a Sidekiq server is alive. At zero replicas
|
|
12
|
+
# nothing polls, nothing is enqueued, and nothing ever raises the queue depth
|
|
13
|
+
# that would wake a worker back up — a closed loop that forces a permanent
|
|
14
|
+
# floor of one replica just to keep a poller alive. Recurring jobs are not
|
|
15
|
+
# caught up afterwards either: `reschedule_grace_period` (60s by default)
|
|
16
|
+
# discards any run older than itself, so a worker started later skips it.
|
|
17
|
+
#
|
|
18
|
+
# The poller has no such requirement of its own. `Sidekiq::Cron::Poller` is a
|
|
19
|
+
# Redis-polling thread and runs in any process holding a Sidekiq config, so
|
|
20
|
+
# hosting it on an always-on singleton — the dedicated exporter pod, see
|
|
21
|
+
# GlobalExporter — breaks the cycle and leaves the workers free to scale to
|
|
22
|
+
# zero.
|
|
23
|
+
#
|
|
24
|
+
# Deliberately Rails-free, like the exporter it sits beside: enqueueing does
|
|
25
|
+
# not need the job classes. When `Job#enqueue!` cannot resolve the class it
|
|
26
|
+
# pushes a plain Sidekiq message naming the ActiveJob wrapper with the job
|
|
27
|
+
# class as a *string*, and the worker — which does have Rails — resolves it.
|
|
28
|
+
# That fallback is only correct for entries declaring `active_job: true`, so
|
|
29
|
+
# `start!` warns about any that would instead be pushed as bare Sidekiq jobs.
|
|
30
|
+
#
|
|
31
|
+
# The poller runs under the shared Supervisor, exactly like SolidQueue's
|
|
32
|
+
# scheduler thread. Its own loop swallows StandardError (`Poller#enqueue` and
|
|
33
|
+
# `#wait` both do), so a Redis blip costs one skipped tick — but anything it
|
|
34
|
+
# does not catch takes the thread down and, since this process is the only
|
|
35
|
+
# scheduler, the schedule with it, silently.
|
|
36
|
+
module GlobalScheduler
|
|
37
|
+
SOURCE = 'rails_pod_kit.global_scheduler'
|
|
38
|
+
|
|
39
|
+
module_function
|
|
40
|
+
|
|
41
|
+
# Loads the schedule and starts the supervisor, which starts the poller on
|
|
42
|
+
# its first (immediate) tick, then returns without blocking. Idempotent: a
|
|
43
|
+
# second call is a no-op rather than a second poller in the same process.
|
|
44
|
+
#
|
|
45
|
+
# `schedule_file:`, `poll_interval:` and `reschedule_grace_period:` override
|
|
46
|
+
# sidekiq-cron's own defaults (`config/schedule.yml`, resolved against the
|
|
47
|
+
# working directory, polled every 30s, catching up runs at most 60s late);
|
|
48
|
+
# `supervision_interval:` is the shared Supervisor's.
|
|
49
|
+
#
|
|
50
|
+
# Raising the grace period is what makes a restart of the single scheduling
|
|
51
|
+
# process free: below it a missed occurrence is caught up on the next poll,
|
|
52
|
+
# above it the run is skipped silently. Size it over the worst restart —
|
|
53
|
+
# eviction, reschedule, image pull, boot — not over the poll interval.
|
|
54
|
+
def start!(schedule_file: nil, poll_interval: nil, reschedule_grace_period: nil,
|
|
55
|
+
supervision_interval: Supervisor::DEFAULT_INTERVAL)
|
|
56
|
+
return @supervisor if @supervisor
|
|
57
|
+
return unless RailsPodKit.scheduler_enabled?('sidekiq-cron poller')
|
|
58
|
+
|
|
59
|
+
require 'sidekiq'
|
|
60
|
+
require 'sidekiq-cron'
|
|
61
|
+
# sidekiq-cron renders the schedule file through ERB without requiring it:
|
|
62
|
+
# under Rails it is always already loaded, here it is not.
|
|
63
|
+
require 'erb'
|
|
64
|
+
|
|
65
|
+
configure!(schedule_file: schedule_file, poll_interval: poll_interval,
|
|
66
|
+
reschedule_grace_period: reschedule_grace_period)
|
|
67
|
+
load_schedule!
|
|
68
|
+
|
|
69
|
+
@supervisor = build_supervisor(supervision_interval).start
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Winds the poller down so an in-flight tick finishes before the process
|
|
73
|
+
# exits, instead of being cut off mid-enqueue by the signal.
|
|
74
|
+
def stop!
|
|
75
|
+
@supervisor&.stop
|
|
76
|
+
@supervisor = nil
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def poller
|
|
80
|
+
@supervisor&.subject
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def build_supervisor(interval)
|
|
84
|
+
Supervisor.new(
|
|
85
|
+
source: SOURCE,
|
|
86
|
+
interval: interval,
|
|
87
|
+
start: -> { build_poller.tap(&:start) },
|
|
88
|
+
alive: method(:poller_alive?),
|
|
89
|
+
stop: :terminate
|
|
90
|
+
)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# `Sidekiq::Scheduled::Poller` keeps its thread in `@thread` and `start` is a
|
|
94
|
+
# no-op once that is set, so there is no public way to ask whether the poller
|
|
95
|
+
# is still running, nor to revive it. Reading the ivar lets the supervisor
|
|
96
|
+
# replace a dead poller wholesale — schedule state lives in Redis, so a fresh
|
|
97
|
+
# one picks up exactly where the old one stopped. An unrecognised shape reads
|
|
98
|
+
# as alive, so an upstream rename costs the supervision, never a restart loop.
|
|
99
|
+
def poller_alive?(poller)
|
|
100
|
+
return true unless poller.instance_variable_defined?(:@thread)
|
|
101
|
+
|
|
102
|
+
!!poller.instance_variable_get(:@thread)&.alive?
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def configure!(schedule_file: nil, poll_interval: nil, reschedule_grace_period: nil)
|
|
106
|
+
::Sidekiq::Cron.configure do |cron|
|
|
107
|
+
cron.cron_schedule_file = schedule_file if schedule_file
|
|
108
|
+
cron.cron_poll_interval = poll_interval if poll_interval
|
|
109
|
+
cron.reschedule_grace_period = reschedule_grace_period if reschedule_grace_period
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# sidekiq-cron reads the schedule file from a Sidekiq server's `:startup`
|
|
114
|
+
# lifecycle event, which never fires here, so load it explicitly.
|
|
115
|
+
def load_schedule!
|
|
116
|
+
return unless ::Sidekiq::Cron.configuration.enabled
|
|
117
|
+
|
|
118
|
+
loader = ::Sidekiq::Cron::ScheduleLoader.new
|
|
119
|
+
return unless loader.has_schedule_file?
|
|
120
|
+
|
|
121
|
+
loader.load_schedule
|
|
122
|
+
warn_unresolvable_entries!
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# sidekiq-cron's own Launcher publishes these two into the Sidekiq config
|
|
126
|
+
# before instantiating the poller, which reads them straight back out.
|
|
127
|
+
# Pinning the process count keeps the poll interval at the configured value:
|
|
128
|
+
# the inherited default counts *live Sidekiq servers*, which here is a count
|
|
129
|
+
# of anything but cron pollers — and is zero while the workers are scaled in.
|
|
130
|
+
def build_poller
|
|
131
|
+
config = ::Sidekiq.default_configuration
|
|
132
|
+
config[:cron_poll_interval] = ::Sidekiq::Cron.configuration.cron_poll_interval.to_i
|
|
133
|
+
config[:cron_poll_process_count] = ::Sidekiq::Cron.configuration.cron_poll_process_count || 1
|
|
134
|
+
|
|
135
|
+
::Sidekiq::Cron::Poller.new(config)
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# An entry whose class this process cannot load and which does not declare
|
|
139
|
+
# `active_job: true` is pushed as a bare Sidekiq job message, so the worker
|
|
140
|
+
# runs `perform` outside ActiveJob — no callbacks, no argument
|
|
141
|
+
# deserialization, no retry bookkeeping.
|
|
142
|
+
def warn_unresolvable_entries!
|
|
143
|
+
names = ::Sidekiq::Cron::Job.all('*').reject { |job| enqueueable_without_class?(job) }.map(&:name)
|
|
144
|
+
return if names.empty?
|
|
145
|
+
|
|
146
|
+
::Sidekiq.logger.warn do
|
|
147
|
+
"[rails_pod_kit] cron entries #{names.join(', ')} name a class this Rails-free process cannot load and " \
|
|
148
|
+
'are not marked `active_job: true`; they would be enqueued as plain Sidekiq jobs.'
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def enqueueable_without_class?(job)
|
|
153
|
+
job.to_hash[:active_job] == '1' || !::Sidekiq::Cron::Support.safe_constantize(job.klass.to_s).nil?
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RailsPodKit
|
|
4
|
+
# Blocks the main thread of an always-on entry point until the orchestrator
|
|
5
|
+
# signals. A self-pipe rather than a Queue or a Mutex: writing to an IO is one
|
|
6
|
+
# of the few things safe to do from a trap handler.
|
|
7
|
+
module Shutdown
|
|
8
|
+
SIGNALS = %w[INT TERM].freeze
|
|
9
|
+
|
|
10
|
+
module_function
|
|
11
|
+
|
|
12
|
+
def await(signals: SIGNALS)
|
|
13
|
+
reader, writer = IO.pipe
|
|
14
|
+
signals.each { |signal| Signal.trap(signal) { writer.puts(signal) } }
|
|
15
|
+
reader.gets
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'rails_pod_kit/config'
|
|
4
|
+
require 'rails_pod_kit/exporter'
|
|
4
5
|
|
|
5
6
|
module RailsPodKit
|
|
6
7
|
# Sidekiq integration. Called from inside `Sidekiq.configure_server`:
|
|
@@ -81,19 +82,11 @@ module RailsPodKit
|
|
|
81
82
|
apply_retries_segmentation!
|
|
82
83
|
end
|
|
83
84
|
|
|
84
|
-
# Starts the background WEBrick exporter
|
|
85
|
-
#
|
|
86
|
-
#
|
|
85
|
+
# Starts the background WEBrick exporter shared with the other non-Puma
|
|
86
|
+
# entry points; guarded there so a re-entrant Sidekiq boot can't double-bind
|
|
87
|
+
# the port.
|
|
87
88
|
def start_metrics_server!
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
ENV['PROMETHEUS_EXPORTER_PORT'] ||= RailsPodKit.config.port.to_s
|
|
91
|
-
# Drop the WEBrick exporter's per-scrape access log (Rack::CommonLogger,
|
|
92
|
-
# which the mmap exporter mounts unless this is exactly 'false'). See
|
|
93
|
-
# Config#silence_exporter_access_log.
|
|
94
|
-
ENV['PROMETHEUS_EXPORTER_LOG_REQUESTS'] = 'false' if RailsPodKit.config.silence_exporter_access_log
|
|
95
|
-
Yabeda::Prometheus::Exporter.start_metrics_server!
|
|
96
|
-
@server_started = true
|
|
89
|
+
RailsPodKit::Exporter.start!
|
|
97
90
|
end
|
|
98
91
|
end
|
|
99
92
|
end
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'rails_pod_kit/config'
|
|
4
|
+
require 'rails_pod_kit/error_reporter'
|
|
5
|
+
|
|
6
|
+
module RailsPodKit
|
|
7
|
+
module SolidQueue
|
|
8
|
+
# DB-backed queue gauges for SolidQueue, in the shape yabeda-sidekiq exposes
|
|
9
|
+
# for Redis: SolidQueue ships no metrics endpoint and there is no
|
|
10
|
+
# `yabeda-solid_queue`.
|
|
11
|
+
#
|
|
12
|
+
# Two series, both per `queue`, both computed at scrape time from a yabeda
|
|
13
|
+
# `collect` block — no background thread, no cached snapshot:
|
|
14
|
+
#
|
|
15
|
+
# solid_queue_backlog how many jobs could be claimed right now
|
|
16
|
+
# solid_queue_latency_seconds how long the oldest of them has been waiting
|
|
17
|
+
#
|
|
18
|
+
# "Claimable right now" is ready executions plus scheduled ones whose time
|
|
19
|
+
# has come (the dispatcher has yet to move them across). Backlog alone misses
|
|
20
|
+
# a small-but-stalled queue and latency alone misses a large-but-moving one,
|
|
21
|
+
# so the pair is what dashboards, alerts and an HPA feed actually need.
|
|
22
|
+
#
|
|
23
|
+
# `::SolidQueue` is the host's — the gem declares no dependency on it and
|
|
24
|
+
# nothing here loads until the host calls `install!`.
|
|
25
|
+
module Metrics
|
|
26
|
+
SOURCE = 'rails_pod_kit.solid_queue_metrics'
|
|
27
|
+
|
|
28
|
+
module_function
|
|
29
|
+
|
|
30
|
+
# Declares the gauges and registers the scrape-time collector. Safe before
|
|
31
|
+
# or after `Yabeda.configure!` (yabeda replays configurators either way).
|
|
32
|
+
#
|
|
33
|
+
# `queues:` pins the zero baseline (see #baseline_queues) instead of
|
|
34
|
+
# discovering it. `fail_scrape_on_error:` makes a collection failure fail
|
|
35
|
+
# the whole response rather than serve the last reading — right when this
|
|
36
|
+
# collector owns the endpoint, wrong when it shares one (see #collect!).
|
|
37
|
+
#
|
|
38
|
+
# Declaration is one-shot but the options are not: the dedicated pod boots
|
|
39
|
+
# the host's initializers before `run_exporter!` runs, so by the time the
|
|
40
|
+
# pod asks for `fail_scrape_on_error` the app's own `install_metrics!` has
|
|
41
|
+
# normally already declared the gauges. An option given here always wins;
|
|
42
|
+
# one left out keeps whatever an earlier call set.
|
|
43
|
+
def install!(queues: nil, fail_scrape_on_error: nil)
|
|
44
|
+
@baseline_queues = queues unless queues.nil?
|
|
45
|
+
@fail_scrape_on_error = fail_scrape_on_error unless fail_scrape_on_error.nil?
|
|
46
|
+
|
|
47
|
+
return false if @installed
|
|
48
|
+
|
|
49
|
+
require 'yabeda'
|
|
50
|
+
declare!
|
|
51
|
+
@installed = true
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def declare!
|
|
55
|
+
Yabeda.configure do
|
|
56
|
+
group :solid_queue do
|
|
57
|
+
gauge :backlog,
|
|
58
|
+
tags: %i[queue],
|
|
59
|
+
comment: 'Jobs claimable right now: ready executions plus scheduled ones whose time has come'
|
|
60
|
+
gauge :latency,
|
|
61
|
+
unit: :seconds,
|
|
62
|
+
tags: %i[queue],
|
|
63
|
+
comment: 'How long the oldest claimable job has been waiting'
|
|
64
|
+
|
|
65
|
+
collect { RailsPodKit::SolidQueue::Metrics.collect! }
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Called by yabeda on every scrape.
|
|
71
|
+
#
|
|
72
|
+
# An error is always reported, and then either swallowed or re-raised
|
|
73
|
+
# depending on who owns the endpoint. Swallowing keeps a transient DB
|
|
74
|
+
# failure from taking the Puma series down with it on a shared endpoint —
|
|
75
|
+
# at the cost of serving the last reading, which a consumer cannot tell
|
|
76
|
+
# apart from a live one. On the dedicated pod (`run_exporter!`) there is
|
|
77
|
+
# nothing else to protect, so failing the scrape is the honest answer: the
|
|
78
|
+
# gauges go to no-data and the scraper's own `up` series carries the
|
|
79
|
+
# failure.
|
|
80
|
+
def collect!
|
|
81
|
+
now = ::Time.now.utc
|
|
82
|
+
with_connection { publish_all(claimable_by_queue(now), now) }
|
|
83
|
+
rescue StandardError => e
|
|
84
|
+
ErrorReporter.report(e, source: SOURCE)
|
|
85
|
+
raise if @fail_scrape_on_error
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Collection runs on the exporter's HTTP thread, which would otherwise
|
|
89
|
+
# check out a connection and pin it there for the life of the process.
|
|
90
|
+
def with_connection(&)
|
|
91
|
+
::ActiveRecord::Base.connection_pool.with_connection(&)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# => { "default" => { backlog: 12, waiting_since: <Time> }, … }
|
|
95
|
+
def claimable_by_queue(now)
|
|
96
|
+
merge(ready_rows, due_rows(now))
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def ready_rows
|
|
100
|
+
rows(::SolidQueue::ReadyExecution.all, :created_at)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# A scheduled execution whose time has come is claimable too. Its wait
|
|
104
|
+
# started at `scheduled_at`, not `created_at` — a job enqueued a week ahead
|
|
105
|
+
# of its slot is not a week late.
|
|
106
|
+
def due_rows(now)
|
|
107
|
+
rows(::SolidQueue::ScheduledExecution.where(scheduled_at: ..now), :scheduled_at)
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Two grouped aggregates over an indexed, normally-small table. Both go
|
|
111
|
+
# through ActiveRecord's calculations so the timestamp comes back
|
|
112
|
+
# type-cast on every adapter.
|
|
113
|
+
def rows(relation, waiting_since_column)
|
|
114
|
+
counts = relation.group(:queue_name).count
|
|
115
|
+
oldest = relation.group(:queue_name).minimum(waiting_since_column)
|
|
116
|
+
|
|
117
|
+
counts.map { |queue, count| [queue, count, oldest[queue]] }
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def merge(*row_sets)
|
|
121
|
+
row_sets.flatten(1).each_with_object({}) do |(queue, count, waiting_since), acc|
|
|
122
|
+
entry = acc[queue] ||= { backlog: 0, waiting_since: nil }
|
|
123
|
+
entry[:backlog] += count
|
|
124
|
+
entry[:waiting_since] = [entry[:waiting_since], waiting_since].compact.min
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def publish_all(queues, now)
|
|
129
|
+
queues.each do |queue, entry|
|
|
130
|
+
publish(queue, backlog: entry[:backlog], latency: age_in_seconds(entry[:waiting_since], now))
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
zero_drained_queues(queues.keys)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def publish(queue, backlog:, latency:)
|
|
137
|
+
Yabeda.solid_queue.backlog.set({ queue: queue }, backlog)
|
|
138
|
+
Yabeda.solid_queue.latency.set({ queue: queue }, latency)
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# A gauge keeps its last value per label set, so a queue that just drained
|
|
142
|
+
# would stay pinned at its final backlog forever — the exact reading that
|
|
143
|
+
# would keep an alert firing on an idle system. Track the label sets this
|
|
144
|
+
# process has published and zero the ones missing from this round.
|
|
145
|
+
def zero_drained_queues(current)
|
|
146
|
+
seen = seen_queues
|
|
147
|
+
(seen - current).each { |queue| publish(queue, backlog: 0, latency: 0) }
|
|
148
|
+
@seen_queues = seen | current
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# Seeded with the baseline, so the zeroing above also covers queues this
|
|
152
|
+
# process has never seen busy. A gauge only exists once it has been set:
|
|
153
|
+
# without the seed an exporter that boots while the queue is empty — the
|
|
154
|
+
# steady state of a scale-to-zero deployment — publishes no series at all,
|
|
155
|
+
# and every consumer reads no-data where it should read 0.
|
|
156
|
+
def seen_queues
|
|
157
|
+
@seen_queues ||= baseline_queues
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
# The queues the app is known to use, pinned by the host or discovered
|
|
161
|
+
# once per process from the jobs table (one index scan, never repeated).
|
|
162
|
+
# Discovery is best-effort by construction: that table is bounded by
|
|
163
|
+
# `clear_finished_jobs_after`, so a queue idle for longer than the
|
|
164
|
+
# retention window leaves no trace. Pin `queues:` where the zero has to be
|
|
165
|
+
# guaranteed.
|
|
166
|
+
def baseline_queues
|
|
167
|
+
@baseline_queues || ::SolidQueue::Job.distinct.pluck(:queue_name).compact
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def age_in_seconds(waiting_since, now)
|
|
171
|
+
return 0 if waiting_since.nil?
|
|
172
|
+
|
|
173
|
+
[(now - waiting_since).to_f, 0].max
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# Test/reset hook — drops the published-label-set memo and the install
|
|
177
|
+
# options.
|
|
178
|
+
def reset!
|
|
179
|
+
@seen_queues = nil
|
|
180
|
+
@baseline_queues = nil
|
|
181
|
+
@fail_scrape_on_error = false
|
|
182
|
+
@installed = false
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
end
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'active_support/configuration_file'
|
|
4
|
+
require 'active_support/core_ext/hash/keys'
|
|
5
|
+
|
|
6
|
+
require 'rails_pod_kit/config'
|
|
7
|
+
require 'rails_pod_kit/supervisor'
|
|
8
|
+
|
|
9
|
+
module RailsPodKit
|
|
10
|
+
module SolidQueue
|
|
11
|
+
# Runs a SolidQueue *scheduler* — and only the scheduler — as a supervised
|
|
12
|
+
# background thread, so the process running jobs can scale to zero.
|
|
13
|
+
#
|
|
14
|
+
# Scaling the executor to zero is otherwise a chicken-and-egg problem: with
|
|
15
|
+
# no executor there is no scheduler, so nothing enqueues the recurring or
|
|
16
|
+
# scheduled jobs that would wake one. A k8s CronJob can't take over either,
|
|
17
|
+
# because it can't own *dynamic* recurring tasks (created and updated at
|
|
18
|
+
# runtime through `SolidQueue.schedule_recurring_task`). Moving just the
|
|
19
|
+
# scheduler onto an always-on process — the web, or the exporter pod —
|
|
20
|
+
# breaks the cycle: it keeps evaluating the crons and enqueueing, and the
|
|
21
|
+
# queue depth wakes the executor.
|
|
22
|
+
#
|
|
23
|
+
# Deliberately *not* the full supervisor (`plugin :solid_queue`): that one
|
|
24
|
+
# forks, and its Puma watchdog takes the host process down when the
|
|
25
|
+
# supervisor exits — which a transient Postgres disconnect is enough to
|
|
26
|
+
# cause (rails/solid_queue#512). Here a DB blip at worst kills the scheduler
|
|
27
|
+
# thread; the supervising timer notices on its next tick and starts a fresh
|
|
28
|
+
# one, and the host process never notices.
|
|
29
|
+
#
|
|
30
|
+
# Running it on every replica is safe: enqueues stay exactly-once via the
|
|
31
|
+
# unique index on `solid_queue_recurring_executions (task_key, run_at)`.
|
|
32
|
+
class SchedulerRunner
|
|
33
|
+
# How often the scheduler re-reads the dynamic tasks from the DB.
|
|
34
|
+
DEFAULT_POLLING_INTERVAL = 5
|
|
35
|
+
# How often we check that the scheduler thread is still alive.
|
|
36
|
+
DEFAULT_SUPERVISION_INTERVAL = 5
|
|
37
|
+
|
|
38
|
+
SOURCE = 'rails_pod_kit.solid_queue_scheduler'
|
|
39
|
+
|
|
40
|
+
def initialize(polling_interval: DEFAULT_POLLING_INTERVAL,
|
|
41
|
+
supervision_interval: DEFAULT_SUPERVISION_INTERVAL,
|
|
42
|
+
recurring_schedule_file: nil)
|
|
43
|
+
@polling_interval = polling_interval
|
|
44
|
+
@supervision_interval = supervision_interval
|
|
45
|
+
@recurring_schedule_file = recurring_schedule_file
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Starts the supervisor, which starts the scheduler on its first (immediate)
|
|
49
|
+
# tick and returns without blocking.
|
|
50
|
+
def start
|
|
51
|
+
@supervisor = build_supervisor.start
|
|
52
|
+
self
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Graceful stop: the supervisor drops its timer before winding the
|
|
56
|
+
# scheduler down (unschedule its timers and deregister the process,
|
|
57
|
+
# instead of leaving a row to expire).
|
|
58
|
+
def stop
|
|
59
|
+
@supervisor&.stop
|
|
60
|
+
@supervisor = nil
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def running?
|
|
64
|
+
!!@supervisor&.running?
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
private
|
|
68
|
+
|
|
69
|
+
def build_supervisor
|
|
70
|
+
Supervisor.new(
|
|
71
|
+
source: SOURCE,
|
|
72
|
+
interval: @supervision_interval,
|
|
73
|
+
start: -> { build_scheduler },
|
|
74
|
+
alive: :alive?,
|
|
75
|
+
stop: :stop
|
|
76
|
+
)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def build_scheduler
|
|
80
|
+
scheduler = ::SolidQueue::Scheduler.new(
|
|
81
|
+
recurring_tasks: static_recurring_tasks,
|
|
82
|
+
dynamic_tasks_enabled: true,
|
|
83
|
+
polling_interval: @polling_interval
|
|
84
|
+
)
|
|
85
|
+
scheduler.mode = :async
|
|
86
|
+
scheduler.start # spawns the scheduler's own thread and returns
|
|
87
|
+
scheduler
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# The static tasks from config/recurring.yml; the dynamic ones come from
|
|
91
|
+
# the DB via `dynamic_tasks_enabled`. `SolidQueue::Configuration#recurring_tasks`
|
|
92
|
+
# is private, so parse the file with the same public helper it uses.
|
|
93
|
+
def static_recurring_tasks
|
|
94
|
+
path = recurring_schedule_file
|
|
95
|
+
return [] unless path && ::File.exist?(path)
|
|
96
|
+
|
|
97
|
+
config = ::ActiveSupport::ConfigurationFile.parse(path).deep_symbolize_keys
|
|
98
|
+
config.fetch(environment.to_sym, {}).filter_map do |key, options|
|
|
99
|
+
::SolidQueue::RecurringTask.from_configuration(key, **options) if options&.key?(:schedule)
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Same resolution order SolidQueue's own CLI uses.
|
|
104
|
+
def recurring_schedule_file
|
|
105
|
+
return @recurring_schedule_file if @recurring_schedule_file
|
|
106
|
+
return nil unless defined?(::Rails) && ::Rails.respond_to?(:root) && ::Rails.root
|
|
107
|
+
|
|
108
|
+
::Rails.root.join(ENV.fetch('SOLID_QUEUE_RECURRING_SCHEDULE', 'config/recurring.yml'))
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def environment
|
|
112
|
+
return ::Rails.env if defined?(::Rails) && ::Rails.respond_to?(:env)
|
|
113
|
+
|
|
114
|
+
ENV.fetch('RAILS_ENV', 'development')
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'rails_pod_kit/config'
|
|
4
|
+
require 'rails_pod_kit/exporter'
|
|
5
|
+
require 'rails_pod_kit/shutdown'
|
|
6
|
+
require 'rails_pod_kit/solid_queue/metrics'
|
|
7
|
+
require 'rails_pod_kit/solid_queue/scheduler_runner'
|
|
8
|
+
|
|
9
|
+
module RailsPodKit
|
|
10
|
+
# SolidQueue integration — the pieces a Rails 8 app needs to run its job
|
|
11
|
+
# executor with scale-to-zero autoscaling (KEDA/HPA). All three are opt-in;
|
|
12
|
+
# requiring this file only defines them.
|
|
13
|
+
#
|
|
14
|
+
# install_metrics! the `solid_queue_backlog` / `solid_queue_latency_seconds`
|
|
15
|
+
# gauges on /metrics (see Metrics)
|
|
16
|
+
# start_scheduler! the SolidQueue scheduler as a supervised background
|
|
17
|
+
# thread on an always-on process (see SchedulerRunner)
|
|
18
|
+
# run_exporter! both of the above in a dedicated always-on pod, with
|
|
19
|
+
# the /metrics server, blocking until SIGTERM
|
|
20
|
+
#
|
|
21
|
+
# Nothing here loads `solid_queue` itself: it is the host's dependency, the
|
|
22
|
+
# same way Puma and Sidekiq are.
|
|
23
|
+
#
|
|
24
|
+
# Wiring the two halves separately, when the always-on process is the web:
|
|
25
|
+
#
|
|
26
|
+
# # config/initializers/rails_pod_kit.rb
|
|
27
|
+
# RailsPodKit::SolidQueue.install_metrics!
|
|
28
|
+
#
|
|
29
|
+
# # config/puma.rb — after_booted only runs in the real Puma process,
|
|
30
|
+
# # never in a console, a rake task or the test suite.
|
|
31
|
+
# after_booted { RailsPodKit::SolidQueue.start_scheduler! }
|
|
32
|
+
# at_exit { RailsPodKit::SolidQueue.stop_scheduler! }
|
|
33
|
+
module SolidQueue
|
|
34
|
+
module_function
|
|
35
|
+
|
|
36
|
+
# Declares the queue gauges. Call from an initializer, in whichever process
|
|
37
|
+
# should publish them — see the README on keeping one source per series.
|
|
38
|
+
#
|
|
39
|
+
# Options are Metrics.install!'s: `queues:` to pin the zero baseline,
|
|
40
|
+
# `fail_scrape_on_error:` to fail the scrape instead of serving the last
|
|
41
|
+
# reading when the DB is unreachable (the default only holds on an endpoint
|
|
42
|
+
# this collector shares with another group).
|
|
43
|
+
def install_metrics!(**)
|
|
44
|
+
Metrics.install!(**)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Starts the supervised scheduler thread and returns the runner. Idempotent:
|
|
48
|
+
# a second call returns the running one rather than starting a second
|
|
49
|
+
# scheduler in the same process.
|
|
50
|
+
#
|
|
51
|
+
# Not gated on `RailsPodKit.enabled?` — that switch owns the metrics
|
|
52
|
+
# exporter, and an app may well want the scheduler with metrics turned off.
|
|
53
|
+
# `scheduler_enabled` is the switch that does own this (see Config); beyond
|
|
54
|
+
# it, the guard against starting one in a console or in specs is *where* you
|
|
55
|
+
# call this from (`after_booted`, or the exporter entrypoint).
|
|
56
|
+
def start_scheduler!(**)
|
|
57
|
+
return scheduler_runner if scheduler_runner
|
|
58
|
+
return unless RailsPodKit.scheduler_enabled?('SolidQueue scheduler')
|
|
59
|
+
|
|
60
|
+
@scheduler_runner = SchedulerRunner.new(**).start
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def stop_scheduler!
|
|
64
|
+
@scheduler_runner&.stop
|
|
65
|
+
@scheduler_runner = nil
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def scheduler_runner
|
|
69
|
+
@scheduler_runner
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Runs the dedicated always-on SolidQueue pod and blocks until SIGTERM:
|
|
73
|
+
# the queue gauges, the scheduler thread and the /metrics server in one
|
|
74
|
+
# 1-replica Deployment, so both survive the web and the executor scaling to
|
|
75
|
+
# zero and each series has exactly one source.
|
|
76
|
+
#
|
|
77
|
+
# Unlike GlobalExporter this one is *not* Rails-free — SolidQueue is
|
|
78
|
+
# ActiveRecord-backed and reads the app's own tables — so the host's
|
|
79
|
+
# entrypoint boots the environment first, e.g. `bin/solid-queue-pod`:
|
|
80
|
+
#
|
|
81
|
+
# #!/usr/bin/env ruby
|
|
82
|
+
# require_relative '../config/environment'
|
|
83
|
+
# RailsPodKit::SolidQueue.run_exporter!
|
|
84
|
+
#
|
|
85
|
+
# Pass `scheduler: false` to serve the gauges only (an app whose web process
|
|
86
|
+
# already hosts the scheduler), and `metrics:` to override the gauge options
|
|
87
|
+
# — this endpoint is the collector's own, so a collection failure fails the
|
|
88
|
+
# scrape here rather than serving the last reading.
|
|
89
|
+
def run_exporter!(scheduler: true, metrics: {}, **scheduler_options)
|
|
90
|
+
install_metrics!(fail_scrape_on_error: true, **metrics)
|
|
91
|
+
start_scheduler!(**scheduler_options) if scheduler
|
|
92
|
+
|
|
93
|
+
warn '[rails_pod_kit] disabled — /metrics not served by the SolidQueue exporter' unless Exporter.start!
|
|
94
|
+
|
|
95
|
+
require 'yabeda'
|
|
96
|
+
Yabeda.configure! unless Yabeda.already_configured?
|
|
97
|
+
|
|
98
|
+
await_shutdown
|
|
99
|
+
stop_scheduler!
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Blocks the main thread (the exporter and the scheduler both run on their
|
|
103
|
+
# own) until the kubelet signals.
|
|
104
|
+
def await_shutdown
|
|
105
|
+
Shutdown.await
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|