cable_room 0.7.0.beta2 → 0.8.0.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,237 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "optparse"
4
- require "etc"
5
- require "logger"
6
- require_relative "version"
7
-
8
- module CableRoom
9
- # The `cable_room` command. Its one command, `server`, is the "rooms host" role from the
10
- # distributed-rooms design: boot the Rails app, become a rooms host, run until SIGTERM or SIGINT,
11
- # then shut every room down cleanly.
12
- #
13
- # It only makes sense with `room_host = :remote` and refuses to run otherwise: in :inline the
14
- # web processes host rooms themselves, so a server here would only race them for locks.
15
- #
16
- # Boot works the way sidekiq and good_job do it: `require` the app's `config/environment.rb`
17
- # from the current directory, or from `--require PATH` (an app directory or a boot file).
18
- # Bundler is already set up because the command runs under `bundle exec`. Nothing from the gem
19
- # beyond this file and `version` is loaded before the app boots, so the app's Gemfile decides
20
- # when the Railtie loads, the same as in a web process.
21
- #
22
- # `--workers N` picks the process model. With N = 1 this process hosts rooms itself. With
23
- # N > 1 (the default is the machine's core count) the app is booted once, here, and then
24
- # `Host::Supervisor` forks N children that each host rooms; this parent only watches them,
25
- # replaces one that dies, and relays SIGTERM and SIGINT. The parent never builds a Host, a Bus
26
- # subscriber, or a Redis connection of its own: those belong to the children, which build them
27
- # after the fork so no process ever shares a socket or a thread with another.
28
- #
29
- # Everything is instance state and `run` returns an exit status, so specs can drive the parsing
30
- # and the refusals in-process; only the exe calls `exit`.
31
- class CLI
32
- STOP_SIGNALS = %w[TERM INT].freeze
33
-
34
- attr_reader :options, :command
35
-
36
- def initialize(argv, stdout: $stdout, stderr: $stderr)
37
- @argv = argv.dup
38
- @stdout = stdout
39
- @stderr = stderr
40
- @options = { workers: nil, require: Dir.pwd }
41
- end
42
-
43
- # Parse and run. Returns the process exit status.
44
- def run
45
- parse!
46
- return 0 if @help_shown
47
-
48
- case command
49
- when "server" then server
50
- when nil
51
- @stderr.puts parser
52
- 1
53
- else
54
- @stderr.puts "Unknown command #{command.inspect}.", parser
55
- 1
56
- end
57
- rescue OptionParser::ParseError => e
58
- @stderr.puts e.message, parser
59
- 1
60
- end
61
-
62
- # How many worker processes `server` runs: `--workers`, or one per core.
63
- def workers
64
- options[:workers] || Etc.nprocessors
65
- end
66
-
67
- private
68
-
69
- def parser
70
- @parser ||= OptionParser.new do |o|
71
- o.banner = "Usage: cable_room server [options]"
72
- o.separator ""
73
- o.separator "Runs a CableRoom rooms host: boots the Rails app in the current directory and"
74
- o.separator "hosts rooms until SIGTERM or SIGINT. Needs CableRoom.config.room_host = :remote."
75
- o.separator ""
76
- o.on("-r", "--require PATH", "App directory (with config/environment.rb) or a boot file to require") do |path|
77
- @options[:require] = path
78
- end
79
- o.on("-w", "--workers N", Integer,
80
- "Worker processes to fork (default: one per core, #{Etc.nprocessors} here). 1 runs in this process.") do |n|
81
- @options[:workers] = n
82
- end
83
- o.on("-v", "--version", "Print the version and exit") do
84
- @stdout.puts CableRoom::VERSION
85
- @help_shown = true
86
- end
87
- o.on("-h", "--help", "Show this help") do
88
- @stdout.puts o
89
- @help_shown = true
90
- end
91
- end
92
- end
93
-
94
- def parse!
95
- rest = parser.parse(@argv)
96
- @command = rest.shift
97
- raise OptionParser::ParseError, "Unexpected arguments: #{rest.join(' ')}" if rest.any?
98
- end
99
-
100
- def server
101
- # Check the cheap things before booting Rails, so a bad flag fails in milliseconds
102
- unless workers >= 1
103
- @stderr.puts "cable_room server: --workers must be at least 1 (got #{workers})."
104
- return 1
105
- end
106
-
107
- boot_app!
108
-
109
- unless CableRoom.config.remote?
110
- @stderr.puts "cable_room server: CableRoom.config.room_host is #{CableRoom.config.room_host.inspect}, but a " \
111
- "rooms host only makes sense with :remote (in :inline the web processes host rooms " \
112
- "themselves). Set CABLE_ROOM_HOST=remote or `c.room_host = :remote` and try again."
113
- return 1
114
- end
115
-
116
- return run_host if workers == 1
117
-
118
- supervise_workers
119
- end
120
-
121
- # Load the app: a directory means its config/environment.rb, anything else is required as-is
122
- def boot_app!
123
- path = File.expand_path(options[:require])
124
- path = File.join(path, "config", "environment.rb") if File.directory?(path)
125
- require path
126
- end
127
-
128
- # Fork `workers` children that each run `run_host`, and watch them until a stop signal has
129
- # come and gone. This parent hosts nothing itself (see the class comment for why).
130
- def supervise_workers
131
- Process.setproctitle("cable_room server: supervisor (#{workers} workers)")
132
- say "supervising #{workers} workers (pid #{Process.pid}, room_host=#{CableRoom.config.room_host}, " \
133
- "broadcaster=#{CableRoom.config.broadcaster})"
134
-
135
- supervisor = CableRoom::Host::Supervisor.new(count: workers, logger: supervisor_logger) do |index|
136
- run_host(worker: index)
137
- end
138
- status = supervisor.run
139
- say "shut down"
140
- status
141
- end
142
-
143
- # Logs from the supervisor go to the same place as the CLI's own messages, in the same style.
144
- # Unbuffered, so a "forked worker" line shows up when it happens, not when the process exits.
145
- def supervisor_logger
146
- @stdout.sync = true if @stdout.respond_to?(:sync=)
147
- Logger.new(@stdout).tap do |logger|
148
- logger.formatter = ->(_severity, _time, _progname, message) { "cable_room server: #{message}\n" }
149
- end
150
- end
151
-
152
- # Host rooms in this process until SIGTERM or SIGINT, then shut them down. This is the whole
153
- # life of a single-process server and of each forked worker (`worker` is the worker's index,
154
- # nil when there's no supervisor). Returns the exit status.
155
- def run_host(worker: nil)
156
- # Trap before starting the Host, so a signal that lands while rooms are still coming up
157
- # (the supervisor relaying a SIGTERM that arrived mid-boot, say) is kept, not fatal
158
- signals = trap_stop_signals
159
-
160
- if worker
161
- Process.setproctitle("cable_room server: worker #{worker}")
162
- tag_logger("cable_room worker #{worker}")
163
- end
164
-
165
- host = CableRoom::Host.start!
166
- say "hosting rooms (pid #{Process.pid}, room_host=#{CableRoom.config.room_host}, " \
167
- "broadcaster=#{CableRoom.config.broadcaster})", worker: worker
168
-
169
- signal = wait_for_stop_signal(signals)
170
- stop_host(host, signal, worker: worker)
171
- say "shut down", worker: worker
172
- 0
173
- ensure
174
- restore_traps
175
- end
176
-
177
- # A stop signal with rooms open is a planned shutdown: hand the rooms to peer hosts before
178
- # leaving (`Host#drain!`, bounded by `drain_timeout`; a room nobody adopts closes with
179
- # `room_closed`). With nothing open there is nothing to move, so just shut down. Both are
180
- # idempotent, so the one SIGTERM a supervisor relays and a later `at_exit` don't collide.
181
- def stop_host(host, signal, worker: nil)
182
- rooms = host.rooms.size
183
- if rooms.zero?
184
- say "got SIG#{signal}, shutting down 0 room(s)", worker: worker
185
- host.shutdown!
186
- return
187
- end
188
-
189
- say "got SIG#{signal}, migrating #{rooms} room(s) to peer hosts", worker: worker
190
- result = host.drain!(reason: "SIG#{signal}")
191
- say "drained: #{result.migrated.size} room(s) migrated, #{result.closed.size} closed, " \
192
- "in #{result.duration.round(1)}s", worker: worker
193
- end
194
-
195
- # Every room on this Host logs through `ActionCable.server.logger` (Host#logger); give that
196
- # logger a tag naming the worker, so lines from the N workers sharing one stdout can be told
197
- # apart. `tagged` without a block returns a logger whose tags apply on every thread, which is
198
- # what we need: rooms log from the worker pool, not from this thread. A logger without tags
199
- # (a plain Logger, or Rails' BroadcastLogger fanning out to several) is left as it is.
200
- def tag_logger(tag)
201
- logger = ActionCable.server.logger
202
- return unless logger.respond_to?(:tagged)
203
-
204
- tagged = logger.tagged(tag)
205
- ActionCable.server.config.logger = tagged if tagged.respond_to?(:info)
206
- end
207
-
208
- def say(message, worker: nil)
209
- prefix = worker ? "cable_room server (worker #{worker})" : "cable_room server"
210
- @stdout.puts "#{prefix}: #{message}"
211
- @stdout.flush
212
- end
213
-
214
- # Signal handlers may only do trivial work, so each one just drops the signal's name on a
215
- # queue for the main thread to pick up. Returns the queue.
216
- def trap_stop_signals
217
- signals = Queue.new
218
- @previous_traps = STOP_SIGNALS.to_h { |sig| [sig, trap(sig) { signals << sig }] }
219
- signals
220
- end
221
-
222
- # Block the main thread until SIGTERM or SIGINT; returns the signal's name. A timed pop, not
223
- # a plain one: if this were the only thread left (a worker whose rooms have all gone quiet,
224
- # say), a blocking pop would trip Ruby's deadlock detector, because a trap isn't a thread.
225
- def wait_for_stop_signal(signals)
226
- loop do
227
- signal = signals.pop(timeout: 1)
228
- return signal if signal
229
- end
230
- end
231
-
232
- def restore_traps
233
- @previous_traps&.each { |sig, handler| trap(sig, handler) }
234
- @previous_traps = nil
235
- end
236
- end
237
- end
@@ -1,112 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module CableRoom
4
- # Holds the app-wide settings for CableRoom. Get the current one with `CableRoom.config`
5
- # and change it with `CableRoom.configure`:
6
- #
7
- # CableRoom.configure do |c|
8
- # c.room_host = :remote # :inline | :remote
9
- # c.broadcaster = :anycable # :action_cable | :anycable
10
- # c.provision_delay_ms = 20
11
- # c.drain_timeout = 10.minutes
12
- # c.handoff_timeout = 10.seconds
13
- # end
14
- #
15
- # The env vars CABLE_ROOM_HOST and CABLE_ROOM_BROADCASTER override `room_host` and
16
- # `broadcaster`, even when the configure block sets them. That lets one build run in
17
- # both modes without a code change. With no configuration at all, every default keeps
18
- # today's behavior: rooms run inline and broadcast through ActionCable.
19
- class Config
20
- class InvalidOption < ArgumentError; end
21
-
22
- ROOM_HOSTS = %i[inline remote].freeze
23
- BROADCASTERS = %i[action_cable anycable].freeze
24
-
25
- ENV_OVERRIDES = {
26
- room_host: "CABLE_ROOM_HOST",
27
- broadcaster: "CABLE_ROOM_BROADCASTER",
28
- }.freeze
29
-
30
- DEFAULTS = {
31
- room_host: :inline,
32
- broadcaster: :action_cable,
33
- provision_delay_ms: 20,
34
- drain_timeout: 10.minutes,
35
- handoff_timeout: 10.seconds,
36
- }.freeze
37
-
38
- attr_reader :room_host, :broadcaster
39
- attr_accessor :provision_delay_ms, :drain_timeout, :handoff_timeout
40
-
41
- def initialize
42
- DEFAULTS.each { |name, value| instance_variable_set("@#{name}", value) }
43
- end
44
-
45
- # Where rooms run. `:inline` keeps them in the web process; `:remote` moves them to a
46
- # dedicated host pool. Accepts a Symbol or String; unknown values raise right away.
47
- def room_host=(value)
48
- @room_host = validate_choice!(:room_host, value, ROOM_HOSTS)
49
- end
50
-
51
- # How a room pushes messages to members. Accepts a Symbol or String; unknown values raise.
52
- def broadcaster=(value)
53
- @broadcaster = validate_choice!(:broadcaster, value, BROADCASTERS)
54
- end
55
-
56
- # Rooms run inside every process that joins them (the web process in a Rails app).
57
- def inline?
58
- room_host == :inline
59
- end
60
-
61
- # Rooms run only in `cable_room server` processes; a web process never hosts one.
62
- def remote?
63
- room_host == :remote
64
- end
65
-
66
- # Copies CABLE_ROOM_HOST and CABLE_ROOM_BROADCASTER onto this config. Blank or missing
67
- # variables leave the current value alone. `env` is injectable for specs.
68
- def apply_env_overrides(env = ENV)
69
- ENV_OVERRIDES.each do |name, var|
70
- value = env[var]
71
- next if value.nil? || value.strip.empty?
72
-
73
- public_send("#{name}=", value)
74
- end
75
- self
76
- end
77
-
78
- # Checks every knob at once. The choice knobs are already checked by their setters, so
79
- # this mostly guards the timing knobs, which are plain accessors.
80
- def validate!
81
- validate_choice!(:room_host, room_host, ROOM_HOSTS)
82
- validate_choice!(:broadcaster, broadcaster, BROADCASTERS)
83
- validate_number!(:provision_delay_ms, provision_delay_ms, min: 0)
84
- validate_number!(:drain_timeout, drain_timeout, min: 0, exclusive: true)
85
- validate_number!(:handoff_timeout, handoff_timeout, min: 0, exclusive: true)
86
- self
87
- end
88
-
89
- def to_h
90
- DEFAULTS.keys.to_h { |name| [name, public_send(name)] }
91
- end
92
-
93
- private
94
-
95
- def validate_choice!(name, value, allowed)
96
- symbol = value.to_s.to_sym unless value.nil?
97
- return symbol if allowed.include?(symbol)
98
-
99
- raise InvalidOption,
100
- "CableRoom config: #{name} must be one of #{allowed.map(&:inspect).join(', ')} " \
101
- "(got #{value.inspect})"
102
- end
103
-
104
- def validate_number!(name, value, min:, exclusive: false)
105
- ok = value.is_a?(Numeric) && (exclusive ? value > min : value >= min)
106
- return value if ok
107
-
108
- bound = exclusive ? "greater than #{min}" : "at least #{min}"
109
- raise InvalidOption, "CableRoom config: #{name} must be a number #{bound} (got #{value.inspect})"
110
- end
111
- end
112
- end
@@ -1,36 +0,0 @@
1
- module CableRoom
2
- class Host
3
- # How rooms receive messages: off the process-wide CableRoom::Bus (see `CableRoom.bus`), on
4
- # the channels `Room::Base.inbound_channel` names. This is the only room-side code that
5
- # touches the inbound transport, and it's the same in `:inline` and `:remote` mode — the only
6
- # difference is which process the Host lives in. Anything with these two methods can be handed
7
- # to `Host.new(inbound:)`.
8
- #
9
- # `subscribe` returns a handle that `unsubscribe` needs back. Handlers get the decoded message
10
- # on the Bus's single subscriber thread, in publish order, so they must be quick and hand the
11
- # real work off; Host::Runner does that by queueing it on the room. `subscribe` blocks until
12
- # Redis confirms the subscription, so `on_live` runs on the caller's thread right before it
13
- # returns, and anything published after that is delivered.
14
- class BusInbound
15
- # `bus` defaults to `CableRoom.bus` (looked up lazily so specs can swap it)
16
- def initialize(bus: nil)
17
- @bus = bus
18
- end
19
-
20
- def subscribe(channel, on_live: nil, &on_message)
21
- bus.subscribe(channel) { |message, _channel| on_message.call(message) }
22
- on_live&.call
23
- on_message
24
- end
25
-
26
- def unsubscribe(channel, _handle)
27
- bus.unsubscribe(channel)
28
- end
29
-
30
- # The Bus behind this transport (the process-wide one unless a spec handed us another)
31
- def bus
32
- @bus || CableRoom.bus
33
- end
34
- end
35
- end
36
- end
@@ -1,275 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module CableRoom
4
- class Host
5
- # The parent process behind `cable_room server --workers N`. It forks N children, runs one
6
- # block in each, replaces a child that dies, and passes SIGTERM and SIGINT on to every child
7
- # before exiting itself. It never hosts a room, never opens Redis, and never starts a Bus:
8
- # each child builds its own after the fork (`CableRoom.after_fork!`), so nothing with a
9
- # socket or a thread behind it is ever shared between processes.
10
- #
11
- # Supervisor.new(count: 2, logger: logger) { |index| run_a_host(index) }.run
12
- #
13
- # The block is the whole child: it runs right after the fork and its return value (an Integer,
14
- # or nil for 0) is the child's exit status. The child leaves with `exit!`, so `at_exit` hooks
15
- # the parent registered before forking don't run a second time in every child (Resque does
16
- # the same for its forked workers). A block that raises is logged and exits 1, which makes
17
- # the supervisor replace it.
18
- #
19
- # Restarts: a child that dies is replaced. One that ran for at least `stable_after` seconds
20
- # comes back at once; one that died sooner is treated as failing to boot and comes back
21
- # after a delay that doubles from `first_backoff` up to `max_backoff`, so a broken app forks
22
- # a few times a minute rather than thousands of times a second.
23
- #
24
- # Signals: SIGTERM and SIGINT to the parent are relayed to every live child and the parent
25
- # then waits, with no time limit of its own, until they've all exited (a child migrating rooms
26
- # may take a while; the container's `timeout -k` is the outer bound). A signal sent to one
27
- # child touches only that child: it exits, and the parent replaces it. The handlers do
28
- # nothing but write a byte to a pipe; the main loop reads the pipe, so no real work runs in
29
- # signal context. SIGCHLD writes the same pipe to wake the loop when a child exits, and the
30
- # loop also polls every `POLL_INTERVAL` seconds in case a wake-up is ever missed.
31
- #
32
- # If the parent itself dies without relaying anything (SIGKILL, a crash), each child notices
33
- # through a second pipe (`watch_parent`) and stops as if it had been sent SIGTERM, so a dead
34
- # supervisor never leaves orphaned workers hosting rooms.
35
- class Supervisor
36
- STOP_SIGNALS = %w[TERM INT].freeze
37
- POLL_INTERVAL = 1
38
-
39
- Worker = Struct.new(:index, :pid, :started_at)
40
-
41
- attr_reader :count, :logger
42
-
43
- def initialize(count:, logger:, stable_after: 5, first_backoff: 1, max_backoff: 30, &body)
44
- raise ArgumentError, "count must be at least 1 (got #{count.inspect})" unless count.is_a?(Integer) && count >= 1
45
- raise ArgumentError, "Supervisor needs a block to run in each worker" unless body
46
-
47
- @count = count
48
- @logger = logger
49
- @stable_after = stable_after
50
- @first_backoff = first_backoff
51
- @max_backoff = max_backoff
52
- @body = body
53
-
54
- @workers = {} # index => Worker, for every live child
55
- @restart_at = {} # index => monotonic time to fork a replacement
56
- @backoff = {} # index => the delay used for that slot's last boot failure
57
- @stopping = false
58
- end
59
-
60
- # Fork the workers and supervise them until a stop signal has arrived and every child has
61
- # exited. Blocks. Returns the process exit status (0).
62
- def run
63
- @signal_reader, @signal_writer = IO.pipe
64
- @lifeline_reader, @lifeline_writer = IO.pipe
65
- previous_traps = install_traps
66
-
67
- count.times { |index| start_worker(index) }
68
-
69
- loop do
70
- wait_for_wakeup
71
- reap_exited_workers
72
- break if @stopping && @workers.empty?
73
- start_due_restarts unless @stopping
74
- end
75
-
76
- logger.info "all workers exited"
77
- 0
78
- ensure
79
- # If we're leaving for any reason other than "every child is gone" (a bug, say), don't
80
- # orphan the children: tell them to stop too. Then hand the signals back.
81
- relay("TERM") if @workers.any?
82
- previous_traps&.each { |sig, handler| trap(sig, handler) }
83
- [@signal_reader, @signal_writer, @lifeline_reader, @lifeline_writer].each { |io| io&.close }
84
- end
85
-
86
- # Ask the supervisor to shut down as if `signal` had arrived: relay it to every worker and
87
- # exit once they're gone. Safe from any thread and from a trap handler; the work happens on
88
- # the thread running `run`.
89
- def stop!(signal = "TERM")
90
- wake(signal)
91
- end
92
-
93
- def stopping?
94
- @stopping
95
- end
96
-
97
- # Pids of the children alive right now.
98
- def worker_pids
99
- @workers.values.map(&:pid)
100
- end
101
-
102
- private
103
-
104
- # ---- The parent ----------------------------------------------------------------------
105
-
106
- def install_traps
107
- (STOP_SIGNALS + %w[CHLD]).to_h do |sig|
108
- [sig, trap(sig) { wake(sig) }]
109
- end
110
- end
111
-
112
- # Runs in signal context, so it does the one thing that's safe there: a non-blocking write
113
- # to the pipe. A full pipe means plenty of wake-ups are already queued, so dropping is fine.
114
- def wake(signal)
115
- @signal_writer&.write_nonblock("#{signal}\n")
116
- rescue IO::WaitWritable, IOError, Errno::EPIPE
117
- nil
118
- end
119
-
120
- # Sleep until a signal or child exit wakes us, a restart is due, or the poll interval
121
- # passes. Any stop signal on the pipe starts the shutdown.
122
- def wait_for_wakeup
123
- timeout = [POLL_INTERVAL, seconds_until_next_restart].compact.min
124
- ready, = IO.select([@signal_reader], nil, nil, timeout)
125
- return unless ready
126
-
127
- pending = begin
128
- @signal_reader.read_nonblock(4096)
129
- rescue IO::WaitReadable, EOFError
130
- ""
131
- end
132
- pending.split("\n").each { |signal| begin_stopping(signal) if STOP_SIGNALS.include?(signal) }
133
- end
134
-
135
- def begin_stopping(signal)
136
- if @stopping
137
- # A second Ctrl-C or TERM: pass it along again, in case a child missed the first
138
- relay(signal)
139
- return
140
- end
141
-
142
- @stopping = true
143
- @restart_at.clear
144
- logger.info "got SIG#{signal}, relaying to #{@workers.size} worker(s) and waiting for them"
145
- relay(signal)
146
- end
147
-
148
- def relay(signal)
149
- @workers.each_value do |worker|
150
- Process.kill(signal, worker.pid)
151
- rescue Errno::ESRCH
152
- nil # already gone; the next reap logs it
153
- end
154
- end
155
-
156
- def reap_exited_workers
157
- @workers.values.each do |worker|
158
- status = begin
159
- _, status = Process.wait2(worker.pid, Process::WNOHANG)
160
- status
161
- rescue Errno::ECHILD
162
- :unknown # someone else reaped it; treat it as gone
163
- end
164
- next if status.nil?
165
-
166
- @workers.delete(worker.index)
167
- uptime = now - worker.started_at
168
- reason = describe_exit(status)
169
-
170
- if @stopping
171
- logger.info "worker #{worker.index} (pid #{worker.pid}) #{reason} after #{uptime.round}s"
172
- else
173
- delay = restart_delay(worker.index, uptime)
174
- @restart_at[worker.index] = now + delay
175
- logger.warn "worker #{worker.index} (pid #{worker.pid}) #{reason} after #{uptime.round}s; " \
176
- "restarting #{delay.zero? ? 'now' : "in #{delay}s"}"
177
- end
178
- end
179
- end
180
-
181
- def describe_exit(status)
182
- return "exited (status unknown)" if status == :unknown
183
- return "killed by SIG#{Signal.signame(status.termsig)}" if status.signaled?
184
-
185
- "exited with status #{status.exitstatus}"
186
- end
187
-
188
- # A worker that lived a while and then died gets replaced right away. One that died
189
- # almost immediately most likely can't boot, so wait, and wait longer each time it happens.
190
- def restart_delay(index, uptime)
191
- if uptime >= @stable_after
192
- @backoff.delete(index)
193
- 0
194
- else
195
- @backoff[index] = @backoff[index] ? [@backoff[index] * 2, @max_backoff].min : @first_backoff
196
- end
197
- end
198
-
199
- def start_due_restarts
200
- due = @restart_at.select { |_, at| at <= now }.keys
201
- due.each do |index|
202
- @restart_at.delete(index)
203
- start_worker(index)
204
- end
205
- end
206
-
207
- def seconds_until_next_restart
208
- next_at = @restart_at.values.min
209
- next_at && [next_at - now, 0].max
210
- end
211
-
212
- def start_worker(index)
213
- pid = fork_worker(index)
214
- @workers[index] = Worker.new(index, pid, now)
215
- logger.info "forked worker #{index} (pid #{pid})"
216
- end
217
-
218
- def now
219
- Process.clock_gettime(Process::CLOCK_MONOTONIC)
220
- end
221
-
222
- # ---- The child -----------------------------------------------------------------------
223
-
224
- def fork_worker(index)
225
- Process.fork do
226
- status = 1
227
- begin
228
- # The child inherits the parent's traps and both ends of its wake-up pipe. Neither
229
- # belongs here: a relayed SIGTERM must reach the body's own handler (or kill the
230
- # child by default) instead of writing to a pipe nobody in this process reads.
231
- (STOP_SIGNALS + %w[CHLD]).each { |sig| trap(sig, "DEFAULT") }
232
- @signal_reader.close
233
- @signal_writer.close
234
- @workers = {}
235
-
236
- # Give up our copy of the lifeline's write end first, so a sibling can never keep the
237
- # pipe open after the parent is gone; then watch the read end for the parent's death
238
- @lifeline_writer.close
239
- watch_parent(@lifeline_reader)
240
-
241
- CableRoom.after_fork!
242
-
243
- result = @body.call(index)
244
- status = result.is_a?(Integer) ? result : 0
245
- rescue SystemExit => e
246
- status = e.status
247
- rescue SignalException
248
- raise # let Ruby end the process with the signal, so the parent sees which one
249
- rescue Exception => e # rubocop:disable Lint/RescueException -- the child must not outlive a broken body
250
- logger.error "worker #{index} (pid #{Process.pid}) crashed: #{e.class}: #{e.message}\n #{e.backtrace&.first(10)&.join("\n ")}"
251
- status = 1
252
- end
253
-
254
- $stdout.flush
255
- $stderr.flush
256
- exit!(status)
257
- end
258
- end
259
-
260
- # Once every child has closed its copy, the parent holds the only write end of the lifeline
261
- # pipe, so a read here returns EOF exactly when the parent is gone: crashed, or SIGKILLed by
262
- # the container's `timeout -k`. A worker must not carry on hosting rooms with nobody
263
- # supervising it, so it stops itself the same way a relayed SIGTERM would stop it.
264
- def watch_parent(reader)
265
- thread = Thread.new do
266
- reader.read(1)
267
- Process.kill("TERM", Process.pid)
268
- end
269
- thread.name = "cable_room-supervisor-lifeline"
270
- thread.report_on_exception = false
271
- thread
272
- end
273
- end
274
- end
275
- end