cable_room 0.7.0.beta3 → 0.8.0.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +0 -1287
- data/cable_room.gemspec +2 -5
- data/lib/cable_room/host/action_cable_inbound.rb +35 -0
- data/lib/cable_room/host/runner.rb +109 -391
- data/lib/cable_room/host/worker_pool.rb +27 -40
- data/lib/cable_room/host.rb +18 -364
- data/lib/cable_room/ports.rb +9 -19
- data/lib/cable_room/railtie.rb +11 -3
- data/lib/cable_room/room/base.rb +41 -40
- data/lib/cable_room/room/callbacks.rb +0 -22
- data/lib/cable_room/room/host_adapter.rb +4 -9
- data/lib/cable_room/room/lifecycle.rb +6 -23
- data/lib/cable_room/room/port_management.rb +0 -50
- data/lib/cable_room/room/reaping.rb +0 -33
- data/lib/cable_room/room/user_management.rb +0 -27
- data/lib/cable_room/room.rb +1 -4
- data/lib/cable_room/room_member.rb +84 -275
- data/lib/cable_room/room_proxy_channel.rb +2 -13
- data/lib/cable_room/version.rb +1 -1
- data/lib/cable_room.rb +0 -55
- metadata +7 -21
- data/CHANGELOG.md +0 -122
- data/exe/cable_room +0 -8
- data/lib/cable_room/broadcaster.rb +0 -116
- data/lib/cable_room/bus.rb +0 -372
- data/lib/cable_room/cli.rb +0 -237
- data/lib/cable_room/config.rb +0 -112
- data/lib/cable_room/host/bus_inbound.rb +0 -36
- data/lib/cable_room/host/supervisor.rb +0 -275
- data/lib/cable_room/membership_store.rb +0 -105
- data/lib/cable_room/migration.rb +0 -586
- data/lib/cable_room/placement.rb +0 -260
- data/lib/cable_room/room/snapshotting.rb +0 -82
- data/lib/cable_room/room_harness.rb +0 -168
- data/lib/cable_room/snapshot.rb +0 -136
data/lib/cable_room/cli.rb
DELETED
|
@@ -1,237 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require "optparse"
|
|
4
|
-
require "etc"
|
|
5
|
-
require "logger"
|
|
6
|
-
require_relative "version"
|
|
7
|
-
|
|
8
|
-
module CableRoom
|
|
9
|
-
# The `cable_room` command. Its one command, `server`, is the "rooms host" role from the
|
|
10
|
-
# distributed-rooms design: boot the Rails app, become a rooms host, run until SIGTERM or SIGINT,
|
|
11
|
-
# then shut every room down cleanly.
|
|
12
|
-
#
|
|
13
|
-
# It only makes sense with `room_host = :remote` and refuses to run otherwise: in :inline the
|
|
14
|
-
# web processes host rooms themselves, so a server here would only race them for locks.
|
|
15
|
-
#
|
|
16
|
-
# Boot works the way sidekiq and good_job do it: `require` the app's `config/environment.rb`
|
|
17
|
-
# from the current directory, or from `--require PATH` (an app directory or a boot file).
|
|
18
|
-
# Bundler is already set up because the command runs under `bundle exec`. Nothing from the gem
|
|
19
|
-
# beyond this file and `version` is loaded before the app boots, so the app's Gemfile decides
|
|
20
|
-
# when the Railtie loads, the same as in a web process.
|
|
21
|
-
#
|
|
22
|
-
# `--workers N` picks the process model. With N = 1 this process hosts rooms itself. With
|
|
23
|
-
# N > 1 (the default is the machine's core count) the app is booted once, here, and then
|
|
24
|
-
# `Host::Supervisor` forks N children that each host rooms; this parent only watches them,
|
|
25
|
-
# replaces one that dies, and relays SIGTERM and SIGINT. The parent never builds a Host, a Bus
|
|
26
|
-
# subscriber, or a Redis connection of its own: those belong to the children, which build them
|
|
27
|
-
# after the fork so no process ever shares a socket or a thread with another.
|
|
28
|
-
#
|
|
29
|
-
# Everything is instance state and `run` returns an exit status, so specs can drive the parsing
|
|
30
|
-
# and the refusals in-process; only the exe calls `exit`.
|
|
31
|
-
class CLI
|
|
32
|
-
STOP_SIGNALS = %w[TERM INT].freeze
|
|
33
|
-
|
|
34
|
-
attr_reader :options, :command
|
|
35
|
-
|
|
36
|
-
def initialize(argv, stdout: $stdout, stderr: $stderr)
|
|
37
|
-
@argv = argv.dup
|
|
38
|
-
@stdout = stdout
|
|
39
|
-
@stderr = stderr
|
|
40
|
-
@options = { workers: nil, require: Dir.pwd }
|
|
41
|
-
end
|
|
42
|
-
|
|
43
|
-
# Parse and run. Returns the process exit status.
|
|
44
|
-
def run
|
|
45
|
-
parse!
|
|
46
|
-
return 0 if @help_shown
|
|
47
|
-
|
|
48
|
-
case command
|
|
49
|
-
when "server" then server
|
|
50
|
-
when nil
|
|
51
|
-
@stderr.puts parser
|
|
52
|
-
1
|
|
53
|
-
else
|
|
54
|
-
@stderr.puts "Unknown command #{command.inspect}.", parser
|
|
55
|
-
1
|
|
56
|
-
end
|
|
57
|
-
rescue OptionParser::ParseError => e
|
|
58
|
-
@stderr.puts e.message, parser
|
|
59
|
-
1
|
|
60
|
-
end
|
|
61
|
-
|
|
62
|
-
# How many worker processes `server` runs: `--workers`, or one per core.
|
|
63
|
-
def workers
|
|
64
|
-
options[:workers] || Etc.nprocessors
|
|
65
|
-
end
|
|
66
|
-
|
|
67
|
-
private
|
|
68
|
-
|
|
69
|
-
def parser
|
|
70
|
-
@parser ||= OptionParser.new do |o|
|
|
71
|
-
o.banner = "Usage: cable_room server [options]"
|
|
72
|
-
o.separator ""
|
|
73
|
-
o.separator "Runs a CableRoom rooms host: boots the Rails app in the current directory and"
|
|
74
|
-
o.separator "hosts rooms until SIGTERM or SIGINT. Needs CableRoom.config.room_host = :remote."
|
|
75
|
-
o.separator ""
|
|
76
|
-
o.on("-r", "--require PATH", "App directory (with config/environment.rb) or a boot file to require") do |path|
|
|
77
|
-
@options[:require] = path
|
|
78
|
-
end
|
|
79
|
-
o.on("-w", "--workers N", Integer,
|
|
80
|
-
"Worker processes to fork (default: one per core, #{Etc.nprocessors} here). 1 runs in this process.") do |n|
|
|
81
|
-
@options[:workers] = n
|
|
82
|
-
end
|
|
83
|
-
o.on("-v", "--version", "Print the version and exit") do
|
|
84
|
-
@stdout.puts CableRoom::VERSION
|
|
85
|
-
@help_shown = true
|
|
86
|
-
end
|
|
87
|
-
o.on("-h", "--help", "Show this help") do
|
|
88
|
-
@stdout.puts o
|
|
89
|
-
@help_shown = true
|
|
90
|
-
end
|
|
91
|
-
end
|
|
92
|
-
end
|
|
93
|
-
|
|
94
|
-
def parse!
|
|
95
|
-
rest = parser.parse(@argv)
|
|
96
|
-
@command = rest.shift
|
|
97
|
-
raise OptionParser::ParseError, "Unexpected arguments: #{rest.join(' ')}" if rest.any?
|
|
98
|
-
end
|
|
99
|
-
|
|
100
|
-
def server
|
|
101
|
-
# Check the cheap things before booting Rails, so a bad flag fails in milliseconds
|
|
102
|
-
unless workers >= 1
|
|
103
|
-
@stderr.puts "cable_room server: --workers must be at least 1 (got #{workers})."
|
|
104
|
-
return 1
|
|
105
|
-
end
|
|
106
|
-
|
|
107
|
-
boot_app!
|
|
108
|
-
|
|
109
|
-
unless CableRoom.config.remote?
|
|
110
|
-
@stderr.puts "cable_room server: CableRoom.config.room_host is #{CableRoom.config.room_host.inspect}, but a " \
|
|
111
|
-
"rooms host only makes sense with :remote (in :inline the web processes host rooms " \
|
|
112
|
-
"themselves). Set CABLE_ROOM_HOST=remote or `c.room_host = :remote` and try again."
|
|
113
|
-
return 1
|
|
114
|
-
end
|
|
115
|
-
|
|
116
|
-
return run_host if workers == 1
|
|
117
|
-
|
|
118
|
-
supervise_workers
|
|
119
|
-
end
|
|
120
|
-
|
|
121
|
-
# Load the app: a directory means its config/environment.rb, anything else is required as-is
|
|
122
|
-
def boot_app!
|
|
123
|
-
path = File.expand_path(options[:require])
|
|
124
|
-
path = File.join(path, "config", "environment.rb") if File.directory?(path)
|
|
125
|
-
require path
|
|
126
|
-
end
|
|
127
|
-
|
|
128
|
-
# Fork `workers` children that each run `run_host`, and watch them until a stop signal has
|
|
129
|
-
# come and gone. This parent hosts nothing itself (see the class comment for why).
|
|
130
|
-
def supervise_workers
|
|
131
|
-
Process.setproctitle("cable_room server: supervisor (#{workers} workers)")
|
|
132
|
-
say "supervising #{workers} workers (pid #{Process.pid}, room_host=#{CableRoom.config.room_host}, " \
|
|
133
|
-
"broadcaster=#{CableRoom.config.broadcaster})"
|
|
134
|
-
|
|
135
|
-
supervisor = CableRoom::Host::Supervisor.new(count: workers, logger: supervisor_logger) do |index|
|
|
136
|
-
run_host(worker: index)
|
|
137
|
-
end
|
|
138
|
-
status = supervisor.run
|
|
139
|
-
say "shut down"
|
|
140
|
-
status
|
|
141
|
-
end
|
|
142
|
-
|
|
143
|
-
# Logs from the supervisor go to the same place as the CLI's own messages, in the same style.
|
|
144
|
-
# Unbuffered, so a "forked worker" line shows up when it happens, not when the process exits.
|
|
145
|
-
def supervisor_logger
|
|
146
|
-
@stdout.sync = true if @stdout.respond_to?(:sync=)
|
|
147
|
-
Logger.new(@stdout).tap do |logger|
|
|
148
|
-
logger.formatter = ->(_severity, _time, _progname, message) { "cable_room server: #{message}\n" }
|
|
149
|
-
end
|
|
150
|
-
end
|
|
151
|
-
|
|
152
|
-
# Host rooms in this process until SIGTERM or SIGINT, then shut them down. This is the whole
|
|
153
|
-
# life of a single-process server and of each forked worker (`worker` is the worker's index,
|
|
154
|
-
# nil when there's no supervisor). Returns the exit status.
|
|
155
|
-
def run_host(worker: nil)
|
|
156
|
-
# Trap before starting the Host, so a signal that lands while rooms are still coming up
|
|
157
|
-
# (the supervisor relaying a SIGTERM that arrived mid-boot, say) is kept, not fatal
|
|
158
|
-
signals = trap_stop_signals
|
|
159
|
-
|
|
160
|
-
if worker
|
|
161
|
-
Process.setproctitle("cable_room server: worker #{worker}")
|
|
162
|
-
tag_logger("cable_room worker #{worker}")
|
|
163
|
-
end
|
|
164
|
-
|
|
165
|
-
host = CableRoom::Host.start!
|
|
166
|
-
say "hosting rooms (pid #{Process.pid}, room_host=#{CableRoom.config.room_host}, " \
|
|
167
|
-
"broadcaster=#{CableRoom.config.broadcaster})", worker: worker
|
|
168
|
-
|
|
169
|
-
signal = wait_for_stop_signal(signals)
|
|
170
|
-
stop_host(host, signal, worker: worker)
|
|
171
|
-
say "shut down", worker: worker
|
|
172
|
-
0
|
|
173
|
-
ensure
|
|
174
|
-
restore_traps
|
|
175
|
-
end
|
|
176
|
-
|
|
177
|
-
# A stop signal with rooms open is a planned shutdown: hand the rooms to peer hosts before
|
|
178
|
-
# leaving (`Host#drain!`, bounded by `drain_timeout`; a room nobody adopts closes with
|
|
179
|
-
# `room_closed`). With nothing open there is nothing to move, so just shut down. Both are
|
|
180
|
-
# idempotent, so the one SIGTERM a supervisor relays and a later `at_exit` don't collide.
|
|
181
|
-
def stop_host(host, signal, worker: nil)
|
|
182
|
-
rooms = host.rooms.size
|
|
183
|
-
if rooms.zero?
|
|
184
|
-
say "got SIG#{signal}, shutting down 0 room(s)", worker: worker
|
|
185
|
-
host.shutdown!
|
|
186
|
-
return
|
|
187
|
-
end
|
|
188
|
-
|
|
189
|
-
say "got SIG#{signal}, migrating #{rooms} room(s) to peer hosts", worker: worker
|
|
190
|
-
result = host.drain!(reason: "SIG#{signal}")
|
|
191
|
-
say "drained: #{result.migrated.size} room(s) migrated, #{result.closed.size} closed, " \
|
|
192
|
-
"in #{result.duration.round(1)}s", worker: worker
|
|
193
|
-
end
|
|
194
|
-
|
|
195
|
-
# Every room on this Host logs through `ActionCable.server.logger` (Host#logger); give that
|
|
196
|
-
# logger a tag naming the worker, so lines from the N workers sharing one stdout can be told
|
|
197
|
-
# apart. `tagged` without a block returns a logger whose tags apply on every thread, which is
|
|
198
|
-
# what we need: rooms log from the worker pool, not from this thread. A logger without tags
|
|
199
|
-
# (a plain Logger, or Rails' BroadcastLogger fanning out to several) is left as it is.
|
|
200
|
-
def tag_logger(tag)
|
|
201
|
-
logger = ActionCable.server.logger
|
|
202
|
-
return unless logger.respond_to?(:tagged)
|
|
203
|
-
|
|
204
|
-
tagged = logger.tagged(tag)
|
|
205
|
-
ActionCable.server.config.logger = tagged if tagged.respond_to?(:info)
|
|
206
|
-
end
|
|
207
|
-
|
|
208
|
-
def say(message, worker: nil)
|
|
209
|
-
prefix = worker ? "cable_room server (worker #{worker})" : "cable_room server"
|
|
210
|
-
@stdout.puts "#{prefix}: #{message}"
|
|
211
|
-
@stdout.flush
|
|
212
|
-
end
|
|
213
|
-
|
|
214
|
-
# Signal handlers may only do trivial work, so each one just drops the signal's name on a
|
|
215
|
-
# queue for the main thread to pick up. Returns the queue.
|
|
216
|
-
def trap_stop_signals
|
|
217
|
-
signals = Queue.new
|
|
218
|
-
@previous_traps = STOP_SIGNALS.to_h { |sig| [sig, trap(sig) { signals << sig }] }
|
|
219
|
-
signals
|
|
220
|
-
end
|
|
221
|
-
|
|
222
|
-
# Block the main thread until SIGTERM or SIGINT; returns the signal's name. A timed pop, not
|
|
223
|
-
# a plain one: if this were the only thread left (a worker whose rooms have all gone quiet,
|
|
224
|
-
# say), a blocking pop would trip Ruby's deadlock detector, because a trap isn't a thread.
|
|
225
|
-
def wait_for_stop_signal(signals)
|
|
226
|
-
loop do
|
|
227
|
-
signal = signals.pop(timeout: 1)
|
|
228
|
-
return signal if signal
|
|
229
|
-
end
|
|
230
|
-
end
|
|
231
|
-
|
|
232
|
-
def restore_traps
|
|
233
|
-
@previous_traps&.each { |sig, handler| trap(sig, handler) }
|
|
234
|
-
@previous_traps = nil
|
|
235
|
-
end
|
|
236
|
-
end
|
|
237
|
-
end
|
data/lib/cable_room/config.rb
DELETED
|
@@ -1,112 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module CableRoom
|
|
4
|
-
# Holds the app-wide settings for CableRoom. Get the current one with `CableRoom.config`
|
|
5
|
-
# and change it with `CableRoom.configure`:
|
|
6
|
-
#
|
|
7
|
-
# CableRoom.configure do |c|
|
|
8
|
-
# c.room_host = :remote # :inline | :remote
|
|
9
|
-
# c.broadcaster = :anycable # :action_cable | :anycable
|
|
10
|
-
# c.provision_delay_ms = 20
|
|
11
|
-
# c.drain_timeout = 10.minutes
|
|
12
|
-
# c.handoff_timeout = 10.seconds
|
|
13
|
-
# end
|
|
14
|
-
#
|
|
15
|
-
# The env vars CABLE_ROOM_HOST and CABLE_ROOM_BROADCASTER override `room_host` and
|
|
16
|
-
# `broadcaster`, even when the configure block sets them. That lets one build run in
|
|
17
|
-
# both modes without a code change. With no configuration at all, every default keeps
|
|
18
|
-
# today's behavior: rooms run inline and broadcast through ActionCable.
|
|
19
|
-
class Config
|
|
20
|
-
class InvalidOption < ArgumentError; end
|
|
21
|
-
|
|
22
|
-
ROOM_HOSTS = %i[inline remote].freeze
|
|
23
|
-
BROADCASTERS = %i[action_cable anycable].freeze
|
|
24
|
-
|
|
25
|
-
ENV_OVERRIDES = {
|
|
26
|
-
room_host: "CABLE_ROOM_HOST",
|
|
27
|
-
broadcaster: "CABLE_ROOM_BROADCASTER",
|
|
28
|
-
}.freeze
|
|
29
|
-
|
|
30
|
-
DEFAULTS = {
|
|
31
|
-
room_host: :inline,
|
|
32
|
-
broadcaster: :action_cable,
|
|
33
|
-
provision_delay_ms: 20,
|
|
34
|
-
drain_timeout: 10.minutes,
|
|
35
|
-
handoff_timeout: 10.seconds,
|
|
36
|
-
}.freeze
|
|
37
|
-
|
|
38
|
-
attr_reader :room_host, :broadcaster
|
|
39
|
-
attr_accessor :provision_delay_ms, :drain_timeout, :handoff_timeout
|
|
40
|
-
|
|
41
|
-
def initialize
|
|
42
|
-
DEFAULTS.each { |name, value| instance_variable_set("@#{name}", value) }
|
|
43
|
-
end
|
|
44
|
-
|
|
45
|
-
# Where rooms run. `:inline` keeps them in the web process; `:remote` moves them to a
|
|
46
|
-
# dedicated host pool. Accepts a Symbol or String; unknown values raise right away.
|
|
47
|
-
def room_host=(value)
|
|
48
|
-
@room_host = validate_choice!(:room_host, value, ROOM_HOSTS)
|
|
49
|
-
end
|
|
50
|
-
|
|
51
|
-
# How a room pushes messages to members. Accepts a Symbol or String; unknown values raise.
|
|
52
|
-
def broadcaster=(value)
|
|
53
|
-
@broadcaster = validate_choice!(:broadcaster, value, BROADCASTERS)
|
|
54
|
-
end
|
|
55
|
-
|
|
56
|
-
# Rooms run inside every process that joins them (the web process in a Rails app).
|
|
57
|
-
def inline?
|
|
58
|
-
room_host == :inline
|
|
59
|
-
end
|
|
60
|
-
|
|
61
|
-
# Rooms run only in `cable_room server` processes; a web process never hosts one.
|
|
62
|
-
def remote?
|
|
63
|
-
room_host == :remote
|
|
64
|
-
end
|
|
65
|
-
|
|
66
|
-
# Copies CABLE_ROOM_HOST and CABLE_ROOM_BROADCASTER onto this config. Blank or missing
|
|
67
|
-
# variables leave the current value alone. `env` is injectable for specs.
|
|
68
|
-
def apply_env_overrides(env = ENV)
|
|
69
|
-
ENV_OVERRIDES.each do |name, var|
|
|
70
|
-
value = env[var]
|
|
71
|
-
next if value.nil? || value.strip.empty?
|
|
72
|
-
|
|
73
|
-
public_send("#{name}=", value)
|
|
74
|
-
end
|
|
75
|
-
self
|
|
76
|
-
end
|
|
77
|
-
|
|
78
|
-
# Checks every knob at once. The choice knobs are already checked by their setters, so
|
|
79
|
-
# this mostly guards the timing knobs, which are plain accessors.
|
|
80
|
-
def validate!
|
|
81
|
-
validate_choice!(:room_host, room_host, ROOM_HOSTS)
|
|
82
|
-
validate_choice!(:broadcaster, broadcaster, BROADCASTERS)
|
|
83
|
-
validate_number!(:provision_delay_ms, provision_delay_ms, min: 0)
|
|
84
|
-
validate_number!(:drain_timeout, drain_timeout, min: 0, exclusive: true)
|
|
85
|
-
validate_number!(:handoff_timeout, handoff_timeout, min: 0, exclusive: true)
|
|
86
|
-
self
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
def to_h
|
|
90
|
-
DEFAULTS.keys.to_h { |name| [name, public_send(name)] }
|
|
91
|
-
end
|
|
92
|
-
|
|
93
|
-
private
|
|
94
|
-
|
|
95
|
-
def validate_choice!(name, value, allowed)
|
|
96
|
-
symbol = value.to_s.to_sym unless value.nil?
|
|
97
|
-
return symbol if allowed.include?(symbol)
|
|
98
|
-
|
|
99
|
-
raise InvalidOption,
|
|
100
|
-
"CableRoom config: #{name} must be one of #{allowed.map(&:inspect).join(', ')} " \
|
|
101
|
-
"(got #{value.inspect})"
|
|
102
|
-
end
|
|
103
|
-
|
|
104
|
-
def validate_number!(name, value, min:, exclusive: false)
|
|
105
|
-
ok = value.is_a?(Numeric) && (exclusive ? value > min : value >= min)
|
|
106
|
-
return value if ok
|
|
107
|
-
|
|
108
|
-
bound = exclusive ? "greater than #{min}" : "at least #{min}"
|
|
109
|
-
raise InvalidOption, "CableRoom config: #{name} must be a number #{bound} (got #{value.inspect})"
|
|
110
|
-
end
|
|
111
|
-
end
|
|
112
|
-
end
|
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
module CableRoom
|
|
2
|
-
class Host
|
|
3
|
-
# How rooms receive messages: off the process-wide CableRoom::Bus (see `CableRoom.bus`), on
|
|
4
|
-
# the channels `Room::Base.inbound_channel` names. This is the only room-side code that
|
|
5
|
-
# touches the inbound transport, and it's the same in `:inline` and `:remote` mode — the only
|
|
6
|
-
# difference is which process the Host lives in. Anything with these two methods can be handed
|
|
7
|
-
# to `Host.new(inbound:)`.
|
|
8
|
-
#
|
|
9
|
-
# `subscribe` returns a handle that `unsubscribe` needs back. Handlers get the decoded message
|
|
10
|
-
# on the Bus's single subscriber thread, in publish order, so they must be quick and hand the
|
|
11
|
-
# real work off; Host::Runner does that by queueing it on the room. `subscribe` blocks until
|
|
12
|
-
# Redis confirms the subscription, so `on_live` runs on the caller's thread right before it
|
|
13
|
-
# returns, and anything published after that is delivered.
|
|
14
|
-
class BusInbound
|
|
15
|
-
# `bus` defaults to `CableRoom.bus` (looked up lazily so specs can swap it)
|
|
16
|
-
def initialize(bus: nil)
|
|
17
|
-
@bus = bus
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
def subscribe(channel, on_live: nil, &on_message)
|
|
21
|
-
bus.subscribe(channel) { |message, _channel| on_message.call(message) }
|
|
22
|
-
on_live&.call
|
|
23
|
-
on_message
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
def unsubscribe(channel, _handle)
|
|
27
|
-
bus.unsubscribe(channel)
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
# The Bus behind this transport (the process-wide one unless a spec handed us another)
|
|
31
|
-
def bus
|
|
32
|
-
@bus || CableRoom.bus
|
|
33
|
-
end
|
|
34
|
-
end
|
|
35
|
-
end
|
|
36
|
-
end
|
|
@@ -1,275 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module CableRoom
|
|
4
|
-
class Host
|
|
5
|
-
# The parent process behind `cable_room server --workers N`. It forks N children, runs one
|
|
6
|
-
# block in each, replaces a child that dies, and passes SIGTERM and SIGINT on to every child
|
|
7
|
-
# before exiting itself. It never hosts a room, never opens Redis, and never starts a Bus:
|
|
8
|
-
# each child builds its own after the fork (`CableRoom.after_fork!`), so nothing with a
|
|
9
|
-
# socket or a thread behind it is ever shared between processes.
|
|
10
|
-
#
|
|
11
|
-
# Supervisor.new(count: 2, logger: logger) { |index| run_a_host(index) }.run
|
|
12
|
-
#
|
|
13
|
-
# The block is the whole child: it runs right after the fork and its return value (an Integer,
|
|
14
|
-
# or nil for 0) is the child's exit status. The child leaves with `exit!`, so `at_exit` hooks
|
|
15
|
-
# the parent registered before forking don't run a second time in every child (Resque does
|
|
16
|
-
# the same for its forked workers). A block that raises is logged and exits 1, which makes
|
|
17
|
-
# the supervisor replace it.
|
|
18
|
-
#
|
|
19
|
-
# Restarts: a child that dies is replaced. One that ran for at least `stable_after` seconds
|
|
20
|
-
# comes back at once; one that died sooner is treated as failing to boot and comes back
|
|
21
|
-
# after a delay that doubles from `first_backoff` up to `max_backoff`, so a broken app forks
|
|
22
|
-
# a few times a minute rather than thousands of times a second.
|
|
23
|
-
#
|
|
24
|
-
# Signals: SIGTERM and SIGINT to the parent are relayed to every live child and the parent
|
|
25
|
-
# then waits, with no time limit of its own, until they've all exited (a child migrating rooms
|
|
26
|
-
# may take a while; the container's `timeout -k` is the outer bound). A signal sent to one
|
|
27
|
-
# child touches only that child: it exits, and the parent replaces it. The handlers do
|
|
28
|
-
# nothing but write a byte to a pipe; the main loop reads the pipe, so no real work runs in
|
|
29
|
-
# signal context. SIGCHLD writes the same pipe to wake the loop when a child exits, and the
|
|
30
|
-
# loop also polls every `POLL_INTERVAL` seconds in case a wake-up is ever missed.
|
|
31
|
-
#
|
|
32
|
-
# If the parent itself dies without relaying anything (SIGKILL, a crash), each child notices
|
|
33
|
-
# through a second pipe (`watch_parent`) and stops as if it had been sent SIGTERM, so a dead
|
|
34
|
-
# supervisor never leaves orphaned workers hosting rooms.
|
|
35
|
-
class Supervisor
|
|
36
|
-
STOP_SIGNALS = %w[TERM INT].freeze
|
|
37
|
-
POLL_INTERVAL = 1
|
|
38
|
-
|
|
39
|
-
Worker = Struct.new(:index, :pid, :started_at)
|
|
40
|
-
|
|
41
|
-
attr_reader :count, :logger
|
|
42
|
-
|
|
43
|
-
def initialize(count:, logger:, stable_after: 5, first_backoff: 1, max_backoff: 30, &body)
|
|
44
|
-
raise ArgumentError, "count must be at least 1 (got #{count.inspect})" unless count.is_a?(Integer) && count >= 1
|
|
45
|
-
raise ArgumentError, "Supervisor needs a block to run in each worker" unless body
|
|
46
|
-
|
|
47
|
-
@count = count
|
|
48
|
-
@logger = logger
|
|
49
|
-
@stable_after = stable_after
|
|
50
|
-
@first_backoff = first_backoff
|
|
51
|
-
@max_backoff = max_backoff
|
|
52
|
-
@body = body
|
|
53
|
-
|
|
54
|
-
@workers = {} # index => Worker, for every live child
|
|
55
|
-
@restart_at = {} # index => monotonic time to fork a replacement
|
|
56
|
-
@backoff = {} # index => the delay used for that slot's last boot failure
|
|
57
|
-
@stopping = false
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
# Fork the workers and supervise them until a stop signal has arrived and every child has
|
|
61
|
-
# exited. Blocks. Returns the process exit status (0).
|
|
62
|
-
def run
|
|
63
|
-
@signal_reader, @signal_writer = IO.pipe
|
|
64
|
-
@lifeline_reader, @lifeline_writer = IO.pipe
|
|
65
|
-
previous_traps = install_traps
|
|
66
|
-
|
|
67
|
-
count.times { |index| start_worker(index) }
|
|
68
|
-
|
|
69
|
-
loop do
|
|
70
|
-
wait_for_wakeup
|
|
71
|
-
reap_exited_workers
|
|
72
|
-
break if @stopping && @workers.empty?
|
|
73
|
-
start_due_restarts unless @stopping
|
|
74
|
-
end
|
|
75
|
-
|
|
76
|
-
logger.info "all workers exited"
|
|
77
|
-
0
|
|
78
|
-
ensure
|
|
79
|
-
# If we're leaving for any reason other than "every child is gone" (a bug, say), don't
|
|
80
|
-
# orphan the children: tell them to stop too. Then hand the signals back.
|
|
81
|
-
relay("TERM") if @workers.any?
|
|
82
|
-
previous_traps&.each { |sig, handler| trap(sig, handler) }
|
|
83
|
-
[@signal_reader, @signal_writer, @lifeline_reader, @lifeline_writer].each { |io| io&.close }
|
|
84
|
-
end
|
|
85
|
-
|
|
86
|
-
# Ask the supervisor to shut down as if `signal` had arrived: relay it to every worker and
|
|
87
|
-
# exit once they're gone. Safe from any thread and from a trap handler; the work happens on
|
|
88
|
-
# the thread running `run`.
|
|
89
|
-
def stop!(signal = "TERM")
|
|
90
|
-
wake(signal)
|
|
91
|
-
end
|
|
92
|
-
|
|
93
|
-
def stopping?
|
|
94
|
-
@stopping
|
|
95
|
-
end
|
|
96
|
-
|
|
97
|
-
# Pids of the children alive right now.
|
|
98
|
-
def worker_pids
|
|
99
|
-
@workers.values.map(&:pid)
|
|
100
|
-
end
|
|
101
|
-
|
|
102
|
-
private
|
|
103
|
-
|
|
104
|
-
# ---- The parent ----------------------------------------------------------------------
|
|
105
|
-
|
|
106
|
-
def install_traps
|
|
107
|
-
(STOP_SIGNALS + %w[CHLD]).to_h do |sig|
|
|
108
|
-
[sig, trap(sig) { wake(sig) }]
|
|
109
|
-
end
|
|
110
|
-
end
|
|
111
|
-
|
|
112
|
-
# Runs in signal context, so it does the one thing that's safe there: a non-blocking write
|
|
113
|
-
# to the pipe. A full pipe means plenty of wake-ups are already queued, so dropping is fine.
|
|
114
|
-
def wake(signal)
|
|
115
|
-
@signal_writer&.write_nonblock("#{signal}\n")
|
|
116
|
-
rescue IO::WaitWritable, IOError, Errno::EPIPE
|
|
117
|
-
nil
|
|
118
|
-
end
|
|
119
|
-
|
|
120
|
-
# Sleep until a signal or child exit wakes us, a restart is due, or the poll interval
|
|
121
|
-
# passes. Any stop signal on the pipe starts the shutdown.
|
|
122
|
-
def wait_for_wakeup
|
|
123
|
-
timeout = [POLL_INTERVAL, seconds_until_next_restart].compact.min
|
|
124
|
-
ready, = IO.select([@signal_reader], nil, nil, timeout)
|
|
125
|
-
return unless ready
|
|
126
|
-
|
|
127
|
-
pending = begin
|
|
128
|
-
@signal_reader.read_nonblock(4096)
|
|
129
|
-
rescue IO::WaitReadable, EOFError
|
|
130
|
-
""
|
|
131
|
-
end
|
|
132
|
-
pending.split("\n").each { |signal| begin_stopping(signal) if STOP_SIGNALS.include?(signal) }
|
|
133
|
-
end
|
|
134
|
-
|
|
135
|
-
def begin_stopping(signal)
|
|
136
|
-
if @stopping
|
|
137
|
-
# A second Ctrl-C or TERM: pass it along again, in case a child missed the first
|
|
138
|
-
relay(signal)
|
|
139
|
-
return
|
|
140
|
-
end
|
|
141
|
-
|
|
142
|
-
@stopping = true
|
|
143
|
-
@restart_at.clear
|
|
144
|
-
logger.info "got SIG#{signal}, relaying to #{@workers.size} worker(s) and waiting for them"
|
|
145
|
-
relay(signal)
|
|
146
|
-
end
|
|
147
|
-
|
|
148
|
-
def relay(signal)
|
|
149
|
-
@workers.each_value do |worker|
|
|
150
|
-
Process.kill(signal, worker.pid)
|
|
151
|
-
rescue Errno::ESRCH
|
|
152
|
-
nil # already gone; the next reap logs it
|
|
153
|
-
end
|
|
154
|
-
end
|
|
155
|
-
|
|
156
|
-
def reap_exited_workers
|
|
157
|
-
@workers.values.each do |worker|
|
|
158
|
-
status = begin
|
|
159
|
-
_, status = Process.wait2(worker.pid, Process::WNOHANG)
|
|
160
|
-
status
|
|
161
|
-
rescue Errno::ECHILD
|
|
162
|
-
:unknown # someone else reaped it; treat it as gone
|
|
163
|
-
end
|
|
164
|
-
next if status.nil?
|
|
165
|
-
|
|
166
|
-
@workers.delete(worker.index)
|
|
167
|
-
uptime = now - worker.started_at
|
|
168
|
-
reason = describe_exit(status)
|
|
169
|
-
|
|
170
|
-
if @stopping
|
|
171
|
-
logger.info "worker #{worker.index} (pid #{worker.pid}) #{reason} after #{uptime.round}s"
|
|
172
|
-
else
|
|
173
|
-
delay = restart_delay(worker.index, uptime)
|
|
174
|
-
@restart_at[worker.index] = now + delay
|
|
175
|
-
logger.warn "worker #{worker.index} (pid #{worker.pid}) #{reason} after #{uptime.round}s; " \
|
|
176
|
-
"restarting #{delay.zero? ? 'now' : "in #{delay}s"}"
|
|
177
|
-
end
|
|
178
|
-
end
|
|
179
|
-
end
|
|
180
|
-
|
|
181
|
-
def describe_exit(status)
|
|
182
|
-
return "exited (status unknown)" if status == :unknown
|
|
183
|
-
return "killed by SIG#{Signal.signame(status.termsig)}" if status.signaled?
|
|
184
|
-
|
|
185
|
-
"exited with status #{status.exitstatus}"
|
|
186
|
-
end
|
|
187
|
-
|
|
188
|
-
# A worker that lived a while and then died gets replaced right away. One that died
|
|
189
|
-
# almost immediately most likely can't boot, so wait, and wait longer each time it happens.
|
|
190
|
-
def restart_delay(index, uptime)
|
|
191
|
-
if uptime >= @stable_after
|
|
192
|
-
@backoff.delete(index)
|
|
193
|
-
0
|
|
194
|
-
else
|
|
195
|
-
@backoff[index] = @backoff[index] ? [@backoff[index] * 2, @max_backoff].min : @first_backoff
|
|
196
|
-
end
|
|
197
|
-
end
|
|
198
|
-
|
|
199
|
-
def start_due_restarts
|
|
200
|
-
due = @restart_at.select { |_, at| at <= now }.keys
|
|
201
|
-
due.each do |index|
|
|
202
|
-
@restart_at.delete(index)
|
|
203
|
-
start_worker(index)
|
|
204
|
-
end
|
|
205
|
-
end
|
|
206
|
-
|
|
207
|
-
def seconds_until_next_restart
|
|
208
|
-
next_at = @restart_at.values.min
|
|
209
|
-
next_at && [next_at - now, 0].max
|
|
210
|
-
end
|
|
211
|
-
|
|
212
|
-
def start_worker(index)
|
|
213
|
-
pid = fork_worker(index)
|
|
214
|
-
@workers[index] = Worker.new(index, pid, now)
|
|
215
|
-
logger.info "forked worker #{index} (pid #{pid})"
|
|
216
|
-
end
|
|
217
|
-
|
|
218
|
-
def now
|
|
219
|
-
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
220
|
-
end
|
|
221
|
-
|
|
222
|
-
# ---- The child -----------------------------------------------------------------------
|
|
223
|
-
|
|
224
|
-
def fork_worker(index)
|
|
225
|
-
Process.fork do
|
|
226
|
-
status = 1
|
|
227
|
-
begin
|
|
228
|
-
# The child inherits the parent's traps and both ends of its wake-up pipe. Neither
|
|
229
|
-
# belongs here: a relayed SIGTERM must reach the body's own handler (or kill the
|
|
230
|
-
# child by default) instead of writing to a pipe nobody in this process reads.
|
|
231
|
-
(STOP_SIGNALS + %w[CHLD]).each { |sig| trap(sig, "DEFAULT") }
|
|
232
|
-
@signal_reader.close
|
|
233
|
-
@signal_writer.close
|
|
234
|
-
@workers = {}
|
|
235
|
-
|
|
236
|
-
# Give up our copy of the lifeline's write end first, so a sibling can never keep the
|
|
237
|
-
# pipe open after the parent is gone; then watch the read end for the parent's death
|
|
238
|
-
@lifeline_writer.close
|
|
239
|
-
watch_parent(@lifeline_reader)
|
|
240
|
-
|
|
241
|
-
CableRoom.after_fork!
|
|
242
|
-
|
|
243
|
-
result = @body.call(index)
|
|
244
|
-
status = result.is_a?(Integer) ? result : 0
|
|
245
|
-
rescue SystemExit => e
|
|
246
|
-
status = e.status
|
|
247
|
-
rescue SignalException
|
|
248
|
-
raise # let Ruby end the process with the signal, so the parent sees which one
|
|
249
|
-
rescue Exception => e # rubocop:disable Lint/RescueException -- the child must not outlive a broken body
|
|
250
|
-
logger.error "worker #{index} (pid #{Process.pid}) crashed: #{e.class}: #{e.message}\n #{e.backtrace&.first(10)&.join("\n ")}"
|
|
251
|
-
status = 1
|
|
252
|
-
end
|
|
253
|
-
|
|
254
|
-
$stdout.flush
|
|
255
|
-
$stderr.flush
|
|
256
|
-
exit!(status)
|
|
257
|
-
end
|
|
258
|
-
end
|
|
259
|
-
|
|
260
|
-
# Once every child has closed its copy, the parent holds the only write end of the lifeline
|
|
261
|
-
# pipe, so a read here returns EOF exactly when the parent is gone: crashed, or SIGKILLed by
|
|
262
|
-
# the container's `timeout -k`. A worker must not carry on hosting rooms with nobody
|
|
263
|
-
# supervising it, so it stops itself the same way a relayed SIGTERM would stop it.
|
|
264
|
-
def watch_parent(reader)
|
|
265
|
-
thread = Thread.new do
|
|
266
|
-
reader.read(1)
|
|
267
|
-
Process.kill("TERM", Process.pid)
|
|
268
|
-
end
|
|
269
|
-
thread.name = "cable_room-supervisor-lifeline"
|
|
270
|
-
thread.report_on_exception = false
|
|
271
|
-
thread
|
|
272
|
-
end
|
|
273
|
-
end
|
|
274
|
-
end
|
|
275
|
-
end
|