cable_room 0.7.0.beta3 → 0.8.0.beta2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +0 -1287
- data/cable_room.gemspec +7 -3
- data/lib/cable_room/bus.rb +45 -75
- data/lib/cable_room/cli.rb +105 -76
- data/lib/cable_room/config.rb +46 -80
- data/lib/cable_room/host/bus_inbound.rb +16 -11
- data/lib/cable_room/host/placement.rb +261 -0
- data/lib/cable_room/host/runner.rb +102 -390
- data/lib/cable_room/host/supervisor.rb +48 -22
- data/lib/cable_room/host/worker_pool.rb +27 -40
- data/lib/cable_room/host.rb +90 -320
- data/lib/cable_room/ports.rb +5 -5
- data/lib/cable_room/railtie.rb +14 -0
- data/lib/cable_room/room/base.rb +32 -28
- data/lib/cable_room/room/callbacks.rb +0 -22
- data/lib/cable_room/room/host_adapter.rb +1 -1
- data/lib/cable_room/room/lifecycle.rb +6 -23
- data/lib/cable_room/room/port_management.rb +0 -50
- data/lib/cable_room/room/reaping.rb +0 -33
- data/lib/cable_room/room/user_management.rb +0 -27
- data/lib/cable_room/room.rb +1 -2
- data/lib/cable_room/room_member.rb +112 -248
- data/lib/cable_room/room_proxy_channel.rb +2 -13
- data/lib/cable_room/version.rb +1 -1
- data/lib/cable_room.rb +50 -37
- metadata +10 -11
- data/CHANGELOG.md +0 -122
- data/lib/cable_room/broadcaster.rb +0 -116
- data/lib/cable_room/membership_store.rb +0 -105
- data/lib/cable_room/migration.rb +0 -586
- data/lib/cable_room/placement.rb +0 -260
- data/lib/cable_room/room/snapshotting.rb +0 -82
- data/lib/cable_room/room_harness.rb +0 -168
- data/lib/cable_room/snapshot.rb +0 -136
data/cable_room.gemspec
CHANGED
|
@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
|
|
|
18
18
|
spec.summary = "Build live Rooms on top of ActionCable"
|
|
19
19
|
spec.homepage = "https://instructure.com"
|
|
20
20
|
|
|
21
|
-
spec.files = Dir["{app,config,db,exe,lib}/**/*", "README.md", "
|
|
21
|
+
spec.files = Dir["{app,config,db,exe,lib}/**/*", "README.md", "*.gemspec"]
|
|
22
22
|
spec.bindir = "exe"
|
|
23
23
|
spec.executables = ["cable_room"]
|
|
24
24
|
spec.require_paths = ['lib']
|
|
@@ -27,8 +27,12 @@ Gem::Specification.new do |spec|
|
|
|
27
27
|
spec.add_dependency "rufus-scheduler", "~> 3.6"
|
|
28
28
|
spec.add_dependency "redlock", "~> 2.0"
|
|
29
29
|
spec.add_dependency "rediconn", "~> 0.1.2"
|
|
30
|
-
#
|
|
31
|
-
|
|
30
|
+
# Workaround approved October 2, 2026: rediconn requires "redis" at load time but only declares
|
|
31
|
+
# it as a development dependency, so cable_room declares it instead. Drop this once rediconn
|
|
32
|
+
# does. The Bus needs >= 5: redis-rb 4 holds its client lock for the whole SUBSCRIBE loop, so
|
|
33
|
+
# another thread can't add a channel to a live session. The ceiling matches ActionCable 8.1's
|
|
34
|
+
# Redis adapter (7.2's is "< 6"), so apps on that adapter still resolve.
|
|
35
|
+
spec.add_dependency "redis", ">= 5", "< 7"
|
|
32
36
|
|
|
33
37
|
spec.add_development_dependency 'rspec', '~> 3'
|
|
34
38
|
end
|
data/lib/cable_room/bus.rb
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
require 'set'
|
|
2
|
+
require 'redis'
|
|
2
3
|
require 'active_support/json'
|
|
3
4
|
|
|
4
5
|
module CableRoom
|
|
5
|
-
# The Redis bus
|
|
6
|
-
#
|
|
7
|
-
# host. It's a thin layer over the `CABLEROOM_*` Redis connection (see `CableRoom.redis_pool`)
|
|
8
|
-
# that adds three things:
|
|
6
|
+
# The Redis pub/sub bus for member-to-room traffic. It's a thin layer over the `CABLEROOM_*`
|
|
7
|
+
# Redis connection (see `CableRoom.redis_pool`) that adds three things:
|
|
9
8
|
#
|
|
10
|
-
# * One place that knows the channel
|
|
9
|
+
# * One place that knows the channel names (`cr:{room_port_key}:in` and friends, and
|
|
10
|
+
# `cr:provision`).
|
|
11
11
|
# * JSON encoding on the way in and decoding on the way out, using the same coder the room
|
|
12
12
|
# ports already use for ActionCable streams, so a bad payload fails at the publisher.
|
|
13
13
|
# * A single subscriber thread that owns a dedicated pub/sub connection and hands each
|
|
@@ -18,6 +18,12 @@ module CableRoom
|
|
|
18
18
|
# blocks until Redis confirms the subscription, so anything published after it returns is
|
|
19
19
|
# delivered.
|
|
20
20
|
#
|
|
21
|
+
# Reconnects: when the pub/sub connection drops, the subscriber thread reports the error and
|
|
22
|
+
# opens a new session for every channel it still has a handler for, backing off from 0.1 s up
|
|
23
|
+
# to 5 s between attempts. Redis pub/sub doesn't queue, so anything published while the
|
|
24
|
+
# session is down is lost. Callers that can't afford that need their own recovery (members
|
|
25
|
+
# re-announce on their ping, for example).
|
|
26
|
+
#
|
|
21
27
|
# Handlers run on the subscriber thread. Keep them short (hand the work to a queue) and never
|
|
22
28
|
# block in them waiting on the bus itself. A handler is allowed to call `subscribe` and
|
|
23
29
|
# `unsubscribe`; those calls return without waiting when made from the subscriber thread,
|
|
@@ -32,42 +38,32 @@ module CableRoom
|
|
|
32
38
|
KEY_PREFIX = "cr".freeze
|
|
33
39
|
DEFAULT_TIMEOUT = 5
|
|
34
40
|
|
|
41
|
+
INITIAL_BACKOFF = 0.1
|
|
42
|
+
MAX_BACKOFF = 5
|
|
43
|
+
|
|
35
44
|
class << self
|
|
36
45
|
# Pub/sub channel that a room's host listens on for member-to-room messages. A room's main
|
|
37
|
-
# inbound port has no suffix (`cr:{
|
|
38
|
-
# (`cr:{
|
|
39
|
-
#
|
|
40
|
-
|
|
41
|
-
|
|
46
|
+
# inbound port has no suffix (`cr:{room_port_key}:in`); a custom inbound port gets one
|
|
47
|
+
# (`cr:{room_port_key}:in:{port}`), so every channel a room listens on shares one prefix.
|
|
48
|
+
# The room part is the room's own stream name, so an app's `broadcasting_for` override
|
|
49
|
+
# (PandaPal's tenant prefix) keeps two tenants' rooms apart here too.
|
|
50
|
+
def inbound_channel(room_class, room_key, port = nil)
|
|
51
|
+
channel = room_channel(room_class.room_port_key(room_key), "in")
|
|
42
52
|
port.nil? ? channel : "#{channel}:#{port}"
|
|
43
53
|
end
|
|
44
54
|
|
|
45
|
-
#
|
|
55
|
+
# The one process-wide channel `create: true` members ask for rooms on, and every rooms
|
|
56
|
+
# Host's Placement listens to (see Host::Placement). It isn't per tenant: the tenant rides in
|
|
57
|
+
# the request, and the Host that claims the room switches into it.
|
|
46
58
|
def provision_channel
|
|
47
59
|
"#{KEY_PREFIX}:provision"
|
|
48
60
|
end
|
|
49
61
|
|
|
50
|
-
# Redis list where a migrating room's old host parks inbound messages until the new host
|
|
51
|
-
# has adopted the room.
|
|
52
|
-
def handoff_list(room_key)
|
|
53
|
-
room_channel(room_key, "handoff")
|
|
54
|
-
end
|
|
55
|
-
|
|
56
|
-
# Redis string (with a TTL) holding the frozen room's snapshot during a migration.
|
|
57
|
-
def snapshot_key(room_key)
|
|
58
|
-
room_channel(room_key, "snapshot")
|
|
59
|
-
end
|
|
60
|
-
|
|
61
|
-
# Pub/sub channel the adopting host publishes on once it has taken over the room.
|
|
62
|
-
def adopted_channel(room_key)
|
|
63
|
-
room_channel(room_key, "adopted")
|
|
64
|
-
end
|
|
65
|
-
|
|
66
62
|
private
|
|
67
63
|
|
|
68
|
-
def room_channel(
|
|
69
|
-
key =
|
|
70
|
-
raise ArgumentError, "
|
|
64
|
+
def room_channel(port_key, suffix)
|
|
65
|
+
key = port_key.to_s
|
|
66
|
+
raise ArgumentError, "room_port_key can't be blank" if key.empty?
|
|
71
67
|
"#{KEY_PREFIX}:#{key}:#{suffix}"
|
|
72
68
|
end
|
|
73
69
|
end
|
|
@@ -87,7 +83,7 @@ module CableRoom
|
|
|
87
83
|
@subscriber_redis = nil
|
|
88
84
|
end
|
|
89
85
|
|
|
90
|
-
# ---- Publisher
|
|
86
|
+
# ---- Publisher --------------------------------------------------------------------------
|
|
91
87
|
|
|
92
88
|
# Publish a JSON-encodable payload. Returns the number of subscribers that received it.
|
|
93
89
|
def publish(channel, payload)
|
|
@@ -95,45 +91,6 @@ module CableRoom
|
|
|
95
91
|
redis { |r| r.publish(channel, encoded) }
|
|
96
92
|
end
|
|
97
93
|
|
|
98
|
-
# Append one or more payloads to the end of a list. Returns the list's new length. With `ttl`
|
|
99
|
-
# (seconds or a Duration) the list's expiry is reset in the same round trip, so a list nobody
|
|
100
|
-
# deletes (a host that died mid-handoff) still goes away on its own.
|
|
101
|
-
def rpush(key, *payloads, ttl: nil)
|
|
102
|
-
encoded = payloads.map { |payload| encode(payload) }
|
|
103
|
-
return redis { |r| r.rpush(key, encoded) } unless ttl
|
|
104
|
-
|
|
105
|
-
seconds = ttl.to_i
|
|
106
|
-
raise ArgumentError, "ttl must be a positive number of seconds" unless seconds.positive?
|
|
107
|
-
length, = redis { |r| r.pipelined { |p| p.rpush(key, encoded); p.expire(key, seconds) } }
|
|
108
|
-
length
|
|
109
|
-
end
|
|
110
|
-
|
|
111
|
-
# The whole list, oldest first, decoded.
|
|
112
|
-
def lrange(key)
|
|
113
|
-
redis { |r| r.lrange(key, 0, -1) }.map { |raw| decode(raw) }
|
|
114
|
-
end
|
|
115
|
-
|
|
116
|
-
# Delete keys. Returns how many existed.
|
|
117
|
-
def del(*keys)
|
|
118
|
-
redis { |r| r.del(*keys) }
|
|
119
|
-
end
|
|
120
|
-
|
|
121
|
-
# Store a payload under a key that expires after `ttl` (seconds or an
|
|
122
|
-
# ActiveSupport::Duration). Migration uses this for snapshots so a dead handoff cleans
|
|
123
|
-
# itself up.
|
|
124
|
-
def set(key, payload, ttl:)
|
|
125
|
-
seconds = ttl.to_i
|
|
126
|
-
raise ArgumentError, "ttl must be a positive number of seconds" unless seconds.positive?
|
|
127
|
-
encoded = encode(payload)
|
|
128
|
-
redis { |r| r.set(key, encoded, ex: seconds) }
|
|
129
|
-
end
|
|
130
|
-
|
|
131
|
-
# The decoded payload stored under a key, or nil when there isn't one.
|
|
132
|
-
def get(key)
|
|
133
|
-
raw = redis { |r| r.get(key) }
|
|
134
|
-
raw.nil? ? nil : decode(raw)
|
|
135
|
-
end
|
|
136
|
-
|
|
137
94
|
# ---- Subscriber -------------------------------------------------------------------------
|
|
138
95
|
|
|
139
96
|
# Register `handler` for `channel` and block until Redis confirms the subscription. The
|
|
@@ -169,11 +126,13 @@ module CableRoom
|
|
|
169
126
|
|
|
170
127
|
# Drop the handler for `channel` and block until Redis confirms the unsubscribe. When no
|
|
171
128
|
# channels are left the subscriber thread exits, and this also waits for that. Returns
|
|
172
|
-
# false if the channel wasn't subscribed.
|
|
173
|
-
|
|
129
|
+
# false if the channel wasn't subscribed. With `handler:`, only unsubscribes while that
|
|
130
|
+
# handler is still the one registered (a later `subscribe` may have replaced it).
|
|
131
|
+
def unsubscribe(channel, handler: nil, timeout: DEFAULT_TIMEOUT)
|
|
174
132
|
channel = channel.to_s
|
|
175
133
|
|
|
176
134
|
@mutex.synchronize do
|
|
135
|
+
return false if handler && !@handlers[channel].equal?(handler)
|
|
177
136
|
return false unless @handlers.delete(channel)
|
|
178
137
|
reconcile
|
|
179
138
|
|
|
@@ -253,7 +212,7 @@ module CableRoom
|
|
|
253
212
|
# channel; a session ends when the last channel is unsubscribed (normal) or the connection
|
|
254
213
|
# fails (reported, then retried with backoff). The loop exits once nothing is wanted.
|
|
255
214
|
def subscriber_loop
|
|
256
|
-
backoff =
|
|
215
|
+
backoff = INITIAL_BACKOFF
|
|
257
216
|
loop do
|
|
258
217
|
channels = @mutex.synchronize do
|
|
259
218
|
if @handlers.empty?
|
|
@@ -266,11 +225,11 @@ module CableRoom
|
|
|
266
225
|
|
|
267
226
|
begin
|
|
268
227
|
run_session(channels)
|
|
269
|
-
backoff =
|
|
228
|
+
backoff = INITIAL_BACKOFF
|
|
270
229
|
rescue StandardError => e
|
|
271
230
|
CableRoom.report_error(e, bus: self, channels: channels)
|
|
272
231
|
sleep backoff
|
|
273
|
-
backoff = [backoff * 2,
|
|
232
|
+
backoff = [backoff * 2, MAX_BACKOFF].min
|
|
274
233
|
end
|
|
275
234
|
end
|
|
276
235
|
end
|
|
@@ -330,6 +289,10 @@ module CableRoom
|
|
|
330
289
|
# Bring the server-side subscription set in line with the handlers we hold. Only possible
|
|
331
290
|
# while a session is live; before that, the thread's first confirmation calls this to pick
|
|
332
291
|
# up anything added or removed while it was connecting. Caller must hold @mutex.
|
|
292
|
+
#
|
|
293
|
+
# The writes below happen under @mutex, which is safe because they only queue a command on
|
|
294
|
+
# the socket: nothing here waits on a reply, and the subscriber thread never needs anything
|
|
295
|
+
# but @mutex to make progress.
|
|
333
296
|
def reconcile
|
|
334
297
|
return unless @session_live
|
|
335
298
|
|
|
@@ -342,6 +305,13 @@ module CableRoom
|
|
|
342
305
|
@requested.delete(channel)
|
|
343
306
|
@subscriber_redis.unsubscribe(channel)
|
|
344
307
|
end
|
|
308
|
+
rescue Redis::BaseConnectionError, RedisClient::ConnectionError
|
|
309
|
+
# The socket died under us. Stop writing to it and let the subscriber thread notice, report
|
|
310
|
+
# it, and open a new session for every channel still wanted. A caller waiting in
|
|
311
|
+
# `subscribe` rides on that session's confirmation (or times out) rather than seeing the
|
|
312
|
+
# connection error for a subscription the bus is about to retry anyway.
|
|
313
|
+
@session_live = false
|
|
314
|
+
@subscriber_redis = nil
|
|
345
315
|
end
|
|
346
316
|
|
|
347
317
|
# Decode and hand one message to its handler. A handler that raises is reported and the
|
data/lib/cable_room/cli.rb
CHANGED
|
@@ -6,38 +6,61 @@ require "logger"
|
|
|
6
6
|
require_relative "version"
|
|
7
7
|
|
|
8
8
|
module CableRoom
|
|
9
|
-
# The `cable_room` command. Its one command, `server`, is the
|
|
10
|
-
#
|
|
11
|
-
# then shut every room down
|
|
9
|
+
# The `cable_room` command. Its one command, `server`, is the rooms process for
|
|
10
|
+
# `room_host = :remote`: boot the Rails app, become this process's rooms Host, run until SIGTERM
|
|
11
|
+
# or SIGINT, then shut every room down (members get room_closed). Rooms aren't migrated
|
|
12
|
+
# anywhere on the way out.
|
|
12
13
|
#
|
|
13
|
-
# It
|
|
14
|
-
#
|
|
14
|
+
# It refuses to run in :inline, where the web processes host rooms themselves and a server here
|
|
15
|
+
# would only race them for locks.
|
|
15
16
|
#
|
|
16
|
-
#
|
|
17
|
-
# from the current directory, or from `--require PATH` (an app directory or a boot file).
|
|
18
|
-
# Bundler is already set up because the command runs under `bundle exec`. Nothing from the gem
|
|
19
|
-
# beyond this file and `version` is loaded before the app boots, so the app's Gemfile decides
|
|
20
|
-
# when the Railtie loads, the same as in a web process.
|
|
17
|
+
# What the rooms process needs from the app:
|
|
21
18
|
#
|
|
22
|
-
#
|
|
23
|
-
#
|
|
24
|
-
#
|
|
25
|
-
#
|
|
26
|
-
#
|
|
27
|
-
#
|
|
19
|
+
# * The app's own config/cable.yml with a Redis adapter. Rooms still talk to members with
|
|
20
|
+
# `ActionCable.server.broadcast`, which reaches web processes only through a shared
|
|
21
|
+
# pubsub; the async and inline adapters keep broadcasts inside this process.
|
|
22
|
+
# * The same CABLEROOM_* Redis settings as the web processes. Members publish to rooms on
|
|
23
|
+
# the Bus there, and room locks live there.
|
|
24
|
+
# * A tenant for every room it starts, in a multi-tenant app. Provision requests carry the
|
|
25
|
+
# member's tenant; a room started here by hand needs `MyRoom.ensure(key, tenant: ...)`.
|
|
28
26
|
#
|
|
29
|
-
#
|
|
30
|
-
#
|
|
27
|
+
# Rooms start here through provisioning: a `create: true` join on the web side publishes a
|
|
28
|
+
# provision request on the Bus, and each rooms Host's Placement (started by `Host.start!`)
|
|
29
|
+
# waits in proportion to its load, then races for the room's lock (see Host::Placement).
|
|
30
|
+
#
|
|
31
|
+
# Boot works the way sidekiq and good_job do it: `require` the app's config/environment.rb from
|
|
32
|
+
# the current directory, or from `--require PATH` (an app directory or a boot file). Bundler is
|
|
33
|
+
# already set up because the command runs under `bundle exec`. Nothing from the gem beyond this
|
|
34
|
+
# file and `version` loads before the app boots, so the app's Gemfile decides when the Railtie
|
|
35
|
+
# loads, the same as in a web process. Nothing subscribes to Redis or starts a thread until
|
|
36
|
+
# the app has booted and been eager-loaded, so a supervisor can fork after that point.
|
|
37
|
+
#
|
|
38
|
+
# `--workers N` picks the process model. With N = 1 this process hosts rooms itself, as a
|
|
39
|
+
# single rooms process always has; the container's restart policy is its supervisor. With
|
|
40
|
+
# N > 1 (the default is the machine's processor count) the app is booted once, here, and then
|
|
41
|
+
# `Host::Supervisor` forks N children that each host rooms with their own Bus connection;
|
|
42
|
+
# this parent only watches them, replaces one that dies, and relays SIGTERM and SIGINT. The
|
|
43
|
+
# parent never builds a Host, subscribes the Bus, or keeps a Redis or database connection:
|
|
44
|
+
# the children build their own after the fork, so no two processes ever share a socket or a
|
|
45
|
+
# thread.
|
|
46
|
+
#
|
|
47
|
+
# `run` returns an exit status rather than exiting, so specs can drive it in-process.
|
|
31
48
|
class CLI
|
|
32
49
|
STOP_SIGNALS = %w[TERM INT].freeze
|
|
33
50
|
|
|
51
|
+
# How long a room's startup waits for Redis to confirm each Bus subscription here. A healthy
|
|
52
|
+
# Redis confirms in milliseconds; this only bounds a sick one, and failing sooner frees the
|
|
53
|
+
# room's lock sooner. The web-side default (Bus::DEFAULT_TIMEOUT, 5 s) is sized for a member
|
|
54
|
+
# join that can afford to wait.
|
|
55
|
+
SUBSCRIBE_TIMEOUT = 2
|
|
56
|
+
|
|
34
57
|
attr_reader :options, :command
|
|
35
58
|
|
|
36
59
|
def initialize(argv, stdout: $stdout, stderr: $stderr)
|
|
37
60
|
@argv = argv.dup
|
|
38
61
|
@stdout = stdout
|
|
39
62
|
@stderr = stderr
|
|
40
|
-
@options = {
|
|
63
|
+
@options = { require: Dir.pwd, workers: nil }
|
|
41
64
|
end
|
|
42
65
|
|
|
43
66
|
# Parse and run. Returns the process exit status.
|
|
@@ -59,7 +82,7 @@ module CableRoom
|
|
|
59
82
|
1
|
|
60
83
|
end
|
|
61
84
|
|
|
62
|
-
# How many worker processes `server` runs: `--workers`, or one per
|
|
85
|
+
# How many worker processes `server` runs: `--workers`, or one per processor
|
|
63
86
|
def workers
|
|
64
87
|
options[:workers] || Etc.nprocessors
|
|
65
88
|
end
|
|
@@ -70,14 +93,15 @@ module CableRoom
|
|
|
70
93
|
@parser ||= OptionParser.new do |o|
|
|
71
94
|
o.banner = "Usage: cable_room server [options]"
|
|
72
95
|
o.separator ""
|
|
73
|
-
o.separator "
|
|
74
|
-
o.separator "
|
|
96
|
+
o.separator "Hosts CableRoom rooms for a Rails app with room_host = :remote: boots the app in"
|
|
97
|
+
o.separator "the current directory and runs until SIGTERM or SIGINT. Needs the app's"
|
|
98
|
+
o.separator "config/cable.yml (a Redis adapter) and the same CABLEROOM_* Redis as the web."
|
|
75
99
|
o.separator ""
|
|
76
100
|
o.on("-r", "--require PATH", "App directory (with config/environment.rb) or a boot file to require") do |path|
|
|
77
101
|
@options[:require] = path
|
|
78
102
|
end
|
|
79
103
|
o.on("-w", "--workers N", Integer,
|
|
80
|
-
"Worker processes to fork (default: one per
|
|
104
|
+
"Worker processes to fork (default: one per processor, #{Etc.nprocessors} here). 1 runs in this process.") do |n|
|
|
81
105
|
@options[:workers] = n
|
|
82
106
|
end
|
|
83
107
|
o.on("-v", "--version", "Print the version and exit") do
|
|
@@ -106,31 +130,49 @@ module CableRoom
|
|
|
106
130
|
|
|
107
131
|
boot_app!
|
|
108
132
|
|
|
109
|
-
unless CableRoom.
|
|
110
|
-
@stderr.puts "cable_room server: CableRoom.
|
|
111
|
-
"
|
|
112
|
-
"themselves). Set CABLE_ROOM_HOST=remote or `
|
|
133
|
+
unless CableRoom.remote?
|
|
134
|
+
@stderr.puts "cable_room server: CableRoom.room_host is #{CableRoom.room_host.inspect}, but a rooms " \
|
|
135
|
+
"process only makes sense with :remote (in :inline the web processes host rooms " \
|
|
136
|
+
"themselves). Set CABLE_ROOM_HOST=remote or `CableRoom.room_host = :remote` and try again."
|
|
113
137
|
return 1
|
|
114
138
|
end
|
|
115
139
|
|
|
140
|
+
warn_about_local_cable_adapter
|
|
116
141
|
return run_host if workers == 1
|
|
117
142
|
|
|
118
143
|
supervise_workers
|
|
119
144
|
end
|
|
120
145
|
|
|
121
|
-
# Load the app: a directory means its config/environment.rb, anything else is required as-is
|
|
146
|
+
# Load the app: a directory means its config/environment.rb, anything else is required as-is.
|
|
147
|
+
# Then eager-load it, as production does anyway, so every room class is loaded before any
|
|
148
|
+
# room starts (and before a supervisor forks).
|
|
122
149
|
def boot_app!
|
|
123
150
|
path = File.expand_path(options[:require])
|
|
124
151
|
path = File.join(path, "config", "environment.rb") if File.directory?(path)
|
|
125
152
|
require path
|
|
153
|
+
|
|
154
|
+
Rails.application.eager_load! if defined?(Rails) && Rails.application && !Rails.application.config.eager_load
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
def warn_about_local_cable_adapter
|
|
158
|
+
cable = ActionCable.server.config.cable || {}
|
|
159
|
+
adapter = (cable[:adapter] || cable["adapter"]).to_s
|
|
160
|
+
return unless %w[async inline test].include?(adapter)
|
|
161
|
+
|
|
162
|
+
@stderr.puts "cable_room server: warning: ActionCable's adapter is #{adapter.inspect}, so room broadcasts " \
|
|
163
|
+
"never leave this process and members won't hear their rooms. Use the app's Redis cable.yml."
|
|
126
164
|
end
|
|
127
165
|
|
|
128
166
|
# Fork `workers` children that each run `run_host`, and watch them until a stop signal has
|
|
129
167
|
# come and gone. This parent hosts nothing itself (see the class comment for why).
|
|
130
168
|
def supervise_workers
|
|
131
169
|
Process.setproctitle("cable_room server: supervisor (#{workers} workers)")
|
|
132
|
-
|
|
133
|
-
|
|
170
|
+
# Fail now, once, if the CABLEROOM_* Redis is unreachable, rather than in N workers that
|
|
171
|
+
# would each die and come back with backoff. Then let go of what the ping (or the app's
|
|
172
|
+
# boot) opened, so the children inherit no Redis or database sockets.
|
|
173
|
+
CableRoom.redis(&:ping)
|
|
174
|
+
release_connections_before_fork
|
|
175
|
+
say "supervising #{workers} workers (pid #{Process.pid}, room_host=#{CableRoom.room_host})"
|
|
134
176
|
|
|
135
177
|
supervisor = CableRoom::Host::Supervisor.new(count: workers, logger: supervisor_logger) do |index|
|
|
136
178
|
run_host(worker: index)
|
|
@@ -140,6 +182,15 @@ module CableRoom
|
|
|
140
182
|
status
|
|
141
183
|
end
|
|
142
184
|
|
|
185
|
+
# Close the parent's idle pooled Redis connections and its ActiveRecord connections. The
|
|
186
|
+
# children would drop their copies anyway (see `CableRoom.after_fork!`); closing them here
|
|
187
|
+
# means there are no copies to drop, and the parent, which only waits on children from now
|
|
188
|
+
# on, holds nothing it won't use. Both pools reconnect lazily if anything asks again.
|
|
189
|
+
def release_connections_before_fork
|
|
190
|
+
CableRoom.redis_pool.reload(&:close)
|
|
191
|
+
ActiveRecord::Base.connection_handler.clear_all_connections!(:all) if defined?(ActiveRecord::Base)
|
|
192
|
+
end
|
|
193
|
+
|
|
143
194
|
# Logs from the supervisor go to the same place as the CLI's own messages, in the same style.
|
|
144
195
|
# Unbuffered, so a "forked worker" line shows up when it happens, not when the process exits.
|
|
145
196
|
def supervisor_logger
|
|
@@ -153,8 +204,8 @@ module CableRoom
|
|
|
153
204
|
# life of a single-process server and of each forked worker (`worker` is the worker's index,
|
|
154
205
|
# nil when there's no supervisor). Returns the exit status.
|
|
155
206
|
def run_host(worker: nil)
|
|
156
|
-
# Trap before
|
|
157
|
-
#
|
|
207
|
+
# Trap before anything slow, so a signal that lands while rooms are still coming up (the
|
|
208
|
+
# supervisor relaying a SIGTERM that arrived mid-boot, say) is kept, not fatal
|
|
158
209
|
signals = trap_stop_signals
|
|
159
210
|
|
|
160
211
|
if worker
|
|
@@ -162,41 +213,29 @@ module CableRoom
|
|
|
162
213
|
tag_logger("cable_room worker #{worker}")
|
|
163
214
|
end
|
|
164
215
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
216
|
+
# Fail now, not at the first room, if the CABLEROOM_* Redis is unreachable
|
|
217
|
+
CableRoom.redis(&:ping)
|
|
218
|
+
|
|
219
|
+
host = CableRoom::Host.start!(subscribe_timeout: SUBSCRIBE_TIMEOUT)
|
|
220
|
+
say "hosting rooms (pid #{Process.pid}, room_host=#{CableRoom.room_host})", worker: worker
|
|
168
221
|
|
|
169
|
-
signal =
|
|
170
|
-
|
|
222
|
+
signal = signals.pop
|
|
223
|
+
say "got SIG#{signal}, shutting down #{host.open_rooms_count} room(s)", worker: worker
|
|
224
|
+
# A second signal from here on gets the previous handler (the default one kills at once)
|
|
225
|
+
restore_signal_handlers
|
|
226
|
+
host.shutdown!
|
|
227
|
+
CableRoom.bus.shutdown
|
|
171
228
|
say "shut down", worker: worker
|
|
172
229
|
0
|
|
173
230
|
ensure
|
|
174
|
-
|
|
175
|
-
end
|
|
176
|
-
|
|
177
|
-
# A stop signal with rooms open is a planned shutdown: hand the rooms to peer hosts before
|
|
178
|
-
# leaving (`Host#drain!`, bounded by `drain_timeout`; a room nobody adopts closes with
|
|
179
|
-
# `room_closed`). With nothing open there is nothing to move, so just shut down. Both are
|
|
180
|
-
# idempotent, so the one SIGTERM a supervisor relays and a later `at_exit` don't collide.
|
|
181
|
-
def stop_host(host, signal, worker: nil)
|
|
182
|
-
rooms = host.rooms.size
|
|
183
|
-
if rooms.zero?
|
|
184
|
-
say "got SIG#{signal}, shutting down 0 room(s)", worker: worker
|
|
185
|
-
host.shutdown!
|
|
186
|
-
return
|
|
187
|
-
end
|
|
188
|
-
|
|
189
|
-
say "got SIG#{signal}, migrating #{rooms} room(s) to peer hosts", worker: worker
|
|
190
|
-
result = host.drain!(reason: "SIG#{signal}")
|
|
191
|
-
say "drained: #{result.migrated.size} room(s) migrated, #{result.closed.size} closed, " \
|
|
192
|
-
"in #{result.duration.round(1)}s", worker: worker
|
|
231
|
+
restore_signal_handlers
|
|
193
232
|
end
|
|
194
233
|
|
|
195
234
|
# Every room on this Host logs through `ActionCable.server.logger` (Host#logger); give that
|
|
196
235
|
# logger a tag naming the worker, so lines from the N workers sharing one stdout can be told
|
|
197
236
|
# apart. `tagged` without a block returns a logger whose tags apply on every thread, which is
|
|
198
237
|
# what we need: rooms log from the worker pool, not from this thread. A logger without tags
|
|
199
|
-
# (a plain Logger,
|
|
238
|
+
# (a plain Logger, say) is left as it is.
|
|
200
239
|
def tag_logger(tag)
|
|
201
240
|
logger = ActionCable.server.logger
|
|
202
241
|
return unless logger.respond_to?(:tagged)
|
|
@@ -205,33 +244,23 @@ module CableRoom
|
|
|
205
244
|
ActionCable.server.config.logger = tagged if tagged.respond_to?(:info)
|
|
206
245
|
end
|
|
207
246
|
|
|
208
|
-
def say(message, worker: nil)
|
|
209
|
-
prefix = worker ? "cable_room server (worker #{worker})" : "cable_room server"
|
|
210
|
-
@stdout.puts "#{prefix}: #{message}"
|
|
211
|
-
@stdout.flush
|
|
212
|
-
end
|
|
213
|
-
|
|
214
247
|
# Signal handlers may only do trivial work, so each one just drops the signal's name on a
|
|
215
|
-
# queue for the main thread to pick up
|
|
248
|
+
# queue for the main thread to pick up
|
|
216
249
|
def trap_stop_signals
|
|
217
250
|
signals = Queue.new
|
|
218
|
-
@
|
|
251
|
+
@previous_handlers = STOP_SIGNALS.to_h { |sig| [sig, trap(sig) { signals << sig }] }
|
|
219
252
|
signals
|
|
220
253
|
end
|
|
221
254
|
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
def wait_for_stop_signal(signals)
|
|
226
|
-
loop do
|
|
227
|
-
signal = signals.pop(timeout: 1)
|
|
228
|
-
return signal if signal
|
|
229
|
-
end
|
|
255
|
+
def restore_signal_handlers
|
|
256
|
+
@previous_handlers&.each { |sig, handler| trap(sig, handler) }
|
|
257
|
+
@previous_handlers = nil
|
|
230
258
|
end
|
|
231
259
|
|
|
232
|
-
def
|
|
233
|
-
|
|
234
|
-
@
|
|
260
|
+
def say(message, worker: nil)
|
|
261
|
+
prefix = worker ? "cable_room server (worker #{worker})" : "cable_room server"
|
|
262
|
+
@stdout.puts "#{prefix}: #{message}"
|
|
263
|
+
@stdout.flush
|
|
235
264
|
end
|
|
236
265
|
end
|
|
237
266
|
end
|