cable_room 0.7.0.beta3 → 0.8.0.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +0 -1287
- data/cable_room.gemspec +2 -5
- data/lib/cable_room/host/action_cable_inbound.rb +35 -0
- data/lib/cable_room/host/runner.rb +109 -391
- data/lib/cable_room/host/worker_pool.rb +27 -40
- data/lib/cable_room/host.rb +18 -364
- data/lib/cable_room/ports.rb +9 -19
- data/lib/cable_room/railtie.rb +11 -3
- data/lib/cable_room/room/base.rb +41 -40
- data/lib/cable_room/room/callbacks.rb +0 -22
- data/lib/cable_room/room/host_adapter.rb +4 -9
- data/lib/cable_room/room/lifecycle.rb +6 -23
- data/lib/cable_room/room/port_management.rb +0 -50
- data/lib/cable_room/room/reaping.rb +0 -33
- data/lib/cable_room/room/user_management.rb +0 -27
- data/lib/cable_room/room.rb +1 -4
- data/lib/cable_room/room_member.rb +84 -275
- data/lib/cable_room/room_proxy_channel.rb +2 -13
- data/lib/cable_room/version.rb +1 -1
- data/lib/cable_room.rb +0 -55
- metadata +7 -21
- data/CHANGELOG.md +0 -122
- data/exe/cable_room +0 -8
- data/lib/cable_room/broadcaster.rb +0 -116
- data/lib/cable_room/bus.rb +0 -372
- data/lib/cable_room/cli.rb +0 -237
- data/lib/cable_room/config.rb +0 -112
- data/lib/cable_room/host/bus_inbound.rb +0 -36
- data/lib/cable_room/host/supervisor.rb +0 -275
- data/lib/cable_room/membership_store.rb +0 -105
- data/lib/cable_room/migration.rb +0 -586
- data/lib/cable_room/placement.rb +0 -260
- data/lib/cable_room/room/snapshotting.rb +0 -82
- data/lib/cable_room/room_harness.rb +0 -168
- data/lib/cable_room/snapshot.rb +0 -136
|
@@ -1,50 +1,37 @@
|
|
|
1
|
-
require 'concurrent'
|
|
2
|
-
|
|
3
1
|
module CableRoom
|
|
4
2
|
class Host
|
|
5
|
-
# The thread pool every room's work runs on.
|
|
6
|
-
#
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
#
|
|
10
|
-
#
|
|
11
|
-
#
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
@executor = Concurrent::ThreadPoolExecutor.new(
|
|
20
|
-
name: "CableRoom",
|
|
21
|
-
min_threads: 1,
|
|
22
|
-
max_threads: max_size,
|
|
23
|
-
max_queue: 0,
|
|
24
|
-
)
|
|
3
|
+
# The thread pool every room's work runs on. It's an ActionCable Worker so the `:work`
|
|
4
|
+
# callbacks Rails installs (the executor wrap, ActiveRecord log tagging) still apply to room
|
|
5
|
+
# work exactly as they did when rooms were channels.
|
|
6
|
+
#
|
|
7
|
+
# The "connection" passed around here is the room's Host::Runner. ActionCable's Worker was
|
|
8
|
+
# written for connections; rooms don't have one, but the runner fills the same role: it's
|
|
9
|
+
# the thing with a logger and an error reporter.
|
|
10
|
+
class WorkerPool < ActionCable::Server::Worker
|
|
11
|
+
set_callback :work, :around do |_, blk|
|
|
12
|
+
pconn = ActionCable::Server::Worker.connection
|
|
13
|
+
ActionCable::Server::Worker.connection = connection
|
|
14
|
+
blk.call
|
|
15
|
+
ensure
|
|
16
|
+
ActionCable::Server::Worker.connection = pconn
|
|
25
17
|
end
|
|
26
18
|
|
|
27
|
-
#
|
|
28
|
-
#
|
|
29
|
-
#
|
|
30
|
-
# Host::Runner in practice (see Runner#post_work); the else branch is a defensive fallback.
|
|
19
|
+
# ActionCable's Worker#invoke reduces every exception to a log line and a no-argument
|
|
20
|
+
# `handle_exception` call, which discards the error itself. Rooms run all of their work
|
|
21
|
+
# through here, so report it properly instead.
|
|
31
22
|
def invoke(receiver, method, *args, connection:, &block)
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
connection.report_work_error
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
23
|
+
work(connection) do
|
|
24
|
+
receiver.send method, *args, &block
|
|
25
|
+
rescue Exception => e
|
|
26
|
+
if connection.respond_to?(:report_work_error)
|
|
27
|
+
connection.report_work_error(e)
|
|
28
|
+
else
|
|
29
|
+
logger.error "There was an exception - #{e.class}(#{e.message})"
|
|
30
|
+
logger.error Array(e.backtrace).join("\n")
|
|
31
|
+
CableRoom.report_error(e, connection: connection)
|
|
32
|
+
end
|
|
40
33
|
end
|
|
41
34
|
end
|
|
42
|
-
|
|
43
|
-
private
|
|
44
|
-
|
|
45
|
-
def logger
|
|
46
|
-
ActionCable.server.logger
|
|
47
|
-
end
|
|
48
35
|
end
|
|
49
36
|
end
|
|
50
37
|
end
|
data/lib/cable_room/host.rb
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
require 'socket'
|
|
2
1
|
require_relative 'host/worker_pool'
|
|
3
|
-
require_relative 'host/
|
|
2
|
+
require_relative 'host/action_cable_inbound'
|
|
4
3
|
require_relative 'host/runner'
|
|
5
|
-
require_relative 'host/supervisor'
|
|
6
4
|
|
|
7
5
|
module CableRoom
|
|
8
6
|
# One Host per process. It owns every Room running here: the worker pool their work runs on,
|
|
@@ -15,112 +13,26 @@ module CableRoom
|
|
|
15
13
|
class Host
|
|
16
14
|
BEAT_INTERVAL = 5.seconds
|
|
17
15
|
|
|
18
|
-
# Raised when a process that doesn't host rooms asks for its Host. With `room_host = :remote`
|
|
19
|
-
# only `cable_room server` hosts rooms; a web process must never start one.
|
|
20
|
-
class NotHosting < StandardError; end
|
|
21
|
-
|
|
22
|
-
NOT_HOSTING_MESSAGE =
|
|
23
|
-
"This process doesn't host rooms: CableRoom.config.room_host is :remote, so rooms run " \
|
|
24
|
-
"under `cable_room server`. Start rooms there (or call CableRoom::Host.start! first if this " \
|
|
25
|
-
"process really should host them).".freeze
|
|
26
|
-
|
|
27
|
-
@start_mutex = Mutex.new
|
|
28
|
-
|
|
29
16
|
class << self
|
|
30
|
-
# The Host this process runs, or nil when it runs none (a web process in :remote, or an
|
|
31
|
-
# :inline process that hasn't started a room yet). Use this for introspection and cleanup,
|
|
32
|
-
# where "no rooms here" is a normal answer.
|
|
33
|
-
def current
|
|
34
|
-
@current
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
# The Host this process runs. In :inline every process hosts rooms, so the first caller
|
|
38
|
-
# builds it. In :remote only a process that declared itself a rooms host with `start!`
|
|
39
|
-
# (`cable_room server`) has one; anyone else gets NotHosting, so a room can never quietly
|
|
40
|
-
# start in the web tier because some code path reached for the Host.
|
|
41
17
|
def instance
|
|
42
|
-
|
|
43
|
-
end
|
|
44
|
-
|
|
45
|
-
# Make this process a rooms host. Builds the Host once and starts its provisioning listener,
|
|
46
|
-
# so this process hears `create: true` requests from then on; later calls return the same
|
|
47
|
-
# Host. `cable_room server` calls this explicitly; in :inline `instance` calls it for you.
|
|
48
|
-
def start!
|
|
49
|
-
@start_mutex.synchronize { @current ||= new.tap(&:start_placement) }
|
|
50
|
-
end
|
|
51
|
-
|
|
52
|
-
# Swap this process's Host for `host` (nil allowed) and return the previous one. Only specs
|
|
53
|
-
# that play two process roles in one Ruby process should need this (see RoomHostHarness):
|
|
54
|
-
# the "web" side has to see no Host while another Host runs the rooms.
|
|
55
|
-
def replace_current(host)
|
|
56
|
-
@start_mutex.synchronize { @current.tap { @current = host } }
|
|
57
|
-
end
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
# How many rooms `drain!` hands off at once before `drain_timeout`. Each handoff costs the
|
|
61
|
-
# adopting peer a full room restore (its startup chain, often database reads), so a host with
|
|
62
|
-
# hundreds of rooms must not offer them all in the same second; and each one holds a thread
|
|
63
|
-
# here for up to `handoff_timeout`. Four keeps a big host draining in well under the default
|
|
64
|
-
# ten minutes (a handoff normally takes well under a second) without swamping the fleet.
|
|
65
|
-
DRAIN_CONCURRENCY = 4
|
|
66
|
-
|
|
67
|
-
# What `drain!` returns: how the rooms ended up, and how long it took. `migrated` and
|
|
68
|
-
# `closed` are Migration objects (see their `state` and `room`).
|
|
69
|
-
DrainResult = Struct.new(:reason, :migrated, :closed, :duration, keyword_init: true) do
|
|
70
|
-
def rooms
|
|
71
|
-
migrated.size + closed.size
|
|
18
|
+
@instance ||= new
|
|
72
19
|
end
|
|
73
20
|
end
|
|
74
21
|
|
|
75
|
-
attr_reader :scheduler, :inbound
|
|
76
|
-
|
|
77
|
-
# True once this host has started shutting down for good and is handing its rooms off (see
|
|
78
|
-
# `drain!`). A draining host never claims a provision request, so it can't take on a room it
|
|
79
|
-
# would only have to hand off again.
|
|
80
|
-
attr_writer :draining
|
|
22
|
+
attr_reader :scheduler, :inbound
|
|
81
23
|
|
|
82
24
|
delegate :logger, to: :cable_server
|
|
83
25
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
# app that scales in through an ASG termination lifecycle hook completes the hook here
|
|
87
|
-
# (`complete-lifecycle-action`), once every room has moved or closed. Blocks run on the
|
|
88
|
-
# draining thread before `drain!` returns; one that raises is reported and doesn't stop
|
|
89
|
-
# the others.
|
|
90
|
-
def after_drain(&blk)
|
|
91
|
-
raise ArgumentError, "after_drain needs a block" unless blk
|
|
92
|
-
after_drain_callbacks << blk
|
|
93
|
-
blk
|
|
94
|
-
end
|
|
95
|
-
|
|
96
|
-
def after_drain_callbacks
|
|
97
|
-
@after_drain_callbacks ||= []
|
|
98
|
-
end
|
|
99
|
-
end
|
|
100
|
-
|
|
101
|
-
# `inbound` is how rooms receive messages: by default the process-wide Bus, so a room hears
|
|
102
|
-
# its members the same way whether it runs inside the web process or on a remote host.
|
|
103
|
-
def initialize(inbound: BusInbound.new)
|
|
26
|
+
# `inbound` is how rooms receive messages. Anything with ActionCableInbound's two methods works.
|
|
27
|
+
def initialize(inbound: ActionCableInbound.new)
|
|
104
28
|
@runners = Set.new
|
|
105
29
|
@monitor = Monitor.new
|
|
106
30
|
@inbound = inbound
|
|
107
|
-
@draining = false
|
|
108
|
-
@drain_result = nil
|
|
109
|
-
@drain_thread = nil
|
|
110
|
-
@migrations = []
|
|
111
31
|
@scheduler = Rufus::Scheduler.new
|
|
112
32
|
|
|
113
|
-
# A rooms host that leaves with rooms still open (a crash on the main thread, say; on a
|
|
114
|
-
# SIGTERM `cable_room server` has already drained by now, so this is a no-op) gives them
|
|
115
|
-
# the same chance to move as a planned shutdown would. In :inline there is no fleet of
|
|
116
|
-
# peers by design (see `drain!`), so exit closes rooms the way it always has.
|
|
117
33
|
at_exit do
|
|
118
34
|
logger.info "Shutting down CableRoom"
|
|
119
|
-
|
|
120
|
-
drain!(reason: "process exit")
|
|
121
|
-
else
|
|
122
|
-
shutdown!
|
|
123
|
-
end
|
|
35
|
+
shutdown!
|
|
124
36
|
end
|
|
125
37
|
|
|
126
38
|
# Every few seconds each room renews its Redlock and checks its watchdog
|
|
@@ -130,118 +42,20 @@ module CableRoom
|
|
|
130
42
|
end
|
|
131
43
|
|
|
132
44
|
def worker_pool
|
|
45
|
+
# TODO Pin Rooms to a specific thread so that they never have to worry about thread safety?
|
|
133
46
|
@worker_pool || @monitor.synchronize do
|
|
134
47
|
@worker_pool ||= WorkerPool.new(max_size: cable_server.config.worker_pool_size)
|
|
135
48
|
end
|
|
136
49
|
end
|
|
137
50
|
|
|
138
|
-
# Start
|
|
139
|
-
#
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
# (or a spec harness) can call it on a Host it owns without any member asking.
|
|
143
|
-
#
|
|
144
|
-
# If a snapshot of the room is on offer (another host is handing it off, see
|
|
145
|
-
# CableRoom::Migration), winning the lock means adopting it: the room comes back here with
|
|
146
|
-
# its members and state rather than starting fresh underneath the old host. The check is
|
|
147
|
-
# made after the lock is won, so it can't be raced. `handoff_only: true` (a handoff request)
|
|
148
|
-
# starts nothing when there is no snapshot to adopt — it has expired, or the old host took
|
|
149
|
-
# the room back — and lets the lock go.
|
|
150
|
-
def ensure_room(room_class, key = nil, handoff_only: false, tenant: nil)
|
|
151
|
-
lock_key = room_class.room_port_key(key)
|
|
152
|
-
lock_info = CableRoom.lock_manager.lock(lock_key, room_class::LOCK_DURATION.in_milliseconds)
|
|
153
|
-
return false unless lock_info
|
|
154
|
-
|
|
155
|
-
snapshot = bus.get(Bus.snapshot_key(lock_key))
|
|
156
|
-
return Migration.adopt(self, room_class, key, lock_info: lock_info, snapshot: snapshot) if snapshot
|
|
157
|
-
|
|
158
|
-
if handoff_only
|
|
159
|
-
logger.info "No snapshot to adopt for #{lock_key}; letting the lock go"
|
|
160
|
-
CableRoom.lock_manager.unlock(lock_info)
|
|
161
|
-
return false
|
|
162
|
-
end
|
|
163
|
-
|
|
164
|
-
start_room(
|
|
165
|
-
room_class,
|
|
166
|
-
key,
|
|
167
|
-
lock_info,
|
|
168
|
-
watchdog_interval: room_class::WATCH_DOG_INTERVAL,
|
|
169
|
-
lock_duration: room_class::LOCK_DURATION,
|
|
170
|
-
tenant: tenant,
|
|
171
|
-
)
|
|
172
|
-
|
|
173
|
-
true
|
|
174
|
-
rescue => e
|
|
175
|
-
CableRoom.lock_manager.unlock(lock_info) if lock_info
|
|
176
|
-
raise e
|
|
177
|
-
end
|
|
178
|
-
|
|
179
|
-
# Start running a room here. The caller (`ensure_room`) has already taken the room's Redlock;
|
|
180
|
-
# the runner renews it from now on and releases it when the room dies. `tenant:` comes from the
|
|
181
|
-
# member's provision request (see RoomMembership#initialize / #maybe_provision_room and
|
|
182
|
-
# Placement#claim), already defaulted there to the ambient tenant if the app didn't pass one
|
|
183
|
-
# explicitly -- nil past that point just means no Apartment multi-tenancy is in play.
|
|
184
|
-
def start_room(room_class, key, lock_info, watchdog_interval:, lock_duration:, tenant: nil)
|
|
185
|
-
runner = Runner.new(self, room_class, key, lock_info, watchdog_interval:, lock_duration:, tenant:)
|
|
51
|
+
# Start running a room here. The caller (Room::Base.ensure) has already taken the room's
|
|
52
|
+
# Redlock; the runner renews it from now on and releases it when the room dies.
|
|
53
|
+
def start_room(room_class, key, lock_info, watchdog_interval:, lock_duration:)
|
|
54
|
+
runner = Runner.new(self, room_class, key, lock_info, watchdog_interval:, lock_duration:)
|
|
186
55
|
runner.start!
|
|
187
56
|
runner
|
|
188
57
|
end
|
|
189
58
|
|
|
190
|
-
# Bring a room back from a CableRoom::Snapshot on this Host, the way a migration's adopter
|
|
191
|
-
# does. With no `lock_info`, takes the room's Redlock first and returns false if someone else
|
|
192
|
-
# holds it — the same rule as `ensure_room`. With one, the caller has already claimed the lock.
|
|
193
|
-
# Raises Snapshot::UnknownVersion or Snapshot::UnknownRoomClass before anything is built if
|
|
194
|
-
# the snapshot can't be used here. `hold_inbound: true` brings the room up frozen, holding
|
|
195
|
-
# its inbound until `Runner#thaw!` (see Runner#restore!).
|
|
196
|
-
#
|
|
197
|
-
# If the room fails to come up, the lock is released either way — the caller's too. A room
|
|
198
|
-
# that can't start here must not keep its key locked: the migration that handed it over
|
|
199
|
-
# takes the lock back and restarts it as soon as the lock is free.
|
|
200
|
-
def restore_room(snapshot, lock_info = nil, hold_inbound: false)
|
|
201
|
-
snapshot = Snapshot.validate!(snapshot)
|
|
202
|
-
room_class = Snapshot.room_class_for(snapshot)
|
|
203
|
-
key = Snapshot.key_for(snapshot)
|
|
204
|
-
lock_key = room_class.room_port_key(key)
|
|
205
|
-
|
|
206
|
-
if lock_info
|
|
207
|
-
unless lock_info[:resource] == lock_key
|
|
208
|
-
raise ArgumentError, "lock_info is for #{lock_info[:resource].inspect}, but the snapshot is #{lock_key}"
|
|
209
|
-
end
|
|
210
|
-
else
|
|
211
|
-
lock_info = CableRoom.lock_manager.lock(lock_key, room_class::LOCK_DURATION.in_milliseconds)
|
|
212
|
-
return false unless lock_info
|
|
213
|
-
end
|
|
214
|
-
|
|
215
|
-
runner = Runner.new(
|
|
216
|
-
self,
|
|
217
|
-
room_class,
|
|
218
|
-
key,
|
|
219
|
-
lock_info,
|
|
220
|
-
watchdog_interval: room_class::WATCH_DOG_INTERVAL,
|
|
221
|
-
lock_duration: room_class::LOCK_DURATION,
|
|
222
|
-
tenant: snapshot["tenant"],
|
|
223
|
-
)
|
|
224
|
-
runner.restore!(snapshot, hold_inbound: hold_inbound)
|
|
225
|
-
runner
|
|
226
|
-
rescue => e
|
|
227
|
-
# `restore!` releases the lock itself when the room fails to come up; this covers a failure
|
|
228
|
-
# before the runner exists
|
|
229
|
-
CableRoom.lock_manager.unlock(lock_info) if lock_info && runner.nil?
|
|
230
|
-
raise e
|
|
231
|
-
end
|
|
232
|
-
|
|
233
|
-
# A name for this host in migration traffic and instrumentation (`from_host`, `to_host`):
|
|
234
|
-
# machine, pid, and a few random characters, so two hosts in one process (a spec) differ too.
|
|
235
|
-
def id
|
|
236
|
-
@id ||= "#{Socket.gethostname}:#{Process.pid}:#{SecureRandom.hex(3)}"
|
|
237
|
-
end
|
|
238
|
-
|
|
239
|
-
# The Bus this host's rooms listen on, for the migration's list and key operations. Falls
|
|
240
|
-
# back to the process-wide Bus when `inbound` isn't Bus-backed.
|
|
241
|
-
def bus
|
|
242
|
-
inbound.respond_to?(:bus) ? inbound.bus : CableRoom.bus
|
|
243
|
-
end
|
|
244
|
-
|
|
245
59
|
def track(runner)
|
|
246
60
|
@monitor.synchronize do
|
|
247
61
|
raise "Cannot add Room after shutdown" if @shutdown
|
|
@@ -262,132 +76,25 @@ module CableRoom
|
|
|
262
76
|
runners.map(&:room)
|
|
263
77
|
end
|
|
264
78
|
|
|
265
|
-
# How many member ports are attached to rooms on this Host right now. This is the load
|
|
266
|
-
# number the design scales the rooms pool on, so an app can publish it as a gauge. It reads
|
|
267
|
-
# each room's port table from outside the room's thread; a count that's a moment stale is
|
|
268
|
-
# fine for a metric.
|
|
269
|
-
def open_ports
|
|
270
|
-
rooms.sum(&:open_port_count)
|
|
271
|
-
end
|
|
272
|
-
|
|
273
|
-
# How many rooms this Host runs right now. This is the load number provisioning weights its
|
|
274
|
-
# delay by: a host with more rooms waits longer before racing for a new one.
|
|
275
|
-
def open_rooms_count
|
|
276
|
-
@monitor.synchronize { @runners.size }
|
|
277
|
-
end
|
|
278
|
-
|
|
279
79
|
def each_runner(&blk)
|
|
280
80
|
runners.each(&blk)
|
|
281
81
|
end
|
|
282
82
|
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
# True once `drain!` has finished (every room moved or closed, and the host is shut down).
|
|
288
|
-
def drained?
|
|
289
|
-
!@drain_result.nil?
|
|
290
|
-
end
|
|
291
|
-
|
|
292
|
-
# The result of the drain, or nil until `drain!` has finished.
|
|
293
|
-
def drain_result
|
|
294
|
-
@drain_result
|
|
295
|
-
end
|
|
296
|
-
|
|
297
|
-
# Every CableRoom::Migration this host's drain has started, oldest room first. Each one's
|
|
298
|
-
# `state` says where it is; the list is complete once `drained?`.
|
|
299
|
-
def migrations
|
|
300
|
-
@monitor.synchronize { @migrations.dup }
|
|
83
|
+
# How many rooms this Host runs right now. Cheap enough to publish as a gauge.
|
|
84
|
+
def open_rooms_count
|
|
85
|
+
@monitor.synchronize { @runners.size }
|
|
301
86
|
end
|
|
302
87
|
|
|
303
|
-
#
|
|
304
|
-
#
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
# only have to move again), and instrument `host_draining.cable_room`.
|
|
308
|
-
# 2. Hand rooms off oldest first, DRAIN_CONCURRENCY at a time. A room nobody adopts within
|
|
309
|
-
# `handoff_timeout` closes with `room_closed`; so does one that can't be snapshotted.
|
|
310
|
-
# 3. When `drain_timeout` passes with rooms still waiting their turn, offer all of them at
|
|
311
|
-
# once — the same path, just no longer throttled — so the drain ends within about one
|
|
312
|
-
# more `handoff_timeout` whatever the fleet does.
|
|
313
|
-
# 4. `shutdown!`, run the `Host.after_drain` callbacks, and return a DrainResult.
|
|
314
|
-
#
|
|
315
|
-
# Blocks until done. Runs once: a second call (from another thread, or `at_exit` after
|
|
316
|
-
# `cable_room server` has already drained) waits for the first and returns its result.
|
|
317
|
-
# `drain_timeout` and `handoff_timeout` default to the config's; specs pass short ones.
|
|
318
|
-
#
|
|
319
|
-
# In :inline the peers are the other web processes hosting rooms, if any. The design gives
|
|
320
|
-
# :inline no fleet of its own, so the gem doesn't drain there on exit (see `at_exit`); an app
|
|
321
|
-
# running several inline processes may still call this from its own signal handling.
|
|
322
|
-
def drain!(reason:, drain_timeout: nil, handoff_timeout: nil, concurrency: DRAIN_CONCURRENCY)
|
|
323
|
-
other = @monitor.synchronize do
|
|
324
|
-
return @drain_result if @drain_result
|
|
325
|
-
@drain_thread.tap { @drain_thread ||= Thread.current }
|
|
326
|
-
end
|
|
327
|
-
if other
|
|
328
|
-
other.join unless other == Thread.current
|
|
329
|
-
return @drain_result
|
|
330
|
-
end
|
|
331
|
-
|
|
332
|
-
drain_timeout ||= CableRoom.config.drain_timeout
|
|
333
|
-
handoff_timeout ||= CableRoom.config.handoff_timeout
|
|
334
|
-
started = monotonic_now
|
|
335
|
-
self.draining = true
|
|
336
|
-
|
|
337
|
-
queued = runners.sort_by(&:started_at)
|
|
338
|
-
migrations = queued.map { |runner| Migration.new(self, runner, reason: reason, handoff_timeout: handoff_timeout) }
|
|
339
|
-
@monitor.synchronize { @migrations = migrations }
|
|
340
|
-
logger.info "Draining #{migrations.size} room(s): #{reason}"
|
|
341
|
-
|
|
342
|
-
payload = { host: self, rooms: queued.map(&:room), reason: reason }
|
|
343
|
-
ActiveSupport::Notifications.instrument("host_draining.cable_room", payload) do |event|
|
|
344
|
-
run_migrations(migrations, concurrency: concurrency, deadline: started + drain_timeout.to_f)
|
|
345
|
-
event[:migrated] = migrations.count(&:migrated?)
|
|
346
|
-
event[:closed] = migrations.count { |m| !m.migrated? }
|
|
347
|
-
end
|
|
348
|
-
|
|
349
|
-
shutdown!
|
|
350
|
-
|
|
351
|
-
result = DrainResult.new(
|
|
352
|
-
reason: reason,
|
|
353
|
-
migrated: migrations.select(&:migrated?),
|
|
354
|
-
closed: migrations.reject(&:migrated?),
|
|
355
|
-
duration: monotonic_now - started,
|
|
356
|
-
)
|
|
357
|
-
logger.info "Drained: #{result.migrated.size} room(s) migrated, #{result.closed.size} closed, in #{result.duration.round(2)}s"
|
|
358
|
-
@monitor.synchronize { @drain_result = result }
|
|
359
|
-
|
|
360
|
-
self.class.after_drain_callbacks.each do |callback|
|
|
361
|
-
callback.call(self, result)
|
|
362
|
-
rescue => e
|
|
363
|
-
CableRoom.report_error(e, host: self, drain_result: result)
|
|
364
|
-
end
|
|
365
|
-
|
|
366
|
-
result
|
|
88
|
+
# How many member ports are attached across every room here. It reads each room's port table
|
|
89
|
+
# from outside the room's thread; a count that's a moment stale is fine for a metric.
|
|
90
|
+
def open_ports
|
|
91
|
+
rooms.sum(&:open_port_count)
|
|
367
92
|
end
|
|
368
93
|
|
|
369
94
|
def shutdown?
|
|
370
95
|
@shutdown
|
|
371
96
|
end
|
|
372
97
|
|
|
373
|
-
# Start listening for provision requests (see CableRoom::Placement). `Host.start!` does this
|
|
374
|
-
# for the process Host; a Host built with `new` (a spec harness) calls it itself. Pass a
|
|
375
|
-
# Placement to use one with non-default settings; calling it again re-arms the current one.
|
|
376
|
-
def start_placement(placement = nil)
|
|
377
|
-
previous = @monitor.synchronize do
|
|
378
|
-
raise "Cannot start placement after shutdown" if @shutdown
|
|
379
|
-
@placement.tap { @placement = placement || @placement || Placement.new(self) }
|
|
380
|
-
end
|
|
381
|
-
previous.stop if previous && previous != @placement
|
|
382
|
-
@placement.start
|
|
383
|
-
@placement
|
|
384
|
-
end
|
|
385
|
-
|
|
386
|
-
# Stop listening and drop any claim still waiting. Nothing starts a room here after this.
|
|
387
|
-
def stop_placement
|
|
388
|
-
@placement&.stop
|
|
389
|
-
end
|
|
390
|
-
|
|
391
98
|
# Stop everything: the beat, every room (gracefully, with a bounded wait), then the pool.
|
|
392
99
|
# Runs once; later calls do nothing.
|
|
393
100
|
def shutdown!
|
|
@@ -396,8 +103,6 @@ module CableRoom
|
|
|
396
103
|
@shutdown = true
|
|
397
104
|
end
|
|
398
105
|
|
|
399
|
-
# Stop claiming first: a room started after this point would only be shut down again
|
|
400
|
-
stop_placement
|
|
401
106
|
scheduler.shutdown
|
|
402
107
|
shutdown_rooms!
|
|
403
108
|
worker_pool.executor.shutdown
|
|
@@ -424,57 +129,6 @@ module CableRoom
|
|
|
424
129
|
|
|
425
130
|
private
|
|
426
131
|
|
|
427
|
-
# Run the migrations `concurrency` at a time until `deadline`, then everything still waiting
|
|
428
|
-
# at once. Each runs on a thread of its own rather than the worker pool: a handoff blocks for
|
|
429
|
-
# up to `handoff_timeout` waiting on a peer, and the rooms still need the pool to run.
|
|
430
|
-
def run_migrations(migrations, concurrency:, deadline:)
|
|
431
|
-
queue = Queue.new
|
|
432
|
-
migrations.each { |migration| queue << migration }
|
|
433
|
-
next_migration = -> { queue.pop(true) rescue nil } # non-blocking: nil once the queue is empty
|
|
434
|
-
|
|
435
|
-
workers = Array.new([concurrency, migrations.size].min) do |i|
|
|
436
|
-
migration_thread("cable_room-drain-#{i}") do
|
|
437
|
-
while (migration = next_migration.call)
|
|
438
|
-
migration.run
|
|
439
|
-
end
|
|
440
|
-
end
|
|
441
|
-
end
|
|
442
|
-
|
|
443
|
-
workers.each do |thread|
|
|
444
|
-
remaining = deadline - monotonic_now
|
|
445
|
-
break if remaining <= 0
|
|
446
|
-
thread.join(remaining)
|
|
447
|
-
end
|
|
448
|
-
|
|
449
|
-
forced = []
|
|
450
|
-
while (migration = next_migration.call)
|
|
451
|
-
forced << migration
|
|
452
|
-
end
|
|
453
|
-
if forced.any?
|
|
454
|
-
logger.warn "drain_timeout passed with #{forced.size} room(s) still waiting; handing them all off now"
|
|
455
|
-
end
|
|
456
|
-
forced_threads = forced.map { |migration| migration_thread("cable_room-drain-forced") { migration.run } }
|
|
457
|
-
|
|
458
|
-
(workers + forced_threads).each(&:join)
|
|
459
|
-
end
|
|
460
|
-
|
|
461
|
-
def migration_thread(name)
|
|
462
|
-
thread = Thread.new do
|
|
463
|
-
yield
|
|
464
|
-
rescue => e
|
|
465
|
-
# Migration#run handles its own failures; this is for a bug in the loop itself, which
|
|
466
|
-
# must not take the whole drain down with it
|
|
467
|
-
CableRoom.report_error(e, host: self)
|
|
468
|
-
end
|
|
469
|
-
thread.name = name
|
|
470
|
-
thread.report_on_exception = false
|
|
471
|
-
thread
|
|
472
|
-
end
|
|
473
|
-
|
|
474
|
-
def monotonic_now
|
|
475
|
-
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
476
|
-
end
|
|
477
|
-
|
|
478
132
|
def cable_server
|
|
479
133
|
ActionCable.server
|
|
480
134
|
end
|
data/lib/cable_room/ports.rb
CHANGED
|
@@ -1,17 +1,4 @@
|
|
|
1
1
|
module CableRoom
|
|
2
|
-
# The `ports[...]` DSL shared by a Room and by a member's RoomMembership. A port is a named
|
|
3
|
-
# lane between a room and its members; `ports[:x] << msg` sends on it and `ports[:x].stream`
|
|
4
|
-
# listens on it. Which transport that touches depends on which side you're on, and each side
|
|
5
|
-
# implements the three methods below:
|
|
6
|
-
#
|
|
7
|
-
# * Member side (RoomMembership): sending publishes on the Bus channel for the room's port
|
|
8
|
-
# (member→room), listening is an ActionCable stream (room→member, fed by the broadcaster).
|
|
9
|
-
# * Room side (Room::Base and Room::HostAdapter): sending goes through the configured
|
|
10
|
-
# CableRoom::Broadcaster, listening subscribes on the Bus through the room's Host.
|
|
11
|
-
#
|
|
12
|
-
# So a member's `ports[:custom] << msg` and a room's `ports[:custom].stream` meet on the Bus,
|
|
13
|
-
# and a room's `ports[:custom] << msg` and a member's `ports[:custom].stream` meet on the
|
|
14
|
-
# broadcaster's streams. Neither side needs to know that.
|
|
15
2
|
module Ports
|
|
16
3
|
extend ActiveSupport::Concern
|
|
17
4
|
|
|
@@ -19,19 +6,22 @@ module CableRoom
|
|
|
19
6
|
@ports_proxy ||= PortsProxy.new(self)
|
|
20
7
|
end
|
|
21
8
|
|
|
22
|
-
#
|
|
23
|
-
#
|
|
9
|
+
# The member side, streaming through its ActionCable channel. Rooms override this (and
|
|
10
|
+
# close_streamed_ports!) in Room::HostAdapter to listen through their Host::Runner instead.
|
|
24
11
|
def stream_port(port, auto_close: true, &blk)
|
|
25
|
-
|
|
12
|
+
@cable_channel.stream_from(room_port_key(port), coder: ActiveSupport::JSON, &blk)
|
|
13
|
+
_streamed_ports << port if auto_close
|
|
26
14
|
end
|
|
27
15
|
|
|
28
16
|
def close_streamed_ports!
|
|
29
|
-
|
|
17
|
+
_streamed_ports.each do |port|
|
|
18
|
+
@cable_channel.stop_stream_from(room_port_key(port))
|
|
19
|
+
end
|
|
20
|
+
_streamed_ports.clear
|
|
30
21
|
end
|
|
31
22
|
|
|
32
|
-
# Send `data` on `port`.
|
|
33
23
|
def port_transmit(port, data)
|
|
34
|
-
|
|
24
|
+
ActionCable.server.broadcast(room_port_key(port), data)
|
|
35
25
|
end
|
|
36
26
|
|
|
37
27
|
protected
|
data/lib/cable_room/railtie.rb
CHANGED
|
@@ -5,12 +5,20 @@ require "active_model/railtie"
|
|
|
5
5
|
|
|
6
6
|
module CableRoom
|
|
7
7
|
class Railtie < Rails::Railtie # :nodoc:
|
|
8
|
+
rake_tasks do
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
console do |app|
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
runner do
|
|
15
|
+
end
|
|
16
|
+
|
|
8
17
|
initializer "cable_room.hook_action_cable_restart" do
|
|
9
18
|
module ActionCableServerExtensions
|
|
10
|
-
# ActionCable restarts when the app reloads; stop every room with it
|
|
11
|
-
# hosts no rooms (a web process in :remote) has nothing to stop.
|
|
19
|
+
# ActionCable restarts when the app reloads; stop every room with it
|
|
12
20
|
def restart
|
|
13
|
-
CableRoom::Host.
|
|
21
|
+
CableRoom::Host.instance.stop_all_rooms!
|
|
14
22
|
super
|
|
15
23
|
end
|
|
16
24
|
end
|