cable_room 0.7.0.beta3 → 0.8.0.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +0 -1287
- data/cable_room.gemspec +2 -5
- data/lib/cable_room/host/action_cable_inbound.rb +35 -0
- data/lib/cable_room/host/runner.rb +109 -391
- data/lib/cable_room/host/worker_pool.rb +27 -40
- data/lib/cable_room/host.rb +18 -364
- data/lib/cable_room/ports.rb +9 -19
- data/lib/cable_room/railtie.rb +11 -3
- data/lib/cable_room/room/base.rb +41 -40
- data/lib/cable_room/room/callbacks.rb +0 -22
- data/lib/cable_room/room/host_adapter.rb +4 -9
- data/lib/cable_room/room/lifecycle.rb +6 -23
- data/lib/cable_room/room/port_management.rb +0 -50
- data/lib/cable_room/room/reaping.rb +0 -33
- data/lib/cable_room/room/user_management.rb +0 -27
- data/lib/cable_room/room.rb +1 -4
- data/lib/cable_room/room_member.rb +84 -275
- data/lib/cable_room/room_proxy_channel.rb +2 -13
- data/lib/cable_room/version.rb +1 -1
- data/lib/cable_room.rb +0 -55
- metadata +7 -21
- data/CHANGELOG.md +0 -122
- data/exe/cable_room +0 -8
- data/lib/cable_room/broadcaster.rb +0 -116
- data/lib/cable_room/bus.rb +0 -372
- data/lib/cable_room/cli.rb +0 -237
- data/lib/cable_room/config.rb +0 -112
- data/lib/cable_room/host/bus_inbound.rb +0 -36
- data/lib/cable_room/host/supervisor.rb +0 -275
- data/lib/cable_room/membership_store.rb +0 -105
- data/lib/cable_room/migration.rb +0 -586
- data/lib/cable_room/placement.rb +0 -260
- data/lib/cable_room/room/snapshotting.rb +0 -82
- data/lib/cable_room/room_harness.rb +0 -168
- data/lib/cable_room/snapshot.rb +0 -136
|
@@ -2,7 +2,7 @@ module CableRoom
|
|
|
2
2
|
class Host
|
|
3
3
|
# Runs one Room for the Host. It holds everything that is per-room but not the room's own
|
|
4
4
|
# business: the ordered work queue, lifecycle state, the Redlock and watchdog, the inbound
|
|
5
|
-
#
|
|
5
|
+
# stream subscriptions, and the periodic timers.
|
|
6
6
|
#
|
|
7
7
|
# A Room talks to its runner (`@runner`) for anything that touches threads or the outside
|
|
8
8
|
# world. The Room DSL (`shutdown!`, `stop!`, `async`, `on_room_thread`, `ports[]`,
|
|
@@ -10,56 +10,40 @@ module CableRoom
|
|
|
10
10
|
#
|
|
11
11
|
# States: :initializing -> :starting -> :started -> :shutting_down -> :dead
|
|
12
12
|
#
|
|
13
|
-
#
|
|
14
|
-
#
|
|
15
|
-
#
|
|
16
|
-
#
|
|
17
|
-
#
|
|
13
|
+
# Locking: `@mutex` guards lifecycle state and the stream and timer lists. It is never held
|
|
14
|
+
# across a call into the inbound transport or into the room's own callbacks, because the
|
|
15
|
+
# transport's threads call back into us and would deadlock against it. Those callbacks only
|
|
16
|
+
# ever take `@queue_lock`, a leaf lock around the work queue that is never held while calling
|
|
17
|
+
# out to anything. Lock order is always `@mutex` then `@queue_lock`.
|
|
18
18
|
class Runner
|
|
19
|
-
FROZEN_STATES = %i[freezing frozen].freeze
|
|
20
|
-
|
|
21
|
-
# Thread-local flag guarding re-entry into a Room's `:work` callbacks (see
|
|
22
|
-
# `with_room_context`). Namespaced so it can never collide with a key an app's own callback
|
|
23
|
-
# (PandaPal's, or anything else touching `Thread.current`) happens to use for its own purposes.
|
|
24
|
-
APP_WORK_KEY = :"cable_room.in_room_work_callbacks"
|
|
25
|
-
|
|
26
19
|
attr_reader :host, :room, :room_class, :key, :uuid, :tenant, :logger
|
|
27
20
|
|
|
28
|
-
# Monotonic time this runner was built. `Host#drain!` migrates rooms oldest first.
|
|
29
|
-
attr_reader :started_at
|
|
30
|
-
|
|
31
21
|
delegate :worker_pool, :inbound, to: :host
|
|
32
22
|
|
|
33
|
-
def initialize(host, room_class, key, lock_info, watchdog_interval:, lock_duration
|
|
23
|
+
def initialize(host, room_class, key, lock_info, watchdog_interval:, lock_duration:)
|
|
34
24
|
@host = host
|
|
35
25
|
@room_class = room_class
|
|
36
26
|
@key = key
|
|
37
27
|
@lock_info = lock_info
|
|
38
28
|
@watchdog_interval = watchdog_interval
|
|
39
29
|
@lock_duration = lock_duration
|
|
40
|
-
@started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
41
30
|
|
|
42
31
|
# Used mainly for logs and being able to follow a specific Room instance
|
|
43
32
|
@uuid = SecureRandom.hex(6)
|
|
44
33
|
|
|
45
34
|
@mutex = Monitor.new
|
|
46
|
-
@
|
|
35
|
+
@queue_lock = Mutex.new
|
|
47
36
|
@current_state = :initializing
|
|
37
|
+
@stopping = false
|
|
48
38
|
@processing_work = false
|
|
49
39
|
@work_queue = []
|
|
50
|
-
@
|
|
51
|
-
@hold_from_start = false
|
|
52
|
-
@streams = {} # stream => the inbound transport's handle, needed to unsubscribe
|
|
53
|
-
@handlers = {} # stream => the room's handler, so a message can be fed in by hand (see `inject`)
|
|
40
|
+
@streams = {}
|
|
54
41
|
@periodic_timers = []
|
|
55
42
|
|
|
56
|
-
#
|
|
57
|
-
#
|
|
58
|
-
#
|
|
59
|
-
|
|
60
|
-
# anything read here would be leftover from whatever this thread ran last, not this room's
|
|
61
|
-
# tenant.
|
|
62
|
-
@tenant = tenant
|
|
43
|
+
# Nothing in the gem reads this, but the runner is the "connection" for its work items
|
|
44
|
+
# (see WorkerPool), and PandaPal's Worker :work hook and broadcasting_for read
|
|
45
|
+
# `connection.tenant` to pick the room's tenant. Don't remove it.
|
|
46
|
+
@tenant = Apartment::Tenant.current if defined?(Apartment)
|
|
63
47
|
|
|
64
48
|
@logger = ActionCable::Connection::TaggedLoggerProxy.new(
|
|
65
49
|
host.logger,
|
|
@@ -79,14 +63,8 @@ module CableRoom
|
|
|
79
63
|
@current_state
|
|
80
64
|
end
|
|
81
65
|
|
|
82
|
-
def
|
|
83
|
-
|
|
84
|
-
end
|
|
85
|
-
|
|
86
|
-
# The Redlock this runner holds, or nil once it has been released or handed off. Read-only:
|
|
87
|
-
# use `release_lock!` to let go of it.
|
|
88
|
-
def lock_info
|
|
89
|
-
@lock_info
|
|
66
|
+
def closed?
|
|
67
|
+
state == :dead || state == :shutting_down
|
|
90
68
|
end
|
|
91
69
|
|
|
92
70
|
# -- Lifecycle ---------------------------------------------------------------------------
|
|
@@ -94,230 +72,74 @@ module CableRoom
|
|
|
94
72
|
# Run the room's startup callbacks and start its class-level timers, on the calling thread.
|
|
95
73
|
# If startup fails the room is torn down and the error re-raised.
|
|
96
74
|
def start!
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
# its timers don't start until `thaw!`. This is how a migration's adopter takes a room: the
|
|
108
|
-
# messages the old host relayed have to run before anything that arrives live here.
|
|
109
|
-
def restore!(snapshot, hold_inbound: false)
|
|
110
|
-
@hold_from_start = hold_inbound
|
|
111
|
-
start_with(final_state: hold_inbound ? :frozen : :started) { room.send(:_restore, snapshot) }
|
|
75
|
+
@current_state = :starting
|
|
76
|
+
with_executor do
|
|
77
|
+
room.send(:_startup)
|
|
78
|
+
start_periodic_timers
|
|
79
|
+
end
|
|
80
|
+
# Startup may already have asked to shut down (a reaper, or a KILL that arrived early)
|
|
81
|
+
@queue_lock.synchronize { @current_state = :started if @current_state == :starting }
|
|
82
|
+
rescue => e
|
|
83
|
+
terminate!
|
|
84
|
+
raise e
|
|
112
85
|
end
|
|
113
86
|
|
|
114
87
|
# Stop right now, on the calling thread: run the room's shutdown callbacks and free
|
|
115
88
|
# everything. Work still on the queue is dropped. `initiate_shutdown` is the graceful version.
|
|
116
|
-
# `reason` is what members see in `room_closed`; nil keeps whatever the room already set.
|
|
117
89
|
#
|
|
118
|
-
# Only the state change happens under the mutex
|
|
119
|
-
#
|
|
120
|
-
|
|
121
|
-
def stop!(reason: nil)
|
|
90
|
+
# Only the state change happens under the mutex (see the class comment). `@stopping` makes
|
|
91
|
+
# sure the shutdown callbacks run once even if, say, a lost lock and a queued stop race.
|
|
92
|
+
def stop!
|
|
122
93
|
@mutex.synchronize do
|
|
123
|
-
return if state == :dead
|
|
124
|
-
@
|
|
94
|
+
return if @stopping || state == :dead
|
|
95
|
+
@stopping = true
|
|
96
|
+
@queue_lock.synchronize { @current_state = :shutting_down }
|
|
125
97
|
end
|
|
126
98
|
|
|
127
|
-
room.send(:_shutdown_reason=, reason) unless reason.nil?
|
|
128
99
|
unsubscribe_all
|
|
129
100
|
begin
|
|
130
|
-
with_executor {
|
|
101
|
+
with_executor { room.send(:_shutdown) }
|
|
131
102
|
ensure
|
|
132
103
|
terminate!
|
|
133
104
|
end
|
|
134
105
|
end
|
|
135
106
|
|
|
136
|
-
# Shut down gracefully: stop listening, then stop once everything already queued has run
|
|
137
|
-
# `reason` reaches the members in `room_closed`.
|
|
107
|
+
# Shut down gracefully: stop listening, then stop once everything already queued has run
|
|
138
108
|
def initiate_shutdown(reason)
|
|
139
109
|
@mutex.synchronize do
|
|
140
110
|
return if closed?
|
|
141
111
|
|
|
142
112
|
logger.info "Initiating shutdown: #{reason}"
|
|
143
|
-
room.send(:_shutdown_reason=, reason)
|
|
144
113
|
|
|
145
|
-
# The actual stop goes behind whatever is already waiting, so those messages still run
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
114
|
+
# The actual stop goes behind whatever is already waiting, so those messages still run.
|
|
115
|
+
# Closing in the same step means nothing posted from here on can land after it.
|
|
116
|
+
@queue_lock.synchronize do
|
|
117
|
+
@work_queue << wrap_work(-> { stop! })
|
|
118
|
+
@current_state = :shutting_down
|
|
119
|
+
end
|
|
151
120
|
end
|
|
152
121
|
|
|
153
|
-
unsubscribe_all
|
|
154
|
-
# A frozen room had stopped taking work off its queue; now that it's shutting down the
|
|
155
|
-
# stop posted above has to actually run
|
|
156
122
|
schedule_work
|
|
123
|
+
unsubscribe_all
|
|
157
124
|
end
|
|
158
125
|
|
|
159
|
-
# Free everything without running the room's shutdown callbacks
|
|
160
|
-
|
|
161
|
-
def terminate!(release_lock: true)
|
|
126
|
+
# Free everything without running the room's shutdown callbacks
|
|
127
|
+
def terminate!
|
|
162
128
|
lock_info = @mutex.synchronize do
|
|
163
|
-
@
|
|
164
|
-
|
|
129
|
+
@queue_lock.synchronize do
|
|
130
|
+
@current_state = :dead
|
|
131
|
+
@work_queue.clear
|
|
132
|
+
end
|
|
165
133
|
@lock_info.tap { @lock_info = nil }
|
|
166
134
|
end
|
|
167
135
|
|
|
136
|
+
stop_periodic_timers
|
|
137
|
+
|
|
168
138
|
# Unsubscribe before releasing the lock: the next runner for this key (which needs the
|
|
169
|
-
# lock) must not subscribe to the same
|
|
139
|
+
# lock) must not subscribe to the same streams before this one has let go of them.
|
|
170
140
|
unsubscribe_all
|
|
171
|
-
CableRoom.lock_manager.unlock(lock_info) if
|
|
141
|
+
CableRoom.lock_manager.unlock(lock_info) if lock_info
|
|
172
142
|
host.untrack(self)
|
|
173
|
-
lock_info
|
|
174
|
-
end
|
|
175
|
-
|
|
176
|
-
# -- Migration ---------------------------------------------------------------------------
|
|
177
|
-
#
|
|
178
|
-
# The pieces the migration protocol (CableRoom::Migration) is built from, in the order it
|
|
179
|
-
# uses them: `freeze!`, `snapshot`, `release_lock!`, and finally `discard!` once another
|
|
180
|
-
# host has restored the room — or `stop!` if none did. Members never hear about any of it.
|
|
181
|
-
|
|
182
|
-
# Bring the room to a standstill so it can be snapshotted: stop the periodic timers, let
|
|
183
|
-
# whatever is already queued finish, and from then on hold inbound messages instead of
|
|
184
|
-
# handling them. The Bus subscription stays up, so the messages that arrive while the room
|
|
185
|
-
# is frozen still land here — in `held_inbound`, or with the block if one is given (called
|
|
186
|
-
# on the Bus thread with `(stream, message)`, so keep it quick), for the migration to relay
|
|
187
|
-
# to the new host. The lock is kept and keeps being renewed.
|
|
188
|
-
#
|
|
189
|
-
# Blocks until the room is quiet, or for `timeout` seconds (nil waits as long as it takes).
|
|
190
|
-
# Returns true once frozen, or false if the room died on the way (a queued message asked it
|
|
191
|
-
# to shut down, say) or the timeout passed — in which case the room is running again as if
|
|
192
|
-
# nothing happened, with the messages held meanwhile back on its queue, so the caller can
|
|
193
|
-
# still `stop!` it cleanly. A room that never goes quiet must not hang a drain.
|
|
194
|
-
def freeze!(timeout: nil, &hold)
|
|
195
|
-
@mutex.synchronize do
|
|
196
|
-
return false if closed?
|
|
197
|
-
return true if frozen?
|
|
198
|
-
|
|
199
|
-
logger.info "Freezing"
|
|
200
|
-
@current_state = :freezing
|
|
201
|
-
@hold_inbound = hold
|
|
202
|
-
stop_periodic_timers
|
|
203
|
-
# Nothing else may be running in this room once we return: wait for the queue to drain
|
|
204
|
-
# and the item in flight to finish. New inbound is already being held, so this ends.
|
|
205
|
-
deadline = timeout && monotonic_now + timeout
|
|
206
|
-
while state == :freezing && (@processing_work || @work_queue.any?)
|
|
207
|
-
remaining = deadline && deadline - monotonic_now
|
|
208
|
-
if remaining && remaining <= 0
|
|
209
|
-
logger.warn "Room did not go quiet within #{timeout}s; not freezing it"
|
|
210
|
-
resume_from_freeze
|
|
211
|
-
return false
|
|
212
|
-
end
|
|
213
|
-
@work_finished.wait(remaining)
|
|
214
|
-
end
|
|
215
|
-
return false unless state == :freezing
|
|
216
|
-
|
|
217
|
-
@current_state = :frozen
|
|
218
|
-
# Messages that arrived while the room was still going quiet were held, not handed to
|
|
219
|
-
# the block: had the freeze timed out they'd have to go back on the queue, and once
|
|
220
|
-
# relayed they'd be gone. Now that the freeze is final, hand them over first — under the
|
|
221
|
-
# mutex, so nothing arriving on the Bus thread can get ahead of them.
|
|
222
|
-
if hold
|
|
223
|
-
held = @held_inbound
|
|
224
|
-
@held_inbound = []
|
|
225
|
-
held.each { |stream, message| hold.call(stream, message) }
|
|
226
|
-
end
|
|
227
|
-
end
|
|
228
|
-
true
|
|
229
|
-
end
|
|
230
|
-
|
|
231
|
-
# Messages that arrived while frozen (and weren't handed to a `freeze!` block), as
|
|
232
|
-
# `[stream, message]` pairs in arrival order.
|
|
233
|
-
def held_inbound
|
|
234
|
-
@mutex.synchronize { @held_inbound.dup }
|
|
235
|
-
end
|
|
236
|
-
|
|
237
|
-
# The streams this runner is subscribed to right now (Bus channel names).
|
|
238
|
-
def subscribed_streams
|
|
239
|
-
@mutex.synchronize { @streams.keys }
|
|
240
|
-
end
|
|
241
|
-
|
|
242
|
-
# Feed `message` into the room as if it had arrived on `stream`, behind whatever is already
|
|
243
|
-
# queued — even while the room is frozen, when a live message would be held instead. This is
|
|
244
|
-
# how a migration's adopter replays what the old host relayed: the messages go onto the
|
|
245
|
-
# queue in the order given and run once the room thaws. Returns false (and drops the
|
|
246
|
-
# message, with a warning) if the room doesn't listen on that stream.
|
|
247
|
-
def inject(stream, message)
|
|
248
|
-
handler = @mutex.synchronize { @handlers[String(stream)] }
|
|
249
|
-
unless handler
|
|
250
|
-
logger.warn "Dropping a relayed message for #{stream}: this room doesn't listen on it"
|
|
251
|
-
return false
|
|
252
|
-
end
|
|
253
|
-
|
|
254
|
-
post_work(async: false, silent: true) { handler.call(message) }
|
|
255
|
-
true
|
|
256
|
-
end
|
|
257
|
-
|
|
258
|
-
# Take a frozen room back to :started. The block gets the inbound held so far (as
|
|
259
|
-
# `[stream, message]` pairs, in arrival order) and returns the pairs to run, in order —
|
|
260
|
-
# a migration's adopter uses it to drop the ones it has already replayed from the handoff
|
|
261
|
-
# list. Those are queued (after anything `inject`ed before), the timers start, and the room
|
|
262
|
-
# runs again. Everything happens under the mutex, so no message can land between the block
|
|
263
|
-
# seeing the held list and the room going live: it is either in the list or queued after.
|
|
264
|
-
# Returns false if the room isn't frozen.
|
|
265
|
-
def thaw!
|
|
266
|
-
@mutex.synchronize do
|
|
267
|
-
return false unless state == :frozen
|
|
268
|
-
|
|
269
|
-
held = @held_inbound
|
|
270
|
-
@held_inbound = []
|
|
271
|
-
@hold_inbound = nil
|
|
272
|
-
@hold_from_start = false
|
|
273
|
-
to_run = block_given? ? yield(held) : held
|
|
274
|
-
to_run.each { |stream, message| inject(stream, message) }
|
|
275
|
-
@current_state = :started
|
|
276
|
-
start_periodic_timers
|
|
277
|
-
ping_watchdog
|
|
278
|
-
end
|
|
279
|
-
logger.info "Thawed"
|
|
280
|
-
schedule_work
|
|
281
|
-
true
|
|
282
|
-
end
|
|
283
|
-
|
|
284
|
-
# The room's CableRoom::Snapshot. Only a frozen room can be snapshotted: that's the one
|
|
285
|
-
# state where nothing else is touching its state.
|
|
286
|
-
def snapshot
|
|
287
|
-
raise "#{room_class.name}[#{key}] must be frozen before it can be snapshotted (state: #{state})" unless state == :frozen
|
|
288
|
-
|
|
289
|
-
with_executor { with_room_context { Snapshot.take(room) } }
|
|
290
|
-
end
|
|
291
|
-
|
|
292
|
-
# Give the room's lock up while staying alive, so another host can claim the room and this
|
|
293
|
-
# one can keep relaying inbound until it has. Returns false if there was no lock to release.
|
|
294
|
-
def release_lock!
|
|
295
|
-
lock_info = @mutex.synchronize { @lock_info.tap { @lock_info = nil } }
|
|
296
|
-
return false unless lock_info
|
|
297
|
-
|
|
298
|
-
CableRoom.lock_manager.unlock(lock_info)
|
|
299
|
-
true
|
|
300
|
-
end
|
|
301
|
-
|
|
302
|
-
# Take the room's lock again after `release_lock!`, when no other host claimed it. Returns
|
|
303
|
-
# false if someone else holds it (or this runner still holds it). With the lock back, `stop!`
|
|
304
|
-
# releases it the normal way.
|
|
305
|
-
def retake_lock!
|
|
306
|
-
return false if @lock_info
|
|
307
|
-
|
|
308
|
-
lock_info = CableRoom.lock_manager.lock(room_class.room_port_key(key), @lock_duration.in_milliseconds)
|
|
309
|
-
return false unless lock_info
|
|
310
|
-
|
|
311
|
-
@mutex.synchronize { @lock_info = lock_info }
|
|
312
|
-
true
|
|
313
|
-
end
|
|
314
|
-
|
|
315
|
-
# Drop a room that now lives somewhere else: no shutdown callbacks, no room_closed, and the
|
|
316
|
-
# lock — if this runner still holds one — is left alone, since it may belong to the new host
|
|
317
|
-
# by now. Returns the lock_info that was still held, or nil, so the caller can decide.
|
|
318
|
-
def discard!
|
|
319
|
-
logger.info "Discarding (the room has moved)"
|
|
320
|
-
terminate!(release_lock: false)
|
|
321
143
|
end
|
|
322
144
|
|
|
323
145
|
# -- Watchdog and lock -------------------------------------------------------------------
|
|
@@ -346,9 +168,6 @@ module CableRoom
|
|
|
346
168
|
return
|
|
347
169
|
end
|
|
348
170
|
|
|
349
|
-
# A frozen room is idle on purpose; its timers are stopped, so nobody pings the watchdog
|
|
350
|
-
return if frozen?
|
|
351
|
-
|
|
352
171
|
unless @last_watchdog_ping_at && @last_watchdog_ping_at > @watchdog_interval.ago
|
|
353
172
|
logger.warn "Watchdog timeout for room #{room_class.name}[#{key}], shutting down"
|
|
354
173
|
initiate_shutdown("Watchdog timeout")
|
|
@@ -369,16 +188,10 @@ module CableRoom
|
|
|
369
188
|
# Errors inside the work are reported (see `report_work_error`), never raised, so one bad
|
|
370
189
|
# message can't take the room down with it.
|
|
371
190
|
def post_work(async: false, silent: false, &blk)
|
|
372
|
-
work = proc do
|
|
373
|
-
worker_pool.invoke(-> { with_room_context(&blk) }, :call, connection: self)
|
|
374
|
-
rescue => e
|
|
375
|
-
report_work_error(e)
|
|
376
|
-
end
|
|
377
|
-
|
|
378
191
|
if async
|
|
379
|
-
worker_pool.executor.post(&
|
|
192
|
+
worker_pool.executor.post(&wrap_work(blk))
|
|
380
193
|
else
|
|
381
|
-
enqueue(
|
|
194
|
+
enqueue(wrap_work(blk), silent: silent)
|
|
382
195
|
end
|
|
383
196
|
end
|
|
384
197
|
|
|
@@ -399,37 +212,44 @@ module CableRoom
|
|
|
399
212
|
|
|
400
213
|
# -- Inbound streams ---------------------------------------------------------------------
|
|
401
214
|
|
|
402
|
-
# Deliver every message published on `stream` to `handler`, on the room's own queue.
|
|
403
|
-
#
|
|
404
|
-
#
|
|
405
|
-
#
|
|
215
|
+
# Deliver every message published on `stream` to `handler`, on the room's own queue.
|
|
216
|
+
# Decoding happens on the room's thread too, so a bad payload is reported like any other
|
|
217
|
+
# work error instead of hurting the transport. `on_live` runs once the transport confirms
|
|
218
|
+
# the subscription; anything published before that may be missed.
|
|
406
219
|
#
|
|
407
|
-
# The subscribe itself happens outside the mutex (
|
|
408
|
-
#
|
|
409
|
-
def subscribe(stream, on_live: nil, &handler)
|
|
220
|
+
# The subscribe itself happens outside the mutex (see the class comment), so a room that
|
|
221
|
+
# stopped meanwhile undoes it.
|
|
222
|
+
def subscribe(stream, coder: ActiveSupport::JSON, on_live: nil, &handler)
|
|
410
223
|
raise ArgumentError, "Block required" unless handler
|
|
411
224
|
|
|
412
225
|
stream = String(stream)
|
|
413
226
|
return if closed?
|
|
414
227
|
|
|
415
|
-
|
|
416
|
-
|
|
228
|
+
confirmed = lambda do
|
|
229
|
+
logger.info "#{room_class.name} is streaming from #{stream}"
|
|
230
|
+
on_live&.call
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
handle = inbound.subscribe(stream, on_live: confirmed) do |raw|
|
|
234
|
+
post_work(async: false, silent: true) do
|
|
235
|
+
handler.call(coder ? coder.decode(raw) : raw)
|
|
236
|
+
end
|
|
417
237
|
end
|
|
418
238
|
|
|
419
239
|
stopped = @mutex.synchronize do
|
|
420
|
-
closed? || (@streams[stream] = handle;
|
|
240
|
+
closed? || (@streams[stream] = handle; false)
|
|
421
241
|
end
|
|
422
242
|
inbound.unsubscribe(stream, handle) if stopped
|
|
423
243
|
end
|
|
424
244
|
|
|
425
245
|
def unsubscribe(stream)
|
|
426
246
|
stream = String(stream)
|
|
427
|
-
handle = @mutex.synchronize { @
|
|
247
|
+
handle = @mutex.synchronize { @streams.delete(stream) }
|
|
428
248
|
inbound.unsubscribe(stream, handle) if handle
|
|
429
249
|
end
|
|
430
250
|
|
|
431
251
|
def unsubscribe_all
|
|
432
|
-
handles = @mutex.synchronize { @
|
|
252
|
+
handles = @mutex.synchronize { @streams.to_a.tap { @streams.clear } }
|
|
433
253
|
handles.each { |stream, handle| inbound.unsubscribe(stream, handle) }
|
|
434
254
|
end
|
|
435
255
|
|
|
@@ -444,96 +264,46 @@ module CableRoom
|
|
|
444
264
|
post_work(async: false, silent: true) { callback.call }
|
|
445
265
|
end
|
|
446
266
|
|
|
447
|
-
PeriodicTimer.new(job)
|
|
267
|
+
timer = PeriodicTimer.new(job)
|
|
268
|
+
@mutex.synchronize { @periodic_timers << timer }
|
|
269
|
+
timer
|
|
448
270
|
end
|
|
449
271
|
|
|
450
272
|
private
|
|
451
273
|
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
# `start!` and `restore!` differ only in what the room does first; everything around it —
|
|
457
|
-
# the executor, the timers, tearing down on failure — is the same. A room restored with
|
|
458
|
-
# `hold_inbound` ends up :frozen with no timers running; `thaw!` starts them.
|
|
459
|
-
def start_with(final_state: :started)
|
|
460
|
-
@current_state = :starting
|
|
461
|
-
with_executor do
|
|
462
|
-
with_room_context do
|
|
463
|
-
yield
|
|
464
|
-
start_periodic_timers unless final_state == :frozen
|
|
465
|
-
end
|
|
466
|
-
end
|
|
467
|
-
@current_state = final_state
|
|
468
|
-
rescue => e
|
|
469
|
-
terminate!
|
|
470
|
-
raise e
|
|
471
|
-
end
|
|
472
|
-
|
|
473
|
-
# A message from the Bus, on the Bus thread. Normally it's queued for the room; while the
|
|
474
|
-
# room is frozen it's held for the migration to relay instead. Decided under the mutex so a
|
|
475
|
-
# message can't slip onto the queue in the moment the room freezes.
|
|
476
|
-
#
|
|
477
|
-
# A room restored with `hold_inbound` holds from its very first subscribe, while it is still
|
|
478
|
-
# :starting: the adopter has to see everything that arrives before it thaws the room.
|
|
479
|
-
def receive_inbound(stream, message, handler)
|
|
480
|
-
disposition = @mutex.synchronize do
|
|
481
|
-
next :queue unless frozen? || @hold_from_start
|
|
482
|
-
# Only a fully frozen room relays; while still :freezing it holds (see `freeze!`)
|
|
483
|
-
next :relay if @hold_inbound && state == :frozen
|
|
484
|
-
|
|
485
|
-
@held_inbound << [stream, message]
|
|
486
|
-
:held
|
|
487
|
-
end
|
|
488
|
-
|
|
489
|
-
case disposition
|
|
490
|
-
when :queue
|
|
491
|
-
# A handoff marker is the migration protocol talking to itself (see Migration); it is
|
|
492
|
-
# never for the room. One can only reach a live room on a failure path, so drop it here.
|
|
493
|
-
return if Migration.marker?(message)
|
|
274
|
+
# Timers declared on the Room class with `periodically`. None if startup already asked the
|
|
275
|
+
# room to shut down.
|
|
276
|
+
def start_periodic_timers
|
|
277
|
+
return if closed?
|
|
494
278
|
|
|
495
|
-
|
|
496
|
-
|
|
279
|
+
room_class.periodic_timers.each do |callback, every|
|
|
280
|
+
start_periodic_timer(-> { room.instance_exec(&callback) }, every: every)
|
|
497
281
|
end
|
|
498
282
|
end
|
|
499
283
|
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
def resume_from_freeze
|
|
504
|
-
held = @held_inbound
|
|
505
|
-
@held_inbound = []
|
|
506
|
-
@hold_inbound = nil
|
|
507
|
-
@current_state = :started
|
|
508
|
-
held.each { |stream, message| inject(stream, message) }
|
|
509
|
-
start_periodic_timers
|
|
510
|
-
ping_watchdog
|
|
511
|
-
end
|
|
512
|
-
|
|
513
|
-
def monotonic_now
|
|
514
|
-
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
284
|
+
def stop_periodic_timers
|
|
285
|
+
timers = @mutex.synchronize { @periodic_timers.slice!(0..) }
|
|
286
|
+
timers.each(&:shutdown)
|
|
515
287
|
end
|
|
516
288
|
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
289
|
+
def wrap_work(blk)
|
|
290
|
+
proc do
|
|
291
|
+
worker_pool.invoke(blk, :call, connection: self)
|
|
292
|
+
rescue => e
|
|
293
|
+
report_work_error(e)
|
|
521
294
|
end
|
|
522
295
|
end
|
|
523
296
|
|
|
524
|
-
def stop_periodic_timers
|
|
525
|
-
@periodic_timers.each(&:shutdown)
|
|
526
|
-
@periodic_timers.clear
|
|
527
|
-
end
|
|
528
|
-
|
|
529
297
|
def enqueue(work, silent:)
|
|
530
|
-
@
|
|
531
|
-
if closed?
|
|
532
|
-
raise "Attempt to post work to dead or shutting down room" unless silent
|
|
533
|
-
return
|
|
534
|
-
end
|
|
535
|
-
|
|
298
|
+
accepted = @queue_lock.synchronize do
|
|
299
|
+
next false if closed?
|
|
536
300
|
@work_queue << work
|
|
301
|
+
true
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
unless accepted
|
|
305
|
+
raise "Attempt to post work to dead or shutting down room" unless silent
|
|
306
|
+
return
|
|
537
307
|
end
|
|
538
308
|
|
|
539
309
|
schedule_work
|
|
@@ -543,28 +313,18 @@ module CableRoom
|
|
|
543
313
|
# room is running already. When it finishes, come back for the next one. This is the whole
|
|
544
314
|
# ordering guarantee: one thread per room at a time, in arrival order.
|
|
545
315
|
def schedule_work
|
|
546
|
-
@
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
begin
|
|
559
|
-
work.call
|
|
560
|
-
ensure
|
|
561
|
-
@mutex.synchronize do
|
|
562
|
-
@processing_work = false
|
|
563
|
-
@work_finished.broadcast
|
|
564
|
-
end
|
|
565
|
-
schedule_work
|
|
566
|
-
end
|
|
567
|
-
end
|
|
316
|
+
work = @queue_lock.synchronize do
|
|
317
|
+
next if @processing_work
|
|
318
|
+
|
|
319
|
+
@work_queue.shift.tap { |w| @processing_work = true if w }
|
|
320
|
+
end
|
|
321
|
+
return unless work
|
|
322
|
+
|
|
323
|
+
worker_pool.executor.post do
|
|
324
|
+
work.call
|
|
325
|
+
ensure
|
|
326
|
+
@queue_lock.synchronize { @processing_work = false }
|
|
327
|
+
schedule_work
|
|
568
328
|
end
|
|
569
329
|
end
|
|
570
330
|
|
|
@@ -579,48 +339,6 @@ module CableRoom
|
|
|
579
339
|
yield
|
|
580
340
|
end
|
|
581
341
|
end
|
|
582
|
-
|
|
583
|
-
# The one seam a Room hooks to run its own around-work logic -- Apartment switching,
|
|
584
|
-
# tracing, whatever -- instead of reaching for ActionCable::Server::Worker's `:work`
|
|
585
|
-
# callback the way PandaPal does (see README's Multi-tenancy section). Deliberately the
|
|
586
|
-
# Room's own `before_work`/`after_work`/`around_work` (see Room::Callbacks), not a second,
|
|
587
|
-
# separately-configured extension point: a Room class already is cable_room's own
|
|
588
|
-
# (non-shared, non-leaky) namespace, ordinary Ruby inheritance already gives "every Room"
|
|
589
|
-
# (define it on your own base Room class, or reopen CableRoom::Room::Base itself) and
|
|
590
|
-
# "just this one" (define it again on a specific subclass) for free, and there's no reason
|
|
591
|
-
# to make an app choose between two different mechanisms for the same thing. Still supports
|
|
592
|
-
# PandaPal's lazy pattern: stage a thread-local here, apply it from an ActiveRecord
|
|
593
|
-
# `checkout` hook, so a message that never queries anything never pays for a schema switch.
|
|
594
|
-
#
|
|
595
|
-
# Every place this Runner touches Room code goes through here: `post_work`'s dispatched
|
|
596
|
-
# work, and the lifecycle methods (`start_with`, `stop!`, `snapshot`) that -- unlike
|
|
597
|
-
# `post_work` -- run directly on the calling thread rather than through the worker pool.
|
|
598
|
-
# A Room's `around_work` never has to know which of those it's wrapping, or guard against
|
|
599
|
-
# being entered twice: called from inside work that's already running (`Room::Lifecycle#stop!`
|
|
600
|
-
# from a message handler, say, or the watchdog's own `stop!`), this just yields through.
|
|
601
|
-
#
|
|
602
|
-
# AR query-log tagging (tag the log with this Room's own tags, the way
|
|
603
|
-
# `ActiveRecordConnectionManagement` did when WorkerPool was still a Worker subclass) wraps
|
|
604
|
-
# every call here regardless of re-entry -- tagging nests safely, unlike `:work` callbacks
|
|
605
|
-
# that stage-and-clear a thread-local.
|
|
606
|
-
def with_room_context(&blk)
|
|
607
|
-
with_ar_log_tagging do
|
|
608
|
-
next yield if Thread.current[APP_WORK_KEY]
|
|
609
|
-
|
|
610
|
-
Thread.current[APP_WORK_KEY] = true
|
|
611
|
-
begin
|
|
612
|
-
room.send(:run_callbacks, :work, &blk)
|
|
613
|
-
ensure
|
|
614
|
-
Thread.current[APP_WORK_KEY] = false
|
|
615
|
-
end
|
|
616
|
-
end
|
|
617
|
-
end
|
|
618
|
-
|
|
619
|
-
def with_ar_log_tagging(&blk)
|
|
620
|
-
return yield unless defined?(ActiveRecord::Base)
|
|
621
|
-
|
|
622
|
-
logger.tag(ActiveRecord::Base.logger, &blk)
|
|
623
|
-
end
|
|
624
342
|
end
|
|
625
343
|
end
|
|
626
344
|
end
|