cable_room 0.7.0.beta3 → 0.8.0.beta2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +0 -1287
- data/cable_room.gemspec +7 -3
- data/lib/cable_room/bus.rb +45 -75
- data/lib/cable_room/cli.rb +105 -76
- data/lib/cable_room/config.rb +46 -80
- data/lib/cable_room/host/bus_inbound.rb +16 -11
- data/lib/cable_room/host/placement.rb +261 -0
- data/lib/cable_room/host/runner.rb +102 -390
- data/lib/cable_room/host/supervisor.rb +48 -22
- data/lib/cable_room/host/worker_pool.rb +27 -40
- data/lib/cable_room/host.rb +90 -320
- data/lib/cable_room/ports.rb +5 -5
- data/lib/cable_room/railtie.rb +14 -0
- data/lib/cable_room/room/base.rb +32 -28
- data/lib/cable_room/room/callbacks.rb +0 -22
- data/lib/cable_room/room/host_adapter.rb +1 -1
- data/lib/cable_room/room/lifecycle.rb +6 -23
- data/lib/cable_room/room/port_management.rb +0 -50
- data/lib/cable_room/room/reaping.rb +0 -33
- data/lib/cable_room/room/user_management.rb +0 -27
- data/lib/cable_room/room.rb +1 -2
- data/lib/cable_room/room_member.rb +112 -248
- data/lib/cable_room/room_proxy_channel.rb +2 -13
- data/lib/cable_room/version.rb +1 -1
- data/lib/cable_room.rb +50 -37
- metadata +10 -11
- data/CHANGELOG.md +0 -122
- data/lib/cable_room/broadcaster.rb +0 -116
- data/lib/cable_room/membership_store.rb +0 -105
- data/lib/cable_room/migration.rb +0 -586
- data/lib/cable_room/placement.rb +0 -260
- data/lib/cable_room/room/snapshotting.rb +0 -82
- data/lib/cable_room/room_harness.rb +0 -168
- data/lib/cable_room/snapshot.rb +0 -136
|
@@ -10,24 +10,14 @@ module CableRoom
|
|
|
10
10
|
#
|
|
11
11
|
# States: :initializing -> :starting -> :started -> :shutting_down -> :dead
|
|
12
12
|
#
|
|
13
|
-
#
|
|
14
|
-
#
|
|
15
|
-
#
|
|
16
|
-
#
|
|
17
|
-
#
|
|
13
|
+
# Locking: `@mutex` guards lifecycle state and the stream and timer lists. It is never held
|
|
14
|
+
# across a call into the inbound transport or into the room's own callbacks, because the
|
|
15
|
+
# transport's threads call back into us and would deadlock against it. Those callbacks only
|
|
16
|
+
# ever take `@queue_lock`, a leaf lock around the work queue that is never held while calling
|
|
17
|
+
# out to anything. Lock order is always `@mutex` then `@queue_lock`.
|
|
18
18
|
class Runner
|
|
19
|
-
FROZEN_STATES = %i[freezing frozen].freeze
|
|
20
|
-
|
|
21
|
-
# Thread-local flag guarding re-entry into a Room's `:work` callbacks (see
|
|
22
|
-
# `with_room_context`). Namespaced so it can never collide with a key an app's own callback
|
|
23
|
-
# (PandaPal's, or anything else touching `Thread.current`) happens to use for its own purposes.
|
|
24
|
-
APP_WORK_KEY = :"cable_room.in_room_work_callbacks"
|
|
25
|
-
|
|
26
19
|
attr_reader :host, :room, :room_class, :key, :uuid, :tenant, :logger
|
|
27
20
|
|
|
28
|
-
# Monotonic time this runner was built. `Host#drain!` migrates rooms oldest first.
|
|
29
|
-
attr_reader :started_at
|
|
30
|
-
|
|
31
21
|
delegate :worker_pool, :inbound, to: :host
|
|
32
22
|
|
|
33
23
|
def initialize(host, room_class, key, lock_info, watchdog_interval:, lock_duration:, tenant: nil)
|
|
@@ -37,28 +27,24 @@ module CableRoom
|
|
|
37
27
|
@lock_info = lock_info
|
|
38
28
|
@watchdog_interval = watchdog_interval
|
|
39
29
|
@lock_duration = lock_duration
|
|
40
|
-
@started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
41
30
|
|
|
42
31
|
# Used mainly for logs and being able to follow a specific Room instance
|
|
43
32
|
@uuid = SecureRandom.hex(6)
|
|
44
33
|
|
|
45
34
|
@mutex = Monitor.new
|
|
46
|
-
@
|
|
35
|
+
@queue_lock = Mutex.new
|
|
47
36
|
@current_state = :initializing
|
|
37
|
+
@stopping = false
|
|
48
38
|
@processing_work = false
|
|
49
39
|
@work_queue = []
|
|
50
|
-
@
|
|
51
|
-
@hold_from_start = false
|
|
52
|
-
@streams = {} # stream => the inbound transport's handle, needed to unsubscribe
|
|
53
|
-
@handlers = {} # stream => the room's handler, so a message can be fed in by hand (see `inject`)
|
|
40
|
+
@streams = {}
|
|
54
41
|
@periodic_timers = []
|
|
55
42
|
|
|
56
|
-
#
|
|
57
|
-
#
|
|
58
|
-
#
|
|
59
|
-
#
|
|
60
|
-
#
|
|
61
|
-
# tenant.
|
|
43
|
+
# Nothing in the gem reads this, but the runner is the "connection" for its work items
|
|
44
|
+
# (see WorkerPool), and PandaPal's Worker :work hook and broadcasting_for read
|
|
45
|
+
# `connection.tenant` to pick the room's tenant. Don't remove it. It's whatever the Host
|
|
46
|
+
# was told (see Host#ensure_room), never read from ambient state here: in a rooms process
|
|
47
|
+
# the thread building the runner has no request tenant of its own.
|
|
62
48
|
@tenant = tenant
|
|
63
49
|
|
|
64
50
|
@logger = ActionCable::Connection::TaggedLoggerProxy.new(
|
|
@@ -79,14 +65,8 @@ module CableRoom
|
|
|
79
65
|
@current_state
|
|
80
66
|
end
|
|
81
67
|
|
|
82
|
-
def
|
|
83
|
-
|
|
84
|
-
end
|
|
85
|
-
|
|
86
|
-
# The Redlock this runner holds, or nil once it has been released or handed off. Read-only:
|
|
87
|
-
# use `release_lock!` to let go of it.
|
|
88
|
-
def lock_info
|
|
89
|
-
@lock_info
|
|
68
|
+
def closed?
|
|
69
|
+
state == :dead || state == :shutting_down
|
|
90
70
|
end
|
|
91
71
|
|
|
92
72
|
# -- Lifecycle ---------------------------------------------------------------------------
|
|
@@ -94,230 +74,74 @@ module CableRoom
|
|
|
94
74
|
# Run the room's startup callbacks and start its class-level timers, on the calling thread.
|
|
95
75
|
# If startup fails the room is torn down and the error re-raised.
|
|
96
76
|
def start!
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
# its timers don't start until `thaw!`. This is how a migration's adopter takes a room: the
|
|
108
|
-
# messages the old host relayed have to run before anything that arrives live here.
|
|
109
|
-
def restore!(snapshot, hold_inbound: false)
|
|
110
|
-
@hold_from_start = hold_inbound
|
|
111
|
-
start_with(final_state: hold_inbound ? :frozen : :started) { room.send(:_restore, snapshot) }
|
|
77
|
+
@current_state = :starting
|
|
78
|
+
with_executor do
|
|
79
|
+
room.send(:_startup)
|
|
80
|
+
start_periodic_timers
|
|
81
|
+
end
|
|
82
|
+
# Startup may already have asked to shut down (a reaper, or a KILL that arrived early)
|
|
83
|
+
@queue_lock.synchronize { @current_state = :started if @current_state == :starting }
|
|
84
|
+
rescue => e
|
|
85
|
+
terminate!
|
|
86
|
+
raise e
|
|
112
87
|
end
|
|
113
88
|
|
|
114
89
|
# Stop right now, on the calling thread: run the room's shutdown callbacks and free
|
|
115
90
|
# everything. Work still on the queue is dropped. `initiate_shutdown` is the graceful version.
|
|
116
|
-
# `reason` is what members see in `room_closed`; nil keeps whatever the room already set.
|
|
117
91
|
#
|
|
118
|
-
# Only the state change happens under the mutex
|
|
119
|
-
#
|
|
120
|
-
|
|
121
|
-
def stop!(reason: nil)
|
|
92
|
+
# Only the state change happens under the mutex (see the class comment). `@stopping` makes
|
|
93
|
+
# sure the shutdown callbacks run once even if, say, a lost lock and a queued stop race.
|
|
94
|
+
def stop!
|
|
122
95
|
@mutex.synchronize do
|
|
123
|
-
return if state == :dead
|
|
124
|
-
@
|
|
96
|
+
return if @stopping || state == :dead
|
|
97
|
+
@stopping = true
|
|
98
|
+
@queue_lock.synchronize { @current_state = :shutting_down }
|
|
125
99
|
end
|
|
126
100
|
|
|
127
|
-
room.send(:_shutdown_reason=, reason) unless reason.nil?
|
|
128
101
|
unsubscribe_all
|
|
129
102
|
begin
|
|
130
|
-
with_executor {
|
|
103
|
+
with_executor { room.send(:_shutdown) }
|
|
131
104
|
ensure
|
|
132
105
|
terminate!
|
|
133
106
|
end
|
|
134
107
|
end
|
|
135
108
|
|
|
136
|
-
# Shut down gracefully: stop listening, then stop once everything already queued has run
|
|
137
|
-
# `reason` reaches the members in `room_closed`.
|
|
109
|
+
# Shut down gracefully: stop listening, then stop once everything already queued has run
|
|
138
110
|
def initiate_shutdown(reason)
|
|
139
111
|
@mutex.synchronize do
|
|
140
112
|
return if closed?
|
|
141
113
|
|
|
142
114
|
logger.info "Initiating shutdown: #{reason}"
|
|
143
|
-
room.send(:_shutdown_reason=, reason)
|
|
144
115
|
|
|
145
|
-
# The actual stop goes behind whatever is already waiting, so those messages still run
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
116
|
+
# The actual stop goes behind whatever is already waiting, so those messages still run.
|
|
117
|
+
# Closing in the same step means nothing posted from here on can land after it.
|
|
118
|
+
@queue_lock.synchronize do
|
|
119
|
+
@work_queue << wrap_work(-> { stop! })
|
|
120
|
+
@current_state = :shutting_down
|
|
121
|
+
end
|
|
151
122
|
end
|
|
152
123
|
|
|
153
|
-
unsubscribe_all
|
|
154
|
-
# A frozen room had stopped taking work off its queue; now that it's shutting down the
|
|
155
|
-
# stop posted above has to actually run
|
|
156
124
|
schedule_work
|
|
125
|
+
unsubscribe_all
|
|
157
126
|
end
|
|
158
127
|
|
|
159
|
-
# Free everything without running the room's shutdown callbacks
|
|
160
|
-
|
|
161
|
-
def terminate!(release_lock: true)
|
|
128
|
+
# Free everything without running the room's shutdown callbacks
|
|
129
|
+
def terminate!
|
|
162
130
|
lock_info = @mutex.synchronize do
|
|
163
|
-
@
|
|
164
|
-
|
|
131
|
+
@queue_lock.synchronize do
|
|
132
|
+
@current_state = :dead
|
|
133
|
+
@work_queue.clear
|
|
134
|
+
end
|
|
165
135
|
@lock_info.tap { @lock_info = nil }
|
|
166
136
|
end
|
|
167
137
|
|
|
138
|
+
stop_periodic_timers
|
|
139
|
+
|
|
168
140
|
# Unsubscribe before releasing the lock: the next runner for this key (which needs the
|
|
169
|
-
# lock) must not subscribe to the same
|
|
141
|
+
# lock) must not subscribe to the same streams before this one has let go of them.
|
|
170
142
|
unsubscribe_all
|
|
171
|
-
CableRoom.lock_manager.unlock(lock_info) if
|
|
143
|
+
CableRoom.lock_manager.unlock(lock_info) if lock_info
|
|
172
144
|
host.untrack(self)
|
|
173
|
-
lock_info
|
|
174
|
-
end
|
|
175
|
-
|
|
176
|
-
# -- Migration ---------------------------------------------------------------------------
|
|
177
|
-
#
|
|
178
|
-
# The pieces the migration protocol (CableRoom::Migration) is built from, in the order it
|
|
179
|
-
# uses them: `freeze!`, `snapshot`, `release_lock!`, and finally `discard!` once another
|
|
180
|
-
# host has restored the room — or `stop!` if none did. Members never hear about any of it.
|
|
181
|
-
|
|
182
|
-
# Bring the room to a standstill so it can be snapshotted: stop the periodic timers, let
|
|
183
|
-
# whatever is already queued finish, and from then on hold inbound messages instead of
|
|
184
|
-
# handling them. The Bus subscription stays up, so the messages that arrive while the room
|
|
185
|
-
# is frozen still land here — in `held_inbound`, or with the block if one is given (called
|
|
186
|
-
# on the Bus thread with `(stream, message)`, so keep it quick), for the migration to relay
|
|
187
|
-
# to the new host. The lock is kept and keeps being renewed.
|
|
188
|
-
#
|
|
189
|
-
# Blocks until the room is quiet, or for `timeout` seconds (nil waits as long as it takes).
|
|
190
|
-
# Returns true once frozen, or false if the room died on the way (a queued message asked it
|
|
191
|
-
# to shut down, say) or the timeout passed — in which case the room is running again as if
|
|
192
|
-
# nothing happened, with the messages held meanwhile back on its queue, so the caller can
|
|
193
|
-
# still `stop!` it cleanly. A room that never goes quiet must not hang a drain.
|
|
194
|
-
def freeze!(timeout: nil, &hold)
|
|
195
|
-
@mutex.synchronize do
|
|
196
|
-
return false if closed?
|
|
197
|
-
return true if frozen?
|
|
198
|
-
|
|
199
|
-
logger.info "Freezing"
|
|
200
|
-
@current_state = :freezing
|
|
201
|
-
@hold_inbound = hold
|
|
202
|
-
stop_periodic_timers
|
|
203
|
-
# Nothing else may be running in this room once we return: wait for the queue to drain
|
|
204
|
-
# and the item in flight to finish. New inbound is already being held, so this ends.
|
|
205
|
-
deadline = timeout && monotonic_now + timeout
|
|
206
|
-
while state == :freezing && (@processing_work || @work_queue.any?)
|
|
207
|
-
remaining = deadline && deadline - monotonic_now
|
|
208
|
-
if remaining && remaining <= 0
|
|
209
|
-
logger.warn "Room did not go quiet within #{timeout}s; not freezing it"
|
|
210
|
-
resume_from_freeze
|
|
211
|
-
return false
|
|
212
|
-
end
|
|
213
|
-
@work_finished.wait(remaining)
|
|
214
|
-
end
|
|
215
|
-
return false unless state == :freezing
|
|
216
|
-
|
|
217
|
-
@current_state = :frozen
|
|
218
|
-
# Messages that arrived while the room was still going quiet were held, not handed to
|
|
219
|
-
# the block: had the freeze timed out they'd have to go back on the queue, and once
|
|
220
|
-
# relayed they'd be gone. Now that the freeze is final, hand them over first — under the
|
|
221
|
-
# mutex, so nothing arriving on the Bus thread can get ahead of them.
|
|
222
|
-
if hold
|
|
223
|
-
held = @held_inbound
|
|
224
|
-
@held_inbound = []
|
|
225
|
-
held.each { |stream, message| hold.call(stream, message) }
|
|
226
|
-
end
|
|
227
|
-
end
|
|
228
|
-
true
|
|
229
|
-
end
|
|
230
|
-
|
|
231
|
-
# Messages that arrived while frozen (and weren't handed to a `freeze!` block), as
|
|
232
|
-
# `[stream, message]` pairs in arrival order.
|
|
233
|
-
def held_inbound
|
|
234
|
-
@mutex.synchronize { @held_inbound.dup }
|
|
235
|
-
end
|
|
236
|
-
|
|
237
|
-
# The streams this runner is subscribed to right now (Bus channel names).
|
|
238
|
-
def subscribed_streams
|
|
239
|
-
@mutex.synchronize { @streams.keys }
|
|
240
|
-
end
|
|
241
|
-
|
|
242
|
-
# Feed `message` into the room as if it had arrived on `stream`, behind whatever is already
|
|
243
|
-
# queued — even while the room is frozen, when a live message would be held instead. This is
|
|
244
|
-
# how a migration's adopter replays what the old host relayed: the messages go onto the
|
|
245
|
-
# queue in the order given and run once the room thaws. Returns false (and drops the
|
|
246
|
-
# message, with a warning) if the room doesn't listen on that stream.
|
|
247
|
-
def inject(stream, message)
|
|
248
|
-
handler = @mutex.synchronize { @handlers[String(stream)] }
|
|
249
|
-
unless handler
|
|
250
|
-
logger.warn "Dropping a relayed message for #{stream}: this room doesn't listen on it"
|
|
251
|
-
return false
|
|
252
|
-
end
|
|
253
|
-
|
|
254
|
-
post_work(async: false, silent: true) { handler.call(message) }
|
|
255
|
-
true
|
|
256
|
-
end
|
|
257
|
-
|
|
258
|
-
# Take a frozen room back to :started. The block gets the inbound held so far (as
|
|
259
|
-
# `[stream, message]` pairs, in arrival order) and returns the pairs to run, in order —
|
|
260
|
-
# a migration's adopter uses it to drop the ones it has already replayed from the handoff
|
|
261
|
-
# list. Those are queued (after anything `inject`ed before), the timers start, and the room
|
|
262
|
-
# runs again. Everything happens under the mutex, so no message can land between the block
|
|
263
|
-
# seeing the held list and the room going live: it is either in the list or queued after.
|
|
264
|
-
# Returns false if the room isn't frozen.
|
|
265
|
-
def thaw!
|
|
266
|
-
@mutex.synchronize do
|
|
267
|
-
return false unless state == :frozen
|
|
268
|
-
|
|
269
|
-
held = @held_inbound
|
|
270
|
-
@held_inbound = []
|
|
271
|
-
@hold_inbound = nil
|
|
272
|
-
@hold_from_start = false
|
|
273
|
-
to_run = block_given? ? yield(held) : held
|
|
274
|
-
to_run.each { |stream, message| inject(stream, message) }
|
|
275
|
-
@current_state = :started
|
|
276
|
-
start_periodic_timers
|
|
277
|
-
ping_watchdog
|
|
278
|
-
end
|
|
279
|
-
logger.info "Thawed"
|
|
280
|
-
schedule_work
|
|
281
|
-
true
|
|
282
|
-
end
|
|
283
|
-
|
|
284
|
-
# The room's CableRoom::Snapshot. Only a frozen room can be snapshotted: that's the one
|
|
285
|
-
# state where nothing else is touching its state.
|
|
286
|
-
def snapshot
|
|
287
|
-
raise "#{room_class.name}[#{key}] must be frozen before it can be snapshotted (state: #{state})" unless state == :frozen
|
|
288
|
-
|
|
289
|
-
with_executor { with_room_context { Snapshot.take(room) } }
|
|
290
|
-
end
|
|
291
|
-
|
|
292
|
-
# Give the room's lock up while staying alive, so another host can claim the room and this
|
|
293
|
-
# one can keep relaying inbound until it has. Returns false if there was no lock to release.
|
|
294
|
-
def release_lock!
|
|
295
|
-
lock_info = @mutex.synchronize { @lock_info.tap { @lock_info = nil } }
|
|
296
|
-
return false unless lock_info
|
|
297
|
-
|
|
298
|
-
CableRoom.lock_manager.unlock(lock_info)
|
|
299
|
-
true
|
|
300
|
-
end
|
|
301
|
-
|
|
302
|
-
# Take the room's lock again after `release_lock!`, when no other host claimed it. Returns
|
|
303
|
-
# false if someone else holds it (or this runner still holds it). With the lock back, `stop!`
|
|
304
|
-
# releases it the normal way.
|
|
305
|
-
def retake_lock!
|
|
306
|
-
return false if @lock_info
|
|
307
|
-
|
|
308
|
-
lock_info = CableRoom.lock_manager.lock(room_class.room_port_key(key), @lock_duration.in_milliseconds)
|
|
309
|
-
return false unless lock_info
|
|
310
|
-
|
|
311
|
-
@mutex.synchronize { @lock_info = lock_info }
|
|
312
|
-
true
|
|
313
|
-
end
|
|
314
|
-
|
|
315
|
-
# Drop a room that now lives somewhere else: no shutdown callbacks, no room_closed, and the
|
|
316
|
-
# lock — if this runner still holds one — is left alone, since it may belong to the new host
|
|
317
|
-
# by now. Returns the lock_info that was still held, or nil, so the caller can decide.
|
|
318
|
-
def discard!
|
|
319
|
-
logger.info "Discarding (the room has moved)"
|
|
320
|
-
terminate!(release_lock: false)
|
|
321
145
|
end
|
|
322
146
|
|
|
323
147
|
# -- Watchdog and lock -------------------------------------------------------------------
|
|
@@ -346,9 +170,6 @@ module CableRoom
|
|
|
346
170
|
return
|
|
347
171
|
end
|
|
348
172
|
|
|
349
|
-
# A frozen room is idle on purpose; its timers are stopped, so nobody pings the watchdog
|
|
350
|
-
return if frozen?
|
|
351
|
-
|
|
352
173
|
unless @last_watchdog_ping_at && @last_watchdog_ping_at > @watchdog_interval.ago
|
|
353
174
|
logger.warn "Watchdog timeout for room #{room_class.name}[#{key}], shutting down"
|
|
354
175
|
initiate_shutdown("Watchdog timeout")
|
|
@@ -369,16 +190,10 @@ module CableRoom
|
|
|
369
190
|
# Errors inside the work are reported (see `report_work_error`), never raised, so one bad
|
|
370
191
|
# message can't take the room down with it.
|
|
371
192
|
def post_work(async: false, silent: false, &blk)
|
|
372
|
-
work = proc do
|
|
373
|
-
worker_pool.invoke(-> { with_room_context(&blk) }, :call, connection: self)
|
|
374
|
-
rescue => e
|
|
375
|
-
report_work_error(e)
|
|
376
|
-
end
|
|
377
|
-
|
|
378
193
|
if async
|
|
379
|
-
worker_pool.executor.post(&
|
|
194
|
+
worker_pool.executor.post(&wrap_work(blk))
|
|
380
195
|
else
|
|
381
|
-
enqueue(
|
|
196
|
+
enqueue(wrap_work(blk), silent: silent)
|
|
382
197
|
end
|
|
383
198
|
end
|
|
384
199
|
|
|
@@ -404,32 +219,37 @@ module CableRoom
|
|
|
404
219
|
# room's handler can't hold up another room's traffic. `on_live` runs once the transport
|
|
405
220
|
# confirms the subscription; anything published before that may be missed.
|
|
406
221
|
#
|
|
407
|
-
# The subscribe itself happens outside the mutex (
|
|
408
|
-
#
|
|
222
|
+
# The subscribe itself happens outside the mutex (see the class comment), so a room that
|
|
223
|
+
# stopped meanwhile undoes it.
|
|
409
224
|
def subscribe(stream, on_live: nil, &handler)
|
|
410
225
|
raise ArgumentError, "Block required" unless handler
|
|
411
226
|
|
|
412
227
|
stream = String(stream)
|
|
413
228
|
return if closed?
|
|
414
229
|
|
|
415
|
-
|
|
416
|
-
|
|
230
|
+
confirmed = lambda do
|
|
231
|
+
logger.info "#{room_class.name} is streaming from #{stream}"
|
|
232
|
+
on_live&.call
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
handle = inbound.subscribe(stream, on_live: confirmed) do |message|
|
|
236
|
+
post_work(async: false, silent: true) { handler.call(message) }
|
|
417
237
|
end
|
|
418
238
|
|
|
419
239
|
stopped = @mutex.synchronize do
|
|
420
|
-
closed? || (@streams[stream] = handle;
|
|
240
|
+
closed? || (@streams[stream] = handle; false)
|
|
421
241
|
end
|
|
422
242
|
inbound.unsubscribe(stream, handle) if stopped
|
|
423
243
|
end
|
|
424
244
|
|
|
425
245
|
def unsubscribe(stream)
|
|
426
246
|
stream = String(stream)
|
|
427
|
-
handle = @mutex.synchronize { @
|
|
247
|
+
handle = @mutex.synchronize { @streams.delete(stream) }
|
|
428
248
|
inbound.unsubscribe(stream, handle) if handle
|
|
429
249
|
end
|
|
430
250
|
|
|
431
251
|
def unsubscribe_all
|
|
432
|
-
handles = @mutex.synchronize { @
|
|
252
|
+
handles = @mutex.synchronize { @streams.to_a.tap { @streams.clear } }
|
|
433
253
|
handles.each { |stream, handle| inbound.unsubscribe(stream, handle) }
|
|
434
254
|
end
|
|
435
255
|
|
|
@@ -444,96 +264,46 @@ module CableRoom
|
|
|
444
264
|
post_work(async: false, silent: true) { callback.call }
|
|
445
265
|
end
|
|
446
266
|
|
|
447
|
-
PeriodicTimer.new(job)
|
|
267
|
+
timer = PeriodicTimer.new(job)
|
|
268
|
+
@mutex.synchronize { @periodic_timers << timer }
|
|
269
|
+
timer
|
|
448
270
|
end
|
|
449
271
|
|
|
450
272
|
private
|
|
451
273
|
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
# `start!` and `restore!` differ only in what the room does first; everything around it —
|
|
457
|
-
# the executor, the timers, tearing down on failure — is the same. A room restored with
|
|
458
|
-
# `hold_inbound` ends up :frozen with no timers running; `thaw!` starts them.
|
|
459
|
-
def start_with(final_state: :started)
|
|
460
|
-
@current_state = :starting
|
|
461
|
-
with_executor do
|
|
462
|
-
with_room_context do
|
|
463
|
-
yield
|
|
464
|
-
start_periodic_timers unless final_state == :frozen
|
|
465
|
-
end
|
|
466
|
-
end
|
|
467
|
-
@current_state = final_state
|
|
468
|
-
rescue => e
|
|
469
|
-
terminate!
|
|
470
|
-
raise e
|
|
471
|
-
end
|
|
472
|
-
|
|
473
|
-
# A message from the Bus, on the Bus thread. Normally it's queued for the room; while the
|
|
474
|
-
# room is frozen it's held for the migration to relay instead. Decided under the mutex so a
|
|
475
|
-
# message can't slip onto the queue in the moment the room freezes.
|
|
476
|
-
#
|
|
477
|
-
# A room restored with `hold_inbound` holds from its very first subscribe, while it is still
|
|
478
|
-
# :starting: the adopter has to see everything that arrives before it thaws the room.
|
|
479
|
-
def receive_inbound(stream, message, handler)
|
|
480
|
-
disposition = @mutex.synchronize do
|
|
481
|
-
next :queue unless frozen? || @hold_from_start
|
|
482
|
-
# Only a fully frozen room relays; while still :freezing it holds (see `freeze!`)
|
|
483
|
-
next :relay if @hold_inbound && state == :frozen
|
|
484
|
-
|
|
485
|
-
@held_inbound << [stream, message]
|
|
486
|
-
:held
|
|
487
|
-
end
|
|
488
|
-
|
|
489
|
-
case disposition
|
|
490
|
-
when :queue
|
|
491
|
-
# A handoff marker is the migration protocol talking to itself (see Migration); it is
|
|
492
|
-
# never for the room. One can only reach a live room on a failure path, so drop it here.
|
|
493
|
-
return if Migration.marker?(message)
|
|
274
|
+
# Timers declared on the Room class with `periodically`. None if startup already asked the
|
|
275
|
+
# room to shut down.
|
|
276
|
+
def start_periodic_timers
|
|
277
|
+
return if closed?
|
|
494
278
|
|
|
495
|
-
|
|
496
|
-
|
|
279
|
+
room_class.periodic_timers.each do |callback, every|
|
|
280
|
+
start_periodic_timer(-> { room.instance_exec(&callback) }, every: every)
|
|
497
281
|
end
|
|
498
282
|
end
|
|
499
283
|
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
def resume_from_freeze
|
|
504
|
-
held = @held_inbound
|
|
505
|
-
@held_inbound = []
|
|
506
|
-
@hold_inbound = nil
|
|
507
|
-
@current_state = :started
|
|
508
|
-
held.each { |stream, message| inject(stream, message) }
|
|
509
|
-
start_periodic_timers
|
|
510
|
-
ping_watchdog
|
|
511
|
-
end
|
|
512
|
-
|
|
513
|
-
def monotonic_now
|
|
514
|
-
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
284
|
+
def stop_periodic_timers
|
|
285
|
+
timers = @mutex.synchronize { @periodic_timers.slice!(0..) }
|
|
286
|
+
timers.each(&:shutdown)
|
|
515
287
|
end
|
|
516
288
|
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
289
|
+
def wrap_work(blk)
|
|
290
|
+
proc do
|
|
291
|
+
worker_pool.invoke(blk, :call, connection: self)
|
|
292
|
+
rescue => e
|
|
293
|
+
report_work_error(e)
|
|
521
294
|
end
|
|
522
295
|
end
|
|
523
296
|
|
|
524
|
-
def stop_periodic_timers
|
|
525
|
-
@periodic_timers.each(&:shutdown)
|
|
526
|
-
@periodic_timers.clear
|
|
527
|
-
end
|
|
528
|
-
|
|
529
297
|
def enqueue(work, silent:)
|
|
530
|
-
@
|
|
531
|
-
if closed?
|
|
532
|
-
raise "Attempt to post work to dead or shutting down room" unless silent
|
|
533
|
-
return
|
|
534
|
-
end
|
|
535
|
-
|
|
298
|
+
accepted = @queue_lock.synchronize do
|
|
299
|
+
next false if closed?
|
|
536
300
|
@work_queue << work
|
|
301
|
+
true
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
unless accepted
|
|
305
|
+
raise "Attempt to post work to dead or shutting down room" unless silent
|
|
306
|
+
return
|
|
537
307
|
end
|
|
538
308
|
|
|
539
309
|
schedule_work
|
|
@@ -543,83 +313,25 @@ module CableRoom
|
|
|
543
313
|
# room is running already. When it finishes, come back for the next one. This is the whole
|
|
544
314
|
# ordering guarantee: one thread per room at a time, in arrival order.
|
|
545
315
|
def schedule_work
|
|
546
|
-
@
|
|
547
|
-
|
|
548
|
-
# A frozen room keeps its queue but runs nothing from it (while :freezing it still
|
|
549
|
-
# drains, so that `freeze!` can return to a quiet room)
|
|
550
|
-
return if state == :frozen
|
|
551
|
-
|
|
552
|
-
work = @work_queue.shift
|
|
553
|
-
return unless work
|
|
554
|
-
|
|
555
|
-
@processing_work = true
|
|
556
|
-
|
|
557
|
-
worker_pool.executor.post do
|
|
558
|
-
begin
|
|
559
|
-
work.call
|
|
560
|
-
ensure
|
|
561
|
-
@mutex.synchronize do
|
|
562
|
-
@processing_work = false
|
|
563
|
-
@work_finished.broadcast
|
|
564
|
-
end
|
|
565
|
-
schedule_work
|
|
566
|
-
end
|
|
567
|
-
end
|
|
568
|
-
end
|
|
569
|
-
end
|
|
316
|
+
work = @queue_lock.synchronize do
|
|
317
|
+
next if @processing_work
|
|
570
318
|
|
|
571
|
-
|
|
572
|
-
# callbacks, which Rails wraps in its executor (database connections, reloading, and so
|
|
573
|
-
# on). Keep that behavior. Nesting is fine: the executor yields straight through when it's
|
|
574
|
-
# already active on this thread.
|
|
575
|
-
def with_executor(&blk)
|
|
576
|
-
if defined?(Rails) && Rails.application
|
|
577
|
-
Rails.application.executor.wrap(&blk)
|
|
578
|
-
else
|
|
579
|
-
yield
|
|
319
|
+
@work_queue.shift.tap { |w| @processing_work = true if w }
|
|
580
320
|
end
|
|
581
|
-
|
|
321
|
+
return unless work
|
|
582
322
|
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
# (non-shared, non-leaky) namespace, ordinary Ruby inheritance already gives "every Room"
|
|
589
|
-
# (define it on your own base Room class, or reopen CableRoom::Room::Base itself) and
|
|
590
|
-
# "just this one" (define it again on a specific subclass) for free, and there's no reason
|
|
591
|
-
# to make an app choose between two different mechanisms for the same thing. Still supports
|
|
592
|
-
# PandaPal's lazy pattern: stage a thread-local here, apply it from an ActiveRecord
|
|
593
|
-
# `checkout` hook, so a message that never queries anything never pays for a schema switch.
|
|
594
|
-
#
|
|
595
|
-
# Every place this Runner touches Room code goes through here: `post_work`'s dispatched
|
|
596
|
-
# work, and the lifecycle methods (`start_with`, `stop!`, `snapshot`) that -- unlike
|
|
597
|
-
# `post_work` -- run directly on the calling thread rather than through the worker pool.
|
|
598
|
-
# A Room's `around_work` never has to know which of those it's wrapping, or guard against
|
|
599
|
-
# being entered twice: called from inside work that's already running (`Room::Lifecycle#stop!`
|
|
600
|
-
# from a message handler, say, or the watchdog's own `stop!`), this just yields through.
|
|
601
|
-
#
|
|
602
|
-
# AR query-log tagging (tag the log with this Room's own tags, the way
|
|
603
|
-
# `ActiveRecordConnectionManagement` did when WorkerPool was still a Worker subclass) wraps
|
|
604
|
-
# every call here regardless of re-entry -- tagging nests safely, unlike `:work` callbacks
|
|
605
|
-
# that stage-and-clear a thread-local.
|
|
606
|
-
def with_room_context(&blk)
|
|
607
|
-
with_ar_log_tagging do
|
|
608
|
-
next yield if Thread.current[APP_WORK_KEY]
|
|
609
|
-
|
|
610
|
-
Thread.current[APP_WORK_KEY] = true
|
|
611
|
-
begin
|
|
612
|
-
room.send(:run_callbacks, :work, &blk)
|
|
613
|
-
ensure
|
|
614
|
-
Thread.current[APP_WORK_KEY] = false
|
|
615
|
-
end
|
|
323
|
+
worker_pool.executor.post do
|
|
324
|
+
work.call
|
|
325
|
+
ensure
|
|
326
|
+
@queue_lock.synchronize { @processing_work = false }
|
|
327
|
+
schedule_work
|
|
616
328
|
end
|
|
617
329
|
end
|
|
618
330
|
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
331
|
+
# Room startup and shutdown used to run inside ActionCable's subscribe/unsubscribe
|
|
332
|
+
# callbacks, which Rails wraps in its executor. Keep that behavior.
|
|
333
|
+
def with_executor(&blk)
|
|
334
|
+
CableRoom.with_app_executor(&blk)
|
|
623
335
|
end
|
|
624
336
|
end
|
|
625
337
|
end
|