@torrent-tv/proxy 2.58.3 → 2.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -0
- package/assets/diagnostics/sctpstate.gdb +29 -0
- package/bin/cli.js +12 -1
- package/package.json +1 -1
- package/services/data-channel-handler.js +36 -4
- package/services/delivery-probe.js +110 -4
- package/services/usrsctp-state.js +146 -0
- package/test/probe-wedge-certainty.test.js +60 -0
- package/test/usrsctp-state.test.js +115 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
## 2.59.0
|
|
2
|
+
|
|
3
|
+
- **New**: a wedge (roadmap item 11) is now declared, and evidence gathered for it automatically, even when the small-message shape means nothing ever queues. The only trigger that existed (`wedgeIsCertain`) requires a nonzero channel queue — confirmed 2026-08-28 by reading the code, not assuming it from the note: `bufferedAmount()` reads 0 the instant our bytes reach usrsctp, whatever usrsctp does with them next, so a wedge carrying only probes and control messages never set it and the last real episode's ring was never saved automatically. The delivery probe already computes a queue-independent verdict (`association-stopped`); it is now wired to the same evidence-gathering, gated on its OWN certainty rule rather than the raw verdict — a connection can sit behind by a bounded, non-growing amount for minutes (measured the same day, a backgrounded tab: gap held at 6-7 probes for 95+ seconds while `seen` kept climbing right along with `sent`) without anything being wrong. What a true wedge shows instead, checked against a session already known to be one: `seen` frozen at one value for over a minute while `sent` climbs unbounded. `probeWedgeIsCertain` asks whether the counter has stopped moving for longer than this connection's own history says a healthy gap ever takes — the same shape `wedgeIsCertain` already uses, applied to the probe's own counter.
|
|
4
|
+
- **New**: `usrsctp`'s live association state (peer receive window, pending data, retransmission timeout, congestion window) can now be read on either wedge declaration, automatically — `node_datachannel.node` ships unstripped, so the read is a gdb attach against the running process, no rebuild. The walk and its healthy baseline are `research/session-2026-08-27-28-freeze-onset-and-sessions.md`, section 1; the script that performs it ships in the package (`assets/diagnostics/sctpstate.gdb`) instead of surviving only as long as someone remembers to copy it back onto a host after a container is recreated. Nine episodes of this item have lacked exactly this reading.
|
|
5
|
+
|
|
1
6
|
## 2.58.3
|
|
2
7
|
|
|
3
8
|
- **New**: the proxy says what it is holding, once a minute — resident memory, heap, external and array buffers, the torrent stores in BYTES, and what the machine has left. It was killed on 2026-08-28 by the kernel's own out-of-memory killer at 2.4 GB resident (`exit code 137`, no core dump, `Out of memory: Killed process ... anon-rss: 2422628kB`) and the log had never recorded a single figure about memory. There was one final reading, taken by the kernel, and no series leading to it.
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
set pagination off
|
|
2
|
+
set confirm off
|
|
3
|
+
set $base = (unsigned long) &system_base_info
|
|
4
|
+
set $hash = *(unsigned long *)($base + 0)
|
|
5
|
+
set $mask = *(unsigned long *)($base + 8)
|
|
6
|
+
printf "asochash=%p mask=%lu\n", $hash, $mask
|
|
7
|
+
set $i = 0
|
|
8
|
+
set $stcb = 0
|
|
9
|
+
while $i <= $mask && $stcb == 0
|
|
10
|
+
set $head = *(unsigned long *)($hash + $i * 8)
|
|
11
|
+
if $head != 0
|
|
12
|
+
set $stcb = $head
|
|
13
|
+
printf "bucket %lu -> stcb=%p\n", $i, $stcb
|
|
14
|
+
end
|
|
15
|
+
set $i = $i + 1
|
|
16
|
+
end
|
|
17
|
+
if $stcb == 0
|
|
18
|
+
printf "no association found (no viewer connected?)\n"
|
|
19
|
+
else
|
|
20
|
+
set $sock = *(unsigned long *)($stcb + 0)
|
|
21
|
+
printf "socket=%p asoc=%p\n", $sock, $stcb + 88
|
|
22
|
+
set $buf = (unsigned long) malloc(512)
|
|
23
|
+
set $lenp = (unsigned long) malloc(8)
|
|
24
|
+
set *(int *)$lenp = 512
|
|
25
|
+
set $rc = (int) usrsctp_getsockopt($sock, 132, 256, $buf, $lenp)
|
|
26
|
+
printf "getsockopt rc=%d len=%d\n", $rc, *(int *)$lenp
|
|
27
|
+
printf "state=%d rwnd=%u unackdata=%u penddata=%u instrms=%u outstrms=%u fragpoint=%u\n", *(int *)($buf+4), *(unsigned int *)($buf+8), *(unsigned short *)($buf+12), *(unsigned short *)($buf+14), *(unsigned short *)($buf+16), *(unsigned short *)($buf+18), *(unsigned int *)($buf+20)
|
|
28
|
+
printf "primary: state=%d cwnd=%u srtt=%u rto=%u mtu=%u\n", *(int *)($buf+24+4+128), *(unsigned int *)($buf+24+4+128+4), *(unsigned int *)($buf+24+4+128+8), *(unsigned int *)($buf+24+4+128+12), *(unsigned int *)($buf+24+4+128+16)
|
|
29
|
+
end
|
package/bin/cli.js
CHANGED
|
@@ -25,6 +25,7 @@ import { createWebRtcManager } from "../services/webrtc-manager.js";
|
|
|
25
25
|
import { createDataChannelHandler } from "../services/data-channel-handler.js";
|
|
26
26
|
import { pruneCoreDumps } from "../services/core-dumps.js";
|
|
27
27
|
import { adoptOrphanRingFiles, createPacketWitness, pruneWitnessCaptures } from "../services/packet-witness.js";
|
|
28
|
+
import { createUsrsctpStateReader } from "../services/usrsctp-state.js";
|
|
28
29
|
import { startMemoryReport } from "../services/memory-report.js";
|
|
29
30
|
import { collectHealthMetrics } from "../services/health-collector.js";
|
|
30
31
|
import { createPortMapper } from "../services/port-mapper.js";
|
|
@@ -188,6 +189,9 @@ let webRtcManager = null;
|
|
|
188
189
|
*/
|
|
189
190
|
let packetWitness = null;
|
|
190
191
|
|
|
192
|
+
/** @type {ReturnType<typeof createUsrsctpStateReader> | null} */
|
|
193
|
+
let usrsctpStateReader = null;
|
|
194
|
+
|
|
191
195
|
/** @type {ReturnType<typeof createDataChannelHandler> | null} */
|
|
192
196
|
let dataChannelHandler = null;
|
|
193
197
|
|
|
@@ -335,6 +339,12 @@ try {
|
|
|
335
339
|
// the pruner recognises BEFORE anything starts a new ring over them.
|
|
336
340
|
void adoptOrphanRingFiles(packetWitness.dir).then(() => pruneWitnessCaptures(packetWitness.dir));
|
|
337
341
|
|
|
342
|
+
// Reads usrsctp's own association state via gdb the moment a wedge is
|
|
343
|
+
// declared (roadmap item 11) — no source rebuild, the module ships
|
|
344
|
+
// unstripped. A host without gdb just never gets a reading, the same way a
|
|
345
|
+
// host without tcpdump never gets a packet capture.
|
|
346
|
+
usrsctpStateReader = createUsrsctpStateReader({ log: (message) => logger.info(message) });
|
|
347
|
+
|
|
338
348
|
// What this process holds, once a minute. The kernel killed the proxy on
|
|
339
349
|
// 2026-08-28 at 2.4 GB resident and the log had never said a word about
|
|
340
350
|
// memory, so the growth that ended in that kill has no shape in any record we
|
|
@@ -464,7 +474,8 @@ try {
|
|
|
464
474
|
getTransportSnapshot: (sessionId) => webRtcManager?.getTransportSnapshot(sessionId) ?? null,
|
|
465
475
|
// Records the wire when a queue stays wedged — how the rare one-way
|
|
466
476
|
// transmit death (roadmap item 10, 2026-08-24) gets its evidence.
|
|
467
|
-
witness: packetWitness
|
|
477
|
+
witness: packetWitness,
|
|
478
|
+
usrsctpState: usrsctpStateReader
|
|
468
479
|
});
|
|
469
480
|
|
|
470
481
|
webRtcManager = createWebRtcManager({
|
package/package.json
CHANGED
|
@@ -159,7 +159,7 @@ export function wedgeIsCertain({ queuedBytes, bytesPerSecond, flatForMs, longest
|
|
|
159
159
|
* @param {DataChannel} channel
|
|
160
160
|
* @returns {() => void} Stops the watch.
|
|
161
161
|
*/
|
|
162
|
-
function makeSendQueueWatcher({ log, getTransportSnapshot, witness }) {
|
|
162
|
+
function makeSendQueueWatcher({ log, getTransportSnapshot, witness, usrsctpState }) {
|
|
163
163
|
// Every channel of one connection reads the SAME transport counters — the
|
|
164
164
|
// snapshot describes the peer connection, not the channel — so the heartbeat
|
|
165
165
|
// belongs to the connection and is printed once for it. Printed per channel
|
|
@@ -479,6 +479,14 @@ function makeSendQueueWatcher({ log, getTransportSnapshot, witness }) {
|
|
|
479
479
|
connection.captureStarted = false;
|
|
480
480
|
}
|
|
481
481
|
}
|
|
482
|
+
if (usrsctpState && verdict.certain && peerStillSending) {
|
|
483
|
+
// Its own single-flight and cooldown, independent of the witness's —
|
|
484
|
+
// one gdb attach per wedge is enough, and a refusal here (already
|
|
485
|
+
// read, cooling down) costs nothing to retry on the next tick.
|
|
486
|
+
usrsctpState.maybeRead(
|
|
487
|
+
`send queue stuck ${queued}B, accepted-byte counter unmoved ${Math.round(flatForMs / 1000)}s`
|
|
488
|
+
);
|
|
489
|
+
}
|
|
482
490
|
}, SEND_QUEUE_SAMPLE_MS);
|
|
483
491
|
|
|
484
492
|
if (typeof timer.unref === "function") {
|
|
@@ -526,7 +534,18 @@ export function encodeFrame(idBytes, bytes, done) {
|
|
|
526
534
|
return frame;
|
|
527
535
|
}
|
|
528
536
|
|
|
529
|
-
export function createDataChannelHandler({
|
|
537
|
+
export function createDataChannelHandler({
|
|
538
|
+
proxyPort,
|
|
539
|
+
onLog,
|
|
540
|
+
getTransportSnapshot,
|
|
541
|
+
sourceRegistry,
|
|
542
|
+
witness,
|
|
543
|
+
// Reads usrsctp's own association state (services/usrsctp-state.js) the
|
|
544
|
+
// moment a wedge is declared, from either detector below. Optional: a host
|
|
545
|
+
// without gdb simply never gets a reading, same as the witness without
|
|
546
|
+
// tcpdump.
|
|
547
|
+
usrsctpState
|
|
548
|
+
}) {
|
|
530
549
|
/**
|
|
531
550
|
* Channels currently interested in one file's subtitle cues, keyed by
|
|
532
551
|
* `sourceKey:fileIndex`. Populated the moment a browser asks for an
|
|
@@ -608,13 +627,26 @@ export function createDataChannelHandler({ proxyPort, onLog, getTransportSnapsho
|
|
|
608
627
|
const { watchSendQueue, readDelivery } = makeSendQueueWatcher({
|
|
609
628
|
log: (message) => log(message),
|
|
610
629
|
getTransportSnapshot,
|
|
611
|
-
witness
|
|
630
|
+
witness,
|
|
631
|
+
usrsctpState
|
|
612
632
|
});
|
|
613
633
|
// Numbered probes on every channel, and the browser's echo of what it saw.
|
|
614
634
|
// The proxy's own counters cannot say whether bytes it handed to usrsctp were
|
|
615
635
|
// ever put on the wire; the far end can, and it keeps answering throughout a
|
|
616
636
|
// freeze. See services/delivery-probe.js.
|
|
617
|
-
|
|
637
|
+
//
|
|
638
|
+
// Also the ONLY wedge signal that does not require a nonzero channel queue —
|
|
639
|
+
// `wedgeIsCertain` above needs `queuedBytes > 0`, which small, infrequent
|
|
640
|
+
// traffic never produces (confirmed 2026-08-28: the 2026-08-27 episode's
|
|
641
|
+
// ring was never saved automatically for exactly this reason). So this is
|
|
642
|
+
// wired to the same evidence-gathering as the queue watcher.
|
|
643
|
+
const deliveryProbe = createDeliveryProbe({
|
|
644
|
+
log: (message) => log(message),
|
|
645
|
+
readDelivery,
|
|
646
|
+
getTransportSnapshot,
|
|
647
|
+
witness,
|
|
648
|
+
usrsctpState
|
|
649
|
+
});
|
|
618
650
|
|
|
619
651
|
/**
|
|
620
652
|
* @param {string} message
|
|
@@ -106,6 +106,32 @@ const ECHO_STALE_FALLBACK_MS = 5_000;
|
|
|
106
106
|
/** How often the probe state is written to the log while nothing changes. */
|
|
107
107
|
const REPORT_INTERVAL_MS = 5_000;
|
|
108
108
|
|
|
109
|
+
/**
|
|
110
|
+
* Whether the `seen` counter has stopped advancing for longer than this
|
|
111
|
+
* connection's own history says a healthy gap between two advances ever
|
|
112
|
+
* takes.
|
|
113
|
+
*
|
|
114
|
+
* The `association-stopped` verdict alone is not enough to act on: a
|
|
115
|
+
* connection can sit BEHIND by a bounded, roughly constant amount for
|
|
116
|
+
* minutes (measured 2026-08-28, session on a backgrounded tab — gap held at
|
|
117
|
+
* 6-7 probes for 95+ seconds while `seen` kept climbing right along with
|
|
118
|
+
* `sent`) without anything being wrong. What a true wedge shows instead,
|
|
119
|
+
* measured the same day against a session already known to be one
|
|
120
|
+
* (`d85ae4f5`): `seen` FROZEN at one value for over a minute while `sent`
|
|
121
|
+
* climbs unbounded. So the question is not "is there a gap" but "has the
|
|
122
|
+
* highest-seen number stopped moving at all, for longer than it has ever
|
|
123
|
+
* legitimately taken this connection to report an advance" — the same shape
|
|
124
|
+
* as {@link wedgeIsCertain} in `data-channel-handler.js`, applied to the
|
|
125
|
+
* probe's own counter instead of the transport's byte counter.
|
|
126
|
+
*
|
|
127
|
+
* @param {{ stuckForMs: number, longestHealthySeenGapMs: number, intervalMs?: number }} state
|
|
128
|
+
* @returns {{ certain: boolean, needMs: number }}
|
|
129
|
+
*/
|
|
130
|
+
export function probeWedgeIsCertain({ stuckForMs, longestHealthySeenGapMs, intervalMs = PROBE_INTERVAL_MS }) {
|
|
131
|
+
const needMs = Math.max(longestHealthySeenGapMs, intervalMs);
|
|
132
|
+
return { certain: stuckForMs >= needMs, needMs };
|
|
133
|
+
}
|
|
134
|
+
|
|
109
135
|
/**
|
|
110
136
|
* One connection's probe state.
|
|
111
137
|
*
|
|
@@ -119,6 +145,9 @@ const REPORT_INTERVAL_MS = 5_000;
|
|
|
119
145
|
* @property {number} echoes - How many echoes have arrived.
|
|
120
146
|
* @property {string} verdict - Last verdict reported, so a change is logged at once.
|
|
121
147
|
* @property {number} reportedAt - When the state was last written to the log.
|
|
148
|
+
* @property {number} lastSeenAdvanceAt - When any label's `seen` value last increased (0 = never yet).
|
|
149
|
+
* @property {number} longestHealthySeenGapMs - The longest gap between two advances this connection has shown while not flagged as a wedge.
|
|
150
|
+
* @property {boolean} probeCaptureStarted - One evidence-gathering attempt per wedge; reset once `seen` advances again.
|
|
122
151
|
* @property {ReturnType<typeof setInterval> | null} timer
|
|
123
152
|
*/
|
|
124
153
|
|
|
@@ -204,6 +233,11 @@ export function readProbeState(state) {
|
|
|
204
233
|
* @param {Object} options
|
|
205
234
|
* @param {(message: string) => void} options.log
|
|
206
235
|
* @param {number} [options.intervalMs]
|
|
236
|
+
* @param {(sessionId: string) => object | null} [options.getTransportSnapshot]
|
|
237
|
+
* Needed only to hand the witness a remote endpoint when this probe is the
|
|
238
|
+
* one declaring a wedge.
|
|
239
|
+
* @param {{ maybeCapture: (trigger: object) => boolean }} [options.witness]
|
|
240
|
+
* @param {{ maybeRead: (reasonText: string) => boolean }} [options.usrsctpState]
|
|
207
241
|
* @returns {{
|
|
208
242
|
* attach: (sessionId: string, tag: string, label: string, channel: import('node-datachannel').DataChannel) => void,
|
|
209
243
|
* detach: (sessionId: string, channel: import('node-datachannel').DataChannel) => void,
|
|
@@ -211,7 +245,14 @@ export function readProbeState(state) {
|
|
|
211
245
|
* dispose: () => void
|
|
212
246
|
* }}
|
|
213
247
|
*/
|
|
214
|
-
export function createDeliveryProbe({
|
|
248
|
+
export function createDeliveryProbe({
|
|
249
|
+
log,
|
|
250
|
+
intervalMs = PROBE_INTERVAL_MS,
|
|
251
|
+
readDelivery,
|
|
252
|
+
getTransportSnapshot,
|
|
253
|
+
witness,
|
|
254
|
+
usrsctpState
|
|
255
|
+
}) {
|
|
215
256
|
/** @type {Map<string, ProbeConnection>} */
|
|
216
257
|
const connections = new Map();
|
|
217
258
|
|
|
@@ -284,6 +325,47 @@ export function createDeliveryProbe({ log, intervalMs = PROBE_INTERVAL_MS, readD
|
|
|
284
325
|
connection.reportedAt = now;
|
|
285
326
|
log(`[dc-probe] ${connection.tag} ${verdict} — ${detail} at=${new Date(now).toISOString()}`);
|
|
286
327
|
}
|
|
328
|
+
|
|
329
|
+
// `association-stopped` alone is not certainty — see probeWedgeIsCertain.
|
|
330
|
+
// A connection that is merely lagging by a bounded amount reaches this
|
|
331
|
+
// verdict too (a hidden tab's own echo cadence, measured 2026-08-28), and
|
|
332
|
+
// `seen` keeps advancing right along with it. Only a `seen` value that has
|
|
333
|
+
// stopped moving ENTIRELY, for longer than this connection has ever shown
|
|
334
|
+
// as a legitimate gap, is the wedge this exists to catch.
|
|
335
|
+
const stuckForMs = connection.lastSeenAdvanceAt === 0 ? 0 : now - connection.lastSeenAdvanceAt;
|
|
336
|
+
if (verdict === "association-stopped") {
|
|
337
|
+
const { certain, needMs } = probeWedgeIsCertain({
|
|
338
|
+
stuckForMs,
|
|
339
|
+
longestHealthySeenGapMs: connection.longestHealthySeenGapMs
|
|
340
|
+
});
|
|
341
|
+
if (certain && !connection.probeCaptureStarted) {
|
|
342
|
+
connection.probeCaptureStarted = true;
|
|
343
|
+
const reasonText =
|
|
344
|
+
`probe seen-counter unmoved ${Math.round(stuckForMs / 1000)}s against the ` +
|
|
345
|
+
`${(needMs / 1000).toFixed(1)}s this connection's own history says is legitimate`;
|
|
346
|
+
if (witness) {
|
|
347
|
+
const snapshot = getTransportSnapshot?.(connection.id) ?? null;
|
|
348
|
+
const started = witness.maybeCapture({
|
|
349
|
+
sessionId: connection.id,
|
|
350
|
+
tag: connection.tag,
|
|
351
|
+
label: "probe",
|
|
352
|
+
remote: snapshot?.remote ?? null,
|
|
353
|
+
queuedBytes: 0,
|
|
354
|
+
stuckForMs
|
|
355
|
+
});
|
|
356
|
+
if (!started) {
|
|
357
|
+
connection.probeCaptureStarted = false;
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
if (usrsctpState) {
|
|
361
|
+
usrsctpState.maybeRead(reasonText);
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
} else {
|
|
365
|
+
// Not association-stopped any more: whatever was flagged has cleared,
|
|
366
|
+
// and a later wedge on the same connection deserves its own attempt.
|
|
367
|
+
connection.probeCaptureStarted = false;
|
|
368
|
+
}
|
|
287
369
|
}
|
|
288
370
|
|
|
289
371
|
return {
|
|
@@ -305,6 +387,9 @@ export function createDeliveryProbe({ log, intervalMs = PROBE_INTERVAL_MS, readD
|
|
|
305
387
|
echoes: 0,
|
|
306
388
|
verdict: "",
|
|
307
389
|
reportedAt: 0,
|
|
390
|
+
lastSeenAdvanceAt: 0,
|
|
391
|
+
longestHealthySeenGapMs: 0,
|
|
392
|
+
probeCaptureStarted: false,
|
|
308
393
|
timer: null
|
|
309
394
|
};
|
|
310
395
|
connections.set(sessionId, connection);
|
|
@@ -342,15 +427,36 @@ export function createDeliveryProbe({ log, intervalMs = PROBE_INTERVAL_MS, readD
|
|
|
342
427
|
if (!connection || !echo || typeof echo !== "object") {
|
|
343
428
|
return;
|
|
344
429
|
}
|
|
430
|
+
const now = Date.now();
|
|
345
431
|
const seen = echo.seen;
|
|
346
432
|
if (seen && typeof seen === "object") {
|
|
433
|
+
let advanced = false;
|
|
347
434
|
for (const [label, value] of Object.entries(seen)) {
|
|
348
|
-
if (Number.isInteger(value)) {
|
|
349
|
-
|
|
435
|
+
if (!Number.isInteger(value)) {
|
|
436
|
+
continue;
|
|
350
437
|
}
|
|
438
|
+
const previous = connection.seen.get(label);
|
|
439
|
+
if (!Number.isInteger(previous) || value > previous) {
|
|
440
|
+
advanced = true;
|
|
441
|
+
}
|
|
442
|
+
connection.seen.set(label, value);
|
|
443
|
+
}
|
|
444
|
+
// What a wedge shows is this counter frozen, not merely behind — see
|
|
445
|
+
// probeWedgeIsCertain. The gap since the last time ANY label moved is
|
|
446
|
+
// this connection's own answer to "how long may a healthy report take
|
|
447
|
+
// to arrive", recorded only while nothing is currently flagged (the
|
|
448
|
+
// same guard `longestHealthyFlatMs` uses): a stretch already under
|
|
449
|
+
// suspicion must not teach the detector to tolerate it.
|
|
450
|
+
if (advanced) {
|
|
451
|
+
if (connection.lastSeenAdvanceAt !== 0 && !connection.probeCaptureStarted) {
|
|
452
|
+
const gap = now - connection.lastSeenAdvanceAt;
|
|
453
|
+
if (gap > connection.longestHealthySeenGapMs) {
|
|
454
|
+
connection.longestHealthySeenGapMs = gap;
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
connection.lastSeenAdvanceAt = now;
|
|
351
458
|
}
|
|
352
459
|
}
|
|
353
|
-
const now = Date.now();
|
|
354
460
|
if (connection.echoAt !== 0) {
|
|
355
461
|
const sinceLast = now - connection.echoAt;
|
|
356
462
|
if (sinceLast > connection.echoIntervalMs) {
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file Read usrsctp's live association state, without a rebuild.
|
|
3
|
+
*
|
|
4
|
+
* Roadmap item 11. `node_datachannel.node` ships unstripped — it carries
|
|
5
|
+
* usrsctp's own local symbols, including `system_base_info` and
|
|
6
|
+
* `usrsctp_getsockopt` — so the association's real state (peer receive
|
|
7
|
+
* window, pending data, retransmission timeout, congestion window) can be
|
|
8
|
+
* read out of THIS live process with a short gdb attach. No source rebuild,
|
|
9
|
+
* no SCTP_DEBUG image, no waiting for a packet capture to be read by eye.
|
|
10
|
+
*
|
|
11
|
+
* The walk (hash the association table, call `usrsctp_getsockopt` with
|
|
12
|
+
* `SCTP_STATUS` through the running process) and the healthy baseline it was
|
|
13
|
+
* checked against are in
|
|
14
|
+
* `research/session-2026-08-27-28-freeze-onset-and-sessions.md`, section 1.
|
|
15
|
+
* The script itself is bundled at {@link SCTPSTATE_SCRIPT_PATH} rather than
|
|
16
|
+
* hand-placed on a host, so it ships with every release instead of surviving
|
|
17
|
+
* only as long as someone remembers to copy it back after a container is
|
|
18
|
+
* recreated.
|
|
19
|
+
*
|
|
20
|
+
* `gdb` attaching with ptrace pauses every thread of the process for the
|
|
21
|
+
* duration of the read — one `getsockopt` call, measured at a fraction of a
|
|
22
|
+
* second in the manual procedure this automates.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { spawn } from "node:child_process";
|
|
26
|
+
import path from "node:path";
|
|
27
|
+
import { fileURLToPath } from "node:url";
|
|
28
|
+
|
|
29
|
+
import { shouldStartCapture, WITNESS_COOLDOWN_MS } from "./packet-witness.js";
|
|
30
|
+
|
|
31
|
+
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
32
|
+
|
|
33
|
+
/** The bundled gdb script that performs the usrsctp state walk. */
|
|
34
|
+
export const SCTPSTATE_SCRIPT_PATH = path.join(HERE, "..", "assets", "diagnostics", "sctpstate.gdb");
|
|
35
|
+
|
|
36
|
+
/** How long gdb may run before it is killed. Generous: this is a rare, one-shot read. */
|
|
37
|
+
export const GDB_TIMEOUT_MS = 15_000;
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Create the reader.
|
|
41
|
+
*
|
|
42
|
+
* Single-flight and cooldown-gated the same way the packet witness is
|
|
43
|
+
* ({@link shouldStartCapture}) — a wedge that stays certain for minutes must
|
|
44
|
+
* not spawn a fresh gdb attach on every tick, and two readings ten seconds
|
|
45
|
+
* apart tell the same story a hundred readings would.
|
|
46
|
+
*
|
|
47
|
+
* @param {Object} options
|
|
48
|
+
* @param {(message: string) => void} options.log
|
|
49
|
+
* @param {typeof spawn} [options.spawnProcess] - Seam for tests.
|
|
50
|
+
* @param {number} [options.pid] - Defaults to this process's own pid.
|
|
51
|
+
* @param {string} [options.scriptPath]
|
|
52
|
+
* @param {number} [options.cooldownMs]
|
|
53
|
+
* @returns {{ maybeRead: (reasonText: string) => boolean }}
|
|
54
|
+
*/
|
|
55
|
+
export function createUsrsctpStateReader({
|
|
56
|
+
log,
|
|
57
|
+
spawnProcess = spawn,
|
|
58
|
+
pid = process.pid,
|
|
59
|
+
scriptPath = SCTPSTATE_SCRIPT_PATH,
|
|
60
|
+
cooldownMs = WITNESS_COOLDOWN_MS
|
|
61
|
+
}) {
|
|
62
|
+
/** @type {{ running: boolean, lastStartedAt: number }} */
|
|
63
|
+
const state = { running: false, lastStartedAt: 0 };
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Read the association state now, if the gating rules allow it.
|
|
67
|
+
*
|
|
68
|
+
* @param {string} reasonText - What declared the wedge, for the log line.
|
|
69
|
+
* @returns {boolean} True when a read actually started.
|
|
70
|
+
*/
|
|
71
|
+
const maybeRead = (reasonText) => {
|
|
72
|
+
if (!shouldStartCapture({ ...state, cooldownMs })) {
|
|
73
|
+
return false;
|
|
74
|
+
}
|
|
75
|
+
state.running = true;
|
|
76
|
+
state.lastStartedAt = Date.now();
|
|
77
|
+
const startedAt = state.lastStartedAt;
|
|
78
|
+
void (async () => {
|
|
79
|
+
let out = "";
|
|
80
|
+
let err = "";
|
|
81
|
+
try {
|
|
82
|
+
await new Promise((resolve) => {
|
|
83
|
+
let settled = false;
|
|
84
|
+
let child;
|
|
85
|
+
const finish = () => {
|
|
86
|
+
if (settled) {
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
settled = true;
|
|
90
|
+
clearTimeout(killer);
|
|
91
|
+
resolve();
|
|
92
|
+
};
|
|
93
|
+
try {
|
|
94
|
+
child = spawnProcess(
|
|
95
|
+
"gdb",
|
|
96
|
+
["-q", "-batch", "-p", String(pid), "-x", scriptPath],
|
|
97
|
+
{ stdio: ["ignore", "pipe", "pipe"] }
|
|
98
|
+
);
|
|
99
|
+
} catch (error) {
|
|
100
|
+
err = `could not start gdb: ${error?.message ?? error}`;
|
|
101
|
+
resolve();
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
child.stdout?.on("data", (chunk) => {
|
|
105
|
+
out += chunk.toString();
|
|
106
|
+
});
|
|
107
|
+
child.stderr?.on("data", (chunk) => {
|
|
108
|
+
err += chunk.toString();
|
|
109
|
+
});
|
|
110
|
+
child.on("error", (error) => {
|
|
111
|
+
err += `${err ? " " : ""}gdb error: ${error?.message ?? error}`;
|
|
112
|
+
finish();
|
|
113
|
+
});
|
|
114
|
+
child.on("close", finish);
|
|
115
|
+
const killer = setTimeout(() => {
|
|
116
|
+
try {
|
|
117
|
+
child.kill("SIGKILL");
|
|
118
|
+
} catch {
|
|
119
|
+
// already gone
|
|
120
|
+
}
|
|
121
|
+
}, GDB_TIMEOUT_MS);
|
|
122
|
+
if (typeof killer.unref === "function") {
|
|
123
|
+
killer.unref();
|
|
124
|
+
}
|
|
125
|
+
});
|
|
126
|
+
} finally {
|
|
127
|
+
const lines = out.trim().length > 0 ? out.trim().split("\n") : [];
|
|
128
|
+
if (lines.length > 0) {
|
|
129
|
+
log(`usrsctp state (${reasonText}): ${lines.join(" | ")}`);
|
|
130
|
+
} else {
|
|
131
|
+
log(`usrsctp state (${reasonText}): no reading — ${err.trim() || "gdb produced no output"}`);
|
|
132
|
+
}
|
|
133
|
+
state.running = false;
|
|
134
|
+
// Same spacing rule as the packet witness: honour the cooldown even
|
|
135
|
+
// when the read ended quickly.
|
|
136
|
+
const earliestNext = startedAt + cooldownMs;
|
|
137
|
+
if (state.lastStartedAt < earliestNext) {
|
|
138
|
+
state.lastStartedAt = earliestNext;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
})();
|
|
142
|
+
return true;
|
|
143
|
+
};
|
|
144
|
+
|
|
145
|
+
return { maybeRead };
|
|
146
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import test from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
|
|
4
|
+
import { probeWedgeIsCertain, readProbeState, PROBE_INTERVAL_MS } from "../services/delivery-probe.js";
|
|
5
|
+
|
|
6
|
+
test("a seen-counter bounded lag is not a wedge, however long it lasts", () => {
|
|
7
|
+
// Session 4dcac61b, field log 2026-08-28: gap held at 6-7 probes for 95+
|
|
8
|
+
// seconds on a backgrounded tab, but `seen` kept climbing right along with
|
|
9
|
+
// `sent` — this connection's own history says gaps up to ~3.5 s (7 probes
|
|
10
|
+
// at 500 ms) are ordinary, so the same 3.5 s stuck must not read as certain.
|
|
11
|
+
const verdict = probeWedgeIsCertain({
|
|
12
|
+
stuckForMs: 3400,
|
|
13
|
+
longestHealthySeenGapMs: 3500
|
|
14
|
+
});
|
|
15
|
+
assert.equal(verdict.certain, false);
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
test("a seen-counter frozen past this connection's own worst legitimate gap is a wedge", () => {
|
|
19
|
+
// Session d85ae4f5, the same field log: `seen` frozen at one value for over
|
|
20
|
+
// a minute while `sent` climbed unbounded — this is the shape the detector
|
|
21
|
+
// exists to catch.
|
|
22
|
+
const verdict = probeWedgeIsCertain({
|
|
23
|
+
stuckForMs: 90_000,
|
|
24
|
+
longestHealthySeenGapMs: 3500
|
|
25
|
+
});
|
|
26
|
+
assert.equal(verdict.certain, true);
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
test("with no healthy history yet, one probe interval is still required", () => {
|
|
30
|
+
const verdict = probeWedgeIsCertain({
|
|
31
|
+
stuckForMs: PROBE_INTERVAL_MS - 1,
|
|
32
|
+
longestHealthySeenGapMs: 0
|
|
33
|
+
});
|
|
34
|
+
assert.equal(verdict.certain, false);
|
|
35
|
+
assert.equal(verdict.needMs, PROBE_INTERVAL_MS);
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
test("readProbeState calls association-stopped only when the unreliable channel agrees", () => {
|
|
39
|
+
// Ordered channels behind, but the unordered/no-retransmit one current:
|
|
40
|
+
// head-of-line blocking in one stream, not the association.
|
|
41
|
+
const streamStuck = readProbeState({
|
|
42
|
+
seq: 100,
|
|
43
|
+
seen: { proxy: 90, "proxy-control": 90, "proxy-fast": 99 },
|
|
44
|
+
labels: ["proxy", "proxy-control", "proxy-fast"],
|
|
45
|
+
echoes: 5,
|
|
46
|
+
echoAgeMs: 100,
|
|
47
|
+
allowed: { proxy: 2, "proxy-control": 2, "proxy-fast": 2 }
|
|
48
|
+
});
|
|
49
|
+
assert.equal(streamStuck.verdict, "stream-stuck");
|
|
50
|
+
|
|
51
|
+
const associationStopped = readProbeState({
|
|
52
|
+
seq: 100,
|
|
53
|
+
seen: { proxy: 90, "proxy-control": 90, "proxy-fast": 90 },
|
|
54
|
+
labels: ["proxy", "proxy-control", "proxy-fast"],
|
|
55
|
+
echoes: 5,
|
|
56
|
+
echoAgeMs: 100,
|
|
57
|
+
allowed: { proxy: 2, "proxy-control": 2, "proxy-fast": 2 }
|
|
58
|
+
});
|
|
59
|
+
assert.equal(associationStopped.verdict, "association-stopped");
|
|
60
|
+
});
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file The usrsctp state reader's gating and command construction.
|
|
3
|
+
*
|
|
4
|
+
* Roadmap item 11: reads usrsctp's live association state via gdb the moment
|
|
5
|
+
* a wedge is declared. Everything here is the part that decides WHEN and WITH
|
|
6
|
+
* WHAT ARGUMENTS — the spawning itself is thin glue around these, the same
|
|
7
|
+
* shape as the packet witness (test/packet-witness.test.js).
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import test from "node:test";
|
|
11
|
+
import assert from "node:assert/strict";
|
|
12
|
+
import { EventEmitter } from "node:events";
|
|
13
|
+
|
|
14
|
+
import { createUsrsctpStateReader, SCTPSTATE_SCRIPT_PATH } from "../services/usrsctp-state.js";
|
|
15
|
+
|
|
16
|
+
class FakeChild extends EventEmitter {
|
|
17
|
+
constructor(command, args) {
|
|
18
|
+
super();
|
|
19
|
+
this.command = command;
|
|
20
|
+
this.args = args;
|
|
21
|
+
this.stdout = new EventEmitter();
|
|
22
|
+
this.stderr = new EventEmitter();
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
kill() {
|
|
26
|
+
// Not exercised by these tests — every fake run finishes on its own.
|
|
27
|
+
return true;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* @param {string} output - What the fake gdb writes to stdout before closing.
|
|
33
|
+
* @returns {{ spawnProcess: Function, calls: Array<{ command: string, args: string[] }> }}
|
|
34
|
+
*/
|
|
35
|
+
function makeSpawn(output) {
|
|
36
|
+
const calls = [];
|
|
37
|
+
const spawnProcess = (command, args) => {
|
|
38
|
+
calls.push({ command, args });
|
|
39
|
+
const child = new FakeChild(command, args);
|
|
40
|
+
setImmediate(() => {
|
|
41
|
+
if (output) {
|
|
42
|
+
child.stdout.emit("data", Buffer.from(output));
|
|
43
|
+
}
|
|
44
|
+
child.emit("close", 0);
|
|
45
|
+
});
|
|
46
|
+
return child;
|
|
47
|
+
};
|
|
48
|
+
return { spawnProcess, calls };
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Wait until `check` holds, rather than for a chosen interval. */
|
|
52
|
+
async function waitFor(check) {
|
|
53
|
+
const deadline = Date.now() + 5_000;
|
|
54
|
+
for (;;) {
|
|
55
|
+
if (check()) {
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
if (Date.now() > deadline) {
|
|
59
|
+
throw new Error("timed out waiting for the reading to be logged");
|
|
60
|
+
}
|
|
61
|
+
await new Promise((resolve) => setTimeout(resolve, 5));
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
test("gdb is invoked attached to this process with the bundled script", async () => {
|
|
66
|
+
const { spawnProcess, calls } = makeSpawn("state=8 rwnd=95890\n");
|
|
67
|
+
const lines = [];
|
|
68
|
+
const reader = createUsrsctpStateReader({
|
|
69
|
+
log: (message) => lines.push(message),
|
|
70
|
+
spawnProcess,
|
|
71
|
+
pid: 4242
|
|
72
|
+
});
|
|
73
|
+
const started = reader.maybeRead("test wedge");
|
|
74
|
+
assert.equal(started, true);
|
|
75
|
+
await waitFor(() => lines.length > 0);
|
|
76
|
+
assert.equal(calls.length, 1);
|
|
77
|
+
assert.deepEqual(calls[0].args, ["-q", "-batch", "-p", "4242", "-x", SCTPSTATE_SCRIPT_PATH]);
|
|
78
|
+
assert.match(lines[0], /state=8 rwnd=95890/);
|
|
79
|
+
assert.match(lines[0], /test wedge/);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
test("no output is reported as no reading, not silence", async () => {
|
|
83
|
+
const { spawnProcess } = makeSpawn("");
|
|
84
|
+
const lines = [];
|
|
85
|
+
const reader = createUsrsctpStateReader({ log: (message) => lines.push(message), spawnProcess, pid: 1 });
|
|
86
|
+
reader.maybeRead("empty case");
|
|
87
|
+
await waitFor(() => lines.length > 0);
|
|
88
|
+
assert.match(lines[0], /no reading/);
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
test("a second read is refused within the cooldown, like the packet witness", async () => {
|
|
92
|
+
const { spawnProcess, calls } = makeSpawn("state=8\n");
|
|
93
|
+
const lines = [];
|
|
94
|
+
const reader = createUsrsctpStateReader({
|
|
95
|
+
log: (message) => lines.push(message),
|
|
96
|
+
spawnProcess,
|
|
97
|
+
pid: 1,
|
|
98
|
+
cooldownMs: 60_000
|
|
99
|
+
});
|
|
100
|
+
assert.equal(reader.maybeRead("first"), true);
|
|
101
|
+
await waitFor(() => lines.length > 0);
|
|
102
|
+
assert.equal(reader.maybeRead("second, too soon"), false);
|
|
103
|
+
assert.equal(calls.length, 1);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
test("a missing gdb is reported, not thrown", async () => {
|
|
107
|
+
const lines = [];
|
|
108
|
+
const spawnProcess = () => {
|
|
109
|
+
throw new Error("spawn gdb ENOENT");
|
|
110
|
+
};
|
|
111
|
+
const reader = createUsrsctpStateReader({ log: (message) => lines.push(message), spawnProcess, pid: 1 });
|
|
112
|
+
reader.maybeRead("no gdb on this host");
|
|
113
|
+
await waitFor(() => lines.length > 0);
|
|
114
|
+
assert.match(lines[0], /could not start gdb/);
|
|
115
|
+
});
|