@torrent-tv/proxy 2.55.8 → 2.55.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/bin/cli.js +30 -1
- package/package.json +1 -1
- package/services/data-channel-handler.js +48 -6
- package/services/packet-witness.js +406 -0
- package/services/webrtc-manager.js +11 -5
- package/test/packet-witness.test.js +120 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,11 @@
|
|
|
1
|
+
## 2.55.10
|
|
2
|
+
|
|
3
|
+
- **Change**: Add `--sctp-debug` (off by default). When enabled, the proxy calls `node-datachannel.initLogger('Verbose')` early so SCTP-level lines (`usrsctp: …`) become visible. Useful only with an image rebuilt with `SCTP_DEBUG=ON` (addon 0.48.0) — there they carry the SACK `a_rwnd` and gap information that separates the two remaining hypotheses for the delivery-side freeze of 2026-08-24/25.
|
|
4
|
+
|
|
5
|
+
## 2.55.9
|
|
6
|
+
|
|
7
|
+
- **New**: A send queue that stays wedged for over 30 s now records the wire itself. Field session 2026-08-24 (`research/dead-channel-2026-08-24.md`): the proxy counted bytes as sent that never reached the viewer's SCTP stack, and every counter above the wire reported success for 88 minutes — the two candidate causes inside SCTP separate by one look at the packets (duplicate SACKs naming a missing TSN with no retransmission vs SACKs advertising `a_rwnd=0`), but occurrences are rare, so waiting to be asked meant waiting forever. When the stuck warning crosses 30 s, the proxy spawns a bounded tcpdump on the WebRTC UDP port filtered to that session's remote address: snaplen 128 B, ring of 4 × 30 s files beside the core dumps (`--state-dir`), killed after 120 s + grace, one capture at a time with a 10-minute cooldown, old captures pruned at startup the way core dumps are. Where no tcpdump exists it degrades to a single log line; the address travels into the filter only as a validated IPv4/IPv6 literal (zone suffixes stripped), spawned as an argv array without a shell.
|
|
8
|
+
|
|
1
9
|
## 2.55.8
|
|
2
10
|
|
|
3
11
|
- **Fix**: The proxy died twice in one evening (2026-08-22, 15:50:12 and 16:12:21 UTC) with no stop order given, and both deaths are the same fault. The torrent worker thread ends itself when its event loop drains — every recurring interval there is unref'd, upload is disabled by default, and idle peer connections close about half a minute after the traffic stops — so when a viewer paused or left, the thread finished ~35 s later on its own and Node began tearing it down. That teardown touched memory already freed or overwritten (SIGSEGV inside `uv_timer_stop`, reached through `PerIsolatePlatformData::Shutdown`; two core dumps captured identical stacks), and a fault in any thread kills the whole process instantly — HTTP server, tunnel and data channels together, with no log line and no way to restart anything from inside. The HA supervisor restarted the container each time (~15 s), but the browser's reconnect ladder had already given up by then. The worker now keeps ONE interval accounted for (no `.unref()`): an empty tick every 5 s costs nothing, the event loop can never drain while the process lives, and the teardown path — with whatever structure is corrupted inside it — stays unreachable, regardless of which native module is guilty; the three earlier crashes of this family (2026-08-18..21) stay documented under roadmap item 1.
|
package/bin/cli.js
CHANGED
|
@@ -24,6 +24,7 @@ import { createTunnelClient } from "../services/tunnel-client.js";
|
|
|
24
24
|
import { createWebRtcManager } from "../services/webrtc-manager.js";
|
|
25
25
|
import { createDataChannelHandler } from "../services/data-channel-handler.js";
|
|
26
26
|
import { pruneCoreDumps } from "../services/core-dumps.js";
|
|
27
|
+
import { createPacketWitness, pruneWitnessCaptures } from "../services/packet-witness.js";
|
|
27
28
|
import { collectHealthMetrics } from "../services/health-collector.js";
|
|
28
29
|
import { createPortMapper } from "../services/port-mapper.js";
|
|
29
30
|
import { classifyNat } from "../services/nat-classifier.js";
|
|
@@ -96,6 +97,7 @@ program
|
|
|
96
97
|
DEFAULT_SEGMENT_FORMAT_ID
|
|
97
98
|
)
|
|
98
99
|
.option("--token <token>", "Registration token", "")
|
|
100
|
+
.option("--sctp-debug", "Enable verbose SCTP debug logging (SCTP_DEBUG build only)")
|
|
99
101
|
.addHelpText("after", HELP_EXAMPLES);
|
|
100
102
|
|
|
101
103
|
program.parse(process.argv);
|
|
@@ -281,6 +283,23 @@ async function shutdown(signal) {
|
|
|
281
283
|
|
|
282
284
|
try {
|
|
283
285
|
logToFile(options.logFile);
|
|
286
|
+
// Diagnostic: verbose SCTP logging. The binary must have been built with
|
|
287
|
+
// SCTP_DEBUG=ON (addon 0.48.0); this flag merely sets the log level.
|
|
288
|
+
if (options.sctpDebug) {
|
|
289
|
+
try {
|
|
290
|
+
const dc = await import("node-datachannel");
|
|
291
|
+
const initLogger =
|
|
292
|
+
// node-datachannel exports initLogger at the top level; be defensive
|
|
293
|
+
// about CJS-default interop.
|
|
294
|
+
dc.initLogger ?? dc.default?.initLogger ?? null;
|
|
295
|
+
if (typeof initLogger === "function") {
|
|
296
|
+
initLogger("Verbose");
|
|
297
|
+
logger.info("SCTP debug verbose logging enabled");
|
|
298
|
+
}
|
|
299
|
+
} catch (error) {
|
|
300
|
+
logger.warn(`Failed to enable SCTP debug: ${error?.message ?? error}`);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
284
303
|
if (transcodeAudio) {
|
|
285
304
|
assertFfmpegAvailability();
|
|
286
305
|
}
|
|
@@ -307,6 +326,13 @@ try {
|
|
|
307
326
|
// field host, and four of them nearly filled a 235 GB disk. Keep the newest
|
|
308
327
|
// two, which are the evidence for the fault still open, and drop the rest.
|
|
309
328
|
void pruneCoreDumps(options.stateDir);
|
|
329
|
+
// Same policy for the packet captures the send-queue witness writes.
|
|
330
|
+
const packetWitness = createPacketWitness({
|
|
331
|
+
log: (message) => logger.info(message),
|
|
332
|
+
dir: options.stateDir || "",
|
|
333
|
+
port: actualPort
|
|
334
|
+
});
|
|
335
|
+
void pruneWitnessCaptures(packetWitness.dir);
|
|
310
336
|
|
|
311
337
|
logger.info(`Starting @torrent-tv/proxy v${PROXY_VERSION}`);
|
|
312
338
|
logger.info(`Local stream endpoint: http://${bindHost}:${actualPort}/stream`);
|
|
@@ -428,7 +454,10 @@ try {
|
|
|
428
454
|
sourceRegistry,
|
|
429
455
|
// Lets a stuck send queue ask the transport what it is doing. Late-bound:
|
|
430
456
|
// the manager is created below, with this handler already in hand.
|
|
431
|
-
getTransportSnapshot: (sessionId) => webRtcManager?.getTransportSnapshot(sessionId) ?? null
|
|
457
|
+
getTransportSnapshot: (sessionId) => webRtcManager?.getTransportSnapshot(sessionId) ?? null,
|
|
458
|
+
// Records the wire when a queue stays wedged — how the rare one-way
|
|
459
|
+
// transmit death (roadmap item 10, 2026-08-24) gets its evidence.
|
|
460
|
+
witness: packetWitness
|
|
432
461
|
});
|
|
433
462
|
|
|
434
463
|
webRtcManager = createWebRtcManager({
|
package/package.json
CHANGED
|
@@ -54,6 +54,15 @@ import { deriveSourceKey } from "./torrent-source-key.js";
|
|
|
54
54
|
* Incoming requests are forwarded to `http://127.0.0.1:{proxyPort}`.
|
|
55
55
|
* @property {(message: string) => void} [onLog]
|
|
56
56
|
* Optional log sink.
|
|
57
|
+
* @property {{ maybeCapture: (trigger: {
|
|
58
|
+
* sessionId: string, tag: string, label: string,
|
|
59
|
+
* remote: { address: string, port: number } | null,
|
|
60
|
+
* queuedBytes: number, stuckForMs: number
|
|
61
|
+
* }) => boolean }} [witness]
|
|
62
|
+
* The packet witness (services/packet-witness.js). When a wedged queue
|
|
63
|
+
* crosses {@linkcode SEND_QUEUE_CAPTURE_AFTER_MS} the watcher hands it the
|
|
64
|
+
* transport snapshot's remote endpoint so a bounded tcpdump can record what
|
|
65
|
+
* the wire actually did. Optional; absent means no captures are taken.
|
|
57
66
|
*/
|
|
58
67
|
|
|
59
68
|
/**
|
|
@@ -108,7 +117,7 @@ import { deriveSourceKey } from "./torrent-source-key.js";
|
|
|
108
117
|
* @param {DataChannel} channel
|
|
109
118
|
* @returns {() => void} Stops the watch.
|
|
110
119
|
*/
|
|
111
|
-
function makeSendQueueWatcher({ log, getTransportSnapshot }) {
|
|
120
|
+
function makeSendQueueWatcher({ log, getTransportSnapshot, witness }) {
|
|
112
121
|
// Every channel of one connection reads the SAME transport counters — the
|
|
113
122
|
// snapshot describes the peer connection, not the channel — so the heartbeat
|
|
114
123
|
// belongs to the connection and is printed once for it. Printed per channel
|
|
@@ -123,8 +132,10 @@ function makeSendQueueWatcher({ log, getTransportSnapshot }) {
|
|
|
123
132
|
// a label is whatever the peer chose and two channels can carry the same one
|
|
124
133
|
// (or none, where `getLabel` is missing and both fall back to "?"), and a
|
|
125
134
|
// Map keyed on that would let one channel evict the other and then, on
|
|
126
|
-
// closing, delete the survivor's entry.
|
|
127
|
-
|
|
135
|
+
// closing, delete the survivor's entry. `captureStarted` rides on the same
|
|
136
|
+
// record: both channels of one wedged connection must ask the witness once,
|
|
137
|
+
// not once per channel.
|
|
138
|
+
/** @type {Map<string, { channels: Map<DataChannel, string>, at: number, previous: object | null, unknown: number, captureStarted: boolean }>} */
|
|
128
139
|
const connections = new Map();
|
|
129
140
|
|
|
130
141
|
/**
|
|
@@ -153,7 +164,7 @@ function makeSendQueueWatcher({ log, getTransportSnapshot }) {
|
|
|
153
164
|
let previous = null;
|
|
154
165
|
let connection = connections.get(sessionId);
|
|
155
166
|
if (!connection) {
|
|
156
|
-
connection = { channels: new Map(), at: 0, previous: null, unknown: 0 };
|
|
167
|
+
connection = { channels: new Map(), at: 0, previous: null, unknown: 0, captureStarted: false };
|
|
157
168
|
connections.set(sessionId, connection);
|
|
158
169
|
}
|
|
159
170
|
connection.channels.set(channel, label);
|
|
@@ -256,6 +267,31 @@ function makeSendQueueWatcher({ log, getTransportSnapshot }) {
|
|
|
256
267
|
`received=${snapshot.bytesReceived}${recvDelta === null ? "" : ` (+${recvDelta})`} ` +
|
|
257
268
|
`rtt=${snapshot.rtt}ms pc=${snapshot.state} ice=${snapshot.iceState} pair=${snapshot.pair}`
|
|
258
269
|
);
|
|
270
|
+
// Roadmap item 10, occurrence 2026-08-24: a wedge this old is the moment
|
|
271
|
+
// the packet-level truth has to be caught, because no counter above the
|
|
272
|
+
// wire can name the cause. One attempt per connection — the witness
|
|
273
|
+
// applies its own single-flight and cooldown rules after that.
|
|
274
|
+
if (
|
|
275
|
+
witness &&
|
|
276
|
+
!connection.captureStarted &&
|
|
277
|
+
now - stuckSince >= SEND_QUEUE_CAPTURE_AFTER_MS
|
|
278
|
+
) {
|
|
279
|
+
connection.captureStarted = true;
|
|
280
|
+
const started = witness.maybeCapture({
|
|
281
|
+
sessionId,
|
|
282
|
+
tag,
|
|
283
|
+
label,
|
|
284
|
+
remote: snapshot.remote ?? null,
|
|
285
|
+
queuedBytes: queued,
|
|
286
|
+
stuckForMs: now - stuckSince
|
|
287
|
+
});
|
|
288
|
+
if (!started) {
|
|
289
|
+
// Refused for now (no remote endpoint yet, capture already running
|
|
290
|
+
// elsewhere, cooldown): let the next tick try again rather than
|
|
291
|
+
// spending the one attempt per wedge on a refusal.
|
|
292
|
+
connection.captureStarted = false;
|
|
293
|
+
}
|
|
294
|
+
}
|
|
259
295
|
}, SEND_QUEUE_SAMPLE_MS);
|
|
260
296
|
|
|
261
297
|
if (typeof timer.unref === "function") {
|
|
@@ -301,7 +337,7 @@ export function encodeFrame(idBytes, bytes, done) {
|
|
|
301
337
|
return frame;
|
|
302
338
|
}
|
|
303
339
|
|
|
304
|
-
export function createDataChannelHandler({ proxyPort, onLog, getTransportSnapshot, sourceRegistry }) {
|
|
340
|
+
export function createDataChannelHandler({ proxyPort, onLog, getTransportSnapshot, sourceRegistry, witness }) {
|
|
305
341
|
/**
|
|
306
342
|
* Channels currently interested in one file's subtitle cues, keyed by
|
|
307
343
|
* `sourceKey:fileIndex`. Populated the moment a browser asks for an
|
|
@@ -380,7 +416,7 @@ export function createDataChannelHandler({ proxyPort, onLog, getTransportSnapsho
|
|
|
380
416
|
/** Request id → its ASCII bytes; see {@link requestIdBytes}. */
|
|
381
417
|
const requestIdCache = new Map();
|
|
382
418
|
|
|
383
|
-
const watchSendQueue = makeSendQueueWatcher({ log: (message) => log(message), getTransportSnapshot });
|
|
419
|
+
const watchSendQueue = makeSendQueueWatcher({ log: (message) => log(message), getTransportSnapshot, witness });
|
|
384
420
|
|
|
385
421
|
/**
|
|
386
422
|
* @param {string} message
|
|
@@ -867,6 +903,12 @@ const TRANSPORT_HEARTBEAT_MS = 5_000;
|
|
|
867
903
|
// registry does not end a healthy watch.
|
|
868
904
|
const TRANSPORT_UNKNOWN_HEARTBEATS = 3;
|
|
869
905
|
const SEND_QUEUE_STUCK_MS = 5_000;
|
|
906
|
+
// How long a wedged queue waits before the packet witness starts recording the
|
|
907
|
+
// wire. Long enough to be certain this is not a slow drain (a 6-11 MB segment
|
|
908
|
+
// leaves in well under a second, measured), short enough that the capture is
|
|
909
|
+
// still running while whatever broke is breaking — the 2026-08-24 episode
|
|
910
|
+
// began minutes before anyone could have asked for a capture by hand.
|
|
911
|
+
const SEND_QUEUE_CAPTURE_AFTER_MS = 30_000;
|
|
870
912
|
const DC_BUFFER_HIGH_WATER = 8 * 1024 * 1024;
|
|
871
913
|
/** Resume sending once the channel buffer drains to this many bytes. */
|
|
872
914
|
const DC_BUFFER_LOW_WATER = 1 * 1024 * 1024;
|
|
@@ -0,0 +1,406 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file Automatic packet witness for a send queue that stops draining.
|
|
3
|
+
*
|
|
4
|
+
* Roadmap item 10, occurrence 2026-08-24 (`research/dead-channel-2026-08-24.md`):
|
|
5
|
+
* the proxy counted bytes as sent that never reached the viewer's SCTP stack,
|
|
6
|
+
* and every counter above the wire reported success for 88 minutes. The two
|
|
7
|
+
* candidate causes separate by ONE look at the packets — repeated duplicate
|
|
8
|
+
* SACKs naming a missing TSN with no retransmission (usrsctp retransmit
|
|
9
|
+
* defect) versus SACKs advertising `a_rwnd=0` with no gaps (receiver stopped
|
|
10
|
+
* reading). Occurrences are rare and unpredictable, so the witness fires by
|
|
11
|
+
* itself: a send queue wedged longer than ~30 s starts a bounded tcpdump on
|
|
12
|
+
* the WebRTC UDP port filtered to that session's remote address, writes a few
|
|
13
|
+
* small rotating files, and stops. Where no tcpdump exists the whole thing
|
|
14
|
+
* degrades to a single log line.
|
|
15
|
+
*
|
|
16
|
+
* Bounded three ways so it can never cost more than an episode is worth: the
|
|
17
|
+
* process is killed after {@linkcode WITNESS_CAPTURE_SECONDS} plus grace; the
|
|
18
|
+
* ring keeps at most {@linkcode WITNESS_RING_FILES} files of
|
|
19
|
+
* {@linkcode WITNESS_ROTATE_SECONDS} each; and one capture runs at a time,
|
|
20
|
+
* with a cooldown before the next. Captures land beside the core dumps
|
|
21
|
+
* (`--state-dir`) and are pruned at startup the same way.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { spawn } from "node:child_process";
|
|
25
|
+
import { readdir, rm, stat } from "node:fs/promises";
|
|
26
|
+
import os from "node:os";
|
|
27
|
+
import path from "node:path";
|
|
28
|
+
|
|
29
|
+
import { dumpsToRemove } from "./core-dumps.js";
|
|
30
|
+
|
|
31
|
+
/** Per-packet capture length. Small: headers are what the signatures need. */
|
|
32
|
+
export const WITNESS_SNAPLEN_BYTES = 128;
|
|
33
|
+
|
|
34
|
+
/** One rotation step of the ring, in seconds. */
|
|
35
|
+
export const WITNESS_ROTATE_SECONDS = 30;
|
|
36
|
+
|
|
37
|
+
/** How many rotation steps stay on disk (30 s × 4 = 120 s of evidence). */
|
|
38
|
+
export const WITNESS_RING_FILES = 4;
|
|
39
|
+
|
|
40
|
+
/** Total wall-clock bound on one capture, seconds. */
|
|
41
|
+
export const WITNESS_CAPTURE_SECONDS = WITNESS_ROTATE_SECONDS * WITNESS_RING_FILES;
|
|
42
|
+
|
|
43
|
+
/** Extra time before the SIGKILL fallback lands on a hanging tcpdump. */
|
|
44
|
+
export const WITNESS_KILL_GRACE_MS = 5_000;
|
|
45
|
+
|
|
46
|
+
/** Minimum spacing between captures, whatever the reason for them. */
|
|
47
|
+
export const WITNESS_COOLDOWN_MS = 10 * 60_000;
|
|
48
|
+
|
|
49
|
+
/** How many old captures survive at startup, newest first. */
|
|
50
|
+
export const WITNESS_CAPTURES_KEPT = 4;
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* The remote endpoint of a transport snapshot, already structured.
|
|
54
|
+
*
|
|
55
|
+
* @typedef {Object} WitnessRemoteEndpoint
|
|
56
|
+
* @property {string} address - IP literal (may carry a `%zone` suffix).
|
|
57
|
+
* @property {number} port
|
|
58
|
+
*/
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* What the watcher hands over when a wedge crosses the capture threshold.
|
|
62
|
+
*
|
|
63
|
+
* @typedef {Object} WitnessTrigger
|
|
64
|
+
* @property {string} sessionId
|
|
65
|
+
* @property {string} tag - Session id, first 8 characters.
|
|
66
|
+
* @property {string} label - Data channel label ("proxy", "proxy-control").
|
|
67
|
+
* @property {WitnessRemoteEndpoint | null} remote
|
|
68
|
+
* @property {number} queuedBytes
|
|
69
|
+
* @property {number} stuckForMs
|
|
70
|
+
*/
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Strip a zone suffix and keep only IPv4/IPv6 literals.
|
|
74
|
+
*
|
|
75
|
+
* The address travels into a tcpdump BPF filter argument. The child is spawned
|
|
76
|
+
* as an argv array (no shell), so injection is not reachable, but a garbage or
|
|
77
|
+
* hostname value would produce a capture of nothing — rejected instead, so the
|
|
78
|
+
* log can say why nothing was written.
|
|
79
|
+
*
|
|
80
|
+
* @param {unknown} raw - Address as libdatachannel reports it.
|
|
81
|
+
* @returns {string | null} A clean literal, or null when not usable.
|
|
82
|
+
*/
|
|
83
|
+
export function normalizeRemoteAddress(raw) {
|
|
84
|
+
if (typeof raw !== "string") {
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
const zoneFree = raw.replace(/%.*$/, "");
|
|
88
|
+
if (zoneFree.length === 0 || zoneFree.length > 45) {
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
const ipv4 = /^(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}$/;
|
|
92
|
+
// Hex groups and at most one "::" compression — enough to keep hostnames,
|
|
93
|
+
// decimal-octet junk and shell metacharacters out of the filter.
|
|
94
|
+
const ipv6 = /^(?:[0-9A-Fa-f]{0,4}:){2,7}[0-9A-Fa-f]{0,4}$/;
|
|
95
|
+
if (ipv4.test(zoneFree) || (zoneFree.includes(":") && ipv6.test(zoneFree))) {
|
|
96
|
+
return zoneFree;
|
|
97
|
+
}
|
|
98
|
+
return null;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* The tcpdump command line for one bounded capture.
|
|
103
|
+
*
|
|
104
|
+
* `-i any` covers hosts where the UDP mux socket is not on a named interface;
|
|
105
|
+
* `-n` keeps DNS out of the hot path; `-S` prints absolute sequence numbers so
|
|
106
|
+
* duplicate SACKs across file rotations compare equal by eye.
|
|
107
|
+
*
|
|
108
|
+
* @param {Object} parts
|
|
109
|
+
* @param {string} parts.host - Validated IP literal.
|
|
110
|
+
* @param {number} parts.port - The WebRTC UDP port.
|
|
111
|
+
* @param {string} parts.filePrefix - Path prefix; tcpdump appends timestamps.
|
|
112
|
+
* @param {number} [parts.snaplen] - Bytes per packet to store.
|
|
113
|
+
* @param {number} [parts.rotateSeconds]
|
|
114
|
+
* @param {number} [parts.ringFiles]
|
|
115
|
+
* @returns {string[]}
|
|
116
|
+
*/
|
|
117
|
+
export function buildTcpdumpArgs({
|
|
118
|
+
host,
|
|
119
|
+
port,
|
|
120
|
+
filePrefix,
|
|
121
|
+
snaplen = WITNESS_SNAPLEN_BYTES,
|
|
122
|
+
rotateSeconds = WITNESS_ROTATE_SECONDS,
|
|
123
|
+
ringFiles = WITNESS_RING_FILES
|
|
124
|
+
}) {
|
|
125
|
+
return [
|
|
126
|
+
"-n",
|
|
127
|
+
"-S",
|
|
128
|
+
"-s",
|
|
129
|
+
String(snaplen),
|
|
130
|
+
"-i",
|
|
131
|
+
"any",
|
|
132
|
+
"-w",
|
|
133
|
+
filePrefix,
|
|
134
|
+
"-G",
|
|
135
|
+
String(rotateSeconds),
|
|
136
|
+
"-W",
|
|
137
|
+
String(ringFiles),
|
|
138
|
+
"udp",
|
|
139
|
+
"and",
|
|
140
|
+
"host",
|
|
141
|
+
host,
|
|
142
|
+
"and",
|
|
143
|
+
"port",
|
|
144
|
+
String(port)
|
|
145
|
+
];
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Whether a new capture may start right now.
|
|
150
|
+
*
|
|
151
|
+
* Pure, so the gating rule is testable without clocks or processes: never two
|
|
152
|
+
* at once, and none within the cooldown of the previous one. `lastStartedAt`
|
|
153
|
+
* of 0 means this process has not captured yet.
|
|
154
|
+
*
|
|
155
|
+
* @param {{ running: boolean, lastStartedAt: number, now?: number, cooldownMs?: number }} state
|
|
156
|
+
* @returns {boolean}
|
|
157
|
+
*/
|
|
158
|
+
export function shouldStartCapture({ running, lastStartedAt, now = Date.now(), cooldownMs = WITNESS_COOLDOWN_MS }) {
|
|
159
|
+
if (running) {
|
|
160
|
+
return false;
|
|
161
|
+
}
|
|
162
|
+
return lastStartedAt <= 0 || now - lastStartedAt >= cooldownMs;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Whether a file name is a capture this module wrote (including the
|
|
167
|
+
* timestamp-suffixed rotations tcpdump appends under `-G`).
|
|
168
|
+
*
|
|
169
|
+
* @param {string} name
|
|
170
|
+
* @returns {boolean}
|
|
171
|
+
*/
|
|
172
|
+
export function isWitnessCapture(name) {
|
|
173
|
+
return typeof name === "string" && /^packet-witness\.[^/\\]+\.pcap/.test(name);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Delete all but the newest few captures in `dir`.
|
|
178
|
+
*
|
|
179
|
+
* Mirrors `pruneCoreDumps`: best-effort, never fatal, one summary line.
|
|
180
|
+
*
|
|
181
|
+
* @param {string} dir
|
|
182
|
+
* @param {number} [keep]
|
|
183
|
+
* @returns {Promise<void>}
|
|
184
|
+
*/
|
|
185
|
+
export async function pruneWitnessCaptures(dir, keep = WITNESS_CAPTURES_KEPT) {
|
|
186
|
+
if (typeof dir !== "string" || dir.length === 0) {
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
/** @type {Array<{ name: string, writtenAt: number, bytes: number }>} */
|
|
190
|
+
const captures = [];
|
|
191
|
+
try {
|
|
192
|
+
for (const name of await readdir(dir)) {
|
|
193
|
+
if (!isWitnessCapture(name)) {
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
try {
|
|
197
|
+
const info = await stat(path.join(dir, name));
|
|
198
|
+
if (info.isFile()) {
|
|
199
|
+
captures.push({ name, writtenAt: info.mtimeMs, bytes: info.size });
|
|
200
|
+
}
|
|
201
|
+
} catch {
|
|
202
|
+
// silent-ok: vanished between listing and reading.
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
} catch {
|
|
206
|
+
return; // No such directory, or unreadable. Nothing to tidy.
|
|
207
|
+
}
|
|
208
|
+
if (captures.length === 0) {
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
const doomed = dumpsToRemove(captures, keep);
|
|
212
|
+
const freed = captures
|
|
213
|
+
.filter((capture) => doomed.includes(capture.name))
|
|
214
|
+
.reduce((total, capture) => total + capture.bytes, 0);
|
|
215
|
+
for (const name of doomed) {
|
|
216
|
+
try {
|
|
217
|
+
await rm(path.join(dir, name), { force: true });
|
|
218
|
+
} catch {
|
|
219
|
+
// silent-ok: best effort, retried at the next start.
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
const keptBytes = captures
|
|
223
|
+
.filter((capture) => !doomed.includes(capture.name))
|
|
224
|
+
.reduce((total, capture) => total + capture.bytes, 0);
|
|
225
|
+
if (freed > 0 || captures.length > keep) {
|
|
226
|
+
logLine(
|
|
227
|
+
`packet witness: ${captures.length} capture(s) present, keeping the newest ${Math.min(keep, captures.length)}, ` +
|
|
228
|
+
`removed ${doomed.length} (${(freed / 1024).toFixed(1)} KB), keeping ${(keptBytes / 1024).toFixed(1)} KB`
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* The logger handed to {@linkcode createPacketWitness}; module-level so the
|
|
235
|
+
* pruner can speak without an options bag threaded everywhere.
|
|
236
|
+
*
|
|
237
|
+
* @type {(message: string) => void}
|
|
238
|
+
*/
|
|
239
|
+
let logLine = () => {};
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* Create the witness.
|
|
243
|
+
*
|
|
244
|
+
* @param {Object} options
|
|
245
|
+
* @param {(message: string) => void} options.log - Log sink (the shared logger).
|
|
246
|
+
* @param {string} [options.dir] - Where captures go; empty means os.tmpdir().
|
|
247
|
+
* @param {number} options.port - The WebRTC UDP port to filter on.
|
|
248
|
+
* @returns {{ dir: string, maybeCapture: (trigger: WitnessTrigger) => boolean }}
|
|
249
|
+
*/
|
|
250
|
+
export function createPacketWitness({ log, dir = "", port }) {
|
|
251
|
+
logLine = typeof log === "function" ? log : logLine;
|
|
252
|
+
const resolvedDir = typeof dir === "string" && dir.length > 0 ? dir : os.tmpdir();
|
|
253
|
+
|
|
254
|
+
/** @type {{ running: boolean, lastStartedAt: number, availability: "unknown" | "yes" | "no" }} */
|
|
255
|
+
const state = { running: false, lastStartedAt: 0, availability: "unknown" };
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* Ask whether tcpdump exists at all — once per process, whichever way it
|
|
259
|
+
* answers. A host without it gets exactly one log line, ever.
|
|
260
|
+
*
|
|
261
|
+
* @returns {Promise<boolean>}
|
|
262
|
+
*/
|
|
263
|
+
const probeAvailability = () =>
|
|
264
|
+
new Promise((resolve) => {
|
|
265
|
+
let settled = false;
|
|
266
|
+
const done = (value) => {
|
|
267
|
+
if (!settled) {
|
|
268
|
+
settled = true;
|
|
269
|
+
resolve(value);
|
|
270
|
+
}
|
|
271
|
+
};
|
|
272
|
+
try {
|
|
273
|
+
const probe = spawn("tcpdump", ["--version"], { stdio: "ignore" });
|
|
274
|
+
probe.on("error", () => done(false));
|
|
275
|
+
probe.on("spawn", () => done(true));
|
|
276
|
+
probe.on("close", () => done(true));
|
|
277
|
+
} catch {
|
|
278
|
+
done(false);
|
|
279
|
+
}
|
|
280
|
+
});
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* Run one bounded capture and report what it wrote.
|
|
284
|
+
*
|
|
285
|
+
* @param {WitnessTrigger} trigger
|
|
286
|
+
* @param {string} address - Validated remote IP literal.
|
|
287
|
+
* @returns {Promise<void>}
|
|
288
|
+
*/
|
|
289
|
+
const runCapture = async (trigger, address) => {
|
|
290
|
+
const prefix = path.join(
|
|
291
|
+
resolvedDir,
|
|
292
|
+
`packet-witness.${trigger.tag}.${Math.floor(Date.now() / 1000)}.pcap`
|
|
293
|
+
);
|
|
294
|
+
const args = buildTcpdumpArgs({ host: address, port, filePrefix: prefix });
|
|
295
|
+
logLine(
|
|
296
|
+
`packet witness: capturing ${WITNESS_CAPTURE_SECONDS}s of udp port ${port} ↔ ${address} ` +
|
|
297
|
+
`(session ${trigger.tag}, "${trigger.label}" queue ${trigger.queuedBytes}B wedged ` +
|
|
298
|
+
`${Math.round(trigger.stuckForMs / 1000)}s) → ${prefix}`
|
|
299
|
+
);
|
|
300
|
+
await new Promise((resolve) => {
|
|
301
|
+
/** @type {ReturnType<typeof setTimeout> | null} */
|
|
302
|
+
let killer = null;
|
|
303
|
+
/** @type {NodeJS.Timeout | null} */
|
|
304
|
+
let hardKill = null;
|
|
305
|
+
let child;
|
|
306
|
+
try {
|
|
307
|
+
child = spawn("tcpdump", args, { stdio: "ignore" });
|
|
308
|
+
} catch (error) {
|
|
309
|
+
logLine(`packet witness: could not start tcpdump: ${error?.message ?? error}`);
|
|
310
|
+
resolve();
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
const finish = () => {
|
|
314
|
+
if (killer) {
|
|
315
|
+
clearTimeout(killer);
|
|
316
|
+
killer = null;
|
|
317
|
+
}
|
|
318
|
+
if (hardKill) {
|
|
319
|
+
clearTimeout(hardKill);
|
|
320
|
+
hardKill = null;
|
|
321
|
+
}
|
|
322
|
+
resolve();
|
|
323
|
+
};
|
|
324
|
+
child.on("error", (error) => {
|
|
325
|
+
state.availability = "no";
|
|
326
|
+
logLine(`packet witness: tcpdump failed to run: ${error?.message ?? error}`);
|
|
327
|
+
finish();
|
|
328
|
+
});
|
|
329
|
+
child.on("close", (code, signal) => {
|
|
330
|
+
logLine(
|
|
331
|
+
`packet witness: capture ended${signal ? ` (${signal})` : ` (exit ${code ?? "?"})`}`
|
|
332
|
+
);
|
|
333
|
+
finish();
|
|
334
|
+
});
|
|
335
|
+
killer = setTimeout(() => {
|
|
336
|
+
try { child.kill("SIGTERM"); } catch { /* already gone */ }
|
|
337
|
+
hardKill = setTimeout(() => {
|
|
338
|
+
try { child.kill("SIGKILL"); } catch { /* already gone */ }
|
|
339
|
+
}, WITNESS_KILL_GRACE_MS);
|
|
340
|
+
}, WITNESS_CAPTURE_SECONDS * 1000);
|
|
341
|
+
});
|
|
342
|
+
// Say what landed on disk, so whoever reads the log later knows whether
|
|
343
|
+
// the evidence exists without listing the directory themselves.
|
|
344
|
+
try {
|
|
345
|
+
const names = (await readdir(resolvedDir)).filter((name) => isWitnessCapture(name));
|
|
346
|
+
const mine = [];
|
|
347
|
+
for (const name of names) {
|
|
348
|
+
if (!name.startsWith(path.basename(prefix))) {
|
|
349
|
+
continue;
|
|
350
|
+
}
|
|
351
|
+
try {
|
|
352
|
+
const info = await stat(path.join(resolvedDir, name));
|
|
353
|
+
mine.push(`${name} ${(info.size / 1024).toFixed(1)} KB`);
|
|
354
|
+
} catch { /* gone between listing and reading */ }
|
|
355
|
+
}
|
|
356
|
+
logLine(`packet witness: wrote ${mine.length} file(s): ${mine.join(", ") || "none"}`);
|
|
357
|
+
} catch {
|
|
358
|
+
logLine("packet witness: could not list the capture directory");
|
|
359
|
+
}
|
|
360
|
+
};
|
|
361
|
+
|
|
362
|
+
/**
|
|
363
|
+
* Start a capture for this trigger, if the rules allow one.
|
|
364
|
+
*
|
|
365
|
+
* @param {WitnessTrigger} trigger
|
|
366
|
+
* @returns {boolean} True when a capture actually started.
|
|
367
|
+
*/
|
|
368
|
+
const maybeCapture = (trigger) => {
|
|
369
|
+
const address = normalizeRemoteAddress(trigger?.remote?.address);
|
|
370
|
+
const portNumber = trigger?.remote?.port;
|
|
371
|
+
if (!address || !Number.isInteger(portNumber) || portNumber < 1 || portNumber > 65535) {
|
|
372
|
+
return false;
|
|
373
|
+
}
|
|
374
|
+
if (!shouldStartCapture(state)) {
|
|
375
|
+
return false;
|
|
376
|
+
}
|
|
377
|
+
state.running = true;
|
|
378
|
+
state.lastStartedAt = Date.now();
|
|
379
|
+
const startedAt = state.lastStartedAt;
|
|
380
|
+
void (async () => {
|
|
381
|
+
try {
|
|
382
|
+
if (state.availability === "unknown") {
|
|
383
|
+
state.availability = (await probeAvailability()) ? "yes" : "no";
|
|
384
|
+
}
|
|
385
|
+
if (state.availability === "no") {
|
|
386
|
+
logLine(
|
|
387
|
+
"packet witness: unavailable — no tcpdump on this host; " +
|
|
388
|
+
"the stuck-queue log lines remain the only evidence here"
|
|
389
|
+
);
|
|
390
|
+
return;
|
|
391
|
+
}
|
|
392
|
+
await runCapture(trigger, address);
|
|
393
|
+
} finally {
|
|
394
|
+
state.running = false;
|
|
395
|
+
// Keep the requested spacing honest even when the capture ended early.
|
|
396
|
+
const earliestNext = startedAt + WITNESS_COOLDOWN_MS;
|
|
397
|
+
if (state.lastStartedAt < earliestNext) {
|
|
398
|
+
state.lastStartedAt = earliestNext;
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
})();
|
|
402
|
+
return true;
|
|
403
|
+
};
|
|
404
|
+
|
|
405
|
+
return { dir: resolvedDir, maybeCapture };
|
|
406
|
+
}
|
|
@@ -443,11 +443,11 @@ export function createWebRtcManager({ sendSignal, onDataChannel, onLog, udpPort,
|
|
|
443
443
|
* collapsed. `bytesReceived` still climbing at the same time proves the
|
|
444
444
|
* peer is alive and its packets still reach us, i.e. the failure is
|
|
445
445
|
* one-directional.
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
446
|
+
* - both flat → nothing crosses in either direction.
|
|
447
|
+
*
|
|
448
|
+
* @param {string} sessionId
|
|
449
|
+
* @returns {{ bytesSent: number, bytesReceived: number, rtt: number, pair: string, remote: { address: string, port: number } | null, state: string, iceState: string } | null}
|
|
450
|
+
*/
|
|
451
451
|
function getTransportSnapshot(sessionId) {
|
|
452
452
|
const pc = peers.get(sessionId);
|
|
453
453
|
if (!pc) {
|
|
@@ -469,6 +469,12 @@ export function createWebRtcManager({ sendSignal, onDataChannel, onLog, udpPort,
|
|
|
469
469
|
? `${pair.local?.address}:${pair.local?.port}->${pair.remote?.address}:${pair.remote?.port}` +
|
|
470
470
|
` (${pair.local?.type}/${pair.remote?.type})`
|
|
471
471
|
: "none",
|
|
472
|
+
// Structured, for consumers that need to act on the endpoint itself (the
|
|
473
|
+
// packet witness builds a tcpdump filter from it) rather than print it.
|
|
474
|
+
remote:
|
|
475
|
+
typeof pair?.remote?.address === "string" && Number.isFinite(pair?.remote?.port)
|
|
476
|
+
? { address: pair.remote.address, port: pair.remote.port }
|
|
477
|
+
: null,
|
|
472
478
|
state: read(() => pc.state(), "?"),
|
|
473
479
|
iceState: read(() => pc.iceState(), "?")
|
|
474
480
|
};
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file The packet witness's gating rules and command construction.
|
|
3
|
+
*
|
|
4
|
+
* Roadmap item 10: a send queue wedged longer than ~30 s must start a bounded
|
|
5
|
+
* tcpdump on its own, because the 2026-08-24 episode proved no counter above
|
|
6
|
+
* the wire can name the cause. Everything here is the part that decides WHEN
|
|
7
|
+
* and WITH WHAT ARGUMENTS — the spawning itself is thin glue around these.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import assert from "node:assert/strict";
|
|
11
|
+
import test from "node:test";
|
|
12
|
+
|
|
13
|
+
import {
|
|
14
|
+
buildTcpdumpArgs,
|
|
15
|
+
createPacketWitness,
|
|
16
|
+
isWitnessCapture,
|
|
17
|
+
normalizeRemoteAddress,
|
|
18
|
+
shouldStartCapture,
|
|
19
|
+
WITNESS_CAPTURE_SECONDS,
|
|
20
|
+
WITNESS_RING_FILES,
|
|
21
|
+
WITNESS_ROTATE_SECONDS
|
|
22
|
+
} from "../services/packet-witness.js";
|
|
23
|
+
|
|
24
|
+
test("an IPv4 literal survives unchanged", () => {
|
|
25
|
+
assert.equal(normalizeRemoteAddress("192.168.178.57"), "192.168.178.57");
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
test("an IPv6 literal survives unchanged", () => {
|
|
29
|
+
assert.equal(
|
|
30
|
+
normalizeRemoteAddress("2001:1c00:a603:2100:a129:a1a1:7f07:3f0b"),
|
|
31
|
+
"2001:1c00:a603:2100:a129:a1a1:7f07:3f0b"
|
|
32
|
+
);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
test("a zone suffix is stripped", () => {
|
|
36
|
+
assert.equal(normalizeRemoteAddress("fe80::2aca:8001:3ba6:f16f%18"), "fe80::2aca:8001:3ba6:f16f");
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test("hostnames, garbage and shell text are rejected", () => {
|
|
40
|
+
assert.equal(normalizeRemoteAddress("homeassistant.local"), null);
|
|
41
|
+
assert.equal(normalizeRemoteAddress("8.8.8.8; rm -rf /"), null);
|
|
42
|
+
assert.equal(normalizeRemoteAddress("999.1.1.1"), null);
|
|
43
|
+
assert.equal(normalizeRemoteAddress(""), null);
|
|
44
|
+
assert.equal(normalizeRemoteAddress(undefined), null);
|
|
45
|
+
assert.equal(normalizeRemoteAddress(42), null);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test("the tcpdump command line is bounded and filtered to one peer", () => {
|
|
49
|
+
const args = buildTcpdumpArgs({
|
|
50
|
+
host: "2001:db8::1",
|
|
51
|
+
port: 9090,
|
|
52
|
+
filePrefix: "/data/packet-witness.e2b5ef39.1787600000.pcap"
|
|
53
|
+
});
|
|
54
|
+
assert.deepEqual(args, [
|
|
55
|
+
"-n",
|
|
56
|
+
"-S",
|
|
57
|
+
"-s",
|
|
58
|
+
"128",
|
|
59
|
+
"-i",
|
|
60
|
+
"any",
|
|
61
|
+
"-w",
|
|
62
|
+
"/data/packet-witness.e2b5ef39.1787600000.pcap",
|
|
63
|
+
"-G",
|
|
64
|
+
String(WITNESS_ROTATE_SECONDS),
|
|
65
|
+
"-W",
|
|
66
|
+
String(WITNESS_RING_FILES),
|
|
67
|
+
"udp",
|
|
68
|
+
"and",
|
|
69
|
+
"host",
|
|
70
|
+
"2001:db8::1",
|
|
71
|
+
"and",
|
|
72
|
+
"port",
|
|
73
|
+
"9090"
|
|
74
|
+
]);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
test("the capture window is what the rotation ring adds up to", () => {
|
|
78
|
+
assert.equal(WITNESS_ROTATE_SECONDS * WITNESS_RING_FILES, WITNESS_CAPTURE_SECONDS);
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
test("one capture runs at a time", () => {
|
|
82
|
+
assert.equal(shouldStartCapture({ running: true, lastStartedAt: Date.now() }), false);
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
test("the first capture of a process is always allowed", () => {
|
|
86
|
+
assert.equal(shouldStartCapture({ running: false, lastStartedAt: 0, now: 1000 }), true);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test("a capture within the cooldown is refused, one after it is allowed", () => {
|
|
90
|
+
const state = { running: false, lastStartedAt: 10_000 };
|
|
91
|
+
assert.equal(shouldStartCapture({ ...state, now: 10_000 + 599_999 }), false);
|
|
92
|
+
assert.equal(shouldStartCapture({ ...state, now: 10_000 + 600_000 }), true);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test("capture files are recognised by name, rotations included", () => {
|
|
96
|
+
assert.equal(isWitnessCapture("packet-witness.e2b5ef39.1787600000.pcap"), true);
|
|
97
|
+
assert.equal(isWitnessCapture("packet-witness.e2b5ef39.1787600000.pcap20260825T120000"), true);
|
|
98
|
+
assert.equal(isWitnessCapture("core.WorkerThread.81.1787600000"), false);
|
|
99
|
+
assert.equal(isWitnessCapture("proxy.log"), false);
|
|
100
|
+
assert.equal(isWitnessCapture("../packet-witness.x.pcap"), false);
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
test("a trigger without a usable remote endpoint is refused without touching anything", () => {
|
|
104
|
+
const lines = [];
|
|
105
|
+
const witness = createPacketWitness({
|
|
106
|
+
log: (message) => lines.push(message),
|
|
107
|
+
dir: "",
|
|
108
|
+
port: 9090
|
|
109
|
+
});
|
|
110
|
+
const started = witness.maybeCapture({
|
|
111
|
+
sessionId: "e2b5ef39-0000",
|
|
112
|
+
tag: "e2b5ef39",
|
|
113
|
+
label: "proxy",
|
|
114
|
+
remote: null,
|
|
115
|
+
queuedBytes: 160_000_000,
|
|
116
|
+
stuckForMs: 31_000
|
|
117
|
+
});
|
|
118
|
+
assert.equal(started, false);
|
|
119
|
+
assert.deepEqual(lines, []);
|
|
120
|
+
});
|