@camstack/server 1.2.268 → 1.2.270

package/dist/api/backup-download-pump.js

@@ -0,0 +1,139 @@
+"use strict";
+/**
+ * backup-download-pump — a backup archive is streamed at the pace the client
+ * READS it, and a client that stops reading is cut, never buffered until
+ * hub-main dies.
+ *
+ * ## What happened
+ *
+ * `/api/backup/download/:locationId/:archiveId` read an archive from the
+ * storage cap in 8 MiB chunks and called `reply.raw.write(chunk)` in a loop
+ * that never looked at the return value and never waited for `drain`. A Node
+ * writable accepts every write; what the socket cannot send it QUEUES, in
+ * hub-main's heap, until the peer reads it. On a fast disk the whole archive
+ * is read in seconds, so for a client on a slow link — a phone over the
+ * Cloudflare tunnel, a browser tab in the background — the queue is the
+ * archive: 464 MB across 9 169 files on this hub, per download, per client,
+ * released only when the socket closes. And not one line said so.
+ *
+ * ## The policy, and why this one
+ *
+ * It is the WebSocket guard's (`ws-slow-consumer-guard.ts`, D444) and the
+ * restreamer's (`rtsp-session.ts`): backpressure measures LIVENESS, not
+ * identity. Nobody is cut for being momentarily behind — a client draining an
+ * 8 MiB chunk over a 2 Mbit link takes half a minute per chunk and is coming
+ * back — and a client that cannot take a single socket buffer in
+ * {@link BACKUP_DOWNLOAD_DRAIN_GRACE_MS} is not behind, it is gone.
+ *
+ * The form differs from the subscription guard in one way that matters: the
+ * respect for `drain` does the bounding on its own. A download is a long,
+ * legitimate flow, not a subscription, so the loop simply does not READ the
+ * next chunk until the socket has taken the previous one — the most the
+ * response can hold is one chunk plus the socket buffer, whatever the client's
+ * speed. The grace exists only for the peer that stops reading altogether:
+ * a half-open TCP connection can sit in that state for the kernel's whole
+ * retransmit budget (minutes), holding the chunk, the storage cap's download
+ * handle and the request, and `drain` never comes.
+ *
+ * ## What is said
+ *
+ * `pumpDownload` returns what happened in numbers — bytes, waits, the longest
+ * wait, how long a cut client was held — and the route turns that into ONE
+ * line per download: `info` for a completed or client-abandoned transfer,
+ * `warn` for a cut client or a failed source. The route previously logged
+ * nothing on success and `console.error` on failure.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BACKUP_DOWNLOAD_DRAIN_GRACE_MS = exports.BACKUP_DOWNLOAD_CHUNK_BYTES = void 0;
+exports.pumpDownload = pumpDownload;
+/** Bytes read from the storage cap per round trip; one chunk is the most the response holds. */
+exports.BACKUP_DOWNLOAD_CHUNK_BYTES = 8 * 1024 * 1024;
+/**
+ * How long a `write` may stay unacknowledged by `drain` before the client is
+ * cut. A `ServerResponse` signals `drain` once the socket has taken its
+ * buffer (16 KiB by default): a client that cannot take 16 KiB in a minute is
+ * under 0.3 KB/s — not a slow link, an absent reader.
+ */
+exports.BACKUP_DOWNLOAD_DRAIN_GRACE_MS = 60_000;
+function waitForDrain(sink, graceMs) {
+    return new Promise((resolve) => {
+        const settle = (outcome) => {
+            sink.off('drain', onDrain);
+            sink.off('close', onClose);
+            clearTimeout(timer);
+            resolve(outcome);
+        };
+        const onDrain = () => settle('drained');
+        const onClose = () => settle('closed');
+        const timer = setTimeout(() => settle('timeout'), graceMs);
+        sink.on('drain', onDrain);
+        sink.on('close', onClose);
+    });
+}
+function errorOf(err) {
+    return err instanceof Error ? err : new Error(String(err));
+}
+/**
+ * Stream `sizeBytes` from `read` into `sink`, one chunk at a time, reading the
+ * next only once the socket has taken the previous. Never throws: every exit
+ * is a {@link PumpResult} the caller can log.
+ */
+async function pumpDownload(options) {
+    const { sizeBytes, read, sink } = options;
+    const chunkBytes = options.chunkBytes ?? exports.BACKUP_DOWNLOAD_CHUNK_BYTES;
+    const graceMs = options.graceMs ?? exports.BACKUP_DOWNLOAD_DRAIN_GRACE_MS;
+    const now = options.now ?? (() => Date.now());
+    const startedAt = now();
+    let offset = 0;
+    let drainWaits = 0;
+    let drainWaitMaxMs = 0;
+    const counters = () => ({
+        bytesWritten: offset,
+        drainWaits,
+        drainWaitMaxMs,
+        durationMs: now() - startedAt,
+    });
+    while (offset < sizeBytes) {
+        if (sink.destroyed)
+            return { status: 'peer-gone', ...counters() };
+        const length = Math.min(chunkBytes, sizeBytes - offset);
+        let chunk;
+        try {
+            chunk = await read(offset, length);
+        }
+        catch (err) {
+            const error = errorOf(err);
+            sink.destroy(error);
+            return { status: 'source-failed', error: error.message, ...counters() };
+        }
+        if (chunk.byteLength === 0) {
+            // A short read before the end is a truncated archive, never "done":
+            // `end()` here would hand the client a file that fails to extract.
+            const error = new Error(`backup download: empty chunk at offset ${offset}/${sizeBytes}`);
+            sink.destroy(error);
+            return { status: 'source-failed', error: error.message, ...counters() };
+        }
+        // The peer may have gone while the storage cap was answering.
+        if (sink.destroyed)
+            return { status: 'peer-gone', ...counters() };
+        const accepted = sink.write(chunk);
+        offset += chunk.byteLength;
+        if (accepted)
+            continue;
+        const waitStartedAt = now();
+        const outcome = await waitForDrain(sink, graceMs);
+        const waitedMs = now() - waitStartedAt;
+        drainWaits += 1;
+        if (waitedMs > drainWaitMaxMs)
+            drainWaitMaxMs = waitedMs;
+        if (outcome === 'closed')
+            return { status: 'peer-gone', ...counters() };
+        if (outcome === 'timeout') {
+            const queuedBytes = sink.writableLength;
+            sink.destroy(new Error(`backup download: client stopped reading — ${queuedBytes} bytes queued for ${waitedMs} ms`));
+            return { status: 'drain-timeout', heldForMs: waitedMs, queuedBytes, ...counters() };
+        }
+    }
+    sink.end();
+    return { status: 'completed', ...counters() };
+}

package/dist/api/backup-download.js

@@ -0,0 +1,109 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.registerBackupDownloadRoute = registerBackupDownloadRoute;
+const backup_download_pump_js_1 = require("./backup-download-pump.js");
+const client_ip_js_1 = require("./trpc/client-ip.js");
+/** Strip what would break the `Content-Disposition` header. Whitespace is fine. */
+function safeAttachmentName(name) {
+    return name.replace(/[\r\n"]/g, '_');
+}
+function registerBackupDownloadRoute(fastify, options) {
+    const { verifier, capabilities, logger } = options;
+    const chunkBytes = options.chunkBytes ?? backup_download_pump_js_1.BACKUP_DOWNLOAD_CHUNK_BYTES;
+    const graceMs = options.graceMs ?? backup_download_pump_js_1.BACKUP_DOWNLOAD_DRAIN_GRACE_MS;
+    fastify.get('/api/backup/download/:locationId/:archiveId', async (request, reply) => {
+        const authHeader = request.headers.authorization;
+        if (!authHeader) {
+            return reply.status(401).send({ error: 'Unauthorized' });
+        }
+        try {
+            const payload = verifier.verifyToken(authHeader.replace('Bearer ', ''));
+            if (!payload.isAdmin) {
+                return reply.status(403).send({ error: 'Admin required' });
+            }
+        }
+        catch {
+            return reply.status(401).send({ error: 'Invalid token' });
+        }
+        const { locationId, archiveId } = request.params;
+        const backup = capabilities.getSingleton('backup');
+        if (!backup?.listArchives) {
+            return reply.status(503).send({ error: 'Backup orchestrator unavailable' });
+        }
+        const archives = await backup.listArchives({ destinationId: locationId });
+        const archive = archives.find((a) => a.id === archiveId);
+        if (!archive) {
+            return reply.status(404).send({ error: 'Archive not found' });
+        }
+        const storage = capabilities.getSingleton('storage');
+        if (!storage?.beginDownload) {
+            return reply.status(503).send({ error: 'Storage cap unavailable' });
+        }
+        const { downloadId, sizeBytes } = await storage.beginDownload({
+            location: locationId,
+            relativePath: archive.filename,
+        });
+        // The response is written on the raw socket, so it is taken out of
+        // Fastify's hands and the headers go on the raw response. The previous
+        // route set them through `reply.header(…)` — which Fastify applies only
+        // on `reply.send()`, never reached here — so no download ever carried a
+        // `content-length` or a `content-disposition`. `sizeBytes` is the storage
+        // cap's figure for the file it is about to serve, not the catalogue's.
+        reply.hijack();
+        reply.raw.writeHead(200, {
+            'content-type': 'application/gzip',
+            'content-length': String(sizeBytes),
+            'content-disposition': `attachment; filename="${safeAttachmentName(`${archive.label ?? archive.id}.tar.gz`)}"`,
+        });
+        const facts = {
+            locationId,
+            archiveId,
+            filename: archive.filename,
+            sizeBytes,
+            ip: (0, client_ip_js_1.extractClientIp)(request.raw) ?? '(unidentified)',
+            chunkBytes,
+            graceMs,
+        };
+        let result;
+        try {
+            result = await (0, backup_download_pump_js_1.pumpDownload)({
+                sizeBytes,
+                chunkBytes,
+                graceMs,
+                sink: reply.raw,
+                read: (offset, length) => storage.readChunk({ downloadId, offset, length }),
+            });
+        }
+        finally {
+            try {
+                await storage.endDownload({ downloadId });
+            }
+            catch (err) {
+                logger.warn('backup archive download: the storage cap refused to close the handle', {
+                    meta: { ...facts, error: err instanceof Error ? err.message : String(err) },
+                });
+            }
+        }
+        logOutcome(logger, facts, result);
+    });
+}
+function logOutcome(logger, facts, result) {
+    const meta = { ...facts, ...result };
+    switch (result.status) {
+        case 'completed':
+            logger.info('backup archive download completed', { meta });
+            return;
+        case 'peer-gone':
+            logger.info('backup archive download abandoned — the client closed the connection', { meta });
+            return;
+        case 'drain-timeout':
+            logger.warn('backup archive download cut — the client stopped reading and the response queue ' +
+                'stayed unsent for the whole grace period (nothing more is read; the queue is released)', { meta });
+            return;
+        case 'source-failed':
+            logger.warn('backup archive download failed — the storage cap could not serve the archive', {
+                meta,
+            });
+            return;
+    }
+}

package/dist/api/trpc/ws-slow-consumer-guard.js

@@ -0,0 +1,166 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WsSlowConsumerGuard = exports.WS_SLOW_CONSUMER_POLL_MS = exports.WS_SLOW_CONSUMER_GRACE_MS = exports.WS_SLOW_CONSUMER_MAX_BUFFERED_BYTES = void 0;
+exports.wsConnectionFacts = wsConnectionFacts;
+const client_ip_js_1 = require("./client-ip.js");
+/** Queued bytes above which a client is a slow consumer. */
+exports.WS_SLOW_CONSUMER_MAX_BUFFERED_BYTES = 16 * 1024 * 1024;
+/** How long the queue may stay above the bound before the socket is terminated. */
+exports.WS_SLOW_CONSUMER_GRACE_MS = 30_000;
+/** How often every connection's queue is read. A getter per socket; free. */
+exports.WS_SLOW_CONSUMER_POLL_MS = 5_000;
+/** WebSocket close code for a socket that was never closed cleanly. */
+const ABNORMAL_CLOSURE = 1006;
+const UNIDENTIFIED = '(unidentified)';
+class WsSlowConsumerGuard {
+    connections = new Set();
+    bySocket = new WeakMap();
+    logger;
+    maxBufferedBytes;
+    graceMs;
+    pollMs;
+    now;
+    timer = null;
+    constructor(options) {
+        this.logger = options.logger;
+        this.maxBufferedBytes = options.maxBufferedBytes ?? exports.WS_SLOW_CONSUMER_MAX_BUFFERED_BYTES;
+        this.graceMs = options.graceMs ?? exports.WS_SLOW_CONSUMER_GRACE_MS;
+        this.pollMs = options.pollMs ?? exports.WS_SLOW_CONSUMER_POLL_MS;
+        this.now = options.now ?? (() => Date.now());
+    }
+    /** Watch one connection until it closes. */
+    attach(socket, facts) {
+        const connection = {
+            socket,
+            facts,
+            openedAt: this.now(),
+            principal: UNIDENTIFIED,
+            overSince: null,
+            peakBufferedBytes: 0,
+            terminated: false,
+        };
+        this.connections.add(connection);
+        this.bySocket.set(socket, connection);
+        socket.once('close', (code, reason) => {
+            this.connections.delete(connection);
+            this.logger.info('tRPC WS connection closed', {
+                meta: {
+                    ...this.describe(connection),
+                    code,
+                    reason: reason.length > 0 ? reason.toString('utf8') : '',
+                    closedBy: connection.terminated
+                        ? 'slow-consumer-guard'
+                        : code === ABNORMAL_CLOSURE
+                            ? 'peer gone or keep-alive PING unanswered'
+                            : 'peer',
+                },
+            });
+        });
+    }
+    /** Name the connection's principal once the WS context resolved it. */
+    identify(socket, principal) {
+        const connection = this.bySocket.get(socket);
+        if (connection !== undefined)
+            connection.principal = principal;
+    }
+    /** Read every queue once. Exposed for the spec; production runs it on {@link start}. */
+    poll() {
+        const at = this.now();
+        for (const connection of this.connections) {
+            try {
+                this.judge(connection, at);
+            }
+            catch (err) {
+                // A diagnostic must never take down what it guards; say so instead.
+                this.logger.warn('tRPC WS slow-consumer guard could not read a socket', {
+                    meta: { ...this.describe(connection), error: errorMessage(err) },
+                });
+            }
+        }
+    }
+    start() {
+        if (this.timer !== null)
+            return;
+        this.timer = setInterval(() => this.poll(), this.pollMs);
+        this.timer.unref?.();
+    }
+    stop() {
+        if (this.timer === null)
+            return;
+        clearInterval(this.timer);
+        this.timer = null;
+    }
+    judge(connection, at) {
+        if (connection.terminated)
+            return;
+        const buffered = connection.socket.bufferedAmount;
+        if (buffered > connection.peakBufferedBytes)
+            connection.peakBufferedBytes = buffered;
+        if (buffered <= this.maxBufferedBytes) {
+            if (connection.overSince !== null) {
+                // It came back: say so, so a merely slow client is visible and not confused
+                // with the ones that are terminated.
+                this.logger.info('tRPC WS client drained a backlog that had crossed the bound', {
+                    meta: {
+                        ...this.describe(connection),
+                        overForMs: at - connection.overSince,
+                        bufferedBytes: buffered,
+                    },
+                });
+                connection.overSince = null;
+                connection.peakBufferedBytes = buffered;
+            }
+            return;
+        }
+        connection.overSince ??= at;
+        const overForMs = at - connection.overSince;
+        if (overForMs < this.graceMs)
+            return;
+        connection.terminated = true;
+        this.logger.warn('tRPC WS client disconnected — slow consumer: it stopped reading and its send queue ' +
+            'stayed above the bound for the whole grace period (a backgrounded viewer keeps the ' +
+            'socket open with its JS suspended; the queue lived in hub-main old_space)', {
+            meta: {
+                ...this.describe(connection),
+                bufferedBytes: buffered,
+                overForMs,
+                boundBytes: this.maxBufferedBytes,
+                graceMs: this.graceMs,
+            },
+        });
+        connection.socket.terminate();
+    }
+    describe(connection) {
+        return {
+            ip: connection.facts.ip ?? UNIDENTIFIED,
+            userAgent: connection.facts.userAgent ?? UNIDENTIFIED,
+            principal: connection.principal,
+            bytesRead: safeCount(connection.facts.bytesRead),
+            bytesWritten: safeCount(connection.facts.bytesWritten),
+            peakBufferedBytes: connection.peakBufferedBytes,
+            connectionAgeMs: this.now() - connection.openedAt,
+        };
+    }
+}
+exports.WsSlowConsumerGuard = WsSlowConsumerGuard;
+function safeCount(read) {
+    try {
+        return read();
+    }
+    catch {
+        // A destroyed socket may have dropped its handle; null is "unknown", never 0.
+        return null;
+    }
+}
+function errorMessage(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+/** The facts a real `ws` upgrade request provides. */
+function wsConnectionFacts(req) {
+    return {
+        ip: (0, client_ip_js_1.extractClientIp)(req),
+        userAgent: (0, client_ip_js_1.extractUserAgent)(req),
+        bytesRead: () => req.socket.bytesRead,
+        bytesWritten: () => req.socket.bytesWritten,
+    };
+}