@camstack/server 1.2.269 → 1.2.270

package/dist/api/backup-download-pump.js

@@ -0,0 +1,139 @@
+"use strict";
+/**
+ * backup-download-pump — a backup archive is streamed at the pace the client
+ * READS it, and a client that stops reading is cut, never buffered until
+ * hub-main dies.
+ *
+ * ## What happened
+ *
+ * `/api/backup/download/:locationId/:archiveId` read an archive from the
+ * storage cap in 8 MiB chunks and called `reply.raw.write(chunk)` in a loop
+ * that never looked at the return value and never waited for `drain`. A Node
+ * writable accepts every write; what the socket cannot send it QUEUES, in
+ * hub-main's heap, until the peer reads it. On a fast disk the whole archive
+ * is read in seconds, so for a client on a slow link — a phone over the
+ * Cloudflare tunnel, a browser tab in the background — the queue is the
+ * archive: 464 MB across 9 169 files on this hub, per download, per client,
+ * released only when the socket closes. And not one line said so.
+ *
+ * ## The policy, and why this one
+ *
+ * It is the WebSocket guard's (`ws-slow-consumer-guard.ts`, D444) and the
+ * restreamer's (`rtsp-session.ts`): backpressure measures LIVENESS, not
+ * identity. Nobody is cut for being momentarily behind — a client draining an
+ * 8 MiB chunk over a 2 Mbit link takes half a minute per chunk and is coming
+ * back — and a client that cannot take a single socket buffer in
+ * {@link BACKUP_DOWNLOAD_DRAIN_GRACE_MS} is not behind, it is gone.
+ *
+ * The form differs from the subscription guard in one way that matters: the
+ * respect for `drain` does the bounding on its own. A download is a long,
+ * legitimate flow, not a subscription, so the loop simply does not READ the
+ * next chunk until the socket has taken the previous one — the most the
+ * response can hold is one chunk plus the socket buffer, whatever the client's
+ * speed. The grace exists only for the peer that stops reading altogether:
+ * a half-open TCP connection can sit in that state for the kernel's whole
+ * retransmit budget (minutes), holding the chunk, the storage cap's download
+ * handle and the request, and `drain` never comes.
+ *
+ * ## What is said
+ *
+ * `pumpDownload` returns what happened in numbers — bytes, waits, the longest
+ * wait, how long a cut client was held — and the route turns that into ONE
+ * line per download: `info` for a completed or client-abandoned transfer,
+ * `warn` for a cut client or a failed source. The route previously logged
+ * nothing on success and `console.error` on failure.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BACKUP_DOWNLOAD_DRAIN_GRACE_MS = exports.BACKUP_DOWNLOAD_CHUNK_BYTES = void 0;
+exports.pumpDownload = pumpDownload;
+/** Bytes read from the storage cap per round trip; one chunk is the most the response holds. */
+exports.BACKUP_DOWNLOAD_CHUNK_BYTES = 8 * 1024 * 1024;
+/**
+ * How long a `write` may stay unacknowledged by `drain` before the client is
+ * cut. A `ServerResponse` signals `drain` once the socket has taken its
+ * buffer (16 KiB by default): a client that cannot take 16 KiB in a minute is
+ * under 0.3 KB/s — not a slow link, an absent reader.
+ */
+exports.BACKUP_DOWNLOAD_DRAIN_GRACE_MS = 60_000;
+function waitForDrain(sink, graceMs) {
+    return new Promise((resolve) => {
+        const settle = (outcome) => {
+            sink.off('drain', onDrain);
+            sink.off('close', onClose);
+            clearTimeout(timer);
+            resolve(outcome);
+        };
+        const onDrain = () => settle('drained');
+        const onClose = () => settle('closed');
+        const timer = setTimeout(() => settle('timeout'), graceMs);
+        sink.on('drain', onDrain);
+        sink.on('close', onClose);
+    });
+}
+function errorOf(err) {
+    return err instanceof Error ? err : new Error(String(err));
+}
+/**
+ * Stream `sizeBytes` from `read` into `sink`, one chunk at a time, reading the
+ * next only once the socket has taken the previous. Never throws: every exit
+ * is a {@link PumpResult} the caller can log.
+ */
+async function pumpDownload(options) {
+    const { sizeBytes, read, sink } = options;
+    const chunkBytes = options.chunkBytes ?? exports.BACKUP_DOWNLOAD_CHUNK_BYTES;
+    const graceMs = options.graceMs ?? exports.BACKUP_DOWNLOAD_DRAIN_GRACE_MS;
+    const now = options.now ?? (() => Date.now());
+    const startedAt = now();
+    let offset = 0;
+    let drainWaits = 0;
+    let drainWaitMaxMs = 0;
+    const counters = () => ({
+        bytesWritten: offset,
+        drainWaits,
+        drainWaitMaxMs,
+        durationMs: now() - startedAt,
+    });
+    while (offset < sizeBytes) {
+        if (sink.destroyed)
+            return { status: 'peer-gone', ...counters() };
+        const length = Math.min(chunkBytes, sizeBytes - offset);
+        let chunk;
+        try {
+            chunk = await read(offset, length);
+        }
+        catch (err) {
+            const error = errorOf(err);
+            sink.destroy(error);
+            return { status: 'source-failed', error: error.message, ...counters() };
+        }
+        if (chunk.byteLength === 0) {
+            // A short read before the end is a truncated archive, never "done":
+            // `end()` here would hand the client a file that fails to extract.
+            const error = new Error(`backup download: empty chunk at offset ${offset}/${sizeBytes}`);
+            sink.destroy(error);
+            return { status: 'source-failed', error: error.message, ...counters() };
+        }
+        // The peer may have gone while the storage cap was answering.
+        if (sink.destroyed)
+            return { status: 'peer-gone', ...counters() };
+        const accepted = sink.write(chunk);
+        offset += chunk.byteLength;
+        if (accepted)
+            continue;
+        const waitStartedAt = now();
+        const outcome = await waitForDrain(sink, graceMs);
+        const waitedMs = now() - waitStartedAt;
+        drainWaits += 1;
+        if (waitedMs > drainWaitMaxMs)
+            drainWaitMaxMs = waitedMs;
+        if (outcome === 'closed')
+            return { status: 'peer-gone', ...counters() };
+        if (outcome === 'timeout') {
+            const queuedBytes = sink.writableLength;
+            sink.destroy(new Error(`backup download: client stopped reading — ${queuedBytes} bytes queued for ${waitedMs} ms`));
+            return { status: 'drain-timeout', heldForMs: waitedMs, queuedBytes, ...counters() };
+        }
+    }
+    sink.end();
+    return { status: 'completed', ...counters() };
+}

package/dist/api/backup-download.js

@@ -0,0 +1,109 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.registerBackupDownloadRoute = registerBackupDownloadRoute;
+const backup_download_pump_js_1 = require("./backup-download-pump.js");
+const client_ip_js_1 = require("./trpc/client-ip.js");
+/** Strip what would break the `Content-Disposition` header. Whitespace is fine. */
+function safeAttachmentName(name) {
+    return name.replace(/[\r\n"]/g, '_');
+}
+function registerBackupDownloadRoute(fastify, options) {
+    const { verifier, capabilities, logger } = options;
+    const chunkBytes = options.chunkBytes ?? backup_download_pump_js_1.BACKUP_DOWNLOAD_CHUNK_BYTES;
+    const graceMs = options.graceMs ?? backup_download_pump_js_1.BACKUP_DOWNLOAD_DRAIN_GRACE_MS;
+    fastify.get('/api/backup/download/:locationId/:archiveId', async (request, reply) => {
+        const authHeader = request.headers.authorization;
+        if (!authHeader) {
+            return reply.status(401).send({ error: 'Unauthorized' });
+        }
+        try {
+            const payload = verifier.verifyToken(authHeader.replace('Bearer ', ''));
+            if (!payload.isAdmin) {
+                return reply.status(403).send({ error: 'Admin required' });
+            }
+        }
+        catch {
+            return reply.status(401).send({ error: 'Invalid token' });
+        }
+        const { locationId, archiveId } = request.params;
+        const backup = capabilities.getSingleton('backup');
+        if (!backup?.listArchives) {
+            return reply.status(503).send({ error: 'Backup orchestrator unavailable' });
+        }
+        const archives = await backup.listArchives({ destinationId: locationId });
+        const archive = archives.find((a) => a.id === archiveId);
+        if (!archive) {
+            return reply.status(404).send({ error: 'Archive not found' });
+        }
+        const storage = capabilities.getSingleton('storage');
+        if (!storage?.beginDownload) {
+            return reply.status(503).send({ error: 'Storage cap unavailable' });
+        }
+        const { downloadId, sizeBytes } = await storage.beginDownload({
+            location: locationId,
+            relativePath: archive.filename,
+        });
+        // The response is written on the raw socket, so it is taken out of
+        // Fastify's hands and the headers go on the raw response. The previous
+        // route set them through `reply.header(…)` — which Fastify applies only
+        // on `reply.send()`, never reached here — so no download ever carried a
+        // `content-length` or a `content-disposition`. `sizeBytes` is the storage
+        // cap's figure for the file it is about to serve, not the catalogue's.
+        reply.hijack();
+        reply.raw.writeHead(200, {
+            'content-type': 'application/gzip',
+            'content-length': String(sizeBytes),
+            'content-disposition': `attachment; filename="${safeAttachmentName(`${archive.label ?? archive.id}.tar.gz`)}"`,
+        });
+        const facts = {
+            locationId,
+            archiveId,
+            filename: archive.filename,
+            sizeBytes,
+            ip: (0, client_ip_js_1.extractClientIp)(request.raw) ?? '(unidentified)',
+            chunkBytes,
+            graceMs,
+        };
+        let result;
+        try {
+            result = await (0, backup_download_pump_js_1.pumpDownload)({
+                sizeBytes,
+                chunkBytes,
+                graceMs,
+                sink: reply.raw,
+                read: (offset, length) => storage.readChunk({ downloadId, offset, length }),
+            });
+        }
+        finally {
+            try {
+                await storage.endDownload({ downloadId });
+            }
+            catch (err) {
+                logger.warn('backup archive download: the storage cap refused to close the handle', {
+                    meta: { ...facts, error: err instanceof Error ? err.message : String(err) },
+                });
+            }
+        }
+        logOutcome(logger, facts, result);
+    });
+}
+function logOutcome(logger, facts, result) {
+    const meta = { ...facts, ...result };
+    switch (result.status) {
+        case 'completed':
+            logger.info('backup archive download completed', { meta });
+            return;
+        case 'peer-gone':
+            logger.info('backup archive download abandoned — the client closed the connection', { meta });
+            return;
+        case 'drain-timeout':
+            logger.warn('backup archive download cut — the client stopped reading and the response queue ' +
+                'stayed unsent for the whole grace period (nothing more is read; the queue is released)', { meta });
+            return;
+        case 'source-failed':
+            logger.warn('backup archive download failed — the storage cap could not serve the archive', {
+                meta,
+            });
+            return;
+    }
+}

package/dist/main.js

@@ -58,6 +58,7 @@ const ws_1 = require("@trpc/server/adapters/ws");
 const ws_2 = require("ws");
 const addon_upload_1 = require("./api/addon-upload");
 const auth_whoami_1 = require("./api/auth-whoami");
+const backup_download_js_1 = require("./api/backup-download.js");
 const health_routes_1 = require("./api/health/health.routes");
 const adaptive_probe_routes_1 = require("./api/health/adaptive-probe.routes");
 const model_distributor_js_1 = require("./api/model-distributor.js");
@@ -303,6 +304,12 @@ async function bootstrap() {
         socketPlane: (0, system_1.createSocketPlaneReader)({
             registry: () => moleculerForEventPlane?.childRegistry ?? null,
         }),
+        // The write queue to every peer NODE, from Moleculer's own writer
+        // sockets. Those are unref'd (`TcpWriter.connect`), so the 2026-09-10
+        // exclusion of "every socket queue" — a sum over `_getActiveHandles()` —
+        // never saw them. With the UDS queues above, every outbound plane this
+        // process writes to is now on the line and judged on every probe (D446).
+        meshQueue: (0, system_1.createMeshQueueReader)(() => moleculerForEventPlane?.broker ?? null),
         // The ArrayBuffer census, armed by the probe itself. Twice on 2026-09-10
         // this process's `arrayBuffers` climbed linearly for an hour (to 1.3 GB
         // and to 5 GB, both LIVE across a forced compaction) and released in one
@@ -821,89 +828,14 @@ async function bootstrap() {
                 });
             }
         });
-        // ── Backup archive download (Phase 4 / Task 24) ────────────────
-        // Streams an archive at `<locationId>/<archiveId>` through the
-        // storage cap's chunked-download protocol straight to an HTTP
-        // response. The admin UI uses an authenticated `fetch` + Blob to
-        // fetch and trigger a save dialog (no cookie auth on this server,
-        // so we can't rely on `window.location.assign`).
-        fastify.get('/api/backup/download/:locationId/:archiveId', async (request, reply) => {
-            const authHeader = request.headers.authorization;
-            if (!authHeader) {
-                return reply.status(401).send({ error: 'Unauthorized' });
-            }
-            try {
-                const token = authHeader.replace('Bearer ', '');
-                const downloadAuth = app.get(auth_service_1.AuthService);
-                const payload = downloadAuth.verifyToken(token);
-                if (!payload.isAdmin) {
-                    return reply.status(403).send({ error: 'Admin required' });
-                }
-            }
-            catch {
-                return reply.status(401).send({ error: 'Invalid token' });
-            }
-            const { locationId, archiveId } = request.params;
-            const backupSingleton = capabilityRegistry.getSingleton('backup');
-            if (!backupSingleton?.listArchives) {
-                return reply.status(503).send({ error: 'Backup orchestrator unavailable' });
-            }
-            const archives = await backupSingleton.listArchives({ destinationId: locationId });
-            const archive = archives.find((a) => a.id === archiveId);
-            if (!archive) {
-                return reply.status(404).send({ error: 'Archive not found' });
-            }
-            const storage = capabilityRegistry.getSingleton('storage');
-            if (!storage?.beginDownload) {
-                return reply.status(503).send({ error: 'Storage cap unavailable' });
-            }
-            const downloadName = `${archive.label ?? archive.id}.tar.gz`;
-            // Sanitize: strip newlines and double-quotes which would break
-            // the Content-Disposition header. Whitespace is fine.
-            const safeName = downloadName.replace(/[\r\n"]/g, '_');
-            reply.header('content-type', 'application/gzip');
-            reply.header('content-length', String(archive.sizeBytes));
-            reply.header('content-disposition', `attachment; filename="${safeName}"`);
-            const { downloadId, sizeBytes } = await storage.beginDownload({
-                location: locationId,
-                relativePath: archive.filename,
-            });
-            const CHUNK = 8 * 1024 * 1024;
-            try {
-                let offset = 0;
-                // Stream the chunked response. Fastify's `reply.raw` is the
-                // Node.js writable; we write each chunk and `end()` once
-                // we've drained the source. Per-write back-pressure is
-                // handled inside the runtime — buffered writes pile up but
-                // never beyond the OS socket buffer.
-                while (offset < sizeBytes) {
-                    const len = Math.min(CHUNK, sizeBytes - offset);
-                    const chunk = await storage.readChunk({ downloadId, offset, length: len });
-                    if (chunk.byteLength === 0) {
-                        throw new Error(`backup download: empty chunk at offset ${offset}/${sizeBytes}`);
-                    }
-                    reply.raw.write(chunk);
-                    offset += chunk.byteLength;
-                }
-                reply.raw.end();
-            }
-            catch (err) {
-                console.error('[backup-download] stream failed:', err);
-                // Headers may already be flushed — abort the socket if so.
-                if (!reply.raw.headersSent) {
-                    return reply.status(500).send({ error: 'Download failed' });
-                }
-                reply.raw.destroy(err instanceof Error ? err : new Error(String(err)));
-            }
-            finally {
-                try {
-                    await storage.endDownload({ downloadId });
-                }
-                catch {
-                    /* best-effort */
-                }
-            }
-            return reply;
+        // ── Backup archive download ─────────────────────────────────────
+        // Streams an archive through the storage cap's chunked-download
+        // protocol at the pace the client reads it (D447). The route, its
+        // auth chain and the drain-respecting pump live in `api/backup-download*`.
+        (0, backup_download_js_1.registerBackupDownloadRoute)(fastify, {
+            verifier: app.get(auth_service_1.AuthService),
+            capabilities: capabilityRegistry,
+            logger: app.get(logging_service_1.LoggingService).createLogger('backup-download'),
         });
         // POST /api/auth/session — upgrade a tRPC-issued JWT to a browser cookie.
         fastify.post('/api/auth/session', async (request, reply) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@camstack/server",
-  "version": "1.2.269",
+  "version": "1.2.270",
   "private": false,
   "files": [
     "dist",