dsh-zen-remote 2.3.11 → 2.4.1

package/lib/relay-server.js

@@ -29,9 +29,9 @@ import { withPeer } from './owner-peer.js';
 import { controlProjectionFrame } from './merge-streams.js';
 import { randomBytes, timingSafeEqual } from 'node:crypto';
 import { Readable, Transform } from 'node:stream';
-import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
 import { createRequire } from 'node:module';
 import { join } from 'node:path';
+import { ensureServerIdMigrated, readStateFile, STATE_FILE_NAME, writeStateFile } from './state-store.js';
 import { decideHttpRoute, decideInvoke, decideStream, decideUploadQuery, parseEventResultBody } from './relay-access.js';
 import { createWorkspaceFollowState, filterControlFrame, filterJobListFrame, filterModelCatalogResult, filterSessionListResult, filterWorkspaceFrame, } from './relay-filter.js';
 import { SESSION_REFERENCE_URI, collectReferenceTexts, decodeSessionReferenceUri } from './session-reference.js';
@@ -60,6 +60,15 @@ export { MAX_UPLOAD_BYTES };
 /** The `/api` route the upload channel dispatches into, verbatim from the
  * host's own registration (RT dsh-client-file-upload lib/index.js:73). */
 export const UPLOAD_HTTP_PATH = '/api/session/uploadFileBinary';
+/**
+ * The duplex uplink's buffering ceiling (TA-1): how many bytes of
+ * not-yet-consumed uplink frames one live stream may hold — the same budget
+ * the host's own stream inbox applies (`DEFAULT_STREAM_INBOX_BYTES`), so a
+ * stalled terminal consumer backs up at exactly the point the local session
+ * would have. Past it the `/v1/stream-uplink` route answers 413
+ * `uplink-overflow` and the stream is aborted.
+ */
+export const MAX_STREAM_UPLINK_BYTES = 262_144;
 /** Error messages forwarded to the client are clipped to this many chars. */
 const MAX_MESSAGE_CHARS = 500;
 /**
@@ -133,31 +142,39 @@ const JOB_CACHE_LIMIT = 1024;
  * handful of servers apart in a client's server list without leaking anything
  * countable about the deployment. */
 const SERVER_ID_PATTERN = /^[0-9a-f]{8}$/;
-const SERVER_ID_FILE = 'zen-remote-server.json';
+/**
+ * The state file the server identity lives in since MS-1 — the same file the
+ * share table (`sessions`) and the paired-servers list (`servers`) use, so
+ * every writer of one field must preserve the other two. `loadServerId`
+ * reads and writes it only through state-store's merged doors.
+ */
+const SERVER_ID_FILE = STATE_FILE_NAME;
 /**
  * Read the persisted server id, creating (and persisting) one on first use.
- * A missing file is the normal first run; an unreadable or damaged file must
- * never take the host down, so every failure degrades to a fresh per-process
- * id that simply is not remembered across restarts.
+ * A pre-multi-server `zen-remote-server.json` identity migrates in first
+ * (and its file is deleted) — an upgraded host keeps its id, so paired
+ * devices never see a stranger. A missing file is the normal first run; an
+ * unreadable or damaged file must never take the host down, so every failure
+ * degrades to a fresh per-process id that simply is not remembered across
+ * restarts.
  */
 export function loadServerId(home) {
     const file = join(home, SERVER_ID_FILE);
+    ensureServerIdMigrated(file);
     try {
-        const parsed = JSON.parse(readFileSync(file, 'utf8'));
-        const id = parsed?.serverId;
-        if (typeof id === 'string' && SERVER_ID_PATTERN.test(id))
-            return id;
+        const state = readStateFile(file);
+        if (state.serverId !== undefined && SERVER_ID_PATTERN.test(state.serverId))
+            return state.serverId;
     }
     catch {
-        // First run, unreadable path, or unparsable JSON — mint a new one below.
+        // Damaged state: mint a new one below (state-store never throws, but the
+        // belt stays on).
     }
     const id = randomBytes(4).toString('hex');
     try {
-        // tmp + rename, the same atomic dance the share table uses: a crash
-        // mid-write must not leave a half-file that would corrupt the next read.
-        mkdirSync(home, { recursive: true });
-        writeFileSync(`${file}.tmp`, JSON.stringify({ serverId: id }));
-        renameSync(`${file}.tmp`, file);
+        // The merged write: sessions/servers already in the file survive, and the
+        // atomic tmp(0600)+rename dance is state-store's job now.
+        writeStateFile(file, (current) => ({ ...current, version: 1, serverId: id }));
     }
     catch {
         // Unwritable home: the id stays process-local. Nothing may throw here.
@@ -520,6 +537,114 @@ function errorOf(error) {
         out.details = details;
     return out;
 }
+/**
+ * The bounded async queue behind one live duplex stream's uplink half (TA-1).
+ * The `/v1/stream-uplink` route pushes the client's frames in; the gateway's
+ * stream implementation (the host's own uplink consumer) pulls them out.
+ *
+ * Three states beyond "open", each with its own door:
+ * - `ended`: the client half-closed (`end: true`). Queued items still drain;
+ *   the iteration finishes cleanly once the buffer is empty. A push AFTER the
+ *   end is a client bug — it records `failure` and aborts the stream.
+ * - `failure`: a post-end push or a buffer overflow. The recorded error is
+ *   thrown to the consumer, and the stream's abort fires (the route cannot
+ *   push anymore, so the downstream is torn down with it).
+ * - `closed`: the stream is over (its own finally, or closeAll). Everything
+ *   is dropped; further pushes are silently swallowed (`'ok'`) — the route's
+ *   answer for a dead stream is the 404 unknown-stream, so the swallow only
+ *   covers the race between the two.
+ *
+ * The waiter is woken on EVERY state change (item, end, close, failure) so a
+ * parked `next()` never sits on a stale promise: the iteration is the host's
+ * consumer, and a missed wake would hang a terminal that just died.
+ */
+class RelayUplinkQueue {
+    maxBytes;
+    onFail;
+    items = [];
+    bufferedBytes = 0;
+    ended = false;
+    closed = false;
+    failure;
+    waiter;
+    constructor(maxBytes, onFail) {
+        this.maxBytes = maxBytes;
+        this.onFail = onFail;
+    }
+    /** One frame in. `frameBytes` is the frame's serialized size — the same
+     * accounting the host's inbox charges, so the ceiling here means the same
+     * thing it means locally. */
+    push(value, frameBytes) {
+        if (this.closed || this.failure !== undefined)
+            return 'ok';
+        if (this.ended) {
+            this.failure = new Error('uplink frame arrived after end');
+            this.wake();
+            this.onFail();
+            return 'after-end';
+        }
+        if (this.bufferedBytes + frameBytes > this.maxBytes) {
+            this.failure = new Error('uplink buffer overflow');
+            this.wake();
+            this.onFail();
+            return 'overflow';
+        }
+        this.items.push({ value, bytes: frameBytes });
+        this.bufferedBytes += frameBytes;
+        this.wake();
+        return 'ok';
+    }
+    /** The client half-closed: queued items still drain, then the iteration
+     * ends. Idempotent. */
+    end() {
+        if (this.ended)
+            return;
+        this.ended = true;
+        this.wake();
+    }
+    /** The whole stream is over: drop everything, finish every parked reader,
+     * swallow all future pushes. Idempotent. */
+    close() {
+        if (this.closed)
+            return;
+        this.closed = true;
+        this.items.length = 0;
+        this.bufferedBytes = 0;
+        this.wake();
+    }
+    wake() {
+        const wake = this.waiter;
+        this.waiter = undefined;
+        wake?.();
+    }
+    [Symbol.asyncIterator]() {
+        return {
+            next: () => this.next(),
+            return: () => {
+                this.close();
+                return Promise.resolve({ value: undefined, done: true });
+            },
+        };
+    }
+    async next() {
+        while (true) {
+            if (this.closed)
+                return { value: undefined, done: true };
+            const entry = this.items.shift();
+            if (entry !== undefined) {
+                this.bufferedBytes -= entry.bytes;
+                return { value: entry.value, done: false };
+            }
+            if (this.failure !== undefined)
+                throw this.failure;
+            if (this.ended)
+                return { value: undefined, done: true };
+            await new Promise((resolve) => {
+                this.waiter = resolve;
+            });
+        }
+    }
+}
 /**
  * Build the relay route handler mounted under {@link RELAY_PREFIX}. Exported
  * as a factory so the route tests can drive it against a plain node:http
@@ -592,6 +717,7 @@ export function createRelayHandler(options) {
      * route searches these for every incoming eventId. A closed
      * subscription removes itself; closeAll empties the set. */
     const liveEventSubscriptions = new Set();
+    const liveUplinkStreams = new Map();
     /** Latest `job/list` rows seen per session — the recent-state half of the
      * job ownership check (the other half is a one-shot list on a cache miss).
      * An unshared session's rows are forgotten (T22b-fix): its jobs are none of
@@ -1991,6 +2117,68 @@ export function createRelayHandler(options) {
             }
             return;
         }
+        if (req.method === 'POST' && pathname === `${RELAY_PREFIX}/v1/stream-uplink`) {
+            // The duplex uplink's post route (TA-1): one JSON body of frames for a
+            // stream opened with `uplink: true`, resolved against the live-uplink
+            // table. Four walls in order, each with its own status: the shape
+            // (400), the stream id and its owning device (404 — unknown-stream,
+            // deliberately indistinguishable between "no such stream" and "not
+            // yours", so a foreign device learns nothing), the share table at post
+            // time (403 — an unshared session's keystrokes must die with the
+            // share), and the sequence counter (409 — replays and reordered posts
+            // would corrupt the host's own input sequencing).
+            const body = await readBodyOrRespond(req, res);
+            if (body === undefined)
+                return;
+            const streamId = body.streamId;
+            const seq = body.seq;
+            const items = body.items;
+            const end = body.end;
+            if (typeof streamId !== 'string' ||
+                streamId === '' ||
+                typeof seq !== 'number' ||
+                !Number.isSafeInteger(seq) ||
+                seq < 1 ||
+                (items !== undefined && !Array.isArray(items)) ||
+                (end !== undefined && typeof end !== 'boolean') ||
+                !((Array.isArray(items) && items.length > 0) || end === true)) {
+                responseJson(res, 400, { ok: false, error: { code: 'bad-request' } });
+                return;
+            }
+            const device = headerValue(req, 'x-zen-remote-device') ?? '';
+            const entry = liveUplinkStreams.get(streamId);
+            if (entry === undefined || entry.device !== device) {
+                responseJson(res, 404, { ok: false, error: { code: 'unknown-stream' } });
+                return;
+            }
+            if (!entry.sessionIds.every((id) => isAccessible(id))) {
+                responseJson(res, 403, { ok: false, error: { code: 'not-shared' } });
+                return;
+            }
+            if (seq !== entry.nextSeq) {
+                responseJson(res, 409, { ok: false, error: { code: 'out-of-order' } });
+                return;
+            }
+            entry.nextSeq += 1;
+            if (Array.isArray(items)) {
+                for (const item of items) {
+                    const itemBytes = Buffer.byteLength(JSON.stringify(item) ?? '', 'utf8');
+                    const status = entry.queue.push(item, itemBytes);
+                    if (status === 'after-end') {
+                        responseJson(res, 400, { ok: false, error: { code: 'uplink-after-end' } });
+                        return;
+                    }
+                    if (status === 'overflow') {
+                        responseJson(res, 413, { ok: false, error: { code: 'uplink-overflow' } });
+                        return;
+                    }
+                }
+            }
+            if (end === true)
+                entry.queue.end();
+            responseJson(res, 200, { ok: true });
+            return;
+        }
         if (req.method === 'POST' && pathname === `${RELAY_PREFIX}/v1/stream`) {
             // Body validation identical to invoke: one JSON object naming the
             // method and carrying object args.
@@ -2017,6 +2205,23 @@ export function createRelayHandler(options) {
                 responseJson(res, 403, { ok: false, error: { code: decision.reason } });
                 return;
             }
+            // TA-1: the client's advertised uplink window, shape-checked ONCE here —
+            // only a well-formed window travels to the gateway; garbage (a zero or
+            // negative budget, a fractional value) is dropped, never refused: the
+            // numbers are the host's own contract with itself, and a missing one
+            // just means the host default applies.
+            let uplinkWindow;
+            if (isPlainObject(body.uplinkWindow)) {
+                const candidate = body.uplinkWindow;
+                if (typeof candidate.maxBytes === 'number' &&
+                    Number.isSafeInteger(candidate.maxBytes) &&
+                    candidate.maxBytes > 0 &&
+                    typeof candidate.frameOverheadBytes === 'number' &&
+                    Number.isSafeInteger(candidate.frameOverheadBytes) &&
+                    candidate.frameOverheadBytes >= 0) {
+                    uplinkWindow = { maxBytes: candidate.maxBytes, frameOverheadBytes: candidate.frameOverheadBytes };
+                }
+            }
             const device = headerValue(req, 'x-zen-remote-device') ?? '';
             if ((streamsByDevice.get(device) ?? 0) >= MAX_STREAMS_PER_DEVICE) {
                 responseJson(res, 429, { ok: false, error: { code: 'too-many-streams' } });
@@ -2178,6 +2383,30 @@ export function createRelayHandler(options) {
                             res.end();
                     });
                 };
+                // ---- TA-1: the duplex uplink -------------------------------------
+                // A duplex decision the client took up: build the bounded queue the
+                // gateway will read keystrokes from, register it for the
+                // /v1/stream-uplink route, and send the handshake line FIRST —
+                // before the upstream is even open — so the client may start posting
+                // the moment it sees the id. The write chain serializes the line
+                // ahead of any frame; registration precedes the open so a post that
+                // races the handshake still finds the entry (its seq order and the
+                // share table keep it honest either way).
+                let uplinkStreamId;
+                let uplinkQueue;
+                if (decision.uplink === true && body.uplink === true) {
+                    uplinkStreamId = randomBytes(16).toString('hex');
+                    uplinkQueue = new RelayUplinkQueue(MAX_STREAM_UPLINK_BYTES, () => hangUp.abort());
+                    liveUplinkStreams.set(uplinkStreamId, {
+                        streamId: uplinkStreamId,
+                        device,
+                        sessionIds: decision.sessionIds,
+                        nextSeq: 1,
+                        queue: uplinkQueue,
+                        abort: () => hangUp.abort(),
+                    });
+                    void writeLine({ type: 'uplink', streamId: uplinkStreamId });
+                }
                 // ---- share-change synchronization (T22b §4) --------------------
                 // The `$zr/events` forwarding bookkeeping (T32): registered for the
                 // answer route the moment this stream starts being served, removed
@@ -2342,10 +2571,19 @@ export function createRelayHandler(options) {
                     else {
                         // `$zr/events` opens through the wire adapter (the ONLY door to
                         // the forwarded-event stream); every other route is a typert
-                        // stream method.
+                        // stream method. TA-1: a duplex stream rides with its uplink
+                        // queue (the field's PRESENCE is the duplex flag the gateway
+                        // reads) and the shape-checked window, both optional.
                         const iterable = eventsSub !== undefined
                             ? await openEventsStream(hangUp.signal)
-                            : await gateway.stream(withPeer({ namespace, method, args, signal: hangUp.signal }, options.getPeer));
+                            : await gateway.stream(withPeer({
+                                namespace,
+                                method,
+                                args,
+                                signal: hangUp.signal,
+                                ...(uplinkQueue !== undefined ? { uplink: uplinkQueue } : {}),
+                                ...(uplinkWindow !== undefined ? { uplinkWindow } : {}),
+                            }, options.getPeer));
                         for await (const frame of iterable) {
                             if (finished || clientGone)
                                 break;
@@ -2376,6 +2614,13 @@ export function createRelayHandler(options) {
                     clearInterval(heartbeat);
                     unsubscribe();
                     tableUnsubscribers.delete(unsubscribe);
+                    // TA-1: the uplink half dies with the stream — deregistered BEFORE
+                    // the queue closes, so a post racing the teardown answers
+                    // unknown-stream instead of feeding a dead queue, and the gateway's
+                    // consumer is finished instead of parked forever.
+                    if (uplinkStreamId !== undefined)
+                        liveUplinkStreams.delete(uplinkStreamId);
+                    uplinkQueue?.close();
                     if (eventsSub !== undefined) {
                         liveEventSubscriptions.delete(eventsSub);
                         eventsSub.events.clear();
@@ -2423,6 +2668,13 @@ export function createRelayHandler(options) {
             sub.events.clear();
         }
         liveEventSubscriptions.clear();
+        // TA-1: the duplex uplinks die with the handler — close() finishes every
+        // parked gateway consumer (a terminal read must not hang on a reload)
+        // and later posts answer unknown-stream; the streams' own finallys then
+        // find their entries already gone and delete nothing.
+        for (const entry of liveUplinkStreams.values())
+            entry.queue.close();
+        liveUplinkStreams.clear();
         // Remember the reason for requests still inside their ownership probe —
         // they read it at the pre-open gate. Killing the open streams runs their
         // normal teardown (error line, res.end, abort), whose finally releases