@sema-agent/server 7.50.0 → 7.52.0

package/dist/device-ws-hub.js

@@ -0,0 +1,1012 @@
+import http from "node:http";
+import { createHash } from "node:crypto";
+import { WebSocketServer } from "ws";
+import { DEVICE_ERROR_CODES, DEVICE_ERROR_STATUS, assertDeviceIdShape, } from "./device-store.js";
+import { DEVICE_MIN_PROTOCOL_VERSION, DEVICE_WS_LIMITS, DEVICE_WS_PATH, DEVICE_PROTOCOL_VERSION, asInstructionKind, buildHelloSignaturePayload, formatGeneration, mintChallengeNonce, mintInstructionId, parseDeviceFrameText, verifyHelloSignature, } from "./device-ws-protocol.js";
+const OUTCOME_UNKNOWN_SENTENCE = "outcome unknown — the command may have already executed on the device; verify its effect before re-running";
+const SETTLED_MEMO_MAX = 512;
+function rejection(code, message, retryAfterSec) {
+    return { code, status: DEVICE_ERROR_STATUS[code], message, ...(retryAfterSec === undefined ? {} : { retryAfterSec }) };
+}
+function capabilityDriftResult(drift) {
+    return {
+        kind: "never_started",
+        outcome: {
+            ok: false,
+            errorCode: "precondition_failed",
+            message: `[device.capability_drift] the connected device declares protective primitives (${drift.join(", ")}) that the caller's environment was minted without — ` +
+                "dispatching a non-atomic form here would be exactly the silent protective downgrade this refusal exists to prevent (#157)",
+        },
+    };
+}
+function outcomeUnknown(why) {
+    return { ok: false, errorCode: "outcome_unknown", message: `${why} — ${OUTCOME_UNKNOWN_SENTENCE}` };
+}
+const WRITE_PAYLOAD_KINDS = new Set(["writeFile", "writeFileExclusive", "writeFileGuarded", "appendFile"]);
+const ALLOWED_SUCCESS_OUTCOME_KINDS = {
+    exec: ["exec"],
+    execStream: ["exec"],
+    readTextFile: ["read"],
+    readTextLines: ["read"],
+    readBinaryFile: ["read"],
+    writeFile: ["write"],
+    writeFileExclusive: ["write"],
+    writeFileGuarded: ["write"],
+    appendFile: ["void"],
+    fileInfo: ["stat"],
+    listDir: ["stat"],
+    canonicalPath: ["stat"],
+    exists: ["stat"],
+    readLink: ["stat"],
+    statBatch: ["stat"],
+    createDir: ["void"],
+    remove: ["void"],
+    createTempDir: ["write"],
+    createTempFile: ["write"],
+};
+export function createDeviceWsHub(options) {
+    const { store, enrollment, replicaId, logger, metrics } = options;
+    const heartbeatIntervalMs = options.heartbeatIntervalMs ?? 15_000;
+    const heartbeatMisses = options.heartbeatMisses ?? 3;
+    const leaseSeconds = options.leaseSeconds ?? 45;
+    const leaseRenewIntervalMs = options.leaseRenewIntervalMs ?? heartbeatIntervalMs;
+    const maxInflightPerDevice = options.maxInflightPerDevice ?? 4;
+    const dispatchTimeoutMs = options.dispatchTimeoutMs ?? 10_000;
+    const execGraceMs = options.execGraceMs ?? 30_000;
+    const cancelGraceMs = options.cancelGraceMs ?? 10_000;
+    const reconnectGraceMs = options.reconnectGraceMs ?? 120_000;
+    const chunkGapTimeoutMs = options.chunkGapTimeoutMs ?? 30_000;
+    const supersede = options.supersede ?? "stale_only";
+    const chunkBufferMaxEntries = options.chunkBufferMaxEntries ?? 64;
+    const chunkBufferMaxBytes = options.chunkBufferMaxBytes ?? 2 * 1024 * 1024;
+    const maxPreAuthConnections = options.maxPreAuthConnections ?? 64;
+    const maxPreAuthConnectionsPerRemote = options.maxPreAuthConnectionsPerRemote ?? 8;
+    const wss = new WebSocketServer({ noServer: true, maxPayload: DEVICE_WS_LIMITS.maxFrameBytes });
+    const conns = new Map();
+    const allConns = new Set();
+    let preAuthCount = 0;
+    const preAuthByRemote = new Map();
+    const pending = new Map();
+    const inflightCount = new Map();
+    const slotWaiters = new Map();
+    const reconnectTimers = new Map();
+    const settledMemo = new Map();
+    let draining = false;
+    let closed = false;
+    function auditFrame(deviceId, owner, reason, detail) {
+        metrics?.inc("device_frame_unexpected_total", { reason });
+        void store
+            .appendAudit({ event: "device_frame_unexpected", deviceId, owner, detail: { reason, ...detail } })
+            .catch((err) => logger?.error("device_audit_write_failed", { event: "device_frame_unexpected", reason, err: err instanceof Error ? err.message : String(err) }));
+    }
+    function auditEvent(event, deviceId, owner, detail) {
+        void store
+            .appendAudit({ event, deviceId, owner, detail })
+            .catch((err) => logger?.error("device_audit_write_failed", { event, err: err instanceof Error ? err.message : String(err) }));
+    }
+    function sendServerFrame(conn, frame) {
+        if (conn.ws.readyState !== conn.ws.OPEN)
+            return false;
+        try {
+            conn.ws.send(JSON.stringify(frame));
+            return true;
+        }
+        catch (err) {
+            logger?.warn("device_ws_send_failed", { deviceId: conn.deviceId, t: frame.t, err: err instanceof Error ? err.message : String(err) });
+            return false;
+        }
+    }
+    function acquireSlot(deviceId) {
+        const cur = inflightCount.get(deviceId) ?? 0;
+        if (cur < maxInflightPerDevice) {
+            inflightCount.set(deviceId, cur + 1);
+            return Promise.resolve(true);
+        }
+        return new Promise((resolve) => {
+            const queue = slotWaiters.get(deviceId) ?? [];
+            const waiter = {
+                resolve,
+                timer: setTimeout(() => {
+                    const idx = queue.indexOf(waiter);
+                    if (idx >= 0)
+                        queue.splice(idx, 1);
+                    resolve(false);
+                }, dispatchTimeoutMs),
+            };
+            queue.push(waiter);
+            slotWaiters.set(deviceId, queue);
+        });
+    }
+    function releaseSlot(deviceId) {
+        const queue = slotWaiters.get(deviceId);
+        const waiter = queue?.shift();
+        if (waiter) {
+            clearTimeout(waiter.timer);
+            waiter.resolve(true);
+            return;
+        }
+        inflightCount.set(deviceId, Math.max(0, (inflightCount.get(deviceId) ?? 1) - 1));
+    }
+    function rememberSettled(p) {
+        if (settledMemo.size >= SETTLED_MEMO_MAX) {
+            const oldest = settledMemo.keys().next();
+            if (!oldest.done)
+                settledMemo.delete(oldest.value);
+        }
+        settledMemo.set(p.instructionId, { deviceId: p.deviceId, generation: p.generation });
+    }
+    function settle(p, outcome) {
+        if (p.settled)
+            return;
+        p.settled = true;
+        p.state = "terminal";
+        if (p.ceilingTimer)
+            clearTimeout(p.ceilingTimer);
+        if (p.cancelTimer)
+            clearTimeout(p.cancelTimer);
+        if (p.chunkGapTimer)
+            clearTimeout(p.chunkGapTimer);
+        p.ceilingTimer = null;
+        p.cancelTimer = null;
+        p.chunkGapTimer = null;
+        pending.delete(p.instructionId);
+        rememberSettled(p);
+        releaseSlot(p.deviceId);
+        p.resolve(outcome);
+    }
+    function settleUnknown(p, why) {
+        settle(p, outcomeUnknown(why));
+    }
+    function pendingOfDevice(deviceId) {
+        return [...pending.values()].filter((p) => p.deviceId === deviceId);
+    }
+    function abandonPendingOfDevice(deviceId, why) {
+        for (const p of pendingOfDevice(deviceId))
+            settleUnknown(p, why);
+    }
+    function releasePreAuthSlot(conn) {
+        if (!conn.holdsPreAuthSlot)
+            return;
+        conn.holdsPreAuthSlot = false;
+        preAuthCount = Math.max(0, preAuthCount - 1);
+        const cur = preAuthByRemote.get(conn.remoteKey) ?? 1;
+        if (cur <= 1)
+            preAuthByRemote.delete(conn.remoteKey);
+        else
+            preAuthByRemote.set(conn.remoteKey, cur - 1);
+    }
+    function clearConnTimers(conn) {
+        if (conn.handshakeTimer)
+            clearTimeout(conn.handshakeTimer);
+        if (conn.watchdog)
+            clearInterval(conn.watchdog);
+        if (conn.leaseTimer)
+            clearInterval(conn.leaseTimer);
+        if (conn.goodbyeTimer)
+            clearTimeout(conn.goodbyeTimer);
+        conn.handshakeTimer = null;
+        conn.watchdog = null;
+        conn.leaseTimer = null;
+        conn.goodbyeTimer = null;
+    }
+    const pendingLeaseReleases = new Map();
+    function releaseLeaseTracked(deviceId, generation) {
+        const p = store
+            .releaseConnection(deviceId, generation)
+            .catch((err) => logger?.warn("device_lease_release_failed", { deviceId, err: err instanceof Error ? err.message : String(err) }))
+            .then(() => undefined)
+            .finally(() => {
+            if (pendingLeaseReleases.get(deviceId) === p)
+                pendingLeaseReleases.delete(deviceId);
+        });
+        pendingLeaseReleases.set(deviceId, p);
+        return p;
+    }
+    function closeConn(conn, opts) {
+        if (conn.phase === "closing")
+            return;
+        conn.phase = "closing";
+        clearConnTimers(conn);
+        releasePreAuthSlot(conn);
+        allConns.delete(conn);
+        if (conn.deviceId !== "" && conns.get(conn.deviceId) === conn) {
+            conns.delete(conn.deviceId);
+            if (opts.releaseLease) {
+                void releaseLeaseTracked(conn.deviceId, conn.generation);
+            }
+            if (opts.abandonPending !== undefined)
+                abandonPendingOfDevice(conn.deviceId, opts.abandonPending);
+            else
+                scheduleReconnectGrace(conn.deviceId);
+        }
+        try {
+            conn.ws.close();
+        }
+        catch (err) {
+            logger?.warn("device_ws_close_failed", { deviceId: conn.deviceId, err: err instanceof Error ? err.message : String(err) });
+        }
+    }
+    function scheduleReconnectGrace(deviceId) {
+        if (closed)
+            return;
+        const prior = reconnectTimers.get(deviceId);
+        if (prior)
+            clearTimeout(prior);
+        const timer = setTimeout(() => {
+            reconnectTimers.delete(deviceId);
+            if (conns.has(deviceId))
+                return;
+            for (const p of pendingOfDevice(deviceId))
+                settleUnknown(p, "the device did not reconnect within the reconnect grace window");
+        }, reconnectGraceMs);
+        reconnectTimers.set(deviceId, timer);
+    }
+    function rejectHello(conn, code, minProtocolVersion) {
+        if (conn.handshakeTimer)
+            clearTimeout(conn.handshakeTimer);
+        conn.handshakeTimer = null;
+        sendServerFrame(conn, { t: "helloReject", code, ...(minProtocolVersion === undefined ? {} : { minProtocolVersion }) });
+        closeConn(conn, { releaseLease: false });
+    }
+    async function fenceConnection(conn) {
+        if (conn.phase === "closing" || conn.deviceId === "")
+            return false;
+        let held = false;
+        try {
+            held = await store.heartbeatConnection({ deviceId: conn.deviceId, generation: conn.generation, leaseSeconds });
+        }
+        catch (err) {
+            logger?.warn("device_lease_renew_failed", { deviceId: conn.deviceId, err: err instanceof Error ? err.message : String(err) });
+            return "unavailable";
+        }
+        if (held) {
+            conn.lastLeaseRenewAt = Date.now();
+            return true;
+        }
+        let revoked = false;
+        try {
+            const row = await store.getDeviceUnscoped(conn.deviceId);
+            revoked = row?.status === "revoked";
+        }
+        catch (err) {
+            logger?.warn("device_row_read_failed", { deviceId: conn.deviceId, err: err instanceof Error ? err.message : String(err) });
+        }
+        metrics?.inc("device_fence_lost_total", { reason: revoked ? "revoked" : "superseded" });
+        sendServerFrame(conn, { t: "bye", gen: conn.genStr, reason: revoked ? "revoked" : "superseded" });
+        closeConn(conn, { releaseLease: false, abandonPending: revoked ? "the device was revoked while this instruction was in flight" : "the device connection lease was taken over elsewhere while this instruction was in flight" });
+        return false;
+    }
+    async function renewLeaseThrottled(conn) {
+        if (conn.phase !== "active")
+            return;
+        const floor = Math.max(50, Math.floor(leaseRenewIntervalMs / 3));
+        if (Date.now() - conn.lastLeaseRenewAt < floor)
+            return;
+        conn.lastLeaseRenewAt = Date.now();
+        await fenceConnection(conn);
+    }
+    function startConnTimers(conn) {
+        conn.watchdog = setInterval(() => {
+            if (Date.now() - conn.lastFrameAt > heartbeatIntervalMs * heartbeatMisses) {
+                logger?.warn("device_heartbeat_lost", { deviceId: conn.deviceId, generation: conn.generation });
+                metrics?.inc("device_heartbeat_lost_total");
+                closeConn(conn, { releaseLease: true });
+            }
+        }, Math.max(50, Math.floor(heartbeatIntervalMs / 2)));
+        conn.leaseTimer = setInterval(() => {
+            if (Date.now() - conn.lastFrameAt > heartbeatIntervalMs * heartbeatMisses)
+                return;
+            conn.lastLeaseRenewAt = Date.now();
+            void fenceConnection(conn);
+        }, leaseRenewIntervalMs);
+    }
+    function handshakeStillLive(conn) {
+        return conn.phase === "authenticating" && conn.ws.readyState === conn.ws.OPEN;
+    }
+    async function handleHello(conn, frame) {
+        if (draining || closed)
+            return rejectHello(conn, "draining");
+        if (frame.protocolVersion < DEVICE_MIN_PROTOCOL_VERSION || frame.protocolVersion > DEVICE_PROTOCOL_VERSION) {
+            return rejectHello(conn, "protocol_version_unsupported", DEVICE_MIN_PROTOCOL_VERSION);
+        }
+        let idShapeOk = true;
+        try {
+            assertDeviceIdShape(frame.deviceId);
+        }
+        catch {
+            idShapeOk = false;
+        }
+        let row = null;
+        if (idShapeOk) {
+            try {
+                row = await store.getDeviceUnscoped(frame.deviceId);
+            }
+            catch (err) {
+                logger?.error("device_hello_store_read_failed", { err: err instanceof Error ? err.message : String(err) });
+                return rejectHello(conn, "auth_failed");
+            }
+        }
+        if (!handshakeStillLive(conn))
+            return closeConn(conn, { releaseLease: false });
+        let payload = null;
+        if (row) {
+            try {
+                payload = buildHelloSignaturePayload({ nonce: conn.nonce, deviceId: frame.deviceId, epoch: frame.epoch, protocolVersion: frame.protocolVersion });
+            }
+            catch {
+                payload = null;
+            }
+        }
+        if (row === null || payload === null)
+            return rejectHello(conn, "auth_failed");
+        const verdict = verifyHelloSignature({ pubkeyB64: row.pubkey, signatureB64: frame.signatureB64, payload });
+        if (!verdict.ok) {
+            logger?.warn("device_hello_signature_rejected", { deviceId: frame.deviceId, reason: verdict.reason });
+            metrics?.inc("device_hello_rejected_total", { code: "auth_failed" });
+            return rejectHello(conn, "auth_failed");
+        }
+        if (row.status === "revoked")
+            return rejectHello(conn, "revoked");
+        const existing = conns.get(row.deviceId);
+        if (existing && existing !== conn) {
+            const fresh = Date.now() - existing.lastFrameAt < heartbeatIntervalMs * heartbeatMisses;
+            if (supersede === "deny" || fresh) {
+                auditEvent("device_attached_elsewhere", row.deviceId, row.owner, { at: "hello", reason: supersede === "deny" ? "supersede knob is deny" : "the existing connection is still fresh", replicaId });
+                return rejectHello(conn, "already_connected");
+            }
+            auditEvent("device_superseded", row.deviceId, row.owner, { at: "hello", supersededGeneration: existing.generation, replicaId });
+            sendServerFrame(existing, { t: "bye", gen: existing.genStr, reason: "superseded" });
+            closeConn(existing, { releaseLease: true });
+        }
+        const inflightRelease = pendingLeaseReleases.get(row.deviceId);
+        if (inflightRelease)
+            await inflightRelease;
+        if (!handshakeStillLive(conn))
+            return closeConn(conn, { releaseLease: false });
+        let acquired;
+        try {
+            acquired = await store.acquireConnection({ deviceId: row.deviceId, owner: row.owner, connEpoch: frame.epoch, replicaId, leaseSeconds });
+        }
+        catch (err) {
+            logger?.error("device_acquire_failed", { deviceId: row.deviceId, err: err instanceof Error ? err.message : String(err) });
+            return rejectHello(conn, "auth_failed");
+        }
+        switch (acquired.outcome) {
+            case "attached_elsewhere":
+                auditEvent("device_attached_elsewhere", row.deviceId, row.owner, { at: "acquire", heldGeneration: acquired.generation, replicaId });
+                return rejectHello(conn, "already_connected");
+            case "device_not_found":
+                return rejectHello(conn, "auth_failed");
+            case "device_revoked":
+                return rejectHello(conn, "revoked");
+            case "acquired":
+                break;
+        }
+        if (!handshakeStillLive(conn)) {
+            void releaseLeaseTracked(row.deviceId, acquired.generation);
+            return closeConn(conn, { releaseLease: false });
+        }
+        const priorTimer = reconnectTimers.get(row.deviceId);
+        if (priorTimer) {
+            clearTimeout(priorTimer);
+            reconnectTimers.delete(row.deviceId);
+        }
+        const pendingIds = [];
+        const chunkWatermarks = {};
+        for (const p of pendingOfDevice(row.deviceId)) {
+            if (p.epoch !== frame.epoch) {
+                settleUnknown(p, "the executor reconnected with a new process epoch, so its in-flight instructions were abandoned");
+                continue;
+            }
+            p.generation = acquired.generation;
+            pendingIds.push(p.instructionId);
+            chunkWatermarks[p.instructionId] = p.nextStreamSeq - 1;
+        }
+        conn.deviceId = row.deviceId;
+        conn.owner = row.owner;
+        conn.epoch = frame.epoch;
+        conn.generation = acquired.generation;
+        conn.genStr = formatGeneration(acquired.generation);
+        conn.workspaceRoot = frame.workspaceRoot;
+        const supported = new Set();
+        for (const s of frame.supports) {
+            const k = asInstructionKind(s);
+            if (k !== null)
+                supported.add(k);
+        }
+        conn.supports = supported;
+        conn.phase = "acquiring";
+        conn.lastLeaseRenewAt = Date.now();
+        conns.set(row.deviceId, conn);
+        if (conn.handshakeTimer)
+            clearTimeout(conn.handshakeTimer);
+        conn.handshakeTimer = null;
+        releasePreAuthSlot(conn);
+        sendServerFrame(conn, { t: "helloAck", ok: true, heartbeatIntervalMs, maxInflight: maxInflightPerDevice, gen: conn.genStr, pendingInstructions: pendingIds, chunkWatermarks });
+        logger?.info?.("device_connected", { deviceId: row.deviceId, generation: acquired.generation, epoch: frame.epoch, replicaId });
+        metrics?.inc("device_connected_total");
+    }
+    function deliverChunks(conn, p) {
+        let delivered = false;
+        for (;;) {
+            const next = p.chunkBuffer.get(p.nextStreamSeq);
+            if (!next)
+                break;
+            p.chunkBuffer.delete(p.nextStreamSeq);
+            p.chunkBufferBytes -= chunkPayloadBytes(next);
+            p.nextStreamSeq += 1;
+            p.lastDeliveredSeq = next.streamSeq;
+            if (next.kind === "exit") {
+                p.exitChunkCount += 1;
+                p.exitChunk = { streamSeq: next.streamSeq, exitCode: next.exitCode ?? 0 };
+            }
+            delivered = true;
+            if (p.onChunk) {
+                try {
+                    p.onChunk({
+                        streamSeq: next.streamSeq,
+                        kind: next.kind,
+                        ...(next.dataB64 === undefined ? {} : { data: Buffer.from(next.dataB64, "base64") }),
+                        ...(next.exitCode === undefined ? {} : { exitCode: next.exitCode }),
+                    });
+                }
+                catch (err) {
+                    logger?.error("device_chunk_consumer_threw", { instructionId: p.instructionId, err: err instanceof Error ? err.message : String(err) });
+                }
+            }
+        }
+        if (delivered)
+            sendServerFrame(conn, { t: "chunkAck", gen: conn.genStr, instructionId: p.instructionId, upToStreamSeq: p.nextStreamSeq - 1 });
+        if (p.chunkBuffer.size > 0 && p.chunkGapTimer === null) {
+            p.chunkGapTimer = setTimeout(() => failOnChunkGap(p), chunkGapTimeoutMs);
+        }
+        else if (p.chunkBuffer.size === 0 && p.chunkGapTimer !== null) {
+            clearTimeout(p.chunkGapTimer);
+            p.chunkGapTimer = null;
+        }
+    }
+    function failOnChunkGap(p) {
+        const missing = p.nextStreamSeq;
+        settle(p, {
+            ok: false,
+            errorCode: "outcome_unknown",
+            message: `the output stream had a gap (missing chunk streamSeq ${missing}) so the result cannot be reported faithfully — ${OUTCOME_UNKNOWN_SENTENCE}`,
+        });
+    }
+    function chunkPayloadBytes(frame) {
+        return frame.dataB64 === undefined ? 0 : Math.floor((frame.dataB64.length * 3) / 4);
+    }
+    function streamTerminatorComplaint(p, outcome) {
+        if (outcome.ok === true && outcome.kind !== "cancelled" && !ALLOWED_SUCCESS_OUTCOME_KINDS[p.kind].includes(outcome.kind)) {
+            return `the device answered a ${p.kind} instruction with a ${outcome.kind} outcome (allowed: ${ALLOWED_SUCCESS_OUTCOME_KINDS[p.kind].join("|")}|cancelled)`;
+        }
+        if (!p.expectsStreamTerminator)
+            return null;
+        if (!(outcome.ok === true && outcome.kind === "exec"))
+            return null;
+        if (p.exitChunkCount === 0)
+            return "the stream never delivered its single exit chunk, so its tail may have been truncated";
+        if (p.exitChunkCount > 1)
+            return `the stream delivered ${p.exitChunkCount} exit chunks (the contract allows exactly one)`;
+        if (p.exitChunk !== null && p.exitChunk.streamSeq !== p.lastDeliveredSeq)
+            return "the exit chunk was not the last chunk of the stream";
+        if (p.exitChunk !== null && p.exitChunk.exitCode !== outcome.exitCode)
+            return `the exit chunk reported exitCode ${p.exitChunk.exitCode} but the result reported ${outcome.exitCode}`;
+        return null;
+    }
+    function matchPending(conn, instructionId, t) {
+        const p = pending.get(instructionId);
+        if (!p) {
+            const memo = settledMemo.get(instructionId);
+            if (!memo) {
+                auditFrame(conn.deviceId, conn.owner, "unknown_instruction", { instructionId, t });
+                return { rejected: true };
+            }
+            if (memo.deviceId !== conn.deviceId) {
+                auditFrame(conn.deviceId, conn.owner, "device_mismatch", { instructionId, boundDeviceId: memo.deviceId, settled: true, t });
+                return { rejected: true };
+            }
+            return { memo: true };
+        }
+        if (p.deviceId !== conn.deviceId) {
+            auditFrame(conn.deviceId, conn.owner, "device_mismatch", { instructionId, boundDeviceId: p.deviceId, t });
+            return { rejected: true };
+        }
+        if (p.state !== "dispatch_committed") {
+            auditFrame(conn.deviceId, conn.owner, "instruction_not_committed", { instructionId, state: p.state, t });
+            return { rejected: true };
+        }
+        return { hit: p };
+    }
+    async function acceptTerminal(conn, p, outcome) {
+        p.terminalFenceInFlight = true;
+        let fenced;
+        try {
+            fenced = await fenceConnection(conn);
+        }
+        finally {
+            p.terminalFenceInFlight = false;
+        }
+        if (p.settled)
+            return;
+        if (p.chunkBuffer.size > 0) {
+            failOnChunkGap(p);
+            return;
+        }
+        if (fenced !== true) {
+            auditFrame(conn.deviceId, conn.owner, "generation_mismatch", {
+                instructionId: p.instructionId,
+                t: "result",
+                detail: fenced === "unavailable" ? "the store was unavailable so the terminal could not be fenced" : "the connection no longer holds the device lease",
+            });
+            settleUnknown(p, fenced === "unavailable" ? "the device lease could not be re-asserted when its result arrived" : "the device lost its lease before its result could be accepted");
+            return;
+        }
+        const complaint = streamTerminatorComplaint(p, outcome);
+        if (complaint !== null) {
+            metrics?.inc("device_stream_terminator_missing_total");
+            sendServerFrame(conn, { t: "resultAck", gen: conn.genStr, instructionId: p.instructionId });
+            settle(p, { ok: false, errorCode: "outcome_unknown", message: `${complaint} — ${OUTCOME_UNKNOWN_SENTENCE}` });
+            return;
+        }
+        sendServerFrame(conn, { t: "resultAck", gen: conn.genStr, instructionId: p.instructionId });
+        settle(p, outcome);
+    }
+    function onDeviceText(conn, text) {
+        conn.lastFrameAt = Date.now();
+        const preAuth = conn.phase === "await_hello" || conn.phase === "authenticating";
+        const cap = preAuth ? DEVICE_WS_LIMITS.maxPreAuthFrameBytes : DEVICE_WS_LIMITS.maxFrameBytes;
+        const parsed = parseDeviceFrameText(text, { maxBytes: cap });
+        if (!parsed.ok) {
+            auditFrame(conn.deviceId === "" ? null : conn.deviceId, conn.owner, parsed.reason, { detail: parsed.detail, phase: conn.phase });
+            if (preAuth)
+                closeConn(conn, { releaseLease: false });
+            return;
+        }
+        const frame = parsed.frame;
+        if (preAuth) {
+            if (conn.phase === "authenticating" || conn.nonceConsumed) {
+                auditFrame(conn.deviceId === "" ? null : conn.deviceId, conn.owner, "pre_hello", { t: frame.t, phase: conn.phase, reason: "a handshake is already in flight on this socket" });
+                return;
+            }
+            if (frame.t !== "hello") {
+                auditFrame(null, null, "pre_hello", { t: frame.t, phase: conn.phase });
+                closeConn(conn, { releaseLease: false });
+                return;
+            }
+            conn.nonceConsumed = true;
+            conn.phase = "authenticating";
+            void handleHello(conn, frame).catch((err) => {
+                logger?.error("device_hello_failed", { err: err instanceof Error ? err.message : String(err) });
+                closeConn(conn, { releaseLease: false });
+            });
+            return;
+        }
+        if (frame.t === "hello") {
+            auditFrame(conn.deviceId, conn.owner, "pre_hello", { t: "hello", phase: conn.phase });
+            return;
+        }
+        if (frame.gen !== conn.genStr || conns.get(conn.deviceId) !== conn) {
+            auditFrame(conn.deviceId, conn.owner, "generation_mismatch", { t: frame.t, frameGen: frame.gen, socketGen: conn.genStr });
+            return;
+        }
+        switch (frame.t) {
+            case "generationAck": {
+                if (conn.phase === "acquiring") {
+                    conn.phase = "active";
+                    startConnTimers(conn);
+                    for (const p of pendingOfDevice(conn.deviceId))
+                        if (p.cancelRequested)
+                            deliverCancel(p);
+                }
+                return;
+            }
+            case "heartbeat": {
+                void renewLeaseThrottled(conn);
+                return;
+            }
+            case "result": {
+                const m = matchPending(conn, frame.instructionId, "result");
+                if ("rejected" in m)
+                    return;
+                if ("memo" in m) {
+                    sendServerFrame(conn, { t: "resultAck", gen: conn.genStr, instructionId: frame.instructionId });
+                    return;
+                }
+                const p = m.hit;
+                if (p.terminalFenceInFlight)
+                    return;
+                if (frame.deviceSeq <= p.lastDeviceSeq) {
+                    sendServerFrame(conn, { t: "resultAck", gen: conn.genStr, instructionId: p.instructionId });
+                    return;
+                }
+                p.lastDeviceSeq = frame.deviceSeq;
+                if (p.chunkBuffer.size > 0) {
+                    failOnChunkGap(p);
+                    return;
+                }
+                void acceptTerminal(conn, p, frame.outcome);
+                return;
+            }
+            case "chunk": {
+                const m = matchPending(conn, frame.instructionId, "chunk");
+                if ("rejected" in m)
+                    return;
+                if ("memo" in m)
+                    return;
+                const p = m.hit;
+                if (frame.streamSeq < p.nextStreamSeq) {
+                    sendServerFrame(conn, { t: "chunkAck", gen: conn.genStr, instructionId: p.instructionId, upToStreamSeq: p.nextStreamSeq - 1 });
+                    return;
+                }
+                if (p.terminalFenceInFlight) {
+                    auditFrame(conn.deviceId, conn.owner, "instruction_not_committed", { instructionId: p.instructionId, t: "chunk", detail: "a new stream chunk arrived after this instruction's terminal result" });
+                    settleUnknown(p, `the device sent stream chunk ${frame.streamSeq} after it had already reported this instruction's terminal result, so the stream is not self-consistent`);
+                    return;
+                }
+                if (p.chunkBuffer.has(frame.streamSeq)) {
+                    sendServerFrame(conn, { t: "chunkAck", gen: conn.genStr, instructionId: p.instructionId, upToStreamSeq: p.nextStreamSeq - 1 });
+                    return;
+                }
+                const wouldBeEntries = p.chunkBuffer.size + 1;
+                const wouldBeBytes = p.chunkBufferBytes + chunkPayloadBytes(frame);
+                if (wouldBeEntries > chunkBufferMaxEntries || wouldBeBytes > chunkBufferMaxBytes) {
+                    auditFrame(conn.deviceId, conn.owner, "bad_shape", {
+                        instructionId: p.instructionId,
+                        detail: `reorder buffer cap exceeded (entries ${wouldBeEntries}/${chunkBufferMaxEntries}, bytes ${wouldBeBytes}/${chunkBufferMaxBytes})`,
+                    });
+                    metrics?.inc("device_chunk_buffer_overflow_total");
+                    settleUnknown(p, `the out-of-order chunk buffer exceeded its cap while waiting for streamSeq ${p.nextStreamSeq}, so the output stream cannot be reconstructed`);
+                    return;
+                }
+                p.chunkBufferBytes = wouldBeBytes;
+                p.chunkBuffer.set(frame.streamSeq, frame);
+                deliverChunks(conn, p);
+                return;
+            }
+            case "goodbye": {
+                conn.goodbye = true;
+                logger?.info?.("device_goodbye", { deviceId: conn.deviceId, reason: frame.reason });
+                const grace = frame.graceMs ?? reconnectGraceMs;
+                if (conn.goodbyeTimer)
+                    clearTimeout(conn.goodbyeTimer);
+                conn.goodbyeTimer = setTimeout(() => closeConn(conn, { releaseLease: true }), grace);
+                return;
+            }
+        }
+    }
+    function refuseUpgrade(socket, status) {
+        const reason = http.STATUS_CODES[status] ?? "Error";
+        socket.write(`HTTP/1.1 ${status} ${reason}\r\nConnection: close\r\nContent-Length: 0\r\n\r\n`);
+        socket.destroy();
+    }
+    function onSocket(ws, remoteKey) {
+        const conn = {
+            ws,
+            nonce: mintChallengeNonce(),
+            nonceConsumed: false,
+            remoteKey,
+            holdsPreAuthSlot: true,
+            deviceId: "",
+            owner: null,
+            epoch: "",
+            generation: 0,
+            genStr: "",
+            phase: "await_hello",
+            supports: new Set(),
+            workspaceRoot: "",
+            lastFrameAt: Date.now(),
+            lastLeaseRenewAt: 0,
+            issuedSeq: 0,
+            goodbye: false,
+            handshakeTimer: null,
+            watchdog: null,
+            leaseTimer: null,
+            goodbyeTimer: null,
+        };
+        allConns.add(conn);
+        ws.on("error", (err) => {
+            logger?.warn("device_ws_socket_error", { deviceId: conn.deviceId, err: err.message });
+            closeConn(conn, { releaseLease: conn.phase === "active" || conn.phase === "acquiring" });
+        });
+        ws.on("close", () => {
+            closeConn(conn, { releaseLease: true });
+        });
+        ws.on("message", (data, isBinary) => {
+            if (isBinary) {
+                auditFrame(conn.deviceId === "" ? null : conn.deviceId, conn.owner, "not_json", { detail: "binary frame on a text-only protocol" });
+                if (conn.phase === "await_hello")
+                    closeConn(conn, { releaseLease: false });
+                return;
+            }
+            onDeviceText(conn, data.toString("utf8"));
+        });
+        conn.handshakeTimer = setTimeout(() => {
+            logger?.warn("device_handshake_timeout", {});
+            closeConn(conn, { releaseLease: false });
+        }, DEVICE_WS_LIMITS.handshakeDeadlineMs);
+        sendServerFrame(conn, { t: "challenge", nonce: conn.nonce, protocolVersion: DEVICE_PROTOCOL_VERSION });
+    }
+    function attach(server) {
+        server.on("upgrade", (req, socket, head) => {
+            const path = (req.url ?? "/").split("?")[0];
+            if (path !== DEVICE_WS_PATH)
+                return refuseUpgrade(socket, 404);
+            if (draining || closed)
+                return refuseUpgrade(socket, 503);
+            const remoteKey = req.socket.remoteAddress ?? "unknown";
+            if (preAuthCount >= maxPreAuthConnections || (preAuthByRemote.get(remoteKey) ?? 0) >= maxPreAuthConnectionsPerRemote) {
+                metrics?.inc("device_preauth_refused_total");
+                logger?.warn("device_preauth_cap_hit", { remoteKey, preAuthCount, cap: maxPreAuthConnections });
+                return refuseUpgrade(socket, 503);
+            }
+            preAuthCount += 1;
+            preAuthByRemote.set(remoteKey, (preAuthByRemote.get(remoteKey) ?? 0) + 1);
+            let handed = false;
+            const refundIfUnhanded = () => {
+                if (handed)
+                    return;
+                handed = true;
+                preAuthCount = Math.max(0, preAuthCount - 1);
+                const cur = preAuthByRemote.get(remoteKey) ?? 1;
+                if (cur <= 1)
+                    preAuthByRemote.delete(remoteKey);
+                else
+                    preAuthByRemote.set(remoteKey, cur - 1);
+            };
+            socket.once("close", refundIfUnhanded);
+            socket.once("error", refundIfUnhanded);
+            wss.handleUpgrade(req, socket, head, (ws) => {
+                handed = true;
+                onSocket(ws, remoteKey);
+            });
+        });
+    }
+    function sendPayloadSequence(conn, instructionId, body, partBytes, sha256) {
+        sendServerFrame(conn, { t: "payload", gen: conn.genStr, instructionId, part: "begin", totalLen: body.length, sha256 });
+        for (let offset = 0; offset < body.length; offset += partBytes) {
+            sendServerFrame(conn, { t: "payload", gen: conn.genStr, instructionId, part: "chunk", offset, dataB64: body.subarray(offset, offset + partBytes).toString("base64") });
+        }
+        sendServerFrame(conn, { t: "payload", gen: conn.genStr, instructionId, part: "commit", totalLen: body.length, sha256 });
+    }
+    async function dispatch(input) {
+        if (input.payloadPartBytes !== undefined) {
+            const n = input.payloadPartBytes;
+            if (!Number.isSafeInteger(n) || n <= 0 || n > DEVICE_WS_LIMITS.maxFrameBytes) {
+                throw new Error(`device-ws: payloadPartBytes must be a positive safe integer ≤ ${DEVICE_WS_LIMITS.maxFrameBytes} (got ${String(n)})`);
+            }
+        }
+        if (!Number.isSafeInteger(input.timeoutMs) || input.timeoutMs <= 0) {
+            throw new Error(`device-ws: timeoutMs must be a positive safe integer (got ${String(input.timeoutMs)})`);
+        }
+        const abortedBeforeCommit = () => ({
+            kind: "never_started",
+            outcome: { ok: false, errorCode: "aborted", message: "the caller aborted before this instruction crossed the dispatch commit" },
+        });
+        const abortedNow = () => input.signal?.aborted === true;
+        if (abortedNow())
+            return abortedBeforeCommit();
+        if (closed)
+            return { kind: "rejected", reject: rejection(DEVICE_ERROR_CODES.BUSY, "this device hub is shutting down", 1) };
+        if (draining)
+            return { kind: "rejected", reject: rejection(DEVICE_ERROR_CODES.BUSY, "this replica is draining for shutdown/upgrade and no longer issues new device instructions", 1) };
+        const admitted = await enrollment.admitSession({ rootSessionId: input.rootSessionId, requester: input.requester });
+        if (!admitted.ok)
+            return { kind: "rejected", reject: admitted.reject };
+        if (abortedNow())
+            return abortedBeforeCommit();
+        const { deviceId, connGeneration } = admitted.value;
+        const conn = conns.get(deviceId);
+        if (!conn || conn.phase !== "active" || conn.goodbye) {
+            return { kind: "rejected", reject: rejection(DEVICE_ERROR_CODES.OFFLINE, "the bound device has no dispatchable connection on this replica") };
+        }
+        if (!conn.supports.has(input.kind)) {
+            return { kind: "never_started", outcome: { ok: false, errorCode: "not_supported", message: `the connected device does not declare support for the ${input.kind} instruction` } };
+        }
+        const driftNow = () => {
+            const cur = conns.get(deviceId);
+            return cur === undefined ? [] : (input.refuseIfDeclared ?? []).filter((k) => cur.supports.has(k));
+        };
+        {
+            const drift = driftNow();
+            if (drift.length > 0)
+                return capabilityDriftResult(drift);
+        }
+        const granted = await acquireSlot(deviceId);
+        if (!granted) {
+            return { kind: "rejected", reject: rejection(DEVICE_ERROR_CODES.BUSY, `the device already has ${maxInflightPerDevice} instructions in flight`, 1) };
+        }
+        if (abortedNow()) {
+            releaseSlot(deviceId);
+            return abortedBeforeCommit();
+        }
+        {
+            const drift = driftNow();
+            if (drift.length > 0) {
+                releaseSlot(deviceId);
+                return capabilityDriftResult(drift);
+            }
+        }
+        if (conn.generation !== connGeneration || conns.get(deviceId) !== conn || conn.phase !== "active") {
+            releaseSlot(deviceId);
+            return { kind: "rejected", reject: rejection(DEVICE_ERROR_CODES.ATTACHED_ELSEWHERE, "the device connection generation moved between admission and dispatch") };
+        }
+        let fenced = false;
+        try {
+            fenced = await store.heartbeatConnection({ deviceId, generation: conn.generation, leaseSeconds });
+        }
+        catch (err) {
+            releaseSlot(deviceId);
+            logger?.error("device_dispatch_fence_failed", { deviceId, err: err instanceof Error ? err.message : String(err) });
+            return { kind: "rejected", reject: rejection(DEVICE_ERROR_CODES.OFFLINE, "the device lease could not be re-asserted before dispatch") };
+        }
+        if (!fenced) {
+            releaseSlot(deviceId);
+            let revoked = false;
+            try {
+                const row = await store.getDeviceUnscoped(deviceId);
+                revoked = row?.status === "revoked";
+            }
+            catch (err) {
+                logger?.warn("device_row_read_failed", { deviceId, err: err instanceof Error ? err.message : String(err) });
+            }
+            sendServerFrame(conn, { t: "bye", gen: conn.genStr, reason: revoked ? "revoked" : "superseded" });
+            closeConn(conn, { releaseLease: false });
+            return { kind: "rejected", reject: rejection(revoked ? DEVICE_ERROR_CODES.REVOKED : DEVICE_ERROR_CODES.ATTACHED_ELSEWHERE, "the device connection lost its lease before this instruction was dispatched") };
+        }
+        if (conn.ws.readyState !== conn.ws.OPEN) {
+            releaseSlot(deviceId);
+            return { kind: "rejected", reject: rejection(DEVICE_ERROR_CODES.OFFLINE, "the device socket closed before dispatch") };
+        }
+        if (abortedNow()) {
+            releaseSlot(deviceId);
+            return abortedBeforeCommit();
+        }
+        const instructionId = mintInstructionId();
+        conn.issuedSeq += 1;
+        let resolve;
+        const settled = new Promise((r) => {
+            resolve = r;
+        });
+        const p = {
+            instructionId,
+            deviceId,
+            rootSessionId: input.rootSessionId,
+            kind: input.kind,
+            epoch: conn.epoch,
+            generation: conn.generation,
+            state: "dispatch_committed",
+            lastDeviceSeq: -1,
+            nextStreamSeq: 0,
+            chunkBuffer: new Map(),
+            chunkBufferBytes: 0,
+            chunkGapTimer: null,
+            ceilingTimer: null,
+            cancelTimer: null,
+            cancelRequested: false,
+            cancelDeliveredOnGeneration: null,
+            settled: false,
+            terminalFenceInFlight: false,
+            expectsStreamTerminator: input.onChunk !== undefined || input.kind === "execStream",
+            exitChunk: null,
+            exitChunkCount: 0,
+            lastDeliveredSeq: -1,
+            ...(input.onChunk === undefined ? {} : { onChunk: input.onChunk }),
+            resolve,
+        };
+        pending.set(instructionId, p);
+        let args = input.args;
+        let body = null;
+        let sha = "";
+        if (input.payload !== undefined && WRITE_PAYLOAD_KINDS.has(input.kind)) {
+            body = Buffer.from(input.payload);
+            sha = createHash("sha256").update(body).digest("hex");
+            args = { ...input.args, contentLen: body.length, sha256: sha };
+        }
+        const wrote = sendServerFrame(conn, {
+            t: "instruction",
+            gen: conn.genStr,
+            instructionId,
+            rootSessionId: input.rootSessionId,
+            issuedSeq: conn.issuedSeq,
+            kind: input.kind,
+            args,
+            ...(input.cwd === undefined ? {} : { cwd: input.cwd }),
+            ...(input.shellEnv === undefined ? {} : { shellEnv: input.shellEnv }),
+            timeoutMs: input.timeoutMs,
+        });
+        if (body !== null)
+            sendPayloadSequence(conn, instructionId, body, input.payloadPartBytes ?? DEVICE_WS_LIMITS.defaultPayloadPartBytes, sha);
+        metrics?.inc("device_instruction_dispatched_total", { kind: input.kind });
+        if (!wrote) {
+            settleUnknown(p, "the instruction could not be written to the device socket after the dispatch commit");
+            return { kind: "settled", instructionId, outcome: await settled };
+        }
+        p.ceilingTimer = setTimeout(() => {
+            metrics?.inc("device_outcome_unknown_total", { cause: "exec_ceiling" });
+            settleUnknown(p, `the device did not report a result within timeoutMs+grace (${input.timeoutMs}+${execGraceMs}ms)`);
+        }, input.timeoutMs + execGraceMs);
+        let onAbort;
+        if (input.signal) {
+            if (input.signal.aborted)
+                void cancel(instructionId);
+            else {
+                onAbort = () => void cancel(instructionId);
+                input.signal.addEventListener("abort", onAbort, { once: true });
+            }
+        }
+        try {
+            return { kind: "settled", instructionId, outcome: await settled };
+        }
+        finally {
+            if (onAbort)
+                input.signal?.removeEventListener("abort", onAbort);
+        }
+    }
+    function deliverCancel(p) {
+        if (p.settled || p.state !== "dispatch_committed")
+            return;
+        const conn = conns.get(p.deviceId);
+        if (!conn || conn.phase !== "active")
+            return;
+        if (p.cancelDeliveredOnGeneration === conn.generation)
+            return;
+        if (!sendServerFrame(conn, { t: "cancel", gen: conn.genStr, instructionId: p.instructionId }))
+            return;
+        p.cancelDeliveredOnGeneration = conn.generation;
+        if (p.cancelTimer)
+            clearTimeout(p.cancelTimer);
+        p.cancelTimer = setTimeout(() => {
+            metrics?.inc("device_outcome_unknown_total", { cause: "cancel_grace" });
+            settleUnknown(p, `the device did not return a terminal receipt within ${cancelGraceMs}ms of the cancel`);
+        }, cancelGraceMs);
+    }
+    async function cancel(instructionId) {
+        const p = pending.get(instructionId);
+        if (!p || p.state !== "dispatch_committed")
+            return;
+        p.cancelRequested = true;
+        deliverCancel(p);
+        return Promise.resolve();
+    }
+    function beginDrain() {
+        draining = true;
+        logger?.info?.("device_hub_draining", { connections: conns.size, pending: pending.size });
+    }
+    async function close() {
+        closed = true;
+        draining = true;
+        for (const timer of reconnectTimers.values())
+            clearTimeout(timer);
+        reconnectTimers.clear();
+        for (const conn of [...allConns]) {
+            if (conn.phase === "active" || conn.phase === "acquiring")
+                sendServerFrame(conn, { t: "bye", gen: conn.genStr, reason: "draining" });
+            closeConn(conn, { releaseLease: true });
+        }
+        for (const p of [...pending.values()])
+            settleUnknown(p, "the device hub shut down while this instruction was in flight");
+        for (const queue of slotWaiters.values()) {
+            for (const w of queue) {
+                clearTimeout(w.timer);
+                w.resolve(false);
+            }
+        }
+        slotWaiters.clear();
+        await new Promise((resolve) => wss.close(() => resolve()));
+    }
+    return {
+        attach,
+        dispatch,
+        cancel,
+        beginDrain,
+        isDraining: () => draining,
+        connectedDeviceIds: () => [...conns.keys()],
+        declaredSupports: (deviceId) => {
+            const conn = conns.get(deviceId);
+            return conn && (conn.phase === "active" || conn.phase === "acquiring") ? conn.supports : undefined;
+        },
+        close,
+    };
+}
+//# sourceMappingURL=device-ws-hub.js.map