crbro-memory 2.10.0 → 2.11.0

package/dist/daemon/endpoint.js

@@ -18,8 +18,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.DAEMON_PROTOCOL = void 0;
+exports.BUILD_ID_FILE = exports.DAEMON_PROTOCOL = void 0;
 exports.packageVersion = packageVersion;
+exports.readBuildId = readBuildId;
 exports.buildId = buildId;
 exports.daemonDir = daemonDir;
 exports.endpointFor = endpointFor;
@@ -37,6 +38,8 @@ exports.writeState = writeState;
 exports.readState = readState;
 exports.removeState = removeState;
 exports.listStates = listStates;
+exports.verifyDaemon = verifyDaemon;
+exports.pruneDaemonDir = pruneDaemonDir;
 exports.proof = proof;
 exports.sameProof = sameProof;
 exports.newNonce = newNonce;
@@ -44,39 +47,76 @@ exports.daemonEnabled = daemonEnabled;
 exports.setDaemonEnabled = setDaemonEnabled;
 const node_crypto_1 = require("node:crypto");
 const node_fs_1 = require("node:fs");
+const node_net_1 = __importDefault(require("node:net"));
 const node_os_1 = __importDefault(require("node:os"));
 const node_path_1 = __importDefault(require("node:path"));
 const shelf_js_1 = require("../engine/shelf.js");
 const version_js_1 = require("../version.js");
+const lines_js_1 = require("./lines.js");
 exports.DAEMON_PROTOCOL = 1;
 const DAEMON_DIR = '.daemon';
 const ENABLED_FLAG = 'enabled';
-/** The package version, read from disk on each call (the same lookup as view=status), '0.0.0' when unreadable. */
+/**
+ * The version of the code this process runs, '0.0.0' when unreadable. Read
+ * once, when the process loaded (version.ts): npx refreshes its cache in
+ * place, and a proxy that re-read package.json at every reconnection computed
+ * the NEXT release's build and launched its daemon — served by code it never
+ * started.
+ */
 function packageVersion() {
-    const v = (0, version_js_1.readPackageVersion)();
-    return v === 'unknown' ? '0.0.0' : v;
+    return version_js_1.RUNNING_VERSION === 'unknown' ? '0.0.0' : version_js_1.RUNNING_VERSION;
 }
+/** Written next to the compiled code by `npm run build` (scripts/build-id.mjs). */
+exports.BUILD_ID_FILE = 'build-id.json';
 /**
- * Version plus the build's own timestamp. The version alone would let a
- * developer's half-built dist serve the clients that launched the published
- * package; the timestamp alone would not survive a reinstall of the same
- * release on another path. Together: you are served by the code you launched.
+ * The build id a compiled tree declares for itself: the version plus a hash
+ * of every compiled .js file, computed when it was built. Two installs of the
+ * same release hold the same bytes and so the same id — one daemon between
+ * them, where the old id (version + the mtime of server.js) gave each copy its
+ * own ~630 MB daemon. Null when the file is missing, malformed, or no longer
+ * describes the code beside it (`tsc --watch` rewrites the code, not the file).
  */
-function buildId() {
-    let stamp = '0';
+function readBuildId(distDir) {
     try {
-        const here = __dirname; // CommonJS output: dist/daemon, or src/daemon under the test runner
-        for (const f of ['../server.js', '../server.ts']) {
-            try {
-                stamp = Math.floor((0, node_fs_1.statSync)(node_path_1.default.join(here, f)).mtimeMs).toString(36);
-                break;
-            }
-            catch { /* next */ }
+        const meta = JSON.parse((0, node_fs_1.readFileSync)(node_path_1.default.join(distDir, exports.BUILD_ID_FILE), 'utf8'));
+        if (typeof meta?.build !== 'string' || !/^[\w.+-]{1,96}$/.test(meta.build))
+            return null;
+        const check = meta.check;
+        if (typeof check?.file !== 'string' || typeof check?.sha256 !== 'string' || check.file.includes('..'))
+            return null;
+        const sha = (0, node_crypto_1.createHash)('sha256').update((0, node_fs_1.readFileSync)(node_path_1.default.join(distDir, check.file))).digest('hex');
+        return sha === check.sha256 ? meta.build : null;
+    }
+    catch {
+        return null;
+    }
+}
+/** A tree with no build-id.json (a developer's src under the test runner, or a dist built by bare tsc): version plus the timestamp of server.js. */
+function devBuildId(here) {
+    let stamp = '0';
+    for (const f of ['../server.js', '../server.ts']) {
+        try {
+            stamp = Math.floor((0, node_fs_1.statSync)(node_path_1.default.join(here, f)).mtimeMs).toString(36);
+            break;
         }
+        catch { /* next */ }
     }
-    catch { /* keep 0 */ }
     return `${packageVersion()}-${stamp}`;
 }
+let ownBuild = null;
+/**
+ * Which build this process is, decided once (the first time it is asked) and
+ * never re-read: you are served by the code you launched. In a published or
+ * built tree it is dist/build-id.json; only a tree without one falls back to
+ * version plus the mtime of server.js.
+ */
+function buildId() {
+    if (ownBuild === null) {
+        const here = __dirname; // CommonJS output: dist/daemon, or src/daemon under the test runner
+        ownBuild = readBuildId(node_path_1.default.join(here, '..')) ?? devBuildId(here);
+    }
+    return ownBuild;
+}
 function daemonDir(brainRoot) {
     return node_path_1.default.join(brainRoot, DAEMON_DIR);
 }
@@ -232,13 +272,30 @@ async function acquireStartLock(brainRoot, build) {
 function newToken() {
     return (0, node_crypto_1.randomBytes)(32).toString('hex');
 }
-async function writeState(brainRoot, state) {
+/**
+ * Write the descriptor atomically. On Windows the rename can fail for a moment
+ * (EPERM/EBUSY while an antivirus or a reading client holds the target), so it
+ * is tried a few times; a temporary that could not become the descriptor is
+ * removed, never left behind with a token in it.
+ */
+async function writeState(brainRoot, state, attempts = 3) {
     const dir = daemonDir(brainRoot);
     await node_fs_1.promises.mkdir(dir, { recursive: true, mode: 0o700 });
     const file = stateFile(brainRoot, state.build);
     const tmp = `${file}.${process.pid}.tmp`;
-    await node_fs_1.promises.writeFile(tmp, JSON.stringify(state), { mode: 0o600 });
-    await node_fs_1.promises.rename(tmp, file);
+    for (let i = 0;; i++) {
+        try {
+            await node_fs_1.promises.writeFile(tmp, JSON.stringify(state), { mode: 0o600 });
+            await node_fs_1.promises.rename(tmp, file);
+            return;
+        }
+        catch (err) {
+            await node_fs_1.promises.rm(tmp, { force: true }).catch(() => undefined);
+            if (i + 1 >= attempts)
+                throw err;
+            await new Promise(r => setTimeout(r, 100 * (i + 1)));
+        }
+    }
 }
 async function readState(brainRoot, build = buildId()) {
     try {
@@ -274,6 +331,150 @@ async function listStates(brainRoot) {
     catch { /* no folder, no daemons */ }
     return out;
 }
+/**
+ * Is the daemon a descriptor names really there? Asked of the endpoint, with
+ * the descriptor's own token — never of the pid: Windows hands pids out again
+ * quickly, and a descriptor weeks old whose pid now belongs to a cmd.exe was
+ * reported "alive" for ever.
+ *
+ *   alive    it answered and proved it holds the token
+ *   refused  nobody listens there (connection refused / no such pipe)
+ *   dead     something answers, but not the daemon this descriptor names
+ *   unknown  no verdict in time (busy, or closed on us): leave it alone
+ */
+function verifyDaemon(state, timeoutMs = 1_500) {
+    return new Promise(resolve => {
+        let done = false;
+        const nonce = newNonce();
+        const s = node_net_1.default.connect(state.endpoint);
+        const finish = (v) => {
+            if (done)
+                return;
+            done = true;
+            clearTimeout(t);
+            s.destroy();
+            resolve(v);
+        };
+        // A late answer that arrived while the loop was busy is read before the verdict.
+        const t = setTimeout(() => setImmediate(() => finish('unknown')), timeoutMs);
+        const reader = new lines_js_1.LineReader(line => {
+            let m = null;
+            try {
+                m = JSON.parse(line);
+            }
+            catch { /* not a daemon */ }
+            finish(m?.crbro === 'hello' && sameProof(String(m.proof || ''), proof(state.token, 'daemon', nonce)) ? 'alive' : 'dead');
+        }, () => finish('dead'), 64 * 1024);
+        s.on('connect', () => s.write(`${JSON.stringify({ crbro: 'hello', protocol: exports.DAEMON_PROTOCOL, nonce })}\n`));
+        s.on('data', chunk => reader.push(typeof chunk === 'string' ? Buffer.from(chunk, 'utf8') : chunk));
+        s.on('error', err => {
+            const code = err.code;
+            finish(code === 'ECONNREFUSED' || code === 'ENOENT' ? 'refused' : 'unknown');
+        });
+        s.on('close', () => finish('unknown'));
+    });
+}
+/** A *.tmp this old was left by a writer that died or failed: nobody will rename it any more. */
+const TMP_STALE_MS = 5 * 60_000;
+/** A start lock is held for the seconds a start takes. */
+const LOCK_STALE_MS = 5 * 60_000;
+/** A blocked marker matters to the proxy waiting right now, not tomorrow. */
+const BLOCKED_STALE_MS = 24 * 3_600_000;
+/**
+ * Sweep .daemon: descriptors whose daemon is not there (checked at the
+ * endpoint, with the token), temporaries, start locks and blocked markers that
+ * nobody will ever come back for. Before 2.11 only `daemon status|stop|off`
+ * removed anything, by pid, and nobody runs those: twelve dead descriptors and
+ * five token-bearing temporaries were found in one brain. `keep` names a build
+ * whose descriptor is not probed (the caller's own).
+ */
+async function pruneDaemonDir(brainRoot, opts = {}) {
+    const dir = daemonDir(brainRoot);
+    const now = opts.now ?? Date.now();
+    const verify = opts.verify ?? verifyDaemon;
+    const out = { removed: [], alive: [], unknown: [] };
+    let names;
+    try {
+        names = await node_fs_1.promises.readdir(dir);
+    }
+    catch {
+        return out;
+    }
+    const keepFile = opts.keep ? node_path_1.default.basename(stateFile(brainRoot, opts.keep)) : null;
+    const rmIfOlder = async (name, maxAgeMs) => {
+        try {
+            const st = await node_fs_1.promises.stat(node_path_1.default.join(dir, name));
+            if (now - st.mtimeMs <= maxAgeMs)
+                return;
+            await node_fs_1.promises.rm(node_path_1.default.join(dir, name), { force: true });
+            out.removed.push(name);
+        }
+        catch { /* gone already */ }
+    };
+    const checks = [];
+    for (const name of names) {
+        if (/\.json\.\d+\.tmp$/.test(name)) {
+            checks.push(rmIfOlder(name, TMP_STALE_MS));
+            continue;
+        }
+        if (/^[0-9a-f]{16}\.lock$/.test(name)) {
+            checks.push(rmIfOlder(name, LOCK_STALE_MS));
+            continue;
+        }
+        if (/^[0-9a-f]{16}\.blocked$/.test(name)) {
+            checks.push(rmIfOlder(name, BLOCKED_STALE_MS));
+            continue;
+        }
+        if (!/^[0-9a-f]{16}\.json$/.test(name) || name === keepFile)
+            continue;
+        checks.push((async () => {
+            const file = node_path_1.default.join(dir, name);
+            let s;
+            try {
+                s = JSON.parse(await node_fs_1.promises.readFile(file, 'utf8'));
+            }
+            catch {
+                return;
+            }
+            // Only descriptors that are what their name says: this brain, this build's own endpoint.
+            // Anything else is not ours to judge — nor a place to connect to.
+            if (s?.protocol !== exports.DAEMON_PROTOCOL || typeof s.build !== 'string' || typeof s.token !== 'string')
+                return;
+            if (node_path_1.default.basename(stateFile(brainRoot, s.build)) !== name || s.endpoint !== endpointFor(brainRoot, s.build))
+                return;
+            const v = await verify(s);
+            if (v === 'alive') {
+                out.alive.push(s);
+                return;
+            }
+            if (v === 'unknown') {
+                out.unknown.push(s);
+                return;
+            }
+            // Removed only if it is still the same descriptor: a replacement may have been written meanwhile.
+            try {
+                const again = JSON.parse(await node_fs_1.promises.readFile(file, 'utf8'));
+                if (again?.token !== s.token)
+                    return;
+                await node_fs_1.promises.rm(file, { force: true });
+                out.removed.push(name);
+            }
+            catch {
+                return;
+            }
+            // A socket file nobody listens on (never a pipe, never one somebody answers on).
+            if (v === 'refused' && process.platform !== 'win32' && node_path_1.default.dirname(s.endpoint) === dir) {
+                try {
+                    await node_fs_1.promises.rm(s.endpoint, { force: true });
+                    out.removed.push(node_path_1.default.basename(s.endpoint));
+                }
+                catch { /* best effort */ }
+            }
+        })());
+    }
+    await Promise.all(checks);
+    return out;
+}
 function proof(token, role, nonce) {
     return (0, node_crypto_1.createHmac)('sha256', token).update(`${role}:${nonce}`).digest('hex');
 }

package/dist/daemon/lines.js

@@ -4,18 +4,38 @@
 // The same framing MCP uses on stdio, so a proxy can hand a client's bytes to
 // the daemon without understanding them. One JSON value per line, UTF-8.
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.LoopbackTransport = exports.SocketServerTransport = exports.LineReader = exports.MAX_LINE_BYTES = void 0;
+exports.LoopbackTransport = exports.SocketServerTransport = exports.NOT_STARTED = exports.LineReader = exports.MAX_LINE_BYTES = void 0;
+exports.idFromHead = idFromHead;
 /** A line bigger than this is not a message, it is a mistake or an attack. */
 exports.MAX_LINE_BYTES = 32 * 1024 * 1024;
-/** Splits a byte stream into lines. Tolerates \r\n and lines split across chunks. */
+/** How much of an over-long line is kept to say which request it was. */
+const HEAD_BYTES = 64 * 1024;
+/**
+ * Splits a byte stream into lines. Tolerates \r\n and lines split across chunks.
+ *
+ * A line longer than the limit is reported once, with its first bytes (enough
+ * to find its id), and everything up to its newline is thrown away. It used to
+ * be cut at the limit and its tail delivered as a line of its own: a broken
+ * message, and an id that never got an answer.
+ */
 class LineReader {
     onLine;
     onOverflow;
+    maxBytes;
     buf = [];
     size = 0;
-    constructor(onLine, onOverflow) {
+    discarding = false;
+    constructor(onLine, onOverflow, maxBytes = exports.MAX_LINE_BYTES) {
         this.onLine = onLine;
         this.onOverflow = onOverflow;
+        this.maxBytes = maxBytes;
+    }
+    overflow(piece) {
+        const parts = [...this.buf, piece];
+        const head = Buffer.concat(parts).subarray(0, HEAD_BYTES).toString('utf8');
+        this.buf = [];
+        this.size = 0;
+        this.onOverflow(head);
     }
     push(chunk) {
         let start = 0;
@@ -23,32 +43,60 @@ class LineReader {
             if (chunk[i] !== 0x0a)
                 continue;
             const piece = chunk.subarray(start, i);
+            start = i + 1;
+            if (this.discarding) {
+                this.discarding = false;
+                continue;
+            } // the end of a line already refused
+            if (this.size + piece.length > this.maxBytes) {
+                this.overflow(piece);
+                continue;
+            }
             const whole = this.buf.length ? Buffer.concat([...this.buf, piece]) : piece;
             this.buf = [];
             this.size = 0;
-            start = i + 1;
             const line = whole.toString('utf8').replace(/\r$/, '');
             if (line)
                 this.onLine(line);
         }
-        if (start < chunk.length) {
+        if (start < chunk.length && !this.discarding) {
             const rest = chunk.subarray(start);
+            if (this.size + rest.length > this.maxBytes) {
+                this.overflow(rest);
+                this.discarding = true; // until the next newline, nothing is a line
+                return;
+            }
             this.buf.push(rest);
             this.size += rest.length;
-            if (this.size > exports.MAX_LINE_BYTES) {
-                this.buf = [];
-                this.size = 0;
-                this.onOverflow();
-            }
         }
     }
 }
 exports.LineReader = LineReader;
+/** The id of a request, read from the first bytes of a line too long to parse. Null when it cannot be told. */
+function idFromHead(head) {
+    const m = /"id"\s*:\s*("(?:[^"\\]|\\.){0,256}"|-?\d{1,16})/.exec(head);
+    if (!m)
+        return null;
+    try {
+        return JSON.parse(m[1]);
+    }
+    catch {
+        return null;
+    }
+}
+const isRequest = (m) => !!m && typeof m.method === 'string' && m.id !== undefined && m.id !== null;
+const isResponse = (m) => !!m && m.method === undefined && m.id !== undefined && m.id !== null;
+/** The error a stopping daemon gives a request it never started. The proxy recognises it and sends the request again elsewhere. */
+exports.NOT_STARTED = 'not-started';
 /**
  * The daemon's side of one client: an MCP Transport over a socket that has
  * already passed the handshake. `scope` wraps every incoming message, which
  * is how per-connection state (the session tally) follows the request through
  * shared engines without any of them knowing about connections.
+ *
+ * It counts the requests it has handed to the server and not yet answered, so
+ * a daemon that is asked to stop can wait for them instead of cutting a write
+ * in half — and, while it waits, refuse new ones in a way the proxy can retry.
  */
 class SocketServerTransport {
     socket;
@@ -59,6 +107,9 @@ class SocketServerTransport {
     closed = false;
     /** Lines that arrived before the server finished connecting, in order. */
     pending = [];
+    inflight = new Set();
+    draining = false;
+    idleWaiters = [];
     constructor(socket, scope) {
         this.socket = socket;
         this.scope = scope;
@@ -74,6 +125,17 @@ class SocketServerTransport {
         this.socket.on('close', () => this.finish());
         this.socket.on('error', err => { this.onerror?.(err); this.finish(); });
     }
+    /** Requests handed to the server and not answered yet. */
+    inFlight() {
+        return this.inflight.size;
+    }
+    /** From now on new requests are refused (NOT_STARTED); resolves once those already running are answered. */
+    drain() {
+        this.draining = true;
+        if (this.inflight.size === 0 || this.closed)
+            return Promise.resolve();
+        return new Promise(resolve => this.idleWaiters.push(resolve));
+    }
     /** Fed by the daemon, which owns the reader (the handshake used it first). */
     receive(line) {
         if (this.pending) {
@@ -88,15 +150,41 @@ class SocketServerTransport {
             this.onerror?.(err);
             return;
         }
+        if (isRequest(message)) {
+            const id = message.id;
+            if (this.draining) {
+                this.write({ jsonrpc: '2.0', id, error: { code: -32000, message: 'CRBRO is restarting: this call was not started.', data: { crbro: exports.NOT_STARTED } } });
+                return;
+            }
+            this.inflight.add(id);
+        }
+        else if (message.method === 'notifications/cancelled') {
+            // A cancelled request is never answered: it is not running any more either.
+            if (this.inflight.delete(message.params?.requestId))
+                queueMicrotask(() => this.settle());
+        }
         this.scope(() => this.onmessage?.(message));
     }
-    async send(message) {
+    write(message) {
         if (this.closed || this.socket.destroyed)
-            return;
-        await new Promise(resolve => {
+            return Promise.resolve();
+        return new Promise(resolve => {
             this.socket.write(`${JSON.stringify(message)}\n`, () => resolve());
         });
     }
+    async send(message) {
+        await this.write(message);
+        if (isResponse(message) && this.inflight.delete(message.id))
+            this.settle();
+    }
+    settle() {
+        if (this.inflight.size > 0 && !this.closed)
+            return;
+        const waiters = this.idleWaiters;
+        this.idleWaiters = [];
+        for (const w of waiters)
+            w();
+    }
     async close() {
         if (!this.socket.destroyed)
             this.socket.end();
@@ -106,6 +194,7 @@ class SocketServerTransport {
         if (this.closed)
             return;
         this.closed = true;
+        this.settle();
         this.onclose?.();
     }
 }