@sema-agent/core 5.21.1 → 5.23.0

package/dist/core/memory-engine/layout.js

@@ -1,4 +1,4 @@
-import { closeSync, constants as fsConstants, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, rmdirSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs";
+import { closeSync, constants as fsConstants, copyFileSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, rmdirSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs";
 const { O_WRONLY, O_CREAT, O_TRUNC, O_NOFOLLOW } = fsConstants;
 import { homedir } from "node:os";
 import { createHash } from "node:crypto";
@@ -696,6 +696,474 @@ export function clearScanFuse(controlDir, keys) {
         return changed ? counts : undefined;
     });
 }
+export const LINEAGE_FILE = "lineage.json";
+export const CHALLENGES_FILE = "challenges.json";
+export const LINEAGE_AUDIT_FILE = "lineage-audit.json";
+export const LINEAGE_AUDIT_MAX_ROWS = 2048;
+export const CHALLENGED_HISTORY_FILE = "usage-challenged-history.json";
+function rollForwardStrictSidecar(file, journal) {
+    let raw;
+    try {
+        raw = readFileSync(journal, "utf8");
+    }
+    catch {
+        return;
+    }
+    try {
+        JSON.parse(raw);
+    }
+    catch {
+        throw new ControlPlaneCorruptError(`control-plane journal is unparseable (fail-closed sidecar): ${journal}`);
+    }
+    atomicWriteFileSync(file, raw);
+    rmSync(journal, { force: true });
+}
+function lockedStrictUpdate(controlDir, fileName, what, coerce, fn) {
+    ensureDirExists(controlDir);
+    const file = join(controlDir, fileName);
+    const journal = `${file}.journal`;
+    const lock = `${file}.lock`;
+    const token = acquireSidecarLock(lock, { onDeadline: "throw" });
+    try {
+        rollForwardStrictSidecar(file, journal);
+        const current = coerce(readStrictSidecarRaw(file, what));
+        const { next, result } = fn(current);
+        if (next !== undefined) {
+            const data = `${JSON.stringify(next, null, 2)}\n`;
+            assertSidecarLockOwnership(lock, token, what);
+            atomicWriteFileSync(journal, data);
+            atomicWriteFileSync(file, data);
+            rmSync(journal, { force: true });
+        }
+        return result;
+    }
+    finally {
+        releaseSidecarLock(lock, token);
+    }
+}
+function readStrictSidecarRaw(file, what) {
+    let raw;
+    try {
+        raw = readFileSync(file, "utf8");
+    }
+    catch (err) {
+        if (err.code === "ENOENT")
+            return undefined;
+        throw new ControlPlaneCorruptError(`${what} unreadable (${err.code ?? "io error"}): ${file}`, { cause: err });
+    }
+    try {
+        return JSON.parse(raw);
+    }
+    catch {
+        throw new ControlPlaneCorruptError(`${what} is unparseable: ${file}`);
+    }
+}
+function readStrictSidecar(controlDir, fileName, what) {
+    const file = join(controlDir, fileName);
+    const journal = `${file}.journal`;
+    let journalRaw;
+    try {
+        journalRaw = readFileSync(journal, "utf8");
+    }
+    catch {
+        journalRaw = undefined;
+    }
+    if (journalRaw !== undefined) {
+        try {
+            return JSON.parse(journalRaw);
+        }
+        catch {
+            throw new ControlPlaneCorruptError(`control-plane journal is unparseable (fail-closed sidecar): ${journal}`);
+        }
+    }
+    return readStrictSidecarRaw(file, what);
+}
+function coerceLineage(raw) {
+    if (raw === undefined)
+        return { version: 1, committed: {}, pending: {} };
+    if (!raw || typeof raw !== "object" || Array.isArray(raw))
+        throw new ControlPlaneCorruptError("memory lineage ledger has the wrong shape");
+    const r = raw;
+    if (r.version !== 1)
+        throw new ControlPlaneCorruptError(`memory lineage ledger has an unrecognized version (${String(r.version)}) — refusing (fail-closed)`);
+    const badShape = (part) => new ControlPlaneCorruptError(`memory lineage ledger ${part} has the wrong shape`);
+    if (!r.committed || typeof r.committed !== "object" || Array.isArray(r.committed))
+        throw badShape("committed set");
+    if (!r.pending || typeof r.pending !== "object" || Array.isArray(r.pending))
+        throw badShape("pending set");
+    for (const [entryId, sessions] of Object.entries(r.committed)) {
+        if (!sessions || typeof sessions !== "object" || Array.isArray(sessions))
+            throw badShape(`committed[${JSON.stringify(entryId)}]`);
+        for (const [sid, c] of Object.entries(sessions)) {
+            const row = c;
+            if (!row || typeof row !== "object" || typeof row.lastRev !== "string" || typeof row.lastAt !== "number") {
+                throw badShape(`committed[${JSON.stringify(entryId)}][${JSON.stringify(sid)}]`);
+            }
+        }
+    }
+    for (const [txnId, t] of Object.entries(r.pending)) {
+        const txn = t;
+        if (!txn || typeof txn !== "object" || typeof txn.sessionId !== "string" || typeof txn.at !== "number" || !Array.isArray(txn.rows)) {
+            throw badShape(`pending[${JSON.stringify(txnId)}]`);
+        }
+        for (const row of txn.rows) {
+            const p = row;
+            if (!p || typeof p !== "object" || typeof p.entryId !== "string" || typeof p.rev !== "string")
+                throw badShape(`pending[${JSON.stringify(txnId)}] row`);
+        }
+        if (txn.credential !== undefined) {
+            const c = txn.credential;
+            if (!c || typeof c !== "object" || !Array.isArray(c.appliedIds) || c.appliedIds.some((x) => typeof x !== "string") || typeof c.at !== "number") {
+                throw badShape(`pending[${JSON.stringify(txnId)}] credential`);
+            }
+        }
+    }
+    return raw;
+}
+export function stageLineagePending(controlDir, txnId, sessionId, rows, now) {
+    lockedStrictUpdate(controlDir, LINEAGE_FILE, "memory lineage ledger", coerceLineage, (rec) => {
+        rec.pending[txnId] = { sessionId, at: now(), rows: [...rows] };
+        return { next: rec, result: undefined };
+    });
+}
+export function discardLineagePending(controlDir, txnId) {
+    lockedStrictUpdate(controlDir, LINEAGE_FILE, "memory lineage ledger", coerceLineage, (rec) => {
+        if (rec.pending[txnId] === undefined)
+            return { result: undefined };
+        delete rec.pending[txnId];
+        return { next: rec, result: undefined };
+    });
+}
+export function recordLineageCredential(controlDir, txnId, appliedIds, now) {
+    lockedStrictUpdate(controlDir, LINEAGE_FILE, "memory lineage ledger", coerceLineage, (rec) => {
+        const txn = rec.pending[txnId];
+        if (txn === undefined)
+            throw new ControlPlaneCorruptError(`memory lineage ledger: credential for unknown pending transaction ${txnId}`);
+        txn.credential = { appliedIds: [...appliedIds], at: now() };
+        return { next: rec, result: undefined };
+    });
+}
+export function promoteLineagePending(controlDir, txnId, now) {
+    return lockedStrictUpdate(controlDir, LINEAGE_FILE, "memory lineage ledger", coerceLineage, (rec) => {
+        const txn = rec.pending[txnId];
+        if (txn === undefined)
+            return { result: [] };
+        if (txn.credential === undefined) {
+            throw new ControlPlaneCorruptError(`memory lineage ledger: refusing to promote uncredentialed pending transaction ${txnId} (the commit outcome is not durably known)`);
+        }
+        const applied = new Set(txn.credential.appliedIds);
+        const at = now();
+        const promoted = [];
+        for (const row of txn.rows) {
+            if (!applied.has(row.entryId))
+                continue;
+            const sessions = (rec.committed[row.entryId] ??= {});
+            sessions[txn.sessionId] = { lastRev: row.rev, lastAt: at };
+            promoted.push({ entryId: row.entryId, sessionId: txn.sessionId, rev: row.rev });
+        }
+        delete rec.pending[txnId];
+        return { next: rec, result: promoted };
+    });
+}
+export function adjudicateLineagePending(controlDir, txnId, action, now) {
+    return lockedStrictUpdate(controlDir, LINEAGE_FILE, "memory lineage ledger", coerceLineage, (rec) => {
+        const txn = rec.pending[txnId];
+        if (txn === undefined)
+            return { result: [] };
+        if (action === "discard") {
+            delete rec.pending[txnId];
+            return { next: rec, result: [] };
+        }
+        const applied = txn.credential !== undefined ? new Set(txn.credential.appliedIds) : undefined;
+        const at = now();
+        const promoted = [];
+        for (const row of txn.rows) {
+            if (applied !== undefined && !applied.has(row.entryId))
+                continue;
+            const sessions = (rec.committed[row.entryId] ??= {});
+            sessions[txn.sessionId] = { lastRev: row.rev, lastAt: at };
+            promoted.push({ entryId: row.entryId, sessionId: txn.sessionId, rev: row.rev });
+        }
+        delete rec.pending[txnId];
+        return { next: rec, result: promoted };
+    });
+}
+export function reconcileLineage(controlDir, now) {
+    return lockedStrictUpdate(controlDir, LINEAGE_FILE, "memory lineage ledger", coerceLineage, (rec) => {
+        const promoted = [];
+        const undecidable = [];
+        let at;
+        let changed = false;
+        for (const [txnId, txn] of Object.entries(rec.pending)) {
+            if (txn.credential === undefined) {
+                undecidable.push({ txnId, sessionId: txn.sessionId, entryIds: txn.rows.map((r) => r.entryId) });
+                continue;
+            }
+            const applied = new Set(txn.credential.appliedIds);
+            for (const row of txn.rows) {
+                if (!applied.has(row.entryId))
+                    continue;
+                at ??= now();
+                const sessions = (rec.committed[row.entryId] ??= {});
+                sessions[txn.sessionId] = { lastRev: row.rev, lastAt: at };
+                promoted.push({ entryId: row.entryId, sessionId: txn.sessionId, rev: row.rev });
+            }
+            delete rec.pending[txnId];
+            changed = true;
+        }
+        return { ...(changed ? { next: rec } : {}), result: { promoted, undecidable } };
+    });
+}
+export function lineageLatchedIds(controlDir) {
+    const rec = coerceLineage(readStrictSidecar(controlDir, LINEAGE_FILE, "memory lineage ledger"));
+    const out = new Set();
+    for (const txn of Object.values(rec.pending))
+        for (const row of txn.rows)
+            out.add(row.entryId);
+    return out;
+}
+export function lineageContributionsOfSession(controlDir, sessionId) {
+    const rec = coerceLineage(readStrictSidecar(controlDir, LINEAGE_FILE, "memory lineage ledger"));
+    const out = [];
+    for (const [entryId, sessions] of Object.entries(rec.committed)) {
+        const c = sessions[sessionId];
+        if (c !== undefined)
+            out.push({ entryId, lastRev: c.lastRev });
+    }
+    return out;
+}
+export function clearLineageForEntries(controlDir, entryIds) {
+    if (entryIds.length === 0)
+        return;
+    lockedStrictUpdate(controlDir, LINEAGE_FILE, "memory lineage ledger", coerceLineage, (rec) => {
+        let changed = false;
+        for (const id of entryIds) {
+            if (rec.committed[id] !== undefined) {
+                delete rec.committed[id];
+                changed = true;
+            }
+        }
+        return { ...(changed ? { next: rec } : {}), result: undefined };
+    });
+}
+export function readLineageRecord(controlDir) {
+    const rec = coerceLineage(readStrictSidecar(controlDir, LINEAGE_FILE, "memory lineage ledger"));
+    return { committed: rec.committed, pending: rec.pending };
+}
+export function appendLineageAudit(controlDir, rows) {
+    if (rows.length === 0)
+        return;
+    lockedJournaledUpdate(controlDir, LINEAGE_AUDIT_FILE, (current) => {
+        const list = Array.isArray(current) ? current.filter((r) => r && typeof r === "object") : [];
+        list.push(...rows);
+        return list.length > LINEAGE_AUDIT_MAX_ROWS ? list.slice(-LINEAGE_AUDIT_MAX_ROWS) : list;
+    });
+}
+function coerceChallenges(raw) {
+    if (raw === undefined)
+        return { version: 1, events: [] };
+    if (!raw || typeof raw !== "object" || Array.isArray(raw))
+        throw new ControlPlaneCorruptError("memory challenge ledger has the wrong shape");
+    const r = raw;
+    if (r.version !== 1)
+        throw new ControlPlaneCorruptError(`memory challenge ledger has an unrecognized version (${String(r.version)}) — refusing (fail-closed)`);
+    if (!Array.isArray(r.events))
+        throw new ControlPlaneCorruptError("memory challenge ledger events list has the wrong shape");
+    for (const e of r.events) {
+        const ev = e;
+        if (!ev ||
+            typeof ev !== "object" ||
+            typeof ev.eventId !== "string" ||
+            typeof ev.entryId !== "string" ||
+            (ev.kind !== "challenge" && ev.kind !== "resolve") ||
+            typeof ev.generation !== "number" ||
+            typeof ev.at !== "number" ||
+            typeof ev.reason !== "string") {
+            throw new ControlPlaneCorruptError("memory challenge ledger event has the wrong shape");
+        }
+    }
+    return raw;
+}
+export const CHALLENGE_LEDGER_MAX_EVENTS = 10_000;
+export function appendChallengeEvents(controlDir, events, now) {
+    if (events.length === 0)
+        return { assignments: [], eventCount: undefined };
+    return lockedStrictUpdate(controlDir, CHALLENGES_FILE, "memory challenge ledger", coerceChallenges, (rec) => {
+        const byEventId = new Map(rec.events.map((e) => [e.eventId, e]));
+        const maxGen = new Map();
+        for (const e of rec.events) {
+            if (e.kind !== "challenge")
+                continue;
+            maxGen.set(e.entryId, Math.max(maxGen.get(e.entryId) ?? 0, e.generation));
+        }
+        const out = [];
+        let changed = false;
+        for (const req of events) {
+            const existing = byEventId.get(req.eventId);
+            if (existing !== undefined) {
+                out.push({ entryId: existing.entryId, eventId: existing.eventId, generation: existing.generation, at: existing.at, replayed: true });
+                continue;
+            }
+            const generation = (maxGen.get(req.entryId) ?? 0) + 1;
+            maxGen.set(req.entryId, generation);
+            const ev = {
+                eventId: req.eventId,
+                entryId: req.entryId,
+                kind: "challenge",
+                generation,
+                at: now(),
+                reason: req.reason,
+                grade: "S",
+                ...(req.challengedRev !== undefined ? { challengedRev: req.challengedRev } : {}),
+            };
+            rec.events.push(ev);
+            byEventId.set(ev.eventId, ev);
+            out.push({ entryId: ev.entryId, eventId: ev.eventId, generation, at: ev.at, replayed: false });
+            changed = true;
+        }
+        return { ...(changed ? { next: rec } : {}), result: { assignments: out, eventCount: rec.events.length } };
+    });
+}
+export function resolveChallengeEvent(controlDir, entryId, generation, reason, now, requestId) {
+    return lockedStrictUpdate(controlDir, CHALLENGES_FILE, "memory challenge ledger", coerceChallenges, (rec) => {
+        const opened = rec.events.some((e) => e.kind === "challenge" && e.entryId === entryId && e.generation === generation);
+        if (!opened)
+            return { result: { resolved: false, eventCount: undefined } };
+        const alreadyResolved = rec.events.some((e) => e.kind === "resolve" && e.entryId === entryId && e.generation === generation);
+        if (alreadyResolved)
+            return { result: { resolved: true, eventCount: undefined } };
+        rec.events.push({
+            eventId: requestId !== undefined && requestId !== "" ? `${requestId}:${entryId}` : `resolve:${entryId}:${generation}`,
+            entryId,
+            kind: "resolve",
+            generation,
+            at: now(),
+            reason,
+        });
+        return { next: rec, result: { resolved: true, eventCount: rec.events.length } };
+    });
+}
+export function challengedEntryIds(controlDir) {
+    const rec = coerceChallenges(readStrictSidecar(controlDir, CHALLENGES_FILE, "memory challenge ledger"));
+    const resolved = new Set();
+    for (const e of rec.events)
+        if (e.kind === "resolve")
+            resolved.add(`${e.entryId}#${e.generation}`);
+    const out = new Map();
+    for (const e of rec.events) {
+        if (e.kind !== "challenge" || resolved.has(`${e.entryId}#${e.generation}`))
+            continue;
+        const prior = out.get(e.entryId);
+        if (prior === undefined || e.generation > prior.generation)
+            out.set(e.entryId, { generation: e.generation, at: e.at });
+    }
+    return out;
+}
+export function readChallengeEvents(controlDir) {
+    return coerceChallenges(readStrictSidecar(controlDir, CHALLENGES_FILE, "memory challenge ledger")).events;
+}
+export function isStrictControlPlaneLedgerCorrupt(controlDir, ledger) {
+    const fileName = ledger === "lineage" ? LINEAGE_FILE : CHALLENGES_FILE;
+    const what = ledger === "lineage" ? "memory lineage ledger" : "memory challenge ledger";
+    const coerce = ledger === "lineage" ? coerceLineage : coerceChallenges;
+    try {
+        coerce(readStrictSidecar(controlDir, fileName, what));
+        return false;
+    }
+    catch (err) {
+        if (err instanceof ControlPlaneCorruptError)
+            return true;
+        throw err;
+    }
+}
+export function rebuildStrictControlPlaneLedger(controlDir, ledger, now) {
+    const fileName = ledger === "lineage" ? LINEAGE_FILE : CHALLENGES_FILE;
+    const what = ledger === "lineage" ? "memory lineage ledger" : "memory challenge ledger";
+    const coerce = ledger === "lineage" ? coerceLineage : coerceChallenges;
+    const empty = ledger === "lineage" ? { version: 1, committed: {}, pending: {} } : { version: 1, events: [] };
+    ensureDirExists(controlDir);
+    const file = join(controlDir, fileName);
+    const journal = `${file}.journal`;
+    const lock = `${file}.lock`;
+    const token = acquireSidecarLock(lock, { onDeadline: "throw" });
+    try {
+        let healthy = false;
+        try {
+            coerce(readStrictSidecar(controlDir, fileName, what));
+            healthy = true;
+        }
+        catch (err) {
+            if (!(err instanceof ControlPlaneCorruptError))
+                throw err;
+        }
+        if (healthy) {
+            const e = new Error(`${what} reads cleanly — refusing to rebuild it. This call exists to recover a ledger whose corruption already refuses every read; resetting a healthy one would drop taint evidence that is still doing its job.`);
+            e.code = "memory.control_plane_not_corrupt";
+            throw e;
+        }
+        const at = now();
+        const quarantinedTo = [];
+        assertSidecarLockOwnership(lock, token, what);
+        for (const path of [file, journal]) {
+            if (!existsSync(path))
+                continue;
+            let dest = `${path}.corrupt-${at}`;
+            for (let n = 2;; n++) {
+                try {
+                    copyFileSync(path, dest, fsConstants.COPYFILE_EXCL);
+                    break;
+                }
+                catch (err) {
+                    if (err.code !== "EEXIST")
+                        throw err;
+                    dest = `${path}.corrupt-${at}-${n}`;
+                }
+            }
+            quarantinedTo.push(dest);
+        }
+        assertSidecarLockOwnership(lock, token, what);
+        atomicWriteFileSync(file, `${JSON.stringify(empty, null, 2)}\n`);
+        rmSync(journal, { force: true });
+        return { ledger, quarantinedTo, at };
+    }
+    finally {
+        releaseSidecarLock(lock, token);
+    }
+}
+function coerceChallengedHistory(raw) {
+    if (!raw || typeof raw !== "object" || Array.isArray(raw))
+        return {};
+    const out = {};
+    for (const [k, v] of Object.entries(raw)) {
+        const row = v;
+        if (row &&
+            typeof row === "object" &&
+            typeof row.count === "number" &&
+            Number.isFinite(row.count) &&
+            row.count > 0 &&
+            typeof row.lastAt === "number" &&
+            (row.lastOp === "update" || row.lastOp === "tombstone")) {
+            out[k] = { count: Math.floor(row.count), lastAt: row.lastAt, lastOp: row.lastOp };
+        }
+    }
+    return out;
+}
+export function recordChallengedHistory(controlDir, rows, now) {
+    if (rows.length === 0)
+        return;
+    const at = now();
+    lockedJournaledUpdate(controlDir, CHALLENGED_HISTORY_FILE, (current) => {
+        const acc = coerceChallengedHistory(current);
+        for (const r of rows) {
+            const prior = acc[r.entryId];
+            acc[r.entryId] = { count: (prior?.count ?? 0) + 1, lastAt: at, lastOp: r.op };
+        }
+        return acc;
+    });
+}
+export function readChallengedHistory(controlDir) {
+    return coerceChallengedHistory(readSidecarJson(controlDir, CHALLENGED_HISTORY_FILE));
+}
 export function writeAllSync(fd, data) {
     const buf = Buffer.from(data, "utf8");
     let written = 0;

package/dist/core/memory-engine/tools.d.ts

@@ -48,6 +48,19 @@ export interface MemoryEnginePlane {
     backend: MemoryBackend;
     scopes: readonly string[];
     recordRetrieved: (ids: readonly string[]) => void;
+    /**
+     * design/180 B-2' — the plane's EXCLUSION view: entry id → why its content is withheld from the
+     * model face (`challenged` = an unresolved S-grade challenge generation; `lineage_pending` = the
+     * dirty latch, its commit bookkeeping is unsettled). THROWS when the underlying ledger's
+     * integrity is unknowable — both tools then REFUSE SERVICE on that plane (fail-closed: a corrupt
+     * ledger must never read as "nothing withheld"). Absent ⇒ a plane with no control plane (direct
+     * hosts / tests) — no exclusion, v1 behavior byte-identical.
+     */
+    challengeExclusions?: () => ReadonlyMap<string, {
+        code: "challenged" | "lineage_pending";
+        generation?: number;
+        at?: number;
+    }>;
 }
 export interface MemoryEngineToolsOptions {
     planes: ReadonlyArray<MemoryEnginePlane>;
@@ -85,6 +98,16 @@ export interface MemoryGetDetails {
     offset?: number;
     lines?: number;
     totalLines?: number;
+    /** design/180 — the OPAQUE withheld refusal's structured half (`reason: "challenged" |
+     *  "lineage_pending"`): the unresolved challenge generation and its timestamp. Deliberately the
+     *  ONLY fields beyond the id — name/description/free reason text are content positions and live
+     *  on the host audit face exclusively. */
+    generation?: number;
+    challengedAt?: number;
+    /** design/180 (intra-line cursor) — byte offset into the (neutralized) single line at `offset`
+     *  this page continued from / where the next page should continue. */
+    lineCursor?: number;
+    nextLineCursor?: number;
 }
 /** Cut `text` to at most `maxBytes` UTF-8 bytes on a CODE POINT boundary (a byte-wise slice would
  *  strand half a character), reporting how many bytes were dropped. Unchanged text reports 0.
@@ -93,4 +116,11 @@ export declare function cutToBytes(text: string, maxBytes: number): {
     text: string;
     omittedBytes: number;
 };
+/** Skip `startBytes` UTF-8 bytes of `text` on a code-point boundary (the intra-line cursor's seek
+ *  half — {@link cutToBytes} is its cut half). A cursor landing mid-character (never minted by this
+ *  module) rounds FORWARD to the character's end, so no half-character is ever emitted. */
+export declare function skipBytes(text: string, startBytes: number): {
+    text: string;
+    skippedBytes: number;
+};
 export declare function createMemoryEngineTools(opts: MemoryEngineToolsOptions): ToolSpec[];