@alisio/core 0.6.0 → 0.8.0

package/dist/proof/recorder.js

@@ -0,0 +1,347 @@
+/**
+ * Proof evidence capture (specs/011 3.3, D3, D6, D7). The runner hands the recorder the RAW
+ * result of a tool call right after it executed, before the model-facing bound. Only the
+ * recorder writes evidence: nothing the model says or a tool prints can create, edit or mark a
+ * criterion, because the only input that counts is the structured result of the four command
+ * tools below and the journal of the call that produced it.
+ *
+ * Without `command`, `file` or `data-check` criteria in the session `capture` returns before doing
+ * anything but one indexed read, and only for the command tools, `read_file`, `data_query` and
+ * `write_file`/`edit_file` (D7).
+ *
+ * A successful `write_file`/`edit_file` is noted per path (D5); the ledger derives from those
+ * times on every read, so evaluation stays pure. `divergences` checks a final answer's claims.
+ */
+import { createHash } from "node:crypto";
+import { isAbsolute, relative, resolve, sep } from "node:path";
+import { redactOutput } from "../eval/secrets.js";
+import { callFingerprint, criterionFingerprints, normalizeCwd, relativeCwd, } from "./command.js";
+import { judgeCoverage } from "./coverage.js";
+import { detectDivergence } from "./divergence.js";
+import { ProofStore } from "./store.js";
+import { treeFingerprint } from "./tree.js";
+import { judgeDataCheck, judgeFile, judgeMissingFile, readsFile, runsCanonicalQuery, } from "./verifiers.js";
+/** Tools whose raw result can back a `command` criterion. */
+const COMMAND_TOOLS = new Set(["run_process", "shell", "bg_run", "bg_output"]);
+/** Tools whose raw result can back a `file` (`read_file`) or `data-check` (`data_query`) criterion. */
+const VERIFIER_TOOLS = new Set(["read_file", "data_query"]);
+/** Criteria bound to these verifier kinds are the ones the recorder captures evidence for. */
+const CAPTURING = new Set(["command", "coverage", "file", "data-check"]);
+/** Tools whose success edits workspace files and so expires evidence (D5). */
+const EDIT_TOOLS = new Set(["write_file", "edit_file"]);
+const EXCERPT_BYTES = 4096;
+/** Background tasks remembered per recorder (oldest dropped first). */
+const MAX_BACKGROUND = 256;
+const sha256 = (text) => createHash("sha256").update(text).digest("hex");
+function textOf(result) {
+    return result.content.map((part) => (part.type === "text" ? part.text : "")).join("");
+}
+function jsonObject(text) {
+    try {
+        const value = JSON.parse(text);
+        return value && typeof value === "object" && !Array.isArray(value)
+            ? value
+            : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/** The last `EXCERPT_BYTES` bytes of the redacted output (the end holds the summary and errors). */
+export function excerptOf(output) {
+    const redacted = redactOutput(output);
+    const bytes = Buffer.from(redacted);
+    if (bytes.length <= EXCERPT_BYTES)
+        return redacted;
+    let start = bytes.length - EXCERPT_BYTES;
+    // Never start inside a multi-byte character: it would decode to a longer replacement char.
+    while (start < bytes.length && ((bytes[start] ?? 0) & 0xc0) === 0x80)
+        start++;
+    return bytes.subarray(start).toString("utf8");
+}
+function passes(verifier, outcome) {
+    if (!(verifier.successExitCodes ?? [0]).includes(outcome.exitCode))
+        return false;
+    if (verifier.outputPattern === undefined)
+        return true;
+    try {
+        return new RegExp(verifier.outputPattern).test(outcome.output);
+    }
+    catch {
+        return false; // an unusable pattern never verifies anything
+    }
+}
+export class ProofRecorder {
+    db;
+    options;
+    store;
+    /** `bg_run` calls whose exit code has not been seen yet, by session and task id. */
+    background = new Map();
+    constructor(db, options = {}) {
+        this.db = db;
+        this.options = options;
+        this.store = new ProofStore(db);
+    }
+    /** Records the evidence a just-executed call yields; returns the ledger changes to announce. */
+    capture(call) {
+        const edit = EDIT_TOOLS.has(call.name);
+        const command = COMMAND_TOOLS.has(call.name);
+        if (!edit && !command && !VERIFIER_TOOLS.has(call.name))
+            return [];
+        const criteria = this.tracked(call.sessionId);
+        if (criteria.length === 0)
+            return [];
+        if (edit)
+            return this.captureEdit(call);
+        const findings = command
+            ? this.commandFindings(call, criteria)
+            : call.name === "read_file"
+                ? this.fileFindings(call, criteria)
+                : this.dataFindings(call, criteria);
+        return this.persist(call, findings);
+    }
+    /**
+     * A `read_file` that threw because the file is not there (the runner reports it here: a throw
+     * never reaches `capture`). It refutes every `file` criterion on that path; any other failure
+     * proves nothing.
+     */
+    captureFailure(call) {
+        if (call.name !== "read_file" || !call.result.isError)
+            return [];
+        const judged = judgeMissingFile(textOf(call.result));
+        if (!judged)
+            return [];
+        const matching = this.tracked(call.sessionId).filter((c) => c.verifier?.kind === "file" && readsFile(c.verifier, call.input.path, call.workspace));
+        return this.persist(call, matching.map((criterion) => this.finding(call, criterion, "file", judged, textOf(call.result))));
+    }
+    /** The criteria of the session the recorder can capture evidence for (not marked N/A). */
+    tracked(sessionId) {
+        return this.store.criteria(sessionId).filter((c) => c.verifier &&
+            CAPTURING.has(c.verifier.kind) &&
+            !c.notApplicableReason &&
+            // Opt-in: with `proof.coverage.enabled` off a coverage criterion is never captured.
+            (c.verifier.kind !== "coverage" || this.options.coverage?.() === true));
+    }
+    finding(call, criterion, verifierKind, judged, rawOutput) {
+        return {
+            criterion,
+            verifierKind,
+            ok: judged.ok,
+            outputHash: sha256(rawOutput),
+            excerpt: excerptOf(judged.summary),
+            durationMs: call.durationMs,
+        };
+    }
+    fileFindings(call, criteria) {
+        if (call.result.isError)
+            return [];
+        const text = textOf(call.result);
+        const body = jsonObject(text);
+        if (!body)
+            return [];
+        return criteria.flatMap((c) => {
+            if (c.verifier?.kind !== "file" || !readsFile(c.verifier, call.input.path, call.workspace))
+                return [];
+            const judged = judgeFile(c.verifier, body);
+            return judged ? [this.finding(call, c, "file", judged, text)] : [];
+        });
+    }
+    dataFindings(call, criteria) {
+        if (call.result.isError)
+            return [];
+        const text = textOf(call.result);
+        return criteria.flatMap((c) => {
+            if (c.verifier?.kind !== "data-check" || !runsCanonicalQuery(c.verifier, call.input))
+                return [];
+            const judged = judgeDataCheck(c.verifier, text);
+            return judged ? [this.finding(call, c, "data-check", judged, text)] : [];
+        });
+    }
+    commandFindings(call, criteria) {
+        const commands = criteria.filter((c) => c.verifier?.kind === "command");
+        const coverages = criteria.filter((c) => c.verifier?.kind === "coverage");
+        if (commands.length === 0 && coverages.length === 0)
+            return [];
+        const outcome = this.outcome(call);
+        if (!outcome)
+            return [];
+        const coverage = coverages
+            .filter((c) => criterionFingerprints(c.verifier.command, ".").includes(outcome.fingerprint))
+            .map((c) => {
+            const judged = judgeCoverage(c.verifier, outcome);
+            return {
+                criterion: c,
+                verifierKind: "coverage",
+                ok: judged.ok,
+                outputHash: sha256(outcome.output),
+                // The verdict goes last: the excerpt keeps the END of the text (see `excerptOf`).
+                excerpt: excerptOf(`${outcome.output ? `${outcome.output}\n` : ""}${judged.summary}`),
+                exitCode: outcome.exitCode,
+                fingerprint: outcome.fingerprint,
+                durationMs: outcome.durationMs,
+            };
+        });
+        return [
+            ...coverage,
+            ...commands
+                .filter((c) => criterionFingerprints(c.verifier.command, normalizeCwd(c.verifier.cwd ?? ".")).includes(outcome.fingerprint))
+                .map((c) => ({
+                criterion: c,
+                verifierKind: "command",
+                ok: passes(c.verifier, outcome),
+                outputHash: sha256(outcome.output),
+                ...(outcome.output ? { excerpt: excerptOf(outcome.output) } : {}),
+                exitCode: outcome.exitCode,
+                fingerprint: outcome.fingerprint,
+                durationMs: outcome.durationMs,
+            })),
+        ];
+    }
+    /** Appends one evidence per finding, once the call is proven to be in the session journal. */
+    persist(call, findings) {
+        if (findings.length === 0 || !this.executed(call))
+            return [];
+        const capturedAt = this.clock();
+        const filesTouched = this.store.edits(call.sessionId).map((e) => e.path);
+        const tree = treeFingerprint(call.workspace, filesTouched);
+        const appended = findings.map((f) => this.store.appendEvidence({
+            criterionId: f.criterion.criterionId,
+            verifierKind: f.verifierKind,
+            runId: call.runId,
+            callId: call.callId,
+            ok: f.ok,
+            ...(f.fingerprint ? { commandFingerprint: f.fingerprint } : {}),
+            ...(f.exitCode !== undefined ? { exitCode: f.exitCode } : {}),
+            durationMs: f.durationMs,
+            outputHash: f.outputHash,
+            ...(f.excerpt ? { excerpt: f.excerpt } : {}),
+            treeFingerprint: tree,
+            filesTouched,
+            capturedAt,
+        }));
+        const ledger = this.store.ledger(call.sessionId, capturedAt);
+        return appended.flatMap((evidence) => {
+            const criterion = ledger.criteria.find((c) => c.criterionId === evidence.criterionId);
+            return criterion
+                ? [
+                    {
+                        criterionId: criterion.criterionId,
+                        status: criterion.status,
+                        stale: criterion.stale ?? false,
+                        evidenceId: evidence.evidenceId,
+                    },
+                ]
+                : [];
+        });
+    }
+    clock() {
+        return (this.options.now?.() ?? new Date()).toISOString();
+    }
+    /**
+     * Notes a successful write/edit of a workspace file (D5) and announces the criteria whose
+     * derived status or staleness changed because of it. Edits outside the workspace cannot touch
+     * a workspace criterion and are ignored.
+     */
+    captureEdit(call) {
+        if (call.result.isError || typeof call.input.path !== "string" || !this.executed(call))
+            return [];
+        const path = relative(call.workspace, resolve(call.workspace, call.input.path));
+        if (!path || path === ".." || path.startsWith(`..${sep}`) || isAbsolute(path))
+            return [];
+        const at = this.clock();
+        const before = this.store.ledger(call.sessionId, at).criteria;
+        this.store.recordEdit(call.sessionId, path.split(sep).join("/"), at);
+        const after = this.store.ledger(call.sessionId, at).criteria;
+        return after.flatMap((criterion) => {
+            const previous = before.find((c) => c.criterionId === criterion.criterionId);
+            const stale = criterion.stale ?? false;
+            if (previous && previous.status === criterion.status && (previous.stale ?? false) === stale)
+                return [];
+            return [{ criterionId: criterion.criterionId, status: criterion.status, stale }];
+        });
+    }
+    /**
+     * The claims of a final answer that the ledger does not back (D8: shown, never acted on). Empty
+     * without criteria, after one indexed read (D7).
+     */
+    divergences(sessionId, finalText) {
+        if (this.store.criteria(sessionId).length === 0)
+            return [];
+        return detectDivergence(finalText, this.store.ledger(sessionId, this.clock()));
+    }
+    /** Reads what a command call proved, or undefined when it proved nothing (yet). */
+    outcome(call) {
+        const body = jsonObject(textOf(call.result));
+        if (!body)
+            return undefined;
+        if (call.name === "run_process" || call.name === "shell")
+            return this.processOutcome(call, body);
+        if (call.name === "bg_run") {
+            this.startedBackground(call, body);
+            return undefined;
+        }
+        return this.backgroundOutcome(call, body);
+    }
+    processOutcome(call, body) {
+        if (typeof body.exitCode !== "number" || !Number.isFinite(body.exitCode))
+            return undefined;
+        const command = call.input.command;
+        if (typeof command !== "string")
+            return undefined;
+        const args = Array.isArray(call.input.args) ? call.input.args.map(String) : [];
+        const stdout = typeof body.stdout === "string" ? body.stdout : "";
+        const stderr = typeof body.stderr === "string" ? body.stderr : "";
+        return {
+            exitCode: body.exitCode,
+            output: [stdout, stderr].filter(Boolean).join("\n"),
+            durationMs: call.durationMs,
+            fingerprint: callFingerprint(call.name, { command, args }, "."),
+        };
+    }
+    startedBackground(call, body) {
+        const command = call.input.command;
+        if (typeof body.id !== "string" || typeof command !== "string")
+            return;
+        const cwd = typeof call.input.cwd === "string" ? call.input.cwd : undefined;
+        this.background.set(this.key(call.sessionId, body.id), {
+            fingerprint: callFingerprint("shell", { command }, relativeCwd(call.workspace, cwd)),
+            startedAt: Date.now() - call.durationMs,
+            recorded: false,
+        });
+        while (this.background.size > MAX_BACKGROUND) {
+            const oldest = this.background.keys().next().value;
+            if (oldest === undefined)
+                break;
+            this.background.delete(oldest);
+        }
+    }
+    /** `bg_output` carries the exit code of a task this recorder saw start; once per task. */
+    backgroundOutcome(call, body) {
+        const id = typeof body.id === "string" ? body.id : undefined;
+        const task = id ? this.background.get(this.key(call.sessionId, id)) : undefined;
+        if (!task || task.recorded)
+            return undefined;
+        if (typeof body.exit_code !== "number" || !Number.isFinite(body.exit_code))
+            return undefined;
+        task.recorded = true;
+        return {
+            exitCode: body.exit_code,
+            output: typeof body.text === "string" ? body.text : "",
+            durationMs: Math.max(0, Date.now() - task.startedAt),
+            fingerprint: task.fingerprint,
+        };
+    }
+    key(sessionId, taskId) {
+        return `${sessionId}\u0000${taskId}`;
+    }
+    /**
+     * D2/D3: the call must exist in the session's tool-call journal under this run and name, which
+     * the runner writes before the tool executes. A forged or foreign `runId`/`callId` fails here.
+     */
+    executed(call) {
+        const row = this.db
+            .prepare("SELECT 1 AS ok FROM tool_calls WHERE session=? AND call_id=? AND run_id=? AND name=?")
+            .get(call.sessionId, call.callId, call.runId, call.name);
+        return row !== undefined && row !== null;
+    }
+}

package/dist/proof/scope.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Scope globs against edited paths (specs/011 3.4, D5). Pure: no filesystem, no clock.
+ *
+ * Paths are workspace-relative with `/` separators. A scope entry covers a path when its glob
+ * matches the path or any ancestor directory of it, so `src` and `src/**` both cover
+ * `src/a/b.ts`. No scope (absent or empty) means the whole workspace. Supported glob syntax:
+ * `*`, `?`, `**`, `[abc]`/`[a-z]`/`[!x]` and `{a,b}`; anything else is literal.
+ */
+/** Forward slashes, no leading `./`, no trailing `/`. */
+export declare function normalizeScopePath(path: string): string;
+/** Whether any scope entry covers `path`. Absent or empty scope: the whole workspace. */
+export declare function scopeMatches(scope: readonly string[] | undefined, path: string): boolean;

package/dist/proof/scope.js

@@ -0,0 +1,121 @@
+/**
+ * Scope globs against edited paths (specs/011 3.4, D5). Pure: no filesystem, no clock.
+ *
+ * Paths are workspace-relative with `/` separators. A scope entry covers a path when its glob
+ * matches the path or any ancestor directory of it, so `src` and `src/**` both cover
+ * `src/a/b.ts`. No scope (absent or empty) means the whole workspace. Supported glob syntax:
+ * `*`, `?`, `**`, `[abc]`/`[a-z]`/`[!x]` and `{a,b}`; anything else is literal.
+ */
+const MAX_PATTERN = 256;
+const MAX_EXPANSIONS = 32;
+/** Forward slashes, no leading `./`, no trailing `/`. */
+export function normalizeScopePath(path) {
+    let out = path.replace(/\\/g, "/");
+    while (out.startsWith("./"))
+        out = out.slice(2);
+    out = out.replace(/\/{2,}/g, "/");
+    while (out.length > 1 && out.endsWith("/"))
+        out = out.slice(0, -1);
+    return out;
+}
+/** Expands `{a,b}` groups (innermost first) into plain patterns, bounded. */
+function expandBraces(pattern) {
+    const open = pattern.search(/\{[^{}]*\}/);
+    if (open < 0)
+        return [pattern];
+    const close = pattern.indexOf("}", open);
+    const head = pattern.slice(0, open);
+    const tail = pattern.slice(close + 1);
+    const out = [];
+    for (const option of pattern.slice(open + 1, close).split(",")) {
+        for (const rest of expandBraces(head + option + tail)) {
+            out.push(rest);
+            if (out.length >= MAX_EXPANSIONS)
+                return out;
+        }
+    }
+    return out;
+}
+const escapeChar = (c) => c.replace(/[\\^$.*+?()[\]{}|/-]/g, "\\$&");
+/** Compiles one brace-free glob to an anchored regular expression. */
+function compile(glob) {
+    let g = glob.replace(/(?:\*\*\/)+/g, "**/").replace(/\*{3,}/g, "**");
+    // A trailing `/**` also matches the directory itself.
+    const tail = g.endsWith("/**") ? "(?:/.*)?" : "";
+    if (tail)
+        g = g.slice(0, -3);
+    let re = "";
+    for (let i = 0; i < g.length; i++) {
+        const c = g.charAt(i);
+        if (c === "*") {
+            if (g[i + 1] === "*") {
+                const slash = g[i + 2] === "/";
+                // `**/` also matches nothing.
+                if (slash) {
+                    re += "(?:.*/)?";
+                    i += 2;
+                }
+                else {
+                    re += ".*";
+                    i += 1;
+                }
+            }
+            else
+                re += "[^/]*";
+        }
+        else if (c === "?")
+            re += "[^/]";
+        else if (c === "[") {
+            const close = g.indexOf("]", i + 2);
+            if (close < 0)
+                re += "\\[";
+            else {
+                const body = g.slice(i + 1, close);
+                const negate = body.startsWith("!") || body.startsWith("^");
+                const set = (negate ? body.slice(1) : body).replace(/[\\\]^]/g, "\\$&");
+                re += `[${negate ? "^/" : ""}${set}]`;
+                i = close;
+            }
+        }
+        else
+            re += escapeChar(c);
+    }
+    return new RegExp(`^${re}${tail}$`);
+}
+const cache = new Map();
+function compiled(entry) {
+    const known = cache.get(entry);
+    if (known !== undefined)
+        return known;
+    let value;
+    try {
+        value = expandBraces(normalizeScopePath(entry)).map(compile);
+    }
+    catch {
+        value = null; // unusable entry: callers fail safe and treat it as matching everything
+    }
+    if (cache.size >= 512)
+        cache.clear();
+    cache.set(entry, value);
+    return value;
+}
+/** Whether any scope entry covers `path`. Absent or empty scope: the whole workspace. */
+export function scopeMatches(scope, path) {
+    if (!scope || scope.length === 0)
+        return true;
+    const target = normalizeScopePath(path);
+    const segments = target.split("/");
+    const prefixes = segments.map((_, i) => segments.slice(0, i + 1).join("/"));
+    for (const entry of scope) {
+        if (entry.length > MAX_PATTERN)
+            return true;
+        const patterns = compiled(entry);
+        if (patterns === null)
+            return true;
+        for (const pattern of patterns) {
+            if (prefixes.some((p) => pattern.test(p)))
+                return true;
+        }
+    }
+    return false;
+}

package/dist/proof/store.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Persistence of the Proof ledger (tables `proof_criteria` and `proof_evidence`, schema v11).
+ *
+ * Evidence is append-only: there is no update path here and a database trigger rejects UPDATE;
+ * rows disappear only with their session (`ON DELETE CASCADE`). The status is never stored (D4):
+ * `ledger()` derives it on every read.
+ */
+import type { ProofEvidence, ProofLedger, SqlDatabase } from "@alisio/sdk";
+import type { CriterionRecord, NewCriterion, NewEvidence } from "./types.ts";
+export declare class ProofStore {
+    private readonly db;
+    constructor(db: SqlDatabase);
+    addCriterion(input: NewCriterion): CriterionRecord;
+    criterion(criterionId: string): CriterionRecord | undefined;
+    /** The criteria of a session in creation order, without any derived field. */
+    criteria(sessionId: string): CriterionRecord[];
+    /** How many criteria a session has (one indexed count; the stream snapshot uses it). */
+    criteriaCount(sessionId: string): number;
+    /** Marks a criterion not applicable. Callers must be the user, never a tool result (O2). */
+    markNotApplicable(criterionId: string, reason: string): void;
+    /**
+     * The user edits a criterion. Only the wording and the required flag change: the verifier and
+     * the scope stay, so the evidence already captured keeps meaning what it meant (add a new
+     * criterion to check something else). Callers must be the user, never a tool result (O2).
+     */
+    updateCriterion(criterionId: string, change: {
+        text?: string;
+        required?: boolean;
+    }): CriterionRecord | undefined;
+    /**
+     * The unbacked claims the last COMPLETED run of the session warned about (`proof_divergence`
+     * events of the event journal). A later run that completes without a warning clears them.
+     * Display only (D8): nothing here changes a status.
+     */
+    divergences(sessionId: string): ProofLedger["divergences"];
+    /** Appends one immutable evidence (D2: it must name its run and its call). */
+    appendEvidence(input: NewEvidence): ProofEvidence;
+    /**
+     * Notes a successful edit of a workspace-relative path (D5). One row per path keeps the newest
+     * time, so the table is bounded by the files touched and an older time never rewinds it.
+     */
+    recordEdit(sessionId: string, path: string, editedAt: string): void;
+    /** The files edited in a session with the time of their latest edit, sorted by path. */
+    edits(sessionId: string): Array<{
+        path: string;
+        editedAt: string;
+    }>;
+    /** The ledger of a session with every status derived now (D4). */
+    ledger(sessionId: string, now?: string): ProofLedger;
+    /** The ledger every surface shows: `ledger()` plus the divergences of the last completed run. */
+    view(sessionId: string, now?: string): ProofLedger;
+}