@cspeach/cli 0.9.0 → 1.1.0

package/dist/commands/plan-audit-evidence.js

@@ -0,0 +1,266 @@
+/**
+ * Ground-truth extractor for the phase auditor.
+ *
+ * The agent loop writes a JSONL audit log of every tool call to
+ * `~/.cspeach/checkpoints/<sessionId>/tool-calls.jsonl` (Layer 2 default
+ * handler in `agent/skill-checkpoint.ts`). The phase auditor must judge a
+ * phase's self-report against evidence the audited model did NOT author —
+ * this module produces that evidence: a bounded, compact excerpt of the
+ * write + verification tool calls (set_source, activate, syntax_check,
+ * transport, snapshot, ...), newest-last.
+ *
+ * Pure read module — no writes, no new dependencies.
+ */
+import { promises as fs } from 'node:fs';
+import * as path from 'node:path';
+import { checkpointsRoot } from '../agent/skill-checkpoint.js';
+/**
+ * PROTECTED evidence — write + verification calls that PROVE a phase's core
+ * claims and must NEVER be evicted by read rows (see the two-tier cap in
+ * extractAuditEvidence). Transport tools are matched by their EXACT
+ * write-relevant names (fix-wave): the registered family is
+ * sap_transport_create / _for_object / _list / _release (tools/transport.ts),
+ * and a bare `sap_transport` prefix let list-shaped read polling
+ * (sap_transport_list, sap_transport_for_object) evict real write rows from
+ * the 60-row cap.
+ *
+ * D-A (2026-07-05): sap_atc_run and sap_inactive_objects are VERIFICATION
+ * tools the audit contract + phase exit gates explicitly demand (ATC clean /
+ * no inactive versions remain), yet they were missing from this filter — so
+ * a phase that legitimately ran ATC or checked for inactive objects had that
+ * evidence stripped, and the auditor false-failed it with "no ATC call
+ * appears anywhere in the evidence". They are reads in the sense of not
+ * mutating source, but they ARE the proof of the verify steps, so they
+ * belong in the evidence excerpt. (Unlike the transport LIST polling, these
+ * fire once per verify, not in a loop, so they do not threaten the row cap.)
+ *
+ * Task 2 (audit-confidence-tiered-redesign, §6): request_approval is admitted
+ * here too. It is the PROOF-of-consent the §6 escalation exception keys on — a
+ * write to an infra base-object (DEVC / transport / number-range) under a
+ * writes:false phase is a sanctioned escalation ONLY IF the evidence carries an
+ * OK request_approval row for it. With the old sap_*-only filter that row was
+ * dropped, so the exception could never be satisfied. Like the existence-read
+ * proofs it is ground truth (the user's recorded consent), and the whole
+ * contract branch depends on it — so it must NEVER be evicted by the row cap:
+ * PROTECTED, not the fillable existence tier. Matched EXACTLY (^…$ anchored) so
+ * a near-miss tool name can never masquerade as consent.
+ */
+const PROTECTED_TOOL_RE = /^sap_(set_source|update_method|create_object|delete_object|activate|syntax_check|atc_run|inactive_objects|transport_create|transport_release|service_binding_publish|snapshot)|^request_approval$/;
+/**
+ * D-F (2026-07-05): EXISTENCE / VERIFY-READ evidence. A verify-only phase
+ * re-run (object already built in a prior attempt; this attempt proves it
+ * exists / is active / is clean and writes NOTHING) records only reads:
+ * sap_object_structure (version=active), sap_get_source (fields match), and
+ * sap_transport_for_object (object locked in a transport). With the old
+ * write-only filter those rows were stripped, the evidence went empty, and
+ * the auditor false-FAILED the re-run ("no write call in evidence"). These
+ * reads are legitimate ground truth: a read against the live harness log
+ * cannot be fabricated — a read of a nonexistent object ERRORS, so an `ok`
+ * existence read IS the system agreeing the object is there. They are
+ * admitted here so the auditor can satisfy a pre-existence claim on
+ * verification evidence alone (contract Rule 1, D-F branch).
+ *
+ * Cap safety (two-tier, see extractAuditEvidence): these reads are also used
+ * heavily in analysis phases and could, in a single-bucket cap, evict real
+ * write rows. They are therefore the FILLABLE tier — the newest of them fill
+ * whatever budget remains after every PROTECTED row is kept; a write row is
+ * never dropped to make room for one of these.
+ */
+const EXISTENCE_READ_RE = /^sap_(object_structure|get_source|transport_for_object)/;
+/** Any row the auditor may see is the union of the two tiers. */
+const EVIDENCE_TOOL_RE = new RegExp(`${PROTECTED_TOOL_RE.source}|${EXISTENCE_READ_RE.source}`);
+const NO_LOG_SENTINEL = '(no tool-call log found for this session)';
+const NO_EVIDENCE_ROWS = '(tool-call log present, but no write/verify calls recorded for this session)';
+const DEFAULT_MAX_ROWS = 60;
+const RESULT_EXCERPT_CHARS = 200;
+const ARGS_COMPACT_CHARS = 80;
+/**
+ * audit-timeout (2026-07-06) — per-tool result caps.
+ *
+ * Live false-FAIL #4: a c1.behavior audit failed with "both sap_inactive_objects
+ * results are truncated (only ZI_FRG3_DC and ZI_FRG_V2DC visible, followed by
+ * ellipsis)". The default 200-char cap beheaded the inactive-objects LIST,
+ * destroying the absence-from-inactive-list proof the D-F activeness arm depends
+ * on — and the auditor was CORRECTLY refusing to certify activeness from clipped
+ * evidence. The fix: render the inactive-objects list generously (it is names
+ * only, bounded in practice) so absence stays judgeable; only if it is
+ * pathologically huge do we clip it AND mark the clip HONESTLY, so the contract
+ * can fall back to the object_structure "version": "active" proof rather than
+ * inferring absence from a beheaded list. ATC results get a modest bump because
+ * the auditor reads finding priorities/counts from them. Everything else keeps
+ * the 200-char cap (object_structure surfaces "version" early, so 200 is fine).
+ */
+const INACTIVE_LIST_CAP = 4000;
+const ATC_RESULT_CAP = 500;
+/** Appended when the inactive-objects list itself is clipped — see the contract. */
+const INACTIVE_CLIP_MARKER = ' … (list clipped — absence not verifiable from this row; use object_structure version)';
+function resultCapFor(toolName) {
+    if (/^sap_inactive_objects/.test(toolName))
+        return INACTIVE_LIST_CAP;
+    if (/^sap_atc_run/.test(toolName))
+        return ATC_RESULT_CAP;
+    return RESULT_EXCERPT_CHARS;
+}
+function normalizeRow(raw) {
+    if (raw === null || typeof raw !== 'object')
+        return null;
+    const r = raw;
+    const toolName = typeof r.toolName === 'string' ? r.toolName
+        : typeof r.tool === 'string' ? r.tool
+            : null;
+    if (!toolName)
+        return null;
+    const isError = typeof r.isError === 'boolean' ? r.isError
+        : typeof r.ok === 'boolean' ? !r.ok
+            : false;
+    const at = typeof r.completedAt === 'string' ? r.completedAt
+        : typeof r.at === 'string' ? r.at
+            : null;
+    // PROTECTED is checked first: it is a strict-enough family that no
+    // existence-read name can collide with it, but ordering makes the tiering
+    // explicit for the reader.
+    const kind = PROTECTED_TOOL_RE.test(toolName) ? 'protected' : 'existence';
+    return { toolName, args: r.args, result: r.result, isError, at, kind };
+}
+/** Render the args as `name/type` when available, else a compact fallback. */
+function renderArgs(args) {
+    if (args === null || args === undefined || typeof args !== 'object')
+        return '';
+    const a = args;
+    const name = a.name ?? a.object_name;
+    const type = a.type ?? a.object_type;
+    if (typeof name === 'string' && name.length > 0) {
+        return typeof type === 'string' && type.length > 0 ? `${name}/${type}` : name;
+    }
+    // No name/type — render what's available, compactly.
+    try {
+        const json = JSON.stringify(a);
+        if (!json || json === '{}')
+            return '';
+        return json.length <= ARGS_COMPACT_CHARS ? json : json.slice(0, ARGS_COMPACT_CHARS) + '…';
+    }
+    catch {
+        return '';
+    }
+}
+/**
+ * Result excerpt, flattened to a single line, capped per-tool (see
+ * resultCapFor). For sap_inactive_objects, a clip is marked honestly so a
+ * beheaded list can never masquerade as a complete absence proof.
+ */
+function renderResult(result, toolName) {
+    let text;
+    if (typeof result === 'string') {
+        text = result;
+    }
+    else if (result === null || result === undefined) {
+        text = '';
+    }
+    else {
+        try {
+            text = JSON.stringify(result) ?? '';
+        }
+        catch {
+            text = String(result);
+        }
+    }
+    const flat = text.replace(/\s+/g, ' ').trim();
+    const cap = resultCapFor(toolName);
+    if (flat.length <= cap)
+        return flat;
+    if (/^sap_inactive_objects/.test(toolName)) {
+        // Absence-from-inactive-list is an activeness PROOF; a bare ellipsis would
+        // silently behead the list and make absence unverifiable. Mark it honestly.
+        return flat.slice(0, cap) + INACTIVE_CLIP_MARKER;
+    }
+    return flat.slice(0, cap);
+}
+function renderRow(row) {
+    const status = row.isError ? 'ERROR' : 'ok';
+    return `${row.toolName}(${renderArgs(row.args)}) → ${status} — ${renderResult(row.result, row.toolName)}`;
+}
+/**
+ * Extract a bounded, auditor-ready excerpt of the session's write +
+ * verification tool calls from the JSONL audit log.
+ *
+ * - Only rows whose toolName matches {@link EVIDENCE_TOOL_RE} are included.
+ * - D-B: when `sinceIso` is given, rows whose timestamp is strictly BEFORE it
+ *   are excluded — so an audit judging attempt N of a phase never sees the
+ *   rows of an earlier attempt that shares the same session JSONL (two
+ *   attempts in one CLI process). Untimestamped rows are kept (fail-open).
+ * - D-F two-tier cap: chronological order is preserved (newest-last). When
+ *   the evidence exceeds `maxRows` (default 60), the OLDEST existence-read
+ *   rows are dropped first; a PROTECTED (write / verify-proof) row is dropped
+ *   only if no existence-read rows remain to drop. So a write row is never
+ *   evicted to make room for a read — the exact regression a single bucket +
+ *   the new existence reads would have introduced. The 60 cap is kept (not
+ *   raised): the tiering, not a bigger budget, is what guarantees write
+ *   survival, and a fixed cap keeps the audit prompt's token cost bounded.
+ * - Missing file/dir returns the sentinel — never throws.
+ * - Malformed JSONL lines are skipped silently.
+ */
+export async function extractAuditEvidence(sessionId, opts) {
+    const maxRows = opts?.maxRows ?? DEFAULT_MAX_ROWS;
+    const sinceIso = opts?.sinceIso;
+    const filePath = path.join(checkpointsRoot(), sessionId, 'tool-calls.jsonl');
+    let content;
+    try {
+        content = await fs.readFile(filePath, 'utf-8');
+    }
+    catch {
+        return NO_LOG_SENTINEL;
+    }
+    const rows = [];
+    for (const line of content.split('\n')) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        let parsed;
+        try {
+            parsed = JSON.parse(trimmed);
+        }
+        catch {
+            continue; // malformed line — skip silently
+        }
+        const row = normalizeRow(parsed);
+        if (!row || !EVIDENCE_TOOL_RE.test(row.toolName))
+            continue;
+        // D-B attempt-window: drop rows recorded before this attempt started.
+        // ISO-8601 UTC strings compare correctly lexicographically. A row with no
+        // timestamp is never excluded (fail-open).
+        if (sinceIso && row.at !== null && row.at < sinceIso)
+            continue;
+        rows.push(row);
+    }
+    if (rows.length === 0)
+        return NO_EVIDENCE_ROWS;
+    return capRows(rows, maxRows).map(renderRow).join('\n');
+}
+/**
+ * D-F two-tier cap. Returns at most `maxRows` rows in chronological order.
+ * Eviction order when over budget: oldest EXISTENCE reads first, then (only
+ * if still over and no reads remain) oldest PROTECTED rows. Writes are thus
+ * never evicted by reads.
+ */
+function capRows(rows, maxRows) {
+    if (rows.length <= maxRows)
+        return rows;
+    let over = rows.length - maxRows;
+    const dropped = new Set();
+    // Pass 1 — drop oldest existence reads.
+    for (let i = 0; i < rows.length && over > 0; i++) {
+        if (rows[i].kind === 'existence') {
+            dropped.add(i);
+            over--;
+        }
+    }
+    // Pass 2 — reads exhausted, still over: drop oldest protected rows (writes
+    // evicting older writes is the original 60-cap behavior, not a read winning).
+    for (let i = 0; i < rows.length && over > 0; i++) {
+        if (!dropped.has(i)) {
+            dropped.add(i);
+            over--;
+        }
+    }
+    return rows.filter((_, i) => !dropped.has(i));
+}