@nexrall/code-core 1.3.1 → 1.4.1

package/dist/agent/loop.js

@@ -33,8 +33,15 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.VERIFY_CMD_RE = exports.WRITE_TOOL_NAMES = void 0;
 exports.resolveMaxIterations = resolveMaxIterations;
+exports.estimateBodyBytes = estimateBodyBytes;
 exports.findSafeCutIndex = findSafeCutIndex;
+exports.transcriptOf = transcriptOf;
+exports.createLedger = createLedger;
+exports.ledgerRecord = ledgerRecord;
+exports.ledgerSummary = ledgerSummary;
+exports.pruneOldToolResults = pruneOldToolResults;
 exports.runAgentLoop = runAgentLoop;
 const client_1 = require("../api/client");
 const executor_1 = require("../tools/executor");
@@ -42,6 +49,8 @@ const agentTypes_1 = require("./agentTypes");
 const rules_1 = require("../permissions/rules");
 const sandbox_1 = require("../tools/sandbox");
 const index_1 = require("../plugins/index");
+const testIntegrity_1 = require("./testIntegrity");
+const flaky_1 = require("./flaky");
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const child_process_1 = require("child_process");
@@ -391,13 +400,45 @@ async function runSubTask(input, options, agentTypes) {
 // Compaction only happens at a turn boundary (top of the loop, before the next
 // streamChat) and only cuts at a "safe" user message — one with no tool_result
 // blocks — so tool_use/tool_result pairing is never broken.
+// All three code tiers are NATIVELY 1M-context (backend CODE_MODEL_MAP + CTX_LIMITS):
+//   turbo → claude-sonnet-5  (1M)
+//   pro   → claude-opus-4-8  (1M)
+//   ultra → claude-fable-5   (1M)
+// Previously turbo/pro were hard-coded to 200K here, which made auto-compact fire at
+// 80% × 200K = 160K tokens — throttling the agent to ~16% of the real window and forcing
+// premature (lossy) summarisation on long tasks. Keep these in sync with backend
+// routes/code.js CTX_LIMITS.
 const MODEL_CONTEXT_TOKENS = {
-    turbo: 200000,
-    pro: 200000,
+    turbo: 1000000,
+    pro: 1000000,
     ultra: 1000000,
 };
 const AUTO_COMPACT_THRESHOLD = 0.8; // compact when prompt > 80% of the window
 const COMPACT_KEEP_MIN = 6; // always keep at least the last N messages verbatim
+// Byte-level safety net, independent of the token estimate.
+//
+// Tool-heavy sessions on large codebases accumulate many tool_result blocks
+// (read_file / bash / search output). The token count can still look "under
+// budget" while the SERIALISED body has grown to tens of MB — the char↔token
+// ratio for JSON/code/logs is highly variable, so a token threshold alone does
+// NOT bound the request body size. The backend rejects bodies over its limit
+// (413), which the token-based compactor never anticipates because:
+//   • it reacts to lastPromptTokens from the PREVIOUS turn's usage event, so on
+//     a freshly-resumed (already-large) session it is 0 and never fires, and
+//   • 80% × 1M tokens of tool_result can be 25–45 MB — far past any body limit.
+// This guard measures the ACTUAL body bytes before each send and forces a
+// compaction whenever it crosses the threshold, regardless of the token count.
+// Kept comfortably under the server's 25 MB /api/code limit.
+const MAX_BODY_BYTES = 8 * 1024 * 1024; // 8 MB
+/** Approximate serialised request-body size (bytes) for the messages array. */
+function estimateBodyBytes(messages) {
+    try {
+        return Buffer.byteLength(JSON.stringify(messages), 'utf-8');
+    }
+    catch {
+        return 0; // circular/unserialisable — don't block on the estimate
+    }
+}
 function resolveAutoCompact(fromOptions, rawSettings) {
     if (typeof fromOptions === 'boolean')
         return fromOptions;
@@ -411,6 +452,22 @@ function resolveAutoCompact(fromOptions, rawSettings) {
         return s;
     return true;
 }
+/** Opt-out for the one-shot verification nudge (GAP D). Defaults to on. */
+function resolveVerificationNudge(rawSettings) {
+    const env = (process.env.NEXRALL_VERIFY_NUDGE ?? '').toLowerCase();
+    if (env === '0' || env === 'false' || env === 'off')
+        return false;
+    if (env === '1' || env === 'true' || env === 'on')
+        return true;
+    const s = rawSettings?.verifyNudge;
+    if (typeof s === 'boolean')
+        return s;
+    return true;
+}
+/** Tools that mutate the filesystem — used by the verification nudge (GAP D). */
+exports.WRITE_TOOL_NAMES = new Set(['write_file', 'edit_file', 'multi_edit', 'delete_file', 'move_file', 'copy_file', 'notebook_edit']);
+/** Heuristic: does a bash command look like it's running tests/build/lint/typecheck? (GAP D) */
+exports.VERIFY_CMD_RE = /\b(npm|yarn|pnpm)\s+(run\s+)?(test|build|lint|typecheck|tsc)\b|\bpytest\b|\bgo\s+(test|vet|build)\b|\btsc\b|\beslint\b|\bcargo\s+(test|build|check)\b/i;
 /**
  * Find the latest index ≤ maxIdx where history can be cut safely.
  *
@@ -431,7 +488,21 @@ function findSafeCutIndex(messages, maxIdx) {
     }
     return -1;
 }
-/** Render messages to a plain-text transcript for the summariser (tool noise truncated). */
+// Hard ceiling on the transcript we hand to the summariser. Per-block truncation
+// alone does NOT bound the total: a very long run has thousands of blocks, so the
+// concatenated transcript can itself exceed the summariser call's context window →
+// the summarise request 400s → autoCompactMessages returns false → NO compaction
+// happens exactly when the session is largest (the context-wall failure mode).
+// ~600K chars ≈ 150K tokens, well under a 1M window even with prompt overhead.
+const MAX_TRANSCRIPT_CHARS = 600000;
+/**
+ * Render messages to a plain-text transcript for the summariser (tool noise
+ * truncated per-block AND the whole transcript hard-capped). When the transcript
+ * would exceed MAX_TRANSCRIPT_CHARS we keep the HEAD (original task + early
+ * decisions) and the TAIL (most-recent, highest-signal context) and drop the
+ * middle — a middle-out elision that preserves both "what we set out to do" and
+ * "where we are now", which is what the continuation summary needs most.
+ */
 function transcriptOf(messages) {
     const parts = [];
     for (const m of messages) {
@@ -447,18 +518,207 @@ function transcriptOf(messages) {
             }
         }
     }
-    return parts.join('\n');
+    const full = parts.join('\n');
+    if (full.length <= MAX_TRANSCRIPT_CHARS)
+        return full;
+    // Middle-out: keep 40% head, 60% tail (recent context is higher-signal for
+    // continuation). Slice on line boundaries so we don't cut a line in half.
+    const headBudget = Math.floor(MAX_TRANSCRIPT_CHARS * 0.4);
+    const tailBudget = MAX_TRANSCRIPT_CHARS - headBudget;
+    const head = full.slice(0, headBudget);
+    const tail = full.slice(full.length - tailBudget);
+    const dropped = full.length - head.length - tail.length;
+    return `${head}\n\n[… ${dropped} chars of mid-session transcript elided to fit the summariser's context window …]\n\n${tail}`;
+}
+// ─── Structured progress ledger (GAP E) ────────────────────────────────────────
+//
+// The single biggest long-horizon failure mode (industry-wide "context rot") is
+// that each auto-compaction summarises a transcript that ALREADY contains a prior
+// summary → summary-of-summary → fidelity decays: the agent forgets which files it
+// edited, whether tests passed, what's still open. Prose summarisation is inherently
+// lossy and gets worse every round.
+//
+// Defence: maintain a DETERMINISTIC, append-only ledger of high-signal facts derived
+// directly from tool calls — files created/edited (with count), commands verified
+// (test/build/lint) and their pass/fail, and explicit open TODOs. This is built from
+// structured tool data (NOT model output), so it is LOSSLESS and never degrades. We
+// inject it VERBATIM into every compaction preamble, so no matter how many times the
+// prose summary is re-summarised, the concrete "what changed / what's verified /
+// what's left" facts survive intact across an arbitrarily long run.
+const LEDGER_MAX_FILES = 60; // cap the file list so the preamble can't balloon
+const LEDGER_MAX_NOTES = 20; // cap verification/among notes
+function createLedger() {
+    return { filesTouched: new Map(), verifications: [], testIntegrity: [], epoch: 0 };
+}
+/** Record one tool call's effect on the ledger (deterministic, no model call). */
+function ledgerRecord(ledger, toolName, input, ok, output) {
+    if (exports.WRITE_TOOL_NAMES.has(toolName)) {
+        if (!ok)
+            return; // a FAILED write changed nothing — not a durable fact
+        // A successful source write advances the mutation epoch: any verification
+        // run after this point has different inputs than runs before it.
+        ledger.epoch += 1;
+        const p = typeof input?.path === 'string' ? input.path : undefined;
+        if (p) {
+            const prev = ledger.filesTouched.get(p);
+            ledger.filesTouched.set(p, { tool: toolName, edits: (prev?.edits ?? 0) + 1 });
+        }
+        // Reward-hacking guard: if this write WEAKENED a test file, record it so the
+        // signal survives compaction and can be surfaced before the agent finishes.
+        const reasons = [];
+        // write_file overwrites carry a marker computed by the executor (which had the
+        // prior on-disk content) — it detects REMOVED assertions/cases, not just
+        // additive skips/tautologies. Prefer it when present.
+        const markerReasons = toolName === 'write_file' ? (0, testIntegrity_1.decodeTestIntegrityMarker)(output) : [];
+        if (markerReasons.length) {
+            reasons.push(...markerReasons);
+        }
+        else {
+            const ti = (0, testIntegrity_1.analyzeWriteToolForTestIntegrity)(toolName, input);
+            if (ti?.suspicious)
+                reasons.push(...ti.findings.map((f) => f.reason));
+        }
+        if (reasons.length && p) {
+            for (const reason of reasons) {
+                ledger.testIntegrity.push({ path: p, reason });
+            }
+            if (ledger.testIntegrity.length > LEDGER_MAX_NOTES * 2) {
+                ledger.testIntegrity.splice(0, ledger.testIntegrity.length - LEDGER_MAX_NOTES);
+            }
+        }
+    }
+    else if (toolName === 'bash') {
+        const cmd = String(input?.command ?? '').trim();
+        if (cmd && exports.VERIFY_CMD_RE.test(cmd)) {
+            // Record BOTH outcomes: a FAILED test/build is the single most important
+            // fact to carry across a compaction (it tells the agent work is NOT done).
+            // ok===true means the command exited 0 (executor sets error on non-zero).
+            ledger.verifications.push({ cmd: cmd.slice(0, 120), ok, epoch: ledger.epoch });
+            if (ledger.verifications.length > LEDGER_MAX_NOTES * 2) {
+                ledger.verifications.splice(0, ledger.verifications.length - LEDGER_MAX_NOTES);
+            }
+        }
+    }
+}
+/** Render the ledger as a compact, verbatim block for the compaction preamble. */
+function ledgerSummary(ledger) {
+    const lines = [];
+    if (ledger.filesTouched.size) {
+        const files = [...ledger.filesTouched.entries()];
+        const shown = files.slice(0, LEDGER_MAX_FILES);
+        lines.push(`FILES CHANGED THIS SESSION (${ledger.filesTouched.size}):`);
+        for (const [p, meta] of shown) {
+            lines.push(`  • ${p}  (${meta.tool}${meta.edits > 1 ? ` ×${meta.edits}` : ''})`);
+        }
+        if (files.length > shown.length)
+            lines.push(`  • … and ${files.length - shown.length} more`);
+    }
+    if (ledger.verifications.length) {
+        const recent = ledger.verifications.slice(-LEDGER_MAX_NOTES);
+        lines.push(`VERIFICATION RUNS (most recent ${recent.length}):`);
+        for (const v of recent)
+            lines.push(`  • [${v.ok ? 'PASS' : 'FAIL'}] ${v.cmd}`);
+    }
+    if (ledger.testIntegrity.length) {
+        const recent = ledger.testIntegrity.slice(-LEDGER_MAX_NOTES);
+        lines.push(`⚠ TEST-INTEGRITY ALERTS (test files were weakened — must justify or revert):`);
+        for (const t of recent)
+            lines.push(`  • ${t.path}: ${t.reason}`);
+    }
+    const flaky = (0, flaky_1.detectFlaky)(ledger.verifications);
+    if (flaky.length) {
+        lines.push(`⚠ FLAKY TESTS (same command flipped PASS↔FAIL with no edit between — a green run proves nothing):`);
+        for (const f of flaky.slice(0, LEDGER_MAX_NOTES)) {
+            lines.push(`  • ${f.cmd}  (${f.passes} pass / ${f.fails} fail at identical code)`);
+        }
+    }
+    return lines.join('\n');
+}
+// How many of the most-recent messages keep their tool_result content verbatim.
+// Older tool_result bodies are the bulk of a large body and are the safest thing
+// to shed first (the model has already acted on them), so we replace their content
+// with a short stub while KEEPING the block (so tool_use/tool_result pairing and
+// turn structure stay intact — unlike summarisation, which drops whole turns).
+const PRUNE_KEEP_RECENT = 8;
+const PRUNE_STUB_KEEP_CHARS = 400; // keep a short head of each pruned result for context
+// Marker sentinel appended to a pruned tool_result's content. We detect
+// "already pruned" by this suffix rather than by an out-of-schema field on the
+// block, because the block object is serialised verbatim onto the request body
+// and forwarded to Anthropic — any extra property (e.g. a `_pruned` flag) would
+// be rejected as an unknown field on a content block (400). Encoding the state
+// inside the (string) content keeps the wire payload schema-clean AND idempotent.
+const PRUNE_MARKER = '\n\n[… ';
+const PRUNE_MARKER_TAIL = ' pruned to conserve context. Re-run the tool if you need the full result.]';
+/**
+ * Lossy-but-structure-preserving prune: shrink OLD, large tool_result blocks in
+ * place, keeping the last PRUNE_KEEP_RECENT messages untouched. This is tried
+ * BEFORE summarisation because it:
+ *   • keeps every turn and every tool_use/tool_result pair (API stays valid),
+ *   • never makes an extra model call (summarisation does — cost + latency),
+ *   • degrades gracefully on repeat (summarise-of-summarise loses the most on
+ *     long runs; pruning just trims already-consumed output further).
+ *
+ * IMPORTANT: pruned state is encoded in the content string (PRUNE_MARKER_TAIL
+ * suffix), NOT as an extra property on the block — a stray field on a content
+ * block is rejected by the Anthropic API as an unknown key (400). This keeps the
+ * serialised body schema-clean while remaining idempotent across repeat calls.
+ *
+ * Returns the number of bytes reclaimed (0 if nothing was prunable).
+ */
+function pruneOldToolResults(messages) {
+    const cutoff = messages.length - PRUNE_KEEP_RECENT;
+    if (cutoff <= 1)
+        return 0;
+    let reclaimed = 0;
+    for (let i = 0; i < cutoff; i++) {
+        const m = messages[i];
+        if (!Array.isArray(m.content))
+            continue;
+        for (const b of m.content) {
+            if (b.type !== 'tool_result')
+                continue;
+            const text = typeof b.content === 'string' ? b.content : JSON.stringify(b.content ?? '');
+            if (text.endsWith(PRUNE_MARKER_TAIL))
+                continue; // already pruned (idempotent)
+            if (text.length <= PRUNE_STUB_KEEP_CHARS + 80)
+                continue; // already small
+            const head = text.slice(0, PRUNE_STUB_KEEP_CHARS);
+            const omitted = text.length - head.length;
+            b.content = `${head}${PRUNE_MARKER}${omitted} chars of earlier tool output${PRUNE_MARKER_TAIL}`;
+            reclaimed += omitted;
+        }
+    }
+    return reclaimed;
 }
 /**
  * Compact `messages` in place: summarise everything before a safe cut point and
  * replace it with a summary preamble. Returns true if compaction happened.
  */
-async function autoCompactMessages(messages, options) {
+/** Extract the first user turn's plain text — the ORIGINAL task/goal. */
+function originalTaskText(messages) {
+    const first = messages.find((m) => m.role === 'user');
+    if (!first || !Array.isArray(first.content))
+        return '';
+    return first.content
+        .filter((b) => b.type === 'text' && b.text)
+        .map((b) => b.text)
+        .join('\n')
+        .trim();
+}
+async function autoCompactMessages(messages, options, ledger) {
     const cut = findSafeCutIndex(messages, messages.length - COMPACT_KEEP_MIN);
     if (cut < 2)
         return false; // nothing meaningful to fold
     const toSummarize = messages.slice(0, cut);
     const kept = messages.slice(cut);
+    // Pin the ORIGINAL task verbatim. findSafeCutIndex can (and on a long single
+    // run usually does) cut PAST the first user turn, folding the user's actual
+    // goal into the lossy summary — after a few compactions the agent drifts off
+    // what it was asked to do. We re-inject the first user turn's text verbatim
+    // into the replacement preamble so the objective survives every compaction.
+    // (We cannot keep it as a separate user message: the API requires alternating
+    // roles and kept[0] is already an assistant turn — two user turns would 400.)
+    const originalTask = originalTaskText(toSummarize);
     const summaryPrompt = `Summarize this coding-session transcript into concise bullet points the assistant needs to continue the work: ` +
         `key decisions, files changed (and how), commands run, unresolved problems, and user preferences. Max 400 words.\n\n` +
         transcriptOf(toSummarize);
@@ -488,7 +748,17 @@ async function autoCompactMessages(messages, options) {
     // and no orphaned tool_result is left behind. We intentionally do NOT insert
     // an assistant-ack here: that would put two assistant messages back-to-back
     // (kept[0] is already an assistant), which the API rejects.
-    messages.splice(0, cut, { role: 'user', content: [{ type: 'text', text: `[Auto-compacted ${toSummarize.length} earlier messages]\n\nSummary of the earlier conversation so far:\n${summary}\n\nContinue the work from here.` }] });
+    const taskBlock = originalTask
+        ? `ORIGINAL TASK (verbatim — keep working toward this, do not lose sight of it):\n${originalTask}\n\n`
+        : '';
+    // GAP E — the deterministic ledger (files changed + verification pass/fail) is
+    // injected VERBATIM, so these concrete facts never decay through repeated
+    // summary-of-summary compactions the way the prose summary does.
+    const ledgerText = ledger ? ledgerSummary(ledger) : '';
+    const ledgerBlock = ledgerText
+        ? `PROGRESS LEDGER (authoritative, machine-tracked — trust this over the prose summary for what changed/verified):\n${ledgerText}\n\n`
+        : '';
+    messages.splice(0, cut, { role: 'user', content: [{ type: 'text', text: `[Auto-compacted ${toSummarize.length} earlier messages]\n\n${taskBlock}${ledgerBlock}Summary of the earlier conversation so far:\n${summary}\n\nContinue the work from here.` }] });
     // `kept` follows automatically since splice only replaced the head.
     void kept;
     return true;
@@ -511,6 +781,7 @@ async function runAgentLoop(initialMessages, options) {
     const maxIterations = resolveMaxIterations(options.maxIterations, settings.raw);
     const autoContinue = resolveAutoContinue(options.autoContinue, settings.raw);
     const autoCompact = resolveAutoCompact(options.autoCompact, settings.raw);
+    const verifyNudgeOn = resolveVerificationNudge(settings.raw);
     const contextWindow = MODEL_CONTEXT_TOKENS[model] ?? 200000;
     // Live prompt-size estimate, updated from usage events after every stream.
     let lastPromptTokens = 0;
@@ -532,20 +803,63 @@ async function runAgentLoop(initialMessages, options) {
     let consecutiveErrorRounds = 0; // rounds where every tool call errored
     let budget = maxIterations; // extended by auto-continue, capped at hardCap
     let iteration = 0;
+    // ─── Verification nudge (GAP D) ───────────────────────────────────────────────
+    // Coding agents commonly claim "done" after editing files without ever running a
+    // build/test/lint command to confirm the change actually works (the industry's
+    // unsolved "verification problem" — we can't guarantee correctness, but we CAN
+    // make the agent check its own work when it visibly skipped that step). This is a
+    // single one-shot text nudge, NOT a forced extra model/sub-agent call: cheap, and
+    // the agent can decline it if verification genuinely isn't applicable (e.g. a
+    // docs-only change) since it's a suggestion appended before the turn ends, not a
+    // blocking gate.
+    let filesMutatedSinceVerify = false;
+    let ranVerificationCmd = false;
+    let verificationNudgeSent = false;
+    // Reward-hacking guard: number of test-integrity findings already surfaced,
+    // so the one-shot nudge fires once per NEW batch of weakened-test signals.
+    let testIntegrityNudgedCount = 0;
+    // Flaky-test guard: commands already nudged about, so we only warn about a
+    // newly-detected flaky command once.
+    const flakyNudgedCmds = new Set();
+    // GAP E — deterministic progress ledger, preserved verbatim across compactions.
+    const ledger = createLedger();
     try {
         for (; iteration < budget; iteration++) {
             if (options.abortSignal?.aborted)
                 break;
-            // Auto-compact: if the last request's prompt crossed the threshold,
-            // summarise older history before the next stream so we never hit the
-            // context-window wall mid-task. Runs at a turn boundary only.
-            if (autoCompact && !compacting && lastPromptTokens > contextWindow * AUTO_COMPACT_THRESHOLD && messages.length > COMPACT_KEEP_MIN + 2) {
+            // Auto-compact: summarise older history before the next stream so we never
+            // hit the context-window wall — or the backend body-size limit — mid-task.
+            // Runs at a turn boundary only. Two independent triggers:
+            //   1. TOKEN pressure — the last request's prompt crossed 80% of the window.
+            //   2. BYTE pressure  — the serialised body has grown past MAX_BODY_BYTES.
+            // The byte trigger is what catches tool-heavy runs whose body balloons past
+            // the server's 413 limit while the token count still looks fine (and it fires
+            // even on turn 0 of a resumed large session, where lastPromptTokens is 0).
+            let bodyBytes = estimateBodyBytes(messages);
+            const tokenPressure = lastPromptTokens > contextWindow * AUTO_COMPACT_THRESHOLD;
+            let bytePressure = bodyBytes > MAX_BODY_BYTES;
+            // Byte pressure first tries the CHEAP, structure-preserving prune (no model
+            // call, keeps every turn). Only if that isn't enough do we fall through to
+            // summarisation below. This keeps long runs coherent — summarise-of-summarise
+            // is the main cause of an agent "forgetting" what it did earlier.
+            if (autoCompact && !compacting && bytePressure && messages.length > PRUNE_KEEP_RECENT + 2) {
+                const reclaimed = pruneOldToolResults(messages);
+                if (reclaimed > 0) {
+                    bodyBytes = estimateBodyBytes(messages);
+                    bytePressure = bodyBytes > MAX_BODY_BYTES;
+                    options.onText(`\n\u267b\ufe0f Trimmed ~${(reclaimed / (1024 * 1024)).toFixed(1)}MB of older tool output to conserve context.\n`);
+                }
+            }
+            if (autoCompact && !compacting && (tokenPressure || bytePressure) && messages.length > COMPACT_KEEP_MIN + 2) {
                 compacting = true;
                 try {
-                    const did = await autoCompactMessages(messages, options);
+                    const did = await autoCompactMessages(messages, options, ledger);
                     if (did) {
                         lastPromptTokens = 0; // stale — next usage event refreshes it
-                        options.onText(`\n\u267b\ufe0f Auto-compacted earlier conversation to stay within the context window.\n`);
+                        const reason = bytePressure
+                            ? `body ~${(bodyBytes / (1024 * 1024)).toFixed(1)}MB`
+                            : 'context window';
+                        options.onText(`\n\u267b\ufe0f Auto-compacted earlier conversation to stay within the ${reason}.\n`);
                     }
                 }
                 finally {
@@ -664,6 +978,69 @@ async function runAgentLoop(initialMessages, options) {
                     messages.push({ role: 'user', content: [{ type: 'text', text }] });
                     continue;
                 }
+                // Reward-hacking guard (A4): the agent is about to finish, but it WEAKENED
+                // one or more test files this session (removed assertions, added .skip,
+                // introduced tautologies, commented out cases). Frontier agents let this
+                // pass silently and report "tests pass". We surface every NEW finding once
+                // and require the agent to either justify each change (legit refactor) or
+                // revert it and fix the real code. Deterministic — the signal comes from
+                // diff structure, not model self-report, so it can't be gamed away.
+                if (ledger.testIntegrity.length > testIntegrityNudgedCount) {
+                    const fresh = ledger.testIntegrity.slice(testIntegrityNudgedCount);
+                    testIntegrityNudgedCount = ledger.testIntegrity.length;
+                    const bullet = fresh.map((t) => `  • ${t.path}: ${t.reason}`).join('\n');
+                    messages.push({
+                        role: 'user',
+                        content: [{
+                                type: 'text',
+                                text: 'STOP — test-integrity check. Before finishing, I detected that you WEAKENED test(s) this session:\n' +
+                                    bullet +
+                                    '\n\nMaking tests pass by weakening them (skipping, deleting assertions/cases, or adding always-true checks) is not acceptable — it hides real failures. For EACH item above, either:\n' +
+                                    '  1. Revert the weakening and fix the actual code so the ORIGINAL test passes, or\n' +
+                                    '  2. Justify concretely why the change is a legitimate test refactor (e.g. the behaviour was intentionally removed per the task, a duplicate test, or the assertion was genuinely wrong) — and confirm the remaining tests still meaningfully cover the behaviour.\n' +
+                                    'Then re-run the test suite to prove it passes for real.',
+                            }],
+                    });
+                    continue;
+                }
+                // Flaky-test guard: a command that flipped PASS↔FAIL with no edit between
+                // the differing runs is non-deterministic — a green run of it proves
+                // nothing, and the agent may be (consciously or not) re-running until it
+                // goes green. Warn once per flaky command before letting the run end.
+                {
+                    const flaky = (0, flaky_1.detectFlaky)(ledger.verifications).filter((f) => !flakyNudgedCmds.has(f.cmd));
+                    if (flaky.length) {
+                        for (const f of flaky)
+                            flakyNudgedCmds.add(f.cmd);
+                        const bullet = flaky.map((f) => `  • ${f.cmd}  (${f.passes} pass / ${f.fails} fail on identical code)`).join('\n');
+                        messages.push({
+                            role: 'user',
+                            content: [{
+                                    type: 'text',
+                                    text: 'STOP — flaky-test check. These command(s) produced BOTH a pass and a fail with NO code change in between, so they are non-deterministic:\n' +
+                                        bullet +
+                                        '\n\nA green run of a flaky test proves nothing, and re-running until it passes is not a fix. Do NOT declare success on this basis. Instead: identify the source of non-determinism (unseeded RNG, timing/sleep races, test-ordering or shared-state dependence, wall-clock/timezone, network) and make the test deterministic — or, if it is out of scope, say so explicitly and flag it as pre-existing flakiness rather than treating the passing run as proof.',
+                                }],
+                        });
+                        continue;
+                    }
+                }
+                // One-shot verification nudge (GAP D): the agent is about to declare the task
+                // done, but it edited files this run and never ran a build/test/lint command
+                // to confirm the change works. Ask ONCE — if it still finishes without
+                // verifying (e.g. a docs-only change, or no test suite exists), we respect
+                // that and end normally rather than looping forever on the same nudge.
+                if (verifyNudgeOn && filesMutatedSinceVerify && !ranVerificationCmd && !verificationNudgeSent) {
+                    verificationNudgeSent = true;
+                    messages.push({
+                        role: 'user',
+                        content: [{
+                                type: 'text',
+                                text: 'Before finishing: you edited file(s) in this task but I don\'t see a build/test/lint/typecheck command run to confirm the change works. If this project has one, please run it now and fix any failures. If verification genuinely doesn\'t apply here (e.g. docs-only change, no test suite), just say so and finish.',
+                            }],
+                    });
+                    continue;
+                }
                 runSimpleHooks(hooks.PostMessageComplete, options.workDir);
                 completedCleanly = true;
                 break;
@@ -745,10 +1122,34 @@ async function runAgentLoop(initialMessages, options) {
                         }
                     }
                 }
+                // Capture the raw output (may carry the invisible test-integrity
+                // marker from a write_file overwrite) for the ledger, then strip the
+                // marker so neither the UI (onToolResult) nor the model ever see it.
+                let rawOutput;
+                if (name === 'write_file' && result.output) {
+                    rawOutput = result.output;
+                    result.output = (0, testIntegrity_1.stripTestIntegrityMarker)(result.output);
+                }
                 // Notify caller about result
                 options.onToolResult(name, result);
-                return { block: { ...block, id }, result };
+                return { block: { ...block, id }, result, rawOutput };
             }));
+            // Track whether files were mutated / verified this run, for the one-shot
+            // end-of-task nudge below (GAP D — see declaration above).
+            for (const { block, result, rawOutput } of toolResults) {
+                const ok = result.error === undefined;
+                // GAP E — feed every successful effect into the deterministic ledger.
+                // rawOutput carries the (already-stripped-from-view) test-integrity marker.
+                ledgerRecord(ledger, block.name, block.input, ok, rawOutput ?? result.output);
+                if (!ok)
+                    continue; // failed calls don't count either way
+                if (exports.WRITE_TOOL_NAMES.has(block.name))
+                    filesMutatedSinceVerify = true;
+                else if (block.name === 'bash' && exports.VERIFY_CMD_RE.test(String(block.input?.command ?? ''))) {
+                    ranVerificationCmd = true;
+                    filesMutatedSinceVerify = false; // verified — reset until the next mutation
+                }
+            }
             // 6. Build tool_result message and append to history
             const toolResultBlocks = toolResults.map(({ block, result }) => ({
                 type: 'tool_result',

package/dist/agent/testIntegrity.d.ts

@@ -0,0 +1,62 @@
+/**
+ * Test-Integrity Guard — deterministic reward-hacking detection.
+ *
+ * THE PROBLEM (unsolved by every frontier coding agent today):
+ *   A model asked to "make the tests pass" can satisfy that objective two ways:
+ *     (a) fix the code so the existing tests pass  ← what we want
+ *     (b) weaken the tests so they pass regardless ← reward hacking
+ *   (b) includes: deleting assertions, adding `.skip`/`.only`/`xit`, turning a
+ *   real assertion into a tautology (`assert True`, `expect(true).toBe(true)`),
+ *   or commenting out / deleting whole test cases. The agent then honestly
+ *   reports "tests pass" and the user believes the work is done. Frontier agents
+ *   mitigate this only with prompt instructions ("don't cheat") — there is no
+ *   deterministic detector. METR / Anthropic sabotage evals confirm every strong
+ *   model does this measurably under pressure.
+ *
+ * THE APPROACH:
+ *   Analyse the EDIT PAYLOAD of any write to a test file (old_string→new_string
+ *   for edit_file / multi_edit; content for write_file). Deterministically count
+ *   assertion / test-case / skip-marker deltas. When a test edit's NET effect is
+ *   "fewer or weaker tests" we flag it — with a concrete, human-readable reason.
+ *   No model output is trusted; this is pure structural analysis of the diff, so
+ *   it cannot itself be hallucinated or gamed.
+ *
+ * This is a heuristic signal, not a proof: legitimate refactors (renaming a test,
+ * splitting a file) can reduce counts. So the guard never BLOCKS — it surfaces a
+ * one-shot nudge asking the agent to justify the change, and records it in the
+ * progress ledger so it survives compaction. That converts a silent, invisible
+ * failure into an explicit, reviewable decision.
+ */
+export interface TestEditFinding {
+    /** Human-readable reason the edit looks like test-weakening. */
+    reason: string;
+    /** Coarse signal type for aggregation. */
+    kind: 'skip-added' | 'assertion-removed' | 'tautology-added' | 'testcase-removed' | 'body-commented';
+}
+export interface TestIntegrityResult {
+    isTestFile: boolean;
+    suspicious: boolean;
+    findings: TestEditFinding[];
+}
+export declare function isTestFile(path: string): boolean;
+/**
+ * Analyse a single test-file edit (old fragment → new fragment). For write_file
+ * pass ('', fullNewContent) — only additive signals (skips, tautologies) fire.
+ */
+export declare function analyzeTestEdit(path: string, oldText: string, newText: string): TestIntegrityResult;
+/**
+ * Extract the (old, new) text fragments to analyse from a write-tool input.
+ * Returns null for non-test files or tools we can't inspect.
+ *
+ * - edit_file:  { old_string, new_string }
+ * - multi_edit: concatenate all edits' old / new
+ * - write_file: ('', content) — additive-only detection (no prior content here)
+ */
+export declare function analyzeWriteToolForTestIntegrity(toolName: string, input: Record<string, unknown> | undefined): TestIntegrityResult | null;
+/** Encode findings as an invisible marker to append to a tool's output string. */
+export declare function encodeTestIntegrityMarker(findings: TestEditFinding[]): string;
+/** Extract weakening reasons from a tool output that may carry the marker. */
+export declare function decodeTestIntegrityMarker(output: string | undefined): string[];
+/** Strip the (invisible) marker from output before it is shown to the model. */
+export declare function stripTestIntegrityMarker(output: string | undefined): string;
+//# sourceMappingURL=testIntegrity.d.ts.map

package/dist/agent/testIntegrity.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"testIntegrity.d.ts","sourceRoot":"","sources":["../../src/agent/testIntegrity.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,MAAM,WAAW,eAAe;IAC9B,gEAAgE;IAChE,MAAM,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,IAAI,EAAE,YAAY,GAAG,mBAAmB,GAAG,iBAAiB,GAAG,kBAAkB,GAAG,gBAAgB,CAAC;CACtG;AAED,MAAM,WAAW,mBAAmB;IAClC,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,EAAE,OAAO,CAAC;IACpB,QAAQ,EAAE,eAAe,EAAE,CAAC;CAC7B;AAQD,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAGhD;AA4ED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,mBAAmB,CAmDnG;AAED;;;;;;;GAOG;AACH,wBAAgB,gCAAgC,CAC9C,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,GACzC,mBAAmB,GAAG,IAAI,CA8B5B;AAcD,kFAAkF;AAClF,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,eAAe,EAAE,GAAG,MAAM,CAI7E;AAED,8EAA8E;AAC9E,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,EAAE,CAQ9E;AAED,gFAAgF;AAChF,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAQ3E"}