@inflexa-ai/harness 0.39.2 → 0.40.0

package/dist/memory/thread-history.js

@@ -3,8 +3,9 @@
  *
  * Conversation-scoped store with an explicit two-method interface and
  * nothing else bundled in (no semantic recall, no working memory, no title
- * generation). It is also the home of context-window management — the read
- * side, `loadRecent`, returns a token-bounded window.
+ * generation). The read side, `loadRecent`, gives the view of the latest
+ * compaction marker: the stored markers decide what a turn sends, and the store
+ * deletes no row.
  *
  * Scope (see the harness-thread-store spec): conversation threads only. A `threadId` is the
  * UI-generated id of one conversation thread (a random UUID — an analysis
@@ -14,11 +15,10 @@
  * conversation-turn shaped on purpose, so reaching for it inside a
  * workflow step feels immediately wrong.
  *
- * The window is always a valid AI SDK model-message sequence: it begins on a
- * `user` message that is genuine user input — never a `tool`-role
- * continuation — and never splits a tool-call/tool-result pair. The turn
- * is the atomic unit; `appendTurn` writes one atomically and `loadRecent`
- * rounds the budget walk to turn boundaries.
+ * The view is always a valid AI SDK model-message sequence: it begins on a
+ * `user` message — never a `tool`-role continuation — and never splits a
+ * tool-call/tool-result pair. A compaction starts only between two rounds, a
+ * drop keeps whole turns, and each write of a group is atomic.
  */
 import { randomUUID } from "node:crypto";
 import { metrics } from "@opentelemetry/api";
@@ -27,7 +27,8 @@ import { stripNulCharacters } from "../input-sanitization.js";
 import { tryMutation, tryQuery, withTransaction } from "../lib/db-result.js";
 import { hasReportedUsage } from "../loop/metrics.js";
 import { countTokens } from "./count-tokens.js";
-import { HARNESS_PROVIDER_NAMESPACE, SYNTHETIC_MESSAGE_KEY, envelopeMessage, isSyntheticUserMessage, parseStoredMessageEnvelope, syntheticRecordMessage, } from "./ai-sdk-message-storage.js";
+import { conversationView, groupTurns, isGenuineUserStart } from "./conversation-view.js";
+import { HARNESS_PROVIDER_NAMESPACE, SYNTHETIC_MESSAGE_KEY, envelopeMessage, parseStoredMessageEnvelope, syntheticRecordMessage, } from "./ai-sdk-message-storage.js";
 import { envelopeDisplayMessages, parseStoredDisplayEnvelope } from "./conversation-display-storage.js";
 /** A resolved `ok(undefined)` ResultAsync — the empty/seed transaction step. */
 function okVoid() {
@@ -51,34 +52,8 @@ export function conversationRecordTurn(text) {
     };
 }
 /**
- * A turn starts on a `user` message that a human actually sent. Two kinds of
- * message carry the `user` role without opening a turn, and both are excluded
- * here:
- *
- * - a tool result, which in AI SDK terms is a `tool`-role message, so a mid-turn
- *   tool continuation never matches in the first place;
- * - a message the LOOP or the HOST synthesized (`syntheticUserMessage`) — the
- *   truncated-reply nudge the loop inserts mid-turn, and the record an embedder
- *   appends between turns for work that happened outside the conversation, such as
- *   an analysis run's outcome. Both carry the `user` role for the wire format and
- *   neither is user input. Reading one as a boundary would split one turn into
- *   two: the token window would evict half a turn, and `retractLastTurn` would cut
- *   its tail in the middle of a turn rather than at its head.
- *
- *   A host-appended record therefore belongs to the turn preceding it, so a tail
- *   retraction that removes that turn removes the record with it. That is the
- *   accepted consequence of the exclusion, not a defect — the alternative, letting
- *   it open a turn, is the exact failure the marker exists to prevent.
- *
- * {@link GENUINE_USER_START_SQL} is the twin of this predicate over stored
- * envelopes; the two are built from the same constants so they cannot drift.
- */
-function isGenuineUserStart(message) {
-    return message.role === "user" && !isSyntheticUserMessage(message);
-}
-/**
- * {@link isGenuineUserStart} expressed over a stored `message_envelope` — the
- * boundary predicate `retractLastTurn` cuts on.
+ * {@link isGenuineUserStart} (`conversation-view.ts`) expressed over a stored
+ * `message_envelope` — the boundary predicate `retractLastTurn` cuts on.
  *
  * Interpolating here, and only here, is safe: both interpolated values are
  * module constants shared with the TypeScript predicate, never caller input, so
@@ -105,35 +80,17 @@ const GENUINE_USER_START_SQL = `message_envelope->'message'->>'role' = 'user'
 function serializeEnvelope(message) {
     return JSON.stringify(envelopeMessage(message), (_key, value) => (typeof value === "string" ? stripNulCharacters(value) : value));
 }
-/**
- * Group rows (oldest-first) into turns at genuine-user-start boundaries.
- * Generic over the row shape so the token-windowed read (`loadRecent`) and
- * the display read (`loadAll`) share it, each supplying its own start
- * predicate.
- */
-function groupTurns(rows, isStart) {
-    const turns = [];
-    for (const row of rows) {
-        if (turns.length === 0 || isStart(row)) {
-            turns.push([row]);
-        }
-        else {
-            turns[turns.length - 1].push(row);
-        }
-    }
-    return turns;
-}
 let instruments;
 function getInstruments() {
     if (instruments === undefined) {
         const meter = metrics.getMeter("cortex.harness.memory");
         instruments = {
             totalTokens: meter.createHistogram("cortex.harness.thread.total_tokens", {
-                description: "Total token count of a conversation thread, sampled on every loadRecent",
+                description: "Total estimated token count of the rows of a conversation thread, sampled on every loadRecent",
                 unit: "{token}",
             }),
             turnsEvicted: meter.createHistogram("cortex.harness.thread.turns_evicted", {
-                description: "Conversation turns dropped by loadRecent's token-budget window",
+                description: "Conversation turns of which the view of loadRecent holds no message",
                 unit: "{turn}",
             }),
         };
@@ -148,27 +105,115 @@ export function __resetThreadHistoryMetricsForTest() {
     instruments = undefined;
 }
 /**
- * Chunked-eviction block size, in whole turns — the granularity at which
- * `loadRecent`'s retained window may advance its START turn.
- *
- * `loadRecent` snaps its eviction count UP to a multiple of this block, so the
- * window's first turn — hence `messages[0]` and the whole tools+system+history
- * prompt-cache prefix — stays byte-identical across a run of appends and shifts
- * only once per block instead of once per turn. That single shift is the only
- * message-cache miss the block costs.
- *
- * The block is at once (a) the cache-miss cadence — one prefix shift per
- * `EVICTION_BLOCK_TURNS` appends once a thread is over budget — and (b) the most
- * extra context ever sacrificed: snapping up drops up to `EVICTION_BLOCK_TURNS -
- * 1` oldest turns the budget alone would have kept, trading a little history for
- * a still prefix. 4 holds the prefix for ~4 turns at a cost of at most 3 turns of
- * headroom — negligible against a ~120k-token budget that spans far more turns,
- * while cutting cache misses on a long thread roughly fourfold. A turn-count
- * block (not a token block) is deliberate: a cache miss is triggered by the START
- * turn moving, a per-turn event, so counting in turns makes the miss cadence
- * exact and independent of how large any individual turn is.
+ * Serialize the writes of one thread — without the lock, two transactions can
+ * both read the same MAX(seq) and collide on the (thread_id, seq) primary key.
+ * Released automatically at COMMIT/ROLLBACK.
  */
-export const EVICTION_BLOCK_TURNS = 4;
+function lockThread(client, threadId, op) {
+    return tryQuery(`${op}.lock`, () => client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [threadId])).map(() => undefined);
+}
+/** The `seq` of the next row of the thread. Read it under {@link lockThread}. */
+function nextSeq(client, threadId, op) {
+    return tryQuery(`${op}.maxSeq`, async () => {
+        const { rows } = await client.query("SELECT COALESCE(MAX(seq), -1)::text AS max_seq FROM messages WHERE thread_id = $1", [
+            threadId,
+        ]);
+        return Number(rows[0].max_seq) + 1;
+    });
+}
+/** Insert the rows of one group, from `startSeq` on. */
+function insertGroup(client, threadId, startSeq, group, op) {
+    const { modelMessages: messages, displayMessages, author } = group;
+    // The display projection rides the group's FIRST row, so one SELECT of a
+    // thread's rows yields every envelope in order with no join and no grouping.
+    // Whichever row that is — a genuine user message opening a turn, or the lone
+    // record row of an out-of-band append — is the row a tail retraction removes
+    // the envelope with, because it is the row the projection describes.
+    const displayEnvelope = displayMessages.length > 0 ? JSON.stringify(envelopeDisplayMessages(displayMessages)) : null;
+    // The author rides the row the display projection rides, and only when that
+    // row is a message a person actually sent — an assistant row, a tool row,
+    // and the synthetic opening row of a record append are not. NUL is dropped
+    // rather than refused: a 0x00 byte in a `text` parameter fails the
+    // statement, and one byte in a label would roll the whole turn back.
+    const turnAuthor = author === undefined ? null : stripNulCharacters(author);
+    const authorRow = messages[0] !== undefined && isGenuineUserStart(messages[0]) ? 0 : -1;
+    // Insert the group's messages in order. Each insert chains off the
+    // prior so the first `err` short-circuits — and `withTransaction`
+    // re-throws it to force ROLLBACK (a returned `err` that does not
+    // reach `withTransaction` would COMMIT silently).
+    return messages.reduce((chain, message, i) => chain.andThen(() => tryMutation(`${op}.insert`, () => client.query(
+    // `message_envelope::json`, never `::jsonb`; `display_envelope::jsonb`,
+    // never `::json` — see the column comments in the state-init DDL.
+    `INSERT INTO messages (thread_id, seq, message_envelope, display_envelope, tokens, author)
+                     VALUES ($1, $2, $3::json, $4::jsonb, $5, $6)
+                     ON CONFLICT (thread_id, seq) DO UPDATE
+                       SET message_envelope = EXCLUDED.message_envelope,
+                           display_envelope = EXCLUDED.display_envelope,
+                           tokens = EXCLUDED.tokens,
+                           author = EXCLUDED.author`, [
+        threadId,
+        startSeq + i,
+        serializeEnvelope(message),
+        i === 0 ? displayEnvelope : null,
+        countTokens(message.content),
+        i === authorRow ? turnAuthor : null,
+    ])).map(() => undefined)), okVoid());
+}
+/** Move `cortex_analysis_threads.updated_at` forward inside the write transaction. */
+function touchThread(client, threadId, op) {
+    // Thread listings sort on `cortex_analysis_threads.updated_at`, and
+    // the only other writer of it is the title update — so without this
+    // touch "most recently updated" degrades to "most recently created
+    // or renamed" and an actively-used older thread sorts last. Writing
+    // it from here (a row the thread store otherwise owns) buys the
+    // guarantee for every host with no wiring, and inside the turn's own
+    // transaction there is no window where the rows exist but the thread
+    // reads stale.
+    //
+    // The breadcrumb is never worth the turn: the touch may fail without
+    // failing the append, and affecting zero rows is equally normal — a
+    // thread with no metadata row, or a soft-deleted one this leaves
+    // alone rather than reviving (every other writer of the table filters
+    // the tombstone too). Tolerating the failure takes the savepoint, not
+    // just the swallowed `err`: Postgres poisons a transaction at its
+    // first failed statement and downgrades the eventual COMMIT to a
+    // ROLLBACK — without a rewind point the turn's inserts would go with
+    // it, and silently, since that COMMIT still reports success.
+    //
+    // `GREATEST(updated_at, clock_timestamp())` because `NOW()` is
+    // transaction-START time while this transaction has since waited on
+    // the advisory lock and spent a round trip per message: a title update
+    // that began later can already have stamped a newer `updated_at`, and
+    // a plain assignment would rewind it. Activity only ever moves the
+    // timestamp forward.
+    return tryMutation(`${op}.touchThread.savepoint`, () => client.query("SAVEPOINT touch_thread"))
+        .andThen(() => tryMutation(`${op}.touchThread`, () => client.query(`UPDATE cortex_analysis_threads
+                        SET updated_at = GREATEST(updated_at, clock_timestamp())
+                      WHERE thread_id = $1 AND deleted_at IS NULL`, [threadId])))
+        .map(() => undefined)
+        .orElse(() => 
+    // Rewind to before the touch so the transaction is usable again
+    // and the write commits. Nothing follows in the chain of a caller,
+    // so COMMIT releases the savepoint on the success path.
+    tryMutation(`${op}.touchThread.rewind`, () => client.query("ROLLBACK TO SAVEPOINT touch_thread"))
+        .map(() => undefined)
+        .orElse(() => okVoid()));
+}
+/** Add the `open` record of a turn. The upsert replaces a record whose rows are gone, because the opening row holds its key now. */
+function openTurnRecord(client, threadId, startSeq) {
+    return tryMutation("thread-history.writeTurn.openRecord", () => client.query(`INSERT INTO cortex_thread_turns (thread_id, start_seq, status)
+             VALUES ($1, $2, 'open')
+             ON CONFLICT (thread_id, start_seq) DO UPDATE
+               SET status = 'open', reason = NULL, reported_usage = NULL, turn_duration_ms = NULL, opened_at = NOW(), closed_at = NULL`, [threadId, startSeq])).map(() => undefined);
+}
+/** Set the outcome of a turn record that is still `open`, thus a second close changes nothing. */
+function closeTurnRecord(client, threadId, startSeq, close) {
+    // The predicate of the loop decides whether a rollup reported a quantity, thus the two cannot drift.
+    const rollup = hasReportedUsage(close.turnUsage) ? JSON.stringify(close.turnUsage) : null;
+    return tryMutation("thread-history.writeTurn.closeRecord", () => client.query(`UPDATE cortex_thread_turns
+                SET status = $3, reason = $4, reported_usage = $5::jsonb, turn_duration_ms = $6, closed_at = NOW()
+              WHERE thread_id = $1 AND start_seq = $2 AND status = 'open'`, [threadId, startSeq, close.status, close.reason === undefined ? null : stripNulCharacters(close.reason), rollup, close.turnDurationMs ?? null])).map(() => undefined);
+}
 /**
  * Create a `ThreadHistory` bound to a Postgres pool — a factory closure
  * capturing `pool` (dependency injection per the harness-durable-runtime spec). The `messages` table is
@@ -181,114 +226,36 @@ export const EVICTION_BLOCK_TURNS = 4;
  */
 export function createThreadHistory(pool, logger) {
     function appendTurn(threadId, turn) {
-        const { modelMessages: messages, displayMessages, turnUsage, turnDurationMs, author } = turn;
-        if (messages.length === 0)
+        if (turn.modelMessages.length === 0)
             return okVoid();
-        // The display projection rides the append's FIRST row, so one SELECT of a
-        // thread's rows yields every envelope in order with no join and no grouping.
-        // Whichever row that is — a genuine user message opening a turn, or the lone
-        // record row of an out-of-band append — is the row a tail retraction removes
-        // the envelope with, because it is the row the projection describes.
-        const displayEnvelope = displayMessages.length > 0 ? JSON.stringify(envelopeDisplayMessages(displayMessages)) : null;
-        // A rollup that reports no quantity is stored as absent, not as a rollup of
-        // absences, so "no figure" has exactly one representation in storage.
-        // `hasReportedUsage` is the loop's own predicate for the same question,
-        // reused rather than restated so the write and the loop cannot drift.
-        const rollup = hasReportedUsage(turnUsage) ? JSON.stringify(turnUsage) : null;
-        // The duration takes no predicate of its own, and it keeps its own column.
-        // A measured zero is a figure, thus only an absent value means that nobody
-        // measured the turn. The rollup goes absent under its own predicate above,
-        // and the duration does not go with it.
-        const duration = turnDurationMs ?? null;
-        // The two figures describe the turn, and the assistant reply is the row a
-        // reader associates with the answer. The LAST assistant row, not any of them: a
-        // serial-tool turn writes one assistant row per step and only the last ends
-        // the turn. -1 (a turn that persisted no reply — an abort before any output)
-        // matches no index below, so nothing is written and the append still succeeds.
-        const lastAssistantRow = messages.reduce((last, m, i) => (m.role === "assistant" ? i : last), -1);
-        // The author rides the row the display projection rides, and only when that
-        // row is a message a person actually sent — an assistant row, a tool row,
-        // and the synthetic opening row of a record append are not. NUL is dropped
-        // rather than refused: a 0x00 byte in a `text` parameter fails the
-        // statement, and one byte in a label would roll the whole turn back.
-        const turnAuthor = author === undefined ? null : stripNulCharacters(author);
-        const authorRow = messages[0] !== undefined && isGenuineUserStart(messages[0]) ? 0 : -1;
-        return withTransaction(pool, "thread-history.appendTurn", (client) => 
-        // Serialize concurrent appends on this thread — without the lock, two
-        // transactions can both read the same MAX(seq) and collide on the
-        // (thread_id, seq) primary key. Released automatically at COMMIT/ROLLBACK.
-        tryQuery("thread-history.appendTurn.lock", () => client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [threadId]))
-            .andThen(() => tryQuery("thread-history.appendTurn.maxSeq", async () => {
-            const { rows } = await client.query("SELECT COALESCE(MAX(seq), -1)::text AS max_seq FROM messages WHERE thread_id = $1", [threadId]);
-            return Number(rows[0].max_seq) + 1;
-        }))
-            .andThen((startSeq) => 
-        // Insert the turn's messages in order. Each insert chains off the
-        // prior so the first `err` short-circuits — and `withTransaction`
-        // re-throws it to force ROLLBACK (a returned `err` that does not
-        // reach `withTransaction` would COMMIT silently).
-        messages.reduce((chain, message, i) => chain.andThen(() => tryMutation("thread-history.appendTurn.insert", () => client.query(
-        // `message_envelope::json`, never `::jsonb`; `display_envelope` and
-        // `reported_usage::jsonb`, never `::json` — see the column comments
-        // in the state-init DDL.
-        `INSERT INTO messages (thread_id, seq, message_envelope, display_envelope, tokens, reported_usage, turn_duration_ms, author)
-                     VALUES ($1, $2, $3::json, $4::jsonb, $5, $6::jsonb, $7, $8)
-                     ON CONFLICT (thread_id, seq) DO UPDATE
-                       SET message_envelope = EXCLUDED.message_envelope,
-                           display_envelope = EXCLUDED.display_envelope,
-                           tokens = EXCLUDED.tokens,
-                           reported_usage = EXCLUDED.reported_usage,
-                           turn_duration_ms = EXCLUDED.turn_duration_ms,
-                           author = EXCLUDED.author`, [
-            threadId,
-            startSeq + i,
-            serializeEnvelope(message),
-            i === 0 ? displayEnvelope : null,
-            countTokens(message.content),
-            i === lastAssistantRow ? rollup : null,
-            i === lastAssistantRow ? duration : null,
-            i === authorRow ? turnAuthor : null,
-        ])).map(() => undefined)), okVoid()))
-            .andThen(() => 
-        // Thread listings sort on `cortex_analysis_threads.updated_at`, and
-        // the only other writer of it is the title update — so without this
-        // touch "most recently updated" degrades to "most recently created
-        // or renamed" and an actively-used older thread sorts last. Writing
-        // it from here (a row the thread store otherwise owns) buys the
-        // guarantee for every host with no wiring, and inside the turn's own
-        // transaction there is no window where the rows exist but the thread
-        // reads stale.
-        //
-        // The breadcrumb is never worth the turn: the touch may fail without
-        // failing the append, and affecting zero rows is equally normal — a
-        // thread with no metadata row, or a soft-deleted one this leaves
-        // alone rather than reviving (every other writer of the table filters
-        // the tombstone too). Tolerating the failure takes the savepoint, not
-        // just the swallowed `err`: Postgres poisons a transaction at its
-        // first failed statement and downgrades the eventual COMMIT to a
-        // ROLLBACK — without a rewind point the turn's inserts would go with
-        // it, and silently, since that COMMIT still reports success.
-        //
-        // `GREATEST(updated_at, clock_timestamp())` because `NOW()` is
-        // transaction-START time while this transaction has since waited on
-        // the advisory lock and spent a round trip per message: a title update
-        // that began later can already have stamped a newer `updated_at`, and
-        // a plain assignment would rewind it. Activity only ever moves the
-        // timestamp forward.
-        tryMutation("thread-history.appendTurn.touchThread.savepoint", () => client.query("SAVEPOINT touch_thread"))
-            .andThen(() => tryMutation("thread-history.appendTurn.touchThread", () => client.query(`UPDATE cortex_analysis_threads
-                        SET updated_at = GREATEST(updated_at, clock_timestamp())
-                      WHERE thread_id = $1 AND deleted_at IS NULL`, [threadId])))
-            .map(() => undefined)
-            .orElse(() => 
-        // Rewind to before the touch so the transaction is usable again
-        // and the turn commits. Nothing follows in this chain, so COMMIT
-        // releases the savepoint on the success path.
-        tryMutation("thread-history.appendTurn.touchThread.rewind", () => client.query("ROLLBACK TO SAVEPOINT touch_thread"))
-            .map(() => undefined)
-            .orElse(() => okVoid()))));
+        return withTransaction(pool, "thread-history.appendTurn", (client) => lockThread(client, threadId, "thread-history.appendTurn")
+            .andThen(() => nextSeq(client, threadId, "thread-history.appendTurn"))
+            .andThen((startSeq) => insertGroup(client, threadId, startSeq, turn, "thread-history.appendTurn"))
+            .andThen(() => touchThread(client, threadId, "thread-history.appendTurn")));
     }
-    function loadRecent(threadId, tokenBudget, options) {
+    function writeTurn(threadId, write) {
+        const { close } = write;
+        const groups = [...(write.opening === undefined ? [] : [write.opening]), ...write.rounds, ...(close?.note === undefined ? [] : [close.note])];
+        return withTransaction(pool, "thread-history.writeTurn", (client) => lockThread(client, threadId, "thread-history.writeTurn")
+            .andThen(() => nextSeq(client, threadId, "thread-history.writeTurn"))
+            .andThen((firstSeq) => {
+            const startSeq = write.opening === undefined ? write.startSeq : firstSeq;
+            const steps = [];
+            let seq = firstSeq;
+            for (const [index, group] of groups.entries()) {
+                const groupSeq = seq;
+                seq += group.modelMessages.length;
+                steps.push(() => insertGroup(client, threadId, groupSeq, group, "thread-history.writeTurn"));
+                if (index === 0 && write.opening !== undefined)
+                    steps.push(() => openTurnRecord(client, threadId, startSeq));
+            }
+            if (close !== undefined)
+                steps.push(() => closeTurnRecord(client, threadId, startSeq, close));
+            steps.push(() => touchThread(client, threadId, "thread-history.writeTurn"));
+            return steps.reduce((chain, step) => chain.andThen(step), okVoid()).map(() => ({ startSeq }));
+        }));
+    }
+    function loadRecent(threadId, options) {
         return tryQuery("thread-history.loadRecent", async () => {
             const { rows } = await pool.query(
             // ORDER BY must qualify `messages.seq` — a bare `seq` would bind to the
@@ -297,72 +264,26 @@ export function createThreadHistory(pool, logger) {
             // "10" before "2". Scrambled order splits a tool-call/tool-result pair
             // across an intervening turn. The qualified name forces the bigint column.
             //
-            // `reported_usage` is deliberately NOT selected. Windowing budgets on
-            // `tokens` — an offline estimate every row carries — and a turn that
-            // reported nothing has no rollup at all, so budgeting on the rollup
-            // would silently stop evicting on exactly those threads. The two are
-            // different measurements sharing a unit; see the column comments.
+            // `reported_usage` is deliberately NOT selected. The metric reads the
+            // `tokens` estimate that every row carries, and a turn that reported
+            // nothing has no rollup at all. The two are different measurements
+            // sharing a unit; see the column comments.
             `SELECT seq::text AS seq, message_envelope, tokens
          FROM messages WHERE thread_id = $1 ORDER BY messages.seq ASC`, [threadId]);
-            const parsed = rows.map((row) => ({
+            const parsed = rows.map((row, index) => ({
+                index,
                 message: parseStoredMessageEnvelope(row.message_envelope, `${threadId}/${row.seq}`).message,
                 tokens: row.tokens,
             }));
+            const view = conversationView(parsed.map((row) => row.message), options ?? {});
+            const held = new Set(view.sources);
             const turns = groupTurns(parsed, (row) => isGenuineUserStart(row.message));
-            const turnTokens = turns.map((turn) => turn.reduce((sum, row) => sum + row.tokens, 0));
-            const threadTotal = turnTokens.reduce((sum, n) => sum + n, 0);
-            // Chunked eviction holds the prompt-cache prefix still. The budget alone
-            // evicts the OLDEST turns newest-first, so the window START — the stable
-            // head of the tools+system+history cache prefix — advances by ~one turn
-            // per appended turn; every turn then ships a shifted `messages[0]` and
-            // rewrites the whole message cache instead of reading it back (watch
-            // `cortex.harness.agent.cache_read_tokens` collapse on long threads).
-            // Instead, take the budget-minimal eviction and snap it UP to a whole
-            // `EVICTION_BLOCK_TURNS` block: `evicted` stays constant while the minimum
-            // sits inside a block and jumps by a block when it crosses, so
-            // `messages[0]` is byte-identical for a block of appends (the cache reads
-            // survive) and shifts only once per block. Snapping UP retains a subset of
-            // the budget-minimal window, so the kept tokens never exceed `tokenBudget`.
-            //
-            // Budget-minimal fill: walk newest-first, always keeping the most recent
-            // turn (a valid over-budget single turn beats an under-budget cut), then
-            // add older turns while they fit.
-            let included = 0;
-            let cumulative = 0;
-            for (let t = turns.length - 1; t >= 0; t--) {
-                if (included > 0 && cumulative + turnTokens[t] > tokenBudget)
-                    break;
-                cumulative += turnTokens[t];
-                included++;
-            }
-            const minimalEvicted = turns.length - included;
-            // Snap the eviction count up to a block boundary, clamped below the last
-            // turn so the most recent turn always ships. Both terms of the min are
-            // >= `minimalEvicted`, so `turnsEvicted >= minimalEvicted`: the retained
-            // suffix is a subset of the budget-minimal one and stays within budget.
-            const turnsEvicted = turns.length === 0 ? 0 : Math.min(Math.ceil(minimalEvicted / EVICTION_BLOCK_TURNS) * EVICTION_BLOCK_TURNS, turns.length - 1);
-            // The retained head. The first turn of a report session carries the
-            // brief and the copy of the working-memory render, and no tail message
-            // replaces them. Thus `keepFirstTurn` puts that turn in front of the
-            // retained suffix, and `messages[0]` never moves again. An eviction of
-            // zero already holds the first turn at the head, thus the option adds
-            // nothing there and the window never carries the turn two times.
-            const keepsFirstTurn = options?.keepFirstTurn === true && turnsEvicted > 0;
-            const { totalTokens, turnsEvicted: turnsEvictedHist } = getInstruments();
-            // A retained head gives one evicted turn back to the window, thus the
-            // histogram reports one turn less than the budget walk cut.
-            const turnsDropped = keepsFirstTurn ? turnsEvicted - 1 : turnsEvicted;
-            // The flag reads the cut of the budget walk, and never the count that
-            // the retained head reduced. The two answer different questions: the
-            // flag says that the window lost a turn, and the histogram says how
-            // many. A walk that cut one block, under a block size of one, drops
-            // its whole cut back into the window. The flag must still report the
-            // eviction, because the thread crossed its budget.
-            const attributes = { eviction: turnsEvicted > 0 };
-            totalTokens.record(threadTotal, attributes);
-            turnsEvictedHist.record(turnsDropped, attributes);
-            const retained = turns.slice(turnsEvicted);
-            return (keepsFirstTurn ? [turns[0], ...retained] : retained).flat().map((row) => row.message);
+            const turnsLeftOut = turns.filter((turn) => !turn.some((row) => held.has(row.index))).length;
+            const { totalTokens, turnsEvicted } = getInstruments();
+            const attributes = { eviction: turnsLeftOut > 0 };
+            totalTokens.record(parsed.reduce((sum, row) => sum + row.tokens, 0), attributes);
+            turnsEvicted.record(turnsLeftOut, attributes);
+            return view.messages;
         });
     }
     // Display reads are turn-bounded, not row-bounded. A serial-tool assistant
@@ -377,8 +298,13 @@ export function createThreadHistory(pool, logger) {
         // `seq::text AS seq` output alias (Postgres resolves an unqualified
         // ORDER BY name to the output column), sorting the bigint as text:
         // "10" before "2". The qualified name forces the bigint column.
-        `SELECT seq::text AS seq, message_envelope, display_envelope, reported_usage, turn_duration_ms::text AS turn_duration_ms, author, created_at
-         FROM messages WHERE thread_id = $1
+        `SELECT messages.seq::text AS seq, messages.message_envelope, messages.display_envelope, messages.reported_usage,
+                messages.turn_duration_ms::text AS turn_duration_ms, messages.author, messages.created_at,
+                turns.status AS turn_status, turns.reason AS turn_reason, turns.reported_usage AS turn_usage,
+                turns.turn_duration_ms::text AS turn_record_duration_ms
+         FROM messages
+         LEFT JOIN cortex_thread_turns turns ON turns.thread_id = messages.thread_id AND turns.start_seq = messages.seq
+         WHERE messages.thread_id = $1
          ORDER BY messages.seq ASC`, [threadId]);
         // Spread rather than `usage: r.reported_usage ?? undefined`, so a row with
         // no rollup carries no `usage` KEY at all — absent, not present-and-undefined.
@@ -388,6 +314,14 @@ export function createThreadHistory(pool, logger) {
         const stored = await Promise.all(rows.map(async (r) => {
             const envelope = parseStoredMessageEnvelope(r.message_envelope, `${threadId}/${r.seq}`);
             const displayEnvelope = r.display_envelope == null ? undefined : await parseStoredDisplayEnvelope(r.display_envelope, `${threadId}/${r.seq}/display`, logger);
+            const turn = r.turn_status === null
+                ? undefined
+                : {
+                    status: r.turn_status,
+                    ...(r.turn_reason === null ? {} : { reason: r.turn_reason }),
+                    ...(r.turn_usage === null ? {} : { usage: r.turn_usage }),
+                    ...(r.turn_record_duration_ms === null ? {} : { durationMs: Number(r.turn_record_duration_ms) }),
+                };
             return {
                 seq: Number(r.seq),
                 envelope,
@@ -395,6 +329,7 @@ export function createThreadHistory(pool, logger) {
                 ...(displayEnvelope ? { displayEnvelope } : {}),
                 ...(r.reported_usage === null ? {} : { usage: r.reported_usage }),
                 ...(r.turn_duration_ms === null ? {} : { durationMs: Number(r.turn_duration_ms) }),
+                ...(turn === undefined ? {} : { turn }),
                 ...(r.author === null ? {} : { author: r.author }),
                 createdAt: r.created_at,
             };
@@ -444,7 +379,7 @@ export function createThreadHistory(pool, logger) {
             // Delete the whole tail turn: every row at or past the boundary seq.
             // `$2::bigint` compares the bigint column against a bigint, never a
             // text projection, keeping the comparison exact for large seqs.
-            return tryMutation("thread-history.retractLastTurn.delete", () => client.query("DELETE FROM messages WHERE thread_id = $1 AND seq >= $2::bigint", [threadId, boundary])).map((res) => ({ kind: "retracted", messages: res.rowCount ?? 0 }));
+            return tryMutation("thread-history.retractLastTurn.delete", () => client.query("DELETE FROM messages WHERE thread_id = $1 AND seq >= $2::bigint", [threadId, boundary])).andThen((res) => tryMutation("thread-history.retractLastTurn.deleteTurnRecords", () => client.query("DELETE FROM cortex_thread_turns WHERE thread_id = $1 AND start_seq >= $2::bigint", [threadId, boundary])).map(() => ({ kind: "retracted", messages: res.rowCount ?? 0 })));
         }));
     }
     function latestSeq(threadId) {
@@ -481,5 +416,5 @@ export function createThreadHistory(pool, logger) {
                 AND seq > $2::bigint
                 AND ${GENUINE_USER_START_SQL}`, [threadId, seq])).map(({ rows }) => Number(rows[0].turns));
     }
-    return { appendTurn, loadRecent, loadAll, retractLastTurn, latestSeq, latestTurnAt, countUserTurnsAfter };
+    return { appendTurn, writeTurn, loadRecent, loadAll, retractLastTurn, latestSeq, latestTurnAt, countUserTurnsAfter };
 }

package/dist/memory/thread-store.d.ts

@@ -22,8 +22,8 @@
  * row and the thread reads as it did; a listing widened with `includeArchived`
  * is the only way to obtain the id it takes, so the recovery is reachable by a
  * host that holds no thread ids it obtained elsewhere.
- * `purgeThread` is not recoverable: it removes the `messages` rows, the report
- * session-state rows, and the metadata rows of the whole subtree in one
+ * `purgeThread` is not recoverable: it removes the `messages` rows, the turn
+ * records, the report session-state rows, the kept tool outputs, and the metadata rows of the whole subtree in one
  * transaction, so a failure partway leaves them all — never a thread stripped of
  * its transcript, nor a transcript with nothing naming it. It is the thread-scoped member of this package's
  * reclamation vocabulary — `purgeAnalysis` reclaims an analysis's whole
@@ -222,7 +222,7 @@ export interface ThreadStore {
     /**
      * Reclaim the subtree's whole footprint: the named thread, every descendant
      * reachable through `parent_thread_id` at any depth, and the `messages` rows
-     * of every one of them, removed together in one transaction. Unrecoverable —
+     * and the turn records of every one of them, removed together in one transaction. Unrecoverable —
      * nothing survives, and no tombstone marks that any of it existed. A
      * `thread_id` with no row succeeds as a no-op.
      *

package/dist/memory/thread-store.js

@@ -22,8 +22,8 @@
  * row and the thread reads as it did; a listing widened with `includeArchived`
  * is the only way to obtain the id it takes, so the recovery is reachable by a
  * host that holds no thread ids it obtained elsewhere.
- * `purgeThread` is not recoverable: it removes the `messages` rows, the report
- * session-state rows, and the metadata rows of the whole subtree in one
+ * `purgeThread` is not recoverable: it removes the `messages` rows, the turn
+ * records, the report session-state rows, the kept tool outputs, and the metadata rows of the whole subtree in one
  * transaction, so a failure partway leaves them all — never a thread stripped of
  * its transcript, nor a transcript with nothing naming it. It is the thread-scoped member of this package's
  * reclamation vocabulary — `purgeAnalysis` reclaims an analysis's whole
@@ -295,8 +295,13 @@ export function createThreadStore(pool) {
         // set that can still change, and after it the rows are gone.
         return withTransaction(pool, "thread-store.purgeThread", (client) => tryMutation("thread-store.purgeThread.messages", () => client.query(`${SUBTREE_CTE}
          DELETE FROM messages WHERE thread_id IN (SELECT thread_id FROM subtree)`, [threadId]))
+            .andThen(() => tryMutation("thread-store.purgeThread.turnRecords", () => client.query(`${SUBTREE_CTE}
+         DELETE FROM cortex_thread_turns WHERE thread_id IN (SELECT thread_id FROM subtree)`, [threadId])))
             .andThen(() => tryMutation("thread-store.purgeThread.reportSessionState", () => client.query(`${SUBTREE_CTE}
          DELETE FROM cortex_report_session_state WHERE thread_id IN (SELECT thread_id FROM subtree)`, [threadId])))
+            // A kept text of a run names no thread, thus it stays for the analysis purge.
+            .andThen(() => tryMutation("thread-store.purgeThread.toolOutputs", () => client.query(`${SUBTREE_CTE}
+         DELETE FROM cortex_tool_outputs WHERE thread_id IN (SELECT thread_id FROM subtree)`, [threadId])))
             .andThen(() => tryMutation("thread-store.purgeThread.thread", () => client.query(`${SUBTREE_CTE}
          DELETE FROM cortex_analysis_threads WHERE thread_id IN (SELECT thread_id FROM subtree)
          RETURNING thread_id`, [threadId])))

package/dist/memory/tool-call-integrity.d.ts

@@ -10,37 +10,31 @@
  * streams before the stop reason arrives, and only a `tool-calls` (and, for its
  * leading calls, a `length`) finish dispatches it.
  *
- * The repair is a STRIP, never an invented result. A call that was not
- * dispatched did not happen, and a fabricated result would record an execution
- * that did not occur — the model would read it as ground truth on the next
- * turn. The prose of the message survives the strip; a message left with
- * nothing the wire can render disappears entirely.
+ * The repair only appends an answer; it never edits or removes a message,
+ * because a signed thinking block and the prompt cache both key on the exact
+ * prior prefix.
  */
-import type { ModelMessage } from "ai";
-/** One tool call removed by {@link stripUnansweredToolCalls}, identified for the caller's diagnostic record. */
-export interface DroppedToolCall {
+import type { ModelMessage, ToolCallPart, ToolResultPart } from "ai";
+export declare const NOT_RUN_TOOL_RESULT = "Not run: the turn ended before this call ran.";
+export declare function notRunResult(call: Pick<ToolCallPart, "toolCallId" | "toolName">): ToolResultPart;
+/** One tool call that {@link answerUnansweredToolCalls} answered, identified for the caller's diagnostic record. */
+export interface AnsweredToolCall {
     readonly toolCallId: string;
     readonly toolName: string;
 }
 /**
- * Remove, in place, every tool-call part at or past `fromIndex` that has no
- * matching tool result anywhere in `messages`. Returns the removed calls in
- * transcript order — empty on the healthy path — so the caller can log what
- * a reader of the stored thread would otherwise never learn.
+ * Answer, in place, every tool call at or past `fromIndex` that has no matching
+ * tool result anywhere in `messages`. No message is removed or changed. Returns
+ * the answered calls in transcript order — empty on the healthy path — so the
+ * caller can log what a reader of the stored thread would otherwise never learn.
  *
  * Results are matched by id across the whole array, not by adjacency: the loop
  * appends a round's results directly after its assistant message, but a
  * deferred-image user message can ride between rounds, and the stored shape is
  * not this function's to assume.
  *
- * A message that keeps a text or file part survives the strip — refusal prose
- * is the one account the reader has of a filtered reply. A message left with
- * no such part (a call-only message, or reasoning beside the removed call) is
- * removed whole: reasoning without the output it preceded is not replayable,
- * and an empty assistant message is itself a wire violation.
- *
  * `fromIndex` bounds the REPAIR, not the result scan. The loop passes its
- * `initial.length` so it never rewrites the caller's prefix; the assembly
- * repair passes nothing and covers the whole stored window.
+ * `initial.length` so it never answers inside the caller's prefix; the history
+ * load passes nothing and covers the whole loaded window.
  */
-export declare function stripUnansweredToolCalls(messages: ModelMessage[], fromIndex?: number): DroppedToolCall[];
+export declare function answerUnansweredToolCalls(messages: ModelMessage[], fromIndex?: number): AnsweredToolCall[];