chainlesschain 0.166.107 → 0.166.109

package/src/lib/context-memory-kernel/live-compaction.js

@@ -13,6 +13,11 @@ import {
 import { JsonlSessionContextPort } from "./jsonl-session-context-port.js";
 import { createCliContextMemoryRuntime } from "./runtime.js";
 import { projectRuntimeTokenUsage } from "../runtime-usage-ledger.js";
+import {
+  projectCanonicalResumeMessages,
+  getDurableSystemMessageProvenance,
+  isRuntimeOnlySessionMessage,
+} from "../session-message-provenance.js";
 
 const rejectedCompactionUsage = new WeakMap();
 
@@ -138,7 +143,7 @@ class CallbackSessionContextPort extends InMemorySessionContextPort {
   }
 }
 
-function kernelBudget(compressor) {
+function kernelBudget(compressor, ephemeralTokens = 0) {
   const inputBudget = Math.max(
     256,
     Math.trunc(Number(compressor?.maxTokens)) || 16_384,
@@ -150,7 +155,9 @@ function kernelBudget(compressor) {
   return {
     modelWindowTokens: inputBudget + reservedOutputTokens,
     reservedOutputTokens,
-    safetyMarginTokens: 0,
+    // Live host policy/retry text still consumes the provider window even
+    // though it is deliberately absent from the durable authority partition.
+    safetyMarginTokens: ephemeralTokens,
     recoveryReserveTokens: Math.min(
       256,
       Math.max(1, Math.floor(inputBudget * 0.05)),
@@ -202,7 +209,7 @@ export async function compactLiveMessagesCanonical(
   messagesInput,
   options = {},
 ) {
-  const messages = Array.isArray(messagesInput) ? [...messagesInput] : [];
+  const liveMessages = Array.isArray(messagesInput) ? [...messagesInput] : [];
   const sessionId = String(options.sessionId || `live-${randomUUID()}`);
   const operationId = String(options.operationId || `compact-${randomUUID()}`);
   const trigger = options.trigger || "auto";
@@ -224,6 +231,24 @@ export async function compactLiveMessagesCanonical(
     options.persist !== false &&
     Boolean(options.sessionId) &&
     sessionExists(sessionId);
+  // Host policy is live-only. Durable summary/lineage systems retain their
+  // private provenance and remain part of the exact authority comparison.
+  const messages = persistentSession
+    ? projectCanonicalResumeMessages(liveMessages, { strict: true })
+    : liveMessages;
+  const ephemeralMessages = [];
+  if (persistentSession) {
+    let sequence = 0;
+    for (const message of liveMessages) {
+      if (
+        isRuntimeOnlySessionMessage(message) ||
+        (message.role === "system" &&
+          !getDurableSystemMessageProvenance(message))
+      )
+        ephemeralMessages.push({ sequence, message });
+      else sequence += 1;
+    }
+  }
   if (typeof options.commit === "function") {
     const bootstrapRuntime = createCliContextMemoryRuntime({
       sessionId,
@@ -323,12 +348,29 @@ export async function compactLiveMessagesCanonical(
   const provider = options.provider || "provider.local";
   const model = options.model || "default";
   const memoryRevision = await runtime.memoryPort.getRevision();
+  let settledOutputItems = null;
+  if (ephemeralMessages.length) {
+    const appendCompaction = runtime.sessionPort.appendCompaction.bind(
+      runtime.sessionPort,
+    );
+    runtime.sessionPort.appendCompaction = async (event, expectedHead) => {
+      const result = await appendCompaction(event, expectedHead);
+      if (result?.ok === true) settledOutputItems = event.outputItems;
+      return result;
+    };
+  }
   let receipt;
   try {
     receipt = await runtime.kernel.compactContext({
       operationId,
       sessionId,
-      ...kernelBudget(compressor),
+      ...kernelBudget(
+        compressor,
+        messagesToContextItems(
+          ephemeralMessages.map(({ message }) => message),
+          { sessionId },
+        ).reduce((total, item) => total + item.tokenEstimate, 0),
+      ),
       sink: provider,
       scopeAdmissions: [{ scope: "session", scopeId: sessionId }],
       policyVersion: "cli-context-policy-v1",
@@ -421,7 +463,7 @@ export async function compactLiveMessagesCanonical(
   }
   if (!["committed", "degraded"].includes(receipt.status)) {
     return {
-      messages,
+      messages: liveMessages,
       stats: {
         ...(summaryStats || {}),
         strategy: "none",
@@ -433,20 +475,41 @@ export async function compactLiveMessagesCanonical(
     };
   }
   const snapshot = await runtime.sessionPort.readSnapshot(sessionId);
-  const outputMessages = contextItemsToMessages(snapshot.items);
+  // Reattach host policy and private retry nudges at their durable sequence
+  // boundaries. They remain live, never enter the receipt, and do not move a
+  // retry instruction behind tool exchanges that happened after that retry.
+  const restoredMessages = [];
+  let ephemeralIndex = 0;
+  for (const item of [...(settledOutputItems || snapshot.items)].sort(
+    (left, right) =>
+      (left.sourceRef.eventSequence ?? Number.MAX_SAFE_INTEGER) -
+        (right.sourceRef.eventSequence ?? Number.MAX_SAFE_INTEGER) ||
+      left.itemId.localeCompare(right.itemId, "en"),
+  )) {
+    const sequence = item.sourceRef.eventSequence ?? Number.MAX_SAFE_INTEGER;
+    while (
+      ephemeralIndex < ephemeralMessages.length &&
+      ephemeralMessages[ephemeralIndex].sequence <= sequence
+    )
+      restoredMessages.push(ephemeralMessages[ephemeralIndex++].message);
+    restoredMessages.push(...contextItemsToMessages([item]));
+  }
+  restoredMessages.push(
+    ...ephemeralMessages.slice(ephemeralIndex).map(({ message }) => message),
+  );
   return {
-    messages: outputMessages,
+    messages: restoredMessages,
     stats: {
       ...(summaryStats || {}),
       strategy:
         receipt.status === "degraded" ? "deterministic-fallback" : "canonical",
-      originalMessages: messages.length,
-      compressedMessages: outputMessages.length,
-      originalTokens: estimatedTokens(messages),
-      compressedTokens: estimatedTokens(outputMessages),
+      originalMessages: liveMessages.length,
+      compressedMessages: restoredMessages.length,
+      originalTokens: estimatedTokens(liveMessages),
+      compressedTokens: estimatedTokens(restoredMessages),
       saved: Math.max(
         0,
-        estimatedTokens(messages) - estimatedTokens(outputMessages),
+        estimatedTokens(liveMessages) - estimatedTokens(restoredMessages),
       ),
       canonicalAlreadySettled: true,
       canonicalReceipt: receipt,

package/src/lib/read-file-loop-guard.js

@@ -32,6 +32,79 @@ export class ReadFileLoopGuard {
     this.batch = null;
     this.largeOutputs = new Set();
     this.editRecoveryReads = new Map();
+    this.visiblePages = null;
+    this.visibleRequest = 0;
+  }
+
+  /** Observe only the request actually selected for the provider, not history. */
+  observeVisibleMessages(messages) {
+    this.visibleRequest++;
+    this.visiblePages = [];
+    const retain = (value) => {
+      if (
+        typeof value?.path !== "string" ||
+        typeof value.fileVersion !== "string"
+      )
+        return;
+      if (typeof value.content === "string" && !value.contentOmitted) {
+        this.visiblePages.push(value);
+      }
+      for (const excerpt of Array.isArray(value.excerpts)
+        ? value.excerpts
+        : []) {
+        if (typeof excerpt?.content === "string")
+          this.visiblePages.push({
+            ...excerpt,
+            path: value.path,
+            fileVersion: value.fileVersion,
+          });
+      }
+    };
+    const retainContext = (context) => {
+      if (
+        typeof context !== "string" ||
+        !context.startsWith("[File excerpts retained across compaction")
+      )
+        return;
+      try {
+        const entries = JSON.parse(context.slice(context.indexOf("\n") + 1));
+        if (Array.isArray(entries)) entries.forEach(retain);
+      } catch {
+        /* A truncated hint provides no confirmed source evidence. */
+      }
+    };
+    for (const message of messages || []) {
+      if (message.role === "assistant") retainContext(message.content);
+      if (message.role !== "tool") continue;
+      let value;
+      try {
+        value =
+          typeof message.content === "string"
+            ? JSON.parse(message.content.split("\n\n[Budget ")[0])
+            : message.content;
+      } catch {
+        continue;
+      }
+      retain(value);
+      for (const context of Array.isArray(value?.retainedReadContext)
+        ? value.retainedReadContext
+        : []) {
+        retainContext(context);
+      }
+    }
+  }
+
+  isPageVisible(page) {
+    return (
+      this.visiblePages?.some(
+        (visible) =>
+          keyFor(visible.path, {}) === keyFor(page.path, {}) &&
+          visible.fileVersion === page.fileVersion &&
+          typeof page.content === "string" &&
+          page.content.length > 0 &&
+          visible.content.includes(page.content),
+      ) === true
+    );
   }
 
   recordEditRecovery(result) {
@@ -94,6 +167,7 @@ export class ReadFileLoopGuard {
         rereads: new Set(),
         summary: null,
         excerpts: [],
+        contextRestoreOffered: false,
       };
     }
     let page = readPage(args);
@@ -108,11 +182,11 @@ export class ReadFileLoopGuard {
         readProgress: { newContent: false },
       };
     }
-    const span = page.readSpan;
+    let span = page.readSpan;
     const covered = entry.spans.some(
       ([start, end]) => start <= span.start && end >= span.end,
     );
-    const targeted =
+    let targeted =
       Number(args.offset) > 0 &&
       Number(args.limit) > 0 &&
       Number(args.limit) <= 200 &&
@@ -122,20 +196,52 @@ export class ReadFileLoopGuard {
     );
     let newContent = !covered;
     const hadOutline = !!entry.summary?.outline;
+    // A planner can evict a read without compacting the transcript. Give one
+    // bounded landing page for that version, then require explicit ranges.
+    // Dropping the same evidence repeatedly must not reopen full-file scans.
+    if (
+      covered &&
+      entry.summary?.reachedEnd === true &&
+      !targeted &&
+      this.visiblePages !== null &&
+      entry.lastReadRequest < this.visibleRequest &&
+      !this.isPageVisible(page) &&
+      !entry.contextRestoreOffered &&
+      entry.rereads.size < 3
+    ) {
+      const offset =
+        Number.isSafeInteger(Number(args.offset)) && Number(args.offset) > 0
+          ? Number(args.offset)
+          : 1;
+      const bounded = readPage({ ...args, offset, limit: 80 });
+      if (!bounded.error && bounded.readSpan && !this.isPageVisible(bounded)) {
+        page = bounded;
+        span = page.readSpan;
+        targeted = true;
+        entry.contextRestoreOffered = true;
+      }
+    }
     if (covered || (coveredPrefix && !targeted)) {
       // Permit a bounded, explicit reread for editing or lost context. Broad
       // rescans and subsequent repeats recover automatically, including when
       // the model varies offset/limit inside an already covered region.
-      const fingerprint = createHash("sha256")
-        .update(JSON.stringify(span))
-        .digest("hex");
+      const restoring =
+        this.visiblePages !== null &&
+        entry.lastReadRequest < this.visibleRequest &&
+        !this.isPageVisible(page);
+      const fingerprint =
+        (restoring ? "restore:" : "review:") +
+        createHash("sha256").update(JSON.stringify(span)).digest("hex");
       if (
         targeted &&
         !entry.rereads.has(fingerprint) &&
         entry.rereads.size < 3
       ) {
         entry.rereads.add(fingerprint);
-        page.readRecovery = { action: "targeted-review" };
+        page.readRecovery = {
+          action: restoring ? "context-restore" : "targeted-review",
+        };
+        if (restoring) page.readRecovery.fileVersion = fileVersion;
       } else {
         const unread = entry.spans[0]?.[0] === 0 ? entry.spans[0][1] : 0;
         if (unread < span.total) {
@@ -162,12 +268,20 @@ export class ReadFileLoopGuard {
             ...(entry.summary?.outline
               ? { outline: entry.summary.outline }
               : {}),
-            hint: "The entire unchanged file has already been read during this run. Do not restart the scan. Use the retained outline/findings to perform the user's task; use search_files and a small explicit offset/limit only for missing details. Reading the file is not task completion.",
+            recoveryRead: {
+              path: filePath,
+              offset: 1,
+              limit: 80,
+              ...(args.hashed === true ? { hashed: true } : {}),
+              ...(args.raw === true ? { raw: true } : {}),
+            },
+            hint: "The unchanged file was read earlier, but historical coverage does not mean its source is visible now. Use any source excerpts actually present in this request. For missing details, use search_files or read_file with a specific offset and limit (at most 80 lines); recoveryRead is a bounded starting range, not permission to restart a full scan. Reading the file is not task completion.",
           };
         }
       }
     }
     entry.spans = mergeSpan(entry.spans, page.readSpan);
+    entry.lastReadRequest = this.visibleRequest;
     // Coverage alone cannot restore source code after compaction. Keep bounded
     // observed pages for every file type, not only large Markdown outlines.
     // Label truncation explicitly; a retained prefix is not the whole range.
@@ -232,6 +346,8 @@ export class ReadFileLoopGuard {
     if (tool === "read_file") {
       if (result?.readRecovery?.action === "edit-anchor-refresh")
         this.batch.advanced = true;
+      if (result?.readRecovery?.action === "context-restore")
+        this.batch.advanced = true;
       if (result?.readProgress?.newContent === true) this.batch.advanced = true;
       if (result?.readProgress?.newContent === false) this.batch.duplicates++;
     } else if (
@@ -308,7 +424,7 @@ export class ReadFileLoopGuard {
 
   get recoveryHint() {
     return this.repeatedBatches >= 1
-      ? "Repeated file reads or identical large command outputs detected. Do not switch to run_code/run_shell to dump the same file again. Filter/count locally and print only relevant rows, totals and source line numbers, or use a focused search. The runtime continues unfinished read_file scans automatically and returns use-findings after EOF. Use the retained outline to perform the user's task. Do not restart a whole-file scan after context compaction; answer once the requested evidence is sufficient."
+      ? "Repeated file reads or identical large command outputs detected. Do not switch to run_code/run_shell to dump the same file again. Filter/count locally and print only relevant rows, totals and source line numbers, or use a focused search. The runtime continues unfinished read_file scans automatically and returns use-findings after EOF. Use source evidence actually visible in this request; recover missing details with a specific offset and limit of at most 80 lines. Do not restart a whole-file scan after context compaction; answer once the requested evidence is sufficient."
       : null;
   }
 

package/src/lib/session-host-snapshot.js

@@ -27,6 +27,7 @@ import {
   sanitizePersistedMessages,
 } from "./session-message-provenance.js";
 import { createSessionTranscriptStructureProjection } from "./session-transcript-structure.js";
+import { projectToolRoundMessages } from "./session-tool-round.js";
 import { normalizeSessionBudgetRootConfig } from "./session-budget-production-root.js";
 import {
   createDeferredQuestionEventReducer,
@@ -86,6 +87,8 @@ function replayMessagesFromVerifiedEvents(events) {
       isReplayableMessage(event.data)
     ) {
       suffix.push(event.data);
+    } else if (event?.type === "tool_round") {
+      suffix.push(...projectToolRoundMessages(event).toReversed());
     } else if (event?.type === "ws_turn") {
       const turn = projectWsTurnMessages(event);
       if (turn) suffix.push(turn.assistant, turn.user);

package/src/lib/session-message-provenance.js

@@ -26,6 +26,7 @@ export const DURABLE_SYSTEM_MESSAGE_KINDS = Object.freeze({
 
 const DURABLE_KINDS = new Set(Object.values(DURABLE_SYSTEM_MESSAGE_KINDS));
 const RUNTIME_PROVENANCE = new WeakMap();
+const RUNTIME_ONLY_MESSAGES = new WeakSet();
 const INVALID_DATA = Symbol("invalid-session-message-data");
 const MAX_DATA_DEPTH = 100;
 const MAX_DATA_NODES = 100_000;
@@ -227,11 +228,11 @@ function validWireTag(value) {
   const snapshot = plainDataSnapshot(value);
   return Boolean(
     snapshot &&
-    snapshot.keys.length === 2 &&
-    snapshot.keys.includes("schema") &&
-    snapshot.keys.includes("kind") &&
-    dataValue(snapshot, "schema") === SESSION_MESSAGE_PROVENANCE_SCHEMA &&
-    DURABLE_KINDS.has(dataValue(snapshot, "kind")),
+      snapshot.keys.length === 2 &&
+      snapshot.keys.includes("schema") &&
+      snapshot.keys.includes("kind") &&
+      dataValue(snapshot, "schema") === SESSION_MESSAGE_PROVENANCE_SCHEMA &&
+      DURABLE_KINDS.has(dataValue(snapshot, "kind")),
   );
 }
 
@@ -248,6 +249,23 @@ export function markDurableSystemMessage(message, kind) {
   return attachRuntimeTag(message, canonicalTag(kind));
 }
 
+// Only host-created retry/result-delivery messages can be excluded from replay.
+// There is deliberately no wire tag: a provider field or a serialized copy
+// cannot suppress a real user message from authority comparison.
+export function markRuntimeOnlySessionMessage(message) {
+  const snapshot = plainDataSnapshot(message);
+  if (!snapshot || !["user", "assistant"].includes(dataValue(snapshot, "role")))
+    throw new TypeError(
+      "Runtime-only nudges must be user or assistant messages",
+    );
+  RUNTIME_ONLY_MESSAGES.add(message);
+  return message;
+}
+
+export function isRuntimeOnlySessionMessage(message) {
+  return RUNTIME_ONLY_MESSAGES.has(message);
+}
+
 export function getDurableSystemMessageProvenance(message) {
   const snapshot = plainDataSnapshot(message);
   if (!snapshot || dataValue(snapshot, "role") !== "system") return null;
@@ -422,6 +440,7 @@ export function projectCanonicalResumeMessages(messages, options = {}) {
       invalid("Canonical resume message must contain JSON-safe data");
       continue;
     }
+    if (isRuntimeOnlySessionMessage(raw)) continue;
     if (role !== "system") {
       projected.push(clean);
     } else if (runtimeTag) {

package/src/lib/session-tool-round.js

@@ -0,0 +1,39 @@
+/** Append-only, complete provider assistant/tool exchanges. No policy or user
+ * messages may enter durable replay through this event. */
+export const TOOL_ROUND_EVENT = "tool_round";
+export const TOOL_ROUND_SCHEMA = "chainlesschain.session-tool-round/v1";
+
+export function assertCompleteToolRounds(messages) {
+  if (!Array.isArray(messages) || messages.length === 0)
+    throw new TypeError("Tool rounds require a non-empty message array");
+  const pending = new Set();
+  for (const message of messages) {
+    if (message?.role === "assistant" && pending.size === 0) {
+      if (!Array.isArray(message.tool_calls) || message.tool_calls.length === 0)
+        throw new TypeError("Tool round assistant must contain tool calls");
+      for (const call of message.tool_calls) {
+        if (typeof call?.id !== "string" || !call.id || pending.has(call.id))
+          throw new TypeError("Malformed or duplicate tool call");
+        pending.add(call.id);
+      }
+    } else if (
+      message?.role === "tool" &&
+      pending.delete(message.tool_call_id)
+    ) {
+      // Every result settles exactly one call in the immediately preceding round.
+    } else {
+      throw new TypeError(
+        "Tool round contains an orphan result or forbidden message",
+      );
+    }
+  }
+  if (pending.size) throw new TypeError("Tool round is incomplete");
+  return messages;
+}
+
+export function projectToolRoundMessages(event) {
+  if (event?.type !== TOOL_ROUND_EVENT) return null;
+  if (event.data?.schema !== TOOL_ROUND_SCHEMA)
+    throw new TypeError("Unsupported session tool round schema");
+  return assertCompleteToolRounds(event.data.messages);
+}

package/src/lib/session-transcript-history.js

@@ -11,6 +11,7 @@ import {
 } from "./session-transcript-page.js";
 import { createSessionHistoryOrigins } from "./session-history-origins.js";
 import { getDurableSystemMessageProvenance } from "./session-message-provenance.js";
+import { projectToolRoundMessages } from "./session-tool-round.js";
 import {
   BRANCH_HISTORY_SCHEMA,
   BRANCH_HISTORY_SYSTEM_ORIGINS_SCHEMA,
@@ -29,6 +30,7 @@ const MAX_HISTORY_RANGES = 16384;
 // Enumerate all possible display sources, including replacement snapshots.
 // Their physical sequence never changes when the selected history rewinds.
 function eventMessages(event, imported = null) {
+  if (event.type === "tool_round") return projectToolRoundMessages(event);
   if (event.type === BRANCH_HISTORY_MESSAGE)
     return imported?.kind === "archive" ? [imported.message] : [];
   if (

package/src/lib/session-transcript-page.js

@@ -1,3 +1,4 @@
+import { projectToolRoundMessages } from "./session-tool-round.js";
 import { createHash } from "node:crypto";
 import {
   readVerifiedProjection,
@@ -120,7 +121,9 @@ export function createSessionTranscriptPageProjection(
         ["user_message", "assistant_message", "system"].includes(event.type)
       )
         append(event.data);
-      else if (event.type === "ws_turn") {
+      else if (event.type === "tool_round") {
+        for (const message of projectToolRoundMessages(event)) append(message);
+      } else if (event.type === "ws_turn") {
         const pair = projectWsTurnMessages(event);
         if (pair) {
           append(pair.user);

package/src/lib/session-transcript-structure.js

@@ -1,3 +1,4 @@
+import { projectToolRoundMessages } from "./session-tool-round.js";
 /**
  * Streaming structural authority for canonical session transcripts.
  *
@@ -72,6 +73,9 @@ export function createSessionTranscriptStructureProjection(
 
   return Object.freeze({
     accept(event) {
+      // Unknown versions and malformed/orphan results fail closed at every
+      // verified projection, even if that projection does not display tools.
+      if (event?.type === "tool_round") projectToolRoundMessages(event);
       const index = eventCount;
       eventCount += 1;
       if (event?.type === "session_start") {

package/src/runtime/__tests__/message-roles.test.js

@@ -2,9 +2,38 @@ import { describe, it, expect } from "vitest";
 import {
   mergeConsecutiveMessages,
   collapseConsecutiveMessagesInPlace,
+  createResumeRoleProjection,
   _internal,
 } from "../message-roles.js";
 
+describe("createResumeRoleProjection", () => {
+  it("normalizes retained resume pairs across clones without folding later feedback", () => {
+    const initial = [
+      { role: "system", content: "policy" },
+      { role: "user", content: "unanswered" },
+      { role: "user", content: "resume" },
+    ];
+    const project = createResumeRoleProjection(initial);
+    const request = [
+      ...structuredClone(initial),
+      { role: "assistant", content: "reply" },
+      { role: "user", content: "feedback" },
+      { role: "user", content: "next input" },
+    ];
+    const before = structuredClone(request);
+    const projected = project(request);
+    expect(request).toEqual(before);
+    expect(initial).toHaveLength(3);
+    expect(
+      projected
+        .filter((message) => message.role === "user")
+        .map((message) => message.content),
+    ).toEqual(["unanswered\n\nresume", "feedback", "next input"]);
+    expect(projected.at(-1)).toBe(request.at(-1));
+    expect(project(request)).toEqual(projected);
+  });
+});
+
 describe("mergeConsecutiveMessages", () => {
   it("is a no-op on a healthy alternating transcript", () => {
     const msgs = [