@poncho-ai/harness 0.53.0 → 0.55.0

package/.turbo/turbo-build.log

@@ -1,5 +1,5 @@
 
-> @poncho-ai/harness@0.53.0 build /home/runner/work/poncho-ai/poncho-ai/packages/harness
+> @poncho-ai/harness@0.55.0 build /home/runner/work/poncho-ai/poncho-ai/packages/harness
 > node scripts/embed-docs.js && tsup src/index.ts --format esm --dts
 
 [embed-docs] Generated poncho-docs.ts with 4 topics
@@ -9,8 +9,8 @@
 CLI Target: es2022
 ESM Build start
 ESM dist/isolate-F2PPSUL6.js 53.82 KB
-ESM dist/index.js            541.42 KB
-ESM ⚡️ Build success in 239ms
+ESM dist/index.js            545.48 KB
+ESM ⚡️ Build success in 258ms
 DTS Build start
-DTS ⚡️ Build success in 7708ms
-DTS dist/index.d.ts 94.04 KB
+DTS ⚡️ Build success in 7628ms
+DTS dist/index.d.ts 94.38 KB

package/CHANGELOG.md

@@ -1,5 +1,46 @@
 # @poncho-ai/harness
 
+## 0.55.0
+
+### Minor Changes
+
+- [#149](https://github.com/cesr/poncho-ai/pull/149) [`f5a8260`](https://github.com/cesr/poncho-ai/commit/f5a8260d0515038afc1797d00507908c334115ff) Thanks [@cesr](https://github.com/cesr)! - compaction: preserve subagent context and prior summaries, harden the split
+
+  Three improvements to context compaction (fires at ~75% context):
+  - **Split safety**: `findSafeSplitPoint` now refuses a split whose compacted
+    side would end on an assistant message with unanswered `tool_calls` (its
+    answering `role:"tool"` result having moved to the preserved side), walking
+    earlier to the next clean `user` boundary. Prevents orphaning a tool-call
+    relationship inside the summary boundary. Still returns `-1` when no safe
+    point exists.
+  - **Subagent ledger**: while compacting, scans for subagent-callback records
+    (metadata `_subagentCallback`/`subagentCallback`, or text starting with
+    `[Subagent Result]`) and any `## Subagents` block embedded in a prior
+    compaction summary, then renders a combined, deduped (by `subagentId`)
+    ledger that is appended VERBATIM after the LLM summary text — so the model
+    can never paraphrase or truncate subagent results away. Cumulative across
+    successive compactions.
+  - **Cumulative summary**: when the first compacted message is itself a prior
+    compaction summary, it is passed to the summarizer in full (not truncated
+    to 1200 chars) and the prompt instructs the model to merge-and-update the
+    prior working state rather than re-summarize it from scratch. All other
+    messages keep the 1200-char truncation.
+
+## 0.54.0
+
+### Minor Changes
+
+- [#147](https://github.com/cesr/poncho-ai/pull/147) [`a3eed14`](https://github.com/cesr/poncho-ai/commit/a3eed142832318b6397cd73819d3296c79d6eff0) Thanks [@cesr](https://github.com/cesr)! - storage: add append-only conversation-entry substrate (unused groundwork)
+
+  Pure entry types + rebuild functions (`buildLlmContext`,
+  `buildDisplaySnapshot`, `getPendingSubagentResults`) for the eventual
+  append-only conversation model that removes the mutable-blob clobber race
+  (the root cause behind lost subagent results). No storage-engine wiring
+  and no live callers yet — additive, deploys nothing behavioral. The
+  rebuild logic (compaction overlay, amendment folding, callback-consumption)
+  is covered by unit tests so the design is proven before the bigger
+  dual-write / migration / cutover PRs.
+
 ## 0.53.0
 
 ### Minor Changes

package/dist/index.d.ts

@@ -96,6 +96,12 @@ declare const estimateTotalTokens: (systemPrompt: string, messages: Message[], t
  * and everything from it onward is preserved. The split always lands just
  * before a `user` message to avoid breaking assistant+tool pairs.
  *
+ * Defensive guard: even at a `user` boundary, refuse a split whose compacted
+ * side would END on an assistant message with unanswered tool_calls (its
+ * `tool` result having moved to the preserved side). Such a split would
+ * orphan the tool_calls inside the summary boundary. When that happens we
+ * walk earlier to the next safe `user` boundary.
+ *
  * Returns -1 if no valid split point is found.
  */
 declare const findSafeSplitPoint: (messages: Message[], keepRecentMessages: number) => number;

package/dist/index.js

@@ -374,6 +374,9 @@ var SUMMARIZATION_PROMPT = `Summarize the following conversation into a structur
 
 Be concise but preserve all information needed to continue the task.
 Omit any section that has no relevant content.`;
+var CUMULATIVE_SUMMARY_PROMPT = `The FIRST message below (tagged [prior-summary]) is an existing working-state summary produced by an earlier compaction. Treat it as the authoritative prior working state: MERGE AND UPDATE it with the newer messages that follow it, carrying forward all still-relevant detail. Do NOT discard or re-compress information from the prior summary just because it is older \u2014 only drop it if the newer messages explicitly supersede it.`;
+var SUBAGENT_DIGEST_CHARS = 500;
+var SUBAGENT_LEDGER_HEADING = "## Subagents";
 var resolveCompactionConfig = (explicit) => {
   if (!explicit) return { ...DEFAULT_COMPACTION_CONFIG };
   return {
@@ -398,32 +401,57 @@ var estimateTotalTokens = (systemPrompt, messages, toolDefinitionsJson) => {
   }
   return tokens;
 };
+var assistantHasToolCalls = (msg) => {
+  if (msg.role !== "assistant") return false;
+  if (typeof msg.content !== "string") return false;
+  if (!msg.content.includes('"tool_calls"')) return false;
+  try {
+    const parsed = JSON.parse(msg.content);
+    return Array.isArray(parsed.tool_calls) && parsed.tool_calls.length > 0;
+  } catch {
+    return false;
+  }
+};
+var splitOrphansToolCalls = (messages, idx) => {
+  if (idx <= 0 || idx >= messages.length) return false;
+  const lastCompacted = messages[idx - 1];
+  return assistantHasToolCalls(lastCompacted);
+};
 var findSafeSplitPoint = (messages, keepRecentMessages) => {
   const candidateIdx = messages.length - keepRecentMessages;
   if (candidateIdx < MIN_COMPACTABLE_MESSAGES) return -1;
   for (let i = candidateIdx; i >= MIN_COMPACTABLE_MESSAGES; i--) {
-    if (messages[i].role === "user") {
+    if (messages[i].role === "user" && !splitOrphansToolCalls(messages, i)) {
       return i;
     }
   }
   for (let i = candidateIdx + 1; i < messages.length - 1; i++) {
-    if (messages[i].role === "user") {
+    if (messages[i].role === "user" && !splitOrphansToolCalls(messages, i)) {
       if (i < MIN_COMPACTABLE_MESSAGES) return -1;
       return i;
     }
   }
   return -1;
 };
+var isCompactionSummary = (msg) => msg.metadata?.isCompactionSummary === true;
 var buildSummarizationMessages = (messagesToCompact, instructions) => {
+  const hasPriorSummary = messagesToCompact.length > 0 && isCompactionSummary(messagesToCompact[0]);
   const conversationLines = [];
-  for (const msg of messagesToCompact) {
+  for (let i = 0; i < messagesToCompact.length; i++) {
+    const msg = messagesToCompact[i];
     const text = getTextContent(msg);
-    const truncated = text.length > SUMMARIZATION_MESSAGE_TRUNCATION_CHARS ? text.slice(0, SUMMARIZATION_MESSAGE_TRUNCATION_CHARS) + "\n...[truncated]" : text;
-    conversationLines.push(`[${msg.role}]: ${truncated}`);
+    const isPrior = i === 0 && hasPriorSummary;
+    const rendered = isPrior || text.length <= SUMMARIZATION_MESSAGE_TRUNCATION_CHARS ? text : text.slice(0, SUMMARIZATION_MESSAGE_TRUNCATION_CHARS) + "\n...[truncated]";
+    const tag = isPrior ? "prior-summary" : msg.role;
+    conversationLines.push(`[${tag}]: ${rendered}`);
   }
-  const prompt = instructions ? `${SUMMARIZATION_PROMPT}
+  let prompt = SUMMARIZATION_PROMPT;
+  if (hasPriorSummary) prompt = `${prompt}
+
+${CUMULATIVE_SUMMARY_PROMPT}`;
+  if (instructions) prompt = `${prompt}
 
-Additional focus: ${instructions}` : SUMMARIZATION_PROMPT;
+Additional focus: ${instructions}`;
   return [
     {
       role: "user",
@@ -435,6 +463,67 @@ ${conversationLines.join("\n\n")}`
     }
   ];
 };
+var SUBAGENT_RESULT_HEADER = /^\[Subagent Result\] Subagent "([^"]*)" \(([^)]*)\) (\S+):/;
+var parseSubagentCallback = (msg) => {
+  if (msg.role !== "user") return null;
+  const meta = msg.metadata ?? {};
+  const text = getTextContent(msg);
+  const hasMetaFlag = meta._subagentCallback === true || meta.subagentCallback === true;
+  const hasTextMarker = text.startsWith("[Subagent Result]");
+  if (!hasMetaFlag && !hasTextMarker) return null;
+  const headerMatch = text.match(SUBAGENT_RESULT_HEADER);
+  const subagentId = typeof meta.subagentId === "string" && meta.subagentId ? meta.subagentId : headerMatch?.[2] ?? "";
+  if (!subagentId) return null;
+  const task = typeof meta.task === "string" && meta.task ? meta.task : headerMatch?.[1] ?? "";
+  const status = headerMatch?.[3] ?? "completed";
+  const bodyStart = text.indexOf("\n\n");
+  const body = bodyStart >= 0 ? text.slice(bodyStart + 2) : text;
+  const digest = body.length > SUBAGENT_DIGEST_CHARS ? body.slice(0, SUBAGENT_DIGEST_CHARS) + "\u2026" : body;
+  return { subagentId, task, status, digest };
+};
+var parsePriorLedger = (summaryText) => {
+  const headingIdx = summaryText.indexOf(SUBAGENT_LEDGER_HEADING);
+  if (headingIdx < 0) return [];
+  const block = summaryText.slice(headingIdx + SUBAGENT_LEDGER_HEADING.length);
+  const entries = [];
+  const entryRe = /^- \*\*(.*?)\*\* \((.+?)\) — (\S+)\n {2}(.*)$/gm;
+  let m;
+  while ((m = entryRe.exec(block)) !== null) {
+    entries.push({
+      task: m[1],
+      subagentId: m[2],
+      status: m[3],
+      digest: m[4]
+    });
+  }
+  return entries;
+};
+var collectSubagentLedger = (messagesToCompact) => {
+  const byId = /* @__PURE__ */ new Map();
+  const order = [];
+  const upsert = (entry) => {
+    if (!byId.has(entry.subagentId)) order.push(entry.subagentId);
+    byId.set(entry.subagentId, entry);
+  };
+  for (const msg of messagesToCompact) {
+    if (isCompactionSummary(msg)) {
+      for (const prior of parsePriorLedger(getTextContent(msg))) upsert(prior);
+      continue;
+    }
+    const entry = parseSubagentCallback(msg);
+    if (entry) upsert(entry);
+  }
+  return order.map((id) => byId.get(id));
+};
+var renderSubagentLedger = (entries) => {
+  if (entries.length === 0) return "";
+  const lines = entries.map(
+    (e) => `- **${e.task}** (${e.subagentId}) \u2014 ${e.status}
+  ${e.digest.replace(/\n/g, " ")}`
+  );
+  return `${SUBAGENT_LEDGER_HEADING}
+${lines.join("\n")}`;
+};
 var buildContinuationMessage = (summary) => ({
   role: "user",
   content: `[CONTEXT COMPACTION] This conversation was automatically compacted. The summary below covers earlier messages.
@@ -483,7 +572,11 @@ var compactMessages = async (model, messages, config, options) => {
         warning: "Summarization returned empty result"
       };
     }
-    const continuationMessage = buildContinuationMessage(summary);
+    const ledger = renderSubagentLedger(collectSubagentLedger(toCompact));
+    const summaryWithLedger = ledger ? `${summary}
+
+${ledger}` : summary;
+    const continuationMessage = buildContinuationMessage(summaryWithLedger);
     const compactedMessages = [continuationMessage, ...toPreserve];
     return {
       compacted: true,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@poncho-ai/harness",
-  "version": "0.53.0",
+  "version": "0.55.0",
   "description": "Agent execution runtime - conversation loop, tool dispatch, streaming",
   "repository": {
     "type": "git",

package/src/compaction.ts

@@ -26,6 +26,20 @@ const SUMMARIZATION_PROMPT = `Summarize the following conversation into a struct
 Be concise but preserve all information needed to continue the task.
 Omit any section that has no relevant content.`;
 
+/**
+ * Extra instruction appended when the first compacted message is itself a
+ * prior compaction summary. The model must treat that block as the existing
+ * working state and produce an updated, merged version rather than
+ * re-summarizing the (already lossy) summary from scratch.
+ */
+const CUMULATIVE_SUMMARY_PROMPT = `The FIRST message below (tagged [prior-summary]) is an existing working-state summary produced by an earlier compaction. Treat it as the authoritative prior working state: MERGE AND UPDATE it with the newer messages that follow it, carrying forward all still-relevant detail. Do NOT discard or re-compress information from the prior summary just because it is older — only drop it if the newer messages explicitly supersede it.`;
+
+/** Max chars of a subagent result text kept verbatim in the ledger digest. */
+const SUBAGENT_DIGEST_CHARS = 500;
+
+/** Heading used for the verbatim, model-proof subagent ledger block. */
+const SUBAGENT_LEDGER_HEADING = "## Subagents";
+
 export const resolveCompactionConfig = (
   explicit?: Partial<CompactionConfig>,
 ): CompactionConfig => {
@@ -78,11 +92,48 @@ export const estimateTotalTokens = (
   return tokens;
 };
 
+/**
+ * Whether an assistant message carries serialized tool_calls. Assistant
+ * tool-call turns serialize their content as a JSON string of the shape
+ * `{"text":...,"tool_calls":[...]}` (see the harness run loop). A plain-text
+ * assistant message returns false.
+ */
+const assistantHasToolCalls = (msg: Message): boolean => {
+  if (msg.role !== "assistant") return false;
+  if (typeof msg.content !== "string") return false;
+  if (!msg.content.includes('"tool_calls"')) return false;
+  try {
+    const parsed = JSON.parse(msg.content) as { tool_calls?: unknown };
+    return Array.isArray(parsed.tool_calls) && parsed.tool_calls.length > 0;
+  } catch {
+    return false;
+  }
+};
+
+/**
+ * Whether splitting at `idx` would orphan a tool-call relationship on the
+ * COMPACTED side — i.e. the last compacted message (`messages[idx-1]`) is an
+ * assistant message with tool_calls whose answering `role:"tool"` result
+ * lives on the PRESERVED side (`messages[idx]`). Folding only the assistant
+ * call into the summary strands the tool_calls with no matching result.
+ */
+const splitOrphansToolCalls = (messages: Message[], idx: number): boolean => {
+  if (idx <= 0 || idx >= messages.length) return false;
+  const lastCompacted = messages[idx - 1]!;
+  return assistantHasToolCalls(lastCompacted);
+};
+
 /**
  * Find the safe split index so that everything before it can be compacted
  * and everything from it onward is preserved. The split always lands just
  * before a `user` message to avoid breaking assistant+tool pairs.
  *
+ * Defensive guard: even at a `user` boundary, refuse a split whose compacted
+ * side would END on an assistant message with unanswered tool_calls (its
+ * `tool` result having moved to the preserved side). Such a split would
+ * orphan the tool_calls inside the summary boundary. When that happens we
+ * walk earlier to the next safe `user` boundary.
+ *
  * Returns -1 if no valid split point is found.
  */
 export const findSafeSplitPoint = (
@@ -92,16 +143,17 @@ export const findSafeSplitPoint = (
   const candidateIdx = messages.length - keepRecentMessages;
   if (candidateIdx < MIN_COMPACTABLE_MESSAGES) return -1;
 
-  // Walk backwards from candidate to find a user message boundary
+  // Walk backwards from candidate to find a user message boundary that does
+  // not orphan a tool-call relationship on the compacted side.
   for (let i = candidateIdx; i >= MIN_COMPACTABLE_MESSAGES; i--) {
-    if (messages[i]!.role === "user") {
+    if (messages[i]!.role === "user" && !splitOrphansToolCalls(messages, i)) {
       return i;
     }
   }
 
-  // Walk forwards from candidate as fallback
+  // Walk forwards from candidate as fallback.
   for (let i = candidateIdx + 1; i < messages.length - 1; i++) {
-    if (messages[i]!.role === "user") {
+    if (messages[i]!.role === "user" && !splitOrphansToolCalls(messages, i)) {
       if (i < MIN_COMPACTABLE_MESSAGES) return -1;
       return i;
     }
@@ -110,25 +162,46 @@ export const findSafeSplitPoint = (
   return -1;
 };
 
+/**
+ * Whether a message is itself a prior compaction summary.
+ */
+const isCompactionSummary = (msg: Message): boolean =>
+  msg.metadata?.isCompactionSummary === true;
+
 /**
  * Build the summarization messages for the generateText call.
+ *
+ * Cumulative behavior: when the FIRST compacted message is itself a prior
+ * compaction summary, it is passed in FULL (not truncated to
+ * SUMMARIZATION_MESSAGE_TRUNCATION_CHARS) and tagged `[prior-summary]`, and
+ * the prompt instructs the model to merge-and-update rather than
+ * re-summarize. All other messages keep the 1200-char truncation.
  */
 const buildSummarizationMessages = (
   messagesToCompact: Message[],
   instructions?: string,
 ): Array<{ role: "user"; content: string }> => {
+  const hasPriorSummary =
+    messagesToCompact.length > 0 && isCompactionSummary(messagesToCompact[0]!);
+
   const conversationLines: string[] = [];
-  for (const msg of messagesToCompact) {
+  for (let i = 0; i < messagesToCompact.length; i++) {
+    const msg = messagesToCompact[i]!;
     const text = getTextContent(msg);
-    const truncated = text.length > SUMMARIZATION_MESSAGE_TRUNCATION_CHARS
-      ? text.slice(0, SUMMARIZATION_MESSAGE_TRUNCATION_CHARS) + "\n...[truncated]"
-      : text;
-    conversationLines.push(`[${msg.role}]: ${truncated}`);
+    const isPrior = i === 0 && hasPriorSummary;
+    // The prior summary is the working state we must not lose — pass it whole.
+    const rendered =
+      isPrior || text.length <= SUMMARIZATION_MESSAGE_TRUNCATION_CHARS
+        ? text
+        : text.slice(0, SUMMARIZATION_MESSAGE_TRUNCATION_CHARS) +
+          "\n...[truncated]";
+    const tag = isPrior ? "prior-summary" : msg.role;
+    conversationLines.push(`[${tag}]: ${rendered}`);
   }
 
-  const prompt = instructions
-    ? `${SUMMARIZATION_PROMPT}\n\nAdditional focus: ${instructions}`
-    : SUMMARIZATION_PROMPT;
+  let prompt = SUMMARIZATION_PROMPT;
+  if (hasPriorSummary) prompt = `${prompt}\n\n${CUMULATIVE_SUMMARY_PROMPT}`;
+  if (instructions) prompt = `${prompt}\n\nAdditional focus: ${instructions}`;
 
   return [
     {
@@ -138,6 +211,121 @@ const buildSummarizationMessages = (
   ];
 };
 
+interface SubagentLedgerEntry {
+  subagentId: string;
+  task: string;
+  status: string;
+  digest: string;
+}
+
+/** Match the header line of an injected subagent callback message. */
+const SUBAGENT_RESULT_HEADER =
+  /^\[Subagent Result\] Subagent "([^"]*)" \(([^)]*)\) (\S+):/;
+
+/**
+ * Parse the metadata + text of a subagent-callback user message into a ledger
+ * entry. Returns null when the message is not a subagent callback.
+ */
+const parseSubagentCallback = (msg: Message): SubagentLedgerEntry | null => {
+  if (msg.role !== "user") return null;
+  const meta = (msg.metadata ?? {}) as Record<string, unknown>;
+  const text = getTextContent(msg);
+  const hasMetaFlag =
+    meta._subagentCallback === true || meta.subagentCallback === true;
+  const hasTextMarker = text.startsWith("[Subagent Result]");
+  if (!hasMetaFlag && !hasTextMarker) return null;
+
+  // Prefer structured metadata, fall back to parsing the header line.
+  const headerMatch = text.match(SUBAGENT_RESULT_HEADER);
+  const subagentId =
+    typeof meta.subagentId === "string" && meta.subagentId
+      ? meta.subagentId
+      : headerMatch?.[2] ?? "";
+  if (!subagentId) return null;
+  const task =
+    typeof meta.task === "string" && meta.task
+      ? meta.task
+      : headerMatch?.[1] ?? "";
+  const status = headerMatch?.[3] ?? "completed";
+
+  // Digest = the body after the header line (the result text), capped.
+  const bodyStart = text.indexOf("\n\n");
+  const body = bodyStart >= 0 ? text.slice(bodyStart + 2) : text;
+  const digest =
+    body.length > SUBAGENT_DIGEST_CHARS
+      ? body.slice(0, SUBAGENT_DIGEST_CHARS) + "…"
+      : body;
+
+  return { subagentId, task, status, digest };
+};
+
+/**
+ * Parse a prior `## Subagents` ledger block out of an existing compaction
+ * summary's content so it can be carried forward cumulatively. The block is
+ * rendered by `renderSubagentLedger`, so we parse that same shape.
+ */
+const parsePriorLedger = (summaryText: string): SubagentLedgerEntry[] => {
+  const headingIdx = summaryText.indexOf(SUBAGENT_LEDGER_HEADING);
+  if (headingIdx < 0) return [];
+  const block = summaryText.slice(headingIdx + SUBAGENT_LEDGER_HEADING.length);
+  const entries: SubagentLedgerEntry[] = [];
+  // Each entry: a bullet line "- **<task>** (<id>) — <status>" then a digest
+  // line. We tolerate missing digest lines.
+  const entryRe =
+    /^- \*\*(.*?)\*\* \((.+?)\) — (\S+)\n {2}(.*)$/gm;
+  let m: RegExpExecArray | null;
+  while ((m = entryRe.exec(block)) !== null) {
+    entries.push({
+      task: m[1]!,
+      subagentId: m[2]!,
+      status: m[3]!,
+      digest: m[4]!,
+    });
+  }
+  return entries;
+};
+
+/**
+ * Scan the messages being compacted for subagent-callback records and any
+ * prior ledger embedded in a compaction summary, returning a combined,
+ * deduped (by subagentId, last-write-wins) list in first-seen order.
+ */
+const collectSubagentLedger = (
+  messagesToCompact: Message[],
+): SubagentLedgerEntry[] => {
+  const byId = new Map<string, SubagentLedgerEntry>();
+  const order: string[] = [];
+  const upsert = (entry: SubagentLedgerEntry) => {
+    if (!byId.has(entry.subagentId)) order.push(entry.subagentId);
+    byId.set(entry.subagentId, entry);
+  };
+
+  for (const msg of messagesToCompact) {
+    if (isCompactionSummary(msg)) {
+      for (const prior of parsePriorLedger(getTextContent(msg))) upsert(prior);
+      continue;
+    }
+    const entry = parseSubagentCallback(msg);
+    if (entry) upsert(entry);
+  }
+
+  return order.map((id) => byId.get(id)!);
+};
+
+/**
+ * Render the subagent ledger as a verbatim markdown block appended to the
+ * summary AFTER the LLM text, so the model can never paraphrase it away.
+ * Returns "" when there are no subagents.
+ */
+const renderSubagentLedger = (entries: SubagentLedgerEntry[]): string => {
+  if (entries.length === 0) return "";
+  const lines = entries.map(
+    (e) =>
+      `- **${e.task}** (${e.subagentId}) — ${e.status}\n  ${e.digest.replace(/\n/g, " ")}`,
+  );
+  return `${SUBAGENT_LEDGER_HEADING}\n${lines.join("\n")}`;
+};
+
 /**
  * Build the continuation message that replaces compacted messages.
  */
@@ -217,7 +405,12 @@ export const compactMessages = async (
       };
     }
 
-    const continuationMessage = buildContinuationMessage(summary);
+    // Append the subagent ledger AFTER the LLM summary, verbatim, so the
+    // model's paraphrasing can never drop or truncate subagent results.
+    const ledger = renderSubagentLedger(collectSubagentLedger(toCompact));
+    const summaryWithLedger = ledger ? `${summary}\n\n${ledger}` : summary;
+
+    const continuationMessage = buildContinuationMessage(summaryWithLedger);
     const compactedMessages = [continuationMessage, ...toPreserve];
 
     return {

package/src/storage/entries.ts

@@ -0,0 +1,204 @@
+import type { Message } from "@poncho-ai/sdk";
+import type { PendingSubagentResult } from "../state.js";
+
+/**
+ * Append-only conversation entries (Phase 3 substrate).
+ *
+ * The eventual replacement for the mutable per-conversation JSON blob: a
+ * conversation becomes an ordered, append-only list of entries, and the
+ * mutable-blob clobber race (two writers serializing a stale whole-blob
+ * snapshot over each other — the root cause behind lost subagent results)
+ * stops being expressible.
+ *
+ * This module is intentionally PURE: it defines the entry shapes and the
+ * functions that rebuild a conversation's LLM context / display transcript
+ * / pending-subagent-results from an entry list. No storage engine, no DB,
+ * no wiring into the live run loop yet — so it deploys nothing and is
+ * fully unit-testable. The engine implementations (append/read on
+ * postgres/sqlite/memory) and the write-site conversions come in later PRs
+ * once this rebuild logic is proven.
+ *
+ * Ordering: every entry carries a monotonic per-conversation `seq`. Entries
+ * are assumed sorted by `seq` ascending when passed to the rebuild fns.
+ */
+
+interface BaseEntry {
+  /** Stable cross-reference id (uuid). */
+  id: string;
+  /** Monotonic per-conversation order. */
+  seq: number;
+  createdAt: number;
+}
+
+/** A user-role display message (incl. typed subagent-callback messages). */
+export interface UserMessageEntry extends BaseEntry {
+  type: "user_message";
+  message: Message;
+  turnId: string;
+  /** Hidden from the display transcript (e.g. a framed job prompt, an
+   *  onboarding seed, or an injected subagent-result message). Still part
+   *  of the record; just not rendered as a chat bubble. */
+  hidden?: boolean;
+}
+
+/** The final assistant bubble for a completed/cancelled/errored turn. */
+export interface AssistantMessageEntry extends BaseEntry {
+  type: "assistant_message";
+  message: Message;
+  turnId: string;
+  runId: string;
+}
+
+/** A post-hoc edit to an already-emitted assistant message — replaces the
+ *  orchestrator/resume "mutate the last assistant message in place" writes
+ *  with an append. Applied at rebuild time. */
+export interface AssistantAmendmentEntry extends BaseEntry {
+  type: "assistant_amendment";
+  targetEntryId: string;
+  appendText?: string;
+}
+
+/** One LLM-transcript message (the model-visible form). Appended from the
+ *  run loop per step — never diffed from an array. */
+export interface HarnessMessageEntry extends BaseEntry {
+  type: "harness_message";
+  message: Message;
+  turnId: string;
+}
+
+/** Compaction overlay: nothing is deleted. At rebuild, the LLM context is
+ *  the latest compaction's `summaryMessage` followed by the harness
+ *  messages from `firstKeptSeq` onward. */
+export interface CompactionEntry extends BaseEntry {
+  type: "compaction";
+  summaryMessage: Message;
+  firstKeptSeq: number;
+  tokensBefore?: number;
+  tokensAfter?: number;
+}
+
+/** A finished subagent's result arriving for the parent. Pending = a
+ *  subagent_result whose seq is not listed in any later callback_started. */
+export interface SubagentResultEntry extends BaseEntry {
+  type: "subagent_result";
+  result: PendingSubagentResult;
+}
+
+/** Marks which subagent_result entries a callback turn consumed (by seq).
+ *  Consumption is an append, never a delete. */
+export interface CallbackStartedEntry extends BaseEntry {
+  type: "callback_started";
+  consumedSeqs: number[];
+}
+
+export type ConversationEntry =
+  | UserMessageEntry
+  | AssistantMessageEntry
+  | AssistantAmendmentEntry
+  | HarnessMessageEntry
+  | CompactionEntry
+  | SubagentResultEntry
+  | CallbackStartedEntry;
+
+/**
+ * Rebuild the LLM-visible message context from the entry log.
+ *
+ * If a compaction overlay exists, the context is its summary message
+ * followed by every harness message with seq >= firstKeptSeq (a later
+ * compaction's firstKeptSeq can point at an earlier summary that was
+ * itself appended as a harness message, so layered compactions just work).
+ * With no compaction, it's every harness message in order.
+ */
+export function buildLlmContext(entries: ConversationEntry[]): Message[] {
+  let latestCompaction: CompactionEntry | undefined;
+  for (const e of entries) {
+    if (e.type === "compaction" && (!latestCompaction || e.seq > latestCompaction.seq)) {
+      latestCompaction = e;
+    }
+  }
+
+  const harnessMsgs = entries.filter(
+    (e): e is HarnessMessageEntry => e.type === "harness_message",
+  );
+
+  if (latestCompaction) {
+    const kept = harnessMsgs
+      .filter((e) => e.seq >= latestCompaction!.firstKeptSeq)
+      .map((e) => e.message);
+    return [latestCompaction.summaryMessage, ...kept];
+  }
+  return harnessMsgs.map((e) => e.message);
+}
+
+export interface DisplaySnapshot {
+  messages: Message[];
+  /** Total display messages available (for pagination UIs). */
+  totalMessages: number;
+  /** seq of the first message returned (a `beforeSeq` pagination cursor). */
+  headSeq: number | null;
+}
+
+/**
+ * Rebuild the display transcript (the user-visible chat) from the entry
+ * log, returning the trailing `tailN` messages. Amendments are folded into
+ * their target assistant message; hidden user messages are dropped.
+ */
+export function buildDisplaySnapshot(
+  entries: ConversationEntry[],
+  tailN: number,
+): DisplaySnapshot {
+  const amendmentsByTarget = new Map<string, AssistantAmendmentEntry[]>();
+  for (const e of entries) {
+    if (e.type === "assistant_amendment") {
+      const list = amendmentsByTarget.get(e.targetEntryId) ?? [];
+      list.push(e);
+      amendmentsByTarget.set(e.targetEntryId, list);
+    }
+  }
+
+  const built: { seq: number; message: Message }[] = [];
+  for (const e of entries) {
+    if (e.type === "user_message") {
+      if (e.hidden) continue;
+      built.push({ seq: e.seq, message: e.message });
+    } else if (e.type === "assistant_message") {
+      let content = typeof e.message.content === "string" ? e.message.content : "";
+      const amendments = amendmentsByTarget.get(e.id);
+      if (amendments) {
+        for (const a of amendments.sort((x, y) => x.seq - y.seq)) {
+          if (a.appendText) content += a.appendText;
+        }
+      }
+      built.push({ seq: e.seq, message: { ...e.message, content } });
+    }
+  }
+
+  const total = built.length;
+  const tail = tailN >= total ? built : built.slice(total - tailN);
+  return {
+    messages: tail.map((b) => b.message),
+    totalMessages: total,
+    headSeq: tail.length > 0 ? tail[0]!.seq : null,
+  };
+}
+
+/**
+ * Subagent results that have arrived but not yet been consumed by a
+ * callback turn — the append-only replacement for the mutable
+ * `pendingSubagentResults` array. A result is pending unless a later
+ * callback_started lists its seq in `consumedSeqs`.
+ */
+export function getPendingSubagentResults(
+  entries: ConversationEntry[],
+): PendingSubagentResult[] {
+  const consumed = new Set<number>();
+  for (const e of entries) {
+    if (e.type === "callback_started") {
+      for (const s of e.consumedSeqs) consumed.add(s);
+    }
+  }
+  return entries
+    .filter((e): e is SubagentResultEntry => e.type === "subagent_result")
+    .filter((e) => !consumed.has(e.seq))
+    .map((e) => e.result);
+}

package/test/compaction.test.ts

@@ -0,0 +1,274 @@
+import { describe, expect, it } from "vitest";
+import type { LanguageModel } from "ai";
+import { MockLanguageModelV3 } from "ai/test";
+import type { Message } from "@poncho-ai/sdk";
+import {
+  compactMessages,
+  findSafeSplitPoint,
+  resolveCompactionConfig,
+} from "../src/compaction.js";
+
+// ── Fake model ──────────────────────────────────────────────────────────
+// A MockLanguageModelV3 whose doGenerate returns a fixed text and records the
+// prompt it was handed, so tests can assert what was sent to the summarizer.
+function fakeModel(summaryText: string): {
+  model: LanguageModel;
+  prompts: string[];
+} {
+  const prompts: string[] = [];
+  const model = new MockLanguageModelV3({
+    doGenerate: async (options) => {
+      // Flatten the prompt text we were given (the user message content).
+      for (const m of options.prompt) {
+        if (Array.isArray(m.content)) {
+          for (const part of m.content) {
+            if (part.type === "text") prompts.push(part.text);
+          }
+        }
+      }
+      return {
+        content: [{ type: "text", text: summaryText }],
+        finishReason: "stop",
+        usage: { inputTokens: 10, outputTokens: 10, totalTokens: 20 },
+        warnings: [],
+      };
+    },
+  });
+  return { model: model as unknown as LanguageModel, prompts };
+}
+
+const userMsg = (text: string, metadata?: Message["metadata"]): Message => ({
+  role: "user",
+  content: text,
+  ...(metadata ? { metadata } : {}),
+});
+const assistantText = (text: string): Message => ({
+  role: "assistant",
+  content: text,
+});
+const assistantToolCall = (text: string, toolName: string): Message => ({
+  role: "assistant",
+  content: JSON.stringify({
+    text,
+    tool_calls: [{ id: "call_1", name: toolName, arguments: {} }],
+  }),
+});
+const toolResult = (text: string): Message => ({ role: "tool", content: text });
+
+describe("findSafeSplitPoint", () => {
+  it("splits at a normal user-message boundary", () => {
+    const messages: Message[] = [
+      userMsg("u0"),
+      assistantText("a0"),
+      userMsg("u1"),
+      assistantText("a1"),
+      userMsg("u2"), // index 4 — a clean user boundary
+      assistantText("a2"),
+      userMsg("u3"),
+      assistantText("a3"),
+    ];
+    const idx = findSafeSplitPoint(messages, 4);
+    // candidate = 8 - 4 = 4, which is already a user message → split there.
+    expect(idx).toBe(4);
+    expect(messages[idx]!.role).toBe("user");
+  });
+
+  it("returns -1 when there are too few messages", () => {
+    const messages: Message[] = [userMsg("u0"), assistantText("a0")];
+    expect(findSafeSplitPoint(messages, 4)).toBe(-1);
+  });
+
+  it("walks earlier when the split would orphan tool_calls being moved", () => {
+    // The candidate user boundary sits right after an assistant tool-call
+    // message whose tool result is on the preserved side — splitting there
+    // would strand the tool_calls in the summary. Guard must walk earlier to
+    // the next clean user boundary (which is still >= MIN_COMPACTABLE_MESSAGES).
+    const messages: Message[] = [
+      userMsg("u0"), // 0
+      assistantText("a0"), // 1
+      userMsg("u1"), // 2
+      assistantText("a1"), // 3
+      userMsg("u2"), // 4  <- safe earlier boundary (>= MIN_COMPACTABLE_MESSAGES)
+      assistantText("a2"), // 5
+      assistantToolCall("calling tool", "search"), // 6  <- would be last-compacted if split at 7
+      userMsg("u3 (tool result delivered as user)"), // 7  <- candidate boundary
+      toolResult("result"), // 8
+      assistantText("a3"), // 9
+    ];
+    // candidate = 10 - 3 = 7 (a user message), but messages[6] is an assistant
+    // with tool_calls → orphan. Must walk back to index 4.
+    const idx = findSafeSplitPoint(messages, 3);
+    expect(idx).toBe(4);
+    // Confirm the chosen split does NOT end the compacted side on a dangling
+    // assistant-with-tool_calls.
+    const lastCompacted = messages[idx - 1]!;
+    expect(
+      typeof lastCompacted.content === "string" &&
+        lastCompacted.content.includes('"tool_calls"'),
+    ).toBe(false);
+  });
+});
+
+describe("compactMessages", () => {
+  const config = resolveCompactionConfig({ keepRecentMessages: 2 });
+
+  it("compacts older messages into a summary continuation message", async () => {
+    const { model } = fakeModel("SUMMARY TEXT");
+    const messages: Message[] = [
+      userMsg("u0"),
+      assistantText("a0"),
+      userMsg("u1"),
+      assistantText("a1"),
+      userMsg("u2"),
+      assistantText("a2"),
+    ];
+    const res = await compactMessages(model, messages, config);
+    expect(res.compacted).toBe(true);
+    expect(res.messages[0]!.metadata?.isCompactionSummary).toBe(true);
+    expect(res.messages[0]!.content).toContain("SUMMARY TEXT");
+    // No subagents → no ledger block.
+    expect(res.messages[0]!.content).not.toContain("## Subagents");
+  });
+
+  it("appends a verbatim subagent ledger after the LLM summary", async () => {
+    const { model } = fakeModel("SUMMARY TEXT");
+    const messages: Message[] = [
+      userMsg("u0"),
+      assistantText("a0"),
+      userMsg(
+        '[Subagent Result] Subagent "research the API" (sub_abc) completed:\n\nFound that the endpoint returns JSON with a data array. Use /v2/items.',
+        {
+          _subagentCallback: true,
+          subagentId: "sub_abc",
+          task: "research the API",
+        } as Message["metadata"],
+      ),
+      assistantText("a1"),
+      userMsg("u2"),
+      assistantText("a2"),
+    ];
+    const res = await compactMessages(model, messages, config);
+    expect(res.compacted).toBe(true);
+    const content = res.messages[0]!.content as string;
+    expect(content).toContain("## Subagents");
+    expect(content).toContain("sub_abc");
+    expect(content).toContain("research the API");
+    // Digest carries the verbatim result body.
+    expect(content).toContain("endpoint returns JSON");
+    // Ledger comes AFTER the summary text.
+    expect(content.indexOf("SUMMARY TEXT")).toBeLessThan(
+      content.indexOf("## Subagents"),
+    );
+  });
+
+  it("detects subagent callbacks by text marker even without metadata", async () => {
+    const { model } = fakeModel("S");
+    const messages: Message[] = [
+      userMsg("u0"),
+      assistantText("a0"),
+      userMsg(
+        '[Subagent Result] Subagent "compile report" (sub_xyz) completed:\n\nThe report is ready.',
+      ),
+      assistantText("a1"),
+      userMsg("u2"),
+      assistantText("a2"),
+    ];
+    const res = await compactMessages(model, messages, config);
+    const content = res.messages[0]!.content as string;
+    expect(content).toContain("sub_xyz");
+    expect(content).toContain("compile report");
+  });
+
+  it("carries forward a prior ledger and dedupes by subagentId", async () => {
+    const { model } = fakeModel("NEW SUMMARY");
+    // First compacted message is itself a prior compaction summary that
+    // already embeds a ## Subagents block for sub_abc and sub_old.
+    const priorSummary: Message = {
+      role: "user",
+      content: [
+        "[CONTEXT COMPACTION] prior.",
+        "<summary>",
+        "Earlier work done.",
+        "",
+        "## Subagents",
+        "- **research the API** (sub_abc) — completed",
+        "  Old digest about the API.",
+        "- **legacy task** (sub_old) — completed",
+        "  Legacy digest text.",
+        "</summary>",
+      ].join("\n"),
+      metadata: { isCompactionSummary: true },
+    };
+    const messages: Message[] = [
+      priorSummary,
+      assistantText("a0"),
+      // A fresh callback for sub_abc should OVERRIDE the prior entry.
+      userMsg(
+        '[Subagent Result] Subagent "research the API" (sub_abc) completed:\n\nUpdated finding: the endpoint moved to /v3/items.',
+        {
+          _subagentCallback: true,
+          subagentId: "sub_abc",
+          task: "research the API",
+        } as Message["metadata"],
+      ),
+      assistantText("a1"),
+      userMsg("u2"),
+      assistantText("a2"),
+    ];
+    const res = await compactMessages(model, messages, config);
+    const content = res.messages[0]!.content as string;
+    // Both subagents present.
+    expect(content).toContain("sub_abc");
+    expect(content).toContain("sub_old");
+    // sub_abc appears exactly once (deduped).
+    const occurrences = content.split("sub_abc").length - 1;
+    expect(occurrences).toBe(1);
+    // The newer digest won.
+    expect(content).toContain("/v3/items");
+    expect(content).not.toContain("Old digest about the API");
+  });
+
+  it("passes a prior summary in full (no 1200-char truncation) and adds the merge instruction", async () => {
+    const { model, prompts } = fakeModel("MERGED");
+    const longPrior = "PRIOR-STATE ".repeat(200); // ~2400 chars, > 1200
+    const priorSummary: Message = {
+      role: "user",
+      content: longPrior,
+      metadata: { isCompactionSummary: true },
+    };
+    const messages: Message[] = [
+      priorSummary,
+      assistantText("a0"),
+      userMsg("u1"),
+      assistantText("a1"),
+      userMsg("u2"),
+      assistantText("a2"),
+    ];
+    await compactMessages(model, messages, config);
+    const sentPrompt = prompts.join("\n");
+    // The whole prior summary text was sent, untruncated.
+    expect(sentPrompt).toContain(longPrior.trim());
+    expect(sentPrompt).not.toContain("[truncated]");
+    // Tagged as prior-summary, with the merge-and-update instruction.
+    expect(sentPrompt).toContain("[prior-summary]");
+    expect(sentPrompt).toContain("MERGE AND UPDATE");
+  });
+
+  it("still truncates non-prior-summary long messages to 1200 chars", async () => {
+    const { model, prompts } = fakeModel("S");
+    const longUser = "X".repeat(3000);
+    const messages: Message[] = [
+      userMsg(longUser),
+      assistantText("a0"),
+      userMsg("u1"),
+      assistantText("a1"),
+      userMsg("u2"),
+      assistantText("a2"),
+    ];
+    await compactMessages(model, messages, config);
+    const sentPrompt = prompts.join("\n");
+    expect(sentPrompt).toContain("[truncated]");
+    // The first message was NOT a prior summary, so no merge instruction.
+    expect(sentPrompt).not.toContain("MERGE AND UPDATE");
+  });
+});

package/test/entries.test.ts

@@ -0,0 +1,125 @@
+import { describe, it, expect } from "vitest";
+import {
+  buildLlmContext,
+  buildDisplaySnapshot,
+  getPendingSubagentResults,
+  type ConversationEntry,
+} from "../src/storage/entries.js";
+import type { Message } from "@poncho-ai/sdk";
+
+const msg = (role: Message["role"], content: string): Message => ({ role, content });
+
+let seq = 0;
+const reset = () => { seq = 0; };
+const next = () => ++seq;
+
+const harness = (content: string, turnId = "t1"): ConversationEntry => ({
+  type: "harness_message", id: `h${seq + 1}`, seq: next(), createdAt: 0,
+  message: msg("assistant", content), turnId,
+});
+const user = (content: string, opts: { hidden?: boolean } = {}): ConversationEntry => ({
+  type: "user_message", id: `u${seq + 1}`, seq: next(), createdAt: 0,
+  message: msg("user", content), turnId: "t1", hidden: opts.hidden,
+});
+const assistant = (id: string, content: string): ConversationEntry => ({
+  type: "assistant_message", id, seq: next(), createdAt: 0,
+  message: msg("assistant", content), turnId: "t1", runId: "r1",
+});
+
+describe("buildLlmContext", () => {
+  it("returns all harness messages in order with no compaction", () => {
+    reset();
+    const entries = [harness("a"), harness("b"), harness("c")];
+    expect(buildLlmContext(entries).map((m) => m.content)).toEqual(["a", "b", "c"]);
+  });
+
+  it("applies a compaction overlay: summary + messages from firstKeptSeq", () => {
+    reset();
+    const h1 = harness("old1"); // seq 1
+    const h2 = harness("old2"); // seq 2
+    const h3 = harness("kept3"); // seq 3
+    const h4 = harness("kept4"); // seq 4
+    const compaction: ConversationEntry = {
+      type: "compaction", id: "c1", seq: next(), createdAt: 0,
+      summaryMessage: msg("user", "[summary]"), firstKeptSeq: 3,
+    };
+    const ctx = buildLlmContext([h1, h2, h3, h4, compaction]);
+    expect(ctx.map((m) => m.content)).toEqual(["[summary]", "kept3", "kept4"]);
+  });
+
+  it("uses the LATEST compaction when several exist (layered)", () => {
+    reset();
+    const h1 = harness("a");
+    const c1: ConversationEntry = { type: "compaction", id: "c1", seq: next(), createdAt: 0, summaryMessage: msg("user", "[sum1]"), firstKeptSeq: 1 };
+    const h2 = harness("b"); // seq 3
+    const c2: ConversationEntry = { type: "compaction", id: "c2", seq: next(), createdAt: 0, summaryMessage: msg("user", "[sum2]"), firstKeptSeq: 3 };
+    const ctx = buildLlmContext([h1, c1, h2, c2]);
+    expect(ctx.map((m) => m.content)).toEqual(["[sum2]", "b"]);
+  });
+});
+
+describe("buildDisplaySnapshot", () => {
+  it("drops hidden user messages and returns the tail", () => {
+    reset();
+    const entries = [
+      user("hidden-framed", { hidden: true }),
+      user("hello"),
+      assistant("a1", "hi"),
+      user("again"),
+      assistant("a2", "yo"),
+    ];
+    const snap = buildDisplaySnapshot(entries, 10);
+    expect(snap.messages.map((m) => m.content)).toEqual(["hello", "hi", "again", "yo"]);
+    expect(snap.totalMessages).toBe(4);
+  });
+
+  it("folds amendments into their target assistant message", () => {
+    reset();
+    const a = assistant("a1", "part1");
+    const amend: ConversationEntry = {
+      type: "assistant_amendment", id: "am1", seq: next(), createdAt: 0,
+      targetEntryId: "a1", appendText: " + part2",
+    };
+    const snap = buildDisplaySnapshot([user("q"), a, amend], 10);
+    expect(snap.messages.map((m) => m.content)).toEqual(["q", "part1 + part2"]);
+  });
+
+  it("returns only the trailing tailN messages", () => {
+    reset();
+    const entries = [user("1"), assistant("a", "2"), user("3"), assistant("b", "4")];
+    const snap = buildDisplaySnapshot(entries, 2);
+    expect(snap.messages.map((m) => m.content)).toEqual(["3", "4"]);
+    expect(snap.totalMessages).toBe(4);
+  });
+});
+
+describe("getPendingSubagentResults", () => {
+  const result = (subagentId: string): ConversationEntry => ({
+    type: "subagent_result", id: `sr-${subagentId}`, seq: next(), createdAt: 0,
+    result: { subagentId, task: "t", status: "completed", timestamp: 0 },
+  });
+
+  it("returns results not yet consumed by a callback", () => {
+    reset();
+    const r1 = result("s1"); // seq 1
+    const r2 = result("s2"); // seq 2
+    const callback: ConversationEntry = {
+      type: "callback_started", id: "cb1", seq: next(), createdAt: 0,
+      consumedSeqs: [1],
+    };
+    const r3 = result("s3"); // seq 4
+    const pending = getPendingSubagentResults([r1, r2, callback, r3]);
+    // s1 consumed; s2 + s3 still pending
+    expect(pending.map((p) => p.subagentId)).toEqual(["s2", "s3"]);
+  });
+
+  it("returns empty when all consumed", () => {
+    reset();
+    const r1 = result("s1");
+    const callback: ConversationEntry = {
+      type: "callback_started", id: "cb1", seq: next(), createdAt: 0,
+      consumedSeqs: [1],
+    };
+    expect(getPendingSubagentResults([r1, callback])).toEqual([]);
+  });
+});