@poncho-ai/harness 0.55.0 → 0.58.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@poncho-ai/harness",
-  "version": "0.55.0",
+  "version": "0.58.0",
   "description": "Agent execution runtime - conversation loop, tool dispatch, streaming",
   "repository": {
     "type": "git",

package/src/index.ts

@@ -21,6 +21,24 @@ export * from "./telemetry.js";
 export * from "./secrets-store.js";
 export * from "./storage/index.js";
 export * from "./storage/store-adapters.js";
+// Append-only conversation entries (Phase 3 substrate): types + rebuild fns.
+// appendEntries/readEntries are reachable on the ConversationStore /
+// StorageEngine.conversations surfaces already exported above.
+export {
+  buildLlmContext,
+  buildDisplaySnapshot,
+  getPendingSubagentResults,
+  type ConversationEntry,
+  type NewConversationEntry,
+  type UserMessageEntry,
+  type AssistantMessageEntry,
+  type AssistantAmendmentEntry,
+  type HarnessMessageEntry,
+  type CompactionEntry,
+  type SubagentResultEntry,
+  type CallbackStartedEntry,
+  type DisplaySnapshot,
+} from "./storage/entries.js";
 export {
   PonchoFsAdapter,
   type VirtualMount,

package/src/orchestrator/entries-dual-write.ts

@@ -0,0 +1,265 @@
+// ---------------------------------------------------------------------------
+// Phase 3b — dual-write + parity checker (instrumentation only)
+//
+// At each conversation WRITE site we ALSO append the corresponding
+// append-only `ConversationEntry`s alongside the existing mutable-blob write.
+// READ paths are untouched: nothing consumes these entries yet, so a bug here
+// can only mislog — it cannot corrupt behavior. The blob remains the source of
+// truth until the read-cutover PR (3c).
+//
+// Two public surfaces:
+//   - `appendEntriesSafe(...)` — fire-and-forget wrapper that swallows every
+//     error (so a dual-write failure never breaks a live turn) and stamps a
+//     uuid `id` on each entry (the engine inserts `entry.id` as a column).
+//   - `verifyEntriesParity(...)` — gated on `PONCHO_VERIFY_ENTRIES === "1"`,
+//     rebuilds LLM context + display snapshot from the entry log and diffs
+//     them against the blob's `_harnessMessages` / `messages`. Logs mismatches
+//     under `[entries-parity]`. Never throws.
+// ---------------------------------------------------------------------------
+
+import { randomUUID } from "node:crypto";
+import { getTextContent, type Message } from "@poncho-ai/sdk";
+import type { Logger } from "@poncho-ai/sdk";
+import type { Conversation, ConversationStore, PendingSubagentResult } from "../state.js";
+import {
+  buildDisplaySnapshot,
+  buildLlmContext,
+  type ConversationEntry,
+  type NewConversationEntry,
+} from "../storage/entries.js";
+
+/** True when dual-write parity verification is opted in via env. */
+export const entriesParityEnabled = (): boolean =>
+  process.env.PONCHO_VERIFY_ENTRIES === "1";
+
+// DISTRIBUTIVE omit (same reasoning as NewConversationEntry in entries.ts): a
+// plain Omit<NewConversationEntry, "id"> over a union collapses to the keys
+// common to every member, dropping `message`/`result`/etc. Distribute over the
+// union so each member keeps its own discriminant fields.
+type NewEntryNoId = NewConversationEntry extends infer T
+  ? T extends NewConversationEntry
+    ? Omit<T, "id">
+    : never
+  : never;
+
+/**
+ * Append entries to the conversation's append-only log, mirroring an existing
+ * blob write. Best-effort and non-blocking by contract:
+ *   - stamps a fresh uuid `id` on each entry (required input column),
+ *   - never throws (logs and returns [] on failure),
+ *   - is safe to `void` (callers needn't await).
+ *
+ * Returns the stored entries (with seq/createdAt) for callers that want them
+ * (e.g. to learn the assistant entry's id for a later amendment), or [] on
+ * empty input / failure.
+ */
+export const appendEntriesSafe = async (
+  store: ConversationStore,
+  conversation: Pick<Conversation, "conversationId" | "ownerId" | "tenantId">,
+  entries: NewEntryNoId[],
+  log: Logger,
+): Promise<ConversationEntry[]> => {
+  if (entries.length === 0) return [];
+  try {
+    const withIds = entries.map(
+      (e) => ({ id: randomUUID(), ...e }) as NewConversationEntry,
+    );
+    return await store.appendEntries(
+      conversation.conversationId,
+      conversation.ownerId,
+      conversation.tenantId ?? null,
+      withIds,
+    );
+  } catch (err) {
+    log.error(
+      `[entries-dual-write] append failed for ${conversation.conversationId}: ${
+        err instanceof Error ? err.message : String(err)
+      }`,
+    );
+    return [];
+  }
+};
+
+// --- entry builders (pure; centralize the best-effort derivation) ----------
+
+export const userMessageEntry = (
+  message: Message,
+  turnId: string,
+  opts?: { hidden?: boolean },
+): NewEntryNoId => ({
+  type: "user_message",
+  message,
+  turnId,
+  ...(opts?.hidden ? { hidden: true } : {}),
+});
+
+export const assistantMessageEntry = (
+  message: Message,
+  turnId: string,
+  runId: string,
+): NewEntryNoId => ({
+  type: "assistant_message",
+  message,
+  turnId,
+  runId,
+});
+
+export const harnessMessageEntries = (
+  messages: Message[],
+  turnId: string,
+): NewEntryNoId[] =>
+  messages.map((message) => ({ type: "harness_message", message, turnId }));
+
+export const compactionEntry = (
+  summaryMessage: Message,
+  firstKeptSeq: number,
+  opts?: { tokensBefore?: number; tokensAfter?: number },
+): NewEntryNoId => ({
+  type: "compaction",
+  summaryMessage,
+  firstKeptSeq,
+  ...(opts?.tokensBefore !== undefined ? { tokensBefore: opts.tokensBefore } : {}),
+  ...(opts?.tokensAfter !== undefined ? { tokensAfter: opts.tokensAfter } : {}),
+});
+
+export const subagentResultEntry = (
+  result: PendingSubagentResult,
+): NewEntryNoId => ({ type: "subagent_result", result });
+
+export const callbackStartedEntry = (consumedSeqs: number[]): NewEntryNoId => ({
+  type: "callback_started",
+  consumedSeqs,
+});
+
+export const assistantAmendmentEntry = (
+  targetEntryId: string,
+  appendText: string,
+): NewEntryNoId => ({
+  type: "assistant_amendment",
+  targetEntryId,
+  ...(appendText ? { appendText } : {}),
+});
+
+// --- "new harness messages this turn" diff ---------------------------------
+
+/**
+ * The harness messages added during the just-finished turn — i.e. the suffix
+ * of the new `_harnessMessages` array beyond what was there before the turn.
+ *
+ * BEST-EFFORT: the blob replaces `_harnessMessages` wholesale (it's not an
+ * append log), so we recover "what's new" by length-diffing prev vs next.
+ * When a compaction collapsed history this turn, `next` can be SHORTER than
+ * `prev`; in that case there's no clean suffix and we return the whole `next`
+ * so the entry log still ends up with the model-visible context (parity will
+ * flag the over-count for review). The compaction entry (appended separately)
+ * is what makes rebuild correct in that case.
+ */
+export const newHarnessMessagesThisTurn = (
+  prev: Message[] | undefined,
+  next: Message[] | undefined,
+): { messages: Message[]; approximate: boolean } => {
+  const prevArr = prev ?? [];
+  const nextArr = next ?? [];
+  if (nextArr.length === 0) return { messages: [], approximate: false };
+  if (prevArr.length === 0) return { messages: nextArr, approximate: false };
+  if (nextArr.length >= prevArr.length) {
+    // Assume the new array is prev + appended suffix (the common case).
+    return { messages: nextArr.slice(prevArr.length), approximate: false };
+  }
+  // next shorter than prev — compaction or a rebuild reshaped the array.
+  return { messages: nextArr, approximate: true };
+};
+
+// --- parity checker ---------------------------------------------------------
+
+/** Normalized text projection for length-insensitive content comparison. */
+const projectText = (m: Message): string => {
+  const role = m.role;
+  const text = getTextContent(m).replace(/\s+/g, " ").trim();
+  return `${role}:${text}`;
+};
+
+const projectAll = (msgs: Message[]): string[] => msgs.map(projectText);
+
+const countMismatch = (label: string, a: number, b: number): string | null =>
+  a === b ? null : `${label} length ${a} (entries) vs ${b} (blob)`;
+
+/**
+ * Rebuild LLM context + display snapshot from the entry log and diff against
+ * the blob. Logs under `[entries-parity]` with the conversationId. Never
+ * throws. No-op unless PONCHO_VERIFY_ENTRIES === "1".
+ */
+export const verifyEntriesParity = async (
+  store: ConversationStore,
+  conversationId: string,
+  blob: { harnessMessages?: Message[]; displayMessages?: Message[] },
+  log: Logger,
+): Promise<void> => {
+  if (!entriesParityEnabled()) return;
+  try {
+    const entries = await store.readEntries(conversationId);
+    const mismatches: string[] = [];
+
+    if (blob.harnessMessages) {
+      const llm = buildLlmContext(entries);
+      const lenMismatch = countMismatch(
+        "llmContext",
+        llm.length,
+        blob.harnessMessages.length,
+      );
+      if (lenMismatch) mismatches.push(lenMismatch);
+      // Compare a trailing normalized text projection. We don't require
+      // byte-equality — metadata, tool-call framing, and exact whitespace
+      // differ by construction between the two representations.
+      const entriesProj = projectAll(llm);
+      const blobProj = projectAll(blob.harnessMessages);
+      const tail = Math.min(entriesProj.length, blobProj.length, 5);
+      for (let i = 1; i <= tail; i++) {
+        const ep = entriesProj[entriesProj.length - i];
+        const bp = blobProj[blobProj.length - i];
+        if (ep !== bp) {
+          mismatches.push(
+            `llmContext tail[-${i}] differs: entries=${JSON.stringify(ep).slice(0, 120)} blob=${JSON.stringify(bp).slice(0, 120)}`,
+          );
+        }
+      }
+    }
+
+    if (blob.displayMessages) {
+      // tailN large enough to cover the whole transcript for the diff.
+      const snap = buildDisplaySnapshot(entries, Number.MAX_SAFE_INTEGER);
+      const lenMismatch = countMismatch(
+        "display",
+        snap.totalMessages,
+        blob.displayMessages.length,
+      );
+      if (lenMismatch) mismatches.push(lenMismatch);
+      const entriesProj = projectAll(snap.messages);
+      const blobProj = projectAll(blob.displayMessages);
+      const tail = Math.min(entriesProj.length, blobProj.length, 5);
+      for (let i = 1; i <= tail; i++) {
+        const ep = entriesProj[entriesProj.length - i];
+        const bp = blobProj[blobProj.length - i];
+        if (ep !== bp) {
+          mismatches.push(
+            `display tail[-${i}] differs: entries=${JSON.stringify(ep).slice(0, 120)} blob=${JSON.stringify(bp).slice(0, 120)}`,
+          );
+        }
+      }
+    }
+
+    if (mismatches.length > 0) {
+      log.warn(
+        `[entries-parity] ${conversationId} MISMATCH (${mismatches.length}): ${mismatches.join(" | ")}`,
+      );
+    } else {
+      log.info(`[entries-parity] ${conversationId} OK`);
+    }
+  } catch (err) {
+    log.error(
+      `[entries-parity] ${conversationId} checker threw (ignored): ${
+        err instanceof Error ? err.message : String(err)
+      }`,
+    );
+  }
+};

package/src/orchestrator/index.ts

@@ -61,3 +61,10 @@ export {
   type RunConversationTurnOpts,
   type RunConversationTurnResult,
 } from "./run-conversation-turn.js";
+
+export {
+  appendEntriesSafe,
+  verifyEntriesParity,
+  entriesParityEnabled,
+  newHarnessMessagesThisTurn,
+} from "./entries-dual-write.js";

package/src/orchestrator/orchestrator.ts

@@ -1,4 +1,4 @@
-import { getTextContent, type AgentEvent, type Message } from "@poncho-ai/sdk";
+import { createLogger, getTextContent, type AgentEvent, type Message } from "@poncho-ai/sdk";
 import type { Conversation, ConversationStore, PendingSubagentResult } from "../state.js";
 import type { AgentHarness } from "../harness.js";
 import type { TelemetryEmitter } from "../telemetry.js";
@@ -27,6 +27,17 @@ import {
   CALLBACK_LOCK_STALE_MS,
   STALE_SUBAGENT_THRESHOLD_MS,
 } from "./subagents.js";
+import {
+  appendEntriesSafe,
+  assistantAmendmentEntry,
+  assistantMessageEntry,
+  callbackStartedEntry,
+  subagentResultEntry,
+  userMessageEntry,
+  verifyEntriesParity,
+} from "./entries-dual-write.js";
+
+const dualWriteLog = createLogger("orchestrator:entries");
 
 // ── Subagent result extraction ──
 
@@ -491,6 +502,11 @@ export class AgentOrchestrator {
     if (!checkpointedRun) {
       const conv = await this.conversationStore.get(conversationId);
       if (conv) {
+        // Track which dual-write branch the blob took: an in-place merge onto
+        // the previous assistant bubble (→ assistant_amendment) or a fresh
+        // bubble (→ assistant_message).
+        let amendmentText: string | undefined;
+        let pushedAssistant: Message | undefined;
         const hasAssistantContent =
           draft.assistantResponse.length > 0 || draft.toolTimeline.length > 0 || draft.sections.length > 0;
         if (hasAssistantContent) {
@@ -519,15 +535,14 @@ export class AgentOrchestrator {
                 } as Message["metadata"],
               },
             ];
+            amendmentText = draft.assistantResponse;
           } else {
-            conv.messages = [
-              ...prevMessages,
-              {
-                role: "assistant" as const,
-                content: draft.assistantResponse,
-                metadata: buildAssistantMetadata(draft),
-              },
-            ];
+            pushedAssistant = {
+              role: "assistant" as const,
+              content: draft.assistantResponse,
+              metadata: buildAssistantMetadata(draft),
+            };
+            conv.messages = [...prevMessages, pushedAssistant];
           }
         }
         applyTurnMetadata(conv, {
@@ -537,6 +552,62 @@ export class AgentOrchestrator {
           harnessMessages: execution?.runHarnessMessages,
         }, { shouldRebuildCanonical: true });
         await this.conversationStore.update(conv);
+
+        // DUAL-WRITE (mirrors the resume merge/push above). In-place merge →
+        // assistant_amendment targeting the latest assistant_message entry
+        // (BEST-EFFORT: the blob carries no entry id, so we resolve the target
+        // by reading the log's last assistant_message). Fresh bubble →
+        // assistant_message. Then parity-check. Fire-and-forget.
+        if (amendmentText !== undefined || pushedAssistant) {
+          const finalConv = conv;
+          const amendText = amendmentText;
+          const pushed = pushedAssistant;
+          void (async () => {
+            try {
+              if (pushed) {
+                await appendEntriesSafe(
+                  this.conversationStore,
+                  finalConv,
+                  [assistantMessageEntry(pushed, `resume-${conversationId}`, latestRunId)],
+                  dualWriteLog,
+                );
+              } else if (amendText !== undefined) {
+                const existing = await this.conversationStore.readEntries(
+                  conversationId,
+                  { types: ["assistant_message"] },
+                );
+                const target = existing[existing.length - 1];
+                if (target) {
+                  await appendEntriesSafe(
+                    this.conversationStore,
+                    finalConv,
+                    [assistantAmendmentEntry(target.id, amendText)],
+                    dualWriteLog,
+                  );
+                } else {
+                  dualWriteLog.warn(
+                    `[entries-dual-write] resume amendment for ${conversationId}: no assistant_message entry to target; skipped`,
+                  );
+                }
+              }
+              await verifyEntriesParity(
+                this.conversationStore,
+                conversationId,
+                {
+                  harnessMessages: finalConv._harnessMessages,
+                  displayMessages: finalConv.messages,
+                },
+                dualWriteLog,
+              );
+            } catch (err) {
+              dualWriteLog.error(
+                `[entries-dual-write] resume finalize append failed for ${conversationId}: ${
+                  err instanceof Error ? err.message : String(err)
+                }`,
+              );
+            }
+          })();
+        }
       }
     } else {
       const conv = await this.conversationStore.get(conversationId);
@@ -1158,6 +1229,9 @@ export class AgentOrchestrator {
     const callbackCount = (conversation.subagentCallbackCount ?? 0) + 1;
     conversation.subagentCallbackCount = callbackCount;
 
+    // Collect the injected callback messages so the dual-write can append them
+    // as hidden user_message entries (mirroring the blob pushes below).
+    const injectedCallbackMessages: Message[] = [];
     for (const pr of pendingResults) {
       // An empty response is recoverable, not a dead end: the subagent's work
       // lives in its transcript even when it produced no closing summary (e.g.
@@ -1172,11 +1246,13 @@ export class AgentOrchestrator {
         : pr.error
           ? `Error: ${pr.error.message}`
           : "(no result)";
-      conversation.messages.push({
+      const injected: Message = {
         role: "user",
         content: `[Subagent Result] Subagent "${pr.task}" (${pr.subagentId}) ${pr.status}:\n\n${resultBody}`,
         metadata: { _subagentCallback: true, subagentId: pr.subagentId, task: pr.task, timestamp: pr.timestamp } as Message["metadata"],
-      });
+      };
+      injectedCallbackMessages.push(injected);
+      conversation.messages.push(injected);
     }
     const processedIds = new Set(pendingResults.map(pr => pr.subagentId));
     const freshForPending = await this.conversationStore.get(conversationId);
@@ -1187,6 +1263,48 @@ export class AgentOrchestrator {
     conversation.updatedAt = Date.now();
     await this.conversationStore.update(conversation);
 
+    // DUAL-WRITE (mirrors the consume-pending + message-push blob writes
+    // above): append a callback_started entry listing the consumed
+    // subagent_result seqs (resolved by matching subagentId against the entry
+    // log — BEST-EFFORT, since the blob's pending array carries no seq), plus
+    // a hidden user_message entry per injected callback message.
+    if (pendingResults.length > 0) {
+      const turnId = `callback-${callbackCount}-${conversation.conversationId}`;
+      void (async () => {
+        try {
+          const resultEntries = await this.conversationStore.readEntries(
+            conversation.conversationId,
+            { types: ["subagent_result"] },
+          );
+          const consumedIds = new Set(pendingResults.map((pr) => pr.subagentId));
+          const consumedSeqs = resultEntries
+            .filter(
+              (e) =>
+                e.type === "subagent_result" &&
+                consumedIds.has(e.result.subagentId),
+            )
+            .map((e) => e.seq);
+          await appendEntriesSafe(
+            this.conversationStore,
+            conversation,
+            [
+              callbackStartedEntry(consumedSeqs),
+              ...injectedCallbackMessages.map((m) =>
+                userMessageEntry(m, turnId, { hidden: true }),
+              ),
+            ],
+            dualWriteLog,
+          );
+        } catch (err) {
+          dualWriteLog.error(
+            `[entries-dual-write] callback_started append failed for ${conversation.conversationId}: ${
+              err instanceof Error ? err.message : String(err)
+            }`,
+          );
+        }
+      })();
+    }
+
     if (callbackCount > MAX_SUBAGENT_CALLBACK_COUNT) {
       console.warn(`[poncho][subagent-callback] Circuit breaker: ${callbackCount} callbacks for ${conversationId}, skipping re-run`);
       conversation.runningCallbackSince = undefined;
@@ -1256,12 +1374,14 @@ export class AgentOrchestrator {
       if (callbackNeedsContinuation || execution.draft.assistantResponse.length > 0 || execution.draft.toolTimeline.length > 0) {
         const freshConv = await this.conversationStore.get(conversationId);
         if (freshConv) {
+          let callbackAssistantMsg: Message | undefined;
           if (!callbackNeedsContinuation) {
-            freshConv.messages.push({
+            callbackAssistantMsg = {
               role: "assistant",
               content: execution.draft.assistantResponse,
               metadata: buildAssistantMetadata(execution.draft),
-            });
+            };
+            freshConv.messages.push(callbackAssistantMsg);
           }
           applyTurnMetadata(freshConv, {
             latestRunId: execution.latestRunId,
@@ -1275,6 +1395,30 @@ export class AgentOrchestrator {
           freshConv.runningCallbackSince = undefined;
           await this.conversationStore.update(freshConv);
 
+          // DUAL-WRITE (mirrors the assistant push above): the callback re-run's
+          // final assistant bubble. Only when not continuing (a continuation has
+          // no final bubble yet). Then run the parity check on the rebuilt
+          // transcript. Fire-and-forget; never blocks.
+          if (callbackAssistantMsg) {
+            const finalMsg = callbackAssistantMsg;
+            void appendEntriesSafe(
+              this.conversationStore,
+              freshConv,
+              [assistantMessageEntry(finalMsg, `callback-${conversationId}`, execution.latestRunId)],
+              dualWriteLog,
+            ).then(() =>
+              verifyEntriesParity(
+                this.conversationStore,
+                conversationId,
+                {
+                  harnessMessages: freshConv._harnessMessages,
+                  displayMessages: freshConv.messages,
+                },
+                dualWriteLog,
+              ),
+            );
+          }
+
           // Proactive messaging notification
           if (freshConv.channelMeta && execution.draft.assistantResponse.length > 0) {
             this.hooks?.onMessagingNotify?.(conversationId, execution.draft.assistantResponse);
@@ -1732,6 +1876,28 @@ export class AgentOrchestrator {
     parentConversationId: string,
     result: PendingSubagentResult,
   ): Promise<boolean> {
+    // DUAL-WRITE (mirrors the appendSubagentResult blob write below): append a
+    // subagent_result entry. Fire-and-forget; needs the parent's owner/tenant,
+    // fetched cheaply. Never blocks or fails the reliable append.
+    void (async () => {
+      try {
+        const parent = await this.conversationStore.get(parentConversationId);
+        if (!parent) return;
+        await appendEntriesSafe(
+          this.conversationStore,
+          parent,
+          [subagentResultEntry(result)],
+          dualWriteLog,
+        );
+      } catch (err) {
+        dualWriteLog.error(
+          `[entries-dual-write] subagent_result append failed for ${parentConversationId}: ${
+            err instanceof Error ? err.message : String(err)
+          }`,
+        );
+      }
+    })();
+
     try {
       await this.conversationStore.appendSubagentResult(parentConversationId, result);
       return true;