@poncho-ai/harness 0.59.2 → 0.59.3

package/src/orchestrator/entries-dual-write.ts

@@ -1,41 +1,29 @@
 // ---------------------------------------------------------------------------
-// Phase 3b — dual-write + parity checker (instrumentation only)
+// Subagent delivery-queue writers.
 //
-// At each conversation WRITE site we ALSO append the corresponding
-// append-only `ConversationEntry`s alongside the existing mutable-blob write.
-// READ paths are untouched: nothing consumes these entries yet, so a bug here
-// can only mislog — it cannot corrupt behavior. The blob remains the source of
-// truth until the read-cutover PR (3c).
+// `conversation_entries` is the append-only queue that carries a finished
+// subagent's result to its parent conversation (see storage/entries.ts for
+// the full rationale — short version: subagent results are the one
+// conversation field with concurrent writers, so they're delivered by
+// INSERT instead of blob read-modify-write). This module owns the write
+// side: a safe append wrapper + the two entry builders.
 //
-// Two public surfaces:
-//   - `appendEntriesSafe(...)` — fire-and-forget wrapper that swallows every
-//     error (so a dual-write failure never breaks a live turn) and stamps a
-//     uuid `id` on each entry (the engine inserts `entry.id` as a column).
-//   - `verifyEntriesParity(...)` — gated on `PONCHO_VERIFY_ENTRIES === "1"`,
-//     rebuilds LLM context + display snapshot from the entry log and diffs
-//     them against the blob's `_harnessMessages` / `messages`. Logs mismatches
-//     under `[entries-parity]`. Never throws.
+// (This file once held a full transcript dual-write + parity checker — the
+// Phase 3 groundwork for replacing the conversation blob. That migration
+// was deliberately abandoned after the 0.58.0 cutover incident; the unread
+// entry types and their writers were deleted rather than maintained as
+// drift-prone dead weight.)
 // ---------------------------------------------------------------------------
 
 import { randomUUID } from "node:crypto";
-import { getTextContent, type Message } from "@poncho-ai/sdk";
 import type { Logger } from "@poncho-ai/sdk";
 import type { Conversation, ConversationStore, PendingSubagentResult } from "../state.js";
-import {
-  buildDisplaySnapshot,
-  buildLlmContext,
-  type ConversationEntry,
-  type NewConversationEntry,
-} from "../storage/entries.js";
-
-/** True when dual-write parity verification is opted in via env. */
-export const entriesParityEnabled = (): boolean =>
-  process.env.PONCHO_VERIFY_ENTRIES === "1";
+import type { ConversationEntry, NewConversationEntry } from "../storage/entries.js";
 
 // DISTRIBUTIVE omit (same reasoning as NewConversationEntry in entries.ts): a
 // plain Omit<NewConversationEntry, "id"> over a union collapses to the keys
-// common to every member, dropping `message`/`result`/etc. Distribute over the
-// union so each member keeps its own discriminant fields.
+// common to every member, dropping `result`/`consumedSeqs`. Distribute over
+// the union so each member keeps its own discriminant fields.
 type NewEntryNoId = NewConversationEntry extends infer T
   ? T extends NewConversationEntry
     ? Omit<T, "id">
@@ -43,15 +31,13 @@ type NewEntryNoId = NewConversationEntry extends infer T
   : never;
 
 /**
- * Append entries to the conversation's append-only log, mirroring an existing
- * blob write. Best-effort and non-blocking by contract:
+ * Append entries to the conversation's queue. Best-effort by contract:
  *   - stamps a fresh uuid `id` on each entry (required input column),
  *   - never throws (logs and returns [] on failure),
- *   - is safe to `void` (callers needn't await).
+ *   - safe to `void` when the caller doesn't need the stored rows.
  *
- * Returns the stored entries (with seq/createdAt) for callers that want them
- * (e.g. to learn the assistant entry's id for a later amendment), or [] on
- * empty input / failure.
+ * Returns the stored entries (with seq/createdAt) — the callback path needs
+ * the seqs to record consumption.
  */
 export const appendEntriesSafe = async (
   store: ConversationStore,
@@ -72,7 +58,7 @@ export const appendEntriesSafe = async (
     );
   } catch (err) {
     log.error(
-      `[entries-dual-write] append failed for ${conversation.conversationId}: ${
+      `[entries-queue] append failed for ${conversation.conversationId}: ${
         err instanceof Error ? err.message : String(err)
       }`,
     );
@@ -80,48 +66,6 @@ export const appendEntriesSafe = async (
   }
 };
 
-// --- entry builders (pure; centralize the best-effort derivation) ----------
-
-export const userMessageEntry = (
-  message: Message,
-  turnId: string,
-  opts?: { hidden?: boolean },
-): NewEntryNoId => ({
-  type: "user_message",
-  message,
-  turnId,
-  ...(opts?.hidden ? { hidden: true } : {}),
-});
-
-export const assistantMessageEntry = (
-  message: Message,
-  turnId: string,
-  runId: string,
-): NewEntryNoId => ({
-  type: "assistant_message",
-  message,
-  turnId,
-  runId,
-});
-
-export const harnessMessageEntries = (
-  messages: Message[],
-  turnId: string,
-): NewEntryNoId[] =>
-  messages.map((message) => ({ type: "harness_message", message, turnId }));
-
-export const compactionEntry = (
-  summaryMessage: Message,
-  firstKeptSeq: number,
-  opts?: { tokensBefore?: number; tokensAfter?: number },
-): NewEntryNoId => ({
-  type: "compaction",
-  summaryMessage,
-  firstKeptSeq,
-  ...(opts?.tokensBefore !== undefined ? { tokensBefore: opts.tokensBefore } : {}),
-  ...(opts?.tokensAfter !== undefined ? { tokensAfter: opts.tokensAfter } : {}),
-});
-
 export const subagentResultEntry = (
   result: PendingSubagentResult,
 ): NewEntryNoId => ({ type: "subagent_result", result });
@@ -130,136 +74,3 @@ export const callbackStartedEntry = (consumedSeqs: number[]): NewEntryNoId => ({
   type: "callback_started",
   consumedSeqs,
 });
-
-export const assistantAmendmentEntry = (
-  targetEntryId: string,
-  appendText: string,
-): NewEntryNoId => ({
-  type: "assistant_amendment",
-  targetEntryId,
-  ...(appendText ? { appendText } : {}),
-});
-
-// --- "new harness messages this turn" diff ---------------------------------
-
-/**
- * The harness messages added during the just-finished turn — i.e. the suffix
- * of the new `_harnessMessages` array beyond what was there before the turn.
- *
- * BEST-EFFORT: the blob replaces `_harnessMessages` wholesale (it's not an
- * append log), so we recover "what's new" by length-diffing prev vs next.
- * When a compaction collapsed history this turn, `next` can be SHORTER than
- * `prev`; in that case there's no clean suffix and we return the whole `next`
- * so the entry log still ends up with the model-visible context (parity will
- * flag the over-count for review). The compaction entry (appended separately)
- * is what makes rebuild correct in that case.
- */
-export const newHarnessMessagesThisTurn = (
-  prev: Message[] | undefined,
-  next: Message[] | undefined,
-): { messages: Message[]; approximate: boolean } => {
-  const prevArr = prev ?? [];
-  const nextArr = next ?? [];
-  if (nextArr.length === 0) return { messages: [], approximate: false };
-  if (prevArr.length === 0) return { messages: nextArr, approximate: false };
-  if (nextArr.length >= prevArr.length) {
-    // Assume the new array is prev + appended suffix (the common case).
-    return { messages: nextArr.slice(prevArr.length), approximate: false };
-  }
-  // next shorter than prev — compaction or a rebuild reshaped the array.
-  return { messages: nextArr, approximate: true };
-};
-
-// --- parity checker ---------------------------------------------------------
-
-/** Normalized text projection for length-insensitive content comparison. */
-const projectText = (m: Message): string => {
-  const role = m.role;
-  const text = getTextContent(m).replace(/\s+/g, " ").trim();
-  return `${role}:${text}`;
-};
-
-const projectAll = (msgs: Message[]): string[] => msgs.map(projectText);
-
-const countMismatch = (label: string, a: number, b: number): string | null =>
-  a === b ? null : `${label} length ${a} (entries) vs ${b} (blob)`;
-
-/**
- * Rebuild LLM context + display snapshot from the entry log and diff against
- * the blob. Logs under `[entries-parity]` with the conversationId. Never
- * throws. No-op unless PONCHO_VERIFY_ENTRIES === "1".
- */
-export const verifyEntriesParity = async (
-  store: ConversationStore,
-  conversationId: string,
-  blob: { harnessMessages?: Message[]; displayMessages?: Message[] },
-  log: Logger,
-): Promise<void> => {
-  if (!entriesParityEnabled()) return;
-  try {
-    const entries = await store.readEntries(conversationId);
-    const mismatches: string[] = [];
-
-    if (blob.harnessMessages) {
-      const llm = buildLlmContext(entries);
-      const lenMismatch = countMismatch(
-        "llmContext",
-        llm.length,
-        blob.harnessMessages.length,
-      );
-      if (lenMismatch) mismatches.push(lenMismatch);
-      // Compare a trailing normalized text projection. We don't require
-      // byte-equality — metadata, tool-call framing, and exact whitespace
-      // differ by construction between the two representations.
-      const entriesProj = projectAll(llm);
-      const blobProj = projectAll(blob.harnessMessages);
-      const tail = Math.min(entriesProj.length, blobProj.length, 5);
-      for (let i = 1; i <= tail; i++) {
-        const ep = entriesProj[entriesProj.length - i];
-        const bp = blobProj[blobProj.length - i];
-        if (ep !== bp) {
-          mismatches.push(
-            `llmContext tail[-${i}] differs: entries=${JSON.stringify(ep).slice(0, 120)} blob=${JSON.stringify(bp).slice(0, 120)}`,
-          );
-        }
-      }
-    }
-
-    if (blob.displayMessages) {
-      // tailN large enough to cover the whole transcript for the diff.
-      const snap = buildDisplaySnapshot(entries, Number.MAX_SAFE_INTEGER);
-      const lenMismatch = countMismatch(
-        "display",
-        snap.totalMessages,
-        blob.displayMessages.length,
-      );
-      if (lenMismatch) mismatches.push(lenMismatch);
-      const entriesProj = projectAll(snap.messages);
-      const blobProj = projectAll(blob.displayMessages);
-      const tail = Math.min(entriesProj.length, blobProj.length, 5);
-      for (let i = 1; i <= tail; i++) {
-        const ep = entriesProj[entriesProj.length - i];
-        const bp = blobProj[blobProj.length - i];
-        if (ep !== bp) {
-          mismatches.push(
-            `display tail[-${i}] differs: entries=${JSON.stringify(ep).slice(0, 120)} blob=${JSON.stringify(bp).slice(0, 120)}`,
-          );
-        }
-      }
-    }
-
-    if (mismatches.length > 0) {
-      log.warn(
-        `[entries-parity] ${conversationId} MISMATCH (${mismatches.length}): ${mismatches.join(" | ")}`,
-      );
-    } else {
-      log.info(`[entries-parity] ${conversationId} OK`);
-    }
-  } catch (err) {
-    log.error(
-      `[entries-parity] ${conversationId} checker threw (ignored): ${
-        err instanceof Error ? err.message : String(err)
-      }`,
-    );
-  }
-};

package/src/orchestrator/index.ts

@@ -62,9 +62,4 @@ export {
   type RunConversationTurnResult,
 } from "./run-conversation-turn.js";
 
-export {
-  appendEntriesSafe,
-  verifyEntriesParity,
-  entriesParityEnabled,
-  newHarnessMessagesThisTurn,
-} from "./entries-dual-write.js";
+export { appendEntriesSafe } from "./entries-dual-write.js";

package/src/orchestrator/orchestrator.ts

@@ -29,15 +29,11 @@ import {
 } from "./subagents.js";
 import {
   appendEntriesSafe,
-  assistantAmendmentEntry,
-  assistantMessageEntry,
   callbackStartedEntry,
   subagentResultEntry,
-  userMessageEntry,
-  verifyEntriesParity,
 } from "./entries-dual-write.js";
 
-const dualWriteLog = createLogger("orchestrator:entries");
+const entriesQueueLog = createLogger("orchestrator:entries");
 
 // ── Subagent result extraction ──
 
@@ -502,11 +498,6 @@ export class AgentOrchestrator {
     if (!checkpointedRun) {
       const conv = await this.conversationStore.get(conversationId);
       if (conv) {
-        // Track which dual-write branch the blob took: an in-place merge onto
-        // the previous assistant bubble (→ assistant_amendment) or a fresh
-        // bubble (→ assistant_message).
-        let amendmentText: string | undefined;
-        let pushedAssistant: Message | undefined;
         const hasAssistantContent =
           draft.assistantResponse.length > 0 || draft.toolTimeline.length > 0 || draft.sections.length > 0;
         if (hasAssistantContent) {
@@ -535,14 +526,15 @@ export class AgentOrchestrator {
                 } as Message["metadata"],
               },
             ];
-            amendmentText = draft.assistantResponse;
           } else {
-            pushedAssistant = {
-              role: "assistant" as const,
-              content: draft.assistantResponse,
-              metadata: buildAssistantMetadata(draft),
-            };
-            conv.messages = [...prevMessages, pushedAssistant];
+            conv.messages = [
+              ...prevMessages,
+              {
+                role: "assistant" as const,
+                content: draft.assistantResponse,
+                metadata: buildAssistantMetadata(draft),
+              },
+            ];
           }
         }
         applyTurnMetadata(conv, {
@@ -552,62 +544,6 @@ export class AgentOrchestrator {
           harnessMessages: execution?.runHarnessMessages,
         }, { shouldRebuildCanonical: true });
         await this.conversationStore.update(conv);
-
-        // DUAL-WRITE (mirrors the resume merge/push above). In-place merge →
-        // assistant_amendment targeting the latest assistant_message entry
-        // (BEST-EFFORT: the blob carries no entry id, so we resolve the target
-        // by reading the log's last assistant_message). Fresh bubble →
-        // assistant_message. Then parity-check. Fire-and-forget.
-        if (amendmentText !== undefined || pushedAssistant) {
-          const finalConv = conv;
-          const amendText = amendmentText;
-          const pushed = pushedAssistant;
-          void (async () => {
-            try {
-              if (pushed) {
-                await appendEntriesSafe(
-                  this.conversationStore,
-                  finalConv,
-                  [assistantMessageEntry(pushed, `resume-${conversationId}`, latestRunId)],
-                  dualWriteLog,
-                );
-              } else if (amendText !== undefined) {
-                const existing = await this.conversationStore.readEntries(
-                  conversationId,
-                  { types: ["assistant_message"] },
-                );
-                const target = existing[existing.length - 1];
-                if (target) {
-                  await appendEntriesSafe(
-                    this.conversationStore,
-                    finalConv,
-                    [assistantAmendmentEntry(target.id, amendText)],
-                    dualWriteLog,
-                  );
-                } else {
-                  dualWriteLog.warn(
-                    `[entries-dual-write] resume amendment for ${conversationId}: no assistant_message entry to target; skipped`,
-                  );
-                }
-              }
-              await verifyEntriesParity(
-                this.conversationStore,
-                conversationId,
-                {
-                  harnessMessages: finalConv._harnessMessages,
-                  displayMessages: finalConv.messages,
-                },
-                dualWriteLog,
-              );
-            } catch (err) {
-              dualWriteLog.error(
-                `[entries-dual-write] resume finalize append failed for ${conversationId}: ${
-                  err instanceof Error ? err.message : String(err)
-                }`,
-              );
-            }
-          })();
-        }
       }
     } else {
       const conv = await this.conversationStore.get(conversationId);
@@ -1263,13 +1199,13 @@ export class AgentOrchestrator {
     conversation.updatedAt = Date.now();
     await this.conversationStore.update(conversation);
 
-    // DUAL-WRITE (mirrors the consume-pending + message-push blob writes
-    // above): append a callback_started entry listing the consumed
-    // subagent_result seqs (resolved by matching subagentId against the entry
-    // log — BEST-EFFORT, since the blob's pending array carries no seq), plus
-    // a hidden user_message entry per injected callback message.
+    // QUEUE CONSUMPTION: append a callback_started entry listing the consumed
+    // subagent_result seqs (resolved by matching subagentId against the queue
+    // — the blob's pending array carries no seq), so the consumed results stop
+    // being "pending" on the entry-sourced read path. Mirrors the blob's
+    // consume-pending write above (kept fresh for the PONCHO_READ_ENTRIES=0
+    // kill-switch).
     if (pendingResults.length > 0) {
-      const turnId = `callback-${callbackCount}-${conversation.conversationId}`;
       void (async () => {
         try {
           const resultEntries = await this.conversationStore.readEntries(
@@ -1287,17 +1223,12 @@ export class AgentOrchestrator {
           await appendEntriesSafe(
             this.conversationStore,
             conversation,
-            [
-              callbackStartedEntry(consumedSeqs),
-              ...injectedCallbackMessages.map((m) =>
-                userMessageEntry(m, turnId, { hidden: true }),
-              ),
-            ],
-            dualWriteLog,
+            [callbackStartedEntry(consumedSeqs)],
+            entriesQueueLog,
           );
         } catch (err) {
-          dualWriteLog.error(
-            `[entries-dual-write] callback_started append failed for ${conversation.conversationId}: ${
+          entriesQueueLog.error(
+            `[entries-queue] callback_started append failed for ${conversation.conversationId}: ${
               err instanceof Error ? err.message : String(err)
             }`,
           );
@@ -1395,30 +1326,6 @@ export class AgentOrchestrator {
           freshConv.runningCallbackSince = undefined;
           await this.conversationStore.update(freshConv);
 
-          // DUAL-WRITE (mirrors the assistant push above): the callback re-run's
-          // final assistant bubble. Only when not continuing (a continuation has
-          // no final bubble yet). Then run the parity check on the rebuilt
-          // transcript. Fire-and-forget; never blocks.
-          if (callbackAssistantMsg) {
-            const finalMsg = callbackAssistantMsg;
-            void appendEntriesSafe(
-              this.conversationStore,
-              freshConv,
-              [assistantMessageEntry(finalMsg, `callback-${conversationId}`, execution.latestRunId)],
-              dualWriteLog,
-            ).then(() =>
-              verifyEntriesParity(
-                this.conversationStore,
-                conversationId,
-                {
-                  harnessMessages: freshConv._harnessMessages,
-                  displayMessages: freshConv.messages,
-                },
-                dualWriteLog,
-              ),
-            );
-          }
-
           // Proactive messaging notification
           if (freshConv.channelMeta && execution.draft.assistantResponse.length > 0) {
             this.hooks?.onMessagingNotify?.(conversationId, execution.draft.assistantResponse);
@@ -1887,11 +1794,11 @@ export class AgentOrchestrator {
           this.conversationStore,
           parent,
           [subagentResultEntry(result)],
-          dualWriteLog,
+          entriesQueueLog,
         );
       } catch (err) {
-        dualWriteLog.error(
-          `[entries-dual-write] subagent_result append failed for ${parentConversationId}: ${
+        entriesQueueLog.error(
+          `[entries-queue] subagent_result append failed for ${parentConversationId}: ${
             err instanceof Error ? err.message : String(err)
           }`,
         );

package/src/orchestrator/run-conversation-turn.ts

@@ -36,15 +36,6 @@ import {
   executeConversationTurn,
   flushTurnDraft,
 } from "./turn.js";
-import {
-  appendEntriesSafe,
-  assistantMessageEntry,
-  compactionEntry,
-  harnessMessageEntries,
-  newHarnessMessagesThisTurn,
-  userMessageEntry,
-  verifyEntriesParity,
-} from "./entries-dual-write.js";
 
 const log = createLogger("orchestrator");
 
@@ -207,14 +198,6 @@ export const runConversationTurn = async (
     await opts.conversationStore.update(conversation);
   };
 
-  // Snapshot the harness-message array as it stood BEFORE this turn so the
-  // finalize path can diff out the messages this turn appended (dual-write).
-  const preTurnHarnessMessages = conversation._harnessMessages
-    ? [...conversation._harnessMessages]
-    : undefined;
-  // The stable per-turn id used to group dual-write entries.
-  const turnId = assistantId;
-
   // Persist the user turn immediately so a crash mid-run still records what
   // the user said. Fire-and-forget — don't block the run.
   conversation.messages = [...historyMessages, userMessage];
@@ -227,15 +210,6 @@ export const runConversationTurn = async (
     );
   });
 
-  // DUAL-WRITE (additive, mirrors the user-turn blob write above): append a
-  // user_message entry. Fire-and-forget — never blocks or breaks the turn.
-  void appendEntriesSafe(
-    opts.conversationStore,
-    conversation,
-    [userMessageEntry(userMessage, turnId)],
-    log,
-  );
-
   try {
     const execution = await executeConversationTurn({
       harness: opts.harness,
@@ -286,48 +260,6 @@ export const runConversationTurn = async (
               ...existingHistory,
               ...preRunMessages.slice(0, removedCount),
             ];
-
-            // DUAL-WRITE (mirrors the compactedHistory blob write above): the
-            // compacted array is [summaryMessage, ...keptMessages]. BEST-EFFORT
-            // firstKeptSeq: the entry-log seqs of the kept harness messages
-            // aren't known here, so we derive a sentinel from the kept-count by
-            // reading the current max harness_message seq and pointing at the
-            // tail. We read the existing entries to compute it.
-            const summaryMessage = event.compactedMessages[0];
-            const keptCount = Math.max(0, event.compactedMessages.length - 1);
-            if (summaryMessage) {
-              void (async () => {
-                try {
-                  const existing = await opts.conversationStore.readEntries(
-                    opts.conversationId,
-                    { types: ["harness_message"] },
-                  );
-                  // firstKeptSeq = seq of the (keptCount)-th-from-last existing
-                  // harness message, so rebuild keeps exactly that many.
-                  const harnessSeqs = existing.map((e) => e.seq);
-                  const firstKeptSeq =
-                    harnessSeqs.length >= keptCount && keptCount > 0
-                      ? harnessSeqs[harnessSeqs.length - keptCount]!
-                      : (harnessSeqs[harnessSeqs.length - 1] ?? 0) + 1;
-                  await appendEntriesSafe(
-                    opts.conversationStore,
-                    conversation,
-                    [
-                      compactionEntry(summaryMessage, firstKeptSeq, {
-                        tokensBefore: conversation.contextTokens,
-                      }),
-                    ],
-                    log,
-                  );
-                } catch (err) {
-                  log.error(
-                    `[entries-dual-write] compaction append failed: ${
-                      err instanceof Error ? err.message : String(err)
-                    }`,
-                  );
-                }
-              })();
-            }
           }
         }
         if (event.type === "step:completed") {
@@ -468,46 +400,6 @@ export const runConversationTurn = async (
         { shouldRebuildCanonical },
       );
       await opts.conversationStore.update(conversation);
-
-      // DUAL-WRITE at finalize (mirrors applyTurnMetadata's _harnessMessages
-      // write + the final assistant bubble in conversation.messages):
-      //   1. harness_message entries for the messages this turn appended,
-      //   2. the final assistant_message entry.
-      // Best-effort + fire-and-forget; never blocks the return.
-      const finalAssistant =
-        conversation.messages[conversation.messages.length - 1];
-      const { messages: newHarness, approximate } = newHarnessMessagesThisTurn(
-        preTurnHarnessMessages,
-        conversation._harnessMessages,
-      );
-      if (approximate) {
-        log.warn(
-          `[entries-dual-write] ${opts.conversationId} harness-message diff approximate ` +
-            `(blob array shrank this turn — likely compaction); appended full context`,
-        );
-      }
-      const finalizeEntries = [
-        ...harnessMessageEntries(newHarness, turnId),
-        ...(finalAssistant && finalAssistant.role === "assistant"
-          ? [assistantMessageEntry(finalAssistant, turnId, latestRunId)]
-          : []),
-      ];
-      void appendEntriesSafe(
-        opts.conversationStore,
-        conversation,
-        finalizeEntries,
-        log,
-      ).then(() =>
-        verifyEntriesParity(
-          opts.conversationStore,
-          opts.conversationId,
-          {
-            harnessMessages: conversation._harnessMessages,
-            displayMessages: conversation.messages,
-          },
-          log,
-        ),
-      );
     }
 
     return {

package/src/state.ts

@@ -278,7 +278,9 @@ export class InMemoryConversationStore implements ConversationStore {
     // Phase 3c read cutover: rebuild reader-facing fields from the entry log
     // (blob fallback for un-migrated conversations). Clone first — the map
     // holds a live mutable reference and the rebuild overrides fields.
-    return rebuildConversationFromEntries({ ...c }, (id) => this.readEntries(id));
+    return rebuildConversationFromEntries({ ...c }, (id) =>
+      this.readEntries(id, { types: ["subagent_result", "callback_started"] }),
+    );
   }
 
   // In-memory stores already hold the full conversation object, so there's