@poncho-ai/harness 0.59.1 → 0.59.3

package/src/storage/entries.ts

@@ -1,27 +1,42 @@
-import { createLogger, type Message } from "@poncho-ai/sdk";
+import { createLogger } from "@poncho-ai/sdk";
 import type { Conversation, PendingSubagentResult } from "../state.js";
 
 const entriesReadLog = createLogger("entries-read");
 
 /**
- * Append-only conversation entries (Phase 3 substrate).
+ * The subagent delivery queue: append-only `conversation_entries` rows that
+ * carry a finished subagent's result to its parent conversation.
  *
- * The eventual replacement for the mutable per-conversation JSON blob: a
- * conversation becomes an ordered, append-only list of entries, and the
- * mutable-blob clobber race (two writers serializing a stale whole-blob
- * snapshot over each other — the root cause behind lost subagent results)
- * stops being expressible.
+ * Why this exists: subagent results are the ONE conversation field with
+ * concurrent writers. A subagent finishes whenever it finishes — possibly
+ * while the parent turn is mid-stream doing whole-blob writes — so a
+ * read-modify-write on the mutable conversation row could serialize a stale
+ * snapshot over the result (the historical "lost subagent result" clobber).
+ * An append-only INSERT can't express that race. Everything single-writer
+ * (message history, metadata) stays on the conversation row, where the
+ * orchestrator's per-conversation turn serialization already makes mutation
+ * safe.
  *
- * This module is intentionally PURE: it defines the entry shapes and the
- * functions that rebuild a conversation's LLM context / display transcript
- * / pending-subagent-results from an entry list. No storage engine, no DB,
- * no wiring into the live run loop yet — so it deploys nothing and is
- * fully unit-testable. The engine implementations (append/read on
- * postgres/sqlite/memory) and the write-site conversions come in later PRs
- * once this rebuild logic is proven.
+ * Two entry types:
+ *   - `subagent_result`: a finished subagent's result, appended by the
+ *     orchestrator's result-delivery path.
+ *   - `callback_started`: marks which result entries a callback turn
+ *     consumed (by seq). Consumption is an append, never a delete — a
+ *     result is "pending" until a later callback_started lists its seq.
  *
- * Ordering: every entry carries a monotonic per-conversation `seq`. Entries
- * are assumed sorted by `seq` ascending when passed to the rebuild fns.
+ * Historical note: this module once defined a full transcript's worth of
+ * entry types (user/assistant/harness messages, compaction overlays) as
+ * groundwork for replacing the conversation blob entirely. The full read
+ * cutover shipped briefly (harness 0.58.0), proved unfaithful for callback
+ * turns, and was reverted; the unread types + dual-writes were then deleted
+ * rather than maintained as drift-prone dead weight. If a future feature
+ * needs real history semantics (editing, branching, audit), design that
+ * migration fresh — and remember the 0.58.0 lesson: an append-only log is
+ * only as good as the completeness of its writers.
+ *
+ * Ordering: every entry carries a monotonic per-conversation `seq`,
+ * assigned by the engine at append time. Entries are sorted by `seq`
+ * ascending when passed to the rebuild fn.
  */
 
 interface BaseEntry {
@@ -32,53 +47,6 @@ interface BaseEntry {
   createdAt: number;
 }
 
-/** A user-role display message (incl. typed subagent-callback messages). */
-export interface UserMessageEntry extends BaseEntry {
-  type: "user_message";
-  message: Message;
-  turnId: string;
-  /** Hidden from the display transcript (e.g. a framed job prompt, an
-   *  onboarding seed, or an injected subagent-result message). Still part
-   *  of the record; just not rendered as a chat bubble. */
-  hidden?: boolean;
-}
-
-/** The final assistant bubble for a completed/cancelled/errored turn. */
-export interface AssistantMessageEntry extends BaseEntry {
-  type: "assistant_message";
-  message: Message;
-  turnId: string;
-  runId: string;
-}
-
-/** A post-hoc edit to an already-emitted assistant message — replaces the
- *  orchestrator/resume "mutate the last assistant message in place" writes
- *  with an append. Applied at rebuild time. */
-export interface AssistantAmendmentEntry extends BaseEntry {
-  type: "assistant_amendment";
-  targetEntryId: string;
-  appendText?: string;
-}
-
-/** One LLM-transcript message (the model-visible form). Appended from the
- *  run loop per step — never diffed from an array. */
-export interface HarnessMessageEntry extends BaseEntry {
-  type: "harness_message";
-  message: Message;
-  turnId: string;
-}
-
-/** Compaction overlay: nothing is deleted. At rebuild, the LLM context is
- *  the latest compaction's `summaryMessage` followed by the harness
- *  messages from `firstKeptSeq` onward. */
-export interface CompactionEntry extends BaseEntry {
-  type: "compaction";
-  summaryMessage: Message;
-  firstKeptSeq: number;
-  tokensBefore?: number;
-  tokensAfter?: number;
-}
-
 /** A finished subagent's result arriving for the parent. Pending = a
  *  subagent_result whose seq is not listed in any later callback_started. */
 export interface SubagentResultEntry extends BaseEntry {
@@ -93,21 +61,14 @@ export interface CallbackStartedEntry extends BaseEntry {
   consumedSeqs: number[];
 }
 
-export type ConversationEntry =
-  | UserMessageEntry
-  | AssistantMessageEntry
-  | AssistantAmendmentEntry
-  | HarnessMessageEntry
-  | CompactionEntry
-  | SubagentResultEntry
-  | CallbackStartedEntry;
+export type ConversationEntry = SubagentResultEntry | CallbackStartedEntry;
 
 /**
  * An entry to append, before the engine assigns `seq` and `createdAt`. This
  * is a DISTRIBUTIVE omit — `Omit<ConversationEntry, K>` over a union would
- * collapse to only the keys common to every member (dropping `message`,
- * `summaryMessage`, etc.), so we distribute over the union with a
- * conditional type to omit those fields from each member individually.
+ * collapse to only the keys common to every member, so we distribute over
+ * the union with a conditional type to omit those fields from each member
+ * individually.
  */
 export type NewConversationEntry = ConversationEntry extends infer T
   ? T extends ConversationEntry
@@ -115,88 +76,6 @@ export type NewConversationEntry = ConversationEntry extends infer T
     : never
   : never;
 
-/**
- * Rebuild the LLM-visible message context from the entry log.
- *
- * If a compaction overlay exists, the context is its summary message
- * followed by every harness message with seq >= firstKeptSeq (a later
- * compaction's firstKeptSeq can point at an earlier summary that was
- * itself appended as a harness message, so layered compactions just work).
- * With no compaction, it's every harness message in order.
- */
-export function buildLlmContext(entries: ConversationEntry[]): Message[] {
-  let latestCompaction: CompactionEntry | undefined;
-  for (const e of entries) {
-    if (e.type === "compaction" && (!latestCompaction || e.seq > latestCompaction.seq)) {
-      latestCompaction = e;
-    }
-  }
-
-  const harnessMsgs = entries.filter(
-    (e): e is HarnessMessageEntry => e.type === "harness_message",
-  );
-
-  if (latestCompaction) {
-    const kept = harnessMsgs
-      .filter((e) => e.seq >= latestCompaction!.firstKeptSeq)
-      .map((e) => e.message);
-    return [latestCompaction.summaryMessage, ...kept];
-  }
-  return harnessMsgs.map((e) => e.message);
-}
-
-export interface DisplaySnapshot {
-  messages: Message[];
-  /** Total display messages available (for pagination UIs). */
-  totalMessages: number;
-  /** seq of the first message returned (a `beforeSeq` pagination cursor). */
-  headSeq: number | null;
-}
-
-/**
- * Rebuild the display transcript (the user-visible chat) from the entry
- * log, returning the trailing `tailN` messages. Amendments are folded into
- * their target assistant message; hidden user messages are dropped.
- */
-export function buildDisplaySnapshot(
-  entries: ConversationEntry[],
-  tailN: number,
-): DisplaySnapshot {
-  const amendmentsByTarget = new Map<string, AssistantAmendmentEntry[]>();
-  for (const e of entries) {
-    if (e.type === "assistant_amendment") {
-      const list = amendmentsByTarget.get(e.targetEntryId) ?? [];
-      list.push(e);
-      amendmentsByTarget.set(e.targetEntryId, list);
-    }
-  }
-
-  const built: { seq: number; message: Message }[] = [];
-  for (const e of entries) {
-    if (e.type === "user_message") {
-      if (e.hidden) continue;
-      built.push({ seq: e.seq, message: e.message });
-    } else if (e.type === "assistant_message") {
-      let content = typeof e.message.content === "string" ? e.message.content : "";
-      const amendments = amendmentsByTarget.get(e.id);
-      if (amendments) {
-        for (const a of amendments.sort((x, y) => x.seq - y.seq)) {
-          if (a.appendText) content += a.appendText;
-        }
-      }
-      built.push({ seq: e.seq, message: { ...e.message, content } });
-    }
-  }
-
-  const total = built.length;
-  const tail = tailN >= total ? built : built.slice(total - tailN);
-  return {
-    messages: tail.map((b) => b.message),
-    totalMessages: total,
-    headSeq: tail.length > 0 ? tail[0]!.seq : null,
-  };
-}
-
 /**
  * Subagent results that have arrived but not yet been consumed by a
  * callback turn — the append-only replacement for the mutable
@@ -219,25 +98,20 @@ export function getPendingSubagentResults(
 }
 
 /**
- * Phase 3c read cutover: rebuild a conversation's reader-facing fields from
- * the append-only entry log, with a blob fallback for conversations that
- * predate dual-write.
- *
- * Call this in every conversation `get`/`getWithArchive` path AFTER the
- * Conversation has been constructed from the stored row/blob. It:
- *   - reads the entry log via `readEntries`,
- *   - if NON-EMPTY, overrides `_harnessMessages`, `messages`, and
- *     `pendingSubagentResults` with entry-derived values,
- *   - if EMPTY (un-migrated conversation), leaves the blob-derived fields
- *     untouched (fallback),
- *   - on ANY error, logs and falls back to the blob (never throws — this is
- *     a hot read path).
+ * Read-path override: rebuild `pendingSubagentResults` from the queue.
  *
- * `_continuationMessages` and `pendingApprovals` are NOT modeled as entries
- * yet and are intentionally left as blob fields.
+ * Called in every conversation `get`/`getWithArchive` path AFTER the
+ * Conversation has been constructed from the stored row/blob. Only
+ * `pendingSubagentResults` is overridden — it's the only field with a write
+ * race; message history is written solely by the serialized turn finalize
+ * and stays on the blob. If the queue is EMPTY (conversation predates it,
+ * or simply has no subagent traffic recorded) the blob-derived value is
+ * left untouched; on ANY error this logs and falls back to the blob (hot
+ * read path — never throws).
  *
  * Kill-switch: set `PONCHO_READ_ENTRIES=0` to instantly revert to pure blob
- * reads without a deploy (rebuild is ON by default).
+ * reads without a deploy (queue reads are ON by default). The blob field is
+ * still dual-written for exactly this reason.
  *
  * NOTE: mutates `conversation` in place and returns it. Callers that hand
  * back a shared/mutable Conversation reference (the in-memory stores) MUST
@@ -247,20 +121,11 @@ export async function rebuildConversationFromEntries(
   conversation: Conversation,
   readEntries: (conversationId: string) => Promise<ConversationEntry[]>,
 ): Promise<Conversation> {
-  // Targeted append-only: only `pendingSubagentResults` is read from the
-  // entry log, because it's the ONLY conversation field with a write race
-  // (a subagent finishing mid-turn vs. the parent turn's whole-blob write).
-  // The message history (`messages` / `_harnessMessages`) is written solely
-  // by the turn finalize, which the orchestrator serializes per
-  // conversation — never raced — so it stays on the blob (known-good, and
-  // far simpler than faithfully rebuilding the LLM transcript from entries).
-  //
-  // Kill-switch: ON by default; PONCHO_READ_ENTRIES="0" reverts to the blob.
   if (process.env.PONCHO_READ_ENTRIES === "0") return conversation;
 
   try {
     const entries = await readEntries(conversation.conversationId);
-    if (entries.length === 0) return conversation; // fallback: pre-dual-write
+    if (entries.length === 0) return conversation; // fallback: pre-queue conversations
     conversation.pendingSubagentResults = getPendingSubagentResults(entries);
     return conversation;
   } catch (err) {

package/src/storage/memory-engine.ts

@@ -113,7 +113,7 @@ export class InMemoryEngine implements StorageEngine {
       // log (blob fallback for un-migrated conversations). Clone first — the
       // map holds a live mutable reference and the rebuild overrides fields.
       return rebuildConversationFromEntries({ ...c }, (id) =>
-        this.conversations.readEntries(id),
+        this.conversations.readEntries(id, { types: ["subagent_result", "callback_started"] }),
       );
     },
 
@@ -123,7 +123,7 @@ export class InMemoryEngine implements StorageEngine {
       const c = this.convs.get(conversationId);
       if (!c) return undefined;
       return rebuildConversationFromEntries({ ...c }, (id) =>
-        this.conversations.readEntries(id),
+        this.conversations.readEntries(id, { types: ["subagent_result", "callback_started"] }),
       );
     },
 

package/src/storage/sql-dialect.ts

@@ -333,7 +333,7 @@ export abstract class SqlStorageEngine implements StorageEngine {
       // append-only entry log, falling back to the blob for un-migrated
       // conversations. parseConversation returns a fresh object, so no clone.
       return rebuildConversationFromEntries(conv, (id) =>
-        this.conversations.readEntries(id),
+        this.conversations.readEntries(id, { types: ["subagent_result", "callback_started"] }),
       );
     },
 
@@ -420,7 +420,7 @@ export abstract class SqlStorageEngine implements StorageEngine {
       // Phase 3c read cutover: rebuild reader-facing fields from the entry
       // log (blob fallback for un-migrated conversations).
       return rebuildConversationFromEntries(conv, (id) =>
-        this.conversations.readEntries(id),
+        this.conversations.readEntries(id, { types: ["subagent_result", "callback_started"] }),
       );
     },
 
@@ -557,11 +557,31 @@ export abstract class SqlStorageEngine implements StorageEngine {
       conversationId: string,
       title: string,
     ): Promise<Conversation | undefined> => {
-      const conv = await this.conversations.get(conversationId);
-      if (!conv) return undefined;
-      conv.title = normalizeTitle(title);
-      await this.conversations.update(conv);
-      return conv;
+      // Targeted update — deliberately NOT get→mutate→update(). The
+      // whole-row read-modify-write races a streaming turn's per-step
+      // draft persist: rename reads the row at T0, the turn persists step
+      // N's draft at T1, rename writes T0's stale blob back at T2 and
+      // silently reverts the turn's progress.
+      //
+      // Title lives in BOTH the `title` column and the `data` blob (reads
+      // parse the blob), so update both — the blob via the database's own
+      // JSON-set function INSIDE the same UPDATE. That keeps the write
+      // atomic and server-side: no stale snapshot is ever serialized back.
+      const normalized = normalizeTitle(title);
+      // Distinct placeholders for the two title occurrences: rewrite()
+      // converts $N → ? positionally for sqlite, so reusing $1 would
+      // desync the param array.
+      const dataExpr = this.dialect.tag === "sqlite"
+        ? `json_set(data, '$.title', $2)`
+        : `jsonb_set(data, '{title}', to_jsonb($2::text))`;
+      await this.executor.run(
+        rewrite(
+          `UPDATE conversations SET title = $1, data = ${dataExpr}, updated_at = $3 WHERE id = $4`,
+          this.dialect,
+        ),
+        [normalized, normalized, new Date().toISOString(), conversationId],
+      );
+      return this.conversations.get(conversationId);
     },
 
     delete: async (conversationId: string): Promise<boolean> => {

package/test/entries-dual-write.test.ts

@@ -1,172 +1,49 @@
 import { describe, it, expect } from "vitest";
-import type { Message } from "@poncho-ai/sdk";
-import { createLogger } from "@poncho-ai/sdk";
 import { InMemoryConversationStore } from "../src/state.js";
-import {
-  buildDisplaySnapshot,
-  buildLlmContext,
-  getPendingSubagentResults,
-} from "../src/storage/entries.js";
+import { createLogger } from "@poncho-ai/sdk";
 import {
   appendEntriesSafe,
-  assistantMessageEntry,
   callbackStartedEntry,
-  compactionEntry,
-  harnessMessageEntries,
-  newHarnessMessagesThisTurn,
   subagentResultEntry,
-  userMessageEntry,
 } from "../src/orchestrator/entries-dual-write.js";
 
 const log = createLogger("test");
-const msg = (role: Message["role"], content: string): Message => ({
-  role,
-  content,
-  metadata: { id: `${role}-${content}` },
-});
-
-const conv = (id: string) => ({
-  conversationId: id,
-  ownerId: "owner-1",
-  tenantId: null as string | null,
-});
+const convRef = { conversationId: "c1", ownerId: "owner", tenantId: null };
 
-describe("entries dual-write", () => {
-  it("rebuilds llm context + display from a simulated chat turn's appends", async () => {
+describe("appendEntriesSafe (queue writer)", () => {
+  it("stamps a uuid id and returns stored entries with seq/createdAt", async () => {
     const store = new InMemoryConversationStore();
-    const c = conv("c1");
-    const turnId = "turn-1";
-
-    // Turn start: user message.
-    await appendEntriesSafe(store, c, [userMessageEntry(msg("user", "hi"), turnId)], log);
-
-    // During the turn the harness produced two model-visible messages and a
-    // final assistant bubble.
-    const harness1 = msg("user", "hi");
-    const harness2 = msg("assistant", "hello there");
-    const finalAssistant = msg("assistant", "hello there");
-    await appendEntriesSafe(
+    const stored = await appendEntriesSafe(
       store,
-      c,
+      convRef,
       [
-        ...harnessMessageEntries([harness1, harness2], turnId),
-        assistantMessageEntry(finalAssistant, turnId, "run-1"),
+        subagentResultEntry({ subagentId: "s1", task: "t", status: "completed", timestamp: 1 }),
+        callbackStartedEntry([1]),
       ],
       log,
     );
-
-    const entries = await store.readEntries("c1");
-
-    // LLM context == the harness messages in order.
-    const llm = buildLlmContext(entries);
-    expect(llm.map((m) => m.content)).toEqual(["hi", "hello there"]);
-
-    // Display == [user, assistant] (final assistant bubble; harness msgs hidden).
-    const snap = buildDisplaySnapshot(entries, 100);
-    expect(snap.messages.map((m) => [m.role, m.content])).toEqual([
-      ["user", "hi"],
-      ["assistant", "hello there"],
-    ]);
-    expect(snap.totalMessages).toBe(2);
+    expect(stored).toHaveLength(2);
+    expect(stored.every((e) => typeof e.id === "string" && e.id.length > 0)).toBe(true);
+    expect(stored.map((e) => e.seq)).toEqual([1, 2]);
   });
 
-  it("compaction overlay keeps summary + tail at rebuild", async () => {
+  it("returns [] on empty input without touching the store", async () => {
     const store = new InMemoryConversationStore();
-    const c = conv("c2");
-
-    await appendEntriesSafe(
-      store,
-      c,
-      harnessMessageEntries(
-        [msg("user", "m1"), msg("assistant", "m2"), msg("user", "m3")],
-        "t",
-      ),
-      log,
-    );
-    const before = await store.readEntries("c2");
-    // Keep only the last harness message (seq 3) after compaction.
-    const firstKeptSeq = before[before.length - 1]!.seq;
-    await appendEntriesSafe(
-      store,
-      c,
-      [compactionEntry(msg("assistant", "SUMMARY"), firstKeptSeq)],
-      log,
-    );
-
-    const llm = buildLlmContext(await store.readEntries("c2"));
-    expect(llm.map((m) => m.content)).toEqual(["SUMMARY", "m3"]);
+    expect(await appendEntriesSafe(store, convRef, [], log)).toEqual([]);
+    expect(await store.readEntries("c1")).toEqual([]);
   });
 
-  it("subagent_result + callback_started track pending consumption", async () => {
+  it("never throws — swallows store failures and returns []", async () => {
     const store = new InMemoryConversationStore();
-    const c = conv("c3");
-
+    store.appendEntries = async () => {
+      throw new Error("boom");
+    };
     const stored = await appendEntriesSafe(
       store,
-      c,
-      [
-        subagentResultEntry({
-          subagentId: "sa-1",
-          task: "do thing",
-          status: "completed",
-          timestamp: 1,
-        }),
-      ],
-      log,
-    );
-    const resultSeq = stored[0]!.seq;
-
-    // Before consumption: pending.
-    expect(getPendingSubagentResults(await store.readEntries("c3"))).toHaveLength(1);
-
-    // The callback consumes it + injects a hidden user message.
-    await appendEntriesSafe(
-      store,
-      c,
-      [
-        callbackStartedEntry([resultSeq]),
-        userMessageEntry(msg("user", "[Subagent Result] ..."), "cb-1", { hidden: true }),
-      ],
-      log,
-    );
-
-    const after = await store.readEntries("c3");
-    expect(getPendingSubagentResults(after)).toHaveLength(0);
-    // Hidden injected message does not appear in the display transcript.
-    expect(buildDisplaySnapshot(after, 100).messages).toHaveLength(0);
-  });
-
-  it("newHarnessMessagesThisTurn diffs the suffix and flags shrinks", () => {
-    const a = msg("user", "a");
-    const b = msg("assistant", "b");
-    const cc = msg("user", "c");
-
-    expect(newHarnessMessagesThisTurn(undefined, [a, b])).toEqual({
-      messages: [a, b],
-      approximate: false,
-    });
-    expect(newHarnessMessagesThisTurn([a], [a, b, cc])).toEqual({
-      messages: [b, cc],
-      approximate: false,
-    });
-    // Shrink (compaction reshaped the array) → approximate, returns full next.
-    const shrink = newHarnessMessagesThisTurn([a, b, cc], [a]);
-    expect(shrink.approximate).toBe(true);
-    expect(shrink.messages).toEqual([a]);
-  });
-
-  it("appendEntriesSafe swallows store errors and returns []", async () => {
-    const brokenStore = {
-      appendEntries: async () => {
-        throw new Error("boom");
-      },
-    } as unknown as InMemoryConversationStore;
-    const result = await appendEntriesSafe(
-      brokenStore,
-      conv("c4"),
-      [userMessageEntry(msg("user", "x"), "t")],
+      convRef,
+      [callbackStartedEntry([1])],
       log,
     );
-    expect(result).toEqual([]);
+    expect(stored).toEqual([]);
   });
 });