@poncho-ai/harness 0.53.0 → 0.57.0

package/src/storage/entries.ts

@@ -0,0 +1,217 @@
+import type { Message } from "@poncho-ai/sdk";
+import type { PendingSubagentResult } from "../state.js";
+
+/**
+ * Append-only conversation entries (Phase 3 substrate).
+ *
+ * The eventual replacement for the mutable per-conversation JSON blob: a
+ * conversation becomes an ordered, append-only list of entries, and the
+ * mutable-blob clobber race (two writers serializing a stale whole-blob
+ * snapshot over each other — the root cause behind lost subagent results)
+ * stops being expressible.
+ *
+ * This module is intentionally PURE: it defines the entry shapes and the
+ * functions that rebuild a conversation's LLM context / display transcript
+ * / pending-subagent-results from an entry list. No storage engine, no DB,
+ * no wiring into the live run loop yet — so it deploys nothing and is
+ * fully unit-testable. The engine implementations (append/read on
+ * postgres/sqlite/memory) and the write-site conversions come in later PRs
+ * once this rebuild logic is proven.
+ *
+ * Ordering: every entry carries a monotonic per-conversation `seq`. Entries
+ * are assumed sorted by `seq` ascending when passed to the rebuild fns.
+ */
+
+interface BaseEntry {
+  /** Stable cross-reference id (uuid). */
+  id: string;
+  /** Monotonic per-conversation order. */
+  seq: number;
+  createdAt: number;
+}
+
+/** A user-role display message (incl. typed subagent-callback messages). */
+export interface UserMessageEntry extends BaseEntry {
+  type: "user_message";
+  message: Message;
+  turnId: string;
+  /** Hidden from the display transcript (e.g. a framed job prompt, an
+   *  onboarding seed, or an injected subagent-result message). Still part
+   *  of the record; just not rendered as a chat bubble. */
+  hidden?: boolean;
+}
+
+/** The final assistant bubble for a completed/cancelled/errored turn. */
+export interface AssistantMessageEntry extends BaseEntry {
+  type: "assistant_message";
+  message: Message;
+  turnId: string;
+  runId: string;
+}
+
+/** A post-hoc edit to an already-emitted assistant message — replaces the
+ *  orchestrator/resume "mutate the last assistant message in place" writes
+ *  with an append. Applied at rebuild time. */
+export interface AssistantAmendmentEntry extends BaseEntry {
+  type: "assistant_amendment";
+  targetEntryId: string;
+  appendText?: string;
+}
+
+/** One LLM-transcript message (the model-visible form). Appended from the
+ *  run loop per step — never diffed from an array. */
+export interface HarnessMessageEntry extends BaseEntry {
+  type: "harness_message";
+  message: Message;
+  turnId: string;
+}
+
+/** Compaction overlay: nothing is deleted. At rebuild, the LLM context is
+ *  the latest compaction's `summaryMessage` followed by the harness
+ *  messages from `firstKeptSeq` onward. */
+export interface CompactionEntry extends BaseEntry {
+  type: "compaction";
+  summaryMessage: Message;
+  firstKeptSeq: number;
+  tokensBefore?: number;
+  tokensAfter?: number;
+}
+
+/** A finished subagent's result arriving for the parent. Pending = a
+ *  subagent_result whose seq is not listed in any later callback_started. */
+export interface SubagentResultEntry extends BaseEntry {
+  type: "subagent_result";
+  result: PendingSubagentResult;
+}
+
+/** Marks which subagent_result entries a callback turn consumed (by seq).
+ *  Consumption is an append, never a delete. */
+export interface CallbackStartedEntry extends BaseEntry {
+  type: "callback_started";
+  consumedSeqs: number[];
+}
+
+export type ConversationEntry =
+  | UserMessageEntry
+  | AssistantMessageEntry
+  | AssistantAmendmentEntry
+  | HarnessMessageEntry
+  | CompactionEntry
+  | SubagentResultEntry
+  | CallbackStartedEntry;
+
+/**
+ * An entry to append, before the engine assigns `seq` and `createdAt`. This
+ * is a DISTRIBUTIVE omit — `Omit<ConversationEntry, K>` over a union would
+ * collapse to only the keys common to every member (dropping `message`,
+ * `summaryMessage`, etc.), so we distribute over the union with a
+ * conditional type to omit those fields from each member individually.
+ */
+export type NewConversationEntry = ConversationEntry extends infer T
+  ? T extends ConversationEntry
+    ? Omit<T, "seq" | "createdAt">
+    : never
+  : never;
+
+/**
+ * Rebuild the LLM-visible message context from the entry log.
+ *
+ * If a compaction overlay exists, the context is its summary message
+ * followed by every harness message with seq >= firstKeptSeq (a later
+ * compaction's firstKeptSeq can point at an earlier summary that was
+ * itself appended as a harness message, so layered compactions just work).
+ * With no compaction, it's every harness message in order.
+ */
+export function buildLlmContext(entries: ConversationEntry[]): Message[] {
+  let latestCompaction: CompactionEntry | undefined;
+  for (const e of entries) {
+    if (e.type === "compaction" && (!latestCompaction || e.seq > latestCompaction.seq)) {
+      latestCompaction = e;
+    }
+  }
+
+  const harnessMsgs = entries.filter(
+    (e): e is HarnessMessageEntry => e.type === "harness_message",
+  );
+
+  if (latestCompaction) {
+    const kept = harnessMsgs
+      .filter((e) => e.seq >= latestCompaction!.firstKeptSeq)
+      .map((e) => e.message);
+    return [latestCompaction.summaryMessage, ...kept];
+  }
+  return harnessMsgs.map((e) => e.message);
+}
+
+export interface DisplaySnapshot {
+  messages: Message[];
+  /** Total display messages available (for pagination UIs). */
+  totalMessages: number;
+  /** seq of the first message returned (a `beforeSeq` pagination cursor). */
+  headSeq: number | null;
+}
+
+/**
+ * Rebuild the display transcript (the user-visible chat) from the entry
+ * log, returning the trailing `tailN` messages. Amendments are folded into
+ * their target assistant message; hidden user messages are dropped.
+ */
+export function buildDisplaySnapshot(
+  entries: ConversationEntry[],
+  tailN: number,
+): DisplaySnapshot {
+  const amendmentsByTarget = new Map<string, AssistantAmendmentEntry[]>();
+  for (const e of entries) {
+    if (e.type === "assistant_amendment") {
+      const list = amendmentsByTarget.get(e.targetEntryId) ?? [];
+      list.push(e);
+      amendmentsByTarget.set(e.targetEntryId, list);
+    }
+  }
+
+  const built: { seq: number; message: Message }[] = [];
+  for (const e of entries) {
+    if (e.type === "user_message") {
+      if (e.hidden) continue;
+      built.push({ seq: e.seq, message: e.message });
+    } else if (e.type === "assistant_message") {
+      let content = typeof e.message.content === "string" ? e.message.content : "";
+      const amendments = amendmentsByTarget.get(e.id);
+      if (amendments) {
+        for (const a of amendments.sort((x, y) => x.seq - y.seq)) {
+          if (a.appendText) content += a.appendText;
+        }
+      }
+      built.push({ seq: e.seq, message: { ...e.message, content } });
+    }
+  }
+
+  const total = built.length;
+  const tail = tailN >= total ? built : built.slice(total - tailN);
+  return {
+    messages: tail.map((b) => b.message),
+    totalMessages: total,
+    headSeq: tail.length > 0 ? tail[0]!.seq : null,
+  };
+}
+
+/**
+ * Subagent results that have arrived but not yet been consumed by a
+ * callback turn — the append-only replacement for the mutable
+ * `pendingSubagentResults` array. A result is pending unless a later
+ * callback_started lists its seq in `consumedSeqs`.
+ */
+export function getPendingSubagentResults(
+  entries: ConversationEntry[],
+): PendingSubagentResult[] {
+  const consumed = new Set<number>();
+  for (const e of entries) {
+    if (e.type === "callback_started") {
+      for (const s of e.consumedSeqs) consumed.add(s);
+    }
+  }
+  return entries
+    .filter((e): e is SubagentResultEntry => e.type === "subagent_result")
+    .filter((e) => !consumed.has(e.seq))
+    .map((e) => e.result);
+}

package/src/storage/memory-engine.ts

@@ -14,6 +14,7 @@ import type { MainMemory } from "../memory.js";
 import type { TodoItem } from "../todo-tools.js";
 import type { Reminder, ReminderCreateInput, ReminderStatus } from "../reminder-store.js";
 import type { StorageEngine, VfsDirEntry, VfsStat } from "./engine.js";
+import type { ConversationEntry, NewConversationEntry } from "./entries.js";
 
 // ---------------------------------------------------------------------------
 // Internal VFS entry type
@@ -59,6 +60,8 @@ export class InMemoryEngine implements StorageEngine {
 
   // Conversation data
   private convs = new Map<string, Conversation>();
+  // Append-only conversation entries (Phase 3 substrate)
+  private entries = new Map<string, ConversationEntry[]>();
   // Memory data
   private mem = new Map<string, MainMemory>();
   // Todos data
@@ -239,6 +242,43 @@ export class InMemoryEngine implements StorageEngine {
       results.sort((a, b) => b.updatedAt - a.updatedAt);
       return results;
     },
+
+    appendEntries: async (
+      conversationId: string,
+      _agentId: string,
+      _tenantId: string | null,
+      entries: NewConversationEntry[],
+    ): Promise<ConversationEntry[]> => {
+      const list = this.entries.get(conversationId) ?? [];
+      // seq is per-conversation: max existing seq + 1, then consecutive.
+      let nextSeq = list.reduce((max, e) => (e.seq > max ? e.seq : max), 0) + 1;
+      const now = Date.now();
+      const stored: ConversationEntry[] = entries.map(
+        (e) => ({ ...e, seq: nextSeq++, createdAt: now }) as ConversationEntry,
+      );
+      this.entries.set(conversationId, [...list, ...stored]);
+      return stored;
+    },
+
+    readEntries: async (
+      conversationId: string,
+      opts?: { types?: string[]; afterSeq?: number; limit?: number },
+    ): Promise<ConversationEntry[]> => {
+      let list = (this.entries.get(conversationId) ?? [])
+        .slice()
+        .sort((a, b) => a.seq - b.seq);
+      if (opts?.types && opts.types.length > 0) {
+        const allowed = new Set(opts.types);
+        list = list.filter((e) => allowed.has(e.type));
+      }
+      if (typeof opts?.afterSeq === "number") {
+        list = list.filter((e) => e.seq > opts.afterSeq!);
+      }
+      if (typeof opts?.limit === "number") {
+        list = list.slice(0, opts.limit);
+      }
+      return list;
+    },
   };
 
   // -----------------------------------------------------------------------

package/src/storage/schema.ts

@@ -214,4 +214,34 @@ export const migrations: Migration[] = [
       ];
     },
   },
+  {
+    version: 8,
+    name: "conversation_entries",
+    // Append-only conversation log (Phase 3 substrate). Additive: no
+    // existing table or behavior changes. `seq` is a per-conversation
+    // monotonic order assigned by the application (NOT an autoincrement
+    // serial), so the same seq space restarts at 1 for every conversation.
+    // The UNIQUE (conversation_id, seq) constraint backstops the
+    // app-assigned ordering against concurrent writers.
+    up: (d) => {
+      const jsonType = d === "sqlite" ? "TEXT" : "JSONB";
+      const tsDefault = d === "sqlite" ? "datetime('now')" : "NOW()";
+      const autoTs = `DEFAULT (${tsDefault})`;
+      return [
+        `CREATE TABLE IF NOT EXISTS conversation_entries (
+          seq INTEGER NOT NULL,
+          id TEXT NOT NULL UNIQUE,
+          agent_id TEXT NOT NULL,
+          tenant_id TEXT NOT NULL DEFAULT '__default__',
+          conversation_id TEXT NOT NULL,
+          type TEXT NOT NULL,
+          payload ${jsonType} NOT NULL,
+          created_at TIMESTAMP ${autoTs},
+          UNIQUE (conversation_id, seq)
+        )`,
+        `CREATE INDEX IF NOT EXISTS idx_conversation_entries_seq
+          ON conversation_entries (conversation_id, seq)`,
+      ];
+    },
+  },
 ];

package/src/storage/sql-dialect.ts

@@ -22,6 +22,7 @@ import type { MainMemory } from "../memory.js";
 import type { TodoItem } from "../todo-tools.js";
 import type { Reminder, ReminderCreateInput, ReminderStatus } from "../reminder-store.js";
 import type { StorageEngine, VfsDirEntry, VfsStat } from "./engine.js";
+import type { ConversationEntry, NewConversationEntry } from "./entries.js";
 import { type DialectTag, migrations } from "./schema.js";
 
 // ---------------------------------------------------------------------------
@@ -628,6 +629,117 @@ export abstract class SqlStorageEngine implements StorageEngine {
       ]);
       return rows.map((r) => this.rowToSummary(r));
     },
+
+    appendEntries: async (
+      conversationId: string,
+      agentId: string,
+      tenantId: string | null,
+      entries: NewConversationEntry[],
+    ): Promise<ConversationEntry[]> => {
+      if (entries.length === 0) return [];
+      const tid = normalizeTenant(tenantId);
+
+      // CONCURRENCY: seq is assigned by the app as COALESCE(MAX(seq),0)+1 per
+      // conversation, inside a transaction. Two concurrent writers can read
+      // the same MAX and collide on the UNIQUE (conversation_id, seq)
+      // constraint; we let the loser's transaction fail and retry the whole
+      // compute-and-insert up to 3 times. This MAX+1+retry scheme is correct
+      // for the modest concurrency here (a single conversation rarely has
+      // simultaneous appenders). A later hardening — a per-conversation
+      // advisory lock — can replace it before any read path depends on these
+      // entries.
+      const maxAttempts = 3;
+      let lastErr: unknown;
+      for (let attempt = 0; attempt < maxAttempts; attempt++) {
+        const stored: ConversationEntry[] = [];
+        try {
+          await this.executor.transaction(async () => {
+            stored.length = 0; // reset on retry within the same closure
+            const row = await this.executor.get<{ max_seq: number | null }>(
+              rewrite(
+                "SELECT MAX(seq) AS max_seq FROM conversation_entries WHERE conversation_id = $1",
+                this.dialect,
+              ),
+              [conversationId],
+            );
+            let nextSeq = Number(row?.max_seq ?? 0) + 1;
+            const now = Date.now();
+            for (const entry of entries) {
+              const seq = nextSeq++;
+              const createdAt = now;
+              // Persist the full entry object (minus seq/createdAt, which are
+              // columns) as the JSON payload. seq, id, created_at are columns.
+              const payload = JSON.stringify(entry);
+              await this.executor.run(
+                rewrite(
+                  `INSERT INTO conversation_entries
+                     (seq, id, agent_id, tenant_id, conversation_id, type, payload, created_at)
+                   VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
+                  this.dialect,
+                ),
+                [
+                  seq,
+                  entry.id,
+                  agentId,
+                  tid,
+                  conversationId,
+                  entry.type,
+                  payload,
+                  new Date(createdAt).toISOString(),
+                ],
+              );
+              stored.push({ ...entry, seq, createdAt } as ConversationEntry);
+            }
+          });
+          return stored;
+        } catch (err) {
+          lastErr = err;
+          // Retry on any failure (the likely cause is a UNIQUE collision from
+          // a concurrent writer); the next attempt re-reads MAX(seq).
+        }
+      }
+      throw lastErr;
+    },
+
+    readEntries: async (
+      conversationId: string,
+      opts?: { types?: string[]; afterSeq?: number; limit?: number },
+    ): Promise<ConversationEntry[]> => {
+      const params: unknown[] = [conversationId];
+      let sql = "SELECT seq, id, payload, created_at FROM conversation_entries WHERE conversation_id = $1";
+      if (opts?.types && opts.types.length > 0) {
+        const placeholders = opts.types.map(
+          (_t, i) => `$${params.length + 1 + i}`,
+        );
+        sql += ` AND type IN (${placeholders.join(", ")})`;
+        params.push(...opts.types);
+      }
+      if (typeof opts?.afterSeq === "number") {
+        sql += ` AND seq > $${params.length + 1}`;
+        params.push(opts.afterSeq);
+      }
+      sql += " ORDER BY seq ASC";
+      if (typeof opts?.limit === "number") {
+        sql += ` LIMIT $${params.length + 1}`;
+        params.push(opts.limit);
+      }
+      const rows = await this.executor.all<{
+        seq: number;
+        id: string;
+        payload: unknown;
+        created_at: string;
+      }>(rewrite(sql, this.dialect), params);
+      return rows.map((r) => {
+        const parsed =
+          typeof r.payload === "string" ? JSON.parse(r.payload) : r.payload;
+        return {
+          ...parsed,
+          id: r.id,
+          seq: Number(r.seq),
+          createdAt: new Date(r.created_at).getTime(),
+        } as ConversationEntry;
+      });
+    },
   };
 
   // -----------------------------------------------------------------------

package/src/storage/store-adapters.ts

@@ -58,6 +58,14 @@ export function createConversationStoreFromEngine(
       engine.conversations.clearCallbackLock(conversationId),
     listThreads: (parentConversationId: string) =>
       engine.conversations.listThreads(parentConversationId),
+    appendEntries: (
+      conversationId: string,
+      agentId: string,
+      tenantId: string | null,
+      entries,
+    ) => engine.conversations.appendEntries(conversationId, agentId, tenantId, entries),
+    readEntries: (conversationId: string, opts) =>
+      engine.conversations.readEntries(conversationId, opts),
   };
 }