@poncho-ai/harness 0.60.1 → 0.61.1

package/src/orchestrator/turn.ts

@@ -1,3 +1,4 @@
+import { randomUUID } from "node:crypto";
 import type { AgentEvent, Message } from "@poncho-ai/sdk";
 import type { Conversation } from "../state.js";
 import type { AgentHarness } from "../harness.js";
@@ -341,6 +342,110 @@ export const buildApprovalCheckpoints = ({
     kind,
   }));
 
+// ── Checkpoint transcript reconstruction (single source of truth) ──
+//
+// Resuming a checkpointed turn requires rebuilding the FULL canonical model
+// transcript from what was stored at the pause. Getting this right — in the
+// canonical message space, not the display one — is subtle, and re-deriving
+// it by hand in each embedder is exactly how the model-facing transcript
+// drifted from the display transcript and silently dropped a turn's user
+// message after an approval. These helpers are that logic, exported so the
+// orchestrator's own resume AND external embedders (e.g. PonchOS, which
+// executes gated tools itself) share one implementation that can't drift.
+
+/** Reconstruct the full canonical transcript a checkpoint resumes from: the
+ *  prior history (before the checkpointed turn) + the checkpoint delta.
+ *  Handles both storage conventions via `normalizeApprovalCheckpoint` — the
+ *  initial checkpoint (base = prior length, delta) and a resume-created one
+ *  (base 0, full canonical). */
+export const assembleCheckpointMessages = (
+  conversation: Conversation,
+  checkpoint: StoredApproval,
+): Message[] => {
+  const n = normalizeApprovalCheckpoint(checkpoint, conversation.messages);
+  const base = n.baseMessageCount != null ? conversation.messages.slice(0, n.baseMessageCount) : [];
+  return [...base, ...(n.checkpointMessages ?? [])];
+};
+
+/** Pair an assistant tool-call message's `tool_calls` with externally-computed
+ *  results into the single `role:"tool"` message a continuation reads.
+ *  Returns undefined when `assistantMsg` isn't an assistant message with
+ *  parseable tool calls. Used to persist the resume's canonical history so it
+ *  matches what `continueFromToolResult` fed the model. Missing results
+ *  default to the deferred-error marker (same text the run uses). */
+export const buildToolResultMessage = (
+  assistantMsg: Message | undefined,
+  toolResults: Array<{ callId: string; toolName: string; result?: unknown; error?: string }>,
+): Message | undefined => {
+  if (assistantMsg?.role !== "assistant") return undefined;
+  let toolCalls: Array<{ id: string; name: string }> = [];
+  try {
+    const parsed = JSON.parse(typeof assistantMsg.content === "string" ? assistantMsg.content : "");
+    toolCalls = parsed.tool_calls ?? [];
+  } catch {
+    return undefined;
+  }
+  if (toolCalls.length === 0) return undefined;
+  const provided = new Map(toolResults.map((r) => [r.callId, r]));
+  return {
+    role: "tool",
+    content: JSON.stringify(
+      toolCalls.map((tc) => {
+        const r = provided.get(tc.id);
+        return {
+          type: "tool_result",
+          tool_use_id: tc.id,
+          tool_name: r?.toolName ?? tc.name,
+          content: r
+            ? (r.error ? `Tool error: ${r.error}` : JSON.stringify(r.result ?? null))
+            : "Tool error: Tool execution deferred (pending approval checkpoint)",
+        };
+      }),
+    ),
+    metadata: { timestamp: Date.now(), id: randomUUID() },
+  };
+};
+
+/** Build the `pendingApprovals` rows for a checkpoint reached DURING a resume
+ *  continuation. Stores the WHOLE canonical history the continuation ran with
+ *  (`priorMessages` = prior + tool result + new delta) with
+ *  `baseMessageCount: 0`. Keeping the full history here — rather than a delta
+ *  + a base count into a different message array — is what lets the next
+ *  resume reconstruct with no index arithmetic. */
+export const buildResumeCheckpoints = ({
+  priorMessages,
+  checkpointEvent,
+  runId,
+  kind = "approval",
+}: {
+  priorMessages: Message[];
+  checkpointEvent: {
+    approvals: ApprovalEventItem[];
+    checkpointMessages: Message[];
+    pendingToolCalls: PendingToolCall[];
+  };
+  runId: string;
+  kind?: "approval" | "device";
+}): NonNullable<Conversation["pendingApprovals"]> =>
+  buildApprovalCheckpoints({
+    approvals: checkpointEvent.approvals,
+    runId,
+    checkpointMessages: [...priorMessages, ...checkpointEvent.checkpointMessages],
+    baseMessageCount: 0,
+    pendingToolCalls: checkpointEvent.pendingToolCalls,
+    kind,
+  });
+
+/** Text of a message for the transcript-integrity guard — flattens the two
+ *  content shapes so a user message can be matched across the display and
+ *  canonical transcripts regardless of how each stored it. */
+const messageText = (m: Message): string =>
+  typeof m.content === "string"
+    ? m.content
+    : Array.isArray(m.content)
+      ? m.content.map((p) => (p as { text?: string }).text ?? "").join("")
+      : "";
+
 // ── Turn metadata persistence ──
 
 export const applyTurnMetadata = (
@@ -373,6 +478,31 @@ export const applyTurnMetadata = (
     conv._harnessMessages = conv.messages;
   }
 
+  // Invariant guard: the model-facing transcript must not drop the current
+  // turn's user message. A resume that reconstructed `_harnessMessages`
+  // incorrectly silently lost it (the display transcript kept it, the model
+  // didn't), so the agent second-guessed an approval it never saw. This is the
+  // one choke point every finalize flows through — assert the latest user
+  // message survived into canonical, and log loudly if not. Matched by content
+  // (robust across the display/canonical shapes) and skipped when compaction
+  // has legitimately summarized history. Log-only; never throws.
+  if (isMessageArray(conv._harnessMessages) && conv._harnessMessages.length > 0) {
+    const canonical = conv._harnessMessages;
+    const summarized = canonical.some((m) => m.metadata?.isCompactionSummary);
+    const lastUser = [...conv.messages].reverse().find((m) => m.role === "user");
+    const lastUserText = lastUser ? messageText(lastUser).trim() : "";
+    if (!summarized && lastUserText) {
+      const present = canonical.some((m) => m.role === "user" && messageText(m).trim() === lastUserText);
+      if (!present) {
+        console.error(
+          `[transcript-guard] conversation ${conv.conversationId}: model-facing transcript ` +
+            `is missing the latest user message — it diverged from the display transcript. ` +
+            `This is a resume/finalize message-assembly bug; the model will not see that turn's input.`,
+        );
+      }
+    }
+  }
+
   if (meta.toolResultArchive !== undefined) {
     conv._toolResultArchive = meta.toolResultArchive;
   }

package/src/state.ts

@@ -17,10 +17,23 @@ export interface StateStore {
   delete(runId: string): Promise<void>;
 }
 
+/**
+ * Whether a subagent actually accomplished its task — distinct from the *run*
+ * status (`status` below), which is "completed" whenever the run merely
+ * finished. A subagent that ran to completion but could not do the task (e.g.
+ * lacked the required tools) is `runStatus:"completed"` but `taskOutcome:
+ * "failed"`. Derived at subagent finalize; "unknown" when it can't be
+ * determined. Consumed by the compaction subagent ledger so a failed task can
+ * never be summarized as "completed".
+ */
+export type SubagentTaskOutcome = "succeeded" | "failed" | "partial" | "unknown";
+
 export interface PendingSubagentResult {
   subagentId: string;
   task: string;
   status: "completed" | "error" | "stopped";
+  /** Did the subagent accomplish its task? See {@link SubagentTaskOutcome}. */
+  taskOutcome: SubagentTaskOutcome;
   result?: import("@poncho-ai/sdk").RunResult;
   error?: import("@poncho-ai/sdk").AgentFailure;
   timestamp: number;

package/test/compaction.test.ts

@@ -5,8 +5,10 @@ import type { Message } from "@poncho-ai/sdk";
 import {
   compactMessages,
   findSafeSplitPoint,
+  findSafeSplitPointByTurns,
   resolveCompactionConfig,
 } from "../src/compaction.js";
+import { deriveTaskOutcome, stripOutcomeVerdict } from "../src/orchestrator/orchestrator.js";
 
 // ── Fake model ──────────────────────────────────────────────────────────
 // A MockLanguageModelV3 whose doGenerate returns a fixed text and records the
@@ -254,9 +256,9 @@ describe("compactMessages", () => {
     expect(sentPrompt).toContain("MERGE AND UPDATE");
   });
 
-  it("still truncates non-prior-summary long messages to 1200 chars", async () => {
+  it("still truncates non-prior-summary long messages to the per-message cap", async () => {
     const { model, prompts } = fakeModel("S");
-    const longUser = "X".repeat(3000);
+    const longUser = "X".repeat(6000); // > SUMMARIZATION_MESSAGE_TRUNCATION_CHARS (4000)
     const messages: Message[] = [
       userMsg(longUser),
       assistantText("a0"),
@@ -272,3 +274,123 @@ describe("compactMessages", () => {
     expect(sentPrompt).not.toContain("MERGE AND UPDATE");
   });
 });
+
+describe("findSafeSplitPointByTurns", () => {
+  const turns = (n: number, assistantText = "a"): Message[] => {
+    const msgs: Message[] = [];
+    for (let i = 0; i < n; i++) {
+      msgs.push(userMsg(`u${i}`));
+      msgs.push({ role: "assistant", content: `${assistantText}${i}` });
+    }
+    return msgs;
+  };
+
+  it("preserves the last N whole turns verbatim", () => {
+    const messages = turns(6); // userIdx = [0,2,4,6,8,10]
+    // keepRecentTurns=4, generous token budget → split before the 4th-from-last
+    // user message (index 4), preserving turns u2..u5.
+    const idx = findSafeSplitPointByTurns(messages, 4, 6, 1_000_000);
+    expect(idx).toBe(4);
+    const preserved = messages.slice(idx);
+    const userCount = preserved.filter((m) => m.role === "user").length;
+    expect(userCount).toBe(4);
+  });
+
+  it("reduces N when N turns exceed the preserved-token budget", () => {
+    // Each assistant turn is large (~460 tokens); 4 turns preserved would blow a
+    // small budget, so it must fall back to fewer turns.
+    const big = "Z".repeat(1600);
+    const messages = turns(6, big); // userIdx=[0,2,4,6,8,10]
+    const idx = findSafeSplitPointByTurns(messages, 4, 6, 1200);
+    // n=4 (~1840 tok) and n=3 (~1380 tok) exceed 1200; n=2 (~920 tok) fits →
+    // split at userIdx[len-2] = index 8, preserving 2 turns.
+    expect(idx).toBe(8);
+    expect(messages.slice(idx).filter((m) => m.role === "user").length).toBe(2);
+  });
+
+  it("returns -1 for a single giant turn with no earlier user boundary", () => {
+    // One user message followed by many tool rounds — no safe boundary to
+    // compact against. (The message-based fallback also finds none.)
+    const messages: Message[] = [userMsg("do a big thing")];
+    for (let i = 0; i < 8; i++) {
+      messages.push(assistantToolCall(`step ${i}`, "run_code"));
+      messages.push(toolResult(`out ${i}`));
+    }
+    expect(findSafeSplitPointByTurns(messages, 4, 6, 1_000_000)).toBe(-1);
+  });
+});
+
+describe("deriveTaskOutcome / stripOutcomeVerdict", () => {
+  it("parses the self-declared verdict", () => {
+    expect(deriveTaskOutcome("did it [[OUTCOME: succeeded]] all good", false)).toBe("succeeded");
+    expect(deriveTaskOutcome("partial [[OUTCOME: partial]] some left", false)).toBe("partial");
+    expect(deriveTaskOutcome("couldn't [[OUTCOME: failed]] no tools", false)).toBe("failed");
+  });
+
+  it("defaults to unknown when no verdict is present (never assumes success)", () => {
+    expect(deriveTaskOutcome("I finished everything.", false)).toBe("unknown");
+  });
+
+  it("treats abnormal ends and empty output as failed", () => {
+    expect(deriveTaskOutcome("whatever", true)).toBe("failed");
+    expect(deriveTaskOutcome("   ", false)).toBe("failed");
+  });
+
+  it("strips the verdict marker from delivered text", () => {
+    expect(stripOutcomeVerdict("Here is the report.\n[[OUTCOME: failed]] missing API key")).toBe(
+      "Here is the report.",
+    );
+    expect(stripOutcomeVerdict("No verdict here")).toBe("No verdict here");
+  });
+});
+
+describe("compactMessages — failure fidelity", () => {
+  const config = resolveCompactionConfig({ keepRecentTurns: 1 });
+
+  it("renders a failed subagent as failed (never 'completed') and keeps its reason", async () => {
+    const { model } = fakeModel("SUMMARY");
+    const messages: Message[] = [
+      userMsg("please read my chats"),
+      assistantText("delegating to a subagent"),
+      userMsg(
+        '[Subagent Result] Subagent "read chats" (sub_fail) failed:\n\nI could not access the LinkedIn tools, so I read nothing.',
+        {
+          _subagentCallback: true,
+          subagentId: "sub_fail",
+          task: "read chats",
+          taskOutcome: "failed",
+        } as Message["metadata"],
+      ),
+      assistantText("continuing"),
+      userMsg("how did it go?"),
+      assistantText("let me check"),
+    ];
+    const res = await compactMessages(model, messages, config);
+    const content = res.messages[0]!.content as string;
+    expect(content).toContain("## Subagents");
+    expect(content).toContain("sub_fail");
+    // Rendered with the failed outcome, not "completed".
+    expect(content).toContain("— failed");
+    expect(content).not.toContain("(sub_fail) — completed");
+    // The failure reason survives verbatim.
+    expect(content).toContain("could not access the LinkedIn tools");
+  });
+
+  it("keeps failure-bearing messages when the input budget forces drops", async () => {
+    const { model, prompts } = fakeModel("S");
+    // Many bulky non-failure messages plus one early failure. The failure must
+    // survive into the summarizer input even if older bulk is dropped.
+    const messages: Message[] = [userMsg("start")];
+    messages.push(toolResult("ERROR: the deploy failed because the token expired"));
+    for (let i = 0; i < 400; i++) {
+      messages.push(assistantText("filler ".repeat(200) + i));
+      messages.push(userMsg(`ok ${i}`));
+    }
+    messages.push(userMsg("final question"));
+    messages.push(assistantText("final answer"));
+    await compactMessages(model, messages, config);
+    const sentPrompt = prompts.join("\n");
+    // The failure line is marked important and never dropped.
+    expect(sentPrompt).toContain("the deploy failed because the token expired");
+  });
+});

package/test/resume-transcript-integrity.test.ts

@@ -0,0 +1,156 @@
+import { describe, expect, it, vi } from "vitest";
+import type { Message } from "@poncho-ai/sdk";
+import {
+  assembleCheckpointMessages,
+  buildToolResultMessage,
+  buildResumeCheckpoints,
+  applyTurnMetadata,
+  type StoredApproval,
+} from "../src/orchestrator/index.js";
+import type { Conversation } from "../src/state.js";
+
+const conv = (overrides: Partial<Conversation> & Pick<Conversation, "messages">): Conversation => ({
+  conversationId: "conv_test",
+  title: "test",
+  ownerId: "owner",
+  tenantId: null,
+  createdAt: Date.now(),
+  updatedAt: Date.now(),
+  ...overrides,
+});
+
+// An assistant tool-call message is JSON `{ text, tool_calls: [...] }` in the
+// canonical transcript.
+const assistantToolCall = (text: string, calls: Array<{ id: string; name: string }>): Message => ({
+  role: "assistant",
+  content: JSON.stringify({ text, tool_calls: calls.map((c) => ({ ...c, input: {} })) }),
+});
+
+const meta = (harnessMessages?: Message[]) => ({
+  latestRunId: "",
+  contextTokens: 0,
+  contextWindow: 0,
+  harnessMessages,
+});
+
+describe("assembleCheckpointMessages", () => {
+  it("initial checkpoint: prior history (sliced by base) + delta", () => {
+    const prior: Message[] = [
+      { role: "user", content: "hi" },
+      { role: "assistant", content: "hello" },
+    ];
+    // Task-bearing turn: the delta carries the turn's user message + the
+    // assistant preamble/tool-call. base = prior length.
+    const delta: Message[] = [
+      { role: "user", content: "yeah those are a lot better" },
+      assistantToolCall("I found the two PE events…", [{ id: "call_1", name: "update_event" }]),
+    ];
+    const checkpoint = { baseMessageCount: prior.length, checkpointMessages: delta } as StoredApproval;
+    const out = assembleCheckpointMessages(conv({ messages: prior }), checkpoint);
+
+    expect(out).toHaveLength(4);
+    expect(out.some((m) => m.role === "user" && m.content === "yeah those are a lot better")).toBe(true);
+    expect(out.some((m) => typeof m.content === "string" && m.content.includes("I found the two PE events"))).toBe(true);
+  });
+
+  it("resume convention: base 0 + full canonical returns the full canonical verbatim", () => {
+    const full: Message[] = [
+      { role: "user", content: "hi" },
+      { role: "user", content: "approve this" },
+      assistantToolCall("preamble", [{ id: "call_1", name: "t" }]),
+    ];
+    const checkpoint = { baseMessageCount: 0, checkpointMessages: full } as StoredApproval;
+    // Display messages are irrelevant when base is 0 — the reconstruction must
+    // NOT depend on them (this is what the old PonchOS hand-merge got wrong).
+    const out = assembleCheckpointMessages(conv({ messages: [{ role: "user", content: "unrelated display" }] }), checkpoint);
+    expect(out).toEqual(full);
+  });
+});
+
+describe("buildToolResultMessage", () => {
+  const asst = assistantToolCall("", [{ id: "call_1", name: "update_event" }]);
+
+  it("pairs a provided result by callId", () => {
+    const msg = buildToolResultMessage(asst, [{ callId: "call_1", toolName: "update_event", result: { ok: true } }]);
+    expect(msg?.role).toBe("tool");
+    expect(msg?.content).toContain("call_1");
+    expect(msg?.content).toContain("ok");
+  });
+
+  it("emits the deferred-error marker when a result is missing", () => {
+    const msg = buildToolResultMessage(asst, []);
+    expect(msg?.content).toContain("Tool execution deferred");
+  });
+
+  it("returns undefined for a non-assistant / non-tool-call message", () => {
+    expect(buildToolResultMessage({ role: "user", content: "hi" }, [])).toBeUndefined();
+    expect(buildToolResultMessage({ role: "assistant", content: "plain text" }, [])).toBeUndefined();
+  });
+});
+
+describe("buildResumeCheckpoints + round-trip (the regression)", () => {
+  it("stores full canonical (base 0) so the NEXT resume reconstructs without dropping the user turn", () => {
+    // The continuation ran with this full canonical (incl. the turn's user
+    // message + preamble) and appended a tool result.
+    const fullWithResults: Message[] = [
+      { role: "user", content: "hi" },
+      { role: "assistant", content: "hello" },
+      { role: "user", content: "approve this" },
+      assistantToolCall("preamble text", [{ id: "call_1", name: "t1" }]),
+      { role: "tool", content: JSON.stringify([{ type: "tool_result", tool_use_id: "call_1", content: "done" }]) },
+    ];
+    // It then gated again on a new tool (the re-checkpoint event delta).
+    const delta2: Message[] = [assistantToolCall("", [{ id: "call_2", name: "t2" }])];
+    const stored = buildResumeCheckpoints({
+      priorMessages: fullWithResults,
+      checkpointEvent: {
+        approvals: [{ approvalId: "a2", tool: "t2", toolCallId: "call_2", input: {} }],
+        checkpointMessages: delta2,
+        pendingToolCalls: [{ id: "call_2", name: "t2", input: {} }],
+      },
+      runId: "run_2",
+    });
+
+    expect(stored[0]!.baseMessageCount).toBe(0);
+    expect(stored[0]!.checkpointMessages).toEqual([...fullWithResults, ...delta2]);
+
+    // Next resume reconstructs from the stored checkpoint — display is
+    // deliberately unrelated to prove no dependence on it.
+    const reconstructed = assembleCheckpointMessages(
+      conv({ messages: [{ role: "user", content: "unrelated" }], pendingApprovals: stored }),
+      stored[0]!,
+    );
+    // The user turn + preamble that PonchOS used to drop MUST survive.
+    expect(reconstructed.some((m) => m.role === "user" && m.content === "approve this")).toBe(true);
+    expect(reconstructed.some((m) => typeof m.content === "string" && m.content.includes("preamble text"))).toBe(true);
+  });
+});
+
+describe("applyTurnMetadata transcript-integrity guard", () => {
+  const latestUser: Message = { role: "user", content: "the latest question", metadata: { id: "u_latest" } };
+
+  it("logs when the latest user message is missing from the canonical transcript", () => {
+    const spy = vi.spyOn(console, "error").mockImplementation(() => {});
+    const c = conv({ messages: [{ role: "assistant", content: "earlier" }, latestUser] });
+    // Canonical is missing "the latest question" — the exact divergence.
+    applyTurnMetadata(c, meta([{ role: "user", content: "hi" }, { role: "assistant", content: "x" }]));
+    expect(spy).toHaveBeenCalledWith(expect.stringContaining("[transcript-guard]"));
+    spy.mockRestore();
+  });
+
+  it("stays silent when the latest user message is present in canonical", () => {
+    const spy = vi.spyOn(console, "error").mockImplementation(() => {});
+    const c = conv({ messages: [latestUser] });
+    applyTurnMetadata(c, meta([{ role: "user", content: "the latest question" }, { role: "assistant", content: "reply" }]));
+    expect(spy).not.toHaveBeenCalled();
+    spy.mockRestore();
+  });
+
+  it("skips the check when canonical was legitimately compacted", () => {
+    const spy = vi.spyOn(console, "error").mockImplementation(() => {});
+    const c = conv({ messages: [latestUser] });
+    applyTurnMetadata(c, meta([{ role: "assistant", content: "summary", metadata: { isCompactionSummary: true } }]));
+    expect(spy).not.toHaveBeenCalled();
+    spy.mockRestore();
+  });
+});