@tt-a1i/openpi 0.4.0 → 0.5.0

package/extensions/workflows/journal.ts

@@ -38,6 +38,148 @@ export interface WorkflowJournal {
   readonly entries: readonly JournalEntry[];
 }
 
+interface EncodedJournalEntry {
+  readonly entry: JournalEntry;
+  /** The entry formatted at the indentation used inside the journal array. */
+  readonly json: string;
+  readonly bytes: number;
+}
+
+const JOURNAL_PREFIX = `{
+  "version": ${JOURNAL_VERSION},
+  "entries": [
+`;
+const JOURNAL_SUFFIX = "\n  ]\n}";
+const JOURNAL_SEPARATOR = ",\n";
+const JOURNAL_EMPTY = JSON.stringify(
+  { version: JOURNAL_VERSION, entries: [] },
+  null,
+  2,
+);
+const JOURNAL_PREFIX_BYTES = Buffer.byteLength(JOURNAL_PREFIX, "utf8");
+const JOURNAL_SUFFIX_BYTES = Buffer.byteLength(JOURNAL_SUFFIX, "utf8");
+const JOURNAL_SEPARATOR_BYTES = Buffer.byteLength(JOURNAL_SEPARATOR, "utf8");
+const JOURNAL_EMPTY_BYTES = Buffer.byteLength(JOURNAL_EMPTY, "utf8");
+
+function encodeJournalEntry(entry: JournalEntry): EncodedJournalEntry {
+  // JSON.stringify on an array serializes an undefined element as null. The
+  // public type only permits objects, but retaining that fallback keeps this
+  // helper's behavior total if a malformed caller reaches it at runtime.
+  const json = JSON.stringify(entry, null, 2) ?? "null";
+  const indented = `    ${json.replaceAll("\n", "\n    ")}`;
+  return {
+    entry,
+    json: indented,
+    bytes: Buffer.byteLength(indented, "utf8"),
+  };
+}
+
+function journalBytes(entryCount: number, entryBytes: number) {
+  if (entryCount === 0) return JOURNAL_EMPTY_BYTES;
+  return (
+    JOURNAL_PREFIX_BYTES +
+    entryBytes +
+    JOURNAL_SUFFIX_BYTES +
+    (entryCount - 1) * JOURNAL_SEPARATOR_BYTES
+  );
+}
+
+/**
+ * Incrementally bounded journal storage for a running Workflow.
+ *
+ * Each entry is encoded once when appended. The active queue uses a head
+ * cursor, so dropping old entries only subtracts their already-known byte
+ * contribution. `toJson()` assembles the complete canonical artifact without
+ * re-stringifying each entry or repeatedly measuring the whole suffix.
+ */
+export interface WorkflowJournalAccumulator {
+  readonly bytes: number;
+  readonly dropped: number;
+  readonly length: number;
+  append(entry: JournalEntry): void;
+  toJournal(): WorkflowJournal;
+  toJson(): string;
+}
+
+export function createJournalAccumulator(
+  maxBytes = JOURNAL_MAX_BYTES,
+): WorkflowJournalAccumulator {
+  if (!Number.isSafeInteger(maxBytes) || maxBytes < JOURNAL_EMPTY_BYTES) {
+    throw new RangeError(
+      `Journal byte budget must be at least ${JOURNAL_EMPTY_BYTES} bytes`,
+    );
+  }
+
+  let queue: Array<EncodedJournalEntry | undefined> = [];
+  let head = 0;
+  let entryBytes = 0;
+  let dropped = 0;
+
+  const activeLength = () => queue.length - head;
+  const activeBytes = () => journalBytes(activeLength(), entryBytes);
+
+  const compact = () => {
+    // Release the encoded payload immediately when it is evicted. Compact the
+    // sparse prefix only occasionally so eviction remains amortized O(1).
+    if (head === queue.length) {
+      queue = [];
+      head = 0;
+      return;
+    }
+    if (head >= 64 && head * 2 >= queue.length) {
+      queue = queue.slice(head);
+      head = 0;
+    }
+  };
+
+  return {
+    get bytes() {
+      return activeBytes();
+    },
+    get dropped() {
+      return dropped;
+    },
+    get length() {
+      return activeLength();
+    },
+    append(entry) {
+      const encoded = encodeJournalEntry(entry);
+      queue.push(encoded);
+      entryBytes += encoded.bytes;
+
+      while (activeLength() > 0 && activeBytes() > maxBytes) {
+        const oldest = queue[head];
+        if (!oldest) break;
+        queue[head] = undefined;
+        head++;
+        entryBytes -= oldest.bytes;
+        dropped++;
+      }
+      compact();
+    },
+    toJournal() {
+      const entries: JournalEntry[] = [];
+      for (let index = head; index < queue.length; index++) {
+        const encoded = queue[index];
+        if (encoded) entries.push(encoded.entry);
+      }
+      return {
+        version: JOURNAL_VERSION,
+        entries,
+      };
+    },
+    toJson() {
+      if (activeLength() === 0) return JOURNAL_EMPTY;
+      const entries: string[] = [];
+      for (let index = head; index < queue.length; index++) {
+        const encoded = queue[index];
+        if (encoded) entries.push(encoded.json);
+      }
+      return JOURNAL_PREFIX + entries.join(JOURNAL_SEPARATOR) + JOURNAL_SUFFIX;
+    },
+  };
+}
+
 /**
  * Only semantic options change what an agent produces, so only these are
  * hashed. Callers pass their whole options object; `label` and `phase` are
@@ -146,19 +288,12 @@ export type ReplayCache = ReturnType<typeof createReplayCache>;
  * journal that serializes to a useless preview string.
  */
 export function boundedJournal(entries: readonly JournalEntry[]) {
-  const size = (kept: readonly JournalEntry[]) =>
-    Buffer.byteLength(
-      JSON.stringify({ version: JOURNAL_VERSION, entries: kept }, null, 2),
-      "utf8",
-    );
-
-  let kept = [...entries];
-  let dropped = 0;
-  while (kept.length > 0 && size(kept) > JOURNAL_MAX_BYTES) {
-    kept = kept.slice(1);
-    dropped++;
-  }
-  return { journal: { version: JOURNAL_VERSION, entries: kept }, dropped };
+  const accumulator = createJournalAccumulator();
+  for (const entry of entries) accumulator.append(entry);
+  return {
+    journal: accumulator.toJournal(),
+    dropped: accumulator.dropped,
+  };
 }
 
 function isRecord(value: unknown): value is Record<string, unknown> {

package/extensions/workflows/model.ts

@@ -5,8 +5,8 @@
 
 import * as os from "node:os";
 import {
-  truncateHead,
   type ExtensionContext,
+  truncateHead,
 } from "@earendil-works/pi-coding-agent";
 import { formatContextUtilization } from "../shared/context-utilization.ts";
 import { spinnerFrame } from "../shared/spinner.ts";
@@ -141,6 +141,23 @@ export interface AgentRecord {
 export interface WorkflowLogEntry {
   at: number;
   text: string;
+  /** Runtime-authored evidence, distinct from free-form script narration. */
+  kind?: "pipeline-drop";
+}
+
+/** Metadata attached only to the bounded in-memory settled-run projection. */
+export interface WorkflowMemoryProjection {
+  readonly kind: "settled";
+  readonly maxBytes: number;
+  readonly bytes: number;
+  readonly truncated: boolean;
+  readonly omitted: {
+    readonly agents: number;
+    readonly logs: number;
+    readonly transcriptEntries: number;
+    readonly result: boolean;
+    readonly graph: boolean;
+  };
 }
 
 export interface WorkflowDetails {
@@ -172,6 +189,54 @@ export interface WorkflowDetails {
   /** Read-only lineage projection; never execution or admission authority. */
   graph?: WorkflowGraphProjection;
   error?: string;
+  /** Present only on the session-memory projection, never canonical history. */
+  memoryProjection?: WorkflowMemoryProjection;
+}
+
+/** Bounded tool-result projection; authoritative details remain in run artifacts. */
+export function compactWorkflowToolDetails(
+  details: WorkflowDetails,
+): WorkflowDetails {
+  return {
+    ...details,
+    ...(details.result !== undefined
+      ? {
+          result: JSON.parse(
+            safeStringify(details.result, { maxBytes: 64 * 1024 }),
+          ),
+        }
+      : {}),
+    agents: details.agents.map((agent) => ({ ...agent, transcript: [] })),
+  };
+}
+
+/**
+ * Bound only the current-session terminal projection. Persisted workflow
+ * records and side artifacts remain the canonical history.
+ */
+export const MAX_SETTLED_RUNS = 32;
+
+/** Evict the oldest terminal projections without touching active-run state. */
+export function evictOldestSettledRuns(
+  settledRuns: Map<string, WorkflowDetails>,
+  maxRuns = MAX_SETTLED_RUNS,
+): string[] {
+  const limit = Number.isFinite(maxRuns)
+    ? Math.max(0, Math.floor(maxRuns))
+    : MAX_SETTLED_RUNS;
+  const evicted: string[] = [];
+  while (settledRuns.size > limit) {
+    let oldest: { runId: string; at: number } | undefined;
+    for (const [runId, details] of settledRuns) {
+      if (details.status === "running") continue;
+      const at = details.finishedAt ?? details.startedAt;
+      if (!oldest || at < oldest.at) oldest = { runId, at };
+    }
+    if (!oldest) break;
+    settledRuns.delete(oldest.runId);
+    evicted.push(oldest.runId);
+  }
+  return evicted;
 }
 
 export function workflowGraphRecords(
@@ -306,11 +371,12 @@ export function appendLog(
   details: WorkflowDetails,
   text: string,
   at: number,
+  kind?: WorkflowLogEntry["kind"],
 ): void {
   const clean = sanitizeLine(text, MAX_LOG_TEXT);
   if (!clean) return;
   const logs = (details.logs ??= []);
-  logs.push({ at, text: clean });
+  logs.push({ at, text: clean, ...(kind ? { kind } : {}) });
   const excess = logs.length - MAX_LOG_ENTRIES;
   if (excess > 0) {
     logs.splice(0, excess);
@@ -432,10 +498,14 @@ export function agentContext(agent: AgentRecord): string {
   });
 }
 
-export function formatElapsed(startedAt: number, finishedAt?: number): string {
+export function formatElapsed(
+  startedAt: number,
+  finishedAt?: number,
+  now = Date.now(),
+): string {
   const totalSeconds = Math.max(
     0,
-    Math.round(((finishedAt ?? Date.now()) - startedAt) / 1000),
+    Math.round(((finishedAt ?? now) - startedAt) / 1000),
   );
   const minutes = Math.floor(totalSeconds / 60);
   const seconds = totalSeconds % 60;

package/extensions/workflows/navigation.ts

@@ -31,6 +31,11 @@ export interface WorkflowStripEntry {
   details: WorkflowDetails;
 }
 
+/** Changes only when the strip needs an immediate lifecycle repaint. */
+export function workflowStripEntryKey(entry: WorkflowStripEntry | undefined) {
+  return entry ? `${entry.runId}:${entry.details.status}` : undefined;
+}
+
 function cleanLine(value: string) {
   return sanitizeTerminalText(value).replace(/\s+/g, " ").trim();
 }
@@ -48,7 +53,7 @@ function statusGlyph(status: WorkflowStatus, theme: Theme, now: number) {
 
 /** Live, one-line Claude-style workflow entry rendered below the editor. */
 export class WorkflowStripWidget {
-  private readonly timer: ReturnType<typeof setInterval>;
+  private timer: ReturnType<typeof setInterval> | undefined;
   private readonly tui: TUI;
   private readonly theme: Theme;
   private readonly strip: BelowEditorStripState;
@@ -64,20 +69,39 @@ export class WorkflowStripWidget {
     this.theme = theme;
     this.strip = strip;
     this.getEntry = getEntry;
-    this.timer = setInterval(
-      () => this.tui.requestRender(),
-      SPINNER_INTERVAL_MS,
-    );
-    this.timer.unref?.();
+    this.syncSpinner();
   }
 
   dispose() {
-    clearInterval(this.timer);
+    if (this.timer) clearInterval(this.timer);
+    this.timer = undefined;
   }
 
-  invalidate() {}
+  invalidate() {
+    this.syncSpinner();
+  }
+
+  private syncSpinner() {
+    if (this.getEntry()?.details.status === "running") {
+      if (this.timer) return;
+      const timer = setInterval(() => {
+        if (this.timer !== timer) return;
+        if (this.getEntry()?.details.status !== "running") {
+          clearInterval(timer);
+          this.timer = undefined;
+        }
+        this.tui.requestRender();
+      }, SPINNER_INTERVAL_MS);
+      this.timer = timer;
+      timer.unref?.();
+      return;
+    }
+    if (this.timer) clearInterval(this.timer);
+    this.timer = undefined;
+  }
 
   render(width: number) {
+    this.syncSpinner();
     const entry = this.getEntry();
     if (!entry || width <= 0) return [];
     const details = entry.details;

package/extensions/workflows/progress-projection.ts

@@ -0,0 +1,306 @@
+import {
+  calculateContextTokens,
+  type AgentSession,
+  estimateTokens,
+} from "@earendil-works/pi-coding-agent";
+import { type AgentUsage, emptyUsage, type TranscriptEntry } from "./model.ts";
+import { safeStringify, truncateUtf8 } from "./serialization.ts";
+
+const TRANSCRIPT_ENTRY_MAX_BYTES = 16 * 1024;
+const TRANSCRIPT_TOTAL_MAX_BYTES = 256 * 1024;
+const TRANSCRIPT_MAX_ENTRIES = 200;
+
+type AgentMessage = AgentSession["messages"][number];
+export type ProgressAssistantMessage = Extract<
+  AgentMessage,
+  { role: "assistant" }
+>;
+
+export interface ProgressToolTiming {
+  startedAt?: number;
+  finishedAt?: number;
+  durationMs?: number;
+}
+
+type ContextTokens = number | null | undefined;
+
+interface ProjectedTranscriptEntry {
+  role: TranscriptEntry["role"];
+  text: string;
+  sourceTextTruncated: boolean;
+  name?: string;
+  sourceToolCallId?: string;
+  isError?: boolean;
+  timestamp?: number;
+}
+
+export interface AgentProgressProjectionSnapshot {
+  preview: string;
+  usage: AgentUsage;
+  latestAssistant?: ProgressAssistantMessage;
+  transcript: TranscriptEntry[];
+}
+
+function validAssistantContextTokens(message: ProgressAssistantMessage) {
+  if (
+    message.stopReason === "aborted" ||
+    message.stopReason === "error" ||
+    !message.usage
+  ) {
+    return undefined;
+  }
+  const tokens = calculateContextTokens(message.usage);
+  return tokens > 0 ? tokens : undefined;
+}
+
+function boundedSourceText(text: string) {
+  const bounded = truncateUtf8(text, TRANSCRIPT_ENTRY_MAX_BYTES);
+  return { text: bounded, sourceTextTruncated: bounded !== text };
+}
+
+function safeJson(value: unknown) {
+  return safeStringify(value, {
+    maxBytes: TRANSCRIPT_ENTRY_MAX_BYTES,
+    maxDepth: 12,
+    maxNodes: 2_000,
+  });
+}
+
+function messageEntries(message: AgentMessage): ProjectedTranscriptEntry[] {
+  if (message.role === "user") {
+    const source =
+      typeof message.content === "string"
+        ? message.content
+        : message.content
+            .map((part) =>
+              part.type === "text" ? part.text : `[image: ${part.mimeType}]`,
+            )
+            .join("\n");
+    if (!source.trim()) return [];
+    return [
+      {
+        role: "user",
+        ...boundedSourceText(source),
+        timestamp: message.timestamp,
+      },
+    ];
+  }
+
+  if (message.role === "assistant") {
+    const entries: ProjectedTranscriptEntry[] = [];
+    for (const part of message.content) {
+      if (part.type === "text" && part.text.trim()) {
+        entries.push({
+          role: "assistant",
+          ...boundedSourceText(part.text),
+          timestamp: message.timestamp,
+        });
+        continue;
+      }
+      if (part.type === "thinking" && part.thinking.trim()) {
+        entries.push({
+          role: "thinking",
+          ...boundedSourceText(part.thinking),
+          timestamp: message.timestamp,
+        });
+        continue;
+      }
+      if (part.type === "toolCall") {
+        entries.push({
+          role: "tool",
+          ...boundedSourceText(safeJson(part.arguments)),
+          name: part.name,
+          sourceToolCallId: part.id,
+          timestamp: message.timestamp,
+        });
+      }
+    }
+    return entries;
+  }
+
+  if (message.role !== "toolResult") return [];
+  const source = message.content
+    .map((part) =>
+      part.type === "text" ? part.text : `[image: ${part.mimeType}]`,
+    )
+    .join("\n");
+  return [
+    {
+      role: "toolResult",
+      ...boundedSourceText(source),
+      name: message.toolName,
+      sourceToolCallId: message.toolCallId,
+      isError: message.isError,
+      timestamp: message.timestamp,
+    },
+  ];
+}
+
+function assistantText(message: ProgressAssistantMessage) {
+  return message.content
+    .filter((part) => part.type === "text")
+    .map((part) => part.text)
+    .join("\n")
+    .trim();
+}
+
+function addAssistantUsage(
+  usage: AgentUsage,
+  message: ProgressAssistantMessage,
+) {
+  usage.turns++;
+  const current = message.usage;
+  if (!current) return;
+  usage.input += current.input || 0;
+  usage.output += current.output || 0;
+  usage.cacheRead += current.cacheRead || 0;
+  usage.cacheWrite += current.cacheWrite || 0;
+  usage.cost += current.cost?.total || 0;
+}
+
+function toolMetadata(
+  toolCallId: string,
+  timings: ReadonlyMap<string, ProgressToolTiming>,
+) {
+  const timing = timings.get(toolCallId);
+  return {
+    toolCallId: truncateUtf8(toolCallId, 1024),
+    ...(timing?.startedAt === undefined ? {} : { startedAt: timing.startedAt }),
+    ...(timing?.finishedAt === undefined
+      ? {}
+      : { finishedAt: timing.finishedAt }),
+    ...(timing?.durationMs === undefined
+      ? {}
+      : { durationMs: timing.durationMs }),
+  };
+}
+
+/**
+ * Bounded, ephemeral projection of Pi's active child messages.
+ *
+ * Pi remains canonical. Normal finalized messages are folded once; startup,
+ * compaction, and terminal settlement replace this state from Pi's messages.
+ */
+export class AgentProgressProjection {
+  private usage = emptyUsage();
+  private preview = "";
+  private latestAssistant?: ProgressAssistantMessage;
+  private firstEntry?: ProjectedTranscriptEntry;
+  private tailEntries: ProjectedTranscriptEntry[] = [];
+  private totalEntries = 0;
+  private contextTokens: ContextTokens;
+
+  constructor(
+    messages: readonly AgentMessage[] = [],
+    contextTokens?: ContextTokens,
+  ) {
+    this.replace(messages, contextTokens);
+  }
+
+  replace(messages: readonly AgentMessage[], contextTokens?: ContextTokens) {
+    this.usage = emptyUsage();
+    this.preview = "";
+    this.latestAssistant = undefined;
+    this.firstEntry = undefined;
+    this.tailEntries = [];
+    this.totalEntries = 0;
+    this.contextTokens = contextTokens;
+    for (const message of messages) this.ingest(message, false);
+  }
+
+  append(message: AgentMessage) {
+    this.ingest(message, true);
+  }
+
+  private ingest(message: AgentMessage, updateContext: boolean) {
+    if (message.role === "assistant") {
+      addAssistantUsage(this.usage, message);
+      this.latestAssistant = message;
+      const text = assistantText(message);
+      if (text) this.preview = text;
+    }
+
+    if (updateContext && this.contextTokens !== undefined) {
+      const assistantTokens =
+        message.role === "assistant"
+          ? validAssistantContextTokens(message)
+          : undefined;
+      if (assistantTokens !== undefined) {
+        this.contextTokens = assistantTokens;
+      } else if (this.contextTokens !== null) {
+        this.contextTokens += estimateTokens(message);
+      }
+    }
+
+    for (const entry of messageEntries(message)) this.pushEntry(entry);
+  }
+
+  private pushEntry(entry: ProjectedTranscriptEntry) {
+    this.totalEntries++;
+    if (!this.firstEntry) {
+      this.firstEntry = entry;
+      return;
+    }
+    this.tailEntries.push(entry);
+    if (this.tailEntries.length >= TRANSCRIPT_MAX_ENTRIES) {
+      this.tailEntries.shift();
+    }
+  }
+
+  snapshot(
+    toolTimings: ReadonlyMap<string, ProgressToolTiming> = new Map(),
+  ): AgentProgressProjectionSnapshot {
+    const selected = this.firstEntry
+      ? [this.firstEntry, ...this.tailEntries]
+      : [];
+    const transcript: TranscriptEntry[] = [];
+    let totalBytes = 0;
+    for (const entry of selected) {
+      const remaining = TRANSCRIPT_TOTAL_MAX_BYTES - totalBytes;
+      if (remaining <= 0) break;
+      const text = truncateUtf8(
+        entry.text,
+        Math.min(TRANSCRIPT_ENTRY_MAX_BYTES, remaining),
+      );
+      totalBytes += Buffer.byteLength(text, "utf8");
+      const truncated = entry.sourceTextTruncated || text !== entry.text;
+      transcript.push({
+        role: entry.role,
+        text: truncated ? `${text}\n[transcript entry truncated]` : text,
+        ...(entry.name === undefined ? {} : { name: entry.name }),
+        ...(entry.sourceToolCallId === undefined
+          ? {}
+          : toolMetadata(entry.sourceToolCallId, toolTimings)),
+        ...(entry.isError === undefined ? {} : { isError: entry.isError }),
+        ...(entry.timestamp === undefined
+          ? {}
+          : { timestamp: entry.timestamp }),
+      });
+    }
+    if (transcript.length < this.totalEntries) {
+      transcript.push({
+        role: "toolResult",
+        name: "transcript",
+        text: `[transcript truncated: retained ${transcript.length} of ${this.totalEntries} entries]`,
+      });
+    }
+    return {
+      preview: this.preview,
+      usage: {
+        ...this.usage,
+        ...(typeof this.contextTokens === "number"
+          ? { contextTokens: this.contextTokens }
+          : {}),
+      },
+      latestAssistant: this.latestAssistant,
+      transcript,
+    };
+  }
+}
+
+export function transcriptFromMessages(
+  messages: AgentMessage[],
+  toolTimings: ReadonlyMap<string, ProgressToolTiming> = new Map(),
+) {
+  return new AgentProgressProjection(messages).snapshot(toolTimings).transcript;
+}