@tt-a1i/openpi 0.3.1 → 0.5.0

package/extensions/user-input-fold/index.ts

@@ -0,0 +1,185 @@
+/**
+ * Fold very long user messages in the transcript display.
+ *
+ * A pasted log, stack trace, or whole file can wipe out several screens of
+ * chat. When a finalized user message exceeds the fold thresholds, the
+ * display keeps a short preview of the prose and of each fenced code block
+ * and closes with a marker line stating how much was folded.
+ *
+ * A fold that would hide nothing (or almost nothing) is skipped and the
+ * message rendered in full: hiding a handful of lines costs a marker row
+ * while saving almost no screen space, and char-heavy messages with few
+ * long lines do not shrink on screen when logical lines are removed —
+ * folding must hide at least MIN_FOLDED_LINES lines to earn its keep.
+ *
+ * This is display-only. The transformer runs through Pi's
+ * `registerMarkdownTransformer` hook, which changes only what the TUI
+ * renders: the session file and the model context keep the full message
+ * untouched, so the model always receives the complete paste. The pure
+ * `foldUserMessage` helper never mutates its input and has no side effects.
+ */
+
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+
+/** Fold a user message longer than this many lines. */
+const MAX_LINES = 20;
+/** Fold a user message longer than this many characters. */
+const MAX_CHARS = 1_200;
+/** Prose lines kept when a message is folded. */
+const PROSE_PREVIEW_LINES = 12;
+/** Content lines kept per fenced code block when a message is folded. */
+const BLOCK_PREVIEW_LINES = 4;
+/** Character budget for the prose part of a folded message. */
+const PROSE_PREVIEW_CHARS = 1_200;
+/** Total rendered lines before the fold marker, across prose and code blocks. */
+const MAX_PREVIEW_LINES = 20;
+/** Only fold when at least this many lines would be hidden. */
+const MIN_FOLDED_LINES = 8;
+
+type Segment =
+  | { kind: "prose"; lines: string[] }
+  | { kind: "code"; open: string; content: string[]; close: string };
+
+const FENCE_OPEN = /^ {0,3}`{3,}/;
+const FENCE_CLOSE = /^ {0,3}`{3,}[ \t]*\r?$/;
+
+function countLines(markdown: string) {
+  const parts = markdown.split("\n");
+  // A trailing newline ends the last line; it does not open a new one.
+  return parts.at(-1) === "" ? parts.length - 1 : parts.length;
+}
+
+function splitLines(markdown: string) {
+  const lines = markdown.split("\n");
+  if (lines.at(-1) === "") lines.pop();
+  return lines;
+}
+
+function parseSegments(lines: string[]): Segment[] {
+  const segments: Segment[] = [];
+  let prose: string[] = [];
+  let i = 0;
+  while (i < lines.length) {
+    if (!FENCE_OPEN.test(lines[i])) {
+      prose.push(lines[i]);
+      i += 1;
+      continue;
+    }
+    if (prose.length > 0) {
+      segments.push({ kind: "prose", lines: prose });
+      prose = [];
+    }
+    const open = lines[i];
+    const content: string[] = [];
+    let close: string | undefined;
+    let j = i + 1;
+    while (j < lines.length && close === undefined) {
+      if (FENCE_CLOSE.test(lines[j])) close = lines[j];
+      else content.push(lines[j]);
+      j += 1;
+    }
+    if (close === undefined) {
+      // Unterminated fence: fold the whole message conservatively as text.
+      return [{ kind: "prose", lines }];
+    }
+    segments.push({ kind: "code", open, content, close });
+    i = j;
+  }
+  if (prose.length > 0) segments.push({ kind: "prose", lines: prose });
+  return segments;
+}
+
+/**
+ * Return the Markdown Pi should render instead of a long user message.
+ * Messages at or below both thresholds — and messages whose fold would
+ * hide fewer than MIN_FOLDED_LINES lines — are returned unchanged. Pure:
+ * the input string is never modified, and the model still sees the original.
+ */
+export function foldUserMessage(markdown: string): string {
+  const totalLines = countLines(markdown);
+  if (totalLines <= MAX_LINES && markdown.length <= MAX_CHARS) return markdown;
+
+  const preview: string[] = [];
+  let proseLinesLeft = PROSE_PREVIEW_LINES;
+  let proseCharsLeft = PROSE_PREVIEW_CHARS;
+  let linesShown = 0;
+
+  for (const segment of parseSegments(splitLines(markdown))) {
+    const remainingLines = MAX_PREVIEW_LINES - preview.length;
+    if (remainingLines <= 0) break;
+    if (segment.kind === "code") {
+      if (segment.content.length === 0) {
+        if (remainingLines < 2) break;
+        preview.push(segment.open, segment.close);
+        linesShown += 2;
+        continue;
+      }
+      // A partial block needs opening/closing fences plus an ellipsis. If that
+      // cannot fit, stop before the block instead of emitting broken Markdown.
+      if (remainingLines < 3) break;
+      let shown = Math.min(
+        segment.content.length,
+        BLOCK_PREVIEW_LINES,
+        remainingLines - 2,
+      );
+      const truncated = () => shown < segment.content.length;
+      while (truncated() && shown + 3 > remainingLines) shown -= 1;
+      preview.push(segment.open, ...segment.content.slice(0, shown));
+      if (truncated()) preview.push("…");
+      preview.push(segment.close);
+      linesShown += 2 + shown;
+      continue;
+    }
+    for (const line of segment.lines) {
+      if (proseLinesLeft <= 0 || preview.length >= MAX_PREVIEW_LINES) break;
+      const cost = line.length + (preview.length > 0 ? 1 : 0);
+      if (preview.length > 0 && cost > proseCharsLeft) break;
+      if (preview.length === 0 && line.length > proseCharsLeft) {
+        // A single giant first line is the only case that cuts mid-line.
+        preview.push(`${line.slice(0, proseCharsLeft)}…`);
+        proseLinesLeft = 0;
+        proseCharsLeft = 0;
+        break;
+      }
+      preview.push(line);
+      linesShown += 1;
+      proseLinesLeft -= 1;
+      proseCharsLeft -= cost;
+    }
+  }
+
+  const foldedLines = totalLines - linesShown;
+  if (foldedLines < MIN_FOLDED_LINES) {
+    // Folding must earn its keep. Hiding fewer lines than MIN_FOLDED_LINES
+    // costs a marker row, hides content, and saves almost nothing on screen
+    // (char-heavy messages with few long lines wrap regardless), so render
+    // the message in full instead.
+    return markdown;
+  }
+  const noun = foldedLines === 1 ? "line" : "lines";
+  preview.push(
+    `… folded ${foldedLines} ${noun} · full content was sent to the model`,
+  );
+  return preview.join("\n");
+}
+
+/**
+ * The registered transformer: fold only finalized user messages and leave
+ * assistant text, thinking blocks, and streaming updates untouched.
+ */
+export function transformUserMarkdown(
+  markdown: string,
+  context: { messageType: string; isStreaming: boolean },
+): string {
+  if (context.messageType !== "user" || context.isStreaming) return markdown;
+  try {
+    return foldUserMessage(markdown);
+  } catch {
+    // Display-only: a folding bug must never break rendering.
+    return markdown;
+  }
+}
+
+export default function (pi: ExtensionAPI) {
+  pi.registerMarkdownTransformer(transformUserMarkdown);
+}

package/extensions/web/index.ts

@@ -0,0 +1,234 @@
+import { spawn as nodeSpawn } from "node:child_process";
+import { dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import type {
+  ExtensionAPI,
+  ExtensionCommandContext,
+} from "@earendil-works/pi-coding-agent";
+
+const DEFAULT_SHUTDOWN_TIMEOUT_MS = 5_000;
+
+export interface WebProcess {
+  readonly exitCode: number | null;
+  readonly signalCode: NodeJS.Signals | null;
+  once(event: "error", listener: (error: Error) => void): this;
+  once(
+    event: "close",
+    listener: (code: number | null, signal: NodeJS.Signals | null) => void,
+  ): this;
+  kill(signal?: NodeJS.Signals): boolean;
+}
+
+interface SpawnWebOptions {
+  cwd: string;
+  env: NodeJS.ProcessEnv;
+  shell: false;
+  stdio: "inherit";
+}
+
+function webProcessEnvironment(cwd: string) {
+  const environment: NodeJS.ProcessEnv = { ...process.env, PWD: cwd };
+  delete environment.OLDPWD;
+  delete environment.INIT_CWD;
+  delete environment.PI_SESSION_ID;
+  delete environment.PI_SESSION_FILE;
+  return environment;
+}
+
+export interface WebCommandDependencies {
+  entrypoint: string;
+  spawn(command: string, args: string[], options: SpawnWebOptions): WebProcess;
+  clearTerminal(): void;
+  holdParentSigint(): () => void;
+  shutdownTimeoutMs: number;
+}
+
+type WebExit =
+  | { kind: "close"; code: number | null; signal: NodeJS.Signals | null }
+  | { kind: "error"; error: Error };
+
+interface ActiveWebProcess {
+  child: WebProcess;
+  closed: Promise<void>;
+}
+
+const defaultDependencies: WebCommandDependencies = {
+  entrypoint: fileURLToPath(new URL("../../bin/openpi.js", import.meta.url)),
+  spawn(command, args, options) {
+    return nodeSpawn(command, args, options);
+  },
+  clearTerminal() {
+    process.stdout.write("\u001b[2J\u001b[H");
+  },
+  holdParentSigint() {
+    const keepPiAlive = () => {};
+    process.on("SIGINT", keepPiAlive);
+    return () => process.removeListener("SIGINT", keepPiAlive);
+  },
+  shutdownTimeoutMs: DEFAULT_SHUTDOWN_TIMEOUT_MS,
+};
+
+function delay(milliseconds: number) {
+  return new Promise<void>((resolve) => {
+    const timer = setTimeout(resolve, milliseconds);
+    timer.unref();
+  });
+}
+
+async function stopWebProcess(active: ActiveWebProcess, timeoutMs: number) {
+  if (active.child.exitCode !== null || active.child.signalCode !== null)
+    return;
+  active.child.kill("SIGTERM");
+  const timedOut = await Promise.race([
+    active.closed.then(() => false),
+    delay(timeoutMs).then(() => true),
+  ]);
+  if (!timedOut) return;
+  if (active.child.exitCode === null && active.child.signalCode === null) {
+    active.child.kill("SIGKILL");
+    await Promise.race([active.closed, delay(timeoutMs)]);
+  }
+}
+
+function runWebInForeground(
+  ctx: ExtensionCommandContext,
+  dependencies: WebCommandDependencies,
+  setActive: (active: ActiveWebProcess | undefined) => void,
+  isShuttingDown: () => boolean,
+) {
+  return ctx.ui.custom<WebExit>((tui, _theme, _keybindings, done) => {
+    let finished = false;
+    let tuiStopped = false;
+    let resolveClosed = () => {};
+    const closed = new Promise<void>((resolve) => {
+      resolveClosed = resolve;
+    });
+    const releaseParentSigint = dependencies.holdParentSigint();
+
+    const finish = (result: WebExit) => {
+      if (finished) return;
+      finished = true;
+      releaseParentSigint();
+      setActive(undefined);
+      resolveClosed();
+      if (tuiStopped && !isShuttingDown()) {
+        tui.start();
+        tui.requestRender(true);
+      }
+      done(result);
+    };
+
+    try {
+      tui.stop();
+      tuiStopped = true;
+      dependencies.clearTerminal();
+      const childCwd = dirname(dependencies.entrypoint);
+      const child = dependencies.spawn(
+        process.execPath,
+        [dependencies.entrypoint, "web", "--no-workspace"],
+        {
+          cwd: childCwd,
+          env: webProcessEnvironment(childCwd),
+          shell: false,
+          stdio: "inherit",
+        },
+      );
+      setActive({ child, closed });
+      child.once("error", (error) => finish({ kind: "error", error }));
+      child.once("close", (code, signal) =>
+        finish({ kind: "close", code, signal }),
+      );
+    } catch (error) {
+      finish({
+        kind: "error",
+        error: error instanceof Error ? error : new Error(String(error)),
+      });
+    }
+
+    return { render: () => [], invalidate: () => {} };
+  });
+}
+
+export default function web(
+  pi: ExtensionAPI,
+  dependencies: WebCommandDependencies = defaultDependencies,
+) {
+  let active: ActiveWebProcess | undefined;
+  let running = false;
+  let shuttingDown = false;
+
+  pi.on("session_start", () => {
+    shuttingDown = false;
+  });
+
+  pi.on("session_shutdown", async () => {
+    shuttingDown = true;
+    if (active) {
+      await stopWebProcess(active, dependencies.shutdownTimeoutMs);
+    }
+  });
+
+  pi.registerCommand("web", {
+    description:
+      "Open the separate OpenPI Web Workbench in this terminal until Ctrl+C",
+    handler: async (args, ctx) => {
+      if (args.trim()) {
+        ctx.ui.notify("Usage: /web", "warning");
+        return;
+      }
+      if (ctx.mode !== "tui") {
+        ctx.ui.notify("/web requires the interactive TUI.", "warning");
+        return;
+      }
+      if (!ctx.isIdle() || ctx.hasPendingMessages()) {
+        ctx.ui.notify(
+          "Wait until the current Pi Session is idle before starting /web.",
+          "warning",
+        );
+        return;
+      }
+      if (running) {
+        ctx.ui.notify("OpenPI Web Workbench is already running.", "warning");
+        return;
+      }
+
+      running = true;
+      try {
+        const result = await runWebInForeground(
+          ctx,
+          dependencies,
+          (next) => {
+            active = next;
+          },
+          () => shuttingDown,
+        );
+        if (shuttingDown) return;
+        if (result.kind === "error") {
+          ctx.ui.notify(
+            `Failed to start OpenPI Web Workbench: ${result.error.message}`,
+            "error",
+          );
+          return;
+        }
+        if (result.signal !== null) {
+          ctx.ui.notify(
+            `OpenPI Web Workbench was terminated by ${result.signal}.`,
+            "error",
+          );
+          return;
+        }
+        if (result.code !== 0) {
+          ctx.ui.notify(
+            `OpenPI Web Workbench exited with code ${result.code ?? "unknown"}.`,
+            "error",
+          );
+          return;
+        }
+        ctx.ui.notify("OpenPI Web Workbench stopped.", "info");
+      } finally {
+        active = undefined;
+        running = false;
+      }
+    },
+  });
+}

package/extensions/workflows/artifacts.ts

@@ -1,31 +1,40 @@
-import {
-  refreshWorkflowGraph,
-  type TranscriptEntry,
-  type WorkflowDetails,
-} from "./model.ts";
+import * as fs from "node:fs";
+import * as path from "node:path";
 import {
   boundedJournal,
-  parseJournal,
+  JOURNAL_MAX_BYTES,
   type JournalEntry,
+  parseJournal,
   type WorkflowJournal,
+  type WorkflowJournalAccumulator,
 } from "./journal.ts";
 import {
+  refreshWorkflowGraph,
+  type TranscriptEntry,
+  type WorkflowDelivery,
+  type WorkflowDetails,
+} from "./model.ts";
+import {
+  encodeCompleteJson,
   safeStringify,
   truncateUtf8,
   writeFileAtomic,
 } from "./serialization.ts";
-import * as fs from "node:fs";
-import * as path from "node:path";
 
 export const JOURNAL_FILE = "journal.json";
 
 const ARTIFACT_TRANSCRIPT_MAX_BYTES = 32 * 1024;
 const ARTIFACT_TRANSCRIPT_ENTRY_MAX_BYTES = 8 * 1024;
+const AGENT_RESULT_ARTIFACT_MAX_BYTES = 2 * 1024 * 1024;
 export const WORKFLOW_CHECKPOINT_INTERVAL_MS = 500;
 const ENTRY_TRUNCATION_MARKER = "\n[entry truncated]";
 const TRANSCRIPT_TRUNCATION_MARKER =
   "[artifact transcript truncated: older entries omitted]";
 
+type WorkflowJournalSource =
+  | readonly JournalEntry[]
+  | WorkflowJournalAccumulator;
+
 function textBytes(text: string) {
   return Buffer.byteLength(text, "utf8");
 }
@@ -100,10 +109,61 @@ function writeRunFile(runDir: string, name: string, content: string) {
   writeFileAtomic(path.join(runDir, name), content);
 }
 
+export function persistWorkflowTerminalState(
+  runDir: string,
+  details: WorkflowDetails,
+) {
+  const terminalManifest: WorkflowDetails = {
+    ...details,
+    agents: details.agents.map((agent) => ({ ...agent, transcript: [] })),
+  };
+  delete terminalManifest.result;
+  delete terminalManifest.resultArtifact;
+  delete terminalManifest.transcriptArtifact;
+  writeRunFile(
+    runDir,
+    "workflow.json",
+    safeStringify(terminalManifest, { maxBytes: 1024 * 1024 }),
+  );
+}
+
+/** Persist one successful child result before any handoff/context projection. */
+export function persistWorkflowAgentResult(
+  runDir: string,
+  index: number,
+  result: { output: string; structured?: unknown },
+) {
+  // Artifact references are persisted as portable workflow paths. The host
+  // filesystem join happens in writeRunFile; handoff validation and replay
+  // consumers intentionally use `/` regardless of the platform.
+  const artifact = `agent-results/agent-${String(index).padStart(4, "0")}.json`;
+  const encoded = encodeCompleteJson(
+    {
+      output: result.output,
+      ...(result.structured !== undefined
+        ? { structured: result.structured }
+        : {}),
+    },
+    {
+      maxBytes: AGENT_RESULT_ARTIFACT_MAX_BYTES,
+      maxDepth: 32,
+      maxNodes: 100_000,
+      maxStringBytes: AGENT_RESULT_ARTIFACT_MAX_BYTES,
+    },
+  );
+  if (!encoded.ok) {
+    throw new Error(
+      `Agent result artifact exceeded the ${AGENT_RESULT_ARTIFACT_MAX_BYTES}-byte budget (${encoded.limit} limit at ${encoded.path})`,
+    );
+  }
+  writeRunFile(runDir, artifact, encoded.json);
+  return artifact;
+}
+
 export function persistWorkflowJson(
   runDir: string,
   details: WorkflowDetails,
-  journal?: readonly JournalEntry[],
+  journal?: WorkflowJournalSource,
 ) {
   refreshWorkflowGraph(details);
   const transcripts = Object.fromEntries(
@@ -112,6 +172,15 @@ export function persistWorkflowJson(
       boundedArtifactTranscript(agent.transcript),
     ]),
   );
+
+  // Publish terminal execution facts before dependent side artifacts. If a
+  // later artifact write fails, readers still see an explained terminal run
+  // instead of the previous `running` manifest. The final manifest below adds
+  // the artifact references once every dependent file has committed.
+  if (details.status !== "running") {
+    persistWorkflowTerminalState(runDir, details);
+  }
+
   writeRunFile(
     runDir,
     "transcripts.json",
@@ -119,15 +188,20 @@ export function persistWorkflowJson(
   );
   // Written alongside the rest so it inherits atomic write, 500ms coalescing,
   // and the final flush. Only present once a call has actually succeeded.
-  // boundedJournal has already brought this under the cap, and plain
-  // JSON.stringify is deliberate: safeStringify would swap an over-cap value
-  // for a preview stub, silently turning the journal into something unusable.
-  if (journal && journal.length > 0) {
-    writeRunFile(
-      runDir,
-      JOURNAL_FILE,
-      JSON.stringify(boundedJournal(journal).journal, null, 2),
-    );
+  // Accumulators already enforce the cap incrementally and can assemble the
+  // canonical JSON from their cached entry fragments. Plain arrays retain the
+  // compatibility path through boundedJournal; plain JSON.stringify is
+  // deliberate because safeStringify would swap an over-cap value for a
+  // preview stub, silently turning the journal into something unusable.
+  if (
+    journal &&
+    (journal.length > 0 || ("toJson" in journal && journal.dropped > 0))
+  ) {
+    const content =
+      "toJson" in journal
+        ? journal.toJson()
+        : JSON.stringify(boundedJournal(journal).journal, null, 2);
+    writeRunFile(runDir, JOURNAL_FILE, content);
   }
   if (details.result !== undefined) {
     writeRunFile(
@@ -151,6 +225,25 @@ export function persistWorkflowJson(
   );
 }
 
+/**
+ * Update only the durable delivery receipt. Completion delivery may retain a
+ * compact memory projection after the run has been evicted, so rewriting the
+ * whole details object here would risk replacing exact result artifacts with
+ * that projection.
+ */
+export function persistWorkflowDeliveryState(
+  runDir: string,
+  delivery: WorkflowDelivery,
+) {
+  const file = path.join(runDir, "workflow.json");
+  const raw: unknown = JSON.parse(fs.readFileSync(file, "utf8"));
+  if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
+    throw new Error(`Invalid persisted workflow details: ${file}`);
+  }
+  const next = { ...(raw as Record<string, unknown>), delivery };
+  writeFileAtomic(file, JSON.stringify(next));
+}
+
 /** Coalesce live checkpoints while keeping final persistence synchronous. */
 export function createWorkflowPersistence(
   runDir: string,
@@ -160,10 +253,10 @@ export function createWorkflowPersistence(
     persist?: (
       runDir: string,
       details: WorkflowDetails,
-      journal?: readonly JournalEntry[],
+      journal?: WorkflowJournalSource,
     ) => void;
-    /** Read at write time so callers only have to append to their own array. */
-    journal?: () => readonly JournalEntry[];
+    /** Read at write time so callers only have to append to their journal. */
+    journal?: () => WorkflowJournalSource;
   } = {},
 ) {
   const intervalMs = Math.max(
@@ -221,10 +314,42 @@ export function createWorkflowPersistence(
  * turn into a new way for a run to fail.
  */
 export function loadJournal(runDir: string): WorkflowJournal | undefined {
+  let descriptor: number | undefined;
   try {
-    const raw = fs.readFileSync(path.join(runDir, JOURNAL_FILE), "utf8");
+    descriptor = fs.openSync(path.join(runDir, JOURNAL_FILE), "r");
+    const initialSize = fs.fstatSync(descriptor).size;
+    if (initialSize > JOURNAL_MAX_BYTES) return undefined;
+
+    // Read at most one byte beyond the cap. The descriptor pins the inspected
+    // file, while the extra byte catches growth after fstat without ever
+    // allocating or parsing an unbounded journal.
+    const bytes = Buffer.allocUnsafe(initialSize + 1);
+    let length = 0;
+    while (length < bytes.length) {
+      const read = fs.readSync(
+        descriptor,
+        bytes,
+        length,
+        bytes.length - length,
+        null,
+      );
+      if (read === 0) break;
+      length += read;
+    }
+    // Filling the extra byte means the file changed after inspection. Reject
+    // that race even when the new size would still fit the ordinary cap.
+    if (length === bytes.length) return undefined;
+    const raw = bytes.toString("utf8", 0, length);
     return parseJournal(JSON.parse(raw));
   } catch {
     return undefined;
+  } finally {
+    if (descriptor !== undefined) {
+      try {
+        fs.closeSync(descriptor);
+      } catch {
+        // Loading a replay cache is optional and always fails open to a miss.
+      }
+    }
   }
 }