@tt-a1i/openpi 0.4.0 → 0.5.0

package/extensions/workflows/artifacts.ts

@@ -1,32 +1,40 @@
-import {
-  refreshWorkflowGraph,
-  type TranscriptEntry,
-  type WorkflowDetails,
-} from "./model.ts";
+import * as fs from "node:fs";
+import * as path from "node:path";
 import {
   boundedJournal,
-  parseJournal,
+  JOURNAL_MAX_BYTES,
   type JournalEntry,
+  parseJournal,
   type WorkflowJournal,
+  type WorkflowJournalAccumulator,
 } from "./journal.ts";
 import {
+  refreshWorkflowGraph,
+  type TranscriptEntry,
+  type WorkflowDelivery,
+  type WorkflowDetails,
+} from "./model.ts";
+import {
+  encodeCompleteJson,
   safeStringify,
-  toSerializable,
   truncateUtf8,
   writeFileAtomic,
 } from "./serialization.ts";
-import * as fs from "node:fs";
-import * as path from "node:path";
 
 export const JOURNAL_FILE = "journal.json";
 
 const ARTIFACT_TRANSCRIPT_MAX_BYTES = 32 * 1024;
 const ARTIFACT_TRANSCRIPT_ENTRY_MAX_BYTES = 8 * 1024;
+const AGENT_RESULT_ARTIFACT_MAX_BYTES = 2 * 1024 * 1024;
 export const WORKFLOW_CHECKPOINT_INTERVAL_MS = 500;
 const ENTRY_TRUNCATION_MARKER = "\n[entry truncated]";
 const TRANSCRIPT_TRUNCATION_MARKER =
   "[artifact transcript truncated: older entries omitted]";
 
+type WorkflowJournalSource =
+  | readonly JournalEntry[]
+  | WorkflowJournalAccumulator;
+
 function textBytes(text: string) {
   return Buffer.byteLength(text, "utf8");
 }
@@ -101,44 +109,61 @@ function writeRunFile(runDir: string, name: string, content: string) {
   writeFileAtomic(path.join(runDir, name), content);
 }
 
+export function persistWorkflowTerminalState(
+  runDir: string,
+  details: WorkflowDetails,
+) {
+  const terminalManifest: WorkflowDetails = {
+    ...details,
+    agents: details.agents.map((agent) => ({ ...agent, transcript: [] })),
+  };
+  delete terminalManifest.result;
+  delete terminalManifest.resultArtifact;
+  delete terminalManifest.transcriptArtifact;
+  writeRunFile(
+    runDir,
+    "workflow.json",
+    safeStringify(terminalManifest, { maxBytes: 1024 * 1024 }),
+  );
+}
+
 /** Persist one successful child result before any handoff/context projection. */
 export function persistWorkflowAgentResult(
   runDir: string,
   index: number,
   result: { output: string; structured?: unknown },
 ) {
-  const artifact = path.join(
-    "agent-results",
-    `agent-${String(index).padStart(4, "0")}.json`,
-  );
-  writeRunFile(
-    runDir,
-    artifact,
-    JSON.stringify(
-      toSerializable(
-        {
-          output: result.output,
-          ...(result.structured !== undefined
-            ? { structured: result.structured }
-            : {}),
-        },
-        {
-          maxDepth: 32,
-          maxNodes: 100_000,
-          maxStringBytes: 2 * 1024 * 1024,
-        },
-      ),
-      null,
-      2,
-    ),
+  // Artifact references are persisted as portable workflow paths. The host
+  // filesystem join happens in writeRunFile; handoff validation and replay
+  // consumers intentionally use `/` regardless of the platform.
+  const artifact = `agent-results/agent-${String(index).padStart(4, "0")}.json`;
+  const encoded = encodeCompleteJson(
+    {
+      output: result.output,
+      ...(result.structured !== undefined
+        ? { structured: result.structured }
+        : {}),
+    },
+    {
+      maxBytes: AGENT_RESULT_ARTIFACT_MAX_BYTES,
+      maxDepth: 32,
+      maxNodes: 100_000,
+      maxStringBytes: AGENT_RESULT_ARTIFACT_MAX_BYTES,
+    },
   );
+  if (!encoded.ok) {
+    throw new Error(
+      `Agent result artifact exceeded the ${AGENT_RESULT_ARTIFACT_MAX_BYTES}-byte budget (${encoded.limit} limit at ${encoded.path})`,
+    );
+  }
+  writeRunFile(runDir, artifact, encoded.json);
   return artifact;
 }
 
 export function persistWorkflowJson(
   runDir: string,
   details: WorkflowDetails,
-  journal?: readonly JournalEntry[],
+  journal?: WorkflowJournalSource,
 ) {
   refreshWorkflowGraph(details);
   const transcripts = Object.fromEntries(
@@ -147,6 +172,15 @@ export function persistWorkflowJson(
       boundedArtifactTranscript(agent.transcript),
     ]),
   );
+
+  // Publish terminal execution facts before dependent side artifacts. If a
+  // later artifact write fails, readers still see an explained terminal run
+  // instead of the previous `running` manifest. The final manifest below adds
+  // the artifact references once every dependent file has committed.
+  if (details.status !== "running") {
+    persistWorkflowTerminalState(runDir, details);
+  }
+
   writeRunFile(
     runDir,
     "transcripts.json",
@@ -154,15 +188,20 @@ export function persistWorkflowJson(
   );
   // Written alongside the rest so it inherits atomic write, 500ms coalescing,
   // and the final flush. Only present once a call has actually succeeded.
-  // boundedJournal has already brought this under the cap, and plain
-  // JSON.stringify is deliberate: safeStringify would swap an over-cap value
-  // for a preview stub, silently turning the journal into something unusable.
-  if (journal && journal.length > 0) {
-    writeRunFile(
-      runDir,
-      JOURNAL_FILE,
-      JSON.stringify(boundedJournal(journal).journal, null, 2),
-    );
+  // Accumulators already enforce the cap incrementally and can assemble the
+  // canonical JSON from their cached entry fragments. Plain arrays retain the
+  // compatibility path through boundedJournal; plain JSON.stringify is
+  // deliberate because safeStringify would swap an over-cap value for a
+  // preview stub, silently turning the journal into something unusable.
+  if (
+    journal &&
+    (journal.length > 0 || ("toJson" in journal && journal.dropped > 0))
+  ) {
+    const content =
+      "toJson" in journal
+        ? journal.toJson()
+        : JSON.stringify(boundedJournal(journal).journal, null, 2);
+    writeRunFile(runDir, JOURNAL_FILE, content);
   }
   if (details.result !== undefined) {
     writeRunFile(
@@ -186,6 +225,25 @@ export function persistWorkflowJson(
   );
 }
 
+/**
+ * Update only the durable delivery receipt. Completion delivery may retain a
+ * compact memory projection after the run has been evicted, so rewriting the
+ * whole details object here would risk replacing exact result artifacts with
+ * that projection.
+ */
+export function persistWorkflowDeliveryState(
+  runDir: string,
+  delivery: WorkflowDelivery,
+) {
+  const file = path.join(runDir, "workflow.json");
+  const raw: unknown = JSON.parse(fs.readFileSync(file, "utf8"));
+  if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
+    throw new Error(`Invalid persisted workflow details: ${file}`);
+  }
+  const next = { ...(raw as Record<string, unknown>), delivery };
+  writeFileAtomic(file, JSON.stringify(next));
+}
+
 /** Coalesce live checkpoints while keeping final persistence synchronous. */
 export function createWorkflowPersistence(
   runDir: string,
@@ -195,10 +253,10 @@ export function createWorkflowPersistence(
     persist?: (
       runDir: string,
       details: WorkflowDetails,
-      journal?: readonly JournalEntry[],
+      journal?: WorkflowJournalSource,
     ) => void;
-    /** Read at write time so callers only have to append to their own array. */
-    journal?: () => readonly JournalEntry[];
+    /** Read at write time so callers only have to append to their journal. */
+    journal?: () => WorkflowJournalSource;
   } = {},
 ) {
   const intervalMs = Math.max(
@@ -256,10 +314,42 @@ export function createWorkflowPersistence(
  * turn into a new way for a run to fail.
  */
 export function loadJournal(runDir: string): WorkflowJournal | undefined {
+  let descriptor: number | undefined;
   try {
-    const raw = fs.readFileSync(path.join(runDir, JOURNAL_FILE), "utf8");
+    descriptor = fs.openSync(path.join(runDir, JOURNAL_FILE), "r");
+    const initialSize = fs.fstatSync(descriptor).size;
+    if (initialSize > JOURNAL_MAX_BYTES) return undefined;
+
+    // Read at most one byte beyond the cap. The descriptor pins the inspected
+    // file, while the extra byte catches growth after fstat without ever
+    // allocating or parsing an unbounded journal.
+    const bytes = Buffer.allocUnsafe(initialSize + 1);
+    let length = 0;
+    while (length < bytes.length) {
+      const read = fs.readSync(
+        descriptor,
+        bytes,
+        length,
+        bytes.length - length,
+        null,
+      );
+      if (read === 0) break;
+      length += read;
+    }
+    // Filling the extra byte means the file changed after inspection. Reject
+    // that race even when the new size would still fit the ordinary cap.
+    if (length === bytes.length) return undefined;
+    const raw = bytes.toString("utf8", 0, length);
     return parseJournal(JSON.parse(raw));
   } catch {
     return undefined;
+  } finally {
+    if (descriptor !== undefined) {
+      try {
+        fs.closeSync(descriptor);
+      } catch {
+        // Loading a replay cache is optional and always fails open to a miss.
+      }
+    }
   }
 }