agent-accelerator 0.5.2 → 0.5.3

package/src/session/store.ts

@@ -32,6 +32,7 @@ import { getModelFromCatalog } from "../models/catalog.ts";
 import type { SubAgentStep } from "../types/response.ts";
 import { SessionLoadError } from "../types/errors.ts";
 import { createSessionId } from "../utils/session.ts";
+import { writeFileDurable, readWithBackup } from "../utils/durable.ts";
 
 /** Cumulative per-session token + cost totals (JSON-safe). */
 export interface SessionTotals {
@@ -61,6 +62,40 @@ export interface PersistedSubAgentTrace {
   error?: string;
 }
 
+/** How a run ended; a run with no ending was cut off (a crash). `abandoned` = a newer run started instead of resuming it. */
+export type SessionRunEnd = "done" | "failed" | "aborted" | "stopped" | "abandoned";
+
+/** Where one run began: `i` is the index of its user message in `messages` (turn boundaries for later phases). */
+export interface SessionRunMark {
+  id: string;
+  i: number;
+  at: number;
+  /** Unset while the run is active or when it was cut off. */
+  end?: SessionRunEnd;
+}
+
+/** The result of a tool call that finished, saved the moment it finished so a crash never repeats it. */
+export interface SessionToolRecord {
+  id: string;
+  name: string;
+  result: unknown;
+  isError?: boolean;
+  /** The tool ended the run with `ctx.stop`. */
+  stopped?: boolean;
+}
+
+/** A prompt that was waiting when the process stopped (its options other than `additionalContext` are not saved). */
+export interface SessionPendingPrompt {
+  prompt: string | ContentPart[];
+  additionalContext?: string;
+}
+
+/** Steers not yet injected into the active run, and queued follow-ups, as last journaled. */
+export interface SessionPending {
+  steer: SessionPendingPrompt[];
+  queue: SessionPendingPrompt[];
+}
+
 /** Single-file session snapshot (v1). Written as one JSON object. */
 export interface PersistedAgentSession {
   version: 1;
@@ -79,6 +114,14 @@ export interface PersistedAgentSession {
   parentSessionId?: string;
   /** Dedicated sub-agent traces, keyed by worker TrackingID (32-char hex). */
   subagents?: Record<string, PersistedSubAgentTrace>;
+  /** Written with a journal (dir mode): only a `journal.ndjson` whose header names this id belongs to this snapshot. */
+  epoch?: string;
+  /** Where each run began, oldest first. */
+  runs?: SessionRunMark[];
+  /** Read from the journal only (never in `session.json`): prompts that were still waiting. */
+  pending?: SessionPending;
+  /** Read from the journal only: results of tool calls finished in the current run (used to resume without repeating them). */
+  tools?: SessionToolRecord[];
 }
 
 export function emptyTotals(): SessionTotals {
@@ -286,6 +329,12 @@ export interface SessionAgentLike {
   instructions?: string;
   parentSessionId?: unknown;
   subagentTraces?: Record<string, PersistedSubAgentTrace>;
+  /** Run boundaries (see {@link SessionRunMark}). */
+  runMarks?: SessionRunMark[];
+  /** Prompts waiting in the steer inbox / follow-up queue; called by the journal on every sync. */
+  pendingPrompts?: () => SessionPending;
+  /** Tool calls finished in the current run (cleared when the next run starts). */
+  toolRecords?: SessionToolRecord[];
   context: { systemPrompt?: string; messages: Message[] };
 }
 
@@ -354,7 +403,7 @@ function stableStringify(value: unknown): string {
  * stored `arguments` (pure duplicate, the common case). Divergent or
  * unparseable wire strings are kept — they carry info parsing lost.
  */
-function stripRedundantRawArguments(messages: Message[]): Message[] {
+export function stripRedundantRawArguments(messages: Message[]): Message[] {
   return messages.map((m) => {
     if (!Array.isArray((m as Message).content)) return m;
     let changed = false;
@@ -432,6 +481,7 @@ export function buildSessionData(agent: SessionAgentLike, telemetry?: SessionTel
     messages,
     ...(parentSessionId ? { parentSessionId } : {}),
     ...(subagents ? { subagents } : {}),
+    ...(agent.runMarks && agent.runMarks.length > 0 ? { runs: agent.runMarks.map((r) => ({ ...r })) } : {}),
   };
 }
 
@@ -619,6 +669,8 @@ function parseSession(raw: string, limits?: SessionLimits): PersistedAgentSessio
       totals: obj.totals,
       parentSessionId: obj.parentSessionId,
       subagents: obj.subagents,
+      ...(typeof obj.epoch === "string" ? { epoch: obj.epoch } : {}),
+      ...(Array.isArray(obj.runs) ? { runs: obj.runs as SessionRunMark[] } : {}),
     };
   } catch {
     return null;
@@ -654,14 +706,14 @@ export function inspectSessionPayload(
 /** Loads a session file (pretty `.json` or legacy `.jsonl`). Returns null when missing/corrupt. */
 export function loadSessionFile(filePath: string, limits?: SessionLimits): PersistedAgentSession | null {
   const resolved = path.resolve(filePath);
-  let raw: string;
-  try {
-    if (!fs.existsSync(resolved)) return null;
-    raw = fs.readFileSync(resolved, "utf8");
-  } catch {
-    return null;
-  }
-  return sessionFromPayload(raw, resolved, limits);
+  const read = readWithBackup(resolved, isLoadableSession);
+  return read ? sessionFromPayload(read.text, resolved, limits) : null;
+}
+
+// True when a stored payload is a usable snapshot (a valid one over the load limits counts); in: raw file text, out: readWithBackup's fallback decision.
+function isLoadableSession(raw: string): boolean {
+  const status = inspectSessionPayload(raw).status;
+  return status === "ok" || status === "over_limit";
 }
 
 /**
@@ -680,25 +732,8 @@ function sessionFromPayload(raw: string, file: string, limits?: SessionLimits):
   return res.session ?? null;
 }
 
-/** Atomic file write (temp + rename) so a crash never leaves half-written JSON. */
-export function writeFileAtomic(filePath: string, content: string): void {
-  const dir = path.dirname(filePath);
-  try {
-    fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
-  } catch {}
-  let tag = "tmp";
-  try {
-    const pid = (globalThis as any)?.process?.pid;
-    const rand = (globalThis as any)?.crypto?.randomUUID?.().slice(0, 8) ?? Math.random().toString(36).slice(2, 8);
-    tag = `${typeof pid === "number" ? pid : "np"}-${rand}`;
-  } catch {
-    tag = Math.random().toString(36).slice(2, 8);
-  }
-  const tmp = `${filePath}.tmp-${tag}`;
-  fs.writeFileSync(tmp, content, { encoding: "utf8", mode: 0o600 });
-  try { fs.chmodSync(tmp, 0o600); } catch {}
-  fs.renameSync(tmp, filePath);
-}
+/** Public name of the durable writer (temp file, fsync, `.bak`, atomic rename): a crash or power cut never leaves half-written JSON. */
+export const writeFileAtomic = writeFileDurable;
 
 /** Session id grammar: no traversal, no slashes, no hidden files. */
 export function assertValidSessionId(sessionId: string): void {
@@ -707,6 +742,15 @@ export function assertValidSessionId(sessionId: string): void {
   }
 }
 
+/** Renames a genuinely corrupt snapshot to `*.corrupt-<ts>` before it is overwritten (a healthy one over the load limits is our own prior save and is just rewritten). */
+export function quarantineIfCorrupt(file: string): void {
+  try {
+    if (inspectSessionPayload(fs.readFileSync(file, "utf8")).status === "corrupt") fs.renameSync(file, `${file}.corrupt-${Date.now()}`);
+  } catch {
+    // why: no file yet, or it cannot be read: nothing to quarantine.
+  }
+}
+
 /** Saves an agent + telemetry snapshot as a pretty-printed `.json` document (2-space indent). */
 export function saveSessionFile(
   filePath: string,
@@ -714,20 +758,7 @@ export function saveSessionFile(
   telemetry?: SessionTelemetry | SessionTotals | null
 ): void {
   const resolved = path.resolve(filePath);
-  // Q-28-partial: never overwrite a corrupt file in place — quarantine it
-  // as `*.corrupt-<ts>` first (load returns null for corrupt payloads).
-  try {
-    if (fs.existsSync(resolved)) {
-      try {
-        const raw = fs.readFileSync(resolved, "utf8");
-        // Only genuinely corrupt payloads are quarantined; a healthy snapshot
-        // over the load limits is simply rewritten (it is our own prior save).
-        if (inspectSessionPayload(raw).status === "corrupt") {
-          try { fs.renameSync(resolved, `${resolved}.corrupt-${Date.now()}`); } catch {}
-        }
-      } catch {}
-    }
-  } catch {}
+  quarantineIfCorrupt(resolved);
   const data = buildSessionData(agent, telemetry);
   try {
     writeFileAtomic(resolved, serializeSession(data) + "\n");
@@ -790,6 +821,28 @@ function extForMedia(mimeType?: string, filename?: string): string {
   return "bin";
 }
 
+// Finds embedded media bytes in a part (binary, `data:` URL or raw base64); remote URLs and local paths are references and give undefined; in: any content part, out: extractMediaToDir and the session journal.
+export function embeddedMedia(part: ContentPart): { key: (typeof MEDIA_PART_KEYS)[number]; bytes: Uint8Array; mimeType?: string; ext: string } | undefined {
+  const p = part as unknown as Record<string, unknown> | null;
+  if (!p || typeof p["type"] !== "string" || !(MEDIA_PART_KEYS as readonly string[]).includes(p["type"])) return undefined;
+  const key = p["type"] as (typeof MEDIA_PART_KEYS)[number];
+  const raw = p[key];
+  let bytes: Uint8Array;
+  let mimeType = typeof p["mimeType"] === "string" ? (p["mimeType"] as string) : undefined;
+  if (raw instanceof Uint8Array) bytes = raw;
+  else if (raw instanceof ArrayBuffer) bytes = new Uint8Array(raw);
+  else if (typeof raw === "string") {
+    if (isHttpRef(raw)) return undefined; // remote reference: the provider fetches it per turn
+    const dataUrl = raw.startsWith("data:") ? parseDataUrl(raw) : undefined;
+    if (dataUrl) {
+      bytes = base64ToBytes(dataUrl.base64);
+      mimeType = mimeType ?? dataUrl.mimeType;
+    } else if (looksLikeEmbeddedBase64(raw)) bytes = base64ToBytes(raw.replace(/\s/g, ""));
+    else return undefined; // local path or short label: keep as reference
+  } else return undefined;
+  return { key, bytes, ...(mimeType ? { mimeType } : {}), ext: extForMedia(mimeType, typeof p["filename"] === "string" ? (p["filename"] as string) : undefined) };
+}
+
 function cloneMessages(messages: Message[]): Message[] {
   try {
     const g: any = globalThis as any;
@@ -830,35 +883,15 @@ export function extractMediaToDir(
   } catch {}
   const wrote: string[] = [];
   const counters: Record<string, number> = {};
-  let hasBinary = false;
 
   for (const msg of cloned) {
     if (!Array.isArray((msg as Message).content)) continue;
     for (const part of (msg as Message).content as ContentPart[]) {
-      const p: any = part as any;
-      if (!p || typeof p.type !== "string") continue;
-      if (!(MEDIA_PART_KEYS as readonly string[]).includes(p.type)) continue;
-      const key = p.type as (typeof MEDIA_PART_KEYS)[number];
-      const raw = p[key] as unknown;
-      let bytes: Uint8Array | undefined;
-      let mimeType: string | undefined = typeof p.mimeType === "string" ? p.mimeType : undefined;
-      if (raw instanceof Uint8Array) bytes = raw;
-      else if (raw instanceof ArrayBuffer) bytes = new Uint8Array(raw);
-      else if (typeof raw === "string") {
-        if (isHttpRef(raw)) continue; // remote reference — provider fetches per turn
-        const dataUrl = raw.startsWith("data:") ? parseDataUrl(raw) : undefined;
-        if (dataUrl) {
-          bytes = base64ToBytes(dataUrl.base64);
-          mimeType = mimeType ?? dataUrl.mimeType;
-        } else if (looksLikeEmbeddedBase64(raw)) {
-          bytes = base64ToBytes(raw.replace(/\s/g, ""));
-        } else continue; // local path or short label — keep as reference
-      } else continue;
-
-      hasBinary = true;
+      const found = embeddedMedia(part);
+      if (!found) continue;
+      const { key, bytes, mimeType } = found;
       const n = (counters[key] = (counters[key] ?? 0) + 1);
-      const ext = extForMedia(mimeType, typeof p.filename === "string" ? p.filename : undefined);
-      const file = `${key}-${String(n).padStart(3, "0")}.${ext}`;
+      const file = `${key}-${String(n).padStart(3, "0")}.${found.ext}`;
       try {
         fs.mkdirSync(mediaDir, { recursive: true, mode: 0o700 });
         const dest = path.resolve(mediaDir, file);
@@ -868,8 +901,9 @@ export function extractMediaToDir(
       } catch {
         continue;
       }
+      const p = part as unknown as Record<string, unknown>;
       p[key] = `media/${file}`;
-      if (mimeType) p.mimeType = mimeType;
+      if (mimeType) p["mimeType"] = mimeType;
       wrote.push(`media/${file}`);
     }
   }
@@ -937,17 +971,7 @@ export function saveSessionDir(
   );
   try {
     const target = path.join(sessionDir, "session.json");
-    // Q-28-partial: quarantine corrupt targets instead of overwriting.
-    try {
-      if (fs.existsSync(target)) {
-        try {
-          const raw = fs.readFileSync(target, "utf8");
-          if (inspectSessionPayload(raw).status === "corrupt") {
-            try { fs.renameSync(target, `${target}.corrupt-${Date.now()}`); } catch {}
-          }
-        } catch {}
-      }
-    } catch {}
+    quarantineIfCorrupt(target);
     writeFileAtomic(target, serializeSession(data) + "\n");
   } catch {
     // persistence must never crash a chat turn
@@ -955,6 +979,67 @@ export function saveSessionDir(
   return sessionDir;
 }
 
+/** Name of the append-only step log next to `session.json` (see `session/journal.ts`). */
+export const JOURNAL_FILE = "journal.ndjson";
+
+// Snapshot fields a `meta` journal record replaces (everything except messages, identity and totals).
+const JOURNAL_META_KEYS = ["model", "subagentModel", "thinkingLevel", "cache", "systemPrompt", "parentSessionId", "subagents"] as const;
+
+// Applies a journal to its snapshot: records of the snapshot's own epoch only, `msg` by index (already-present ones skipped), `run` adds a run boundary, `end` closes one, `tool` keeps a finished tool result, `pending` replaces the waiting prompts, `meta` replaces config fields, unknown record types ignored; a torn last line is dropped, damage earlier stops the replay with a bus warning; in: the parsed snapshot + journal text, out: loadSessionDir.
+function replayJournal(snapshot: PersistedAgentSession, text: string, file: string): PersistedAgentSession {
+  if (!snapshot.epoch) return snapshot;
+  const lines = text.split("\n");
+  const session: PersistedAgentSession = { ...snapshot, messages: [...snapshot.messages] };
+  let sawHeader = false;
+  const stop = (reason: string, line: number): void => {
+    telemetryBus.emitWarning("session.journal_corrupt", `[Agent Accelerator] ${path.basename(file)} line ${line + 1}: ${reason}; the rest of the journal is ignored.`, { file, line: line + 1 });
+  };
+  for (let k = 0; k < lines.length; k++) {
+    const line = lines[k]!.trim();
+    if (!line) continue;
+    let rec: { t?: unknown; e?: unknown; i?: unknown; m?: unknown; p?: unknown; id?: unknown; at?: unknown; status?: unknown; call?: unknown; name?: unknown; result?: unknown; isError?: unknown; stopped?: unknown };
+    try {
+      rec = JSON.parse(line);
+    } catch {
+      if (lines.slice(k + 1).some((l) => l.trim())) stop("not valid JSON", k); // a torn LAST line is just a cut-off write
+      break;
+    }
+    if (!sawHeader) {
+      sawHeader = true;
+      if (rec.t !== "base" || rec.e !== snapshot.epoch) return snapshot; // a journal from before this snapshot
+      continue;
+    }
+    if (rec.t === "msg" && typeof rec.i === "number") {
+      if (rec.i < session.messages.length) continue; // already in the snapshot
+      if (rec.i > session.messages.length || findInvalidMessage([rec.m]) !== -1) {
+        stop(rec.i > session.messages.length ? "a message is missing before this one" : "invalid message", k);
+        break;
+      }
+      session.messages.push(rec.m as Message);
+    } else if (rec.t === "run" && typeof rec.id === "string" && typeof rec.i === "number" && typeof rec.at === "number") {
+      if (!session.runs?.some((r) => r.id === rec.id)) session.runs = [...(session.runs ?? []), { id: rec.id, i: rec.i, at: rec.at }];
+    } else if (rec.t === "end" && typeof rec.id === "string" && typeof rec.status === "string") {
+      const mark = session.runs?.find((r) => r.id === rec.id);
+      if (mark) session.runs = session.runs!.map((r) => (r === mark ? { ...r, end: rec.status as SessionRunEnd } : r));
+    } else if (rec.t === "tool" && typeof rec.call === "string" && typeof rec.name === "string") {
+      session.tools = [...(session.tools ?? []).filter((x) => x.id !== rec.call), { id: rec.call, name: rec.name, result: rec.result, ...(rec.isError ? { isError: true } : {}), ...(rec.stopped ? { stopped: true } : {}) }];
+    } else if (rec.t === "pending" && rec.p && typeof rec.p === "object") {
+      const p = rec.p as SessionPending; // latest record wins
+      if (Array.isArray(p.steer) && Array.isArray(p.queue)) {
+        if (p.steer.length + p.queue.length > 0) session.pending = p;
+        else delete session.pending;
+      }
+    } else if (rec.t === "meta" && rec.m && typeof rec.m === "object") {
+      const meta = rec.m as Record<string, unknown>;
+      for (const key of JOURNAL_META_KEYS) {
+        if (key in meta) (session as unknown as Record<string, unknown>)[key] = meta[key];
+        else delete session[key];
+      }
+    }
+  }
+  return session;
+}
+
 /** A loaded session folder: snapshot (media paths resolved) + folder location. */
 export interface LoadedSessionDir {
   session: PersistedAgentSession;
@@ -985,9 +1070,17 @@ export function loadSessionDir(sessionPath: string, limits?: SessionLimits): Loa
       // path does not exist yet — fall through to session.json lookup
     }
     const file = path.join(dir, "session.json");
-    if (!fs.existsSync(file)) return null;
-    const data = sessionFromPayload(fs.readFileSync(file, "utf8"), file, limits);
-    if (!data) return null;
+    const read = readWithBackup(file, isLoadableSession);
+    if (!read) return null;
+    const snapshot = sessionFromPayload(read.text, file, limits);
+    if (!snapshot) return null;
+    let journal: string | undefined;
+    try {
+      journal = fs.readFileSync(path.join(dir, JOURNAL_FILE), "utf8");
+    } catch {
+      journal = undefined; // no journal: the snapshot is the whole session
+    }
+    const data = journal ? replayJournal(snapshot, journal, path.join(dir, JOURNAL_FILE)) : snapshot;
     data.messages = resolveMediaPaths(data.messages, dir);
     return { session: data, dir };
   } catch (err) {
@@ -1007,7 +1100,13 @@ export function findLatestSessionDir(rootDir: string): string | null {
       try {
         const st = fs.statSync(file);
         if (!st.isFile()) continue;
-        if (!best || st.mtimeMs > best.mtime) best = { dir: path.join(root, entry), mtime: st.mtimeMs };
+        let mtime = st.mtimeMs;
+        try {
+          mtime = Math.max(mtime, fs.statSync(path.join(root, entry, JOURNAL_FILE)).mtimeMs); // steps land in the journal
+        } catch {
+          // why: no journal yet.
+        }
+        if (!best || mtime > best.mtime) best = { dir: path.join(root, entry), mtime };
       } catch {
         continue;
       }

package/src/session/tool-log.ts

@@ -0,0 +1,62 @@
+/**
+ * One log file per tool call, inside the session folder: `tool-logs/<n>-<tool>.ndjson`. The executor writes its own
+ * steps into it (input, attempts, retries, result or error, duration) and a tool adds its own with `ctx.log(...)`.
+ * Every line is flushed to disk as it is written, so after a crash the file shows how far the call got. The file is for
+ * developers and harnesses: it is never sent to the model, and nothing in the SDK reads it back.
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { fsyncFolder } from "../utils/durable.ts";
+
+/** The log of one tool call, as the executor sees it. */
+export interface ToolLogWriter {
+  /** Path relative to the session folder (`tool-logs/003-send_email.ndjson`); saved on the call's result. */
+  readonly file: string;
+  /** Appends one line (`timestamp` is added) and flushes it. Never throws. */
+  write(entry: Record<string, unknown>): void;
+  /** Closes the file; later writes (a tool still running after a timeout) are ignored. */
+  close(): void;
+}
+
+/** Hands out the log files of one session folder, numbering them in the order the calls start. */
+export class ToolLogs {
+  private next: number | undefined;
+
+  constructor(private readonly sessionDir: string) {}
+
+  // Opens `tool-logs/<n>-<tool>.ndjson` for one call; the numbering continues after files from earlier runs and restarts; in: a tool call as the executor starts it, out: the writer the executor and `ctx.log` write through.
+  open(call: { name: string }): ToolLogWriter | undefined {
+    const dir = path.join(this.sessionDir, "tool-logs");
+    try {
+      fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
+      this.next ??= fs.readdirSync(dir).reduce((max, f) => Math.max(max, Number(/^(\d+)-/.exec(f)?.[1] ?? 0)), 0) + 1;
+      const name = `${String(this.next++).padStart(3, "0")}-${call.name.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 64)}.ndjson`;
+      const fd = fs.openSync(path.join(dir, name), "a", 0o600);
+      fsyncFolder(dir);
+      let open = true;
+      return {
+        file: `tool-logs/${name}`,
+        write(entry) {
+          if (!open) return;
+          try {
+            fs.writeSync(fd, JSON.stringify({ timestamp: Date.now(), ...entry }) + "\n");
+            fs.fdatasyncSync(fd);
+          } catch {
+            // why: a full disk must never fail the tool call being logged.
+          }
+        },
+        close() {
+          if (!open) return;
+          open = false;
+          try {
+            fs.closeSync(fd);
+          } catch {
+            // why: nothing left to flush.
+          }
+        },
+      };
+    } catch {
+      return undefined; // logging is best effort: the call runs without a log
+    }
+  }
+}

package/src/telemetry-file.ts

@@ -4,12 +4,14 @@
  * Each event is a single synchronous append, so a process crash keeps every
  * event already emitted: a `tool.call_started` with no ending event was in
  * flight, and the last `tool.progress` shows how far it got. Power loss is not
- * covered (no fsync). `turn.thinking` (full reasoning text) is never written.
+ * covered (no fsync, by design: this is observability, not state). `turn.thinking`
+ * (full reasoning text) is never written. Takes the home's one-process lock.
  */
 import * as fs from "node:fs";
 import * as path from "node:path";
 import { telemetryBus } from "./telemetry.ts";
 import { safeStringify } from "./utils/serialization.ts";
+import { lockFolder } from "./utils/durable.ts";
 
 const sinks = new Set<string>();
 
@@ -17,6 +19,7 @@ const sinks = new Set<string>();
 export function persistTelemetry(home: string): void {
   const dir = path.join(home, "telemetry");
   if (sinks.has(dir)) return;
+  lockFolder(home);
   sinks.add(dir);
   telemetryBus.onEvent((event) => {
     if (event.code === "turn.thinking") return;