@mono-agent/agent-runtime 0.20.14 → 0.21.0

package/src/ai/providers/pi-native/session-lifecycle.js

@@ -10,13 +10,15 @@
 // createSessionLiveness primitives so the await-free spans are enforced by
 // construction rather than by inline sequencing.
 
-import { JsonlSessionRepo, MemorySessionRepo } from "@earendil-works/pi-agent-core";
+import { JsonlSessionRepo, MemorySessionRepo, laneConfig, laneState, operationMeta, operationResult, operationState } from "@earendil-works/pi-agent-core";
+import { validRecoveryProjection } from "./terminal-recovery.js";
 import { NodeExecutionEnv } from "@earendil-works/pi-agent-core/node";
-import { access, open } from "node:fs/promises";
-import { dirname, resolve } from "node:path";
+import { createHash } from "node:crypto";
+import { access, open, readdir, unlink } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
 import { createSessionRegistry } from "../../runtime/sessions.js";
 import { createSessionLiveness } from "../../runtime/session-liveness.js";
-import { createPiSessionAdapter, PI_CONTEXT } from "./harness-adapter.js";
+import { buildPiSessionContext, createPiSessionAdapter, PI_CONTEXT } from "./harness-adapter.js";
 
 async function syncPath(path) {
   const handle = await open(path, "r");
@@ -28,6 +30,7 @@ async function syncPath(path) {
 }
 
 async function syncDurableTranscript(entry) {
+  if (entry.recoveryPending) throw new Error("Pi terminal recovery is pending");
   if (!entry.durable) return;
   const path = entry.metadata?.path;
   if (typeof path !== "string" || !path) {
@@ -91,7 +94,7 @@ async function closeAndDeleteSession(session, repo, knownMetadata) {
 // reach over native pi sessions that the legacy bridge had.
 const nativeSessionRepo = new MemorySessionRepo();
 const nativeSessions = createSessionRegistry({
-  isBusy: (entry) => entry.busy === true,
+  isBusy: (entry) => entry.busy === true || entry.recoveryPending === true,
   onSync: syncDurableTranscript,
   onEvict: async (entry, reason) => {
     // Ordinary disposal/TTL only drops registry metadata so durable sessions
@@ -128,11 +131,13 @@ export function resolveDurableNativeSessionRepo(piSessionsRoot) {
 }
 
 /**
- * Permanently retire every durable Pi transcript with this exact logical id.
- * This is intentionally stronger than live-session invalidation: history
- * rotation and retention can retire an epoch after its registry entry was
- * already evicted or after a process restart. Absence is success; any cleanup
- * or verification uncertainty rejects so canonical history remains reachable.
+ * Retire every currently materialized durable Pi transcript with this exact
+ * logical id. This is intentionally stronger than live-session invalidation:
+ * history rotation and retention can retire an epoch after its registry entry
+ * was already evicted or after a process restart. When an active old run later
+ * recreates its pathname, its post-runtime caller retries this operation. The
+ * canonical epoch has already rotated, so that old id is never resumable in the
+ * interim. Absence is success; cleanup or verification uncertainty rejects.
  */
 export async function retireDurableNativeSession(providerSessionId, piSessionsRoot) {
   if (!isSafeSessionId(providerSessionId)) {
@@ -142,7 +147,15 @@ export async function retireDurableNativeSession(providerSessionId, piSessionsRo
     throw new TypeError("piSessionsRoot must be a non-empty path");
   }
 
-  // First guarantee this process cannot keep writing through a stale handle.
+  // First guarantee this process cannot resume through a stale registry entry.
+  // A host cancellation can rotate canonical history while the provider is
+  // still unwinding an open Pi session. The Pi repo deliberately refuses to
+  // delete an open session, so detach the registry and unlink its current file.
+  // Pi appends by pathname and can recreate a headerless orphan if the old run
+  // writes later; the raw exact-name sweep below removes those files on the
+  // post-runtime retry. The rotated canonical epoch never refers to this id.
+  const liveEntry = nativeSessions.get(providerSessionId);
+  const providerStillUnwinding = liveEntry?.busy === true;
   // Other processes are serialized by the history coordinator and must pass
   // the same cold-refresh barrier before their next turn.
   await nativeSessions.refresh(providerSessionId);
@@ -155,16 +168,66 @@ export async function retireDurableNativeSession(providerSessionId, piSessionsRo
     if (typeof metadata?.path !== "string" || !metadata.path) {
       throw new Error(`Durable Pi session ${providerSessionId} has invalid metadata`);
     }
-    await repo.delete(metadata, PI_CONTEXT);
+    if (providerStillUnwinding) {
+      try {
+        await unlink(metadata.path);
+      } catch (error) {
+        if (error?.code !== "ENOENT") throw error;
+      }
+    } else {
+      await repo.delete(metadata, PI_CONTEXT);
+    }
     changedDirectories.add(dirname(metadata.path));
   }
+  // A prior active-session unlink can be followed by Pi recreating the same
+  // pathname with a transaction line but no header. JsonlSessionRepo.list()
+  // intentionally ignores that invalid file, so scan only Pi's fixed
+  // root/directory/filename layout to make the later retirement retry reclaim
+  // it. Safe ids are single filename components and symlink directories/files
+  // are ignored.
+  for (const path of await exactDurableSessionFiles(piSessionsRoot, providerSessionId)) {
+    try {
+      await unlink(path);
+    } catch (error) {
+      if (error?.code !== "ENOENT") throw error;
+    }
+    changedDirectories.add(dirname(path));
+  }
   for (const directory of changedDirectories) await syncPath(directory);
   if (changedDirectories.size > 0) await syncPath(resolve(piSessionsRoot));
 
   const remaining = (await repo.list(undefined, PI_CONTEXT)).filter((entry) => entry?.id === providerSessionId);
-  if (remaining.length > 0) {
+  const remainingPaths = await exactDurableSessionFiles(piSessionsRoot, providerSessionId);
+  if (!providerStillUnwinding && (remaining.length > 0 || remainingPaths.length > 0)) {
     throw new Error(`Durable Pi session ${providerSessionId} could not be retired completely`);
   }
+  // An active Pi write can race the final exact-name check after the sweep.
+  // Canonical history is already preparing a fresh epoch, so the retired id is
+  // unreachable; the harness retries retirement when that old run returns.
+}
+
+async function exactDurableSessionFiles(piSessionsRoot, providerSessionId) {
+  const root = resolve(piSessionsRoot);
+  const suffix = `_${encodeURIComponent(providerSessionId)}.jsonl`;
+  let rootEntries;
+  try {
+    rootEntries = await readdir(root, { withFileTypes: true });
+  } catch (error) {
+    if (error?.code === "ENOENT") return [];
+    throw error;
+  }
+  const paths = [];
+  for (const directory of rootEntries) {
+    if (!directory.isDirectory()) continue;
+    const directoryPath = join(root, directory.name);
+    const entries = await readdir(directoryPath, { withFileTypes: true });
+    for (const entry of entries) {
+      if (entry.isFile() && entry.name.endsWith(suffix)) {
+        paths.push(join(directoryPath, entry.name));
+      }
+    }
+  }
+  return paths;
 }
 
 // Defense in depth (R4): create-on-miss passes the caller-controlled session id
@@ -355,7 +418,7 @@ export async function resolveSession(runState, {
       // (F4) safe. Do not introduce any await in this span or the TOCTOU window
       // reopens. `claim` re-reads the same registry entry and sets busy in one
       // await-free step; a busy entry loses and returns session_busy.
-      const claimed = liveness.claim(requestedSessionId);
+      const claimed = entry.recoveryPending ? { ok: /** @type {const} */ (false), reason: "busy" } : liveness.claim(requestedSessionId);
       if (!claimed.ok) {
         // claim() can lose two ways: "busy" (the entry adopted above is
         // mid-turn) or "missing" (no live entry). "missing" is UNREACHABLE on
@@ -386,6 +449,7 @@ export async function resolveSession(runState, {
         };
       }
       runState.sessionEntry = claimed.entry;
+      delete claimed.entry.recovery;
       try {
         runState.session = createPiSessionAdapter(await claimed.entry.repo.open(
           claimed.entry.metadata,
@@ -403,10 +467,11 @@ export async function resolveSession(runState, {
       }
     }
   } else {
-    // Fresh runs persist into the durable jsonl repo when piSessionsRoot is
-    // set, so a kept-alive session can be reopened from disk after the live
-    // entry is evicted; otherwise the in-memory repo is used.
-    runState.session = createPiSessionAdapter(await (durableRepo || nativeSessionRepo)
+    // Attribution may equal a live primary's id on a stateless retry/backup.
+    // A private ephemeral repo prevents both create collisions and cleanup of
+    // that primary's transcript. Only keep-alive calls use a shared repository.
+    if (options.sessionKeepAlive !== true) runState.ephemeralSessionRepo = new MemorySessionRepo();
+    runState.session = createPiSessionAdapter(await (runState.ephemeralSessionRepo || durableRepo || nativeSessionRepo)
       .create({ id: providerSessionId, cwd: cwd || process.cwd() }, PI_CONTEXT));
   }
   return { done: false };
@@ -428,7 +493,7 @@ export async function discardUncommittedSession(runState, { durableRepo }) {
   if (runState.session && !runState.sessionEntry) {
     await closeAndDeleteSession(
       runState.session,
-      durableRepo || nativeSessionRepo,
+      runState.ephemeralSessionRepo || durableRepo || nativeSessionRepo,
     );
   }
   // Drop the create-on-miss BUSY reservation too, else the busy placeholder
@@ -457,7 +522,7 @@ export async function commitSession(runState, {
   onEvent,
 }) {
   const { session, sessionEntry, baselineLeafId, reservation } = runState;
-  if (options.sessionKeepAlive === true && !externalAbort && !errorMessage) {
+  if (options.sessionKeepAlive === true && ((!externalAbort && !errorMessage) || runState.retainRecoveryTail)) {
     try {
       if (sessionEntry) {
         // Resumed run: the harness appended this run's turns onto the live
@@ -480,6 +545,8 @@ export async function commitSession(runState, {
         else nativeSessions.set(providerSessionId, entry, { idleTimeoutMs: sessionTtlMs });
         runState.registeredSessionEntry = entry;
       }
+      const retained = runState.sessionEntry || runState.registeredSessionEntry;
+      if (runState.retainRecoveryTail && retained) retained.recoveryPending = !!externalAbort || !!errorMessage;
     } catch (err) {
       // Session persistence must never fail the run; drop the (now
       // inconsistent) session instead of resuming from a broken transcript.
@@ -513,7 +580,7 @@ export async function commitSession(runState, {
     // (the success keep-alive path overwrites it with the finalized entry, so
     // it is only this drop branch that must clean it up).
     if (reservation) reservation.release();
-    await closeAndDeleteSession(session, durableRepo || nativeSessionRepo);
+    await closeAndDeleteSession(session, runState.ephemeralSessionRepo || durableRepo || nativeSessionRepo);
   }
 }
 
@@ -534,8 +601,9 @@ export async function rollbackAbortedTurn(runState, { requestedSessionId, provid
     }
     nativeSessions.delete(requestedSessionId);
   } else {
-    nativeSessions.delete(providerSessionId);
-    await closeAndDeleteSession(session, durableRepo || nativeSessionRepo);
+    // A stateless call never registered this id; it may belong to the primary.
+    if (!runState.ephemeralSessionRepo) nativeSessions.delete(providerSessionId);
+    await closeAndDeleteSession(session, runState.ephemeralSessionRepo || durableRepo || nativeSessionRepo);
   }
 }
 
@@ -558,7 +626,7 @@ export async function cleanupSessionOnThrow(runState, { durableRepo }) {
   // resumed user session here would be data loss) and never when the throw
   // preceded session create.
   if (session && !sessionEntry) {
-    await closeAndDeleteSession(session, durableRepo || nativeSessionRepo);
+    await closeAndDeleteSession(session, runState.ephemeralSessionRepo || durableRepo || nativeSessionRepo);
   }
   // Drop a create-on-miss BUSY placeholder (R8) left in the registry by a throw
   // during/after the reservation — including a throw inside the create await
@@ -582,3 +650,78 @@ export async function cleanupSessionOnThrow(runState, { durableRepo }) {
     try { await session.close(); } catch { /* best-effort */ }
   }
 }
+
+/** Capture only after close; pending entries cannot be driven by another turn. */
+export async function captureSessionRecovery(runState, { options, providerSessionId, modelKey, model, pending }) {
+  const entry = runState.sessionEntry || runState.registeredSessionEntry;
+  if (!entry?.durable) return undefined;
+  try {
+    if (!Array.isArray(runState.recoveryInputIds)) throw new Error("Pi session recovery input identities are unavailable");
+    const tipId = await runState.session.getLeafId();
+    if (typeof tipId !== "string" || !tipId) throw new Error("Pi session recovery tip is unavailable");
+    const ancestry = createHash("sha256").update(JSON.stringify(await runState.session.getEntries())).digest("hex");
+    await runState.session.close();
+    const receipt = { runId: options.sessionRecovery.runId, revision: options.sessionRecovery.revision, providerSessionId, modelKey, tipId };
+    entry.recovery = { receipt: { ...receipt }, model: { ...model, input: [...model.input] }, ancestry, operationId: runState.recoveryOperationId, baselineTipId: runState.recoveryBaselineTipId, inputIds: runState.recoveryInputIds };
+    entry.recoveryPending = pending || !!options.abortSignal?.aborted;
+    return receipt;
+  } catch (error) {
+    // The run's outer catch performs legacy rollback/close or fresh deletion.
+    // Release provisional recovery state first so failed capture cannot strand
+    // an entry as busy without a receipt that could settle it.
+    entry.recoveryPending = false;
+    delete entry.recovery;
+    throw error;
+  }
+}
+
+/** Read-only settlement: never drive an operation or append host-authored prose. */
+export async function recoverDurableNativeSession(receipt, context) {
+  const entry = nativeSessions.get(receipt?.providerSessionId);
+  const proof = entry?.recovery;
+  if (!entry?.durable || entry.busy || !proof
+    || !["runId", "revision", "providerSessionId", "modelKey", "tipId"].every((key) => receipt[key] === proof.receipt[key])
+    || !Array.isArray(context?.appliedInputIds)
+    || proof.inputIds.some((id) => typeof id !== "string")
+    || JSON.stringify([...proof.inputIds].sort()) !== JSON.stringify([...context.appliedInputIds].sort())) return false;
+  entry.busy = true;
+  let raw;
+  try {
+    const matches = (await entry.repo.list(undefined, PI_CONTEXT)).filter((record) => record.id === receipt.providerSessionId);
+    if (matches.length !== 1 || matches[0].path !== entry.metadata.path) return false;
+    raw = await entry.repo.open(matches[0], PI_CONTEXT);
+    const branch = await raw.branch("main", PI_CONTEXT);
+    if (!branch || await branch.getTipId(PI_CONTEXT) !== receipt.tipId) return false;
+    const state = (await raw.getValue(laneState("main"), PI_CONTEXT))?.value;
+    const config = (await raw.getValue(laneConfig("main"), PI_CONTEXT))?.value;
+    const terminal = (await raw.getValue(operationResult(proof.operationId), PI_CONTEXT))?.value;
+    const meta = (await raw.getValue(operationMeta(proof.operationId), PI_CONTEXT))?.value;
+    if (!state || state.currentOperationId !== null || state.lastOperationId !== proof.operationId || state.inbox.length !== 0
+      || config?.model.provider !== proof.model.provider || config?.model.modelId !== proof.model.id
+      || !terminal || !["completed", "failed", "aborted"].includes(terminal.status) || terminal.tipId !== receipt.tipId
+      || meta !== undefined || terminal.kind !== "run" || terminal.fromTipId !== proof.baselineTipId
+      || await raw.getValue(operationState(proof.operationId), PI_CONTEXT)) return false;
+    const entries = await branch.findEntries({ order: "oldestFirst" }, PI_CONTEXT);
+    if (createHash("sha256").update(JSON.stringify(entries)).digest("hex") !== proof.ancestry) return false;
+    const baseline = proof.baselineTipId === null ? -1 : entries.findIndex((item) => item.id === proof.baselineTipId);
+    if (proof.baselineTipId !== null && baseline < 0) return false;
+    const tail = entries.slice(baseline + 1);
+    if (tail[0]?.type !== "message" || tail[0].message.role !== "user"
+      || tail.filter((item) => item.type === "message" && item.message.role === "user").length !== 1 + proof.inputIds.length) return false;
+    if (!validRecoveryProjection(buildPiSessionContext(entries), proof.model)) return false;
+    await raw.close(PI_CONTEXT);
+    raw = undefined;
+    // Pending is cleared only after persistence is certain. Bypass the ordinary
+    // sync guard while keeping the public busy reservation throughout the fsync.
+    await syncPath(entry.metadata.path);
+    await syncPath(dirname(entry.metadata.path));
+    entry.recoveryPending = false;
+    delete entry.recovery;
+    return true;
+  } catch {
+    return false;
+  } finally {
+    try { await raw?.close(PI_CONTEXT); } catch { /* already failed closed */ }
+    entry.busy = false;
+  }
+}

package/src/ai/providers/pi-native/stream-subscriber.js

@@ -32,12 +32,30 @@ function toolResultOutcome(result) {
     code: "code",
     backend: "backend",
     signal: "signal",
+    contentKind: "content_kind",
+    charset: "charset",
+    charsetSource: "charset_source",
+    extractionStage: "extraction_stage",
+    renderReason: "render_reason",
+    retryAt: "retry_at",
+    nextAction: "next_action",
   };
   const numbers = {
     attempts: "attempts",
+    queueWaitMs: "queue_wait_ms",
+    backendDurationMs: "backend_duration_ms",
+    retryAfterMs: "retry_after_ms",
+    maxRequestsPerRun: "max_requests_per_run",
+    requestsThisCall: "requests_this_call",
+    requestsUsed: "requests_used",
+    requestsRemaining: "requests_remaining",
+    cooldownSkipCount: "cooldown_skip_count",
+    quotaSkipCount: "quota_skip_count",
     bytes: "bytes",
     exitCode: "exit_code",
     statusCode: "status_code",
+    redirectCount: "redirect_count",
+    parserFailureCount: "parser_failure_count",
   };
   const booleans = {
     retryable: "retryable",
@@ -45,6 +63,11 @@ function toolResultOutcome(result) {
     truncated: "truncated",
     timedOut: "timed_out",
     rendered: "rendered",
+    renderFailed: "render_failed",
+    browserRecommended: "browser_recommended",
+    retryInRun: "retry_in_run",
+    fallbackUsed: "fallback_used",
+    hadDecodingReplacement: "had_decoding_replacement",
   };
   for (const [input, output] of Object.entries(strings)) {
     if (typeof source[input] === "string") bounded[output] = source[input].slice(0, 120);
@@ -110,18 +133,23 @@ export function createStreamSubscriber(runState, { onEvent, options, toolLimits,
     } else if (event.type === "message_end" && event.message?.role === "assistant") {
       const contextUsage = contextUsageFromAssistantMessage(event.message);
       if (contextUsage) {
+        const { costUsd, ...contextTokens } = contextUsage;
         const contextWindow = Number(harness?.getModel?.()?.contextWindow) || 0;
         const measurementId = typeof event.message?.id === "string" && event.message.id.trim().length > 0
           ? event.message.id
           : undefined;
         onEvent({
           type: "context_usage",
+          ...harness?.getPromptCacheRequest?.(),
           sdk,
           model,
           timestamp: Date.now(),
           ...(measurementId === undefined ? {} : { measurementId }),
           ...(contextWindow > 0 ? { contextWindow } : {}),
-          tokens: contextUsage,
+          tokens: contextTokens,
+          costUsd,
+          providerCostUsd: typeof event.message.usage?.cost?.total === "number" && Number.isFinite(event.message.usage.cost.total) && event.message.usage.cost.total >= 0 ? event.message.usage.cost.total : null,
+          costSource: "pi_usage",
         });
       }
       const hasVisibleContent = Array.isArray(event.message?.content)
@@ -216,7 +244,7 @@ export function createStreamSubscriber(runState, { onEvent, options, toolLimits,
       });
     } else if (event.type === "turn_end") {
       runState.turnCount += 1;
-      // NON-DELEGABLE (verified against @earendil-works/pi-agent-core 0.85.0).
+      // NON-DELEGABLE (verified against @earendil-works/pi-agent-core 0.85.1).
       // pi's only after-turn stop hook is `shouldStopAfterTurn` on the LOW-LEVEL
       // `AgentLoopConfig` (dist/types.d.ts) — the config passed to the raw
       // `agentLoop`. It is NOT surfaced on `AgentHarnessOptions`

package/src/ai/providers/pi-native/terminal-recovery.js

@@ -0,0 +1,40 @@
+// @ts-check
+import { transformMessages } from "@earendil-works/pi-ai/api/transform-messages";
+
+/** Validate the effective provider projection; Pi supplies honest missing results. */
+export function validRecoveryProjection(messages, model) {
+  try {
+    const projected = transformMessages(messages, model);
+    const seen = new Set();
+    const pending = new Map();
+    for (const message of projected) {
+      if (!["user", "assistant", "toolResult"].includes(message?.role)) return false;
+      if (message.role !== "toolResult" && pending.size) return false;
+      if (message.role === "user" && typeof message.content === "string") continue;
+      if (!Array.isArray(message.content)) return false;
+      if (message.role === "toolResult") {
+        if (!pending.has(message.toolCallId) || pending.get(message.toolCallId) !== message.toolName) return false;
+        pending.delete(message.toolCallId);
+      }
+      if (message.role === "assistant" && ["error", "aborted", "deferred"].includes(message.stopReason)) return false;
+      for (const block of message.content) {
+        if (block?.type === "text") { if (typeof block.text !== "string") return false; }
+        else if (block?.type === "image") { if (typeof block.data !== "string" || typeof block.mimeType !== "string") return false; }
+        else if (block?.type === "thinking") {
+          if (message.role !== "assistant" || typeof block.thinking !== "string"
+            || (block.thinkingSignature !== undefined && (typeof block.thinkingSignature !== "string" || !block.thinkingSignature))) return false;
+          if (block.thinkingSignature && String(model.api).includes("responses")) {
+            const signature = JSON.parse(block.thinkingSignature);
+            if (signature?.type !== "reasoning" || typeof signature.id !== "string" || !Array.isArray(signature.summary)) return false;
+          }
+        } else if (block?.type === "toolCall") {
+          if (message.role !== "assistant" || typeof block.id !== "string" || !block.id || seen.has(block.id)
+            || typeof block.name !== "string" || !block.name || !block.arguments || typeof block.arguments !== "object") return false;
+          seen.add(block.id);
+          pending.set(block.id, block.name);
+        } else return false;
+      }
+    }
+    return pending.size === 0;
+  } catch { return false; }
+}