@abot-ai/runtime 1.5.0 → 1.7.0

package/plugins/exec/source/handlers.ts

@@ -1,4 +1,5 @@
 import { runningResult, finalizeExecResult } from "./result-presentation.js";
+import { retainedOutputFailure } from "./retained-output-failure.js";
 import { ExecRequestResources } from "./request-resources.js";
 import {
   boundText,
@@ -15,6 +16,8 @@ import { execFailureFromError, ExecPluginError } from "./errors.js";
 import { captureExecFilesystemSnapshot } from "./filesystem-observer.js";
 import { resolveExecWorkingDirectory } from "./paths.js";
 import { createExecProcessManager } from "./process-manager.js";
+import { createExecOutputStore } from "./output-store.js";
+import { materializeExecOutput } from "./output-snapshot.js";
 import type { ExecSettings } from "./settings.js";
 import type { ExecProcessSnapshot, PendingExecExecution } from "./types.js";
 import { readExecCommand } from "./validation.js";
@@ -37,7 +40,12 @@ export function createExecHandlers(
   settings: ExecSettings,
   preparation?: ToolRequestPreparationContext,
 ): ExecHandlers {
-  const processManager = createExecProcessManager();
+  const outputStore = createExecOutputStore({
+    stateDir: pluginContext.stateDir,
+    retentionMs: settings.outputRetentionMs,
+  });
+  outputStore.startMaintenance();
+  const processManager = createExecProcessManager(outputStore);
   const pendingExecutions = new Map<string, PendingExecExecution>();
   const requestResources = new ExecRequestResources(
     processManager,
@@ -50,16 +58,65 @@ export function createExecHandlers(
     execution: PendingExecExecution;
     scope: string;
     cancellationIsSuccess?: boolean;
+    context?: ToolExecutionContext;
   }): Promise<ToolImplementationOutput> => {
+    pendingExecutions.set(params.snapshot.processId, params.execution);
     try {
-      return await finalizeExecResult(params);
-    } finally {
+      if (params.snapshot.outputFailure) {
+        return retainedOutputFailure(params.snapshot, params.context);
+      }
+      const result = await finalizeExecResult({
+        ...params,
+        resultBudget: params.context?.resultBudget,
+      });
+      if (result.errorCode === "plugin_result_too_large") {
+        return retainedOutputFailure(params.snapshot, params.context);
+      }
+      try {
+        if (!requestResources.isRestored(params.snapshot.processId)) {
+          await processManager.accept(params.snapshot, params.scope);
+        }
+      } catch (error) {
+        return retainedOutputFailure(
+          {
+            ...params.snapshot,
+            outputFailure: {
+              code:
+                error instanceof Error
+                  ? error.message
+                  : "exec_output_acknowledgement_failed",
+              stdoutBytes: params.snapshot.stdout.body?.byteLength ?? 0,
+              stderrBytes: params.snapshot.stderr.body?.byteLength ?? 0,
+            },
+          },
+          params.context,
+        );
+      }
       pendingExecutions.delete(params.snapshot.processId);
       requestResources.forget(params.snapshot.processId);
       await processManager.release(params.snapshot.processId, params.scope);
+      return result;
+    } finally {
+      requestResources.rejectTerminal(params.snapshot);
+      await processManager.reject(params.snapshot, params.scope);
     }
   };
 
+  const deliverRunning = async (
+    snapshot: ExecProcessSnapshot,
+    execution: PendingExecExecution,
+    scope: string,
+    context?: ToolExecutionContext,
+  ): Promise<ToolImplementationOutput> => {
+    if (snapshot.outputFailure) return retainedOutputFailure(snapshot, context);
+    const result = runningResult(snapshot, execution, context?.resultBudget);
+    if (result.errorCode === "plugin_result_too_large") {
+      return retainedOutputFailure(snapshot, context);
+    }
+    await processManager.accept(snapshot, scope);
+    return result;
+  };
+
   const exec: ToolImplementation = async (params, context) => {
     try {
       const command = readExecCommand(params.command);
@@ -82,14 +139,12 @@ export function createExecHandlers(
         filesystemStateBefore,
         hardTimeoutMs: settings.hardTimeoutMs,
         idleTimeoutMs: Math.min(settings.idleTimeoutMs, settings.hardTimeoutMs),
-        outputMaxChars: settings.outputMaxChars,
       });
       const scope = processScope(context);
       const snapshot = await processManager.start({
         scope,
         shellCommand: command,
         cwd: cwd.absolutePath,
-        outputMaxChars: settings.outputMaxChars,
         yieldAfterMs: Math.min(settings.yieldAfterMs, settings.hardTimeoutMs),
         idleTimeoutMs: execution.idleTimeoutMs,
         hardTimeoutMs: execution.hardTimeoutMs,
@@ -103,9 +158,9 @@ export function createExecHandlers(
       });
       if (snapshot.status === "running") {
         pendingExecutions.set(snapshot.processId, execution);
-        return runningResult(snapshot, execution);
+        return deliverRunning(snapshot, execution, scope, context);
       }
-      return finalizeAndRelease({ snapshot, execution, scope });
+      return finalizeAndRelease({ snapshot, execution, scope, context });
     } catch (error) {
       return execFailureFromError(error, "exec");
     }
@@ -130,17 +185,18 @@ export function createExecHandlers(
         );
       }
       const scope = processScope(context);
-      const snapshot =
-        requestResources.terminal(processId, scope, cursor) ??
-        (await processManager.wait({
-          processId,
-          scope,
-          cursor,
-          waitMs: settings.yieldAfterMs,
-        }));
+      const restored = requestResources.terminal(processId, scope, cursor);
+      const snapshot = restored
+        ? await materializeExecOutput(outputStore, scope, restored)
+        : await processManager.wait({
+            processId,
+            scope,
+            cursor,
+            waitMs: settings.yieldAfterMs,
+          });
       return snapshot.status === "running"
-        ? runningResult(snapshot, execution)
-        : finalizeAndRelease({ snapshot, execution, scope });
+        ? deliverRunning(snapshot, execution, scope, context)
+        : finalizeAndRelease({ snapshot, execution, scope, context });
     } catch (error) {
       return execFailureFromError(error, "exec_wait");
     }
@@ -160,14 +216,16 @@ export function createExecHandlers(
         );
       }
       const scope = processScope(context);
-      const snapshot =
-        requestResources.terminal(processId, scope) ??
-        (await processManager.cancel({ processId, scope }));
+      const restored = requestResources.terminal(processId, scope);
+      const snapshot = restored
+        ? await materializeExecOutput(outputStore, scope, restored)
+        : await processManager.cancel({ processId, scope });
       return finalizeAndRelease({
         snapshot,
         execution,
         scope,
         cancellationIsSuccess: true,
+        context,
       });
     } catch (error) {
       return execFailureFromError(error, "exec_cancel");

package/plugins/exec/source/output-contracts.ts

@@ -0,0 +1,45 @@
+import type { Writable } from "node:stream";
+
+export const DEFAULT_EXEC_OUTPUT_RETENTION_MS = 7 * 24 * 60 * 60_000;
+/** Resource bound for one disk operation, independent of model context budgets. */
+export const EXEC_OUTPUT_IO_CHUNK_BYTES = 64 * 1024;
+
+export type ExecOutputStream = "stdout" | "stderr";
+export type ExecOutputBody = Readonly<{
+  id: string;
+  stream: ExecOutputStream;
+  startByte: number;
+  endByte: number;
+  byteLength: number;
+  sha256: string;
+}>;
+export type ExecOutputRange = Readonly<{
+  offset: number;
+  length: number;
+  maxChunkBytes: number;
+  signal?: AbortSignal;
+}>;
+export type ExecOutputWriter = Readonly<{
+  sink: Writable;
+  /** Bytes whose write callbacks completed; pending pipe buffers are excluded. */
+  byteLength: number;
+  /** Committed prefix that does not end inside an incomplete UTF8 code point. */
+  utf8ByteLength: number;
+  completion: Promise<void>;
+  snapshot(
+    range?: Readonly<{ startByte: number; endByte?: number }>,
+  ): Promise<ExecOutputBody>;
+  /** Closes the handle only; retained output remains readable. */
+  dispose(): Promise<void>;
+}>;
+export type ExecOutputSnapshotRecord = Readonly<{
+  kind: "exec_output_snapshot_v1";
+  ownerId: string;
+  streamId: string;
+  body: ExecOutputBody;
+}>;
+export type ExecOutputOwnerRecord = Readonly<{
+  kind: "exec_output_owner_v1";
+  ownerId: string;
+  completedAt: number | null;
+}>;

package/plugins/exec/source/output-files.ts

@@ -0,0 +1,161 @@
+import { createHash, randomUUID } from "node:crypto";
+import { mkdir, open, readFile, rename, rm } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { ExecPluginError } from "./errors.js";
+import type {
+  ExecOutputBody,
+  ExecOutputOwnerRecord,
+  ExecOutputSnapshotRecord,
+} from "./output-contracts.js";
+
+const ID = /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/;
+export function outputStorageRoot(stateDir: string): string {
+  if (!stateDir.trim()) throw outputFailure("state_dir_invalid");
+  return resolve(stateDir, "output");
+}
+export function outputOwnerDirectory(root: string, ownerId: string): string {
+  if (!ownerId.trim()) throw outputFailure("owner_invalid");
+  return join(root, createHash("sha256").update(ownerId).digest("hex"));
+}
+export function outputStreamPath(directory: string, id: string): string {
+  assertOutputId(id);
+  return join(directory, `${id}.bytes`);
+}
+export function outputSnapshotPath(directory: string, id: string): string {
+  assertOutputId(id);
+  return join(directory, `${id}.snapshot.json`);
+}
+export function assertOutputId(id: string): void {
+  if (typeof id !== "string" || !ID.test(id)) throw outputFailure("id_invalid");
+}
+export function outputFailure(code: string, cause?: unknown): ExecPluginError {
+  const error = new ExecPluginError(
+    `exec_output_${code}`,
+    `Exec output storage failed: ${code}.`,
+  );
+  if (cause !== undefined)
+    Object.defineProperty(error, "cause", { value: cause });
+  return error;
+}
+export async function writeOutputMetadata(
+  path: string,
+  value: unknown,
+): Promise<void> {
+  const temporary = `${path}.${randomUUID()}.tmp`;
+  try {
+    const file = await open(temporary, "wx", 0o600);
+    try {
+      await file.writeFile(JSON.stringify(value));
+      await file.sync();
+    } finally {
+      await file.close();
+    }
+    await rename(temporary, path);
+    const directory = await open(dirname(path), "r");
+    try {
+      await directory.sync();
+    } finally {
+      await directory.close();
+    }
+  } catch (error) {
+    await rm(temporary, { force: true }).catch(() => undefined);
+    throw outputFailure("metadata_write_failed", error);
+  }
+}
+export async function prepareOutputOwnerDirectory(
+  directory: string,
+): Promise<void> {
+  try {
+    await mkdir(directory, { recursive: true, mode: 0o700 });
+  } catch (error) {
+    throw outputFailure("write_failed", error);
+  }
+}
+export async function readOutputOwner(
+  directory: string,
+): Promise<ExecOutputOwnerRecord | undefined> {
+  let value: unknown;
+  try {
+    value = JSON.parse(await readFile(join(directory, "owner.json"), "utf8"));
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined;
+    throw outputFailure("owner_read_failed", error);
+  }
+  if (!isOutputOwnerRecord(value)) throw outputFailure("owner_invalid");
+  return value;
+}
+function isOutputOwnerRecord(value: unknown): value is ExecOutputOwnerRecord {
+  if (!value || typeof value !== "object") return false;
+  const owner = value as ExecOutputOwnerRecord;
+  if (owner.kind !== "exec_output_owner_v1") return false;
+  if (typeof owner.ownerId !== "string" || !owner.ownerId.trim()) return false;
+  return (
+    owner.completedAt === null ||
+    (Number.isSafeInteger(owner.completedAt) && owner.completedAt >= 0)
+  );
+}
+export async function lookupOutputSnapshot(
+  directory: string,
+  ownerId: string,
+  id: string,
+): Promise<ExecOutputSnapshotRecord> {
+  let value: unknown;
+  try {
+    value = JSON.parse(
+      await readFile(outputSnapshotPath(directory, id), "utf8"),
+    );
+  } catch (error) {
+    throw outputFailure("snapshot_unavailable", error);
+  }
+  if (!isStoredOutputSnapshot(value, ownerId, id))
+    throw outputFailure("snapshot_binding_invalid");
+  return value;
+}
+function isStoredOutputSnapshot(
+  value: unknown,
+  ownerId: string,
+  id: string,
+): value is ExecOutputSnapshotRecord {
+  if (!value || typeof value !== "object") return false;
+  const snapshot = value as ExecOutputSnapshotRecord;
+  if (
+    snapshot.kind !== "exec_output_snapshot_v1" ||
+    snapshot.ownerId !== ownerId
+  )
+    return false;
+  if (typeof snapshot.streamId !== "string" || !ID.test(snapshot.streamId))
+    return false;
+  const body = snapshot.body;
+  if (!body || typeof body !== "object" || body.id !== id) return false;
+  return isExecOutputBody(body);
+}
+export function isExecOutputBody(value: unknown): value is ExecOutputBody {
+  if (!value || typeof value !== "object") return false;
+  const body = value as ExecOutputBody;
+  if (typeof body.id !== "string" || !ID.test(body.id)) return false;
+  if (body.stream !== "stdout" && body.stream !== "stderr") return false;
+  if (!Number.isSafeInteger(body.startByte) || body.startByte < 0) return false;
+  if (!Number.isSafeInteger(body.endByte) || body.endByte < body.startByte)
+    return false;
+  if (body.byteLength !== body.endByte - body.startByte) return false;
+  return (
+    typeof body.sha256 === "string" && /^sha256:[a-f0-9]{64}$/.test(body.sha256)
+  );
+}
+export async function readOutputSnapshot(
+  directory: string,
+  ownerId: string,
+  body: ExecOutputBody,
+): Promise<ExecOutputSnapshotRecord> {
+  const snapshot = await lookupOutputSnapshot(directory, ownerId, body.id);
+  const stored = snapshot.body;
+  if (
+    stored.stream !== body.stream ||
+    stored.startByte !== body.startByte ||
+    stored.endByte !== body.endByte ||
+    stored.byteLength !== body.byteLength ||
+    stored.sha256 !== body.sha256
+  )
+    throw outputFailure("snapshot_binding_invalid");
+  return snapshot;
+}

package/plugins/exec/source/output-maintenance.ts

@@ -0,0 +1,60 @@
+import { readdir } from "node:fs/promises";
+
+const MAX_CLEANUP_INTERVAL_MS = 60 * 60_000;
+type CleanupJob = {
+  cleanup: () => Promise<number>;
+  intervalMs: number;
+  requested: boolean;
+};
+const jobs = new Map<string, CleanupJob>();
+
+/** One unreferenced job per storage root, retired when the storage is empty. */
+export function scheduleExecOutputCleanup(
+  root: string,
+  retentionMs: number,
+  cleanup: () => Promise<number>,
+): void {
+  const intervalMs = Math.max(
+    1,
+    Math.min(retentionMs, MAX_CLEANUP_INTERVAL_MS),
+  );
+  const previous = jobs.get(root);
+  if (previous) {
+    previous.cleanup = cleanup;
+    previous.intervalMs = intervalMs;
+    previous.requested = true;
+    return;
+  }
+  const job: CleanupJob = { cleanup, intervalMs, requested: false };
+  jobs.set(root, job);
+  const sweep = async (): Promise<void> => {
+    job.requested = false;
+    let empty = false;
+    try {
+      await job.cleanup();
+      empty =
+        (
+          await readdir(root).catch((error: NodeJS.ErrnoException) => {
+            if (error.code === "ENOENT") return [];
+            throw error;
+          })
+        ).length === 0;
+    } catch {
+      process.emitWarning(
+        "EXEC output cleanup failed; a later sweep will retry.",
+        {
+          code: "EXEC_OUTPUT_CLEANUP_FAILED",
+        },
+      );
+    }
+    if (empty && !job.requested) {
+      jobs.delete(root);
+      return;
+    }
+    const timer = setTimeout(() => {
+      void sweep();
+    }, job.intervalMs);
+    timer.unref();
+  };
+  void sweep();
+}

package/plugins/exec/source/output-range-reader.ts

@@ -0,0 +1,97 @@
+import { createHash } from "node:crypto";
+import { open } from "node:fs/promises";
+import { ExecPluginError } from "./errors.js";
+import {
+  EXEC_OUTPUT_IO_CHUNK_BYTES,
+  type ExecOutputBody,
+  type ExecOutputRange,
+} from "./output-contracts.js";
+import { outputFailure } from "./output-files.js";
+
+export function assertOutputRange(
+  startByte: number,
+  endByte: number,
+  availableBytes: number,
+): void {
+  if (!Number.isSafeInteger(startByte) || startByte < 0)
+    throw outputFailure("range_invalid");
+  if (!Number.isSafeInteger(endByte) || endByte < startByte)
+    throw outputFailure("range_invalid");
+  if (endByte > availableBytes) throw outputFailure("range_invalid");
+}
+export async function* readOutputFileRange(
+  path: string,
+  range: ExecOutputRange,
+): AsyncIterable<Uint8Array> {
+  assertOutputRange(
+    range.offset,
+    range.offset + range.length,
+    Number.MAX_SAFE_INTEGER,
+  );
+  if (
+    !Number.isSafeInteger(range.maxChunkBytes) ||
+    range.maxChunkBytes < 1 ||
+    range.maxChunkBytes > EXEC_OUTPUT_IO_CHUNK_BYTES
+  )
+    throw outputFailure("chunk_size_invalid");
+  range.signal?.throwIfAborted();
+  const handle = await open(path, "r").catch((error) => {
+    throw outputFailure("read_failed", error);
+  });
+  try {
+    const end = range.offset + range.length;
+    let position = range.offset;
+    while (position < end) {
+      range.signal?.throwIfAborted();
+      const buffer = Buffer.allocUnsafe(
+        Math.min(range.maxChunkBytes, end - position),
+      );
+      const { bytesRead } = await handle.read(
+        buffer,
+        0,
+        buffer.length,
+        position,
+      );
+      if (bytesRead === 0) throw outputFailure("body_incomplete");
+      position += bytesRead;
+      yield buffer.subarray(0, bytesRead);
+    }
+    range.signal?.throwIfAborted();
+  } catch (error) {
+    if (range.signal?.aborted) throw error;
+    if (error instanceof ExecPluginError) throw error;
+    throw outputFailure("read_failed", error);
+  } finally {
+    await handle.close();
+  }
+}
+export async function hashOutputRange(
+  path: string,
+  startByte: number,
+  endByte: number,
+  signal?: AbortSignal,
+): Promise<string> {
+  const hash = createHash("sha256");
+  for await (const chunk of readOutputFileRange(path, {
+    offset: startByte,
+    length: endByte - startByte,
+    maxChunkBytes: EXEC_OUTPUT_IO_CHUNK_BYTES,
+    ...(signal ? { signal } : {}),
+  }))
+    hash.update(chunk);
+  return `sha256:${hash.digest("hex")}`;
+}
+export async function assertOutputBodyHash(
+  path: string,
+  body: ExecOutputBody,
+  signal?: AbortSignal,
+): Promise<void> {
+  assertOutputRange(body.startByte, body.endByte, Number.MAX_SAFE_INTEGER);
+  if (body.byteLength !== body.endByte - body.startByte)
+    throw outputFailure("snapshot_binding_invalid");
+  if (
+    (await hashOutputRange(path, body.startByte, body.endByte, signal)) !==
+    body.sha256
+  )
+    throw outputFailure("body_hash_mismatch");
+}

package/plugins/exec/source/output-retention.ts

@@ -0,0 +1,34 @@
+import { readdir, stat } from "node:fs/promises";
+import { join } from "node:path";
+import type { ExecOutputOwnerRecord } from "./output-contracts.js";
+
+/** Old or interrupted captures may lack a closed timestamp. Reads do not renew TTL. */
+export async function outputCaptureCompletedAt(
+  directory: string,
+  owner?: ExecOutputOwnerRecord,
+): Promise<number> {
+  if (owner?.completedAt !== null && owner?.completedAt !== undefined)
+    return owner.completedAt;
+  const entries = await readdir(directory, { withFileTypes: true });
+  const captures = entries.filter(
+    (entry) => entry.isFile() && entry.name.endsWith(".bytes"),
+  );
+  if (!captures.length) return (await stat(directory)).mtimeMs;
+  const times = await Promise.all(
+    captures.map(
+      async (entry) => (await stat(join(directory, entry.name))).mtimeMs,
+    ),
+  );
+  return Math.max(...times);
+}
+
+export async function hasOutputExpired(
+  directory: string,
+  owner: ExecOutputOwnerRecord | undefined,
+  now: number,
+  retentionMs: number,
+): Promise<boolean> {
+  return (
+    now - (await outputCaptureCompletedAt(directory, owner)) >= retentionMs
+  );
+}

package/plugins/exec/source/output-snapshot.ts

@@ -0,0 +1,54 @@
+import { ExecPluginError } from "./errors.js";
+import { StringDecoder } from "node:string_decoder";
+import { EXEC_OUTPUT_IO_CHUNK_BYTES } from "./output-contracts.js";
+import { createExecOutputStore } from "./output-store.js";
+import { ExecOutputBuffer } from "./process-output-buffer.js";
+import type { ExecProcessSnapshot, ExecStreamSnapshot } from "./types.js";
+
+export type ExecOutputStore = ReturnType<typeof createExecOutputStore>;
+
+export function execOutputOwner(scope: string, processId: string): string {
+  return JSON.stringify([scope, processId]);
+}
+
+/** Materialize only the observation being delivered, after disk capture. */
+export async function materializeExecOutput(
+  store: ExecOutputStore,
+  scope: string,
+  snapshot: ExecProcessSnapshot,
+): Promise<ExecProcessSnapshot> {
+  if (snapshot.outputFailure) return snapshot;
+  const ownerId = execOutputOwner(scope, snapshot.processId);
+  async function read(stream: ExecStreamSnapshot): Promise<ExecStreamSnapshot> {
+    if (!stream.body) return stream; // Previously persisted inline receipts.
+    const decoder = new StringDecoder("utf8");
+    const buffer = new ExecOutputBuffer();
+    for await (const chunk of store.readRange({
+      ownerId,
+      body: stream.body,
+      offset: 0,
+      length: stream.body.byteLength,
+      maxChunkBytes: EXEC_OUTPUT_IO_CHUNK_BYTES,
+    }))
+      buffer.append(decoder.write(Buffer.from(chunk)));
+    buffer.append(decoder.end());
+    return { ...buffer.peek(), sawOutput: stream.sawOutput, body: stream.body };
+  }
+  try {
+    const stdout = await read(snapshot.stdout);
+    const stderr = await read(snapshot.stderr);
+    return { ...snapshot, stdout, stderr };
+  } catch (error) {
+    return {
+      ...snapshot,
+      outputFailure: {
+        code:
+          error instanceof ExecPluginError
+            ? error.code
+            : "exec_output_read_failed",
+        stdoutBytes: snapshot.stdout.body?.byteLength ?? 0,
+        stderrBytes: snapshot.stderr.body?.byteLength ?? 0,
+      },
+    };
+  }
+}