@bjoernrochel/sevro 0.1.0-rc.7 → 1.0.0

package/src/hosts/codex-app-server.ts

@@ -1,8 +1,11 @@
-import { isRecord } from "../value-guards";
-import { writeFile } from "node:fs/promises";
-import { join } from "node:path";
+import { redactDiagnosticText } from "../host-diagnostic-redaction";
+import { isRecord, requireRecord } from "../value-guards";
 import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
 import { createInterface } from "node:readline";
+import {
+  HostProcessDiagnostics,
+  type HostDiagnostics,
+} from "../host-diagnostics";
 interface HarnessRunRequest {
   repoDir: string;
   prompt: string;
@@ -14,9 +17,7 @@ interface HarnessRunRequest {
 
 type RecordValue = Record<string, unknown>;
 function record(value: unknown): RecordValue {
-  if (!value || typeof value !== "object" || Array.isArray(value))
-    throw new Error("Malformed app-server object");
-  return value as RecordValue;
+  return requireRecord(value, "Malformed app-server object");
 }
 function identifier(value: unknown): string {
   if (typeof value !== "string" || !value)
@@ -70,15 +71,7 @@ function publicErrorMessage(value: unknown): {
   messageTruncated?: boolean;
 } {
   if (typeof value !== "string") return { message: null };
-  const redacted = value
-    .slice(0, 20000)
-    .replace(/(authorization\s*:\s*)[^\r\n]+/gi, "$1[redacted]")
-    .replace(/\bBearer\s+\S+/gi, "Bearer [redacted]")
-    .replace(/\bsk-[A-Za-z0-9_-]+/g, "[redacted]")
-    .replace(
-      /((?:api[_-]?key|access[_-]?token|refresh[_-]?token)\s*[=:]\s*)\S+/gi,
-      "$1[redacted]",
-    );
+  const redacted = redactDiagnosticText(value.slice(0, 20000), []);
   return {
     message: redacted.slice(0, 2000),
     ...(value.length > 20000 || redacted.length > 2000
@@ -145,7 +138,7 @@ class AppServerRpc {
     {
       resolve: (value: RecordValue) => void;
       reject: (error: Error) => void;
-      timer: ReturnType<typeof setTimeout>;
+      method: string;
     }
   >();
   private wake?: () => void;
@@ -155,8 +148,15 @@ class AppServerRpc {
   private reads: Promise<void>;
   private errors: Promise<void>;
   stderr = "";
+  readonly capture: HostProcessDiagnostics;
 
-  constructor(argv: string[], cwd: string, env: Record<string, string>) {
+  constructor(
+    argv: string[],
+    cwd: string,
+    env: Record<string, string>,
+    diagnostics: HostDiagnostics,
+  ) {
+    this.capture = new HostProcessDiagnostics(diagnostics);
     this.process = spawn(identifier(argv[0]), argv.slice(1), {
       detached: process.platform !== "win32",
       cwd,
@@ -166,10 +166,14 @@ class AppServerRpc {
     this.exited = new Promise((resolve) => {
       this.process.once("exit", resolve);
       this.process.once("error", (error) => {
+        if (this.process.pid === undefined) this.capture.recordUnstarted();
         this.fail(error);
         resolve(null);
       });
     });
+    this.process.stdout.on("data", (chunk: Buffer) => {
+      this.capture.stdout.add(chunk);
+    });
     this.reads = this.readLines().catch((error: unknown) => {
       this.fail(error);
     });
@@ -177,7 +181,8 @@ class AppServerRpc {
       this.fail(error);
     });
     this.errors = new Promise((resolve) => {
-      this.process.stderr.on("data", (chunk) => {
+      this.process.stderr.on("data", (chunk: Buffer) => {
+        this.capture.stderr.add(chunk);
         this.stderr = (this.stderr + String(chunk)).slice(-16000);
       });
       this.process.stderr.on("end", resolve);
@@ -195,10 +200,7 @@ class AppServerRpc {
 
   fail(error: unknown): void {
     this.failure ??= error instanceof Error ? error : new Error(String(error));
-    for (const pending of this.pending.values()) {
-      clearTimeout(pending.timer);
-      pending.reject(this.failure);
-    }
+    for (const pending of this.pending.values()) pending.reject(this.failure);
     this.pending.clear();
     this.wake?.();
   }
@@ -263,7 +265,6 @@ class AppServerRpc {
     // Validate before removing the waiter so malformed responses reject it.
     const value = record(message.error ?? message.result);
     this.pending.delete(id);
-    clearTimeout(pending.timer);
     if (message.error)
       pending.reject(
         new Error(`App-server RPC failed (${String(value.code)})`),
@@ -293,14 +294,12 @@ class AppServerRpc {
     this.wake?.();
   }
 
+  /** The session's configured time bound is the only deadline, including startup. */
   async call(method: string, params: RecordValue): Promise<RecordValue> {
     if (this.failure) throw this.failure;
     const id = this.nextId++;
     return new Promise((resolve, reject) => {
-      const timer = setTimeout(() => {
-        this.fail(new Error(`App-server RPC timeout: ${method}`));
-      }, 45000);
-      this.pending.set(id, { resolve, reject, timer });
+      this.pending.set(id, { resolve, reject, method });
       try {
         this.send({ id, method, params });
       } catch (error) {
@@ -309,6 +308,10 @@ class AppServerRpc {
     });
   }
 
+  pendingMethods(): string[] {
+    return Array.from(this.pending.values(), (pending) => pending.method);
+  }
+
   async initialize(): Promise<void> {
     await this.call("initialize", {
       clientInfo: { name: "sevro_eval", version: "0.1.0" },
@@ -332,11 +335,13 @@ class AppServerRpc {
   async close(): Promise<void> {
     this.closing = true;
     this.process.stdin.end();
+    let shutdownError: NodeJS.ErrnoException | undefined;
     const terminate = (signal: NodeJS.Signals) => {
       try {
         if (this.process.pid) process.kill(-this.process.pid, signal);
       } catch (error) {
-        if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;
+        if ((error as NodeJS.ErrnoException).code !== "ESRCH")
+          shutdownError ??= error as NodeJS.ErrnoException;
       }
     };
     const term = setTimeout(() => {
@@ -353,6 +358,7 @@ class AppServerRpc {
       clearTimeout(term);
       clearTimeout(kill);
     }
+    if (shutdownError !== undefined) throw shutdownError;
   }
 }
 
@@ -471,6 +477,7 @@ function requireGoalObjective(objective: unknown): void {
 }
 
 export interface AppServerRunOptions {
+  diagnostics: HostDiagnostics;
   vettedHooks?: boolean;
   request: HarnessRunRequest;
   argv: string[];
@@ -492,9 +499,9 @@ class AppServerSession {
     goals: [],
   };
   readonly out: string[] = [];
+  finalMessage: string | null = null;
   private threadId = "";
   private goalSeen = false;
-  private usage: unknown;
   private emittedItems = new Set<string>();
   private followUp?: string;
 
@@ -621,14 +628,6 @@ class AppServerSession {
         if (params.willRetry !== true)
           throw new Error("App-server reported a fatal turn error");
         break;
-      case "thread/tokenUsage/updated": {
-        const total = record(record(params.tokenUsage).total);
-        this.usage = {
-          input_tokens: total.inputTokens,
-          output_tokens: total.outputTokens,
-        };
-        break;
-      }
       case "item/completed":
         this.emitItem(params.item);
         break;
@@ -651,13 +650,10 @@ class AppServerSession {
     const turnId = this.observeTurn(turn);
     this.requireCompletedTurn(turn, turnId);
     const status = await this.completedGoal(turnId);
-    this.emit({
-      type: "turn.completed",
-      ...(this.usage ? { usage: this.usage } : {}),
-    });
+    this.emit({ type: "turn.completed" });
     if (this.followUp !== undefined)
       return this.deliverFeedback(turnId, status, this.followUp);
-    return status !== "active" && (await this.saveFinal(turnId));
+    return status !== "active" && (await this.captureFinal(turnId));
   }
 
   private requireCompletedTurn(turn: RecordValue, turnId: string): void {
@@ -686,7 +682,7 @@ class AppServerSession {
   ): Promise<boolean> {
     // User input does not require a terminal goal. Preserve the completed
     // response and actual boundary before delivering the declared follow-up.
-    if (!(await this.saveFinal(turnId, true))) return false;
+    if (!(await this.captureFinal(turnId, true))) return false;
     const boundary = protocolMessage(
       await this.options.followUpBoundary(this.threadId, {
         nativeGoalObserved: this.goalSeen,
@@ -704,7 +700,7 @@ class AppServerSession {
     return false;
   }
 
-  private async saveFinal(
+  private async captureFinal(
     turnId: string,
     forFeedback = false,
   ): Promise<boolean> {
@@ -718,10 +714,7 @@ class AppServerSession {
     if (!latestTurnReady(latest, turnId)) return false;
     const final = appServerFinal(thread, turnId);
     for (const value of turns) this.backfillItems(record(value));
-    await writeFile(
-      join(this.options.request.repoDir, ".git", "last-message.md"),
-      final,
-    );
+    this.finalMessage = final;
     this.evidence.finalTurnId = turnId;
     this.emit({
       type: "item.completed",
@@ -861,10 +854,16 @@ function requireAppServerRun(request: HarnessRunRequest, bound: number): void {
     throw new Error("Invalid app-server time bound");
 }
 function sessionFailure(session: AppServerSession, error: unknown): void {
+  session.options.diagnostics.failure = error;
   session.evidence.failure =
     error instanceof Error ? error.message : String(error);
   session.emit({ type: "turn.failed" });
 }
+/** Name an unanswered request so a stalled startup stays distinguishable. */
+function timeBoundFailure(pending: string[]): string {
+  const reached = "App-server evaluation time bound reached";
+  return pending.length ? `${reached} awaiting ${pending.join(", ")}` : reached;
+}
 async function completeSession(
   session: AppServerSession,
   rpc: AppServerRpc,
@@ -876,19 +875,32 @@ async function completeSession(
   };
   request.signal?.addEventListener("abort", cancel, { once: true });
   const deadline = setTimeout(() => {
-    rpc.fail(new Error("App-server evaluation time bound reached"));
+    rpc.fail(new Error(timeBoundFailure(rpc.pendingMethods())));
   }, bound);
+  let code = 0;
   try {
     await session.run();
     session.assertSettled();
-    return 0;
   } catch (error) {
     sessionFailure(session, error);
-    return 1;
+    code = 1;
   } finally {
     clearTimeout(deadline);
     request.signal?.removeEventListener("abort", cancel);
+  }
+  return closeSession(session, rpc, code);
+}
+async function closeSession(
+  session: AppServerSession,
+  rpc: AppServerRpc,
+  code: number,
+): Promise<number> {
+  try {
     await rpc.close();
+    return code;
+  } catch (error) {
+    if (code === 0) sessionFailure(session, error);
+    return 1;
   }
 }
 function settlementCode(session: AppServerSession, code: number): number {
@@ -906,20 +918,34 @@ export async function runCodexAppServer(options: AppServerRunOptions): Promise<{
   out: string;
   err: string;
   code: number;
+  finalMessage: string | null;
   evidence: AppServerEvidence;
 }> {
   const { request, argv, env } = options;
   const bound = request.control?.appServerTimeoutMs ?? 3600000;
   requireAppServerRun(request, bound);
-  const rpc = new AppServerRpc(argv, request.repoDir, env);
+  const rpc = new AppServerRpc(argv, request.repoDir, env, options.diagnostics);
   const session = new AppServerSession(rpc, options);
   const code = await completeSession(session, rpc, request, bound);
   // Flush stdout before accepting success, including trailing data and terminal errors.
   const finalCode = settlementCode(session, code);
+  if (rpc.process.pid !== undefined)
+    await rpc.capture.record(
+      { exited: rpc.exited, signalCode: rpc.process.signalCode },
+      finalCode === 0 ? "exit" : "error",
+    );
   return {
     out: session.out.join("\n"),
     err: rpc.stderr,
     code: finalCode,
+    finalMessage: completedMessage(session, finalCode),
     evidence: session.evidence,
   };
 }
+
+function completedMessage(
+  session: AppServerSession,
+  code: number,
+): string | null {
+  return code === 0 ? session.finalMessage : null;
+}

package/src/hosts/codex-configuration.ts

@@ -1,7 +1,8 @@
 import type { Stats } from "node:fs";
 import { isRecord } from "../value-guards";
 import { lstat, readFile, realpath, stat } from "node:fs/promises";
-import { dirname, isAbsolute, join, relative, sep } from "node:path";
+import { dirname, isAbsolute, join } from "node:path";
+import { insideRoot } from "../paths";
 import { parse } from "smol-toml";
 
 export async function configurationRoot(root: string): Promise<string> {
@@ -52,7 +53,7 @@ async function readConfiguration(
     const info = await lstat(configPath);
     declared = true;
     const canonical = await realpath(configPath);
-    validateConfigurationFile(info, relative(root, canonical));
+    validateConfigurationFile(info, insideRoot(root, canonical));
     source = await readFile(configPath, "utf8");
   } catch (error) {
     if (!declared && (error as NodeJS.ErrnoException).code === "ENOENT")
@@ -97,15 +98,9 @@ function concurrencyLimit(limit: unknown, configPath: string): number | null {
   return Number(limit);
 }
 
-function validateConfigurationFile(info: Stats, location: string): void {
-  if (!info.isFile() || info.size > 64 * 1024 || escapedRoot(location))
+function validateConfigurationFile(info: Stats, contained: boolean): void {
+  if (!info.isFile() || info.size > 64 * 1024 || !contained)
     throw new Error(
       "configuration must be a contained regular file up to 64 KiB",
     );
 }
-
-function escapedRoot(location: string): boolean {
-  return (
-    location === ".." || location.startsWith(`..${sep}`) || isAbsolute(location)
-  );
-}