@bjoernrochel/sevro 0.1.0-rc.1

package/src/graders/shell.ts

@@ -0,0 +1,277 @@
+import { mkdir, stat } from "node:fs/promises";
+import { delimiter, join } from "node:path";
+import { prepareMacSandboxCommand } from "../hosts/mac-sandbox";
+
+const DEFAULT_TIMEOUT_MS = 30_000;
+const MAX_TIMEOUT_MS = 120_000;
+const MAX_OUTPUT_BYTES = 1024 * 1024;
+
+export interface ShellCheckDeclaration {
+  id: string;
+  grader: "sevro.shell";
+  configuration: Record<string, unknown>;
+}
+
+export interface PreparedShellCheck {
+  id: string;
+  run: string;
+  expectedExitCode: number;
+  timeoutMs: number;
+  expectExact?: string;
+  expectRegex?: RegExp;
+  notRegex?: RegExp;
+  captureStdout: boolean;
+}
+
+export function prepareShellChecks(
+  declarations: ShellCheckDeclaration[],
+): PreparedShellCheck[] {
+  return declarations.map(({ id, configuration }) => {
+    if (!id || !configuration || typeof configuration !== "object")
+      throw new Error("invalid shell check declaration");
+    if (
+      Object.keys(configuration).some(
+        (key) =>
+          ![
+            "run",
+            "expectedExitCode",
+            "timeoutMs",
+            "expectExact",
+            "expectRegex",
+            "notRegex",
+            "flags",
+          ].includes(key),
+      )
+    )
+      throw new Error("unsupported shell check configuration");
+    const {
+      run,
+      expectedExitCode = 0,
+      timeoutMs = DEFAULT_TIMEOUT_MS,
+      expectExact,
+      expectRegex,
+      notRegex,
+      flags = "",
+    } = configuration;
+    if (typeof run !== "string" || !run.trim() || run.length > 4096)
+      throw new Error("shell check requires a bounded command");
+    if (
+      !Number.isSafeInteger(expectedExitCode) ||
+      (expectedExitCode as number) < 0 ||
+      (expectedExitCode as number) > 255
+    )
+      throw new Error("invalid expected shell exit code");
+    if (
+      !Number.isSafeInteger(timeoutMs) ||
+      (timeoutMs as number) < 1 ||
+      (timeoutMs as number) > MAX_TIMEOUT_MS
+    )
+      throw new Error("invalid shell check timeout");
+    if (
+      expectExact !== undefined &&
+      (typeof expectExact !== "string" ||
+        Buffer.byteLength(expectExact, "utf8") > MAX_OUTPUT_BYTES)
+    )
+      throw new Error("invalid exact shell output expectation");
+    if (
+      flags !== "" &&
+      (typeof flags !== "string" ||
+        !/^[isu]*$/.test(flags) ||
+        new Set(flags).size !== flags.length)
+    )
+      throw new Error("invalid shell regex flags");
+    if (flags && expectRegex === undefined && notRegex === undefined)
+      throw new Error("shell regex flags require a pattern");
+    const pattern = (value: unknown): RegExp | undefined => {
+      if (value === undefined) return undefined;
+      if (typeof value !== "string" || value.length > 4096)
+        throw new Error("invalid shell regex pattern");
+      try {
+        return new RegExp(value, `m${flags}`);
+      } catch {
+        throw new Error("invalid shell regex pattern or flags");
+      }
+    };
+    const expectedPattern = pattern(expectRegex);
+    const forbiddenPattern = pattern(notRegex);
+    return {
+      id,
+      run,
+      expectedExitCode: expectedExitCode as number,
+      timeoutMs: timeoutMs as number,
+      expectExact: expectExact as string | undefined,
+      expectRegex: expectedPattern,
+      notRegex: forbiddenPattern,
+      captureStdout:
+        expectExact !== undefined ||
+        expectedPattern !== undefined ||
+        forbiddenPattern !== undefined,
+    };
+  });
+}
+
+export interface ShellCheckResult {
+  exitCode: number;
+  stdout: string | null;
+}
+
+/** Grade bounded shell output without exposing it in retained evidence. */
+export function assessShellCheck(
+  check: PreparedShellCheck,
+  result: ShellCheckResult,
+): { passed: boolean; detail: string } {
+  if (result.exitCode !== check.expectedExitCode)
+    return {
+      passed: false,
+      detail: `exit code ${result.exitCode} (expected ${check.expectedExitCode})`,
+    };
+  const out = result.stdout;
+  if (check.captureStdout && out === null)
+    throw new Error("shell stdout observation is missing");
+  if (
+    check.expectExact !== undefined &&
+    (out!.endsWith("\n") ? out!.slice(0, -1) : out) !== check.expectExact
+  )
+    return { passed: false, detail: "exact shell output did not match" };
+  if (check.expectRegex && !check.expectRegex.test(out!))
+    return {
+      passed: false,
+      detail: "expected shell output pattern did not match",
+    };
+  if (check.notRegex && check.notRegex.test(out!))
+    return { passed: false, detail: "forbidden shell output pattern matched" };
+  return { passed: true, detail: `exit code ${result.exitCode}` };
+}
+
+async function boundedOutput(
+  stream: ReadableStream<Uint8Array>,
+): Promise<string> {
+  const chunks: Uint8Array[] = [];
+  let size = 0;
+  for await (const chunk of stream) {
+    size += chunk.byteLength;
+    if (size > MAX_OUTPUT_BYTES) throw new Error("shell stdout exceeds 1 MiB");
+    chunks.push(chunk);
+  }
+  return new TextDecoder("utf-8", { fatal: true }).decode(
+    Buffer.concat(chunks),
+  );
+}
+
+function stopProcess(proc: Bun.Subprocess): void {
+  if (process.platform !== "win32") {
+    try {
+      process.kill(-proc.pid, "SIGKILL");
+      return;
+    } catch {
+      // The group may already be gone.
+    }
+  }
+  try {
+    proc.kill("SIGKILL");
+  } catch {
+    // The process already exited.
+  }
+}
+
+/** Run a check without inherited credentials or retaining raw process output. */
+export async function runShellCheck(
+  check: PreparedShellCheck,
+  options: {
+    workspace: string;
+    fixtureBinDir?: string;
+    toolchainBinDir?: string;
+    uvRuntimeCache?: boolean;
+    protectedRoots: string[];
+    protectedRootsCanonical?: boolean;
+    privateStateRoot: string;
+    signal?: AbortSignal;
+  },
+): Promise<ShellCheckResult> {
+  if (options.signal?.aborted) throw new Error("shell check cancelled");
+  const runtimeRoot = join(options.workspace, ".git", "sevro-runtime");
+  const home = join(runtimeRoot, "check-home");
+  const temp = join(runtimeRoot, "check-tmp");
+  await Promise.all([
+    mkdir(home, { recursive: true, mode: 0o700 }),
+    mkdir(temp, { recursive: true, mode: 0o700 }),
+  ]);
+  const isolated = await prepareMacSandboxCommand({
+    argv: ["/bin/sh", "-e", "-c", check.run],
+    workspace: options.workspace,
+    protectedRoots: options.protectedRoots,
+    protectedRootsCanonical: options.protectedRootsCanonical,
+    privateStateRoot: options.privateStateRoot,
+    denyNetwork: true,
+  });
+  const uvCache = join(runtimeRoot, "uv-cache");
+  if (options.uvRuntimeCache && !(await stat(uvCache)).isDirectory())
+    throw new Error("isolated UV cache is missing");
+  let proc: Bun.Subprocess | undefined;
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  let cancel: (() => void) | undefined;
+  try {
+    proc = Bun.spawn(isolated.argv, {
+      cwd: options.workspace,
+      env: {
+        PATH: [
+          options.fixtureBinDir,
+          options.toolchainBinDir,
+          "/usr/bin:/bin:/usr/sbin:/sbin",
+        ]
+          .filter(Boolean)
+          .join(delimiter),
+        HOME: home,
+        TMPDIR: temp,
+        LANG: process.env.LANG ?? "C",
+        GIT_CONFIG_GLOBAL: "/dev/null",
+        GIT_CONFIG_NOSYSTEM: "1",
+        ...(options.uvRuntimeCache
+          ? {
+              UV_CACHE_DIR: uvCache,
+              UV_PROJECT_ENVIRONMENT: join(runtimeRoot, "project-environment"),
+              UV_OFFLINE: "1",
+              PYTHONDONTWRITEBYTECODE: "1",
+            }
+          : {}),
+      },
+      detached: true,
+      stdin: "ignore",
+      stdout: check.captureStdout ? "pipe" : "ignore",
+      stderr: "ignore",
+    });
+    const running = proc;
+    const outputStream = running.stdout;
+    if (check.captureStdout && !(outputStream instanceof ReadableStream))
+      throw new Error("shell stdout pipe is unavailable");
+    const timeout = new Promise<never>((_resolve, reject) => {
+      timer = setTimeout(
+        () => reject(new Error("shell check timed out")),
+        check.timeoutMs,
+      );
+    });
+    const aborted = new Promise<never>((_resolve, reject) => {
+      if (!options.signal) return;
+      cancel = () => reject(new Error("shell check cancelled"));
+      if (options.signal.aborted) cancel();
+      else options.signal.addEventListener("abort", cancel, { once: true });
+    });
+    const completed = Promise.all([
+      running.exited,
+      check.captureStdout
+        ? boundedOutput(outputStream as ReadableStream<Uint8Array>)
+        : Promise.resolve(null),
+    ]).then(([exitCode, stdout]) => ({ exitCode, stdout }));
+    return await Promise.race([completed, timeout, aborted]);
+  } catch (error) {
+    if (proc) {
+      stopProcess(proc);
+      await proc.exited;
+    }
+    throw error;
+  } finally {
+    if (timer) clearTimeout(timer);
+    if (cancel) options.signal?.removeEventListener("abort", cancel);
+    await isolated.release();
+  }
+}

package/src/hosts/claude-continuation.ts

@@ -0,0 +1,94 @@
+import { randomUUID } from "node:crypto";
+import { summarizeClaudeEvents } from "./claude-events";
+import { workspaceFingerprint } from "./workspace-fingerprint";
+
+export type ClaudeProcessResult = { code: number; out: string; err: string };
+export type ClaudeSession = { option: "--session-id" | "--resume"; id: string };
+type Observation = {
+  id: string;
+  completeness: "complete" | "partial";
+  data: Record<string, unknown>;
+};
+type ClaudeTurns = ClaudeProcessResult & {
+  initialOut?: string;
+  followUpOut?: string;
+  continuation?: Observation;
+  sessionResultsBound?: boolean;
+};
+
+function boundResult(stream: string, session: string): boolean {
+  try {
+    const results = stream
+      .split("\n")
+      .filter((line) => line.trim())
+      .map((line) => JSON.parse(line))
+      .filter((entry) => entry?.type === "result");
+    return results.length === 1 && results[0].session_id === session;
+  } catch {
+    return false;
+  }
+}
+
+/** Resume a declared follow-up in the same isolated native Claude session. */
+export async function runClaudeTurns(options: {
+  prompt: string;
+  followUpPrompt?: string;
+  workspace: string;
+  run: (
+    prompt: string,
+    session?: ClaudeSession,
+  ) => Promise<ClaudeProcessResult>;
+}): Promise<ClaudeTurns> {
+  if (options.followUpPrompt === undefined) return options.run(options.prompt);
+  const sessionId = randomUUID();
+  const initialFingerprint = await workspaceFingerprint(options.workspace);
+  const initial = await options.run(options.prompt, {
+    option: "--session-id",
+    id: sessionId,
+  });
+  const summary = summarizeClaudeEvents(initial.out, initial.code);
+  const initialBound = boundResult(initial.out, sessionId);
+  if (!summary.complete || !initialBound)
+    return {
+      ...initial,
+      code: summary.complete ? 1 : initial.code,
+      sessionResultsBound: initialBound,
+    };
+  const beforeFollowUp = await workspaceFingerprint(options.workspace);
+  const followUp = await options.run(options.followUpPrompt, {
+    option: "--resume",
+    id: sessionId,
+  });
+  const followUpSummary = summarizeClaudeEvents(followUp.out, followUp.code);
+  const sessionResultsBound = boundResult(followUp.out, sessionId);
+  const bound = followUpSummary.complete && sessionResultsBound;
+  const measured = initialFingerprint !== null && beforeFollowUp !== null;
+  const out = `${initial.out.trimEnd()}\n${followUp.out.trimStart()}`;
+  const err = [initial.err.trimEnd(), followUp.err.trimStart()]
+    .filter(Boolean)
+    .join("\n");
+  if (
+    Buffer.byteLength(out, "utf8") > 8 * 1024 * 1024 ||
+    Buffer.byteLength(err, "utf8") > 64 * 1024
+  )
+    throw new Error("Claude combined process output exceeds its limit");
+  return {
+    code: bound ? 0 : followUp.code || 1,
+    out,
+    err,
+    initialOut: initial.out,
+    followUpOut: followUp.out,
+    sessionResultsBound,
+    continuation: {
+      id: "sevro.claude.continuation",
+      completeness: bound && measured ? "complete" : "partial",
+      data: {
+        method: "same_session_resume",
+        sessionId,
+        preFollowUpWorktreeUnchanged: measured
+          ? initialFingerprint === beforeFollowUp
+          : null,
+      },
+    },
+  };
+}

package/src/hosts/claude-credential.ts

@@ -0,0 +1,73 @@
+import { existsSync } from "node:fs";
+import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+const MAX_CREDENTIAL_BYTES = 1024 * 1024;
+
+async function keychainCredential(): Promise<Buffer> {
+  if (process.platform !== "darwin" || !existsSync("/usr/bin/security"))
+    throw new Error("Claude keychain credential is unavailable");
+  const account = process.env.USER ?? process.env.LOGNAME;
+  const proc = Bun.spawn(
+    [
+      "/usr/bin/security",
+      "find-generic-password",
+      "-s",
+      "Claude Code-credentials",
+      ...(account ? ["-a", account] : []),
+      "-w",
+    ],
+    { stdout: "pipe", stderr: "ignore" },
+  );
+  const timer = setTimeout(() => proc.kill("SIGKILL"), 10_000);
+  const reader = proc.stdout.getReader();
+  const chunks: Uint8Array[] = [];
+  let size = 0;
+  try {
+    while (true) {
+      const { value, done } = await reader.read();
+      if (done) break;
+      size += value.byteLength;
+      if (size > MAX_CREDENTIAL_BYTES)
+        throw new Error("Claude credential exceeds the size limit");
+      chunks.push(value);
+    }
+    if ((await proc.exited) !== 0)
+      throw new Error("Claude keychain credential is unavailable");
+    return Buffer.concat(chunks);
+  } catch {
+    proc.kill("SIGKILL");
+    await proc.exited;
+    throw new Error("Claude keychain credential is unavailable");
+  } finally {
+    clearTimeout(timer);
+    reader.releaseLock();
+  }
+}
+
+/** Copy only a bounded login credential into one private host state. */
+export async function stageClaudeCredential(
+  configRoot: string,
+  sourceFile?: string,
+): Promise<string> {
+  await mkdir(configRoot, { recursive: true, mode: 0o700 });
+  let credential: Buffer;
+  try {
+    if (sourceFile) {
+      if ((await stat(sourceFile)).size > MAX_CREDENTIAL_BYTES)
+        throw new Error("oversized credential");
+      credential = await readFile(sourceFile);
+    } else credential = await keychainCredential();
+  } catch {
+    throw new Error("Claude credential is unreadable or unavailable");
+  }
+  if (
+    credential.byteLength === 0 ||
+    credential.byteLength > MAX_CREDENTIAL_BYTES ||
+    !credential.toString("utf8").trim()
+  )
+    throw new Error("Claude credential is empty or oversized");
+  const target = join(configRoot, ".credentials.json");
+  await writeFile(target, credential, { flag: "wx", mode: 0o600 });
+  return target;
+}

package/src/hosts/claude-events.ts

@@ -0,0 +1,128 @@
+const MAX_EVENT_STREAM_BYTES = 8 * 1024 * 1024;
+
+export class ClaudeEventError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ClaudeEventError";
+  }
+}
+
+type Entry = Record<string, unknown>;
+
+function record(value: unknown): value is Entry {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+function nonnegative(value: unknown): value is number {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0;
+}
+
+function validAssistant(value: Entry): boolean {
+  if (!record(value.message) || !Array.isArray(value.message.content))
+    return false;
+  return value.message.content.every((block: unknown) => {
+    if (!record(block)) return false;
+    if (block.type !== "tool_use" || block.name !== "Skill") return true;
+    return (
+      record(block.input) &&
+      typeof block.input.skill === "string" &&
+      !!block.input.skill.split(":").at(-1)?.trim()
+    );
+  });
+}
+
+function validResult(value: Entry): boolean {
+  return (
+    typeof value.subtype === "string" &&
+    !!value.subtype.trim() &&
+    typeof value.is_error === "boolean" &&
+    typeof value.result === "string"
+  );
+}
+
+function validEvent(value: unknown): value is Entry {
+  if (!record(value)) return false;
+  if (value.type === "result") return validResult(value);
+  if (value.type === "assistant") return validAssistant(value);
+  return typeof value.type === "string";
+}
+
+interface Usage {
+  inputTokens: number;
+  outputTokens: number;
+}
+
+function usage(value: unknown): Usage | null {
+  if (!record(value)) return null;
+  const input = value.input_tokens;
+  const output = value.output_tokens;
+  const cacheCreation = value.cache_creation_input_tokens ?? 0;
+  const cacheRead = value.cache_read_input_tokens ?? 0;
+  if (
+    !nonnegative(input) ||
+    !nonnegative(output) ||
+    !nonnegative(cacheCreation) ||
+    !nonnegative(cacheRead)
+  )
+    return null;
+  return {
+    inputTokens: input + cacheCreation + cacheRead,
+    outputTokens: output,
+  };
+}
+
+export interface ClaudeEventSummary {
+  complete: boolean;
+  finalMessage: string | null;
+  inputTokens: number | null;
+  outputTokens: number | null;
+  costUsd: number | null;
+  usageComplete: boolean;
+}
+
+/** Parse bounded Claude stream JSON without retaining its private event bodies. */
+export function summarizeClaudeEvents(
+  stream: string,
+  exitCode: number,
+): ClaudeEventSummary {
+  if (Buffer.byteLength(stream, "utf8") > MAX_EVENT_STREAM_BYTES)
+    throw new ClaudeEventError("Claude event stream exceeds 8 MiB");
+  const results: Entry[] = [];
+  let malformed = false;
+  for (const line of stream.split("\n")) {
+    if (!line.trim()) continue;
+    try {
+      const event: unknown = JSON.parse(line);
+      if (!validEvent(event)) malformed = true;
+      else if (event.type === "result") results.push(event);
+    } catch {
+      malformed = true;
+    }
+  }
+  const final = results.at(-1);
+  const usages = results.map((result) => usage(result.usage));
+  const usageComplete =
+    !malformed && results.length > 0 && usages.every(Boolean);
+  const costs = results.map((result) => result.total_cost_usd);
+  return {
+    complete:
+      !malformed &&
+      exitCode === 0 &&
+      results.length > 0 &&
+      results.every(
+        (result) => result.subtype === "success" && result.is_error === false,
+      ),
+    finalMessage: typeof final?.result === "string" ? final.result : null,
+    inputTokens: usageComplete
+      ? usages.reduce((total, value) => total + value!.inputTokens, 0)
+      : null,
+    outputTokens: usageComplete
+      ? usages.reduce((total, value) => total + value!.outputTokens, 0)
+      : null,
+    costUsd:
+      !malformed && results.length && costs.every(nonnegative)
+        ? costs.reduce((total, value) => total + (value as number), 0)
+        : null,
+    usageComplete,
+  };
+}