@bjoernrochel/sevro 0.1.0-rc.1

package/src/hosts/isolation-roots.ts

@@ -0,0 +1,60 @@
+import { readdir, realpath } from "node:fs/promises";
+import { homedir, tmpdir, userInfo } from "node:os";
+import { isAbsolute, join, resolve } from "node:path";
+import { protectedWorktrees } from "./protected-worktrees";
+
+/** Include source, results, host configuration, and every active peer fixture. */
+export async function evaluationProtectedRoots(options: {
+  workspace: string;
+  projectRoot: string;
+  resultsRoot: string;
+  additionalRoots: string[];
+}): Promise<string[]> {
+  const workspace = await realpath(options.workspace);
+  const optionalConfigRoots = [
+    process.env.CODEX_HOME,
+    process.env.CLAUDE_CONFIG_DIR,
+  ].filter((value): value is string => Boolean(value));
+  const candidates = [
+    resolve(import.meta.dir, "../.."),
+    options.projectRoot,
+    options.resultsRoot,
+    homedir(),
+    userInfo().homedir,
+    ...options.additionalRoots,
+    ...optionalConfigRoots,
+  ];
+  if (candidates.some((root) => !isAbsolute(root)))
+    throw new Error("evaluation protected roots must be absolute");
+  const peers = (await readdir(tmpdir(), { withFileTypes: true }))
+    .filter(
+      (entry) => entry.isDirectory() && entry.name.startsWith("sevro-case-"),
+    )
+    .map((entry) => join(tmpdir(), entry.name));
+  const roots: string[] = [];
+  for (const candidate of [...candidates, ...peers]) {
+    let canonical: string;
+    try {
+      canonical = await realpath(candidate);
+    } catch (error) {
+      if (
+        (peers.includes(candidate) ||
+          optionalConfigRoots.includes(candidate)) &&
+        (error as NodeJS.ErrnoException).code === "ENOENT"
+      )
+        continue;
+      throw new Error("evaluation protected root is unreadable");
+    }
+    if (canonical !== workspace) {
+      roots.push(canonical);
+      // Peers are denied directly; their Git preparation may still be in progress.
+      if (!peers.includes(candidate))
+        roots.push(
+          ...(await protectedWorktrees(canonical)).filter(
+            (root) => root !== workspace,
+          ),
+        );
+    }
+  }
+  return [...new Set(roots)];
+}

package/src/hosts/mac-sandbox.ts

@@ -0,0 +1,108 @@
+import { randomUUID } from "node:crypto";
+import { existsSync } from "node:fs";
+import { mkdir, realpath, rm, writeFile } from "node:fs/promises";
+import { isAbsolute, relative, resolve, sep } from "node:path";
+
+const SANDBOX_EXEC = "/usr/bin/sandbox-exec";
+
+export class HostIsolationError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "HostIsolationError";
+  }
+}
+
+function inside(root: string, path: string): boolean {
+  const child = relative(root, path);
+  return (
+    child === "" ||
+    (child !== ".." && !child.startsWith(`..${sep}`) && !child.startsWith(sep))
+  );
+}
+
+function quoted(path: string): string {
+  if (/[\x00-\x1f\x7f]/.test(path))
+    throw new HostIsolationError("sandbox path contains a control character");
+  return path.replaceAll("\\", "\\\\").replaceAll('"', '\\"');
+}
+
+/** Deny both observation and mutation of every protected canonical root. */
+export function macSandboxProfile(
+  deniedRoots: string[],
+  denyNetwork = false,
+): string {
+  if (!deniedRoots.length || deniedRoots.some((path) => !path.startsWith("/")))
+    throw new HostIsolationError("sandbox roots must be absolute and nonempty");
+  return [
+    "(version 1)",
+    "(allow default)",
+    ...(denyNetwork ? ["(deny network*)"] : []),
+    ...[...new Set(deniedRoots)]
+      .sort()
+      .flatMap((path) => [
+        `(deny file-read* (subpath "${quoted(path)}"))`,
+        `(deny file-write* (subpath "${quoted(path)}"))`,
+      ]),
+    "",
+  ].join("\n");
+}
+
+export interface IsolatedCommand {
+  argv: string[];
+  release(): Promise<void>;
+}
+
+/** Build the outer boundary before launching an agent or candidate-controlled code. */
+export async function prepareMacSandboxCommand(options: {
+  argv: string[];
+  workspace: string;
+  protectedRoots: string[];
+  protectedRootsCanonical?: boolean;
+  privateStateRoot: string;
+  denyNetwork?: boolean;
+}): Promise<IsolatedCommand> {
+  if (process.platform !== "darwin" || !existsSync(SANDBOX_EXEC))
+    throw new HostIsolationError("macOS sandbox-exec isolation is unavailable");
+  if (!options.argv.length || options.argv.some((part) => !part))
+    throw new HostIsolationError("isolated command must be nonempty");
+  if (!options.protectedRoots.length)
+    throw new HostIsolationError("isolated command needs protected roots");
+  if (
+    !isAbsolute(options.workspace) ||
+    !isAbsolute(options.privateStateRoot) ||
+    options.protectedRoots.some((path) => !isAbsolute(path))
+  )
+    throw new HostIsolationError("isolation paths must be absolute");
+  const workspace = await realpath(options.workspace);
+  const roots = options.protectedRootsCanonical
+    ? options.protectedRoots
+    : await Promise.all(options.protectedRoots.map((path) => realpath(path)));
+  await mkdir(options.privateStateRoot, { recursive: true, mode: 0o700 });
+  const stateRoot = await realpath(options.privateStateRoot);
+  if (
+    roots.some((root) => inside(root, workspace)) ||
+    inside(stateRoot, workspace)
+  )
+    throw new HostIsolationError(
+      "protected root includes the candidate workspace",
+    );
+  if (inside(workspace, stateRoot))
+    throw new HostIsolationError(
+      "sandbox state is inside the candidate workspace",
+    );
+  const profilePath = resolve(stateRoot, `host-${randomUUID()}.sb`);
+  await writeFile(
+    profilePath,
+    macSandboxProfile([...roots, stateRoot], options.denyNetwork),
+    {
+      flag: "wx",
+      mode: 0o600,
+    },
+  );
+  return {
+    argv: [SANDBOX_EXEC, "-f", profilePath, ...options.argv],
+    release: async () => {
+      await rm(profilePath, { force: true });
+    },
+  };
+}

package/src/hosts/native-controls.ts

@@ -0,0 +1,124 @@
+import type { codexNativeCallObservation } from "./codex-native-calls";
+import { summarizeClaudeEvents } from "./claude-events";
+
+const MAX_CALLS = 128;
+type Call = { ordinal: number; namespace: string; name: string };
+type Entry = Record<string, unknown>;
+
+export interface NativeControlObservation {
+  id: "sevro.host.native-controls";
+  completeness: "complete" | "partial" | "unavailable";
+  data: {
+    method: "native_control_calls";
+    calls: Call[];
+    acceptedAgentCount: number | null;
+    submittedExecCalls: number | null;
+    truncated: boolean | null;
+  };
+}
+
+function record(value: unknown): value is Entry {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+function label(value: unknown, max: number): value is string {
+  return (
+    typeof value === "string" &&
+    value.length > 0 &&
+    value.length <= max &&
+    /^[A-Za-z0-9_.:-]+$/.test(value)
+  );
+}
+
+export function codexNativeControls(
+  observation: Awaited<ReturnType<typeof codexNativeCallObservation>>,
+): NativeControlObservation {
+  const complete = observation.completeness === "complete";
+  return {
+    id: "sevro.host.native-controls",
+    completeness: observation.completeness,
+    data: {
+      method: "native_control_calls",
+      calls: observation.data.toolCalls.map(({ ordinal, namespace, name }) => ({
+        ordinal,
+        namespace,
+        name,
+      })),
+      acceptedAgentCount: complete
+        ? observation.data.acceptedSpawns.length
+        : null,
+      submittedExecCalls: complete ? observation.data.submittedExecCalls : null,
+      truncated: complete ? false : null,
+    },
+  };
+}
+
+/** Observe all native labels without retaining arguments or interpreting policy. */
+export function claudeNativeControls(
+  stream: string,
+  exitCode: number,
+): NativeControlObservation {
+  const summary = summarizeClaudeEvents(stream, exitCode);
+  const calls: Call[] = [];
+  const ids = new Set<string>();
+  let malformed = false;
+  let truncated = false;
+  lines: for (const line of stream.split("\n")) {
+    if (!line.trim()) continue;
+    let event: unknown;
+    try {
+      event = JSON.parse(line);
+    } catch {
+      malformed = true;
+      continue;
+    }
+    if (!record(event) || event.type !== "assistant") continue;
+    const content = record(event.message) ? event.message.content : null;
+    if (!Array.isArray(content)) {
+      malformed = true;
+      continue;
+    }
+    for (const block of content) {
+      if (
+        !record(block) ||
+        !["text", "thinking", "redacted_thinking", "tool_use"].includes(
+          String(block.type),
+        )
+      ) {
+        malformed = true;
+        continue;
+      }
+      if (block.type !== "tool_use") continue;
+      if (
+        !label(block.id, 128) ||
+        !label(block.name, 256) ||
+        ids.has(block.id)
+      ) {
+        malformed = true;
+        continue;
+      }
+      ids.add(block.id);
+      if (calls.length >= MAX_CALLS) {
+        truncated = true;
+        break lines;
+      }
+      calls.push({
+        ordinal: calls.length,
+        namespace: "claude",
+        name: block.name,
+      });
+    }
+  }
+  return {
+    id: "sevro.host.native-controls",
+    completeness:
+      summary.complete && !malformed && !truncated ? "complete" : "partial",
+    data: {
+      method: "native_control_calls",
+      calls,
+      acceptedAgentCount: null,
+      submittedExecCalls: null,
+      truncated,
+    },
+  };
+}

package/src/hosts/protected-worktrees.ts

@@ -0,0 +1,56 @@
+import { lstat, realpath, stat } from "node:fs/promises";
+import { dirname, isAbsolute, join } from "node:path";
+
+async function containingRepository(root: string): Promise<string | null> {
+  let directory = (await stat(root)).isDirectory() ? root : dirname(root);
+  while (true) {
+    try {
+      await lstat(join(directory, ".git"));
+      return directory;
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT")
+        throw new Error("protected repository metadata is unreadable");
+    }
+    const parent = dirname(directory);
+    if (parent === directory) return null;
+    directory = parent;
+  }
+}
+
+/** Protect the primary checkout and linked worktrees without requiring Git. */
+export async function protectedWorktrees(root: string): Promise<string[]> {
+  const repository = await containingRepository(root);
+  if (!repository) return [];
+  const result = Bun.spawnSync(
+    ["git", "-C", repository, "worktree", "list", "--porcelain", "-z"],
+    {
+      env: {
+        PATH: process.env.PATH ?? "/usr/bin:/bin",
+        LC_ALL: "C",
+        GIT_CONFIG_NOSYSTEM: "1",
+        GIT_CONFIG_GLOBAL: "/dev/null",
+      },
+      stdout: "pipe",
+      stderr: "ignore",
+    },
+  );
+  if (result.exitCode !== 0 || result.stdout.byteLength > 1024 * 1024)
+    throw new Error("protected repository worktrees are unavailable");
+  const paths = result.stdout
+    .toString()
+    .split("\0")
+    .filter((field) => field.startsWith("worktree "))
+    .map((field) => field.slice(9));
+  if (!paths.length || paths.some((path) => !isAbsolute(path)))
+    throw new Error("protected repository worktree paths are invalid");
+  const existing: string[] = [];
+  for (const path of paths) {
+    try {
+      existing.push(await realpath(path));
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT")
+        throw new Error("protected repository worktree is unreadable");
+    }
+  }
+  return existing;
+}

package/src/hosts/workspace-fingerprint.ts

@@ -0,0 +1,55 @@
+import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { lstat, readdir, readlink } from "node:fs/promises";
+import { join, relative, sep } from "node:path";
+
+const MAX_ENTRIES = 10_000;
+const MAX_FILE_BYTES = 128 * 1024 * 1024;
+
+/** Hash visible worktree contents without retaining paths or file data. */
+export async function workspaceFingerprint(
+  root: string,
+): Promise<string | null> {
+  const identity = createHash("sha256");
+  let entries = 0;
+  let fileBytes = 0;
+  async function visit(path: string): Promise<void> {
+    entries++;
+    if (entries > MAX_ENTRIES) throw new Error("workspace entry limit");
+    const name = relative(root, path).split(sep).join("/");
+    const info = await lstat(path);
+    if (info.isDirectory()) {
+      identity.update(`dir\0${name}\0`);
+      for (const child of (await readdir(path)).sort()) {
+        await visit(join(path, child));
+      }
+      return;
+    }
+    if (info.isSymbolicLink()) {
+      identity.update(`link\0${name}\0${await readlink(path)}\0`);
+      return;
+    }
+    if (!info.isFile() || !Number.isSafeInteger(info.size))
+      throw new Error("unsupported workspace entry");
+    if (fileBytes + info.size > MAX_FILE_BYTES)
+      throw new Error("workspace byte limit");
+    const content = createHash("sha256");
+    for await (const chunk of createReadStream(path)) {
+      fileBytes += chunk.byteLength;
+      if (fileBytes > MAX_FILE_BYTES) throw new Error("workspace byte limit");
+      content.update(chunk);
+    }
+    identity.update(
+      `file\0${name}\0${info.mode & 0o111 ? "x" : "-"}\0${content.digest("hex")}\0`,
+    );
+  }
+  try {
+    for (const entry of (await readdir(root)).sort()) {
+      if (entry === ".git") continue;
+      await visit(join(root, entry));
+    }
+    return identity.digest("hex");
+  } catch {
+    return null;
+  }
+}

package/src/identity.ts

@@ -0,0 +1,156 @@
+import { createHash } from "node:crypto";
+
+const ALGORITHM = "sevro.identity.v1";
+const DIGEST = /^[a-f0-9]{64}$/;
+const DIMENSION_KEYS = [
+  "runnerBuildDigest",
+  "projectDigest",
+  "configurationDigest",
+  "extensionDigest",
+  "extensionProtocol",
+  "caseDigest",
+  "fixtureDigest",
+  "checksDigest",
+  "requiredEvidenceDigest",
+  "evaluatorDigest",
+  "graderDigest",
+  "instrumentationDigest",
+  "routeDigest",
+  "condition",
+  "trialCount",
+  "passThreshold",
+] as const;
+
+export interface IdentityDimensions {
+  runnerBuildDigest: string;
+  projectDigest: string;
+  configurationDigest: string;
+  extensionDigest: string | null;
+  extensionProtocol: "sevro.extension.v1" | null;
+  caseDigest: string;
+  fixtureDigest: string;
+  checksDigest: string;
+  requiredEvidenceDigest: string;
+  evaluatorDigest: string;
+  graderDigest: string;
+  instrumentationDigest: string;
+  routeDigest: string;
+  condition: "passive" | "enforced";
+  trialCount: number;
+  passThreshold: number;
+}
+
+function assertUnicode(value: string): void {
+  for (let index = 0; index < value.length; index++) {
+    const code = value.charCodeAt(index);
+    if (code >= 0xd800 && code <= 0xdbff) {
+      const next = value.charCodeAt(++index);
+      if (!(next >= 0xdc00 && next <= 0xdfff))
+        throw new Error("JSON string contains an unpaired surrogate");
+    } else if (code >= 0xdc00 && code <= 0xdfff) {
+      throw new Error("JSON string contains an unpaired surrogate");
+    }
+  }
+}
+
+/** RFC 8785 JSON canonicalization for ordinary JSON values. */
+export function canonicalJson(value: unknown): string {
+  const visiting = new WeakSet<object>();
+  function write(item: unknown): string {
+    if (item === null) return "null";
+    if (typeof item === "string") {
+      assertUnicode(item);
+      return JSON.stringify(item);
+    }
+    if (typeof item === "number") {
+      if (!Number.isFinite(item)) throw new Error("JSON number must be finite");
+      return JSON.stringify(item);
+    }
+    if (typeof item === "boolean") return item ? "true" : "false";
+    if (typeof item !== "object") throw new Error("value is not JSON data");
+    if (visiting.has(item)) throw new Error("JSON value contains a cycle");
+    visiting.add(item);
+    try {
+      if (Array.isArray(item)) return `[${item.map(write).join(",")}]`;
+      const prototype = Object.getPrototypeOf(item);
+      if (prototype !== Object.prototype && prototype !== null)
+        throw new Error("value is not a plain JSON object");
+      if (Object.getOwnPropertySymbols(item).length)
+        throw new Error("JSON object has symbol keys");
+      const record = item as Record<string, unknown>;
+      const entries = Object.keys(record)
+        .sort()
+        .map((key) => {
+          assertUnicode(key);
+          const descriptor = Object.getOwnPropertyDescriptor(record, key);
+          if (!descriptor || !Object.hasOwn(descriptor, "value"))
+            throw new Error("JSON object has an accessor property");
+          return `${JSON.stringify(key)}:${write(descriptor.value)}`;
+        });
+      return `{${entries.join(",")}}`;
+    } finally {
+      visiting.delete(item);
+    }
+  }
+  return write(value);
+}
+
+export function hashJson(value: unknown): string {
+  return createHash("sha256")
+    .update(canonicalJson(value), "utf8")
+    .digest("hex");
+}
+
+function validatedDimensions(value: unknown): IdentityDimensions {
+  if (!value || typeof value !== "object" || Array.isArray(value))
+    throw new Error("identity dimensions must be a JSON object");
+  const record = value as Record<string, unknown>;
+  for (const key of DIMENSION_KEYS)
+    if (!Object.hasOwn(record, key))
+      throw new Error(`missing identity dimension: ${key}`);
+  const unexpected = Object.keys(record).filter(
+    (key) => !DIMENSION_KEYS.includes(key as (typeof DIMENSION_KEYS)[number]),
+  );
+  if (unexpected.length)
+    throw new Error(`unexpected identity dimension: ${unexpected[0]}`);
+  for (const key of DIMENSION_KEYS.filter((name) => name.endsWith("Digest"))) {
+    if (key === "extensionDigest" && record[key] === null) continue;
+    if (typeof record[key] !== "string" || !DIGEST.test(record[key]))
+      throw new Error(`invalid identity dimension: ${key}`);
+  }
+  if (
+    (record.extensionDigest === null) !== (record.extensionProtocol === null) ||
+    (record.extensionProtocol !== null &&
+      record.extensionProtocol !== "sevro.extension.v1")
+  )
+    throw new Error("extension identity requires its matching protocol");
+  if (record.condition !== "passive" && record.condition !== "enforced")
+    throw new Error("invalid identity condition");
+  if (
+    !Number.isSafeInteger(record.trialCount) ||
+    (record.trialCount as number) < 1
+  )
+    throw new Error("invalid identity trial count");
+  if (
+    typeof record.passThreshold !== "number" ||
+    !Number.isFinite(record.passThreshold) ||
+    record.passThreshold <= 0 ||
+    record.passThreshold > 1
+  )
+    throw new Error("invalid identity pass threshold");
+  return record as unknown as IdentityDimensions;
+}
+
+/** Digest the exact comparison dimensions with a versioned domain prefix. */
+export function createEvaluationIdentity(dimensions: unknown): {
+  algorithm: typeof ALGORITHM;
+  digest: string;
+  dimensions: IdentityDimensions;
+} {
+  const checked = validatedDimensions(dimensions);
+  const digest = createHash("sha256")
+    .update(`${ALGORITHM}\n`, "utf8")
+    .update(canonicalJson(checked), "utf8")
+    .digest("hex");
+  return { algorithm: ALGORITHM, digest, dimensions: checked };
+}

package/src/instrumentation.ts

@@ -0,0 +1,106 @@
+import { canonicalJson } from "./identity";
+
+export interface InstrumentationRequest {
+  id: string;
+  configuration: Record<string, unknown>;
+}
+
+export interface InstrumentationCapability {
+  id: string;
+  executionChanging: boolean;
+}
+
+export class InstrumentationEvidenceError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "InstrumentationEvidenceError";
+  }
+}
+
+const ID = /^[a-z][a-z0-9]*(?:[.-][a-z0-9]+)+$/;
+
+function validRequest(value: InstrumentationRequest): boolean {
+  return (
+    Boolean(value && ID.test(value.id)) &&
+    Boolean(
+      value.configuration &&
+      typeof value.configuration === "object" &&
+      !Array.isArray(value.configuration),
+    )
+  );
+}
+
+/** Validate an extension's request before the candidate host starts. */
+export function prepareInstrumentation(
+  requested: InstrumentationRequest[],
+  supported: InstrumentationCapability[],
+  negotiated: string[],
+  condition: "passive" | "enforced",
+): InstrumentationRequest[] {
+  const supportedIds = new Set(supported.map((item) => item.id));
+  if (
+    supported.some(
+      (item) =>
+        !ID.test(item.id) || typeof item.executionChanging !== "boolean",
+    ) ||
+    supportedIds.size !== supported.length
+  )
+    throw new Error("host instrumentation capabilities are invalid");
+  const requestedIds = new Set(requested.map((item) => item.id));
+  if (
+    requested.some((item) => !validRequest(item)) ||
+    requestedIds.size !== requested.length
+  )
+    throw new Error("requested instrumentation is invalid or duplicated");
+  for (const item of requested) {
+    const capability = supported.find((entry) => entry.id === item.id);
+    if (!capability || !negotiated.includes(item.id))
+      throw new Error(
+        "extension preparation requested unsupported instrumentation",
+      );
+    if (condition === "passive" && capability.executionChanging)
+      throw new Error(
+        "passive condition cannot apply execution-changing instrumentation",
+      );
+  }
+  canonicalJson(requested);
+  return requested;
+}
+
+/** A host must report exactly the instrumentation it applied. */
+export function verifyAppliedInstrumentation(
+  requested: InstrumentationRequest[],
+  applied: InstrumentationRequest[] | undefined,
+  requestedCondition: "passive" | "enforced",
+  actualCondition: "passive" | "enforced" | "unknown" | undefined,
+): InstrumentationRequest[] {
+  const actual = applied ?? [];
+  if (
+    !Array.isArray(actual) ||
+    actual.some((item) => !validRequest(item)) ||
+    new Set(actual.map((item) => item.id)).size !== actual.length ||
+    canonicalJson(actual) !== canonicalJson(requested)
+  )
+    throw new InstrumentationEvidenceError(
+      "host applied instrumentation does not match the request",
+    );
+  if (
+    actualCondition !== undefined &&
+    actualCondition !== "unknown" &&
+    actualCondition !== "passive" &&
+    actualCondition !== "enforced"
+  )
+    throw new InstrumentationEvidenceError(
+      "host condition evidence is invalid",
+    );
+  if (
+    (actualCondition !== undefined &&
+      actualCondition !== "unknown" &&
+      actualCondition !== requestedCondition) ||
+    (requested.length > 0 && actualCondition !== requestedCondition)
+  )
+    throw new InstrumentationEvidenceError(
+      "host did not confirm the requested condition",
+    );
+  return actual;
+}