@bjoernrochel/sevro 0.1.0-rc.8 → 1.1.0

package/src/extension-client.ts

@@ -1,6 +1,12 @@
+import { spawn, type Subprocess } from "./runtime-process";
 import { randomUUID } from "node:crypto";
 import validExchange from "./generated/extension.cjs";
 import { isRecord, isStringArray } from "./value-guards";
+import {
+  captureBoundedBytes,
+  ProcessDeadline,
+  terminateOwnedProcessGroup,
+} from "./process-lifecycle";
 
 const MAX_MESSAGE_BYTES = 8 * 1024 * 1024;
 const MAX_DIAGNOSTIC_BYTES = 64 * 1024;
@@ -35,61 +41,6 @@ export interface ExchangeOptions {
   signal?: AbortSignal;
 }
 
-function boundedChunk(
-  value: Uint8Array,
-  size: number,
-  limit: number,
-  truncate: boolean,
-): Uint8Array {
-  if (!truncate && size + value.byteLength > limit)
-    throw new ExtensionProtocolError("extension message exceeds 8 MiB");
-  return value.subarray(0, Math.max(0, limit - size));
-}
-
-async function readBounded(
-  stream: ReadableStream<Uint8Array>,
-  limit: number,
-  truncate: boolean,
-): Promise<Uint8Array> {
-  const reader = stream.getReader();
-  const chunks: Uint8Array[] = [];
-  let size = 0;
-  for (;;) {
-    const { value, done } = await reader.read();
-    if (done) break;
-    const kept = boundedChunk(value, size, limit, truncate);
-    if (kept.byteLength) chunks.push(kept);
-    size += kept.byteLength;
-  }
-  return concatenateChunks(chunks, size);
-}
-
-function concatenateChunks(chunks: Uint8Array[], size: number): Uint8Array {
-  const result = new Uint8Array(size);
-  let offset = 0;
-  for (const chunk of chunks) {
-    result.set(chunk, offset);
-    offset += chunk.byteLength;
-  }
-  return result;
-}
-
-function terminate(proc: Bun.Subprocess): void {
-  if (process.platform !== "win32") {
-    try {
-      process.kill(-proc.pid, "SIGKILL");
-      return;
-    } catch {
-      // The group may have exited while the runner was handling the error.
-    }
-  }
-  try {
-    proc.kill("SIGKILL");
-  } catch {
-    // The process already exited.
-  }
-}
-
 interface SuccessfulExtensionResponse extends ExtensionResponse {
   result: Record<string, unknown>;
   error?: never;
@@ -121,9 +72,9 @@ function extensionPayload(request: ExtensionRequest): Uint8Array {
 function spawnExtension(
   command: string[],
   options: ExchangeOptions,
-): Bun.Subprocess {
+): Subprocess {
   try {
-    return Bun.spawn(command, {
+    return spawn(command, {
       cwd: options.cwd,
       stdin: "pipe",
       stdout: "pipe",
@@ -137,22 +88,15 @@ function spawnExtension(
   }
 }
 
-function extensionWriter(proc: Bun.Subprocess) {
+function extensionWriter(proc: Subprocess) {
   const stdin = proc.stdin;
   if (!stdin || typeof stdin === "number")
     throw new ExtensionProtocolError("extension pipes are unavailable");
   return stdin;
 }
 
-function extensionReaders(proc: Bun.Subprocess) {
+function extensionReaders(proc: Subprocess) {
   const { stdout, stderr } = proc;
-  if (
-    !stdout ||
-    typeof stdout === "number" ||
-    !stderr ||
-    typeof stderr === "number"
-  )
-    throw new ExtensionProtocolError("extension pipes are unavailable");
   return { stdout, stderr };
 }
 
@@ -217,13 +161,23 @@ function assertSuccessfulResponse(
 }
 
 class ExtensionExchange {
-  private timer: ReturnType<typeof setTimeout> | undefined;
-  private cancel: (() => void) | undefined;
+  private readonly deadline: ProcessDeadline;
   constructor(
-    private readonly proc: Bun.Subprocess,
-    private readonly options: ExchangeOptions,
-    private readonly timeoutMs: number,
-  ) {}
+    private readonly proc: Subprocess,
+    options: ExchangeOptions,
+    timeoutMs: number,
+  ) {
+    this.deadline = new ProcessDeadline({
+      timeoutMs,
+      signal: options.signal,
+      interruption: (reason) =>
+        new ExtensionProtocolError(
+          reason === "timeout"
+            ? "extension timed out"
+            : "extension request cancelled",
+        ),
+    });
+  }
 
   private async output(
     request: ExtensionRequest,
@@ -232,9 +186,17 @@ class ExtensionExchange {
     const stdin = extensionWriter(this.proc);
     const { stdout, stderr } = extensionReaders(this.proc);
     const write = writeExtensionRequest(stdin, payload);
-    const [bytes, , exitCode] = await Promise.all([
-      readBounded(stdout, MAX_MESSAGE_BYTES, false),
-      readBounded(stderr, MAX_DIAGNOSTIC_BYTES, true),
+    const [{ bytes }, , exitCode] = await Promise.all([
+      captureBoundedBytes(stdout, {
+        limit: MAX_MESSAGE_BYTES,
+        overflow: "reject",
+        exceeded: () =>
+          new ExtensionProtocolError("extension message exceeds 8 MiB"),
+      }),
+      captureBoundedBytes(stderr, {
+        limit: MAX_DIAGNOSTIC_BYTES,
+        overflow: "truncate",
+      }),
       this.proc.exited,
       write,
     ]);
@@ -248,48 +210,18 @@ class ExtensionExchange {
     return response;
   }
 
-  private timeout(): Promise<never> {
-    return new Promise((_resolve, reject) => {
-      this.timer = setTimeout(() => {
-        reject(new ExtensionProtocolError("extension timed out"));
-      }, this.timeoutMs);
-    });
-  }
-
-  private aborted(): Promise<never> {
-    return new Promise((_resolve, reject) => {
-      const signal = this.options.signal;
-      if (!signal) return;
-      this.cancel = () => {
-        reject(new ExtensionProtocolError("extension request cancelled"));
-      };
-      if (signal.aborted) this.cancel();
-      else signal.addEventListener("abort", this.cancel, { once: true });
-    });
-  }
-
-  private dispose(): void {
-    if (this.timer) clearTimeout(this.timer);
-    if (this.cancel)
-      this.options.signal?.removeEventListener("abort", this.cancel);
-  }
-
   async run(
     request: ExtensionRequest,
     payload: Uint8Array,
   ): Promise<SuccessfulExtensionResponse> {
     try {
-      return await Promise.race([
-        this.output(request, payload),
-        this.timeout(),
-        this.aborted(),
-      ]);
+      return await this.deadline.wait(this.output(request, payload));
     } catch (cause) {
-      terminate(this.proc);
+      terminateOwnedProcessGroup(this.proc);
       if (cause instanceof ExtensionProtocolError) throw cause;
       throw new ExtensionProtocolError("extension transport failed", { cause });
     } finally {
-      this.dispose();
+      this.deadline.dispose();
     }
   }
 }

package/src/extension-session.ts

@@ -4,9 +4,20 @@ import { createHash, randomUUID } from "node:crypto";
 import { createReadStream } from "node:fs";
 import { isAbsolute } from "node:path";
 import { canonicalJson, hashJson } from "./identity";
-import type { GeneratedFixture } from "./generated-fixture";
-import type { RepositoryFixture } from "./repository-fixture";
-import type { FixtureSetupDeclaration } from "./fixture-setup";
+import type { GeneratedFixture } from "./fixtures/generated";
+import type { RepositoryFixture } from "./fixtures/repository";
+import type { FixtureSetupDeclaration } from "./fixtures/setup";
+import type {
+  ArtifactReference,
+  Assessment,
+  CheckDeclaration,
+  DomainOutcome,
+  ExtensionCheckOutcome,
+  MetricEvidence,
+  ObservationEvidence,
+  TaskVerdict,
+} from "./contract-types";
+import type { InstrumentationRequest } from "./instrumentation";
 import {
   ExtensionProtocolError,
   exchangeExtension,
@@ -32,11 +43,7 @@ export interface ExtensionCase {
     | { kind: "inline"; files: Record<string, string> }
     | RepositoryFixture
     | GeneratedFixture;
-  checks: {
-    id: string;
-    grader: string;
-    configuration: Record<string, unknown>;
-  }[];
+  checks: CheckDeclaration[];
   requiredEvidence: string[];
   extensionData: Record<string, unknown>;
 }
@@ -51,10 +58,7 @@ export interface PreparationResult {
     gitExclude?: boolean;
     executable?: boolean;
   }[];
-  requestedInstrumentation: {
-    id: string;
-    configuration: Record<string, unknown>;
-  }[];
+  requestedInstrumentation: InstrumentationRequest[];
   fixtureSetup?: FixtureSetupDeclaration;
   codexMarketplace?: {
     artifactRoot: string;
@@ -77,48 +81,20 @@ export interface PreparationResult {
 
 export interface EvaluationRequest {
   caseId: string;
-  execution: {
-    status: "completed" | "failed" | "cancelled" | "not_run";
+  execution: Assessment["execution"] & {
     errorCode?: string;
   };
-  observations: {
-    id: string;
-    source: string;
-    completeness: "complete" | "partial" | "unavailable";
-    data: Record<string, unknown>;
-  }[];
-  builtinChecks: {
-    id: string;
-    status: "passed" | "failed" | "unavailable";
-    detail?: string;
-    evidenceRefs: string[];
-  }[];
-  artifacts: {
-    id: string;
-    path: string;
-    sha256: string;
-    gitExclude?: boolean;
-    executable?: boolean;
-  }[];
+  observations: ObservationEvidence[];
+  builtinChecks: ExtensionCheckOutcome[];
+  artifacts: ArtifactReference[];
   extensionData: Record<string, unknown>;
 }
 
 export interface EvaluationResult {
-  checks: {
-    id: string;
-    status: "passed" | "failed" | "unavailable";
-    detail?: string;
-    evidenceRefs: string[];
-  }[];
-  metrics: { id: string; value: number | null; unit: string }[];
-  domainOutcomes?: {
-    id: string;
-    status: "passed" | "failed" | "unavailable";
-    detail?: string;
-    evidenceRefs: string[];
-    data?: Record<string, unknown>;
-  }[];
-  taskVerdictRecommendation?: "passed" | "failed" | "not_assessed";
+  checks: ExtensionCheckOutcome[];
+  metrics: MetricEvidence[];
+  domainOutcomes?: DomainOutcome[];
+  taskVerdictRecommendation?: TaskVerdict;
 }
 
 export interface ExtensionSessionOptions extends ExchangeOptions {

package/src/{advisory-fixture.ts → fixtures/advisory.ts}

@@ -1,5 +1,5 @@
 import type { Dirent, Stats } from "node:fs";
-import { isMissingFile } from "./fixture-tools";
+import { isMissingFile } from "./tools";
 import {
   lstat,
   mkdir,
@@ -11,7 +11,9 @@ import {
   copyFile,
 } from "node:fs/promises";
 import { allocateWorkspace, removeWorkspace } from "./workspace-state";
-import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+import { dirname, isAbsolute, join, resolve } from "node:path";
+import { insideRoot as contained } from "../paths";
+import { runGit } from "../git";
 import { fixtureParts } from "./preparation";
 
 const MAX_GIT_OUTPUT = 16 * 1024 * 1024;
@@ -19,87 +21,25 @@ const MAX_UNTRACKED_BYTES = 32 * 1024 * 1024;
 const MAX_UNTRACKED_FILES = 1024;
 const HIDDEN_ROOTS = [".agents", ".claude", ".codex", ".git"];
 
-function contained(root: string, path: string): boolean {
-  const child = relative(root, path);
-  return (
-    child === "" ||
-    (child !== ".." && !child.startsWith(`..${sep}`) && !isAbsolute(child))
-  );
-}
-
 function excluded(path: string, roots: string[]): boolean {
   return roots.some((root) => path === root || path.startsWith(`${root}/`));
 }
 
-async function bounded(stream: ReadableStream<Uint8Array>): Promise<Buffer> {
-  const chunks: Uint8Array[] = [];
-  let size = 0;
-  for await (const chunk of stream) {
-    size += chunk.byteLength;
-    if (size > MAX_GIT_OUTPUT)
-      throw new Error("advisory fixture Git output is too large");
-    chunks.push(chunk);
-  }
-  return Buffer.concat(chunks);
-}
-
-function nullGitConfig(): string {
-  return process.platform === "win32" ? "NUL" : "/dev/null";
-}
-function gitEnvironment(): Record<string, string> {
-  return {
-    PATH: process.env.PATH ?? "/usr/bin:/bin",
-    ...(process.env.SystemRoot ? { SystemRoot: process.env.SystemRoot } : {}),
-    GIT_CONFIG_NOSYSTEM: "1",
-    GIT_CONFIG_GLOBAL: nullGitConfig(),
-    GIT_TERMINAL_PROMPT: "0",
-    GIT_AUTHOR_NAME: "Sevro Advisory",
-    GIT_AUTHOR_EMAIL: "advisory@sevro.invalid",
-    GIT_COMMITTER_NAME: "Sevro Advisory",
-    GIT_COMMITTER_EMAIL: "advisory@sevro.invalid",
-    GIT_AUTHOR_DATE: "2000-01-01T00:00:00+00:00",
-    GIT_COMMITTER_DATE: "2000-01-01T00:00:00+00:00",
-  };
-}
-
 async function git(
   cwd: string,
   args: string[],
   input?: Uint8Array,
 ): Promise<Buffer> {
-  const proc = Bun.spawn(
-    ["git", "-c", `core.hooksPath=${nullGitConfig()}`, ...args],
-    {
-      cwd,
-      stdin: input ? "pipe" : "ignore",
-      stdout: "pipe",
-      stderr: "pipe",
-      env: gitEnvironment(),
-    },
-  );
   try {
-    const write = (async () => {
-      if (!input) return;
-      if (!proc.stdin || typeof proc.stdin === "number")
-        throw new Error("advisory fixture Git input unavailable");
-      await proc.stdin.write(input);
-      await proc.stdin.end();
-    })();
-    const [out, , code] = await Promise.all([
-      bounded(proc.stdout),
-      bounded(proc.stderr),
-      proc.exited,
-      write,
-    ]);
-    if (code !== 0) throw new Error("advisory fixture Git operation failed");
-    return out;
+    return await runGit({
+      cwd,
+      args,
+      input,
+      identity: "advisory",
+      label: "advisory fixture",
+      maxOutputBytes: MAX_GIT_OUTPUT,
+    });
   } catch (cause) {
-    try {
-      proc.kill("SIGKILL");
-    } catch {
-      // The process may already have exited.
-    }
-    await proc.exited;
     throw new Error("advisory fixture Git operation failed", { cause });
   }
 }

package/src/{fixture-bin.ts → fixtures/bin.ts}

@@ -2,7 +2,7 @@ import {
   prepareFixtureTools,
   installFixtureTools,
   isMissingFile,
-} from "./fixture-tools";
+} from "./tools";
 import { lstat, mkdir, realpath } from "node:fs/promises";
 import { join } from "node:path";
 

package/src/{fixture-cleanup.ts → fixtures/cleanup.ts}

@@ -15,6 +15,18 @@ async function restoreDirectoryAccess(path: string): Promise<void> {
       await restoreDirectoryAccess(join(path, child.name));
 }
 
+/** Remove owned disposable state without following cache or fixture symlinks. */
+export async function removeDisposableDirectory(path: string): Promise<void> {
+  try {
+    await rm(path, { recursive: true, force: true });
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOTDIR") return;
+    if (!permissionFailure(error)) throw error;
+    await restoreDirectoryAccess(path);
+    await rm(path, { recursive: true, force: true });
+  }
+}
+
 /** Clear a completed fixture while preserving its reserved root for active peers. */
 export async function clearFixtureContents(workspace: string): Promise<void> {
   const root = await lstat(workspace);
@@ -26,13 +38,7 @@ export async function clearFixtureContents(workspace: string): Promise<void> {
   await Promise.all(
     names.map(async (name) => {
       const path = join(workspace, name);
-      try {
-        await rm(path, { recursive: true, force: true });
-      } catch (error) {
-        if (!permissionFailure(error)) throw error;
-        await restoreDirectoryAccess(path);
-        await rm(path, { recursive: true, force: true });
-      }
+      await removeDisposableDirectory(path);
     }),
   );
 }

package/src/fixtures/create.ts

@@ -0,0 +1,183 @@
+import { appendFile, mkdir, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { allocateWorkspace, removeWorkspace } from "./workspace-state";
+import { fixtureBinDirectory, installFixtureBin } from "./bin";
+import { installGitHooks } from "./git-hooks";
+import {
+  materializeGeneratedFixture,
+  type GeneratedFixture,
+} from "./generated";
+import { runFixtureSetup, type FixtureSetup } from "./setup";
+import {
+  fixtureParts,
+  safePreparationTarget,
+  type InlineArtifact,
+  type PreparationSources,
+} from "./preparation";
+import {
+  applyRepositoryOverlay,
+  cloneRepositorySource,
+  type RepositoryFixture,
+  type RepositorySource,
+} from "./repository";
+import {
+  EvaluationConfigurationError,
+  type ResolvedCase,
+} from "../evaluation/types";
+
+export type FixtureSource<Repository = RepositorySource> =
+  | { kind: "inline"; files: ResolvedCase["fixture"]["files"] }
+  | { kind: "generated"; fixture: GeneratedFixture }
+  | { kind: "repository"; fixture: RepositoryFixture; repository: Repository };
+
+interface FixtureOptions {
+  artifacts?: InlineArtifact[];
+  sources?: PreparationSources;
+  setup?: FixtureSetup | null;
+  projectRoot: string;
+  signal?: AbortSignal;
+  reservedWorkspace?: string;
+  candidateRuntimeRoot?: string;
+}
+
+export async function createFixture(
+  source: FixtureSource,
+  options: FixtureOptions,
+): Promise<string> {
+  const paths = inlineFixturePaths(source);
+  const artifacts = options.artifacts ?? [];
+  const workspace = options.reservedWorkspace ?? (await allocateWorkspace());
+  try {
+    await initializeFixtureHistory(source, options.sources, workspace);
+    await installDeclaredFixtureTools(source, workspace);
+    await writeInlineFixtureFiles(paths, workspace);
+    await setupFixture(
+      options.setup,
+      workspace,
+      options.projectRoot,
+      options.signal,
+      options.candidateRuntimeRoot,
+    );
+    await mountPreparationArtifacts(artifacts, workspace);
+    await excludePreparationArtifacts(artifacts, workspace);
+    await fixtureBinDirectory(workspace);
+    return workspace;
+  } catch (error) {
+    if (!options.reservedWorkspace) await removeWorkspace(workspace);
+    throw error;
+  }
+}
+
+function inlineFixturePaths(source: FixtureSource) {
+  return Object.entries(
+    source.kind === "inline" ? (source.files ?? {}) : {},
+  ).map(([path, content]) => ({ path, parts: fixtureParts(path), content }));
+}
+
+async function cloneFixtureRepository(
+  source: Extract<FixtureSource, { kind: "repository" }>,
+  sources: PreparationSources | undefined,
+  workspace: string,
+): Promise<void> {
+  if (!sources)
+    throw new EvaluationConfigurationError(
+      "repository source root is required and must be absolute",
+    );
+  await cloneRepositorySource(
+    source.fixture.sourceRef,
+    sources,
+    source.repository,
+    workspace,
+  );
+}
+
+async function initializeFixtureHistory(
+  source: FixtureSource,
+  sources: PreparationSources | undefined,
+  workspace: string,
+): Promise<void> {
+  switch (source.kind) {
+    case "repository":
+      await cloneFixtureRepository(source, sources, workspace);
+      await applyRepositoryOverlay(source.fixture, workspace);
+      break;
+    case "generated":
+      await materializeGeneratedFixture(source.fixture, workspace);
+      break;
+    case "inline":
+      break;
+  }
+}
+
+async function installDeclaredFixtureTools(
+  source: FixtureSource,
+  workspace: string,
+): Promise<void> {
+  const fixture = source.kind === "inline" ? undefined : source.fixture;
+  await installFixtureBin(fixture?.bin, workspace);
+  await installGitHooks(fixture?.hooks, workspace);
+}
+
+async function writeInlineFixtureFiles(
+  files: ReturnType<typeof inlineFixturePaths>,
+  workspace: string,
+): Promise<void> {
+  for (const file of files) {
+    const target = join(workspace, ...file.parts);
+    await mkdir(dirname(target), { recursive: true, mode: 0o700 });
+    await writeFile(target, file.content, { flag: "wx", mode: 0o600 });
+  }
+}
+
+async function setupFixture(
+  setup: FixtureSetup | null | undefined,
+  workspace: string,
+  projectRoot: string,
+  signal?: AbortSignal,
+  candidateRuntimeRoot?: string,
+): Promise<void> {
+  if (!setup) return;
+  await runFixtureSetup(setup, {
+    workspace,
+    projectRoot,
+    fixtureBinDir: (await fixtureBinDirectory(workspace)) ?? undefined,
+    signal,
+    candidateRuntimeRoot,
+  });
+}
+
+async function mountPreparationArtifacts(
+  artifacts: InlineArtifact[],
+  workspace: string,
+): Promise<void> {
+  for (const artifact of artifacts) {
+    const target = await safePreparationTarget(
+      workspace,
+      artifact.relativePath,
+    );
+    await writeFile(target, artifact.bytes, {
+      flag: "wx",
+      mode: artifact.executable ? 0o700 : 0o600,
+    });
+  }
+}
+
+async function excludePreparationArtifacts(
+  artifacts: InlineArtifact[],
+  workspace: string,
+): Promise<void> {
+  const gitExcluded = artifacts.filter((artifact) => artifact.gitExclude);
+  if (!gitExcluded.length) return;
+  const special = new Set(["\\", "*", "?", "[", "]", "#", "!", " "]);
+  const patterns = gitExcluded.map((artifact) =>
+    Array.from(artifact.relativePath)
+      .map((character) =>
+        special.has(character) ? `\\${character}` : character,
+      )
+      .join(""),
+  );
+  await appendFile(
+    join(workspace, ".git", "info", "exclude"),
+    `\n${patterns.map((path) => `/${path}`).join("\n")}\n`,
+  );
+}