@bjoernrochel/sevro 0.1.0-rc.8 → 1.0.0

package/src/extension-session.ts

@@ -4,9 +4,20 @@ import { createHash, randomUUID } from "node:crypto";
 import { createReadStream } from "node:fs";
 import { isAbsolute } from "node:path";
 import { canonicalJson, hashJson } from "./identity";
-import type { GeneratedFixture } from "./generated-fixture";
-import type { RepositoryFixture } from "./repository-fixture";
-import type { FixtureSetupDeclaration } from "./fixture-setup";
+import type { GeneratedFixture } from "./fixtures/generated";
+import type { RepositoryFixture } from "./fixtures/repository";
+import type { FixtureSetupDeclaration } from "./fixtures/setup";
+import type {
+  ArtifactReference,
+  Assessment,
+  CheckDeclaration,
+  DomainOutcome,
+  ExtensionCheckOutcome,
+  MetricEvidence,
+  ObservationEvidence,
+  TaskVerdict,
+} from "./contract-types";
+import type { InstrumentationRequest } from "./instrumentation";
 import {
   ExtensionProtocolError,
   exchangeExtension,
@@ -32,11 +43,7 @@ export interface ExtensionCase {
     | { kind: "inline"; files: Record<string, string> }
     | RepositoryFixture
     | GeneratedFixture;
-  checks: {
-    id: string;
-    grader: string;
-    configuration: Record<string, unknown>;
-  }[];
+  checks: CheckDeclaration[];
   requiredEvidence: string[];
   extensionData: Record<string, unknown>;
 }
@@ -51,10 +58,7 @@ export interface PreparationResult {
     gitExclude?: boolean;
     executable?: boolean;
   }[];
-  requestedInstrumentation: {
-    id: string;
-    configuration: Record<string, unknown>;
-  }[];
+  requestedInstrumentation: InstrumentationRequest[];
   fixtureSetup?: FixtureSetupDeclaration;
   codexMarketplace?: {
     artifactRoot: string;
@@ -77,48 +81,20 @@ export interface PreparationResult {
 
 export interface EvaluationRequest {
   caseId: string;
-  execution: {
-    status: "completed" | "failed" | "cancelled" | "not_run";
+  execution: Assessment["execution"] & {
     errorCode?: string;
   };
-  observations: {
-    id: string;
-    source: string;
-    completeness: "complete" | "partial" | "unavailable";
-    data: Record<string, unknown>;
-  }[];
-  builtinChecks: {
-    id: string;
-    status: "passed" | "failed" | "unavailable";
-    detail?: string;
-    evidenceRefs: string[];
-  }[];
-  artifacts: {
-    id: string;
-    path: string;
-    sha256: string;
-    gitExclude?: boolean;
-    executable?: boolean;
-  }[];
+  observations: ObservationEvidence[];
+  builtinChecks: ExtensionCheckOutcome[];
+  artifacts: ArtifactReference[];
   extensionData: Record<string, unknown>;
 }
 
 export interface EvaluationResult {
-  checks: {
-    id: string;
-    status: "passed" | "failed" | "unavailable";
-    detail?: string;
-    evidenceRefs: string[];
-  }[];
-  metrics: { id: string; value: number | null; unit: string }[];
-  domainOutcomes?: {
-    id: string;
-    status: "passed" | "failed" | "unavailable";
-    detail?: string;
-    evidenceRefs: string[];
-    data?: Record<string, unknown>;
-  }[];
-  taskVerdictRecommendation?: "passed" | "failed" | "not_assessed";
+  checks: ExtensionCheckOutcome[];
+  metrics: MetricEvidence[];
+  domainOutcomes?: DomainOutcome[];
+  taskVerdictRecommendation?: TaskVerdict;
 }
 
 export interface ExtensionSessionOptions extends ExchangeOptions {

package/src/{advisory-fixture.ts → fixtures/advisory.ts}

@@ -1,5 +1,5 @@
 import type { Dirent, Stats } from "node:fs";
-import { isMissingFile } from "./fixture-tools";
+import { isMissingFile } from "./tools";
 import {
   lstat,
   mkdir,
@@ -11,7 +11,9 @@ import {
   copyFile,
 } from "node:fs/promises";
 import { allocateWorkspace, removeWorkspace } from "./workspace-state";
-import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+import { dirname, isAbsolute, join, resolve } from "node:path";
+import { insideRoot as contained } from "../paths";
+import { runGit } from "../git";
 import { fixtureParts } from "./preparation";
 
 const MAX_GIT_OUTPUT = 16 * 1024 * 1024;
@@ -19,87 +21,25 @@ const MAX_UNTRACKED_BYTES = 32 * 1024 * 1024;
 const MAX_UNTRACKED_FILES = 1024;
 const HIDDEN_ROOTS = [".agents", ".claude", ".codex", ".git"];
 
-function contained(root: string, path: string): boolean {
-  const child = relative(root, path);
-  return (
-    child === "" ||
-    (child !== ".." && !child.startsWith(`..${sep}`) && !isAbsolute(child))
-  );
-}
-
 function excluded(path: string, roots: string[]): boolean {
   return roots.some((root) => path === root || path.startsWith(`${root}/`));
 }
 
-async function bounded(stream: ReadableStream<Uint8Array>): Promise<Buffer> {
-  const chunks: Uint8Array[] = [];
-  let size = 0;
-  for await (const chunk of stream) {
-    size += chunk.byteLength;
-    if (size > MAX_GIT_OUTPUT)
-      throw new Error("advisory fixture Git output is too large");
-    chunks.push(chunk);
-  }
-  return Buffer.concat(chunks);
-}
-
-function nullGitConfig(): string {
-  return process.platform === "win32" ? "NUL" : "/dev/null";
-}
-function gitEnvironment(): Record<string, string> {
-  return {
-    PATH: process.env.PATH ?? "/usr/bin:/bin",
-    ...(process.env.SystemRoot ? { SystemRoot: process.env.SystemRoot } : {}),
-    GIT_CONFIG_NOSYSTEM: "1",
-    GIT_CONFIG_GLOBAL: nullGitConfig(),
-    GIT_TERMINAL_PROMPT: "0",
-    GIT_AUTHOR_NAME: "Sevro Advisory",
-    GIT_AUTHOR_EMAIL: "advisory@sevro.invalid",
-    GIT_COMMITTER_NAME: "Sevro Advisory",
-    GIT_COMMITTER_EMAIL: "advisory@sevro.invalid",
-    GIT_AUTHOR_DATE: "2000-01-01T00:00:00+00:00",
-    GIT_COMMITTER_DATE: "2000-01-01T00:00:00+00:00",
-  };
-}
-
 async function git(
   cwd: string,
   args: string[],
   input?: Uint8Array,
 ): Promise<Buffer> {
-  const proc = Bun.spawn(
-    ["git", "-c", `core.hooksPath=${nullGitConfig()}`, ...args],
-    {
-      cwd,
-      stdin: input ? "pipe" : "ignore",
-      stdout: "pipe",
-      stderr: "pipe",
-      env: gitEnvironment(),
-    },
-  );
   try {
-    const write = (async () => {
-      if (!input) return;
-      if (!proc.stdin || typeof proc.stdin === "number")
-        throw new Error("advisory fixture Git input unavailable");
-      await proc.stdin.write(input);
-      await proc.stdin.end();
-    })();
-    const [out, , code] = await Promise.all([
-      bounded(proc.stdout),
-      bounded(proc.stderr),
-      proc.exited,
-      write,
-    ]);
-    if (code !== 0) throw new Error("advisory fixture Git operation failed");
-    return out;
+    return await runGit({
+      cwd,
+      args,
+      input,
+      identity: "advisory",
+      label: "advisory fixture",
+      maxOutputBytes: MAX_GIT_OUTPUT,
+    });
   } catch (cause) {
-    try {
-      proc.kill("SIGKILL");
-    } catch {
-      // The process may already have exited.
-    }
-    await proc.exited;
     throw new Error("advisory fixture Git operation failed", { cause });
   }
 }

package/src/{fixture-bin.ts → fixtures/bin.ts}

@@ -2,7 +2,7 @@ import {
   prepareFixtureTools,
   installFixtureTools,
   isMissingFile,
-} from "./fixture-tools";
+} from "./tools";
 import { lstat, mkdir, realpath } from "node:fs/promises";
 import { join } from "node:path";
 

package/src/{fixture-cleanup.ts → fixtures/cleanup.ts}

@@ -15,6 +15,18 @@ async function restoreDirectoryAccess(path: string): Promise<void> {
       await restoreDirectoryAccess(join(path, child.name));
 }
 
+/** Remove owned disposable state without following cache or fixture symlinks. */
+export async function removeDisposableDirectory(path: string): Promise<void> {
+  try {
+    await rm(path, { recursive: true, force: true });
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOTDIR") return;
+    if (!permissionFailure(error)) throw error;
+    await restoreDirectoryAccess(path);
+    await rm(path, { recursive: true, force: true });
+  }
+}
+
 /** Clear a completed fixture while preserving its reserved root for active peers. */
 export async function clearFixtureContents(workspace: string): Promise<void> {
   const root = await lstat(workspace);
@@ -26,13 +38,7 @@ export async function clearFixtureContents(workspace: string): Promise<void> {
   await Promise.all(
     names.map(async (name) => {
       const path = join(workspace, name);
-      try {
-        await rm(path, { recursive: true, force: true });
-      } catch (error) {
-        if (!permissionFailure(error)) throw error;
-        await restoreDirectoryAccess(path);
-        await rm(path, { recursive: true, force: true });
-      }
+      await removeDisposableDirectory(path);
     }),
   );
 }

package/src/fixtures/create.ts

@@ -0,0 +1,183 @@
+import { appendFile, mkdir, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { allocateWorkspace, removeWorkspace } from "./workspace-state";
+import { fixtureBinDirectory, installFixtureBin } from "./bin";
+import { installGitHooks } from "./git-hooks";
+import {
+  materializeGeneratedFixture,
+  type GeneratedFixture,
+} from "./generated";
+import { runFixtureSetup, type FixtureSetup } from "./setup";
+import {
+  fixtureParts,
+  safePreparationTarget,
+  type InlineArtifact,
+  type PreparationSources,
+} from "./preparation";
+import {
+  applyRepositoryOverlay,
+  cloneRepositorySource,
+  type RepositoryFixture,
+  type RepositorySource,
+} from "./repository";
+import {
+  EvaluationConfigurationError,
+  type ResolvedCase,
+} from "../evaluation/types";
+
+export type FixtureSource<Repository = RepositorySource> =
+  | { kind: "inline"; files: ResolvedCase["fixture"]["files"] }
+  | { kind: "generated"; fixture: GeneratedFixture }
+  | { kind: "repository"; fixture: RepositoryFixture; repository: Repository };
+
+interface FixtureOptions {
+  artifacts?: InlineArtifact[];
+  sources?: PreparationSources;
+  setup?: FixtureSetup | null;
+  projectRoot: string;
+  signal?: AbortSignal;
+  reservedWorkspace?: string;
+  candidateRuntimeRoot?: string;
+}
+
+export async function createFixture(
+  source: FixtureSource,
+  options: FixtureOptions,
+): Promise<string> {
+  const paths = inlineFixturePaths(source);
+  const artifacts = options.artifacts ?? [];
+  const workspace = options.reservedWorkspace ?? (await allocateWorkspace());
+  try {
+    await initializeFixtureHistory(source, options.sources, workspace);
+    await installDeclaredFixtureTools(source, workspace);
+    await writeInlineFixtureFiles(paths, workspace);
+    await setupFixture(
+      options.setup,
+      workspace,
+      options.projectRoot,
+      options.signal,
+      options.candidateRuntimeRoot,
+    );
+    await mountPreparationArtifacts(artifacts, workspace);
+    await excludePreparationArtifacts(artifacts, workspace);
+    await fixtureBinDirectory(workspace);
+    return workspace;
+  } catch (error) {
+    if (!options.reservedWorkspace) await removeWorkspace(workspace);
+    throw error;
+  }
+}
+
+function inlineFixturePaths(source: FixtureSource) {
+  return Object.entries(
+    source.kind === "inline" ? (source.files ?? {}) : {},
+  ).map(([path, content]) => ({ path, parts: fixtureParts(path), content }));
+}
+
+async function cloneFixtureRepository(
+  source: Extract<FixtureSource, { kind: "repository" }>,
+  sources: PreparationSources | undefined,
+  workspace: string,
+): Promise<void> {
+  if (!sources)
+    throw new EvaluationConfigurationError(
+      "repository source root is required and must be absolute",
+    );
+  await cloneRepositorySource(
+    source.fixture.sourceRef,
+    sources,
+    source.repository,
+    workspace,
+  );
+}
+
+async function initializeFixtureHistory(
+  source: FixtureSource,
+  sources: PreparationSources | undefined,
+  workspace: string,
+): Promise<void> {
+  switch (source.kind) {
+    case "repository":
+      await cloneFixtureRepository(source, sources, workspace);
+      await applyRepositoryOverlay(source.fixture, workspace);
+      break;
+    case "generated":
+      await materializeGeneratedFixture(source.fixture, workspace);
+      break;
+    case "inline":
+      break;
+  }
+}
+
+async function installDeclaredFixtureTools(
+  source: FixtureSource,
+  workspace: string,
+): Promise<void> {
+  const fixture = source.kind === "inline" ? undefined : source.fixture;
+  await installFixtureBin(fixture?.bin, workspace);
+  await installGitHooks(fixture?.hooks, workspace);
+}
+
+async function writeInlineFixtureFiles(
+  files: ReturnType<typeof inlineFixturePaths>,
+  workspace: string,
+): Promise<void> {
+  for (const file of files) {
+    const target = join(workspace, ...file.parts);
+    await mkdir(dirname(target), { recursive: true, mode: 0o700 });
+    await writeFile(target, file.content, { flag: "wx", mode: 0o600 });
+  }
+}
+
+async function setupFixture(
+  setup: FixtureSetup | null | undefined,
+  workspace: string,
+  projectRoot: string,
+  signal?: AbortSignal,
+  candidateRuntimeRoot?: string,
+): Promise<void> {
+  if (!setup) return;
+  await runFixtureSetup(setup, {
+    workspace,
+    projectRoot,
+    fixtureBinDir: (await fixtureBinDirectory(workspace)) ?? undefined,
+    signal,
+    candidateRuntimeRoot,
+  });
+}
+
+async function mountPreparationArtifacts(
+  artifacts: InlineArtifact[],
+  workspace: string,
+): Promise<void> {
+  for (const artifact of artifacts) {
+    const target = await safePreparationTarget(
+      workspace,
+      artifact.relativePath,
+    );
+    await writeFile(target, artifact.bytes, {
+      flag: "wx",
+      mode: artifact.executable ? 0o700 : 0o600,
+    });
+  }
+}
+
+async function excludePreparationArtifacts(
+  artifacts: InlineArtifact[],
+  workspace: string,
+): Promise<void> {
+  const gitExcluded = artifacts.filter((artifact) => artifact.gitExclude);
+  if (!gitExcluded.length) return;
+  const special = new Set(["\\", "*", "?", "[", "]", "#", "!", " "]);
+  const patterns = gitExcluded.map((artifact) =>
+    Array.from(artifact.relativePath)
+      .map((character) =>
+        special.has(character) ? `\\${character}` : character,
+      )
+      .join(""),
+  );
+  await appendFile(
+    join(workspace, ".git", "info", "exclude"),
+    `\n${patterns.map((path) => `/${path}`).join("\n")}\n`,
+  );
+}

package/src/{generated-fixture.ts → fixtures/generated.ts}

@@ -1,9 +1,10 @@
 import { mkdir, writeFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
-import { canonicalJson } from "./identity";
-import { prepareFixtureBin } from "./fixture-bin";
+import { canonicalJson } from "../identity";
+import { prepareFixtureBin } from "./bin";
 import { prepareGitHooks } from "./git-hooks";
 import { fixtureParts } from "./preparation";
+import { runGit } from "../git";
 
 const MAX_COMMITS = 128;
 const MAX_FILES = 1024;
@@ -197,60 +198,16 @@ export function prepareGeneratedFixture(value: unknown): GeneratedFixture {
   return JSON.parse(canonicalJson(fixture)) as GeneratedFixture;
 }
 
-function gitEnvironment() {
-  const nullDevice = process.platform === "win32" ? "NUL" : "/dev/null";
-  return {
-    PATH: process.env.PATH ?? "/usr/bin:/bin",
-    ...(process.env.SystemRoot ? { SystemRoot: process.env.SystemRoot } : {}),
-    GIT_CONFIG_NOSYSTEM: "1",
-    GIT_CONFIG_GLOBAL: nullDevice,
-    GIT_TERMINAL_PROMPT: "0",
-    GIT_AUTHOR_NAME: "Sevro Fixture",
-    GIT_AUTHOR_EMAIL: "fixture@sevro.invalid",
-    GIT_COMMITTER_NAME: "Sevro Fixture",
-    GIT_COMMITTER_EMAIL: "fixture@sevro.invalid",
-    GIT_AUTHOR_DATE: "2000-01-01T00:00:00+00:00",
-    GIT_COMMITTER_DATE: "2000-01-01T00:00:00+00:00",
-  };
-}
-
 async function git(workspace: string, args: string[]): Promise<void> {
-  async function bounded(stream: ReadableStream<Uint8Array>): Promise<void> {
-    let size = 0;
-    for await (const chunk of stream) {
-      size += chunk.byteLength;
-      if (size > MAX_GIT_OUTPUT)
-        throw new Error("generated fixture Git output is too large");
-    }
-  }
-  const proc = Bun.spawn(
-    [
-      "git",
-      "-c",
-      `core.hooksPath=${process.platform === "win32" ? "NUL" : "/dev/null"}`,
-      ...args,
-    ],
-    {
-      cwd: workspace,
-      stdout: "pipe",
-      stderr: "pipe",
-      env: gitEnvironment(),
-    },
-  );
   try {
-    const [, , code] = await Promise.all([
-      bounded(proc.stdout),
-      bounded(proc.stderr),
-      proc.exited,
-    ]);
-    if (code !== 0) throw new Error();
+    await runGit({
+      cwd: workspace,
+      args,
+      identity: "fixture",
+      label: "generated fixture",
+      maxOutputBytes: MAX_GIT_OUTPUT,
+    });
   } catch (cause) {
-    try {
-      proc.kill("SIGKILL");
-    } catch {
-      // The process may have exited before the failure was observed.
-    }
-    await proc.exited;
     throw new Error("generated fixture Git operation failed", { cause });
   }
 }

package/src/{git-hooks.ts → fixtures/git-hooks.ts}

@@ -1,5 +1,5 @@
 import { join } from "node:path";
-import { prepareFixtureTools, installFixtureTools } from "./fixture-tools";
+import { prepareFixtureTools, installFixtureTools } from "./tools";
 
 /** Bound hook declarations and keep every name inside Git's hooks directory. */
 export function prepareGitHooks(value: unknown): Record<string, string> | null {

package/src/{preparation.ts → fixtures/preparation.ts}

@@ -1,7 +1,9 @@
-import { isMissingFile } from "./fixture-tools";
-import { sha256 } from "./identity";
+import { isMissingFile } from "./tools";
+import { sha256 } from "../identity";
 import { lstat, mkdir, readFile, realpath, stat } from "node:fs/promises";
-import { isAbsolute, join, relative, sep } from "node:path";
+import { isAbsolute, join } from "node:path";
+import { insideRoot } from "../paths";
+import { containsControl as containsControlCharacter } from "../value-guards";
 import { fileURLToPath } from "node:url";
 
 const MAX_ARTIFACT_BYTES = 8 * 1024 * 1024;
@@ -114,12 +116,6 @@ function requireArtifactFlags(item: PreparationArtifact): void {
   if (item.executable !== undefined && typeof item.executable !== "boolean")
     throw new Error("preparation artifact executable must be boolean");
 }
-function containsControlCharacter(path: string): boolean {
-  return Array.from(path).some(
-    (character) =>
-      character.charCodeAt(0) < 0x20 || character.charCodeAt(0) === 0x7f,
-  );
-}
 function requireInlineSource(
   item: PreparationArtifact,
 ): asserts item is PreparationArtifact & { contentBase64: string } {
@@ -220,13 +216,7 @@ async function preparationSourcePath(url: string): Promise<string> {
   }
 }
 function requireContainedSource(root: string, path: string): void {
-  const child = relative(root, path);
-  if (
-    !child ||
-    child === ".." ||
-    child.startsWith(`..${sep}`) ||
-    isAbsolute(child)
-  )
+  if (root === path || !insideRoot(root, path))
     throw new Error("preparation source escapes its declared root");
 }
 async function preparationSourceSize(path: string): Promise<number> {