@bjoernrochel/sevro 0.1.0-rc.1

package/src/extension-client.ts

@@ -0,0 +1,257 @@
+import { randomUUID } from "node:crypto";
+import Ajv2020 from "ajv/dist/2020.js";
+import extensionSchema from "../schemas/extension-v1.schema.json";
+
+const MAX_MESSAGE_BYTES = 8 * 1024 * 1024;
+const MAX_DIAGNOSTIC_BYTES = 64 * 1024;
+const DEFAULT_TIMEOUT_MS = 30_000;
+const PROTOCOL = "sevro.extension.v1";
+
+const ajv = new Ajv2020({
+  strict: true,
+  strictRequired: false,
+  strictTypes: false,
+});
+const validExchange = ajv.compile(extensionSchema);
+
+export class ExtensionProtocolError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ExtensionProtocolError";
+  }
+}
+
+export interface ExtensionRequest {
+  protocol: string;
+  id: string;
+  method: "describe" | "resolve" | "prepare" | "evaluate";
+  params: Record<string, unknown>;
+}
+
+export interface ExtensionResponse {
+  protocol: string;
+  id: string;
+  method: ExtensionRequest["method"];
+  result?: Record<string, unknown>;
+  error?: { code: string; message: string };
+}
+
+export interface ExchangeOptions {
+  cwd?: string;
+  timeoutMs?: number;
+  signal?: AbortSignal;
+}
+
+async function readBounded(
+  stream: ReadableStream<Uint8Array>,
+  limit: number,
+  truncate: boolean,
+): Promise<Uint8Array> {
+  const reader = stream.getReader();
+  const chunks: Uint8Array[] = [];
+  let size = 0;
+  while (true) {
+    const { value, done } = await reader.read();
+    if (done) break;
+    if (!truncate && size + value.byteLength > limit)
+      throw new ExtensionProtocolError("extension message exceeds 8 MiB");
+    const kept = value.subarray(0, Math.max(0, limit - size));
+    if (kept.byteLength) chunks.push(kept);
+    size += kept.byteLength;
+  }
+  const result = new Uint8Array(size);
+  let offset = 0;
+  for (const chunk of chunks) {
+    result.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return result;
+}
+
+function terminate(proc: Bun.Subprocess): void {
+  if (process.platform !== "win32") {
+    try {
+      process.kill(-proc.pid, "SIGKILL");
+      return;
+    } catch {
+      // The group may have exited while the runner was handling the error.
+    }
+  }
+  try {
+    proc.kill("SIGKILL");
+  } catch {
+    // The process already exited.
+  }
+}
+
+/** Run exactly one request in a fresh extension process. */
+export async function exchangeExtension(
+  command: string[],
+  request: ExtensionRequest,
+  options: ExchangeOptions = {},
+): Promise<ExtensionResponse> {
+  if (!command.length || command.some((part) => !part))
+    throw new ExtensionProtocolError(
+      "extension command must be a nonempty argv array",
+    );
+  if (!validExchange(request))
+    throw new ExtensionProtocolError("invalid extension request");
+  if (options.signal?.aborted)
+    throw new ExtensionProtocolError("extension request cancelled");
+  const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0)
+    throw new ExtensionProtocolError(
+      "extension timeout must be a positive integer",
+    );
+  const payload = new TextEncoder().encode(JSON.stringify(request));
+  if (payload.byteLength > MAX_MESSAGE_BYTES)
+    throw new ExtensionProtocolError("extension request exceeds 8 MiB");
+
+  let proc: Bun.Subprocess;
+  try {
+    proc = Bun.spawn(command, {
+      cwd: options.cwd,
+      stdin: "pipe",
+      stdout: "pipe",
+      stderr: "pipe",
+      detached: process.platform !== "win32",
+    });
+  } catch {
+    throw new ExtensionProtocolError("could not start extension process");
+  }
+
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  let cancel: (() => void) | undefined;
+  try {
+    const { stdin, stdout, stderr } = proc;
+    if (
+      !stdin ||
+      typeof stdin === "number" ||
+      !stdout ||
+      typeof stdout === "number" ||
+      !stderr ||
+      typeof stderr === "number"
+    )
+      throw new ExtensionProtocolError("extension pipes are unavailable");
+    const write = (async () => {
+      await stdin.write(payload);
+      await stdin.end();
+    })();
+    const output = (async () => {
+      const [responseBytes, _diagnostic, exitCode] = await Promise.all([
+        readBounded(stdout, MAX_MESSAGE_BYTES, false),
+        readBounded(stderr, MAX_DIAGNOSTIC_BYTES, true),
+        proc.exited,
+        write,
+      ]);
+      if (exitCode !== 0)
+        throw new ExtensionProtocolError(
+          `extension exited with code ${exitCode}`,
+        );
+      let response: unknown;
+      try {
+        response = JSON.parse(
+          new TextDecoder("utf-8", { fatal: true }).decode(responseBytes),
+        );
+      } catch {
+        throw new ExtensionProtocolError(
+          "extension did not return one UTF-8 JSON response",
+        );
+      }
+      if (!validExchange(response))
+        throw new ExtensionProtocolError("invalid extension response");
+      const envelope = response as unknown as ExtensionResponse;
+      if (
+        envelope.id !== request.id ||
+        envelope.protocol !== request.protocol ||
+        envelope.method !== request.method
+      )
+        throw new ExtensionProtocolError(
+          "extension response does not match request",
+        );
+      if (envelope.error)
+        throw new ExtensionProtocolError(
+          `extension reported ${envelope.error.code}`,
+        );
+      return envelope;
+    })();
+    const timeout = new Promise<never>((_resolve, reject) => {
+      timer = setTimeout(
+        () => reject(new ExtensionProtocolError("extension timed out")),
+        timeoutMs,
+      );
+    });
+    const aborted = new Promise<never>((_resolve, reject) => {
+      if (!options.signal) return;
+      cancel = () =>
+        reject(new ExtensionProtocolError("extension request cancelled"));
+      if (options.signal.aborted) cancel();
+      else options.signal.addEventListener("abort", cancel, { once: true });
+    });
+    return await Promise.race([output, timeout, aborted]);
+  } catch (error) {
+    terminate(proc);
+    if (error instanceof ExtensionProtocolError) throw error;
+    throw new ExtensionProtocolError("extension transport failed");
+  } finally {
+    if (timer) clearTimeout(timer);
+    if (cancel) options.signal?.removeEventListener("abort", cancel);
+  }
+}
+
+export interface NegotiatedExtension {
+  id: string;
+  version: string;
+  protocol: string;
+  capabilities: string[];
+  graders: string[];
+  taskVerdictPolicies: string[];
+}
+
+/** Discover an extension and select the v1 protocol and available capabilities. */
+export async function negotiateExtension(
+  command: string[],
+  available: { engineCapabilities: string[]; hostCapabilities: string[] },
+  options: ExchangeOptions = {},
+): Promise<NegotiatedExtension> {
+  const response = await exchangeExtension(
+    command,
+    {
+      protocol: "sevro.discovery.v1",
+      id: randomUUID(),
+      method: "describe",
+      params: {
+        protocols: [PROTOCOL],
+        engineCapabilities: available.engineCapabilities,
+        hostCapabilities: available.hostCapabilities,
+      },
+    },
+    options,
+  );
+  const result = response.result!;
+  const offered = result.protocols as string[];
+  if (!offered.includes(PROTOCOL))
+    throw new ExtensionProtocolError("extension has no compatible protocol");
+  const supported = new Set([
+    ...available.engineCapabilities,
+    ...available.hostCapabilities,
+  ]);
+  const required = result.requiredCapabilities as string[];
+  for (const capability of required)
+    if (!supported.has(capability))
+      throw new ExtensionProtocolError(
+        `unsupported required capability: ${capability}`,
+      );
+  const optional = (result.optionalCapabilities as string[]).filter((item) =>
+    supported.has(item),
+  );
+  const extension = result.extension as { id: string; version: string };
+  return {
+    id: extension.id,
+    version: extension.version,
+    protocol: PROTOCOL,
+    capabilities: [...new Set([...required, ...optional])],
+    graders: result.graders as string[],
+    taskVerdictPolicies: result.taskVerdictPolicies as string[],
+  };
+}

package/src/extension-session.ts

@@ -0,0 +1,335 @@
+import { createHash, randomUUID } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { isAbsolute } from "node:path";
+import { canonicalJson, hashJson } from "./identity";
+import type { GeneratedFixture } from "./generated-fixture";
+import type { RepositoryFixture } from "./repository-fixture";
+import type { FixtureSetupDeclaration } from "./fixture-setup";
+import {
+  ExtensionProtocolError,
+  exchangeExtension,
+  negotiateExtension,
+  type ExchangeOptions,
+  type NegotiatedExtension,
+} from "./extension-client";
+
+const PROTOCOL = "sevro.extension.v1";
+
+export interface ExtensionCase {
+  id: string;
+  prompt: string;
+  followUpPrompt?: string;
+  fixture:
+    | { kind: "inline"; files: Record<string, string> }
+    | RepositoryFixture
+    | GeneratedFixture;
+  checks: {
+    id: string;
+    grader: string;
+    configuration: Record<string, unknown>;
+  }[];
+  requiredEvidence: string[];
+  extensionData: Record<string, unknown>;
+}
+
+export interface PreparationResult {
+  artifacts: {
+    id: string;
+    relativePath: string;
+    sha256: string;
+    contentBase64?: string;
+    sourceRef?: string;
+    gitExclude?: boolean;
+    executable?: boolean;
+  }[];
+  requestedInstrumentation: {
+    id: string;
+    configuration: Record<string, unknown>;
+  }[];
+  fixtureSetup?: FixtureSetupDeclaration;
+  codexMarketplace?: {
+    artifactRoot: string;
+    marketplaceName: string;
+    pluginNames: string[];
+  };
+  claudePluginDirs?: { artifactRoots: string[] };
+  codexSkillInvocation?: {
+    pluginName: string;
+    skillName: string;
+  };
+  codexRepositorySkillInvocation?: { skillName: string };
+  claudeRepositorySkillInvocation?: { skillName: string };
+  claudeSkillInvocation?: {
+    pluginName: string;
+    skillName: string;
+  };
+  extensionData: Record<string, unknown>;
+}
+
+export interface EvaluationRequest {
+  caseId: string;
+  execution: {
+    status: "completed" | "failed" | "cancelled" | "not_run";
+    errorCode?: string;
+  };
+  observations: {
+    id: string;
+    source: string;
+    completeness: "complete" | "partial" | "unavailable";
+    data: Record<string, unknown>;
+  }[];
+  builtinChecks: {
+    id: string;
+    status: "passed" | "failed" | "unavailable";
+    detail?: string;
+    evidenceRefs: string[];
+  }[];
+  artifacts: {
+    id: string;
+    path: string;
+    sha256: string;
+    gitExclude?: boolean;
+    executable?: boolean;
+  }[];
+  extensionData: Record<string, unknown>;
+}
+
+export interface EvaluationResult {
+  checks: {
+    id: string;
+    status: "passed" | "failed" | "unavailable";
+    detail?: string;
+    evidenceRefs: string[];
+  }[];
+  metrics: { id: string; value: number | null; unit: string }[];
+  domainOutcomes?: {
+    id: string;
+    status: "passed" | "failed" | "unavailable";
+    detail?: string;
+    evidenceRefs: string[];
+    data?: Record<string, unknown>;
+  }[];
+  taskVerdictRecommendation?: "passed" | "failed" | "not_assessed";
+}
+
+export interface ExtensionSessionOptions extends ExchangeOptions {
+  command: string[];
+  sourceFiles: string[];
+  configuration: Record<string, unknown>;
+  redactedConfiguration: Record<string, unknown>;
+  engineCapabilities: string[];
+  hostCapabilities: string[];
+  taskVerdictPolicy?: string;
+  replaceBuiltinGraders?: string[];
+}
+
+export interface ExtensionIdentity extends NegotiatedExtension {
+  sourceDigest: string;
+  configurationDigest: string;
+  selectedTaskVerdictPolicy: string | null;
+  replacedBuiltinGraders: string[];
+}
+
+async function digestSources(paths: string[]): Promise<string> {
+  const hashes: string[] = [];
+  for (const path of paths) {
+    try {
+      const hash = createHash("sha256");
+      for await (const chunk of createReadStream(path)) hash.update(chunk);
+      hashes.push(hash.digest("hex"));
+    } catch {
+      throw new ExtensionProtocolError(
+        "declared extension source is unreadable",
+      );
+    }
+  }
+  return hashJson(hashes);
+}
+
+function jsonCopy<T>(value: T): T {
+  return JSON.parse(canonicalJson(value)) as T;
+}
+
+/** Bind a stateless extension command to a negotiated identity and source snapshot. */
+export async function openExtensionSession(options: ExtensionSessionOptions) {
+  const command = [...options.command];
+  const sourceFiles = [...options.sourceFiles];
+  const sourceClosure = [...new Set([command[0], ...sourceFiles])];
+  const taskVerdictPolicy = options.taskVerdictPolicy;
+  const replacedBuiltinGraders = [
+    ...(options.replaceBuiltinGraders ?? []),
+  ].sort();
+  const exchangeOptions: ExchangeOptions = {
+    cwd: options.cwd,
+    timeoutMs: options.timeoutMs,
+    signal: options.signal,
+  };
+  if (
+    !command.length ||
+    !isAbsolute(command[0] ?? "") ||
+    !sourceFiles.length ||
+    sourceFiles.some((path) => !isAbsolute(path)) ||
+    new Set(sourceFiles).size !== sourceFiles.length
+  )
+    throw new ExtensionProtocolError(
+      "extension source files must be unique absolute paths",
+    );
+  if (new Set(replacedBuiltinGraders).size !== replacedBuiltinGraders.length)
+    throw new ExtensionProtocolError("duplicate built-in grader replacement");
+  const configuration = jsonCopy(options.configuration);
+  const redactedConfiguration = jsonCopy(options.redactedConfiguration);
+  const sourceDigest = await digestSources(sourceClosure);
+  const negotiated = await negotiateExtension(
+    command,
+    {
+      engineCapabilities: options.engineCapabilities,
+      hostCapabilities: options.hostCapabilities,
+    },
+    exchangeOptions,
+  );
+  if ((await digestSources(sourceClosure)) !== sourceDigest)
+    throw new ExtensionProtocolError(
+      "extension source changed during discovery",
+    );
+  if (
+    taskVerdictPolicy &&
+    !negotiated.taskVerdictPolicies.includes(taskVerdictPolicy)
+  )
+    throw new ExtensionProtocolError(
+      "extension did not advertise the selected task policy",
+    );
+  const identity: ExtensionIdentity = {
+    ...negotiated,
+    sourceDigest,
+    configurationDigest: hashJson({
+      command,
+      configuration: redactedConfiguration,
+      taskVerdictPolicy: taskVerdictPolicy ?? null,
+      replacedBuiltinGraders,
+    }),
+    selectedTaskVerdictPolicy: taskVerdictPolicy ?? null,
+    replacedBuiltinGraders,
+  };
+  Object.freeze(identity.capabilities);
+  Object.freeze(identity.graders);
+  Object.freeze(identity.taskVerdictPolicies);
+  Object.freeze(identity.replacedBuiltinGraders);
+  Object.freeze(identity);
+
+  async function call(
+    method: "resolve" | "prepare" | "evaluate",
+    params: object,
+  ) {
+    if ((await digestSources(sourceClosure)) !== sourceDigest)
+      throw new ExtensionProtocolError("extension source changed during run");
+    const response = await exchangeExtension(
+      command,
+      {
+        protocol: PROTOCOL,
+        id: randomUUID(),
+        method,
+        params: { ...params, configuration },
+      },
+      exchangeOptions,
+    );
+    if ((await digestSources(sourceClosure)) !== sourceDigest)
+      throw new ExtensionProtocolError("extension source changed during run");
+    return response.result!;
+  }
+
+  return {
+    identity,
+    get redactedConfiguration() {
+      return jsonCopy(redactedConfiguration);
+    },
+    async resolve(
+      projectRoot: string,
+      selectors: Record<string, unknown>,
+      host?: {
+        id: string;
+        model: string;
+        effort: string;
+        capabilities: string[];
+      },
+    ): Promise<ExtensionCase[]> {
+      const result = await call("resolve", {
+        projectRoot,
+        selectors,
+        ...(host ? { host } : {}),
+      });
+      const cases = result.cases as ExtensionCase[];
+      const ids = cases.map((item) => item.id);
+      if (new Set(ids).size !== ids.length)
+        throw new ExtensionProtocolError(
+          "extension resolved duplicate case IDs",
+        );
+      return cases;
+    },
+    async prepare(
+      resolvedCase: ExtensionCase,
+      host: { id: string; capabilities: string[] },
+      condition: "passive" | "enforced",
+    ): Promise<PreparationResult> {
+      return (await call("prepare", {
+        case: resolvedCase,
+        host,
+        condition,
+      })) as unknown as PreparationResult;
+    },
+    async evaluate(request: EvaluationRequest): Promise<EvaluationResult> {
+      const result = (await call("evaluate", {
+        ...request,
+        ...(taskVerdictPolicy
+          ? { selectedTaskVerdictPolicy: taskVerdictPolicy }
+          : {}),
+      })) as unknown as EvaluationResult;
+      const outcomes = result.domainOutcomes ?? [];
+      const ids = [
+        ...result.checks.map((check) => check.id),
+        ...outcomes.map((outcome) => outcome.id),
+      ];
+      if (
+        new Set(ids).size !== ids.length ||
+        ids.some((id) => !id.startsWith(`${identity.id}.`))
+      )
+        throw new ExtensionProtocolError(
+          "extension returned duplicate or foreign result IDs",
+        );
+      if (result.taskVerdictRecommendation && !taskVerdictPolicy)
+        throw new ExtensionProtocolError(
+          "extension task policy was not selected",
+        );
+      if (taskVerdictPolicy && !result.taskVerdictRecommendation)
+        throw new ExtensionProtocolError(
+          "extension omitted the selected task policy recommendation",
+        );
+      const available = new Map(
+        request.observations.map((item) => [item.id, item.completeness]),
+      );
+      for (const item of request.artifacts) available.set(item.id, "complete");
+      for (const item of request.builtinChecks)
+        available.set(
+          item.id,
+          item.status === "unavailable" ? "unavailable" : "complete",
+        );
+      for (const item of [...result.checks, ...outcomes]) {
+        if (item.status === "passed" && item.evidenceRefs.length === 0)
+          throw new ExtensionProtocolError(
+            "extension passed a result without evidence",
+          );
+        if (item.evidenceRefs.some((id) => !available.has(id)))
+          throw new ExtensionProtocolError(
+            "extension result cites unknown evidence",
+          );
+        if (
+          item.status === "passed" &&
+          item.evidenceRefs.some((id) => available.get(id) !== "complete")
+        )
+          throw new ExtensionProtocolError(
+            "extension passed a result with incomplete evidence",
+          );
+      }
+      return result;
+    },
+  };
+}

package/src/fixture-bin.ts

@@ -0,0 +1,87 @@
+import { chmod, lstat, mkdir, realpath, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+const MAX_TOOLS = 64;
+const MAX_BYTES = 1024 * 1024;
+
+/** Validate executable fixture tools without allowing path traversal. */
+export function prepareFixtureBin(
+  value: unknown,
+): Record<string, string> | null {
+  if (value === undefined) return null;
+  if (!value || typeof value !== "object" || Array.isArray(value))
+    throw new Error("invalid fixture binaries");
+  const bin = value as Record<string, unknown>;
+  const entries = Object.entries(bin);
+  if (
+    entries.length > MAX_TOOLS ||
+    entries.some(
+      ([name, content]) =>
+        !/^[A-Za-z][A-Za-z0-9._-]*$/.test(name) ||
+        name.length > 64 ||
+        typeof content !== "string" ||
+        !content ||
+        content.includes("\0"),
+    ) ||
+    entries.reduce(
+      (size, [name, content]) =>
+        size +
+        Buffer.byteLength(name, "utf8") +
+        Buffer.byteLength(content as string, "utf8"),
+      0,
+    ) > MAX_BYTES
+  )
+    throw new Error("invalid fixture binaries");
+  return bin as Record<string, string>;
+}
+
+/** Install declared tools inside the fixture's Git metadata, after commits. */
+export async function installFixtureBin(
+  bin: Record<string, string> | undefined,
+  workspace: string,
+): Promise<void> {
+  if (!bin || Object.keys(bin).length === 0) return;
+  const directory = join(workspace, ".git", "fixture-bin");
+  await mkdir(directory, { recursive: true, mode: 0o700 });
+  const root = await lstat(directory);
+  if (!root.isDirectory() || root.isSymbolicLink())
+    throw new Error("fixture binary directory is invalid");
+  for (const [name, content] of Object.entries(bin)) {
+    const target = join(directory, name);
+    try {
+      const existing = await lstat(target);
+      if (!existing.isFile() || existing.isSymbolicLink())
+        throw new Error("fixture binary target is invalid");
+    } catch (error) {
+      if (!(
+        error instanceof Error &&
+        "code" in error &&
+        error.code === "ENOENT"
+      ))
+        throw error;
+    }
+    await writeFile(target, content, { mode: 0o755 });
+    await chmod(target, 0o755);
+  }
+}
+
+/** Setup may also install trusted tools, such as a repository ticket stub. */
+export async function fixtureBinDirectory(
+  workspace: string,
+): Promise<string | null> {
+  const path = join(workspace, ".git", "fixture-bin");
+  try {
+    const entry = await lstat(path);
+    if (!entry.isDirectory() || entry.isSymbolicLink())
+      throw new Error("fixture binary directory is invalid");
+    const canonicalWorkspace = await realpath(workspace);
+    const canonicalBin = await realpath(path);
+    if (canonicalBin !== join(canonicalWorkspace, ".git", "fixture-bin"))
+      throw new Error("fixture binary directory escapes the workspace");
+    return path;
+  } catch (error) {
+    if (error instanceof Error && "code" in error && error.code === "ENOENT")
+      return null;
+    throw error;
+  }
+}