@wrongstack/bench 0.260.0

package/dist/index.d.ts

@@ -0,0 +1,606 @@
+/**
+ * Core contracts for the model-independent benchmark harness.
+ *
+ * The guiding principle: WrongStack is the *harness* (system prompt + tool set
+ * + agent loop + scaffolding). The model is the swappable variable. Grading is
+ * deterministic (the suite's own tests decide pass/fail — never an LLM), and
+ * every report is stamped with a {@link HarnessFingerprint} so rows are only
+ * comparable when the harness is identical.
+ */
+/** A single model under test — one column in the leaderboard. */
+interface ModelCell {
+    /** Short human label shown in the report (e.g. "opus-4.8"). Must be unique. */
+    label: string;
+    /** Provider id passed to `wstack --provider` (e.g. "anthropic"). */
+    provider: string;
+    /** Model id passed to `wstack --model` (e.g. "claude-opus-4-8"). */
+    model: string;
+}
+/** Loaded `bench.config.json`. */
+interface BenchConfig {
+    /** Per-task iteration cap (seeded into the isolated config). Default 40. */
+    maxIterations: number;
+    /** How many cells/tasks run concurrently. Default 4. */
+    concurrency: number;
+    /** Per-task wall-clock timeout in milliseconds. Default 600_000 (10m). */
+    timeoutMs: number;
+    /** The models to benchmark. At least one. */
+    cells: ModelCell[];
+}
+/** One unit of work: a single benchmark exercise/issue. */
+interface BenchTask {
+    /** Stable id, unique within the suite (e.g. "polyglot/python/bowling"). */
+    id: string;
+    /** Suite this task belongs to. */
+    suite: SuiteId;
+    /** The instruction text handed to the agent via `--prompt`. */
+    prompt: string;
+    /**
+     * Absolute path to a template directory. The runner copies it into an
+     * isolated workdir before each cell so parallel runs never collide.
+     */
+    templateDir: string;
+    /**
+     * Top-level entry names to omit when copying the template (e.g. `.meta` so
+     * the agent never sees the reference solution). Matched against each path's
+     * segments. Defaults to none.
+     */
+    templateExclude?: string[] | undefined;
+    /** Opaque per-suite data the grader needs (test command, language, etc.). */
+    meta: Record<string, unknown>;
+}
+type SuiteId = 'polyglot' | 'swebench';
+/** A suite knows how to enumerate its tasks and grade a finished workdir. */
+interface BenchSuite {
+    id: SuiteId;
+    /** Discover tasks. `limit` caps the count (for cheap smoke runs). */
+    loadTasks(opts: {
+        limit?: number | undefined;
+    }): Promise<BenchTask[]>;
+    /** A stable id for the exact task subset, folded into the fingerprint. */
+    subsetId(tasks: BenchTask[]): string;
+}
+/** Deterministic grader verdict for one finished workdir. */
+interface GradeResult {
+    /** Did the suite's own tests pass? This is the headline correctness signal. */
+    passed: boolean;
+    /**
+     * Whether a verdict was actually produced. Defaults to true. SWE-bench sets
+     * this false when it only exported a prediction for offline grading by the
+     * official harness — such rows are excluded from the pass rate so they don't
+     * masquerade as failures.
+     */
+    graded?: boolean | undefined;
+    /** Optional detail (failing test names, compiler error, etc.). */
+    detail?: string | undefined;
+}
+/** Raw telemetry parsed from a single `wstack` subprocess run. */
+interface RawRun {
+    /** RunResult.status from `--output-json`, or a harness-level status. */
+    status: 'completed' | 'failed' | 'aborted' | 'max_iterations' | 'timeout' | 'crashed';
+    finalText: string | null;
+    iterations: number;
+    tokensIn: number;
+    tokensOut: number;
+    costUsd: number;
+    elapsedMs: number;
+    /** Process exit code (null when killed by timeout). */
+    exitCode: number | null;
+}
+/** Per-(task × cell) result: telemetry + deterministic grade + tool metrics. */
+interface TaskResult {
+    taskId: string;
+    cell: ModelCell;
+    run: RawRun;
+    grade: GradeResult;
+    /** Tool-call metrics parsed from the isolated session JSONL. */
+    tools: ToolMetrics;
+}
+/** Tool-level metrics derived from the session log (model-free). */
+interface ToolMetrics {
+    totalCalls: number;
+    /** edit/write tool invocations. */
+    editCalls: number;
+    /** edit/write invocations that returned an error (failed to apply). */
+    editErrors: number;
+    /** provider 429 / retry events. */
+    rateLimitRetries: number;
+}
+/** Folded results for one model cell across all its tasks. */
+interface CellResult {
+    cell: ModelCell;
+    taskCount: number;
+    /** How many tasks produced an actual graded verdict (graded !== false). */
+    gradedCount: number;
+    /** Fraction in [0,1] of GRADED tasks whose grader passed (pass@1). */
+    passRate: number;
+    /** Fraction in [0,1] of edit/write calls that applied cleanly. */
+    editApplyRate: number;
+    avgCostUsd: number;
+    avgTokensIn: number;
+    avgTokensOut: number;
+    /** Median iterations across tasks. */
+    p50Iterations: number;
+    /** Median wall-clock per task, ms. */
+    p50ElapsedMs: number;
+    /** Fraction in [0,1] of tasks that hit the timeout. */
+    timeoutRate: number;
+    totalRateLimitRetries: number;
+}
+/**
+ * Identifies the harness configuration. Two reports are only comparable when
+ * their fingerprints match; a prompt/tool/version change flips the hash and
+ * marks older rows stale.
+ */
+interface HarnessFingerprint {
+    cliVersion: string;
+    /** Sorted, comma-joined tool names available to the agent. */
+    toolNames: string[];
+    maxIterations: number;
+    yolo: boolean;
+    /** Suite subset id (the exact task set). */
+    subsetId: string;
+    /** sha256 hex (first 12 chars) of the above. */
+    hash: string;
+}
+/** The full report artifact written to disk. */
+interface BenchReport {
+    suite: SuiteId;
+    /** ISO timestamp the run finished (stamped by the caller, not the harness). */
+    finishedAt: string;
+    fingerprint: HarnessFingerprint;
+    cells: CellResult[];
+    /** Every per-(task × cell) row, for reproducibility. */
+    results: TaskResult[];
+}
+
+/**
+ * Fold every per-(task × cell) result for ONE cell into its leaderboard row.
+ * All metrics are derived from deterministic signals (grader pass/fail, the
+ * `--output-json` usage block, and session-log tool counts) — nothing here
+ * consults a model.
+ */
+declare function aggregateCell(cell: ModelCell, results: TaskResult[]): CellResult;
+/** Group all results by cell label and aggregate each group. */
+declare function aggregateAll(cells: ModelCell[], results: TaskResult[]): CellResult[];
+/** Median of a numeric array (0 for empty). Exported for tests. */
+declare function median(values: number[]): number;
+
+/**
+ * Parse and validate a raw `bench.config.json` object. Throws a descriptive
+ * Error on any structural problem so the CLI can surface it cleanly instead of
+ * failing deep inside the runner.
+ */
+declare function parseBenchConfig(raw: unknown): BenchConfig;
+/** Load and validate a `bench.config.json` from disk. */
+declare function loadBenchConfig(path: string): Promise<BenchConfig>;
+
+interface ExecResult {
+    exitCode: number | null;
+    stdout: string;
+    stderr: string;
+    timedOut: boolean;
+}
+/**
+ * Run a command (argv form) in a directory and capture its result. Used by the
+ * deterministic graders to run a suite's own test command — the run's exit code
+ * is the pass/fail signal, no LLM involved.
+ *
+ * Never rejects; a spawn failure surfaces as exitCode null with the error on
+ * stderr.
+ */
+declare function execCommand(opts: {
+    command: string;
+    args: string[];
+    cwd: string;
+    timeoutMs: number;
+    env?: NodeJS.ProcessEnv | undefined;
+    /**
+     * Run through a shell. Needed for launchers that resolve platform wrappers
+     * (`npm` → npm.cmd on Windows, `./gradlew`). Defaults to true. Pass false for
+     * real executables (git, python, node, cargo, go) to avoid the shell entirely
+     * — no metacharacter interpretation, no DEP0190, no injection surface.
+     */
+    shell?: boolean | undefined;
+}): Promise<ExecResult>;
+
+/**
+ * Compute the harness fingerprint. This is what makes a report
+ * "model-independent": every cell in a run shares one fingerprint, so the only
+ * thing that varies across leaderboard rows is the model. Change the CLI
+ * version, the tool roster, the iteration cap, the yolo flag, or the task
+ * subset and the hash changes — which is exactly when old numbers stop being
+ * comparable.
+ *
+ * The hash is intentionally cheap and reproducible: no timestamps, no random
+ * salt. Same inputs → same hash, on any machine.
+ */
+declare function computeHarnessFingerprint(input: {
+    cliVersion: string;
+    toolNames: string[];
+    maxIterations: number;
+    yolo: boolean;
+    subsetId: string;
+}): HarnessFingerprint;
+/** One-line human label for report headers: `wrongstack@0.255 · fp:a3f9c7 · maxIter=40 · yolo`. */
+declare function fingerprintLabel(fp: HarnessFingerprint): string;
+
+/**
+ * Deterministic polyglot grader: run the exercise's own test command in the
+ * finished workdir. Exit code 0 → passed. No LLM, no judgement — this is the
+ * invariant that keeps the report model-independent.
+ *
+ * For languages with a dependency-install step (JS `npm install`, etc.) the
+ * setup command runs first; if setup fails the task is graded as not-passed
+ * with the setup error as detail (it cannot be the model's fault, but the run
+ * is genuinely ungradeable, so it counts as a fail rather than crashing).
+ */
+declare function gradePolyglot(opts: {
+    workdir: string;
+    task: BenchTask;
+    /** Per-step timeout for setup/test commands. */
+    timeoutMs: number;
+}): Promise<GradeResult>;
+
+/** Injectable command runner (defaults to execCommand) for testability. */
+type Exec = (opts: {
+    command: string;
+    args: string[];
+    cwd: string;
+    timeoutMs: number;
+    shell?: boolean | undefined;
+}) => Promise<ExecResult>;
+/**
+ * Extract the model's patch from a finished SWE-bench workdir.
+ *
+ * The workdir is a git checkout at the instance's base commit; the agent's
+ * edits are uncommitted. `git add -A` stages new/modified/deleted files, then
+ * `git diff --cached` produces a unified diff in the exact form the official
+ * SWE-bench harness expects as `model_patch`.
+ *
+ * Changes to files touched by the held-out `test_patch` are stripped: the
+ * harness applies the model patch and then the test patch, and the agent is
+ * told not to edit tests — dropping those sections keeps the model patch from
+ * conflicting with (or sneaking changes into) the graded tests.
+ */
+declare function extractModelPatch(opts: {
+    workdir: string;
+    /** The instance's held-out test patch, used to exclude test-file edits. */
+    testPatch?: string | undefined;
+    timeoutMs: number;
+    exec?: Exec | undefined;
+}): Promise<string>;
+/**
+ * Collect the file paths a unified diff touches. Reads both the
+ * `diff --git a/<p> b/<p>` header and the `+++ b/<p>` / `--- a/<p>` lines so it
+ * works on patches produced by git or by `diff -u`.
+ */
+declare function extractPatchPaths(patch: string): Set<string>;
+/**
+ * Drop every per-file section of `patch` whose target path is in `exclude`.
+ * Sections are delimited by `diff --git` headers (git's format).
+ */
+declare function filterPatchExcludingPaths(patch: string, exclude: Set<string>): string;
+/**
+ * Drop each `diff --git` section for which `shouldDrop(aPath, bPath)` is true.
+ */
+declare function filterPatchSections(patch: string, shouldDrop: (aPath: string, bPath: string) => boolean): string;
+
+/**
+ * Inline (Docker) grader hook. Given an instance's model patch and held-out
+ * test data, return whether the issue is resolved — or `undefined` if it could
+ * not produce a verdict. Left injectable so the heavy, version-sensitive Docker
+ * execution is plugged in by the host (the official SWE-bench harness) rather
+ * than re-implemented and guessed-at here.
+ */
+type SwebenchExternalGrade = (args: {
+    instanceId: string;
+    patch: string;
+    image?: string | undefined;
+    failToPass: string[];
+    passToPass: string[];
+    testPatch?: string | undefined;
+    workdir: string;
+    timeoutMs: number;
+}) => Promise<boolean | undefined>;
+/**
+ * SWE-bench grader.
+ *
+ * Resolution is decided deterministically by the instance's own tests
+ * (`FAIL_TO_PASS` must pass, `PASS_TO_PASS` must still pass) inside its pinned
+ * Docker image — never an LLM. We do NOT re-implement that Docker evaluation
+ * (the official `princeton-nlp/SWE-bench` harness owns it and it is version
+ * sensitive). Instead this grader:
+ *
+ *   1. Extracts the model patch from the finished workdir (`git diff`),
+ *      excluding any edits to the held-out test files.
+ *   2. Writes a conformant per-instance prediction so the run can be graded by
+ *      the official harness (`--predictions_path`).
+ *   3. If an inline grader is supplied (Docker available), runs it and returns a
+ *      real pass/fail verdict; otherwise marks the row "exported, ungraded"
+ *      (graded:false) so it is excluded from pass@1 rather than counted as a
+ *      failure.
+ */
+declare function gradeSwebench(opts: {
+    workdir: string;
+    task: BenchTask;
+    cell: ModelCell;
+    timeoutMs: number;
+    /** Where per-instance prediction files are written. */
+    predictionsDir: string;
+    exec?: Exec | undefined;
+    externalGrade?: SwebenchExternalGrade | undefined;
+}): Promise<GradeResult>;
+
+/**
+ * Per-run isolation. Each benchmark run gets one sandbox directory tree:
+ *
+ *   <sandbox>/
+ *     home/        → isolated WRONGSTACK_HOME (config seed + all session JSONL)
+ *     work/<id>/   → one copy of a task template per (task × cell)
+ *
+ * The isolated home keeps the bench off the developer's real ~/.wrongstack
+ * (config, sessions, models cache). Each task workdir hashes to its own
+ * project slug under home/projects/, so concurrent runs never share a session
+ * file even though they share one home.
+ */
+interface Sandbox {
+    /** Root sandbox dir. */
+    root: string;
+    /** Isolated WRONGSTACK_HOME. */
+    homeDir: string;
+    /** Directory that holds per-task workdirs. */
+    workRoot: string;
+}
+/** Create the sandbox tree and seed the isolated home's config.json. */
+declare function createSandbox(opts: {
+    /** Where to create the sandbox. Defaults to an OS temp dir. */
+    baseDir?: string | undefined;
+    maxIterations: number;
+    yolo: boolean;
+}): Promise<Sandbox>;
+/**
+ * Copy a task template into a fresh workdir. The directory name embeds the
+ * cell label and task id so it is both unique (parallel-safe) and debuggable.
+ */
+declare function prepareWorkdir(sandbox: Sandbox, templateDir: string, taskId: string, cellLabel: string, exclude?: string[] | undefined): Promise<string>;
+/** Remove the whole sandbox tree. Best-effort. */
+declare function cleanupSandbox(sandbox: Sandbox): Promise<void>;
+
+interface RunBenchmarkOptions {
+    suite: BenchSuite;
+    /** Suite-specific deterministic grader. */
+    grade: (args: {
+        workdir: string;
+        task: BenchTask;
+        cell: ModelCell;
+        timeoutMs: number;
+    }) => Promise<GradeResult>;
+    config: BenchConfig;
+    cliVersion: string;
+    /** Tool names available to the agent — folded into the fingerprint. */
+    toolNames: string[];
+    /** Node executable. */
+    nodeBin: string;
+    /** Path to the wstack CLI entry. */
+    wstackEntry: string;
+    /** Cap the number of tasks (cheap smoke runs). */
+    limit?: number | undefined;
+    /** Where the sandbox is created (default OS temp). */
+    sandboxBaseDir?: string | undefined;
+    /** Extra env for the subprocess (provider keys are inherited from process.env). */
+    env?: NodeJS.ProcessEnv | undefined;
+    /** Keep the sandbox on disk after the run (debugging). */
+    keepSandbox?: boolean | undefined;
+    /** Progress callback (one line per event). */
+    onProgress?: ((msg: string) => void) | undefined;
+    /** Injected clock for the report timestamp (tests pass a fixed value). */
+    now?: (() => string) | undefined;
+}
+/**
+ * Run the full benchmark: load the task subset, fan every (task × cell) cell
+ * out through isolated subprocesses, grade deterministically, and fold into a
+ * fingerprint-stamped report.
+ */
+declare function runBenchmark(opts: RunBenchmarkOptions): Promise<BenchReport>;
+
+/**
+ * Write the machine-readable report artifacts:
+ *   - results.jsonl  → one line per (task × cell), for reproducibility
+ *   - summary.json   → fingerprint + folded cell results
+ *
+ * The markdown report is derived from summary.json (see report/markdown.ts), so
+ * `wstack bench report` can re-render without re-running anything.
+ */
+declare function writeJsonArtifacts(outDir: string, report: BenchReport): Promise<void>;
+/** Read back a summary.json into the partial report shape markdown needs. */
+declare function readSummary(outDir: string): Promise<Pick<BenchReport, 'suite' | 'finishedAt' | 'fingerprint' | 'cells'>>;
+
+/**
+ * Render the human-facing leaderboard. The header carries the harness
+ * fingerprint: rows are only comparable across reports that share it. The body
+ * sorts cells by pass rate (the headline correctness metric), highest first.
+ */
+declare function renderMarkdownReport(report: Pick<BenchReport, 'suite' | 'finishedAt' | 'fingerprint' | 'cells'>): string;
+/** Build the full fingerprint-stamped header line for terminal echo. */
+declare function reportHeaderLine(fp: HarnessFingerprint): string;
+
+/**
+ * One SWE-bench prediction row, in the exact shape the official harness
+ * (`princeton-nlp/SWE-bench`) consumes via `--predictions_path`.
+ */
+interface SwebenchPrediction {
+    instance_id: string;
+    /** The model/system label — becomes a column in the official report. */
+    model_name_or_path: string;
+    /** The unified diff the agent produced. */
+    model_patch: string;
+}
+/**
+ * Write a `predictions.jsonl` for one model cell. SWE-bench grading is delegated
+ * to the canonical, version-sensitive harness rather than re-implemented here:
+ * we own running the agent and producing a conformant patch; the official tool
+ * owns the Docker execution and pass/fail verdict.
+ *
+ * Returns the file path written.
+ */
+declare function writePredictionsJsonl(outDir: string, cellLabel: string, predictions: SwebenchPrediction[]): Promise<string>;
+/**
+ * Write one instance's prediction to its own file under
+ * `<predictionsDir>/<cell>/<instance>.json`. Distinct files per instance are
+ * concurrency-safe — the SWE-bench grader runs inside the orchestrator's
+ * parallel fan-out, so appending to a shared jsonl would race. Call
+ * {@link collectCellPredictions} after the run to merge them.
+ */
+declare function writeInstancePrediction(predictionsDir: string, cellLabel: string, prediction: SwebenchPrediction): Promise<void>;
+/** Read back every per-instance prediction written for one cell. */
+declare function collectCellPredictions(predictionsDir: string, cellLabel: string): Promise<SwebenchPrediction[]>;
+/**
+ * Parse an official SWE-bench evaluation report JSON for the set of resolved
+ * instance ids. The harness writes `resolved_ids` (newer) or a per-instance
+ * `{ resolved: bool }` map; both shapes are handled so this keeps working across
+ * harness versions.
+ */
+declare function parseResolvedIds(reportJson: unknown): Set<string>;
+
+/** Everything needed to run one (task × cell) subprocess. */
+interface RunWstackOptions {
+    /** Node executable (process.execPath). */
+    nodeBin: string;
+    /** Path to the wstack CLI entry (dist/index.js) — or a fake in tests. */
+    wstackEntry: string;
+    /** Isolated WRONGSTACK_HOME for this run. */
+    homeDir: string;
+    /** Task workdir (becomes the subprocess cwd / projectRoot). */
+    workdir: string;
+    cell: ModelCell;
+    prompt: string;
+    timeoutMs: number;
+    /** Extra env merged over process.env (e.g. provider keys already present). */
+    env?: NodeJS.ProcessEnv | undefined;
+    /** Extra CLI args appended after the standard set (Phase 2 hooks). */
+    extraArgs?: string[] | undefined;
+}
+/**
+ * Run the real `wstack` binary once, in single-shot `--output-json` mode, and
+ * parse its machine-readable result. This is the heart of the model-independent
+ * design: the subprocess is the *whole* harness (real wiring, real tools), and
+ * the only thing that varies between calls is `--provider`/`--model`.
+ *
+ * Never rejects — a crash, non-JSON output, or timeout becomes a RawRun with an
+ * explanatory status so the grader still produces a row.
+ */
+declare function runWstack(opts: RunWstackOptions): Promise<RawRun>;
+/**
+ * Map `items` through `fn` with at most `concurrency` in flight at once.
+ * Results preserve input order. A tiny dependency-free p-limit.
+ */
+declare function mapWithConcurrency<T, R>(items: T[], concurrency: number, fn: (item: T, index: number) => Promise<R>): Promise<R[]>;
+
+/**
+ * Derive model-free tool metrics from the isolated session JSONL the
+ * subprocess wrote. Everything here comes from `tool_call_end` events
+ * (`{ name, ok }`) and provider retry/error events — no LLM, no heuristics.
+ *
+ * Returns zeroed metrics (never throws) when the session log is missing or
+ * unreadable: a crashed run still produces a valid, gradeable TaskResult.
+ */
+declare function readToolMetrics(opts: {
+    homeDir: string;
+    /** The task workdir — the subprocess used this as its projectRoot. */
+    workdir: string;
+}): Promise<ToolMetrics>;
+
+/**
+ * Aider polyglot benchmark loader.
+ *
+ * The polyglot-benchmark repo (https://github.com/Aider-AI/polyglot-benchmark)
+ * lays exercises out in Exercism form:
+ *
+ *   <root>/<language>/exercises/practice/<slug>/
+ *     .docs/instructions.md          ← problem statement
+ *     .meta/config.json              ← Exercism file manifest (solution/test/example)
+ *     .meta/example.<ext>            ← reference solution (EXCLUDED from the agent)
+ *     <solution files>               ← stubs the agent edits
+ *     <test files>                   ← the hidden tests the grader runs
+ *
+ * We do NOT vendor the exercises (225 across 6 languages); the caller points
+ * `--polyglot-dir` at a local checkout.
+ */
+/** Per-language test command + optional dependency-install step. */
+interface LanguageRunner {
+    /** Directory name under the polyglot root. */
+    dir: string;
+    /** argv for the test command, run in the workdir. */
+    test: (testFiles: string[]) => {
+        command: string;
+        args: string[];
+    };
+    /** argv for an optional setup/install step run before tests. */
+    setup?: {
+        command: string;
+        args: string[];
+    } | undefined;
+}
+declare const LANGUAGE_RUNNERS: Record<string, LanguageRunner>;
+/** Metadata the polyglot grader reads off each task. */
+interface PolyglotMeta {
+    language: string;
+    solutionFiles: string[];
+    testFiles: string[];
+    testCommand: {
+        command: string;
+        args: string[];
+    };
+    setupCommand?: {
+        command: string;
+        args: string[];
+    } | undefined;
+}
+declare function createPolyglotSuite(opts: {
+    /** Local checkout of the polyglot-benchmark repo. */
+    polyglotDir: string;
+    /** Restrict to these languages (default: all present). */
+    languages?: string[] | undefined;
+}): BenchSuite;
+
+/**
+ * SWE-bench Verified adapter (Phase 2).
+ *
+ * Unlike polyglot (a self-contained directory of exercises), SWE-bench requires
+ * a materialized repo at a specific base commit AND the task's pinned execution
+ * environment — which the official harness ships as per-instance Docker images.
+ * Running it without Docker is not meaningful, so this adapter is wired and
+ * fingerprint-aware but gated: `loadTasks` throws an actionable error unless a
+ * prepared dataset directory is supplied and `docker` is enabled.
+ *
+ * The fixed instance subset lives in `subsets/swe-bench-verified-50.json` so a
+ * model comparison always grades the exact same issues (reproducibility).
+ */
+interface SwebenchOptions {
+    /**
+     * Directory of prepared instances. Expected layout (produced by your
+     * SWE-bench setup step — see packages/bench/README.md):
+     *   <datasetDir>/<instance_id>/
+     *     repo/                 ← git checkout at base_commit
+     *     instance.json         ← { problem_statement, test_patch, FAIL_TO_PASS, PASS_TO_PASS, image }
+     */
+    datasetDir?: string | undefined;
+    /** Must be true to actually build tasks — the env is Docker-backed. */
+    docker?: boolean | undefined;
+    /** Override the committed subset file. */
+    subsetFile?: string | undefined;
+}
+interface SwebenchMeta {
+    instanceId: string;
+    instanceDir: string;
+    image?: string | undefined;
+    failToPass: string[];
+    passToPass: string[];
+    testPatch?: string | undefined;
+}
+/** Read the pinned instance-id subset (the canonical task set). */
+declare function loadSubset(subsetFile?: string): Promise<string[]>;
+declare function createSwebenchSuite(opts?: SwebenchOptions): BenchSuite;
+
+export { type BenchConfig, type BenchReport, type BenchSuite, type BenchTask, type CellResult, type Exec, type ExecResult, type GradeResult, type HarnessFingerprint, LANGUAGE_RUNNERS, type ModelCell, type PolyglotMeta, type RawRun, type RunBenchmarkOptions, type RunWstackOptions, type Sandbox, type SuiteId, type SwebenchExternalGrade, type SwebenchMeta, type SwebenchOptions, type SwebenchPrediction, type TaskResult, type ToolMetrics, aggregateAll, aggregateCell, cleanupSandbox, collectCellPredictions, computeHarnessFingerprint, createPolyglotSuite, createSandbox, createSwebenchSuite, execCommand, extractModelPatch, extractPatchPaths, filterPatchExcludingPaths, filterPatchSections, fingerprintLabel, gradePolyglot, gradeSwebench, loadBenchConfig, loadSubset, mapWithConcurrency, median, parseBenchConfig, parseResolvedIds, prepareWorkdir, readSummary, readToolMetrics, renderMarkdownReport, reportHeaderLine, runBenchmark, runWstack, writeInstancePrediction, writeJsonArtifacts, writePredictionsJsonl };