@agentgrader/core 1.0.0

package/dist/index.d.ts

@@ -0,0 +1,756 @@
+import { z } from 'zod';
+import { AgrDb } from '@agentgrader/store';
+
+declare const SuccessCriterionSchema: z.ZodUnion<[z.ZodObject<{
+    run: z.ZodString;
+    expect: z.ZodObject<{
+        exit_code: z.ZodDefault<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        exit_code: number;
+    }, {
+        exit_code?: number | undefined;
+    }>;
+}, "strip", z.ZodTypeAny, {
+    run: string;
+    expect: {
+        exit_code: number;
+    };
+}, {
+    run: string;
+    expect: {
+        exit_code?: number | undefined;
+    };
+}>, z.ZodObject<{
+    assert: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    assert: string;
+}, {
+    assert: string;
+}>]>;
+type SuccessCriterion = z.infer<typeof SuccessCriterionSchema>;
+declare const TestCaseSchema: z.ZodObject<{
+    id: z.ZodOptional<z.ZodString>;
+    name: z.ZodString;
+    description: z.ZodOptional<z.ZodString>;
+    fixture: z.ZodString;
+    prompt: z.ZodString;
+    success: z.ZodArray<z.ZodUnion<[z.ZodObject<{
+        run: z.ZodString;
+        expect: z.ZodObject<{
+            exit_code: z.ZodDefault<z.ZodNumber>;
+        }, "strip", z.ZodTypeAny, {
+            exit_code: number;
+        }, {
+            exit_code?: number | undefined;
+        }>;
+    }, "strip", z.ZodTypeAny, {
+        run: string;
+        expect: {
+            exit_code: number;
+        };
+    }, {
+        run: string;
+        expect: {
+            exit_code?: number | undefined;
+        };
+    }>, z.ZodObject<{
+        assert: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        assert: string;
+    }, {
+        assert: string;
+    }>]>, "many">;
+    timeout_seconds: z.ZodDefault<z.ZodNumber>;
+    tags: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    test_command: z.ZodOptional<z.ZodString>;
+    fail_to_pass: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    pass_to_pass: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    forbid_modified: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    expected_files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    solution: z.ZodOptional<z.ZodString>;
+    test_patch: z.ZodOptional<z.ZodString>;
+    created_at: z.ZodOptional<z.ZodString>;
+    image: z.ZodOptional<z.ZodString>;
+    toolkits: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+}, "strip", z.ZodTypeAny, {
+    name: string;
+    fixture: string;
+    prompt: string;
+    success: ({
+        run: string;
+        expect: {
+            exit_code: number;
+        };
+    } | {
+        assert: string;
+    })[];
+    timeout_seconds: number;
+    id?: string | undefined;
+    description?: string | undefined;
+    tags?: string[] | undefined;
+    test_command?: string | undefined;
+    fail_to_pass?: string[] | undefined;
+    pass_to_pass?: string[] | undefined;
+    forbid_modified?: string[] | undefined;
+    expected_files?: string[] | undefined;
+    solution?: string | undefined;
+    test_patch?: string | undefined;
+    created_at?: string | undefined;
+    image?: string | undefined;
+    toolkits?: string[] | undefined;
+}, {
+    name: string;
+    fixture: string;
+    prompt: string;
+    success: ({
+        run: string;
+        expect: {
+            exit_code?: number | undefined;
+        };
+    } | {
+        assert: string;
+    })[];
+    id?: string | undefined;
+    description?: string | undefined;
+    timeout_seconds?: number | undefined;
+    tags?: string[] | undefined;
+    test_command?: string | undefined;
+    fail_to_pass?: string[] | undefined;
+    pass_to_pass?: string[] | undefined;
+    forbid_modified?: string[] | undefined;
+    expected_files?: string[] | undefined;
+    solution?: string | undefined;
+    test_patch?: string | undefined;
+    created_at?: string | undefined;
+    image?: string | undefined;
+    toolkits?: string[] | undefined;
+}>;
+type TestCase = z.infer<typeof TestCaseSchema>;
+
+declare const AgentConfigSchema: z.ZodObject<{
+    id: z.ZodOptional<z.ZodString>;
+    name: z.ZodString;
+    model: z.ZodString;
+    max_steps: z.ZodDefault<z.ZodNumber>;
+    temperature: z.ZodOptional<z.ZodNumber>;
+    system_prompt: z.ZodOptional<z.ZodString>;
+    tools: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    toolkits: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    mcp_servers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodObject<{
+        command: z.ZodString;
+        args: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    }, "strip", z.ZodTypeAny, {
+        command: string;
+        args?: string[] | undefined;
+        env?: Record<string, string> | undefined;
+    }, {
+        command: string;
+        args?: string[] | undefined;
+        env?: Record<string, string> | undefined;
+    }>, z.ZodObject<{
+        type: z.ZodOptional<z.ZodEnum<["http", "sse"]>>;
+        url: z.ZodString;
+        headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    }, "strip", z.ZodTypeAny, {
+        url: string;
+        type?: "http" | "sse" | undefined;
+        headers?: Record<string, string> | undefined;
+    }, {
+        url: string;
+        type?: "http" | "sse" | undefined;
+        headers?: Record<string, string> | undefined;
+    }>]>>>;
+}, "strip", z.ZodTypeAny, {
+    name: string;
+    model: string;
+    max_steps: number;
+    id?: string | undefined;
+    toolkits?: string[] | undefined;
+    temperature?: number | undefined;
+    system_prompt?: string | undefined;
+    tools?: string[] | undefined;
+    mcp_servers?: Record<string, {
+        command: string;
+        args?: string[] | undefined;
+        env?: Record<string, string> | undefined;
+    } | {
+        url: string;
+        type?: "http" | "sse" | undefined;
+        headers?: Record<string, string> | undefined;
+    }> | undefined;
+}, {
+    name: string;
+    model: string;
+    id?: string | undefined;
+    toolkits?: string[] | undefined;
+    max_steps?: number | undefined;
+    temperature?: number | undefined;
+    system_prompt?: string | undefined;
+    tools?: string[] | undefined;
+    mcp_servers?: Record<string, {
+        command: string;
+        args?: string[] | undefined;
+        env?: Record<string, string> | undefined;
+    } | {
+        url: string;
+        type?: "http" | "sse" | undefined;
+        headers?: Record<string, string> | undefined;
+    }> | undefined;
+}>;
+type AgentConfig = z.infer<typeof AgentConfigSchema>;
+
+/**
+ * Frontmatter for a Claude Agent Skill (`SKILL.md`).
+ *
+ * This is intentionally a conservative subset of the published Agent Skills
+ * spec: `name` and `description` are the two fields that are solidly
+ * documented as always loaded into context for skill discovery (progressive
+ * disclosure - the rest of SKILL.md is only read on demand). `allowed-tools`,
+ * `disallowed-tools`, and `license` are reasonably well documented optional
+ * fields. Anything else is passed through unvalidated via `.passthrough()`
+ * so we don't reject SKILL.md files that use additional frontmatter we
+ * haven't verified.
+ */
+declare const SkillFrontmatterSchema: z.ZodObject<{
+    /** lowercase letters, numbers, hyphens; max 64 chars */
+    name: z.ZodString;
+    /** third-person description of what the skill does and when to use it; max 1024 chars */
+    description: z.ZodString;
+    "allowed-tools": z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    "disallowed-tools": z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    license: z.ZodOptional<z.ZodString>;
+}, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+    /** lowercase letters, numbers, hyphens; max 64 chars */
+    name: z.ZodString;
+    /** third-person description of what the skill does and when to use it; max 1024 chars */
+    description: z.ZodString;
+    "allowed-tools": z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    "disallowed-tools": z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    license: z.ZodOptional<z.ZodString>;
+}, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+    /** lowercase letters, numbers, hyphens; max 64 chars */
+    name: z.ZodString;
+    /** third-person description of what the skill does and when to use it; max 1024 chars */
+    description: z.ZodString;
+    "allowed-tools": z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    "disallowed-tools": z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    license: z.ZodOptional<z.ZodString>;
+}, z.ZodTypeAny, "passthrough">>;
+type SkillFrontmatter = z.infer<typeof SkillFrontmatterSchema>;
+/** A discovered skill: parsed frontmatter, markdown body, and its location on disk. */
+interface Skill {
+    frontmatter: SkillFrontmatter;
+    /** markdown body of SKILL.md, with the frontmatter block stripped */
+    body: string;
+    /** absolute path to the SKILL.md file */
+    path: string;
+    /** absolute path to the skill's directory (for resolving bundled scripts/resources) */
+    dir: string;
+}
+/**
+ * MCP server configuration, mirroring the `mcpServers` entries used by
+ * `.mcp.json` configs:
+ *
+ *  - stdio servers are launched as a local subprocess and spoken to over
+ *    stdin/stdout (`command` + optional `args`/`env`).
+ *  - http/sse servers are remote endpoints reached over HTTP(S) (`url` +
+ *    optional `headers`).
+ */
+declare const McpServerConfigSchema: z.ZodUnion<[z.ZodObject<{
+    command: z.ZodString;
+    args: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+}, "strip", z.ZodTypeAny, {
+    command: string;
+    args?: string[] | undefined;
+    env?: Record<string, string> | undefined;
+}, {
+    command: string;
+    args?: string[] | undefined;
+    env?: Record<string, string> | undefined;
+}>, z.ZodObject<{
+    type: z.ZodOptional<z.ZodEnum<["http", "sse"]>>;
+    url: z.ZodString;
+    headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+}, "strip", z.ZodTypeAny, {
+    url: string;
+    type?: "http" | "sse" | undefined;
+    headers?: Record<string, string> | undefined;
+}, {
+    url: string;
+    type?: "http" | "sse" | undefined;
+    headers?: Record<string, string> | undefined;
+}>]>;
+type McpServerConfig = z.infer<typeof McpServerConfigSchema>;
+
+declare const RunSchema: z.ZodObject<{
+    id: z.ZodString;
+    testCaseId: z.ZodString;
+    agentConfigId: z.ZodString;
+    sandboxProvider: z.ZodString;
+    status: z.ZodEnum<["running", "completed", "failed"]>;
+    passed: z.ZodOptional<z.ZodBoolean>;
+    score: z.ZodOptional<z.ZodNumber>;
+    stepsCount: z.ZodDefault<z.ZodNumber>;
+    tokensIn: z.ZodDefault<z.ZodNumber>;
+    tokensOut: z.ZodDefault<z.ZodNumber>;
+    costUsd: z.ZodDefault<z.ZodNumber>;
+    durationMs: z.ZodDefault<z.ZodNumber>;
+    error: z.ZodOptional<z.ZodString>;
+    finalDiff: z.ZodOptional<z.ZodString>;
+    metrics: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+    createdAt: z.ZodNumber;
+    completedAt: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    status: "running" | "completed" | "failed";
+    id: string;
+    testCaseId: string;
+    agentConfigId: string;
+    sandboxProvider: string;
+    stepsCount: number;
+    tokensIn: number;
+    tokensOut: number;
+    costUsd: number;
+    durationMs: number;
+    createdAt: number;
+    passed?: boolean | undefined;
+    score?: number | undefined;
+    error?: string | undefined;
+    finalDiff?: string | undefined;
+    metrics?: Record<string, any> | undefined;
+    completedAt?: number | undefined;
+}, {
+    status: "running" | "completed" | "failed";
+    id: string;
+    testCaseId: string;
+    agentConfigId: string;
+    sandboxProvider: string;
+    createdAt: number;
+    passed?: boolean | undefined;
+    score?: number | undefined;
+    stepsCount?: number | undefined;
+    tokensIn?: number | undefined;
+    tokensOut?: number | undefined;
+    costUsd?: number | undefined;
+    durationMs?: number | undefined;
+    error?: string | undefined;
+    finalDiff?: string | undefined;
+    metrics?: Record<string, any> | undefined;
+    completedAt?: number | undefined;
+}>;
+type Run = z.infer<typeof RunSchema>;
+
+declare const StepEventSchema: z.ZodObject<{
+    index: z.ZodNumber;
+    kind: z.ZodEnum<["tool_call", "tool_result", "message", "thinking"]>;
+    tool: z.ZodOptional<z.ZodString>;
+    tokensIn: z.ZodDefault<z.ZodNumber>;
+    tokensOut: z.ZodDefault<z.ZodNumber>;
+    costUsd: z.ZodDefault<z.ZodNumber>;
+    timestamp: z.ZodNumber;
+    content: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    tokensIn: number;
+    tokensOut: number;
+    costUsd: number;
+    index: number;
+    kind: "message" | "tool_call" | "tool_result" | "thinking";
+    timestamp: number;
+    tool?: string | undefined;
+    content?: string | undefined;
+}, {
+    index: number;
+    kind: "message" | "tool_call" | "tool_result" | "thinking";
+    timestamp: number;
+    tokensIn?: number | undefined;
+    tokensOut?: number | undefined;
+    costUsd?: number | undefined;
+    tool?: string | undefined;
+    content?: string | undefined;
+}>;
+type StepEvent = z.infer<typeof StepEventSchema>;
+declare const TraceSchema: z.ZodObject<{
+    runId: z.ZodString;
+    steps: z.ZodArray<z.ZodObject<{
+        index: z.ZodNumber;
+        kind: z.ZodEnum<["tool_call", "tool_result", "message", "thinking"]>;
+        tool: z.ZodOptional<z.ZodString>;
+        tokensIn: z.ZodDefault<z.ZodNumber>;
+        tokensOut: z.ZodDefault<z.ZodNumber>;
+        costUsd: z.ZodDefault<z.ZodNumber>;
+        timestamp: z.ZodNumber;
+        content: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        tokensIn: number;
+        tokensOut: number;
+        costUsd: number;
+        index: number;
+        kind: "message" | "tool_call" | "tool_result" | "thinking";
+        timestamp: number;
+        tool?: string | undefined;
+        content?: string | undefined;
+    }, {
+        index: number;
+        kind: "message" | "tool_call" | "tool_result" | "thinking";
+        timestamp: number;
+        tokensIn?: number | undefined;
+        tokensOut?: number | undefined;
+        costUsd?: number | undefined;
+        tool?: string | undefined;
+        content?: string | undefined;
+    }>, "many">;
+}, "strip", z.ZodTypeAny, {
+    runId: string;
+    steps: {
+        tokensIn: number;
+        tokensOut: number;
+        costUsd: number;
+        index: number;
+        kind: "message" | "tool_call" | "tool_result" | "thinking";
+        timestamp: number;
+        tool?: string | undefined;
+        content?: string | undefined;
+    }[];
+}, {
+    runId: string;
+    steps: {
+        index: number;
+        kind: "message" | "tool_call" | "tool_result" | "thinking";
+        timestamp: number;
+        tokensIn?: number | undefined;
+        tokensOut?: number | undefined;
+        costUsd?: number | undefined;
+        tool?: string | undefined;
+        content?: string | undefined;
+    }[];
+}>;
+type Trace = z.infer<typeof TraceSchema>;
+
+interface PatchApplyResult {
+    /** true if the patch was applied successfully (with or without repair) */
+    applied: boolean;
+    /** true if a fallback/repair strategy (3-way merge or `patch --fuzz`) was needed */
+    repaired: boolean;
+    /** combined output/diagnostics from the apply attempt(s) */
+    output: string;
+}
+interface SandboxHandle {
+    exec(cmd: string): Promise<{
+        stdout: string;
+        stderr: string;
+        exitCode: number;
+    }>;
+    writeFile(path: string, content: string): Promise<void>;
+    readFile(path: string): Promise<string>;
+    gitDiff(): Promise<string>;
+    /**
+     * Applies a unified diff to the sandbox's working tree.
+     *
+     * Mirrors SWE-bench's patch-application robustness: tries `git apply`
+     * first, then falls back to `git apply --3way`, then `patch --fuzz=3`.
+     * Implementations should report whether a fallback ("repair") was needed.
+     */
+    applyPatch(diff: string): Promise<PatchApplyResult>;
+    destroy(): Promise<void>;
+}
+interface SandboxProvider {
+    readonly name: string;
+    create(opts: {
+        image?: string;
+        gitSnapshot?: string;
+        /**
+         * Absolute paths to local "toolkit" directories to inject into the
+         * sandbox, in addition to `gitSnapshot`. A toolkit may contain a `bin/`
+         * directory (custom CLI tools, made executable and put on `PATH`) and a
+         * `.claude/skills/` directory (Agent Skills documentation, see
+         * `runner/skills.ts`).
+         */
+        toolkits?: string[];
+    }): Promise<SandboxHandle>;
+}
+
+interface AgentResult {
+    finished: boolean;
+    finalDiff: string;
+}
+interface AgentAdapter {
+    readonly name: string;
+    /**
+     * run the agent against a problem in a prepared sandbox.
+     * the adapter doesn't know about scoring — it just solves and emits
+     * step events. the framework measures everything from the outside.
+     */
+    solve(input: {
+        prompt: string;
+        sandbox: SandboxHandle;
+        config: AgentConfig;
+        onStep: (step: StepEvent) => void;
+    }): Promise<AgentResult>;
+}
+
+interface ScorerResult {
+    passed: boolean;
+    detail: string;
+    score?: number;
+}
+interface Scorer {
+    readonly name: string;
+    score(input: {
+        testCase: TestCase;
+        result: AgentResult;
+        trace: Trace;
+        sandbox: SandboxHandle;
+    }): Promise<ScorerResult>;
+}
+
+/**
+ * Parses raw test-runner output into a per-test status map, so scorers can
+ * compare individual test outcomes against FAIL_TO_PASS / PASS_TO_PASS lists
+ * (SWE-bench style).
+ */
+type TestStatus = "PASS" | "FAIL" | "SKIP";
+/** Maps a test's display name to its outcome. */
+type TestStatusMap = Record<string, TestStatus>;
+interface TestResultParser {
+    readonly name: string;
+    parse(output: string): TestStatusMap;
+}
+/**
+ * Parses TAP (Test Anything Protocol) output, as produced by
+ * `node --test --test-reporter=tap` (or `tsx --test --test-reporter=tap`).
+ *
+ * Handles lines of the form:
+ *   ok 1 - should succeed on first attempt
+ *   not ok 2 - should retry on failure and succeed
+ *   ok 3 - should be skipped # SKIP
+ */
+declare class TapTestResultParser implements TestResultParser {
+    readonly name = "tap";
+    parse(output: string): TestStatusMap;
+}
+
+declare class CommandScorer implements Scorer {
+    readonly name = "CommandScorer";
+    score(input: {
+        testCase: TestCase;
+        sandbox: SandboxHandle;
+    }): Promise<ScorerResult>;
+}
+
+declare class AssertionScorer implements Scorer {
+    readonly name = "AssertionScorer";
+    score(input: {
+        testCase: TestCase;
+        trace: Trace;
+    }): Promise<ScorerResult>;
+}
+
+/**
+ * SWE-bench style regression scorer.
+ *
+ * - FAIL_TO_PASS: tests that were failing before the agent's patch and MUST
+ *   pass afterwards. This is the actual "did the agent fix the bug" signal.
+ * - PASS_TO_PASS: tests that were passing before the agent's patch and MUST
+ *   keep passing afterwards (no regressions introduced).
+ * - forbid_modified: acts as a tamper guard - if the agent edited test files
+ *   (or other forbidden paths) to make the suite pass trivially, fail hard.
+ *
+ * If `test_command` / fail_to_pass / pass_to_pass are not configured on the
+ * test case, this scorer is a no-op pass (keeps it backwards compatible).
+ */
+declare class RegressionScorer implements Scorer {
+    readonly name = "RegressionScorer";
+    score(input: {
+        testCase: TestCase;
+        sandbox: SandboxHandle;
+        /** pre-patch test status map, used to avoid penalizing already-broken PASS_TO_PASS tests */
+        baseline?: TestStatusMap;
+    }): Promise<ScorerResult>;
+}
+
+interface DiffStats {
+    filesChanged: string[];
+    insertions: number;
+    deletions: number;
+    linesChanged: number;
+}
+/** Parses a unified diff (as produced by `git diff`) into basic stats. */
+declare function parseDiffStats(diff: string): DiffStats;
+/**
+ * Scores the "scope" of an agent's patch.
+ *
+ * SWE-bench analyses showed agents frequently produce patches that are far
+ * larger / touch far more files than the gold patch - editing unrelated
+ * code, leaving debug statements, etc. This scorer reports diff stats and,
+ * when a gold `solution` patch is available (loaded as raw diff content by
+ * the CLI's `loadTestCase` helper), penalizes patches that are much larger
+ * than the gold patch in terms of total changed lines.
+ */
+declare class DiffScorer implements Scorer {
+    readonly name = "DiffScorer";
+    score(input: {
+        testCase: TestCase;
+        result: AgentResult;
+    }): Promise<ScorerResult>;
+}
+
+/**
+ * Measures whether the agent edited the "right" files, using
+ * `expected_files` glob patterns (typically derived from the gold patch).
+ *
+ * - precision: fraction of files the agent touched that match an expected pattern
+ * - recall: fraction of expected patterns that were matched by at least one touched file
+ * - f1: harmonic mean of precision and recall
+ */
+declare class LocalizationScorer implements Scorer {
+    readonly name = "LocalizationScorer";
+    score(input: {
+        testCase: TestCase;
+        result: AgentResult;
+    }): Promise<ScorerResult>;
+}
+
+interface RunSingleInput {
+    testCase: TestCase;
+    agentConfig: AgentConfig;
+    adapter: AgentAdapter;
+    sandboxProvider: SandboxProvider;
+    db?: AgrDb;
+    runId: string;
+}
+interface RunSingleResult {
+    runId: string;
+    passed: boolean;
+    score?: number;
+    stepsCount: number;
+    tokensIn: number;
+    tokensOut: number;
+    costUsd: number;
+    durationMs: number;
+    error?: string;
+    finalDiff?: string;
+    metrics?: Record<string, any>;
+}
+declare function runSingle(input: RunSingleInput): Promise<RunSingleResult>;
+
+interface BenchmarkInput {
+    testCases: TestCase[];
+    agentConfigs: AgentConfig[];
+    adapter: AgentAdapter;
+    sandboxProvider: SandboxProvider;
+    db?: AgrDb;
+    concurrency?: number;
+    onRunUpdate?: (run: RunSingleResult & {
+        testCaseId: string;
+        agentConfigId: string;
+        status: "running" | "completed" | "failed";
+    }) => void;
+}
+interface BenchmarkResult {
+    runs: RunSingleResult[];
+}
+declare function runBenchmark(input: BenchmarkInput): Promise<BenchmarkResult>;
+
+/**
+ * Minimal glob matcher (no external dependency) supporting:
+ *   - `*`  matches any sequence of characters except `/`
+ *   - `**` matches any sequence of characters, including `/`
+ *   - `?`  matches a single character except `/`
+ *
+ * Used for `forbid_modified` / `expected_files` patterns in test cases.
+ */
+declare function matchGlob(pattern: string, filePath: string): boolean;
+declare function matchAnyGlob(patterns: string[], filePath: string): boolean;
+
+/**
+ * Computes a stable SHA-256 hash over the contents (and relative paths) of a
+ * fixture directory. Used to key cached baseline test results - if the
+ * fixture changes, the baseline must be recomputed.
+ */
+declare function hashFixture(fixtureDir: string): string;
+
+interface BaselineResult {
+    fixtureHash: string;
+    statusMap: TestStatusMap;
+    /** true if this baseline came from the cache rather than a fresh run */
+    cached: boolean;
+}
+/**
+ * Computes (or loads from cache) the pre-patch test status map for a test
+ * case's fixture. Used by RegressionScorer so PASS_TO_PASS tests that were
+ * already broken before the agent touched anything don't unfairly penalize
+ * the run.
+ *
+ * Returns `undefined` if the test case has no `test_command` configured -
+ * baseline computation is then skipped entirely.
+ */
+declare function getOrComputeBaseline(input: {
+    testCase: TestCase;
+    sandboxProvider: SandboxProvider;
+    db?: AgrDb;
+}): Promise<BaselineResult | undefined>;
+
+interface ValidationCheck {
+    name: string;
+    passed: boolean;
+    detail: string;
+}
+interface ValidationReport {
+    ok: boolean;
+    checks: ValidationCheck[];
+}
+/**
+ * validates a test case the way SWE-bench validates a candidate task
+ * instance before it's added to the benchmark:
+ *
+ * 1. static check required fields are present and internally consistent.
+ * 2. pre-patch run - FAIL_TO_PASS tests must currently be FAILING and
+ *    PASS_TO_PASS tests must currently be PASSING (on the raw fixture, with
+ *    `test_patch` applied if present).
+ * 3. post-patch run - if a gold `solution` patch is provided, apply it and
+ *    verify FAIL_TO_PASS tests now PASS and PASS_TO_PASS tests still PASS.
+ *
+ * this catches the most common authoring mistakes: typo'd test names,
+ * tests that pass/fail for the wrong reason, gold patches that don't
+ * actually fix the issue, and forbidden-file globs that never match.
+ */
+declare function validateTestCase(input: {
+    testCase: TestCase;
+    sandboxProvider: SandboxProvider;
+}): Promise<ValidationReport>;
+
+/**
+ * Parses a SKILL.md file's content into validated frontmatter + body.
+ *
+ * Throws if the file has no `---`-delimited YAML frontmatter block, or if
+ * the frontmatter doesn't satisfy `SkillFrontmatterSchema` (e.g. missing
+ * `name`/`description`).
+ */
+declare function parseSkillMarkdown(content: string, path: string, dir: string): Skill;
+/**
+ * Discovers all skills bundled in a toolkit directory, i.e. every
+ * `<toolkitDir>/.claude/skills/<skill-name>/SKILL.md`.
+ *
+ * Returns an empty array if the toolkit has no `.claude/skills` directory.
+ */
+declare function discoverSkills(toolkitDir: string): Skill[];
+/** Discovers skills across multiple toolkit directories, in order. */
+declare function discoverSkillsForToolkits(toolkitDirs: string[]): Skill[];
+/**
+ * Builds a system-prompt addendum that tells the agent which skills are
+ * available, mirroring the "progressive disclosure" model used by Claude
+ * Agent Skills: only the skill's `name` + `description` are injected up
+ * front, and the full SKILL.md body is read on demand (via the agent's
+ * `readFile` tool) once the agent decides a skill is relevant.
+ *
+ * Assumes toolkits are injected into the sandbox at `/app`, so a skill at
+ * `<toolkitDir>/.claude/skills/<name>/SKILL.md` is readable at
+ * `/app/.claude/skills/<name>/SKILL.md` (see DockerSandboxProvider).
+ *
+ * Returns an empty string if there are no skills (so callers can append it
+ * unconditionally without producing an empty trailing section).
+ */
+declare function buildSkillsPromptAddendum(skills: Skill[]): string;
+
+export { type AgentAdapter, type AgentConfig, AgentConfigSchema, type AgentResult, AssertionScorer, type BaselineResult, type BenchmarkInput, type BenchmarkResult, CommandScorer, DiffScorer, type DiffStats, LocalizationScorer, type McpServerConfig, McpServerConfigSchema, type PatchApplyResult, RegressionScorer, type Run, RunSchema, type RunSingleInput, type RunSingleResult, type SandboxHandle, type SandboxProvider, type Scorer, type ScorerResult, type Skill, type SkillFrontmatter, SkillFrontmatterSchema, type StepEvent, StepEventSchema, type SuccessCriterion, SuccessCriterionSchema, TapTestResultParser, type TestCase, TestCaseSchema, type TestResultParser, type TestStatus, type TestStatusMap, type Trace, TraceSchema, type ValidationCheck, type ValidationReport, buildSkillsPromptAddendum, discoverSkills, discoverSkillsForToolkits, getOrComputeBaseline, hashFixture, matchAnyGlob, matchGlob, parseDiffStats, parseSkillMarkdown, runBenchmark, runSingle, validateTestCase };