@tangle-network/agent-eval 0.23.1 → 0.25.0

package/dist/test-graded-scenario-B2kWEdh9.d.ts

@@ -0,0 +1,146 @@
+import { T as TraceEmitter } from './emitter-DP_cSSiw.js';
+import { R as Run, F as FailureClass, T as TraceStore } from './store-Db2Bv8Cf.js';
+
+/**
+ * SandboxHarness — executes a scenario in an isolated environment and
+ * emits a rich SandboxSpan into the trace.
+ *
+ * Two built-in drivers:
+ *   - `SubprocessSandboxDriver` — spawn in a local cwd with env vars.
+ *     Fast, no dependencies, fine for unit tests and most CI gates.
+ *   - `DockerSandboxDriver` — lifted from tangle-router's sandbox path;
+ *     shells out to `docker run`. Stronger isolation, slower startup.
+ *
+ * Consumers implement `SandboxDriver` for custom backends (Firecracker,
+ * Cloudflare sandbox product, etc.). The harness doesn't care which.
+ */
+
+interface HarnessConfig {
+    /** Setup command (e.g. "pnpm install"). Non-zero exit fails the run. */
+    setupCommand?: string;
+    /** Run command (e.g. "pnpm build"). */
+    runCommand?: string;
+    /** Test command (e.g. "pnpm test --run"). Drives the test count + pass count. */
+    testCommand?: string;
+    /** Absolute cwd for the subprocess driver. Ignored by docker driver. */
+    cwd?: string;
+    /** Max wall-clock per phase in ms. Default 10 minutes. */
+    timeoutMs?: number;
+    /** Image for the docker driver. */
+    image?: string;
+    /** Extra env vars (validated; shell-escaped). */
+    env?: Record<string, string>;
+    /** Parser for the test output — maps stdout/stderr/exit code → pass count. */
+    testParser?: TestOutputParser;
+}
+interface TestOutputParser {
+    id: string;
+    parse(stdout: string, stderr: string, exitCode: number): {
+        testsTotal: number;
+        testsPassed: number;
+    } | undefined;
+}
+interface SandboxResult {
+    phase: 'setup' | 'run' | 'test';
+    exitCode: number;
+    stdout: string;
+    stderr: string;
+    wallMs: number;
+    testsTotal?: number;
+    testsPassed?: number;
+}
+interface SandboxDriver {
+    id: string;
+    exec(phase: SandboxResult['phase'], command: string, config: HarnessConfig): Promise<SandboxResult>;
+}
+/** Vitest default summary line: "Tests  X passed | Y failed". */
+declare const vitestTestParser: TestOutputParser;
+/** Pytest default: "collected N items" + " X passed, Y failed". */
+declare const pytestTestParser: TestOutputParser;
+/** Jest: "Tests: X passed, Y total" (and optional failed). */
+declare const jestTestParser: TestOutputParser;
+/** Composite parser — tries a list of parsers in order. */
+declare function composeParsers(...parsers: TestOutputParser[]): TestOutputParser;
+interface SubprocessSandboxDriverOptions {
+    /**
+     * Default cwd for all `exec` calls. Used when the per-call `HarnessConfig`
+     * does not set its own `cwd`. Lets callers bind the driver to a working
+     * directory once instead of spreading cwd into every harness config —
+     * useful when the harness config is constructed far from the call site
+     * (e.g. starter-foundry's promoter passes a static HarnessConfig per
+     * family taxonomy but needs a per-run composed-scaffold cwd).
+     */
+    cwd?: string;
+    /**
+     * Default env merged into every `exec` call's env (per-call `HarnessConfig.env`
+     * still wins on key collision). Same ergonomic rationale as `cwd` above.
+     */
+    env?: Record<string, string>;
+}
+declare class SubprocessSandboxDriver implements SandboxDriver {
+    id: string;
+    private defaultCwd?;
+    private defaultEnv?;
+    constructor(options?: SubprocessSandboxDriverOptions);
+    exec(phase: SandboxResult['phase'], command: string, config: HarnessConfig): Promise<SandboxResult>;
+}
+declare class DockerSandboxDriver implements SandboxDriver {
+    id: string;
+    exec(phase: SandboxResult['phase'], command: string, config: HarnessConfig): Promise<SandboxResult>;
+}
+interface SandboxHarnessResult {
+    passed: boolean;
+    setup?: SandboxResult;
+    run?: SandboxResult;
+    test?: SandboxResult;
+    totalWallMs: number;
+    /** Final score — 0 when no tests; otherwise testsPassed/testsTotal. */
+    score: number;
+}
+declare class SandboxHarness {
+    private driver;
+    constructor(driver?: SandboxDriver);
+    run(config: HarnessConfig, emitter: TraceEmitter): Promise<SandboxHarnessResult>;
+}
+
+/**
+ * TestGradedScenario — a scenario whose score comes from a test suite.
+ *
+ * This is the SWE-bench pattern generalized. The scenario ships:
+ *   - fixture data (setup instructions)
+ *   - a test command the harness runs
+ *   - optional assertion overrides
+ *
+ * The runner emits a run, delegates to SandboxHarness, records the
+ * outcome, and returns a structured verdict. Consumers bind their own
+ * agent execution to this contract.
+ */
+
+interface TestGradedScenario {
+    id: string;
+    description?: string;
+    harness: HarnessConfig;
+    /** Optional pass threshold in 0..1 (default 1.0 = all tests must pass). */
+    passThreshold?: number;
+    /** Provenance for dataset tracking. */
+    datasetVersion?: string;
+    /** Free-form tags (difficulty, category, etc.). */
+    tags?: Record<string, string>;
+}
+interface TestGradedRunOptions {
+    variantId?: string;
+    driver?: SandboxDriver;
+    /** Metadata recorded on the Run (codeSha, promptSha, modelFingerprint, seed). */
+    provenance?: Pick<Run, 'codeSha' | 'promptSha' | 'modelFingerprint' | 'seed' | 'envFingerprint'>;
+}
+interface TestGradedRunResult {
+    runId: string;
+    scenario: TestGradedScenario;
+    harness: SandboxHarnessResult;
+    pass: boolean;
+    score: number;
+    failureClass?: FailureClass;
+}
+declare function runTestGradedScenario(scenario: TestGradedScenario, store: TraceStore, options?: TestGradedRunOptions): Promise<TestGradedRunResult>;
+
+export { DockerSandboxDriver as D, type HarnessConfig as H, type SandboxDriver as S, type TestGradedScenario as T, type SandboxHarnessResult as a, type TestGradedRunResult as b, SandboxHarness as c, type SandboxResult as d, SubprocessSandboxDriver as e, type SubprocessSandboxDriverOptions as f, type TestGradedRunOptions as g, type TestOutputParser as h, composeParsers as i, jestTestParser as j, pytestTestParser as p, runTestGradedScenario as r, vitestTestParser as v };