@attest-ai/vitest 0.9.0 → 1.0.0-rc.2

package/dist/reporter.d.ts

@@ -1,3 +1,4 @@
+import type { Reporter, TestModule, Vitest } from "vitest/node";
 interface ReporterContext {
     printInfo: (message: string) => void;
 }
@@ -12,5 +13,41 @@ interface ReporterContext {
 export declare class AttestCostReporter {
     onFinished(_files: unknown, _errors: unknown, context?: ReporterContext): void;
 }
-export {};
+export interface AttestReporterOptions {
+    /**
+     * Path of the report-v2 JSON file, relative to the Vitest root. Overrides
+     * `--outputFile.attest=<path>`.
+     */
+    outputFile?: string;
+}
+/**
+ * Vitest reporter that writes a report-v2 file (proto/attest/v1/report-v2.schema.json)
+ * when the run ends: every test with its outcome, file and line, and every Attest
+ * assertion result tagged with the test that requested it. Feed the file to
+ * `attest policy evaluate --report` and the pr-annotate action.
+ *
+ * Writes only when an output file is set: `new AttestReporter({ outputFile })` in the
+ * Vitest config, or on the command line, the counterpart of pytest's
+ * `--attest-report=<path>`:
+ * `vitest run --reporter=default --reporter=@attest-ai/vitest/reporter --outputFile.attest=<path>`. Turns on `includeTaskLocation` so each test carries its
+ * line. Requires Vitest 3+.
+ *
+ * At the end of the run it prints, with or without an output file, one line per
+ * `trials()` test ("<test_id>: <passed>/<evaluated> trials passed (<n> not evaluated)")
+ * and the "attest usage: …" line when the run made or refused an attest model call.
+ */
+export declare class AttestReporter implements Reporter {
+    private outputFile;
+    private root;
+    private vitest;
+    private usageDir;
+    constructor(options?: AttestReporterOptions);
+    onInit(vitest: Vitest): void;
+    onTestRunStart(): void;
+    /** Sum and remove the usage files workers wrote for evaluations outside any test. */
+    private takeUnattachedUsage;
+    onTestRunEnd(testModules: ReadonlyArray<TestModule>): void;
+}
+/** `vitest run --reporter=@attest-ai/vitest/reporter` loads the default export. */
+export default AttestReporter;
 //# sourceMappingURL=reporter.d.ts.map

package/dist/reporter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../src/reporter.ts"],"names":[],"mappings":"AAEA,UAAU,eAAe;IACvB,SAAS,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACtC;AAED;;;;;;;GAOG;AACH,qBAAa,kBAAkB;IAC7B,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,IAAI;CAwC/E"}
+{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../src/reporter.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAIhE,UAAU,eAAe;IACvB,SAAS,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACtC;AAED;;;;;;;GAOG;AACH,qBAAa,kBAAkB;IAC7B,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,IAAI;CAwC/E;AAED,MAAM,WAAW,qBAAqB;IACpC;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,cAAe,YAAW,QAAQ;IAC7C,OAAO,CAAC,UAAU,CAAqB;IACvC,OAAO,CAAC,IAAI,CAAiB;IAC7B,OAAO,CAAC,MAAM,CAAqB;IACnC,OAAO,CAAC,QAAQ,CAAqB;gBAEzB,OAAO,GAAE,qBAA0B;IAI/C,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAe5B,cAAc,IAAI,IAAI;IAItB,qFAAqF;IACrF,OAAO,CAAC,mBAAmB;IAS3B,YAAY,CAAC,WAAW,EAAE,aAAa,CAAC,UAAU,CAAC,GAAG,IAAI;CAmC3D;AAED,mFAAmF;AACnF,eAAe,cAAc,CAAC"}

package/dist/reporter.js

@@ -1,4 +1,9 @@
-import { renderDiagnostics, renderSummary } from "@attest-ai/core";
+import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, relative, resolve, sep } from "node:path";
+import { isSimulationMode, parseUsage, renderDiagnostics, renderSummary, sumUsage, } from "@attest-ai/core";
+import { buildReport, summaryLines } from "./report.js";
+import { ROOT_ENV, USAGE_DIR_ENV } from "./test-state.js";
 /**
  * Vitest reporter that prints the cost report and a hierarchical
  * diagnostic block for every failed Attest evaluation in the run.
@@ -30,4 +35,94 @@ export class AttestCostReporter {
         }
     }
 }
+/**
+ * Vitest reporter that writes a report-v2 file (proto/attest/v1/report-v2.schema.json)
+ * when the run ends: every test with its outcome, file and line, and every Attest
+ * assertion result tagged with the test that requested it. Feed the file to
+ * `attest policy evaluate --report` and the pr-annotate action.
+ *
+ * Writes only when an output file is set: `new AttestReporter({ outputFile })` in the
+ * Vitest config, or on the command line, the counterpart of pytest's
+ * `--attest-report=<path>`:
+ * `vitest run --reporter=default --reporter=@attest-ai/vitest/reporter --outputFile.attest=<path>`. Turns on `includeTaskLocation` so each test carries its
+ * line. Requires Vitest 3+.
+ *
+ * At the end of the run it prints, with or without an output file, one line per
+ * `trials()` test ("<test_id>: <passed>/<evaluated> trials passed (<n> not evaluated)")
+ * and the "attest usage: …" line when the run made or refused an attest model call.
+ */
+export class AttestReporter {
+    outputFile;
+    root = process.cwd();
+    vitest;
+    usageDir;
+    constructor(options = {}) {
+        this.outputFile = options.outputFile;
+    }
+    onInit(vitest) {
+        this.vitest = vitest;
+        this.root = vitest.config.root;
+        const configured = vitest.config.outputFile;
+        if (this.outputFile === undefined && typeof configured === "object") {
+            this.outputFile = configured["attest"];
+        }
+        vitest.config.includeTaskLocation = true;
+        // Workers inherit these: test ids hash paths relative to this root, and
+        // evaluations outside any test drop their usage in usageDir.
+        this.usageDir = mkdtempSync(join(tmpdir(), "attest-usage-"));
+        process.env[ROOT_ENV] = this.root;
+        process.env[USAGE_DIR_ENV] = this.usageDir;
+    }
+    onTestRunStart() {
+        if (this.usageDir !== undefined)
+            mkdirSync(this.usageDir, { recursive: true });
+    }
+    /** Sum and remove the usage files workers wrote for evaluations outside any test. */
+    takeUnattachedUsage() {
+        const dir = this.usageDir;
+        if (dir === undefined)
+            return sumUsage([]);
+        const files = readdirSync(dir).filter((f) => f.endsWith(".json"));
+        const usage = sumUsage(files.map((f) => parseUsage(JSON.parse(readFileSync(join(dir, f), "utf8")))));
+        rmSync(dir, { recursive: true, force: true });
+        return usage;
+    }
+    onTestRunEnd(testModules) {
+        const outputFile = this.outputFile;
+        const tests = [];
+        for (const testModule of testModules) {
+            const file = relative(this.root, testModule.moduleId).split(sep).join("/");
+            for (const testCase of testModule.children.allTests()) {
+                const location = testCase.location;
+                // The location matters only for the report file; summary lines need none.
+                if (location === undefined && outputFile !== undefined) {
+                    throw new Error(`AttestReporter: no source location for "${testCase.fullName}". Vitest records ` +
+                        "locations only when test.includeTaskLocation is true and the test file path matches " +
+                        "the root; set test.root to the real path (no symlinks) if it is set.");
+                }
+                const state = testCase.result().state;
+                tests.push({
+                    testId: `${file} > ${testCase.fullName}`,
+                    name: testCase.name,
+                    file,
+                    line: location?.line ?? 0,
+                    outcome: state === "passed" || state === "failed" ? state : "skipped",
+                    durationMs: testCase.diagnostic()?.duration ?? 0,
+                    attest: testCase.meta().attest,
+                });
+            }
+        }
+        const unattached = this.takeUnattachedUsage();
+        const lines = summaryLines(tests, unattached);
+        if (lines.length > 0)
+            this.vitest?.logger.log(lines.join("\n"));
+        if (outputFile === undefined)
+            return;
+        const path = resolve(this.root, outputFile);
+        mkdirSync(dirname(path), { recursive: true });
+        writeFileSync(path, `${JSON.stringify(buildReport(tests, isSimulationMode(), unattached), null, 2)}\n`);
+    }
+}
+/** `vitest run --reporter=@attest-ai/vitest/reporter` loads the default export. */
+export default AttestReporter;
 //# sourceMappingURL=reporter.js.map

package/dist/reporter.js.map

@@ -1 +1 @@
-{"version":3,"file":"reporter.js","sourceRoot":"","sources":["../src/reporter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAMnE;;;;;;;GAOG;AACH,MAAM,OAAO,kBAAkB;IAC7B,UAAU,CAAC,MAAe,EAAE,OAAgB,EAAE,OAAyB;QACrE,MAAM,IAAI,GAAG,UAAU,CAAC,uBAAuB,IAAI,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,UAAU,CAAC,gCAAgC,IAAI,CAAC,CAAC;QACtE,MAAM,QAAQ,GAAG,UAAU,CAAC,2BAA2B,IAAI,EAAE,CAAC;QAE9D,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CACR,EAAE,EACF,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EACd,sBAAsB,EACtB,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CACf,CAAC;YACF,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;gBAC9B,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC;gBACtC,KAAK,CAAC,IAAI,CAAC,cAAc,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;QAED,KAAK,CAAC,IAAI,CACR,EAAE,EACF,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EACd,sBAAsB,EACtB,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EACd,mCAAmC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,EACxD,mCAAmC,YAAY,EAAE,EACjD,mCAAmC,QAAQ,CAAC,MAAM,EAAE,EACpD,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EACd,EAAE,CACH,CAAC;QAEF,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEhC,IAAI,OAAO,EAAE,SAAS,EAAE,CAAC;YACvB,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"reporter.js","sourceRoot":"","sources":["../src/reporter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnG,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAClE,OAAO,EACL,gBAAgB,EAChB,UAAU,EACV,iBAAiB,EACjB,aAAa,EACb,QAAQ,GAET,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,WAAW,EAAE,YAAY,EAAmB,MAAM,aAAa,CAAC;AACzE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAM1D;;;;;;;GAOG;AACH,MAAM,OAAO,kBAAkB;IAC7B,UAAU,CAAC,MAAe,EAAE,OAAgB,EAAE,OAAyB;QACrE,MAAM,IAAI,GAAG,UAAU,CAAC,uBAAuB,IAAI,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,UAAU,CAAC,gCAAgC,IAAI,CAAC,CAAC;QACtE,MAAM,QAAQ,GAAG,UAAU,CAAC,2BAA2B,IAAI,EAAE,CAAC;QAE9D,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CACR,EAAE,EACF,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EACd,sBAAsB,EACtB,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CACf,CAAC;YACF,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;gBAC9B,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC;gBACtC,KAAK,CAAC,IAAI,CAAC,cAAc,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;QAED,KAAK,CAAC,IAAI,CACR,EAAE,EACF,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EACd,sBAAsB,EACtB,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EACd,mCAAmC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,EACxD,mCAAmC,YAAY,EAAE,EACjD,mCAAmC,QAAQ,CAAC,MAAM,EAAE,EACpD,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EACd,EAAE,CACH,CAAC;QAEF,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEhC,IAAI,OAAO,EAAE,SAAS,EAAE,CAAC;YACvB,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;CACF;AAUD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,OAAO,cAAc;IACjB,UAAU,CAAqB;IAC/B,IAAI,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IACrB,MAAM,CAAqB;IAC3B,QAAQ,CAAqB;IAErC,YAAY,UAAiC,EAAE;QAC7C,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;IACvC,CAAC;IAED,MAAM,CAAC,MAAc;QACnB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;QAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC;QAC5C,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;YACpE,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACzC,CAAC;QACD,MAAM,CAAC,MAAM,CAAC,mBAAmB,GAAG,IAAI,CAAC;QACzC,wEAAwE;QACxE,6DAA6D;QAC7D,IAAI,CAAC,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,eAAe,CAAC,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC;IAC7C,CAAC;IAED,cAAc;QACZ,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS;YAAE,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACjF,CAAC;IAED,qFAAqF;IAC7E,mBAAmB;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC1B,IAAI,GAAG,KAAK,SAAS;YAAE,OAAO,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAClE,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrG,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,OAAO,KAAK,CAAC;IACf,CAAC;IAED,YAAY,CAAC,WAAsC;QACjD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACnC,MAAM,KAAK,GAAiB,EAAE,CAAC;QAC/B,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC3E,KAAK,MAAM,QAAQ,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC;gBACtD,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;gBACnC,0EAA0E;gBAC1E,IAAI,QAAQ,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;oBACvD,MAAM,IAAI,KAAK,CACb,2CAA2C,QAAQ,CAAC,QAAQ,oBAAoB;wBAC9E,sFAAsF;wBACtF,sEAAsE,CACzE,CAAC;gBACJ,CAAC;gBACD,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC;gBACtC,KAAK,CAAC,IAAI,CAAC;oBACT,MAAM,EAAE,GAAG,IAAI,MAAM,QAAQ,CAAC,QAAQ,EAAE;oBACxC,IAAI,EAAE,QAAQ,CAAC,IAAI;oBACnB,IAAI;oBACJ,IAAI,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC;oBACzB,OAAO,EAAE,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;oBACrE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,EAAE,QAAQ,IAAI,CAAC;oBAChD,MAAM,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM;iBAC/B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,MAAM,UAAU,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAC9C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAChE,IAAI,UAAU,KAAK,SAAS;YAAE,OAAO;QACrC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAC5C,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,aAAa,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,KAAK,EAAE,gBAAgB,EAAE,EAAE,UAAU,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IAC1G,CAAC;CACF;AAED,mFAAmF;AACnF,eAAe,cAAc,CAAC"}

package/dist/test-state.d.ts

@@ -0,0 +1,100 @@
+/**
+ * Per-test state shared by evaluate/inspect and trials: deterministic assertion
+ * IDs, the per-test budget remainder, and the results recorded on task meta.
+ */
+import { AsyncLocalStorage } from "node:async_hooks";
+import type { TestContext } from "vitest";
+import { type AgentResult, type Assertion, type BatchBudget, type Trace, type Usage } from "@attest-ai/core";
+import type { AttestTaskMeta } from "./report.js";
+export type RunningTest = NonNullable<TestContext["task"]>;
+/** Limits for the attest model calls of one test (all its evaluations and trials). */
+export interface TestBudget {
+    /** Judge-provider requests the test may make. */
+    maxJudgeCalls?: number;
+    /** USD the test may spend: attest judge/embedding cost plus agent spend (trace `metadata.cost_usd`). */
+    maxCostUsd?: number;
+}
+/** The trials() call a body runs in; `trial` is undefined for a single run (N = 1). */
+export interface TrialScope {
+    test: RunningTest;
+    trial: number | undefined;
+    budget: TestBudget | undefined;
+    /** Evaluations of the current trial. */
+    results: AgentResult[];
+}
+export declare const trialScope: AsyncLocalStorage<TrialScope>;
+/** True while the code runs inside trial i of a trials() call with N > 1. */
+export declare function inTrial(): boolean;
+interface AttemptState {
+    attempt: number;
+    /** Assertions given an ID in this test (this trial, under trials()). */
+    ordinal: number;
+    /** Attest usage of this attempt so far. */
+    usage: Usage;
+    /** Sum of `metadata.cost_usd` of the traces evaluated in this attempt. */
+    agentSpendUsd: number;
+    traceIds: Set<string>;
+}
+/** State of the running attempt of `test`; a retried test starts afresh. */
+export declare function attemptState(test: RunningTest): AttemptState;
+/**
+ * The running test: `TestRunner.getCurrentTest()` on Vitest 4.1+, `vitest/suite`
+ * before that (loaded only there, because Vitest 4.1 warns when it is imported).
+ */
+export declare function runningTest(): Promise<RunningTest | undefined>;
+/**
+ * The test an evaluation belongs to: the passed context, else the enclosing
+ * trials() call, else Vitest's running test. Under test.concurrent the running
+ * test may be another test, so outside trials() the caller must pass `context`.
+ */
+export declare function resolveTest(context: TestContext | undefined): Promise<RunningTest | undefined>;
+/**
+ * Set by AttestReporter in the main process before workers start: the Vitest
+ * root its report paths are relative to, and the directory where workers drop
+ * the usage of evaluations that belong to no test.
+ */
+export declare const ROOT_ENV = "ATTEST_VITEST_ROOT";
+export declare const USAGE_DIR_ENV = "ATTEST_VITEST_USAGE_DIR";
+/**
+ * The report's test_id of `test`: "<file> > <suites> > <name>", the same string
+ * AttestReporter writes. The file is relative to the Vitest root AttestReporter
+ * publishes (in a multi-project setup, not the project root); without
+ * AttestReporter it is the file task's name, relative to the project root.
+ */
+export declare function testIdOf(test: RunningTest): string;
+/**
+ * Hand the usage of an evaluation that belongs to no test (beforeAll, a script)
+ * to AttestReporter: one file per evaluation in its usage directory, so the
+ * session usage counts it once. Without AttestReporter there is no report to add it to.
+ */
+export declare function recordUnattachedUsage(usage: Usage): void;
+/** `assert_` + the first 12 hex chars of sha256("<test_id>#<ordinal>"). */
+export declare function deterministicAssertionId(testId: string, ordinal: number): string;
+/** Give each assertion the next deterministic ID of the test. */
+export declare function assignAssertionIds(test: RunningTest, assertions: readonly Assertion[]): Assertion[];
+/** Reject a budget with unknown keys or negative/non-numeric limits. */
+export declare function validateBudget(budget: TestBudget): TestBudget;
+/**
+ * What is left of the test budget for the next evaluate_batch: judge calls not
+ * yet used, and USD not yet spent by attest or the agent, each floored at 0.
+ * Undefined when the budget sets no limit.
+ */
+export declare function remainingBudget(budget: TestBudget, usage: Usage, agentSpendUsd: number): BatchBudget | undefined;
+/** True when attest cost plus agent spend has reached the test's cost limit. */
+export declare function costBudgetSpent(budget: TestBudget | undefined, state: AttemptState): boolean;
+/** Count the agent's own spend of a trace once per attempt. */
+export declare function recordAgentSpend(test: RunningTest, trace: Trace): void;
+/**
+ * The meta of the running attempt. Results from an earlier attempt of a retried
+ * test are dropped, so the report matches the outcome Vitest reports; their
+ * usage is kept, because those model calls were made.
+ */
+export declare function attemptMeta(test: RunningTest): AttestTaskMeta;
+/**
+ * Attach the results to the test's meta so AttestReporter, in the main process,
+ * can write them with the test's file and line; rows of trial i carry `trial`.
+ */
+export declare function recordOnTest(test: RunningTest, result: AgentResult, trial: number | undefined): void;
+export declare function strictFromEnv(): boolean;
+export {};
+//# sourceMappingURL=test-state.d.ts.map

package/dist/test-state.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-state.d.ts","sourceRoot":"","sources":["../src/test-state.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAKrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AAC1C,OAAO,EAGL,KAAK,WAAW,EAChB,KAAK,SAAS,EACd,KAAK,WAAW,EAChB,KAAK,KAAK,EACV,KAAK,KAAK,EACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,MAAM,MAAM,WAAW,GAAG,WAAW,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;AAE3D,sFAAsF;AACtF,MAAM,WAAW,UAAU;IACzB,iDAAiD;IACjD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,wGAAwG;IACxG,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,uFAAuF;AACvF,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,WAAW,CAAC;IAClB,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,MAAM,EAAE,UAAU,GAAG,SAAS,CAAC;IAC/B,wCAAwC;IACxC,OAAO,EAAE,WAAW,EAAE,CAAC;CACxB;AAED,eAAO,MAAM,UAAU,+BAAsC,CAAC;AAE9D,6EAA6E;AAC7E,wBAAgB,OAAO,IAAI,OAAO,CAEjC;AAED,UAAU,YAAY;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,wEAAwE;IACxE,OAAO,EAAE,MAAM,CAAC;IAChB,2CAA2C;IAC3C,KAAK,EAAE,KAAK,CAAC;IACb,0EAA0E;IAC1E,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;CACvB;AAQD,4EAA4E;AAC5E,wBAAgB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,YAAY,CAQ5D;AAED;;;GAGG;AACH,wBAAsB,WAAW,IAAI,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAKpE;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,SAAS,GAAG,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAYpG;AAED;;;;GAIG;AACH,eAAO,MAAM,QAAQ,uBAAuB,CAAC;AAC7C,eAAO,MAAM,aAAa,4BAA4B,CAAC;AAEvD;;;;;GAKG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,WAAW,GAAG,MAAM,CAWlD;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAMxD;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAEhF;AAED,iEAAiE;AACjE,wBAAgB,kBAAkB,CAChC,IAAI,EAAE,WAAW,EACjB,UAAU,EAAE,SAAS,SAAS,EAAE,GAC/B,SAAS,EAAE,CAIb;AAED,wEAAwE;AACxE,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,UAAU,CAc7D;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAC7B,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,KAAK,EACZ,aAAa,EAAE,MAAM,GACpB,WAAW,GAAG,SAAS,CASzB;AAED,gFAAgF;AAChF,wBAAgB,eAAe,CAAC,MAAM,EAAE,UAAU,GAAG,SAAS,EAAE,KAAK,EAAE,YAAY,GAAG,OAAO,CAI5F;AAED,+DAA+D;AAC/D,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI,CAKtE;AAED;;;;GAIG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,WAAW,GAAG,cAAc,CAc7D;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAUpG;AAED,wBAAgB,aAAa,IAAI,OAAO,CAGvC"}

package/dist/test-state.js

@@ -0,0 +1,187 @@
+/**
+ * Per-test state shared by evaluate/inspect and trials: deterministic assertion
+ * IDs, the per-test budget remainder, and the results recorded on task meta.
+ */
+import { AsyncLocalStorage } from "node:async_hooks";
+import { createHash, randomUUID } from "node:crypto";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join, relative, sep } from "node:path";
+import * as vitest from "vitest";
+import { addUsage, zeroUsage, } from "@attest-ai/core";
+export const trialScope = new AsyncLocalStorage();
+/** True while the code runs inside trial i of a trials() call with N > 1. */
+export function inTrial() {
+    return trialScope.getStore()?.trial !== undefined;
+}
+const attempts = new WeakMap();
+function attemptOf(test) {
+    return test.result?.retryCount ?? 0;
+}
+/** State of the running attempt of `test`; a retried test starts afresh. */
+export function attemptState(test) {
+    const attempt = attemptOf(test);
+    let state = attempts.get(test);
+    if (state === undefined || state.attempt !== attempt) {
+        state = { attempt, ordinal: 0, usage: zeroUsage(), agentSpendUsd: 0, traceIds: new Set() };
+        attempts.set(test, state);
+    }
+    return state;
+}
+/**
+ * The running test: `TestRunner.getCurrentTest()` on Vitest 4.1+, `vitest/suite`
+ * before that (loaded only there, because Vitest 4.1 warns when it is imported).
+ */
+export async function runningTest() {
+    const runner = vitest.TestRunner;
+    if (runner !== undefined)
+        return runner.getCurrentTest();
+    const suite = await import("vitest/suite");
+    return suite.getCurrentTest();
+}
+/**
+ * The test an evaluation belongs to: the passed context, else the enclosing
+ * trials() call, else Vitest's running test. Under test.concurrent the running
+ * test may be another test, so outside trials() the caller must pass `context`.
+ */
+export async function resolveTest(context) {
+    if (context !== undefined)
+        return context.task;
+    const scope = trialScope.getStore();
+    if (scope !== undefined)
+        return scope.test;
+    const test = await runningTest();
+    if (test !== undefined && test.concurrent === true) {
+        throw new Error("Attest results from a test.concurrent test — pass the test context: " +
+            "inspect(chain, { context }), evaluate(chain, { context }) or trials(body, { context })");
+    }
+    return test;
+}
+/**
+ * Set by AttestReporter in the main process before workers start: the Vitest
+ * root its report paths are relative to, and the directory where workers drop
+ * the usage of evaluations that belong to no test.
+ */
+export const ROOT_ENV = "ATTEST_VITEST_ROOT";
+export const USAGE_DIR_ENV = "ATTEST_VITEST_USAGE_DIR";
+/**
+ * The report's test_id of `test`: "<file> > <suites> > <name>", the same string
+ * AttestReporter writes. The file is relative to the Vitest root AttestReporter
+ * publishes (in a multi-project setup, not the project root); without
+ * AttestReporter it is the file task's name, relative to the project root.
+ */
+export function testIdOf(test) {
+    const names = [test.name];
+    for (let suite = test.suite; suite !== undefined && suite !== test.file; suite = suite.suite) {
+        names.unshift(suite.name);
+    }
+    const root = process.env[ROOT_ENV];
+    const file = root === undefined || test.file.filepath === undefined
+        ? test.file.name
+        : relative(root, test.file.filepath).split(sep).join("/");
+    return `${file} > ${names.join(" > ")}`;
+}
+/**
+ * Hand the usage of an evaluation that belongs to no test (beforeAll, a script)
+ * to AttestReporter: one file per evaluation in its usage directory, so the
+ * session usage counts it once. Without AttestReporter there is no report to add it to.
+ */
+export function recordUnattachedUsage(usage) {
+    const dir = process.env[USAGE_DIR_ENV];
+    if (dir === undefined)
+        return;
+    if (Object.values(usage).every((v) => v === 0))
+        return;
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(join(dir, `${randomUUID()}.json`), JSON.stringify(usage));
+}
+/** `assert_` + the first 12 hex chars of sha256("<test_id>#<ordinal>"). */
+export function deterministicAssertionId(testId, ordinal) {
+    return `assert_${createHash("sha256").update(`${testId}#${ordinal}`).digest("hex").slice(0, 12)}`;
+}
+/** Give each assertion the next deterministic ID of the test. */
+export function assignAssertionIds(test, assertions) {
+    const state = attemptState(test);
+    const testId = testIdOf(test);
+    return assertions.map((a) => ({ ...a, assertion_id: deterministicAssertionId(testId, state.ordinal++) }));
+}
+/** Reject a budget with unknown keys or negative/non-numeric limits. */
+export function validateBudget(budget) {
+    for (const key of Object.keys(budget)) {
+        if (key !== "maxJudgeCalls" && key !== "maxCostUsd") {
+            throw new Error(`Attest budget: unknown option "${key}" (expected maxJudgeCalls, maxCostUsd)`);
+        }
+    }
+    const { maxJudgeCalls, maxCostUsd } = budget;
+    if (maxJudgeCalls !== undefined && !(Number.isInteger(maxJudgeCalls) && maxJudgeCalls >= 0)) {
+        throw new Error(`Attest budget: maxJudgeCalls must be an integer >= 0, got ${String(maxJudgeCalls)}`);
+    }
+    if (maxCostUsd !== undefined && !(Number.isFinite(maxCostUsd) && maxCostUsd >= 0)) {
+        throw new Error(`Attest budget: maxCostUsd must be a number >= 0, got ${String(maxCostUsd)}`);
+    }
+    return budget;
+}
+/**
+ * What is left of the test budget for the next evaluate_batch: judge calls not
+ * yet used, and USD not yet spent by attest or the agent, each floored at 0.
+ * Undefined when the budget sets no limit.
+ */
+export function remainingBudget(budget, usage, agentSpendUsd) {
+    const wire = {};
+    if (budget.maxJudgeCalls !== undefined) {
+        wire.max_judge_calls = Math.max(0, budget.maxJudgeCalls - usage.judge_calls);
+    }
+    if (budget.maxCostUsd !== undefined) {
+        wire.max_cost_usd = Math.max(0, budget.maxCostUsd - (usage.cost_usd + agentSpendUsd));
+    }
+    return Object.keys(wire).length === 0 ? undefined : wire;
+}
+/** True when attest cost plus agent spend has reached the test's cost limit. */
+export function costBudgetSpent(budget, state) {
+    return (budget?.maxCostUsd !== undefined && state.usage.cost_usd + state.agentSpendUsd >= budget.maxCostUsd);
+}
+/** Count the agent's own spend of a trace once per attempt. */
+export function recordAgentSpend(test, trace) {
+    const state = attemptState(test);
+    if (state.traceIds.has(trace.trace_id))
+        return;
+    state.traceIds.add(trace.trace_id);
+    state.agentSpendUsd += trace.metadata?.cost_usd ?? 0;
+}
+/**
+ * The meta of the running attempt. Results from an earlier attempt of a retried
+ * test are dropped, so the report matches the outcome Vitest reports; their
+ * usage is kept, because those model calls were made.
+ */
+export function attemptMeta(test) {
+    const attempt = attemptOf(test);
+    let meta = test.meta.attest;
+    if (meta === undefined || meta.attempt !== attempt) {
+        meta = {
+            attempt,
+            results: [],
+            totalCost: 0,
+            totalDurationMs: 0,
+            usage: meta?.usage ?? zeroUsage(),
+        };
+        test.meta.attest = meta;
+    }
+    return meta;
+}
+/**
+ * Attach the results to the test's meta so AttestReporter, in the main process,
+ * can write them with the test's file and line; rows of trial i carry `trial`.
+ */
+export function recordOnTest(test, result, trial) {
+    const state = attemptState(test);
+    state.usage = addUsage(state.usage, result.usage);
+    const meta = attemptMeta(test);
+    meta.results.push(...result.assertionResults.map((r) => (trial === undefined ? r : { ...r, trial })));
+    meta.totalCost += result.totalCost;
+    meta.totalDurationMs += result.totalDurationMs;
+    meta.usage = addUsage(meta.usage, result.usage);
+}
+export function strictFromEnv() {
+    const v = (process.env["ATTEST_STRICT"] ?? "").trim().toLowerCase();
+    return v === "1" || v === "true" || v === "yes";
+}
+//# sourceMappingURL=test-state.js.map

package/dist/test-state.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-state.js","sourceRoot":"","sources":["../src/test-state.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AAEjC,OAAO,EACL,QAAQ,EACR,SAAS,GAMV,MAAM,iBAAiB,CAAC;AAsBzB,MAAM,CAAC,MAAM,UAAU,GAAG,IAAI,iBAAiB,EAAc,CAAC;AAE9D,6EAA6E;AAC7E,MAAM,UAAU,OAAO;IACrB,OAAO,UAAU,CAAC,QAAQ,EAAE,EAAE,KAAK,KAAK,SAAS,CAAC;AACpD,CAAC;AAaD,MAAM,QAAQ,GAAG,IAAI,OAAO,EAA6B,CAAC;AAE1D,SAAS,SAAS,CAAC,IAAiB;IAClC,OAAO,IAAI,CAAC,MAAM,EAAE,UAAU,IAAI,CAAC,CAAC;AACtC,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,YAAY,CAAC,IAAiB;IAC5C,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;QACrD,KAAK,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,aAAa,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;QAC3F,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW;IAC/B,MAAM,MAAM,GAAI,MAAiC,CAAC,UAAU,CAAC;IAC7D,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,cAAc,EAAE,CAAC;IACzD,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;IAC3C,OAAO,KAAK,CAAC,cAAc,EAAE,CAAC;AAChC,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAAgC;IAChE,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,OAAO,CAAC,IAAI,CAAC;IAC/C,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;IACpC,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC;IAC3C,MAAM,IAAI,GAAG,MAAM,WAAW,EAAE,CAAC;IACjC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CACb,sEAAsE;YACpE,wFAAwF,CAC3F,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,oBAAoB,CAAC;AAC7C,MAAM,CAAC,MAAM,aAAa,GAAG,yBAAyB,CAAC;AAEvD;;;;;GAKG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAiB;IACxC,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,CAAC,IAAI,EAAE,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;QAC7F,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,IAAI,GACR,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS;QACpD,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;QAChB,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9D,OAAO,GAAG,IAAI,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;AAC1C,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAY;IAChD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IACvC,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO;IAC9B,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;QAAE,OAAO;IACvD,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;AAC1E,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,wBAAwB,CAAC,MAAc,EAAE,OAAe;IACtE,OAAO,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;AACpG,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,kBAAkB,CAChC,IAAiB,EACjB,UAAgC;IAEhC,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC9B,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,YAAY,EAAE,wBAAwB,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5G,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,cAAc,CAAC,MAAkB;IAC/C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACtC,IAAI,GAAG,KAAK,eAAe,IAAI,GAAG,KAAK,YAAY,EAAE,CAAC;YACpD,MAAM,IAAI,KAAK,CAAC,kCAAkC,GAAG,wCAAwC,CAAC,CAAC;QACjG,CAAC;IACH,CAAC;IACD,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;IAC7C,IAAI,aAAa,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,IAAI,aAAa,IAAI,CAAC,CAAC,EAAE,CAAC;QAC5F,MAAM,IAAI,KAAK,CAAC,6DAA6D,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IACxG,CAAC;IACD,IAAI,UAAU,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC;QAClF,MAAM,IAAI,KAAK,CAAC,wDAAwD,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IAChG,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAC7B,MAAkB,EAClB,KAAY,EACZ,aAAqB;IAErB,MAAM,IAAI,GAAwD,EAAE,CAAC;IACrE,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QACvC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QACpC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,GAAG,CAAC,KAAK,CAAC,QAAQ,GAAG,aAAa,CAAC,CAAC,CAAC;IACxF,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;AAC3D,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,eAAe,CAAC,MAA8B,EAAE,KAAmB;IACjF,OAAO,CACL,MAAM,EAAE,UAAU,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,aAAa,IAAI,MAAM,CAAC,UAAU,CACpG,CAAC;AACJ,CAAC;AAED,+DAA+D;AAC/D,MAAM,UAAU,gBAAgB,CAAC,IAAiB,EAAE,KAAY;IAC9D,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC;QAAE,OAAO;IAC/C,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACnC,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,QAAQ,EAAE,QAAQ,IAAI,CAAC,CAAC;AACvD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,IAAiB;IAC3C,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;IAC5B,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;QACnD,IAAI,GAAG;YACL,OAAO;YACP,OAAO,EAAE,EAAE;YACX,SAAS,EAAE,CAAC;YACZ,eAAe,EAAE,CAAC;YAClB,KAAK,EAAE,IAAI,EAAE,KAAK,IAAI,SAAS,EAAE;SAClC,CAAC;QACF,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IAC1B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,IAAiB,EAAE,MAAmB,EAAE,KAAyB;IAC5F,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,KAAK,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,CAAC,OAAO,CAAC,IAAI,CACf,GAAG,MAAM,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CACnF,CAAC;IACF,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;IACnC,IAAI,CAAC,eAAe,IAAI,MAAM,CAAC,eAAe,CAAC;IAC/C,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;AAClD,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACpE,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,KAAK,CAAC;AAClD,CAAC"}

package/dist/trials.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Fixed-N trials: the Vitest counterpart of pytest's `--attest-trials=N`.
+ */
+import type { TestContext } from "vitest";
+import { type TestBudget } from "./test-state.js";
+/** How attestGlobalSetup({ minPassRate }) reaches the workers that run trials(). */
+export declare const MIN_PASS_RATE_ENV = "ATTEST_MIN_PASS_RATE";
+export interface TrialsOptions {
+    /** Number of trials N; defaults to the ATTEST_TRIALS environment variable, else 1. */
+    n?: number;
+    /**
+     * The running test's context (`it("…", async (context) => …)`); required under
+     * `test.concurrent` and on Vitest before 4.1.
+     */
+    context?: TestContext;
+    /**
+     * Limits for the test's attest model calls across all trials; see
+     * InspectOptions.testBudget. Before each trial after the first, the trials stop
+     * when attest cost plus agent spend has reached `maxCostUsd`.
+     */
+    budget?: TestBudget;
+    /**
+     * Fail the test when passed / evaluated trials is below this rate (0..1).
+     * Overrides the run's value: attestGlobalSetup({ minPassRate }) or the
+     * ATTEST_MIN_PASS_RATE environment variable, else 1.0 (any failing trial fails).
+     */
+    minPassRate?: number;
+}
+/** A pass rate in [0, 1]; anything else is an error naming where it came from. */
+export declare function validateMinPassRate(rate: number, source: string): number;
+/**
+ * Run `body` N times inside the current test (N = `options.n`, else the
+ * ATTEST_TRIALS environment variable, else 1) and enforce only the aggregate.
+ *
+ * With N = 1 the body runs once as a plain test body: evaluate() enforces as
+ * usual. With N > 1, during trial i (1-based): evaluate() does not enforce
+ * (it behaves as inspect()), enforceOutcome() returns its argument, every
+ * result row is reported with `trial = i`, and assertion IDs restart so each
+ * assertion keeps its ID across trials. A trial that throws counts as failed:
+ * its rows are replaced by one hard_fail row (failure_class broken_code) per
+ * assertion ID the test evaluated; when no trial evaluated anything, the first
+ * error is rethrown. A trial whose own outcome is skip counts as not evaluated,
+ * also under ATTEST_STRICT. After the last trial the test is skipped when no trial was
+ * evaluated (failed under ATTEST_STRICT), fails when passed / evaluated
+ * trials is below the minimum pass rate (default 1.0: any failing trial
+ * fails; not-evaluated trials are left out of the rate), and passes otherwise. AttestReporter prints "<test_id>: <passed>/<evaluated>
+ * trials passed (<n> not evaluated)".
+ *
+ * This is a pass/fail gate on N runs, not a statistical verdict: whether the
+ * pass rate regressed (pass / regressed / inconclusive) comes from
+ * `attest policy evaluate --baseline <tag>` with a `regression` rule over the
+ * report's per-trial rows.
+ */
+export declare function trials(body: (trial: number) => Promise<void> | void, options?: TrialsOptions): Promise<void>;
+//# sourceMappingURL=trials.d.ts.map

package/dist/trials.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"trials.d.ts","sourceRoot":"","sources":["../src/trials.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AAS1C,OAAO,EASL,KAAK,UAAU,EAChB,MAAM,iBAAiB,CAAC;AAEzB,oFAAoF;AACpF,eAAO,MAAM,iBAAiB,yBAAyB,CAAC;AAExD,MAAM,WAAW,aAAa;IAC5B,sFAAsF;IACtF,CAAC,CAAC,EAAE,MAAM,CAAC;IACX;;;OAGG;IACH,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB;;;;OAIG;IACH,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAoBD,kFAAkF;AAClF,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAKxE;AA0ED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAsB,MAAM,CAC1B,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,EAC7C,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,IAAI,CAAC,CAqEf"}