@attest-ai/vitest 1.0.0-rc.1 → 1.0.0-rc.2

package/dist/fixtures.d.ts

@@ -1,8 +1,17 @@
 import type { TestContext } from "vitest";
 import { type ExpectChain, AgentResult, AttestClient } from "@attest-ai/core";
+import { type TestBudget } from "./test-state.js";
 export interface InspectOptions {
     /** Throw when the cumulative session cost exceeds this USD amount. */
     budget?: number;
+    /**
+     * Limits for this test's attest model calls. Each evaluation sends what is
+     * left: judge calls not yet used in the test, and `maxCostUsd` minus the
+     * test's attest cost and agent spend so far (trace `metadata.cost_usd`). A
+     * call that does not fit is not sent and its assertion is not_evaluated with
+     * reason "budget_exhausted". Inside trials(), defaults to the trials budget.
+     */
+    testBudget?: TestBudget;
     /**
      * The running test's context (`it("…", async (context) => …)`). Used to
      * attach results to the right test for AttestReporter and to skip it;
@@ -21,7 +30,8 @@ export interface EvaluateOptions extends InspectOptions {
  * Fail or skip the running Vitest test according to the batch outcome:
  * any hard_fail throws with rendered diagnostics; otherwise any
  * not_evaluated skips with "not evaluated: <reasons>" (throws when strict);
- * soft failures pass.
+ * soft failures pass. Inside a trial of trials() (N > 1) it does nothing and
+ * returns `result`: trials() enforces the aggregate outcome after the last trial.
  */
 export declare function enforceOutcome(result: AgentResult, strict: boolean, context?: TestContext): AgentResult;
 export declare class AttestEngineFixture {
@@ -48,8 +58,11 @@ export declare class AttestEngineFixture {
     evaluate(chain: ExpectChain, options?: EvaluateOptions): Promise<AgentResult>;
 }
 export declare function useAttest(): AttestEngineFixture;
-/** Evaluate and enforce: hard_fail throws, not_evaluated skips (strict: throws). */
+/**
+ * Evaluate and enforce: hard_fail throws, not_evaluated skips (strict: throws).
+ * Inside a trial of trials() (N > 1) it does not enforce, like inspect().
+ */
 export declare function evaluate(chain: ExpectChain, options?: EvaluateOptions): Promise<AgentResult>;
-/** Evaluate without enforcing; use for trial loops and custom checks. */
+/** Evaluate without enforcing; use for custom checks. For repeated runs use trials(). */
 export declare function inspect(chain: ExpectChain, options?: InspectOptions): Promise<AgentResult>;
 //# sourceMappingURL=fixtures.d.ts.map

package/dist/fixtures.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fixtures.d.ts","sourceRoot":"","sources":["../src/fixtures.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AAC1C,OAAO,EACL,KAAK,WAAW,EAChB,WAAW,EACX,YAAY,EAQb,MAAM,iBAAiB,CAAC;AAEzB,MAAM,WAAW,cAAc;IAC7B,sEAAsE;IACtE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,OAAO,CAAC,EAAE,WAAW,CAAC;CACvB;AAED,MAAM,WAAW,eAAgB,SAAQ,cAAc;IACrD;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AA8CD;;;;;GAKG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,WAAW,EACnB,MAAM,EAAE,OAAO,EACf,OAAO,CAAC,EAAE,WAAW,GACpB,WAAW,CAUb;AAmBD,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,OAAO,CAA4B;IAC3C,OAAO,CAAC,OAAO,CAA2B;IAC1C,OAAO,CAAC,QAAQ,CAA4B;gBAEhC,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,YAAY,CAAA;KAAE;IAWvF;;;;OAIG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAoBtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B,IAAI,MAAM,IAAI,YAAY,CAKzB;YAEa,aAAa;IAM3B,wEAAwE;IAClE,OAAO,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,CAAC;IA4CjF,kEAAkE;IAC5D,QAAQ,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,WAAW,CAAC;CAIpF;AAOD,wBAAgB,SAAS,IAAI,mBAAmB,CAO/C;AAED,oFAAoF;AACpF,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,WAAW,EAClB,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,WAAW,CAAC,CAEtB;AAED,yEAAyE;AACzE,wBAAsB,OAAO,CAC3B,KAAK,EAAE,WAAW,EAClB,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,WAAW,CAAC,CAEtB"}
+{"version":3,"file":"fixtures.d.ts","sourceRoot":"","sources":["../src/fixtures.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AAC1C,OAAO,EACL,KAAK,WAAW,EAChB,WAAW,EAEX,YAAY,EASb,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAYL,KAAK,UAAU,EAChB,MAAM,iBAAiB,CAAC;AAEzB,MAAM,WAAW,cAAc;IAC7B,sEAAsE;IACtE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB;;;;OAIG;IACH,OAAO,CAAC,EAAE,WAAW,CAAC;CACvB;AAED,MAAM,WAAW,eAAgB,SAAQ,cAAc;IACrD;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,WAAW,EACnB,MAAM,EAAE,OAAO,EACf,OAAO,CAAC,EAAE,WAAW,GACpB,WAAW,CAab;AAmBD,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,OAAO,CAA4B;IAC3C,OAAO,CAAC,OAAO,CAA2B;IAC1C,OAAO,CAAC,QAAQ,CAA4B;gBAEhC,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,YAAY,CAAA;KAAE;IAWvF;;;;OAIG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAoBtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B,IAAI,MAAM,IAAI,YAAY,CAKzB;YAEa,aAAa;IAM3B,wEAAwE;IAClE,OAAO,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,CAAC;IA4EjF,kEAAkE;IAC5D,QAAQ,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,WAAW,CAAC;CAIpF;AAOD,wBAAgB,SAAS,IAAI,mBAAmB,CAO/C;AAED;;;GAGG;AACH,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,WAAW,EAClB,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,WAAW,CAAC,CAEtB;AAED,yFAAyF;AACzF,wBAAsB,OAAO,CAC3B,KAAK,EAAE,WAAW,EAClB,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,WAAW,CAAC,CAEtB"}

package/dist/fixtures.js

@@ -1,58 +1,23 @@
 import * as vitest from "vitest";
-import { AgentResult, AttestClient, EngineManager, isSimulationMode, renderDiagnostics, requireSimulationCapability, OUTCOME_FAIL, OUTCOME_SKIP, STATUS_SOFT_FAIL, } from "@attest-ai/core";
-/**
- * The running test: `TestRunner.getCurrentTest()` on Vitest 4.1+, `vitest/suite`
- * before that (loaded only there, because Vitest 4.1 warns when it is imported).
- */
-async function runningTest() {
-    const runner = vitest.TestRunner;
-    if (runner !== undefined)
-        return runner.getCurrentTest();
-    const suite = await import("vitest/suite");
-    return suite.getCurrentTest();
-}
-/**
- * Attach the results to the running test's meta so AttestReporter, in the main
- * process, can write them with the test's file and line. Results from an earlier
- * attempt of a retried test are dropped, so the report matches the outcome Vitest
- * reports. Outside a test (e.g. in a beforeAll hook) there is no test to attach to.
- */
-async function recordOnTest(result, context) {
-    const test = context?.task ?? (await runningTest());
-    if (test === undefined)
-        return;
-    if (context === undefined && test.concurrent === true) {
-        throw new Error("Attest results from a test.concurrent test — pass the test context: " +
-            "inspect(chain, { context }) or evaluate(chain, { context })");
-    }
-    const attempt = test.result?.retryCount ?? 0;
-    let meta = test.meta.attest;
-    if (meta === undefined || meta.attempt !== attempt) {
-        meta = { attempt, results: [], totalCost: 0, totalDurationMs: 0 };
-        test.meta.attest = meta;
-    }
-    meta.results.push(...result.assertionResults);
-    meta.totalCost += result.totalCost;
-    meta.totalDurationMs += result.totalDurationMs;
-}
-function strictFromEnv() {
-    const v = (process.env["ATTEST_STRICT"] ?? "").trim().toLowerCase();
-    return v === "1" || v === "true" || v === "yes";
-}
+import { AgentResult, STATUS_SOFT_FAIL, AttestClient, EngineManager, isSimulationMode, renderDiagnostics, requireSimulationCapability, OUTCOME_FAIL, OUTCOME_SKIP, zeroUsage, } from "@attest-ai/core";
+import { assignAssertionIds, attemptState, inTrial, recordAgentSpend, recordOnTest, recordUnattachedUsage, remainingBudget, resolveTest, strictFromEnv, trialScope, validateBudget, } from "./test-state.js";
 /**
  * Fail or skip the running Vitest test according to the batch outcome:
  * any hard_fail throws with rendered diagnostics; otherwise any
  * not_evaluated skips with "not evaluated: <reasons>" (throws when strict);
- * soft failures pass.
+ * soft failures pass. Inside a trial of trials() (N > 1) it does nothing and
+ * returns `result`: trials() enforces the aggregate outcome after the last trial.
  */
 export function enforceOutcome(result, strict, context) {
+    if (inTrial())
+        return result;
     const outcome = result.outcome({ strict });
     if (outcome === OUTCOME_FAIL) {
         const header = result.hardFailures.length === 0 ? `strict: ${result.skipReason()}\n` : "";
         throw new Error(header + renderDiagnostics(result).replace(/^\n/, ""));
     }
     if (outcome === OUTCOME_SKIP) {
-        (context ?? currentTestContext(result)).skip(result.skipReason());
+        (context ?? trialScope.getStore()?.test.context ?? currentTestContext(result)).skip(result.skipReason());
     }
     return result;
 }
@@ -131,14 +96,44 @@ export class AttestEngineFixture {
     /** Evaluate collected assertions without enforcing the test outcome. */
     async inspect(chain, options) {
         await this.ensureStarted();
-        const result = await this.client.evaluateBatch(chain.trace, chain.assertions);
+        // Inside a test, assertion IDs are deterministic (the test's report test_id
+        // and the assertion's ordinal in the test, restarting each trial) so trials
+        // and baselines match rows by ID. Outside a test they stay random.
+        const test = await resolveTest(options?.context);
+        const scope = trialScope.getStore();
+        const trial = scope !== undefined && scope.test === test ? scope.trial : undefined;
+        const budget = options?.testBudget !== undefined
+            ? validateBudget(options.testBudget)
+            : scope !== undefined && scope.test === test
+                ? scope.budget
+                : undefined;
+        let assertions = chain.assertions;
+        let batchBudget;
+        if (test !== undefined) {
+            assertions = assignAssertionIds(test, assertions);
+            recordAgentSpend(test, chain.trace);
+            if (budget !== undefined) {
+                const state = attemptState(test);
+                batchBudget = remainingBudget(budget, state.usage, state.agentSpendUsd);
+            }
+        }
+        else if (budget !== undefined) {
+            // Outside a test the budget covers this one evaluation.
+            batchBudget = remainingBudget(budget, zeroUsage(), chain.trace.metadata?.cost_usd ?? 0);
+        }
+        const result = await this.client.evaluateBatch(chain.trace, assertions, batchBudget === undefined ? undefined : { budget: batchBudget });
         globalThis.__attest_session_cost__ =
             (globalThis.__attest_session_cost__ ?? 0) + (result.total_cost ?? 0);
         const softFailCount = (result.results ?? []).filter((r) => r.status === STATUS_SOFT_FAIL).length;
         globalThis.__attest_session_soft_failures__ =
             (globalThis.__attest_session_soft_failures__ ?? 0) + softFailCount;
-        const agentResult = new AgentResult(chain.trace, result.results ?? [], result.total_cost ?? 0, result.total_duration_ms ?? 0);
-        await recordOnTest(agentResult, options?.context);
+        const agentResult = new AgentResult(chain.trace, result.results ?? [], result.total_cost ?? 0, result.total_duration_ms ?? 0, result.usage);
+        if (test !== undefined)
+            recordOnTest(test, agentResult, trial);
+        else
+            recordUnattachedUsage(agentResult.usage);
+        if (scope !== undefined && scope.test === test)
+            scope.results.push(agentResult);
         // Track non-passing AgentResults so AttestCostReporter can render the
         // diagnostic block at the end of the run. Bound size because long
         // suites with many failures should not balloon RAM.
@@ -173,11 +168,14 @@ export function useAttest() {
     workerFixture ??= new AttestEngineFixture();
     return workerFixture;
 }
-/** Evaluate and enforce: hard_fail throws, not_evaluated skips (strict: throws). */
+/**
+ * Evaluate and enforce: hard_fail throws, not_evaluated skips (strict: throws).
+ * Inside a trial of trials() (N > 1) it does not enforce, like inspect().
+ */
 export async function evaluate(chain, options) {
     return useAttest().evaluate(chain, options);
 }
-/** Evaluate without enforcing; use for trial loops and custom checks. */
+/** Evaluate without enforcing; use for custom checks. For repeated runs use trials(). */
 export async function inspect(chain, options) {
     return useAttest().inspect(chain, options);
 }

package/dist/fixtures.js.map

@@ -1 +1 @@
-{"version":3,"file":"fixtures.js","sourceRoot":"","sources":["../src/fixtures.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AAEjC,OAAO,EAEL,WAAW,EACX,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,2BAA2B,EAC3B,YAAY,EACZ,YAAY,EACZ,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AAuBzB;;;GAGG;AACH,KAAK,UAAU,WAAW;IACxB,MAAM,MAAM,GAAI,MAAiC,CAAC,UAAU,CAAC;IAC7D,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,cAAc,EAAE,CAAC;IACzD,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;IAC3C,OAAO,KAAK,CAAC,cAAc,EAAE,CAAC;AAChC,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,YAAY,CAAC,MAAmB,EAAE,OAAqB;IACpE,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,CAAC,MAAM,WAAW,EAAE,CAAC,CAAC;IACpD,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO;IAC/B,IAAI,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CACb,sEAAsE;YACpE,6DAA6D,CAChE,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,UAAU,IAAI,CAAC,CAAC;IAC7C,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;IAC5B,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;QACnD,IAAI,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC;QAClE,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IAC1B,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAC9C,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;IACnC,IAAI,CAAC,eAAe,IAAI,MAAM,CAAC,eAAe,CAAC;AACjD,CAAC;AAED,SAAS,aAAa;IACpB,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACpE,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,KAAK,CAAC;AAClD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAC5B,MAAmB,EACnB,MAAe,EACf,OAAqB;IAErB,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAC3C,IAAI,OAAO,KAAK,YAAY,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,MAAM,IAAI,KAAK,CAAC,MAAM,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;IACzE,CAAC;IACD,IAAI,OAAO,KAAK,YAAY,EAAE,CAAC;QAC7B,CAAC,OAAO,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,MAAmB;IAC7C,MAAM,MAAM,GAAI,MAAiC,CAAC,UAAU,CAAC;IAC7D,MAAM,IAAI,GAAG,MAAM,EAAE,cAAc,EAAE,CAAC;IACtC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CACb,GAAG,MAAM,CAAC,UAAU,EAAE,8CAA8C;YAClE,8EAA8E,CACjF,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC,OAAO,CAAC;AACtB,CAAC;AAED,MAAM,OAAO,mBAAmB;IACtB,OAAO,CAA4B;IACnC,OAAO,CAA2B;IAClC,QAAQ,CAA4B;IAE5C,YAAY,OAA2E;QACrF,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,OAAO;QACT,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,aAAa,CAAC;YAC/B,UAAU,EAAE,OAAO,EAAE,UAAU;YAC/B,QAAQ,EAAE,OAAO,EAAE,QAAQ,IAAI,MAAM;SACtC,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QACD,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAC;QACtC,IAAI,UAAU,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,GAAG,CAAC;QACzC,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACxC,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,2BAA2B,CAAC,IAAI,CAAC,CAAC;YACpC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC1B,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAC1B,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;YACzB,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,IAAI,MAAM;QACR,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS;YAAE,OAAO;QACvC,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC;IACtB,CAAC;IAED,wEAAwE;IACxE,KAAK,CAAC,OAAO,CAAC,KAAkB,EAAE,OAAwB;QACxD,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;QAE9E,UAAU,CAAC,uBAAuB;YAChC,CAAC,UAAU,CAAC,uBAAuB,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC;QAEvE,MAAM,aAAa,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,CACjD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,gBAAgB,CACrC,CAAC,MAAM,CAAC;QACT,UAAU,CAAC,gCAAgC;YACzC,CAAC,UAAU,CAAC,gCAAgC,IAAI,CAAC,CAAC,GAAG,aAAa,CAAC;QAErE,MAAM,WAAW,GAAG,IAAI,WAAW,CACjC,KAAK,CAAC,KAAK,EACX,MAAM,CAAC,OAAO,IAAI,EAAE,EACpB,MAAM,CAAC,UAAU,IAAI,CAAC,EACtB,MAAM,CAAC,iBAAiB,IAAI,CAAC,CAC9B,CAAC;QACF,MAAM,YAAY,CAAC,WAAW,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAElD,sEAAsE;QACtE,kEAAkE;QAClE,oDAAoD;QACpD,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,CAAC,UAAU,CAAC,2BAA2B,KAAK,EAAE,CAAC,CAAC;YAC7D,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACtB,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QAED,IACE,OAAO,EAAE,MAAM,KAAK,SAAS;YAC7B,UAAU,CAAC,uBAAuB,GAAG,OAAO,CAAC,MAAM,EACnD,CAAC;YACD,MAAM,IAAI,KAAK,CACb,iCAAiC,UAAU,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;gBAC9E,cAAc,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAC5C,CAAC;QACJ,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,kEAAkE;IAClE,KAAK,CAAC,QAAQ,CAAC,KAAkB,EAAE,OAAyB;QAC1D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAClD,OAAO,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,aAAa,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IACtF,CAAC;CACF;AAED,yEAAyE;AACzE,+EAA+E;AAC/E,sEAAsE;AACtE,IAAI,aAA8C,CAAC;AAEnD,MAAM,UAAU,SAAS;IACvB,MAAM,MAAM,GAAG,UAAU,CAAC,iBAAiB,CAAC;IAC5C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,IAAI,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAC7C,CAAC;IACD,aAAa,KAAK,IAAI,mBAAmB,EAAE,CAAC;IAC5C,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,oFAAoF;AACpF,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAkB,EAClB,OAAyB;IAEzB,OAAO,SAAS,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC9C,CAAC;AAED,yEAAyE;AACzE,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,KAAkB,EAClB,OAAwB;IAExB,OAAO,SAAS,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC7C,CAAC"}
+{"version":3,"file":"fixtures.js","sourceRoot":"","sources":["../src/fixtures.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AAEjC,OAAO,EAEL,WAAW,EACX,gBAAgB,EAChB,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,2BAA2B,EAC3B,YAAY,EACZ,YAAY,EACZ,SAAS,GAEV,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,kBAAkB,EAClB,YAAY,EACZ,OAAO,EACP,gBAAgB,EAChB,YAAY,EACZ,qBAAqB,EACrB,eAAe,EACf,WAAW,EACX,aAAa,EACb,UAAU,EACV,cAAc,GAEf,MAAM,iBAAiB,CAAC;AA6BzB;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAC5B,MAAmB,EACnB,MAAe,EACf,OAAqB;IAErB,IAAI,OAAO,EAAE;QAAE,OAAO,MAAM,CAAC;IAC7B,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAC3C,IAAI,OAAO,KAAK,YAAY,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,MAAM,IAAI,KAAK,CAAC,MAAM,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;IACzE,CAAC;IACD,IAAI,OAAO,KAAK,YAAY,EAAE,CAAC;QAC7B,CAAC,OAAO,IAAI,UAAU,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CACjF,MAAM,CAAC,UAAU,EAAE,CACpB,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,MAAmB;IAC7C,MAAM,MAAM,GAAI,MAAiC,CAAC,UAAU,CAAC;IAC7D,MAAM,IAAI,GAAG,MAAM,EAAE,cAAc,EAAE,CAAC;IACtC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CACb,GAAG,MAAM,CAAC,UAAU,EAAE,8CAA8C;YAClE,8EAA8E,CACjF,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC,OAAO,CAAC;AACtB,CAAC;AAED,MAAM,OAAO,mBAAmB;IACtB,OAAO,CAA4B;IACnC,OAAO,CAA2B;IAClC,QAAQ,CAA4B;IAE5C,YAAY,OAA2E;QACrF,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,OAAO;QACT,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,aAAa,CAAC;YAC/B,UAAU,EAAE,OAAO,EAAE,UAAU;YAC/B,QAAQ,EAAE,OAAO,EAAE,QAAQ,IAAI,MAAM;SACtC,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QACD,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAC;QACtC,IAAI,UAAU,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,GAAG,CAAC;QACzC,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACxC,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,2BAA2B,CAAC,IAAI,CAAC,CAAC;YACpC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC1B,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAC1B,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;YACzB,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,IAAI,MAAM;QACR,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS;YAAE,OAAO;QACvC,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC;IACtB,CAAC;IAED,wEAAwE;IACxE,KAAK,CAAC,OAAO,CAAC,KAAkB,EAAE,OAAwB;QACxD,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC3B,4EAA4E;QAC5E,4EAA4E;QAC5E,mEAAmE;QACnE,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QACnF,MAAM,MAAM,GACV,OAAO,EAAE,UAAU,KAAK,SAAS;YAC/B,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,UAAU,CAAC;YACpC,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI;gBAC1C,CAAC,CAAC,KAAK,CAAC,MAAM;gBACd,CAAC,CAAC,SAAS,CAAC;QAClB,IAAI,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;QAClC,IAAI,WAAoC,CAAC;QACzC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,UAAU,GAAG,kBAAkB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAClD,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;YACpC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;gBACjC,WAAW,GAAG,eAAe,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;aAAM,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YAChC,wDAAwD;YACxD,WAAW,GAAG,eAAe,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,IAAI,CAAC,CAAC,CAAC;QAC1F,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAC5C,KAAK,CAAC,KAAK,EACX,UAAU,EACV,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAChE,CAAC;QAEF,UAAU,CAAC,uBAAuB;YAChC,CAAC,UAAU,CAAC,uBAAuB,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC;QAEvE,MAAM,aAAa,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,CACjD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,gBAAgB,CACrC,CAAC,MAAM,CAAC;QACT,UAAU,CAAC,gCAAgC;YACzC,CAAC,UAAU,CAAC,gCAAgC,IAAI,CAAC,CAAC,GAAG,aAAa,CAAC;QAErE,MAAM,WAAW,GAAG,IAAI,WAAW,CACjC,KAAK,CAAC,KAAK,EACX,MAAM,CAAC,OAAO,IAAI,EAAE,EACpB,MAAM,CAAC,UAAU,IAAI,CAAC,EACtB,MAAM,CAAC,iBAAiB,IAAI,CAAC,EAC7B,MAAM,CAAC,KAAK,CACb,CAAC;QACF,IAAI,IAAI,KAAK,SAAS;YAAE,YAAY,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;;YAC1D,qBAAqB,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAC9C,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI;YAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEhF,sEAAsE;QACtE,kEAAkE;QAClE,oDAAoD;QACpD,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,CAAC,UAAU,CAAC,2BAA2B,KAAK,EAAE,CAAC,CAAC;YAC7D,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACtB,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QAED,IACE,OAAO,EAAE,MAAM,KAAK,SAAS;YAC7B,UAAU,CAAC,uBAAuB,GAAG,OAAO,CAAC,MAAM,EACnD,CAAC;YACD,MAAM,IAAI,KAAK,CACb,iCAAiC,UAAU,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;gBAC9E,cAAc,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAC5C,CAAC;QACJ,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,kEAAkE;IAClE,KAAK,CAAC,QAAQ,CAAC,KAAkB,EAAE,OAAyB;QAC1D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAClD,OAAO,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,aAAa,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IACtF,CAAC;CACF;AAED,yEAAyE;AACzE,+EAA+E;AAC/E,sEAAsE;AACtE,IAAI,aAA8C,CAAC;AAEnD,MAAM,UAAU,SAAS;IACvB,MAAM,MAAM,GAAG,UAAU,CAAC,iBAAiB,CAAC;IAC5C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,IAAI,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAC7C,CAAC;IACD,aAAa,KAAK,IAAI,mBAAmB,EAAE,CAAC;IAC5C,OAAO,aAAa,CAAC;AACvB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAkB,EAClB,OAAyB;IAEzB,OAAO,SAAS,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC9C,CAAC;AAED,yFAAyF;AACzF,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,KAAkB,EAClB,OAAwB;IAExB,OAAO,SAAS,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC7C,CAAC"}

package/dist/index.d.ts

@@ -2,9 +2,12 @@ export { attestGlobalSetup } from "./plugin.js";
 export type { AttestGlobalSetupOptions } from "./plugin.js";
 export { AttestEngineFixture, useAttest, evaluate, inspect, enforceOutcome } from "./fixtures.js";
 export type { EvaluateOptions, InspectOptions } from "./fixtures.js";
+export { trials, MIN_PASS_RATE_ENV } from "./trials.js";
+export type { TrialsOptions } from "./trials.js";
+export type { TestBudget } from "./test-state.js";
 export { AttestCostReporter, AttestReporter } from "./reporter.js";
 export type { AttestReporterOptions } from "./reporter.js";
-export { buildReport, REPORT_VERSION } from "./report.js";
-export type { AttestTaskMeta, TestOutcome, TestRecord } from "./report.js";
+export { buildReport, REPORT_VERSION, sessionUsage, summaryLines } from "./report.js";
+export type { AttestTaskMeta, RecordedResult, TestOutcome, TestRecord, TrialsMeta } from "./report.js";
 export { filterByTier } from "./tier-filter.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,YAAY,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAClG,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACrE,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACnE,YAAY,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC1D,YAAY,EAAE,cAAc,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,YAAY,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAClG,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACrE,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACxD,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,YAAY,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACnE,YAAY,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACtF,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACvG,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC"}

package/dist/index.js

@@ -1,6 +1,7 @@
 export { attestGlobalSetup } from "./plugin.js";
 export { AttestEngineFixture, useAttest, evaluate, inspect, enforceOutcome } from "./fixtures.js";
+export { trials, MIN_PASS_RATE_ENV } from "./trials.js";
 export { AttestCostReporter, AttestReporter } from "./reporter.js";
-export { buildReport, REPORT_VERSION } from "./report.js";
+export { buildReport, REPORT_VERSION, sessionUsage, summaryLines } from "./report.js";
 export { filterByTier } from "./tier-filter.js";
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAElG,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAEnE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE1D,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAElG,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAGxD,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAEnE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEtF,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC"}

package/dist/plugin.d.ts

@@ -2,6 +2,12 @@ import { EngineManager, AttestClient, type AgentResult } from "@attest-ai/core";
 export interface AttestGlobalSetupOptions {
     enginePath?: string;
     logLevel?: string;
+    /**
+     * Minimum passed / evaluated trials for every trials() test of the run (0..1,
+     * default 1.0); trials(body, { minPassRate }) overrides it per test. Reaches
+     * the workers as the ATTEST_MIN_PASS_RATE environment variable.
+     */
+    minPassRate?: number;
 }
 declare global {
     var __attest_engine__: EngineManager | undefined;

package/dist/plugin.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,aAAa,EACb,YAAY,EAGZ,KAAK,WAAW,EACjB,MAAM,iBAAiB,CAAC;AAKzB,MAAM,WAAW,wBAAwB;IACvC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,OAAO,CAAC,MAAM,CAAC;IAEb,IAAI,iBAAiB,EAAE,aAAa,GAAG,SAAS,CAAC;IAEjD,IAAI,iBAAiB,EAAE,YAAY,GAAG,SAAS,CAAC;IAEhD,IAAI,uBAAuB,EAAE,MAAM,CAAC;IAEpC,IAAI,gCAAgC,EAAE,MAAM,CAAC;IAE7C,IAAI,2BAA2B,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC;CAC5D;AAED,wBAAgB,iBAAiB,CAAC,OAAO,CAAC,EAAE,wBAAwB;;;EA8CnE"}
+{"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,aAAa,EACb,YAAY,EAGZ,KAAK,WAAW,EACjB,MAAM,iBAAiB,CAAC;AAMzB,MAAM,WAAW,wBAAwB;IACvC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,OAAO,CAAC,MAAM,CAAC;IAEb,IAAI,iBAAiB,EAAE,aAAa,GAAG,SAAS,CAAC;IAEjD,IAAI,iBAAiB,EAAE,YAAY,GAAG,SAAS,CAAC;IAEhD,IAAI,uBAAuB,EAAE,MAAM,CAAC;IAEpC,IAAI,gCAAgC,EAAE,MAAM,CAAC;IAE7C,IAAI,2BAA2B,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC;CAC5D;AAED,wBAAgB,iBAAiB,CAAC,OAAO,CAAC,EAAE,wBAAwB;;;EAmDnE"}

package/dist/plugin.js

@@ -1,9 +1,15 @@
 import { EngineManager, AttestClient, isSimulationMode, requireSimulationCapability, } from "@attest-ai/core";
+import { MIN_PASS_RATE_ENV, validateMinPassRate } from "./trials.js";
 let engineManager;
 let attestClient;
 export function attestGlobalSetup(options) {
+    const minPassRate = options?.minPassRate === undefined
+        ? undefined
+        : validateMinPassRate(options.minPassRate, `minPassRate=${options.minPassRate}`);
     return {
         async setup() {
+            if (minPassRate !== undefined)
+                process.env[MIN_PASS_RATE_ENV] = String(minPassRate);
             globalThis.__attest_session_cost__ = 0;
             globalThis.__attest_session_soft_failures__ = 0;
             globalThis.__attest_session_failures__ = [];

package/dist/plugin.js.map

@@ -1 +1 @@
-{"version":3,"file":"plugin.js","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,aAAa,EACb,YAAY,EACZ,gBAAgB,EAChB,2BAA2B,GAE5B,MAAM,iBAAiB,CAAC;AAEzB,IAAI,aAAwC,CAAC;AAC7C,IAAI,YAAsC,CAAC;AAoB3C,MAAM,UAAU,iBAAiB,CAAC,OAAkC;IAClE,OAAO;QACL,KAAK,CAAC,KAAK;YACT,UAAU,CAAC,uBAAuB,GAAG,CAAC,CAAC;YACvC,UAAU,CAAC,gCAAgC,GAAG,CAAC,CAAC;YAChD,UAAU,CAAC,2BAA2B,GAAG,EAAE,CAAC;YAE5C,IAAI,gBAAgB,EAAE,EAAE,CAAC;gBACvB,iEAAiE;gBACjE,iDAAiD;gBACjD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,GAAG,CAAC;YACzC,CAAC;YAED,aAAa,GAAG,IAAI,aAAa,CAAC;gBAChC,UAAU,EAAE,OAAO,EAAE,UAAU;gBAC/B,QAAQ,EAAE,OAAO,EAAE,QAAQ,IAAI,MAAM;aACtC,CAAC,CAAC;YAEH,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,CAAC;YACzC,IAAI,gBAAgB,EAAE,EAAE,CAAC;gBACvB,IAAI,CAAC;oBACH,2BAA2B,CAAC,IAAI,CAAC,CAAC;gBACpC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;oBAC3B,aAAa,GAAG,SAAS,CAAC;oBAC1B,MAAM,GAAG,CAAC;gBACZ,CAAC;YACH,CAAC;YACD,YAAY,GAAG,IAAI,YAAY,CAAC,aAAa,CAAC,CAAC;YAE/C,UAAU,CAAC,iBAAiB,GAAG,aAAa,CAAC;YAC7C,UAAU,CAAC,iBAAiB,GAAG,YAAY,CAAC;QAC9C,CAAC;QAED,KAAK,CAAC,QAAQ;YACZ,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;gBAC3B,aAAa,GAAG,SAAS,CAAC;gBAC1B,YAAY,GAAG,SAAS,CAAC;YAC3B,CAAC;YAED,UAAU,CAAC,iBAAiB,GAAG,SAAS,CAAC;YACzC,UAAU,CAAC,iBAAiB,GAAG,SAAS,CAAC;YACzC,UAAU,CAAC,2BAA2B,GAAG,SAAS,CAAC;QACrD,CAAC;KACF,CAAC;AACJ,CAAC"}
+{"version":3,"file":"plugin.js","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,aAAa,EACb,YAAY,EACZ,gBAAgB,EAChB,2BAA2B,GAE5B,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAErE,IAAI,aAAwC,CAAC;AAC7C,IAAI,YAAsC,CAAC;AA0B3C,MAAM,UAAU,iBAAiB,CAAC,OAAkC;IAClE,MAAM,WAAW,GACf,OAAO,EAAE,WAAW,KAAK,SAAS;QAChC,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,mBAAmB,CAAC,OAAO,CAAC,WAAW,EAAE,eAAe,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IACrF,OAAO;QACL,KAAK,CAAC,KAAK;YACT,IAAI,WAAW,KAAK,SAAS;gBAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;YACpF,UAAU,CAAC,uBAAuB,GAAG,CAAC,CAAC;YACvC,UAAU,CAAC,gCAAgC,GAAG,CAAC,CAAC;YAChD,UAAU,CAAC,2BAA2B,GAAG,EAAE,CAAC;YAE5C,IAAI,gBAAgB,EAAE,EAAE,CAAC;gBACvB,iEAAiE;gBACjE,iDAAiD;gBACjD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,GAAG,CAAC;YACzC,CAAC;YAED,aAAa,GAAG,IAAI,aAAa,CAAC;gBAChC,UAAU,EAAE,OAAO,EAAE,UAAU;gBAC/B,QAAQ,EAAE,OAAO,EAAE,QAAQ,IAAI,MAAM;aACtC,CAAC,CAAC;YAEH,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,CAAC;YACzC,IAAI,gBAAgB,EAAE,EAAE,CAAC;gBACvB,IAAI,CAAC;oBACH,2BAA2B,CAAC,IAAI,CAAC,CAAC;gBACpC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;oBAC3B,aAAa,GAAG,SAAS,CAAC;oBAC1B,MAAM,GAAG,CAAC;gBACZ,CAAC;YACH,CAAC;YACD,YAAY,GAAG,IAAI,YAAY,CAAC,aAAa,CAAC,CAAC;YAE/C,UAAU,CAAC,iBAAiB,GAAG,aAAa,CAAC;YAC7C,UAAU,CAAC,iBAAiB,GAAG,YAAY,CAAC;QAC9C,CAAC;QAED,KAAK,CAAC,QAAQ;YACZ,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;gBAC3B,aAAa,GAAG,SAAS,CAAC;gBAC1B,YAAY,GAAG,SAAS,CAAC;YAC3B,CAAC;YAED,UAAU,CAAC,iBAAiB,GAAG,SAAS,CAAC;YACzC,UAAU,CAAC,iBAAiB,GAAG,SAAS,CAAC;YACzC,UAAU,CAAC,2BAA2B,GAAG,SAAS,CAAC;QACrD,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/report.d.ts

@@ -6,16 +6,35 @@
  * outcome. `attest policy evaluate` gates on `results`; the pr-annotate action
  * places inline comments with `file` and `line`.
  */
-import { type AssertionResult } from "@attest-ai/core";
+import { type AssertionResult, type Usage } from "@attest-ai/core";
 export declare const REPORT_VERSION = 2;
 export type TestOutcome = "passed" | "failed" | "skipped";
+/** An assertion result as recorded for the report; `trial` is set under trials() with N > 1. */
+export type RecordedResult = AssertionResult & {
+    readonly trial?: number;
+};
+/** How a trials() call with N > 1 went. */
+export interface TrialsMeta {
+    /** N. */
+    requested: number;
+    /** Trials actually run; fewer than N when the test's cost budget was spent. */
+    run: number;
+    passed: number;
+    /** Trials that passed, failed or raised (not counting not-evaluated ones). */
+    evaluated: number;
+    notEvaluated: number;
+    stoppedByBudget: boolean;
+}
 /** Attest results a test recorded, carried from the worker in task meta. */
 export interface AttestTaskMeta {
     /** `retryCount` of the attempt that produced `results`. */
     attempt: number;
-    results: AssertionResult[];
+    results: RecordedResult[];
     totalCost: number;
     totalDurationMs: number;
+    /** Attest model usage of every attempt of the test, including retried ones. */
+    usage: Usage;
+    trials?: TrialsMeta;
 }
 declare module "vitest" {
     interface TaskMeta {
@@ -33,5 +52,18 @@ export interface TestRecord {
     durationMs: number;
     attest: AttestTaskMeta | undefined;
 }
-export declare function buildReport(tests: readonly TestRecord[], simulated: boolean): Record<string, unknown>;
+/**
+ * `unattached` is the usage of evaluations that belong to no test (beforeAll,
+ * scripts); it counts in `summary.usage`.
+ */
+export declare function buildReport(tests: readonly TestRecord[], simulated: boolean, unattached?: Usage): Record<string, unknown>;
+/** Attest usage summed over every test of the run plus the evaluations that belong to no test. */
+export declare function sessionUsage(tests: readonly TestRecord[], unattached?: Usage): Usage;
+/**
+ * Lines for the end of the run: one per trials test
+ * ("<test_id>: <passed>/<evaluated> trials passed (<n> not evaluated)", plus a
+ * note when the test budget stopped it early), then the "attest usage: …" line
+ * when the session made or refused a model call.
+ */
+export declare function summaryLines(tests: readonly TestRecord[], unattached?: Usage): string[];
 //# sourceMappingURL=report.d.ts.map

package/dist/report.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAKL,KAAK,eAAe,EACrB,MAAM,iBAAiB,CAAC;AAEzB,eAAO,MAAM,cAAc,IAAI,CAAC;AAEhC,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE1D,4EAA4E;AAC5E,MAAM,WAAW,cAAc;IAC7B,2DAA2D;IAC3D,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,QAAQ;QAChB,MAAM,CAAC,EAAE,cAAc,CAAC;KACzB;CACF;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,oDAAoD;IACpD,IAAI,EAAE,MAAM,CAAC;IACb,2CAA2C;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,cAAc,GAAG,SAAS,CAAC;CACpC;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,EAAE,SAAS,EAAE,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CA8CrG"}
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAQL,KAAK,eAAe,EACpB,KAAK,KAAK,EACX,MAAM,iBAAiB,CAAC;AAEzB,eAAO,MAAM,cAAc,IAAI,CAAC;AAEhC,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE1D,gGAAgG;AAChG,MAAM,MAAM,cAAc,GAAG,eAAe,GAAG;IAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAE3E,2CAA2C;AAC3C,MAAM,WAAW,UAAU;IACzB,SAAS;IACT,SAAS,EAAE,MAAM,CAAC;IAClB,+EAA+E;IAC/E,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,8EAA8E;IAC9E,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,OAAO,CAAC;CAC1B;AAED,4EAA4E;AAC5E,MAAM,WAAW,cAAc;IAC7B,2DAA2D;IAC3D,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,+EAA+E;IAC/E,KAAK,EAAE,KAAK,CAAC;IACb,MAAM,CAAC,EAAE,UAAU,CAAC;CACrB;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,QAAQ;QAChB,MAAM,CAAC,EAAE,cAAc,CAAC;KACzB;CACF;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,oDAAoD;IACpD,IAAI,EAAE,MAAM,CAAC;IACb,2CAA2C;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,cAAc,GAAG,SAAS,CAAC;CACpC;AAED;;;GAGG;AACH,wBAAgB,WAAW,CACzB,KAAK,EAAE,SAAS,UAAU,EAAE,EAC5B,SAAS,EAAE,OAAO,EAClB,UAAU,GAAE,KAAmB,GAC9B,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAgDzB;AAED,kGAAkG;AAClG,wBAAgB,YAAY,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,EAAE,UAAU,GAAE,KAAmB,GAAG,KAAK,CAEjG;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,EAAE,UAAU,GAAE,KAAmB,GAAG,MAAM,EAAE,CAiBpG"}

package/dist/report.js

@@ -6,9 +6,13 @@
  * outcome. `attest policy evaluate` gates on `results`; the pr-annotate action
  * places inline comments with `file` and `line`.
  */
-import { STATUS_HARD_FAIL, STATUS_NOT_EVALUATED, STATUS_PASS, STATUS_SOFT_FAIL, } from "@attest-ai/core";
+import { STATUS_HARD_FAIL, STATUS_NOT_EVALUATED, STATUS_PASS, STATUS_SOFT_FAIL, sumUsage, usageSummaryLine, zeroUsage, } from "@attest-ai/core";
 export const REPORT_VERSION = 2;
-export function buildReport(tests, simulated) {
+/**
+ * `unattached` is the usage of evaluations that belong to no test (beforeAll,
+ * scripts); it counts in `summary.usage`.
+ */
+export function buildReport(tests, simulated, unattached = zeroUsage()) {
     const rows = [];
     const testRows = [];
     let totalCost = 0;
@@ -16,13 +20,13 @@ export function buildReport(tests, simulated) {
     for (const test of tests) {
         if (test.line < 1)
             throw new Error(`${test.testId}: line must be >= 1, got ${test.line}`);
-        const assertionIds = [];
+        const assertionIds = new Set();
         const reasons = [];
         totalCost += test.attest?.totalCost ?? 0;
         totalDurationMs += test.attest?.totalDurationMs ?? 0;
         for (const result of test.attest?.results ?? []) {
             rows.push({ ...result, test_id: test.testId, file: test.file, line: test.line });
-            assertionIds.push(result.assertion_id);
+            assertionIds.add(result.assertion_id);
             if (result.status === STATUS_NOT_EVALUATED && !reasons.includes(String(result.reason))) {
                 reasons.push(String(result.reason));
             }
@@ -34,8 +38,9 @@ export function buildReport(tests, simulated) {
             line: test.line,
             outcome: test.outcome,
             ...(reasons.length > 0 ? { not_evaluated_reasons: reasons } : {}),
-            assertion_ids: assertionIds,
+            assertion_ids: [...assertionIds],
             duration_ms: Math.round(test.durationMs),
+            ...(test.attest?.trials !== undefined ? { trials: test.attest.trials.run } : {}),
         });
     }
     const count = (status) => rows.filter((r) => r["status"] === status).length;
@@ -49,6 +54,7 @@ export function buildReport(tests, simulated) {
             soft_fail: count(STATUS_SOFT_FAIL),
             hard_fail: count(STATUS_HARD_FAIL),
             not_evaluated: count(STATUS_NOT_EVALUATED),
+            usage: sessionUsage(tests, unattached),
         },
         results: rows,
         tests: testRows,
@@ -56,4 +62,30 @@ export function buildReport(tests, simulated) {
         total_duration_ms: totalDurationMs,
     };
 }
+/** Attest usage summed over every test of the run plus the evaluations that belong to no test. */
+export function sessionUsage(tests, unattached = zeroUsage()) {
+    return sumUsage([unattached, ...tests.flatMap((t) => (t.attest === undefined ? [] : [t.attest.usage]))]);
+}
+/**
+ * Lines for the end of the run: one per trials test
+ * ("<test_id>: <passed>/<evaluated> trials passed (<n> not evaluated)", plus a
+ * note when the test budget stopped it early), then the "attest usage: …" line
+ * when the session made or refused a model call.
+ */
+export function summaryLines(tests, unattached = zeroUsage()) {
+    const lines = [];
+    for (const test of tests) {
+        const trials = test.attest?.trials;
+        if (trials === undefined)
+            continue;
+        lines.push(`${test.testId}: ${trials.passed}/${trials.evaluated} trials passed (${trials.notEvaluated} not evaluated)`);
+        if (trials.stoppedByBudget) {
+            lines.push(`${test.testId}: stopped after ${trials.run} of ${trials.requested} trials: test budget spent`);
+        }
+    }
+    const usage = usageSummaryLine(sessionUsage(tests, unattached));
+    if (usage !== undefined)
+        lines.push(usage);
+    return lines;
+}
 //# sourceMappingURL=report.js.map

package/dist/report.js.map

@@ -1 +1 @@
-{"version":3,"file":"report.js","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EACL,gBAAgB,EAChB,oBAAoB,EACpB,WAAW,EACX,gBAAgB,GAEjB,MAAM,iBAAiB,CAAC;AAEzB,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC;AA+BhC,MAAM,UAAU,WAAW,CAAC,KAA4B,EAAE,SAAkB;IAC1E,MAAM,IAAI,GAA8B,EAAE,CAAC;IAC3C,MAAM,QAAQ,GAA8B,EAAE,CAAC;IAC/C,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,MAAM,4BAA4B,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1F,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,SAAS,IAAI,IAAI,CAAC,MAAM,EAAE,SAAS,IAAI,CAAC,CAAC;QACzC,eAAe,IAAI,IAAI,CAAC,MAAM,EAAE,eAAe,IAAI,CAAC,CAAC;QACrD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC;YAChD,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YACjF,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YACvC,IAAI,MAAM,CAAC,MAAM,KAAK,oBAAoB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;gBACvF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC;YACZ,OAAO,EAAE,IAAI,CAAC,MAAM;YACpB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,qBAAqB,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,aAAa,EAAE,YAAY;YAC3B,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC;SACzC,CAAC,CAAC;IACL,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IACpF,OAAO;QACL,cAAc,EAAE,cAAc;QAC9B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;QAC7D,SAAS;QACT,OAAO,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC;YAC1B,SAAS,EAAE,KAAK,CAAC,gBAAgB,CAAC;YAClC,SAAS,EAAE,KAAK,CAAC,gBAAgB,CAAC;YAClC,aAAa,EAAE,KAAK,CAAC,oBAAoB,CAAC;SAC3C;QACD,OAAO,EAAE,IAAI;QACb,KAAK,EAAE,QAAQ;QACf,UAAU,EAAE,SAAS;QACrB,iBAAiB,EAAE,eAAe;KACnC,CAAC;AACJ,CAAC"}
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EACL,gBAAgB,EAChB,oBAAoB,EACpB,WAAW,EACX,gBAAgB,EAChB,QAAQ,EACR,gBAAgB,EAChB,SAAS,GAGV,MAAM,iBAAiB,CAAC;AAEzB,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC;AAkDhC;;;GAGG;AACH,MAAM,UAAU,WAAW,CACzB,KAA4B,EAC5B,SAAkB,EAClB,aAAoB,SAAS,EAAE;IAE/B,MAAM,IAAI,GAA8B,EAAE,CAAC;IAC3C,MAAM,QAAQ,GAA8B,EAAE,CAAC;IAC/C,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,MAAM,4BAA4B,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1F,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QACvC,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,SAAS,IAAI,IAAI,CAAC,MAAM,EAAE,SAAS,IAAI,CAAC,CAAC;QACzC,eAAe,IAAI,IAAI,CAAC,MAAM,EAAE,eAAe,IAAI,CAAC,CAAC;QACrD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC;YAChD,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YACjF,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YACtC,IAAI,MAAM,CAAC,MAAM,KAAK,oBAAoB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;gBACvF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC;YACZ,OAAO,EAAE,IAAI,CAAC,MAAM;YACpB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,qBAAqB,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,aAAa,EAAE,CAAC,GAAG,YAAY,CAAC;YAChC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC;YACxC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACjF,CAAC,CAAC;IACL,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IACpF,OAAO;QACL,cAAc,EAAE,cAAc;QAC9B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;QAC7D,SAAS;QACT,OAAO,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC;YAC1B,SAAS,EAAE,KAAK,CAAC,gBAAgB,CAAC;YAClC,SAAS,EAAE,KAAK,CAAC,gBAAgB,CAAC;YAClC,aAAa,EAAE,KAAK,CAAC,oBAAoB,CAAC;YAC1C,KAAK,EAAE,YAAY,CAAC,KAAK,EAAE,UAAU,CAAC;SACvC;QACD,OAAO,EAAE,IAAI;QACb,KAAK,EAAE,QAAQ;QACf,UAAU,EAAE,SAAS;QACrB,iBAAiB,EAAE,eAAe;KACnC,CAAC;AACJ,CAAC;AAED,kGAAkG;AAClG,MAAM,UAAU,YAAY,CAAC,KAA4B,EAAE,aAAoB,SAAS,EAAE;IACxF,OAAO,QAAQ,CAAC,CAAC,UAAU,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC3G,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,KAA4B,EAAE,aAAoB,SAAS,EAAE;IACxF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC;QACnC,IAAI,MAAM,KAAK,SAAS;YAAE,SAAS;QACnC,KAAK,CAAC,IAAI,CACR,GAAG,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,SAAS,mBAAmB,MAAM,CAAC,YAAY,iBAAiB,CAC5G,CAAC;QACF,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CACR,GAAG,IAAI,CAAC,MAAM,mBAAmB,MAAM,CAAC,GAAG,OAAO,MAAM,CAAC,SAAS,4BAA4B,CAC/F,CAAC;QACJ,CAAC;IACH,CAAC;IACD,MAAM,KAAK,GAAG,gBAAgB,CAAC,YAAY,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC;IAChE,IAAI,KAAK,KAAK,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3C,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/reporter.d.ts

@@ -31,12 +31,21 @@ export interface AttestReporterOptions {
  * `--attest-report=<path>`:
  * `vitest run --reporter=default --reporter=@attest-ai/vitest/reporter --outputFile.attest=<path>`. Turns on `includeTaskLocation` so each test carries its
  * line. Requires Vitest 3+.
+ *
+ * At the end of the run it prints, with or without an output file, one line per
+ * `trials()` test ("<test_id>: <passed>/<evaluated> trials passed (<n> not evaluated)")
+ * and the "attest usage: …" line when the run made or refused an attest model call.
  */
 export declare class AttestReporter implements Reporter {
     private outputFile;
     private root;
+    private vitest;
+    private usageDir;
     constructor(options?: AttestReporterOptions);
     onInit(vitest: Vitest): void;
+    onTestRunStart(): void;
+    /** Sum and remove the usage files workers wrote for evaluations outside any test. */
+    private takeUnattachedUsage;
     onTestRunEnd(testModules: ReadonlyArray<TestModule>): void;
 }
 /** `vitest run --reporter=@attest-ai/vitest/reporter` loads the default export. */

package/dist/reporter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../src/reporter.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGhE,UAAU,eAAe;IACvB,SAAS,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACtC;AAED;;;;;;;GAOG;AACH,qBAAa,kBAAkB;IAC7B,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,IAAI;CAwC/E;AAED,MAAM,WAAW,qBAAqB;IACpC;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,cAAe,YAAW,QAAQ;IAC7C,OAAO,CAAC,UAAU,CAAqB;IACvC,OAAO,CAAC,IAAI,CAAiB;gBAEjB,OAAO,GAAE,qBAA0B;IAI/C,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAS5B,YAAY,CAAC,WAAW,EAAE,aAAa,CAAC,UAAU,CAAC,GAAG,IAAI;CA+B3D;AAED,mFAAmF;AACnF,eAAe,cAAc,CAAC"}
+{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../src/reporter.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAIhE,UAAU,eAAe;IACvB,SAAS,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACtC;AAED;;;;;;;GAOG;AACH,qBAAa,kBAAkB;IAC7B,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,IAAI;CAwC/E;AAED,MAAM,WAAW,qBAAqB;IACpC;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,cAAe,YAAW,QAAQ;IAC7C,OAAO,CAAC,UAAU,CAAqB;IACvC,OAAO,CAAC,IAAI,CAAiB;IAC7B,OAAO,CAAC,MAAM,CAAqB;IACnC,OAAO,CAAC,QAAQ,CAAqB;gBAEzB,OAAO,GAAE,qBAA0B;IAI/C,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAe5B,cAAc,IAAI,IAAI;IAItB,qFAAqF;IACrF,OAAO,CAAC,mBAAmB;IAS3B,YAAY,CAAC,WAAW,EAAE,aAAa,CAAC,UAAU,CAAC,GAAG,IAAI;CAmC3D;AAED,mFAAmF;AACnF,eAAe,cAAc,CAAC"}

package/dist/reporter.js

@@ -1,7 +1,9 @@
-import { mkdirSync, writeFileSync } from "node:fs";
-import { dirname, relative, resolve, sep } from "node:path";
-import { isSimulationMode, renderDiagnostics, renderSummary } from "@attest-ai/core";
-import { buildReport } from "./report.js";
+import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, relative, resolve, sep } from "node:path";
+import { isSimulationMode, parseUsage, renderDiagnostics, renderSummary, sumUsage, } from "@attest-ai/core";
+import { buildReport, summaryLines } from "./report.js";
+import { ROOT_ENV, USAGE_DIR_ENV } from "./test-state.js";
 /**
  * Vitest reporter that prints the cost report and a hierarchical
  * diagnostic block for every failed Attest evaluation in the run.
@@ -44,31 +46,56 @@ export class AttestCostReporter {
  * `--attest-report=<path>`:
  * `vitest run --reporter=default --reporter=@attest-ai/vitest/reporter --outputFile.attest=<path>`. Turns on `includeTaskLocation` so each test carries its
  * line. Requires Vitest 3+.
+ *
+ * At the end of the run it prints, with or without an output file, one line per
+ * `trials()` test ("<test_id>: <passed>/<evaluated> trials passed (<n> not evaluated)")
+ * and the "attest usage: …" line when the run made or refused an attest model call.
  */
 export class AttestReporter {
     outputFile;
     root = process.cwd();
+    vitest;
+    usageDir;
     constructor(options = {}) {
         this.outputFile = options.outputFile;
     }
     onInit(vitest) {
+        this.vitest = vitest;
         this.root = vitest.config.root;
         const configured = vitest.config.outputFile;
         if (this.outputFile === undefined && typeof configured === "object") {
             this.outputFile = configured["attest"];
         }
         vitest.config.includeTaskLocation = true;
+        // Workers inherit these: test ids hash paths relative to this root, and
+        // evaluations outside any test drop their usage in usageDir.
+        this.usageDir = mkdtempSync(join(tmpdir(), "attest-usage-"));
+        process.env[ROOT_ENV] = this.root;
+        process.env[USAGE_DIR_ENV] = this.usageDir;
+    }
+    onTestRunStart() {
+        if (this.usageDir !== undefined)
+            mkdirSync(this.usageDir, { recursive: true });
+    }
+    /** Sum and remove the usage files workers wrote for evaluations outside any test. */
+    takeUnattachedUsage() {
+        const dir = this.usageDir;
+        if (dir === undefined)
+            return sumUsage([]);
+        const files = readdirSync(dir).filter((f) => f.endsWith(".json"));
+        const usage = sumUsage(files.map((f) => parseUsage(JSON.parse(readFileSync(join(dir, f), "utf8")))));
+        rmSync(dir, { recursive: true, force: true });
+        return usage;
     }
     onTestRunEnd(testModules) {
         const outputFile = this.outputFile;
-        if (outputFile === undefined)
-            return;
         const tests = [];
         for (const testModule of testModules) {
             const file = relative(this.root, testModule.moduleId).split(sep).join("/");
             for (const testCase of testModule.children.allTests()) {
                 const location = testCase.location;
-                if (location === undefined) {
+                // The location matters only for the report file; summary lines need none.
+                if (location === undefined && outputFile !== undefined) {
                     throw new Error(`AttestReporter: no source location for "${testCase.fullName}". Vitest records ` +
                         "locations only when test.includeTaskLocation is true and the test file path matches " +
                         "the root; set test.root to the real path (no symlinks) if it is set.");
@@ -78,16 +105,22 @@ export class AttestReporter {
                     testId: `${file} > ${testCase.fullName}`,
                     name: testCase.name,
                     file,
-                    line: location.line,
+                    line: location?.line ?? 0,
                     outcome: state === "passed" || state === "failed" ? state : "skipped",
                     durationMs: testCase.diagnostic()?.duration ?? 0,
                     attest: testCase.meta().attest,
                 });
             }
         }
+        const unattached = this.takeUnattachedUsage();
+        const lines = summaryLines(tests, unattached);
+        if (lines.length > 0)
+            this.vitest?.logger.log(lines.join("\n"));
+        if (outputFile === undefined)
+            return;
         const path = resolve(this.root, outputFile);
         mkdirSync(dirname(path), { recursive: true });
-        writeFileSync(path, `${JSON.stringify(buildReport(tests, isSimulationMode()), null, 2)}\n`);
+        writeFileSync(path, `${JSON.stringify(buildReport(tests, isSimulationMode(), unattached), null, 2)}\n`);
     }
 }
 /** `vitest run --reporter=@attest-ai/vitest/reporter` loads the default export. */

package/dist/reporter.js.map

@@ -1 +1 @@
-{"version":3,"file":"reporter.js","sourceRoot":"","sources":["../src/reporter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAErF,OAAO,EAAE,WAAW,EAAmB,MAAM,aAAa,CAAC;AAM3D;;;;;;;GAOG;AACH,MAAM,OAAO,kBAAkB;IAC7B,UAAU,CAAC,MAAe,EAAE,OAAgB,EAAE,OAAyB;QACrE,MAAM,IAAI,GAAG,UAAU,CAAC,uBAAuB,IAAI,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,UAAU,CAAC,gCAAgC,IAAI,CAAC,CAAC;QACtE,MAAM,QAAQ,GAAG,UAAU,CAAC,2BAA2B,IAAI,EAAE,CAAC;QAE9D,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CACR,EAAE,EACF,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EACd,sBAAsB,EACtB,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CACf,CAAC;YACF,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;gBAC9B,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC;gBACtC,KAAK,CAAC,IAAI,CAAC,cAAc,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;QAED,KAAK,CAAC,IAAI,CACR,EAAE,EACF,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EACd,sBAAsB,EACtB,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EACd,mCAAmC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,EACxD,mCAAmC,YAAY,EAAE,EACjD,mCAAmC,QAAQ,CAAC,MAAM,EAAE,EACpD,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EACd,EAAE,CACH,CAAC;QAEF,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEhC,IAAI,OAAO,EAAE,SAAS,EAAE,CAAC;YACvB,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;CACF;AAUD;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,cAAc;IACjB,UAAU,CAAqB;IAC/B,IAAI,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAE7B,YAAY,UAAiC,EAAE;QAC7C,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;IACvC,CAAC;IAED,MAAM,CAAC,MAAc;QACnB,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;QAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC;QAC5C,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;YACpE,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACzC,CAAC;QACD,MAAM,CAAC,MAAM,CAAC,mBAAmB,GAAG,IAAI,CAAC;IAC3C,CAAC;IAED,YAAY,CAAC,WAAsC;QACjD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACnC,IAAI,UAAU,KAAK,SAAS;YAAE,OAAO;QACrC,MAAM,KAAK,GAAiB,EAAE,CAAC;QAC/B,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC3E,KAAK,MAAM,QAAQ,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC;gBACtD,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;gBACnC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;oBAC3B,MAAM,IAAI,KAAK,CACb,2CAA2C,QAAQ,CAAC,QAAQ,oBAAoB;wBAC9E,sFAAsF;wBACtF,sEAAsE,CACzE,CAAC;gBACJ,CAAC;gBACD,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC;gBACtC,KAAK,CAAC,IAAI,CAAC;oBACT,MAAM,EAAE,GAAG,IAAI,MAAM,QAAQ,CAAC,QAAQ,EAAE;oBACxC,IAAI,EAAE,QAAQ,CAAC,IAAI;oBACnB,IAAI;oBACJ,IAAI,EAAE,QAAQ,CAAC,IAAI;oBACnB,OAAO,EAAE,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;oBACrE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,EAAE,QAAQ,IAAI,CAAC;oBAChD,MAAM,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM;iBAC/B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAC5C,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,aAAa,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,KAAK,EAAE,gBAAgB,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IAC9F,CAAC;CACF;AAED,mFAAmF;AACnF,eAAe,cAAc,CAAC"}
+{"version":3,"file":"reporter.js","sourceRoot":"","sources":["../src/reporter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnG,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAClE,OAAO,EACL,gBAAgB,EAChB,UAAU,EACV,iBAAiB,EACjB,aAAa,EACb,QAAQ,GAET,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,WAAW,EAAE,YAAY,EAAmB,MAAM,aAAa,CAAC;AACzE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAM1D;;;;;;;GAOG;AACH,MAAM,OAAO,kBAAkB;IAC7B,UAAU,CAAC,MAAe,EAAE,OAAgB,EAAE,OAAyB;QACrE,MAAM,IAAI,GAAG,UAAU,CAAC,uBAAuB,IAAI,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,UAAU,CAAC,gCAAgC,IAAI,CAAC,CAAC;QACtE,MAAM,QAAQ,GAAG,UAAU,CAAC,2BAA2B,IAAI,EAAE,CAAC;QAE9D,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CACR,EAAE,EACF,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EACd,sBAAsB,EACtB,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CACf,CAAC;YACF,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;gBAC9B,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC;gBACtC,KAAK,CAAC,IAAI,CAAC,cAAc,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;QAED,KAAK,CAAC,IAAI,CACR,EAAE,EACF,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EACd,sBAAsB,EACtB,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EACd,mCAAmC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,EACxD,mCAAmC,YAAY,EAAE,EACjD,mCAAmC,QAAQ,CAAC,MAAM,EAAE,EACpD,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EACd,EAAE,CACH,CAAC;QAEF,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEhC,IAAI,OAAO,EAAE,SAAS,EAAE,CAAC;YACvB,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;CACF;AAUD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,OAAO,cAAc;IACjB,UAAU,CAAqB;IAC/B,IAAI,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IACrB,MAAM,CAAqB;IAC3B,QAAQ,CAAqB;IAErC,YAAY,UAAiC,EAAE;QAC7C,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;IACvC,CAAC;IAED,MAAM,CAAC,MAAc;QACnB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;QAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC;QAC5C,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;YACpE,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACzC,CAAC;QACD,MAAM,CAAC,MAAM,CAAC,mBAAmB,GAAG,IAAI,CAAC;QACzC,wEAAwE;QACxE,6DAA6D;QAC7D,IAAI,CAAC,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,eAAe,CAAC,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC;IAC7C,CAAC;IAED,cAAc;QACZ,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS;YAAE,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACjF,CAAC;IAED,qFAAqF;IAC7E,mBAAmB;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC1B,IAAI,GAAG,KAAK,SAAS;YAAE,OAAO,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAClE,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrG,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,OAAO,KAAK,CAAC;IACf,CAAC;IAED,YAAY,CAAC,WAAsC;QACjD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACnC,MAAM,KAAK,GAAiB,EAAE,CAAC;QAC/B,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC3E,KAAK,MAAM,QAAQ,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC;gBACtD,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;gBACnC,0EAA0E;gBAC1E,IAAI,QAAQ,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;oBACvD,MAAM,IAAI,KAAK,CACb,2CAA2C,QAAQ,CAAC,QAAQ,oBAAoB;wBAC9E,sFAAsF;wBACtF,sEAAsE,CACzE,CAAC;gBACJ,CAAC;gBACD,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC;gBACtC,KAAK,CAAC,IAAI,CAAC;oBACT,MAAM,EAAE,GAAG,IAAI,MAAM,QAAQ,CAAC,QAAQ,EAAE;oBACxC,IAAI,EAAE,QAAQ,CAAC,IAAI;oBACnB,IAAI;oBACJ,IAAI,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC;oBACzB,OAAO,EAAE,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;oBACrE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,EAAE,QAAQ,IAAI,CAAC;oBAChD,MAAM,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM;iBAC/B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,MAAM,UAAU,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAC9C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAChE,IAAI,UAAU,KAAK,SAAS;YAAE,OAAO;QACrC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAC5C,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,aAAa,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,KAAK,EAAE,gBAAgB,EAAE,EAAE,UAAU,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IAC1G,CAAC;CACF;AAED,mFAAmF;AACnF,eAAe,cAAc,CAAC"}

package/dist/test-state.d.ts

@@ -0,0 +1,100 @@
+/**
+ * Per-test state shared by evaluate/inspect and trials: deterministic assertion
+ * IDs, the per-test budget remainder, and the results recorded on task meta.
+ */
+import { AsyncLocalStorage } from "node:async_hooks";
+import type { TestContext } from "vitest";
+import { type AgentResult, type Assertion, type BatchBudget, type Trace, type Usage } from "@attest-ai/core";
+import type { AttestTaskMeta } from "./report.js";
+export type RunningTest = NonNullable<TestContext["task"]>;
+/** Limits for the attest model calls of one test (all its evaluations and trials). */
+export interface TestBudget {
+    /** Judge-provider requests the test may make. */
+    maxJudgeCalls?: number;
+    /** USD the test may spend: attest judge/embedding cost plus agent spend (trace `metadata.cost_usd`). */
+    maxCostUsd?: number;
+}
+/** The trials() call a body runs in; `trial` is undefined for a single run (N = 1). */
+export interface TrialScope {
+    test: RunningTest;
+    trial: number | undefined;
+    budget: TestBudget | undefined;
+    /** Evaluations of the current trial. */
+    results: AgentResult[];
+}
+export declare const trialScope: AsyncLocalStorage<TrialScope>;
+/** True while the code runs inside trial i of a trials() call with N > 1. */
+export declare function inTrial(): boolean;
+interface AttemptState {
+    attempt: number;
+    /** Assertions given an ID in this test (this trial, under trials()). */
+    ordinal: number;
+    /** Attest usage of this attempt so far. */
+    usage: Usage;
+    /** Sum of `metadata.cost_usd` of the traces evaluated in this attempt. */
+    agentSpendUsd: number;
+    traceIds: Set<string>;
+}
+/** State of the running attempt of `test`; a retried test starts afresh. */
+export declare function attemptState(test: RunningTest): AttemptState;
+/**
+ * The running test: `TestRunner.getCurrentTest()` on Vitest 4.1+, `vitest/suite`
+ * before that (loaded only there, because Vitest 4.1 warns when it is imported).
+ */
+export declare function runningTest(): Promise<RunningTest | undefined>;
+/**
+ * The test an evaluation belongs to: the passed context, else the enclosing
+ * trials() call, else Vitest's running test. Under test.concurrent the running
+ * test may be another test, so outside trials() the caller must pass `context`.
+ */
+export declare function resolveTest(context: TestContext | undefined): Promise<RunningTest | undefined>;
+/**
+ * Set by AttestReporter in the main process before workers start: the Vitest
+ * root its report paths are relative to, and the directory where workers drop
+ * the usage of evaluations that belong to no test.
+ */
+export declare const ROOT_ENV = "ATTEST_VITEST_ROOT";
+export declare const USAGE_DIR_ENV = "ATTEST_VITEST_USAGE_DIR";
+/**
+ * The report's test_id of `test`: "<file> > <suites> > <name>", the same string
+ * AttestReporter writes. The file is relative to the Vitest root AttestReporter
+ * publishes (in a multi-project setup, not the project root); without
+ * AttestReporter it is the file task's name, relative to the project root.
+ */
+export declare function testIdOf(test: RunningTest): string;
+/**
+ * Hand the usage of an evaluation that belongs to no test (beforeAll, a script)
+ * to AttestReporter: one file per evaluation in its usage directory, so the
+ * session usage counts it once. Without AttestReporter there is no report to add it to.
+ */
+export declare function recordUnattachedUsage(usage: Usage): void;
+/** `assert_` + the first 12 hex chars of sha256("<test_id>#<ordinal>"). */
+export declare function deterministicAssertionId(testId: string, ordinal: number): string;
+/** Give each assertion the next deterministic ID of the test. */
+export declare function assignAssertionIds(test: RunningTest, assertions: readonly Assertion[]): Assertion[];
+/** Reject a budget with unknown keys or negative/non-numeric limits. */
+export declare function validateBudget(budget: TestBudget): TestBudget;
+/**
+ * What is left of the test budget for the next evaluate_batch: judge calls not
+ * yet used, and USD not yet spent by attest or the agent, each floored at 0.
+ * Undefined when the budget sets no limit.
+ */
+export declare function remainingBudget(budget: TestBudget, usage: Usage, agentSpendUsd: number): BatchBudget | undefined;
+/** True when attest cost plus agent spend has reached the test's cost limit. */
+export declare function costBudgetSpent(budget: TestBudget | undefined, state: AttemptState): boolean;
+/** Count the agent's own spend of a trace once per attempt. */
+export declare function recordAgentSpend(test: RunningTest, trace: Trace): void;
+/**
+ * The meta of the running attempt. Results from an earlier attempt of a retried
+ * test are dropped, so the report matches the outcome Vitest reports; their
+ * usage is kept, because those model calls were made.
+ */
+export declare function attemptMeta(test: RunningTest): AttestTaskMeta;
+/**
+ * Attach the results to the test's meta so AttestReporter, in the main process,
+ * can write them with the test's file and line; rows of trial i carry `trial`.
+ */
+export declare function recordOnTest(test: RunningTest, result: AgentResult, trial: number | undefined): void;
+export declare function strictFromEnv(): boolean;
+export {};
+//# sourceMappingURL=test-state.d.ts.map

package/dist/test-state.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-state.d.ts","sourceRoot":"","sources":["../src/test-state.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAKrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AAC1C,OAAO,EAGL,KAAK,WAAW,EAChB,KAAK,SAAS,EACd,KAAK,WAAW,EAChB,KAAK,KAAK,EACV,KAAK,KAAK,EACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,MAAM,MAAM,WAAW,GAAG,WAAW,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;AAE3D,sFAAsF;AACtF,MAAM,WAAW,UAAU;IACzB,iDAAiD;IACjD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,wGAAwG;IACxG,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,uFAAuF;AACvF,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,WAAW,CAAC;IAClB,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,MAAM,EAAE,UAAU,GAAG,SAAS,CAAC;IAC/B,wCAAwC;IACxC,OAAO,EAAE,WAAW,EAAE,CAAC;CACxB;AAED,eAAO,MAAM,UAAU,+BAAsC,CAAC;AAE9D,6EAA6E;AAC7E,wBAAgB,OAAO,IAAI,OAAO,CAEjC;AAED,UAAU,YAAY;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,wEAAwE;IACxE,OAAO,EAAE,MAAM,CAAC;IAChB,2CAA2C;IAC3C,KAAK,EAAE,KAAK,CAAC;IACb,0EAA0E;IAC1E,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;CACvB;AAQD,4EAA4E;AAC5E,wBAAgB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,YAAY,CAQ5D;AAED;;;GAGG;AACH,wBAAsB,WAAW,IAAI,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAKpE;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,SAAS,GAAG,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAYpG;AAED;;;;GAIG;AACH,eAAO,MAAM,QAAQ,uBAAuB,CAAC;AAC7C,eAAO,MAAM,aAAa,4BAA4B,CAAC;AAEvD;;;;;GAKG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,WAAW,GAAG,MAAM,CAWlD;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAMxD;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAEhF;AAED,iEAAiE;AACjE,wBAAgB,kBAAkB,CAChC,IAAI,EAAE,WAAW,EACjB,UAAU,EAAE,SAAS,SAAS,EAAE,GAC/B,SAAS,EAAE,CAIb;AAED,wEAAwE;AACxE,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,UAAU,CAc7D;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAC7B,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,KAAK,EACZ,aAAa,EAAE,MAAM,GACpB,WAAW,GAAG,SAAS,CASzB;AAED,gFAAgF;AAChF,wBAAgB,eAAe,CAAC,MAAM,EAAE,UAAU,GAAG,SAAS,EAAE,KAAK,EAAE,YAAY,GAAG,OAAO,CAI5F;AAED,+DAA+D;AAC/D,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI,CAKtE;AAED;;;;GAIG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,WAAW,GAAG,cAAc,CAc7D;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAUpG;AAED,wBAAgB,aAAa,IAAI,OAAO,CAGvC"}

package/dist/test-state.js

@@ -0,0 +1,187 @@
+/**
+ * Per-test state shared by evaluate/inspect and trials: deterministic assertion
+ * IDs, the per-test budget remainder, and the results recorded on task meta.
+ */
+import { AsyncLocalStorage } from "node:async_hooks";
+import { createHash, randomUUID } from "node:crypto";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join, relative, sep } from "node:path";
+import * as vitest from "vitest";
+import { addUsage, zeroUsage, } from "@attest-ai/core";
+export const trialScope = new AsyncLocalStorage();
+/** True while the code runs inside trial i of a trials() call with N > 1. */
+export function inTrial() {
+    return trialScope.getStore()?.trial !== undefined;
+}
+const attempts = new WeakMap();
+function attemptOf(test) {
+    return test.result?.retryCount ?? 0;
+}
+/** State of the running attempt of `test`; a retried test starts afresh. */
+export function attemptState(test) {
+    const attempt = attemptOf(test);
+    let state = attempts.get(test);
+    if (state === undefined || state.attempt !== attempt) {
+        state = { attempt, ordinal: 0, usage: zeroUsage(), agentSpendUsd: 0, traceIds: new Set() };
+        attempts.set(test, state);
+    }
+    return state;
+}
+/**
+ * The running test: `TestRunner.getCurrentTest()` on Vitest 4.1+, `vitest/suite`
+ * before that (loaded only there, because Vitest 4.1 warns when it is imported).
+ */
+export async function runningTest() {
+    const runner = vitest.TestRunner;
+    if (runner !== undefined)
+        return runner.getCurrentTest();
+    const suite = await import("vitest/suite");
+    return suite.getCurrentTest();
+}
+/**
+ * The test an evaluation belongs to: the passed context, else the enclosing
+ * trials() call, else Vitest's running test. Under test.concurrent the running
+ * test may be another test, so outside trials() the caller must pass `context`.
+ */
+export async function resolveTest(context) {
+    if (context !== undefined)
+        return context.task;
+    const scope = trialScope.getStore();
+    if (scope !== undefined)
+        return scope.test;
+    const test = await runningTest();
+    if (test !== undefined && test.concurrent === true) {
+        throw new Error("Attest results from a test.concurrent test — pass the test context: " +
+            "inspect(chain, { context }), evaluate(chain, { context }) or trials(body, { context })");
+    }
+    return test;
+}
+/**
+ * Set by AttestReporter in the main process before workers start: the Vitest
+ * root its report paths are relative to, and the directory where workers drop
+ * the usage of evaluations that belong to no test.
+ */
+export const ROOT_ENV = "ATTEST_VITEST_ROOT";
+export const USAGE_DIR_ENV = "ATTEST_VITEST_USAGE_DIR";
+/**
+ * The report's test_id of `test`: "<file> > <suites> > <name>", the same string
+ * AttestReporter writes. The file is relative to the Vitest root AttestReporter
+ * publishes (in a multi-project setup, not the project root); without
+ * AttestReporter it is the file task's name, relative to the project root.
+ */
+export function testIdOf(test) {
+    const names = [test.name];
+    for (let suite = test.suite; suite !== undefined && suite !== test.file; suite = suite.suite) {
+        names.unshift(suite.name);
+    }
+    const root = process.env[ROOT_ENV];
+    const file = root === undefined || test.file.filepath === undefined
+        ? test.file.name
+        : relative(root, test.file.filepath).split(sep).join("/");
+    return `${file} > ${names.join(" > ")}`;
+}
+/**
+ * Hand the usage of an evaluation that belongs to no test (beforeAll, a script)
+ * to AttestReporter: one file per evaluation in its usage directory, so the
+ * session usage counts it once. Without AttestReporter there is no report to add it to.
+ */
+export function recordUnattachedUsage(usage) {
+    const dir = process.env[USAGE_DIR_ENV];
+    if (dir === undefined)
+        return;
+    if (Object.values(usage).every((v) => v === 0))
+        return;
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(join(dir, `${randomUUID()}.json`), JSON.stringify(usage));
+}
+/** `assert_` + the first 12 hex chars of sha256("<test_id>#<ordinal>"). */
+export function deterministicAssertionId(testId, ordinal) {
+    return `assert_${createHash("sha256").update(`${testId}#${ordinal}`).digest("hex").slice(0, 12)}`;
+}
+/** Give each assertion the next deterministic ID of the test. */
+export function assignAssertionIds(test, assertions) {
+    const state = attemptState(test);
+    const testId = testIdOf(test);
+    return assertions.map((a) => ({ ...a, assertion_id: deterministicAssertionId(testId, state.ordinal++) }));
+}
+/** Reject a budget with unknown keys or negative/non-numeric limits. */
+export function validateBudget(budget) {
+    for (const key of Object.keys(budget)) {
+        if (key !== "maxJudgeCalls" && key !== "maxCostUsd") {
+            throw new Error(`Attest budget: unknown option "${key}" (expected maxJudgeCalls, maxCostUsd)`);
+        }
+    }
+    const { maxJudgeCalls, maxCostUsd } = budget;
+    if (maxJudgeCalls !== undefined && !(Number.isInteger(maxJudgeCalls) && maxJudgeCalls >= 0)) {
+        throw new Error(`Attest budget: maxJudgeCalls must be an integer >= 0, got ${String(maxJudgeCalls)}`);
+    }
+    if (maxCostUsd !== undefined && !(Number.isFinite(maxCostUsd) && maxCostUsd >= 0)) {
+        throw new Error(`Attest budget: maxCostUsd must be a number >= 0, got ${String(maxCostUsd)}`);
+    }
+    return budget;
+}
+/**
+ * What is left of the test budget for the next evaluate_batch: judge calls not
+ * yet used, and USD not yet spent by attest or the agent, each floored at 0.
+ * Undefined when the budget sets no limit.
+ */
+export function remainingBudget(budget, usage, agentSpendUsd) {
+    const wire = {};
+    if (budget.maxJudgeCalls !== undefined) {
+        wire.max_judge_calls = Math.max(0, budget.maxJudgeCalls - usage.judge_calls);
+    }
+    if (budget.maxCostUsd !== undefined) {
+        wire.max_cost_usd = Math.max(0, budget.maxCostUsd - (usage.cost_usd + agentSpendUsd));
+    }
+    return Object.keys(wire).length === 0 ? undefined : wire;
+}
+/** True when attest cost plus agent spend has reached the test's cost limit. */
+export function costBudgetSpent(budget, state) {
+    return (budget?.maxCostUsd !== undefined && state.usage.cost_usd + state.agentSpendUsd >= budget.maxCostUsd);
+}
+/** Count the agent's own spend of a trace once per attempt. */
+export function recordAgentSpend(test, trace) {
+    const state = attemptState(test);
+    if (state.traceIds.has(trace.trace_id))
+        return;
+    state.traceIds.add(trace.trace_id);
+    state.agentSpendUsd += trace.metadata?.cost_usd ?? 0;
+}
+/**
+ * The meta of the running attempt. Results from an earlier attempt of a retried
+ * test are dropped, so the report matches the outcome Vitest reports; their
+ * usage is kept, because those model calls were made.
+ */
+export function attemptMeta(test) {
+    const attempt = attemptOf(test);
+    let meta = test.meta.attest;
+    if (meta === undefined || meta.attempt !== attempt) {
+        meta = {
+            attempt,
+            results: [],
+            totalCost: 0,
+            totalDurationMs: 0,
+            usage: meta?.usage ?? zeroUsage(),
+        };
+        test.meta.attest = meta;
+    }
+    return meta;
+}
+/**
+ * Attach the results to the test's meta so AttestReporter, in the main process,
+ * can write them with the test's file and line; rows of trial i carry `trial`.
+ */
+export function recordOnTest(test, result, trial) {
+    const state = attemptState(test);
+    state.usage = addUsage(state.usage, result.usage);
+    const meta = attemptMeta(test);
+    meta.results.push(...result.assertionResults.map((r) => (trial === undefined ? r : { ...r, trial })));
+    meta.totalCost += result.totalCost;
+    meta.totalDurationMs += result.totalDurationMs;
+    meta.usage = addUsage(meta.usage, result.usage);
+}
+export function strictFromEnv() {
+    const v = (process.env["ATTEST_STRICT"] ?? "").trim().toLowerCase();
+    return v === "1" || v === "true" || v === "yes";
+}
+//# sourceMappingURL=test-state.js.map

package/dist/test-state.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-state.js","sourceRoot":"","sources":["../src/test-state.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AAEjC,OAAO,EACL,QAAQ,EACR,SAAS,GAMV,MAAM,iBAAiB,CAAC;AAsBzB,MAAM,CAAC,MAAM,UAAU,GAAG,IAAI,iBAAiB,EAAc,CAAC;AAE9D,6EAA6E;AAC7E,MAAM,UAAU,OAAO;IACrB,OAAO,UAAU,CAAC,QAAQ,EAAE,EAAE,KAAK,KAAK,SAAS,CAAC;AACpD,CAAC;AAaD,MAAM,QAAQ,GAAG,IAAI,OAAO,EAA6B,CAAC;AAE1D,SAAS,SAAS,CAAC,IAAiB;IAClC,OAAO,IAAI,CAAC,MAAM,EAAE,UAAU,IAAI,CAAC,CAAC;AACtC,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,YAAY,CAAC,IAAiB;IAC5C,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;QACrD,KAAK,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,aAAa,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;QAC3F,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW;IAC/B,MAAM,MAAM,GAAI,MAAiC,CAAC,UAAU,CAAC;IAC7D,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,cAAc,EAAE,CAAC;IACzD,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;IAC3C,OAAO,KAAK,CAAC,cAAc,EAAE,CAAC;AAChC,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAAgC;IAChE,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,OAAO,CAAC,IAAI,CAAC;IAC/C,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;IACpC,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC;IAC3C,MAAM,IAAI,GAAG,MAAM,WAAW,EAAE,CAAC;IACjC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CACb,sEAAsE;YACpE,wFAAwF,CAC3F,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,oBAAoB,CAAC;AAC7C,MAAM,CAAC,MAAM,aAAa,GAAG,yBAAyB,CAAC;AAEvD;;;;;GAKG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAiB;IACxC,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,CAAC,IAAI,EAAE,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;QAC7F,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,IAAI,GACR,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS;QACpD,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;QAChB,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9D,OAAO,GAAG,IAAI,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;AAC1C,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAY;IAChD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IACvC,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO;IAC9B,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;QAAE,OAAO;IACvD,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;AAC1E,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,wBAAwB,CAAC,MAAc,EAAE,OAAe;IACtE,OAAO,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;AACpG,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,kBAAkB,CAChC,IAAiB,EACjB,UAAgC;IAEhC,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC9B,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,YAAY,EAAE,wBAAwB,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5G,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,cAAc,CAAC,MAAkB;IAC/C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACtC,IAAI,GAAG,KAAK,eAAe,IAAI,GAAG,KAAK,YAAY,EAAE,CAAC;YACpD,MAAM,IAAI,KAAK,CAAC,kCAAkC,GAAG,wCAAwC,CAAC,CAAC;QACjG,CAAC;IACH,CAAC;IACD,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;IAC7C,IAAI,aAAa,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,IAAI,aAAa,IAAI,CAAC,CAAC,EAAE,CAAC;QAC5F,MAAM,IAAI,KAAK,CAAC,6DAA6D,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IACxG,CAAC;IACD,IAAI,UAAU,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC;QAClF,MAAM,IAAI,KAAK,CAAC,wDAAwD,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IAChG,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAC7B,MAAkB,EAClB,KAAY,EACZ,aAAqB;IAErB,MAAM,IAAI,GAAwD,EAAE,CAAC;IACrE,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QACvC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QACpC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,GAAG,CAAC,KAAK,CAAC,QAAQ,GAAG,aAAa,CAAC,CAAC,CAAC;IACxF,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;AAC3D,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,eAAe,CAAC,MAA8B,EAAE,KAAmB;IACjF,OAAO,CACL,MAAM,EAAE,UAAU,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,aAAa,IAAI,MAAM,CAAC,UAAU,CACpG,CAAC;AACJ,CAAC;AAED,+DAA+D;AAC/D,MAAM,UAAU,gBAAgB,CAAC,IAAiB,EAAE,KAAY;IAC9D,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC;QAAE,OAAO;IAC/C,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACnC,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,QAAQ,EAAE,QAAQ,IAAI,CAAC,CAAC;AACvD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,IAAiB;IAC3C,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;IAC5B,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;QACnD,IAAI,GAAG;YACL,OAAO;YACP,OAAO,EAAE,EAAE;YACX,SAAS,EAAE,CAAC;YACZ,eAAe,EAAE,CAAC;YAClB,KAAK,EAAE,IAAI,EAAE,KAAK,IAAI,SAAS,EAAE;SAClC,CAAC;QACF,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IAC1B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,IAAiB,EAAE,MAAmB,EAAE,KAAyB;IAC5F,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,KAAK,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,CAAC,OAAO,CAAC,IAAI,CACf,GAAG,MAAM,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CACnF,CAAC;IACF,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;IACnC,IAAI,CAAC,eAAe,IAAI,MAAM,CAAC,eAAe,CAAC;IAC/C,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;AAClD,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACpE,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,KAAK,CAAC;AAClD,CAAC"}

package/dist/trials.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Fixed-N trials: the Vitest counterpart of pytest's `--attest-trials=N`.
+ */
+import type { TestContext } from "vitest";
+import { type TestBudget } from "./test-state.js";
+/** How attestGlobalSetup({ minPassRate }) reaches the workers that run trials(). */
+export declare const MIN_PASS_RATE_ENV = "ATTEST_MIN_PASS_RATE";
+export interface TrialsOptions {
+    /** Number of trials N; defaults to the ATTEST_TRIALS environment variable, else 1. */
+    n?: number;
+    /**
+     * The running test's context (`it("…", async (context) => …)`); required under
+     * `test.concurrent` and on Vitest before 4.1.
+     */
+    context?: TestContext;
+    /**
+     * Limits for the test's attest model calls across all trials; see
+     * InspectOptions.testBudget. Before each trial after the first, the trials stop
+     * when attest cost plus agent spend has reached `maxCostUsd`.
+     */
+    budget?: TestBudget;
+    /**
+     * Fail the test when passed / evaluated trials is below this rate (0..1).
+     * Overrides the run's value: attestGlobalSetup({ minPassRate }) or the
+     * ATTEST_MIN_PASS_RATE environment variable, else 1.0 (any failing trial fails).
+     */
+    minPassRate?: number;
+}
+/** A pass rate in [0, 1]; anything else is an error naming where it came from. */
+export declare function validateMinPassRate(rate: number, source: string): number;
+/**
+ * Run `body` N times inside the current test (N = `options.n`, else the
+ * ATTEST_TRIALS environment variable, else 1) and enforce only the aggregate.
+ *
+ * With N = 1 the body runs once as a plain test body: evaluate() enforces as
+ * usual. With N > 1, during trial i (1-based): evaluate() does not enforce
+ * (it behaves as inspect()), enforceOutcome() returns its argument, every
+ * result row is reported with `trial = i`, and assertion IDs restart so each
+ * assertion keeps its ID across trials. A trial that throws counts as failed:
+ * its rows are replaced by one hard_fail row (failure_class broken_code) per
+ * assertion ID the test evaluated; when no trial evaluated anything, the first
+ * error is rethrown. A trial whose own outcome is skip counts as not evaluated,
+ * also under ATTEST_STRICT. After the last trial the test is skipped when no trial was
+ * evaluated (failed under ATTEST_STRICT), fails when passed / evaluated
+ * trials is below the minimum pass rate (default 1.0: any failing trial
+ * fails; not-evaluated trials are left out of the rate), and passes otherwise. AttestReporter prints "<test_id>: <passed>/<evaluated>
+ * trials passed (<n> not evaluated)".
+ *
+ * This is a pass/fail gate on N runs, not a statistical verdict: whether the
+ * pass rate regressed (pass / regressed / inconclusive) comes from
+ * `attest policy evaluate --baseline <tag>` with a `regression` rule over the
+ * report's per-trial rows.
+ */
+export declare function trials(body: (trial: number) => Promise<void> | void, options?: TrialsOptions): Promise<void>;
+//# sourceMappingURL=trials.d.ts.map

package/dist/trials.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"trials.d.ts","sourceRoot":"","sources":["../src/trials.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AAS1C,OAAO,EASL,KAAK,UAAU,EAChB,MAAM,iBAAiB,CAAC;AAEzB,oFAAoF;AACpF,eAAO,MAAM,iBAAiB,yBAAyB,CAAC;AAExD,MAAM,WAAW,aAAa;IAC5B,sFAAsF;IACtF,CAAC,CAAC,EAAE,MAAM,CAAC;IACX;;;OAGG;IACH,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB;;;;OAIG;IACH,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAoBD,kFAAkF;AAClF,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAKxE;AA0ED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAsB,MAAM,CAC1B,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,EAC7C,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,IAAI,CAAC,CAqEf"}

package/dist/trials.js

@@ -0,0 +1,184 @@
+import { AgentResult, FailureClass, STATUS_HARD_FAIL, STATUS_NOT_EVALUATED, renderDiagnostics, } from "@attest-ai/core";
+import { attemptMeta, attemptState, costBudgetSpent, runningTest, strictFromEnv, trialScope, validateBudget, } from "./test-state.js";
+/** How attestGlobalSetup({ minPassRate }) reaches the workers that run trials(). */
+export const MIN_PASS_RATE_ENV = "ATTEST_MIN_PASS_RATE";
+function trialCount(n) {
+    const raw = n ?? Number(process.env["ATTEST_TRIALS"] ?? 1);
+    if (!Number.isInteger(raw) || raw < 1) {
+        const source = n === undefined ? `ATTEST_TRIALS=${process.env["ATTEST_TRIALS"]}` : `n=${n}`;
+        throw new Error(`trials(): N must be an integer >= 1, got ${source}`);
+    }
+    return raw;
+}
+/** A pass rate in [0, 1]; anything else is an error naming where it came from. */
+export function validateMinPassRate(rate, source) {
+    if (!Number.isFinite(rate) || rate < 0 || rate > 1) {
+        throw new Error(`minPassRate must be a number between 0 and 1, got ${source}`);
+    }
+    return rate;
+}
+/** The run's minimum pass rate: ATTEST_MIN_PASS_RATE (set by attestGlobalSetup), else 1.0. */
+function runMinPassRate() {
+    const raw = process.env[MIN_PASS_RATE_ENV];
+    if (raw === undefined)
+        return 1;
+    return validateMinPassRate(raw.trim() === "" ? Number.NaN : Number(raw), `${MIN_PASS_RATE_ENV}=${raw}`);
+}
+function describeError(error) {
+    return error instanceof Error ? `${error.name}: ${error.message}` : `${typeof error}: ${String(error)}`;
+}
+/** A trial's own outcome: raised or any hard_fail fails; else not_evaluated (or nothing evaluated) skips. */
+function trialOutcome(trial) {
+    if (trial.error !== undefined)
+        return "fail";
+    const rows = trial.results.flatMap((r) => r.assertionResults);
+    if (rows.some((r) => r.status === STATUS_HARD_FAIL))
+        return "fail";
+    if (rows.length === 0)
+        return "skip";
+    if (rows.some((r) => r.status === STATUS_NOT_EVALUATED))
+        return "skip";
+    return "pass";
+}
+/**
+ * Replace the rows of each trial that threw with exactly one hard_fail row per
+ * assertion ID of the test, so pass rates count the trial as failed once per ID.
+ * When the trial had evaluated the ID before throwing, the explanation keeps
+ * the engine's status.
+ */
+function thrownTrialRows(test, trials) {
+    const meta = attemptMeta(test);
+    const ids = [...new Set(trials.flatMap((t) => t.results.flatMap((r) => r.assertionResults)).map((r) => r.assertion_id))];
+    for (const trial of trials) {
+        if (trial.error === undefined)
+            continue;
+        const before = new Map(trial.results.flatMap((r) => r.assertionResults.map((row) => [row.assertion_id, row.status])));
+        meta.results = meta.results.filter((row) => row.trial !== trial.index);
+        const raised = `trial ${trial.index} raised ${describeError(trial.error.value)}`;
+        for (const id of ids) {
+            const status = before.get(id);
+            const row = {
+                assertion_id: id,
+                status: STATUS_HARD_FAIL,
+                score: 0,
+                explanation: status === undefined ? raised : `${raised} (engine status before the error: ${status})`,
+                failure_class: FailureClass.BrokenCode,
+                trial: trial.index,
+            };
+            meta.results.push(row);
+        }
+    }
+}
+function failureMessage(summary, trials, minPassRate) {
+    const rate = summary.passed / summary.evaluated;
+    const parts = [
+        `${summary.passed}/${summary.evaluated} trials passed (pass rate ${rate.toFixed(2)}) ` +
+            `below min pass rate ${minPassRate.toFixed(2)}`,
+    ];
+    for (const trial of trials) {
+        if (trialOutcome(trial) !== "fail")
+            continue;
+        if (trial.error !== undefined) {
+            parts.push(`trial ${trial.index} raised ${describeError(trial.error.value)}`);
+            continue;
+        }
+        for (const result of trial.results) {
+            if (result.hardFailures.length === 0)
+                continue;
+            parts.push(`trial ${trial.index}:${renderDiagnostics(result)}`);
+        }
+    }
+    return parts.join("\n");
+}
+/**
+ * Run `body` N times inside the current test (N = `options.n`, else the
+ * ATTEST_TRIALS environment variable, else 1) and enforce only the aggregate.
+ *
+ * With N = 1 the body runs once as a plain test body: evaluate() enforces as
+ * usual. With N > 1, during trial i (1-based): evaluate() does not enforce
+ * (it behaves as inspect()), enforceOutcome() returns its argument, every
+ * result row is reported with `trial = i`, and assertion IDs restart so each
+ * assertion keeps its ID across trials. A trial that throws counts as failed:
+ * its rows are replaced by one hard_fail row (failure_class broken_code) per
+ * assertion ID the test evaluated; when no trial evaluated anything, the first
+ * error is rethrown. A trial whose own outcome is skip counts as not evaluated,
+ * also under ATTEST_STRICT. After the last trial the test is skipped when no trial was
+ * evaluated (failed under ATTEST_STRICT), fails when passed / evaluated
+ * trials is below the minimum pass rate (default 1.0: any failing trial
+ * fails; not-evaluated trials are left out of the rate), and passes otherwise. AttestReporter prints "<test_id>: <passed>/<evaluated>
+ * trials passed (<n> not evaluated)".
+ *
+ * This is a pass/fail gate on N runs, not a statistical verdict: whether the
+ * pass rate regressed (pass / regressed / inconclusive) comes from
+ * `attest policy evaluate --baseline <tag>` with a `regression` rule over the
+ * report's per-trial rows.
+ */
+export async function trials(body, options) {
+    const n = trialCount(options?.n);
+    const minPassRate = options?.minPassRate === undefined
+        ? runMinPassRate()
+        : validateMinPassRate(options.minPassRate, `minPassRate=${options.minPassRate}`);
+    const budget = options?.budget === undefined ? undefined : validateBudget(options.budget);
+    const test = options?.context?.task ?? (await runningTest());
+    if (test === undefined)
+        throw new Error("trials() must be called inside a test");
+    if (options?.context === undefined && test.concurrent === true) {
+        throw new Error("trials() in a test.concurrent test — pass the test context: trials(body, { context })");
+    }
+    if (n === 1) {
+        await trialScope.run({ test, trial: undefined, budget, results: [] }, () => body(1));
+        return;
+    }
+    const state = attemptState(test);
+    const records = [];
+    let stoppedByBudget = false;
+    for (let index = 1; index <= n; index++) {
+        if (index > 1 && costBudgetSpent(budget, state)) {
+            stoppedByBudget = true;
+            break;
+        }
+        state.ordinal = 0;
+        const record = { index, results: [], error: undefined };
+        records.push(record);
+        try {
+            await trialScope.run({ test, trial: index, budget, results: record.results }, () => body(index));
+        }
+        catch (error) {
+            record.error = { value: error };
+        }
+    }
+    const evaluatedAny = records.some((r) => r.results.some((res) => res.assertionResults.length > 0));
+    const firstError = records.find((r) => r.error !== undefined)?.error;
+    if (!evaluatedAny && firstError !== undefined)
+        throw firstError.value;
+    thrownTrialRows(test, records);
+    const strict = strictFromEnv();
+    const outcomes = records.map((r) => trialOutcome(r));
+    const passed = outcomes.filter((o) => o === "pass").length;
+    const failed = outcomes.filter((o) => o === "fail").length;
+    const summary = {
+        requested: n,
+        run: records.length,
+        passed,
+        evaluated: passed + failed,
+        notEvaluated: outcomes.length - passed - failed,
+        stoppedByBudget,
+    };
+    attemptMeta(test).trials = summary;
+    if (summary.evaluated === 0) {
+        const all = records.flatMap((r) => r.results);
+        const rows = all.flatMap((r) => r.assertionResults);
+        const first = all[0];
+        const reason = first === undefined || rows.length === 0
+            ? `not evaluated: no assertions in ${records.length} trials`
+            : new AgentResult(first.trace, rows).skipReason();
+        if (strict)
+            throw new Error(`strict: ${reason}`);
+        (options?.context ?? test.context).skip(reason);
+        return;
+    }
+    if (passed / summary.evaluated < minPassRate) {
+        throw new Error(failureMessage(summary, records, minPassRate));
+    }
+}
+//# sourceMappingURL=trials.js.map

package/dist/trials.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"trials.js","sourceRoot":"","sources":["../src/trials.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,WAAW,EACX,YAAY,EACZ,gBAAgB,EAChB,oBAAoB,EACpB,iBAAiB,GAClB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,WAAW,EACX,YAAY,EACZ,eAAe,EACf,WAAW,EACX,aAAa,EACb,UAAU,EACV,cAAc,GAGf,MAAM,iBAAiB,CAAC;AAEzB,oFAAoF;AACpF,MAAM,CAAC,MAAM,iBAAiB,GAAG,sBAAsB,CAAC;AAiCxD,SAAS,UAAU,CAAC,CAAqB;IACvC,MAAM,GAAG,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3D,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,iBAAiB,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5F,MAAM,IAAI,KAAK,CAAC,4CAA4C,MAAM,EAAE,CAAC,CAAC;IACxE,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,kFAAkF;AAClF,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,MAAc;IAC9D,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CAAC,qDAAqD,MAAM,EAAE,CAAC,CAAC;IACjF,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8FAA8F;AAC9F,SAAS,cAAc;IACrB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IAC3C,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,CAAC,CAAC;IAChC,OAAO,mBAAmB,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,GAAG,iBAAiB,IAAI,GAAG,EAAE,CAAC,CAAC;AAC1G,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,KAAK,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;AAC1G,CAAC;AAED,6GAA6G;AAC7G,SAAS,YAAY,CAAC,KAAkB;IACtC,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC;IAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC;IAC9D,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,gBAAgB,CAAC;QAAE,OAAO,MAAM,CAAC;IACnE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IACrC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,oBAAoB,CAAC;QAAE,OAAO,MAAM,CAAC;IACvE,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,SAAS,eAAe,CAAC,IAAiB,EAAE,MAA8B;IACxE,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACzH,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;YAAE,SAAS;QACxC,MAAM,MAAM,GAAG,IAAI,GAAG,CACpB,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,YAAY,EAAE,GAAG,CAAC,MAAM,CAAU,CAAC,CAAC,CACvG,CAAC;QACF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,CAAC,CAAC;QACvE,MAAM,MAAM,GAAG,SAAS,KAAK,CAAC,KAAK,WAAW,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;QACjF,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACrB,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC9B,MAAM,GAAG,GAAmB;gBAC1B,YAAY,EAAE,EAAE;gBAChB,MAAM,EAAE,gBAAgB;gBACxB,KAAK,EAAE,CAAC;gBACR,WAAW,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,qCAAqC,MAAM,GAAG;gBACpG,aAAa,EAAE,YAAY,CAAC,UAAU;gBACtC,KAAK,EAAE,KAAK,CAAC,KAAK;aACnB,CAAC;YACF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,OAAmB,EAAE,MAA8B,EAAE,WAAmB;IAC9F,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAChD,MAAM,KAAK,GAAG;QACZ,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,SAAS,6BAA6B,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;YACpF,uBAAuB,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;KAClD,CAAC;IACF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,YAAY,CAAC,KAAK,CAAC,KAAK,MAAM;YAAE,SAAS;QAC7C,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC9B,KAAK,CAAC,IAAI,CAAC,SAAS,KAAK,CAAC,KAAK,WAAW,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC9E,SAAS;QACX,CAAC;QACD,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAC/C,KAAK,CAAC,IAAI,CAAC,SAAS,KAAK,CAAC,KAAK,IAAI,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,IAA6C,EAC7C,OAAuB;IAEvB,MAAM,CAAC,GAAG,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IACjC,MAAM,WAAW,GACf,OAAO,EAAE,WAAW,KAAK,SAAS;QAChC,CAAC,CAAC,cAAc,EAAE;QAClB,CAAC,CAAC,mBAAmB,CAAC,OAAO,CAAC,WAAW,EAAE,eAAe,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IACrF,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1F,MAAM,IAAI,GAAG,OAAO,EAAE,OAAO,EAAE,IAAI,IAAI,CAAC,MAAM,WAAW,EAAE,CAAC,CAAC;IAC7D,IAAI,IAAI,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IACjF,IAAI,OAAO,EAAE,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;QAC/D,MAAM,IAAI,KAAK,CAAC,uFAAuF,CAAC,CAAC;IAC3G,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACZ,MAAM,UAAU,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACrF,OAAO;IACT,CAAC;IAED,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,IAAI,eAAe,GAAG,KAAK,CAAC;IAC5B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;QACxC,IAAI,KAAK,GAAG,CAAC,IAAI,eAAe,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;YAChD,eAAe,GAAG,IAAI,CAAC;YACvB,MAAM;QACR,CAAC;QACD,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC;QAClB,MAAM,MAAM,GAAgB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;QACrE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC;YACH,MAAM,UAAU,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QACnG,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QAClC,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IACnG,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,EAAE,KAAK,CAAC;IACrE,IAAI,CAAC,YAAY,IAAI,UAAU,KAAK,SAAS;QAAE,MAAM,UAAU,CAAC,KAAK,CAAC;IACtE,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE/B,MAAM,MAAM,GAAG,aAAa,EAAE,CAAC;IAC/B,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;IACrD,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IAC3D,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IAC3D,MAAM,OAAO,GAAe;QAC1B,SAAS,EAAE,CAAC;QACZ,GAAG,EAAE,OAAO,CAAC,MAAM;QACnB,MAAM;QACN,SAAS,EAAE,MAAM,GAAG,MAAM;QAC1B,YAAY,EAAE,QAAQ,CAAC,MAAM,GAAG,MAAM,GAAG,MAAM;QAC/C,eAAe;KAChB,CAAC;IACF,WAAW,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC;IAEnC,IAAI,OAAO,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC;QACpD,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,MAAM,GACV,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YACtC,CAAC,CAAC,mCAAmC,OAAO,CAAC,MAAM,SAAS;YAC5D,CAAC,CAAC,IAAI,WAAW,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;QACtD,IAAI,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,WAAW,MAAM,EAAE,CAAC,CAAC;QACjD,CAAC,OAAO,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChD,OAAO;IACT,CAAC;IACD,IAAI,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,WAAW,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC;IACjE,CAAC;AACH,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@attest-ai/vitest",
-  "version": "1.0.0-rc.1",
+  "version": "1.0.0-rc.2",
   "description": "Attest vitest integration — global setup, fixtures, cost reporter, tier filtering",
   "type": "module",
   "main": "./dist/index.js",
@@ -50,7 +50,7 @@
     "ajv": "^8.20.0",
     "typescript": "^5.9.3",
     "vitest": "^4.1.10",
-    "@attest-ai/core": "1.0.0-rc.1"
+    "@attest-ai/core": "1.0.0-rc.2"
   },
   "scripts": {
     "build": "tsc",