@attest-ai/vitest 1.0.0-rc.1 → 1.0.0-rc.3

package/dist/fixtures.d.ts

@@ -1,8 +1,17 @@
 import type { TestContext } from "vitest";
 import { type ExpectChain, AgentResult, AttestClient } from "@attest-ai/core";
+import { type TestBudget } from "./test-state.js";
 export interface InspectOptions {
     /** Throw when the cumulative session cost exceeds this USD amount. */
     budget?: number;
+    /**
+     * Limits for this test's attest model calls. Each evaluation sends what is
+     * left: judge calls not yet used in the test, and `maxCostUsd` minus the
+     * test's attest cost and agent spend so far (trace `metadata.cost_usd`). A
+     * call that does not fit is not sent and its assertion is not_evaluated with
+     * reason "budget_exhausted". Inside trials(), defaults to the trials budget.
+     */
+    testBudget?: TestBudget;
     /**
      * The running test's context (`it("…", async (context) => …)`). Used to
      * attach results to the right test for AttestReporter and to skip it;
@@ -21,7 +30,8 @@ export interface EvaluateOptions extends InspectOptions {
  * Fail or skip the running Vitest test according to the batch outcome:
  * any hard_fail throws with rendered diagnostics; otherwise any
  * not_evaluated skips with "not evaluated: <reasons>" (throws when strict);
- * soft failures pass.
+ * soft failures pass. Inside a trial of trials() (N > 1) it does nothing and
+ * returns `result`: trials() enforces the aggregate outcome after the last trial.
  */
 export declare function enforceOutcome(result: AgentResult, strict: boolean, context?: TestContext): AgentResult;
 export declare class AttestEngineFixture {
@@ -48,8 +58,11 @@ export declare class AttestEngineFixture {
     evaluate(chain: ExpectChain, options?: EvaluateOptions): Promise<AgentResult>;
 }
 export declare function useAttest(): AttestEngineFixture;
-/** Evaluate and enforce: hard_fail throws, not_evaluated skips (strict: throws). */
+/**
+ * Evaluate and enforce: hard_fail throws, not_evaluated skips (strict: throws).
+ * Inside a trial of trials() (N > 1) it does not enforce, like inspect().
+ */
 export declare function evaluate(chain: ExpectChain, options?: EvaluateOptions): Promise<AgentResult>;
-/** Evaluate without enforcing; use for trial loops and custom checks. */
+/** Evaluate without enforcing; use for custom checks. For repeated runs use trials(). */
 export declare function inspect(chain: ExpectChain, options?: InspectOptions): Promise<AgentResult>;
 //# sourceMappingURL=fixtures.d.ts.map

package/dist/fixtures.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fixtures.d.ts","sourceRoot":"","sources":["../src/fixtures.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AAC1C,OAAO,EACL,KAAK,WAAW,EAChB,WAAW,EACX,YAAY,EAQb,MAAM,iBAAiB,CAAC;AAEzB,MAAM,WAAW,cAAc;IAC7B,sEAAsE;IACtE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,OAAO,CAAC,EAAE,WAAW,CAAC;CACvB;AAED,MAAM,WAAW,eAAgB,SAAQ,cAAc;IACrD;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AA8CD;;;;;GAKG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,WAAW,EACnB,MAAM,EAAE,OAAO,EACf,OAAO,CAAC,EAAE,WAAW,GACpB,WAAW,CAUb;AAmBD,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,OAAO,CAA4B;IAC3C,OAAO,CAAC,OAAO,CAA2B;IAC1C,OAAO,CAAC,QAAQ,CAA4B;gBAEhC,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,YAAY,CAAA;KAAE;IAWvF;;;;OAIG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAoBtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B,IAAI,MAAM,IAAI,YAAY,CAKzB;YAEa,aAAa;IAM3B,wEAAwE;IAClE,OAAO,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,CAAC;IA4CjF,kEAAkE;IAC5D,QAAQ,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,WAAW,CAAC;CAIpF;AAOD,wBAAgB,SAAS,IAAI,mBAAmB,CAO/C;AAED,oFAAoF;AACpF,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,WAAW,EAClB,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,WAAW,CAAC,CAEtB;AAED,yEAAyE;AACzE,wBAAsB,OAAO,CAC3B,KAAK,EAAE,WAAW,EAClB,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,WAAW,CAAC,CAEtB"}
+{"version":3,"file":"fixtures.d.ts","sourceRoot":"","sources":["../src/fixtures.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AAC1C,OAAO,EACL,KAAK,WAAW,EAChB,WAAW,EAEX,YAAY,EASb,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAYL,KAAK,UAAU,EAChB,MAAM,iBAAiB,CAAC;AAEzB,MAAM,WAAW,cAAc;IAC7B,sEAAsE;IACtE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB;;;;OAIG;IACH,OAAO,CAAC,EAAE,WAAW,CAAC;CACvB;AAED,MAAM,WAAW,eAAgB,SAAQ,cAAc;IACrD;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,WAAW,EACnB,MAAM,EAAE,OAAO,EACf,OAAO,CAAC,EAAE,WAAW,GACpB,WAAW,CAab;AAmBD,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,OAAO,CAA4B;IAC3C,OAAO,CAAC,OAAO,CAA2B;IAC1C,OAAO,CAAC,QAAQ,CAA4B;gBAEhC,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,YAAY,CAAA;KAAE;IAWvF;;;;OAIG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAoBtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B,IAAI,MAAM,IAAI,YAAY,CAKzB;YAEa,aAAa;IAM3B,wEAAwE;IAClE,OAAO,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,CAAC;IA4EjF,kEAAkE;IAC5D,QAAQ,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,WAAW,CAAC;CAIpF;AAOD,wBAAgB,SAAS,IAAI,mBAAmB,CAO/C;AAED;;;GAGG;AACH,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,WAAW,EAClB,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,WAAW,CAAC,CAEtB;AAED,yFAAyF;AACzF,wBAAsB,OAAO,CAC3B,KAAK,EAAE,WAAW,EAClB,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,WAAW,CAAC,CAEtB"}

package/dist/fixtures.js

@@ -1,58 +1,23 @@
 import * as vitest from "vitest";
-import { AgentResult, AttestClient, EngineManager, isSimulationMode, renderDiagnostics, requireSimulationCapability, OUTCOME_FAIL, OUTCOME_SKIP, STATUS_SOFT_FAIL, } from "@attest-ai/core";
-/**
- * The running test: `TestRunner.getCurrentTest()` on Vitest 4.1+, `vitest/suite`
- * before that (loaded only there, because Vitest 4.1 warns when it is imported).
- */
-async function runningTest() {
-    const runner = vitest.TestRunner;
-    if (runner !== undefined)
-        return runner.getCurrentTest();
-    const suite = await import("vitest/suite");
-    return suite.getCurrentTest();
-}
-/**
- * Attach the results to the running test's meta so AttestReporter, in the main
- * process, can write them with the test's file and line. Results from an earlier
- * attempt of a retried test are dropped, so the report matches the outcome Vitest
- * reports. Outside a test (e.g. in a beforeAll hook) there is no test to attach to.
- */
-async function recordOnTest(result, context) {
-    const test = context?.task ?? (await runningTest());
-    if (test === undefined)
-        return;
-    if (context === undefined && test.concurrent === true) {
-        throw new Error("Attest results from a test.concurrent test — pass the test context: " +
-            "inspect(chain, { context }) or evaluate(chain, { context })");
-    }
-    const attempt = test.result?.retryCount ?? 0;
-    let meta = test.meta.attest;
-    if (meta === undefined || meta.attempt !== attempt) {
-        meta = { attempt, results: [], totalCost: 0, totalDurationMs: 0 };
-        test.meta.attest = meta;
-    }
-    meta.results.push(...result.assertionResults);
-    meta.totalCost += result.totalCost;
-    meta.totalDurationMs += result.totalDurationMs;
-}
-function strictFromEnv() {
-    const v = (process.env["ATTEST_STRICT"] ?? "").trim().toLowerCase();
-    return v === "1" || v === "true" || v === "yes";
-}
+import { AgentResult, STATUS_SOFT_FAIL, AttestClient, EngineManager, isSimulationMode, renderDiagnostics, requireSimulationCapability, OUTCOME_FAIL, OUTCOME_SKIP, zeroUsage, } from "@attest-ai/core";
+import { assignAssertionIds, attemptState, inTrial, recordAgentSpend, recordOnTest, recordUnattachedUsage, remainingBudget, resolveTest, strictFromEnv, trialScope, validateBudget, } from "./test-state.js";
 /**
  * Fail or skip the running Vitest test according to the batch outcome:
  * any hard_fail throws with rendered diagnostics; otherwise any
  * not_evaluated skips with "not evaluated: <reasons>" (throws when strict);
- * soft failures pass.
+ * soft failures pass. Inside a trial of trials() (N > 1) it does nothing and
+ * returns `result`: trials() enforces the aggregate outcome after the last trial.
  */
 export function enforceOutcome(result, strict, context) {
+    if (inTrial())
+        return result;
     const outcome = result.outcome({ strict });
     if (outcome === OUTCOME_FAIL) {
         const header = result.hardFailures.length === 0 ? `strict: ${result.skipReason()}\n` : "";
         throw new Error(header + renderDiagnostics(result).replace(/^\n/, ""));
     }
     if (outcome === OUTCOME_SKIP) {
-        (context ?? currentTestContext(result)).skip(result.skipReason());
+        (context ?? trialScope.getStore()?.test.context ?? currentTestContext(result)).skip(result.skipReason());
     }
     return result;
 }
@@ -131,14 +96,44 @@ export class AttestEngineFixture {
     /** Evaluate collected assertions without enforcing the test outcome. */
     async inspect(chain, options) {
         await this.ensureStarted();
-        const result = await this.client.evaluateBatch(chain.trace, chain.assertions);
+        // Inside a test, assertion IDs are deterministic (the test's report test_id
+        // and the assertion's ordinal in the test, restarting each trial) so trials
+        // and baselines match rows by ID. Outside a test they stay random.
+        const test = await resolveTest(options?.context);
+        const scope = trialScope.getStore();
+        const trial = scope !== undefined && scope.test === test ? scope.trial : undefined;
+        const budget = options?.testBudget !== undefined
+            ? validateBudget(options.testBudget)
+            : scope !== undefined && scope.test === test
+                ? scope.budget
+                : undefined;
+        let assertions = chain.assertions;
+        let batchBudget;
+        if (test !== undefined) {
+            assertions = assignAssertionIds(test, assertions);
+            recordAgentSpend(test, chain.trace);
+            if (budget !== undefined) {
+                const state = attemptState(test);
+                batchBudget = remainingBudget(budget, state.usage, state.agentSpendUsd);
+            }
+        }
+        else if (budget !== undefined) {
+            // Outside a test the budget covers this one evaluation.
+            batchBudget = remainingBudget(budget, zeroUsage(), chain.trace.metadata?.cost_usd ?? 0);
+        }
+        const result = await this.client.evaluateBatch(chain.trace, assertions, batchBudget === undefined ? undefined : { budget: batchBudget });
         globalThis.__attest_session_cost__ =
             (globalThis.__attest_session_cost__ ?? 0) + (result.total_cost ?? 0);
         const softFailCount = (result.results ?? []).filter((r) => r.status === STATUS_SOFT_FAIL).length;
         globalThis.__attest_session_soft_failures__ =
             (globalThis.__attest_session_soft_failures__ ?? 0) + softFailCount;
-        const agentResult = new AgentResult(chain.trace, result.results ?? [], result.total_cost ?? 0, result.total_duration_ms ?? 0);
-        await recordOnTest(agentResult, options?.context);
+        const agentResult = new AgentResult(chain.trace, result.results ?? [], result.total_cost ?? 0, result.total_duration_ms ?? 0, result.usage);
+        if (test !== undefined)
+            recordOnTest(test, agentResult, trial);
+        else
+            recordUnattachedUsage(agentResult.usage);
+        if (scope !== undefined && scope.test === test)
+            scope.results.push(agentResult);
         // Track non-passing AgentResults so AttestCostReporter can render the
         // diagnostic block at the end of the run. Bound size because long
         // suites with many failures should not balloon RAM.
@@ -173,11 +168,14 @@ export function useAttest() {
     workerFixture ??= new AttestEngineFixture();
     return workerFixture;
 }
-/** Evaluate and enforce: hard_fail throws, not_evaluated skips (strict: throws). */
+/**
+ * Evaluate and enforce: hard_fail throws, not_evaluated skips (strict: throws).
+ * Inside a trial of trials() (N > 1) it does not enforce, like inspect().
+ */
 export async function evaluate(chain, options) {
     return useAttest().evaluate(chain, options);
 }
-/** Evaluate without enforcing; use for trial loops and custom checks. */
+/** Evaluate without enforcing; use for custom checks. For repeated runs use trials(). */
 export async function inspect(chain, options) {
     return useAttest().inspect(chain, options);
 }

package/dist/fixtures.js.map

@@ -1 +1 @@
-{"version":3,"file":"fixtures.js","sourceRoot":"","sources":["../src/fixtures.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AAEjC,OAAO,EAEL,WAAW,EACX,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,2BAA2B,EAC3B,YAAY,EACZ,YAAY,EACZ,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AAuBzB;;;GAGG;AACH,KAAK,UAAU,WAAW;IACxB,MAAM,MAAM,GAAI,MAAiC,CAAC,UAAU,CAAC;IAC7D,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,cAAc,EAAE,CAAC;IACzD,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;IAC3C,OAAO,KAAK,CAAC,cAAc,EAAE,CAAC;AAChC,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,YAAY,CAAC,MAAmB,EAAE,OAAqB;IACpE,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,CAAC,MAAM,WAAW,EAAE,CAAC,CAAC;IACpD,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO;IAC/B,IAAI,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CACb,sEAAsE;YACpE,6DAA6D,CAChE,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,UAAU,IAAI,CAAC,CAAC;IAC7C,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;IAC5B,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;QACnD,IAAI,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC;QAClE,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IAC1B,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAC9C,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;IACnC,IAAI,CAAC,eAAe,IAAI,MAAM,CAAC,eAAe,CAAC;AACjD,CAAC;AAED,SAAS,aAAa;IACpB,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACpE,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,KAAK,CAAC;AAClD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAC5B,MAAmB,EACnB,MAAe,EACf,OAAqB;IAErB,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAC3C,IAAI,OAAO,KAAK,YAAY,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,MAAM,IAAI,KAAK,CAAC,MAAM,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;IACzE,CAAC;IACD,IAAI,OAAO,KAAK,YAAY,EAAE,CAAC;QAC7B,CAAC,OAAO,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,MAAmB;IAC7C,MAAM,MAAM,GAAI,MAAiC,CAAC,UAAU,CAAC;IAC7D,MAAM,IAAI,GAAG,MAAM,EAAE,cAAc,EAAE,CAAC;IACtC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CACb,GAAG,MAAM,CAAC,UAAU,EAAE,8CAA8C;YAClE,8EAA8E,CACjF,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC,OAAO,CAAC;AACtB,CAAC;AAED,MAAM,OAAO,mBAAmB;IACtB,OAAO,CAA4B;IACnC,OAAO,CAA2B;IAClC,QAAQ,CAA4B;IAE5C,YAAY,OAA2E;QACrF,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,OAAO;QACT,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,aAAa,CAAC;YAC/B,UAAU,EAAE,OAAO,EAAE,UAAU;YAC/B,QAAQ,EAAE,OAAO,EAAE,QAAQ,IAAI,MAAM;SACtC,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QACD,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAC;QACtC,IAAI,UAAU,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,GAAG,CAAC;QACzC,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACxC,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,2BAA2B,CAAC,IAAI,CAAC,CAAC;YACpC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC1B,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAC1B,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;YACzB,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,IAAI,MAAM;QACR,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS;YAAE,OAAO;QACvC,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC;IACtB,CAAC;IAED,wEAAwE;IACxE,KAAK,CAAC,OAAO,CAAC,KAAkB,EAAE,OAAwB;QACxD,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;QAE9E,UAAU,CAAC,uBAAuB;YAChC,CAAC,UAAU,CAAC,uBAAuB,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC;QAEvE,MAAM,aAAa,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,CACjD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,gBAAgB,CACrC,CAAC,MAAM,CAAC;QACT,UAAU,CAAC,gCAAgC;YACzC,CAAC,UAAU,CAAC,gCAAgC,IAAI,CAAC,CAAC,GAAG,aAAa,CAAC;QAErE,MAAM,WAAW,GAAG,IAAI,WAAW,CACjC,KAAK,CAAC,KAAK,EACX,MAAM,CAAC,OAAO,IAAI,EAAE,EACpB,MAAM,CAAC,UAAU,IAAI,CAAC,EACtB,MAAM,CAAC,iBAAiB,IAAI,CAAC,CAC9B,CAAC;QACF,MAAM,YAAY,CAAC,WAAW,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAElD,sEAAsE;QACtE,kEAAkE;QAClE,oDAAoD;QACpD,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,CAAC,UAAU,CAAC,2BAA2B,KAAK,EAAE,CAAC,CAAC;YAC7D,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACtB,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QAED,IACE,OAAO,EAAE,MAAM,KAAK,SAAS;YAC7B,UAAU,CAAC,uBAAuB,GAAG,OAAO,CAAC,MAAM,EACnD,CAAC;YACD,MAAM,IAAI,KAAK,CACb,iCAAiC,UAAU,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;gBAC9E,cAAc,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAC5C,CAAC;QACJ,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,kEAAkE;IAClE,KAAK,CAAC,QAAQ,CAAC,KAAkB,EAAE,OAAyB;QAC1D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAClD,OAAO,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,aAAa,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IACtF,CAAC;CACF;AAED,yEAAyE;AACzE,+EAA+E;AAC/E,sEAAsE;AACtE,IAAI,aAA8C,CAAC;AAEnD,MAAM,UAAU,SAAS;IACvB,MAAM,MAAM,GAAG,UAAU,CAAC,iBAAiB,CAAC;IAC5C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,IAAI,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAC7C,CAAC;IACD,aAAa,KAAK,IAAI,mBAAmB,EAAE,CAAC;IAC5C,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,oFAAoF;AACpF,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAkB,EAClB,OAAyB;IAEzB,OAAO,SAAS,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC9C,CAAC;AAED,yEAAyE;AACzE,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,KAAkB,EAClB,OAAwB;IAExB,OAAO,SAAS,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC7C,CAAC"}
+{"version":3,"file":"fixtures.js","sourceRoot":"","sources":["../src/fixtures.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AAEjC,OAAO,EAEL,WAAW,EACX,gBAAgB,EAChB,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,2BAA2B,EAC3B,YAAY,EACZ,YAAY,EACZ,SAAS,GAEV,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,kBAAkB,EAClB,YAAY,EACZ,OAAO,EACP,gBAAgB,EAChB,YAAY,EACZ,qBAAqB,EACrB,eAAe,EACf,WAAW,EACX,aAAa,EACb,UAAU,EACV,cAAc,GAEf,MAAM,iBAAiB,CAAC;AA6BzB;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAC5B,MAAmB,EACnB,MAAe,EACf,OAAqB;IAErB,IAAI,OAAO,EAAE;QAAE,OAAO,MAAM,CAAC;IAC7B,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAC3C,IAAI,OAAO,KAAK,YAAY,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,MAAM,IAAI,KAAK,CAAC,MAAM,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;IACzE,CAAC;IACD,IAAI,OAAO,KAAK,YAAY,EAAE,CAAC;QAC7B,CAAC,OAAO,IAAI,UAAU,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CACjF,MAAM,CAAC,UAAU,EAAE,CACpB,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,MAAmB;IAC7C,MAAM,MAAM,GAAI,MAAiC,CAAC,UAAU,CAAC;IAC7D,MAAM,IAAI,GAAG,MAAM,EAAE,cAAc,EAAE,CAAC;IACtC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CACb,GAAG,MAAM,CAAC,UAAU,EAAE,8CAA8C;YAClE,8EAA8E,CACjF,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC,OAAO,CAAC;AACtB,CAAC;AAED,MAAM,OAAO,mBAAmB;IACtB,OAAO,CAA4B;IACnC,OAAO,CAA2B;IAClC,QAAQ,CAA4B;IAE5C,YAAY,OAA2E;QACrF,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,OAAO;QACT,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,aAAa,CAAC;YAC/B,UAAU,EAAE,OAAO,EAAE,UAAU;YAC/B,QAAQ,EAAE,OAAO,EAAE,QAAQ,IAAI,MAAM;SACtC,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QACD,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAC;QACtC,IAAI,UAAU,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,GAAG,CAAC;QACzC,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACxC,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,2BAA2B,CAAC,IAAI,CAAC,CAAC;YACpC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC1B,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAC1B,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;YACzB,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,IAAI,MAAM;QACR,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS;YAAE,OAAO;QACvC,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC;IACtB,CAAC;IAED,wEAAwE;IACxE,KAAK,CAAC,OAAO,CAAC,KAAkB,EAAE,OAAwB;QACxD,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC3B,4EAA4E;QAC5E,4EAA4E;QAC5E,mEAAmE;QACnE,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QACnF,MAAM,MAAM,GACV,OAAO,EAAE,UAAU,KAAK,SAAS;YAC/B,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,UAAU,CAAC;YACpC,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI;gBAC1C,CAAC,CAAC,KAAK,CAAC,MAAM;gBACd,CAAC,CAAC,SAAS,CAAC;QAClB,IAAI,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;QAClC,IAAI,WAAoC,CAAC;QACzC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,UAAU,GAAG,kBAAkB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAClD,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;YACpC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;gBACjC,WAAW,GAAG,eAAe,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;aAAM,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YAChC,wDAAwD;YACxD,WAAW,GAAG,eAAe,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,IAAI,CAAC,CAAC,CAAC;QAC1F,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAC5C,KAAK,CAAC,KAAK,EACX,UAAU,EACV,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAChE,CAAC;QAEF,UAAU,CAAC,uBAAuB;YAChC,CAAC,UAAU,CAAC,uBAAuB,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC;QAEvE,MAAM,aAAa,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,CACjD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,gBAAgB,CACrC,CAAC,MAAM,CAAC;QACT,UAAU,CAAC,gCAAgC;YACzC,CAAC,UAAU,CAAC,gCAAgC,IAAI,CAAC,CAAC,GAAG,aAAa,CAAC;QAErE,MAAM,WAAW,GAAG,IAAI,WAAW,CACjC,KAAK,CAAC,KAAK,EACX,MAAM,CAAC,OAAO,IAAI,EAAE,EACpB,MAAM,CAAC,UAAU,IAAI,CAAC,EACtB,MAAM,CAAC,iBAAiB,IAAI,CAAC,EAC7B,MAAM,CAAC,KAAK,CACb,CAAC;QACF,IAAI,IAAI,KAAK,SAAS;YAAE,YAAY,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;;YAC1D,qBAAqB,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAC9C,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI;YAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEhF,sEAAsE;QACtE,kEAAkE;QAClE,oDAAoD;QACpD,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,CAAC,UAAU,CAAC,2BAA2B,KAAK,EAAE,CAAC,CAAC;YAC7D,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACtB,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QAED,IACE,OAAO,EAAE,MAAM,KAAK,SAAS;YAC7B,UAAU,CAAC,uBAAuB,GAAG,OAAO,CAAC,MAAM,EACnD,CAAC;YACD,MAAM,IAAI,KAAK,CACb,iCAAiC,UAAU,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;gBAC9E,cAAc,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAC5C,CAAC;QACJ,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,kEAAkE;IAClE,KAAK,CAAC,QAAQ,CAAC,KAAkB,EAAE,OAAyB;QAC1D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAClD,OAAO,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,aAAa,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IACtF,CAAC;CACF;AAED,yEAAyE;AACzE,+EAA+E;AAC/E,sEAAsE;AACtE,IAAI,aAA8C,CAAC;AAEnD,MAAM,UAAU,SAAS;IACvB,MAAM,MAAM,GAAG,UAAU,CAAC,iBAAiB,CAAC;IAC5C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,IAAI,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAC7C,CAAC;IACD,aAAa,KAAK,IAAI,mBAAmB,EAAE,CAAC;IAC5C,OAAO,aAAa,CAAC;AACvB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAkB,EAClB,OAAyB;IAEzB,OAAO,SAAS,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC9C,CAAC;AAED,yFAAyF;AACzF,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,KAAkB,EAClB,OAAwB;IAExB,OAAO,SAAS,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC7C,CAAC"}

package/dist/index.d.ts

@@ -2,9 +2,17 @@ export { attestGlobalSetup } from "./plugin.js";
 export type { AttestGlobalSetupOptions } from "./plugin.js";
 export { AttestEngineFixture, useAttest, evaluate, inspect, enforceOutcome } from "./fixtures.js";
 export type { EvaluateOptions, InspectOptions } from "./fixtures.js";
+export { trials, MIN_PASS_RATE_ENV } from "./trials.js";
+export type { TrialsOptions } from "./trials.js";
+export type { TestBudget } from "./test-state.js";
 export { AttestCostReporter, AttestReporter } from "./reporter.js";
 export type { AttestReporterOptions } from "./reporter.js";
-export { buildReport, REPORT_VERSION } from "./report.js";
-export type { AttestTaskMeta, TestOutcome, TestRecord } from "./report.js";
+export { buildReport, REPORT_VERSION, sessionUsage, summaryLines } from "./report.js";
+export type { AttestTaskMeta, RecordedResult, TestOutcome, TestRecord, TrialsMeta } from "./report.js";
 export { filterByTier } from "./tier-filter.js";
+export { useWorld } from "./world.js";
+export { useReplay } from "./replay.js";
+export type { ReplayHandle, ReplayMeta, UseReplayOptions } from "./replay.js";
+export { EgressBlockedError, REASON_REPLAY_MISS, ReplayError, ReplayMissError, ReplayNotEvaluatedError, egressBlockRequested, firstMiss, installEgressGuard, isProviderHost, liveLine, resolveMode, startReplay, startWorld, uninstallEgressGuard, WorldError, } from "@attest-ai/core";
+export type { ReplaySession, ReplaySummary, WorldOptions, WorldSession, WorldSnapshot } from "@attest-ai/core";
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,YAAY,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAClG,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACrE,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACnE,YAAY,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC1D,YAAY,EAAE,cAAc,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,YAAY,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAClG,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACrE,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACxD,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,YAAY,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACnE,YAAY,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACtF,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACvG,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC9E,OAAO,EACL,kBAAkB,EAClB,kBAAkB,EAClB,WAAW,EACX,eAAe,EACf,uBAAuB,EACvB,oBAAoB,EACpB,SAAS,EACT,kBAAkB,EAClB,cAAc,EACd,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,oBAAoB,EACpB,UAAU,GACX,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC"}

package/dist/index.js

@@ -1,6 +1,10 @@
 export { attestGlobalSetup } from "./plugin.js";
 export { AttestEngineFixture, useAttest, evaluate, inspect, enforceOutcome } from "./fixtures.js";
+export { trials, MIN_PASS_RATE_ENV } from "./trials.js";
 export { AttestCostReporter, AttestReporter } from "./reporter.js";
-export { buildReport, REPORT_VERSION } from "./report.js";
+export { buildReport, REPORT_VERSION, sessionUsage, summaryLines } from "./report.js";
 export { filterByTier } from "./tier-filter.js";
+export { useWorld } from "./world.js";
+export { useReplay } from "./replay.js";
+export { EgressBlockedError, REASON_REPLAY_MISS, ReplayError, ReplayMissError, ReplayNotEvaluatedError, egressBlockRequested, firstMiss, installEgressGuard, isProviderHost, liveLine, resolveMode, startReplay, startWorld, uninstallEgressGuard, WorldError, } from "@attest-ai/core";
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAElG,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAEnE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE1D,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAElG,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAGxD,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAEnE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEtF,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC,OAAO,EACL,kBAAkB,EAClB,kBAAkB,EAClB,WAAW,EACX,eAAe,EACf,uBAAuB,EACvB,oBAAoB,EACpB,SAAS,EACT,kBAAkB,EAClB,cAAc,EACd,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,oBAAoB,EACpB,UAAU,GACX,MAAM,iBAAiB,CAAC"}

package/dist/plugin.d.ts

@@ -2,6 +2,12 @@ import { EngineManager, AttestClient, type AgentResult } from "@attest-ai/core";
 export interface AttestGlobalSetupOptions {
     enginePath?: string;
     logLevel?: string;
+    /**
+     * Minimum passed / evaluated trials for every trials() test of the run (0..1,
+     * default 1.0); trials(body, { minPassRate }) overrides it per test. Reaches
+     * the workers as the ATTEST_MIN_PASS_RATE environment variable.
+     */
+    minPassRate?: number;
 }
 declare global {
     var __attest_engine__: EngineManager | undefined;

package/dist/plugin.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,aAAa,EACb,YAAY,EAGZ,KAAK,WAAW,EACjB,MAAM,iBAAiB,CAAC;AAKzB,MAAM,WAAW,wBAAwB;IACvC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,OAAO,CAAC,MAAM,CAAC;IAEb,IAAI,iBAAiB,EAAE,aAAa,GAAG,SAAS,CAAC;IAEjD,IAAI,iBAAiB,EAAE,YAAY,GAAG,SAAS,CAAC;IAEhD,IAAI,uBAAuB,EAAE,MAAM,CAAC;IAEpC,IAAI,gCAAgC,EAAE,MAAM,CAAC;IAE7C,IAAI,2BAA2B,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC;CAC5D;AAED,wBAAgB,iBAAiB,CAAC,OAAO,CAAC,EAAE,wBAAwB;;;EA8CnE"}
+{"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,aAAa,EACb,YAAY,EAGZ,KAAK,WAAW,EACjB,MAAM,iBAAiB,CAAC;AAMzB,MAAM,WAAW,wBAAwB;IACvC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,OAAO,CAAC,MAAM,CAAC;IAEb,IAAI,iBAAiB,EAAE,aAAa,GAAG,SAAS,CAAC;IAEjD,IAAI,iBAAiB,EAAE,YAAY,GAAG,SAAS,CAAC;IAEhD,IAAI,uBAAuB,EAAE,MAAM,CAAC;IAEpC,IAAI,gCAAgC,EAAE,MAAM,CAAC;IAE7C,IAAI,2BAA2B,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC;CAC5D;AAED,wBAAgB,iBAAiB,CAAC,OAAO,CAAC,EAAE,wBAAwB;;;EAmDnE"}

package/dist/plugin.js

@@ -1,9 +1,15 @@
 import { EngineManager, AttestClient, isSimulationMode, requireSimulationCapability, } from "@attest-ai/core";
+import { MIN_PASS_RATE_ENV, validateMinPassRate } from "./trials.js";
 let engineManager;
 let attestClient;
 export function attestGlobalSetup(options) {
+    const minPassRate = options?.minPassRate === undefined
+        ? undefined
+        : validateMinPassRate(options.minPassRate, `minPassRate=${options.minPassRate}`);
     return {
         async setup() {
+            if (minPassRate !== undefined)
+                process.env[MIN_PASS_RATE_ENV] = String(minPassRate);
             globalThis.__attest_session_cost__ = 0;
             globalThis.__attest_session_soft_failures__ = 0;
             globalThis.__attest_session_failures__ = [];

package/dist/plugin.js.map

@@ -1 +1 @@
-{"version":3,"file":"plugin.js","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,aAAa,EACb,YAAY,EACZ,gBAAgB,EAChB,2BAA2B,GAE5B,MAAM,iBAAiB,CAAC;AAEzB,IAAI,aAAwC,CAAC;AAC7C,IAAI,YAAsC,CAAC;AAoB3C,MAAM,UAAU,iBAAiB,CAAC,OAAkC;IAClE,OAAO;QACL,KAAK,CAAC,KAAK;YACT,UAAU,CAAC,uBAAuB,GAAG,CAAC,CAAC;YACvC,UAAU,CAAC,gCAAgC,GAAG,CAAC,CAAC;YAChD,UAAU,CAAC,2BAA2B,GAAG,EAAE,CAAC;YAE5C,IAAI,gBAAgB,EAAE,EAAE,CAAC;gBACvB,iEAAiE;gBACjE,iDAAiD;gBACjD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,GAAG,CAAC;YACzC,CAAC;YAED,aAAa,GAAG,IAAI,aAAa,CAAC;gBAChC,UAAU,EAAE,OAAO,EAAE,UAAU;gBAC/B,QAAQ,EAAE,OAAO,EAAE,QAAQ,IAAI,MAAM;aACtC,CAAC,CAAC;YAEH,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,CAAC;YACzC,IAAI,gBAAgB,EAAE,EAAE,CAAC;gBACvB,IAAI,CAAC;oBACH,2BAA2B,CAAC,IAAI,CAAC,CAAC;gBACpC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;oBAC3B,aAAa,GAAG,SAAS,CAAC;oBAC1B,MAAM,GAAG,CAAC;gBACZ,CAAC;YACH,CAAC;YACD,YAAY,GAAG,IAAI,YAAY,CAAC,aAAa,CAAC,CAAC;YAE/C,UAAU,CAAC,iBAAiB,GAAG,aAAa,CAAC;YAC7C,UAAU,CAAC,iBAAiB,GAAG,YAAY,CAAC;QAC9C,CAAC;QAED,KAAK,CAAC,QAAQ;YACZ,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;gBAC3B,aAAa,GAAG,SAAS,CAAC;gBAC1B,YAAY,GAAG,SAAS,CAAC;YAC3B,CAAC;YAED,UAAU,CAAC,iBAAiB,GAAG,SAAS,CAAC;YACzC,UAAU,CAAC,iBAAiB,GAAG,SAAS,CAAC;YACzC,UAAU,CAAC,2BAA2B,GAAG,SAAS,CAAC;QACrD,CAAC;KACF,CAAC;AACJ,CAAC"}
+{"version":3,"file":"plugin.js","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,aAAa,EACb,YAAY,EACZ,gBAAgB,EAChB,2BAA2B,GAE5B,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAErE,IAAI,aAAwC,CAAC;AAC7C,IAAI,YAAsC,CAAC;AA0B3C,MAAM,UAAU,iBAAiB,CAAC,OAAkC;IAClE,MAAM,WAAW,GACf,OAAO,EAAE,WAAW,KAAK,SAAS;QAChC,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,mBAAmB,CAAC,OAAO,CAAC,WAAW,EAAE,eAAe,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IACrF,OAAO;QACL,KAAK,CAAC,KAAK;YACT,IAAI,WAAW,KAAK,SAAS;gBAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;YACpF,UAAU,CAAC,uBAAuB,GAAG,CAAC,CAAC;YACvC,UAAU,CAAC,gCAAgC,GAAG,CAAC,CAAC;YAChD,UAAU,CAAC,2BAA2B,GAAG,EAAE,CAAC;YAE5C,IAAI,gBAAgB,EAAE,EAAE,CAAC;gBACvB,iEAAiE;gBACjE,iDAAiD;gBACjD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,GAAG,CAAC;YACzC,CAAC;YAED,aAAa,GAAG,IAAI,aAAa,CAAC;gBAChC,UAAU,EAAE,OAAO,EAAE,UAAU;gBAC/B,QAAQ,EAAE,OAAO,EAAE,QAAQ,IAAI,MAAM;aACtC,CAAC,CAAC;YAEH,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,CAAC;YACzC,IAAI,gBAAgB,EAAE,EAAE,CAAC;gBACvB,IAAI,CAAC;oBACH,2BAA2B,CAAC,IAAI,CAAC,CAAC;gBACpC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;oBAC3B,aAAa,GAAG,SAAS,CAAC;oBAC1B,MAAM,GAAG,CAAC;gBACZ,CAAC;YACH,CAAC;YACD,YAAY,GAAG,IAAI,YAAY,CAAC,aAAa,CAAC,CAAC;YAE/C,UAAU,CAAC,iBAAiB,GAAG,aAAa,CAAC;YAC7C,UAAU,CAAC,iBAAiB,GAAG,YAAY,CAAC;QAC9C,CAAC;QAED,KAAK,CAAC,QAAQ;YACZ,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;gBAC3B,aAAa,GAAG,SAAS,CAAC;gBAC1B,YAAY,GAAG,SAAS,CAAC;YAC3B,CAAC;YAED,UAAU,CAAC,iBAAiB,GAAG,SAAS,CAAC;YACzC,UAAU,CAAC,iBAAiB,GAAG,SAAS,CAAC;YACzC,UAAU,CAAC,2BAA2B,GAAG,SAAS,CAAC;QACrD,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/replay.d.ts

@@ -0,0 +1,41 @@
+import { type ReplayModeSetting, type ReplayOptions, type ReplaySummary } from "@attest-ai/core";
+/** Replay provenance of one test, carried from the worker in task meta. */
+export interface ReplayMeta {
+    fixtures: string[];
+    mode: "strict" | "auto";
+    replayed: number;
+    live: number;
+    misses: number;
+    /** First miss message. */
+    miss?: string;
+    liveCostUsd: number;
+    unpricedLiveCalls: number;
+    /** The test was skipped as `not_evaluated: replay_miss`. */
+    notEvaluated: boolean;
+}
+export interface UseReplayOptions extends Omit<ReplayOptions, "mode" | "noLive"> {
+    /** `strict`, `auto` or `off`; default `ATTEST_REPLAY`, else `strict` when `CI` is set, else `auto`. */
+    mode?: ReplayModeSetting;
+    /** Block in-process connections to model-provider hosts (default: `ATTEST_EGRESS=block`). */
+    egress?: "block" | "allow";
+    /** Fail (instead of skip) when auto mode misses. Default: the ATTEST_STRICT variable. */
+    strict?: boolean;
+}
+export interface ReplayHandle {
+    /** `off` when replay is disabled and nothing was started. */
+    readonly mode: "strict" | "auto" | "off";
+    /** The session's environment (empty when off). */
+    readonly env: Record<string, string>;
+    /** Stop the replay now and return its summary (`undefined` when off). The test's outcome is still decided at its end. */
+    finish(): Promise<ReplaySummary | undefined>;
+}
+/**
+ * Start replaying fixture `name` for the running test. Call it before constructing the
+ * model client (clients read `OPENAI_BASE_URL` / `ANTHROPIC_BASE_URL` at construction).
+ * When the test ends the replay stops and the test's outcome follows the summary:
+ * - strict: any miss fails the test with the miss message, whatever the agent threw;
+ * - auto: a miss with no provider key skips the test as `not evaluated: replay_miss`
+ *   (a failure under `strict: true` / `ATTEST_STRICT=1`); with a key the call went live.
+ */
+export declare function useReplay(name: string, opts?: UseReplayOptions): Promise<ReplayHandle>;
+//# sourceMappingURL=replay.d.ts.map

package/dist/replay.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"replay.d.ts","sourceRoot":"","sources":["../src/replay.ts"],"names":[],"mappings":"AAKA,OAAO,EAOL,KAAK,iBAAiB,EACtB,KAAK,aAAa,EAElB,KAAK,aAAa,EACnB,MAAM,iBAAiB,CAAC;AAGzB,2EAA2E;AAC3E,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,IAAI,EAAE,QAAQ,GAAG,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,0BAA0B;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,4DAA4D;IAC5D,YAAY,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,gBAAiB,SAAQ,IAAI,CAAC,aAAa,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC9E,uGAAuG;IACvG,IAAI,CAAC,EAAE,iBAAiB,CAAC;IACzB,6FAA6F;IAC7F,MAAM,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC;IAC3B,yFAAyF;IACzF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,6DAA6D;IAC7D,QAAQ,CAAC,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,KAAK,CAAC;IACzC,kDAAkD;IAClD,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,yHAAyH;IACzH,MAAM,IAAI,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAAC;CAC9C;AAUD;;;;;;;GAOG;AACH,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,gBAAqB,GAAG,OAAO,CAAC,YAAY,CAAC,CA4BhG"}

package/dist/replay.js

@@ -0,0 +1,98 @@
+/**
+ * `useReplay(name)`: run the test's real agent code with model responses served from
+ * fixture `<name>` by the engine's replay proxy, and report how each request was served.
+ */
+import { onTestFinished } from "vitest";
+import { ReplayNotEvaluatedError, egressBlockRequested, firstMiss, installEgressGuard, resolveMode, startReplay, } from "@attest-ai/core";
+import { strictFromEnv } from "./test-state.js";
+/**
+ * Start replaying fixture `name` for the running test. Call it before constructing the
+ * model client (clients read `OPENAI_BASE_URL` / `ANTHROPIC_BASE_URL` at construction).
+ * When the test ends the replay stops and the test's outcome follows the summary:
+ * - strict: any miss fails the test with the miss message, whatever the agent threw;
+ * - auto: a miss with no provider key skips the test as `not evaluated: replay_miss`
+ *   (a failure under `strict: true` / `ATTEST_STRICT=1`); with a key the call went live.
+ */
+export async function useReplay(name, opts = {}) {
+    const mode = resolveMode(opts.mode);
+    if (mode === "off")
+        return { mode, env: {}, finish: () => Promise.resolve(undefined) };
+    const blocking = (opts.egress ?? (egressBlockRequested() ? "block" : "allow")) === "block";
+    const uninstallGuard = blocking ? installEgressGuard() : undefined;
+    let session;
+    try {
+        session = await startReplay(name, { ...opts, mode, noLive: blocking });
+    }
+    catch (err) {
+        uninstallGuard?.();
+        throw err;
+    }
+    let summary;
+    const finish = async () => {
+        try {
+            summary = (await session.finish()) ?? undefined;
+            if (summary === undefined)
+                throw new Error("replay session returned no summary");
+            return summary;
+        }
+        finally {
+            uninstallGuard?.();
+        }
+    };
+    onTestFinished(async ({ task }) => {
+        const done = await finish();
+        settle(task, name, mode, done, session, opts.strict ?? strictFromEnv());
+    });
+    return { mode, env: session.env, finish };
+}
+function settle(task, name, mode, s, session, strict) {
+    const miss = firstMiss(s) ?? undefined;
+    const skip = s.misses > 0 && mode === "auto" && !strict;
+    const meta = {
+        fixtures: [name],
+        mode,
+        replayed: s.replayed,
+        live: s.live,
+        misses: s.misses,
+        ...(miss !== undefined ? { miss } : {}),
+        liveCostUsd: s.live_cost_usd,
+        unpricedLiveCalls: s.unpriced_live_calls,
+        // An auto miss is a not_evaluated row in the report even when --strict fails the test.
+        notEvaluated: mode === "auto" && s.misses > 0,
+    };
+    const prior = task.meta["attestReplay"];
+    task.meta["attestReplay"] = prior === undefined ? meta : mergeMeta(prior, meta);
+    if (miss === undefined)
+        return;
+    if (skip) {
+        // The agent's own error is a consequence of the miss; the test is not evaluated.
+        task.result = { ...task.result, state: "skip", errors: undefined };
+        return;
+    }
+    const error = session.error() ?? new ReplayNotEvaluatedError(miss, s);
+    // The runner flips a failed `it.fails` test to a pass after the hooks run; a miss must stay a failure.
+    task.fails = false;
+    if (task.result?.errors !== undefined && task.result.errors.length > 0) {
+        // Whatever the agent threw, the user sees the miss.
+        task.result.errors = [error];
+        task.result.state = "fail";
+        return;
+    }
+    throw error;
+}
+/** Fold a later session of the same test into the earlier one, as the Python report does. */
+function mergeMeta(a, b) {
+    const miss = a.miss ?? b.miss;
+    return {
+        fixtures: [...new Set([...a.fixtures, ...b.fixtures])],
+        mode: a.mode === b.mode ? a.mode : "auto",
+        replayed: a.replayed + b.replayed,
+        live: a.live + b.live,
+        misses: a.misses + b.misses,
+        ...(miss !== undefined ? { miss } : {}),
+        liveCostUsd: a.liveCostUsd + b.liveCostUsd,
+        unpricedLiveCalls: a.unpricedLiveCalls + b.unpricedLiveCalls,
+        notEvaluated: a.notEvaluated || b.notEvaluated,
+    };
+}
+//# sourceMappingURL=replay.js.map

package/dist/replay.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"replay.js","sourceRoot":"","sources":["../src/replay.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AACxC,OAAO,EACL,uBAAuB,EACvB,oBAAoB,EACpB,SAAS,EACT,kBAAkB,EAClB,WAAW,EACX,WAAW,GAKZ,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AA2ChD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,OAAyB,EAAE;IACvE,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpC,IAAI,IAAI,KAAK,KAAK;QAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;IAEvF,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,OAAO,CAAC;IAC3F,MAAM,cAAc,GAAG,QAAQ,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACnE,IAAI,OAAsB,CAAC;IAC3B,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;IACzE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,cAAc,EAAE,EAAE,CAAC;QACnB,MAAM,GAAG,CAAC;IACZ,CAAC;IACD,IAAI,OAAkC,CAAC;IACvC,MAAM,MAAM,GAAG,KAAK,IAA4B,EAAE;QAChD,IAAI,CAAC;YACH,OAAO,GAAG,CAAC,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,CAAC;YAChD,IAAI,OAAO,KAAK,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;YACjF,OAAO,OAAO,CAAC;QACjB,CAAC;gBAAS,CAAC;YACT,cAAc,EAAE,EAAE,CAAC;QACrB,CAAC;IACH,CAAC,CAAC;IACF,cAAc,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;QAChC,MAAM,IAAI,GAAG,MAAM,MAAM,EAAE,CAAC;QAC5B,MAAM,CAAC,IAA+B,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,IAAI,aAAa,EAAE,CAAC,CAAC;IACrG,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,CAAC;AAC5C,CAAC;AAED,SAAS,MAAM,CACb,IAAkB,EAClB,IAAY,EACZ,IAAuB,EACvB,CAAgB,EAChB,OAAsB,EACtB,MAAe;IAEf,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC;IACvC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,KAAK,MAAM,IAAI,CAAC,MAAM,CAAC;IACxD,MAAM,IAAI,GAAe;QACvB,QAAQ,EAAE,CAAC,IAAI,CAAC;QAChB,IAAI;QACJ,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvC,WAAW,EAAE,CAAC,CAAC,aAAa;QAC5B,iBAAiB,EAAE,CAAC,CAAC,mBAAmB;QACxC,uFAAuF;QACvF,YAAY,EAAE,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC;KAC9C,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,CAA2B,CAAC;IAClE,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAChF,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO;IAC/B,IAAI,IAAI,EAAE,CAAC;QACT,iFAAiF;QACjF,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QACnE,OAAO;IACT,CAAC;IACD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,IAAI,IAAI,uBAAuB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACtE,uGAAuG;IACvG,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACnB,IAAI,IAAI,CAAC,MAAM,EAAE,MAAM,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvE,oDAAoD;QACpD,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC;QAC3B,OAAO;IACT,CAAC;IACD,MAAM,KAAK,CAAC;AACd,CAAC;AAED,6FAA6F;AAC7F,SAAS,SAAS,CAAC,CAAa,EAAE,CAAa;IAC7C,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC;IAC9B,OAAO;QACL,QAAQ,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QACtD,IAAI,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM;QACzC,QAAQ,EAAE,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ;QACjC,IAAI,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI;QACrB,MAAM,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM;QAC3B,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvC,WAAW,EAAE,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW;QAC1C,iBAAiB,EAAE,CAAC,CAAC,iBAAiB,GAAG,CAAC,CAAC,iBAAiB;QAC5D,YAAY,EAAE,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,YAAY;KAC/C,CAAC;AACJ,CAAC"}

package/dist/report.d.ts

@@ -6,20 +6,41 @@
  * outcome. `attest policy evaluate` gates on `results`; the pr-annotate action
  * places inline comments with `file` and `line`.
  */
-import { type AssertionResult } from "@attest-ai/core";
+import { type AssertionResult, type Usage } from "@attest-ai/core";
+import type { ReplayMeta } from "./replay.js";
 export declare const REPORT_VERSION = 2;
 export type TestOutcome = "passed" | "failed" | "skipped";
+/** An assertion result as recorded for the report; `trial` is set under trials() with N > 1. */
+export type RecordedResult = AssertionResult & {
+    readonly trial?: number;
+};
+/** How a trials() call with N > 1 went. */
+export interface TrialsMeta {
+    /** N. */
+    requested: number;
+    /** Trials actually run; fewer than N when the test's cost budget was spent. */
+    run: number;
+    passed: number;
+    /** Trials that passed, failed or raised (not counting not-evaluated ones). */
+    evaluated: number;
+    notEvaluated: number;
+    stoppedByBudget: boolean;
+}
 /** Attest results a test recorded, carried from the worker in task meta. */
 export interface AttestTaskMeta {
     /** `retryCount` of the attempt that produced `results`. */
     attempt: number;
-    results: AssertionResult[];
+    results: RecordedResult[];
     totalCost: number;
     totalDurationMs: number;
+    /** Attest model usage of every attempt of the test, including retried ones. */
+    usage: Usage;
+    trials?: TrialsMeta;
 }
 declare module "vitest" {
     interface TaskMeta {
         attest?: AttestTaskMeta;
+        attestReplay?: ReplayMeta;
     }
 }
 export interface TestRecord {
@@ -32,6 +53,20 @@ export interface TestRecord {
     outcome: TestOutcome;
     durationMs: number;
     attest: AttestTaskMeta | undefined;
+    replay?: ReplayMeta;
 }
-export declare function buildReport(tests: readonly TestRecord[], simulated: boolean): Record<string, unknown>;
+/**
+ * `unattached` is the usage of evaluations that belong to no test (beforeAll,
+ * scripts); it counts in `summary.usage`.
+ */
+export declare function buildReport(tests: readonly TestRecord[], simulated: boolean, unattached?: Usage): Record<string, unknown>;
+/** Attest usage summed over every test of the run plus the evaluations that belong to no test. */
+export declare function sessionUsage(tests: readonly TestRecord[], unattached?: Usage): Usage;
+/**
+ * Lines for the end of the run: one per trials test
+ * ("<test_id>: <passed>/<evaluated> trials passed (<n> not evaluated)", plus a
+ * note when the test budget stopped it early), then the "attest usage: …" line
+ * when the session made or refused a model call.
+ */
+export declare function summaryLines(tests: readonly TestRecord[], unattached?: Usage): string[];
 //# sourceMappingURL=report.d.ts.map

package/dist/report.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAKL,KAAK,eAAe,EACrB,MAAM,iBAAiB,CAAC;AAEzB,eAAO,MAAM,cAAc,IAAI,CAAC;AAEhC,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE1D,4EAA4E;AAC5E,MAAM,WAAW,cAAc;IAC7B,2DAA2D;IAC3D,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,QAAQ;QAChB,MAAM,CAAC,EAAE,cAAc,CAAC;KACzB;CACF;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,oDAAoD;IACpD,IAAI,EAAE,MAAM,CAAC;IACb,2CAA2C;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,cAAc,GAAG,SAAS,CAAC;CACpC;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,EAAE,SAAS,EAAE,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CA8CrG"}
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EASL,KAAK,eAAe,EACpB,KAAK,KAAK,EACX,MAAM,iBAAiB,CAAC;AAEzB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAG9C,eAAO,MAAM,cAAc,IAAI,CAAC;AAEhC,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE1D,gGAAgG;AAChG,MAAM,MAAM,cAAc,GAAG,eAAe,GAAG;IAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAE3E,2CAA2C;AAC3C,MAAM,WAAW,UAAU;IACzB,SAAS;IACT,SAAS,EAAE,MAAM,CAAC;IAClB,+EAA+E;IAC/E,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,8EAA8E;IAC9E,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,OAAO,CAAC;CAC1B;AAED,4EAA4E;AAC5E,MAAM,WAAW,cAAc;IAC7B,2DAA2D;IAC3D,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,+EAA+E;IAC/E,KAAK,EAAE,KAAK,CAAC;IACb,MAAM,CAAC,EAAE,UAAU,CAAC;CACrB;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,QAAQ;QAChB,MAAM,CAAC,EAAE,cAAc,CAAC;QACxB,YAAY,CAAC,EAAE,UAAU,CAAC;KAC3B;CACF;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,oDAAoD;IACpD,IAAI,EAAE,MAAM,CAAC;IACb,2CAA2C;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,cAAc,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,UAAU,CAAC;CACrB;AAED;;;GAGG;AACH,wBAAgB,WAAW,CACzB,KAAK,EAAE,SAAS,UAAU,EAAE,EAC5B,SAAS,EAAE,OAAO,EAClB,UAAU,GAAE,KAAmB,GAC9B,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CA8EzB;AAkBD,kGAAkG;AAClG,wBAAgB,YAAY,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,EAAE,UAAU,GAAE,KAAmB,GAAG,KAAK,CAEjG;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,EAAE,UAAU,GAAE,KAAmB,GAAG,MAAM,EAAE,CA0BpG"}