@attest-ai/vitest 1.0.0-rc.3 → 1.0.0-rc.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/fixtures.d.ts +11 -2
- package/dist/fixtures.d.ts.map +1 -1
- package/dist/fixtures.js +16 -5
- package/dist/fixtures.js.map +1 -1
- package/dist/report.d.ts +23 -2
- package/dist/report.d.ts.map +1 -1
- package/dist/report.js +51 -2
- package/dist/report.js.map +1 -1
- package/dist/reporter.js +2 -2
- package/dist/reporter.js.map +1 -1
- package/dist/test-state.d.ts +8 -2
- package/dist/test-state.d.ts.map +1 -1
- package/dist/test-state.js +28 -3
- package/dist/test-state.js.map +1 -1
- package/package.json +2 -2
package/dist/fixtures.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { TestContext } from "vitest";
|
|
2
|
-
import { type ExpectChain, AgentResult, AttestClient } from "@attest-ai/core";
|
|
2
|
+
import { type ExpectChain, AgentResult, AttestClient, type Verdict } from "@attest-ai/core";
|
|
3
3
|
import { type TestBudget } from "./test-state.js";
|
|
4
4
|
export interface InspectOptions {
|
|
5
5
|
/** Throw when the cumulative session cost exceeds this USD amount. */
|
|
@@ -33,7 +33,7 @@ export interface EvaluateOptions extends InspectOptions {
|
|
|
33
33
|
* soft failures pass. Inside a trial of trials() (N > 1) it does nothing and
|
|
34
34
|
* returns `result`: trials() enforces the aggregate outcome after the last trial.
|
|
35
35
|
*/
|
|
36
|
-
export declare function enforceOutcome(result:
|
|
36
|
+
export declare function enforceOutcome<V extends Verdict>(result: V, strict: boolean, context?: TestContext): V;
|
|
37
37
|
export declare class AttestEngineFixture {
|
|
38
38
|
private manager;
|
|
39
39
|
private _client;
|
|
@@ -56,6 +56,15 @@ export declare class AttestEngineFixture {
|
|
|
56
56
|
inspect(chain: ExpectChain, options?: InspectOptions): Promise<AgentResult>;
|
|
57
57
|
/** Evaluate collected assertions and enforce the test outcome. */
|
|
58
58
|
evaluate(chain: ExpectChain, options?: EvaluateOptions): Promise<AgentResult>;
|
|
59
|
+
/**
|
|
60
|
+
* Fail or skip the running test according to a verdict's outcome: an
|
|
61
|
+
* `AgentResult` from `inspect()` or the aggregate of `scenario()` /
|
|
62
|
+
* `repeat()`. See `enforceOutcome`.
|
|
63
|
+
*/
|
|
64
|
+
enforce<V extends Verdict>(verdict: V, options?: {
|
|
65
|
+
strict?: boolean;
|
|
66
|
+
context?: TestContext;
|
|
67
|
+
}): V;
|
|
59
68
|
}
|
|
60
69
|
export declare function useAttest(): AttestEngineFixture;
|
|
61
70
|
/**
|
package/dist/fixtures.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fixtures.d.ts","sourceRoot":"","sources":["../src/fixtures.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AAC1C,OAAO,EACL,KAAK,WAAW,EAChB,WAAW,EAEX,YAAY,
|
|
1
|
+
{"version":3,"file":"fixtures.d.ts","sourceRoot":"","sources":["../src/fixtures.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AAC1C,OAAO,EACL,KAAK,WAAW,EAChB,WAAW,EAEX,YAAY,EAIZ,KAAK,OAAO,EAMb,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAaL,KAAK,UAAU,EAChB,MAAM,iBAAiB,CAAC;AAEzB,MAAM,WAAW,cAAc;IAC7B,sEAAsE;IACtE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB;;;;OAIG;IACH,OAAO,CAAC,EAAE,WAAW,CAAC;CACvB;AAED,MAAM,WAAW,eAAgB,SAAQ,cAAc;IACrD;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,CAAC,SAAS,OAAO,EAC9C,MAAM,EAAE,CAAC,EACT,MAAM,EAAE,OAAO,EACf,OAAO,CAAC,EAAE,WAAW,GACpB,CAAC,CAaH;AAmBD,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,OAAO,CAA4B;IAC3C,OAAO,CAAC,OAAO,CAA2B;IAC1C,OAAO,CAAC,QAAQ,CAA4B;gBAEhC,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,YAAY,CAAA;KAAE;IAWvF;;;;OAIG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAoBtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B,IAAI,MAAM,IAAI,YAAY,CAKzB;YAEa,aAAa;IAM3B,wEAAwE;IAClE,OAAO,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,CAAC;IA2EjF,kEAAkE;IAC5D,QAAQ,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,WAAW,CAAC;IAKnF;;;;OAIG;IACH,OAAO,CAAC,CAAC,SAAS,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,WAAW,CAAA;KAAE,GAAG,CAAC;CAGjG;AAOD,wBAAgB,SAAS,IAAI,mBAAmB,CAO/C;AAED;;;GAGG;AACH,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,WAAW,EAClB,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,WAAW,CAAC,CAEtB;AAED,yFAAyF;AACzF,wBAAsB,OAAO,CAC3B,KAAK,EAAE,WAAW,EAClB,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,WAAW,CAAC,CAEtB"}
|
package/dist/fixtures.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as vitest from "vitest";
|
|
2
|
-
import { AgentResult, STATUS_SOFT_FAIL, AttestClient, EngineManager, isSimulationMode,
|
|
3
|
-
import { assignAssertionIds, attemptState, inTrial, recordAgentSpend, recordOnTest, recordUnattachedUsage, remainingBudget, resolveTest, strictFromEnv, trialScope, validateBudget, } from "./test-state.js";
|
|
2
|
+
import { AgentResult, STATUS_SOFT_FAIL, AttestClient, EngineManager, isSimulationMode, renderVerdictDiagnostics, requireSimulationCapability, OUTCOME_FAIL, OUTCOME_SKIP, zeroUsage, } from "@attest-ai/core";
|
|
3
|
+
import { assignAssertionIds, batchContextOf, attemptState, inTrial, recordAgentSpend, recordOnTest, recordUnattachedUsage, remainingBudget, resolveTest, strictFromEnv, trialScope, validateBudget, } from "./test-state.js";
|
|
4
4
|
/**
|
|
5
5
|
* Fail or skip the running Vitest test according to the batch outcome:
|
|
6
6
|
* any hard_fail throws with rendered diagnostics; otherwise any
|
|
@@ -13,8 +13,8 @@ export function enforceOutcome(result, strict, context) {
|
|
|
13
13
|
return result;
|
|
14
14
|
const outcome = result.outcome({ strict });
|
|
15
15
|
if (outcome === OUTCOME_FAIL) {
|
|
16
|
-
const header = result.
|
|
17
|
-
throw new Error(header +
|
|
16
|
+
const header = result.outcome({ strict: false }) === OUTCOME_FAIL ? "" : `strict: ${result.skipReason()}\n`;
|
|
17
|
+
throw new Error(header + renderVerdictDiagnostics(result).replace(/^\n/, ""));
|
|
18
18
|
}
|
|
19
19
|
if (outcome === OUTCOME_SKIP) {
|
|
20
20
|
(context ?? trialScope.getStore()?.test.context ?? currentTestContext(result)).skip(result.skipReason());
|
|
@@ -121,7 +121,10 @@ export class AttestEngineFixture {
|
|
|
121
121
|
// Outside a test the budget covers this one evaluation.
|
|
122
122
|
batchBudget = remainingBudget(budget, zeroUsage(), chain.trace.metadata?.cost_usd ?? 0);
|
|
123
123
|
}
|
|
124
|
-
const result = await this.client.evaluateBatch(chain.trace, assertions,
|
|
124
|
+
const result = await this.client.evaluateBatch(chain.trace, assertions, {
|
|
125
|
+
...(batchBudget === undefined ? {} : { budget: batchBudget }),
|
|
126
|
+
...(test === undefined ? {} : { context: batchContextOf(test) }),
|
|
127
|
+
});
|
|
125
128
|
globalThis.__attest_session_cost__ =
|
|
126
129
|
(globalThis.__attest_session_cost__ ?? 0) + (result.total_cost ?? 0);
|
|
127
130
|
const softFailCount = (result.results ?? []).filter((r) => r.status === STATUS_SOFT_FAIL).length;
|
|
@@ -155,6 +158,14 @@ export class AttestEngineFixture {
|
|
|
155
158
|
const result = await this.inspect(chain, options);
|
|
156
159
|
return enforceOutcome(result, options?.strict ?? strictFromEnv(), options?.context);
|
|
157
160
|
}
|
|
161
|
+
/**
|
|
162
|
+
* Fail or skip the running test according to a verdict's outcome: an
|
|
163
|
+
* `AgentResult` from `inspect()` or the aggregate of `scenario()` /
|
|
164
|
+
* `repeat()`. See `enforceOutcome`.
|
|
165
|
+
*/
|
|
166
|
+
enforce(verdict, options) {
|
|
167
|
+
return enforceOutcome(verdict, options?.strict ?? strictFromEnv(), options?.context);
|
|
168
|
+
}
|
|
158
169
|
}
|
|
159
170
|
// Vitest runs globalSetup in the main process and tests in workers, so a
|
|
160
171
|
// worker that cannot see the global client starts its own engine on first use.
|
package/dist/fixtures.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fixtures.js","sourceRoot":"","sources":["../src/fixtures.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AAEjC,OAAO,EAEL,WAAW,EACX,gBAAgB,EAChB,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,
|
|
1
|
+
{"version":3,"file":"fixtures.js","sourceRoot":"","sources":["../src/fixtures.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AAEjC,OAAO,EAEL,WAAW,EACX,gBAAgB,EAChB,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,wBAAwB,EAExB,2BAA2B,EAC3B,YAAY,EACZ,YAAY,EACZ,SAAS,GAEV,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,kBAAkB,EAClB,cAAc,EACd,YAAY,EACZ,OAAO,EACP,gBAAgB,EAChB,YAAY,EACZ,qBAAqB,EACrB,eAAe,EACf,WAAW,EACX,aAAa,EACb,UAAU,EACV,cAAc,GAEf,MAAM,iBAAiB,CAAC;AA6BzB;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAC5B,MAAS,EACT,MAAe,EACf,OAAqB;IAErB,IAAI,OAAO,EAAE;QAAE,OAAO,MAAM,CAAC;IAC7B,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAC3C,IAAI,OAAO,KAAK,YAAY,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,KAAK,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC;QAC5G,MAAM,IAAI,KAAK,CAAC,MAAM,GAAG,wBAAwB,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;IAChF,CAAC;IACD,IAAI,OAAO,KAAK,YAAY,EAAE,CAAC;QAC7B,CAAC,OAAO,IAAI,UAAU,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CACjF,MAAM,CAAC,UAAU,EAAE,CACpB,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,MAAe;IACzC,MAAM,MAAM,GAAI,MAAiC,CAAC,UAAU,CAAC;IAC7D,MAAM,IAAI,GAAG,MAAM,EAAE,cAAc,EAAE,CAAC;IACtC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CACb,GAAG,MAAM,CAAC,UAAU,EAAE,8CAA8C;YAClE,8EAA8E,CACjF,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC,OAAO,CAAC;AACtB,CAAC;AAED,MAAM,OAAO,mBAAmB;IACtB,OAAO,CAA4B;IACnC,OAAO,CAA2B;IAClC,QAAQ,CAA4B;IAE5C,YAAY,OAA2E;QACrF,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,OAAO;QACT,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,aAAa,CAAC;YAC/B,UAAU,EAAE,OAAO,EAAE,UAAU;YAC/B,QAAQ,EAAE,OAAO,EAAE,QAAQ,IAAI,MAAM;SACtC,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QACD,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAC;QACtC,IAAI,UAAU,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,GAAG,CAAC;QACzC,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACxC,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,2BAA2B,CAAC,IAAI,CAAC,CAAC;YACpC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC1B,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAC1B,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;YACzB,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,IAAI,MAAM;QACR,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS;YAAE,OAAO;QACvC,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC;IACtB,CAAC;IAED,wEAAwE;IACxE,KAAK,CAAC,OAAO,CAAC,KAAkB,EAAE,OAAwB;QACxD,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC3B,4EAA4E;QAC5E,4EAA4E;QAC5E,mEAAmE;QACnE,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QACnF,MAAM,MAAM,GACV,OAAO,EAAE,UAAU,KAAK,SAAS;YAC/B,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,UAAU,CAAC;YACpC,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI;gBAC1C,CAAC,CAAC,KAAK,CAAC,MAAM;gBACd,CAAC,CAAC,SAAS,CAAC;QAClB,IAAI,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;QAClC,IAAI,WAAoC,CAAC;QACzC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,UAAU,GAAG,kBAAkB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAClD,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;YACpC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;gBACjC,WAAW,GAAG,eAAe,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;aAAM,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YAChC,wDAAwD;YACxD,WAAW,GAAG,eAAe,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,IAAI,CAAC,CAAC,CAAC;QAC1F,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,UAAU,EAAE;YACtE,GAAG,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;YAC7D,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;SACjE,CAAC,CAAC;QAEH,UAAU,CAAC,uBAAuB;YAChC,CAAC,UAAU,CAAC,uBAAuB,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC;QAEvE,MAAM,aAAa,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,CACjD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,gBAAgB,CACrC,CAAC,MAAM,CAAC;QACT,UAAU,CAAC,gCAAgC;YACzC,CAAC,UAAU,CAAC,gCAAgC,IAAI,CAAC,CAAC,GAAG,aAAa,CAAC;QAErE,MAAM,WAAW,GAAG,IAAI,WAAW,CACjC,KAAK,CAAC,KAAK,EACX,MAAM,CAAC,OAAO,IAAI,EAAE,EACpB,MAAM,CAAC,UAAU,IAAI,CAAC,EACtB,MAAM,CAAC,iBAAiB,IAAI,CAAC,EAC7B,MAAM,CAAC,KAAK,CACb,CAAC;QACF,IAAI,IAAI,KAAK,SAAS;YAAE,YAAY,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;;YAC1D,qBAAqB,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAC9C,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI;YAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEhF,sEAAsE;QACtE,kEAAkE;QAClE,oDAAoD;QACpD,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,CAAC,UAAU,CAAC,2BAA2B,KAAK,EAAE,CAAC,CAAC;YAC7D,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACtB,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QAED,IACE,OAAO,EAAE,MAAM,KAAK,SAAS;YAC7B,UAAU,CAAC,uBAAuB,GAAG,OAAO,CAAC,MAAM,EACnD,CAAC;YACD,MAAM,IAAI,KAAK,CACb,iCAAiC,UAAU,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;gBAC9E,cAAc,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAC5C,CAAC;QACJ,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,kEAAkE;IAClE,KAAK,CAAC,QAAQ,CAAC,KAAkB,EAAE,OAAyB;QAC1D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAClD,OAAO,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,aAAa,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IACtF,CAAC;IAED;;;;OAIG;IACH,OAAO,CAAoB,OAAU,EAAE,OAAqD;QAC1F,OAAO,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,aAAa,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IACvF,CAAC;CACF;AAED,yEAAyE;AACzE,+EAA+E;AAC/E,sEAAsE;AACtE,IAAI,aAA8C,CAAC;AAEnD,MAAM,UAAU,SAAS;IACvB,MAAM,MAAM,GAAG,UAAU,CAAC,iBAAiB,CAAC;IAC5C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,IAAI,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAC7C,CAAC;IACD,aAAa,KAAK,IAAI,mBAAmB,EAAE,CAAC;IAC5C,OAAO,aAAa,CAAC;AACvB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAkB,EAClB,OAAyB;IAEzB,OAAO,SAAS,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC9C,CAAC;AAED,yFAAyF;AACzF,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,KAAkB,EAClB,OAAwB;IAExB,OAAO,SAAS,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC7C,CAAC"}
|
package/dist/report.d.ts
CHANGED
|
@@ -6,14 +6,33 @@
|
|
|
6
6
|
* outcome. `attest policy evaluate` gates on `results`; the pr-annotate action
|
|
7
7
|
* places inline comments with `file` and `line`.
|
|
8
8
|
*/
|
|
9
|
-
import { type AssertionResult, type Usage } from "@attest-ai/core";
|
|
9
|
+
import { type AssertionResult, type Trace, type Usage } from "@attest-ai/core";
|
|
10
10
|
import type { ReplayMeta } from "./replay.js";
|
|
11
11
|
export declare const REPORT_VERSION = 2;
|
|
12
12
|
export type TestOutcome = "passed" | "failed" | "skipped";
|
|
13
13
|
/** An assertion result as recorded for the report; `trial` is set under trials() with N > 1. */
|
|
14
14
|
export type RecordedResult = AssertionResult & {
|
|
15
15
|
readonly trial?: number;
|
|
16
|
+
/** Key into `AttestTaskMeta.traces`; set only when `ATTEST_REPORT_TRACES=1`. */
|
|
17
|
+
readonly trace_id?: string;
|
|
16
18
|
};
|
|
19
|
+
/**
|
|
20
|
+
* Set `ATTEST_REPORT_TRACES=1` to embed the evaluated traces in the report (`traces` and each
|
|
21
|
+
* row's `trace_id`) for `attest view`. Off by default: traces hold prompts, model outputs and
|
|
22
|
+
* tool arguments.
|
|
23
|
+
*/
|
|
24
|
+
export declare const REPORT_TRACES_ENV = "ATTEST_REPORT_TRACES";
|
|
25
|
+
export declare function reportTracesEnabled(): boolean;
|
|
26
|
+
/**
|
|
27
|
+
* The report's `traces` map: one entry per distinct trace. Equal traces share an id; distinct
|
|
28
|
+
* traces sharing a `trace_id` get `<id>#2`, `<id>#3`, ... and a trace without an id gets
|
|
29
|
+
* `trace-<n>`, so a row's `trace_id` names exactly the trace it was evaluated against.
|
|
30
|
+
*/
|
|
31
|
+
export declare class TraceTable {
|
|
32
|
+
readonly traces: Record<string, Trace>;
|
|
33
|
+
constructor(traces?: Record<string, Trace>);
|
|
34
|
+
add(trace: Trace): string;
|
|
35
|
+
}
|
|
17
36
|
/** How a trials() call with N > 1 went. */
|
|
18
37
|
export interface TrialsMeta {
|
|
19
38
|
/** N. */
|
|
@@ -36,6 +55,8 @@ export interface AttestTaskMeta {
|
|
|
36
55
|
/** Attest model usage of every attempt of the test, including retried ones. */
|
|
37
56
|
usage: Usage;
|
|
38
57
|
trials?: TrialsMeta;
|
|
58
|
+
/** Traces the results were evaluated against, by `trace_id` (only with `ATTEST_REPORT_TRACES=1`). */
|
|
59
|
+
traces?: Record<string, Trace>;
|
|
39
60
|
}
|
|
40
61
|
declare module "vitest" {
|
|
41
62
|
interface TaskMeta {
|
|
@@ -59,7 +80,7 @@ export interface TestRecord {
|
|
|
59
80
|
* `unattached` is the usage of evaluations that belong to no test (beforeAll,
|
|
60
81
|
* scripts); it counts in `summary.usage`.
|
|
61
82
|
*/
|
|
62
|
-
export declare function buildReport(tests: readonly TestRecord[], simulated: boolean, unattached?: Usage): Record<string, unknown>;
|
|
83
|
+
export declare function buildReport(tests: readonly TestRecord[], simulated: boolean, unattached?: Usage, includeTraces?: boolean): Record<string, unknown>;
|
|
63
84
|
/** Attest usage summed over every test of the run plus the evaluations that belong to no test. */
|
|
64
85
|
export declare function sessionUsage(tests: readonly TestRecord[], unattached?: Usage): Usage;
|
|
65
86
|
/**
|
package/dist/report.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EASL,KAAK,eAAe,EACpB,KAAK,KAAK,EACX,MAAM,iBAAiB,CAAC;AAEzB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAG9C,eAAO,MAAM,cAAc,IAAI,CAAC;AAEhC,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE1D,gGAAgG;AAChG,MAAM,MAAM,cAAc,GAAG,eAAe,GAAG;
|
|
1
|
+
{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EASL,KAAK,eAAe,EACpB,KAAK,KAAK,EACV,KAAK,KAAK,EACX,MAAM,iBAAiB,CAAC;AAEzB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAG9C,eAAO,MAAM,cAAc,IAAI,CAAC;AAEhC,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE1D,gGAAgG;AAChG,MAAM,MAAM,cAAc,GAAG,eAAe,GAAG;IAC7C,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,gFAAgF;IAChF,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,iBAAiB,yBAAyB,CAAC;AAExD,wBAAgB,mBAAmB,IAAI,OAAO,CAE7C;AAED;;;;GAIG;AACH,qBAAa,UAAU;IACT,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC;gBAA7B,MAAM,GAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAM;IAEvD,GAAG,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM;CAY1B;AAED,2CAA2C;AAC3C,MAAM,WAAW,UAAU;IACzB,SAAS;IACT,SAAS,EAAE,MAAM,CAAC;IAClB,+EAA+E;IAC/E,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,8EAA8E;IAC9E,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,OAAO,CAAC;CAC1B;AAED,4EAA4E;AAC5E,MAAM,WAAW,cAAc;IAC7B,2DAA2D;IAC3D,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,+EAA+E;IAC/E,KAAK,EAAE,KAAK,CAAC;IACb,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,qGAAqG;IACrG,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;CAChC;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,QAAQ;QAChB,MAAM,CAAC,EAAE,cAAc,CAAC;QACxB,YAAY,CAAC,EAAE,UAAU,CAAC;KAC3B;CACF;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,oDAAoD;IACpD,IAAI,EAAE,MAAM,CAAC;IACb,2CAA2C;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,cAAc,GAAG,SAAS,CAAC;IACnC,MAAM,CAAC,EAAE,UAAU,CAAC;CACrB;AAED;;;GAGG;AACH,wBAAgB,WAAW,CACzB,KAAK,EAAE,SAAS,UAAU,EAAE,EAC5B,SAAS,EAAE,OAAO,EAClB,UAAU,GAAE,KAAmB,EAC/B,aAAa,UAAQ,GACpB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CA6FzB;AAkBD,kGAAkG;AAClG,wBAAgB,YAAY,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,EAAE,UAAU,GAAE,KAAmB,GAAG,KAAK,CAEjG;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,EAAE,UAAU,GAAE,KAAmB,GAAG,MAAM,EAAE,CA0BpG"}
|
package/dist/report.js
CHANGED
|
@@ -9,11 +9,45 @@
|
|
|
9
9
|
import { REASON_REPLAY_MISS, STATUS_HARD_FAIL, STATUS_NOT_EVALUATED, STATUS_PASS, STATUS_SOFT_FAIL, sumUsage, usageSummaryLine, zeroUsage, } from "@attest-ai/core";
|
|
10
10
|
import { liveLine } from "@attest-ai/core";
|
|
11
11
|
export const REPORT_VERSION = 2;
|
|
12
|
+
/**
|
|
13
|
+
* Set `ATTEST_REPORT_TRACES=1` to embed the evaluated traces in the report (`traces` and each
|
|
14
|
+
* row's `trace_id`) for `attest view`. Off by default: traces hold prompts, model outputs and
|
|
15
|
+
* tool arguments.
|
|
16
|
+
*/
|
|
17
|
+
export const REPORT_TRACES_ENV = "ATTEST_REPORT_TRACES";
|
|
18
|
+
export function reportTracesEnabled() {
|
|
19
|
+
return (process.env[REPORT_TRACES_ENV] ?? "").trim() === "1";
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* The report's `traces` map: one entry per distinct trace. Equal traces share an id; distinct
|
|
23
|
+
* traces sharing a `trace_id` get `<id>#2`, `<id>#3`, ... and a trace without an id gets
|
|
24
|
+
* `trace-<n>`, so a row's `trace_id` names exactly the trace it was evaluated against.
|
|
25
|
+
*/
|
|
26
|
+
export class TraceTable {
|
|
27
|
+
traces;
|
|
28
|
+
constructor(traces = {}) {
|
|
29
|
+
this.traces = traces;
|
|
30
|
+
}
|
|
31
|
+
add(trace) {
|
|
32
|
+
const base = trace.trace_id === "" ? `trace-${Object.keys(this.traces).length + 1}` : trace.trace_id;
|
|
33
|
+
const doc = JSON.stringify(trace);
|
|
34
|
+
let key = base;
|
|
35
|
+
for (let n = 1; key in this.traces;) {
|
|
36
|
+
if (JSON.stringify(this.traces[key]) === doc)
|
|
37
|
+
return key;
|
|
38
|
+
n += 1;
|
|
39
|
+
key = `${base}#${n}`;
|
|
40
|
+
}
|
|
41
|
+
this.traces[key] = trace;
|
|
42
|
+
return key;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
12
45
|
/**
|
|
13
46
|
* `unattached` is the usage of evaluations that belong to no test (beforeAll,
|
|
14
47
|
* scripts); it counts in `summary.usage`.
|
|
15
48
|
*/
|
|
16
|
-
export function buildReport(tests, simulated, unattached = zeroUsage()) {
|
|
49
|
+
export function buildReport(tests, simulated, unattached = zeroUsage(), includeTraces = false) {
|
|
50
|
+
const table = includeTraces ? new TraceTable() : undefined;
|
|
17
51
|
const rows = [];
|
|
18
52
|
const testRows = [];
|
|
19
53
|
let totalCost = 0;
|
|
@@ -25,8 +59,22 @@ export function buildReport(tests, simulated, unattached = zeroUsage()) {
|
|
|
25
59
|
const reasons = [];
|
|
26
60
|
totalCost += test.attest?.totalCost ?? 0;
|
|
27
61
|
totalDurationMs += test.attest?.totalDurationMs ?? 0;
|
|
62
|
+
// The worker keyed traces per test; re-key them into the report-wide table.
|
|
63
|
+
const traceKeys = new Map();
|
|
64
|
+
for (const [local, trace] of Object.entries(test.attest?.traces ?? {})) {
|
|
65
|
+
if (table !== undefined)
|
|
66
|
+
traceKeys.set(local, table.add(trace));
|
|
67
|
+
}
|
|
28
68
|
for (const result of test.attest?.results ?? []) {
|
|
29
|
-
|
|
69
|
+
const { trace_id: local, ...rest } = result;
|
|
70
|
+
const traceId = local === undefined ? undefined : traceKeys.get(local);
|
|
71
|
+
rows.push({
|
|
72
|
+
...rest,
|
|
73
|
+
...(traceId === undefined ? {} : { trace_id: traceId }),
|
|
74
|
+
test_id: test.testId,
|
|
75
|
+
file: test.file,
|
|
76
|
+
line: test.line,
|
|
77
|
+
});
|
|
30
78
|
assertionIds.add(result.assertion_id);
|
|
31
79
|
if (result.status === STATUS_NOT_EVALUATED && !reasons.includes(String(result.reason))) {
|
|
32
80
|
reasons.push(String(result.reason));
|
|
@@ -92,6 +140,7 @@ export function buildReport(tests, simulated, unattached = zeroUsage()) {
|
|
|
92
140
|
tests: testRows,
|
|
93
141
|
total_cost: totalCost,
|
|
94
142
|
total_duration_ms: totalDurationMs,
|
|
143
|
+
...(table === undefined ? {} : { traces: table.traces }),
|
|
95
144
|
};
|
|
96
145
|
}
|
|
97
146
|
/** `summary.replay` for the tests that ran in replay mode; absent when none did. */
|
package/dist/report.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"report.js","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EACL,kBAAkB,EAClB,gBAAgB,EAChB,oBAAoB,EACpB,WAAW,EACX,gBAAgB,EAChB,QAAQ,EACR,gBAAgB,EAChB,SAAS,
|
|
1
|
+
{"version":3,"file":"report.js","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EACL,kBAAkB,EAClB,gBAAgB,EAChB,oBAAoB,EACpB,WAAW,EACX,gBAAgB,EAChB,QAAQ,EACR,gBAAgB,EAChB,SAAS,GAIV,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAE3C,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC;AAWhC;;;;GAIG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,sBAAsB,CAAC;AAExD,MAAM,UAAU,mBAAmB;IACjC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC;AAC/D,CAAC;AAED;;;;GAIG;AACH,MAAM,OAAO,UAAU;IACA;IAArB,YAAqB,SAAgC,EAAE;QAAlC,WAAM,GAAN,MAAM,CAA4B;IAAG,CAAC;IAE3D,GAAG,CAAC,KAAY;QACd,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;QACrG,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,GAAG,GAAG,IAAI,CAAC;QACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,IAAI,CAAC,MAAM,GAAI,CAAC;YACrC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG;gBAAE,OAAO,GAAG,CAAC;YACzD,CAAC,IAAI,CAAC,CAAC;YACP,GAAG,GAAG,GAAG,IAAI,IAAI,CAAC,EAAE,CAAC;QACvB,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACzB,OAAO,GAAG,CAAC;IACb,CAAC;CACF;AAiDD;;;GAGG;AACH,MAAM,UAAU,WAAW,CACzB,KAA4B,EAC5B,SAAkB,EAClB,aAAoB,SAAS,EAAE,EAC/B,aAAa,GAAG,KAAK;IAErB,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,IAAI,UAAU,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3D,MAAM,IAAI,GAA8B,EAAE,CAAC;IAC3C,MAAM,QAAQ,GAA8B,EAAE,CAAC;IAC/C,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,MAAM,4BAA4B,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1F,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QACvC,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,SAAS,IAAI,IAAI,CAAC,MAAM,EAAE,SAAS,IAAI,CAAC,CAAC;QACzC,eAAe,IAAI,IAAI,CAAC,MAAM,EAAE,eAAe,IAAI,CAAC,CAAC;QACrD,4EAA4E;QAC5E,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC5C,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;YACvE,IAAI,KAAK,KAAK,SAAS;gBAAE,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;QAClE,CAAC;QACD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC;YAChD,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC;YAC5C,MAAM,OAAO,GAAG,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACvE,IAAI,CAAC,IAAI,CAAC;gBACR,GAAG,IAAI;gBACP,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;gBACvD,OAAO,EAAE,IAAI,CAAC,MAAM;gBACpB,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;aAChB,CAAC,CAAC;YACH,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YACtC,IAAI,MAAM,CAAC,MAAM,KAAK,oBAAoB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;gBACvF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,MAAM,EAAE,YAAY,KAAK,IAAI,EAAE,CAAC;YAClC,IAAI,CAAC,IAAI,CAAC;gBACR,YAAY,EAAE,UAAU,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,SAAS,EAAE;gBACzD,MAAM,EAAE,oBAAoB;gBAC5B,MAAM,EAAE,kBAAkB;gBAC1B,KAAK,EAAE,CAAC;gBACR,WAAW,EAAE,MAAM,CAAC,IAAI,IAAI,aAAa;gBACzC,IAAI,EAAE,CAAC;gBACP,WAAW,EAAE,CAAC;gBACd,OAAO,EAAE,IAAI,CAAC,MAAM;gBACpB,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;aAChB,CAAC,CAAC;YACH,YAAY,CAAC,GAAG,CAAC,UAAU,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,SAAS,EAAE,CAAC,CAAC;YAC9D,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC;gBAAE,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC9E,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC;YACZ,OAAO,EAAE,IAAI,CAAC,MAAM;YACpB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,qBAAqB,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,aAAa,EAAE,CAAC,GAAG,YAAY,CAAC;YAChC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC;YACxC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAChF,GAAG,CAAC,MAAM,KAAK,SAAS;gBACtB,CAAC,CAAC;oBACE,MAAM,EAAE;wBACN,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,IAAI,EAAE,MAAM,CAAC,IAAI;wBACjB,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,IAAI,EAAE,MAAM,CAAC,IAAI;wBACjB,MAAM,EAAE,MAAM,CAAC,MAAM;wBACrB,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;qBAC5D;iBACF;gBACH,CAAC,CAAC,EAAE,CAAC;SACR,CAAC,CAAC;IACL,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IACpF,OAAO;QACL,cAAc,EAAE,cAAc;QAC9B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;QAC7D,SAAS;QACT,OAAO,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC;YAC1B,SAAS,EAAE,KAAK,CAAC,gBAAgB,CAAC;YAClC,SAAS,EAAE,KAAK,CAAC,gBAAgB,CAAC;YAClC,aAAa,EAAE,KAAK,CAAC,oBAAoB,CAAC;YAC1C,KAAK,EAAE,YAAY,CAAC,KAAK,EAAE,UAAU,CAAC;YACtC,GAAG,aAAa,CAAC,KAAK,CAAC;SACxB;QACD,OAAO,EAAE,IAAI;QACb,KAAK,EAAE,QAAQ;QACf,UAAU,EAAE,SAAS;QACrB,iBAAiB,EAAE,eAAe;QAClC,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;KACzD,CAAC;AACJ,CAAC;AAED,oFAAoF;AACpF,SAAS,aAAa,CAAC,KAA4B;IACjD,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACjF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACpC,MAAM,GAAG,GAAG,CAAC,CAA4B,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACpF,OAAO;QACL,MAAM,EAAE;YACN,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;YAChC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;YACxB,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;YAC5B,aAAa,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC;YACxC,mBAAmB,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC;SACrD;KACF,CAAC;AACJ,CAAC;AAED,kGAAkG;AAClG,MAAM,UAAU,YAAY,CAAC,KAA4B,EAAE,aAAoB,SAAS,EAAE;IACxF,OAAO,QAAQ,CAAC,CAAC,UAAU,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC3G,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,KAA4B,EAAE,aAAoB,SAAS,EAAE;IACxF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC;QACnC,IAAI,MAAM,KAAK,SAAS;YAAE,SAAS;QACnC,KAAK,CAAC,IAAI,CACR,GAAG,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,SAAS,mBAAmB,MAAM,CAAC,YAAY,iBAAiB,CAC5G,CAAC;QACF,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CACR,GAAG,IAAI,CAAC,MAAM,mBAAmB,MAAM,CAAC,GAAG,OAAO,MAAM,CAAC,SAAS,4BAA4B,CAC/F,CAAC;QACJ,CAAC;IACH,CAAC;IACD,MAAM,KAAK,GAAG,gBAAgB,CAAC,YAAY,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC;IAChE,IAAI,KAAK,KAAK,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACjF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,CAA4B,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAClF,KAAK,CAAC,IAAI,CACR,kBAAkB,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,cAAc;YAC1D,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;YAClF,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAC5E,CAAC;IACJ,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
package/dist/reporter.js
CHANGED
|
@@ -2,7 +2,7 @@ import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSyn
|
|
|
2
2
|
import { tmpdir } from "node:os";
|
|
3
3
|
import { dirname, join, relative, resolve, sep } from "node:path";
|
|
4
4
|
import { isSimulationMode, parseUsage, renderDiagnostics, renderSummary, sumUsage, } from "@attest-ai/core";
|
|
5
|
-
import { buildReport, summaryLines } from "./report.js";
|
|
5
|
+
import { buildReport, reportTracesEnabled, summaryLines } from "./report.js";
|
|
6
6
|
import { ROOT_ENV, USAGE_DIR_ENV } from "./test-state.js";
|
|
7
7
|
/**
|
|
8
8
|
* Vitest reporter that prints the cost report and a hierarchical
|
|
@@ -121,7 +121,7 @@ export class AttestReporter {
|
|
|
121
121
|
return;
|
|
122
122
|
const path = resolve(this.root, outputFile);
|
|
123
123
|
mkdirSync(dirname(path), { recursive: true });
|
|
124
|
-
writeFileSync(path, `${JSON.stringify(buildReport(tests, isSimulationMode(), unattached), null, 2)}\n`);
|
|
124
|
+
writeFileSync(path, `${JSON.stringify(buildReport(tests, isSimulationMode(), unattached, reportTracesEnabled()), null, 2)}\n`);
|
|
125
125
|
}
|
|
126
126
|
}
|
|
127
127
|
/** `vitest run --reporter=@attest-ai/vitest/reporter` loads the default export. */
|
package/dist/reporter.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reporter.js","sourceRoot":"","sources":["../src/reporter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnG,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAClE,OAAO,EACL,gBAAgB,EAChB,UAAU,EACV,iBAAiB,EACjB,aAAa,EACb,QAAQ,GAET,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,WAAW,EAAE,YAAY,EAAmB,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"reporter.js","sourceRoot":"","sources":["../src/reporter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnG,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAClE,OAAO,EACL,gBAAgB,EAChB,UAAU,EACV,iBAAiB,EACjB,aAAa,EACb,QAAQ,GAET,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,WAAW,EAAE,mBAAmB,EAAE,YAAY,EAAmB,MAAM,aAAa,CAAC;AAC9F,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAM1D;;;;;;;GAOG;AACH,MAAM,OAAO,kBAAkB;IAC7B,UAAU,CAAC,MAAe,EAAE,OAAgB,EAAE,OAAyB;QACrE,MAAM,IAAI,GAAG,UAAU,CAAC,uBAAuB,IAAI,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,UAAU,CAAC,gCAAgC,IAAI,CAAC,CAAC;QACtE,MAAM,QAAQ,GAAG,UAAU,CAAC,2BAA2B,IAAI,EAAE,CAAC;QAE9D,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CACR,EAAE,EACF,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EACd,sBAAsB,EACtB,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CACf,CAAC;YACF,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;gBAC9B,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC;gBACtC,KAAK,CAAC,IAAI,CAAC,cAAc,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;QAED,KAAK,CAAC,IAAI,CACR,EAAE,EACF,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EACd,sBAAsB,EACtB,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EACd,mCAAmC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,EACxD,mCAAmC,YAAY,EAAE,EACjD,mCAAmC,QAAQ,CAAC,MAAM,EAAE,EACpD,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EACd,EAAE,CACH,CAAC;QAEF,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEhC,IAAI,OAAO,EAAE,SAAS,EAAE,CAAC;YACvB,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;CACF;AAUD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,OAAO,cAAc;IACjB,UAAU,CAAqB;IAC/B,IAAI,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IACrB,MAAM,CAAqB;IAC3B,QAAQ,CAAqB;IAErC,YAAY,UAAiC,EAAE;QAC7C,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;IACvC,CAAC;IAED,MAAM,CAAC,MAAc;QACnB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;QAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC;QAC5C,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;YACpE,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACzC,CAAC;QACD,MAAM,CAAC,MAAM,CAAC,mBAAmB,GAAG,IAAI,CAAC;QACzC,wEAAwE;QACxE,6DAA6D;QAC7D,IAAI,CAAC,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,eAAe,CAAC,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC;IAC7C,CAAC;IAED,cAAc;QACZ,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS;YAAE,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACjF,CAAC;IAED,qFAAqF;IAC7E,mBAAmB;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC1B,IAAI,GAAG,KAAK,SAAS;YAAE,OAAO,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAClE,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrG,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,OAAO,KAAK,CAAC;IACf,CAAC;IAED,YAAY,CAAC,WAAsC;QACjD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACnC,MAAM,KAAK,GAAiB,EAAE,CAAC;QAC/B,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC3E,KAAK,MAAM,QAAQ,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC;gBACtD,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;gBACnC,0EAA0E;gBAC1E,IAAI,QAAQ,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;oBACvD,MAAM,IAAI,KAAK,CACb,2CAA2C,QAAQ,CAAC,QAAQ,oBAAoB;wBAC9E,sFAAsF;wBACtF,sEAAsE,CACzE,CAAC;gBACJ,CAAC;gBACD,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC;gBACtC,KAAK,CAAC,IAAI,CAAC;oBACT,MAAM,EAAE,GAAG,IAAI,MAAM,QAAQ,CAAC,QAAQ,EAAE;oBACxC,IAAI,EAAE,QAAQ,CAAC,IAAI;oBACnB,IAAI;oBACJ,IAAI,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC;oBACzB,OAAO,EAAE,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;oBACrE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,EAAE,QAAQ,IAAI,CAAC;oBAChD,MAAM,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM;oBAC9B,MAAM,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,YAAY;iBACrC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,MAAM,UAAU,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAC9C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAChE,IAAI,UAAU,KAAK,SAAS;YAAE,OAAO;QACrC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAC5C,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,aAAa,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,KAAK,EAAE,gBAAgB,EAAE,EAAE,UAAU,EAAE,mBAAmB,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IACjI,CAAC;CACF;AAED,mFAAmF;AACnF,eAAe,cAAc,CAAC"}
|
package/dist/test-state.d.ts
CHANGED
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
6
6
|
import type { TestContext } from "vitest";
|
|
7
|
-
import { type AgentResult, type Assertion, type BatchBudget, type Trace, type Usage } from "@attest-ai/core";
|
|
8
|
-
import type
|
|
7
|
+
import { type AgentResult, type Assertion, type BatchBudget, type BatchContext, type Trace, type Usage } from "@attest-ai/core";
|
|
8
|
+
import { type AttestTaskMeta } from "./report.js";
|
|
9
9
|
export type RunningTest = NonNullable<TestContext["task"]>;
|
|
10
10
|
/** Limits for the attest model calls of one test (all its evaluations and trials). */
|
|
11
11
|
export interface TestBudget {
|
|
@@ -62,6 +62,12 @@ export declare const USAGE_DIR_ENV = "ATTEST_VITEST_USAGE_DIR";
|
|
|
62
62
|
* AttestReporter it is the file task's name, relative to the project root.
|
|
63
63
|
*/
|
|
64
64
|
export declare function testIdOf(test: RunningTest): string;
|
|
65
|
+
/**
|
|
66
|
+
* The `evaluate_batch` context naming `test`: its report test_id, its file and
|
|
67
|
+
* the line it starts on. The line is present only when Vitest records task
|
|
68
|
+
* locations (`test.includeTaskLocation`).
|
|
69
|
+
*/
|
|
70
|
+
export declare function batchContextOf(test: RunningTest): BatchContext;
|
|
65
71
|
/**
|
|
66
72
|
* Hand the usage of an evaluation that belongs to no test (beforeAll, a script)
|
|
67
73
|
* to AttestReporter: one file per evaluation in its usage directory, so the
|
package/dist/test-state.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"test-state.d.ts","sourceRoot":"","sources":["../src/test-state.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAKrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AAC1C,OAAO,EAGL,KAAK,WAAW,EAChB,KAAK,SAAS,EACd,KAAK,WAAW,EAChB,KAAK,KAAK,EACV,KAAK,KAAK,EACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"test-state.d.ts","sourceRoot":"","sources":["../src/test-state.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAKrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AAC1C,OAAO,EAGL,KAAK,WAAW,EAChB,KAAK,SAAS,EACd,KAAK,WAAW,EAChB,KAAK,YAAY,EACjB,KAAK,KAAK,EACV,KAAK,KAAK,EACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAmC,KAAK,cAAc,EAAE,MAAM,aAAa,CAAC;AAEnF,MAAM,MAAM,WAAW,GAAG,WAAW,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;AAE3D,sFAAsF;AACtF,MAAM,WAAW,UAAU;IACzB,iDAAiD;IACjD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,wGAAwG;IACxG,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,uFAAuF;AACvF,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,WAAW,CAAC;IAClB,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,MAAM,EAAE,UAAU,GAAG,SAAS,CAAC;IAC/B,wCAAwC;IACxC,OAAO,EAAE,WAAW,EAAE,CAAC;CACxB;AAED,eAAO,MAAM,UAAU,+BAAsC,CAAC;AAE9D,6EAA6E;AAC7E,wBAAgB,OAAO,IAAI,OAAO,CAEjC;AAED,UAAU,YAAY;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,wEAAwE;IACxE,OAAO,EAAE,MAAM,CAAC;IAChB,2CAA2C;IAC3C,KAAK,EAAE,KAAK,CAAC;IACb,0EAA0E;IAC1E,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;CACvB;AAQD,4EAA4E;AAC5E,wBAAgB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,YAAY,CAQ5D;AAED;;;GAGG;AACH,wBAAsB,WAAW,IAAI,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAKpE;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,SAAS,GAAG,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAYpG;AAED;;;;GAIG;AACH,eAAO,MAAM,QAAQ,uBAAuB,CAAC;AAC7C,eAAO,MAAM,aAAa,4BAA4B,CAAC;AAEvD;;;;;GAKG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,WAAW,GAAG,MAAM,CAMlD;AASD;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,WAAW,GAAG,YAAY,CAO9D;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAMxD;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAEhF;AAED,iEAAiE;AACjE,wBAAgB,kBAAkB,CAChC,IAAI,EAAE,WAAW,EACjB,UAAU,EAAE,SAAS,SAAS,EAAE,GAC/B,SAAS,EAAE,CAIb;AAED,wEAAwE;AACxE,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,UAAU,CAc7D;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAC7B,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,KAAK,EACZ,aAAa,EAAE,MAAM,GACpB,WAAW,GAAG,SAAS,CASzB;AAED,gFAAgF;AAChF,wBAAgB,eAAe,CAAC,MAAM,EAAE,UAAU,GAAG,SAAS,EAAE,KAAK,EAAE,YAAY,GAAG,OAAO,CAI5F;AAED,+DAA+D;AAC/D,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI,CAKtE;AAED;;;;GAIG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,WAAW,GAAG,cAAc,CAc7D;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAmBpG;AAED,wBAAgB,aAAa,IAAI,OAAO,CAGvC"}
|
package/dist/test-state.js
CHANGED
|
@@ -8,6 +8,7 @@ import { mkdirSync, writeFileSync } from "node:fs";
|
|
|
8
8
|
import { join, relative, sep } from "node:path";
|
|
9
9
|
import * as vitest from "vitest";
|
|
10
10
|
import { addUsage, zeroUsage, } from "@attest-ai/core";
|
|
11
|
+
import { reportTracesEnabled, TraceTable } from "./report.js";
|
|
11
12
|
export const trialScope = new AsyncLocalStorage();
|
|
12
13
|
/** True while the code runs inside trial i of a trials() call with N > 1. */
|
|
13
14
|
export function inTrial() {
|
|
@@ -74,11 +75,26 @@ export function testIdOf(test) {
|
|
|
74
75
|
for (let suite = test.suite; suite !== undefined && suite !== test.file; suite = suite.suite) {
|
|
75
76
|
names.unshift(suite.name);
|
|
76
77
|
}
|
|
78
|
+
return `${testFileOf(test)} > ${names.join(" > ")}`;
|
|
79
|
+
}
|
|
80
|
+
function testFileOf(test) {
|
|
77
81
|
const root = process.env[ROOT_ENV];
|
|
78
|
-
|
|
82
|
+
return root === undefined || test.file.filepath === undefined
|
|
79
83
|
? test.file.name
|
|
80
84
|
: relative(root, test.file.filepath).split(sep).join("/");
|
|
81
|
-
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* The `evaluate_batch` context naming `test`: its report test_id, its file and
|
|
88
|
+
* the line it starts on. The line is present only when Vitest records task
|
|
89
|
+
* locations (`test.includeTaskLocation`).
|
|
90
|
+
*/
|
|
91
|
+
export function batchContextOf(test) {
|
|
92
|
+
const line = test.location?.line;
|
|
93
|
+
return {
|
|
94
|
+
test_id: testIdOf(test),
|
|
95
|
+
file: testFileOf(test),
|
|
96
|
+
...(line !== undefined ? { line } : {}),
|
|
97
|
+
};
|
|
82
98
|
}
|
|
83
99
|
/**
|
|
84
100
|
* Hand the usage of an evaluation that belongs to no test (beforeAll, a script)
|
|
@@ -175,7 +191,16 @@ export function recordOnTest(test, result, trial) {
|
|
|
175
191
|
const state = attemptState(test);
|
|
176
192
|
state.usage = addUsage(state.usage, result.usage);
|
|
177
193
|
const meta = attemptMeta(test);
|
|
178
|
-
|
|
194
|
+
let traceId;
|
|
195
|
+
if (reportTracesEnabled()) {
|
|
196
|
+
meta.traces ??= {};
|
|
197
|
+
traceId = new TraceTable(meta.traces).add(result.trace);
|
|
198
|
+
}
|
|
199
|
+
meta.results.push(...result.assertionResults.map((r) => ({
|
|
200
|
+
...r,
|
|
201
|
+
...(trial === undefined ? {} : { trial }),
|
|
202
|
+
...(traceId === undefined ? {} : { trace_id: traceId }),
|
|
203
|
+
})));
|
|
179
204
|
meta.totalCost += result.totalCost;
|
|
180
205
|
meta.totalDurationMs += result.totalDurationMs;
|
|
181
206
|
meta.usage = addUsage(meta.usage, result.usage);
|
package/dist/test-state.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"test-state.js","sourceRoot":"","sources":["../src/test-state.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AAEjC,OAAO,EACL,QAAQ,EACR,SAAS,
|
|
1
|
+
{"version":3,"file":"test-state.js","sourceRoot":"","sources":["../src/test-state.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AAEjC,OAAO,EACL,QAAQ,EACR,SAAS,GAOV,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,mBAAmB,EAAE,UAAU,EAAuB,MAAM,aAAa,CAAC;AAqBnF,MAAM,CAAC,MAAM,UAAU,GAAG,IAAI,iBAAiB,EAAc,CAAC;AAE9D,6EAA6E;AAC7E,MAAM,UAAU,OAAO;IACrB,OAAO,UAAU,CAAC,QAAQ,EAAE,EAAE,KAAK,KAAK,SAAS,CAAC;AACpD,CAAC;AAaD,MAAM,QAAQ,GAAG,IAAI,OAAO,EAA6B,CAAC;AAE1D,SAAS,SAAS,CAAC,IAAiB;IAClC,OAAO,IAAI,CAAC,MAAM,EAAE,UAAU,IAAI,CAAC,CAAC;AACtC,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,YAAY,CAAC,IAAiB;IAC5C,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;QACrD,KAAK,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,aAAa,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;QAC3F,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW;IAC/B,MAAM,MAAM,GAAI,MAAiC,CAAC,UAAU,CAAC;IAC7D,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,cAAc,EAAE,CAAC;IACzD,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;IAC3C,OAAO,KAAK,CAAC,cAAc,EAAE,CAAC;AAChC,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAAgC;IAChE,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,OAAO,CAAC,IAAI,CAAC;IAC/C,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;IACpC,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC;IAC3C,MAAM,IAAI,GAAG,MAAM,WAAW,EAAE,CAAC;IACjC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CACb,sEAAsE;YACpE,wFAAwF,CAC3F,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,oBAAoB,CAAC;AAC7C,MAAM,CAAC,MAAM,aAAa,GAAG,yBAAyB,CAAC;AAEvD;;;;;GAKG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAiB;IACxC,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,CAAC,IAAI,EAAE,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;QAC7F,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;AACtD,CAAC;AAED,SAAS,UAAU,CAAC,IAAiB;IACnC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACnC,OAAO,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS;QAC3D,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;QAChB,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC9D,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,IAAiB;IAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC;IACjC,OAAO;QACL,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC;QACvB,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC;QACtB,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACxC,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAY;IAChD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IACvC,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO;IAC9B,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;QAAE,OAAO;IACvD,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;AAC1E,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,wBAAwB,CAAC,MAAc,EAAE,OAAe;IACtE,OAAO,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;AACpG,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,kBAAkB,CAChC,IAAiB,EACjB,UAAgC;IAEhC,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC9B,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,YAAY,EAAE,wBAAwB,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5G,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,cAAc,CAAC,MAAkB;IAC/C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACtC,IAAI,GAAG,KAAK,eAAe,IAAI,GAAG,KAAK,YAAY,EAAE,CAAC;YACpD,MAAM,IAAI,KAAK,CAAC,kCAAkC,GAAG,wCAAwC,CAAC,CAAC;QACjG,CAAC;IACH,CAAC;IACD,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;IAC7C,IAAI,aAAa,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,IAAI,aAAa,IAAI,CAAC,CAAC,EAAE,CAAC;QAC5F,MAAM,IAAI,KAAK,CAAC,6DAA6D,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IACxG,CAAC;IACD,IAAI,UAAU,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC;QAClF,MAAM,IAAI,KAAK,CAAC,wDAAwD,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IAChG,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAC7B,MAAkB,EAClB,KAAY,EACZ,aAAqB;IAErB,MAAM,IAAI,GAAwD,EAAE,CAAC;IACrE,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QACvC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QACpC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,GAAG,CAAC,KAAK,CAAC,QAAQ,GAAG,aAAa,CAAC,CAAC,CAAC;IACxF,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;AAC3D,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,eAAe,CAAC,MAA8B,EAAE,KAAmB;IACjF,OAAO,CACL,MAAM,EAAE,UAAU,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,aAAa,IAAI,MAAM,CAAC,UAAU,CACpG,CAAC;AACJ,CAAC;AAED,+DAA+D;AAC/D,MAAM,UAAU,gBAAgB,CAAC,IAAiB,EAAE,KAAY;IAC9D,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC;QAAE,OAAO;IAC/C,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACnC,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,QAAQ,EAAE,QAAQ,IAAI,CAAC,CAAC;AACvD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,IAAiB;IAC3C,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;IAC5B,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;QACnD,IAAI,GAAG;YACL,OAAO;YACP,OAAO,EAAE,EAAE;YACX,SAAS,EAAE,CAAC;YACZ,eAAe,EAAE,CAAC;YAClB,KAAK,EAAE,IAAI,EAAE,KAAK,IAAI,SAAS,EAAE;SAClC,CAAC;QACF,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IAC1B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,IAAiB,EAAE,MAAmB,EAAE,KAAyB;IAC5F,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,KAAK,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,OAA2B,CAAC;IAChC,IAAI,mBAAmB,EAAE,EAAE,CAAC;QAC1B,IAAI,CAAC,MAAM,KAAK,EAAE,CAAC;QACnB,OAAO,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,IAAI,CACf,GAAG,MAAM,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACrC,GAAG,CAAC;QACJ,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;QACzC,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;KACxD,CAAC,CAAC,CACJ,CAAC;IACF,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;IACnC,IAAI,CAAC,eAAe,IAAI,MAAM,CAAC,eAAe,CAAC;IAC/C,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;AAClD,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACpE,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,KAAK,CAAC;AAClD,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@attest-ai/vitest",
|
|
3
|
-
"version": "1.0.0-rc.
|
|
3
|
+
"version": "1.0.0-rc.5",
|
|
4
4
|
"description": "Attest vitest integration — global setup, fixtures, cost reporter, tier filtering",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
"ajv": "^8.20.0",
|
|
51
51
|
"typescript": "^5.9.3",
|
|
52
52
|
"vitest": "^4.1.10",
|
|
53
|
-
"@attest-ai/core": "1.0.0-rc.
|
|
53
|
+
"@attest-ai/core": "1.0.0-rc.5"
|
|
54
54
|
},
|
|
55
55
|
"scripts": {
|
|
56
56
|
"build": "tsc",
|