@attest-ai/vitest 0.9.0 → 1.0.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/fixtures.d.ts +54 -6
- package/dist/fixtures.d.ts.map +1 -1
- package/dist/fixtures.js +130 -31
- package/dist/fixtures.js.map +1 -1
- package/dist/index.d.ts +9 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -2
- package/dist/index.js.map +1 -1
- package/dist/plugin.d.ts +6 -1
- package/dist/plugin.d.ts.map +1 -1
- package/dist/plugin.js +21 -5
- package/dist/plugin.js.map +1 -1
- package/dist/report.d.ts +69 -0
- package/dist/report.d.ts.map +1 -0
- package/dist/report.js +91 -0
- package/dist/report.js.map +1 -0
- package/dist/reporter.d.ts +38 -1
- package/dist/reporter.d.ts.map +1 -1
- package/dist/reporter.js +96 -1
- package/dist/reporter.js.map +1 -1
- package/dist/test-state.d.ts +100 -0
- package/dist/test-state.d.ts.map +1 -0
- package/dist/test-state.js +187 -0
- package/dist/test-state.js.map +1 -0
- package/dist/trials.d.ts +55 -0
- package/dist/trials.d.ts.map +1 -0
- package/dist/trials.js +184 -0
- package/dist/trials.js.map +1 -0
- package/package.json +11 -6
package/dist/fixtures.d.ts
CHANGED
|
@@ -1,20 +1,68 @@
|
|
|
1
|
+
import type { TestContext } from "vitest";
|
|
1
2
|
import { type ExpectChain, AgentResult, AttestClient } from "@attest-ai/core";
|
|
3
|
+
import { type TestBudget } from "./test-state.js";
|
|
4
|
+
export interface InspectOptions {
|
|
5
|
+
/** Throw when the cumulative session cost exceeds this USD amount. */
|
|
6
|
+
budget?: number;
|
|
7
|
+
/**
|
|
8
|
+
* Limits for this test's attest model calls. Each evaluation sends what is
|
|
9
|
+
* left: judge calls not yet used in the test, and `maxCostUsd` minus the
|
|
10
|
+
* test's attest cost and agent spend so far (trace `metadata.cost_usd`). A
|
|
11
|
+
* call that does not fit is not sent and its assertion is not_evaluated with
|
|
12
|
+
* reason "budget_exhausted". Inside trials(), defaults to the trials budget.
|
|
13
|
+
*/
|
|
14
|
+
testBudget?: TestBudget;
|
|
15
|
+
/**
|
|
16
|
+
* The running test's context (`it("…", async (context) => …)`). Used to
|
|
17
|
+
* attach results to the right test for AttestReporter and to skip it;
|
|
18
|
+
* required under `test.concurrent` and, for skips, on Vitest before 4.1.
|
|
19
|
+
*/
|
|
20
|
+
context?: TestContext;
|
|
21
|
+
}
|
|
22
|
+
export interface EvaluateOptions extends InspectOptions {
|
|
23
|
+
/**
|
|
24
|
+
* Fail (instead of skip) when an assertion was not evaluated. Defaults to
|
|
25
|
+
* the ATTEST_STRICT environment variable ("1", "true" or "yes").
|
|
26
|
+
*/
|
|
27
|
+
strict?: boolean;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Fail or skip the running Vitest test according to the batch outcome:
|
|
31
|
+
* any hard_fail throws with rendered diagnostics; otherwise any
|
|
32
|
+
* not_evaluated skips with "not evaluated: <reasons>" (throws when strict);
|
|
33
|
+
* soft failures pass. Inside a trial of trials() (N > 1) it does nothing and
|
|
34
|
+
* returns `result`: trials() enforces the aggregate outcome after the last trial.
|
|
35
|
+
*/
|
|
36
|
+
export declare function enforceOutcome(result: AgentResult, strict: boolean, context?: TestContext): AgentResult;
|
|
2
37
|
export declare class AttestEngineFixture {
|
|
3
38
|
private manager;
|
|
4
39
|
private _client;
|
|
40
|
+
private starting;
|
|
5
41
|
constructor(options?: {
|
|
6
42
|
enginePath?: string;
|
|
7
43
|
logLevel?: string;
|
|
44
|
+
client?: AttestClient;
|
|
8
45
|
});
|
|
46
|
+
/**
|
|
47
|
+
* Start the engine. In simulation mode the engine is started with
|
|
48
|
+
* ATTEST_SIMULATION=1 so it evaluates layers 1-4 and reports layers 5-6
|
|
49
|
+
* as not evaluated; an engine without that capability is rejected.
|
|
50
|
+
*/
|
|
9
51
|
start(): Promise<void>;
|
|
10
52
|
stop(): Promise<void>;
|
|
11
53
|
get client(): AttestClient;
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
54
|
+
private ensureStarted;
|
|
55
|
+
/** Evaluate collected assertions without enforcing the test outcome. */
|
|
56
|
+
inspect(chain: ExpectChain, options?: InspectOptions): Promise<AgentResult>;
|
|
57
|
+
/** Evaluate collected assertions and enforce the test outcome. */
|
|
58
|
+
evaluate(chain: ExpectChain, options?: EvaluateOptions): Promise<AgentResult>;
|
|
15
59
|
}
|
|
16
60
|
export declare function useAttest(): AttestEngineFixture;
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
61
|
+
/**
|
|
62
|
+
* Evaluate and enforce: hard_fail throws, not_evaluated skips (strict: throws).
|
|
63
|
+
* Inside a trial of trials() (N > 1) it does not enforce, like inspect().
|
|
64
|
+
*/
|
|
65
|
+
export declare function evaluate(chain: ExpectChain, options?: EvaluateOptions): Promise<AgentResult>;
|
|
66
|
+
/** Evaluate without enforcing; use for custom checks. For repeated runs use trials(). */
|
|
67
|
+
export declare function inspect(chain: ExpectChain, options?: InspectOptions): Promise<AgentResult>;
|
|
20
68
|
//# sourceMappingURL=fixtures.d.ts.map
|
package/dist/fixtures.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fixtures.d.ts","sourceRoot":"","sources":["../src/fixtures.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"fixtures.d.ts","sourceRoot":"","sources":["../src/fixtures.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AAC1C,OAAO,EACL,KAAK,WAAW,EAChB,WAAW,EAEX,YAAY,EASb,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAYL,KAAK,UAAU,EAChB,MAAM,iBAAiB,CAAC;AAEzB,MAAM,WAAW,cAAc;IAC7B,sEAAsE;IACtE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB;;;;OAIG;IACH,OAAO,CAAC,EAAE,WAAW,CAAC;CACvB;AAED,MAAM,WAAW,eAAgB,SAAQ,cAAc;IACrD;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,WAAW,EACnB,MAAM,EAAE,OAAO,EACf,OAAO,CAAC,EAAE,WAAW,GACpB,WAAW,CAab;AAmBD,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,OAAO,CAA4B;IAC3C,OAAO,CAAC,OAAO,CAA2B;IAC1C,OAAO,CAAC,QAAQ,CAA4B;gBAEhC,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,YAAY,CAAA;KAAE;IAWvF;;;;OAIG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAoBtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B,IAAI,MAAM,IAAI,YAAY,CAKzB;YAEa,aAAa;IAM3B,wEAAwE;IAClE,OAAO,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,CAAC;IA4EjF,kEAAkE;IAC5D,QAAQ,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,WAAW,CAAC;CAIpF;AAOD,wBAAgB,SAAS,IAAI,mBAAmB,CAO/C;AAED;;;GAGG;AACH,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,WAAW,EAClB,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,WAAW,CAAC,CAEtB;AAED,yFAAyF;AACzF,wBAAsB,OAAO,CAC3B,KAAK,EAAE,WAAW,EAClB,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,WAAW,CAAC,CAEtB"}
|
package/dist/fixtures.js
CHANGED
|
@@ -1,22 +1,77 @@
|
|
|
1
|
-
import
|
|
1
|
+
import * as vitest from "vitest";
|
|
2
|
+
import { AgentResult, STATUS_SOFT_FAIL, AttestClient, EngineManager, isSimulationMode, renderDiagnostics, requireSimulationCapability, OUTCOME_FAIL, OUTCOME_SKIP, zeroUsage, } from "@attest-ai/core";
|
|
3
|
+
import { assignAssertionIds, attemptState, inTrial, recordAgentSpend, recordOnTest, recordUnattachedUsage, remainingBudget, resolveTest, strictFromEnv, trialScope, validateBudget, } from "./test-state.js";
|
|
4
|
+
/**
|
|
5
|
+
* Fail or skip the running Vitest test according to the batch outcome:
|
|
6
|
+
* any hard_fail throws with rendered diagnostics; otherwise any
|
|
7
|
+
* not_evaluated skips with "not evaluated: <reasons>" (throws when strict);
|
|
8
|
+
* soft failures pass. Inside a trial of trials() (N > 1) it does nothing and
|
|
9
|
+
* returns `result`: trials() enforces the aggregate outcome after the last trial.
|
|
10
|
+
*/
|
|
11
|
+
export function enforceOutcome(result, strict, context) {
|
|
12
|
+
if (inTrial())
|
|
13
|
+
return result;
|
|
14
|
+
const outcome = result.outcome({ strict });
|
|
15
|
+
if (outcome === OUTCOME_FAIL) {
|
|
16
|
+
const header = result.hardFailures.length === 0 ? `strict: ${result.skipReason()}\n` : "";
|
|
17
|
+
throw new Error(header + renderDiagnostics(result).replace(/^\n/, ""));
|
|
18
|
+
}
|
|
19
|
+
if (outcome === OUTCOME_SKIP) {
|
|
20
|
+
(context ?? trialScope.getStore()?.test.context ?? currentTestContext(result)).skip(result.skipReason());
|
|
21
|
+
}
|
|
22
|
+
return result;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* The running test's context when Vitest can name it unambiguously:
|
|
26
|
+
* TestRunner exists from Vitest 4.1, and under test.concurrent the
|
|
27
|
+
* "current test" may be another test, so the caller must pass `context`.
|
|
28
|
+
*/
|
|
29
|
+
function currentTestContext(result) {
|
|
30
|
+
const runner = vitest.TestRunner;
|
|
31
|
+
const test = runner?.getCurrentTest();
|
|
32
|
+
if (test === undefined || test.concurrent === true) {
|
|
33
|
+
throw new Error(`${result.skipReason()} — pass the test context to skip this test: ` +
|
|
34
|
+
"evaluate(chain, { context }) (required for test.concurrent and Vitest < 4.1)");
|
|
35
|
+
}
|
|
36
|
+
return test.context;
|
|
37
|
+
}
|
|
2
38
|
export class AttestEngineFixture {
|
|
3
39
|
manager;
|
|
4
40
|
_client;
|
|
41
|
+
starting;
|
|
5
42
|
constructor(options) {
|
|
6
|
-
if (
|
|
7
|
-
this.
|
|
8
|
-
|
|
9
|
-
logLevel: options?.logLevel ?? "warn",
|
|
10
|
-
});
|
|
43
|
+
if (options?.client !== undefined) {
|
|
44
|
+
this._client = options.client;
|
|
45
|
+
return;
|
|
11
46
|
}
|
|
47
|
+
this.manager = new EngineManager({
|
|
48
|
+
enginePath: options?.enginePath,
|
|
49
|
+
logLevel: options?.logLevel ?? "warn",
|
|
50
|
+
});
|
|
12
51
|
}
|
|
52
|
+
/**
|
|
53
|
+
* Start the engine. In simulation mode the engine is started with
|
|
54
|
+
* ATTEST_SIMULATION=1 so it evaluates layers 1-4 and reports layers 5-6
|
|
55
|
+
* as not evaluated; an engine without that capability is rejected.
|
|
56
|
+
*/
|
|
13
57
|
async start() {
|
|
14
|
-
if (isSimulationMode())
|
|
15
|
-
return;
|
|
16
58
|
if (this.manager === undefined) {
|
|
17
59
|
throw new Error("AttestEngineFixture already stopped.");
|
|
18
60
|
}
|
|
19
|
-
|
|
61
|
+
const simulation = isSimulationMode();
|
|
62
|
+
if (simulation) {
|
|
63
|
+
process.env["ATTEST_SIMULATION"] = "1";
|
|
64
|
+
}
|
|
65
|
+
const init = await this.manager.start();
|
|
66
|
+
if (simulation) {
|
|
67
|
+
try {
|
|
68
|
+
requireSimulationCapability(init);
|
|
69
|
+
}
|
|
70
|
+
catch (err) {
|
|
71
|
+
await this.manager.stop();
|
|
72
|
+
throw err;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
20
75
|
this._client = new AttestClient(this.manager);
|
|
21
76
|
}
|
|
22
77
|
async stop() {
|
|
@@ -27,28 +82,61 @@ export class AttestEngineFixture {
|
|
|
27
82
|
}
|
|
28
83
|
}
|
|
29
84
|
get client() {
|
|
30
|
-
if (this._client === undefined
|
|
85
|
+
if (this._client === undefined) {
|
|
31
86
|
throw new Error("Engine not started. Call start() first.");
|
|
32
87
|
}
|
|
33
88
|
return this._client;
|
|
34
89
|
}
|
|
35
|
-
async
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
90
|
+
async ensureStarted() {
|
|
91
|
+
if (this._client !== undefined)
|
|
92
|
+
return;
|
|
93
|
+
this.starting ??= this.start();
|
|
94
|
+
await this.starting;
|
|
95
|
+
}
|
|
96
|
+
/** Evaluate collected assertions without enforcing the test outcome. */
|
|
97
|
+
async inspect(chain, options) {
|
|
98
|
+
await this.ensureStarted();
|
|
99
|
+
// Inside a test, assertion IDs are deterministic (the test's report test_id
|
|
100
|
+
// and the assertion's ordinal in the test, restarting each trial) so trials
|
|
101
|
+
// and baselines match rows by ID. Outside a test they stay random.
|
|
102
|
+
const test = await resolveTest(options?.context);
|
|
103
|
+
const scope = trialScope.getStore();
|
|
104
|
+
const trial = scope !== undefined && scope.test === test ? scope.trial : undefined;
|
|
105
|
+
const budget = options?.testBudget !== undefined
|
|
106
|
+
? validateBudget(options.testBudget)
|
|
107
|
+
: scope !== undefined && scope.test === test
|
|
108
|
+
? scope.budget
|
|
109
|
+
: undefined;
|
|
110
|
+
let assertions = chain.assertions;
|
|
111
|
+
let batchBudget;
|
|
112
|
+
if (test !== undefined) {
|
|
113
|
+
assertions = assignAssertionIds(test, assertions);
|
|
114
|
+
recordAgentSpend(test, chain.trace);
|
|
115
|
+
if (budget !== undefined) {
|
|
116
|
+
const state = attemptState(test);
|
|
117
|
+
batchBudget = remainingBudget(budget, state.usage, state.agentSpendUsd);
|
|
118
|
+
}
|
|
39
119
|
}
|
|
40
|
-
else {
|
|
41
|
-
|
|
120
|
+
else if (budget !== undefined) {
|
|
121
|
+
// Outside a test the budget covers this one evaluation.
|
|
122
|
+
batchBudget = remainingBudget(budget, zeroUsage(), chain.trace.metadata?.cost_usd ?? 0);
|
|
42
123
|
}
|
|
124
|
+
const result = await this.client.evaluateBatch(chain.trace, assertions, batchBudget === undefined ? undefined : { budget: batchBudget });
|
|
43
125
|
globalThis.__attest_session_cost__ =
|
|
44
126
|
(globalThis.__attest_session_cost__ ?? 0) + (result.total_cost ?? 0);
|
|
45
127
|
const softFailCount = (result.results ?? []).filter((r) => r.status === STATUS_SOFT_FAIL).length;
|
|
46
128
|
globalThis.__attest_session_soft_failures__ =
|
|
47
129
|
(globalThis.__attest_session_soft_failures__ ?? 0) + softFailCount;
|
|
48
|
-
const agentResult = new AgentResult(chain.trace, result.results ?? [], result.total_cost ?? 0, result.total_duration_ms ?? 0);
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
130
|
+
const agentResult = new AgentResult(chain.trace, result.results ?? [], result.total_cost ?? 0, result.total_duration_ms ?? 0, result.usage);
|
|
131
|
+
if (test !== undefined)
|
|
132
|
+
recordOnTest(test, agentResult, trial);
|
|
133
|
+
else
|
|
134
|
+
recordUnattachedUsage(agentResult.usage);
|
|
135
|
+
if (scope !== undefined && scope.test === test)
|
|
136
|
+
scope.results.push(agentResult);
|
|
137
|
+
// Track non-passing AgentResults so AttestCostReporter can render the
|
|
138
|
+
// diagnostic block at the end of the run. Bound size because long
|
|
139
|
+
// suites with many failures should not balloon RAM.
|
|
52
140
|
if (!agentResult.passed) {
|
|
53
141
|
const list = (globalThis.__attest_session_failures__ ??= []);
|
|
54
142
|
if (list.length < 200) {
|
|
@@ -62,22 +150,33 @@ export class AttestEngineFixture {
|
|
|
62
150
|
}
|
|
63
151
|
return agentResult;
|
|
64
152
|
}
|
|
153
|
+
/** Evaluate collected assertions and enforce the test outcome. */
|
|
154
|
+
async evaluate(chain, options) {
|
|
155
|
+
const result = await this.inspect(chain, options);
|
|
156
|
+
return enforceOutcome(result, options?.strict ?? strictFromEnv(), options?.context);
|
|
157
|
+
}
|
|
65
158
|
}
|
|
159
|
+
// Vitest runs globalSetup in the main process and tests in workers, so a
|
|
160
|
+
// worker that cannot see the global client starts its own engine on first use.
|
|
161
|
+
// The engine exits when its stdin closes, i.e. when the worker exits.
|
|
162
|
+
let workerFixture;
|
|
66
163
|
export function useAttest() {
|
|
67
|
-
if (isSimulationMode() || globalThis.__attest_simulation_mode__) {
|
|
68
|
-
return new AttestEngineFixture();
|
|
69
|
-
}
|
|
70
164
|
const client = globalThis.__attest_client__;
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
throw new Error("Attest engine not available. Configure attestGlobalSetup() in vitest.config.ts globalSetup.");
|
|
165
|
+
if (client !== undefined) {
|
|
166
|
+
return new AttestEngineFixture({ client });
|
|
74
167
|
}
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
return fixture;
|
|
168
|
+
workerFixture ??= new AttestEngineFixture();
|
|
169
|
+
return workerFixture;
|
|
78
170
|
}
|
|
171
|
+
/**
|
|
172
|
+
* Evaluate and enforce: hard_fail throws, not_evaluated skips (strict: throws).
|
|
173
|
+
* Inside a trial of trials() (N > 1) it does not enforce, like inspect().
|
|
174
|
+
*/
|
|
79
175
|
export async function evaluate(chain, options) {
|
|
80
|
-
|
|
81
|
-
|
|
176
|
+
return useAttest().evaluate(chain, options);
|
|
177
|
+
}
|
|
178
|
+
/** Evaluate without enforcing; use for custom checks. For repeated runs use trials(). */
|
|
179
|
+
export async function inspect(chain, options) {
|
|
180
|
+
return useAttest().inspect(chain, options);
|
|
82
181
|
}
|
|
83
182
|
//# sourceMappingURL=fixtures.js.map
|
package/dist/fixtures.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fixtures.js","sourceRoot":"","sources":["../src/fixtures.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"fixtures.js","sourceRoot":"","sources":["../src/fixtures.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AAEjC,OAAO,EAEL,WAAW,EACX,gBAAgB,EAChB,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,2BAA2B,EAC3B,YAAY,EACZ,YAAY,EACZ,SAAS,GAEV,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,kBAAkB,EAClB,YAAY,EACZ,OAAO,EACP,gBAAgB,EAChB,YAAY,EACZ,qBAAqB,EACrB,eAAe,EACf,WAAW,EACX,aAAa,EACb,UAAU,EACV,cAAc,GAEf,MAAM,iBAAiB,CAAC;AA6BzB;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAC5B,MAAmB,EACnB,MAAe,EACf,OAAqB;IAErB,IAAI,OAAO,EAAE;QAAE,OAAO,MAAM,CAAC;IAC7B,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAC3C,IAAI,OAAO,KAAK,YAAY,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,MAAM,IAAI,KAAK,CAAC,MAAM,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;IACzE,CAAC;IACD,IAAI,OAAO,KAAK,YAAY,EAAE,CAAC;QAC7B,CAAC,OAAO,IAAI,UAAU,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CACjF,MAAM,CAAC,UAAU,EAAE,CACpB,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,MAAmB;IAC7C,MAAM,MAAM,GAAI,MAAiC,CAAC,UAAU,CAAC;IAC7D,MAAM,IAAI,GAAG,MAAM,EAAE,cAAc,EAAE,CAAC;IACtC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CACb,GAAG,MAAM,CAAC,UAAU,EAAE,8CAA8C;YAClE,8EAA8E,CACjF,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC,OAAO,CAAC;AACtB,CAAC;AAED,MAAM,OAAO,mBAAmB;IACtB,OAAO,CAA4B;IACnC,OAAO,CAA2B;IAClC,QAAQ,CAA4B;IAE5C,YAAY,OAA2E;QACrF,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,OAAO;QACT,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,aAAa,CAAC;YAC/B,UAAU,EAAE,OAAO,EAAE,UAAU;YAC/B,QAAQ,EAAE,OAAO,EAAE,QAAQ,IAAI,MAAM;SACtC,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QACD,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAC;QACtC,IAAI,UAAU,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,GAAG,CAAC;QACzC,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACxC,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,2BAA2B,CAAC,IAAI,CAAC,CAAC;YACpC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC1B,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAC1B,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;YACzB,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,IAAI,MAAM;QACR,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS;YAAE,OAAO;QACvC,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC;IACtB,CAAC;IAED,wEAAwE;IACxE,KAAK,CAAC,OAAO,CAAC,KAAkB,EAAE,OAAwB;QACxD,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC3B,4EAA4E;QAC5E,4EAA4E;QAC5E,mEAAmE;QACnE,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QACnF,MAAM,MAAM,GACV,OAAO,EAAE,UAAU,KAAK,SAAS;YAC/B,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,UAAU,CAAC;YACpC,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI;gBAC1C,CAAC,CAAC,KAAK,CAAC,MAAM;gBACd,CAAC,CAAC,SAAS,CAAC;QAClB,IAAI,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;QAClC,IAAI,WAAoC,CAAC;QACzC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,UAAU,GAAG,kBAAkB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAClD,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;YACpC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;gBACjC,WAAW,GAAG,eAAe,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;aAAM,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YAChC,wDAAwD;YACxD,WAAW,GAAG,eAAe,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,IAAI,CAAC,CAAC,CAAC;QAC1F,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAC5C,KAAK,CAAC,KAAK,EACX,UAAU,EACV,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAChE,CAAC;QAEF,UAAU,CAAC,uBAAuB;YAChC,CAAC,UAAU,CAAC,uBAAuB,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC;QAEvE,MAAM,aAAa,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,CACjD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,gBAAgB,CACrC,CAAC,MAAM,CAAC;QACT,UAAU,CAAC,gCAAgC;YACzC,CAAC,UAAU,CAAC,gCAAgC,IAAI,CAAC,CAAC,GAAG,aAAa,CAAC;QAErE,MAAM,WAAW,GAAG,IAAI,WAAW,CACjC,KAAK,CAAC,KAAK,EACX,MAAM,CAAC,OAAO,IAAI,EAAE,EACpB,MAAM,CAAC,UAAU,IAAI,CAAC,EACtB,MAAM,CAAC,iBAAiB,IAAI,CAAC,EAC7B,MAAM,CAAC,KAAK,CACb,CAAC;QACF,IAAI,IAAI,KAAK,SAAS;YAAE,YAAY,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;;YAC1D,qBAAqB,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAC9C,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI;YAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEhF,sEAAsE;QACtE,kEAAkE;QAClE,oDAAoD;QACpD,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,CAAC,UAAU,CAAC,2BAA2B,KAAK,EAAE,CAAC,CAAC;YAC7D,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACtB,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QAED,IACE,OAAO,EAAE,MAAM,KAAK,SAAS;YAC7B,UAAU,CAAC,uBAAuB,GAAG,OAAO,CAAC,MAAM,EACnD,CAAC;YACD,MAAM,IAAI,KAAK,CACb,iCAAiC,UAAU,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;gBAC9E,cAAc,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAC5C,CAAC;QACJ,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,kEAAkE;IAClE,KAAK,CAAC,QAAQ,CAAC,KAAkB,EAAE,OAAyB;QAC1D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAClD,OAAO,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,aAAa,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IACtF,CAAC;CACF;AAED,yEAAyE;AACzE,+EAA+E;AAC/E,sEAAsE;AACtE,IAAI,aAA8C,CAAC;AAEnD,MAAM,UAAU,SAAS;IACvB,MAAM,MAAM,GAAG,UAAU,CAAC,iBAAiB,CAAC;IAC5C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,IAAI,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAC7C,CAAC;IACD,aAAa,KAAK,IAAI,mBAAmB,EAAE,CAAC;IAC5C,OAAO,aAAa,CAAC;AACvB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAkB,EAClB,OAAyB;IAEzB,OAAO,SAAS,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC9C,CAAC;AAED,yFAAyF;AACzF,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,KAAkB,EAClB,OAAwB;IAExB,OAAO,SAAS,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC7C,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
export { attestGlobalSetup } from "./plugin.js";
|
|
2
2
|
export type { AttestGlobalSetupOptions } from "./plugin.js";
|
|
3
|
-
export { AttestEngineFixture, useAttest, evaluate } from "./fixtures.js";
|
|
4
|
-
export {
|
|
3
|
+
export { AttestEngineFixture, useAttest, evaluate, inspect, enforceOutcome } from "./fixtures.js";
|
|
4
|
+
export type { EvaluateOptions, InspectOptions } from "./fixtures.js";
|
|
5
|
+
export { trials, MIN_PASS_RATE_ENV } from "./trials.js";
|
|
6
|
+
export type { TrialsOptions } from "./trials.js";
|
|
7
|
+
export type { TestBudget } from "./test-state.js";
|
|
8
|
+
export { AttestCostReporter, AttestReporter } from "./reporter.js";
|
|
9
|
+
export type { AttestReporterOptions } from "./reporter.js";
|
|
10
|
+
export { buildReport, REPORT_VERSION, sessionUsage, summaryLines } from "./report.js";
|
|
11
|
+
export type { AttestTaskMeta, RecordedResult, TestOutcome, TestRecord, TrialsMeta } from "./report.js";
|
|
5
12
|
export { filterByTier } from "./tier-filter.js";
|
|
6
13
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,YAAY,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,YAAY,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAClG,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACrE,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACxD,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,YAAY,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACnE,YAAY,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACtF,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACvG,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
export { attestGlobalSetup } from "./plugin.js";
|
|
2
|
-
export { AttestEngineFixture, useAttest, evaluate } from "./fixtures.js";
|
|
3
|
-
export {
|
|
2
|
+
export { AttestEngineFixture, useAttest, evaluate, inspect, enforceOutcome } from "./fixtures.js";
|
|
3
|
+
export { trials, MIN_PASS_RATE_ENV } from "./trials.js";
|
|
4
|
+
export { AttestCostReporter, AttestReporter } from "./reporter.js";
|
|
5
|
+
export { buildReport, REPORT_VERSION, sessionUsage, summaryLines } from "./report.js";
|
|
4
6
|
export { filterByTier } from "./tier-filter.js";
|
|
5
7
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAElG,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAGxD,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAEnE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEtF,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC"}
|
package/dist/plugin.d.ts
CHANGED
|
@@ -2,13 +2,18 @@ import { EngineManager, AttestClient, type AgentResult } from "@attest-ai/core";
|
|
|
2
2
|
export interface AttestGlobalSetupOptions {
|
|
3
3
|
enginePath?: string;
|
|
4
4
|
logLevel?: string;
|
|
5
|
+
/**
|
|
6
|
+
* Minimum passed / evaluated trials for every trials() test of the run (0..1,
|
|
7
|
+
* default 1.0); trials(body, { minPassRate }) overrides it per test. Reaches
|
|
8
|
+
* the workers as the ATTEST_MIN_PASS_RATE environment variable.
|
|
9
|
+
*/
|
|
10
|
+
minPassRate?: number;
|
|
5
11
|
}
|
|
6
12
|
declare global {
|
|
7
13
|
var __attest_engine__: EngineManager | undefined;
|
|
8
14
|
var __attest_client__: AttestClient | undefined;
|
|
9
15
|
var __attest_session_cost__: number;
|
|
10
16
|
var __attest_session_soft_failures__: number;
|
|
11
|
-
var __attest_simulation_mode__: boolean;
|
|
12
17
|
var __attest_session_failures__: AgentResult[] | undefined;
|
|
13
18
|
}
|
|
14
19
|
export declare function attestGlobalSetup(options?: AttestGlobalSetupOptions): {
|
package/dist/plugin.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,aAAa,EACb,YAAY,
|
|
1
|
+
{"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,aAAa,EACb,YAAY,EAGZ,KAAK,WAAW,EACjB,MAAM,iBAAiB,CAAC;AAMzB,MAAM,WAAW,wBAAwB;IACvC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,OAAO,CAAC,MAAM,CAAC;IAEb,IAAI,iBAAiB,EAAE,aAAa,GAAG,SAAS,CAAC;IAEjD,IAAI,iBAAiB,EAAE,YAAY,GAAG,SAAS,CAAC;IAEhD,IAAI,uBAAuB,EAAE,MAAM,CAAC;IAEpC,IAAI,gCAAgC,EAAE,MAAM,CAAC;IAE7C,IAAI,2BAA2B,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC;CAC5D;AAED,wBAAgB,iBAAiB,CAAC,OAAO,CAAC,EAAE,wBAAwB;;;EAmDnE"}
|
package/dist/plugin.js
CHANGED
|
@@ -1,21 +1,38 @@
|
|
|
1
|
-
import { EngineManager, AttestClient, isSimulationMode, } from "@attest-ai/core";
|
|
1
|
+
import { EngineManager, AttestClient, isSimulationMode, requireSimulationCapability, } from "@attest-ai/core";
|
|
2
|
+
import { MIN_PASS_RATE_ENV, validateMinPassRate } from "./trials.js";
|
|
2
3
|
let engineManager;
|
|
3
4
|
let attestClient;
|
|
4
5
|
export function attestGlobalSetup(options) {
|
|
6
|
+
const minPassRate = options?.minPassRate === undefined
|
|
7
|
+
? undefined
|
|
8
|
+
: validateMinPassRate(options.minPassRate, `minPassRate=${options.minPassRate}`);
|
|
5
9
|
return {
|
|
6
10
|
async setup() {
|
|
11
|
+
if (minPassRate !== undefined)
|
|
12
|
+
process.env[MIN_PASS_RATE_ENV] = String(minPassRate);
|
|
7
13
|
globalThis.__attest_session_cost__ = 0;
|
|
8
14
|
globalThis.__attest_session_soft_failures__ = 0;
|
|
9
15
|
globalThis.__attest_session_failures__ = [];
|
|
10
16
|
if (isSimulationMode()) {
|
|
11
|
-
|
|
12
|
-
|
|
17
|
+
// The engine decides simulation at startup from its environment;
|
|
18
|
+
// this also covers config({ simulation: true }).
|
|
19
|
+
process.env["ATTEST_SIMULATION"] = "1";
|
|
13
20
|
}
|
|
14
21
|
engineManager = new EngineManager({
|
|
15
22
|
enginePath: options?.enginePath,
|
|
16
23
|
logLevel: options?.logLevel ?? "warn",
|
|
17
24
|
});
|
|
18
|
-
await engineManager.start();
|
|
25
|
+
const init = await engineManager.start();
|
|
26
|
+
if (isSimulationMode()) {
|
|
27
|
+
try {
|
|
28
|
+
requireSimulationCapability(init);
|
|
29
|
+
}
|
|
30
|
+
catch (err) {
|
|
31
|
+
await engineManager.stop();
|
|
32
|
+
engineManager = undefined;
|
|
33
|
+
throw err;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
19
36
|
attestClient = new AttestClient(engineManager);
|
|
20
37
|
globalThis.__attest_engine__ = engineManager;
|
|
21
38
|
globalThis.__attest_client__ = attestClient;
|
|
@@ -28,7 +45,6 @@ export function attestGlobalSetup(options) {
|
|
|
28
45
|
}
|
|
29
46
|
globalThis.__attest_engine__ = undefined;
|
|
30
47
|
globalThis.__attest_client__ = undefined;
|
|
31
|
-
globalThis.__attest_simulation_mode__ = false;
|
|
32
48
|
globalThis.__attest_session_failures__ = undefined;
|
|
33
49
|
},
|
|
34
50
|
};
|
package/dist/plugin.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"plugin.js","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,aAAa,EACb,YAAY,EACZ,gBAAgB,
|
|
1
|
+
{"version":3,"file":"plugin.js","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,aAAa,EACb,YAAY,EACZ,gBAAgB,EAChB,2BAA2B,GAE5B,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAErE,IAAI,aAAwC,CAAC;AAC7C,IAAI,YAAsC,CAAC;AA0B3C,MAAM,UAAU,iBAAiB,CAAC,OAAkC;IAClE,MAAM,WAAW,GACf,OAAO,EAAE,WAAW,KAAK,SAAS;QAChC,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,mBAAmB,CAAC,OAAO,CAAC,WAAW,EAAE,eAAe,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IACrF,OAAO;QACL,KAAK,CAAC,KAAK;YACT,IAAI,WAAW,KAAK,SAAS;gBAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;YACpF,UAAU,CAAC,uBAAuB,GAAG,CAAC,CAAC;YACvC,UAAU,CAAC,gCAAgC,GAAG,CAAC,CAAC;YAChD,UAAU,CAAC,2BAA2B,GAAG,EAAE,CAAC;YAE5C,IAAI,gBAAgB,EAAE,EAAE,CAAC;gBACvB,iEAAiE;gBACjE,iDAAiD;gBACjD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,GAAG,CAAC;YACzC,CAAC;YAED,aAAa,GAAG,IAAI,aAAa,CAAC;gBAChC,UAAU,EAAE,OAAO,EAAE,UAAU;gBAC/B,QAAQ,EAAE,OAAO,EAAE,QAAQ,IAAI,MAAM;aACtC,CAAC,CAAC;YAEH,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,CAAC;YACzC,IAAI,gBAAgB,EAAE,EAAE,CAAC;gBACvB,IAAI,CAAC;oBACH,2BAA2B,CAAC,IAAI,CAAC,CAAC;gBACpC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;oBAC3B,aAAa,GAAG,SAAS,CAAC;oBAC1B,MAAM,GAAG,CAAC;gBACZ,CAAC;YACH,CAAC;YACD,YAAY,GAAG,IAAI,YAAY,CAAC,aAAa,CAAC,CAAC;YAE/C,UAAU,CAAC,iBAAiB,GAAG,aAAa,CAAC;YAC7C,UAAU,CAAC,iBAAiB,GAAG,YAAY,CAAC;QAC9C,CAAC;QAED,KAAK,CAAC,QAAQ;YACZ,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;gBAC3B,aAAa,GAAG,SAAS,CAAC;gBAC1B,YAAY,GAAG,SAAS,CAAC;YAC3B,CAAC;YAED,UAAU,CAAC,iBAAiB,GAAG,SAAS,CAAC;YACzC,UAAU,CAAC,iBAAiB,GAAG,SAAS,CAAC;YACzC,UAAU,CAAC,2BAA2B,GAAG,SAAS,CAAC;QACrD,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/dist/report.d.ts
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* report-v2 builder (proto/attest/v1/report-v2.schema.json, contract C3).
|
|
3
|
+
*
|
|
4
|
+
* One row per assertion in `results`, each carrying the test id, file and line
|
|
5
|
+
* of the test that requested it, and one row per test in `tests` with its
|
|
6
|
+
* outcome. `attest policy evaluate` gates on `results`; the pr-annotate action
|
|
7
|
+
* places inline comments with `file` and `line`.
|
|
8
|
+
*/
|
|
9
|
+
import { type AssertionResult, type Usage } from "@attest-ai/core";
|
|
10
|
+
export declare const REPORT_VERSION = 2;
|
|
11
|
+
export type TestOutcome = "passed" | "failed" | "skipped";
|
|
12
|
+
/** An assertion result as recorded for the report; `trial` is set under trials() with N > 1. */
|
|
13
|
+
export type RecordedResult = AssertionResult & {
|
|
14
|
+
readonly trial?: number;
|
|
15
|
+
};
|
|
16
|
+
/** How a trials() call with N > 1 went. */
|
|
17
|
+
export interface TrialsMeta {
|
|
18
|
+
/** N. */
|
|
19
|
+
requested: number;
|
|
20
|
+
/** Trials actually run; fewer than N when the test's cost budget was spent. */
|
|
21
|
+
run: number;
|
|
22
|
+
passed: number;
|
|
23
|
+
/** Trials that passed, failed or raised (not counting not-evaluated ones). */
|
|
24
|
+
evaluated: number;
|
|
25
|
+
notEvaluated: number;
|
|
26
|
+
stoppedByBudget: boolean;
|
|
27
|
+
}
|
|
28
|
+
/** Attest results a test recorded, carried from the worker in task meta. */
|
|
29
|
+
export interface AttestTaskMeta {
|
|
30
|
+
/** `retryCount` of the attempt that produced `results`. */
|
|
31
|
+
attempt: number;
|
|
32
|
+
results: RecordedResult[];
|
|
33
|
+
totalCost: number;
|
|
34
|
+
totalDurationMs: number;
|
|
35
|
+
/** Attest model usage of every attempt of the test, including retried ones. */
|
|
36
|
+
usage: Usage;
|
|
37
|
+
trials?: TrialsMeta;
|
|
38
|
+
}
|
|
39
|
+
declare module "vitest" {
|
|
40
|
+
interface TaskMeta {
|
|
41
|
+
attest?: AttestTaskMeta;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
export interface TestRecord {
|
|
45
|
+
testId: string;
|
|
46
|
+
name: string;
|
|
47
|
+
/** Relative to the Vitest root, forward slashes. */
|
|
48
|
+
file: string;
|
|
49
|
+
/** 1-based line of the test definition. */
|
|
50
|
+
line: number;
|
|
51
|
+
outcome: TestOutcome;
|
|
52
|
+
durationMs: number;
|
|
53
|
+
attest: AttestTaskMeta | undefined;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* `unattached` is the usage of evaluations that belong to no test (beforeAll,
|
|
57
|
+
* scripts); it counts in `summary.usage`.
|
|
58
|
+
*/
|
|
59
|
+
export declare function buildReport(tests: readonly TestRecord[], simulated: boolean, unattached?: Usage): Record<string, unknown>;
|
|
60
|
+
/** Attest usage summed over every test of the run plus the evaluations that belong to no test. */
|
|
61
|
+
export declare function sessionUsage(tests: readonly TestRecord[], unattached?: Usage): Usage;
|
|
62
|
+
/**
|
|
63
|
+
* Lines for the end of the run: one per trials test
|
|
64
|
+
* ("<test_id>: <passed>/<evaluated> trials passed (<n> not evaluated)", plus a
|
|
65
|
+
* note when the test budget stopped it early), then the "attest usage: …" line
|
|
66
|
+
* when the session made or refused a model call.
|
|
67
|
+
*/
|
|
68
|
+
export declare function summaryLines(tests: readonly TestRecord[], unattached?: Usage): string[];
|
|
69
|
+
//# sourceMappingURL=report.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAQL,KAAK,eAAe,EACpB,KAAK,KAAK,EACX,MAAM,iBAAiB,CAAC;AAEzB,eAAO,MAAM,cAAc,IAAI,CAAC;AAEhC,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE1D,gGAAgG;AAChG,MAAM,MAAM,cAAc,GAAG,eAAe,GAAG;IAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAE3E,2CAA2C;AAC3C,MAAM,WAAW,UAAU;IACzB,SAAS;IACT,SAAS,EAAE,MAAM,CAAC;IAClB,+EAA+E;IAC/E,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,8EAA8E;IAC9E,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,OAAO,CAAC;CAC1B;AAED,4EAA4E;AAC5E,MAAM,WAAW,cAAc;IAC7B,2DAA2D;IAC3D,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,+EAA+E;IAC/E,KAAK,EAAE,KAAK,CAAC;IACb,MAAM,CAAC,EAAE,UAAU,CAAC;CACrB;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,QAAQ;QAChB,MAAM,CAAC,EAAE,cAAc,CAAC;KACzB;CACF;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,oDAAoD;IACpD,IAAI,EAAE,MAAM,CAAC;IACb,2CAA2C;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,cAAc,GAAG,SAAS,CAAC;CACpC;AAED;;;GAGG;AACH,wBAAgB,WAAW,CACzB,KAAK,EAAE,SAAS,UAAU,EAAE,EAC5B,SAAS,EAAE,OAAO,EAClB,UAAU,GAAE,KAAmB,GAC9B,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAgDzB;AAED,kGAAkG;AAClG,wBAAgB,YAAY,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,EAAE,UAAU,GAAE,KAAmB,GAAG,KAAK,CAEjG;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,EAAE,UAAU,GAAE,KAAmB,GAAG,MAAM,EAAE,CAiBpG"}
|
package/dist/report.js
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* report-v2 builder (proto/attest/v1/report-v2.schema.json, contract C3).
|
|
3
|
+
*
|
|
4
|
+
* One row per assertion in `results`, each carrying the test id, file and line
|
|
5
|
+
* of the test that requested it, and one row per test in `tests` with its
|
|
6
|
+
* outcome. `attest policy evaluate` gates on `results`; the pr-annotate action
|
|
7
|
+
* places inline comments with `file` and `line`.
|
|
8
|
+
*/
|
|
9
|
+
import { STATUS_HARD_FAIL, STATUS_NOT_EVALUATED, STATUS_PASS, STATUS_SOFT_FAIL, sumUsage, usageSummaryLine, zeroUsage, } from "@attest-ai/core";
|
|
10
|
+
export const REPORT_VERSION = 2;
|
|
11
|
+
/**
|
|
12
|
+
* `unattached` is the usage of evaluations that belong to no test (beforeAll,
|
|
13
|
+
* scripts); it counts in `summary.usage`.
|
|
14
|
+
*/
|
|
15
|
+
export function buildReport(tests, simulated, unattached = zeroUsage()) {
|
|
16
|
+
const rows = [];
|
|
17
|
+
const testRows = [];
|
|
18
|
+
let totalCost = 0;
|
|
19
|
+
let totalDurationMs = 0;
|
|
20
|
+
for (const test of tests) {
|
|
21
|
+
if (test.line < 1)
|
|
22
|
+
throw new Error(`${test.testId}: line must be >= 1, got ${test.line}`);
|
|
23
|
+
const assertionIds = new Set();
|
|
24
|
+
const reasons = [];
|
|
25
|
+
totalCost += test.attest?.totalCost ?? 0;
|
|
26
|
+
totalDurationMs += test.attest?.totalDurationMs ?? 0;
|
|
27
|
+
for (const result of test.attest?.results ?? []) {
|
|
28
|
+
rows.push({ ...result, test_id: test.testId, file: test.file, line: test.line });
|
|
29
|
+
assertionIds.add(result.assertion_id);
|
|
30
|
+
if (result.status === STATUS_NOT_EVALUATED && !reasons.includes(String(result.reason))) {
|
|
31
|
+
reasons.push(String(result.reason));
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
testRows.push({
|
|
35
|
+
test_id: test.testId,
|
|
36
|
+
name: test.name,
|
|
37
|
+
file: test.file,
|
|
38
|
+
line: test.line,
|
|
39
|
+
outcome: test.outcome,
|
|
40
|
+
...(reasons.length > 0 ? { not_evaluated_reasons: reasons } : {}),
|
|
41
|
+
assertion_ids: [...assertionIds],
|
|
42
|
+
duration_ms: Math.round(test.durationMs),
|
|
43
|
+
...(test.attest?.trials !== undefined ? { trials: test.attest.trials.run } : {}),
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
const count = (status) => rows.filter((r) => r["status"] === status).length;
|
|
47
|
+
return {
|
|
48
|
+
report_version: REPORT_VERSION,
|
|
49
|
+
timestamp: new Date().toISOString().replace(/\.\d{3}Z$/, "Z"),
|
|
50
|
+
simulated,
|
|
51
|
+
summary: {
|
|
52
|
+
total: rows.length,
|
|
53
|
+
passed: count(STATUS_PASS),
|
|
54
|
+
soft_fail: count(STATUS_SOFT_FAIL),
|
|
55
|
+
hard_fail: count(STATUS_HARD_FAIL),
|
|
56
|
+
not_evaluated: count(STATUS_NOT_EVALUATED),
|
|
57
|
+
usage: sessionUsage(tests, unattached),
|
|
58
|
+
},
|
|
59
|
+
results: rows,
|
|
60
|
+
tests: testRows,
|
|
61
|
+
total_cost: totalCost,
|
|
62
|
+
total_duration_ms: totalDurationMs,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
/** Attest usage summed over every test of the run plus the evaluations that belong to no test. */
|
|
66
|
+
export function sessionUsage(tests, unattached = zeroUsage()) {
|
|
67
|
+
return sumUsage([unattached, ...tests.flatMap((t) => (t.attest === undefined ? [] : [t.attest.usage]))]);
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Lines for the end of the run: one per trials test
|
|
71
|
+
* ("<test_id>: <passed>/<evaluated> trials passed (<n> not evaluated)", plus a
|
|
72
|
+
* note when the test budget stopped it early), then the "attest usage: …" line
|
|
73
|
+
* when the session made or refused a model call.
|
|
74
|
+
*/
|
|
75
|
+
export function summaryLines(tests, unattached = zeroUsage()) {
|
|
76
|
+
const lines = [];
|
|
77
|
+
for (const test of tests) {
|
|
78
|
+
const trials = test.attest?.trials;
|
|
79
|
+
if (trials === undefined)
|
|
80
|
+
continue;
|
|
81
|
+
lines.push(`${test.testId}: ${trials.passed}/${trials.evaluated} trials passed (${trials.notEvaluated} not evaluated)`);
|
|
82
|
+
if (trials.stoppedByBudget) {
|
|
83
|
+
lines.push(`${test.testId}: stopped after ${trials.run} of ${trials.requested} trials: test budget spent`);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
const usage = usageSummaryLine(sessionUsage(tests, unattached));
|
|
87
|
+
if (usage !== undefined)
|
|
88
|
+
lines.push(usage);
|
|
89
|
+
return lines;
|
|
90
|
+
}
|
|
91
|
+
//# sourceMappingURL=report.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"report.js","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EACL,gBAAgB,EAChB,oBAAoB,EACpB,WAAW,EACX,gBAAgB,EAChB,QAAQ,EACR,gBAAgB,EAChB,SAAS,GAGV,MAAM,iBAAiB,CAAC;AAEzB,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC;AAkDhC;;;GAGG;AACH,MAAM,UAAU,WAAW,CACzB,KAA4B,EAC5B,SAAkB,EAClB,aAAoB,SAAS,EAAE;IAE/B,MAAM,IAAI,GAA8B,EAAE,CAAC;IAC3C,MAAM,QAAQ,GAA8B,EAAE,CAAC;IAC/C,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,MAAM,4BAA4B,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1F,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QACvC,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,SAAS,IAAI,IAAI,CAAC,MAAM,EAAE,SAAS,IAAI,CAAC,CAAC;QACzC,eAAe,IAAI,IAAI,CAAC,MAAM,EAAE,eAAe,IAAI,CAAC,CAAC;QACrD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC;YAChD,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YACjF,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YACtC,IAAI,MAAM,CAAC,MAAM,KAAK,oBAAoB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;gBACvF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC;YACZ,OAAO,EAAE,IAAI,CAAC,MAAM;YACpB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,qBAAqB,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,aAAa,EAAE,CAAC,GAAG,YAAY,CAAC;YAChC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC;YACxC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACjF,CAAC,CAAC;IACL,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IACpF,OAAO;QACL,cAAc,EAAE,cAAc;QAC9B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;QAC7D,SAAS;QACT,OAAO,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC;YAC1B,SAAS,EAAE,KAAK,CAAC,gBAAgB,CAAC;YAClC,SAAS,EAAE,KAAK,CAAC,gBAAgB,CAAC;YAClC,aAAa,EAAE,KAAK,CAAC,oBAAoB,CAAC;YAC1C,KAAK,EAAE,YAAY,CAAC,KAAK,EAAE,UAAU,CAAC;SACvC;QACD,OAAO,EAAE,IAAI;QACb,KAAK,EAAE,QAAQ;QACf,UAAU,EAAE,SAAS;QACrB,iBAAiB,EAAE,eAAe;KACnC,CAAC;AACJ,CAAC;AAED,kGAAkG;AAClG,MAAM,UAAU,YAAY,CAAC,KAA4B,EAAE,aAAoB,SAAS,EAAE;IACxF,OAAO,QAAQ,CAAC,CAAC,UAAU,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC3G,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,KAA4B,EAAE,aAAoB,SAAS,EAAE;IACxF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC;QACnC,IAAI,MAAM,KAAK,SAAS;YAAE,SAAS;QACnC,KAAK,CAAC,IAAI,CACR,GAAG,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,SAAS,mBAAmB,MAAM,CAAC,YAAY,iBAAiB,CAC5G,CAAC;QACF,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CACR,GAAG,IAAI,CAAC,MAAM,mBAAmB,MAAM,CAAC,GAAG,OAAO,MAAM,CAAC,SAAS,4BAA4B,CAC/F,CAAC;QACJ,CAAC;IACH,CAAC;IACD,MAAM,KAAK,GAAG,gBAAgB,CAAC,YAAY,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC;IAChE,IAAI,KAAK,KAAK,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3C,OAAO,KAAK,CAAC;AACf,CAAC"}
|