elasticdash-test 0.1.20-alpha-18 → 0.1.20-alpha-20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ci/benchmark.d.ts +16 -2
- package/dist/ci/benchmark.d.ts.map +1 -1
- package/dist/ci/benchmark.js +102 -1
- package/dist/ci/benchmark.js.map +1 -1
- package/dist/ci/ed-runner.d.ts +12 -0
- package/dist/ci/ed-runner.d.ts.map +1 -1
- package/dist/ci/ed-runner.js +29 -9
- package/dist/ci/ed-runner.js.map +1 -1
- package/dist/ci/test-registry.d.ts +16 -0
- package/dist/ci/test-registry.d.ts.map +1 -1
- package/dist/ci/test-registry.js.map +1 -1
- package/dist/ci/upload-client.d.ts +16 -1
- package/dist/ci/upload-client.d.ts.map +1 -1
- package/dist/ci/upload-client.js +78 -20
- package/dist/ci/upload-client.js.map +1 -1
- package/dist/index.cjs +189 -24
- package/package.json +1 -1
- package/src/ci/benchmark.ts +118 -3
- package/src/ci/ed-runner.ts +45 -9
- package/src/ci/test-registry.ts +17 -0
- package/src/ci/upload-client.ts +101 -20
package/dist/ci/benchmark.d.ts
CHANGED
|
@@ -1,15 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* benchmark.ts
|
|
3
|
+
*
|
|
4
|
+
* Compares recorded trace measurements against user-defined benchmarks.
|
|
5
|
+
* Supports duration, token, output-contains, and LLM-as-a-judge evaluations.
|
|
6
|
+
*
|
|
7
|
+
* Generated/updated on 2026-04-20.
|
|
8
|
+
*/
|
|
1
9
|
import type { TestMeasurement } from './measurement.js';
|
|
2
10
|
import type { TestBenchmarks } from './test-registry.js';
|
|
11
|
+
export type MetricName = 'duration_ms' | 'tokens_total' | 'output_contains' | 'output_not_contains' | 'llm_judge';
|
|
3
12
|
export interface MetricResult {
|
|
4
|
-
name:
|
|
13
|
+
name: MetricName;
|
|
5
14
|
value: number;
|
|
6
15
|
threshold: number;
|
|
7
16
|
passed: boolean;
|
|
17
|
+
detail?: string;
|
|
8
18
|
}
|
|
9
19
|
export interface BenchmarkResult {
|
|
10
20
|
passed: boolean;
|
|
11
21
|
failure_reason?: string;
|
|
12
22
|
metrics: MetricResult[];
|
|
13
23
|
}
|
|
14
|
-
|
|
24
|
+
/**
|
|
25
|
+
* Compare a measurement against benchmarks. Async because llm_judge requires
|
|
26
|
+
* an LLM call. The step's output is needed for output_contains/llm_judge checks.
|
|
27
|
+
*/
|
|
28
|
+
export declare function compareBenchmarks(measurement: TestMeasurement, benchmarks: TestBenchmarks, stepOutput?: unknown): Promise<BenchmarkResult>;
|
|
15
29
|
//# sourceMappingURL=benchmark.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"benchmark.d.ts","sourceRoot":"","sources":["../../src/ci/benchmark.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AACvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AAExD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,
|
|
1
|
+
{"version":3,"file":"benchmark.d.ts","sourceRoot":"","sources":["../../src/ci/benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AACvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AAExD,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,cAAc,GAAG,iBAAiB,GAAG,qBAAqB,GAAG,WAAW,CAAA;AAEjH,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,UAAU,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,OAAO,CAAA;IACf,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,OAAO,EAAE,YAAY,EAAE,CAAA;CACxB;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CACrC,WAAW,EAAE,eAAe,EAC5B,UAAU,EAAE,cAAc,EAC1B,UAAU,CAAC,EAAE,OAAO,GACnB,OAAO,CAAC,eAAe,CAAC,CA+H1B"}
|
package/dist/ci/benchmark.js
CHANGED
|
@@ -1,4 +1,17 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* benchmark.ts
|
|
3
|
+
*
|
|
4
|
+
* Compares recorded trace measurements against user-defined benchmarks.
|
|
5
|
+
* Supports duration, token, output-contains, and LLM-as-a-judge evaluations.
|
|
6
|
+
*
|
|
7
|
+
* Generated/updated on 2026-04-20.
|
|
8
|
+
*/
|
|
9
|
+
import { callProviderLLM } from '../matchers/index.js';
|
|
10
|
+
/**
|
|
11
|
+
* Compare a measurement against benchmarks. Async because llm_judge requires
|
|
12
|
+
* an LLM call. The step's output is needed for output_contains/llm_judge checks.
|
|
13
|
+
*/
|
|
14
|
+
export async function compareBenchmarks(measurement, benchmarks, stepOutput) {
|
|
2
15
|
const metrics = [];
|
|
3
16
|
let firstFailure;
|
|
4
17
|
if (benchmarks.max_duration_ms !== undefined) {
|
|
@@ -26,6 +39,86 @@ export function compareBenchmarks(measurement, benchmarks) {
|
|
|
26
39
|
firstFailure = `tokens_total (${value}) exceeded max threshold (${benchmarks.max_tokens_total})`;
|
|
27
40
|
}
|
|
28
41
|
}
|
|
42
|
+
// ─── Output contains ────────────────────────────────────────
|
|
43
|
+
if (benchmarks.output_contains !== undefined) {
|
|
44
|
+
const outputStr = stringifyOutput(stepOutput);
|
|
45
|
+
const passed = outputStr.toLowerCase().includes(benchmarks.output_contains.toLowerCase());
|
|
46
|
+
metrics.push({
|
|
47
|
+
name: 'output_contains',
|
|
48
|
+
value: passed ? 1 : 0,
|
|
49
|
+
threshold: 1,
|
|
50
|
+
passed,
|
|
51
|
+
detail: passed ? undefined : `output does not contain "${benchmarks.output_contains}"`,
|
|
52
|
+
});
|
|
53
|
+
if (!passed && !firstFailure) {
|
|
54
|
+
firstFailure = `output does not contain "${benchmarks.output_contains}"`;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
// ─── Output not contains ────────────────────────────────────
|
|
58
|
+
if (benchmarks.output_not_contains !== undefined) {
|
|
59
|
+
const outputStr = stringifyOutput(stepOutput);
|
|
60
|
+
const passed = !outputStr.toLowerCase().includes(benchmarks.output_not_contains.toLowerCase());
|
|
61
|
+
metrics.push({
|
|
62
|
+
name: 'output_not_contains',
|
|
63
|
+
value: passed ? 1 : 0,
|
|
64
|
+
threshold: 1,
|
|
65
|
+
passed,
|
|
66
|
+
detail: passed ? undefined : `output unexpectedly contains "${benchmarks.output_not_contains}"`,
|
|
67
|
+
});
|
|
68
|
+
if (!passed && !firstFailure) {
|
|
69
|
+
firstFailure = `output unexpectedly contains "${benchmarks.output_not_contains}"`;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
// ─── LLM-as-a-judge ────────────────────────────────────────
|
|
73
|
+
if (benchmarks.llm_judge) {
|
|
74
|
+
const judge = benchmarks.llm_judge;
|
|
75
|
+
const outputStr = stringifyOutput(stepOutput);
|
|
76
|
+
const threshold = judge.judge_score_threshold ?? 7;
|
|
77
|
+
try {
|
|
78
|
+
const provider = judge.judge_provider ?? 'openai';
|
|
79
|
+
const evalPrompt = `${judge.judge_prompt}\n\nOutput to evaluate:\n${outputStr}\n\nScore this output on a scale of 0-10. Respond with only the number.`;
|
|
80
|
+
const result = await callProviderLLM(evalPrompt, { provider, model: judge.judge_model ?? undefined }, 'You are an expert test judge. Return only a number between 0 and 10.', 16, 0);
|
|
81
|
+
const score = parseFloat(result.content.match(/-?\d+(?:\.\d+)?/)?.[0] ?? '');
|
|
82
|
+
if (isNaN(score)) {
|
|
83
|
+
metrics.push({
|
|
84
|
+
name: 'llm_judge',
|
|
85
|
+
value: 0,
|
|
86
|
+
threshold,
|
|
87
|
+
passed: false,
|
|
88
|
+
detail: `Could not parse score from LLM response: "${result.content}"`,
|
|
89
|
+
});
|
|
90
|
+
if (!firstFailure) {
|
|
91
|
+
firstFailure = `llm_judge: could not parse score from response`;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
else {
|
|
95
|
+
const passed = score >= threshold;
|
|
96
|
+
metrics.push({
|
|
97
|
+
name: 'llm_judge',
|
|
98
|
+
value: score,
|
|
99
|
+
threshold,
|
|
100
|
+
passed,
|
|
101
|
+
detail: `Score: ${score}/${threshold}`,
|
|
102
|
+
});
|
|
103
|
+
if (!passed && !firstFailure) {
|
|
104
|
+
firstFailure = `llm_judge score (${score}) below threshold (${threshold})`;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
catch (err) {
|
|
109
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
110
|
+
metrics.push({
|
|
111
|
+
name: 'llm_judge',
|
|
112
|
+
value: 0,
|
|
113
|
+
threshold,
|
|
114
|
+
passed: false,
|
|
115
|
+
detail: `LLM judge error: ${errMsg}`,
|
|
116
|
+
});
|
|
117
|
+
if (!firstFailure) {
|
|
118
|
+
firstFailure = `llm_judge error: ${errMsg}`;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
29
122
|
const allPassed = metrics.every(m => m.passed);
|
|
30
123
|
return {
|
|
31
124
|
passed: allPassed,
|
|
@@ -33,4 +126,12 @@ export function compareBenchmarks(measurement, benchmarks) {
|
|
|
33
126
|
metrics,
|
|
34
127
|
};
|
|
35
128
|
}
|
|
129
|
+
/** Converts step output to a string for text-based assertions. */
|
|
130
|
+
function stringifyOutput(output) {
|
|
131
|
+
if (output === null || output === undefined)
|
|
132
|
+
return '';
|
|
133
|
+
if (typeof output === 'string')
|
|
134
|
+
return output;
|
|
135
|
+
return JSON.stringify(output);
|
|
136
|
+
}
|
|
36
137
|
//# sourceMappingURL=benchmark.js.map
|
package/dist/ci/benchmark.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/ci/benchmark.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/ci/benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAoBtD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,WAA4B,EAC5B,UAA0B,EAC1B,UAAoB;IAEpB,MAAM,OAAO,GAAmB,EAAE,CAAA;IAClC,IAAI,YAAgC,CAAA;IAEpC,IAAI,UAAU,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,WAAW,CAAC,WAAW,IAAI,UAAU,CAAC,eAAe,CAAA;QACpE,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,aAAa;YACnB,KAAK,EAAE,WAAW,CAAC,WAAW;YAC9B,SAAS,EAAE,UAAU,CAAC,eAAe;YACrC,MAAM;SACP,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,YAAY,GAAG,gBAAgB,WAAW,CAAC,WAAW,6BAA6B,UAAU,CAAC,eAAe,GAAG,CAAA;QAClH,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,WAAW,CAAC,YAAY,IAAI,CAAC,CAAA;QAC3C,MAAM,MAAM,GAAG,KAAK,IAAI,UAAU,CAAC,gBAAgB,CAAA;QACnD,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,cAAc;YACpB,KAAK;YACL,SAAS,EAAE,UAAU,CAAC,gBAAgB;YACtC,MAAM;SACP,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,YAAY,GAAG,iBAAiB,KAAK,6BAA6B,UAAU,CAAC,gBAAgB,GAAG,CAAA;QAClG,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,IAAI,UAAU,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QAC7C,MAAM,SAAS,GAAG,eAAe,CAAC,UAAU,CAAC,CAAA;QAC7C,MAAM,MAAM,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC,CAAA;QACzF,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,iBAAiB;YACvB,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,SAAS,EAAE,CAAC;YACZ,MAAM;YACN,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B,UAAU,CAAC,eAAe,GAAG;SACvF,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,YAAY,GAAG,4BAA4B,UAAU,CAAC,eAAe,GAAG,CAAA;QAC1E,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,IAAI,UAAU,CAAC,mBAAmB,KAAK,SAAS,EAAE,CAAC;QACjD,MAAM,SAAS,GAAG,eAAe,CAAC,UAAU,CAAC,CAAA;QAC7C,MAAM,MAAM,GAAG,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,mBAAmB,CAAC,WAAW,EAAE,CAAC,CAAA;QAC9F,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,qBAAqB;YAC3B,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,SAAS,EAAE,CAAC;YACZ,MAAM;YACN,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,iCAAiC,UAAU,CAAC,mBAAmB,GAAG;SAChG,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,YAAY,GAAG,iCAAiC,UAAU,CAAC,mBAAmB,GAAG,CAAA;QACnF,CAAC;IACH,CAAC;IAED,8DAA8D;IAC9D,IAAI,UAAU,CAAC,SAAS,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,CAAA;QAClC,MAAM,SAAS,GAAG,eAAe,CAAC,UAAU,CAAC,CAAA;QAC7C,MAAM,SAAS,GAAG,KAAK,CAAC,qBAAqB,IAAI,CAAC,CAAA;QAElD,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,IAAI,QAAQ,CAAA;YACjD,MAAM,UAAU,GAAG,GAAG,KAAK,CAAC,YAAY,4BAA4B,SAAS,yEAAyE,CAAA;YAEtJ,MAAM,MAAM,GAAG,MAAM,eAAe,CAClC,UAAU,EACV,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,WAAW,IAAI,SAAS,EAAE,EACnD,sEAAsE,EACtE,EAAE,EACF,CAAC,CACF,CAAA;YAED,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;YAC5E,IAAI,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjB,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,WAAW;oBACjB,KAAK,EAAE,CAAC;oBACR,SAAS;oBACT,MAAM,EAAE,KAAK;oBACb,MAAM,EAAE,6CAA6C,MAAM,CAAC,OAAO,GAAG;iBACvE,CAAC,CAAA;gBACF,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClB,YAAY,GAAG,gDAAgD,CAAA;gBACjE,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,GAAG,KAAK,IAAI,SAAS,CAAA;gBACjC,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,WAAW;oBACjB,KAAK,EAAE,KAAK;oBACZ,SAAS;oBACT,MAAM;oBACN,MAAM,EAAE,UAAU,KAAK,IAAI,SAAS,EAAE;iBACvC,CAAC,CAAA;gBACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;oBAC7B,YAAY,GAAG,oBAAoB,KAAK,sBAAsB,SAAS,GAAG,CAAA;gBAC5E,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YAC/D,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,WAAW;gBACjB,KAAK,EAAE,CAAC;gBACR,SAAS;gBACT,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,oBAAoB,MAAM,EAAE;aACrC,CAAC,CAAA;YACF,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,YAAY,GAAG,oBAAoB,MAAM,EAAE,CAAA;YAC7C,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;IAC9C,OAAO;QACL,MAAM,EAAE,SAAS;QACjB,cAAc,EAAE,YAAY;QAC5B,OAAO;KACR,CAAA;AACH,CAAC;AAED,kEAAkE;AAClE,SAAS,eAAe,CAAC,MAAe;IACtC,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,EAAE,CAAA;IACtD,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAA;IAC7C,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;AAC/B,CAAC"}
|
package/dist/ci/ed-runner.d.ts
CHANGED
|
@@ -9,6 +9,16 @@ export interface EdTestRunOptions {
|
|
|
9
9
|
/** Number of times to run each test. Passes if any run succeeds. Defaults to 1. */
|
|
10
10
|
runs?: number;
|
|
11
11
|
}
|
|
12
|
+
export interface EdSingleRunResult {
|
|
13
|
+
status: 'pass' | 'fail';
|
|
14
|
+
failureReason?: string;
|
|
15
|
+
measurement?: TestMeasurement;
|
|
16
|
+
benchmarkResult?: BenchmarkResult;
|
|
17
|
+
output?: unknown;
|
|
18
|
+
durationMs: number;
|
|
19
|
+
startedAt: string;
|
|
20
|
+
finishedAt: string;
|
|
21
|
+
}
|
|
12
22
|
export interface EdTestResult {
|
|
13
23
|
testId: string;
|
|
14
24
|
testName: string;
|
|
@@ -24,6 +34,8 @@ export interface EdTestResult {
|
|
|
24
34
|
input?: unknown;
|
|
25
35
|
output?: unknown;
|
|
26
36
|
durationMs: number;
|
|
37
|
+
/** All individual run results when --runs > 1 */
|
|
38
|
+
singleRuns?: EdSingleRunResult[];
|
|
27
39
|
}
|
|
28
40
|
export interface EdTestRunResult {
|
|
29
41
|
runId: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ed-runner.d.ts","sourceRoot":"","sources":["../../src/ci/ed-runner.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAOrD,MAAM,WAAW,gBAAgB;IAC/B,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,QAAQ,CAAC,EAAE,SAAS,GAAG,MAAM,GAAG,OAAO,CAAA;IACvC,mFAAmF;IACnF,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,GAAG,MAAM,CAAA;IACvB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,WAAW,CAAC,EAAE,eAAe,CAAA;IAC7B,eAAe,CAAC,EAAE,eAAe,CAAA;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAA;IAC1C,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;
|
|
1
|
+
{"version":3,"file":"ed-runner.d.ts","sourceRoot":"","sources":["../../src/ci/ed-runner.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAOrD,MAAM,WAAW,gBAAgB;IAC/B,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,QAAQ,CAAC,EAAE,SAAS,GAAG,MAAM,GAAG,OAAO,CAAA;IACvC,mFAAmF;IACnF,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAA;IACvB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,WAAW,CAAC,EAAE,eAAe,CAAA;IAC7B,eAAe,CAAC,EAAE,eAAe,CAAA;IACjC,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,GAAG,MAAM,CAAA;IACvB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,WAAW,CAAC,EAAE,eAAe,CAAA;IAC7B,eAAe,CAAC,EAAE,eAAe,CAAA;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAA;IAC1C,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,iDAAiD;IACjD,UAAU,CAAC,EAAE,iBAAiB,EAAE,CAAA;CACjC;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,EAAE,YAAY,EAAE,CAAA;IACvB,UAAU,EAAE,MAAM,CAAA;CACnB;AAID,wBAAsB,UAAU,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC,CAsFrF"}
|
package/dist/ci/ed-runner.js
CHANGED
|
@@ -19,6 +19,7 @@ export async function runEdTests(options) {
|
|
|
19
19
|
status: 'fail',
|
|
20
20
|
failureReason: `validation error: ${err.message}`,
|
|
21
21
|
durationMs: 0,
|
|
22
|
+
singleRuns: [],
|
|
22
23
|
});
|
|
23
24
|
}
|
|
24
25
|
// Filter tests if pattern provided
|
|
@@ -29,22 +30,41 @@ export async function runEdTests(options) {
|
|
|
29
30
|
}
|
|
30
31
|
const maxRuns = Math.max(1, options?.runs ?? 1);
|
|
31
32
|
for (const test of testsToRun) {
|
|
33
|
+
const allRuns = [];
|
|
32
34
|
let bestResult = null;
|
|
33
35
|
for (let attempt = 1; attempt <= maxRuns; attempt++) {
|
|
36
|
+
const runStartedAt = new Date().toISOString();
|
|
34
37
|
const result = await runSingleTest(test);
|
|
38
|
+
const runFinishedAt = new Date().toISOString();
|
|
35
39
|
if (attempt > 1) {
|
|
36
40
|
console.log(` [ed-test] ${test.name}: run ${attempt}/${maxRuns} — ${result.status}`);
|
|
37
41
|
}
|
|
38
|
-
//
|
|
42
|
+
// Collect every run for upload
|
|
43
|
+
allRuns.push({
|
|
44
|
+
status: result.status,
|
|
45
|
+
failureReason: result.failureReason,
|
|
46
|
+
measurement: result.measurement,
|
|
47
|
+
benchmarkResult: result.benchmarkResult,
|
|
48
|
+
output: result.output,
|
|
49
|
+
durationMs: result.durationMs,
|
|
50
|
+
startedAt: runStartedAt,
|
|
51
|
+
finishedAt: runFinishedAt,
|
|
52
|
+
});
|
|
53
|
+
// Keep the first passing result, or the last failure (for aggregate status)
|
|
39
54
|
if (!bestResult || result.status === 'pass') {
|
|
40
55
|
bestResult = result;
|
|
41
56
|
}
|
|
42
|
-
// Stop retrying on pass
|
|
43
|
-
if (result.status === 'pass')
|
|
44
|
-
break;
|
|
45
57
|
}
|
|
46
|
-
|
|
47
|
-
|
|
58
|
+
// Aggregate: fail if ANY run failed
|
|
59
|
+
const anyFailed = allRuns.some(r => r.status === 'fail');
|
|
60
|
+
const failedRun = allRuns.find(r => r.status === 'fail');
|
|
61
|
+
results.push({
|
|
62
|
+
...bestResult,
|
|
63
|
+
status: anyFailed ? 'fail' : 'pass',
|
|
64
|
+
failureReason: anyFailed ? (failedRun?.failureReason || bestResult.failureReason) : undefined,
|
|
65
|
+
singleRuns: allRuns,
|
|
66
|
+
});
|
|
67
|
+
if (options?.failFast && anyFailed) {
|
|
48
68
|
break;
|
|
49
69
|
}
|
|
50
70
|
}
|
|
@@ -112,8 +132,8 @@ async function runSingleTest(test) {
|
|
|
112
132
|
durationMs: Date.now() - startMs,
|
|
113
133
|
};
|
|
114
134
|
}
|
|
115
|
-
// Compare against benchmarks
|
|
116
|
-
const benchmarkResult = compareBenchmarks(measurement, test.benchmarks);
|
|
135
|
+
// Compare against benchmarks (async to support llm_judge)
|
|
136
|
+
const benchmarkResult = await compareBenchmarks(measurement, test.benchmarks, targetStep?.output);
|
|
117
137
|
return {
|
|
118
138
|
...base,
|
|
119
139
|
testId: test.name,
|
|
@@ -132,7 +152,7 @@ async function runSingleTest(test) {
|
|
|
132
152
|
const traceMeasurement = extractMeasurementFromTrace(test);
|
|
133
153
|
if (traceMeasurement) {
|
|
134
154
|
console.log(` [ed-test] ${test.name}: run() failed (${err instanceof Error ? err.message : String(err)}), using trace measurement fallback`);
|
|
135
|
-
const benchmarkResult = compareBenchmarks(traceMeasurement, test.benchmarks);
|
|
155
|
+
const benchmarkResult = await compareBenchmarks(traceMeasurement, test.benchmarks, targetStep?.output);
|
|
136
156
|
return {
|
|
137
157
|
...base,
|
|
138
158
|
testId: test.name,
|
package/dist/ci/ed-runner.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ed-runner.js","sourceRoot":"","sources":["../../src/ci/ed-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC5C,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAClG,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;
|
|
1
|
+
{"version":3,"file":"ed-runner.js","sourceRoot":"","sources":["../../src/ci/ed-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC5C,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAClG,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AAsDlD,+DAA+D;AAE/D,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAA0B;IACzD,MAAM,GAAG,GAAG,OAAO,EAAE,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAA;IACzC,MAAM,KAAK,GAAG,UAAU,EAAE,CAAA;IAC1B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAC1C,MAAM,OAAO,GAAmB,EAAE,CAAA;IAElC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAA;IAElD,2CAA2C;IAC3C,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC;YACX,MAAM,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS;YACjC,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS;YACnC,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,qBAAqB,GAAG,CAAC,OAAO,EAAE;YACjD,UAAU,EAAE,CAAC;YACb,UAAU,EAAE,EAAE;SACf,CAAC,CAAA;IACJ,CAAC;IAED,mCAAmC;IACnC,IAAI,UAAU,GAAoB,KAAK,CAAA;IACvC,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QACpB,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAA;QAC9B,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAA;IAC5D,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,IAAI,CAAC,CAAC,CAAA;IAE/C,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAwB,EAAE,CAAA;QACvC,IAAI,UAAU,GAAwB,IAAI,CAAA;QAE1C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YACpD,MAAM,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YAC7C,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAA;YACxC,MAAM,aAAa,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YAE9C,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,SAAS,OAAO,IAAI,OAAO,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;YACvF,CAAC;YAED,+BAA+B;YAC/B,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,eAAe,EAAE,MAAM,CAAC,eAAe;gBACvC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,SAAS,EAAE,YAAY;gBACvB,UAAU,EAAE,aAAa;aAC1B,CAAC,CAAA;YAEF,4EAA4E;YAC5E,IAAI,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;gBAC5C,UAAU,GAAG,MAAM,CAAA;YACrB,CAAC;QAEP,CAAC;QAEG,oCAAoC;QACpC,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAA;QACxD,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAA;QAExD,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,UAAW;YACd,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YACnC,aAAa,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,aAAa,IAAI,UAAW,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS;YAC9F,UAAU,EAAE,OAAO;SACpB,CAAC,CAAA;QAEF,IAAI,OAAO,EAAE,QAAQ,IAAI,SAAS,EAAE,CAAC;YACnC,MAAK;QACP,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAE3C,OAAO;QACL,KAAK;QACL,SAAS;QACT,UAAU;QACV,OAAO;QACP,UAAU,EAAE,WAAW;KACxB,CAAA;AACH,CAAC;AAED,+DAA+D;AAE/D,KAAK,UAAU,aAAa,CAAC,IAAmB;IAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC1B,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACpF,MAAM,IAAI,GAA0B;QAClC,MAAM,EAAE,IAAI,CAAC,IAAI;QACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;QACnB,QAAQ,EAAE,IAAI,CAAC,KAAK;QACpB,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;QAChE,KAAK,EAAE,UAAU,EAAE,KAAK;QACxB,MAAM,EAAE,UAAU,EAAE,MAAM;KAC3B,CAAA;IAED,4BAA4B;IAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,OAAO,IAAI,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;QAChD,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,0BAA0B;YACzC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAA;IACH,CAAC;IAED,MAAM,SAAS,GAAG,mBAAmB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAC1E,aAAa,CAAC,SAAS,CAAC,CAAA;IAExB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,IAAI,KAAK,CAAA;QAE1C,MAAM,OAAO,CAAC,IAAI,CAAC;YACjB,IAAI,CAAC,GAAG,EAAE;YACV,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAC/B,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CACjE;SACF,CAAC,CAAA;QAEF,4CAA4C;QAC5C,gEAAgE;QAChE,sEAAsE;QACtE,qEAAqE;QACrE,sEAAsE;QACtE,uEAAuE;QACvE,2CAA2C;QAC3C,IAAI,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAC/C,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,WAAW,GAAG,2BAA2B,CAAC,IAAI,CAAC,IAAI,IAAI,CAAA;YACvD,IAAI,WAAW,EAAE,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,yDAAyD,CAAC,CAAA;YAChG,CAAC;QACH,CAAC;QACD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,gBAAgB,IAAI,CAAC,MAAM,CAAC,OAAO,qCAAqC;gBACvF,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QAED,0DAA0D;QAC1D,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,MAAM,CAAC,CAAA;QAEjG,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAChD,aAAa,EAAE,eAAe,CAAC,cAAc;YAC7C,WAAW;YACX,eAAe;YACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAA;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,4EAA4E;QAC5E,4EAA4E;QAC5E,iEAAiE;QACjE,MAAM,gBAAgB,GAAG,2BAA2B,CAAC,IAAI,CAAC,CAAA;QAC1D,IAAI,gBAAgB,EAAE,CAAC;YACrB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,mBAAmB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAA;YAC7I,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,MAAM,CAAC,CAAA;YACtG,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;gBAChD,aAAa,EAAE,eAAe,CAAC,cAAc;gBAC7C,WAAW,EAAE,gBAAgB;gBAC7B,eAAe;gBACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QAED,IAAI,GAAG,YAAY,eAAe,EAAE,CAAC;YACnC,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,gBAAgB,GAAG,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,EAAE;gBAC9D,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QACD,IAAI,GAAG,YAAY,YAAY,EAAE,CAAC;YAChC,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,wBAAwB,GAAG,CAAC,SAAS,IAAI;gBACxD,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QACD,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,oBAAoB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YACrF,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAA;IACH,CAAC;YAAS,CAAC;QACT,eAAe,EAAE,CAAA;IACnB,CAAC;AACH,CAAC;AAED,+DAA+D;AAE/D;;;;;;GAMG;AACH,SAAS,2BAA2B,CAAC,IAAmB;IACtD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAC9E,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAA;IAE3B,MAAM,MAAM,GAAoB;QAC9B,WAAW,EAAE,IAAI,CAAC,WAAW;KAC9B,CAAA;IAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAA;QACvC,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAA;QACzC,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAA;IACzC,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED,+DAA+D;AAE/D,MAAM,YAAa,SAAQ,KAAK;IACX;IAAnB,YAAmB,SAAiB;QAClC,KAAK,CAAC,wBAAwB,SAAS,IAAI,CAAC,CAAA;QAD3B,cAAS,GAAT,SAAS,CAAQ;QAElC,IAAI,CAAC,IAAI,GAAG,cAAc,CAAA;IAC5B,CAAC;CACF;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,OAAe;IAC9C,0CAA0C;IAC1C,MAAM,KAAK,GAAG,IAAI,MAAM,CACtB,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,GAAG,CAClG,CAAA;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACzB,CAAC"}
|
|
@@ -2,9 +2,25 @@ export interface TestTarget {
|
|
|
2
2
|
type: 'tool_call' | 'ai_call';
|
|
3
3
|
step_id: string;
|
|
4
4
|
}
|
|
5
|
+
export interface LLMJudgeBenchmark {
|
|
6
|
+
/** The prompt sent to the LLM judge to evaluate the step's output. */
|
|
7
|
+
judge_prompt: string;
|
|
8
|
+
/** Minimum score (0-10) required to pass. Defaults to 7. */
|
|
9
|
+
judge_score_threshold?: number;
|
|
10
|
+
/** LLM provider to use for judging. Defaults to 'openai'. */
|
|
11
|
+
judge_provider?: 'openai' | 'claude' | 'gemini' | 'grok' | 'kimi';
|
|
12
|
+
/** Model override for the judge LLM. */
|
|
13
|
+
judge_model?: string;
|
|
14
|
+
}
|
|
5
15
|
export interface TestBenchmarks {
|
|
6
16
|
max_duration_ms?: number;
|
|
7
17
|
max_tokens_total?: number;
|
|
18
|
+
/** Assert that the step's output contains this substring. */
|
|
19
|
+
output_contains?: string;
|
|
20
|
+
/** Assert that the step's output does NOT contain this substring. */
|
|
21
|
+
output_not_contains?: string;
|
|
22
|
+
/** LLM-as-a-judge evaluation of the step's output quality. */
|
|
23
|
+
llm_judge?: LLMJudgeBenchmark;
|
|
8
24
|
}
|
|
9
25
|
export interface TestDefinition {
|
|
10
26
|
name: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"test-registry.d.ts","sourceRoot":"","sources":["../../src/ci/test-registry.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,WAAW,GAAG,SAAS,CAAA;IAC7B,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,gBAAgB,CAAC,EAAE,MAAM,CAAA;
|
|
1
|
+
{"version":3,"file":"test-registry.d.ts","sourceRoot":"","sources":["../../src/ci/test-registry.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,WAAW,GAAG,SAAS,CAAA;IAC7B,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,sEAAsE;IACtE,YAAY,EAAE,MAAM,CAAA;IACpB,4DAA4D;IAC5D,qBAAqB,CAAC,EAAE,MAAM,CAAA;IAC9B,6DAA6D;IAC7D,cAAc,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,CAAA;IACjE,wCAAwC;IACxC,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,6DAA6D;IAC7D,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,qEAAqE;IACrE,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,8DAA8D;IAC9D,SAAS,CAAC,EAAE,iBAAiB,CAAA;CAC9B;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,UAAU,CAAA;IAClB,UAAU,EAAE,cAAc,CAAA;IAC1B,2FAA2F;IAC3F,GAAG,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IACzB,2DAA2D;IAC3D,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAWD,wBAAgB,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,GAAG,IAAI,CAEzE;AAED,wBAAgB,eAAe,IAAI,cAAc,EAAE,CAElD;AAED,wBAAgB,iBAAiB,IAAI,IAAI,CAExC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"test-registry.js","sourceRoot":"","sources":["../../src/ci/test-registry.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"test-registry.js","sourceRoot":"","sources":["../../src/ci/test-registry.ts"],"names":[],"mappings":"AAwCA,4EAA4E;AAC5E,6EAA6E;AAC7E,iCAAiC;AACjC,MAAM,CAAC,GAAG,UAAqC,CAAA;AAC/C,MAAM,oBAAoB,GAAG,kCAAkC,CAAA;AAC/D,MAAM,QAAQ,GACX,CAAC,CAAC,oBAAoB,CAAsB,IAAI,EAAE,CAAA;AACrD,IAAI,CAAC,CAAC,CAAC,oBAAoB,CAAC;IAAE,CAAC,CAAC,oBAAoB,CAAC,GAAG,QAAQ,CAAA;AAEhE,MAAM,UAAU,UAAU,CAAC,GAAwC;IACjE,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,GAAG,EAAE,CAAC,CAAA;AAC3B,CAAC;AAED,MAAM,UAAU,eAAe;IAC7B,OAAO,CAAC,GAAG,QAAQ,CAAC,CAAA;AACtB,CAAC;AAED,MAAM,UAAU,iBAAiB;IAC/B,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAA;AACrB,CAAC"}
|
|
@@ -31,18 +31,33 @@ export interface UploadTestResult {
|
|
|
31
31
|
step_id: string;
|
|
32
32
|
};
|
|
33
33
|
duration_ms: number;
|
|
34
|
+
reruns_count: number;
|
|
34
35
|
input?: unknown;
|
|
35
36
|
output?: unknown;
|
|
36
37
|
metrics: UploadMetric[];
|
|
38
|
+
reruns?: UploadRerun[];
|
|
37
39
|
}
|
|
38
40
|
export interface UploadMetric {
|
|
39
|
-
name: 'duration_ms' | 'tokens_total' | 'tokens_input' | 'tokens_output';
|
|
41
|
+
name: 'duration_ms' | 'tokens_total' | 'tokens_input' | 'tokens_output' | 'output_contains' | 'output_not_contains' | 'llm_judge';
|
|
40
42
|
value: number;
|
|
41
43
|
passed: boolean;
|
|
42
44
|
benchmark?: {
|
|
43
45
|
type: string;
|
|
44
46
|
threshold: number;
|
|
45
47
|
};
|
|
48
|
+
/** Per-rerun measurements (when reruns > 1) */
|
|
49
|
+
measurements?: {
|
|
50
|
+
rerun_index: number;
|
|
51
|
+
value: number;
|
|
52
|
+
passed: boolean;
|
|
53
|
+
}[];
|
|
54
|
+
}
|
|
55
|
+
export interface UploadRerun {
|
|
56
|
+
rerun_index: number;
|
|
57
|
+
input?: unknown;
|
|
58
|
+
output?: unknown;
|
|
59
|
+
started_at?: string;
|
|
60
|
+
finished_at?: string;
|
|
46
61
|
}
|
|
47
62
|
export declare function buildUploadPayload(runResult: EdTestRunResult): UploadPayload;
|
|
48
63
|
export interface UploadOptions {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"upload-client.d.ts","sourceRoot":"","sources":["../../src/ci/upload-client.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,eAAe,
|
|
1
|
+
{"version":3,"file":"upload-client.d.ts","sourceRoot":"","sources":["../../src/ci/upload-client.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,eAAe,EAAmC,MAAM,gBAAgB,CAAA;AAKtF,MAAM,WAAW,aAAa;IAC5B,cAAc,EAAE,CAAC,CAAA;IACjB,GAAG,EAAE;QACH,MAAM,EAAE,MAAM,CAAA;QACd,UAAU,EAAE,MAAM,CAAA;QAClB,WAAW,EAAE,MAAM,CAAA;QACnB,WAAW,EAAE,MAAM,CAAA;QACnB,GAAG,EAAE;YACH,UAAU,EAAE,MAAM,CAAA;YAClB,MAAM,EAAE,MAAM,CAAA;YACd,WAAW,CAAC,EAAE,MAAM,CAAA;YACpB,SAAS,CAAC,EAAE,MAAM,CAAA;YAClB,IAAI,CAAC,EAAE,MAAM,CAAA;SACd,CAAA;QACD,EAAE,EAAE;YACF,QAAQ,CAAC,EAAE,MAAM,CAAA;YACjB,OAAO,CAAC,EAAE,MAAM,CAAA;SACjB,CAAA;KACF,CAAA;IACD,OAAO,EAAE,gBAAgB,EAAE,CAAA;CAC5B;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,GAAG,MAAM,CAAA;IACvB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAA;IAC1C,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,OAAO,EAAE,YAAY,EAAE,CAAA;IACvB,MAAM,CAAC,EAAE,WAAW,EAAE,CAAA;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,aAAa,GAAG,cAAc,GAAG,cAAc,GAAG,eAAe,GAAG,iBAAiB,GAAG,qBAAqB,GAAG,WAAW,CAAA;IACjI,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,OAAO,CAAA;IACf,SAAS,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAA;IAC/C,+CAA+C;IAC/C,YAAY,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,EAAE,CAAA;CACzE;AAED,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAID,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,eAAe,GAAG,aAAa,CAwB5E;AAyHD,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,cAAc;IAE7B,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,UAAU,GAAG,kBAAkB,CAAA;IACvC,YAAY,EAAE,MAAM,CAAA;CACrB;AAED,wBAAsB,aAAa,CACjC,OAAO,EAAE,aAAa,EACtB,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,cAAc,CAAC,CAgDzB;AAID,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,aAAa,EACtB,KAAK,EAAE,MAAM,EACb,GAAG,CAAC,EAAE,MAAM,GACX,OAAO,CAAC,IAAI,CAAC,CASf"}
|
package/dist/ci/upload-client.js
CHANGED
|
@@ -28,10 +28,11 @@ export function buildUploadPayload(runResult) {
|
|
|
28
28
|
results: runResult.results.map(r => mapTestResult(r)),
|
|
29
29
|
};
|
|
30
30
|
}
|
|
31
|
-
function
|
|
31
|
+
function buildMetricsFromRun(run, benchmarkResult) {
|
|
32
32
|
const metrics = [];
|
|
33
|
-
|
|
34
|
-
|
|
33
|
+
const br = run.benchmarkResult ?? benchmarkResult;
|
|
34
|
+
if (br) {
|
|
35
|
+
for (const m of br.metrics) {
|
|
35
36
|
metrics.push({
|
|
36
37
|
name: m.name,
|
|
37
38
|
value: m.value,
|
|
@@ -40,28 +41,83 @@ function mapTestResult(result) {
|
|
|
40
41
|
});
|
|
41
42
|
}
|
|
42
43
|
}
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
44
|
+
if (run.measurement) {
|
|
45
|
+
if (!metrics.some(m => m.name === 'duration_ms')) {
|
|
46
|
+
metrics.push({ name: 'duration_ms', value: run.measurement.duration_ms, passed: true });
|
|
47
|
+
}
|
|
48
|
+
if (run.measurement.tokens_input !== undefined && !metrics.some(m => m.name === 'tokens_input')) {
|
|
49
|
+
metrics.push({ name: 'tokens_input', value: run.measurement.tokens_input, passed: true });
|
|
48
50
|
}
|
|
49
|
-
if (
|
|
50
|
-
|
|
51
|
-
if (!has)
|
|
52
|
-
metrics.push({ name: 'tokens_input', value: result.measurement.tokens_input, passed: true });
|
|
51
|
+
if (run.measurement.tokens_output !== undefined && !metrics.some(m => m.name === 'tokens_output')) {
|
|
52
|
+
metrics.push({ name: 'tokens_output', value: run.measurement.tokens_output, passed: true });
|
|
53
53
|
}
|
|
54
|
-
if (
|
|
55
|
-
|
|
56
|
-
if (!has)
|
|
57
|
-
metrics.push({ name: 'tokens_output', value: result.measurement.tokens_output, passed: true });
|
|
54
|
+
if (run.measurement.tokens_total !== undefined && !metrics.some(m => m.name === 'tokens_total')) {
|
|
55
|
+
metrics.push({ name: 'tokens_total', value: run.measurement.tokens_total, passed: true });
|
|
58
56
|
}
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
57
|
+
}
|
|
58
|
+
return metrics;
|
|
59
|
+
}
|
|
60
|
+
function mapTestResult(result) {
|
|
61
|
+
const singleRuns = result.singleRuns || [];
|
|
62
|
+
const rerunsCount = singleRuns.length || 1;
|
|
63
|
+
// Single run (or fallback) — flat metrics, no per-rerun data
|
|
64
|
+
if (rerunsCount <= 1) {
|
|
65
|
+
const run = singleRuns[0];
|
|
66
|
+
const metrics = run
|
|
67
|
+
? buildMetricsFromRun(run, result.benchmarkResult)
|
|
68
|
+
: buildMetricsFromRun(result, result.benchmarkResult);
|
|
69
|
+
return {
|
|
70
|
+
test_id: result.testId,
|
|
71
|
+
test_name: result.testName,
|
|
72
|
+
status: result.status,
|
|
73
|
+
failure_reason: result.failureReason,
|
|
74
|
+
trace_ref: result.traceRef,
|
|
75
|
+
target: result.target,
|
|
76
|
+
duration_ms: result.durationMs,
|
|
77
|
+
reruns_count: 1,
|
|
78
|
+
input: result.input,
|
|
79
|
+
output: result.output,
|
|
80
|
+
metrics,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
// Multiple runs — use measurements[] format with rerun_index
|
|
84
|
+
// Collect all unique metric names across all runs
|
|
85
|
+
const metricNameSet = new Set();
|
|
86
|
+
const perRunMetrics = singleRuns.map(run => {
|
|
87
|
+
const m = buildMetricsFromRun(run, result.benchmarkResult);
|
|
88
|
+
m.forEach(metric => metricNameSet.add(metric.name));
|
|
89
|
+
return m;
|
|
90
|
+
});
|
|
91
|
+
// Build metrics with measurements[] arrays
|
|
92
|
+
const metrics = [];
|
|
93
|
+
for (const name of metricNameSet) {
|
|
94
|
+
const measurements = [];
|
|
95
|
+
let benchmark;
|
|
96
|
+
for (let i = 0; i < singleRuns.length; i++) {
|
|
97
|
+
const metric = perRunMetrics[i].find(m => m.name === name);
|
|
98
|
+
if (metric) {
|
|
99
|
+
measurements.push({ rerun_index: i, value: metric.value, passed: metric.passed });
|
|
100
|
+
if (!benchmark && metric.benchmark)
|
|
101
|
+
benchmark = metric.benchmark;
|
|
102
|
+
}
|
|
63
103
|
}
|
|
104
|
+
// Aggregate: use first run's value as the top-level value
|
|
105
|
+
const firstMeasurement = measurements[0];
|
|
106
|
+
metrics.push({
|
|
107
|
+
name: name,
|
|
108
|
+
value: firstMeasurement?.value ?? 0,
|
|
109
|
+
passed: measurements.every(m => m.passed),
|
|
110
|
+
benchmark,
|
|
111
|
+
measurements,
|
|
112
|
+
});
|
|
64
113
|
}
|
|
114
|
+
// Build reruns[] snapshots
|
|
115
|
+
const reruns = singleRuns.map((run, i) => ({
|
|
116
|
+
rerun_index: i,
|
|
117
|
+
output: run.output,
|
|
118
|
+
started_at: run.startedAt,
|
|
119
|
+
finished_at: run.finishedAt,
|
|
120
|
+
}));
|
|
65
121
|
return {
|
|
66
122
|
test_id: result.testId,
|
|
67
123
|
test_name: result.testName,
|
|
@@ -70,9 +126,11 @@ function mapTestResult(result) {
|
|
|
70
126
|
trace_ref: result.traceRef,
|
|
71
127
|
target: result.target,
|
|
72
128
|
duration_ms: result.durationMs,
|
|
129
|
+
reruns_count: rerunsCount,
|
|
73
130
|
input: result.input,
|
|
74
131
|
output: result.output,
|
|
75
132
|
metrics,
|
|
133
|
+
reruns,
|
|
76
134
|
};
|
|
77
135
|
}
|
|
78
136
|
export async function uploadResults(payload, options) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"upload-client.js","sourceRoot":"","sources":["../../src/ci/upload-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"upload-client.js","sourceRoot":"","sources":["../../src/ci/upload-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AA4D7C,+DAA+D;AAE/D,MAAM,UAAU,kBAAkB,CAAC,SAA0B;IAC3D,MAAM,OAAO,GAAG,aAAa,EAAE,CAAA;IAE/B,OAAO;QACL,cAAc,EAAE,CAAC;QACjB,GAAG,EAAE;YACH,MAAM,EAAE,SAAS,CAAC,KAAK;YACvB,UAAU,EAAE,SAAS,CAAC,SAAS;YAC/B,WAAW,EAAE,SAAS,CAAC,UAAU;YACjC,WAAW,EAAE,SAAS,CAAC,UAAU;YACjC,GAAG,EAAE;gBACH,UAAU,EAAE,OAAO,CAAC,MAAM,IAAI,SAAS;gBACvC,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,SAAS;gBACnC,WAAW,EAAE,OAAO,CAAC,UAAU;gBAC/B,SAAS,EAAE,OAAO,CAAC,QAAQ;gBAC3B,IAAI,EAAE,OAAO,CAAC,IAAI;aACnB;YACD,EAAE,EAAE;gBACF,QAAQ,EAAE,OAAO,CAAC,UAAU;gBAC5B,OAAO,EAAE,OAAO,CAAC,QAAQ;aAC1B;SACF;QACD,OAAO,EAAE,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;KACtD,CAAA;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,GAAsB,EAAE,eAAwC;IAC3F,MAAM,OAAO,GAAmB,EAAE,CAAA;IAClC,MAAM,EAAE,GAAG,GAAG,CAAC,eAAe,IAAI,eAAe,CAAA;IAEjD,IAAI,EAAE,EAAE,CAAC;QACP,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;YAC3B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,SAAS,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE;aACnD,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;QACpB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,aAAa,CAAC,EAAE,CAAC;YACjD,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,GAAG,CAAC,WAAW,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QACzF,CAAC;QACD,IAAI,GAAG,CAAC,WAAW,CAAC,YAAY,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,cAAc,CAAC,EAAE,CAAC;YAChG,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,GAAG,CAAC,WAAW,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QAC3F,CAAC;QACD,IAAI,GAAG,CAAC,WAAW,CAAC,aAAa,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,eAAe,CAAC,EAAE,CAAC;YAClG,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,CAAC,WAAW,CAAC,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QAC7F,CAAC;QACD,IAAI,GAAG,CAAC,WAAW,CAAC,YAAY,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,cAAc,CAAC,EAAE,CAAC;YAChG,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,GAAG,CAAC,WAAW,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QAC3F,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,SAAS,aAAa,CAAC,MAAoB;IACzC,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,EAAE,CAAA;IAC1C,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,IAAI,CAAC,CAAA;IAE1C,6DAA6D;IAC7D,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QACrB,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;QACzB,MAAM,OAAO,GAAG,GAAG;YACjB,CAAC,CAAC,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,eAAe,CAAC;YAClD,CAAC,CAAC,mBAAmB,CAAC,MAAa,EAAE,MAAM,CAAC,eAAe,CAAC,CAAA;QAE9D,OAAO;YACL,OAAO,EAAE,MAAM,CAAC,MAAM;YACtB,SAAS,EAAE,MAAM,CAAC,QAAQ;YAC1B,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,cAAc,EAAE,MAAM,CAAC,aAAa;YACpC,SAAS,EAAE,MAAM,CAAC,QAAQ;YAC1B,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,WAAW,EAAE,MAAM,CAAC,UAAU;YAC9B,YAAY,EAAE,CAAC;YACf,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,OAAO;SACR,CAAA;IACH,CAAC;IAED,6DAA6D;IAC7D,kDAAkD;IAClD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAA;IACvC,MAAM,aAAa,GAAqB,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;QAC3D,MAAM,CAAC,GAAG,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,eAAe,CAAC,CAAA;QAC1D,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;QACnD,OAAO,CAAC,CAAA;IACV,CAAC,CAAC,CAAA;IAEF,2CAA2C;IAC3C,MAAM,OAAO,GAAmB,EAAE,CAAA;IAClC,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;QACjC,MAAM,YAAY,GAA8D,EAAE,CAAA;QAClF,IAAI,SAA0D,CAAA;QAE9D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;YAC1D,IAAI,MAAM,EAAE,CAAC;gBACX,YAAY,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;gBACjF,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS;oBAAE,SAAS,GAAG,MAAM,CAAC,SAAS,CAAA;YAClE,CAAC;QACH,CAAC;QAED,0DAA0D;QAC1D,MAAM,gBAAgB,GAAG,YAAY,CAAC,CAAC,CAAC,CAAA;QACxC,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,IAA4B;YAClC,KAAK,EAAE,gBAAgB,EAAE,KAAK,IAAI,CAAC;YACnC,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;YACzC,SAAS;YACT,YAAY;SACb,CAAC,CAAA;IACJ,CAAC;IAED,2BAA2B;IAC3B,MAAM,MAAM,GAAkB,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACxD,WAAW,EAAE,CAAC;QACd,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,UAAU,EAAE,GAAG,CAAC,SAAS;QACzB,WAAW,EAAE,GAAG,CAAC,UAAU;KAC5B,CAAC,CAAC,CAAA;IAEH,OAAO;QACL,OAAO,EAAE,MAAM,CAAC,MAAM;QACtB,SAAS,EAAE,MAAM,CAAC,QAAQ;QAC1B,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,cAAc,EAAE,MAAM,CAAC,aAAa;QACpC,SAAS,EAAE,MAAM,CAAC,QAAQ;QAC1B,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,WAAW,EAAE,MAAM,CAAC,UAAU;QAC9B,YAAY,EAAE,WAAW;QACzB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,OAAO;QACP,MAAM;KACP,CAAA;AACH,CAAC;AAgBD,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAsB,EACtB,OAAsB;IAEtB,MAAM,GAAG,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,0BAA0B,CAAA;IAC9E,MAAM,OAAO,GAA2B;QACtC,cAAc,EAAE,kBAAkB;QAClC,SAAS,EAAE,OAAO,CAAC,MAAM;QACzB,kBAAkB,EAAE,UAAU,EAAE;KACjC,CAAA;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;IACpC,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,aAAa,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAA;IAC5H,MAAM,UAAU,GAAG,CAAC,CAAA;IACpB,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;IAEpC,IAAI,SAA4B,CAAA;IAEhC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,gBAAgB,EAAE,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;YAE5E,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;gBACX,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAkD,CAAA;gBAC7E,OAAO,IAAI,CAAC,MAAM,CAAA;YACpB,CAAC;YAED,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAA;YAE7C,yCAAyC;YACzC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,EAAE,CAAC;gBAClC,MAAM,IAAI,KAAK,CAAC,sBAAsB,MAAM,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;YAC5E,CAAC;YAED,cAAc;YACd,SAAS,GAAG,IAAI,KAAK,CAAC,sBAAsB,MAAM,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;QAClF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,sBAAsB,CAAC,EAAE,CAAC;gBAC3E,MAAM,GAAG,CAAA,CAAC,kBAAkB;YAC9B,CAAC;YACD,SAAS,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;QACjE,CAAC;QAED,2CAA2C;QAC3C,IAAI,OAAO,GAAG,UAAU,EAAE,CAAC;YACzB,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;QACvE,CAAC;IACH,CAAC;IAED,MAAM,SAAS,IAAI,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAA;AAC7D,CAAC;AAED,+DAA+D;AAE/D,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAAsB,EACtB,KAAa,EACb,GAAY;IAEZ,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,EAAE,YAAY,EAAE,gBAAgB,CAAC,CAAA;IACtE,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,CAAA;QACxD,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;IACtH,CAAC;IAAC,MAAM,CAAC;QACP,4BAA4B;IAC9B,CAAC;AACH,CAAC"}
|