elasticdash-test 0.1.20-alpha-18 → 0.1.20-alpha-19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ci/ed-runner.d.ts +12 -0
- package/dist/ci/ed-runner.d.ts.map +1 -1
- package/dist/ci/ed-runner.js +26 -6
- package/dist/ci/ed-runner.js.map +1 -1
- package/dist/ci/upload-client.d.ts +15 -0
- package/dist/ci/upload-client.d.ts.map +1 -1
- package/dist/ci/upload-client.js +78 -20
- package/dist/ci/upload-client.js.map +1 -1
- package/dist/index.cjs +94 -21
- package/package.json +1 -1
- package/src/ci/ed-runner.ts +42 -6
- package/src/ci/upload-client.ts +100 -19
package/dist/ci/ed-runner.d.ts
CHANGED
|
@@ -9,6 +9,16 @@ export interface EdTestRunOptions {
|
|
|
9
9
|
/** Number of times to run each test. Passes if any run succeeds. Defaults to 1. */
|
|
10
10
|
runs?: number;
|
|
11
11
|
}
|
|
12
|
+
export interface EdSingleRunResult {
|
|
13
|
+
status: 'pass' | 'fail';
|
|
14
|
+
failureReason?: string;
|
|
15
|
+
measurement?: TestMeasurement;
|
|
16
|
+
benchmarkResult?: BenchmarkResult;
|
|
17
|
+
output?: unknown;
|
|
18
|
+
durationMs: number;
|
|
19
|
+
startedAt: string;
|
|
20
|
+
finishedAt: string;
|
|
21
|
+
}
|
|
12
22
|
export interface EdTestResult {
|
|
13
23
|
testId: string;
|
|
14
24
|
testName: string;
|
|
@@ -24,6 +34,8 @@ export interface EdTestResult {
|
|
|
24
34
|
input?: unknown;
|
|
25
35
|
output?: unknown;
|
|
26
36
|
durationMs: number;
|
|
37
|
+
/** All individual run results when --runs > 1 */
|
|
38
|
+
singleRuns?: EdSingleRunResult[];
|
|
27
39
|
}
|
|
28
40
|
export interface EdTestRunResult {
|
|
29
41
|
runId: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ed-runner.d.ts","sourceRoot":"","sources":["../../src/ci/ed-runner.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAOrD,MAAM,WAAW,gBAAgB;IAC/B,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,QAAQ,CAAC,EAAE,SAAS,GAAG,MAAM,GAAG,OAAO,CAAA;IACvC,mFAAmF;IACnF,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,GAAG,MAAM,CAAA;IACvB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,WAAW,CAAC,EAAE,eAAe,CAAA;IAC7B,eAAe,CAAC,EAAE,eAAe,CAAA;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAA;IAC1C,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;
|
|
1
|
+
{"version":3,"file":"ed-runner.d.ts","sourceRoot":"","sources":["../../src/ci/ed-runner.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAOrD,MAAM,WAAW,gBAAgB;IAC/B,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,QAAQ,CAAC,EAAE,SAAS,GAAG,MAAM,GAAG,OAAO,CAAA;IACvC,mFAAmF;IACnF,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAA;IACvB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,WAAW,CAAC,EAAE,eAAe,CAAA;IAC7B,eAAe,CAAC,EAAE,eAAe,CAAA;IACjC,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,GAAG,MAAM,CAAA;IACvB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,WAAW,CAAC,EAAE,eAAe,CAAA;IAC7B,eAAe,CAAC,EAAE,eAAe,CAAA;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAA;IAC1C,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,iDAAiD;IACjD,UAAU,CAAC,EAAE,iBAAiB,EAAE,CAAA;CACjC;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,EAAE,YAAY,EAAE,CAAA;IACvB,UAAU,EAAE,MAAM,CAAA;CACnB;AAID,wBAAsB,UAAU,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC,CAsFrF"}
|
package/dist/ci/ed-runner.js
CHANGED
|
@@ -19,6 +19,7 @@ export async function runEdTests(options) {
|
|
|
19
19
|
status: 'fail',
|
|
20
20
|
failureReason: `validation error: ${err.message}`,
|
|
21
21
|
durationMs: 0,
|
|
22
|
+
singleRuns: [],
|
|
22
23
|
});
|
|
23
24
|
}
|
|
24
25
|
// Filter tests if pattern provided
|
|
@@ -29,22 +30,41 @@ export async function runEdTests(options) {
|
|
|
29
30
|
}
|
|
30
31
|
const maxRuns = Math.max(1, options?.runs ?? 1);
|
|
31
32
|
for (const test of testsToRun) {
|
|
33
|
+
const allRuns = [];
|
|
32
34
|
let bestResult = null;
|
|
33
35
|
for (let attempt = 1; attempt <= maxRuns; attempt++) {
|
|
36
|
+
const runStartedAt = new Date().toISOString();
|
|
34
37
|
const result = await runSingleTest(test);
|
|
38
|
+
const runFinishedAt = new Date().toISOString();
|
|
35
39
|
if (attempt > 1) {
|
|
36
40
|
console.log(` [ed-test] ${test.name}: run ${attempt}/${maxRuns} — ${result.status}`);
|
|
37
41
|
}
|
|
38
|
-
//
|
|
42
|
+
// Collect every run for upload
|
|
43
|
+
allRuns.push({
|
|
44
|
+
status: result.status,
|
|
45
|
+
failureReason: result.failureReason,
|
|
46
|
+
measurement: result.measurement,
|
|
47
|
+
benchmarkResult: result.benchmarkResult,
|
|
48
|
+
output: result.output,
|
|
49
|
+
durationMs: result.durationMs,
|
|
50
|
+
startedAt: runStartedAt,
|
|
51
|
+
finishedAt: runFinishedAt,
|
|
52
|
+
});
|
|
53
|
+
// Keep the first passing result, or the last failure (for aggregate status)
|
|
39
54
|
if (!bestResult || result.status === 'pass') {
|
|
40
55
|
bestResult = result;
|
|
41
56
|
}
|
|
42
|
-
// Stop retrying on pass
|
|
43
|
-
if (result.status === 'pass')
|
|
44
|
-
break;
|
|
45
57
|
}
|
|
46
|
-
|
|
47
|
-
|
|
58
|
+
// Aggregate: fail if ANY run failed
|
|
59
|
+
const anyFailed = allRuns.some(r => r.status === 'fail');
|
|
60
|
+
const failedRun = allRuns.find(r => r.status === 'fail');
|
|
61
|
+
results.push({
|
|
62
|
+
...bestResult,
|
|
63
|
+
status: anyFailed ? 'fail' : 'pass',
|
|
64
|
+
failureReason: anyFailed ? (failedRun?.failureReason || bestResult.failureReason) : undefined,
|
|
65
|
+
singleRuns: allRuns,
|
|
66
|
+
});
|
|
67
|
+
if (options?.failFast && anyFailed) {
|
|
48
68
|
break;
|
|
49
69
|
}
|
|
50
70
|
}
|
package/dist/ci/ed-runner.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ed-runner.js","sourceRoot":"","sources":["../../src/ci/ed-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC5C,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAClG,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;
|
|
1
|
+
{"version":3,"file":"ed-runner.js","sourceRoot":"","sources":["../../src/ci/ed-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC5C,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAClG,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AAsDlD,+DAA+D;AAE/D,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAA0B;IACzD,MAAM,GAAG,GAAG,OAAO,EAAE,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAA;IACzC,MAAM,KAAK,GAAG,UAAU,EAAE,CAAA;IAC1B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAC1C,MAAM,OAAO,GAAmB,EAAE,CAAA;IAElC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAA;IAElD,2CAA2C;IAC3C,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC;YACX,MAAM,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS;YACjC,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS;YACnC,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,qBAAqB,GAAG,CAAC,OAAO,EAAE;YACjD,UAAU,EAAE,CAAC;YACb,UAAU,EAAE,EAAE;SACf,CAAC,CAAA;IACJ,CAAC;IAED,mCAAmC;IACnC,IAAI,UAAU,GAAoB,KAAK,CAAA;IACvC,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QACpB,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAA;QAC9B,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAA;IAC5D,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,IAAI,CAAC,CAAC,CAAA;IAE/C,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAwB,EAAE,CAAA;QACvC,IAAI,UAAU,GAAwB,IAAI,CAAA;QAE1C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YACpD,MAAM,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YAC7C,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAA;YACxC,MAAM,aAAa,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YAE9C,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,SAAS,OAAO,IAAI,OAAO,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;YACvF,CAAC;YAED,+BAA+B;YAC/B,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,eAAe,EAAE,MAAM,CAAC,eAAe;gBACvC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,SAAS,EAAE,YAAY;gBACvB,UAAU,EAAE,aAAa;aAC1B,CAAC,CAAA;YAEF,4EAA4E;YAC5E,IAAI,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;gBAC5C,UAAU,GAAG,MAAM,CAAA;YACrB,CAAC;QAEP,CAAC;QAEG,oCAAoC;QACpC,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAA;QACxD,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAA;QAExD,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,UAAW;YACd,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YACnC,aAAa,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,aAAa,IAAI,UAAW,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS;YAC9F,UAAU,EAAE,OAAO;SACpB,CAAC,CAAA;QAEF,IAAI,OAAO,EAAE,QAAQ,IAAI,SAAS,EAAE,CAAC;YACnC,MAAK;QACP,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAE3C,OAAO;QACL,KAAK;QACL,SAAS;QACT,UAAU;QACV,OAAO;QACP,UAAU,EAAE,WAAW;KACxB,CAAA;AACH,CAAC;AAED,+DAA+D;AAE/D,KAAK,UAAU,aAAa,CAAC,IAAmB;IAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC1B,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACpF,MAAM,IAAI,GAA0B;QAClC,MAAM,EAAE,IAAI,CAAC,IAAI;QACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;QACnB,QAAQ,EAAE,IAAI,CAAC,KAAK;QACpB,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;QAChE,KAAK,EAAE,UAAU,EAAE,KAAK;QACxB,MAAM,EAAE,UAAU,EAAE,MAAM;KAC3B,CAAA;IAED,4BAA4B;IAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,OAAO,IAAI,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;QAChD,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,0BAA0B;YACzC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAA;IACH,CAAC;IAED,MAAM,SAAS,GAAG,mBAAmB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAC1E,aAAa,CAAC,SAAS,CAAC,CAAA;IAExB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,IAAI,KAAK,CAAA;QAE1C,MAAM,OAAO,CAAC,IAAI,CAAC;YACjB,IAAI,CAAC,GAAG,EAAE;YACV,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAC/B,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CACjE;SACF,CAAC,CAAA;QAEF,4CAA4C;QAC5C,gEAAgE;QAChE,sEAAsE;QACtE,qEAAqE;QACrE,sEAAsE;QACtE,uEAAuE;QACvE,2CAA2C;QAC3C,IAAI,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAC/C,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,WAAW,GAAG,2BAA2B,CAAC,IAAI,CAAC,IAAI,IAAI,CAAA;YACvD,IAAI,WAAW,EAAE,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,yDAAyD,CAAC,CAAA;YAChG,CAAC;QACH,CAAC;QACD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,gBAAgB,IAAI,CAAC,MAAM,CAAC,OAAO,qCAAqC;gBACvF,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QAED,6BAA6B;QAC7B,MAAM,eAAe,GAAG,iBAAiB,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC,CAAA;QAEvE,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAChD,aAAa,EAAE,eAAe,CAAC,cAAc;YAC7C,WAAW;YACX,eAAe;YACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAA;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,4EAA4E;QAC5E,4EAA4E;QAC5E,iEAAiE;QACjE,MAAM,gBAAgB,GAAG,2BAA2B,CAAC,IAAI,CAAC,CAAA;QAC1D,IAAI,gBAAgB,EAAE,CAAC;YACrB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,mBAAmB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAA;YAC7I,MAAM,eAAe,GAAG,iBAAiB,CAAC,gBAAgB,EAAE,IAAI,CAAC,UAAU,CAAC,CAAA;YAC5E,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;gBAChD,aAAa,EAAE,eAAe,CAAC,cAAc;gBAC7C,WAAW,EAAE,gBAAgB;gBAC7B,eAAe;gBACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QAED,IAAI,GAAG,YAAY,eAAe,EAAE,CAAC;YACnC,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,gBAAgB,GAAG,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,EAAE;gBAC9D,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QACD,IAAI,GAAG,YAAY,YAAY,EAAE,CAAC;YAChC,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,wBAAwB,GAAG,CAAC,SAAS,IAAI;gBACxD,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QACD,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,oBAAoB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YACrF,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAA;IACH,CAAC;YAAS,CAAC;QACT,eAAe,EAAE,CAAA;IACnB,CAAC;AACH,CAAC;AAED,+DAA+D;AAE/D;;;;;;GAMG;AACH,SAAS,2BAA2B,CAAC,IAAmB;IACtD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAC9E,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAA;IAE3B,MAAM,MAAM,GAAoB;QAC9B,WAAW,EAAE,IAAI,CAAC,WAAW;KAC9B,CAAA;IAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAA;QACvC,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAA;QACzC,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAA;IACzC,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED,+DAA+D;AAE/D,MAAM,YAAa,SAAQ,KAAK;IACX;IAAnB,YAAmB,SAAiB;QAClC,KAAK,CAAC,wBAAwB,SAAS,IAAI,CAAC,CAAA;QAD3B,cAAS,GAAT,SAAS,CAAQ;QAElC,IAAI,CAAC,IAAI,GAAG,cAAc,CAAA;IAC5B,CAAC;CACF;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,OAAe;IAC9C,0CAA0C;IAC1C,MAAM,KAAK,GAAG,IAAI,MAAM,CACtB,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,GAAG,CAClG,CAAA;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACzB,CAAC"}
|
|
@@ -31,9 +31,11 @@ export interface UploadTestResult {
|
|
|
31
31
|
step_id: string;
|
|
32
32
|
};
|
|
33
33
|
duration_ms: number;
|
|
34
|
+
reruns_count: number;
|
|
34
35
|
input?: unknown;
|
|
35
36
|
output?: unknown;
|
|
36
37
|
metrics: UploadMetric[];
|
|
38
|
+
reruns?: UploadRerun[];
|
|
37
39
|
}
|
|
38
40
|
export interface UploadMetric {
|
|
39
41
|
name: 'duration_ms' | 'tokens_total' | 'tokens_input' | 'tokens_output';
|
|
@@ -43,6 +45,19 @@ export interface UploadMetric {
|
|
|
43
45
|
type: string;
|
|
44
46
|
threshold: number;
|
|
45
47
|
};
|
|
48
|
+
/** Per-rerun measurements (when reruns > 1) */
|
|
49
|
+
measurements?: {
|
|
50
|
+
rerun_index: number;
|
|
51
|
+
value: number;
|
|
52
|
+
passed: boolean;
|
|
53
|
+
}[];
|
|
54
|
+
}
|
|
55
|
+
export interface UploadRerun {
|
|
56
|
+
rerun_index: number;
|
|
57
|
+
input?: unknown;
|
|
58
|
+
output?: unknown;
|
|
59
|
+
started_at?: string;
|
|
60
|
+
finished_at?: string;
|
|
46
61
|
}
|
|
47
62
|
export declare function buildUploadPayload(runResult: EdTestRunResult): UploadPayload;
|
|
48
63
|
export interface UploadOptions {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"upload-client.d.ts","sourceRoot":"","sources":["../../src/ci/upload-client.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,eAAe,
|
|
1
|
+
{"version":3,"file":"upload-client.d.ts","sourceRoot":"","sources":["../../src/ci/upload-client.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,eAAe,EAAmC,MAAM,gBAAgB,CAAA;AAKtF,MAAM,WAAW,aAAa;IAC5B,cAAc,EAAE,CAAC,CAAA;IACjB,GAAG,EAAE;QACH,MAAM,EAAE,MAAM,CAAA;QACd,UAAU,EAAE,MAAM,CAAA;QAClB,WAAW,EAAE,MAAM,CAAA;QACnB,WAAW,EAAE,MAAM,CAAA;QACnB,GAAG,EAAE;YACH,UAAU,EAAE,MAAM,CAAA;YAClB,MAAM,EAAE,MAAM,CAAA;YACd,WAAW,CAAC,EAAE,MAAM,CAAA;YACpB,SAAS,CAAC,EAAE,MAAM,CAAA;YAClB,IAAI,CAAC,EAAE,MAAM,CAAA;SACd,CAAA;QACD,EAAE,EAAE;YACF,QAAQ,CAAC,EAAE,MAAM,CAAA;YACjB,OAAO,CAAC,EAAE,MAAM,CAAA;SACjB,CAAA;KACF,CAAA;IACD,OAAO,EAAE,gBAAgB,EAAE,CAAA;CAC5B;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,GAAG,MAAM,CAAA;IACvB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAA;IAC1C,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,OAAO,EAAE,YAAY,EAAE,CAAA;IACvB,MAAM,CAAC,EAAE,WAAW,EAAE,CAAA;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,aAAa,GAAG,cAAc,GAAG,cAAc,GAAG,eAAe,CAAA;IACvE,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,OAAO,CAAA;IACf,SAAS,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAA;IAC/C,+CAA+C;IAC/C,YAAY,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,EAAE,CAAA;CACzE;AAED,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAID,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,eAAe,GAAG,aAAa,CAwB5E;AAyHD,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,cAAc;IAE7B,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,UAAU,GAAG,kBAAkB,CAAA;IACvC,YAAY,EAAE,MAAM,CAAA;CACrB;AAED,wBAAsB,aAAa,CACjC,OAAO,EAAE,aAAa,EACtB,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,cAAc,CAAC,CAgDzB;AAID,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,aAAa,EACtB,KAAK,EAAE,MAAM,EACb,GAAG,CAAC,EAAE,MAAM,GACX,OAAO,CAAC,IAAI,CAAC,CASf"}
|
package/dist/ci/upload-client.js
CHANGED
|
@@ -28,10 +28,11 @@ export function buildUploadPayload(runResult) {
|
|
|
28
28
|
results: runResult.results.map(r => mapTestResult(r)),
|
|
29
29
|
};
|
|
30
30
|
}
|
|
31
|
-
function
|
|
31
|
+
function buildMetricsFromRun(run, benchmarkResult) {
|
|
32
32
|
const metrics = [];
|
|
33
|
-
|
|
34
|
-
|
|
33
|
+
const br = run.benchmarkResult ?? benchmarkResult;
|
|
34
|
+
if (br) {
|
|
35
|
+
for (const m of br.metrics) {
|
|
35
36
|
metrics.push({
|
|
36
37
|
name: m.name,
|
|
37
38
|
value: m.value,
|
|
@@ -40,28 +41,83 @@ function mapTestResult(result) {
|
|
|
40
41
|
});
|
|
41
42
|
}
|
|
42
43
|
}
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
44
|
+
if (run.measurement) {
|
|
45
|
+
if (!metrics.some(m => m.name === 'duration_ms')) {
|
|
46
|
+
metrics.push({ name: 'duration_ms', value: run.measurement.duration_ms, passed: true });
|
|
47
|
+
}
|
|
48
|
+
if (run.measurement.tokens_input !== undefined && !metrics.some(m => m.name === 'tokens_input')) {
|
|
49
|
+
metrics.push({ name: 'tokens_input', value: run.measurement.tokens_input, passed: true });
|
|
48
50
|
}
|
|
49
|
-
if (
|
|
50
|
-
|
|
51
|
-
if (!has)
|
|
52
|
-
metrics.push({ name: 'tokens_input', value: result.measurement.tokens_input, passed: true });
|
|
51
|
+
if (run.measurement.tokens_output !== undefined && !metrics.some(m => m.name === 'tokens_output')) {
|
|
52
|
+
metrics.push({ name: 'tokens_output', value: run.measurement.tokens_output, passed: true });
|
|
53
53
|
}
|
|
54
|
-
if (
|
|
55
|
-
|
|
56
|
-
if (!has)
|
|
57
|
-
metrics.push({ name: 'tokens_output', value: result.measurement.tokens_output, passed: true });
|
|
54
|
+
if (run.measurement.tokens_total !== undefined && !metrics.some(m => m.name === 'tokens_total')) {
|
|
55
|
+
metrics.push({ name: 'tokens_total', value: run.measurement.tokens_total, passed: true });
|
|
58
56
|
}
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
57
|
+
}
|
|
58
|
+
return metrics;
|
|
59
|
+
}
|
|
60
|
+
function mapTestResult(result) {
|
|
61
|
+
const singleRuns = result.singleRuns || [];
|
|
62
|
+
const rerunsCount = singleRuns.length || 1;
|
|
63
|
+
// Single run (or fallback) — flat metrics, no per-rerun data
|
|
64
|
+
if (rerunsCount <= 1) {
|
|
65
|
+
const run = singleRuns[0];
|
|
66
|
+
const metrics = run
|
|
67
|
+
? buildMetricsFromRun(run, result.benchmarkResult)
|
|
68
|
+
: buildMetricsFromRun(result, result.benchmarkResult);
|
|
69
|
+
return {
|
|
70
|
+
test_id: result.testId,
|
|
71
|
+
test_name: result.testName,
|
|
72
|
+
status: result.status,
|
|
73
|
+
failure_reason: result.failureReason,
|
|
74
|
+
trace_ref: result.traceRef,
|
|
75
|
+
target: result.target,
|
|
76
|
+
duration_ms: result.durationMs,
|
|
77
|
+
reruns_count: 1,
|
|
78
|
+
input: result.input,
|
|
79
|
+
output: result.output,
|
|
80
|
+
metrics,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
// Multiple runs — use measurements[] format with rerun_index
|
|
84
|
+
// Collect all unique metric names across all runs
|
|
85
|
+
const metricNameSet = new Set();
|
|
86
|
+
const perRunMetrics = singleRuns.map(run => {
|
|
87
|
+
const m = buildMetricsFromRun(run, result.benchmarkResult);
|
|
88
|
+
m.forEach(metric => metricNameSet.add(metric.name));
|
|
89
|
+
return m;
|
|
90
|
+
});
|
|
91
|
+
// Build metrics with measurements[] arrays
|
|
92
|
+
const metrics = [];
|
|
93
|
+
for (const name of metricNameSet) {
|
|
94
|
+
const measurements = [];
|
|
95
|
+
let benchmark;
|
|
96
|
+
for (let i = 0; i < singleRuns.length; i++) {
|
|
97
|
+
const metric = perRunMetrics[i].find(m => m.name === name);
|
|
98
|
+
if (metric) {
|
|
99
|
+
measurements.push({ rerun_index: i, value: metric.value, passed: metric.passed });
|
|
100
|
+
if (!benchmark && metric.benchmark)
|
|
101
|
+
benchmark = metric.benchmark;
|
|
102
|
+
}
|
|
63
103
|
}
|
|
104
|
+
// Aggregate: use first run's value as the top-level value
|
|
105
|
+
const firstMeasurement = measurements[0];
|
|
106
|
+
metrics.push({
|
|
107
|
+
name: name,
|
|
108
|
+
value: firstMeasurement?.value ?? 0,
|
|
109
|
+
passed: measurements.every(m => m.passed),
|
|
110
|
+
benchmark,
|
|
111
|
+
measurements,
|
|
112
|
+
});
|
|
64
113
|
}
|
|
114
|
+
// Build reruns[] snapshots
|
|
115
|
+
const reruns = singleRuns.map((run, i) => ({
|
|
116
|
+
rerun_index: i,
|
|
117
|
+
output: run.output,
|
|
118
|
+
started_at: run.startedAt,
|
|
119
|
+
finished_at: run.finishedAt,
|
|
120
|
+
}));
|
|
65
121
|
return {
|
|
66
122
|
test_id: result.testId,
|
|
67
123
|
test_name: result.testName,
|
|
@@ -70,9 +126,11 @@ function mapTestResult(result) {
|
|
|
70
126
|
trace_ref: result.traceRef,
|
|
71
127
|
target: result.target,
|
|
72
128
|
duration_ms: result.durationMs,
|
|
129
|
+
reruns_count: rerunsCount,
|
|
73
130
|
input: result.input,
|
|
74
131
|
output: result.output,
|
|
75
132
|
metrics,
|
|
133
|
+
reruns,
|
|
76
134
|
};
|
|
77
135
|
}
|
|
78
136
|
export async function uploadResults(payload, options) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"upload-client.js","sourceRoot":"","sources":["../../src/ci/upload-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"upload-client.js","sourceRoot":"","sources":["../../src/ci/upload-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AA4D7C,+DAA+D;AAE/D,MAAM,UAAU,kBAAkB,CAAC,SAA0B;IAC3D,MAAM,OAAO,GAAG,aAAa,EAAE,CAAA;IAE/B,OAAO;QACL,cAAc,EAAE,CAAC;QACjB,GAAG,EAAE;YACH,MAAM,EAAE,SAAS,CAAC,KAAK;YACvB,UAAU,EAAE,SAAS,CAAC,SAAS;YAC/B,WAAW,EAAE,SAAS,CAAC,UAAU;YACjC,WAAW,EAAE,SAAS,CAAC,UAAU;YACjC,GAAG,EAAE;gBACH,UAAU,EAAE,OAAO,CAAC,MAAM,IAAI,SAAS;gBACvC,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,SAAS;gBACnC,WAAW,EAAE,OAAO,CAAC,UAAU;gBAC/B,SAAS,EAAE,OAAO,CAAC,QAAQ;gBAC3B,IAAI,EAAE,OAAO,CAAC,IAAI;aACnB;YACD,EAAE,EAAE;gBACF,QAAQ,EAAE,OAAO,CAAC,UAAU;gBAC5B,OAAO,EAAE,OAAO,CAAC,QAAQ;aAC1B;SACF;QACD,OAAO,EAAE,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;KACtD,CAAA;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,GAAsB,EAAE,eAAwC;IAC3F,MAAM,OAAO,GAAmB,EAAE,CAAA;IAClC,MAAM,EAAE,GAAG,GAAG,CAAC,eAAe,IAAI,eAAe,CAAA;IAEjD,IAAI,EAAE,EAAE,CAAC;QACP,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;YAC3B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,SAAS,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE;aACnD,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;QACpB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,aAAa,CAAC,EAAE,CAAC;YACjD,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,GAAG,CAAC,WAAW,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QACzF,CAAC;QACD,IAAI,GAAG,CAAC,WAAW,CAAC,YAAY,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,cAAc,CAAC,EAAE,CAAC;YAChG,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,GAAG,CAAC,WAAW,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QAC3F,CAAC;QACD,IAAI,GAAG,CAAC,WAAW,CAAC,aAAa,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,eAAe,CAAC,EAAE,CAAC;YAClG,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,CAAC,WAAW,CAAC,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QAC7F,CAAC;QACD,IAAI,GAAG,CAAC,WAAW,CAAC,YAAY,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,cAAc,CAAC,EAAE,CAAC;YAChG,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,GAAG,CAAC,WAAW,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QAC3F,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,SAAS,aAAa,CAAC,MAAoB;IACzC,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,EAAE,CAAA;IAC1C,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,IAAI,CAAC,CAAA;IAE1C,6DAA6D;IAC7D,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QACrB,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;QACzB,MAAM,OAAO,GAAG,GAAG;YACjB,CAAC,CAAC,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,eAAe,CAAC;YAClD,CAAC,CAAC,mBAAmB,CAAC,MAAa,EAAE,MAAM,CAAC,eAAe,CAAC,CAAA;QAE9D,OAAO;YACL,OAAO,EAAE,MAAM,CAAC,MAAM;YACtB,SAAS,EAAE,MAAM,CAAC,QAAQ;YAC1B,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,cAAc,EAAE,MAAM,CAAC,aAAa;YACpC,SAAS,EAAE,MAAM,CAAC,QAAQ;YAC1B,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,WAAW,EAAE,MAAM,CAAC,UAAU;YAC9B,YAAY,EAAE,CAAC;YACf,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,OAAO;SACR,CAAA;IACH,CAAC;IAED,6DAA6D;IAC7D,kDAAkD;IAClD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAA;IACvC,MAAM,aAAa,GAAqB,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;QAC3D,MAAM,CAAC,GAAG,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,eAAe,CAAC,CAAA;QAC1D,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;QACnD,OAAO,CAAC,CAAA;IACV,CAAC,CAAC,CAAA;IAEF,2CAA2C;IAC3C,MAAM,OAAO,GAAmB,EAAE,CAAA;IAClC,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;QACjC,MAAM,YAAY,GAA8D,EAAE,CAAA;QAClF,IAAI,SAA0D,CAAA;QAE9D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;YAC1D,IAAI,MAAM,EAAE,CAAC;gBACX,YAAY,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;gBACjF,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS;oBAAE,SAAS,GAAG,MAAM,CAAC,SAAS,CAAA;YAClE,CAAC;QACH,CAAC;QAED,0DAA0D;QAC1D,MAAM,gBAAgB,GAAG,YAAY,CAAC,CAAC,CAAC,CAAA;QACxC,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,IAA4B;YAClC,KAAK,EAAE,gBAAgB,EAAE,KAAK,IAAI,CAAC;YACnC,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;YACzC,SAAS;YACT,YAAY;SACb,CAAC,CAAA;IACJ,CAAC;IAED,2BAA2B;IAC3B,MAAM,MAAM,GAAkB,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACxD,WAAW,EAAE,CAAC;QACd,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,UAAU,EAAE,GAAG,CAAC,SAAS;QACzB,WAAW,EAAE,GAAG,CAAC,UAAU;KAC5B,CAAC,CAAC,CAAA;IAEH,OAAO;QACL,OAAO,EAAE,MAAM,CAAC,MAAM;QACtB,SAAS,EAAE,MAAM,CAAC,QAAQ;QAC1B,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,cAAc,EAAE,MAAM,CAAC,aAAa;QACpC,SAAS,EAAE,MAAM,CAAC,QAAQ;QAC1B,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,WAAW,EAAE,MAAM,CAAC,UAAU;QAC9B,YAAY,EAAE,WAAW;QACzB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,OAAO;QACP,MAAM;KACP,CAAA;AACH,CAAC;AAgBD,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAsB,EACtB,OAAsB;IAEtB,MAAM,GAAG,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,0BAA0B,CAAA;IAC9E,MAAM,OAAO,GAA2B;QACtC,cAAc,EAAE,kBAAkB;QAClC,SAAS,EAAE,OAAO,CAAC,MAAM;QACzB,kBAAkB,EAAE,UAAU,EAAE;KACjC,CAAA;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;IACpC,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,aAAa,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAA;IAC5H,MAAM,UAAU,GAAG,CAAC,CAAA;IACpB,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;IAEpC,IAAI,SAA4B,CAAA;IAEhC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,gBAAgB,EAAE,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;YAE5E,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;gBACX,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAkD,CAAA;gBAC7E,OAAO,IAAI,CAAC,MAAM,CAAA;YACpB,CAAC;YAED,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAA;YAE7C,yCAAyC;YACzC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,EAAE,CAAC;gBAClC,MAAM,IAAI,KAAK,CAAC,sBAAsB,MAAM,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;YAC5E,CAAC;YAED,cAAc;YACd,SAAS,GAAG,IAAI,KAAK,CAAC,sBAAsB,MAAM,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;QAClF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,sBAAsB,CAAC,EAAE,CAAC;gBAC3E,MAAM,GAAG,CAAA,CAAC,kBAAkB;YAC9B,CAAC;YACD,SAAS,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;QACjE,CAAC;QAED,2CAA2C;QAC3C,IAAI,OAAO,GAAG,UAAU,EAAE,CAAC;YACzB,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;QACvE,CAAC;IACH,CAAC;IAED,MAAM,SAAS,IAAI,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAA;AAC7D,CAAC;AAED,+DAA+D;AAE/D,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAAsB,EACtB,KAAa,EACb,GAAY;IAEZ,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,EAAE,YAAY,EAAE,gBAAgB,CAAC,CAAA;IACtE,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,CAAA;QACxD,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;IACtH,CAAC;IAAC,MAAM,CAAC;QACP,4BAA4B;IAC9B,CAAC;AACH,CAAC"}
|
package/dist/index.cjs
CHANGED
|
@@ -6358,7 +6358,8 @@ async function runEdTests(options) {
|
|
|
6358
6358
|
testName: err.testName ?? "unknown",
|
|
6359
6359
|
status: "fail",
|
|
6360
6360
|
failureReason: `validation error: ${err.message}`,
|
|
6361
|
-
durationMs: 0
|
|
6361
|
+
durationMs: 0,
|
|
6362
|
+
singleRuns: []
|
|
6362
6363
|
});
|
|
6363
6364
|
}
|
|
6364
6365
|
let testsToRun = tests;
|
|
@@ -6368,19 +6369,38 @@ async function runEdTests(options) {
|
|
|
6368
6369
|
}
|
|
6369
6370
|
const maxRuns = Math.max(1, options?.runs ?? 1);
|
|
6370
6371
|
for (const test of testsToRun) {
|
|
6372
|
+
const allRuns = [];
|
|
6371
6373
|
let bestResult = null;
|
|
6372
6374
|
for (let attempt = 1; attempt <= maxRuns; attempt++) {
|
|
6375
|
+
const runStartedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
6373
6376
|
const result = await runSingleTest(test);
|
|
6377
|
+
const runFinishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
6374
6378
|
if (attempt > 1) {
|
|
6375
6379
|
console.log(` [ed-test] ${test.name}: run ${attempt}/${maxRuns} \u2014 ${result.status}`);
|
|
6376
6380
|
}
|
|
6381
|
+
allRuns.push({
|
|
6382
|
+
status: result.status,
|
|
6383
|
+
failureReason: result.failureReason,
|
|
6384
|
+
measurement: result.measurement,
|
|
6385
|
+
benchmarkResult: result.benchmarkResult,
|
|
6386
|
+
output: result.output,
|
|
6387
|
+
durationMs: result.durationMs,
|
|
6388
|
+
startedAt: runStartedAt,
|
|
6389
|
+
finishedAt: runFinishedAt
|
|
6390
|
+
});
|
|
6377
6391
|
if (!bestResult || result.status === "pass") {
|
|
6378
6392
|
bestResult = result;
|
|
6379
6393
|
}
|
|
6380
|
-
if (result.status === "pass") break;
|
|
6381
6394
|
}
|
|
6382
|
-
|
|
6383
|
-
|
|
6395
|
+
const anyFailed = allRuns.some((r) => r.status === "fail");
|
|
6396
|
+
const failedRun = allRuns.find((r) => r.status === "fail");
|
|
6397
|
+
results.push({
|
|
6398
|
+
...bestResult,
|
|
6399
|
+
status: anyFailed ? "fail" : "pass",
|
|
6400
|
+
failureReason: anyFailed ? failedRun?.failureReason || bestResult.failureReason : void 0,
|
|
6401
|
+
singleRuns: allRuns
|
|
6402
|
+
});
|
|
6403
|
+
if (options?.failFast && anyFailed) {
|
|
6384
6404
|
break;
|
|
6385
6405
|
}
|
|
6386
6406
|
}
|
|
@@ -6555,10 +6575,11 @@ function buildUploadPayload(runResult) {
|
|
|
6555
6575
|
results: runResult.results.map((r) => mapTestResult(r))
|
|
6556
6576
|
};
|
|
6557
6577
|
}
|
|
6558
|
-
function
|
|
6578
|
+
function buildMetricsFromRun(run, benchmarkResult) {
|
|
6559
6579
|
const metrics = [];
|
|
6560
|
-
|
|
6561
|
-
|
|
6580
|
+
const br = run.benchmarkResult ?? benchmarkResult;
|
|
6581
|
+
if (br) {
|
|
6582
|
+
for (const m of br.metrics) {
|
|
6562
6583
|
metrics.push({
|
|
6563
6584
|
name: m.name,
|
|
6564
6585
|
value: m.value,
|
|
@@ -6567,24 +6588,74 @@ function mapTestResult(result) {
|
|
|
6567
6588
|
});
|
|
6568
6589
|
}
|
|
6569
6590
|
}
|
|
6570
|
-
if (
|
|
6571
|
-
|
|
6572
|
-
|
|
6573
|
-
metrics.push({ name: "duration_ms", value: result.measurement.duration_ms, passed: true });
|
|
6591
|
+
if (run.measurement) {
|
|
6592
|
+
if (!metrics.some((m) => m.name === "duration_ms")) {
|
|
6593
|
+
metrics.push({ name: "duration_ms", value: run.measurement.duration_ms, passed: true });
|
|
6574
6594
|
}
|
|
6575
|
-
if (
|
|
6576
|
-
|
|
6577
|
-
if (!has) metrics.push({ name: "tokens_input", value: result.measurement.tokens_input, passed: true });
|
|
6595
|
+
if (run.measurement.tokens_input !== void 0 && !metrics.some((m) => m.name === "tokens_input")) {
|
|
6596
|
+
metrics.push({ name: "tokens_input", value: run.measurement.tokens_input, passed: true });
|
|
6578
6597
|
}
|
|
6579
|
-
if (
|
|
6580
|
-
|
|
6581
|
-
if (!has) metrics.push({ name: "tokens_output", value: result.measurement.tokens_output, passed: true });
|
|
6598
|
+
if (run.measurement.tokens_output !== void 0 && !metrics.some((m) => m.name === "tokens_output")) {
|
|
6599
|
+
metrics.push({ name: "tokens_output", value: run.measurement.tokens_output, passed: true });
|
|
6582
6600
|
}
|
|
6583
|
-
if (
|
|
6584
|
-
|
|
6585
|
-
if (!has) metrics.push({ name: "tokens_total", value: result.measurement.tokens_total, passed: true });
|
|
6601
|
+
if (run.measurement.tokens_total !== void 0 && !metrics.some((m) => m.name === "tokens_total")) {
|
|
6602
|
+
metrics.push({ name: "tokens_total", value: run.measurement.tokens_total, passed: true });
|
|
6586
6603
|
}
|
|
6587
6604
|
}
|
|
6605
|
+
return metrics;
|
|
6606
|
+
}
|
|
6607
|
+
function mapTestResult(result) {
|
|
6608
|
+
const singleRuns = result.singleRuns || [];
|
|
6609
|
+
const rerunsCount = singleRuns.length || 1;
|
|
6610
|
+
if (rerunsCount <= 1) {
|
|
6611
|
+
const run = singleRuns[0];
|
|
6612
|
+
const metrics2 = run ? buildMetricsFromRun(run, result.benchmarkResult) : buildMetricsFromRun(result, result.benchmarkResult);
|
|
6613
|
+
return {
|
|
6614
|
+
test_id: result.testId,
|
|
6615
|
+
test_name: result.testName,
|
|
6616
|
+
status: result.status,
|
|
6617
|
+
failure_reason: result.failureReason,
|
|
6618
|
+
trace_ref: result.traceRef,
|
|
6619
|
+
target: result.target,
|
|
6620
|
+
duration_ms: result.durationMs,
|
|
6621
|
+
reruns_count: 1,
|
|
6622
|
+
input: result.input,
|
|
6623
|
+
output: result.output,
|
|
6624
|
+
metrics: metrics2
|
|
6625
|
+
};
|
|
6626
|
+
}
|
|
6627
|
+
const metricNameSet = /* @__PURE__ */ new Set();
|
|
6628
|
+
const perRunMetrics = singleRuns.map((run) => {
|
|
6629
|
+
const m = buildMetricsFromRun(run, result.benchmarkResult);
|
|
6630
|
+
m.forEach((metric) => metricNameSet.add(metric.name));
|
|
6631
|
+
return m;
|
|
6632
|
+
});
|
|
6633
|
+
const metrics = [];
|
|
6634
|
+
for (const name of metricNameSet) {
|
|
6635
|
+
const measurements = [];
|
|
6636
|
+
let benchmark;
|
|
6637
|
+
for (let i = 0; i < singleRuns.length; i++) {
|
|
6638
|
+
const metric = perRunMetrics[i].find((m) => m.name === name);
|
|
6639
|
+
if (metric) {
|
|
6640
|
+
measurements.push({ rerun_index: i, value: metric.value, passed: metric.passed });
|
|
6641
|
+
if (!benchmark && metric.benchmark) benchmark = metric.benchmark;
|
|
6642
|
+
}
|
|
6643
|
+
}
|
|
6644
|
+
const firstMeasurement = measurements[0];
|
|
6645
|
+
metrics.push({
|
|
6646
|
+
name,
|
|
6647
|
+
value: firstMeasurement?.value ?? 0,
|
|
6648
|
+
passed: measurements.every((m) => m.passed),
|
|
6649
|
+
benchmark,
|
|
6650
|
+
measurements
|
|
6651
|
+
});
|
|
6652
|
+
}
|
|
6653
|
+
const reruns = singleRuns.map((run, i) => ({
|
|
6654
|
+
rerun_index: i,
|
|
6655
|
+
output: run.output,
|
|
6656
|
+
started_at: run.startedAt,
|
|
6657
|
+
finished_at: run.finishedAt
|
|
6658
|
+
}));
|
|
6588
6659
|
return {
|
|
6589
6660
|
test_id: result.testId,
|
|
6590
6661
|
test_name: result.testName,
|
|
@@ -6593,9 +6664,11 @@ function mapTestResult(result) {
|
|
|
6593
6664
|
trace_ref: result.traceRef,
|
|
6594
6665
|
target: result.target,
|
|
6595
6666
|
duration_ms: result.durationMs,
|
|
6667
|
+
reruns_count: rerunsCount,
|
|
6596
6668
|
input: result.input,
|
|
6597
6669
|
output: result.output,
|
|
6598
|
-
metrics
|
|
6670
|
+
metrics,
|
|
6671
|
+
reruns
|
|
6599
6672
|
};
|
|
6600
6673
|
}
|
|
6601
6674
|
async function uploadResults(payload, options) {
|
package/package.json
CHANGED
package/src/ci/ed-runner.ts
CHANGED
|
@@ -22,6 +22,17 @@ export interface EdTestRunOptions {
|
|
|
22
22
|
runs?: number
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
+
export interface EdSingleRunResult {
|
|
26
|
+
status: 'pass' | 'fail'
|
|
27
|
+
failureReason?: string
|
|
28
|
+
measurement?: TestMeasurement
|
|
29
|
+
benchmarkResult?: BenchmarkResult
|
|
30
|
+
output?: unknown
|
|
31
|
+
durationMs: number
|
|
32
|
+
startedAt: string
|
|
33
|
+
finishedAt: string
|
|
34
|
+
}
|
|
35
|
+
|
|
25
36
|
export interface EdTestResult {
|
|
26
37
|
testId: string
|
|
27
38
|
testName: string
|
|
@@ -34,6 +45,8 @@ export interface EdTestResult {
|
|
|
34
45
|
input?: unknown
|
|
35
46
|
output?: unknown
|
|
36
47
|
durationMs: number
|
|
48
|
+
/** All individual run results when --runs > 1 */
|
|
49
|
+
singleRuns?: EdSingleRunResult[]
|
|
37
50
|
}
|
|
38
51
|
|
|
39
52
|
export interface EdTestRunResult {
|
|
@@ -62,6 +75,7 @@ export async function runEdTests(options?: EdTestRunOptions): Promise<EdTestRunR
|
|
|
62
75
|
status: 'fail',
|
|
63
76
|
failureReason: `validation error: ${err.message}`,
|
|
64
77
|
durationMs: 0,
|
|
78
|
+
singleRuns: [],
|
|
65
79
|
})
|
|
66
80
|
}
|
|
67
81
|
|
|
@@ -75,27 +89,49 @@ export async function runEdTests(options?: EdTestRunOptions): Promise<EdTestRunR
|
|
|
75
89
|
const maxRuns = Math.max(1, options?.runs ?? 1)
|
|
76
90
|
|
|
77
91
|
for (const test of testsToRun) {
|
|
92
|
+
const allRuns: EdSingleRunResult[] = []
|
|
78
93
|
let bestResult: EdTestResult | null = null
|
|
79
94
|
|
|
80
95
|
for (let attempt = 1; attempt <= maxRuns; attempt++) {
|
|
96
|
+
const runStartedAt = new Date().toISOString()
|
|
81
97
|
const result = await runSingleTest(test)
|
|
98
|
+
const runFinishedAt = new Date().toISOString()
|
|
82
99
|
|
|
83
100
|
if (attempt > 1) {
|
|
84
101
|
console.log(` [ed-test] ${test.name}: run ${attempt}/${maxRuns} — ${result.status}`)
|
|
85
102
|
}
|
|
86
103
|
|
|
87
|
-
//
|
|
104
|
+
// Collect every run for upload
|
|
105
|
+
allRuns.push({
|
|
106
|
+
status: result.status,
|
|
107
|
+
failureReason: result.failureReason,
|
|
108
|
+
measurement: result.measurement,
|
|
109
|
+
benchmarkResult: result.benchmarkResult,
|
|
110
|
+
output: result.output,
|
|
111
|
+
durationMs: result.durationMs,
|
|
112
|
+
startedAt: runStartedAt,
|
|
113
|
+
finishedAt: runFinishedAt,
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
// Keep the first passing result, or the last failure (for aggregate status)
|
|
88
117
|
if (!bestResult || result.status === 'pass') {
|
|
89
118
|
bestResult = result
|
|
90
119
|
}
|
|
91
120
|
|
|
92
|
-
|
|
93
|
-
if (result.status === 'pass') break
|
|
94
|
-
}
|
|
121
|
+
}
|
|
95
122
|
|
|
96
|
-
|
|
123
|
+
// Aggregate: fail if ANY run failed
|
|
124
|
+
const anyFailed = allRuns.some(r => r.status === 'fail')
|
|
125
|
+
const failedRun = allRuns.find(r => r.status === 'fail')
|
|
126
|
+
|
|
127
|
+
results.push({
|
|
128
|
+
...bestResult!,
|
|
129
|
+
status: anyFailed ? 'fail' : 'pass',
|
|
130
|
+
failureReason: anyFailed ? (failedRun?.failureReason || bestResult!.failureReason) : undefined,
|
|
131
|
+
singleRuns: allRuns,
|
|
132
|
+
})
|
|
97
133
|
|
|
98
|
-
if (options?.failFast &&
|
|
134
|
+
if (options?.failFast && anyFailed) {
|
|
99
135
|
break
|
|
100
136
|
}
|
|
101
137
|
}
|
package/src/ci/upload-client.ts
CHANGED
|
@@ -3,8 +3,8 @@ import { mkdir, writeFile } from 'node:fs/promises'
|
|
|
3
3
|
import { join } from 'node:path'
|
|
4
4
|
import { getOriginalFetch } from '../interceptors/http.js'
|
|
5
5
|
import { detectGitInfo } from './git-info.js'
|
|
6
|
-
import type { EdTestRunResult, EdTestResult } from './ed-runner.js'
|
|
7
|
-
import type { BenchmarkResult
|
|
6
|
+
import type { EdTestRunResult, EdTestResult, EdSingleRunResult } from './ed-runner.js'
|
|
7
|
+
import type { BenchmarkResult } from './benchmark.js'
|
|
8
8
|
|
|
9
9
|
// ─── Upload payload types (aligned with POST /api/v1/test-runs/create) ──
|
|
10
10
|
|
|
@@ -38,9 +38,11 @@ export interface UploadTestResult {
|
|
|
38
38
|
trace_ref?: string
|
|
39
39
|
target?: { type: string; step_id: string }
|
|
40
40
|
duration_ms: number
|
|
41
|
+
reruns_count: number
|
|
41
42
|
input?: unknown
|
|
42
43
|
output?: unknown
|
|
43
44
|
metrics: UploadMetric[]
|
|
45
|
+
reruns?: UploadRerun[]
|
|
44
46
|
}
|
|
45
47
|
|
|
46
48
|
export interface UploadMetric {
|
|
@@ -48,6 +50,16 @@ export interface UploadMetric {
|
|
|
48
50
|
value: number
|
|
49
51
|
passed: boolean
|
|
50
52
|
benchmark?: { type: string; threshold: number }
|
|
53
|
+
/** Per-rerun measurements (when reruns > 1) */
|
|
54
|
+
measurements?: { rerun_index: number; value: number; passed: boolean }[]
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export interface UploadRerun {
|
|
58
|
+
rerun_index: number
|
|
59
|
+
input?: unknown
|
|
60
|
+
output?: unknown
|
|
61
|
+
started_at?: string
|
|
62
|
+
finished_at?: string
|
|
51
63
|
}
|
|
52
64
|
|
|
53
65
|
// ─── Payload assembly ───────────────────────────────────────
|
|
@@ -78,11 +90,12 @@ export function buildUploadPayload(runResult: EdTestRunResult): UploadPayload {
|
|
|
78
90
|
}
|
|
79
91
|
}
|
|
80
92
|
|
|
81
|
-
function
|
|
93
|
+
function buildMetricsFromRun(run: EdSingleRunResult, benchmarkResult?: BenchmarkResult | null): UploadMetric[] {
|
|
82
94
|
const metrics: UploadMetric[] = []
|
|
95
|
+
const br = run.benchmarkResult ?? benchmarkResult
|
|
83
96
|
|
|
84
|
-
if (
|
|
85
|
-
for (const m of
|
|
97
|
+
if (br) {
|
|
98
|
+
for (const m of br.metrics) {
|
|
86
99
|
metrics.push({
|
|
87
100
|
name: m.name,
|
|
88
101
|
value: m.value,
|
|
@@ -92,26 +105,92 @@ function mapTestResult(result: EdTestResult): UploadTestResult {
|
|
|
92
105
|
}
|
|
93
106
|
}
|
|
94
107
|
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
108
|
+
if (run.measurement) {
|
|
109
|
+
if (!metrics.some(m => m.name === 'duration_ms')) {
|
|
110
|
+
metrics.push({ name: 'duration_ms', value: run.measurement.duration_ms, passed: true })
|
|
111
|
+
}
|
|
112
|
+
if (run.measurement.tokens_input !== undefined && !metrics.some(m => m.name === 'tokens_input')) {
|
|
113
|
+
metrics.push({ name: 'tokens_input', value: run.measurement.tokens_input, passed: true })
|
|
100
114
|
}
|
|
101
|
-
if (
|
|
102
|
-
|
|
103
|
-
if (!has) metrics.push({ name: 'tokens_input', value: result.measurement.tokens_input, passed: true })
|
|
115
|
+
if (run.measurement.tokens_output !== undefined && !metrics.some(m => m.name === 'tokens_output')) {
|
|
116
|
+
metrics.push({ name: 'tokens_output', value: run.measurement.tokens_output, passed: true })
|
|
104
117
|
}
|
|
105
|
-
if (
|
|
106
|
-
|
|
107
|
-
|
|
118
|
+
if (run.measurement.tokens_total !== undefined && !metrics.some(m => m.name === 'tokens_total')) {
|
|
119
|
+
metrics.push({ name: 'tokens_total', value: run.measurement.tokens_total, passed: true })
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
return metrics
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function mapTestResult(result: EdTestResult): UploadTestResult {
|
|
127
|
+
const singleRuns = result.singleRuns || []
|
|
128
|
+
const rerunsCount = singleRuns.length || 1
|
|
129
|
+
|
|
130
|
+
// Single run (or fallback) — flat metrics, no per-rerun data
|
|
131
|
+
if (rerunsCount <= 1) {
|
|
132
|
+
const run = singleRuns[0]
|
|
133
|
+
const metrics = run
|
|
134
|
+
? buildMetricsFromRun(run, result.benchmarkResult)
|
|
135
|
+
: buildMetricsFromRun(result as any, result.benchmarkResult)
|
|
136
|
+
|
|
137
|
+
return {
|
|
138
|
+
test_id: result.testId,
|
|
139
|
+
test_name: result.testName,
|
|
140
|
+
status: result.status,
|
|
141
|
+
failure_reason: result.failureReason,
|
|
142
|
+
trace_ref: result.traceRef,
|
|
143
|
+
target: result.target,
|
|
144
|
+
duration_ms: result.durationMs,
|
|
145
|
+
reruns_count: 1,
|
|
146
|
+
input: result.input,
|
|
147
|
+
output: result.output,
|
|
148
|
+
metrics,
|
|
108
149
|
}
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// Multiple runs — use measurements[] format with rerun_index
|
|
153
|
+
// Collect all unique metric names across all runs
|
|
154
|
+
const metricNameSet = new Set<string>()
|
|
155
|
+
const perRunMetrics: UploadMetric[][] = singleRuns.map(run => {
|
|
156
|
+
const m = buildMetricsFromRun(run, result.benchmarkResult)
|
|
157
|
+
m.forEach(metric => metricNameSet.add(metric.name))
|
|
158
|
+
return m
|
|
159
|
+
})
|
|
160
|
+
|
|
161
|
+
// Build metrics with measurements[] arrays
|
|
162
|
+
const metrics: UploadMetric[] = []
|
|
163
|
+
for (const name of metricNameSet) {
|
|
164
|
+
const measurements: { rerun_index: number; value: number; passed: boolean }[] = []
|
|
165
|
+
let benchmark: { type: string; threshold: number } | undefined
|
|
166
|
+
|
|
167
|
+
for (let i = 0; i < singleRuns.length; i++) {
|
|
168
|
+
const metric = perRunMetrics[i].find(m => m.name === name)
|
|
169
|
+
if (metric) {
|
|
170
|
+
measurements.push({ rerun_index: i, value: metric.value, passed: metric.passed })
|
|
171
|
+
if (!benchmark && metric.benchmark) benchmark = metric.benchmark
|
|
172
|
+
}
|
|
112
173
|
}
|
|
174
|
+
|
|
175
|
+
// Aggregate: use first run's value as the top-level value
|
|
176
|
+
const firstMeasurement = measurements[0]
|
|
177
|
+
metrics.push({
|
|
178
|
+
name: name as UploadMetric['name'],
|
|
179
|
+
value: firstMeasurement?.value ?? 0,
|
|
180
|
+
passed: measurements.every(m => m.passed),
|
|
181
|
+
benchmark,
|
|
182
|
+
measurements,
|
|
183
|
+
})
|
|
113
184
|
}
|
|
114
185
|
|
|
186
|
+
// Build reruns[] snapshots
|
|
187
|
+
const reruns: UploadRerun[] = singleRuns.map((run, i) => ({
|
|
188
|
+
rerun_index: i,
|
|
189
|
+
output: run.output,
|
|
190
|
+
started_at: run.startedAt,
|
|
191
|
+
finished_at: run.finishedAt,
|
|
192
|
+
}))
|
|
193
|
+
|
|
115
194
|
return {
|
|
116
195
|
test_id: result.testId,
|
|
117
196
|
test_name: result.testName,
|
|
@@ -120,9 +199,11 @@ function mapTestResult(result: EdTestResult): UploadTestResult {
|
|
|
120
199
|
trace_ref: result.traceRef,
|
|
121
200
|
target: result.target,
|
|
122
201
|
duration_ms: result.durationMs,
|
|
202
|
+
reruns_count: rerunsCount,
|
|
123
203
|
input: result.input,
|
|
124
204
|
output: result.output,
|
|
125
205
|
metrics,
|
|
206
|
+
reruns,
|
|
126
207
|
}
|
|
127
208
|
}
|
|
128
209
|
|