elasticdash-test 0.1.20-alpha-19 → 0.1.20-alpha-20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (17) hide show
  1. package/dist/ci/benchmark.d.ts +16 -2
  2. package/dist/ci/benchmark.d.ts.map +1 -1
  3. package/dist/ci/benchmark.js +102 -1
  4. package/dist/ci/benchmark.js.map +1 -1
  5. package/dist/ci/ed-runner.js +3 -3
  6. package/dist/ci/ed-runner.js.map +1 -1
  7. package/dist/ci/test-registry.d.ts +16 -0
  8. package/dist/ci/test-registry.d.ts.map +1 -1
  9. package/dist/ci/test-registry.js.map +1 -1
  10. package/dist/ci/upload-client.d.ts +1 -1
  11. package/dist/ci/upload-client.d.ts.map +1 -1
  12. package/dist/index.cjs +95 -3
  13. package/package.json +1 -1
  14. package/src/ci/benchmark.ts +118 -3
  15. package/src/ci/ed-runner.ts +3 -3
  16. package/src/ci/test-registry.ts +17 -0
  17. package/src/ci/upload-client.ts +1 -1
package/dist/ci/benchmark.d.ts CHANGED
@@ -1,15 +1,29 @@
1
+ /**
2
+ * benchmark.ts
3
+ *
4
+ * Compares recorded trace measurements against user-defined benchmarks.
5
+ * Supports duration, token, output-contains, and LLM-as-a-judge evaluations.
6
+ *
7
+ * Generated/updated on 2026-04-20.
8
+ */
1
9
  import type { TestMeasurement } from './measurement.js';
2
10
  import type { TestBenchmarks } from './test-registry.js';
11
+ export type MetricName = 'duration_ms' | 'tokens_total' | 'output_contains' | 'output_not_contains' | 'llm_judge';
3
12
  export interface MetricResult {
4
- name: 'duration_ms' | 'tokens_total';
13
+ name: MetricName;
5
14
  value: number;
6
15
  threshold: number;
7
16
  passed: boolean;
17
+ detail?: string;
8
18
  }
9
19
  export interface BenchmarkResult {
10
20
  passed: boolean;
11
21
  failure_reason?: string;
12
22
  metrics: MetricResult[];
13
23
  }
14
- export declare function compareBenchmarks(measurement: TestMeasurement, benchmarks: TestBenchmarks): BenchmarkResult;
24
+ /**
25
+ * Compare a measurement against benchmarks. Async because llm_judge requires
26
+ * an LLM call. The step's output is needed for output_contains/llm_judge checks.
27
+ */
28
+ export declare function compareBenchmarks(measurement: TestMeasurement, benchmarks: TestBenchmarks, stepOutput?: unknown): Promise<BenchmarkResult>;
15
29
  //# sourceMappingURL=benchmark.d.ts.map
package/dist/ci/benchmark.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"benchmark.d.ts","sourceRoot":"","sources":["../../src/ci/benchmark.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AACvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AAExD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,aAAa,GAAG,cAAc,CAAA;IACpC,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,OAAO,CAAA;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,OAAO,CAAA;IACf,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,OAAO,EAAE,YAAY,EAAE,CAAA;CACxB;AAED,wBAAgB,iBAAiB,CAC/B,WAAW,EAAE,eAAe,EAC5B,UAAU,EAAE,cAAc,GACzB,eAAe,CAqCjB"}
1
+ {"version":3,"file":"benchmark.d.ts","sourceRoot":"","sources":["../../src/ci/benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AACvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AAExD,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,cAAc,GAAG,iBAAiB,GAAG,qBAAqB,GAAG,WAAW,CAAA;AAEjH,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,UAAU,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,OAAO,CAAA;IACf,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,OAAO,EAAE,YAAY,EAAE,CAAA;CACxB;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CACrC,WAAW,EAAE,eAAe,EAC5B,UAAU,EAAE,cAAc,EAC1B,UAAU,CAAC,EAAE,OAAO,GACnB,OAAO,CAAC,eAAe,CAAC,CA+H1B"}
package/dist/ci/benchmark.js CHANGED
@@ -1,4 +1,17 @@
1
- export function compareBenchmarks(measurement, benchmarks) {
1
+ /**
2
+ * benchmark.ts
3
+ *
4
+ * Compares recorded trace measurements against user-defined benchmarks.
5
+ * Supports duration, token, output-contains, and LLM-as-a-judge evaluations.
6
+ *
7
+ * Generated/updated on 2026-04-20.
8
+ */
9
+ import { callProviderLLM } from '../matchers/index.js';
10
+ /**
11
+ * Compare a measurement against benchmarks. Async because llm_judge requires
12
+ * an LLM call. The step's output is needed for output_contains/llm_judge checks.
13
+ */
14
+ export async function compareBenchmarks(measurement, benchmarks, stepOutput) {
2
15
  const metrics = [];
3
16
  let firstFailure;
4
17
  if (benchmarks.max_duration_ms !== undefined) {
@@ -26,6 +39,86 @@ export function compareBenchmarks(measurement, benchmarks) {
26
39
  firstFailure = `tokens_total (${value}) exceeded max threshold (${benchmarks.max_tokens_total})`;
27
40
  }
28
41
  }
42
+ // ─── Output contains ────────────────────────────────────────
43
+ if (benchmarks.output_contains !== undefined) {
44
+ const outputStr = stringifyOutput(stepOutput);
45
+ const passed = outputStr.toLowerCase().includes(benchmarks.output_contains.toLowerCase());
46
+ metrics.push({
47
+ name: 'output_contains',
48
+ value: passed ? 1 : 0,
49
+ threshold: 1,
50
+ passed,
51
+ detail: passed ? undefined : `output does not contain "${benchmarks.output_contains}"`,
52
+ });
53
+ if (!passed && !firstFailure) {
54
+ firstFailure = `output does not contain "${benchmarks.output_contains}"`;
55
+ }
56
+ }
57
+ // ─── Output not contains ────────────────────────────────────
58
+ if (benchmarks.output_not_contains !== undefined) {
59
+ const outputStr = stringifyOutput(stepOutput);
60
+ const passed = !outputStr.toLowerCase().includes(benchmarks.output_not_contains.toLowerCase());
61
+ metrics.push({
62
+ name: 'output_not_contains',
63
+ value: passed ? 1 : 0,
64
+ threshold: 1,
65
+ passed,
66
+ detail: passed ? undefined : `output unexpectedly contains "${benchmarks.output_not_contains}"`,
67
+ });
68
+ if (!passed && !firstFailure) {
69
+ firstFailure = `output unexpectedly contains "${benchmarks.output_not_contains}"`;
70
+ }
71
+ }
72
+ // ─── LLM-as-a-judge ────────────────────────────────────────
73
+ if (benchmarks.llm_judge) {
74
+ const judge = benchmarks.llm_judge;
75
+ const outputStr = stringifyOutput(stepOutput);
76
+ const threshold = judge.judge_score_threshold ?? 7;
77
+ try {
78
+ const provider = judge.judge_provider ?? 'openai';
79
+ const evalPrompt = `${judge.judge_prompt}\n\nOutput to evaluate:\n${outputStr}\n\nScore this output on a scale of 0-10. Respond with only the number.`;
80
+ const result = await callProviderLLM(evalPrompt, { provider, model: judge.judge_model ?? undefined }, 'You are an expert test judge. Return only a number between 0 and 10.', 16, 0);
81
+ const score = parseFloat(result.content.match(/-?\d+(?:\.\d+)?/)?.[0] ?? '');
82
+ if (isNaN(score)) {
83
+ metrics.push({
84
+ name: 'llm_judge',
85
+ value: 0,
86
+ threshold,
87
+ passed: false,
88
+ detail: `Could not parse score from LLM response: "${result.content}"`,
89
+ });
90
+ if (!firstFailure) {
91
+ firstFailure = `llm_judge: could not parse score from response`;
92
+ }
93
+ }
94
+ else {
95
+ const passed = score >= threshold;
96
+ metrics.push({
97
+ name: 'llm_judge',
98
+ value: score,
99
+ threshold,
100
+ passed,
101
+ detail: `Score: ${score}/${threshold}`,
102
+ });
103
+ if (!passed && !firstFailure) {
104
+ firstFailure = `llm_judge score (${score}) below threshold (${threshold})`;
105
+ }
106
+ }
107
+ }
108
+ catch (err) {
109
+ const errMsg = err instanceof Error ? err.message : String(err);
110
+ metrics.push({
111
+ name: 'llm_judge',
112
+ value: 0,
113
+ threshold,
114
+ passed: false,
115
+ detail: `LLM judge error: ${errMsg}`,
116
+ });
117
+ if (!firstFailure) {
118
+ firstFailure = `llm_judge error: ${errMsg}`;
119
+ }
120
+ }
121
+ }
29
122
  const allPassed = metrics.every(m => m.passed);
30
123
  return {
31
124
  passed: allPassed,
@@ -33,4 +126,12 @@ export function compareBenchmarks(measurement, benchmarks) {
33
126
  metrics,
34
127
  };
35
128
  }
129
+ /** Converts step output to a string for text-based assertions. */
130
+ function stringifyOutput(output) {
131
+ if (output === null || output === undefined)
132
+ return '';
133
+ if (typeof output === 'string')
134
+ return output;
135
+ return JSON.stringify(output);
136
+ }
36
137
  //# sourceMappingURL=benchmark.js.map
package/dist/ci/benchmark.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/ci/benchmark.ts"],"names":[],"mappings":"AAgBA,MAAM,UAAU,iBAAiB,CAC/B,WAA4B,EAC5B,UAA0B;IAE1B,MAAM,OAAO,GAAmB,EAAE,CAAA;IAClC,IAAI,YAAgC,CAAA;IAEpC,IAAI,UAAU,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,WAAW,CAAC,WAAW,IAAI,UAAU,CAAC,eAAe,CAAA;QACpE,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,aAAa;YACnB,KAAK,EAAE,WAAW,CAAC,WAAW;YAC9B,SAAS,EAAE,UAAU,CAAC,eAAe;YACrC,MAAM;SACP,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,YAAY,GAAG,gBAAgB,WAAW,CAAC,WAAW,6BAA6B,UAAU,CAAC,eAAe,GAAG,CAAA;QAClH,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,WAAW,CAAC,YAAY,IAAI,CAAC,CAAA;QAC3C,MAAM,MAAM,GAAG,KAAK,IAAI,UAAU,CAAC,gBAAgB,CAAA;QACnD,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,cAAc;YACpB,KAAK;YACL,SAAS,EAAE,UAAU,CAAC,gBAAgB;YACtC,MAAM;SACP,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,YAAY,GAAG,iBAAiB,KAAK,6BAA6B,UAAU,CAAC,gBAAgB,GAAG,CAAA;QAClG,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;IAC9C,OAAO;QACL,MAAM,EAAE,SAAS;QACjB,cAAc,EAAE,YAAY;QAC5B,OAAO;KACR,CAAA;AACH,CAAC"}
1
+ {"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/ci/benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAoBtD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,WAA4B,EAC5B,UAA0B,EAC1B,UAAoB;IAEpB,MAAM,OAAO,GAAmB,EAAE,CAAA;IAClC,IAAI,YAAgC,CAAA;IAEpC,IAAI,UAAU,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,WAAW,CAAC,WAAW,IAAI,UAAU,CAAC,eAAe,CAAA;QACpE,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,aAAa;YACnB,KAAK,EAAE,WAAW,CAAC,WAAW;YAC9B,SAAS,EAAE,UAAU,CAAC,eAAe;YACrC,MAAM;SACP,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,YAAY,GAAG,gBAAgB,WAAW,CAAC,WAAW,6BAA6B,UAAU,CAAC,eAAe,GAAG,CAAA;QAClH,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,WAAW,CAAC,YAAY,IAAI,CAAC,CAAA;QAC3C,MAAM,MAAM,GAAG,KAAK,IAAI,UAAU,CAAC,gBAAgB,CAAA;QACnD,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,cAAc;YACpB,KAAK;YACL,SAAS,EAAE,UAAU,CAAC,gBAAgB;YACtC,MAAM;SACP,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,YAAY,GAAG,iBAAiB,KAAK,6BAA6B,UAAU,CAAC,gBAAgB,GAAG,CAAA;QAClG,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,IAAI,UAAU,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QAC7C,MAAM,SAAS,GAAG,eAAe,CAAC,UAAU,CAAC,CAAA;QAC7C,MAAM,MAAM,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC,CAAA;QACzF,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,iBAAiB;YACvB,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,SAAS,EAAE,CAAC;YACZ,MAAM;YACN,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B,UAAU,CAAC,eAAe,GAAG;SACvF,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,YAAY,GAAG,4BAA4B,UAAU,CAAC,eAAe,GAAG,CAAA;QAC1E,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,IAAI,UAAU,CAAC,mBAAmB,KAAK,SAAS,EAAE,CAAC;QACjD,MAAM,SAAS,GAAG,eAAe,CAAC,UAAU,CAAC,CAAA;QAC7C,MAAM,MAAM,GAAG,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,mBAAmB,CAAC,WAAW,EAAE,CAAC,CAAA;QAC9F,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,qBAAqB;YAC3B,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,SAAS,EAAE,CAAC;YACZ,MAAM;YACN,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,iCAAiC,UAAU,CAAC,mBAAmB,GAAG;SAChG,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,YAAY,GAAG,iCAAiC,UAAU,CAAC,mBAAmB,GAAG,CAAA;QACnF,CAAC;IACH,CAAC;IAED,8DAA8D;IAC9D,IAAI,UAAU,CAAC,SAAS,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,CAAA;QAClC,MAAM,SAAS,GAAG,eAAe,CAAC,UAAU,CAAC,CAAA;QAC7C,MAAM,SAAS,GAAG,KAAK,CAAC,qBAAqB,IAAI,CAAC,CAAA;QAElD,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,IAAI,QAAQ,CAAA;YACjD,MAAM,UAAU,GAAG,GAAG,KAAK,CAAC,YAAY,4BAA4B,SAAS,yEAAyE,CAAA;YAEtJ,MAAM,MAAM,GAAG,MAAM,eAAe,CAClC,UAAU,EACV,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,WAAW,IAAI,SAAS,EAAE,EACnD,sEAAsE,EACtE,EAAE,EACF,CAAC,CACF,CAAA;YAED,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;YAC5E,IAAI,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjB,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,WAAW;oBACjB,KAAK,EAAE,CAAC;oBACR,SAAS;oBACT,MAAM,EAAE,KAAK;oBACb,MAAM,EAAE,6CAA6C,MAAM,CAAC,OAAO,GAAG;iBACvE,CAAC,CAAA;gBACF,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClB,YAAY,GAAG,gDAAgD,CAAA;gBACjE,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,GAAG,KAAK,IAAI,SAAS,CAAA;gBACjC,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,WAAW;oBACjB,KAAK,EAAE,KAAK;oBACZ,SAAS;oBACT,MAAM;oBACN,MAAM,EAAE,UAAU,KAAK,IAAI,SAAS,EAAE;iBACvC,CAAC,CAAA;gBACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;oBAC7B,YAAY,GAAG,oBAAoB,KAAK,sBAAsB,SAAS,GAAG,CAAA;gBAC5E,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YAC/D,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,WAAW;gBACjB,KAAK,EAAE,CAAC;gBACR,SAAS;gBACT,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,oBAAoB,MAAM,EAAE;aACrC,CAAC,CAAA;YACF,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,YAAY,GAAG,oBAAoB,MAAM,EAAE,CAAA;YAC7C,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;IAC9C,OAAO;QACL,MAAM,EAAE,SAAS;QACjB,cAAc,EAAE,YAAY;QAC5B,OAAO;KACR,CAAA;AACH,CAAC;AAED,kEAAkE;AAClE,SAAS,eAAe,CAAC,MAAe;IACtC,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,EAAE,CAAA;IACtD,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAA;IAC7C,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;AAC/B,CAAC"}
package/dist/ci/ed-runner.js CHANGED
@@ -132,8 +132,8 @@ async function runSingleTest(test) {
132
132
  durationMs: Date.now() - startMs,
133
133
  };
134
134
  }
135
- // Compare against benchmarks
136
- const benchmarkResult = compareBenchmarks(measurement, test.benchmarks);
135
+ // Compare against benchmarks (async to support llm_judge)
136
+ const benchmarkResult = await compareBenchmarks(measurement, test.benchmarks, targetStep?.output);
137
137
  return {
138
138
  ...base,
139
139
  testId: test.name,
@@ -152,7 +152,7 @@ async function runSingleTest(test) {
152
152
  const traceMeasurement = extractMeasurementFromTrace(test);
153
153
  if (traceMeasurement) {
154
154
  console.log(` [ed-test] ${test.name}: run() failed (${err instanceof Error ? err.message : String(err)}), using trace measurement fallback`);
155
- const benchmarkResult = compareBenchmarks(traceMeasurement, test.benchmarks);
155
+ const benchmarkResult = await compareBenchmarks(traceMeasurement, test.benchmarks, targetStep?.output);
156
156
  return {
157
157
  ...base,
158
158
  testId: test.name,
package/dist/ci/ed-runner.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"ed-runner.js","sourceRoot":"","sources":["../../src/ci/ed-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC5C,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAClG,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AAsDlD,+DAA+D;AAE/D,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAA0B;IACzD,MAAM,GAAG,GAAG,OAAO,EAAE,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAA;IACzC,MAAM,KAAK,GAAG,UAAU,EAAE,CAAA;IAC1B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAC1C,MAAM,OAAO,GAAmB,EAAE,CAAA;IAElC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAA;IAElD,2CAA2C;IAC3C,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC;YACX,MAAM,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS;YACjC,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS;YACnC,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,qBAAqB,GAAG,CAAC,OAAO,EAAE;YACjD,UAAU,EAAE,CAAC;YACb,UAAU,EAAE,EAAE;SACf,CAAC,CAAA;IACJ,CAAC;IAED,mCAAmC;IACnC,IAAI,UAAU,GAAoB,KAAK,CAAA;IACvC,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QACpB,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAA;QAC9B,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAA;IAC5D,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,IAAI,CAAC,CAAC,CAAA;IAE/C,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAwB,EAAE,CAAA;QACvC,IAAI,UAAU,GAAwB,IAAI,CAAA;QAE1C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YACpD,MAAM,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YAC7C,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAA;YACxC,MAAM,aAAa,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YAE9C,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,SAAS,OAAO,IAAI,OAAO,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;YACvF,CAAC;YAED,+BAA+B;YAC/B,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,eAAe,EAAE,MAAM,CAAC,eAAe;gBACvC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,SAAS,EAAE,YAAY;gBACvB,UAAU,EAAE,aAAa;aAC1B,CAAC,CAAA;YAEF,4EAA4E;YAC5E,IAAI,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;gBAC5C,UAAU,GAAG,MAAM,CAAA;YACrB,CAAC;QAEP,CAAC;QAEG,oCAAoC;QACpC,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAA;QACxD,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAA;QAExD,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,UAAW;YACd,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YACnC,aAAa,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,aAAa,IAAI,UAAW,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS;YAC9F,UAAU,EAAE,OAAO;SACpB,CAAC,CAAA;QAEF,IAAI,OAAO,EAAE,QAAQ,IAAI,SAAS,EAAE,CAAC;YACnC,MAAK;QACP,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAE3C,OAAO;QACL,KAAK;QACL,SAAS;QACT,UAAU;QACV,OAAO;QACP,UAAU,EAAE,WAAW;KACxB,CAAA;AACH,CAAC;AAED,+DAA+D;AAE/D,KAAK,UAAU,aAAa,CAAC,IAAmB;IAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC1B,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACpF,MAAM,IAAI,GAA0B;QAClC,MAAM,EAAE,IAAI,CAAC,IAAI;QACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;QACnB,QAAQ,EAAE,IAAI,CAAC,KAAK;QACpB,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;QAChE,KAAK,EAAE,UAAU,EAAE,KAAK;QACxB,MAAM,EAAE,UAAU,EAAE,MAAM;KAC3B,CAAA;IAED,4BAA4B;IAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,OAAO,IAAI,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;QAChD,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,0BAA0B;YACzC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAA;IACH,CAAC;IAED,MAAM,SAAS,GAAG,mBAAmB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAC1E,aAAa,CAAC,SAAS,CAAC,CAAA;IAExB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,IAAI,KAAK,CAAA;QAE1C,MAAM,OAAO,CAAC,IAAI,CAAC;YACjB,IAAI,CAAC,GAAG,EAAE;YACV,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAC/B,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CACjE;SACF,CAAC,CAAA;QAEF,4CAA4C;QAC5C,gEAAgE;QAChE,sEAAsE;QACtE,qEAAqE;QACrE,sEAAsE;QACtE,uEAAuE;QACvE,2CAA2C;QAC3C,IAAI,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAC/C,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,WAAW,GAAG,2BAA2B,CAAC,IAAI,CAAC,IAAI,IAAI,CAAA;YACvD,IAAI,WAAW,EAAE,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,yDAAyD,CAAC,CAAA;YAChG,CAAC;QACH,CAAC;QACD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,gBAAgB,IAAI,CAAC,MAAM,CAAC,OAAO,qCAAqC;gBACvF,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QAED,6BAA6B;QAC7B,MAAM,eAAe,GAAG,iBAAiB,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC,CAAA;QAEvE,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAChD,aAAa,EAAE,eAAe,CAAC,cAAc;YAC7C,WAAW;YACX,eAAe;YACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAA;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,4EAA4E;QAC5E,4EAA4E;QAC5E,iEAAiE;QACjE,MAAM,gBAAgB,GAAG,2BAA2B,CAAC,IAAI,CAAC,CAAA;QAC1D,IAAI,gBAAgB,EAAE,CAAC;YACrB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,mBAAmB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAA;YAC7I,MAAM,eAAe,GAAG,iBAAiB,CAAC,gBAAgB,EAAE,IAAI,CAAC,UAAU,CAAC,CAAA;YAC5E,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;gBAChD,aAAa,EAAE,eAAe,CAAC,cAAc;gBAC7C,WAAW,EAAE,gBAAgB;gBAC7B,eAAe;gBACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QAED,IAAI,GAAG,YAAY,eAAe,EAAE,CAAC;YACnC,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,gBAAgB,GAAG,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,EAAE;gBAC9D,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QACD,IAAI,GAAG,YAAY,YAAY,EAAE,CAAC;YAChC,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,wBAAwB,GAAG,CAAC,SAAS,IAAI;gBACxD,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QACD,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,oBAAoB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YACrF,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAA;IACH,CAAC;YAAS,CAAC;QACT,eAAe,EAAE,CAAA;IACnB,CAAC;AACH,CAAC;AAED,+DAA+D;AAE/D;;;;;;GAMG;AACH,SAAS,2BAA2B,CAAC,IAAmB;IACtD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAC9E,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAA;IAE3B,MAAM,MAAM,GAAoB;QAC9B,WAAW,EAAE,IAAI,CAAC,WAAW;KAC9B,CAAA;IAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAA;QACvC,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAA;QACzC,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAA;IACzC,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED,+DAA+D;AAE/D,MAAM,YAAa,SAAQ,KAAK;IACX;IAAnB,YAAmB,SAAiB;QAClC,KAAK,CAAC,wBAAwB,SAAS,IAAI,CAAC,CAAA;QAD3B,cAAS,GAAT,SAAS,CAAQ;QAElC,IAAI,CAAC,IAAI,GAAG,cAAc,CAAA;IAC5B,CAAC;CACF;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,OAAe;IAC9C,0CAA0C;IAC1C,MAAM,KAAK,GAAG,IAAI,MAAM,CACtB,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,GAAG,CAClG,CAAA;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACzB,CAAC"}
1
+ {"version":3,"file":"ed-runner.js","sourceRoot":"","sources":["../../src/ci/ed-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC5C,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAClG,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AAsDlD,+DAA+D;AAE/D,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAA0B;IACzD,MAAM,GAAG,GAAG,OAAO,EAAE,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAA;IACzC,MAAM,KAAK,GAAG,UAAU,EAAE,CAAA;IAC1B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAC1C,MAAM,OAAO,GAAmB,EAAE,CAAA;IAElC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAA;IAElD,2CAA2C;IAC3C,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC;YACX,MAAM,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS;YACjC,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS;YACnC,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,qBAAqB,GAAG,CAAC,OAAO,EAAE;YACjD,UAAU,EAAE,CAAC;YACb,UAAU,EAAE,EAAE;SACf,CAAC,CAAA;IACJ,CAAC;IAED,mCAAmC;IACnC,IAAI,UAAU,GAAoB,KAAK,CAAA;IACvC,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QACpB,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAA;QAC9B,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAA;IAC5D,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,IAAI,CAAC,CAAC,CAAA;IAE/C,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAwB,EAAE,CAAA;QACvC,IAAI,UAAU,GAAwB,IAAI,CAAA;QAE1C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YACpD,MAAM,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YAC7C,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAA;YACxC,MAAM,aAAa,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YAE9C,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,SAAS,OAAO,IAAI,OAAO,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;YACvF,CAAC;YAED,+BAA+B;YAC/B,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,eAAe,EAAE,MAAM,CAAC,eAAe;gBACvC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,SAAS,EAAE,YAAY;gBACvB,UAAU,EAAE,aAAa;aAC1B,CAAC,CAAA;YAEF,4EAA4E;YAC5E,IAAI,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;gBAC5C,UAAU,GAAG,MAAM,CAAA;YACrB,CAAC;QAEP,CAAC;QAEG,oCAAoC;QACpC,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAA;QACxD,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAA;QAExD,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,UAAW;YACd,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YACnC,aAAa,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,aAAa,IAAI,UAAW,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS;YAC9F,UAAU,EAAE,OAAO;SACpB,CAAC,CAAA;QAEF,IAAI,OAAO,EAAE,QAAQ,IAAI,SAAS,EAAE,CAAC;YACnC,MAAK;QACP,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAE3C,OAAO;QACL,KAAK;QACL,SAAS;QACT,UAAU;QACV,OAAO;QACP,UAAU,EAAE,WAAW;KACxB,CAAA;AACH,CAAC;AAED,+DAA+D;AAE/D,KAAK,UAAU,aAAa,CAAC,IAAmB;IAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC1B,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACpF,MAAM,IAAI,GAA0B;QAClC,MAAM,EAAE,IAAI,CAAC,IAAI;QACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;QACnB,QAAQ,EAAE,IAAI,CAAC,KAAK;QACpB,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;QAChE,KAAK,EAAE,UAAU,EAAE,KAAK;QACxB,MAAM,EAAE,UAAU,EAAE,MAAM;KAC3B,CAAA;IAED,4BAA4B;IAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,OAAO,IAAI,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;QAChD,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,0BAA0B;YACzC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAA;IACH,CAAC;IAED,MAAM,SAAS,GAAG,mBAAmB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAC1E,aAAa,CAAC,SAAS,CAAC,CAAA;IAExB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,IAAI,KAAK,CAAA;QAE1C,MAAM,OAAO,CAAC,IAAI,CAAC;YACjB,IAAI,CAAC,GAAG,EAAE;YACV,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAC/B,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CACjE;SACF,CAAC,CAAA;QAEF,4CAA4C;QAC5C,gEAAgE;QAChE,sEAAsE;QACtE,qEAAqE;QACrE,sEAAsE;QACtE,uEAAuE;QACvE,2CAA2C;QAC3C,IAAI,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAC/C,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,WAAW,GAAG,2BAA2B,CAAC,IAAI,CAAC,IAAI,IAAI,CAAA;YACvD,IAAI,WAAW,EAAE,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,yDAAyD,CAAC,CAAA;YAChG,CAAC;QACH,CAAC;QACD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,gBAAgB,IAAI,CAAC,MAAM,CAAC,OAAO,qCAAqC;gBACvF,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QAED,0DAA0D;QAC1D,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,MAAM,CAAC,CAAA;QAEjG,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAChD,aAAa,EAAE,eAAe,CAAC,cAAc;YAC7C,WAAW;YACX,eAAe;YACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAA;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,4EAA4E;QAC5E,4EAA4E;QAC5E,iEAAiE;QACjE,MAAM,gBAAgB,GAAG,2BAA2B,CAAC,IAAI,CAAC,CAAA;QAC1D,IAAI,gBAAgB,EAAE,CAAC;YACrB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,mBAAmB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAA;YAC7I,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,MAAM,CAAC,CAAA;YACtG,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;gBAChD,aAAa,EAAE,eAAe,CAAC,cAAc;gBAC7C,WAAW,EAAE,gBAAgB;gBAC7B,eAAe;gBACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QAED,IAAI,GAAG,YAAY,eAAe,EAAE,CAAC;YACnC,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,gBAAgB,GAAG,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,EAAE;gBAC9D,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QACD,IAAI,GAAG,YAAY,YAAY,EAAE,CAAC;YAChC,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,wBAAwB,GAAG,CAAC,SAAS,IAAI;gBACxD,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QACD,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,oBAAoB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YACrF,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAA;IACH,CAAC;YAAS,CAAC;QACT,eAAe,EAAE,CAAA;IACnB,CAAC;AACH,CAAC;AAED,+DAA+D;AAE/D;;;;;;GAMG;AACH,SAAS,2BAA2B,CAAC,IAAmB;IACtD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAC9E,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAA;IAE3B,MAAM,MAAM,GAAoB;QAC9B,WAAW,EAAE,IAAI,CAAC,WAAW;KAC9B,CAAA;IAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAA;QACvC,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAA;QACzC,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAA;IACzC,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED,+DAA+D;AAE/D,MAAM,YAAa,SAAQ,KAAK;IACX;IAAnB,YAAmB,SAAiB;QAClC,KAAK,CAAC,wBAAwB,SAAS,IAAI,CAAC,CAAA;QAD3B,cAAS,GAAT,SAAS,CAAQ;QAElC,IAAI,CAAC,IAAI,GAAG,cAAc,CAAA;IAC5B,CAAC;CACF;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,OAAe;IAC9C,0CAA0C;IAC1C,MAAM,KAAK,GAAG,IAAI,MAAM,CACtB,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,GAAG,CAClG,CAAA;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACzB,CAAC"}
package/dist/ci/test-registry.d.ts CHANGED
@@ -2,9 +2,25 @@ export interface TestTarget {
2
2
  type: 'tool_call' | 'ai_call';
3
3
  step_id: string;
4
4
  }
5
+ export interface LLMJudgeBenchmark {
6
+ /** The prompt sent to the LLM judge to evaluate the step's output. */
7
+ judge_prompt: string;
8
+ /** Minimum score (0-10) required to pass. Defaults to 7. */
9
+ judge_score_threshold?: number;
10
+ /** LLM provider to use for judging. Defaults to 'openai'. */
11
+ judge_provider?: 'openai' | 'claude' | 'gemini' | 'grok' | 'kimi';
12
+ /** Model override for the judge LLM. */
13
+ judge_model?: string;
14
+ }
5
15
  export interface TestBenchmarks {
6
16
  max_duration_ms?: number;
7
17
  max_tokens_total?: number;
18
+ /** Assert that the step's output contains this substring. */
19
+ output_contains?: string;
20
+ /** Assert that the step's output does NOT contain this substring. */
21
+ output_not_contains?: string;
22
+ /** LLM-as-a-judge evaluation of the step's output quality. */
23
+ llm_judge?: LLMJudgeBenchmark;
8
24
  }
9
25
  export interface TestDefinition {
10
26
  name: string;
package/dist/ci/test-registry.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"test-registry.d.ts","sourceRoot":"","sources":["../../src/ci/test-registry.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,WAAW,GAAG,SAAS,CAAA;IAC7B,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,gBAAgB,CAAC,EAAE,MAAM,CAAA;CAC1B;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,UAAU,CAAA;IAClB,UAAU,EAAE,cAAc,CAAA;IAC1B,2FAA2F;IAC3F,GAAG,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IACzB,2DAA2D;IAC3D,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAWD,wBAAgB,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,GAAG,IAAI,CAEzE;AAED,wBAAgB,eAAe,IAAI,cAAc,EAAE,CAElD;AAED,wBAAgB,iBAAiB,IAAI,IAAI,CAExC"}
1
+ {"version":3,"file":"test-registry.d.ts","sourceRoot":"","sources":["../../src/ci/test-registry.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,WAAW,GAAG,SAAS,CAAA;IAC7B,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,sEAAsE;IACtE,YAAY,EAAE,MAAM,CAAA;IACpB,4DAA4D;IAC5D,qBAAqB,CAAC,EAAE,MAAM,CAAA;IAC9B,6DAA6D;IAC7D,cAAc,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,CAAA;IACjE,wCAAwC;IACxC,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,6DAA6D;IAC7D,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,qEAAqE;IACrE,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,8DAA8D;IAC9D,SAAS,CAAC,EAAE,iBAAiB,CAAA;CAC9B;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,UAAU,CAAA;IAClB,UAAU,EAAE,cAAc,CAAA;IAC1B,2FAA2F;IAC3F,GAAG,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IACzB,2DAA2D;IAC3D,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAWD,wBAAgB,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,GAAG,IAAI,CAEzE;AAED,wBAAgB,eAAe,IAAI,cAAc,EAAE,CAElD;AAED,wBAAgB,iBAAiB,IAAI,IAAI,CAExC"}
package/dist/ci/test-registry.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"test-registry.js","sourceRoot":"","sources":["../../src/ci/test-registry.ts"],"names":[],"mappings":"AAuBA,4EAA4E;AAC5E,6EAA6E;AAC7E,iCAAiC;AACjC,MAAM,CAAC,GAAG,UAAqC,CAAA;AAC/C,MAAM,oBAAoB,GAAG,kCAAkC,CAAA;AAC/D,MAAM,QAAQ,GACX,CAAC,CAAC,oBAAoB,CAAsB,IAAI,EAAE,CAAA;AACrD,IAAI,CAAC,CAAC,CAAC,oBAAoB,CAAC;IAAE,CAAC,CAAC,oBAAoB,CAAC,GAAG,QAAQ,CAAA;AAEhE,MAAM,UAAU,UAAU,CAAC,GAAwC;IACjE,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,GAAG,EAAE,CAAC,CAAA;AAC3B,CAAC;AAED,MAAM,UAAU,eAAe;IAC7B,OAAO,CAAC,GAAG,QAAQ,CAAC,CAAA;AACtB,CAAC;AAED,MAAM,UAAU,iBAAiB;IAC/B,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAA;AACrB,CAAC"}
1
+ {"version":3,"file":"test-registry.js","sourceRoot":"","sources":["../../src/ci/test-registry.ts"],"names":[],"mappings":"AAwCA,4EAA4E;AAC5E,6EAA6E;AAC7E,iCAAiC;AACjC,MAAM,CAAC,GAAG,UAAqC,CAAA;AAC/C,MAAM,oBAAoB,GAAG,kCAAkC,CAAA;AAC/D,MAAM,QAAQ,GACX,CAAC,CAAC,oBAAoB,CAAsB,IAAI,EAAE,CAAA;AACrD,IAAI,CAAC,CAAC,CAAC,oBAAoB,CAAC;IAAE,CAAC,CAAC,oBAAoB,CAAC,GAAG,QAAQ,CAAA;AAEhE,MAAM,UAAU,UAAU,CAAC,GAAwC;IACjE,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,GAAG,EAAE,CAAC,CAAA;AAC3B,CAAC;AAED,MAAM,UAAU,eAAe;IAC7B,OAAO,CAAC,GAAG,QAAQ,CAAC,CAAA;AACtB,CAAC;AAED,MAAM,UAAU,iBAAiB;IAC/B,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAA;AACrB,CAAC"}
package/dist/ci/upload-client.d.ts CHANGED
@@ -38,7 +38,7 @@ export interface UploadTestResult {
38
38
  reruns?: UploadRerun[];
39
39
  }
40
40
  export interface UploadMetric {
41
- name: 'duration_ms' | 'tokens_total' | 'tokens_input' | 'tokens_output';
41
+ name: 'duration_ms' | 'tokens_total' | 'tokens_input' | 'tokens_output' | 'output_contains' | 'output_not_contains' | 'llm_judge';
42
42
  value: number;
43
43
  passed: boolean;
44
44
  benchmark?: {
package/dist/ci/upload-client.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"upload-client.d.ts","sourceRoot":"","sources":["../../src/ci/upload-client.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,eAAe,EAAmC,MAAM,gBAAgB,CAAA;AAKtF,MAAM,WAAW,aAAa;IAC5B,cAAc,EAAE,CAAC,CAAA;IACjB,GAAG,EAAE;QACH,MAAM,EAAE,MAAM,CAAA;QACd,UAAU,EAAE,MAAM,CAAA;QAClB,WAAW,EAAE,MAAM,CAAA;QACnB,WAAW,EAAE,MAAM,CAAA;QACnB,GAAG,EAAE;YACH,UAAU,EAAE,MAAM,CAAA;YAClB,MAAM,EAAE,MAAM,CAAA;YACd,WAAW,CAAC,EAAE,MAAM,CAAA;YACpB,SAAS,CAAC,EAAE,MAAM,CAAA;YAClB,IAAI,CAAC,EAAE,MAAM,CAAA;SACd,CAAA;QACD,EAAE,EAAE;YACF,QAAQ,CAAC,EAAE,MAAM,CAAA;YACjB,OAAO,CAAC,EAAE,MAAM,CAAA;SACjB,CAAA;KACF,CAAA;IACD,OAAO,EAAE,gBAAgB,EAAE,CAAA;CAC5B;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,GAAG,MAAM,CAAA;IACvB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAA;IAC1C,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,OAAO,EAAE,YAAY,EAAE,CAAA;IACvB,MAAM,CAAC,EAAE,WAAW,EAAE,CAAA;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,aAAa,GAAG,cAAc,GAAG,cAAc,GAAG,eAAe,CAAA;IACvE,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,OAAO,CAAA;IACf,SAAS,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAA;IAC/C,+CAA+C;IAC/C,YAAY,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,EAAE,CAAA;CACzE;AAED,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAID,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,eAAe,GAAG,aAAa,CAwB5E;AAyHD,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,cAAc;IAE7B,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,UAAU,GAAG,kBAAkB,CAAA;IACvC,YAAY,EAAE,MAAM,CAAA;CACrB;AAED,wBAAsB,aAAa,CACjC,OAAO,EAAE,aAAa,EACtB,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,cAAc,CAAC,CAgDzB;AAID,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,aAAa,EACtB,KAAK,EAAE,MAAM,EACb,GAAG,CAAC,EAAE,MAAM,GACX,OAAO,CAAC,IAAI,CAAC,CASf"}
1
+ {"version":3,"file":"upload-client.d.ts","sourceRoot":"","sources":["../../src/ci/upload-client.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,eAAe,EAAmC,MAAM,gBAAgB,CAAA;AAKtF,MAAM,WAAW,aAAa;IAC5B,cAAc,EAAE,CAAC,CAAA;IACjB,GAAG,EAAE;QACH,MAAM,EAAE,MAAM,CAAA;QACd,UAAU,EAAE,MAAM,CAAA;QAClB,WAAW,EAAE,MAAM,CAAA;QACnB,WAAW,EAAE,MAAM,CAAA;QACnB,GAAG,EAAE;YACH,UAAU,EAAE,MAAM,CAAA;YAClB,MAAM,EAAE,MAAM,CAAA;YACd,WAAW,CAAC,EAAE,MAAM,CAAA;YACpB,SAAS,CAAC,EAAE,MAAM,CAAA;YAClB,IAAI,CAAC,EAAE,MAAM,CAAA;SACd,CAAA;QACD,EAAE,EAAE;YACF,QAAQ,CAAC,EAAE,MAAM,CAAA;YACjB,OAAO,CAAC,EAAE,MAAM,CAAA;SACjB,CAAA;KACF,CAAA;IACD,OAAO,EAAE,gBAAgB,EAAE,CAAA;CAC5B;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,GAAG,MAAM,CAAA;IACvB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAA;IAC1C,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,OAAO,EAAE,YAAY,EAAE,CAAA;IACvB,MAAM,CAAC,EAAE,WAAW,EAAE,CAAA;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,aAAa,GAAG,cAAc,GAAG,cAAc,GAAG,eAAe,GAAG,iBAAiB,GAAG,qBAAqB,GAAG,WAAW,CAAA;IACjI,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,OAAO,CAAA;IACf,SAAS,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAA;IAC/C,+CAA+C;IAC/C,YAAY,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,EAAE,CAAA;CACzE;AAED,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAID,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,eAAe,GAAG,aAAa,CAwB5E;AAyHD,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,cAAc;IAE7B,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,UAAU,GAAG,kBAAkB,CAAA;IACvC,YAAY,EAAE,MAAM,CAAA;CACrB;AAED,wBAAsB,aAAa,CACjC,OAAO,EAAE,aAAa,EACtB,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,cAAc,CAAC,CAgDzB;AAID,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,aAAa,EACtB,KAAK,EAAE,MAAM,EACb,GAAG,CAAC,EAAE,MAAM,GACX,OAAO,CAAC,IAAI,CAAC,CASf"}
package/dist/index.cjs CHANGED
@@ -6309,7 +6309,8 @@ function collectMeasurement(ctx) {
6309
6309
  init_trace_schema();
6310
6310
 
6311
6311
  // src/ci/benchmark.ts
6312
- function compareBenchmarks(measurement, benchmarks) {
6312
+ init_matchers();
6313
+ async function compareBenchmarks(measurement, benchmarks, stepOutput) {
6313
6314
  const metrics = [];
6314
6315
  let firstFailure;
6315
6316
  if (benchmarks.max_duration_ms !== void 0) {
@@ -6337,6 +6338,92 @@ function compareBenchmarks(measurement, benchmarks) {
6337
6338
  firstFailure = `tokens_total (${value}) exceeded max threshold (${benchmarks.max_tokens_total})`;
6338
6339
  }
6339
6340
  }
6341
+ if (benchmarks.output_contains !== void 0) {
6342
+ const outputStr = stringifyOutput(stepOutput);
6343
+ const passed = outputStr.toLowerCase().includes(benchmarks.output_contains.toLowerCase());
6344
+ metrics.push({
6345
+ name: "output_contains",
6346
+ value: passed ? 1 : 0,
6347
+ threshold: 1,
6348
+ passed,
6349
+ detail: passed ? void 0 : `output does not contain "${benchmarks.output_contains}"`
6350
+ });
6351
+ if (!passed && !firstFailure) {
6352
+ firstFailure = `output does not contain "${benchmarks.output_contains}"`;
6353
+ }
6354
+ }
6355
+ if (benchmarks.output_not_contains !== void 0) {
6356
+ const outputStr = stringifyOutput(stepOutput);
6357
+ const passed = !outputStr.toLowerCase().includes(benchmarks.output_not_contains.toLowerCase());
6358
+ metrics.push({
6359
+ name: "output_not_contains",
6360
+ value: passed ? 1 : 0,
6361
+ threshold: 1,
6362
+ passed,
6363
+ detail: passed ? void 0 : `output unexpectedly contains "${benchmarks.output_not_contains}"`
6364
+ });
6365
+ if (!passed && !firstFailure) {
6366
+ firstFailure = `output unexpectedly contains "${benchmarks.output_not_contains}"`;
6367
+ }
6368
+ }
6369
+ if (benchmarks.llm_judge) {
6370
+ const judge = benchmarks.llm_judge;
6371
+ const outputStr = stringifyOutput(stepOutput);
6372
+ const threshold = judge.judge_score_threshold ?? 7;
6373
+ try {
6374
+ const provider = judge.judge_provider ?? "openai";
6375
+ const evalPrompt = `${judge.judge_prompt}
6376
+
6377
+ Output to evaluate:
6378
+ ${outputStr}
6379
+
6380
+ Score this output on a scale of 0-10. Respond with only the number.`;
6381
+ const result = await callProviderLLM(
6382
+ evalPrompt,
6383
+ { provider, model: judge.judge_model ?? void 0 },
6384
+ "You are an expert test judge. Return only a number between 0 and 10.",
6385
+ 16,
6386
+ 0
6387
+ );
6388
+ const score = parseFloat(result.content.match(/-?\d+(?:\.\d+)?/)?.[0] ?? "");
6389
+ if (isNaN(score)) {
6390
+ metrics.push({
6391
+ name: "llm_judge",
6392
+ value: 0,
6393
+ threshold,
6394
+ passed: false,
6395
+ detail: `Could not parse score from LLM response: "${result.content}"`
6396
+ });
6397
+ if (!firstFailure) {
6398
+ firstFailure = `llm_judge: could not parse score from response`;
6399
+ }
6400
+ } else {
6401
+ const passed = score >= threshold;
6402
+ metrics.push({
6403
+ name: "llm_judge",
6404
+ value: score,
6405
+ threshold,
6406
+ passed,
6407
+ detail: `Score: ${score}/${threshold}`
6408
+ });
6409
+ if (!passed && !firstFailure) {
6410
+ firstFailure = `llm_judge score (${score}) below threshold (${threshold})`;
6411
+ }
6412
+ }
6413
+ } catch (err) {
6414
+ const errMsg = err instanceof Error ? err.message : String(err);
6415
+ metrics.push({
6416
+ name: "llm_judge",
6417
+ value: 0,
6418
+ threshold,
6419
+ passed: false,
6420
+ detail: `LLM judge error: ${errMsg}`
6421
+ });
6422
+ if (!firstFailure) {
6423
+ firstFailure = `llm_judge error: ${errMsg}`;
6424
+ }
6425
+ }
6426
+ }
6340
6427
  const allPassed = metrics.every((m) => m.passed);
6341
6428
  return {
6342
6429
  passed: allPassed,
@@ -6344,6 +6431,11 @@ function compareBenchmarks(measurement, benchmarks) {
6344
6431
  metrics
6345
6432
  };
6346
6433
  }
6434
+ function stringifyOutput(output) {
6435
+ if (output === null || output === void 0) return "";
6436
+ if (typeof output === "string") return output;
6437
+ return JSON.stringify(output);
6438
+ }
6347
6439
 
6348
6440
  // src/ci/ed-runner.ts
6349
6441
  async function runEdTests(options) {
@@ -6461,7 +6553,7 @@ async function runSingleTest(test) {
6461
6553
  durationMs: Date.now() - startMs
6462
6554
  };
6463
6555
  }
6464
- const benchmarkResult = compareBenchmarks(measurement, test.benchmarks);
6556
+ const benchmarkResult = await compareBenchmarks(measurement, test.benchmarks, targetStep?.output);
6465
6557
  return {
6466
6558
  ...base,
6467
6559
  testId: test.name,
@@ -6476,7 +6568,7 @@ async function runSingleTest(test) {
6476
6568
  const traceMeasurement = extractMeasurementFromTrace(test);
6477
6569
  if (traceMeasurement) {
6478
6570
  console.log(` [ed-test] ${test.name}: run() failed (${err instanceof Error ? err.message : String(err)}), using trace measurement fallback`);
6479
- const benchmarkResult = compareBenchmarks(traceMeasurement, test.benchmarks);
6571
+ const benchmarkResult = await compareBenchmarks(traceMeasurement, test.benchmarks, targetStep?.output);
6480
6572
  return {
6481
6573
  ...base,
6482
6574
  testId: test.name,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "elasticdash-test",
3
- "version": "0.1.20-alpha-19",
3
+ "version": "0.1.20-alpha-20",
4
4
  "description": "AI-native test runner for ElasticDash workflow testing",
5
5
  "type": "module",
6
6
  "bin": {
package/src/ci/benchmark.ts CHANGED
@@ -1,11 +1,24 @@
1
+ /**
2
+ * benchmark.ts
3
+ *
4
+ * Compares recorded trace measurements against user-defined benchmarks.
5
+ * Supports duration, token, output-contains, and LLM-as-a-judge evaluations.
6
+ *
7
+ * Generated/updated on 2026-04-20.
8
+ */
9
+
10
+ import { callProviderLLM } from '../matchers/index.js'
1
11
  import type { TestMeasurement } from './measurement.js'
2
12
  import type { TestBenchmarks } from './test-registry.js'
3
13
 
14
+ export type MetricName = 'duration_ms' | 'tokens_total' | 'output_contains' | 'output_not_contains' | 'llm_judge'
15
+
4
16
  export interface MetricResult {
5
- name: 'duration_ms' | 'tokens_total'
17
+ name: MetricName
6
18
  value: number
7
19
  threshold: number
8
20
  passed: boolean
21
+ detail?: string
9
22
  }
10
23
 
11
24
  export interface BenchmarkResult {
@@ -14,10 +27,15 @@ export interface BenchmarkResult {
14
27
  metrics: MetricResult[]
15
28
  }
16
29
 
17
- export function compareBenchmarks(
30
+ /**
31
+ * Compare a measurement against benchmarks. Async because llm_judge requires
32
+ * an LLM call. The step's output is needed for output_contains/llm_judge checks.
33
+ */
34
+ export async function compareBenchmarks(
18
35
  measurement: TestMeasurement,
19
36
  benchmarks: TestBenchmarks,
20
- ): BenchmarkResult {
37
+ stepOutput?: unknown,
38
+ ): Promise<BenchmarkResult> {
21
39
  const metrics: MetricResult[] = []
22
40
  let firstFailure: string | undefined
23
41
 
@@ -48,6 +66,96 @@ export function compareBenchmarks(
48
66
  }
49
67
  }
50
68
 
69
+ // ─── Output contains ────────────────────────────────────────
70
+ if (benchmarks.output_contains !== undefined) {
71
+ const outputStr = stringifyOutput(stepOutput)
72
+ const passed = outputStr.toLowerCase().includes(benchmarks.output_contains.toLowerCase())
73
+ metrics.push({
74
+ name: 'output_contains',
75
+ value: passed ? 1 : 0,
76
+ threshold: 1,
77
+ passed,
78
+ detail: passed ? undefined : `output does not contain "${benchmarks.output_contains}"`,
79
+ })
80
+ if (!passed && !firstFailure) {
81
+ firstFailure = `output does not contain "${benchmarks.output_contains}"`
82
+ }
83
+ }
84
+
85
+ // ─── Output not contains ────────────────────────────────────
86
+ if (benchmarks.output_not_contains !== undefined) {
87
+ const outputStr = stringifyOutput(stepOutput)
88
+ const passed = !outputStr.toLowerCase().includes(benchmarks.output_not_contains.toLowerCase())
89
+ metrics.push({
90
+ name: 'output_not_contains',
91
+ value: passed ? 1 : 0,
92
+ threshold: 1,
93
+ passed,
94
+ detail: passed ? undefined : `output unexpectedly contains "${benchmarks.output_not_contains}"`,
95
+ })
96
+ if (!passed && !firstFailure) {
97
+ firstFailure = `output unexpectedly contains "${benchmarks.output_not_contains}"`
98
+ }
99
+ }
100
+
101
+ // ─── LLM-as-a-judge ────────────────────────────────────────
102
+ if (benchmarks.llm_judge) {
103
+ const judge = benchmarks.llm_judge
104
+ const outputStr = stringifyOutput(stepOutput)
105
+ const threshold = judge.judge_score_threshold ?? 7
106
+
107
+ try {
108
+ const provider = judge.judge_provider ?? 'openai'
109
+ const evalPrompt = `${judge.judge_prompt}\n\nOutput to evaluate:\n${outputStr}\n\nScore this output on a scale of 0-10. Respond with only the number.`
110
+
111
+ const result = await callProviderLLM(
112
+ evalPrompt,
113
+ { provider, model: judge.judge_model ?? undefined },
114
+ 'You are an expert test judge. Return only a number between 0 and 10.',
115
+ 16,
116
+ 0,
117
+ )
118
+
119
+ const score = parseFloat(result.content.match(/-?\d+(?:\.\d+)?/)?.[0] ?? '')
120
+ if (isNaN(score)) {
121
+ metrics.push({
122
+ name: 'llm_judge',
123
+ value: 0,
124
+ threshold,
125
+ passed: false,
126
+ detail: `Could not parse score from LLM response: "${result.content}"`,
127
+ })
128
+ if (!firstFailure) {
129
+ firstFailure = `llm_judge: could not parse score from response`
130
+ }
131
+ } else {
132
+ const passed = score >= threshold
133
+ metrics.push({
134
+ name: 'llm_judge',
135
+ value: score,
136
+ threshold,
137
+ passed,
138
+ detail: `Score: ${score}/${threshold}`,
139
+ })
140
+ if (!passed && !firstFailure) {
141
+ firstFailure = `llm_judge score (${score}) below threshold (${threshold})`
142
+ }
143
+ }
144
+ } catch (err) {
145
+ const errMsg = err instanceof Error ? err.message : String(err)
146
+ metrics.push({
147
+ name: 'llm_judge',
148
+ value: 0,
149
+ threshold,
150
+ passed: false,
151
+ detail: `LLM judge error: ${errMsg}`,
152
+ })
153
+ if (!firstFailure) {
154
+ firstFailure = `llm_judge error: ${errMsg}`
155
+ }
156
+ }
157
+ }
158
+
51
159
  const allPassed = metrics.every(m => m.passed)
52
160
  return {
53
161
  passed: allPassed,
@@ -55,3 +163,10 @@ export function compareBenchmarks(
55
163
  metrics,
56
164
  }
57
165
  }
166
+
167
+ /** Converts step output to a string for text-based assertions. */
168
+ function stringifyOutput(output: unknown): string {
169
+ if (output === null || output === undefined) return ''
170
+ if (typeof output === 'string') return output
171
+ return JSON.stringify(output)
172
+ }
package/src/ci/ed-runner.ts CHANGED
@@ -211,8 +211,8 @@ async function runSingleTest(test: ValidatedTest): Promise<EdTestResult> {
211
211
  }
212
212
  }
213
213
 
214
- // Compare against benchmarks
215
- const benchmarkResult = compareBenchmarks(measurement, test.benchmarks)
214
+ // Compare against benchmarks (async to support llm_judge)
215
+ const benchmarkResult = await compareBenchmarks(measurement, test.benchmarks, targetStep?.output)
216
216
 
217
217
  return {
218
218
  ...base,
@@ -231,7 +231,7 @@ async function runSingleTest(test: ValidatedTest): Promise<EdTestResult> {
231
231
  const traceMeasurement = extractMeasurementFromTrace(test)
232
232
  if (traceMeasurement) {
233
233
  console.log(` [ed-test] ${test.name}: run() failed (${err instanceof Error ? err.message : String(err)}), using trace measurement fallback`)
234
- const benchmarkResult = compareBenchmarks(traceMeasurement, test.benchmarks)
234
+ const benchmarkResult = await compareBenchmarks(traceMeasurement, test.benchmarks, targetStep?.output)
235
235
  return {
236
236
  ...base,
237
237
  testId: test.name,
package/src/ci/test-registry.ts CHANGED
@@ -3,9 +3,26 @@ export interface TestTarget {
3
3
  step_id: string
4
4
  }
5
5
 
6
+ export interface LLMJudgeBenchmark {
7
+ /** The prompt sent to the LLM judge to evaluate the step's output. */
8
+ judge_prompt: string
9
+ /** Minimum score (0-10) required to pass. Defaults to 7. */
10
+ judge_score_threshold?: number
11
+ /** LLM provider to use for judging. Defaults to 'openai'. */
12
+ judge_provider?: 'openai' | 'claude' | 'gemini' | 'grok' | 'kimi'
13
+ /** Model override for the judge LLM. */
14
+ judge_model?: string
15
+ }
16
+
6
17
  export interface TestBenchmarks {
7
18
  max_duration_ms?: number
8
19
  max_tokens_total?: number
20
+ /** Assert that the step's output contains this substring. */
21
+ output_contains?: string
22
+ /** Assert that the step's output does NOT contain this substring. */
23
+ output_not_contains?: string
24
+ /** LLM-as-a-judge evaluation of the step's output quality. */
25
+ llm_judge?: LLMJudgeBenchmark
9
26
  }
10
27
 
11
28
  export interface TestDefinition {
package/src/ci/upload-client.ts CHANGED
@@ -46,7 +46,7 @@ export interface UploadTestResult {
46
46
  }
47
47
 
48
48
  export interface UploadMetric {
49
- name: 'duration_ms' | 'tokens_total' | 'tokens_input' | 'tokens_output'
49
+ name: 'duration_ms' | 'tokens_total' | 'tokens_input' | 'tokens_output' | 'output_contains' | 'output_not_contains' | 'llm_judge'
50
50
  value: number
51
51
  passed: boolean
52
52
  benchmark?: { type: string; threshold: number }