elasticdash-test 0.1.20-alpha-20 → 0.1.20-alpha-22

package/dist/ci/api-client.d.ts

@@ -20,4 +20,19 @@ export declare function submitTestRun(serverUrl: string, apiKey: string, testGro
 export declare function createBatch(serverUrl: string, apiKey: string, payload: Record<string, unknown>): Promise<{
     id: number;
 }>;
+/**
+ * Resolved evaluator configuration from the backend.
+ * Provider/model/apiKey may be null if the user has not configured an evaluator.
+ */
+export interface EvaluatorConfig {
+    provider: string | null;
+    model: string | null;
+    apiKey: string | null;
+}
+/**
+ * Fetch the project's evaluator config (with user-level fallback).
+ * Used by ed-test llm_judge benchmarks when judge_provider/judge_model
+ * are not specified in the test definition.
+ */
+export declare function fetchEvaluatorConfig(serverUrl: string, apiKey: string): Promise<EvaluatorConfig>;
 //# sourceMappingURL=api-client.d.ts.map

package/dist/ci/api-client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"api-client.d.ts","sourceRoot":"","sources":["../../src/ci/api-client.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AA0C9C;;;GAGG;AACH,wBAAsB,eAAe,CACnC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE;IAAE,YAAY,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GACpE,OAAO,CAAC,YAAY,EAAE,CAAC,CAUzB;AAED;;GAEG;AACH,wBAAsB,aAAa,CACjC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,CAOzB;AAED;;GAEG;AACH,wBAAsB,WAAW,CAC/B,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,CAOzB"}
+{"version":3,"file":"api-client.d.ts","sourceRoot":"","sources":["../../src/ci/api-client.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AA0C9C;;;GAGG;AACH,wBAAsB,eAAe,CACnC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE;IAAE,YAAY,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GACpE,OAAO,CAAC,YAAY,EAAE,CAAC,CAUzB;AAED;;GAEG;AACH,wBAAsB,aAAa,CACjC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,CAOzB;AAED;;GAEG;AACH,wBAAsB,WAAW,CAC/B,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,CAOzB;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;CACtB;AAED;;;;GAIG;AACH,wBAAsB,oBAAoB,CACxC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,eAAe,CAAC,CAI1B"}

package/dist/ci/api-client.js

@@ -68,4 +68,14 @@ export async function createBatch(serverUrl, apiKey, payload) {
         body: JSON.stringify(payload),
     });
 }
+/**
+ * Fetch the project's evaluator config (with user-level fallback).
+ * Used by ed-test llm_judge benchmarks when judge_provider/judge_model
+ * are not specified in the test definition.
+ */
+export async function fetchEvaluatorConfig(serverUrl, apiKey) {
+    const base = normalizeBase(serverUrl);
+    const url = `${base}/api/test-runs/evaluator-config`;
+    return apiRequest(url, apiKey);
+}
 //# sourceMappingURL=api-client.js.map

package/dist/ci/api-client.js.map

@@ -1 +1 @@
-{"version":3,"file":"api-client.js","sourceRoot":"","sources":["../../src/ci/api-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAG1D,gEAAgE;AAChE,sDAAsD;AAEtD,iGAAiG;AACjG,SAAS,aAAa,CAAC,SAAiB;IACtC,OAAO,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;AAC5D,CAAC;AAED,SAAS,OAAO,CAAC,MAAc;IAC7B,OAAO;QACL,cAAc,EAAE,kBAAkB;QAClC,SAAS,EAAE,MAAM,IAAI,EAAE;QACvB,kBAAkB,EAAE,UAAU,EAAE;KACjC,CAAA;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,GAAW,EACX,MAAc,EACd,UAAuB,EAAE;IAEzB,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,WAAW,EAAE,CAAA;IACtD,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,IAAI,GAAG,EAAE,CAAC,CAAA;IAEhD,MAAM,GAAG,GAAG,MAAM,gBAAgB,EAAE,CAAC,GAAG,EAAE;QACxC,GAAG,OAAO;QACV,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,OAAiC,IAAI,EAAE,CAAC,EAAE;KACtF,CAAC,CAAA;IAEF,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAA;QAC7C,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;QAC1F,MAAM,IAAI,KAAK,CAAC,OAAO,GAAG,CAAC,MAAM,KAAK,IAAI,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;IACjE,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAA8B,CAAA;IACzD,kFAAkF;IAClF,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAM,CAAA;AAChD,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,SAAiB,EACjB,MAAc,EACd,OAAqE;IAErE,MAAM,IAAI,GAAG,aAAa,CAAC,SAAS,CAAC,CAAA;IACrC,MAAM,MAAM,GAAG,IAAI,eAAe,EAAE,CAAA;IACpC,IAAI,OAAO,EAAE,YAAY;QAAE,MAAM,CAAC,GAAG,CAAC,cAAc,EAAE,OAAO,CAAC,YAAY,CAAC,CAAA;IAC3E,IAAI,OAAO,EAAE,IAAI,EAAE,MAAM;QAAE,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;IACrE,IAAI,OAAO,EAAE,MAAM;QAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;IAEzD,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;IAC5B,MAAM,GAAG,GAAG,GAAG,IAAI,6BAA6B,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAA;IACpE,OAAO,UAAU,CAAiB,GAAG,EAAE,MAAM,CAAC,CAAA;AAChD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAiB,EACjB,MAAc,EACd,WAAmB,EACnB,OAAgC;IAEhC,MAAM,IAAI,GAAG,aAAa,CAAC,SAAS,CAAC,CAAA;IACrC,MAAM,GAAG,GAAG,GAAG,IAAI,mBAAmB,WAAW,OAAO,CAAA;IACxD,OAAO,UAAU,CAAiB,GAAG,EAAE,MAAM,EAAE;QAC7C,MAAM,EAAE,MAAM;QACd,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;KAC9B,CAAC,CAAA;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,SAAiB,EACjB,MAAc,EACd,OAAgC;IAEhC,MAAM,IAAI,GAAG,aAAa,CAAC,SAAS,CAAC,CAAA;IACrC,MAAM,GAAG,GAAG,GAAG,IAAI,yBAAyB,CAAA;IAC5C,OAAO,UAAU,CAAiB,GAAG,EAAE,MAAM,EAAE;QAC7C,MAAM,EAAE,MAAM;QACd,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;KAC9B,CAAC,CAAA;AACJ,CAAC"}
+{"version":3,"file":"api-client.js","sourceRoot":"","sources":["../../src/ci/api-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAG1D,gEAAgE;AAChE,sDAAsD;AAEtD,iGAAiG;AACjG,SAAS,aAAa,CAAC,SAAiB;IACtC,OAAO,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;AAC5D,CAAC;AAED,SAAS,OAAO,CAAC,MAAc;IAC7B,OAAO;QACL,cAAc,EAAE,kBAAkB;QAClC,SAAS,EAAE,MAAM,IAAI,EAAE;QACvB,kBAAkB,EAAE,UAAU,EAAE;KACjC,CAAA;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,GAAW,EACX,MAAc,EACd,UAAuB,EAAE;IAEzB,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,WAAW,EAAE,CAAA;IACtD,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,IAAI,GAAG,EAAE,CAAC,CAAA;IAEhD,MAAM,GAAG,GAAG,MAAM,gBAAgB,EAAE,CAAC,GAAG,EAAE;QACxC,GAAG,OAAO;QACV,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,OAAiC,IAAI,EAAE,CAAC,EAAE;KACtF,CAAC,CAAA;IAEF,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAA;QAC7C,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;QAC1F,MAAM,IAAI,KAAK,CAAC,OAAO,GAAG,CAAC,MAAM,KAAK,IAAI,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;IACjE,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAA8B,CAAA;IACzD,kFAAkF;IAClF,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAM,CAAA;AAChD,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,SAAiB,EACjB,MAAc,EACd,OAAqE;IAErE,MAAM,IAAI,GAAG,aAAa,CAAC,SAAS,CAAC,CAAA;IACrC,MAAM,MAAM,GAAG,IAAI,eAAe,EAAE,CAAA;IACpC,IAAI,OAAO,EAAE,YAAY;QAAE,MAAM,CAAC,GAAG,CAAC,cAAc,EAAE,OAAO,CAAC,YAAY,CAAC,CAAA;IAC3E,IAAI,OAAO,EAAE,IAAI,EAAE,MAAM;QAAE,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;IACrE,IAAI,OAAO,EAAE,MAAM;QAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;IAEzD,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;IAC5B,MAAM,GAAG,GAAG,GAAG,IAAI,6BAA6B,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAA;IACpE,OAAO,UAAU,CAAiB,GAAG,EAAE,MAAM,CAAC,CAAA;AAChD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAiB,EACjB,MAAc,EACd,WAAmB,EACnB,OAAgC;IAEhC,MAAM,IAAI,GAAG,aAAa,CAAC,SAAS,CAAC,CAAA;IACrC,MAAM,GAAG,GAAG,GAAG,IAAI,mBAAmB,WAAW,OAAO,CAAA;IACxD,OAAO,UAAU,CAAiB,GAAG,EAAE,MAAM,EAAE;QAC7C,MAAM,EAAE,MAAM;QACd,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;KAC9B,CAAC,CAAA;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,SAAiB,EACjB,MAAc,EACd,OAAgC;IAEhC,MAAM,IAAI,GAAG,aAAa,CAAC,SAAS,CAAC,CAAA;IACrC,MAAM,GAAG,GAAG,GAAG,IAAI,yBAAyB,CAAA;IAC5C,OAAO,UAAU,CAAiB,GAAG,EAAE,MAAM,EAAE;QAC7C,MAAM,EAAE,MAAM;QACd,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;KAC9B,CAAC,CAAA;AACJ,CAAC;AAYD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,SAAiB,EACjB,MAAc;IAEd,MAAM,IAAI,GAAG,aAAa,CAAC,SAAS,CAAC,CAAA;IACrC,MAAM,GAAG,GAAG,GAAG,IAAI,iCAAiC,CAAA;IACpD,OAAO,UAAU,CAAkB,GAAG,EAAE,MAAM,CAAC,CAAA;AACjD,CAAC"}

package/dist/ci/benchmark.d.ts

@@ -8,6 +8,7 @@
  */
 import type { TestMeasurement } from './measurement.js';
 import type { TestBenchmarks } from './test-registry.js';
+import type { EvaluatorConfig } from './api-client.js';
 export type MetricName = 'duration_ms' | 'tokens_total' | 'output_contains' | 'output_not_contains' | 'llm_judge';
 export interface MetricResult {
     name: MetricName;
@@ -24,6 +25,9 @@ export interface BenchmarkResult {
 /**
  * Compare a measurement against benchmarks. Async because llm_judge requires
  * an LLM call. The step's output is needed for output_contains/llm_judge checks.
+ *
+ * @param evaluatorConfig - Optional backend evaluator config used as fallback
+ *   when the test does not specify judge_provider/judge_model.
  */
-export declare function compareBenchmarks(measurement: TestMeasurement, benchmarks: TestBenchmarks, stepOutput?: unknown): Promise<BenchmarkResult>;
+export declare function compareBenchmarks(measurement: TestMeasurement, benchmarks: TestBenchmarks, stepOutput?: unknown, evaluatorConfig?: EvaluatorConfig | null): Promise<BenchmarkResult>;
 //# sourceMappingURL=benchmark.d.ts.map

package/dist/ci/benchmark.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"benchmark.d.ts","sourceRoot":"","sources":["../../src/ci/benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AACvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AAExD,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,cAAc,GAAG,iBAAiB,GAAG,qBAAqB,GAAG,WAAW,CAAA;AAEjH,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,UAAU,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,OAAO,CAAA;IACf,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,OAAO,EAAE,YAAY,EAAE,CAAA;CACxB;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CACrC,WAAW,EAAE,eAAe,EAC5B,UAAU,EAAE,cAAc,EAC1B,UAAU,CAAC,EAAE,OAAO,GACnB,OAAO,CAAC,eAAe,CAAC,CA+H1B"}
+{"version":3,"file":"benchmark.d.ts","sourceRoot":"","sources":["../../src/ci/benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AACvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AAEtD,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,cAAc,GAAG,iBAAiB,GAAG,qBAAqB,GAAG,WAAW,CAAA;AAEjH,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,UAAU,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,OAAO,CAAA;IACf,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,OAAO,EAAE,YAAY,EAAE,CAAA;CACxB;AAaD;;;;;;GAMG;AACH,wBAAsB,iBAAiB,CACrC,WAAW,EAAE,eAAe,EAC5B,UAAU,EAAE,cAAc,EAC1B,UAAU,CAAC,EAAE,OAAO,EACpB,eAAe,CAAC,EAAE,eAAe,GAAG,IAAI,GACvC,OAAO,CAAC,eAAe,CAAC,CA4J1B"}

package/dist/ci/benchmark.js

@@ -7,11 +7,23 @@
  * Generated/updated on 2026-04-20.
  */
 import { callProviderLLM } from '../matchers/index.js';
+/** Maps backend provider names to SDK provider names used by callProviderLLM. */
+const PROVIDER_NAME_MAP = {
+    anthropic: 'claude',
+    moonshot: 'kimi',
+};
+/** Normalize provider name from backend format to SDK format. */
+function normalizeSdkProvider(provider) {
+    return PROVIDER_NAME_MAP[provider] ?? provider;
+}
 /**
  * Compare a measurement against benchmarks. Async because llm_judge requires
  * an LLM call. The step's output is needed for output_contains/llm_judge checks.
+ *
+ * @param evaluatorConfig - Optional backend evaluator config used as fallback
+ *   when the test does not specify judge_provider/judge_model.
  */
-export async function compareBenchmarks(measurement, benchmarks, stepOutput) {
+export async function compareBenchmarks(measurement, benchmarks, stepOutput, evaluatorConfig) {
     const metrics = [];
     let firstFailure;
     if (benchmarks.max_duration_ms !== undefined) {
@@ -74,10 +86,35 @@ export async function compareBenchmarks(measurement, benchmarks, stepOutput) {
         const judge = benchmarks.llm_judge;
         const outputStr = stringifyOutput(stepOutput);
         const threshold = judge.judge_score_threshold ?? 7;
+        // Resolve provider/model: test definition takes priority, then backend
+        // evaluator config, then fall back to 'openai' default.
+        const resolvedProvider = normalizeSdkProvider(judge.judge_provider ?? evaluatorConfig?.provider ?? 'openai');
+        const resolvedModel = judge.judge_model ?? evaluatorConfig?.model ?? undefined;
+        // If the backend provided an API key and we're using its provider,
+        // set it in the environment so callProviderLLM can pick it up.
+        const envKeyMap = {
+            openai: 'OPENAI_API_KEY',
+            claude: 'ANTHROPIC_API_KEY',
+            gemini: 'GEMINI_API_KEY',
+            grok: 'GROK_API_KEY',
+            kimi: 'KIMI_API_KEY',
+        };
+        const envKey = envKeyMap[resolvedProvider];
+        let restoreEnv;
+        if (evaluatorConfig?.apiKey && envKey && !judge.judge_provider && !process.env[envKey]) {
+            const prev = process.env[envKey];
+            process.env[envKey] = evaluatorConfig.apiKey;
+            restoreEnv = () => {
+                if (prev === undefined)
+                    delete process.env[envKey];
+                else
+                    process.env[envKey] = prev;
+            };
+        }
         try {
-            const provider = judge.judge_provider ?? 'openai';
             const evalPrompt = `${judge.judge_prompt}\n\nOutput to evaluate:\n${outputStr}\n\nScore this output on a scale of 0-10. Respond with only the number.`;
-            const result = await callProviderLLM(evalPrompt, { provider, model: judge.judge_model ?? undefined }, 'You are an expert test judge. Return only a number between 0 and 10.', 16, 0);
+            const result = await callProviderLLM(evalPrompt, { provider: resolvedProvider, model: resolvedModel }, 'You are an expert test judge. Return only a number between 0 and 10.', 16, 0);
+            restoreEnv?.();
             const score = parseFloat(result.content.match(/-?\d+(?:\.\d+)?/)?.[0] ?? '');
             if (isNaN(score)) {
                 metrics.push({
@@ -106,6 +143,7 @@ export async function compareBenchmarks(measurement, benchmarks, stepOutput) {
             }
         }
         catch (err) {
+            restoreEnv?.();
             const errMsg = err instanceof Error ? err.message : String(err);
             metrics.push({
                 name: 'llm_judge',

package/dist/ci/benchmark.js.map

@@ -1 +1 @@
-{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/ci/benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAoBtD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,WAA4B,EAC5B,UAA0B,EAC1B,UAAoB;IAEpB,MAAM,OAAO,GAAmB,EAAE,CAAA;IAClC,IAAI,YAAgC,CAAA;IAEpC,IAAI,UAAU,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,WAAW,CAAC,WAAW,IAAI,UAAU,CAAC,eAAe,CAAA;QACpE,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,aAAa;YACnB,KAAK,EAAE,WAAW,CAAC,WAAW;YAC9B,SAAS,EAAE,UAAU,CAAC,eAAe;YACrC,MAAM;SACP,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,YAAY,GAAG,gBAAgB,WAAW,CAAC,WAAW,6BAA6B,UAAU,CAAC,eAAe,GAAG,CAAA;QAClH,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,WAAW,CAAC,YAAY,IAAI,CAAC,CAAA;QAC3C,MAAM,MAAM,GAAG,KAAK,IAAI,UAAU,CAAC,gBAAgB,CAAA;QACnD,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,cAAc;YACpB,KAAK;YACL,SAAS,EAAE,UAAU,CAAC,gBAAgB;YACtC,MAAM;SACP,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,YAAY,GAAG,iBAAiB,KAAK,6BAA6B,UAAU,CAAC,gBAAgB,GAAG,CAAA;QAClG,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,IAAI,UAAU,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QAC7C,MAAM,SAAS,GAAG,eAAe,CAAC,UAAU,CAAC,CAAA;QAC7C,MAAM,MAAM,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC,CAAA;QACzF,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,iBAAiB;YACvB,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,SAAS,EAAE,CAAC;YACZ,MAAM;YACN,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B,UAAU,CAAC,eAAe,GAAG;SACvF,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,YAAY,GAAG,4BAA4B,UAAU,CAAC,eAAe,GAAG,CAAA;QAC1E,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,IAAI,UAAU,CAAC,mBAAmB,KAAK,SAAS,EAAE,CAAC;QACjD,MAAM,SAAS,GAAG,eAAe,CAAC,UAAU,CAAC,CAAA;QAC7C,MAAM,MAAM,GAAG,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,mBAAmB,CAAC,WAAW,EAAE,CAAC,CAAA;QAC9F,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,qBAAqB;YAC3B,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,SAAS,EAAE,CAAC;YACZ,MAAM;YACN,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,iCAAiC,UAAU,CAAC,mBAAmB,GAAG;SAChG,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,YAAY,GAAG,iCAAiC,UAAU,CAAC,mBAAmB,GAAG,CAAA;QACnF,CAAC;IACH,CAAC;IAED,8DAA8D;IAC9D,IAAI,UAAU,CAAC,SAAS,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,CAAA;QAClC,MAAM,SAAS,GAAG,eAAe,CAAC,UAAU,CAAC,CAAA;QAC7C,MAAM,SAAS,GAAG,KAAK,CAAC,qBAAqB,IAAI,CAAC,CAAA;QAElD,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,IAAI,QAAQ,CAAA;YACjD,MAAM,UAAU,GAAG,GAAG,KAAK,CAAC,YAAY,4BAA4B,SAAS,yEAAyE,CAAA;YAEtJ,MAAM,MAAM,GAAG,MAAM,eAAe,CAClC,UAAU,EACV,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,WAAW,IAAI,SAAS,EAAE,EACnD,sEAAsE,EACtE,EAAE,EACF,CAAC,CACF,CAAA;YAED,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;YAC5E,IAAI,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjB,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,WAAW;oBACjB,KAAK,EAAE,CAAC;oBACR,SAAS;oBACT,MAAM,EAAE,KAAK;oBACb,MAAM,EAAE,6CAA6C,MAAM,CAAC,OAAO,GAAG;iBACvE,CAAC,CAAA;gBACF,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClB,YAAY,GAAG,gDAAgD,CAAA;gBACjE,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,GAAG,KAAK,IAAI,SAAS,CAAA;gBACjC,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,WAAW;oBACjB,KAAK,EAAE,KAAK;oBACZ,SAAS;oBACT,MAAM;oBACN,MAAM,EAAE,UAAU,KAAK,IAAI,SAAS,EAAE;iBACvC,CAAC,CAAA;gBACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;oBAC7B,YAAY,GAAG,oBAAoB,KAAK,sBAAsB,SAAS,GAAG,CAAA;gBAC5E,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YAC/D,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,WAAW;gBACjB,KAAK,EAAE,CAAC;gBACR,SAAS;gBACT,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,oBAAoB,MAAM,EAAE;aACrC,CAAC,CAAA;YACF,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,YAAY,GAAG,oBAAoB,MAAM,EAAE,CAAA;YAC7C,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;IAC9C,OAAO;QACL,MAAM,EAAE,SAAS;QACjB,cAAc,EAAE,YAAY;QAC5B,OAAO;KACR,CAAA;AACH,CAAC;AAED,kEAAkE;AAClE,SAAS,eAAe,CAAC,MAAe;IACtC,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,EAAE,CAAA;IACtD,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAA;IAC7C,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;AAC/B,CAAC"}
+{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/ci/benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAqBtD,iFAAiF;AACjF,MAAM,iBAAiB,GAA2B;IAChD,SAAS,EAAE,QAAQ;IACnB,QAAQ,EAAE,MAAM;CACjB,CAAA;AAED,iEAAiE;AACjE,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,OAAO,iBAAiB,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAA;AAChD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,WAA4B,EAC5B,UAA0B,EAC1B,UAAoB,EACpB,eAAwC;IAExC,MAAM,OAAO,GAAmB,EAAE,CAAA;IAClC,IAAI,YAAgC,CAAA;IAEpC,IAAI,UAAU,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,WAAW,CAAC,WAAW,IAAI,UAAU,CAAC,eAAe,CAAA;QACpE,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,aAAa;YACnB,KAAK,EAAE,WAAW,CAAC,WAAW;YAC9B,SAAS,EAAE,UAAU,CAAC,eAAe;YACrC,MAAM;SACP,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,YAAY,GAAG,gBAAgB,WAAW,CAAC,WAAW,6BAA6B,UAAU,CAAC,eAAe,GAAG,CAAA;QAClH,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,WAAW,CAAC,YAAY,IAAI,CAAC,CAAA;QAC3C,MAAM,MAAM,GAAG,KAAK,IAAI,UAAU,CAAC,gBAAgB,CAAA;QACnD,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,cAAc;YACpB,KAAK;YACL,SAAS,EAAE,UAAU,CAAC,gBAAgB;YACtC,MAAM;SACP,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,YAAY,GAAG,iBAAiB,KAAK,6BAA6B,UAAU,CAAC,gBAAgB,GAAG,CAAA;QAClG,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,IAAI,UAAU,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QAC7C,MAAM,SAAS,GAAG,eAAe,CAAC,UAAU,CAAC,CAAA;QAC7C,MAAM,MAAM,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC,CAAA;QACzF,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,iBAAiB;YACvB,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,SAAS,EAAE,CAAC;YACZ,MAAM;YACN,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B,UAAU,CAAC,eAAe,GAAG;SACvF,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,YAAY,GAAG,4BAA4B,UAAU,CAAC,eAAe,GAAG,CAAA;QAC1E,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,IAAI,UAAU,CAAC,mBAAmB,KAAK,SAAS,EAAE,CAAC;QACjD,MAAM,SAAS,GAAG,eAAe,CAAC,UAAU,CAAC,CAAA;QAC7C,MAAM,MAAM,GAAG,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,mBAAmB,CAAC,WAAW,EAAE,CAAC,CAAA;QAC9F,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,qBAAqB;YAC3B,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,SAAS,EAAE,CAAC;YACZ,MAAM;YACN,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,iCAAiC,UAAU,CAAC,mBAAmB,GAAG;SAChG,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,YAAY,GAAG,iCAAiC,UAAU,CAAC,mBAAmB,GAAG,CAAA;QACnF,CAAC;IACH,CAAC;IAED,8DAA8D;IAC9D,IAAI,UAAU,CAAC,SAAS,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,CAAA;QAClC,MAAM,SAAS,GAAG,eAAe,CAAC,UAAU,CAAC,CAAA;QAC7C,MAAM,SAAS,GAAG,KAAK,CAAC,qBAAqB,IAAI,CAAC,CAAA;QAElD,uEAAuE;QACvE,wDAAwD;QACxD,MAAM,gBAAgB,GAAG,oBAAoB,CAC3C,KAAK,CAAC,cAAc,IAAI,eAAe,EAAE,QAAQ,IAAI,QAAQ,CAC9D,CAAA;QACD,MAAM,aAAa,GAAG,KAAK,CAAC,WAAW,IAAI,eAAe,EAAE,KAAK,IAAI,SAAS,CAAA;QAE9E,mEAAmE;QACnE,+DAA+D;QAC/D,MAAM,SAAS,GAA2B;YACxC,MAAM,EAAE,gBAAgB;YACxB,MAAM,EAAE,mBAAmB;YAC3B,MAAM,EAAE,gBAAgB;YACxB,IAAI,EAAE,cAAc;YACpB,IAAI,EAAE,cAAc;SACrB,CAAA;QACD,MAAM,MAAM,GAAG,SAAS,CAAC,gBAAgB,CAAC,CAAA;QAC1C,IAAI,UAAoC,CAAA;QACxC,IAAI,eAAe,EAAE,MAAM,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACvF,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YAChC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,eAAe,CAAC,MAAM,CAAA;YAC5C,UAAU,GAAG,GAAG,EAAE;gBAChB,IAAI,IAAI,KAAK,SAAS;oBAAE,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;;oBAC7C,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,CAAA;YACjC,CAAC,CAAA;QACH,CAAC;QAED,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,GAAG,KAAK,CAAC,YAAY,4BAA4B,SAAS,yEAAyE,CAAA;YAEtJ,MAAM,MAAM,GAAG,MAAM,eAAe,CAClC,UAAU,EACV,EAAE,QAAQ,EAAE,gBAAoE,EAAE,KAAK,EAAE,aAAa,EAAE,EACxG,sEAAsE,EACtE,EAAE,EACF,CAAC,CACF,CAAA;YAED,UAAU,EAAE,EAAE,CAAA;YAEd,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;YAC5E,IAAI,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjB,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,WAAW;oBACjB,KAAK,EAAE,CAAC;oBACR,SAAS;oBACT,MAAM,EAAE,KAAK;oBACb,MAAM,EAAE,6CAA6C,MAAM,CAAC,OAAO,GAAG;iBACvE,CAAC,CAAA;gBACF,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClB,YAAY,GAAG,gDAAgD,CAAA;gBACjE,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,GAAG,KAAK,IAAI,SAAS,CAAA;gBACjC,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,WAAW;oBACjB,KAAK,EAAE,KAAK;oBACZ,SAAS;oBACT,MAAM;oBACN,MAAM,EAAE,UAAU,KAAK,IAAI,SAAS,EAAE;iBACvC,CAAC,CAAA;gBACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;oBAC7B,YAAY,GAAG,oBAAoB,KAAK,sBAAsB,SAAS,GAAG,CAAA;gBAC5E,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,UAAU,EAAE,EAAE,CAAA;YACd,MAAM,MAAM,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YAC/D,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,WAAW;gBACjB,KAAK,EAAE,CAAC;gBACR,SAAS;gBACT,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,oBAAoB,MAAM,EAAE;aACrC,CAAC,CAAA;YACF,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,YAAY,GAAG,oBAAoB,MAAM,EAAE,CAAA;YAC7C,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;IAC9C,OAAO;QACL,MAAM,EAAE,SAAS;QACjB,cAAc,EAAE,YAAY;QAC5B,OAAO;KACR,CAAA;AACH,CAAC;AAED,kEAAkE;AAClE,SAAS,eAAe,CAAC,MAAe;IACtC,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,EAAE,CAAA;IACtD,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAA;IAC7C,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;AAC/B,CAAC"}

package/dist/ci/ed-runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ed-runner.d.ts","sourceRoot":"","sources":["../../src/ci/ed-runner.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAOrD,MAAM,WAAW,gBAAgB;IAC/B,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,QAAQ,CAAC,EAAE,SAAS,GAAG,MAAM,GAAG,OAAO,CAAA;IACvC,mFAAmF;IACnF,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAA;IACvB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,WAAW,CAAC,EAAE,eAAe,CAAA;IAC7B,eAAe,CAAC,EAAE,eAAe,CAAA;IACjC,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,GAAG,MAAM,CAAA;IACvB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,WAAW,CAAC,EAAE,eAAe,CAAA;IAC7B,eAAe,CAAC,EAAE,eAAe,CAAA;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAA;IAC1C,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,iDAAiD;IACjD,UAAU,CAAC,EAAE,iBAAiB,EAAE,CAAA;CACjC;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,EAAE,YAAY,EAAE,CAAA;IACvB,UAAU,EAAE,MAAM,CAAA;CACnB;AAID,wBAAsB,UAAU,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC,CAsFrF"}
+{"version":3,"file":"ed-runner.d.ts","sourceRoot":"","sources":["../../src/ci/ed-runner.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAOrD,MAAM,WAAW,gBAAgB;IAC/B,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,QAAQ,CAAC,EAAE,SAAS,GAAG,MAAM,GAAG,OAAO,CAAA;IACvC,mFAAmF;IACnF,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAA;IACvB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,WAAW,CAAC,EAAE,eAAe,CAAA;IAC7B,eAAe,CAAC,EAAE,eAAe,CAAA;IACjC,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,GAAG,MAAM,CAAA;IACvB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,WAAW,CAAC,EAAE,eAAe,CAAA;IAC7B,eAAe,CAAC,EAAE,eAAe,CAAA;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAA;IAC1C,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,iDAAiD;IACjD,UAAU,CAAC,EAAE,iBAAiB,EAAE,CAAA;CACjC;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,EAAE,YAAY,EAAE,CAAA;IACvB,UAAU,EAAE,MAAM,CAAA;CACnB;AAID,wBAAsB,UAAU,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC,CAyGrF"}

package/dist/ci/ed-runner.js

@@ -4,6 +4,7 @@ import { createReplayContext, installReplay, uninstallReplay, ReplayMissError }
 import { collectMeasurement } from './measurement.js';
 import { SDK_VERSION } from './trace-schema.js';
 import { compareBenchmarks } from './benchmark.js';
+import { fetchEvaluatorConfig } from './api-client.js';
 // ─── Runner ─────────────────────────────────────────────────
 export async function runEdTests(options) {
     const cwd = options?.cwd ?? process.cwd();
@@ -29,12 +30,29 @@ export async function runEdTests(options) {
         testsToRun = tests.filter(t => matchGlob(t.name, pattern));
     }
     const maxRuns = Math.max(1, options?.runs ?? 1);
+    // Fetch evaluator config from backend if any test uses llm_judge without
+    // explicit provider/model. Cached for the entire run to avoid repeated calls.
+    let evaluatorConfig = null;
+    const needsEvaluatorConfig = testsToRun.some(t => t.benchmarks.llm_judge && (!t.benchmarks.llm_judge.judge_provider || !t.benchmarks.llm_judge.judge_model));
+    if (needsEvaluatorConfig) {
+        const serverUrl = process.env.ELASTICDASH_API_URL ?? process.env.ELASTICDASH_SERVER ?? '';
+        const apiKey = process.env.ELASTICDASH_API_KEY ?? '';
+        if (serverUrl && apiKey) {
+            try {
+                evaluatorConfig = await fetchEvaluatorConfig(serverUrl, apiKey);
+                console.log(`[ed-test] Evaluator config: provider=${evaluatorConfig.provider}, model=${evaluatorConfig.model}, hasKey=${!!evaluatorConfig.apiKey}`);
+            }
+            catch (err) {
+                console.warn(`[ed-test] Could not fetch evaluator config: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+    }
     for (const test of testsToRun) {
         const allRuns = [];
         let bestResult = null;
         for (let attempt = 1; attempt <= maxRuns; attempt++) {
             const runStartedAt = new Date().toISOString();
-            const result = await runSingleTest(test);
+            const result = await runSingleTest(test, evaluatorConfig);
             const runFinishedAt = new Date().toISOString();
             if (attempt > 1) {
                 console.log(`  [ed-test] ${test.name}: run ${attempt}/${maxRuns} — ${result.status}`);
@@ -78,15 +96,21 @@ export async function runEdTests(options) {
     };
 }
 // ─── Single test execution ──────────────────────────────────
-async function runSingleTest(test) {
+async function resolveCustomInput(input) {
+    return typeof input === 'function' ? await input() : input;
+}
+async function runSingleTest(test, evaluatorConfig) {
     const startMs = Date.now();
     const targetStep = test.traceData.steps.find(s => s.step_id === test.target.step_id);
+    const resolvedInput = test.input !== undefined
+        ? await resolveCustomInput(test.input)
+        : targetStep?.input;
     const base = {
         testId: test.name,
         testName: test.name,
         traceRef: test.trace,
         target: { type: test.target.type, step_id: test.target.step_id },
-        input: targetStep?.input,
+        input: resolvedInput,
         output: targetStep?.output,
     };
     // Check run function exists
@@ -133,7 +157,7 @@ async function runSingleTest(test) {
             };
         }
         // Compare against benchmarks (async to support llm_judge)
-        const benchmarkResult = await compareBenchmarks(measurement, test.benchmarks, targetStep?.output);
+        const benchmarkResult = await compareBenchmarks(measurement, test.benchmarks, targetStep?.output, evaluatorConfig);
         return {
             ...base,
             testId: test.name,
@@ -152,7 +176,7 @@ async function runSingleTest(test) {
         const traceMeasurement = extractMeasurementFromTrace(test);
         if (traceMeasurement) {
             console.log(`  [ed-test] ${test.name}: run() failed (${err instanceof Error ? err.message : String(err)}), using trace measurement fallback`);
-            const benchmarkResult = await compareBenchmarks(traceMeasurement, test.benchmarks, targetStep?.output);
+            const benchmarkResult = await compareBenchmarks(traceMeasurement, test.benchmarks, targetStep?.output, evaluatorConfig);
             return {
                 ...base,
                 testId: test.name,

package/dist/ci/ed-runner.js.map

@@ -1 +1 @@
-{"version":3,"file":"ed-runner.js","sourceRoot":"","sources":["../../src/ci/ed-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC5C,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAClG,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AAsDlD,+DAA+D;AAE/D,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAA0B;IACzD,MAAM,GAAG,GAAG,OAAO,EAAE,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAA;IACzC,MAAM,KAAK,GAAG,UAAU,EAAE,CAAA;IAC1B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAC1C,MAAM,OAAO,GAAmB,EAAE,CAAA;IAElC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAA;IAElD,2CAA2C;IAC3C,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC;YACX,MAAM,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS;YACjC,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS;YACnC,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,qBAAqB,GAAG,CAAC,OAAO,EAAE;YACjD,UAAU,EAAE,CAAC;YACb,UAAU,EAAE,EAAE;SACf,CAAC,CAAA;IACJ,CAAC;IAED,mCAAmC;IACnC,IAAI,UAAU,GAAoB,KAAK,CAAA;IACvC,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QACpB,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAA;QAC9B,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAA;IAC5D,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,IAAI,CAAC,CAAC,CAAA;IAE/C,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAwB,EAAE,CAAA;QACvC,IAAI,UAAU,GAAwB,IAAI,CAAA;QAE1C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YACpD,MAAM,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YAC7C,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAA;YACxC,MAAM,aAAa,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YAE9C,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,SAAS,OAAO,IAAI,OAAO,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;YACvF,CAAC;YAED,+BAA+B;YAC/B,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,eAAe,EAAE,MAAM,CAAC,eAAe;gBACvC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,SAAS,EAAE,YAAY;gBACvB,UAAU,EAAE,aAAa;aAC1B,CAAC,CAAA;YAEF,4EAA4E;YAC5E,IAAI,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;gBAC5C,UAAU,GAAG,MAAM,CAAA;YACrB,CAAC;QAEP,CAAC;QAEG,oCAAoC;QACpC,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAA;QACxD,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAA;QAExD,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,UAAW;YACd,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YACnC,aAAa,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,aAAa,IAAI,UAAW,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS;YAC9F,UAAU,EAAE,OAAO;SACpB,CAAC,CAAA;QAEF,IAAI,OAAO,EAAE,QAAQ,IAAI,SAAS,EAAE,CAAC;YACnC,MAAK;QACP,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAE3C,OAAO;QACL,KAAK;QACL,SAAS;QACT,UAAU;QACV,OAAO;QACP,UAAU,EAAE,WAAW;KACxB,CAAA;AACH,CAAC;AAED,+DAA+D;AAE/D,KAAK,UAAU,aAAa,CAAC,IAAmB;IAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC1B,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACpF,MAAM,IAAI,GAA0B;QAClC,MAAM,EAAE,IAAI,CAAC,IAAI;QACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;QACnB,QAAQ,EAAE,IAAI,CAAC,KAAK;QACpB,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;QAChE,KAAK,EAAE,UAAU,EAAE,KAAK;QACxB,MAAM,EAAE,UAAU,EAAE,MAAM;KAC3B,CAAA;IAED,4BAA4B;IAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,OAAO,IAAI,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;QAChD,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,0BAA0B;YACzC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAA;IACH,CAAC;IAED,MAAM,SAAS,GAAG,mBAAmB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAC1E,aAAa,CAAC,SAAS,CAAC,CAAA;IAExB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,IAAI,KAAK,CAAA;QAE1C,MAAM,OAAO,CAAC,IAAI,CAAC;YACjB,IAAI,CAAC,GAAG,EAAE;YACV,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAC/B,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CACjE;SACF,CAAC,CAAA;QAEF,4CAA4C;QAC5C,gEAAgE;QAChE,sEAAsE;QACtE,qEAAqE;QACrE,sEAAsE;QACtE,uEAAuE;QACvE,2CAA2C;QAC3C,IAAI,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAC/C,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,WAAW,GAAG,2BAA2B,CAAC,IAAI,CAAC,IAAI,IAAI,CAAA;YACvD,IAAI,WAAW,EAAE,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,yDAAyD,CAAC,CAAA;YAChG,CAAC;QACH,CAAC;QACD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,gBAAgB,IAAI,CAAC,MAAM,CAAC,OAAO,qCAAqC;gBACvF,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QAED,0DAA0D;QAC1D,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,MAAM,CAAC,CAAA;QAEjG,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAChD,aAAa,EAAE,eAAe,CAAC,cAAc;YAC7C,WAAW;YACX,eAAe;YACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAA;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,4EAA4E;QAC5E,4EAA4E;QAC5E,iEAAiE;QACjE,MAAM,gBAAgB,GAAG,2BAA2B,CAAC,IAAI,CAAC,CAAA;QAC1D,IAAI,gBAAgB,EAAE,CAAC;YACrB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,mBAAmB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAA;YAC7I,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,MAAM,CAAC,CAAA;YACtG,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;gBAChD,aAAa,EAAE,eAAe,CAAC,cAAc;gBAC7C,WAAW,EAAE,gBAAgB;gBAC7B,eAAe;gBACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QAED,IAAI,GAAG,YAAY,eAAe,EAAE,CAAC;YACnC,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,gBAAgB,GAAG,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,EAAE;gBAC9D,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QACD,IAAI,GAAG,YAAY,YAAY,EAAE,CAAC;YAChC,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,wBAAwB,GAAG,CAAC,SAAS,IAAI;gBACxD,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QACD,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,oBAAoB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YACrF,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAA;IACH,CAAC;YAAS,CAAC;QACT,eAAe,EAAE,CAAA;IACnB,CAAC;AACH,CAAC;AAED,+DAA+D;AAE/D;;;;;;GAMG;AACH,SAAS,2BAA2B,CAAC,IAAmB;IACtD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAC9E,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAA;IAE3B,MAAM,MAAM,GAAoB;QAC9B,WAAW,EAAE,IAAI,CAAC,WAAW;KAC9B,CAAA;IAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAA;QACvC,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAA;QACzC,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAA;IACzC,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED,+DAA+D;AAE/D,MAAM,YAAa,SAAQ,KAAK;IACX;IAAnB,YAAmB,SAAiB;QAClC,KAAK,CAAC,wBAAwB,SAAS,IAAI,CAAC,CAAA;QAD3B,cAAS,GAAT,SAAS,CAAQ;QAElC,IAAI,CAAC,IAAI,GAAG,cAAc,CAAA;IAC5B,CAAC;CACF;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,OAAe;IAC9C,0CAA0C;IAC1C,MAAM,KAAK,GAAG,IAAI,MAAM,CACtB,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,GAAG,CAClG,CAAA;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACzB,CAAC"}
+{"version":3,"file":"ed-runner.js","sourceRoot":"","sources":["../../src/ci/ed-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC5C,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAClG,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AAClD,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAA;AAuDtD,+DAA+D;AAE/D,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAA0B;IACzD,MAAM,GAAG,GAAG,OAAO,EAAE,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAA;IACzC,MAAM,KAAK,GAAG,UAAU,EAAE,CAAA;IAC1B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAC1C,MAAM,OAAO,GAAmB,EAAE,CAAA;IAElC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAA;IAElD,2CAA2C;IAC3C,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC;YACX,MAAM,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS;YACjC,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS;YACnC,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,qBAAqB,GAAG,CAAC,OAAO,EAAE;YACjD,UAAU,EAAE,CAAC;YACb,UAAU,EAAE,EAAE;SACf,CAAC,CAAA;IACJ,CAAC;IAED,mCAAmC;IACnC,IAAI,UAAU,GAAoB,KAAK,CAAA;IACvC,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QACpB,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAA;QAC9B,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAA;IAC5D,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,IAAI,CAAC,CAAC,CAAA;IAE/C,yEAAyE;IACzE,8EAA8E;IAC9E,IAAI,eAAe,GAA2B,IAAI,CAAA;IAClD,MAAM,oBAAoB,GAAG,UAAU,CAAC,IAAI,CAC1C,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,cAAc,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,WAAW,CAAC,CAC/G,CAAA;IACD,IAAI,oBAAoB,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,EAAE,CAAA;QACzF,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,EAAE,CAAA;QACpD,IAAI,SAAS,IAAI,MAAM,EAAE,CAAC;YACxB,IAAI,CAAC;gBACH,eAAe,GAAG,MAAM,oBAAoB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;gBAC/D,OAAO,CAAC,GAAG,CAAC,wCAAwC,eAAe,CAAC,QAAQ,WAAW,eAAe,CAAC,KAAK,YAAY,CAAC,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,CAAA;YACrJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,IAAI,CAAC,+CAA+C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;YACjH,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAwB,EAAE,CAAA;QACvC,IAAI,UAAU,GAAwB,IAAI,CAAA;QAE1C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YACpD,MAAM,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YAC7C,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,eAAe,CAAC,CAAA;YACzD,MAAM,aAAa,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YAE9C,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,SAAS,OAAO,IAAI,OAAO,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;YACvF,CAAC;YAED,+BAA+B;YAC/B,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,eAAe,EAAE,MAAM,CAAC,eAAe;gBACvC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,SAAS,EAAE,YAAY;gBACvB,UAAU,EAAE,aAAa;aAC1B,CAAC,CAAA;YAEF,4EAA4E;YAC5E,IAAI,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;gBAC5C,UAAU,GAAG,MAAM,CAAA;YACrB,CAAC;QAEP,CAAC;QAEG,oCAAoC;QACpC,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAA;QACxD,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAA;QAExD,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,UAAW;YACd,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YACnC,aAAa,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,aAAa,IAAI,UAAW,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS;YAC9F,UAAU,EAAE,OAAO;SACpB,CAAC,CAAA;QAEF,IAAI,OAAO,EAAE,QAAQ,IAAI,SAAS,EAAE,CAAC;YACnC,MAAK;QACP,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAE3C,OAAO;QACL,KAAK;QACL,SAAS;QACT,UAAU;QACV,OAAO;QACP,UAAU,EAAE,WAAW;KACxB,CAAA;AACH,CAAC;AAED,+DAA+D;AAE/D,KAAK,UAAU,kBAAkB,CAAC,KAAmD;IACnF,OAAO,OAAO,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,MAAO,KAA0C,EAAE,CAAC,CAAC,CAAC,KAAK,CAAA;AAClG,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,IAAmB,EAAE,eAAwC;IACxF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC1B,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAEpF,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,KAAK,SAAS;QAC5C,CAAC,CAAC,MAAM,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC;QACtC,CAAC,CAAC,UAAU,EAAE,KAAK,CAAA;IAErB,MAAM,IAAI,GAA0B;QAClC,MAAM,EAAE,IAAI,CAAC,IAAI;QACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;QACnB,QAAQ,EAAE,IAAI,CAAC,KAAK;QACpB,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;QAChE,KAAK,EAAE,aAAa;QACpB,MAAM,EAAE,UAAU,EAAE,MAAM;KAC3B,CAAA;IAED,4BAA4B;IAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,OAAO,IAAI,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;QAChD,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,0BAA0B;YACzC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAA;IACH,CAAC;IAED,MAAM,SAAS,GAAG,mBAAmB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAC1E,aAAa,CAAC,SAAS,CAAC,CAAA;IAExB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,IAAI,KAAK,CAAA;QAE1C,MAAM,OAAO,CAAC,IAAI,CAAC;YACjB,IAAI,CAAC,GAAG,EAAE;YACV,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAC/B,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CACjE;SACF,CAAC,CAAA;QAEF,4CAA4C;QAC5C,gEAAgE;QAChE,sEAAsE;QACtE,qEAAqE;QACrE,sEAAsE;QACtE,uEAAuE;QACvE,2CAA2C;QAC3C,IAAI,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAC/C,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,WAAW,GAAG,2BAA2B,CAAC,IAAI,CAAC,IAAI,IAAI,CAAA;YACvD,IAAI,WAAW,EAAE,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,yDAAyD,CAAC,CAAA;YAChG,CAAC;QACH,CAAC;QACD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,gBAAgB,IAAI,CAAC,MAAM,CAAC,OAAO,qCAAqC;gBACvF,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QAED,0DAA0D;QAC1D,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,MAAM,EAAE,eAAe,CAAC,CAAA;QAElH,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAChD,aAAa,EAAE,eAAe,CAAC,cAAc;YAC7C,WAAW;YACX,eAAe;YACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAA;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,4EAA4E;QAC5E,4EAA4E;QAC5E,iEAAiE;QACjE,MAAM,gBAAgB,GAAG,2BAA2B,CAAC,IAAI,CAAC,CAAA;QAC1D,IAAI,gBAAgB,EAAE,CAAC;YACrB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,mBAAmB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAA;YAC7I,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,MAAM,EAAE,eAAe,CAAC,CAAA;YACvH,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;gBAChD,aAAa,EAAE,eAAe,CAAC,cAAc;gBAC7C,WAAW,EAAE,gBAAgB;gBAC7B,eAAe;gBACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QAED,IAAI,GAAG,YAAY,eAAe,EAAE,CAAC;YACnC,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,gBAAgB,GAAG,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,EAAE;gBAC9D,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QACD,IAAI,GAAG,YAAY,YAAY,EAAE,CAAC;YAChC,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,wBAAwB,GAAG,CAAC,SAAS,IAAI;gBACxD,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QACD,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,oBAAoB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YACrF,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAA;IACH,CAAC;YAAS,CAAC;QACT,eAAe,EAAE,CAAA;IACnB,CAAC;AACH,CAAC;AAED,+DAA+D;AAE/D;;;;;;GAMG;AACH,SAAS,2BAA2B,CAAC,IAAmB;IACtD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAC9E,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAA;IAE3B,MAAM,MAAM,GAAoB;QAC9B,WAAW,EAAE,IAAI,CAAC,WAAW;KAC9B,CAAA;IAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAA;QACvC,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAA;QACzC,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAA;IACzC,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED,+DAA+D;AAE/D,MAAM,YAAa,SAAQ,KAAK;IACX;IAAnB,YAAmB,SAAiB;QAClC,KAAK,CAAC,wBAAwB,SAAS,IAAI,CAAC,CAAA;QAD3B,cAAS,GAAT,SAAS,CAAQ;QAElC,IAAI,CAAC,IAAI,GAAG,cAAc,CAAA;IAC5B,CAAC;CACF;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,OAAe;IAC9C,0CAA0C;IAC1C,MAAM,KAAK,GAAG,IAAI,MAAM,CACtB,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,GAAG,CAClG,CAAA;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACzB,CAAC"}

package/dist/ci/test-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"test-loader.d.ts","sourceRoot":"","sources":["../../src/ci/test-loader.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAA;AAElD,MAAM,WAAW,aAAc,SAAQ,cAAc;IACnD,iBAAiB,EAAE,MAAM,CAAA;IACzB,SAAS,EAAE,SAAS,CAAA;CACrB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,aAAa,EAAE,CAAA;IACtB,MAAM,EAAE,eAAe,EAAE,CAAA;CAC1B;AAED,wBAAsB,SAAS,CAAC,OAAO,CAAC,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,eAAe,CAAC,CA+JpF"}
+{"version":3,"file":"test-loader.d.ts","sourceRoot":"","sources":["../../src/ci/test-loader.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAA;AAElD,MAAM,WAAW,aAAc,SAAQ,cAAc;IACnD,iBAAiB,EAAE,MAAM,CAAA;IACzB,SAAS,EAAE,SAAS,CAAA;CACrB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,aAAa,EAAE,CAAA;IACtB,MAAM,EAAE,eAAe,EAAE,CAAA;CAC1B;AAED,wBAAsB,SAAS,CAAC,OAAO,CAAC,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,eAAe,CAAC,CAkKpF"}

package/dist/ci/test-loader.js

@@ -100,7 +100,10 @@ export async function loadTests(options) {
         }
         const hasMaxDuration = def.benchmarks.max_duration_ms !== undefined;
         const hasMaxTokens = def.benchmarks.max_tokens_total !== undefined;
-        if (!hasMaxDuration && !hasMaxTokens) {
+        const hasOutputContains = def.benchmarks.output_contains !== undefined;
+        const hasOutputNotContains = def.benchmarks.output_not_contains !== undefined;
+        const hasLLMJudge = def.benchmarks.llm_judge !== undefined;
+        if (!hasMaxDuration && !hasMaxTokens && !hasOutputContains && !hasOutputNotContains && !hasLLMJudge) {
             errors.push({ file, testName: def.name, message: 'At least one benchmark must be specified' });
             continue;
         }

package/dist/ci/test-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"test-loader.js","sourceRoot":"","sources":["../../src/ci/test-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AACxC,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAA;AACvD,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AAoBvE,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAA0B;IACxD,MAAM,GAAG,GAAG,OAAO,EAAE,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAA;IACzC,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,EAAE,GAAG,EAAE,CAAC,CAAA;IAE9C,MAAM,cAAc,GAAqB,EAAE,CAAA;IAC3C,MAAM,MAAM,GAAsB,EAAE,CAAA;IAEpC,yDAAyD;IACzD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,iBAAiB,EAAE,CAAA;QACnB,IAAI,CAAC;YACH,yEAAyE;YACzE,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,EAAE,CAAA;YAC7D,MAAM,MAAM,CAAC,OAAO,CAAC,CAAA;QACvB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI;gBACJ,OAAO,EAAE,+BAA+B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;aAC3F,CAAC,CAAA;YACF,SAAQ;QACV,CAAC;QACD,MAAM,IAAI,GAAG,eAAe,EAAE,CAAA;QAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,GAAG,CAAC,WAAW,GAAG,IAAI,CAAA;QACxB,CAAC;QACD,cAAc,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAA;IAC9B,CAAC;IACD,iBAAiB,EAAE,CAAA;IAEnB,gCAAgC;IAChC,MAAM,KAAK,GAAoB,EAAE,CAAA;IACjC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAA,CAAC,qBAAqB;IAEjE,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,GAAG,CAAC,WAAY,CAAA;QAE7B,gBAAgB;QAChB,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9E,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,sCAAsC,EAAE,CAAC,CAAA;YAC1F,SAAQ;QACV,CAAC;QAED,mBAAmB;QACnB,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI;gBACJ,QAAQ,EAAE,GAAG,CAAC,IAAI;gBAClB,OAAO,EAAE,wBAAwB,GAAG,CAAC,IAAI,sBAAsB,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG;aAC1F,CAAC,CAAA;YACF,SAAQ;QACV,CAAC;QACD,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;QAE7B,2DAA2D;QAC3D,MAAM,iBAAiB,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAA;QAE3D,4BAA4B;QAC5B,IAAI,SAAoB,CAAA;QACxB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAA;YACtD,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAc,CAAA;QAC1C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI;gBACJ,QAAQ,EAAE,GAAG,CAAC,IAAI;gBAClB,OAAO,EAAE,iCAAiC,GAAG,CAAC,KAAK,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;aAC5G,CAAC,CAAA;YACF,SAAQ;QACV,CAAC;QAED,iCAAiC;QACjC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YACpC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,eAAe,GAAG,CAAC,KAAK,sBAAsB,EAAE,CAAC,CAAA;YAClG,SAAQ;QACV,CAAC;QAED,kBAAkB;QAClB,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YAC3D,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,mCAAmC,EAAE,CAAC,CAAA;YACvF,SAAQ;QACV,CAAC;QAED,MAAM,WAAW,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QAC/E,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI;gBACJ,QAAQ,EAAE,GAAG,CAAC,IAAI;gBAClB,OAAO,EAAE,YAAY,GAAG,CAAC,MAAM,CAAC,OAAO,yBAAyB,GAAG,CAAC,KAAK,GAAG;aAC7E,CAAC,CAAA;YACF,SAAQ;QACV,CAAC;QAED,IAAI,WAAW,CAAC,IAAI,KAAK,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACzC,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI;gBACJ,QAAQ,EAAE,GAAG,CAAC,IAAI;gBAClB,OAAO,EAAE,gBAAgB,GAAG,CAAC,MAAM,CAAC,IAAI,+BAA+B,WAAW,CAAC,IAAI,kBAAkB,GAAG,CAAC,MAAM,CAAC,OAAO,GAAG;aAC/H,CAAC,CAAA;YACF,SAAQ;QACV,CAAC;QAED,sBAAsB;QACtB,IAAI,CAAC,GAAG,CAAC,UAAU,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;YAC1D,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,8BAA8B,EAAE,CAAC,CAAA;YAClF,SAAQ;QACV,CAAC;QAED,MAAM,cAAc,GAAG,GAAG,CAAC,UAAU,CAAC,eAAe,KAAK,SAAS,CAAA;QACnE,MAAM,YAAY,GAAG,GAAG,CAAC,UAAU,CAAC,gBAAgB,KAAK,SAAS,CAAA;QAElE,IAAI,CAAC,cAAc,IAAI,CAAC,YAAY,EAAE,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,0CAA0C,EAAE,CAAC,CAAA;YAC9F,SAAQ;QACV,CAAC;QAED,IAAI,KAAK,GAAG,IAAI,CAAA;QAEhB,IAAI,cAAc,EAAE,CAAC;YACnB,IAAI,OAAO,GAAG,CAAC,UAAU,CAAC,eAAe,KAAK,QAAQ,IAAI,GAAG,CAAC,UAAU,CAAC,eAAe,IAAI,CAAC,EAAE,CAAC;gBAC9F,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,2CAA2C,EAAE,CAAC,CAAA;gBAC/F,KAAK,GAAG,KAAK,CAAA;YACf,CAAC;QACH,CAAC;QAED,IAAI,YAAY,EAAE,CAAC;YACjB,IAAI,OAAO,GAAG,CAAC,UAAU,CAAC,gBAAgB,KAAK,QAAQ,IAAI,GAAG,CAAC,UAAU,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;gBAChG,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,4CAA4C,EAAE,CAAC,CAAA;gBAChG,KAAK,GAAG,KAAK,CAAA;YACf,CAAC;YACD,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,wDAAwD,EAAE,CAAC,CAAA;gBAC5G,KAAK,GAAG,KAAK,CAAA;YACf,CAAC;QACH,CAAC;QAED,IAAI,CAAC,KAAK;YAAE,SAAQ;QAEpB,4DAA4D;QAC5D,IAAI,GAAG,CAAC,GAAG,KAAK,SAAS,IAAI,OAAO,GAAG,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;YAC3D,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,wBAAwB,EAAE,CAAC,CAAA;YAC5E,SAAQ;QACV,CAAC;QAED,0EAA0E;QAC1E,IAAI,GAAG,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YACjC,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,IAAI,GAAG,CAAC,UAAU,IAAI,CAAC,EAAE,CAAC;gBAC9D,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,sCAAsC,EAAE,CAAC,CAAA;gBAC1F,SAAQ;YACV,CAAC;QACH,CAAC;QAED,KAAK,CAAC,IAAI,CAAC;YACT,GAAG,GAAG;YACN,iBAAiB;YACjB,SAAS;SACV,CAAC,CAAA;IACJ,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAA;AAC1B,CAAC"}
+{"version":3,"file":"test-loader.js","sourceRoot":"","sources":["../../src/ci/test-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AACxC,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAA;AACvD,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AAoBvE,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAA0B;IACxD,MAAM,GAAG,GAAG,OAAO,EAAE,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAA;IACzC,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,EAAE,GAAG,EAAE,CAAC,CAAA;IAE9C,MAAM,cAAc,GAAqB,EAAE,CAAA;IAC3C,MAAM,MAAM,GAAsB,EAAE,CAAA;IAEpC,yDAAyD;IACzD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,iBAAiB,EAAE,CAAA;QACnB,IAAI,CAAC;YACH,yEAAyE;YACzE,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,EAAE,CAAA;YAC7D,MAAM,MAAM,CAAC,OAAO,CAAC,CAAA;QACvB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI;gBACJ,OAAO,EAAE,+BAA+B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;aAC3F,CAAC,CAAA;YACF,SAAQ;QACV,CAAC;QACD,MAAM,IAAI,GAAG,eAAe,EAAE,CAAA;QAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,GAAG,CAAC,WAAW,GAAG,IAAI,CAAA;QACxB,CAAC;QACD,cAAc,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAA;IAC9B,CAAC;IACD,iBAAiB,EAAE,CAAA;IAEnB,gCAAgC;IAChC,MAAM,KAAK,GAAoB,EAAE,CAAA;IACjC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAA,CAAC,qBAAqB;IAEjE,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,GAAG,CAAC,WAAY,CAAA;QAE7B,gBAAgB;QAChB,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9E,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,sCAAsC,EAAE,CAAC,CAAA;YAC1F,SAAQ;QACV,CAAC;QAED,mBAAmB;QACnB,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI;gBACJ,QAAQ,EAAE,GAAG,CAAC,IAAI;gBAClB,OAAO,EAAE,wBAAwB,GAAG,CAAC,IAAI,sBAAsB,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG;aAC1F,CAAC,CAAA;YACF,SAAQ;QACV,CAAC;QACD,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;QAE7B,2DAA2D;QAC3D,MAAM,iBAAiB,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAA;QAE3D,4BAA4B;QAC5B,IAAI,SAAoB,CAAA;QACxB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAA;YACtD,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAc,CAAA;QAC1C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI;gBACJ,QAAQ,EAAE,GAAG,CAAC,IAAI;gBAClB,OAAO,EAAE,iCAAiC,GAAG,CAAC,KAAK,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;aAC5G,CAAC,CAAA;YACF,SAAQ;QACV,CAAC;QAED,iCAAiC;QACjC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YACpC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,eAAe,GAAG,CAAC,KAAK,sBAAsB,EAAE,CAAC,CAAA;YAClG,SAAQ;QACV,CAAC;QAED,kBAAkB;QAClB,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YAC3D,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,mCAAmC,EAAE,CAAC,CAAA;YACvF,SAAQ;QACV,CAAC;QAED,MAAM,WAAW,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QAC/E,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI;gBACJ,QAAQ,EAAE,GAAG,CAAC,IAAI;gBAClB,OAAO,EAAE,YAAY,GAAG,CAAC,MAAM,CAAC,OAAO,yBAAyB,GAAG,CAAC,KAAK,GAAG;aAC7E,CAAC,CAAA;YACF,SAAQ;QACV,CAAC;QAED,IAAI,WAAW,CAAC,IAAI,KAAK,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACzC,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI;gBACJ,QAAQ,EAAE,GAAG,CAAC,IAAI;gBAClB,OAAO,EAAE,gBAAgB,GAAG,CAAC,MAAM,CAAC,IAAI,+BAA+B,WAAW,CAAC,IAAI,kBAAkB,GAAG,CAAC,MAAM,CAAC,OAAO,GAAG;aAC/H,CAAC,CAAA;YACF,SAAQ;QACV,CAAC;QAED,sBAAsB;QACtB,IAAI,CAAC,GAAG,CAAC,UAAU,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;YAC1D,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,8BAA8B,EAAE,CAAC,CAAA;YAClF,SAAQ;QACV,CAAC;QAED,MAAM,cAAc,GAAG,GAAG,CAAC,UAAU,CAAC,eAAe,KAAK,SAAS,CAAA;QACnE,MAAM,YAAY,GAAG,GAAG,CAAC,UAAU,CAAC,gBAAgB,KAAK,SAAS,CAAA;QAClE,MAAM,iBAAiB,GAAG,GAAG,CAAC,UAAU,CAAC,eAAe,KAAK,SAAS,CAAA;QACtE,MAAM,oBAAoB,GAAG,GAAG,CAAC,UAAU,CAAC,mBAAmB,KAAK,SAAS,CAAA;QAC7E,MAAM,WAAW,GAAG,GAAG,CAAC,UAAU,CAAC,SAAS,KAAK,SAAS,CAAA;QAE1D,IAAI,CAAC,cAAc,IAAI,CAAC,YAAY,IAAI,CAAC,iBAAiB,IAAI,CAAC,oBAAoB,IAAI,CAAC,WAAW,EAAE,CAAC;YACpG,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,0CAA0C,EAAE,CAAC,CAAA;YAC9F,SAAQ;QACV,CAAC;QAED,IAAI,KAAK,GAAG,IAAI,CAAA;QAEhB,IAAI,cAAc,EAAE,CAAC;YACnB,IAAI,OAAO,GAAG,CAAC,UAAU,CAAC,eAAe,KAAK,QAAQ,IAAI,GAAG,CAAC,UAAU,CAAC,eAAe,IAAI,CAAC,EAAE,CAAC;gBAC9F,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,2CAA2C,EAAE,CAAC,CAAA;gBAC/F,KAAK,GAAG,KAAK,CAAA;YACf,CAAC;QACH,CAAC;QAED,IAAI,YAAY,EAAE,CAAC;YACjB,IAAI,OAAO,GAAG,CAAC,UAAU,CAAC,gBAAgB,KAAK,QAAQ,IAAI,GAAG,CAAC,UAAU,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;gBAChG,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,4CAA4C,EAAE,CAAC,CAAA;gBAChG,KAAK,GAAG,KAAK,CAAA;YACf,CAAC;YACD,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,wDAAwD,EAAE,CAAC,CAAA;gBAC5G,KAAK,GAAG,KAAK,CAAA;YACf,CAAC;QACH,CAAC;QAED,IAAI,CAAC,KAAK;YAAE,SAAQ;QAEpB,4DAA4D;QAC5D,IAAI,GAAG,CAAC,GAAG,KAAK,SAAS,IAAI,OAAO,GAAG,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;YAC3D,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,wBAAwB,EAAE,CAAC,CAAA;YAC5E,SAAQ;QACV,CAAC;QAED,0EAA0E;QAC1E,IAAI,GAAG,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YACjC,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,IAAI,GAAG,CAAC,UAAU,IAAI,CAAC,EAAE,CAAC;gBAC9D,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,sCAAsC,EAAE,CAAC,CAAA;gBAC1F,SAAQ;YACV,CAAC;QACH,CAAC;QAED,KAAK,CAAC,IAAI,CAAC;YACT,GAAG,GAAG;YACN,iBAAiB;YACjB,SAAS;SACV,CAAC,CAAA;IACJ,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAA;AAC1B,CAAC"}

package/dist/ci/test-registry.d.ts

@@ -27,6 +27,8 @@ export interface TestDefinition {
     trace: string;
     target: TestTarget;
     benchmarks: TestBenchmarks;
+    /** Custom input that overrides the trace's recorded input. Can be a static value or an async function for dynamic resolution (e.g. fetching from a database or API). */
+    input?: unknown | (() => Promise<unknown> | unknown);
     /** The function that invokes the workflow under test. Required for execution (Phase 3). */
     run?: () => Promise<void>;
     /** Per-test timeout in milliseconds. Defaults to 60000. */

package/dist/ci/test-registry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"test-registry.d.ts","sourceRoot":"","sources":["../../src/ci/test-registry.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,WAAW,GAAG,SAAS,CAAA;IAC7B,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,sEAAsE;IACtE,YAAY,EAAE,MAAM,CAAA;IACpB,4DAA4D;IAC5D,qBAAqB,CAAC,EAAE,MAAM,CAAA;IAC9B,6DAA6D;IAC7D,cAAc,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,CAAA;IACjE,wCAAwC;IACxC,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,6DAA6D;IAC7D,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,qEAAqE;IACrE,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,8DAA8D;IAC9D,SAAS,CAAC,EAAE,iBAAiB,CAAA;CAC9B;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,UAAU,CAAA;IAClB,UAAU,EAAE,cAAc,CAAA;IAC1B,2FAA2F;IAC3F,GAAG,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IACzB,2DAA2D;IAC3D,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAWD,wBAAgB,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,GAAG,IAAI,CAEzE;AAED,wBAAgB,eAAe,IAAI,cAAc,EAAE,CAElD;AAED,wBAAgB,iBAAiB,IAAI,IAAI,CAExC"}
+{"version":3,"file":"test-registry.d.ts","sourceRoot":"","sources":["../../src/ci/test-registry.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,WAAW,GAAG,SAAS,CAAA;IAC7B,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,sEAAsE;IACtE,YAAY,EAAE,MAAM,CAAA;IACpB,4DAA4D;IAC5D,qBAAqB,CAAC,EAAE,MAAM,CAAA;IAC9B,6DAA6D;IAC7D,cAAc,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,CAAA;IACjE,wCAAwC;IACxC,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,6DAA6D;IAC7D,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,qEAAqE;IACrE,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,8DAA8D;IAC9D,SAAS,CAAC,EAAE,iBAAiB,CAAA;CAC9B;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,UAAU,CAAA;IAClB,UAAU,EAAE,cAAc,CAAA;IAC1B,wKAAwK;IACxK,KAAK,CAAC,EAAE,OAAO,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,CAAA;IACpD,2FAA2F;IAC3F,GAAG,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IACzB,2DAA2D;IAC3D,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAWD,wBAAgB,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,GAAG,IAAI,CAEzE;AAED,wBAAgB,eAAe,IAAI,cAAc,EAAE,CAElD;AAED,wBAAgB,iBAAiB,IAAI,IAAI,CAExC"}

package/dist/ci/test-registry.js.map

@@ -1 +1 @@
-{"version":3,"file":"test-registry.js","sourceRoot":"","sources":["../../src/ci/test-registry.ts"],"names":[],"mappings":"AAwCA,4EAA4E;AAC5E,6EAA6E;AAC7E,iCAAiC;AACjC,MAAM,CAAC,GAAG,UAAqC,CAAA;AAC/C,MAAM,oBAAoB,GAAG,kCAAkC,CAAA;AAC/D,MAAM,QAAQ,GACX,CAAC,CAAC,oBAAoB,CAAsB,IAAI,EAAE,CAAA;AACrD,IAAI,CAAC,CAAC,CAAC,oBAAoB,CAAC;IAAE,CAAC,CAAC,oBAAoB,CAAC,GAAG,QAAQ,CAAA;AAEhE,MAAM,UAAU,UAAU,CAAC,GAAwC;IACjE,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,GAAG,EAAE,CAAC,CAAA;AAC3B,CAAC;AAED,MAAM,UAAU,eAAe;IAC7B,OAAO,CAAC,GAAG,QAAQ,CAAC,CAAA;AACtB,CAAC;AAED,MAAM,UAAU,iBAAiB;IAC/B,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAA;AACrB,CAAC"}
+{"version":3,"file":"test-registry.js","sourceRoot":"","sources":["../../src/ci/test-registry.ts"],"names":[],"mappings":"AA0CA,4EAA4E;AAC5E,6EAA6E;AAC7E,iCAAiC;AACjC,MAAM,CAAC,GAAG,UAAqC,CAAA;AAC/C,MAAM,oBAAoB,GAAG,kCAAkC,CAAA;AAC/D,MAAM,QAAQ,GACX,CAAC,CAAC,oBAAoB,CAAsB,IAAI,EAAE,CAAA;AACrD,IAAI,CAAC,CAAC,CAAC,oBAAoB,CAAC;IAAE,CAAC,CAAC,oBAAoB,CAAC,GAAG,QAAQ,CAAA;AAEhE,MAAM,UAAU,UAAU,CAAC,GAAwC;IACjE,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,GAAG,EAAE,CAAC,CAAA;AAC3B,CAAC;AAED,MAAM,UAAU,eAAe;IAC7B,OAAO,CAAC,GAAG,QAAQ,CAAC,CAAA;AACtB,CAAC;AAED,MAAM,UAAU,iBAAiB;IAC/B,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAA;AACrB,CAAC"}

package/dist/index.cjs

@@ -4392,6 +4392,7 @@ __export(index_exports, {
   expect: () => import_expect.expect,
   extractTaskOutputs: () => extractTaskOutputs,
   fetchCapturedTrace: () => fetchCapturedTrace,
+  fetchEvaluatorConfig: () => fetchEvaluatorConfig,
   fetchTestGroups: () => fetchTestGroups,
   getCaptureContext: () => getCaptureContext,
   getCurrentTrace: () => getCurrentTrace,
@@ -6246,7 +6247,10 @@ async function loadTests(options) {
     }
     const hasMaxDuration = def.benchmarks.max_duration_ms !== void 0;
     const hasMaxTokens = def.benchmarks.max_tokens_total !== void 0;
-    if (!hasMaxDuration && !hasMaxTokens) {
+    const hasOutputContains = def.benchmarks.output_contains !== void 0;
+    const hasOutputNotContains = def.benchmarks.output_not_contains !== void 0;
+    const hasLLMJudge = def.benchmarks.llm_judge !== void 0;
+    if (!hasMaxDuration && !hasMaxTokens && !hasOutputContains && !hasOutputNotContains && !hasLLMJudge) {
       errors.push({ file, testName: def.name, message: "At least one benchmark must be specified" });
       continue;
     }
@@ -6288,7 +6292,7 @@ async function loadTests(options) {
 }
 
 // src/ci/ed-runner.ts
-var import_node_crypto8 = require("node:crypto");
+var import_node_crypto9 = require("node:crypto");
 
 // src/ci/measurement.ts
 function collectMeasurement(ctx) {
@@ -6310,7 +6314,14 @@ init_trace_schema();
 
 // src/ci/benchmark.ts
 init_matchers();
-async function compareBenchmarks(measurement, benchmarks, stepOutput) {
+var PROVIDER_NAME_MAP = {
+  anthropic: "claude",
+  moonshot: "kimi"
+};
+function normalizeSdkProvider(provider) {
+  return PROVIDER_NAME_MAP[provider] ?? provider;
+}
+async function compareBenchmarks(measurement, benchmarks, stepOutput, evaluatorConfig) {
   const metrics = [];
   let firstFailure;
   if (benchmarks.max_duration_ms !== void 0) {
@@ -6370,8 +6381,28 @@ async function compareBenchmarks(measurement, benchmarks, stepOutput) {
     const judge = benchmarks.llm_judge;
     const outputStr = stringifyOutput(stepOutput);
     const threshold = judge.judge_score_threshold ?? 7;
+    const resolvedProvider = normalizeSdkProvider(
+      judge.judge_provider ?? evaluatorConfig?.provider ?? "openai"
+    );
+    const resolvedModel = judge.judge_model ?? evaluatorConfig?.model ?? void 0;
+    const envKeyMap = {
+      openai: "OPENAI_API_KEY",
+      claude: "ANTHROPIC_API_KEY",
+      gemini: "GEMINI_API_KEY",
+      grok: "GROK_API_KEY",
+      kimi: "KIMI_API_KEY"
+    };
+    const envKey = envKeyMap[resolvedProvider];
+    let restoreEnv;
+    if (evaluatorConfig?.apiKey && envKey && !judge.judge_provider && !process.env[envKey]) {
+      const prev = process.env[envKey];
+      process.env[envKey] = evaluatorConfig.apiKey;
+      restoreEnv = () => {
+        if (prev === void 0) delete process.env[envKey];
+        else process.env[envKey] = prev;
+      };
+    }
     try {
-      const provider = judge.judge_provider ?? "openai";
       const evalPrompt = `${judge.judge_prompt}
 
 Output to evaluate:
@@ -6380,11 +6411,12 @@ ${outputStr}
 Score this output on a scale of 0-10. Respond with only the number.`;
       const result = await callProviderLLM(
         evalPrompt,
-        { provider, model: judge.judge_model ?? void 0 },
+        { provider: resolvedProvider, model: resolvedModel },
         "You are an expert test judge. Return only a number between 0 and 10.",
         16,
         0
       );
+      restoreEnv?.();
       const score = parseFloat(result.content.match(/-?\d+(?:\.\d+)?/)?.[0] ?? "");
       if (isNaN(score)) {
         metrics.push({
@@ -6411,6 +6443,7 @@ Score this output on a scale of 0-10. Respond with only the number.`;
         }
       }
     } catch (err) {
+      restoreEnv?.();
       const errMsg = err instanceof Error ? err.message : String(err);
       metrics.push({
         name: "llm_judge",
@@ -6437,10 +6470,70 @@ function stringifyOutput(output) {
   return JSON.stringify(output);
 }
 
+// src/ci/api-client.ts
+var import_node_crypto8 = require("node:crypto");
+init_http();
+function normalizeBase(serverUrl) {
+  return serverUrl.replace(/\/+$/, "").replace(/\/api$/, "");
+}
+function headers(apiKey) {
+  return {
+    "Content-Type": "application/json",
+    "api-key": apiKey || "",
+    "X-Correlation-ID": (0, import_node_crypto8.randomUUID)()
+  };
+}
+async function apiRequest(url, apiKey, options = {}) {
+  const method = (options.method || "GET").toUpperCase();
+  console.log(`[elasticdash ci] ${method} ${url}`);
+  const res = await getOriginalFetch()(url, {
+    ...options,
+    headers: { ...headers(apiKey), ...options.headers ?? {} }
+  });
+  if (!res.ok) {
+    const text = await res.text().catch(() => "");
+    console.log(`[elasticdash ci] ${method} ${url} \u2192 ${res.status} ${text.substring(0, 200)}`);
+    throw new Error(`API ${res.status}: ${text || res.statusText}`);
+  }
+  const json = await res.json();
+  return json.result ?? json.data ?? json;
+}
+async function fetchTestGroups(serverUrl, apiKey, filters) {
+  const base = normalizeBase(serverUrl);
+  const params = new URLSearchParams();
+  if (filters?.workflowName) params.set("workflowName", filters.workflowName);
+  if (filters?.tags?.length) params.set("tags", filters.tags.join(","));
+  if (filters?.status) params.set("status", filters.status);
+  const qs = params.toString();
+  const url = `${base}/api/testgroups/by-project${qs ? `?${qs}` : ""}`;
+  return apiRequest(url, apiKey);
+}
+async function submitTestRun(serverUrl, apiKey, testGroupId, payload) {
+  const base = normalizeBase(serverUrl);
+  const url = `${base}/api/testgroups/${testGroupId}/runs`;
+  return apiRequest(url, apiKey, {
+    method: "POST",
+    body: JSON.stringify(payload)
+  });
+}
+async function createBatch(serverUrl, apiKey, payload) {
+  const base = normalizeBase(serverUrl);
+  const url = `${base}/api/testgroups/batches`;
+  return apiRequest(url, apiKey, {
+    method: "POST",
+    body: JSON.stringify(payload)
+  });
+}
+async function fetchEvaluatorConfig(serverUrl, apiKey) {
+  const base = normalizeBase(serverUrl);
+  const url = `${base}/api/test-runs/evaluator-config`;
+  return apiRequest(url, apiKey);
+}
+
 // src/ci/ed-runner.ts
 async function runEdTests(options) {
   const cwd = options?.cwd ?? process.cwd();
-  const runId = (0, import_node_crypto8.randomUUID)();
+  const runId = (0, import_node_crypto9.randomUUID)();
   const startedAt = (/* @__PURE__ */ new Date()).toISOString();
   const results = [];
   const { tests, errors } = await loadTests({ cwd });
@@ -6460,12 +6553,28 @@ async function runEdTests(options) {
     testsToRun = tests.filter((t) => matchGlob(t.name, pattern));
   }
   const maxRuns = Math.max(1, options?.runs ?? 1);
+  let evaluatorConfig = null;
+  const needsEvaluatorConfig = testsToRun.some(
+    (t) => t.benchmarks.llm_judge && (!t.benchmarks.llm_judge.judge_provider || !t.benchmarks.llm_judge.judge_model)
+  );
+  if (needsEvaluatorConfig) {
+    const serverUrl = process.env.ELASTICDASH_API_URL ?? process.env.ELASTICDASH_SERVER ?? "";
+    const apiKey = process.env.ELASTICDASH_API_KEY ?? "";
+    if (serverUrl && apiKey) {
+      try {
+        evaluatorConfig = await fetchEvaluatorConfig(serverUrl, apiKey);
+        console.log(`[ed-test] Evaluator config: provider=${evaluatorConfig.provider}, model=${evaluatorConfig.model}, hasKey=${!!evaluatorConfig.apiKey}`);
+      } catch (err) {
+        console.warn(`[ed-test] Could not fetch evaluator config: ${err instanceof Error ? err.message : String(err)}`);
+      }
+    }
+  }
   for (const test of testsToRun) {
     const allRuns = [];
     let bestResult = null;
     for (let attempt = 1; attempt <= maxRuns; attempt++) {
       const runStartedAt = (/* @__PURE__ */ new Date()).toISOString();
-      const result = await runSingleTest(test);
+      const result = await runSingleTest(test, evaluatorConfig);
       const runFinishedAt = (/* @__PURE__ */ new Date()).toISOString();
       if (attempt > 1) {
         console.log(`  [ed-test] ${test.name}: run ${attempt}/${maxRuns} \u2014 ${result.status}`);
@@ -6505,15 +6614,19 @@ async function runEdTests(options) {
     sdkVersion: SDK_VERSION
   };
 }
-async function runSingleTest(test) {
+async function resolveCustomInput(input) {
+  return typeof input === "function" ? await input() : input;
+}
+async function runSingleTest(test, evaluatorConfig) {
   const startMs = Date.now();
   const targetStep = test.traceData.steps.find((s) => s.step_id === test.target.step_id);
+  const resolvedInput = test.input !== void 0 ? await resolveCustomInput(test.input) : targetStep?.input;
   const base = {
     testId: test.name,
     testName: test.name,
     traceRef: test.trace,
     target: { type: test.target.type, step_id: test.target.step_id },
-    input: targetStep?.input,
+    input: resolvedInput,
     output: targetStep?.output
   };
   if (!test.run || typeof test.run !== "function") {
@@ -6553,7 +6666,7 @@ async function runSingleTest(test) {
         durationMs: Date.now() - startMs
       };
     }
-    const benchmarkResult = await compareBenchmarks(measurement, test.benchmarks, targetStep?.output);
+    const benchmarkResult = await compareBenchmarks(measurement, test.benchmarks, targetStep?.output, evaluatorConfig);
     return {
       ...base,
       testId: test.name,
@@ -6568,7 +6681,7 @@ async function runSingleTest(test) {
     const traceMeasurement = extractMeasurementFromTrace(test);
     if (traceMeasurement) {
       console.log(`  [ed-test] ${test.name}: run() failed (${err instanceof Error ? err.message : String(err)}), using trace measurement fallback`);
-      const benchmarkResult = await compareBenchmarks(traceMeasurement, test.benchmarks, targetStep?.output);
+      const benchmarkResult = await compareBenchmarks(traceMeasurement, test.benchmarks, targetStep?.output, evaluatorConfig);
       return {
         ...base,
         testId: test.name,
@@ -6640,7 +6753,7 @@ function matchGlob(name, pattern) {
 }
 
 // src/ci/upload-client.ts
-var import_node_crypto9 = require("node:crypto");
+var import_node_crypto10 = require("node:crypto");
 init_http();
 init_git_info();
 function buildUploadPayload(runResult) {
@@ -6768,7 +6881,7 @@ async function uploadResults(payload, options) {
   const headers2 = {
     "Content-Type": "application/json",
     "api-key": options.apiKey,
-    "X-Correlation-ID": (0, import_node_crypto9.randomUUID)()
+    "X-Correlation-ID": (0, import_node_crypto10.randomUUID)()
   };
   const body = JSON.stringify(payload);
   console.log(`[elasticdash] Uploading to ${url}, api-key=${options.apiKey ? options.apiKey.slice(0, 10) + "..." : "(none)"}`);
@@ -6804,61 +6917,6 @@ async function uploadResults(payload, options) {
 // src/ci/runner.ts
 var import_chalk2 = __toESM(require("chalk"), 1);
 
-// src/ci/api-client.ts
-var import_node_crypto10 = require("node:crypto");
-init_http();
-function normalizeBase(serverUrl) {
-  return serverUrl.replace(/\/+$/, "").replace(/\/api$/, "");
-}
-function headers(apiKey) {
-  return {
-    "Content-Type": "application/json",
-    "api-key": apiKey || "",
-    "X-Correlation-ID": (0, import_node_crypto10.randomUUID)()
-  };
-}
-async function apiRequest(url, apiKey, options = {}) {
-  const method = (options.method || "GET").toUpperCase();
-  console.log(`[elasticdash ci] ${method} ${url}`);
-  const res = await getOriginalFetch()(url, {
-    ...options,
-    headers: { ...headers(apiKey), ...options.headers ?? {} }
-  });
-  if (!res.ok) {
-    const text = await res.text().catch(() => "");
-    console.log(`[elasticdash ci] ${method} ${url} \u2192 ${res.status} ${text.substring(0, 200)}`);
-    throw new Error(`API ${res.status}: ${text || res.statusText}`);
-  }
-  const json = await res.json();
-  return json.result ?? json.data ?? json;
-}
-async function fetchTestGroups(serverUrl, apiKey, filters) {
-  const base = normalizeBase(serverUrl);
-  const params = new URLSearchParams();
-  if (filters?.workflowName) params.set("workflowName", filters.workflowName);
-  if (filters?.tags?.length) params.set("tags", filters.tags.join(","));
-  if (filters?.status) params.set("status", filters.status);
-  const qs = params.toString();
-  const url = `${base}/api/testgroups/by-project${qs ? `?${qs}` : ""}`;
-  return apiRequest(url, apiKey);
-}
-async function submitTestRun(serverUrl, apiKey, testGroupId, payload) {
-  const base = normalizeBase(serverUrl);
-  const url = `${base}/api/testgroups/${testGroupId}/runs`;
-  return apiRequest(url, apiKey, {
-    method: "POST",
-    body: JSON.stringify(payload)
-  });
-}
-async function createBatch(serverUrl, apiKey, payload) {
-  const base = normalizeBase(serverUrl);
-  const url = `${base}/api/testgroups/batches`;
-  return apiRequest(url, apiKey, {
-    method: "POST",
-    body: JSON.stringify(payload)
-  });
-}
-
 // src/ci/executor.ts
 init_portal_executor();
 init_tool_runner();
@@ -7586,6 +7644,7 @@ tryAutoInitHttpContext().catch(() => {
   expect,
   extractTaskOutputs,
   fetchCapturedTrace,
+  fetchEvaluatorConfig,
   fetchTestGroups,
   getCaptureContext,
   getCurrentTrace,

package/dist/index.d.ts

@@ -52,7 +52,7 @@ export type { EdTestRunOptions, EdTestResult, EdTestRunResult } from './ci/ed-ru
 export { uploadResults, buildUploadPayload } from './ci/upload-client.js';
 export type { UploadPayload, UploadTestResult } from './ci/upload-client.js';
 export { runCI } from './ci/runner.js';
-export { fetchTestGroups, submitTestRun, createBatch } from './ci/api-client.js';
+export { fetchTestGroups, submitTestRun, createBatch, fetchEvaluatorConfig } from './ci/api-client.js';
 export { detectGitInfo } from './ci/git-info.js';
 export type { CIRunConfig, CIRunSummary, CITestResult, CISingleRunResult, CIExpectationResult } from './ci/types.js';
 export type { GitInfo } from './ci/git-info.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AACnH,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAA;AACtD,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAC5C,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAGxE,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AACnH,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAA;AAGnI,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAA;AAC3F,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAA;AACtD,YAAY,EAAE,aAAa,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AACzF,YAAY,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AAG3D,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AACjD,OAAO,EAAE,MAAM,EAAE,MAAM,+BAA+B,CAAA;AAGtD,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,kBAAkB,EAClB,sBAAsB,EACtB,gBAAgB,EAChB,6BAA6B,EAC7B,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,kCAAkC,CAAA;AACzC,YAAY,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAA;AAGtE,OAAO,EACL,WAAW,EACX,aAAa,EACb,iBAAiB,EACjB,qBAAqB,EACrB,eAAe,EACf,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,8BAA8B,CAAA;AACrC,YAAY,EACV,WAAW,EACX,YAAY,EACZ,mBAAmB,EACnB,oBAAoB,EACpB,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,8BAA8B,CAAA;AAGrC,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAC3G,OAAO,EAAE,wBAAwB,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAA;AAGhG,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AACzF,YAAY,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAA;AAGlE,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAA;AAGjH,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,kCAAkC,CAAA;AAG/F,OAAO,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACjH,YAAY,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACnF,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAA;AACzD,YAAY,EAAE,uBAAuB,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAA;AACjG,YAAY,EAAE,oBAAoB,EAAE,MAAM,kCAAkC,CAAA;AAC5E,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AACjF,YAAY,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAA;AAG9D,OAAO,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAC/E,YAAY,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAA;AAGnE,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAG1E,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAA;AAC7C,OAAO,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,qCAAqC,CAAA;AAGlF,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAClD,YAAY,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AAGjF,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AACxH,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAG1G,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AACxD,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAG5H,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAC/C,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AACvF,YAAY,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAA;AACzE,YAAY,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AAGpE,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAA;AAC9C,YAAY,EAAE,gBAAgB,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACxF,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAA;AACzE,YAAY,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AAG5E,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAA;AACtC,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAA;AAChF,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAChD,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAA;AACpH,YAAY,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AACnH,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAA;AACtD,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAC5C,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAGxE,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AACnH,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAA;AAGnI,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAA;AAC3F,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAA;AACtD,YAAY,EAAE,aAAa,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AACzF,YAAY,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AAG3D,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AACjD,OAAO,EAAE,MAAM,EAAE,MAAM,+BAA+B,CAAA;AAGtD,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,kBAAkB,EAClB,sBAAsB,EACtB,gBAAgB,EAChB,6BAA6B,EAC7B,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,kCAAkC,CAAA;AACzC,YAAY,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAA;AAGtE,OAAO,EACL,WAAW,EACX,aAAa,EACb,iBAAiB,EACjB,qBAAqB,EACrB,eAAe,EACf,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,8BAA8B,CAAA;AACrC,YAAY,EACV,WAAW,EACX,YAAY,EACZ,mBAAmB,EACnB,oBAAoB,EACpB,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,8BAA8B,CAAA;AAGrC,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAC3G,OAAO,EAAE,wBAAwB,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAA;AAGhG,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AACzF,YAAY,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAA;AAGlE,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAA;AAGjH,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,kCAAkC,CAAA;AAG/F,OAAO,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACjH,YAAY,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACnF,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAA;AACzD,YAAY,EAAE,uBAAuB,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAA;AACjG,YAAY,EAAE,oBAAoB,EAAE,MAAM,kCAAkC,CAAA;AAC5E,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AACjF,YAAY,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAA;AAG9D,OAAO,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAC/E,YAAY,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAA;AAGnE,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAG1E,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAA;AAC7C,OAAO,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,qCAAqC,CAAA;AAGlF,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAClD,YAAY,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AAGjF,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AACxH,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAG1G,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AACxD,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAG5H,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAC/C,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AACvF,YAAY,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAA;AACzE,YAAY,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AAGpE,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAA;AAC9C,YAAY,EAAE,gBAAgB,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACxF,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAA;AACzE,YAAY,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AAG5E,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAA;AACtC,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AACtG,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAChD,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAA;AACpH,YAAY,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAA"}

package/dist/index.js

@@ -53,7 +53,7 @@ export { runEdTests } from './ci/ed-runner.js';
 export { uploadResults, buildUploadPayload } from './ci/upload-client.js';
 // CI runner (programmatic API)
 export { runCI } from './ci/runner.js';
-export { fetchTestGroups, submitTestRun, createBatch } from './ci/api-client.js';
+export { fetchTestGroups, submitTestRun, createBatch, fetchEvaluatorConfig } from './ci/api-client.js';
 export { detectGitInfo } from './ci/git-info.js';
 // ─── Eager auto-init ────────────────────────────────────────
 // When ELASTICDASH_API_KEY is set, automatically initialise observability mode

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAE1C,iBAAiB;AACjB,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AACnH,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAA;AACtD,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAG5C,gBAAgB;AAChB,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAGnH,4BAA4B;AAC5B,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAA;AAC3F,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAA;AAItD,yBAAyB;AACzB,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AACjD,OAAO,EAAE,MAAM,EAAE,MAAM,+BAA+B,CAAA;AAEtD,oEAAoE;AACpE,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,kBAAkB,EAClB,sBAAsB,EACtB,gBAAgB,EAChB,6BAA6B,EAC7B,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,kCAAkC,CAAA;AAGzC,4DAA4D;AAC5D,OAAO,EACL,WAAW,EACX,aAAa,EACb,iBAAiB,EACjB,qBAAqB,EACrB,eAAe,EACf,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,8BAA8B,CAAA;AAUrC,wBAAwB;AACxB,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAC3G,OAAO,EAAE,wBAAwB,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAA;AAEhG,4BAA4B;AAC5B,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAGzF,2BAA2B;AAC3B,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAA;AAEjH,sCAAsC;AACtC,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,kCAAkC,CAAA;AAE/F,gBAAgB;AAChB,OAAO,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AAEjH,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAA;AAGzD,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAGjF,qDAAqD;AACrD,OAAO,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAG/E,YAAY;AACZ,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAE1E,UAAU;AACV,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAA;AAC7C,OAAO,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,qCAAqC,CAAA;AAElF,kBAAkB;AAClB,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAGlD,yBAAyB;AACzB,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AAGxH,8BAA8B;AAC9B,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AAGxD,kDAAkD;AAClD,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAK/C,iCAAiC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAA;AAE9C,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAA;AAGzE,+BAA+B;AAC/B,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAA;AACtC,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAA;AAChF,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAIhD,+DAA+D;AAC/D,+EAA+E;AAC/E,+EAA+E;AAC/E,+EAA+E;AAC/E,iFAAiF;AACjF,OAAO,EAAE,sBAAsB,EAAE,MAAM,kCAAkC,CAAA;AACzE,sBAAsB,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAE1C,iBAAiB;AACjB,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AACnH,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAA;AACtD,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAG5C,gBAAgB;AAChB,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAGnH,4BAA4B;AAC5B,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAA;AAC3F,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAA;AAItD,yBAAyB;AACzB,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AACjD,OAAO,EAAE,MAAM,EAAE,MAAM,+BAA+B,CAAA;AAEtD,oEAAoE;AACpE,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,kBAAkB,EAClB,sBAAsB,EACtB,gBAAgB,EAChB,6BAA6B,EAC7B,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,kCAAkC,CAAA;AAGzC,4DAA4D;AAC5D,OAAO,EACL,WAAW,EACX,aAAa,EACb,iBAAiB,EACjB,qBAAqB,EACrB,eAAe,EACf,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,8BAA8B,CAAA;AAUrC,wBAAwB;AACxB,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAC3G,OAAO,EAAE,wBAAwB,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAA;AAEhG,4BAA4B;AAC5B,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAGzF,2BAA2B;AAC3B,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAA;AAEjH,sCAAsC;AACtC,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,kCAAkC,CAAA;AAE/F,gBAAgB;AAChB,OAAO,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AAEjH,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAA;AAGzD,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAGjF,qDAAqD;AACrD,OAAO,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAG/E,YAAY;AACZ,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAE1E,UAAU;AACV,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAA;AAC7C,OAAO,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,qCAAqC,CAAA;AAElF,kBAAkB;AAClB,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAGlD,yBAAyB;AACzB,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AAGxH,8BAA8B;AAC9B,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AAGxD,kDAAkD;AAClD,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAK/C,iCAAiC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAA;AAE9C,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAA;AAGzE,+BAA+B;AAC/B,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAA;AACtC,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AACtG,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAIhD,+DAA+D;AAC/D,+EAA+E;AAC/E,+EAA+E;AAC/E,+EAA+E;AAC/E,iFAAiF;AACjF,OAAO,EAAE,sBAAsB,EAAE,MAAM,kCAAkC,CAAA;AACzE,sBAAsB,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "elasticdash-test",
-  "version": "0.1.20-alpha-20",
+  "version": "0.1.20-alpha-22",
   "description": "AI-native test runner for ElasticDash workflow testing",
   "type": "module",
   "bin": {

package/src/ci/api-client.ts

@@ -94,3 +94,27 @@ export async function createBatch(
     body: JSON.stringify(payload),
   })
 }
+
+/**
+ * Resolved evaluator configuration from the backend.
+ * Provider/model/apiKey may be null if the user has not configured an evaluator.
+ */
+export interface EvaluatorConfig {
+  provider: string | null
+  model: string | null
+  apiKey: string | null
+}
+
+/**
+ * Fetch the project's evaluator config (with user-level fallback).
+ * Used by ed-test llm_judge benchmarks when judge_provider/judge_model
+ * are not specified in the test definition.
+ */
+export async function fetchEvaluatorConfig(
+  serverUrl: string,
+  apiKey: string,
+): Promise<EvaluatorConfig> {
+  const base = normalizeBase(serverUrl)
+  const url = `${base}/api/test-runs/evaluator-config`
+  return apiRequest<EvaluatorConfig>(url, apiKey)
+}

package/src/ci/benchmark.ts

@@ -10,6 +10,7 @@
 import { callProviderLLM } from '../matchers/index.js'
 import type { TestMeasurement } from './measurement.js'
 import type { TestBenchmarks } from './test-registry.js'
+import type { EvaluatorConfig } from './api-client.js'
 
 export type MetricName = 'duration_ms' | 'tokens_total' | 'output_contains' | 'output_not_contains' | 'llm_judge'
 
@@ -27,14 +28,29 @@ export interface BenchmarkResult {
   metrics: MetricResult[]
 }
 
+/** Maps backend provider names to SDK provider names used by callProviderLLM. */
+const PROVIDER_NAME_MAP: Record<string, string> = {
+  anthropic: 'claude',
+  moonshot: 'kimi',
+}
+
+/** Normalize provider name from backend format to SDK format. */
+function normalizeSdkProvider(provider: string): string {
+  return PROVIDER_NAME_MAP[provider] ?? provider
+}
+
 /**
  * Compare a measurement against benchmarks. Async because llm_judge requires
  * an LLM call. The step's output is needed for output_contains/llm_judge checks.
+ *
+ * @param evaluatorConfig - Optional backend evaluator config used as fallback
+ *   when the test does not specify judge_provider/judge_model.
  */
 export async function compareBenchmarks(
   measurement: TestMeasurement,
   benchmarks: TestBenchmarks,
   stepOutput?: unknown,
+  evaluatorConfig?: EvaluatorConfig | null,
 ): Promise<BenchmarkResult> {
   const metrics: MetricResult[] = []
   let firstFailure: string | undefined
@@ -104,18 +120,46 @@ export async function compareBenchmarks(
     const outputStr = stringifyOutput(stepOutput)
     const threshold = judge.judge_score_threshold ?? 7
 
+    // Resolve provider/model: test definition takes priority, then backend
+    // evaluator config, then fall back to 'openai' default.
+    const resolvedProvider = normalizeSdkProvider(
+      judge.judge_provider ?? evaluatorConfig?.provider ?? 'openai'
+    )
+    const resolvedModel = judge.judge_model ?? evaluatorConfig?.model ?? undefined
+
+    // If the backend provided an API key and we're using its provider,
+    // set it in the environment so callProviderLLM can pick it up.
+    const envKeyMap: Record<string, string> = {
+      openai: 'OPENAI_API_KEY',
+      claude: 'ANTHROPIC_API_KEY',
+      gemini: 'GEMINI_API_KEY',
+      grok: 'GROK_API_KEY',
+      kimi: 'KIMI_API_KEY',
+    }
+    const envKey = envKeyMap[resolvedProvider]
+    let restoreEnv: (() => void) | undefined
+    if (evaluatorConfig?.apiKey && envKey && !judge.judge_provider && !process.env[envKey]) {
+      const prev = process.env[envKey]
+      process.env[envKey] = evaluatorConfig.apiKey
+      restoreEnv = () => {
+        if (prev === undefined) delete process.env[envKey]
+        else process.env[envKey] = prev
+      }
+    }
+
     try {
-      const provider = judge.judge_provider ?? 'openai'
       const evalPrompt = `${judge.judge_prompt}\n\nOutput to evaluate:\n${outputStr}\n\nScore this output on a scale of 0-10. Respond with only the number.`
 
       const result = await callProviderLLM(
         evalPrompt,
-        { provider, model: judge.judge_model ?? undefined },
+        { provider: resolvedProvider as 'openai' | 'claude' | 'gemini' | 'grok' | 'kimi', model: resolvedModel },
         'You are an expert test judge. Return only a number between 0 and 10.',
         16,
         0,
       )
 
+      restoreEnv?.()
+
       const score = parseFloat(result.content.match(/-?\d+(?:\.\d+)?/)?.[0] ?? '')
       if (isNaN(score)) {
         metrics.push({
@@ -142,6 +186,7 @@ export async function compareBenchmarks(
         }
       }
     } catch (err) {
+      restoreEnv?.()
       const errMsg = err instanceof Error ? err.message : String(err)
       metrics.push({
         name: 'llm_judge',

package/src/ci/ed-runner.ts

@@ -4,6 +4,8 @@ import { createReplayContext, installReplay, uninstallReplay, ReplayMissError }
 import { collectMeasurement } from './measurement.js'
 import { SDK_VERSION } from './trace-schema.js'
 import { compareBenchmarks } from './benchmark.js'
+import { fetchEvaluatorConfig } from './api-client.js'
+import type { EvaluatorConfig } from './api-client.js'
 import type { TestMeasurement } from './measurement.js'
 import type { BenchmarkResult } from './benchmark.js'
 import type { ValidatedTest } from './test-loader.js'
@@ -88,13 +90,32 @@ export async function runEdTests(options?: EdTestRunOptions): Promise<EdTestRunR
 
   const maxRuns = Math.max(1, options?.runs ?? 1)
 
+  // Fetch evaluator config from backend if any test uses llm_judge without
+  // explicit provider/model. Cached for the entire run to avoid repeated calls.
+  let evaluatorConfig: EvaluatorConfig | null = null
+  const needsEvaluatorConfig = testsToRun.some(
+    t => t.benchmarks.llm_judge && (!t.benchmarks.llm_judge.judge_provider || !t.benchmarks.llm_judge.judge_model)
+  )
+  if (needsEvaluatorConfig) {
+    const serverUrl = process.env.ELASTICDASH_API_URL ?? process.env.ELASTICDASH_SERVER ?? ''
+    const apiKey = process.env.ELASTICDASH_API_KEY ?? ''
+    if (serverUrl && apiKey) {
+      try {
+        evaluatorConfig = await fetchEvaluatorConfig(serverUrl, apiKey)
+        console.log(`[ed-test] Evaluator config: provider=${evaluatorConfig.provider}, model=${evaluatorConfig.model}, hasKey=${!!evaluatorConfig.apiKey}`)
+      } catch (err) {
+        console.warn(`[ed-test] Could not fetch evaluator config: ${err instanceof Error ? err.message : String(err)}`)
+      }
+    }
+  }
+
   for (const test of testsToRun) {
     const allRuns: EdSingleRunResult[] = []
     let bestResult: EdTestResult | null = null
 
     for (let attempt = 1; attempt <= maxRuns; attempt++) {
       const runStartedAt = new Date().toISOString()
-      const result = await runSingleTest(test)
+      const result = await runSingleTest(test, evaluatorConfig)
       const runFinishedAt = new Date().toISOString()
 
       if (attempt > 1) {
@@ -149,15 +170,24 @@ export async function runEdTests(options?: EdTestRunOptions): Promise<EdTestRunR
 
 // ─── Single test execution ──────────────────────────────────
 
-async function runSingleTest(test: ValidatedTest): Promise<EdTestResult> {
+async function resolveCustomInput(input: unknown | (() => Promise<unknown> | unknown)): Promise<unknown> {
+  return typeof input === 'function' ? await (input as () => Promise<unknown> | unknown)() : input
+}
+
+async function runSingleTest(test: ValidatedTest, evaluatorConfig?: EvaluatorConfig | null): Promise<EdTestResult> {
   const startMs = Date.now()
   const targetStep = test.traceData.steps.find(s => s.step_id === test.target.step_id)
+
+  const resolvedInput = test.input !== undefined
+    ? await resolveCustomInput(test.input)
+    : targetStep?.input
+
   const base: Partial<EdTestResult> = {
     testId: test.name,
     testName: test.name,
     traceRef: test.trace,
     target: { type: test.target.type, step_id: test.target.step_id },
-    input: targetStep?.input,
+    input: resolvedInput,
     output: targetStep?.output,
   }
 
@@ -212,7 +242,7 @@ async function runSingleTest(test: ValidatedTest): Promise<EdTestResult> {
     }
 
     // Compare against benchmarks (async to support llm_judge)
-    const benchmarkResult = await compareBenchmarks(measurement, test.benchmarks, targetStep?.output)
+    const benchmarkResult = await compareBenchmarks(measurement, test.benchmarks, targetStep?.output, evaluatorConfig)
 
     return {
       ...base,
@@ -231,7 +261,7 @@ async function runSingleTest(test: ValidatedTest): Promise<EdTestResult> {
     const traceMeasurement = extractMeasurementFromTrace(test)
     if (traceMeasurement) {
       console.log(`  [ed-test] ${test.name}: run() failed (${err instanceof Error ? err.message : String(err)}), using trace measurement fallback`)
-      const benchmarkResult = await compareBenchmarks(traceMeasurement, test.benchmarks, targetStep?.output)
+      const benchmarkResult = await compareBenchmarks(traceMeasurement, test.benchmarks, targetStep?.output, evaluatorConfig)
       return {
         ...base,
         testId: test.name,

package/src/ci/test-loader.ts

@@ -131,8 +131,11 @@ export async function loadTests(options?: { cwd?: string }): Promise<LoadTestsRe
 
     const hasMaxDuration = def.benchmarks.max_duration_ms !== undefined
     const hasMaxTokens = def.benchmarks.max_tokens_total !== undefined
+    const hasOutputContains = def.benchmarks.output_contains !== undefined
+    const hasOutputNotContains = def.benchmarks.output_not_contains !== undefined
+    const hasLLMJudge = def.benchmarks.llm_judge !== undefined
 
-    if (!hasMaxDuration && !hasMaxTokens) {
+    if (!hasMaxDuration && !hasMaxTokens && !hasOutputContains && !hasOutputNotContains && !hasLLMJudge) {
       errors.push({ file, testName: def.name, message: 'At least one benchmark must be specified' })
       continue
     }

package/src/ci/test-registry.ts

@@ -30,6 +30,8 @@ export interface TestDefinition {
   trace: string
   target: TestTarget
   benchmarks: TestBenchmarks
+  /** Custom input that overrides the trace's recorded input. Can be a static value or an async function for dynamic resolution (e.g. fetching from a database or API). */
+  input?: unknown | (() => Promise<unknown> | unknown)
   /** The function that invokes the workflow under test. Required for execution (Phase 3). */
   run?: () => Promise<void>
   /** Per-test timeout in milliseconds. Defaults to 60000. */

package/src/index.ts

@@ -120,7 +120,7 @@ export type { UploadPayload, UploadTestResult } from './ci/upload-client.js'
 
 // CI runner (programmatic API)
 export { runCI } from './ci/runner.js'
-export { fetchTestGroups, submitTestRun, createBatch } from './ci/api-client.js'
+export { fetchTestGroups, submitTestRun, createBatch, fetchEvaluatorConfig } from './ci/api-client.js'
 export { detectGitInfo } from './ci/git-info.js'
 export type { CIRunConfig, CIRunSummary, CITestResult, CISingleRunResult, CIExpectationResult } from './ci/types.js'
 export type { GitInfo } from './ci/git-info.js'