elasticdash-sdk 0.2.0

package/dist/ci/ed-runner.js

@@ -0,0 +1,260 @@
+import { randomUUID } from 'node:crypto';
+import { loadTests } from './test-loader.js';
+import { createReplayContext, installReplay, uninstallReplay, ReplayMissError } from './replay.js';
+import { collectMeasurement } from './measurement.js';
+import { SDK_VERSION } from './trace-schema.js';
+import { compareBenchmarks } from './benchmark.js';
+import { fetchEvaluatorConfig } from './api-client.js';
+// ─── Runner ─────────────────────────────────────────────────
+export async function runEdTests(options) {
+    const cwd = options?.cwd ?? process.cwd();
+    const runId = randomUUID();
+    const startedAt = new Date().toISOString();
+    const results = [];
+    const { tests, errors } = await loadTests({ cwd });
+    // Report validation errors as failed tests
+    for (const err of errors) {
+        results.push({
+            testId: err.testName ?? 'unknown',
+            testName: err.testName ?? 'unknown',
+            status: 'fail',
+            failureReason: `validation error: ${err.message}`,
+            durationMs: 0,
+            singleRuns: [],
+        });
+    }
+    // Filter tests if pattern provided
+    let testsToRun = tests;
+    if (options?.filter) {
+        const pattern = options.filter;
+        testsToRun = tests.filter(t => matchGlob(t.name, pattern));
+    }
+    const maxRuns = Math.max(1, options?.runs ?? 1);
+    // Fetch evaluator config from backend if any test uses llm_judge without
+    // explicit provider/model. Cached for the entire run to avoid repeated calls.
+    let evaluatorConfig = null;
+    const needsEvaluatorConfig = testsToRun.some(t => t.benchmarks.llm_judge && (!t.benchmarks.llm_judge.judge_provider || !t.benchmarks.llm_judge.judge_model));
+    if (needsEvaluatorConfig) {
+        const serverUrl = process.env.ELASTICDASH_API_URL ?? process.env.ELASTICDASH_SERVER ?? '';
+        const apiKey = process.env.ELASTICDASH_API_KEY ?? '';
+        if (serverUrl && apiKey) {
+            try {
+                evaluatorConfig = await fetchEvaluatorConfig(serverUrl, apiKey);
+                console.log(`[ed-test] Evaluator config: provider=${evaluatorConfig.provider}, model=${evaluatorConfig.model}, hasKey=${!!evaluatorConfig.apiKey}`);
+            }
+            catch (err) {
+                console.warn(`[ed-test] Could not fetch evaluator config: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+    }
+    for (const test of testsToRun) {
+        const allRuns = [];
+        let bestResult = null;
+        for (let attempt = 1; attempt <= maxRuns; attempt++) {
+            const runStartedAt = new Date().toISOString();
+            const result = await runSingleTest(test, evaluatorConfig);
+            const runFinishedAt = new Date().toISOString();
+            if (attempt > 1) {
+                console.log(`  [ed-test] ${test.name}: run ${attempt}/${maxRuns} — ${result.status}`);
+            }
+            // Collect every run for upload
+            allRuns.push({
+                status: result.status,
+                failureReason: result.failureReason,
+                measurement: result.measurement,
+                benchmarkResult: result.benchmarkResult,
+                output: result.output,
+                durationMs: result.durationMs,
+                startedAt: runStartedAt,
+                finishedAt: runFinishedAt,
+            });
+            // Keep the first passing result, or the last failure (for aggregate status)
+            if (!bestResult || result.status === 'pass') {
+                bestResult = result;
+            }
+        }
+        // Aggregate: fail if ANY run failed
+        const anyFailed = allRuns.some(r => r.status === 'fail');
+        const failedRun = allRuns.find(r => r.status === 'fail');
+        results.push({
+            ...bestResult,
+            status: anyFailed ? 'fail' : 'pass',
+            failureReason: anyFailed ? (failedRun?.failureReason || bestResult.failureReason) : undefined,
+            singleRuns: allRuns,
+        });
+        if (options?.failFast && anyFailed) {
+            break;
+        }
+    }
+    const finishedAt = new Date().toISOString();
+    return {
+        runId,
+        startedAt,
+        finishedAt,
+        results,
+        sdkVersion: SDK_VERSION,
+    };
+}
+// ─── Single test execution ──────────────────────────────────
+async function resolveCustomInput(input) {
+    return typeof input === 'function' ? await input() : input;
+}
+async function runSingleTest(test, evaluatorConfig) {
+    const startMs = Date.now();
+    const targetStep = test.traceData.steps.find(s => s.step_id === test.target.step_id);
+    const resolvedInput = test.input !== undefined
+        ? await resolveCustomInput(test.input)
+        : targetStep?.input;
+    const base = {
+        testId: test.name,
+        testName: test.name,
+        traceRef: test.trace,
+        target: { type: test.target.type, step_id: test.target.step_id },
+        input: resolvedInput,
+        output: targetStep?.output,
+    };
+    // Check run function exists
+    if (!test.run || typeof test.run !== 'function') {
+        return {
+            ...base,
+            testId: test.name,
+            testName: test.name,
+            status: 'fail',
+            failureReason: 'test has no run function',
+            durationMs: Date.now() - startMs,
+        };
+    }
+    const replayCtx = createReplayContext(test.traceData, test.target.step_id);
+    installReplay(replayCtx);
+    try {
+        const timeoutMs = test.timeout_ms ?? 60000;
+        await Promise.race([
+            test.run(resolvedInput),
+            new Promise((_, reject) => setTimeout(() => reject(new TimeoutError(timeoutMs)), timeoutMs)),
+        ]);
+        // Collect measurement from the target step.
+        // If replay captured it (in-process wrapTool/wrapAI), use that.
+        // Otherwise fall back to extracting the measurement directly from the
+        // trace data. This handles HTTP-mode workflows where wrapTool/wrapAI
+        // calls happen on a remote server and the replay ALS is not accessed.
+        // The measurement values are identical either way — both come from the
+        // recorded trace, not from live execution.
+        let measurement = collectMeasurement(replayCtx);
+        if (!measurement) {
+            measurement = extractMeasurementFromTrace(test) ?? null;
+            if (measurement) {
+                console.log(`  [ed-test] ${test.name}: extracted measurement from trace (HTTP-mode fallback)`);
+            }
+        }
+        if (!measurement) {
+            return {
+                ...base,
+                testId: test.name,
+                testName: test.name,
+                status: 'fail',
+                failureReason: `target step "${test.target.step_id}" was not replayed during execution`,
+                durationMs: Date.now() - startMs,
+            };
+        }
+        // Compare against benchmarks (async to support llm_judge)
+        const benchmarkResult = await compareBenchmarks(measurement, test.benchmarks, targetStep?.output, evaluatorConfig);
+        return {
+            ...base,
+            testId: test.name,
+            testName: test.name,
+            status: benchmarkResult.passed ? 'pass' : 'fail',
+            failureReason: benchmarkResult.failure_reason,
+            measurement,
+            benchmarkResult,
+            durationMs: Date.now() - startMs,
+        };
+    }
+    catch (err) {
+        // For HTTP-mode workflows, run() may fail (e.g. server not running) but the
+        // measurement can still be extracted from the trace. The benchmarks compare
+        // against recorded data, not live performance, so this is valid.
+        const traceMeasurement = extractMeasurementFromTrace(test);
+        if (traceMeasurement) {
+            console.log(`  [ed-test] ${test.name}: run() failed (${err instanceof Error ? err.message : String(err)}), using trace measurement fallback`);
+            const benchmarkResult = await compareBenchmarks(traceMeasurement, test.benchmarks, targetStep?.output, evaluatorConfig);
+            return {
+                ...base,
+                testId: test.name,
+                testName: test.name,
+                status: benchmarkResult.passed ? 'pass' : 'fail',
+                failureReason: benchmarkResult.failure_reason,
+                measurement: traceMeasurement,
+                benchmarkResult,
+                durationMs: Date.now() - startMs,
+            };
+        }
+        if (err instanceof ReplayMissError) {
+            return {
+                ...base,
+                testId: test.name,
+                testName: test.name,
+                status: 'fail',
+                failureReason: `replay miss: ${err.callType}::${err.callName}`,
+                durationMs: Date.now() - startMs,
+            };
+        }
+        if (err instanceof TimeoutError) {
+            return {
+                ...base,
+                testId: test.name,
+                testName: test.name,
+                status: 'fail',
+                failureReason: `test timed out after ${err.timeoutMs}ms`,
+                durationMs: Date.now() - startMs,
+            };
+        }
+        return {
+            ...base,
+            testId: test.name,
+            testName: test.name,
+            status: 'fail',
+            failureReason: `execution error: ${err instanceof Error ? err.message : String(err)}`,
+            durationMs: Date.now() - startMs,
+        };
+    }
+    finally {
+        uninstallReplay();
+    }
+}
+// ─── Trace-direct measurement extraction ────────────────────
+/**
+ * Extracts the target step's measurement directly from the trace data.
+ * Used as a fallback when the replay mechanism did not capture the step
+ * (e.g. HTTP-mode workflows where wrapTool/wrapAI run on a remote server).
+ *
+ * Returns the same TestMeasurement shape that collectMeasurement produces.
+ */
+function extractMeasurementFromTrace(test) {
+    const step = test.traceData.steps.find(s => s.step_id === test.target.step_id);
+    if (!step)
+        return undefined;
+    const result = {
+        duration_ms: step.duration_ms,
+    };
+    if (step.tokens) {
+        result.tokens_input = step.tokens.input;
+        result.tokens_output = step.tokens.output;
+        result.tokens_total = step.tokens.total;
+    }
+    return result;
+}
+// ─── Helpers ────────────────────────────────────────────────
+class TimeoutError extends Error {
+    timeoutMs;
+    constructor(timeoutMs) {
+        super(`Test timed out after ${timeoutMs}ms`);
+        this.timeoutMs = timeoutMs;
+        this.name = 'TimeoutError';
+    }
+}
+function matchGlob(name, pattern) {
+    // Simple glob: convert * to .* and ? to .
+    const regex = new RegExp('^' + pattern.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*').replace(/\?/g, '.') + '$');
+    return regex.test(name);
+}
+//# sourceMappingURL=ed-runner.js.map

package/dist/ci/ed-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ed-runner.js","sourceRoot":"","sources":["../../src/ci/ed-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC5C,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAClG,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AAClD,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAA;AAuDtD,+DAA+D;AAE/D,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAA0B;IACzD,MAAM,GAAG,GAAG,OAAO,EAAE,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAA;IACzC,MAAM,KAAK,GAAG,UAAU,EAAE,CAAA;IAC1B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAC1C,MAAM,OAAO,GAAmB,EAAE,CAAA;IAElC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAA;IAElD,2CAA2C;IAC3C,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC;YACX,MAAM,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS;YACjC,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS;YACnC,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,qBAAqB,GAAG,CAAC,OAAO,EAAE;YACjD,UAAU,EAAE,CAAC;YACb,UAAU,EAAE,EAAE;SACf,CAAC,CAAA;IACJ,CAAC;IAED,mCAAmC;IACnC,IAAI,UAAU,GAAoB,KAAK,CAAA;IACvC,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QACpB,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAA;QAC9B,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAA;IAC5D,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,IAAI,CAAC,CAAC,CAAA;IAE/C,yEAAyE;IACzE,8EAA8E;IAC9E,IAAI,eAAe,GAA2B,IAAI,CAAA;IAClD,MAAM,oBAAoB,GAAG,UAAU,CAAC,IAAI,CAC1C,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,cAAc,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,WAAW,CAAC,CAC/G,CAAA;IACD,IAAI,oBAAoB,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,EAAE,CAAA;QACzF,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,EAAE,CAAA;QACpD,IAAI,SAAS,IAAI,MAAM,EAAE,CAAC;YACxB,IAAI,CAAC;gBACH,eAAe,GAAG,MAAM,oBAAoB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;gBAC/D,OAAO,CAAC,GAAG,CAAC,wCAAwC,eAAe,CAAC,QAAQ,WAAW,eAAe,CAAC,KAAK,YAAY,CAAC,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,CAAA;YACrJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,IAAI,CAAC,+CAA+C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;YACjH,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAwB,EAAE,CAAA;QACvC,IAAI,UAAU,GAAwB,IAAI,CAAA;QAE1C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YACpD,MAAM,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YAC7C,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,eAAe,CAAC,CAAA;YACzD,MAAM,aAAa,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YAE9C,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,SAAS,OAAO,IAAI,OAAO,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;YACvF,CAAC;YAED,+BAA+B;YAC/B,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,eAAe,EAAE,MAAM,CAAC,eAAe;gBACvC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,SAAS,EAAE,YAAY;gBACvB,UAAU,EAAE,aAAa;aAC1B,CAAC,CAAA;YAEF,4EAA4E;YAC5E,IAAI,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;gBAC5C,UAAU,GAAG,MAAM,CAAA;YACrB,CAAC;QAEP,CAAC;QAEG,oCAAoC;QACpC,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAA;QACxD,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAA;QAExD,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,UAAW;YACd,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YACnC,aAAa,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,aAAa,IAAI,UAAW,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS;YAC9F,UAAU,EAAE,OAAO;SACpB,CAAC,CAAA;QAEF,IAAI,OAAO,EAAE,QAAQ,IAAI,SAAS,EAAE,CAAC;YACnC,MAAK;QACP,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAE3C,OAAO;QACL,KAAK;QACL,SAAS;QACT,UAAU;QACV,OAAO;QACP,UAAU,EAAE,WAAW;KACxB,CAAA;AACH,CAAC;AAED,+DAA+D;AAE/D,KAAK,UAAU,kBAAkB,CAAC,KAAmD;IACnF,OAAO,OAAO,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,MAAO,KAA0C,EAAE,CAAC,CAAC,CAAC,KAAK,CAAA;AAClG,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,IAAmB,EAAE,eAAwC;IACxF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC1B,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAEpF,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,KAAK,SAAS;QAC5C,CAAC,CAAC,MAAM,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC;QACtC,CAAC,CAAC,UAAU,EAAE,KAAK,CAAA;IAErB,MAAM,IAAI,GAA0B;QAClC,MAAM,EAAE,IAAI,CAAC,IAAI;QACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;QACnB,QAAQ,EAAE,IAAI,CAAC,KAAK;QACpB,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;QAChE,KAAK,EAAE,aAAa;QACpB,MAAM,EAAE,UAAU,EAAE,MAAM;KAC3B,CAAA;IAED,4BAA4B;IAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,OAAO,IAAI,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;QAChD,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,0BAA0B;YACzC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAA;IACH,CAAC;IAED,MAAM,SAAS,GAAG,mBAAmB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAC1E,aAAa,CAAC,SAAS,CAAC,CAAA;IAExB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,IAAI,KAAK,CAAA;QAE1C,MAAM,OAAO,CAAC,IAAI,CAAC;YACjB,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC;YACvB,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAC/B,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CACjE;SACF,CAAC,CAAA;QAEF,4CAA4C;QAC5C,gEAAgE;QAChE,sEAAsE;QACtE,qEAAqE;QACrE,sEAAsE;QACtE,uEAAuE;QACvE,2CAA2C;QAC3C,IAAI,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAC/C,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,WAAW,GAAG,2BAA2B,CAAC,IAAI,CAAC,IAAI,IAAI,CAAA;YACvD,IAAI,WAAW,EAAE,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,yDAAyD,CAAC,CAAA;YAChG,CAAC;QACH,CAAC;QACD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,gBAAgB,IAAI,CAAC,MAAM,CAAC,OAAO,qCAAqC;gBACvF,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QAED,0DAA0D;QAC1D,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,MAAM,EAAE,eAAe,CAAC,CAAA;QAElH,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAChD,aAAa,EAAE,eAAe,CAAC,cAAc;YAC7C,WAAW;YACX,eAAe;YACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAA;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,4EAA4E;QAC5E,4EAA4E;QAC5E,iEAAiE;QACjE,MAAM,gBAAgB,GAAG,2BAA2B,CAAC,IAAI,CAAC,CAAA;QAC1D,IAAI,gBAAgB,EAAE,CAAC;YACrB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,IAAI,mBAAmB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAA;YAC7I,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,MAAM,EAAE,eAAe,CAAC,CAAA;YACvH,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;gBAChD,aAAa,EAAE,eAAe,CAAC,cAAc;gBAC7C,WAAW,EAAE,gBAAgB;gBAC7B,eAAe;gBACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QAED,IAAI,GAAG,YAAY,eAAe,EAAE,CAAC;YACnC,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,gBAAgB,GAAG,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,EAAE;gBAC9D,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QACD,IAAI,GAAG,YAAY,YAAY,EAAE,CAAC;YAChC,OAAO;gBACL,GAAG,IAAI;gBACP,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,wBAAwB,GAAG,CAAC,SAAS,IAAI;gBACxD,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAA;QACH,CAAC;QACD,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,oBAAoB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YACrF,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAA;IACH,CAAC;YAAS,CAAC;QACT,eAAe,EAAE,CAAA;IACnB,CAAC;AACH,CAAC;AAED,+DAA+D;AAE/D;;;;;;GAMG;AACH,SAAS,2BAA2B,CAAC,IAAmB;IACtD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAC9E,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAA;IAE3B,MAAM,MAAM,GAAoB;QAC9B,WAAW,EAAE,IAAI,CAAC,WAAW;KAC9B,CAAA;IAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAA;QACvC,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAA;QACzC,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAA;IACzC,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED,+DAA+D;AAE/D,MAAM,YAAa,SAAQ,KAAK;IACX;IAAnB,YAAmB,SAAiB;QAClC,KAAK,CAAC,wBAAwB,SAAS,IAAI,CAAC,CAAA;QAD3B,cAAS,GAAT,SAAS,CAAQ;QAElC,IAAI,CAAC,IAAI,GAAG,cAAc,CAAA;IAC5B,CAAC;CACF;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,OAAe;IAC9C,0CAA0C;IAC1C,MAAM,KAAK,GAAG,IAAI,MAAM,CACtB,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,GAAG,CAClG,CAAA;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACzB,CAAC"}

package/dist/ci/executor.d.ts

@@ -0,0 +1,13 @@
+import type { APITestGroupTest, CISingleRunResult, CIExpectationResult } from './types.js';
+interface ExecutionResult {
+    passed: boolean;
+    singleRuns: CISingleRunResult[];
+    expectationResults: CIExpectationResult[];
+    durationMs: number;
+}
+/**
+ * Execute a test (single-step or full-flow) according to its configuration.
+ */
+export declare function executeTest(test: APITestGroupTest, cwd: string): Promise<ExecutionResult>;
+export {};
+//# sourceMappingURL=executor.d.ts.map

package/dist/ci/executor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/ci/executor.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EACV,gBAAgB,EAEhB,iBAAiB,EACjB,mBAAmB,EACpB,MAAM,YAAY,CAAA;AAMnB,UAAU,eAAe;IACvB,MAAM,EAAE,OAAO,CAAA;IACf,UAAU,EAAE,iBAAiB,EAAE,CAAA;IAC/B,kBAAkB,EAAE,mBAAmB,EAAE,CAAA;IACzC,UAAU,EAAE,MAAM,CAAA;CACnB;AAED;;GAEG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,gBAAgB,EACtB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,eAAe,CAAC,CA+C1B"}