elasticdash-sdk 0.2.0

package/dist/ci/api-client.js

@@ -0,0 +1,96 @@
+import { randomUUID } from 'node:crypto';
+import { getOriginalFetch } from '../interceptors/http.js';
+// ─── API Client ──────────────────────────────────────────────
+// Uses getOriginalFetch() to bypass SDK interceptors.
+/** Normalize serverUrl: strip trailing slash and trailing /api to avoid double /api/api paths */
+function normalizeBase(serverUrl) {
+    return serverUrl.replace(/\/+$/, '').replace(/\/api$/, '');
+}
+function headers(apiKey) {
+    return {
+        'Content-Type': 'application/json',
+        'api-key': apiKey || '',
+        'X-Correlation-ID': randomUUID(),
+    };
+}
+async function apiRequest(url, apiKey, options = {}) {
+    const method = (options.method || 'GET').toUpperCase();
+    console.log(`[elasticdash ci] ${method} ${url}`);
+    const res = await getOriginalFetch()(url, {
+        ...options,
+        headers: { ...headers(apiKey), ...(options.headers ?? {}) },
+    });
+    if (!res.ok) {
+        const text = await res.text().catch(() => '');
+        console.log(`[elasticdash ci] ${method} ${url} → ${res.status} ${text.substring(0, 200)}`);
+        throw new Error(`API ${res.status}: ${text || res.statusText}`);
+    }
+    const json = await res.json();
+    // Backend wraps responses in generalApiResponseSender which sends { result: ... }
+    // Use 'in' check instead of nullish coalescing — result:null is valid data, not "missing"
+    const result = 'result' in json ? json.result : ('data' in json ? json.data : json);
+    console.log(`[elasticdash ci] ${method} ${url} → ${res.status} (result keys: ${result && typeof result === 'object' ? Object.keys(result).join(',') : typeof result})`);
+    return result;
+}
+/**
+ * Fetch all active test groups (with nested tests & expectations) for the project
+ * scoped to the API key.
+ */
+export async function fetchTestGroups(serverUrl, apiKey, filters) {
+    const base = normalizeBase(serverUrl);
+    const params = new URLSearchParams();
+    if (filters?.workflowName)
+        params.set('workflowName', filters.workflowName);
+    if (filters?.tags?.length)
+        params.set('tags', filters.tags.join(','));
+    if (filters?.status)
+        params.set('status', filters.status);
+    const qs = params.toString();
+    const url = `${base}/api/testgroups/by-project${qs ? `?${qs}` : ''}`;
+    return apiRequest(url, apiKey);
+}
+/**
+ * Submit a test run result to the backend.
+ */
+export async function submitTestRun(serverUrl, apiKey, testGroupId, payload) {
+    const base = normalizeBase(serverUrl);
+    const url = `${base}/api/testgroups/${testGroupId}/runs`;
+    return apiRequest(url, apiKey, {
+        method: 'POST',
+        body: JSON.stringify(payload),
+    });
+}
+/**
+ * Create a batch grouping multiple test run IDs.
+ */
+export async function createBatch(serverUrl, apiKey, payload) {
+    const base = normalizeBase(serverUrl);
+    const url = `${base}/api/testgroups/batches`;
+    return apiRequest(url, apiKey, {
+        method: 'POST',
+        body: JSON.stringify(payload),
+    });
+}
+/** Maps backend llmProviderId to provider name. */
+const LLM_PROVIDER_MAP = {
+    1: 'openai',
+    2: 'gemini',
+    3: 'anthropic',
+    4: 'moonshot',
+};
+/**
+ * Fetch the user's default evaluator config.
+ * Used by ed-test llm_judge benchmarks when judge_provider/judge_model
+ * are not specified in the test definition.
+ */
+export async function fetchEvaluatorConfig(serverUrl, apiKey) {
+    const base = normalizeBase(serverUrl);
+    const url = `${base}/api/user/settings/llm/default-evaluator`;
+    const raw = await apiRequest(url, apiKey);
+    return {
+        provider: raw?.llmProviderId ? (LLM_PROVIDER_MAP[raw.llmProviderId] ?? null) : null,
+        model: raw?.model ?? null,
+        apiKey: null,
+    };
+}
+//# sourceMappingURL=api-client.js.map

package/dist/ci/api-client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"api-client.js","sourceRoot":"","sources":["../../src/ci/api-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAG1D,gEAAgE;AAChE,sDAAsD;AAEtD,iGAAiG;AACjG,SAAS,aAAa,CAAC,SAAiB;IACtC,OAAO,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;AAC5D,CAAC;AAED,SAAS,OAAO,CAAC,MAAc;IAC7B,OAAO;QACL,cAAc,EAAE,kBAAkB;QAClC,SAAS,EAAE,MAAM,IAAI,EAAE;QACvB,kBAAkB,EAAE,UAAU,EAAE;KACjC,CAAA;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,GAAW,EACX,MAAc,EACd,UAAuB,EAAE;IAEzB,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,WAAW,EAAE,CAAA;IACtD,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,IAAI,GAAG,EAAE,CAAC,CAAA;IAEhD,MAAM,GAAG,GAAG,MAAM,gBAAgB,EAAE,CAAC,GAAG,EAAE;QACxC,GAAG,OAAO;QACV,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,OAAiC,IAAI,EAAE,CAAC,EAAE;KACtF,CAAC,CAAA;IAEF,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAA;QAC7C,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;QAC1F,MAAM,IAAI,KAAK,CAAC,OAAO,GAAG,CAAC,MAAM,KAAK,IAAI,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;IACjE,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAA6B,CAAA;IACxD,kFAAkF;IAClF,0FAA0F;IAC1F,MAAM,MAAM,GAAG,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;IACnF,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,MAAM,kBAAkB,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,MAAM,GAAG,CAAC,CAAA;IACvK,OAAO,MAAW,CAAA;AACpB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,SAAiB,EACjB,MAAc,EACd,OAAqE;IAErE,MAAM,IAAI,GAAG,aAAa,CAAC,SAAS,CAAC,CAAA;IACrC,MAAM,MAAM,GAAG,IAAI,eAAe,EAAE,CAAA;IACpC,IAAI,OAAO,EAAE,YAAY;QAAE,MAAM,CAAC,GAAG,CAAC,cAAc,EAAE,OAAO,CAAC,YAAY,CAAC,CAAA;IAC3E,IAAI,OAAO,EAAE,IAAI,EAAE,MAAM;QAAE,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;IACrE,IAAI,OAAO,EAAE,MAAM;QAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;IAEzD,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;IAC5B,MAAM,GAAG,GAAG,GAAG,IAAI,6BAA6B,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAA;IACpE,OAAO,UAAU,CAAiB,GAAG,EAAE,MAAM,CAAC,CAAA;AAChD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAiB,EACjB,MAAc,EACd,WAAmB,EACnB,OAAgC;IAEhC,MAAM,IAAI,GAAG,aAAa,CAAC,SAAS,CAAC,CAAA;IACrC,MAAM,GAAG,GAAG,GAAG,IAAI,mBAAmB,WAAW,OAAO,CAAA;IACxD,OAAO,UAAU,CAAiB,GAAG,EAAE,MAAM,EAAE;QAC7C,MAAM,EAAE,MAAM;QACd,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;KAC9B,CAAC,CAAA;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,SAAiB,EACjB,MAAc,EACd,OAAgC;IAEhC,MAAM,IAAI,GAAG,aAAa,CAAC,SAAS,CAAC,CAAA;IACrC,MAAM,GAAG,GAAG,GAAG,IAAI,yBAAyB,CAAA;IAC5C,OAAO,UAAU,CAAiB,GAAG,EAAE,MAAM,EAAE;QAC7C,MAAM,EAAE,MAAM;QACd,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;KAC9B,CAAC,CAAA;AACJ,CAAC;AAYD,mDAAmD;AACnD,MAAM,gBAAgB,GAA2B;IAC/C,CAAC,EAAE,QAAQ;IACX,CAAC,EAAE,QAAQ;IACX,CAAC,EAAE,WAAW;IACd,CAAC,EAAE,UAAU;CACd,CAAA;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,SAAiB,EACjB,MAAc;IAEd,MAAM,IAAI,GAAG,aAAa,CAAC,SAAS,CAAC,CAAA;IACrC,MAAM,GAAG,GAAG,GAAG,IAAI,0CAA0C,CAAA;IAC7D,MAAM,GAAG,GAAG,MAAM,UAAU,CAA6C,GAAG,EAAE,MAAM,CAAC,CAAA;IACrF,OAAO;QACL,QAAQ,EAAE,GAAG,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;QACnF,KAAK,EAAE,GAAG,EAAE,KAAK,IAAI,IAAI;QACzB,MAAM,EAAE,IAAI;KACb,CAAA;AACH,CAAC"}

package/dist/ci/benchmark.d.ts

@@ -0,0 +1,33 @@
+/**
+ * benchmark.ts
+ *
+ * Compares recorded trace measurements against user-defined benchmarks.
+ * Supports duration, token, output-contains, and LLM-as-a-judge evaluations.
+ *
+ * Generated/updated on 2026-04-20.
+ */
+import type { TestMeasurement } from './measurement.js';
+import type { TestBenchmarks } from './test-registry.js';
+import type { EvaluatorConfig } from './api-client.js';
+export type MetricName = 'duration_ms' | 'tokens_total' | 'output_contains' | 'output_not_contains' | 'llm_judge';
+export interface MetricResult {
+    name: MetricName;
+    value: number;
+    threshold: number;
+    passed: boolean;
+    detail?: string;
+}
+export interface BenchmarkResult {
+    passed: boolean;
+    failure_reason?: string;
+    metrics: MetricResult[];
+}
+/**
+ * Compare a measurement against benchmarks. Async because llm_judge requires
+ * an LLM call. The step's output is needed for output_contains/llm_judge checks.
+ *
+ * @param evaluatorConfig - Optional backend evaluator config used as fallback
+ *   when the test does not specify judge_provider/judge_model.
+ */
+export declare function compareBenchmarks(measurement: TestMeasurement, benchmarks: TestBenchmarks, stepOutput?: unknown, evaluatorConfig?: EvaluatorConfig | null): Promise<BenchmarkResult>;
+//# sourceMappingURL=benchmark.d.ts.map

package/dist/ci/benchmark.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark.d.ts","sourceRoot":"","sources":["../../src/ci/benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AACvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AAEtD,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,cAAc,GAAG,iBAAiB,GAAG,qBAAqB,GAAG,WAAW,CAAA;AAEjH,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,UAAU,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,OAAO,CAAA;IACf,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,OAAO,EAAE,YAAY,EAAE,CAAA;CACxB;AAuCD;;;;;;GAMG;AACH,wBAAsB,iBAAiB,CACrC,WAAW,EAAE,eAAe,EAC5B,UAAU,EAAE,cAAc,EAC1B,UAAU,CAAC,EAAE,OAAO,EACpB,eAAe,CAAC,EAAE,eAAe,GAAG,IAAI,GACvC,OAAO,CAAC,eAAe,CAAC,CAyK1B"}

package/dist/ci/benchmark.js

@@ -0,0 +1,213 @@
+/**
+ * benchmark.ts
+ *
+ * Compares recorded trace measurements against user-defined benchmarks.
+ * Supports duration, token, output-contains, and LLM-as-a-judge evaluations.
+ *
+ * Generated/updated on 2026-04-20.
+ */
+import { callProviderLLM } from '../matchers/index.js';
+import { prepareOutputForJudge } from '../core/judge-utils.js';
+/** Maps backend provider names to SDK provider names used by callProviderLLM. */
+const PROVIDER_NAME_MAP = {
+    anthropic: 'claude',
+    moonshot: 'kimi',
+};
+/** Default model for each provider, used when no explicit model is set or
+ *  when the evaluator config model doesn't belong to the resolved provider. */
+const DEFAULT_PROVIDER_MODELS = {
+    openai: 'gpt-4o',
+    claude: 'claude-sonnet-4-20250514',
+    gemini: 'gemini-2.0-flash',
+    grok: 'grok-3',
+    kimi: 'moonshot-v1-auto',
+};
+/** Known model prefixes per provider — used to check if a model belongs to a provider. */
+const PROVIDER_MODEL_PREFIXES = {
+    openai: ['gpt-', 'o1-', 'o3-', 'o4-', 'chatgpt-', 'omni-'],
+    claude: ['claude-'],
+    gemini: ['gemini-'],
+    grok: ['grok-'],
+    kimi: ['moonshot-', 'kimi-'],
+};
+/** Check if a model name belongs to the given provider. */
+function isModelForProvider(model, provider) {
+    const prefixes = PROVIDER_MODEL_PREFIXES[provider];
+    if (!prefixes)
+        return false;
+    return prefixes.some(p => model.toLowerCase().startsWith(p));
+}
+/** Normalize provider name from backend format to SDK format. */
+function normalizeSdkProvider(provider) {
+    return PROVIDER_NAME_MAP[provider] ?? provider;
+}
+/**
+ * Compare a measurement against benchmarks. Async because llm_judge requires
+ * an LLM call. The step's output is needed for output_contains/llm_judge checks.
+ *
+ * @param evaluatorConfig - Optional backend evaluator config used as fallback
+ *   when the test does not specify judge_provider/judge_model.
+ */
+export async function compareBenchmarks(measurement, benchmarks, stepOutput, evaluatorConfig) {
+    const metrics = [];
+    let firstFailure;
+    if (benchmarks.max_duration_ms !== undefined) {
+        const passed = measurement.duration_ms <= benchmarks.max_duration_ms;
+        metrics.push({
+            name: 'duration_ms',
+            value: measurement.duration_ms,
+            threshold: benchmarks.max_duration_ms,
+            passed,
+        });
+        if (!passed && !firstFailure) {
+            firstFailure = `duration_ms (${measurement.duration_ms}) exceeded max threshold (${benchmarks.max_duration_ms})`;
+        }
+    }
+    if (benchmarks.max_tokens_total !== undefined) {
+        const value = measurement.tokens_total ?? 0;
+        const passed = value <= benchmarks.max_tokens_total;
+        metrics.push({
+            name: 'tokens_total',
+            value,
+            threshold: benchmarks.max_tokens_total,
+            passed,
+        });
+        if (!passed && !firstFailure) {
+            firstFailure = `tokens_total (${value}) exceeded max threshold (${benchmarks.max_tokens_total})`;
+        }
+    }
+    // ─── Output contains ────────────────────────────────────────
+    if (benchmarks.output_contains !== undefined) {
+        const outputStr = stringifyOutput(stepOutput);
+        const passed = outputStr.toLowerCase().includes(benchmarks.output_contains.toLowerCase());
+        metrics.push({
+            name: 'output_contains',
+            value: passed ? 1 : 0,
+            threshold: 1,
+            passed,
+            detail: passed ? undefined : `output does not contain "${benchmarks.output_contains}"`,
+        });
+        if (!passed && !firstFailure) {
+            firstFailure = `output does not contain "${benchmarks.output_contains}"`;
+        }
+    }
+    // ─── Output not contains ────────────────────────────────────
+    if (benchmarks.output_not_contains !== undefined) {
+        const outputStr = stringifyOutput(stepOutput);
+        const passed = !outputStr.toLowerCase().includes(benchmarks.output_not_contains.toLowerCase());
+        metrics.push({
+            name: 'output_not_contains',
+            value: passed ? 1 : 0,
+            threshold: 1,
+            passed,
+            detail: passed ? undefined : `output unexpectedly contains "${benchmarks.output_not_contains}"`,
+        });
+        if (!passed && !firstFailure) {
+            firstFailure = `output unexpectedly contains "${benchmarks.output_not_contains}"`;
+        }
+    }
+    // ─── LLM-as-a-judge ────────────────────────────────────────
+    if (benchmarks.llm_judge) {
+        const judge = benchmarks.llm_judge;
+        const outputStr = stringifyOutput(stepOutput);
+        const threshold = judge.judge_score_threshold ?? 7;
+        // Resolve provider/model: test definition takes priority, then backend
+        // evaluator config, then fall back to 'openai' default.
+        const resolvedProvider = normalizeSdkProvider(judge.judge_provider ?? evaluatorConfig?.provider ?? 'openai');
+        // Model resolution: judge_model > evaluatorConfig.model (if compatible) > provider default
+        let resolvedModel = judge.judge_model ?? undefined;
+        if (!resolvedModel && evaluatorConfig?.model) {
+            // Only use the evaluator config model if it belongs to the resolved provider
+            if (isModelForProvider(evaluatorConfig.model, resolvedProvider)) {
+                resolvedModel = evaluatorConfig.model;
+            }
+        }
+        if (!resolvedModel) {
+            resolvedModel = DEFAULT_PROVIDER_MODELS[resolvedProvider];
+        }
+        // If the backend provided an API key and we're using its provider,
+        // set it in the environment so callProviderLLM can pick it up.
+        const envKeyMap = {
+            openai: 'OPENAI_API_KEY',
+            claude: 'ANTHROPIC_API_KEY',
+            gemini: 'GEMINI_API_KEY',
+            grok: 'GROK_API_KEY',
+            kimi: 'KIMI_API_KEY',
+        };
+        const envKey = envKeyMap[resolvedProvider];
+        let restoreEnv;
+        if (evaluatorConfig?.apiKey && envKey && !judge.judge_provider && !process.env[envKey]) {
+            const prev = process.env[envKey];
+            process.env[envKey] = evaluatorConfig.apiKey;
+            restoreEnv = () => {
+                if (prev === undefined)
+                    delete process.env[envKey];
+                else
+                    process.env[envKey] = prev;
+            };
+        }
+        try {
+            console.log(`  [llm_judge] provider=${resolvedProvider}, model=${resolvedModel ?? '(default)'}`);
+            const preparedOutput = prepareOutputForJudge(outputStr, judge.judge_prompt);
+            const evalPrompt = `${judge.judge_prompt}\n\n<output>\n${preparedOutput}\n</output>\n\nBased on the evaluation criteria above, score this output on a scale of 0-10. Respond with only the number.`;
+            const result = await callProviderLLM(evalPrompt, { provider: resolvedProvider, model: resolvedModel }, 'You are an expert test judge. Return only a number between 0 and 10.', 4096, 0);
+            restoreEnv?.();
+            console.log(`  [llm_judge] raw response: "${result.content}"`);
+            const score = parseFloat(result.content.match(/-?\d+(?:\.\d+)?/)?.[0] ?? '');
+            if (isNaN(score)) {
+                metrics.push({
+                    name: 'llm_judge',
+                    value: 0,
+                    threshold,
+                    passed: false,
+                    detail: `Could not parse score from LLM response: "${result.content}"`,
+                });
+                if (!firstFailure) {
+                    firstFailure = `llm_judge: could not parse score from response`;
+                }
+            }
+            else {
+                const passed = score >= threshold;
+                metrics.push({
+                    name: 'llm_judge',
+                    value: score,
+                    threshold,
+                    passed,
+                    detail: `Score: ${score}/${threshold}`,
+                });
+                if (!passed && !firstFailure) {
+                    firstFailure = `llm_judge score (${score}) below threshold (${threshold})`;
+                }
+            }
+        }
+        catch (err) {
+            restoreEnv?.();
+            const errMsg = err instanceof Error ? err.message : String(err);
+            metrics.push({
+                name: 'llm_judge',
+                value: 0,
+                threshold,
+                passed: false,
+                detail: `LLM judge error: ${errMsg}`,
+            });
+            if (!firstFailure) {
+                firstFailure = `llm_judge error: ${errMsg}`;
+            }
+        }
+    }
+    const allPassed = metrics.every(m => m.passed);
+    return {
+        passed: allPassed,
+        failure_reason: firstFailure,
+        metrics,
+    };
+}
+/** Converts step output to a string for text-based assertions. */
+function stringifyOutput(output) {
+    if (output === null || output === undefined)
+        return '';
+    if (typeof output === 'string')
+        return output;
+    return JSON.stringify(output);
+}
+//# sourceMappingURL=benchmark.js.map

package/dist/ci/benchmark.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/ci/benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AACtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAA;AAqB9D,iFAAiF;AACjF,MAAM,iBAAiB,GAA2B;IAChD,SAAS,EAAE,QAAQ;IACnB,QAAQ,EAAE,MAAM;CACjB,CAAA;AAED;+EAC+E;AAC/E,MAAM,uBAAuB,GAA2B;IACtD,MAAM,EAAE,QAAQ;IAChB,MAAM,EAAE,0BAA0B;IAClC,MAAM,EAAE,kBAAkB;IAC1B,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,kBAAkB;CACzB,CAAA;AAED,0FAA0F;AAC1F,MAAM,uBAAuB,GAA6B;IACxD,MAAM,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC;IAC1D,MAAM,EAAE,CAAC,SAAS,CAAC;IACnB,MAAM,EAAE,CAAC,SAAS,CAAC;IACnB,IAAI,EAAE,CAAC,OAAO,CAAC;IACf,IAAI,EAAE,CAAC,WAAW,EAAE,OAAO,CAAC;CAC7B,CAAA;AAED,2DAA2D;AAC3D,SAAS,kBAAkB,CAAC,KAAa,EAAE,QAAgB;IACzD,MAAM,QAAQ,GAAG,uBAAuB,CAAC,QAAQ,CAAC,CAAA;IAClD,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAA;IAC3B,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAA;AAC9D,CAAC;AAED,iEAAiE;AACjE,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,OAAO,iBAAiB,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAA;AAChD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,WAA4B,EAC5B,UAA0B,EAC1B,UAAoB,EACpB,eAAwC;IAExC,MAAM,OAAO,GAAmB,EAAE,CAAA;IAClC,IAAI,YAAgC,CAAA;IAEpC,IAAI,UAAU,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,WAAW,CAAC,WAAW,IAAI,UAAU,CAAC,eAAe,CAAA;QACpE,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,aAAa;YACnB,KAAK,EAAE,WAAW,CAAC,WAAW;YAC9B,SAAS,EAAE,UAAU,CAAC,eAAe;YACrC,MAAM;SACP,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,YAAY,GAAG,gBAAgB,WAAW,CAAC,WAAW,6BAA6B,UAAU,CAAC,eAAe,GAAG,CAAA;QAClH,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,WAAW,CAAC,YAAY,IAAI,CAAC,CAAA;QAC3C,MAAM,MAAM,GAAG,KAAK,IAAI,UAAU,CAAC,gBAAgB,CAAA;QACnD,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,cAAc;YACpB,KAAK;YACL,SAAS,EAAE,UAAU,CAAC,gBAAgB;YACtC,MAAM;SACP,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,YAAY,GAAG,iBAAiB,KAAK,6BAA6B,UAAU,CAAC,gBAAgB,GAAG,CAAA;QAClG,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,IAAI,UAAU,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QAC7C,MAAM,SAAS,GAAG,eAAe,CAAC,UAAU,CAAC,CAAA;QAC7C,MAAM,MAAM,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC,CAAA;QACzF,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,iBAAiB;YACvB,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,SAAS,EAAE,CAAC;YACZ,MAAM;YACN,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B,UAAU,CAAC,eAAe,GAAG;SACvF,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,YAAY,GAAG,4BAA4B,UAAU,CAAC,eAAe,GAAG,CAAA;QAC1E,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,IAAI,UAAU,CAAC,mBAAmB,KAAK,SAAS,EAAE,CAAC;QACjD,MAAM,SAAS,GAAG,eAAe,CAAC,UAAU,CAAC,CAAA;QAC7C,MAAM,MAAM,GAAG,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,mBAAmB,CAAC,WAAW,EAAE,CAAC,CAAA;QAC9F,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,qBAAqB;YAC3B,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,SAAS,EAAE,CAAC;YACZ,MAAM;YACN,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,iCAAiC,UAAU,CAAC,mBAAmB,GAAG;SAChG,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,YAAY,GAAG,iCAAiC,UAAU,CAAC,mBAAmB,GAAG,CAAA;QACnF,CAAC;IACH,CAAC;IAED,8DAA8D;IAC9D,IAAI,UAAU,CAAC,SAAS,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,CAAA;QAClC,MAAM,SAAS,GAAG,eAAe,CAAC,UAAU,CAAC,CAAA;QAC7C,MAAM,SAAS,GAAG,KAAK,CAAC,qBAAqB,IAAI,CAAC,CAAA;QAElD,uEAAuE;QACvE,wDAAwD;QACxD,MAAM,gBAAgB,GAAG,oBAAoB,CAC3C,KAAK,CAAC,cAAc,IAAI,eAAe,EAAE,QAAQ,IAAI,QAAQ,CAC9D,CAAA;QACD,2FAA2F;QAC3F,IAAI,aAAa,GAAG,KAAK,CAAC,WAAW,IAAI,SAAS,CAAA;QAClD,IAAI,CAAC,aAAa,IAAI,eAAe,EAAE,KAAK,EAAE,CAAC;YAC7C,6EAA6E;YAC7E,IAAI,kBAAkB,CAAC,eAAe,CAAC,KAAK,EAAE,gBAAgB,CAAC,EAAE,CAAC;gBAChE,aAAa,GAAG,eAAe,CAAC,KAAK,CAAA;YACvC,CAAC;QACH,CAAC;QACD,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,aAAa,GAAG,uBAAuB,CAAC,gBAAgB,CAAC,CAAA;QAC3D,CAAC;QAED,mEAAmE;QACnE,+DAA+D;QAC/D,MAAM,SAAS,GAA2B;YACxC,MAAM,EAAE,gBAAgB;YACxB,MAAM,EAAE,mBAAmB;YAC3B,MAAM,EAAE,gBAAgB;YACxB,IAAI,EAAE,cAAc;YACpB,IAAI,EAAE,cAAc;SACrB,CAAA;QACD,MAAM,MAAM,GAAG,SAAS,CAAC,gBAAgB,CAAC,CAAA;QAC1C,IAAI,UAAoC,CAAA;QACxC,IAAI,eAAe,EAAE,MAAM,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACvF,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YAChC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,eAAe,CAAC,MAAM,CAAA;YAC5C,UAAU,GAAG,GAAG,EAAE;gBAChB,IAAI,IAAI,KAAK,SAAS;oBAAE,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;;oBAC7C,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,CAAA;YACjC,CAAC,CAAA;QACH,CAAC;QAED,IAAI,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,0BAA0B,gBAAgB,WAAW,aAAa,IAAI,WAAW,EAAE,CAAC,CAAA;YAChG,MAAM,cAAc,GAAG,qBAAqB,CAAC,SAAS,EAAE,KAAK,CAAC,YAAY,CAAC,CAAA;YAC3E,MAAM,UAAU,GAAG,GAAG,KAAK,CAAC,YAAY,iBAAiB,cAAc,4HAA4H,CAAA;YAEnM,MAAM,MAAM,GAAG,MAAM,eAAe,CAClC,UAAU,EACV,EAAE,QAAQ,EAAE,gBAAoE,EAAE,KAAK,EAAE,aAAa,EAAE,EACxG,sEAAsE,EACtE,IAAI,EACJ,CAAC,CACF,CAAA;YAED,UAAU,EAAE,EAAE,CAAA;YAEd,OAAO,CAAC,GAAG,CAAC,gCAAgC,MAAM,CAAC,OAAO,GAAG,CAAC,CAAA;YAC9D,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;YAC5E,IAAI,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjB,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,WAAW;oBACjB,KAAK,EAAE,CAAC;oBACR,SAAS;oBACT,MAAM,EAAE,KAAK;oBACb,MAAM,EAAE,6CAA6C,MAAM,CAAC,OAAO,GAAG;iBACvE,CAAC,CAAA;gBACF,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClB,YAAY,GAAG,gDAAgD,CAAA;gBACjE,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,GAAG,KAAK,IAAI,SAAS,CAAA;gBACjC,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,WAAW;oBACjB,KAAK,EAAE,KAAK;oBACZ,SAAS;oBACT,MAAM;oBACN,MAAM,EAAE,UAAU,KAAK,IAAI,SAAS,EAAE;iBACvC,CAAC,CAAA;gBACF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;oBAC7B,YAAY,GAAG,oBAAoB,KAAK,sBAAsB,SAAS,GAAG,CAAA;gBAC5E,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,UAAU,EAAE,EAAE,CAAA;YACd,MAAM,MAAM,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YAC/D,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,WAAW;gBACjB,KAAK,EAAE,CAAC;gBACR,SAAS;gBACT,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,oBAAoB,MAAM,EAAE;aACrC,CAAC,CAAA;YACF,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,YAAY,GAAG,oBAAoB,MAAM,EAAE,CAAA;YAC7C,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;IAC9C,OAAO;QACL,MAAM,EAAE,SAAS;QACjB,cAAc,EAAE,YAAY;QAC5B,OAAO;KACR,CAAA;AACH,CAAC;AAED,kEAAkE;AAClE,SAAS,eAAe,CAAC,MAAe;IACtC,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,EAAE,CAAA;IACtD,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAA;IAC7C,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;AAC/B,CAAC"}

package/dist/ci/ed-runner.d.ts

@@ -0,0 +1,48 @@
+import type { TestMeasurement } from './measurement.js';
+import type { BenchmarkResult } from './benchmark.js';
+export interface EdTestRunOptions {
+    cwd?: string;
+    filter?: string;
+    failFast?: boolean;
+    noUpload?: boolean;
+    reporter?: 'default' | 'json' | 'junit';
+    /** Number of times to run each test. Passes if any run succeeds. Defaults to 1. */
+    runs?: number;
+}
+export interface EdSingleRunResult {
+    status: 'pass' | 'fail';
+    failureReason?: string;
+    measurement?: TestMeasurement;
+    benchmarkResult?: BenchmarkResult;
+    output?: unknown;
+    durationMs: number;
+    startedAt: string;
+    finishedAt: string;
+}
+export interface EdTestResult {
+    testId: string;
+    testName: string;
+    status: 'pass' | 'fail';
+    failureReason?: string;
+    measurement?: TestMeasurement;
+    benchmarkResult?: BenchmarkResult;
+    traceRef?: string;
+    target?: {
+        type: string;
+        step_id: string;
+    };
+    input?: unknown;
+    output?: unknown;
+    durationMs: number;
+    /** All individual run results when --runs > 1 */
+    singleRuns?: EdSingleRunResult[];
+}
+export interface EdTestRunResult {
+    runId: string;
+    startedAt: string;
+    finishedAt: string;
+    results: EdTestResult[];
+    sdkVersion: string;
+}
+export declare function runEdTests(options?: EdTestRunOptions): Promise<EdTestRunResult>;
+//# sourceMappingURL=ed-runner.d.ts.map

package/dist/ci/ed-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ed-runner.d.ts","sourceRoot":"","sources":["../../src/ci/ed-runner.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAOrD,MAAM,WAAW,gBAAgB;IAC/B,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,QAAQ,CAAC,EAAE,SAAS,GAAG,MAAM,GAAG,OAAO,CAAA;IACvC,mFAAmF;IACnF,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAA;IACvB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,WAAW,CAAC,EAAE,eAAe,CAAA;IAC7B,eAAe,CAAC,EAAE,eAAe,CAAA;IACjC,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,GAAG,MAAM,CAAA;IACvB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,WAAW,CAAC,EAAE,eAAe,CAAA;IAC7B,eAAe,CAAC,EAAE,eAAe,CAAA;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAA;IAC1C,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,iDAAiD;IACjD,UAAU,CAAC,EAAE,iBAAiB,EAAE,CAAA;CACjC;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,EAAE,YAAY,EAAE,CAAA;IACvB,UAAU,EAAE,MAAM,CAAA;CACnB;AAID,wBAAsB,UAAU,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC,CAyGrF"}