elasticdash-test 0.1.26 → 0.1.27

package/src/tool-registry.ts

@@ -0,0 +1,94 @@
+/**
+ * Global registry for tools defined via edTool().
+ *
+ * Tools registered here are discoverable for reruns regardless of which
+ * module they live in. The helper also applies wrapTool() so telemetry
+ * fires automatically — symmetric with the Python @ed_tool decorator.
+ */
+
+import { wrapTool } from './interceptors/tool.js'
+
+export interface RegisteredTool {
+  name: string
+  fn: (...args: unknown[]) => unknown | Promise<unknown>
+  wrapped: (...args: unknown[]) => Promise<unknown>
+  isAsync: boolean
+  signature: string
+  sourceFile: string | null
+  lineNumber: number | null
+}
+
+const _registry: Map<string, RegisteredTool> = new Map()
+
+export function getRegisteredTools(): RegisteredTool[] {
+  return Array.from(_registry.values())
+}
+
+export function getRegisteredTool(name: string): RegisteredTool | undefined {
+  return _registry.get(name)
+}
+
+export function clearToolRegistry(): void {
+  _registry.clear()
+}
+
+function inferCallerLocation(): { file: string | null; line: number | null } {
+  const stack = new Error().stack
+  if (!stack) return { file: null, line: null }
+  const lines = stack.split('\n')
+  for (let i = 2; i < lines.length; i++) {
+    const m = lines[i].match(/\((.*?):(\d+):\d+\)|at\s+(.*?):(\d+):\d+/)
+    if (!m) continue
+    const file = m[1] ?? m[3]
+    const line = parseInt(m[2] ?? m[4], 10)
+    if (file && !file.includes('tool-registry')) {
+      return { file: file.replace(/^file:\/\//, ''), line: isNaN(line) ? null : line }
+    }
+  }
+  return { file: null, line: null }
+}
+
+function inferSignature(fn: Function): string {
+  const src = fn.toString()
+  const m = src.match(/^[^(]*\(([^)]*)\)/)
+  if (!m) return '()'
+  const params = m[1]
+    .split(',')
+    .map(p => p.trim().split(/[\s=:]/)[0])
+    .filter(Boolean)
+  return `(${params.join(', ')})`
+}
+
+/**
+ * Register a function as a rerunnable tool.
+ *
+ *   export const myTool = edTool('my_tool', async (query: string) => { ... })
+ *
+ * The returned function is the telemetry-wrapped version, so calling it from
+ * normal code paths still produces traces. The CLI `run-tool` command and the
+ * ElasticDash MCP `run_tool` tool will resolve the original by name.
+ */
+export function edTool<Args extends unknown[], R>(
+  name: string,
+  fn: (...args: Args) => R | Promise<R>,
+): (...args: Args) => Promise<R> {
+  const isAsync = fn.constructor.name === 'AsyncFunction'
+  const signature = inferSignature(fn)
+  const { file, line } = inferCallerLocation()
+
+  const wrapped = wrapTool(name, fn as (...args: unknown[]) => Promise<R>) as unknown as (...args: Args) => Promise<R>
+
+  _registry.set(name, {
+    name,
+    fn: fn as RegisteredTool['fn'],
+    wrapped: wrapped as RegisteredTool['wrapped'],
+    isAsync,
+    signature,
+    sourceFile: file,
+    lineNumber: line,
+  })
+
+  return wrapped
+}
+
+export const defineTool = edTool

package/src/tool-runner-worker.ts

@@ -205,9 +205,24 @@ async function main() {
       originalExit(1)
       return
     }
-    const fn = mod[toolName]
+
+    // Registry first: covers tools defined via edTool() anywhere in the project,
+    // as long as their containing module is reachable from toolsModulePath's
+    // import graph. Falls back to ed_tools-style module export lookup.
+    let fn: ((...a: unknown[]) => unknown) | undefined
+    try {
+      const reg = await import('./tool-registry.js')
+      const registered = reg.getRegisteredTool(toolName)
+      if (registered) fn = registered.wrapped
+    } catch {
+      // Registry module not available (older SDK build); fall through to export lookup.
+    }
+    if (!fn) {
+      const exported = mod[toolName]
+      if (typeof exported === 'function') fn = exported
+    }
     if (typeof fn !== 'function') {
-      await writeResult({ ok: false, error: `"${toolName}" is not an exported function in the module.` })
+      await writeResult({ ok: false, error: `"${toolName}" not found via edTool() registry or as an exported function in the module.` })
       originalExit(1)
       return
     }

package/src/utils/debug.ts

@@ -1,8 +1,8 @@
 const DEBUG_KEY = 'ELASTICDASH_DEBUG'
 
-/** Log only when ELASTICDASH_DEBUG=1 is set. Drop-in replacement for console.log in interceptors. */
+/** Log only when ELASTICDASH_DEBUG=1 is set. Writes to stderr so callers parsing stdout (e.g. `elasticdash run-tool`) get a clean JSON channel. */
 export function debugLog(...args: unknown[]): void {
   if (typeof process !== 'undefined' && process.env?.[DEBUG_KEY] === '1') {
-    console.log(...args)
+    console.error(...args)
   }
 }

package/dist/cloud-client.d.ts

@@ -1,34 +0,0 @@
-/**
- * Cloud API client — communicates with the ElasticDash backend.
- * Used by the test-group CLI to fetch definitions and push results.
- */
-import type { TestGroup, TestGroupRunResult } from './evaluators/types.js';
-export interface CloudClientConfig {
-    apiUrl: string;
-    apiKey: string;
-    projectId: number;
-}
-export interface FetchFilters {
-    workflowName?: string;
-    tags?: string[];
-}
-export declare function fetchTestGroups(config: CloudClientConfig, filters?: FetchFilters): Promise<TestGroup[]>;
-export declare function pushRunResult(config: CloudClientConfig, testGroupId: number, result: TestGroupRunResult): Promise<{
-    id: number;
-}>;
-export declare function pushBatchResult(config: CloudClientConfig, batch: {
-    testGroupRunIds: number[];
-    gitBranch?: string;
-    gitCommit?: string;
-    passed?: boolean;
-    summary?: string;
-}): Promise<{
-    id: number;
-}>;
-export declare function exportTestGroups(config: CloudClientConfig): Promise<TestGroup[]>;
-export declare function resolveCloudConfig(options: {
-    apiUrl?: string;
-    apiKey?: string;
-    projectId?: string | number;
-}): CloudClientConfig;
-//# sourceMappingURL=cloud-client.d.ts.map

package/dist/cloud-client.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"cloud-client.d.ts","sourceRoot":"","sources":["../src/cloud-client.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAA;AAE1E,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;CAChB;AAyED,wBAAsB,eAAe,CACnC,MAAM,EAAE,iBAAiB,EACzB,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC,SAAS,EAAE,CAAC,CAKtB;AAED,wBAAsB,aAAa,CACjC,MAAM,EAAE,iBAAiB,EACzB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,kBAAkB,GACzB,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,CAMzB;AAED,wBAAsB,eAAe,CACnC,MAAM,EAAE,iBAAiB,EACzB,KAAK,EAAE;IACL,eAAe,EAAE,MAAM,EAAE,CAAA;IACzB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB,GACA,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,CAMzB;AAED,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,iBAAiB,GACxB,OAAO,CAAC,SAAS,EAAE,CAAC,CAItB;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE;IAC1C,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;CAC5B,GAAG,iBAAiB,CAyBpB"}

package/dist/cloud-client.js

@@ -1,103 +0,0 @@
-/**
- * Cloud API client — communicates with the ElasticDash backend.
- * Used by the test-group CLI to fetch definitions and push results.
- */
-class CloudClientError extends Error {
-    statusCode;
-    responseBody;
-    constructor(message, statusCode, responseBody) {
-        super(message);
-        this.statusCode = statusCode;
-        this.responseBody = responseBody;
-        this.name = 'CloudClientError';
-    }
-}
-async function request(url, apiKey, options = {}) {
-    const { method = 'GET', body, retries = 2 } = options;
-    let lastError;
-    for (let attempt = 0; attempt <= retries; attempt++) {
-        if (attempt > 0) {
-            const delayMs = Math.min(1000 * 2 ** (attempt - 1), 5000);
-            await new Promise((r) => setTimeout(r, delayMs));
-        }
-        try {
-            const res = await fetch(url, {
-                method,
-                headers: {
-                    'Authorization': `Bearer ${apiKey}`,
-                    'Content-Type': 'application/json',
-                },
-                body: body ? JSON.stringify(body) : undefined,
-                signal: AbortSignal.timeout(30_000),
-            });
-            if (res.status === 401) {
-                throw new CloudClientError('Authentication failed. Check that your ELASTICDASH_API_KEY is valid and active.', 401);
-            }
-            if (!res.ok) {
-                const text = await res.text().catch(() => '');
-                throw new CloudClientError(`API request failed: ${res.status} ${res.statusText}`, res.status, text);
-            }
-            const json = await res.json();
-            // The backend wraps responses in { status, data } via generalApiResponseSender
-            return (json.data !== undefined ? json.data : json);
-        }
-        catch (err) {
-            lastError = err;
-            // Don't retry auth errors or client errors
-            if (err instanceof CloudClientError && err.statusCode && err.statusCode < 500) {
-                throw err;
-            }
-            // Retry on network/timeout/5xx errors
-            if (attempt === retries) {
-                throw err;
-            }
-        }
-    }
-    throw lastError ?? new Error('Request failed');
-}
-export async function fetchTestGroups(config, filters) {
-    const url = new URL(`${config.apiUrl}/api/testgroups/by-project/${encodeURIComponent(String(config.projectId))}`);
-    if (filters?.workflowName)
-        url.searchParams.set('workflowName', filters.workflowName);
-    if (filters?.tags?.length)
-        url.searchParams.set('tags', filters.tags.join(','));
-    return request(url.toString(), config.apiKey);
-}
-export async function pushRunResult(config, testGroupId, result) {
-    const url = `${config.apiUrl}/api/testgroups/${encodeURIComponent(String(testGroupId))}/runs`;
-    return request(url, config.apiKey, {
-        method: 'POST',
-        body: result,
-    });
-}
-export async function pushBatchResult(config, batch) {
-    const url = `${config.apiUrl}/api/testgroups/batches`;
-    return request(url, config.apiKey, {
-        method: 'POST',
-        body: batch,
-    });
-}
-export async function exportTestGroups(config) {
-    const url = new URL(`${config.apiUrl}/api/testgroups/export`);
-    url.searchParams.set('projectId', String(config.projectId));
-    return request(url.toString(), config.apiKey);
-}
-export function resolveCloudConfig(options) {
-    const apiUrl = options.apiUrl ||
-        process.env.ELASTICDASH_API_URL ||
-        '';
-    const apiKey = options.apiKey ||
-        process.env.ELASTICDASH_API_KEY ||
-        '';
-    const projectId = Number(options.projectId ||
-        process.env.ELASTICDASH_PROJECT_ID ||
-        '1');
-    if (!apiUrl) {
-        throw new CloudClientError('Missing API URL. Set --api-url or ELASTICDASH_API_URL.');
-    }
-    if (!apiKey) {
-        throw new CloudClientError('Missing API key. Set --api-key or ELASTICDASH_API_KEY.');
-    }
-    return { apiUrl: apiUrl.replace(/\/+$/, ''), apiKey, projectId };
-}
-//# sourceMappingURL=cloud-client.js.map

package/dist/cloud-client.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"cloud-client.js","sourceRoot":"","sources":["../src/cloud-client.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAeH,MAAM,gBAAiB,SAAQ,KAAK;IAGzB;IACA;IAHT,YACE,OAAe,EACR,UAAmB,EACnB,YAAqB;QAE5B,KAAK,CAAC,OAAO,CAAC,CAAA;QAHP,eAAU,GAAV,UAAU,CAAS;QACnB,iBAAY,GAAZ,YAAY,CAAS;QAG5B,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAA;IAChC,CAAC;CACF;AAED,KAAK,UAAU,OAAO,CACpB,GAAW,EACX,MAAc,EACd,UAAiE,EAAE;IAEnE,MAAM,EAAE,MAAM,GAAG,KAAK,EAAE,IAAI,EAAE,OAAO,GAAG,CAAC,EAAE,GAAG,OAAO,CAAA;IACrD,IAAI,SAA4B,CAAA;IAEhC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACpD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;YACzD,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAA;QAClD,CAAC;QAED,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAC3B,MAAM;gBACN,OAAO,EAAE;oBACP,eAAe,EAAE,UAAU,MAAM,EAAE;oBACnC,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;gBAC7C,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;aACpC,CAAC,CAAA;YAEF,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBACvB,MAAM,IAAI,gBAAgB,CACxB,iFAAiF,EACjF,GAAG,CACJ,CAAA;YACH,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAA;gBAC7C,MAAM,IAAI,gBAAgB,CACxB,uBAAuB,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,EACrD,GAAG,CAAC,MAAM,EACV,IAAI,CACL,CAAA;YACH,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAA6B,CAAA;YACxD,+EAA+E;YAC/E,OAAO,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAM,CAAA;QAC1D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,GAAG,GAAY,CAAA;YACxB,2CAA2C;YAC3C,IAAI,GAAG,YAAY,gBAAgB,IAAI,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC;gBAC9E,MAAM,GAAG,CAAA;YACX,CAAC;YACD,sCAAsC;YACtC,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;gBACxB,MAAM,GAAG,CAAA;YACX,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,IAAI,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAA;AAChD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAyB,EACzB,OAAsB;IAEtB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,8BAA8B,kBAAkB,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAA;IACjH,IAAI,OAAO,EAAE,YAAY;QAAE,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,cAAc,EAAE,OAAO,CAAC,YAAY,CAAC,CAAA;IACrF,IAAI,OAAO,EAAE,IAAI,EAAE,MAAM;QAAE,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;IAC/E,OAAO,OAAO,CAAc,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;AAC5D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAyB,EACzB,WAAmB,EACnB,MAA0B;IAE1B,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,MAAM,mBAAmB,kBAAkB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,OAAO,CAAA;IAC7F,OAAO,OAAO,CAAiB,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE;QACjD,MAAM,EAAE,MAAM;QACd,IAAI,EAAE,MAAM;KACb,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAyB,EACzB,KAMC;IAED,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,MAAM,yBAAyB,CAAA;IACrD,OAAO,OAAO,CAAiB,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE;QACjD,MAAM,EAAE,MAAM;QACd,IAAI,EAAE,KAAK;KACZ,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,MAAyB;IAEzB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,wBAAwB,CAAC,CAAA;IAC7D,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAA;IAC3D,OAAO,OAAO,CAAc,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;AAC5D,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,OAIlC;IACC,MAAM,MAAM,GACV,OAAO,CAAC,MAAM;QACd,OAAO,CAAC,GAAG,CAAC,mBAAmB;QAC/B,EAAE,CAAA;IAEJ,MAAM,MAAM,GACV,OAAO,CAAC,MAAM;QACd,OAAO,CAAC,GAAG,CAAC,mBAAmB;QAC/B,EAAE,CAAA;IAEJ,MAAM,SAAS,GAAG,MAAM,CACtB,OAAO,CAAC,SAAS;QACjB,OAAO,CAAC,GAAG,CAAC,sBAAsB;QAClC,GAAG,CACJ,CAAA;IAED,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,gBAAgB,CAAC,wDAAwD,CAAC,CAAA;IACtF,CAAC;IACD,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,gBAAgB,CAAC,wDAAwD,CAAC,CAAA;IACtF,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAA;AAClE,CAAC"}

package/dist/evaluators/determinism.d.ts

@@ -1,3 +0,0 @@
-import type { Expectation, SingleRunData, ExpectationResult } from './types.js';
-export declare function evaluateDeterminism(expectation: Expectation, runs: SingleRunData[]): Promise<ExpectationResult>;
-//# sourceMappingURL=determinism.d.ts.map

package/dist/evaluators/determinism.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"determinism.d.ts","sourceRoot":"","sources":["../../src/evaluators/determinism.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,iBAAiB,EAAgB,MAAM,YAAY,CAAA;AA+D7F,wBAAsB,mBAAmB,CACvC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,aAAa,EAAE,GACpB,OAAO,CAAC,iBAAiB,CAAC,CA4D5B"}

package/dist/evaluators/determinism.js

@@ -1,116 +0,0 @@
-function stringify(value) {
-    if (typeof value === 'string')
-        return value;
-    if (value == null)
-        return '';
-    return JSON.stringify(value);
-}
-function stringSimilarity(a, b) {
-    if (a === b)
-        return 1.0;
-    if (a.length === 0 || b.length === 0)
-        return 0.0;
-    // Simple character-level Jaccard similarity for fast local comparison
-    const setA = new Set(a.split(' '));
-    const setB = new Set(b.split(' '));
-    const intersection = new Set([...setA].filter((x) => setB.has(x)));
-    const union = new Set([...setA, ...setB]);
-    return union.size > 0 ? intersection.size / union.size : 0;
-}
-async function llmSimilarity(a, b) {
-    const apiKey = process.env.OPENAI_API_KEY;
-    if (!apiKey) {
-        // Fallback to string similarity if no API key
-        return stringSimilarity(a, b);
-    }
-    try {
-        const res = await fetch('https://api.openai.com/v1/chat/completions', {
-            method: 'POST',
-            headers: {
-                'Authorization': `Bearer ${apiKey}`,
-                'Content-Type': 'application/json',
-            },
-            body: JSON.stringify({
-                model: 'gpt-4o-mini',
-                messages: [
-                    {
-                        role: 'system',
-                        content: 'You compare two outputs for semantic similarity. Respond with ONLY a number between 0.0 and 1.0 where 1.0 means identical meaning and 0.0 means completely different.',
-                    },
-                    {
-                        role: 'user',
-                        content: `Output A:\n${a}\n\nOutput B:\n${b}\n\nSimilarity score (0.0-1.0):`,
-                    },
-                ],
-                max_tokens: 10,
-                temperature: 0,
-            }),
-            signal: AbortSignal.timeout(15_000),
-        });
-        if (!res.ok)
-            return stringSimilarity(a, b);
-        const json = await res.json();
-        const content = json.choices?.[0]?.message?.content ?? '';
-        const score = parseFloat(content.trim());
-        return isNaN(score) ? stringSimilarity(a, b) : Math.max(0, Math.min(1, score));
-    }
-    catch {
-        return stringSimilarity(a, b);
-    }
-}
-export async function evaluateDeterminism(expectation, runs) {
-    const threshold = expectation.similarityThreshold ?? 0.8;
-    if (runs.length < 2) {
-        return {
-            expectationId: expectation.id,
-            type: 'determinism',
-            passed: true,
-            detail: 'Only 1 run, determinism check skipped',
-        };
-    }
-    const outputs = runs.map((r) => stringify(r.output));
-    // Pairwise comparison (for N ≤ 5, compare all pairs; for larger N, sample)
-    const pairs = [];
-    if (runs.length <= 5) {
-        for (let i = 0; i < runs.length; i++) {
-            for (let j = i + 1; j < runs.length; j++) {
-                pairs.push([i, j]);
-            }
-        }
-    }
-    else {
-        // Sample: compare each run against the first run
-        for (let i = 1; i < runs.length; i++) {
-            pairs.push([0, i]);
-        }
-    }
-    const scores = [];
-    for (const [i, j] of pairs) {
-        const score = await llmSimilarity(outputs[i], outputs[j]);
-        scores.push(score);
-    }
-    const avgScore = scores.reduce((a, b) => a + b, 0) / scores.length;
-    const allAboveThreshold = scores.every((s) => s >= threshold);
-    const perRun = runs.map((run, i) => {
-        // For each run, report its average similarity to other runs
-        const relevantScores = pairs
-            .map(([a, b], idx) => (a === i || b === i ? scores[idx] : null))
-            .filter((s) => s !== null);
-        const runAvg = relevantScores.length > 0
-            ? relevantScores.reduce((a, b) => a + b, 0) / relevantScores.length
-            : 1.0;
-        return {
-            runIndex: run.runIndex,
-            passed: relevantScores.every((s) => s >= threshold),
-            value: parseFloat(runAvg.toFixed(3)),
-        };
-    });
-    return {
-        expectationId: expectation.id,
-        type: 'determinism',
-        passed: allAboveThreshold,
-        detail: `avg similarity ${avgScore.toFixed(3)} (threshold: ${threshold}), ${scores.filter((s) => s >= threshold).length}/${scores.length} pairs pass`,
-        perRun,
-    };
-}
-//# sourceMappingURL=determinism.js.map

package/dist/evaluators/determinism.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"determinism.js","sourceRoot":"","sources":["../../src/evaluators/determinism.ts"],"names":[],"mappings":"AAEA,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAA;IAC3C,IAAI,KAAK,IAAI,IAAI;QAAE,OAAO,EAAE,CAAA;IAC5B,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;AAC9B,CAAC;AAED,SAAS,gBAAgB,CAAC,CAAS,EAAE,CAAS;IAC5C,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,GAAG,CAAA;IACvB,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,GAAG,CAAA;IAEhD,sEAAsE;IACtE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAA;IAClC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAA;IAClC,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAClE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,CAAC,CAAA;IACzC,OAAO,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;AAC5D,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,CAAS,EAAE,CAAS;IAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAA;IACzC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,8CAA8C;QAC9C,OAAO,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAC/B,CAAC;IAED,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,4CAA4C,EAAE;YACpE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,eAAe,EAAE,UAAU,MAAM,EAAE;gBACnC,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK,EAAE,aAAa;gBACpB,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,QAAQ;wBACd,OAAO,EAAE,uKAAuK;qBACjL;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,cAAc,CAAC,kBAAkB,CAAC,iCAAiC;qBAC7E;iBACF;gBACD,UAAU,EAAE,EAAE;gBACd,WAAW,EAAE,CAAC;aACf,CAAC;YACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;SACpC,CAAC,CAAA;QAEF,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,OAAO,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;QAE1C,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAS,CAAA;QACpC,MAAM,OAAO,GAAW,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,CAAA;QACjE,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAA;QACxC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAA;IAChF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAC/B,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,WAAwB,EACxB,IAAqB;IAErB,MAAM,SAAS,GAAG,WAAW,CAAC,mBAAmB,IAAI,GAAG,CAAA;IAExD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpB,OAAO;YACL,aAAa,EAAE,WAAW,CAAC,EAAE;YAC7B,IAAI,EAAE,aAAa;YACnB,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,uCAAuC;SAChD,CAAA;IACH,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;IAEpD,2EAA2E;IAC3E,MAAM,KAAK,GAA4B,EAAE,CAAA;IACzC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YACpB,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,iDAAiD;QACjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QACpB,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;QACzD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACpB,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;IAClE,MAAM,iBAAiB,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,SAAS,CAAC,CAAA;IAE7D,MAAM,MAAM,GAAmB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;QACjD,4DAA4D;QAC5D,MAAM,cAAc,GAAG,KAAK;aACzB,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;aAC/D,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAA;QACzC,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,GAAG,CAAC;YACtC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,cAAc,CAAC,MAAM;YACnE,CAAC,CAAC,GAAG,CAAA;QACP,OAAO;YACL,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,MAAM,EAAE,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,SAAS,CAAC;YACnD,KAAK,EAAE,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;SACrC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,OAAO;QACL,aAAa,EAAE,WAAW,CAAC,EAAE;QAC7B,IAAI,EAAE,aAAa;QACnB,MAAM,EAAE,iBAAiB;QACzB,MAAM,EAAE,kBAAkB,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,gBAAgB,SAAS,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,aAAa;QACrJ,MAAM;KACP,CAAA;AACH,CAAC"}

package/dist/evaluators/index.d.ts

@@ -1,4 +0,0 @@
-import type { Expectation, SingleRunData, EvaluationResult } from './types.js';
-export { type Expectation, type SingleRunData, type ExpectationResult, type EvaluationResult } from './types.js';
-export declare function evaluateExpectations(expectations: Expectation[], runs: SingleRunData[], passThreshold?: 'all' | number): Promise<EvaluationResult>;
-//# sourceMappingURL=index.d.ts.map

package/dist/evaluators/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/evaluators/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,aAAa,EAAqB,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAQjG,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,aAAa,EAAE,KAAK,iBAAiB,EAAE,KAAK,gBAAgB,EAAE,MAAM,YAAY,CAAA;AA6BhH,wBAAsB,oBAAoB,CACxC,YAAY,EAAE,WAAW,EAAE,EAC3B,IAAI,EAAE,aAAa,EAAE,EACrB,aAAa,GAAE,KAAK,GAAG,MAAc,GACpC,OAAO,CAAC,gBAAgB,CAAC,CAmC3B"}

package/dist/evaluators/index.js

@@ -1,61 +0,0 @@
-import { evaluateTokenBudget } from './token-budget.js';
-import { evaluateLatencyBudget } from './latency-budget.js';
-import { evaluateOutputContains } from './output-contains.js';
-import { evaluateOutputSchema } from './output-schema.js';
-import { evaluateLLMJudge } from './llm-judge.js';
-import { evaluateDeterminism } from './determinism.js';
-async function evaluateOne(expectation, runs) {
-    switch (expectation.type) {
-        case 'token-budget':
-            return evaluateTokenBudget(expectation, runs);
-        case 'latency-budget':
-            return evaluateLatencyBudget(expectation, runs);
-        case 'output-contains':
-            return evaluateOutputContains(expectation, runs);
-        case 'output-schema':
-            return evaluateOutputSchema(expectation, runs);
-        case 'llm-judge':
-            return evaluateLLMJudge(expectation, runs);
-        case 'determinism':
-            return evaluateDeterminism(expectation, runs);
-        default:
-            return {
-                expectationId: expectation.id,
-                type: expectation.type,
-                passed: false,
-                detail: `Unknown expectation type: ${expectation.type}`,
-            };
-    }
-}
-export async function evaluateExpectations(expectations, runs, passThreshold = 'all') {
-    const results = [];
-    for (const expectation of expectations) {
-        const result = await evaluateOne(expectation, runs);
-        results.push(result);
-    }
-    // Determine overall pass/fail
-    const passedExpectations = results.filter((r) => r.passed).length;
-    const totalExpectations = results.length;
-    let passed;
-    if (passThreshold === 'all') {
-        passed = results.every((r) => r.passed);
-    }
-    else {
-        passed = passedExpectations >= passThreshold;
-    }
-    // Build summary
-    const parts = [];
-    parts.push(`${passedExpectations}/${totalExpectations} expectations passed`);
-    const failedResults = results.filter((r) => !r.passed);
-    if (failedResults.length > 0) {
-        for (const fr of failedResults) {
-            parts.push(`  ${fr.type}: ${fr.detail}`);
-        }
-    }
-    return {
-        passed,
-        summary: parts[0],
-        results,
-    };
-}
-//# sourceMappingURL=index.js.map

package/dist/evaluators/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/evaluators/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAA;AACvD,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAA;AAC3D,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAA;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AACjD,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAA;AAItD,KAAK,UAAU,WAAW,CACxB,WAAwB,EACxB,IAAqB;IAErB,QAAQ,WAAW,CAAC,IAAI,EAAE,CAAC;QACzB,KAAK,cAAc;YACjB,OAAO,mBAAmB,CAAC,WAAW,EAAE,IAAI,CAAC,CAAA;QAC/C,KAAK,gBAAgB;YACnB,OAAO,qBAAqB,CAAC,WAAW,EAAE,IAAI,CAAC,CAAA;QACjD,KAAK,iBAAiB;YACpB,OAAO,sBAAsB,CAAC,WAAW,EAAE,IAAI,CAAC,CAAA;QAClD,KAAK,eAAe;YAClB,OAAO,oBAAoB,CAAC,WAAW,EAAE,IAAI,CAAC,CAAA;QAChD,KAAK,WAAW;YACd,OAAO,gBAAgB,CAAC,WAAW,EAAE,IAAI,CAAC,CAAA;QAC5C,KAAK,aAAa;YAChB,OAAO,mBAAmB,CAAC,WAAW,EAAE,IAAI,CAAC,CAAA;QAC/C;YACE,OAAO;gBACL,aAAa,EAAE,WAAW,CAAC,EAAE;gBAC7B,IAAI,EAAE,WAAW,CAAC,IAAI;gBACtB,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,6BAA6B,WAAW,CAAC,IAAI,EAAE;aACxD,CAAA;IACL,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,YAA2B,EAC3B,IAAqB,EACrB,gBAAgC,KAAK;IAErC,MAAM,OAAO,GAAwB,EAAE,CAAA;IAEvC,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,WAAW,EAAE,IAAI,CAAC,CAAA;QACnD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IACtB,CAAC;IAED,8BAA8B;IAC9B,MAAM,kBAAkB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAA;IACjE,MAAM,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAA;IACxC,IAAI,MAAe,CAAA;IAEnB,IAAI,aAAa,KAAK,KAAK,EAAE,CAAC;QAC5B,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;IACzC,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,kBAAkB,IAAI,aAAa,CAAA;IAC9C,CAAC;IAED,gBAAgB;IAChB,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,KAAK,CAAC,IAAI,CAAC,GAAG,kBAAkB,IAAI,iBAAiB,sBAAsB,CAAC,CAAA;IAE5E,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;IACtD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,MAAM,EAAE,IAAI,aAAa,EAAE,CAAC;YAC/B,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;QAC1C,CAAC;IACH,CAAC;IAED,OAAO;QACL,MAAM;QACN,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QACjB,OAAO;KACR,CAAA;AACH,CAAC"}

package/dist/evaluators/latency-budget.d.ts

@@ -1,3 +0,0 @@
-import type { Expectation, SingleRunData, ExpectationResult } from './types.js';
-export declare function evaluateLatencyBudget(expectation: Expectation, runs: SingleRunData[]): ExpectationResult;
-//# sourceMappingURL=latency-budget.d.ts.map

package/dist/evaluators/latency-budget.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"latency-budget.d.ts","sourceRoot":"","sources":["../../src/evaluators/latency-budget.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,iBAAiB,EAAgB,MAAM,YAAY,CAAA;AAE7F,wBAAgB,qBAAqB,CACnC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,aAAa,EAAE,GACpB,iBAAiB,CAgDnB"}

package/dist/evaluators/latency-budget.js

@@ -1,45 +0,0 @@
-export function evaluateLatencyBudget(expectation, runs) {
-    const perRun = [];
-    let allPassed = true;
-    // Per-run check
-    if (expectation.maxDurationMs != null) {
-        for (const run of runs) {
-            const passed = run.durationMs <= expectation.maxDurationMs;
-            if (!passed)
-                allPassed = false;
-            perRun.push({ runIndex: run.runIndex, passed, value: run.durationMs });
-        }
-    }
-    // Total check
-    const totalDuration = runs.reduce((sum, r) => sum + r.durationMs, 0);
-    if (expectation.maxTotalDurationMs != null && totalDuration > expectation.maxTotalDurationMs) {
-        allPassed = false;
-    }
-    // If neither constraint is set, pass by default
-    if (expectation.maxDurationMs == null && expectation.maxTotalDurationMs == null) {
-        for (const run of runs) {
-            perRun.push({ runIndex: run.runIndex, passed: true, value: run.durationMs });
-        }
-    }
-    // Build detail
-    const avg = runs.length > 0 ? Math.round(totalDuration / runs.length) : 0;
-    const formatMs = (ms) => ms >= 1000 ? `${(ms / 1000).toFixed(1)}s` : `${ms}ms`;
-    const parts = [];
-    if (expectation.maxDurationMs != null) {
-        const passedCount = perRun.filter((r) => r.passed).length;
-        parts.push(`${passedCount}/${runs.length} under ${formatMs(expectation.maxDurationMs)}`);
-    }
-    if (expectation.maxTotalDurationMs != null) {
-        const totalPassed = totalDuration <= expectation.maxTotalDurationMs;
-        parts.push(`total ${formatMs(totalDuration)} ${totalPassed ? 'within' : 'exceeds'} ${formatMs(expectation.maxTotalDurationMs)} budget`);
-    }
-    parts.push(`avg ${formatMs(avg)}`);
-    return {
-        expectationId: expectation.id,
-        type: 'latency-budget',
-        passed: allPassed,
-        detail: parts.join(', '),
-        perRun: perRun.length > 0 ? perRun : undefined,
-    };
-}
-//# sourceMappingURL=latency-budget.js.map

package/dist/evaluators/latency-budget.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"latency-budget.js","sourceRoot":"","sources":["../../src/evaluators/latency-budget.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,qBAAqB,CACnC,WAAwB,EACxB,IAAqB;IAErB,MAAM,MAAM,GAAmB,EAAE,CAAA;IACjC,IAAI,SAAS,GAAG,IAAI,CAAA;IAEpB,gBAAgB;IAChB,IAAI,WAAW,CAAC,aAAa,IAAI,IAAI,EAAE,CAAC;QACtC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,IAAI,WAAW,CAAC,aAAa,CAAA;YAC1D,IAAI,CAAC,MAAM;gBAAE,SAAS,GAAG,KAAK,CAAA;YAC9B,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;QACxE,CAAC;IACH,CAAC;IAED,cAAc;IACd,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAA;IACpE,IAAI,WAAW,CAAC,kBAAkB,IAAI,IAAI,IAAI,aAAa,GAAG,WAAW,CAAC,kBAAkB,EAAE,CAAC;QAC7F,SAAS,GAAG,KAAK,CAAA;IACnB,CAAC;IAED,gDAAgD;IAChD,IAAI,WAAW,CAAC,aAAa,IAAI,IAAI,IAAI,WAAW,CAAC,kBAAkB,IAAI,IAAI,EAAE,CAAC;QAChF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;QAC9E,CAAC;IACH,CAAC;IAED,eAAe;IACf,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACzE,MAAM,QAAQ,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,CAAA;IACtF,MAAM,KAAK,GAAa,EAAE,CAAA;IAE1B,IAAI,WAAW,CAAC,aAAa,IAAI,IAAI,EAAE,CAAC;QACtC,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAA;QACzD,KAAK,CAAC,IAAI,CAAC,GAAG,WAAW,IAAI,IAAI,CAAC,MAAM,UAAU,QAAQ,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;IAC1F,CAAC;IACD,IAAI,WAAW,CAAC,kBAAkB,IAAI,IAAI,EAAE,CAAC;QAC3C,MAAM,WAAW,GAAG,aAAa,IAAI,WAAW,CAAC,kBAAkB,CAAA;QACnE,KAAK,CAAC,IAAI,CAAC,SAAS,QAAQ,CAAC,aAAa,CAAC,IAAI,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,IAAI,QAAQ,CAAC,WAAW,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAA;IACzI,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,OAAO,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAElC,OAAO;QACL,aAAa,EAAE,WAAW,CAAC,EAAE;QAC7B,IAAI,EAAE,gBAAgB;QACtB,MAAM,EAAE,SAAS;QACjB,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;QACxB,MAAM,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;KAC/C,CAAA;AACH,CAAC"}

package/dist/evaluators/llm-judge.d.ts

@@ -1,3 +0,0 @@
-import type { Expectation, SingleRunData, ExpectationResult } from './types.js';
-export declare function evaluateLLMJudge(expectation: Expectation, runs: SingleRunData[]): Promise<ExpectationResult>;
-//# sourceMappingURL=llm-judge.d.ts.map

package/dist/evaluators/llm-judge.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"llm-judge.d.ts","sourceRoot":"","sources":["../../src/evaluators/llm-judge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,iBAAiB,EAAgB,MAAM,YAAY,CAAA;AAqG7F,wBAAsB,gBAAgB,CACpC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,aAAa,EAAE,GACpB,OAAO,CAAC,iBAAiB,CAAC,CA+C5B"}