elasticdash-test 0.1.16 → 0.1.17-alpha-2

package/dist/evaluators/llm-judge.d.ts

@@ -0,0 +1,3 @@
+import type { Expectation, SingleRunData, ExpectationResult } from './types.js';
+export declare function evaluateLLMJudge(expectation: Expectation, runs: SingleRunData[]): Promise<ExpectationResult>;
+//# sourceMappingURL=llm-judge.d.ts.map

package/dist/evaluators/llm-judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-judge.d.ts","sourceRoot":"","sources":["../../src/evaluators/llm-judge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,iBAAiB,EAAgB,MAAM,YAAY,CAAA;AAqG7F,wBAAsB,gBAAgB,CACpC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,aAAa,EAAE,GACpB,OAAO,CAAC,iBAAiB,CAAC,CA+C5B"}

package/dist/evaluators/llm-judge.js

@@ -0,0 +1,122 @@
+async function callJudgeLLM(prompt, output, model, provider) {
+    const outputStr = typeof output === 'string' ? output : JSON.stringify(output, null, 2);
+    const systemPrompt = `You are an evaluation judge. You will be given an output from an AI workflow and a criteria to evaluate it against. Respond with exactly YES or NO on the first line, followed by a brief reasoning on the next line.`;
+    const userMessage = `## Criteria\n${prompt}\n\n## Output to evaluate\n${outputStr}\n\nDoes the output meet the criteria? Answer YES or NO, then explain briefly.`;
+    if (provider === 'openai' || provider === 'default') {
+        const apiKey = process.env.OPENAI_API_KEY;
+        if (!apiKey)
+            throw new Error('OPENAI_API_KEY not set (needed for LLM judge)');
+        const res = await fetch('https://api.openai.com/v1/chat/completions', {
+            method: 'POST',
+            headers: {
+                'Authorization': `Bearer ${apiKey}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                model,
+                messages: [
+                    { role: 'system', content: systemPrompt },
+                    { role: 'user', content: userMessage },
+                ],
+                max_tokens: 200,
+                temperature: 0,
+            }),
+            signal: AbortSignal.timeout(30_000),
+        });
+        if (!res.ok) {
+            throw new Error(`OpenAI API error: ${res.status} ${res.statusText}`);
+        }
+        const json = await res.json();
+        const content = json.choices?.[0]?.message?.content ?? '';
+        return parseVerdict(content);
+    }
+    if (provider === 'anthropic') {
+        const apiKey = process.env.ANTHROPIC_API_KEY;
+        if (!apiKey)
+            throw new Error('ANTHROPIC_API_KEY not set (needed for LLM judge)');
+        const res = await fetch('https://api.anthropic.com/v1/messages', {
+            method: 'POST',
+            headers: {
+                'x-api-key': apiKey,
+                'content-type': 'application/json',
+                'anthropic-version': '2023-06-01',
+            },
+            body: JSON.stringify({
+                model,
+                system: systemPrompt,
+                messages: [{ role: 'user', content: userMessage }],
+                max_tokens: 200,
+                temperature: 0,
+            }),
+            signal: AbortSignal.timeout(30_000),
+        });
+        if (!res.ok) {
+            throw new Error(`Anthropic API error: ${res.status} ${res.statusText}`);
+        }
+        const json = await res.json();
+        const content = json.content?.[0]?.text ?? '';
+        return parseVerdict(content);
+    }
+    throw new Error(`Unsupported judge provider: ${provider}. Use 'openai' or 'anthropic'.`);
+}
+function parseVerdict(content) {
+    const lines = content.trim().split('\n');
+    const firstLine = lines[0]?.trim().toUpperCase() ?? '';
+    const passed = firstLine.startsWith('YES');
+    const reasoning = lines.slice(1).join('\n').trim() || (passed ? 'Criteria met' : 'Criteria not met');
+    return { passed, reasoning };
+}
+function resolveModel(expectation) {
+    const provider = expectation.judgeProvider || detectProvider(expectation.judgeModel);
+    const model = expectation.judgeModel || 'gpt-4o-mini';
+    return { model, provider };
+}
+function detectProvider(model) {
+    if (!model)
+        return 'default';
+    if (model.startsWith('claude'))
+        return 'anthropic';
+    return 'default';
+}
+export async function evaluateLLMJudge(expectation, runs) {
+    if (!expectation.judgePrompt) {
+        return {
+            expectationId: expectation.id,
+            type: 'llm-judge',
+            passed: false,
+            detail: 'No judge prompt provided',
+        };
+    }
+    const { model, provider } = resolveModel(expectation);
+    const perRun = [];
+    let allPassed = true;
+    for (const run of runs) {
+        try {
+            const verdict = await callJudgeLLM(expectation.judgePrompt, run.output, model, provider === 'default' ? 'openai' : provider);
+            if (!verdict.passed)
+                allPassed = false;
+            perRun.push({
+                runIndex: run.runIndex,
+                passed: verdict.passed,
+                value: verdict.reasoning,
+            });
+        }
+        catch (err) {
+            allPassed = false;
+            perRun.push({
+                runIndex: run.runIndex,
+                passed: false,
+                value: `error: ${err.message}`,
+            });
+        }
+    }
+    const passedCount = perRun.filter((r) => r.passed).length;
+    return {
+        expectationId: expectation.id,
+        type: 'llm-judge',
+        passed: allPassed,
+        detail: `${passedCount}/${runs.length} passed judge evaluation (${model})`,
+        perRun,
+    };
+}
+//# sourceMappingURL=llm-judge.js.map

package/dist/evaluators/llm-judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-judge.js","sourceRoot":"","sources":["../../src/evaluators/llm-judge.ts"],"names":[],"mappings":"AAOA,KAAK,UAAU,YAAY,CACzB,MAAc,EACd,MAAe,EACf,KAAa,EACb,QAAgB;IAEhB,MAAM,SAAS,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAA;IACvF,MAAM,YAAY,GAAG,uNAAuN,CAAA;IAC5O,MAAM,WAAW,GAAG,gBAAgB,MAAM,8BAA8B,SAAS,gFAAgF,CAAA;IAEjK,IAAI,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACpD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAA;QACzC,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAA;QAE7E,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,4CAA4C,EAAE;YACpE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,eAAe,EAAE,UAAU,MAAM,EAAE;gBACnC,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK;gBACL,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE;oBACzC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE;iBACvC;gBACD,UAAU,EAAE,GAAG;gBACf,WAAW,EAAE,CAAC;aACf,CAAC;YACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;SACpC,CAAC,CAAA;QAEF,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,qBAAqB,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;QACtE,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAS,CAAA;QACpC,MAAM,OAAO,GAAW,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,CAAA;QACjE,OAAO,YAAY,CAAC,OAAO,CAAC,CAAA;IAC9B,CAAC;IAED,IAAI,QAAQ,KAAK,WAAW,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAA;QAC5C,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAA;QAEhF,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,uCAAuC,EAAE;YAC/D,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,WAAW,EAAE,MAAM;gBACnB,cAAc,EAAE,kBAAkB;gBAClC,mBAAmB,EAAE,YAAY;aAClC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK;gBACL,MAAM,EAAE,YAAY;gBACpB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;gBAClD,UAAU,EAAE,GAAG;gBACf,WAAW,EAAE,CAAC;aACf,CAAC;YACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;SACpC,CAAC,CAAA;QAEF,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;QACzE,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAS,CAAA;QACpC,MAAM,OAAO,GAAW,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,EAAE,CAAA;QACrD,OAAO,YAAY,CAAC,OAAO,CAAC,CAAA;IAC9B,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,gCAAgC,CAAC,CAAA;AAC1F,CAAC;AAED,SAAS,YAAY,CAAC,OAAe;IACnC,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACxC,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAA;IACtD,MAAM,MAAM,GAAG,SAAS,CAAC,UAAU,CAAC,KAAK,CAAC,CAAA;IAC1C,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAA;IACpG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAA;AAC9B,CAAC;AAED,SAAS,YAAY,CAAC,WAAwB;IAC5C,MAAM,QAAQ,GAAG,WAAW,CAAC,aAAa,IAAI,cAAc,CAAC,WAAW,CAAC,UAAU,CAAC,CAAA;IACpF,MAAM,KAAK,GAAG,WAAW,CAAC,UAAU,IAAI,aAAa,CAAA;IACrD,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAA;AAC5B,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAA;IAC5B,IAAI,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,WAAW,CAAA;IAClD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,WAAwB,EACxB,IAAqB;IAErB,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;QAC7B,OAAO;YACL,aAAa,EAAE,WAAW,CAAC,EAAE;YAC7B,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,0BAA0B;SACnC,CAAA;IACH,CAAC;IAED,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,YAAY,CAAC,WAAW,CAAC,CAAA;IACrD,MAAM,MAAM,GAAmB,EAAE,CAAA;IACjC,IAAI,SAAS,GAAG,IAAI,CAAA;IAEpB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,YAAY,CAChC,WAAW,CAAC,WAAW,EACvB,GAAG,CAAC,MAAM,EACV,KAAK,EACL,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAC7C,CAAA;YACD,IAAI,CAAC,OAAO,CAAC,MAAM;gBAAE,SAAS,GAAG,KAAK,CAAA;YACtC,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,KAAK,EAAE,OAAO,CAAC,SAAS;aACzB,CAAC,CAAA;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,GAAG,KAAK,CAAA;YACjB,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,UAAW,GAAa,CAAC,OAAO,EAAE;aAC1C,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAA;IAEzD,OAAO;QACL,aAAa,EAAE,WAAW,CAAC,EAAE;QAC7B,IAAI,EAAE,WAAW;QACjB,MAAM,EAAE,SAAS;QACjB,MAAM,EAAE,GAAG,WAAW,IAAI,IAAI,CAAC,MAAM,6BAA6B,KAAK,GAAG;QAC1E,MAAM;KACP,CAAA;AACH,CAAC"}

package/dist/evaluators/output-contains.d.ts

@@ -0,0 +1,3 @@
+import type { Expectation, SingleRunData, ExpectationResult } from './types.js';
+export declare function evaluateOutputContains(expectation: Expectation, runs: SingleRunData[]): ExpectationResult;
+//# sourceMappingURL=output-contains.d.ts.map

package/dist/evaluators/output-contains.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"output-contains.d.ts","sourceRoot":"","sources":["../../src/evaluators/output-contains.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,iBAAiB,EAAgB,MAAM,YAAY,CAAA;AAQ7F,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,aAAa,EAAE,GACpB,iBAAiB,CAgDnB"}

package/dist/evaluators/output-contains.js

@@ -0,0 +1,52 @@
+function stringify(value) {
+    if (typeof value === 'string')
+        return value;
+    if (value == null)
+        return '';
+    return JSON.stringify(value);
+}
+export function evaluateOutputContains(expectation, runs) {
+    const perRun = [];
+    let allPassed = true;
+    for (const run of runs) {
+        let output = stringify(run.output);
+        let contains = expectation.containsText ?? '';
+        let notContains = expectation.notContainsText ?? '';
+        if (expectation.caseInsensitive) {
+            output = output.toLowerCase();
+            contains = contains.toLowerCase();
+            notContains = notContains.toLowerCase();
+        }
+        let passed = true;
+        const reasons = [];
+        if (expectation.containsText && !output.includes(contains)) {
+            passed = false;
+            reasons.push(`missing "${expectation.containsText}"`);
+        }
+        if (expectation.notContainsText && output.includes(notContains)) {
+            passed = false;
+            reasons.push(`contains forbidden "${expectation.notContainsText}"`);
+        }
+        if (!passed)
+            allPassed = false;
+        perRun.push({
+            runIndex: run.runIndex,
+            passed,
+            value: passed ? 'ok' : reasons.join('; '),
+        });
+    }
+    const passedCount = perRun.filter((r) => r.passed).length;
+    const parts = [`${passedCount}/${runs.length} passed`];
+    if (expectation.containsText)
+        parts.push(`contains: "${expectation.containsText}"`);
+    if (expectation.notContainsText)
+        parts.push(`excludes: "${expectation.notContainsText}"`);
+    return {
+        expectationId: expectation.id,
+        type: 'output-contains',
+        passed: allPassed,
+        detail: parts.join(', '),
+        perRun,
+    };
+}
+//# sourceMappingURL=output-contains.js.map

package/dist/evaluators/output-contains.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"output-contains.js","sourceRoot":"","sources":["../../src/evaluators/output-contains.ts"],"names":[],"mappings":"AAEA,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAA;IAC3C,IAAI,KAAK,IAAI,IAAI;QAAE,OAAO,EAAE,CAAA;IAC5B,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;AAC9B,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,WAAwB,EACxB,IAAqB;IAErB,MAAM,MAAM,GAAmB,EAAE,CAAA;IACjC,IAAI,SAAS,GAAG,IAAI,CAAA;IAEpB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;QAClC,IAAI,QAAQ,GAAG,WAAW,CAAC,YAAY,IAAI,EAAE,CAAA;QAC7C,IAAI,WAAW,GAAG,WAAW,CAAC,eAAe,IAAI,EAAE,CAAA;QAEnD,IAAI,WAAW,CAAC,eAAe,EAAE,CAAC;YAChC,MAAM,GAAG,MAAM,CAAC,WAAW,EAAE,CAAA;YAC7B,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAA;YACjC,WAAW,GAAG,WAAW,CAAC,WAAW,EAAE,CAAA;QACzC,CAAC;QAED,IAAI,MAAM,GAAG,IAAI,CAAA;QACjB,MAAM,OAAO,GAAa,EAAE,CAAA;QAE5B,IAAI,WAAW,CAAC,YAAY,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3D,MAAM,GAAG,KAAK,CAAA;YACd,OAAO,CAAC,IAAI,CAAC,YAAY,WAAW,CAAC,YAAY,GAAG,CAAC,CAAA;QACvD,CAAC;QAED,IAAI,WAAW,CAAC,eAAe,IAAI,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YAChE,MAAM,GAAG,KAAK,CAAA;YACd,OAAO,CAAC,IAAI,CAAC,uBAAuB,WAAW,CAAC,eAAe,GAAG,CAAC,CAAA;QACrE,CAAC;QAED,IAAI,CAAC,MAAM;YAAE,SAAS,GAAG,KAAK,CAAA;QAC9B,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,MAAM;YACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;SAC1C,CAAC,CAAA;IACJ,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAA;IACzD,MAAM,KAAK,GAAa,CAAC,GAAG,WAAW,IAAI,IAAI,CAAC,MAAM,SAAS,CAAC,CAAA;IAChE,IAAI,WAAW,CAAC,YAAY;QAAE,KAAK,CAAC,IAAI,CAAC,cAAc,WAAW,CAAC,YAAY,GAAG,CAAC,CAAA;IACnF,IAAI,WAAW,CAAC,eAAe;QAAE,KAAK,CAAC,IAAI,CAAC,cAAc,WAAW,CAAC,eAAe,GAAG,CAAC,CAAA;IAEzF,OAAO;QACL,aAAa,EAAE,WAAW,CAAC,EAAE;QAC7B,IAAI,EAAE,iBAAiB;QACvB,MAAM,EAAE,SAAS;QACjB,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;QACxB,MAAM;KACP,CAAA;AACH,CAAC"}

package/dist/evaluators/output-schema.d.ts

@@ -0,0 +1,3 @@
+import type { Expectation, SingleRunData, ExpectationResult } from './types.js';
+export declare function evaluateOutputSchema(expectation: Expectation, runs: SingleRunData[]): Promise<ExpectationResult>;
+//# sourceMappingURL=output-schema.d.ts.map

package/dist/evaluators/output-schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"output-schema.d.ts","sourceRoot":"","sources":["../../src/evaluators/output-schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,iBAAiB,EAAgB,MAAM,YAAY,CAAA;AAE7F,wBAAsB,oBAAoB,CACxC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,aAAa,EAAE,GACpB,OAAO,CAAC,iBAAiB,CAAC,CA4D5B"}

package/dist/evaluators/output-schema.js

@@ -0,0 +1,58 @@
+export async function evaluateOutputSchema(expectation, runs) {
+    if (!expectation.jsonSchema) {
+        return {
+            expectationId: expectation.id,
+            type: 'output-schema',
+            passed: true,
+            detail: 'No JSON schema provided, skipped',
+        };
+    }
+    // Dynamic import ajv — it's an optional dependency
+    let Ajv;
+    try {
+        // @ts-ignore — ajv is an optional peer dependency
+        Ajv = (await import('ajv')).default;
+    }
+    catch {
+        return {
+            expectationId: expectation.id,
+            type: 'output-schema',
+            passed: false,
+            detail: 'ajv package not installed. Run: npm install ajv',
+        };
+    }
+    const ajv = new Ajv({ allErrors: true });
+    const validate = ajv.compile(expectation.jsonSchema);
+    const perRun = [];
+    let allPassed = true;
+    for (const run of runs) {
+        let data = run.output;
+        // If output is a string, try to parse it as JSON
+        if (typeof data === 'string') {
+            try {
+                data = JSON.parse(data);
+            }
+            catch {
+                // leave as string — will likely fail validation
+            }
+        }
+        const valid = validate(data);
+        if (!valid) {
+            allPassed = false;
+            const errors = validate.errors?.map((e) => `${e.instancePath || '/'} ${e.message}`).join('; ') ?? 'unknown error';
+            perRun.push({ runIndex: run.runIndex, passed: false, value: errors });
+        }
+        else {
+            perRun.push({ runIndex: run.runIndex, passed: true, value: 'valid' });
+        }
+    }
+    const passedCount = perRun.filter((r) => r.passed).length;
+    return {
+        expectationId: expectation.id,
+        type: 'output-schema',
+        passed: allPassed,
+        detail: `${passedCount}/${runs.length} outputs match schema`,
+        perRun,
+    };
+}
+//# sourceMappingURL=output-schema.js.map

package/dist/evaluators/output-schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"output-schema.js","sourceRoot":"","sources":["../../src/evaluators/output-schema.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,WAAwB,EACxB,IAAqB;IAErB,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;QAC5B,OAAO;YACL,aAAa,EAAE,WAAW,CAAC,EAAE;YAC7B,IAAI,EAAE,eAAe;YACrB,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,kCAAkC;SAC3C,CAAA;IACH,CAAC;IAED,mDAAmD;IACnD,IAAI,GAAQ,CAAA;IACZ,IAAI,CAAC;QACH,kDAAkD;QAClD,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAA;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,aAAa,EAAE,WAAW,CAAC,EAAE;YAC7B,IAAI,EAAE,eAAe;YACrB,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,iDAAiD;SAC1D,CAAA;IACH,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACxC,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,CAAA;IAEpD,MAAM,MAAM,GAAmB,EAAE,CAAA;IACjC,IAAI,SAAS,GAAG,IAAI,CAAA;IAEpB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,IAAI,GAAG,GAAG,CAAC,MAAM,CAAA;QACrB,iDAAiD;QACjD,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;YACzB,CAAC;YAAC,MAAM,CAAC;gBACP,gDAAgD;YAClD,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;QAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,SAAS,GAAG,KAAK,CAAA;YACjB,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,YAAY,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,eAAe,CAAA;YACtH,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAA;QACvE,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAA;QACvE,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAA;IAEzD,OAAO;QACL,aAAa,EAAE,WAAW,CAAC,EAAE;QAC7B,IAAI,EAAE,eAAe;QACrB,MAAM,EAAE,SAAS;QACjB,MAAM,EAAE,GAAG,WAAW,IAAI,IAAI,CAAC,MAAM,uBAAuB;QAC5D,MAAM;KACP,CAAA;AACH,CAAC"}

package/dist/evaluators/token-budget.d.ts

@@ -0,0 +1,3 @@
+import type { Expectation, SingleRunData, ExpectationResult } from './types.js';
+export declare function evaluateTokenBudget(expectation: Expectation, runs: SingleRunData[]): ExpectationResult;
+//# sourceMappingURL=token-budget.d.ts.map

package/dist/evaluators/token-budget.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-budget.d.ts","sourceRoot":"","sources":["../../src/evaluators/token-budget.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,iBAAiB,EAAgB,MAAM,YAAY,CAAA;AAE7F,wBAAgB,mBAAmB,CACjC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,aAAa,EAAE,GACpB,iBAAiB,CAgDnB"}

package/dist/evaluators/token-budget.js

@@ -0,0 +1,45 @@
+export function evaluateTokenBudget(expectation, runs) {
+    const perRun = [];
+    let allPassed = true;
+    // Per-run check
+    if (expectation.maxTokensPerRun != null) {
+        for (const run of runs) {
+            const tokens = run.tokenUsage.totalTokens;
+            const passed = tokens <= expectation.maxTokensPerRun;
+            if (!passed)
+                allPassed = false;
+            perRun.push({ runIndex: run.runIndex, passed, value: tokens });
+        }
+    }
+    // Total check
+    const totalTokens = runs.reduce((sum, r) => sum + r.tokenUsage.totalTokens, 0);
+    if (expectation.maxTotalTokens != null && totalTokens > expectation.maxTotalTokens) {
+        allPassed = false;
+    }
+    // If neither constraint is set, pass by default but populate per-run
+    if (expectation.maxTokensPerRun == null && expectation.maxTotalTokens == null) {
+        for (const run of runs) {
+            perRun.push({ runIndex: run.runIndex, passed: true, value: run.tokenUsage.totalTokens });
+        }
+    }
+    // Build detail string
+    const avg = runs.length > 0 ? Math.round(totalTokens / runs.length) : 0;
+    const parts = [];
+    if (expectation.maxTokensPerRun != null) {
+        const passedCount = perRun.filter((r) => r.passed).length;
+        parts.push(`${passedCount}/${runs.length} runs under ${expectation.maxTokensPerRun} tokens/run`);
+    }
+    if (expectation.maxTotalTokens != null) {
+        const totalPassed = totalTokens <= expectation.maxTotalTokens;
+        parts.push(`total ${totalTokens} tokens ${totalPassed ? 'within' : 'exceeds'} ${expectation.maxTotalTokens} budget`);
+    }
+    parts.push(`avg ${avg} tokens/run`);
+    return {
+        expectationId: expectation.id,
+        type: 'token-budget',
+        passed: allPassed,
+        detail: parts.join(', '),
+        perRun: perRun.length > 0 ? perRun : undefined,
+    };
+}
+//# sourceMappingURL=token-budget.js.map

package/dist/evaluators/token-budget.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-budget.js","sourceRoot":"","sources":["../../src/evaluators/token-budget.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,mBAAmB,CACjC,WAAwB,EACxB,IAAqB;IAErB,MAAM,MAAM,GAAmB,EAAE,CAAA;IACjC,IAAI,SAAS,GAAG,IAAI,CAAA;IAEpB,gBAAgB;IAChB,IAAI,WAAW,CAAC,eAAe,IAAI,IAAI,EAAE,CAAC;QACxC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,WAAW,CAAA;YACzC,MAAM,MAAM,GAAG,MAAM,IAAI,WAAW,CAAC,eAAe,CAAA;YACpD,IAAI,CAAC,MAAM;gBAAE,SAAS,GAAG,KAAK,CAAA;YAC9B,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAA;QAChE,CAAC;IACH,CAAC;IAED,cAAc;IACd,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;IAC9E,IAAI,WAAW,CAAC,cAAc,IAAI,IAAI,IAAI,WAAW,GAAG,WAAW,CAAC,cAAc,EAAE,CAAC;QACnF,SAAS,GAAG,KAAK,CAAA;IACnB,CAAC;IAED,qEAAqE;IACrE,IAAI,WAAW,CAAC,eAAe,IAAI,IAAI,IAAI,WAAW,CAAC,cAAc,IAAI,IAAI,EAAE,CAAC;QAC9E,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC,CAAA;QAC1F,CAAC;IACH,CAAC;IAED,sBAAsB;IACtB,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACvE,MAAM,KAAK,GAAa,EAAE,CAAA;IAE1B,IAAI,WAAW,CAAC,eAAe,IAAI,IAAI,EAAE,CAAC;QACxC,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAA;QACzD,KAAK,CAAC,IAAI,CAAC,GAAG,WAAW,IAAI,IAAI,CAAC,MAAM,eAAe,WAAW,CAAC,eAAe,aAAa,CAAC,CAAA;IAClG,CAAC;IACD,IAAI,WAAW,CAAC,cAAc,IAAI,IAAI,EAAE,CAAC;QACvC,MAAM,WAAW,GAAG,WAAW,IAAI,WAAW,CAAC,cAAc,CAAA;QAC7D,KAAK,CAAC,IAAI,CAAC,SAAS,WAAW,WAAW,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,IAAI,WAAW,CAAC,cAAc,SAAS,CAAC,CAAA;IACtH,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,aAAa,CAAC,CAAA;IAEnC,OAAO;QACL,aAAa,EAAE,WAAW,CAAC,EAAE;QAC7B,IAAI,EAAE,cAAc;QACpB,MAAM,EAAE,SAAS;QACjB,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;QACxB,MAAM,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;KAC/C,CAAA;AACH,CAAC"}

package/dist/evaluators/types.d.ts

@@ -0,0 +1,103 @@
+/**
+ * Evaluator types — shared interfaces for the test group evaluation system.
+ * Field names match the backend DB schema (camelCase versions of snake_case columns).
+ */
+export type ExpectationType = 'llm-judge' | 'token-budget' | 'latency-budget' | 'output-contains' | 'output-schema' | 'determinism';
+export interface Expectation {
+    id: number;
+    testGroupId: number;
+    type: ExpectationType;
+    judgePrompt?: string;
+    judgeModel?: string;
+    judgeProvider?: string;
+    maxTotalTokens?: number;
+    maxTokensPerRun?: number;
+    maxDurationMs?: number;
+    maxTotalDurationMs?: number;
+    containsText?: string;
+    notContainsText?: string;
+    caseInsensitive?: boolean;
+    jsonSchema?: Record<string, unknown>;
+    similarityThreshold?: number;
+}
+export interface TokenUsage {
+    inputTokens: number;
+    outputTokens: number;
+    totalTokens: number;
+}
+export interface SingleRunData {
+    runIndex: number;
+    passed: boolean;
+    durationMs: number;
+    tokenUsage: TokenUsage;
+    output: unknown;
+    trace?: unknown;
+    error?: string;
+    startedAt?: string;
+    completedAt?: string;
+}
+export interface PerRunResult {
+    runIndex: number;
+    passed: boolean;
+    value: unknown;
+}
+export interface ExpectationResult {
+    expectationId: number;
+    type: ExpectationType;
+    passed: boolean;
+    detail: string;
+    perRun?: PerRunResult[];
+}
+export interface EvaluationResult {
+    passed: boolean;
+    summary: string;
+    results: ExpectationResult[];
+}
+export interface TestGroup {
+    id: number;
+    name: string;
+    description?: string;
+    projectId: number;
+    workflowName: string;
+    frozenEvents: unknown[];
+    targetStepId: number;
+    targetStepType: 'ai' | 'tool';
+    mockInput?: unknown;
+    promptMocks?: Record<string, string>;
+    toolMocks?: Record<string, unknown>;
+    expectations: Expectation[];
+    runCount: number;
+    passThreshold: 'all' | number;
+    timeoutMs: number;
+    tags?: string[];
+    createdAt?: string;
+    updatedAt?: string;
+}
+export interface GitContext {
+    branch: string;
+    commit: string;
+    commitMessage?: string;
+    prNumber?: number;
+    prUrl?: string;
+}
+export interface TestGroupRunResult {
+    testGroupId: number;
+    triggeredBy: 'dashboard' | 'ci' | 'api' | 'schedule' | 'local';
+    passed: boolean;
+    summary: string;
+    singleRuns: SingleRunData[];
+    expectationResults: ExpectationResult[];
+    gitContext?: GitContext;
+    startedAt: string;
+    completedAt: string;
+}
+export interface BatchResult {
+    passed: boolean;
+    summary: string;
+    testGroupRunIds: number[];
+    results: TestGroupRunResult[];
+    gitContext?: GitContext;
+    startedAt: string;
+    completedAt: string;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/evaluators/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/evaluators/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,MAAM,eAAe,GACvB,WAAW,GACX,cAAc,GACd,gBAAgB,GAChB,iBAAiB,GACjB,eAAe,GACf,aAAa,CAAA;AAEjB,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAA;IACV,WAAW,EAAE,MAAM,CAAA;IACnB,IAAI,EAAE,eAAe,CAAA;IAGrB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,aAAa,CAAC,EAAE,MAAM,CAAA;IAGtB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,eAAe,CAAC,EAAE,MAAM,CAAA;IAGxB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAG3B,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,eAAe,CAAC,EAAE,OAAO,CAAA;IAGzB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAGpC,mBAAmB,CAAC,EAAE,MAAM,CAAA;CAC7B;AAED,MAAM,WAAW,UAAU;IACzB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,OAAO,CAAA;IACf,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,UAAU,CAAA;IACtB,MAAM,EAAE,OAAO,CAAA;IACf,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,OAAO,CAAA;IACf,KAAK,EAAE,OAAO,CAAA;CACf;AAED,MAAM,WAAW,iBAAiB;IAChC,aAAa,EAAE,MAAM,CAAA;IACrB,IAAI,EAAE,eAAe,CAAA;IACrB,MAAM,EAAE,OAAO,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,YAAY,EAAE,CAAA;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,OAAO,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,iBAAiB,EAAE,CAAA;CAC7B;AAED,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,SAAS,EAAE,MAAM,CAAA;IACjB,YAAY,EAAE,MAAM,CAAA;IAEpB,YAAY,EAAE,OAAO,EAAE,CAAA;IACvB,YAAY,EAAE,MAAM,CAAA;IACpB,cAAc,EAAE,IAAI,GAAG,MAAM,CAAA;IAE7B,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAEnC,YAAY,EAAE,WAAW,EAAE,CAAA;IAE3B,QAAQ,EAAE,MAAM,CAAA;IAChB,aAAa,EAAE,KAAK,GAAG,MAAM,CAAA;IAC7B,SAAS,EAAE,MAAM,CAAA;IACjB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IAEf,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,WAAW,GAAG,IAAI,GAAG,KAAK,GAAG,UAAU,GAAG,OAAO,CAAA;IAC9D,MAAM,EAAE,OAAO,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,UAAU,EAAE,aAAa,EAAE,CAAA;IAC3B,kBAAkB,EAAE,iBAAiB,EAAE,CAAA;IACvC,UAAU,CAAC,EAAE,UAAU,CAAA;IACvB,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,OAAO,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,eAAe,EAAE,MAAM,EAAE,CAAA;IACzB,OAAO,EAAE,kBAAkB,EAAE,CAAA;IAC7B,UAAU,CAAC,EAAE,UAAU,CAAA;IACvB,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;CACpB"}

package/dist/evaluators/types.js

@@ -0,0 +1,6 @@
+/**
+ * Evaluator types — shared interfaces for the test group evaluation system.
+ * Field names match the backend DB schema (camelCase versions of snake_case columns).
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/evaluators/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/evaluators/types.ts"],"names":[],"mappings":"AAAA;;;GAGG"}