@goreal-ai/echo-pdk 0.6.0 → 0.8.0

package/dist/eval/assertions.js

@@ -0,0 +1,349 @@
+/**
+ * @fileoverview Assertion implementations for eval tests
+ *
+ * Each assertion takes a text output and returns a pass/fail result.
+ * Assertions are organized by category:
+ * - Text assertions (contains, equals, matches, etc.)
+ * - Structural assertions (json_valid, json_schema)
+ * - AI/Semantic assertions (llm_judge, similar_to, sentiment)
+ * - Performance assertions (latency, token_count, cost)
+ */
+/**
+ * Run a single assertion against the given context.
+ */
+export async function runAssertion(assertion, ctx) {
+    // Extract the operator (first key of the assertion object)
+    const entries = Object.entries(assertion);
+    if (entries.length === 0) {
+        return { operator: 'unknown', status: 'error', message: 'Empty assertion' };
+    }
+    const [operator, value] = entries[0];
+    const handler = assertionHandlers[operator];
+    if (!handler) {
+        return {
+            operator,
+            status: 'error',
+            message: `Unknown assertion operator: ${operator}`,
+        };
+    }
+    try {
+        return await handler(value, ctx);
+    }
+    catch (err) {
+        return {
+            operator,
+            status: 'error',
+            message: `Assertion error: ${err.message}`,
+        };
+    }
+}
+/**
+ * Run multiple assertions against the given context.
+ */
+export async function runAssertions(assertions, ctx) {
+    const results = [];
+    for (const assertion of assertions) {
+        results.push(await runAssertion(assertion, ctx));
+    }
+    return results;
+}
+const assertionHandlers = {
+    // Text assertions
+    contains: async (value, ctx) => {
+        const expected = String(value);
+        const pass = ctx.text.includes(expected);
+        return {
+            operator: 'contains',
+            status: pass ? 'pass' : 'fail',
+            expected,
+            actual: pass ? undefined : truncate(ctx.text, 100),
+            message: pass ? undefined : `Expected output to contain "${expected}"`,
+        };
+    },
+    not_contains: async (value, ctx) => {
+        const expected = String(value);
+        const pass = !ctx.text.includes(expected);
+        return {
+            operator: 'not_contains',
+            status: pass ? 'pass' : 'fail',
+            expected: `not "${expected}"`,
+            message: pass ? undefined : `Expected output NOT to contain "${expected}"`,
+        };
+    },
+    equals: async (value, ctx) => {
+        const expected = String(value);
+        const pass = ctx.text.trim() === expected.trim();
+        return {
+            operator: 'equals',
+            status: pass ? 'pass' : 'fail',
+            expected,
+            actual: pass ? undefined : truncate(ctx.text, 100),
+            message: pass ? undefined : 'Output does not match expected text',
+        };
+    },
+    matches: async (value, ctx) => {
+        const pattern = String(value);
+        const regex = new RegExp(pattern);
+        const pass = regex.test(ctx.text);
+        return {
+            operator: 'matches',
+            status: pass ? 'pass' : 'fail',
+            expected: pattern,
+            message: pass ? undefined : `Output does not match pattern: ${pattern}`,
+        };
+    },
+    starts_with: async (value, ctx) => {
+        const expected = String(value);
+        const pass = ctx.text.trimStart().startsWith(expected);
+        return {
+            operator: 'starts_with',
+            status: pass ? 'pass' : 'fail',
+            expected,
+            message: pass ? undefined : `Expected output to start with "${expected}"`,
+        };
+    },
+    ends_with: async (value, ctx) => {
+        const expected = String(value);
+        const pass = ctx.text.trimEnd().endsWith(expected);
+        return {
+            operator: 'ends_with',
+            status: pass ? 'pass' : 'fail',
+            expected,
+            message: pass ? undefined : `Expected output to end with "${expected}"`,
+        };
+    },
+    length: async (value, ctx) => {
+        const opts = value;
+        const len = ctx.text.length;
+        let pass = true;
+        const messages = [];
+        if (opts.min != null && len < opts.min) {
+            pass = false;
+            messages.push(`length ${len} < min ${opts.min}`);
+        }
+        if (opts.max != null && len > opts.max) {
+            pass = false;
+            messages.push(`length ${len} > max ${opts.max}`);
+        }
+        return {
+            operator: 'length',
+            status: pass ? 'pass' : 'fail',
+            expected: `${opts.min ?? '0'}..${opts.max ?? '∞'}`,
+            actual: String(len),
+            message: pass ? undefined : messages.join(', '),
+        };
+    },
+    word_count: async (value, ctx) => {
+        const opts = value;
+        const words = ctx.text.trim().split(/\s+/).filter(Boolean).length;
+        let pass = true;
+        const messages = [];
+        if (opts.min != null && words < opts.min) {
+            pass = false;
+            messages.push(`word count ${words} < min ${opts.min}`);
+        }
+        if (opts.max != null && words > opts.max) {
+            pass = false;
+            messages.push(`word count ${words} > max ${opts.max}`);
+        }
+        return {
+            operator: 'word_count',
+            status: pass ? 'pass' : 'fail',
+            expected: `${opts.min ?? '0'}..${opts.max ?? '∞'} words`,
+            actual: `${words} words`,
+            message: pass ? undefined : messages.join(', '),
+        };
+    },
+    // Structural assertions
+    json_valid: async (_value, ctx) => {
+        try {
+            JSON.parse(ctx.text);
+            return { operator: 'json_valid', status: 'pass' };
+        }
+        catch {
+            return {
+                operator: 'json_valid',
+                status: 'fail',
+                message: 'Output is not valid JSON',
+            };
+        }
+    },
+    json_schema: async (_value, ctx) => {
+        // Schema validation is a stretch goal — for now just validate JSON
+        try {
+            JSON.parse(ctx.text);
+            return {
+                operator: 'json_schema',
+                status: 'pass',
+                message: 'JSON is valid (schema validation not yet implemented)',
+            };
+        }
+        catch {
+            return {
+                operator: 'json_schema',
+                status: 'fail',
+                message: 'Output is not valid JSON',
+            };
+        }
+    },
+    // AI/Semantic assertions (require LLM provider)
+    llm_judge: async (value, ctx) => {
+        const question = String(value);
+        if (!ctx.llmProvider) {
+            return {
+                operator: 'llm_judge',
+                status: 'error',
+                message: 'LLM provider not configured — cannot run llm_judge assertion',
+            };
+        }
+        const { pass, reasoning } = await ctx.llmProvider.judge(ctx.text, question);
+        return {
+            operator: 'llm_judge',
+            status: pass ? 'pass' : 'fail',
+            expected: question,
+            actual: reasoning,
+            message: pass
+                ? `LLM judge: ${reasoning}`
+                : `LLM judge answered "no": ${reasoning}`,
+        };
+    },
+    similar_to: async (value, ctx) => {
+        const opts = value;
+        if (!ctx.embeddingSimilarity && !ctx.llmProvider) {
+            return {
+                operator: 'similar_to',
+                status: 'error',
+                message: 'No embedding or LLM provider configured — cannot run similar_to assertion',
+            };
+        }
+        if (!ctx.loadGolden) {
+            return {
+                operator: 'similar_to',
+                status: 'error',
+                message: 'Dataset loader not configured — cannot run similar_to assertion',
+            };
+        }
+        const golden = await ctx.loadGolden(opts.dataset);
+        if (!golden) {
+            return {
+                operator: 'similar_to',
+                status: 'error',
+                message: `No golden response found in dataset "${opts.dataset}"`,
+            };
+        }
+        // Prefer embeddings over LLM — deterministic, cheaper, faster
+        let score;
+        let method;
+        if (ctx.embeddingSimilarity) {
+            score = await ctx.embeddingSimilarity(ctx.text, golden);
+            method = 'embeddings';
+        }
+        else {
+            score = await ctx.llmProvider.similarity(ctx.text, golden);
+            method = 'llm';
+        }
+        const pass = score >= opts.threshold;
+        return {
+            operator: 'similar_to',
+            status: pass ? 'pass' : 'fail',
+            expected: `similarity >= ${opts.threshold}`,
+            actual: `similarity = ${score.toFixed(3)} (${method})`,
+            message: pass ? undefined : `Similarity ${score.toFixed(3)} below threshold ${opts.threshold}`,
+        };
+    },
+    sentiment: async (value, ctx) => {
+        const expected = String(value);
+        if (!ctx.llmProvider) {
+            return {
+                operator: 'sentiment',
+                status: 'error',
+                message: 'LLM provider not configured — cannot run sentiment assertion',
+            };
+        }
+        const question = `Does this text have a ${expected} sentiment or tone?`;
+        const { pass, reasoning } = await ctx.llmProvider.judge(ctx.text, question);
+        return {
+            operator: 'sentiment',
+            status: pass ? 'pass' : 'fail',
+            expected,
+            actual: reasoning,
+            message: pass ? undefined : `Sentiment does not match expected: ${expected}. ${reasoning}`,
+        };
+    },
+    // Performance assertions (use LLM response metadata)
+    latency: async (value, ctx) => {
+        const opts = value;
+        const actual = ctx.llmResponse?.latencyMs;
+        if (actual == null) {
+            return {
+                operator: 'latency',
+                status: 'error',
+                message: 'No latency data available',
+            };
+        }
+        const pass = actual <= opts.max;
+        return {
+            operator: 'latency',
+            status: pass ? 'pass' : 'fail',
+            expected: `<= ${opts.max}ms`,
+            actual: `${actual}ms`,
+            message: pass ? undefined : `Latency ${actual}ms exceeds max ${opts.max}ms`,
+        };
+    },
+    token_count: async (value, ctx) => {
+        const opts = value;
+        const tokens = ctx.llmResponse?.tokens?.completion;
+        if (tokens == null) {
+            return {
+                operator: 'token_count',
+                status: 'error',
+                message: 'No token count data available',
+            };
+        }
+        let pass = true;
+        const messages = [];
+        if (opts.max != null && tokens > opts.max) {
+            pass = false;
+            messages.push(`tokens ${tokens} > max ${opts.max}`);
+        }
+        if (opts.min != null && tokens < opts.min) {
+            pass = false;
+            messages.push(`tokens ${tokens} < min ${opts.min}`);
+        }
+        return {
+            operator: 'token_count',
+            status: pass ? 'pass' : 'fail',
+            expected: `${opts.min ?? '0'}..${opts.max ?? '∞'} tokens`,
+            actual: `${tokens} tokens`,
+            message: pass ? undefined : messages.join(', '),
+        };
+    },
+    cost: async (value, ctx) => {
+        const opts = value;
+        const actual = ctx.llmResponse?.costUsd;
+        if (actual == null) {
+            return {
+                operator: 'cost',
+                status: 'error',
+                message: 'No cost data available',
+            };
+        }
+        const pass = actual <= opts.max;
+        return {
+            operator: 'cost',
+            status: pass ? 'pass' : 'fail',
+            expected: `<= $${opts.max}`,
+            actual: `$${actual.toFixed(4)}`,
+            message: pass ? undefined : `Cost $${actual.toFixed(4)} exceeds max $${opts.max}`,
+        };
+    },
+};
+// =============================================================================
+// HELPERS
+// =============================================================================
+function truncate(text, maxLen) {
+    if (text.length <= maxLen)
+        return text;
+    return text.slice(0, maxLen) + '...';
+}
+//# sourceMappingURL=assertions.js.map

package/dist/eval/assertions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"assertions.js","sourceRoot":"","sources":["../../src/eval/assertions.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAwBH;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,SAAoB,EACpB,GAAqB;IAErB,2DAA2D;IAC3D,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC1C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC9E,CAAC;IAED,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,CAAsB,CAAC;IAC1D,MAAM,OAAO,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAE5C,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO;YACL,QAAQ;YACR,MAAM,EAAE,OAAO;YACf,OAAO,EAAE,+BAA+B,QAAQ,EAAE;SACnD,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACnC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,QAAQ;YACR,MAAM,EAAE,OAAO;YACf,OAAO,EAAE,oBAAqB,GAAa,CAAC,OAAO,EAAE;SACtD,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAAuB,EACvB,GAAqB;IAErB,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,OAAO,CAAC,IAAI,CAAC,MAAM,YAAY,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAWD,MAAM,iBAAiB,GAAqC;IAC1D,kBAAkB;IAClB,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QAC7B,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACzC,OAAO;YACL,QAAQ,EAAE,UAAU;YACpB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC9B,QAAQ;YACR,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC;YAClD,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,+BAA+B,QAAQ,GAAG;SACvE,CAAC;IACJ,CAAC;IAED,YAAY,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACjC,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC1C,OAAO;YACL,QAAQ,EAAE,cAAc;YACxB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC9B,QAAQ,EAAE,QAAQ,QAAQ,GAAG;YAC7B,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,mCAAmC,QAAQ,GAAG;SAC3E,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,QAAQ,CAAC,IAAI,EAAE,CAAC;QACjD,OAAO;YACL,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC9B,QAAQ;YACR,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC;YAClD,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,qCAAqC;SAClE,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QAC5B,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,OAAO;YACL,QAAQ,EAAE,SAAS;YACnB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC9B,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,kCAAkC,OAAO,EAAE;SACxE,CAAC;IACJ,CAAC;IAED,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QAChC,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QACvD,OAAO;YACL,QAAQ,EAAE,aAAa;YACvB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC9B,QAAQ;YACR,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,kCAAkC,QAAQ,GAAG;SAC1E,CAAC;IACJ,CAAC;IAED,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QAC9B,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACnD,OAAO;YACL,QAAQ,EAAE,WAAW;YACrB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC9B,QAAQ;YACR,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,gCAAgC,QAAQ,GAAG;SACxE,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QAC3B,MAAM,IAAI,GAAG,KAAuC,CAAC;QACrD,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC;QAC5B,IAAI,IAAI,GAAG,IAAI,CAAC;QAChB,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvC,IAAI,GAAG,KAAK,CAAC;YACb,QAAQ,CAAC,IAAI,CAAC,UAAU,GAAG,UAAU,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACnD,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvC,IAAI,GAAG,KAAK,CAAC;YACb,QAAQ,CAAC,IAAI,CAAC,UAAU,GAAG,UAAU,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACnD,CAAC;QAED,OAAO;YACL,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC9B,QAAQ,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI,GAAG,KAAK,IAAI,CAAC,GAAG,IAAI,GAAG,EAAE;YAClD,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC;YACnB,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;SAChD,CAAC;IACJ,CAAC;IAED,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QAC/B,MAAM,IAAI,GAAG,KAAuC,CAAC;QACrD,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;QAClE,IAAI,IAAI,GAAG,IAAI,CAAC;QAChB,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,IAAI,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzC,IAAI,GAAG,KAAK,CAAC;YACb,QAAQ,CAAC,IAAI,CAAC,cAAc,KAAK,UAAU,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,IAAI,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzC,IAAI,GAAG,KAAK,CAAC;YACb,QAAQ,CAAC,IAAI,CAAC,cAAc,KAAK,UAAU,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,OAAO;YACL,QAAQ,EAAE,YAAY;YACtB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC9B,QAAQ,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI,GAAG,KAAK,IAAI,CAAC,GAAG,IAAI,GAAG,QAAQ;YACxD,MAAM,EAAE,GAAG,KAAK,QAAQ;YACxB,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;SAChD,CAAC;IACJ,CAAC;IAED,wBAAwB;IACxB,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE;QAChC,IAAI,CAAC;YACH,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACrB,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;QACpD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,QAAQ,EAAE,YAAY;gBACtB,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,0BAA0B;aACpC,CAAC;QACJ,CAAC;IACH,CAAC;IAED,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE;QACjC,mEAAmE;QACnE,IAAI,CAAC;YACH,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACrB,OAAO;gBACL,QAAQ,EAAE,aAAa;gBACvB,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,uDAAuD;aACjE,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,QAAQ,EAAE,aAAa;gBACvB,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,0BAA0B;aACpC,CAAC;QACJ,CAAC;IACH,CAAC;IAED,gDAAgD;IAChD,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QAC9B,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YACrB,OAAO;gBACL,QAAQ,EAAE,WAAW;gBACrB,MAAM,EAAE,OAAO;gBACf,OAAO,EAAE,8DAA8D;aACxE,CAAC;QACJ,CAAC;QAED,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,MAAM,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC5E,OAAO;YACL,QAAQ,EAAE,WAAW;YACrB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC9B,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,IAAI;gBACX,CAAC,CAAC,cAAc,SAAS,EAAE;gBAC3B,CAAC,CAAC,4BAA4B,SAAS,EAAE;SAC5C,CAAC;IACJ,CAAC;IAED,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QAC/B,MAAM,IAAI,GAAG,KAA+C,CAAC;QAC7D,IAAI,CAAC,GAAG,CAAC,mBAAmB,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YACjD,OAAO;gBACL,QAAQ,EAAE,YAAY;gBACtB,MAAM,EAAE,OAAO;gBACf,OAAO,EAAE,2EAA2E;aACrF,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO;gBACL,QAAQ,EAAE,YAAY;gBACtB,MAAM,EAAE,OAAO;gBACf,OAAO,EAAE,iEAAiE;aAC3E,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO;gBACL,QAAQ,EAAE,YAAY;gBACtB,MAAM,EAAE,OAAO;gBACf,OAAO,EAAE,wCAAwC,IAAI,CAAC,OAAO,GAAG;aACjE,CAAC;QACJ,CAAC;QAED,8DAA8D;QAC9D,IAAI,KAAa,CAAC;QAClB,IAAI,MAAc,CAAC;QACnB,IAAI,GAAG,CAAC,mBAAmB,EAAE,CAAC;YAC5B,KAAK,GAAG,MAAM,GAAG,CAAC,mBAAmB,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACxD,MAAM,GAAG,YAAY,CAAC;QACxB,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,MAAM,GAAG,CAAC,WAAY,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC5D,MAAM,GAAG,KAAK,CAAC;QACjB,CAAC;QAED,MAAM,IAAI,GAAG,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC;QACrC,OAAO;YACL,QAAQ,EAAE,YAAY;YACtB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC9B,QAAQ,EAAE,iBAAiB,IAAI,CAAC,SAAS,EAAE;YAC3C,MAAM,EAAE,gBAAgB,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,MAAM,GAAG;YACtD,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,oBAAoB,IAAI,CAAC,SAAS,EAAE;SAC/F,CAAC;IACJ,CAAC;IAED,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QAC9B,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YACrB,OAAO;gBACL,QAAQ,EAAE,WAAW;gBACrB,MAAM,EAAE,OAAO;gBACf,OAAO,EAAE,8DAA8D;aACxE,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG,yBAAyB,QAAQ,qBAAqB,CAAC;QACxE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,MAAM,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC5E,OAAO;YACL,QAAQ,EAAE,WAAW;YACrB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC9B,QAAQ;YACR,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,sCAAsC,QAAQ,KAAK,SAAS,EAAE;SAC3F,CAAC;IACJ,CAAC;IAED,qDAAqD;IACrD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QAC5B,MAAM,IAAI,GAAG,KAAwB,CAAC;QACtC,MAAM,MAAM,GAAG,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC;QAC1C,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;YACnB,OAAO;gBACL,QAAQ,EAAE,SAAS;gBACnB,MAAM,EAAE,OAAO;gBACf,OAAO,EAAE,2BAA2B;aACrC,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC;QAChC,OAAO;YACL,QAAQ,EAAE,SAAS;YACnB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC9B,QAAQ,EAAE,MAAM,IAAI,CAAC,GAAG,IAAI;YAC5B,MAAM,EAAE,GAAG,MAAM,IAAI;YACrB,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,MAAM,kBAAkB,IAAI,CAAC,GAAG,IAAI;SAC5E,CAAC;IACJ,CAAC;IAED,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QAChC,MAAM,IAAI,GAAG,KAAuC,CAAC;QACrD,MAAM,MAAM,GAAG,GAAG,CAAC,WAAW,EAAE,MAAM,EAAE,UAAU,CAAC;QACnD,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;YACnB,OAAO;gBACL,QAAQ,EAAE,aAAa;gBACvB,MAAM,EAAE,OAAO;gBACf,OAAO,EAAE,+BAA+B;aACzC,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,GAAG,IAAI,CAAC;QAChB,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,IAAI,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC1C,IAAI,GAAG,KAAK,CAAC;YACb,QAAQ,CAAC,IAAI,CAAC,UAAU,MAAM,UAAU,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACtD,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,IAAI,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC1C,IAAI,GAAG,KAAK,CAAC;YACb,QAAQ,CAAC,IAAI,CAAC,UAAU,MAAM,UAAU,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACtD,CAAC;QAED,OAAO;YACL,QAAQ,EAAE,aAAa;YACvB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC9B,QAAQ,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI,GAAG,KAAK,IAAI,CAAC,GAAG,IAAI,GAAG,SAAS;YACzD,MAAM,EAAE,GAAG,MAAM,SAAS;YAC1B,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;SAChD,CAAC;IACJ,CAAC;IAED,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACzB,MAAM,IAAI,GAAG,KAAwB,CAAC;QACtC,MAAM,MAAM,GAAG,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC;QACxC,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;YACnB,OAAO;gBACL,QAAQ,EAAE,MAAM;gBAChB,MAAM,EAAE,OAAO;gBACf,OAAO,EAAE,wBAAwB;aAClC,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC;QAChC,OAAO;YACL,QAAQ,EAAE,MAAM;YAChB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC9B,QAAQ,EAAE,OAAO,IAAI,CAAC,GAAG,EAAE;YAC3B,MAAM,EAAE,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;YAC/B,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,iBAAiB,IAAI,CAAC,GAAG,EAAE;SAClF,CAAC;IACJ,CAAC;CACF,CAAC;AAEF,gFAAgF;AAChF,UAAU;AACV,gFAAgF;AAEhF,SAAS,QAAQ,CAAC,IAAY,EAAE,MAAc;IAC5C,IAAI,IAAI,CAAC,MAAM,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IACvC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,KAAK,CAAC;AACvC,CAAC"}

package/dist/eval/dataset.d.ts

@@ -0,0 +1,42 @@
+/**
+ * @fileoverview Dataset operations for eval system
+ *
+ * Handles loading datasets from .dset files, resolving parameter sets,
+ * and recording golden responses.
+ */
+import type { EvalDataset, LLMResponse } from './types.js';
+/**
+ * Manages datasets within a prompt's eval/datasets/ directory.
+ */
+export declare class DatasetManager {
+    /** Path to the prompt directory (contains eval/datasets/) */
+    private promptDir;
+    private cache;
+    constructor(
+    /** Path to the prompt directory (contains eval/datasets/) */
+    promptDir: string);
+    /**
+     * Load a dataset by name.
+     * Looks for eval/datasets/{name}.dset
+     */
+    load(name: string): Promise<EvalDataset>;
+    /**
+     * Get a specific parameter set from a dataset.
+     */
+    getParams(datasetName: string, paramsName: string): Promise<Record<string, unknown>>;
+    /**
+     * Get the golden response from a dataset.
+     */
+    getGolden(datasetName: string): Promise<string | undefined>;
+    /**
+     * Record a golden response into a dataset.
+     * Creates the dataset if it doesn't exist.
+     */
+    recordGolden(datasetName: string, response: string, llmResponse: LLMResponse): Promise<void>;
+    /**
+     * Save a dataset to disk.
+     */
+    save(name: string, dataset: EvalDataset): Promise<void>;
+    private getDatasetPath;
+}
+//# sourceMappingURL=dataset.d.ts.map

package/dist/eval/dataset.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/eval/dataset.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAM3D;;GAEG;AACH,qBAAa,cAAc;IAIvB,6DAA6D;IAC7D,OAAO,CAAC,SAAS;IAJnB,OAAO,CAAC,KAAK,CAAkC;;IAG7C,6DAA6D;IACrD,SAAS,EAAE,MAAM;IAG3B;;;OAGG;IACG,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAU9C;;OAEG;IACG,SAAS,CACb,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAgBnC;;OAEG;IACG,SAAS,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAKjE;;;OAGG;IACG,YAAY,CAChB,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,WAAW,GACvB,OAAO,CAAC,IAAI,CAAC;IA6BhB;;OAEG;IACG,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAM7D,OAAO,CAAC,cAAc;CAGvB"}

package/dist/eval/dataset.js

@@ -0,0 +1,101 @@
+/**
+ * @fileoverview Dataset operations for eval system
+ *
+ * Handles loading datasets from .dset files, resolving parameter sets,
+ * and recording golden responses.
+ */
+import { writeFile } from 'fs/promises';
+import { join, resolve } from 'path';
+import { stringify as stringifyYaml } from 'yaml';
+import { loadDatasetFile } from './loader.js';
+// =============================================================================
+// DATASET MANAGER
+// =============================================================================
+/**
+ * Manages datasets within a prompt's eval/datasets/ directory.
+ */
+export class DatasetManager {
+    promptDir;
+    cache = new Map();
+    constructor(
+    /** Path to the prompt directory (contains eval/datasets/) */
+    promptDir) {
+        this.promptDir = promptDir;
+    }
+    /**
+     * Load a dataset by name.
+     * Looks for eval/datasets/{name}.dset
+     */
+    async load(name) {
+        const cached = this.cache.get(name);
+        if (cached)
+            return cached;
+        const filePath = this.getDatasetPath(name);
+        const dataset = await loadDatasetFile(filePath);
+        this.cache.set(name, dataset);
+        return dataset;
+    }
+    /**
+     * Get a specific parameter set from a dataset.
+     */
+    async getParams(datasetName, paramsName) {
+        const dataset = await this.load(datasetName);
+        const paramSet = dataset.parameters.find((p) => p.name === paramsName);
+        if (!paramSet) {
+            throw new Error(`Parameter set "${paramsName}" not found in dataset "${datasetName}". ` +
+                `Available: ${dataset.parameters.map((p) => p.name).join(', ')}`);
+        }
+        // Return all fields except 'name' as the variable context
+        const { name: _name, ...vars } = paramSet;
+        return vars;
+    }
+    /**
+     * Get the golden response from a dataset.
+     */
+    async getGolden(datasetName) {
+        const dataset = await this.load(datasetName);
+        return dataset.golden?.response;
+    }
+    /**
+     * Record a golden response into a dataset.
+     * Creates the dataset if it doesn't exist.
+     */
+    async recordGolden(datasetName, response, llmResponse) {
+        let dataset;
+        try {
+            dataset = await this.load(datasetName);
+        }
+        catch (_err) {
+            // Dataset doesn't exist — create a minimal one
+            dataset = {
+                name: datasetName,
+                parameters: [],
+            };
+        }
+        // Update golden
+        dataset.golden = {
+            response,
+            model: llmResponse.model,
+            recorded_at: new Date().toISOString(),
+            metadata: {
+                tokens: llmResponse.tokens?.total,
+                latency_ms: llmResponse.latencyMs,
+            },
+        };
+        // Write back
+        await this.save(datasetName, dataset);
+        this.cache.set(datasetName, dataset);
+    }
+    /**
+     * Save a dataset to disk.
+     */
+    async save(name, dataset) {
+        const filePath = this.getDatasetPath(name);
+        const content = stringifyYaml(dataset, { lineWidth: 120 });
+        await writeFile(filePath, content, 'utf-8');
+    }
+    getDatasetPath(name) {
+        return resolve(join(this.promptDir, 'eval', 'datasets', `${name}.dset`));
+    }
+}
+//# sourceMappingURL=dataset.js.map

package/dist/eval/dataset.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/eval/dataset.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,SAAS,IAAI,aAAa,EAAE,MAAM,MAAM,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAG9C,gFAAgF;AAChF,kBAAkB;AAClB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,OAAO,cAAc;IAKf;IAJF,KAAK,GAAG,IAAI,GAAG,EAAuB,CAAC;IAE/C;IACE,6DAA6D;IACrD,SAAiB;QAAjB,cAAS,GAAT,SAAS,CAAQ;IACxB,CAAC;IAEJ;;;OAGG;IACH,KAAK,CAAC,IAAI,CAAC,IAAY;QACrB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;QAE1B,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAC;QAChD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC9B,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,CACb,WAAmB,EACnB,UAAkB;QAElB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC7C,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;QAEvE,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CACb,kBAAkB,UAAU,2BAA2B,WAAW,KAAK;gBACrE,cAAc,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACnE,CAAC;QACJ,CAAC;QAED,0DAA0D;QAC1D,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAC;QAC1C,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,CAAC,WAAmB;QACjC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC7C,OAAO,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC;IAClC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAChB,WAAmB,EACnB,QAAgB,EAChB,WAAwB;QAExB,IAAI,OAAoB,CAAC;QAEzB,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,IAAI,EAAE,CAAC;YACd,+CAA+C;YAC/C,OAAO,GAAG;gBACR,IAAI,EAAE,WAAW;gBACjB,UAAU,EAAE,EAAE;aACf,CAAC;QACJ,CAAC;QAED,gBAAgB;QAChB,OAAO,CAAC,MAAM,GAAG;YACf,QAAQ;YACR,KAAK,EAAE,WAAW,CAAC,KAAK;YACxB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,QAAQ,EAAE;gBACR,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,KAAK;gBACjC,UAAU,EAAE,WAAW,CAAC,SAAS;aAClC;SACF,CAAC;QAEF,aAAa;QACb,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI,CAAC,IAAY,EAAE,OAAoB;QAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,OAAO,GAAG,aAAa,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3D,MAAM,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC9C,CAAC;IAEO,cAAc,CAAC,IAAY;QACjC,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,IAAI,OAAO,CAAC,CAAC,CAAC;IAC3E,CAAC;CACF"}

package/dist/eval/index.d.ts

@@ -0,0 +1,14 @@
+/**
+ * @fileoverview Echo PDK Eval Module - Public API
+ *
+ * Exports the eval runner, loader, assertions, dataset manager,
+ * and reporter for prompt evaluation and testing.
+ */
+export type { EvalSuite, EvalSuiteConfig, EvalTest, Assertion, AssertionOperator, EvalDataset, EvalGolden, EvalParameterSet, EvalSuiteResult, EvalTestResult, AssertionResult, EvalSummary, EvalStatus, EvalRunnerConfig, LLMProvider, LLMResponse, } from './types.js';
+export { loadEvalFile, parseEvalContent, loadDatasetFile, parseDatasetContent, EvalLoadError, } from './loader.js';
+export { runAssertion, runAssertions } from './assertions.js';
+export type { AssertionContext } from './assertions.js';
+export { DatasetManager } from './dataset.js';
+export { runEvalFile, runEvalSuite } from './runner.js';
+export { formatConsole, formatJson, formatJunit, formatResults, } from './reporter.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/eval/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,YAAY,EACV,SAAS,EACT,eAAe,EACf,QAAQ,EACR,SAAS,EACT,iBAAiB,EACjB,WAAW,EACX,UAAU,EACV,gBAAgB,EAChB,eAAe,EACf,cAAc,EACd,eAAe,EACf,WAAW,EACX,UAAU,EACV,gBAAgB,EAChB,WAAW,EACX,WAAW,GACZ,MAAM,YAAY,CAAC;AAGpB,OAAO,EACL,YAAY,EACZ,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EACnB,aAAa,GACd,MAAM,aAAa,CAAC;AAGrB,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAC9D,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAGxD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAG9C,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAGxD,OAAO,EACL,aAAa,EACb,UAAU,EACV,WAAW,EACX,aAAa,GACd,MAAM,eAAe,CAAC"}

package/dist/eval/index.js

@@ -0,0 +1,17 @@
+/**
+ * @fileoverview Echo PDK Eval Module - Public API
+ *
+ * Exports the eval runner, loader, assertions, dataset manager,
+ * and reporter for prompt evaluation and testing.
+ */
+// Loader
+export { loadEvalFile, parseEvalContent, loadDatasetFile, parseDatasetContent, EvalLoadError, } from './loader.js';
+// Assertions
+export { runAssertion, runAssertions } from './assertions.js';
+// Dataset
+export { DatasetManager } from './dataset.js';
+// Runner
+export { runEvalFile, runEvalSuite } from './runner.js';
+// Reporter
+export { formatConsole, formatJson, formatJunit, formatResults, } from './reporter.js';
+//# sourceMappingURL=index.js.map

package/dist/eval/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAsBH,SAAS;AACT,OAAO,EACL,YAAY,EACZ,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EACnB,aAAa,GACd,MAAM,aAAa,CAAC;AAErB,aAAa;AACb,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAG9D,UAAU;AACV,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,SAAS;AACT,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAExD,WAAW;AACX,OAAO,EACL,aAAa,EACb,UAAU,EACV,WAAW,EACX,aAAa,GACd,MAAM,eAAe,CAAC"}

package/dist/eval/loader.d.ts

@@ -0,0 +1,30 @@
+/**
+ * @fileoverview Loader for .eval and .dset YAML files
+ *
+ * Parses and validates eval suite definitions and dataset files.
+ */
+import type { EvalSuite, EvalDataset } from './types.js';
+/**
+ * Load and validate an .eval file from disk.
+ */
+export declare function loadEvalFile(filePath: string): Promise<EvalSuite>;
+/**
+ * Parse .eval YAML content into an EvalSuite.
+ */
+export declare function parseEvalContent(content: string, source?: string): EvalSuite;
+/**
+ * Load and validate a .dset file from disk.
+ */
+export declare function loadDatasetFile(filePath: string): Promise<EvalDataset>;
+/**
+ * Parse .dset YAML content into an EvalDataset.
+ */
+export declare function parseDatasetContent(content: string, source?: string): EvalDataset;
+/**
+ * Error thrown when loading/parsing eval or dataset files.
+ */
+export declare class EvalLoadError extends Error {
+    source?: string | undefined;
+    constructor(message: string, source?: string | undefined);
+}
+//# sourceMappingURL=loader.d.ts.map

package/dist/eval/loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../src/eval/loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,KAAK,EACV,SAAS,EAIT,WAAW,EAGZ,MAAM,YAAY,CAAC;AAMpB;;GAEG;AACH,wBAAsB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAGvE;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CA6B5E;AA6GD;;GAEG;AACH,wBAAsB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAG5E;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,WAAW,CA6CjF;AAMD;;GAEG;AACH,qBAAa,aAAc,SAAQ,KAAK;IAG7B,MAAM,CAAC,EAAE,MAAM;gBADtB,OAAO,EAAE,MAAM,EACR,MAAM,CAAC,EAAE,MAAM,YAAA;CAKzB"}