@rudderjs/ai 1.5.0 → 1.6.0

package/dist/eval/index.js

@@ -0,0 +1,510 @@
+/**
+ * `@rudderjs/ai/eval` — built-in eval framework for #A5 Phase 1.
+ *
+ * Define a suite of input cases + assertions, run them against any
+ * `Agent`, get a console report with pass/fail + cost + tokens. Same
+ * `Agent` instances as your app code — one source of truth.
+ *
+ * @example
+ * ```ts
+ * // evals/support-agent.eval.ts
+ * import { evalSuite, llmJudge, exactMatch, regex } from '@rudderjs/ai/eval'
+ * import { SupportAgent } from '../app/Agents/SupportAgent.js'
+ *
+ * export default evalSuite('SupportAgent', {
+ *   agent: () => new SupportAgent(),
+ *   cases: [
+ *     { name: 'password reset', input: 'How do I reset my password?',
+ *       assert: llmJudge('mentions a password reset link') },
+ *     { name: 'price', input: 'How much?', assert: exactMatch('$99/month') },
+ *     { name: 'support email', input: 'Contact?', assert: regex(/support@/) },
+ *   ],
+ * })
+ * ```
+ *
+ * Run programmatically via `runSuite(suite)` from this entry, or via
+ * `pnpm rudder ai:eval` once Phase 2 lands.
+ *
+ * Built-in metrics: `exactMatch`, `regex`, `llmJudge`, `jsonShape`,
+ * `semanticMatch`, `tokenCost`. Compose multiple via `compose(...)`.
+ * User-defined metrics work today — any `(response, ctx) =>
+ * MetricResult` qualifies.
+ */
+import { agent } from '../agent.js';
+import { Output } from '../output.js';
+import { AI } from '../facade.js';
+import { aiObservers } from '../observers.js';
+import { estimateCost } from '../budget/pricing.js';
+import { z } from 'zod';
+export { reportJson } from './json-reporter.js';
+export { stepsFromResponse } from './fixtures.js';
+export { reportHtml } from './html-reporter.js';
+// ─── Suite definition ─────────────────────────────────────
+/**
+ * Define an eval suite. Returns a frozen `EvalSuite` ready to pass
+ * into {@link runSuite} or to default-export from an `evals/*.eval.ts`
+ * file (Phase 2's CLI auto-discovers those).
+ *
+ * The shape is deliberately a function rather than a class — keeps the
+ * file's default export trivially serializable (Phase 2 needs to load
+ * suites via dynamic import) and avoids the "did you forget `new`?"
+ * footgun.
+ */
+export function evalSuite(name, spec) {
+    if (!name)
+        throw new Error('[RudderJS AI] evalSuite() requires a name.');
+    if (!spec || typeof spec.agent !== 'function') {
+        throw new Error('[RudderJS AI] evalSuite() requires { agent: () => Agent, cases: [...] }.');
+    }
+    if (!Array.isArray(spec.cases) || spec.cases.length === 0) {
+        throw new Error('[RudderJS AI] evalSuite() requires at least one case.');
+    }
+    return Object.freeze({ name, spec });
+}
+// ─── Built-in metrics ─────────────────────────────────────
+/** Exact string equality against `response.text`. */
+export function exactMatch(expected) {
+    return (response) => {
+        const actual = response.text;
+        if (actual === expected)
+            return { pass: true, score: 1 };
+        return {
+            pass: false,
+            score: 0,
+            reason: `expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`,
+        };
+    };
+}
+/** Pattern match against `response.text`. */
+export function regex(pattern) {
+    return (response) => {
+        if (pattern.test(response.text))
+            return { pass: true, score: 1 };
+        return {
+            pass: false,
+            score: 0,
+            reason: `pattern ${pattern} did not match ${JSON.stringify(response.text.slice(0, 120))}${response.text.length > 120 ? '…' : ''}`,
+        };
+    };
+}
+/**
+ * LLM-as-judge: ask a small model whether the response satisfies a
+ * natural-language criterion. Returns the judge's reasoning in
+ * `reason` so failures are debuggable.
+ *
+ * Design: the judge runs as a one-shot anonymous agent (no recursion
+ * concern — default `remembers()` is `false`). Output is shaped via
+ * `Output.object({ schema })` for deterministic parsing. Failures
+ * (network, parse, unhandled judge error) bubble as `pass: false`
+ * with the error in `reason` — a broken judge is not a passing case.
+ *
+ * Pitfall: the judge model has the same biases as any LLM. Use it
+ * for fuzzy "did the answer mention X?" assertions; for exact
+ * structural checks prefer `jsonShape` (Phase 3) or `regex`.
+ */
+export function llmJudge(criterion, opts = {}) {
+    const wrapper = Output.object({
+        schema: z.object({
+            pass: z.boolean(),
+            reason: z.string(),
+        }),
+    });
+    return async (response, ctx) => {
+        try {
+            const judge = agent({
+                instructions: `${JUDGE_INSTRUCTIONS}\n\n${wrapper.toSystemPrompt()}`,
+                ...(opts.model ? { model: opts.model } : {}),
+            });
+            const prompt = [
+                `Criterion: ${criterion}`,
+                '',
+                `User input: ${JSON.stringify(ctx.input)}`,
+                `Agent response: ${JSON.stringify(response.text)}`,
+                '',
+                'Does the response satisfy the criterion? Return strictly valid JSON.',
+            ].join('\n');
+            const judgeResponse = await judge.prompt(prompt);
+            const parsed = wrapper.parse(judgeResponse.text);
+            // Tag the judge's token usage onto the response so the runner
+            // can include it in the cost rollup. This is a side-channel
+            // since the metric signature doesn't surface usage natively.
+            attachExtraUsage(response, judgeResponse.usage.totalTokens);
+            return {
+                pass: parsed.pass,
+                score: parsed.pass ? 1 : 0,
+                reason: parsed.reason,
+            };
+        }
+        catch (err) {
+            return {
+                pass: false,
+                score: 0,
+                reason: `judge failed: ${err instanceof Error ? err.message : String(err)}`,
+            };
+        }
+    };
+}
+const JUDGE_INSTRUCTIONS = [
+    'You are an evaluator judging whether an agent response satisfies a natural-language criterion.',
+    'Be precise: only return pass=true if the criterion is plainly met.',
+    'Provide a short reason for your decision (1-2 sentences) so the developer can debug failures.',
+].join(' ');
+/**
+ * Strict structural assertion: parse `response.text` as JSON
+ * (stripping ```json fences) and run it through a zod schema.
+ *
+ * Pairs naturally with `Output.object({ schema })` on the agent —
+ * if the agent declares the same schema, this metric verifies the
+ * output actually conforms. Failures surface the zod issue path
+ * (e.g. `customer.email`) so debugging doesn't require a separate
+ * console log.
+ */
+export function jsonShape(schema) {
+    return (response) => {
+        const stripped = stripCodeFences(response.text);
+        let parsed;
+        try {
+            parsed = JSON.parse(stripped);
+        }
+        catch (err) {
+            return {
+                pass: false,
+                score: 0,
+                reason: `not JSON: ${err instanceof Error ? err.message : String(err)}`,
+            };
+        }
+        const result = schema.safeParse(parsed);
+        if (result.success)
+            return { pass: true, score: 1 };
+        const first = result.error.issues[0];
+        const path = first?.path.join('.') || '<root>';
+        return {
+            pass: false,
+            score: 0,
+            reason: `schema mismatch at ${path}: ${first?.message ?? 'unknown error'}`,
+        };
+    };
+}
+/**
+ * Embedding-based fuzzy match. Embeds both `reference` and
+ * `response.text` via `AI.embed()`, computes cosine similarity,
+ * passes when >= `threshold` (default `0.85` — tighter than
+ * `EmbeddingUserMemory`'s 0.5 retrieval-rank floor since this is
+ * an assertion, not a ranking).
+ *
+ * Uses ≤ 2 embedding calls per case; embed tokens roll into the
+ * case's cost rollup via the same side-channel `llmJudge` uses.
+ *
+ * Pitfall: requires a provider that implements `createEmbedding()`
+ * (openai / google / mistral / cohere / jina). Failures (no
+ * provider, network, etc.) surface as `pass: false` with the
+ * error in `reason` — a broken embed is not a passing case.
+ */
+export function semanticMatch(reference, opts = {}) {
+    const threshold = opts.threshold ?? 0.85;
+    return async (response) => {
+        try {
+            const inputs = [reference, response.text];
+            const embedOpts = {};
+            if (opts.model)
+                embedOpts.model = opts.model;
+            const result = await AI.embed(inputs, embedOpts);
+            const [refVec, respVec] = result.embeddings;
+            if (!refVec || !respVec) {
+                return { pass: false, score: 0, reason: 'embed returned no vectors' };
+            }
+            attachExtraUsage(response, result.usage.totalTokens);
+            const score = cosineSimilarity(refVec, respVec);
+            const pass = score >= threshold;
+            return {
+                pass,
+                score,
+                reason: pass
+                    ? `cosine ${score.toFixed(3)} >= ${threshold}`
+                    : `cosine ${score.toFixed(3)} < ${threshold} (reference: ${JSON.stringify(reference.slice(0, 80))}${reference.length > 80 ? '…' : ''})`,
+            };
+        }
+        catch (err) {
+            return {
+                pass: false,
+                score: 0,
+                reason: `embed failed: ${err instanceof Error ? err.message : String(err)}`,
+            };
+        }
+    };
+}
+/**
+ * Token budget guard. Passes when `response.usage.totalTokens
+ * <= threshold`. Pair with cost-conscious agents to detect prompt-
+ * size regressions before they show up as a billing surprise.
+ *
+ * `response.usage` is the multi-step rollup, so it's meaningful
+ * even when the agent runs tools across several provider calls.
+ */
+export function tokenCost(threshold) {
+    return (response) => {
+        const used = response.usage.totalTokens;
+        const pass = used <= threshold;
+        return {
+            pass,
+            score: pass ? 1 : 0,
+            reason: pass
+                ? `${used} tokens <= ${threshold}`
+                : `${used} tokens > ${threshold}`,
+        };
+    };
+}
+/**
+ * Compose multiple metrics into one assertion. Runs them in order
+ * and short-circuits on the first failure — failure `reason` is
+ * surfaced; success returns `{ pass: true, score: 1 }`.
+ *
+ * @example
+ *   { input: '…',
+ *     assert: compose(
+ *       jsonShape(SummarySchema),
+ *       tokenCost(800),
+ *     ),
+ *   }
+ */
+export function compose(...metrics) {
+    return async (response, ctx) => {
+        for (const m of metrics) {
+            const result = await m(response, ctx);
+            if (!result.pass)
+                return result;
+        }
+        return { pass: true, score: 1 };
+    };
+}
+/** Local cosine — kept inline so `eval/` doesn't pull in `memory-embedding` (which depends on `@rudderjs/orm`). */
+function cosineSimilarity(a, b) {
+    if (a.length !== b.length)
+        return 0;
+    let dot = 0;
+    let magA = 0;
+    let magB = 0;
+    for (let i = 0; i < a.length; i++) {
+        const ai = a[i];
+        const bi = b[i];
+        dot += ai * bi;
+        magA += ai * ai;
+        magB += bi * bi;
+    }
+    if (magA === 0 || magB === 0)
+        return 0;
+    return dot / (Math.sqrt(magA) * Math.sqrt(magB));
+}
+function stripCodeFences(text) {
+    return text
+        .replace(/^```(?:json)?\s*\n?/m, '')
+        .replace(/\n?```\s*$/m, '')
+        .trim();
+}
+// ─── Runner ───────────────────────────────────────────────
+/**
+ * Run every case in the suite, in declaration order. Returns the
+ * full report; never throws (assertion errors become `failed` cases,
+ * not exceptions).
+ *
+ * Phase 1 runs serially. Parallel execution lands in a follow-up
+ * once we understand the rate-limit shape of real-world judge
+ * models — sequential is correct under any rate limit.
+ */
+export async function runSuite(suite) {
+    const start = performance.now();
+    const cases = [];
+    let passed = 0;
+    let failed = 0;
+    let skipped = 0;
+    for (let i = 0; i < suite.spec.cases.length; i++) {
+        const c = suite.spec.cases[i];
+        const name = c.name ?? `case-${i}`;
+        if (c.skip) {
+            const skipResult = {
+                name,
+                status: 'skipped',
+                reason: typeof c.skip === 'string' ? c.skip : 'skipped',
+                duration: 0,
+                tokens: 0,
+                cost: 0,
+                input: c.input,
+            };
+            cases.push(skipResult);
+            emitEvalCompleted(suite.name, skipResult);
+            skipped++;
+            continue;
+        }
+        const result = await runCase(suite, c, name);
+        cases.push(result);
+        emitEvalCompleted(suite.name, result);
+        if (result.status === 'passed')
+            passed++;
+        else if (result.status === 'failed')
+            failed++;
+    }
+    const duration = performance.now() - start;
+    const report = {
+        suite: suite.name,
+        cases,
+        passed,
+        failed,
+        skipped,
+        duration,
+        cost: cases.reduce((sum, c) => sum + c.cost, 0),
+        tokens: cases.reduce((sum, c) => sum + c.tokens, 0),
+    };
+    if (suite.spec.metadata)
+        report.metadata = suite.spec.metadata;
+    return report;
+}
+function emitEvalCompleted(suiteName, result) {
+    const event = {
+        kind: 'agent.eval.completed',
+        suite: suiteName,
+        case: result.name,
+        status: result.status,
+        pass: result.status === 'passed',
+        tokens: result.tokens,
+        cost: result.cost,
+        duration: result.duration,
+    };
+    if (result.metric?.score !== undefined)
+        event.score = result.metric.score;
+    const reason = result.status === 'skipped' ? result.reason : result.metric?.reason;
+    if (reason)
+        event.reason = reason;
+    aiObservers.emit(event);
+}
+async function runCase(suite, c, name) {
+    const factory = c.agent ?? suite.spec.agent;
+    const ag = factory();
+    const timeout = c.timeout ?? suite.spec.timeout;
+    const start = performance.now();
+    let response;
+    try {
+        response = await runWithTimeout(() => ag.prompt(c.input), timeout);
+    }
+    catch (err) {
+        return {
+            name,
+            status: 'failed',
+            metric: { pass: false, reason: err instanceof Error ? err.message : String(err) },
+            duration: performance.now() - start,
+            tokens: 0,
+            cost: 0,
+            input: c.input,
+        };
+    }
+    let metric;
+    try {
+        metric = await c.assert(response, { input: c.input, caseName: name });
+    }
+    catch (err) {
+        metric = { pass: false, reason: `assert threw: ${err instanceof Error ? err.message : String(err)}` };
+    }
+    const extraTokens = consumeExtraUsage(response);
+    const totalTokens = response.usage.totalTokens + extraTokens;
+    return {
+        name,
+        status: metric.pass ? 'passed' : 'failed',
+        metric,
+        duration: performance.now() - start,
+        tokens: totalTokens,
+        cost: estimateCost(modelStringFor(ag), response.usage.promptTokens, response.usage.completionTokens)
+            + estimateCost(modelStringFor(ag), 0, extraTokens), // judge/embed cost approximated as completion-side
+        input: c.input,
+        responseText: response.text,
+    };
+}
+function modelStringFor(ag) {
+    // `Agent.model()` may return undefined → callers fall back to the
+    // registry default. We don't have a stable hook for the default
+    // here without importing the registry; the eval flow doesn't
+    // strictly need the resolved model for cost estimation as long as
+    // the user's agent declares one. When it doesn't, costs fall back
+    // to an unknown-model rate (zero in Phase 1).
+    return ag.model() ?? 'unknown/unknown';
+}
+async function runWithTimeout(fn, ms) {
+    if (!ms || ms <= 0)
+        return fn();
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => reject(new Error(`timeout after ${ms}ms`)), ms);
+        fn().then(v => { clearTimeout(timer); resolve(v); }, e => { clearTimeout(timer); reject(e); });
+    });
+}
+// ─── Pricing ──────────────────────────────────────────────
+// Pricing catalog + estimator live in `../budget/pricing.ts` and are
+// re-exported here so `import { estimateCost } from '@rudderjs/ai/eval'`
+// continues to work. (Note: estimateCost is also imported at the top
+// of this file for use by `runSuite`.)
+export { estimateCost, ModelPricing } from '../budget/pricing.js';
+// ─── Console reporter ─────────────────────────────────────
+/**
+ * Default reporter — prints a colorless ANSI-aware table to a
+ * caller-supplied `console`-like sink. Uses Unicode pass/fail glyphs
+ * for visual scanning. JSON / HTML reporters land in Phase 2 / 5.
+ *
+ * Returns the report unchanged so chains compose: `await
+ * reportConsole(await runSuite(suite))`.
+ */
+export function reportConsole(report, sink = console) {
+    const lines = [];
+    const summary = `${report.suite} (${report.cases.length} cases, ${formatMs(report.duration)}, ${formatCost(report.cost)})`;
+    lines.push(summary);
+    for (const c of report.cases) {
+        const glyph = c.status === 'passed' ? '✓' : c.status === 'failed' ? '✗' : '○';
+        const meta = c.status === 'skipped'
+            ? `skip: ${c.reason ?? 'skipped'}`
+            : `${formatMs(c.duration)}   ${formatCost(c.cost)}   tokens: ${c.tokens}`;
+        lines.push(`  ${glyph} ${padName(c.name)} ${meta}`);
+        if (c.status === 'failed' && c.metric?.reason) {
+            // Indent reason on its own line so long messages don't break alignment.
+            for (const line of c.metric.reason.split('\n')) {
+                lines.push(`      ${line}`);
+            }
+        }
+    }
+    lines.push('');
+    lines.push(`  ${report.passed} passed, ${report.failed} failed${report.skipped > 0 ? `, ${report.skipped} skipped` : ''}`);
+    lines.push(`  total: ${formatCost(report.cost)}  •  cumulative tokens: ${report.tokens}`);
+    for (const line of lines)
+        sink.log(line);
+    return report;
+}
+function padName(s) {
+    return s.padEnd(28);
+}
+function formatMs(ms) {
+    if (ms < 1000)
+        return `${Math.round(ms)}ms`;
+    return `${(ms / 1000).toFixed(1)}s`;
+}
+function formatCost(cents) {
+    if (cents === 0)
+        return '$0.000';
+    if (cents < 0.001)
+        return '<$0.001';
+    return `$${cents.toFixed(3)}`;
+}
+// ─── Internal: extra-usage side-channel ───────────────────
+//
+// `Metric` doesn't surface token usage in its return type — the
+// signature is `(response, ctx) => MetricResult`. To roll auxiliary
+// token costs (llmJudge's judge model, semanticMatch's embeddings)
+// into the case's cost report, the metrics stamp usage onto a
+// Symbol-keyed slot on the response object and the runner consumes
+// it. Internal-only; never exported.
+const EXTRA_USAGE_KEY = Symbol.for('rudderjs.ai.eval.extraUsage');
+function attachExtraUsage(response, tokens) {
+    const carrier = response;
+    carrier[EXTRA_USAGE_KEY] = (carrier[EXTRA_USAGE_KEY] ?? 0) + tokens;
+}
+function consumeExtraUsage(response) {
+    const carrier = response;
+    const tokens = carrier[EXTRA_USAGE_KEY] ?? 0;
+    delete carrier[EXTRA_USAGE_KEY];
+    return tokens;
+}
+//# sourceMappingURL=index.js.map

package/dist/eval/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAA;AAGnC,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAA;AACrC,OAAO,EAAE,EAAE,EAAE,MAAM,cAAc,CAAA;AACjC,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAA;AACnD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAA;AAE/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAA;AAEjD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAA;AAsI/C,6DAA6D;AAE7D;;;;;;;;;GASG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,IAAmB;IACzD,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAA;IACxE,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,0EAA0E,CAAC,CAAA;IAC7F,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAA;IAC1E,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;AACtC,CAAC;AAED,6DAA6D;AAE7D,qDAAqD;AACrD,MAAM,UAAU,UAAU,CAAC,QAAgB;IACzC,OAAO,CAAC,QAAQ,EAAgB,EAAE;QAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAA;QAC5B,IAAI,MAAM,KAAK,QAAQ;YAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;QACxD,OAAO;YACL,IAAI,EAAI,KAAK;YACb,KAAK,EAAG,CAAC;YACT,MAAM,EAAE,YAAY,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE;SAC9E,CAAA;IACH,CAAC,CAAA;AACH,CAAC;AAED,6CAA6C;AAC7C,MAAM,UAAU,KAAK,CAAC,OAAe;IACnC,OAAO,CAAC,QAAQ,EAAgB,EAAE;QAChC,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;QAChE,OAAO;YACL,IAAI,EAAI,KAAK;YACb,KAAK,EAAG,CAAC;YACT,MAAM,EAAE,WAAW,OAAO,kBAAkB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;SAClI,CAAA;IACH,CAAC,CAAA;AACH,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,QAAQ,CAAC,SAAiB,EAAE,OAA2B,EAAE;IACvE,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;YACf,IAAI,EAAI,CAAC,CAAC,OAAO,EAAE;YACnB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;SACnB,CAAC;KACH,CAAC,CAAA;IAEF,OAAO,KAAK,EAAE,QAAQ,EAAE,GAAG,EAAyB,EAAE;QACpD,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,KAAK,CAAC;gBAClB,YAAY,EAAE,GAAG,kBAAkB,OAAO,OAAO,CAAC,cAAc,EAAE,EAAE;gBACpE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC7C,CAAC,CAAA;YAEF,MAAM,MAAM,GAAG;gBACb,cAAc,SAAS,EAAE;gBACzB,EAAE;gBACF,eAAe,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;gBAC1C,mBAAmB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;gBAClD,EAAE;gBACF,sEAAsE;aACvE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEZ,MAAM,aAAa,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;YAChD,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,CAAA;YAEhD,8DAA8D;YAC9D,4DAA4D;YAC5D,6DAA6D;YAC7D,gBAAgB,CAAC,QAAQ,EAAE,aAAa,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;YAE3D,OAAO;gBACL,IAAI,EAAI,MAAM,CAAC,IAAI;gBACnB,KAAK,EAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3B,MAAM,EAAE,MAAM,CAAC,MAAM;aACtB,CAAA;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,IAAI,EAAI,KAAK;gBACb,KAAK,EAAG,CAAC;gBACT,MAAM,EAAE,iBAAiB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;aAC5E,CAAA;QACH,CAAC;IACH,CAAC,CAAA;AACH,CAAC;AAED,MAAM,kBAAkB,GAAG;IACzB,gGAAgG;IAChG,oEAAoE;IACpE,+FAA+F;CAChG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AAEX;;;;;;;;;GASG;AACH,MAAM,UAAU,SAAS,CAAI,MAAoB;IAC/C,OAAO,CAAC,QAAQ,EAAgB,EAAE;QAChC,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QAC/C,IAAI,MAAe,CAAA;QACnB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;QAC/B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,IAAI,EAAI,KAAK;gBACb,KAAK,EAAG,CAAC;gBACT,MAAM,EAAE,aAAa,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;aACxE,CAAA;QACH,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;QACvC,IAAI,MAAM,CAAC,OAAO;YAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;QACnD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;QACpC,MAAM,IAAI,GAAI,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAA;QAC/C,OAAO;YACL,IAAI,EAAI,KAAK;YACb,KAAK,EAAG,CAAC;YACT,MAAM,EAAE,sBAAsB,IAAI,KAAK,KAAK,EAAE,OAAO,IAAI,eAAe,EAAE;SAC3E,CAAA;IACH,CAAC,CAAA;AACH,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,aAAa,CAC3B,SAAiB,EACjB,OAA+C,EAAE;IAEjD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAA;IACxC,OAAO,KAAK,EAAE,QAAQ,EAAyB,EAAE;QAC/C,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAA;YACzC,MAAM,SAAS,GAAuB,EAAE,CAAA;YACxC,IAAI,IAAI,CAAC,KAAK;gBAAE,SAAS,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAA;YAC5C,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;YAChD,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,UAAU,CAAA;YAC3C,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;gBACxB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,2BAA2B,EAAE,CAAA;YACvE,CAAC;YACD,gBAAgB,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;YAEpD,MAAM,KAAK,GAAG,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;YAC/C,MAAM,IAAI,GAAI,KAAK,IAAI,SAAS,CAAA;YAChC,OAAO;gBACL,IAAI;gBACJ,KAAK;gBACL,MAAM,EAAE,IAAI;oBACV,CAAC,CAAC,UAAU,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,SAAS,EAAE;oBAC9C,CAAC,CAAC,UAAU,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,SAAS,gBAAgB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG;aAC1I,CAAA;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,IAAI,EAAI,KAAK;gBACb,KAAK,EAAG,CAAC;gBACT,MAAM,EAAE,iBAAiB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;aAC5E,CAAA;QACH,CAAC;IACH,CAAC,CAAA;AACH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,SAAS,CAAC,SAAiB;IACzC,OAAO,CAAC,QAAQ,EAAgB,EAAE;QAChC,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAA;QACvC,MAAM,IAAI,GAAG,IAAI,IAAI,SAAS,CAAA;QAC9B,OAAO;YACL,IAAI;YACJ,KAAK,EAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,MAAM,EAAE,IAAI;gBACV,CAAC,CAAC,GAAG,IAAI,cAAc,SAAS,EAAE;gBAClC,CAAC,CAAC,GAAG,IAAI,aAAa,SAAS,EAAE;SACpC,CAAA;IACH,CAAC,CAAA;AACH,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,OAAO,CAAC,GAAG,OAAiB;IAC1C,OAAO,KAAK,EAAE,QAAQ,EAAE,GAAG,EAAyB,EAAE;QACpD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;YACrC,IAAI,CAAC,MAAM,CAAC,IAAI;gBAAE,OAAO,MAAM,CAAA;QACjC,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;IACjC,CAAC,CAAA;AACH,CAAC;AAED,mHAAmH;AACnH,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;QAAE,OAAO,CAAC,CAAA;IACnC,IAAI,GAAG,GAAI,CAAC,CAAA;IACZ,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAE,CAAA;QAChB,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAE,CAAA;QAChB,GAAG,IAAK,EAAE,GAAG,EAAE,CAAA;QACf,IAAI,IAAI,EAAE,GAAG,EAAE,CAAA;QACf,IAAI,IAAI,EAAE,GAAG,EAAE,CAAA;IACjB,CAAC;IACD,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAA;IACtC,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;AAClD,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACnC,OAAO,IAAI;SACR,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC;SACnC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC;SAC1B,IAAI,EAAE,CAAA;AACX,CAAC;AAED,6DAA6D;AAE7D;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,KAAgB;IAC7C,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;IAC/B,MAAM,KAAK,GAAiB,EAAE,CAAA;IAC9B,IAAI,MAAM,GAAI,CAAC,CAAA;IACf,IAAI,MAAM,GAAI,CAAC,CAAA;IACf,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACjD,MAAM,CAAC,GAAM,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAA;QACjC,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,QAAQ,CAAC,EAAE,CAAA;QAElC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;YACX,MAAM,UAAU,GAAe;gBAC7B,IAAI;gBACJ,MAAM,EAAI,SAAS;gBACnB,MAAM,EAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBACzD,QAAQ,EAAE,CAAC;gBACX,MAAM,EAAI,CAAC;gBACX,IAAI,EAAM,CAAC;gBACX,KAAK,EAAK,CAAC,CAAC,KAAK;aAClB,CAAA;YACD,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;YACtB,iBAAiB,CAAC,KAAK,CAAC,IAAI,EAAE,UAAU,CAAC,CAAA;YACzC,OAAO,EAAE,CAAA;YACT,SAAQ;QACV,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,EAAE,IAAI,CAAC,CAAA;QAC5C,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAClB,iBAAiB,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;QACrC,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ;YAAE,MAAM,EAAE,CAAA;aACnC,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ;YAAE,MAAM,EAAE,CAAA;IAC/C,CAAC;IAED,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAA;IAE1C,MAAM,MAAM,GAAgB;QAC1B,KAAK,EAAE,KAAK,CAAC,IAAI;QACjB,KAAK;QACL,MAAM;QACN,MAAM;QACN,OAAO;QACP,QAAQ;QACR,IAAI,EAAI,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAI,CAAC,CAAC;QACnD,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;KACpD,CAAA;IACD,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ;QAAE,MAAM,CAAC,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAA;IAC9D,OAAO,MAAM,CAAA;AACf,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAiB,EAAE,MAAkB;IAC9D,MAAM,KAAK,GAA2C;QACpD,IAAI,EAAM,sBAAsB;QAChC,KAAK,EAAK,SAAS;QACnB,IAAI,EAAM,MAAM,CAAC,IAAI;QACrB,MAAM,EAAI,MAAM,CAAC,MAAM;QACvB,IAAI,EAAM,MAAM,CAAC,MAAM,KAAK,QAAQ;QACpC,MAAM,EAAI,MAAM,CAAC,MAAM;QACvB,IAAI,EAAM,MAAM,CAAC,IAAI;QACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ;KAC1B,CAAA;IACD,IAAI,MAAM,CAAC,MAAM,EAAE,KAAK,KAAK,SAAS;QAAE,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAA;IACzE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAA;IAClF,IAAI,MAAM;QAAE,KAAK,CAAC,MAAM,GAAG,MAAM,CAAA;IACjC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;AACzB,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,KAAgB,EAAE,CAAW,EAAE,IAAY;IAChE,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAA;IAC3C,MAAM,EAAE,GAAQ,OAAO,EAAE,CAAA;IACzB,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAA;IAE/C,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;IAC/B,IAAI,QAAuB,CAAA;IAC3B,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,CAAA;IACpE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,IAAI;YACJ,MAAM,EAAI,QAAQ;YAClB,MAAM,EAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YACnF,QAAQ,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK;YACnC,MAAM,EAAI,CAAC;YACX,IAAI,EAAM,CAAC;YACX,KAAK,EAAK,CAAC,CAAC,KAAK;SAClB,CAAA;IACH,CAAC;IAED,IAAI,MAAoB,CAAA;IACxB,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;IACvE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,CAAA;IACvG,CAAC;IAED,MAAM,WAAW,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAA;IAC/C,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,WAAW,GAAG,WAAW,CAAA;IAE5D,OAAO;QACL,IAAI;QACJ,MAAM,EAAQ,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;QAC/C,MAAM;QACN,QAAQ,EAAM,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK;QACvC,MAAM,EAAQ,WAAW;QACzB,IAAI,EAAU,YAAY,CAAC,cAAc,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAC,YAAY,EAAE,QAAQ,CAAC,KAAK,CAAC,gBAAgB,CAAC;cAC9F,YAAY,CAAC,cAAc,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,WAAW,CAAC,EAAG,mDAAmD;QACpH,KAAK,EAAS,CAAC,CAAC,KAAK;QACrB,YAAY,EAAE,QAAQ,CAAC,IAAI;KAC5B,CAAA;AACH,CAAC;AAED,SAAS,cAAc,CAAC,EAAS;IAC/B,kEAAkE;IAClE,gEAAgE;IAChE,6DAA6D;IAC7D,kEAAkE;IAClE,kEAAkE;IAClE,8CAA8C;IAC9C,OAAO,EAAE,CAAC,KAAK,EAAE,IAAI,iBAAiB,CAAA;AACxC,CAAC;AAED,KAAK,UAAU,cAAc,CAAI,EAAoB,EAAE,EAAsB;IAC3E,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC;QAAE,OAAO,EAAE,EAAE,CAAA;IAC/B,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACxC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QAC9E,EAAE,EAAE,CAAC,IAAI,CACP,CAAC,CAAC,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA,CAAC,CAAC,EACxC,CAAC,CAAC,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA,CAAC,CAAC,CACxC,CAAA;IACH,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,6DAA6D;AAE7D,qEAAqE;AACrE,yEAAyE;AACzE,qEAAqE;AACrE,uCAAuC;AACvC,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAA;AAGjE,6DAA6D;AAE7D;;;;;;;GAOG;AACH,MAAM,UAAU,aAAa,CAAC,MAAmB,EAAE,OAAqC,OAAO;IAC7F,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,MAAM,OAAO,GAAG,GAAG,MAAM,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK,CAAC,MAAM,WAAW,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAA;IAC1H,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IAEnB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAA;QAC7E,MAAM,IAAI,GAAI,CAAC,CAAC,MAAM,KAAK,SAAS;YAClC,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,IAAI,SAAS,EAAE;YAClC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,MAAM,EAAE,CAAA;QAC3E,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,IAAI,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAA;QACnD,IAAI,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;YAC9C,wEAAwE;YACxE,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC/C,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,CAAA;YAC7B,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACd,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,MAAM,YAAY,MAAM,CAAC,MAAM,UAAU,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,OAAO,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IAC1H,KAAK,CAAC,IAAI,CAAC,YAAY,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IAEzF,KAAK,MAAM,IAAI,IAAI,KAAK;QAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IACxC,OAAO,MAAM,CAAA;AACf,CAAC;AAED,SAAS,OAAO,CAAC,CAAS;IACxB,OAAO,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;AACrB,CAAC;AAED,SAAS,QAAQ,CAAC,EAAU;IAC1B,IAAI,EAAE,GAAG,IAAI;QAAE,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAA;IAC3C,OAAO,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAA;AACrC,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAA;IAChC,IAAI,KAAK,GAAG,KAAK;QAAE,OAAO,SAAS,CAAA;IACnC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAA;AAC/B,CAAC;AAED,6DAA6D;AAC7D,EAAE;AACF,gEAAgE;AAChE,oEAAoE;AACpE,mEAAmE;AACnE,8DAA8D;AAC9D,mEAAmE;AACnE,qCAAqC;AAErC,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAA;AAMjE,SAAS,gBAAgB,CAAC,QAAuB,EAAE,MAAc;IAC/D,MAAM,OAAO,GAAG,QAA6C,CAAA;IAC7D,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAA;AACrE,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAuB;IAChD,MAAM,OAAO,GAAG,QAA6C,CAAA;IAC7D,MAAM,MAAM,GAAI,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;IAC7C,OAAO,OAAO,CAAC,eAAe,CAAC,CAAA;IAC/B,OAAO,MAAM,CAAA;AACf,CAAC"}

package/dist/eval/json-reporter.d.ts

@@ -0,0 +1,43 @@
+import type { SuiteReport } from './index.js';
+/**
+ * Single case row in the JSON output. `metric.pass`/`metric.score`/
+ * `metric.reason` are flattened up so consumers don't have to reach
+ * through nested objects in CI scripts.
+ */
+export interface SuiteJsonCase {
+    name: string;
+    status: 'passed' | 'failed' | 'skipped';
+    pass: boolean;
+    score?: number;
+    reason?: string;
+    duration: number;
+    tokens: number;
+    cost: number;
+}
+/**
+ * Machine-readable suite output emitted by `pnpm rudder ai:eval --json`.
+ * Stable shape — bumping fields here is a minor (additive) bump for
+ * `@rudderjs/ai`. Removing or renaming fields is a major.
+ */
+export interface SuiteJson {
+    suite: string;
+    passed: number;
+    failed: number;
+    skipped: number;
+    duration: number;
+    cost: number;
+    tokens: number;
+    cases: SuiteJsonCase[];
+}
+/**
+ * JSON reporter — flattens a `SuiteReport` for CI consumption.
+ *
+ * Mirrors the `command_run` MCP tool envelope shape so the boost
+ * agent surface and the eval CLI feel like one family.
+ *
+ * @example
+ *   const report = await runSuite(suite)
+ *   process.stdout.write(JSON.stringify(reportJson(report)))
+ */
+export declare function reportJson(report: SuiteReport): SuiteJson;
+//# sourceMappingURL=json-reporter.d.ts.map

package/dist/eval/json-reporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-reporter.d.ts","sourceRoot":"","sources":["../../src/eval/json-reporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAE7C;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAO,MAAM,CAAA;IACjB,MAAM,EAAK,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAA;IAC1C,IAAI,EAAO,OAAO,CAAA;IAClB,KAAK,CAAC,EAAK,MAAM,CAAA;IACjB,MAAM,CAAC,EAAI,MAAM,CAAA;IACjB,QAAQ,EAAG,MAAM,CAAA;IACjB,MAAM,EAAK,MAAM,CAAA;IACjB,IAAI,EAAO,MAAM,CAAA;CAClB;AAED;;;;GAIG;AACH,MAAM,WAAW,SAAS;IACxB,KAAK,EAAK,MAAM,CAAA;IAChB,MAAM,EAAI,MAAM,CAAA;IAChB,MAAM,EAAI,MAAM,CAAA;IAChB,OAAO,EAAG,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,IAAI,EAAM,MAAM,CAAA;IAChB,MAAM,EAAI,MAAM,CAAA;IAChB,KAAK,EAAK,aAAa,EAAE,CAAA;CAC1B;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,WAAW,GAAG,SAAS,CAWzD"}

package/dist/eval/json-reporter.js

@@ -0,0 +1,40 @@
+/**
+ * JSON reporter — flattens a `SuiteReport` for CI consumption.
+ *
+ * Mirrors the `command_run` MCP tool envelope shape so the boost
+ * agent surface and the eval CLI feel like one family.
+ *
+ * @example
+ *   const report = await runSuite(suite)
+ *   process.stdout.write(JSON.stringify(reportJson(report)))
+ */
+export function reportJson(report) {
+    return {
+        suite: report.suite,
+        passed: report.passed,
+        failed: report.failed,
+        skipped: report.skipped,
+        duration: report.duration,
+        cost: report.cost,
+        tokens: report.tokens,
+        cases: report.cases.map(toJsonCase),
+    };
+}
+function toJsonCase(c) {
+    const out = {
+        name: c.name,
+        status: c.status,
+        pass: c.status === 'passed',
+        duration: c.duration,
+        tokens: c.tokens,
+        cost: c.cost,
+    };
+    if (c.metric?.score !== undefined)
+        out.score = c.metric.score;
+    if (c.status === 'skipped' && c.reason)
+        out.reason = c.reason;
+    else if (c.metric?.reason)
+        out.reason = c.metric.reason;
+    return out;
+}
+//# sourceMappingURL=json-reporter.js.map

package/dist/eval/json-reporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-reporter.js","sourceRoot":"","sources":["../../src/eval/json-reporter.ts"],"names":[],"mappings":"AAkCA;;;;;;;;;GASG;AACH,MAAM,UAAU,UAAU,CAAC,MAAmB;IAC5C,OAAO;QACL,KAAK,EAAK,MAAM,CAAC,KAAK;QACtB,MAAM,EAAI,MAAM,CAAC,MAAM;QACvB,MAAM,EAAI,MAAM,CAAC,MAAM;QACvB,OAAO,EAAG,MAAM,CAAC,OAAO;QACxB,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,IAAI,EAAM,MAAM,CAAC,IAAI;QACrB,MAAM,EAAI,MAAM,CAAC,MAAM;QACvB,KAAK,EAAK,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC;KACvC,CAAA;AACH,CAAC;AAED,SAAS,UAAU,CAAC,CAA+B;IACjD,MAAM,GAAG,GAAkB;QACzB,IAAI,EAAM,CAAC,CAAC,IAAI;QAChB,MAAM,EAAI,CAAC,CAAC,MAAM;QAClB,IAAI,EAAM,CAAC,CAAC,MAAM,KAAK,QAAQ;QAC/B,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,MAAM,EAAI,CAAC,CAAC,MAAM;QAClB,IAAI,EAAM,CAAC,CAAC,IAAI;KACjB,CAAA;IACD,IAAI,CAAC,CAAC,MAAM,EAAE,KAAK,KAAK,SAAS;QAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAA;IAC7D,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM;QAAE,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAA;SACxD,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM;QAAE,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAA;IACvD,OAAO,GAAG,CAAA;AACZ,CAAC"}