@rudderjs/ai 1.4.0 → 1.6.0

package/dist/eval/index.d.ts

@@ -0,0 +1,271 @@
+/**
+ * `@rudderjs/ai/eval` — built-in eval framework for #A5 Phase 1.
+ *
+ * Define a suite of input cases + assertions, run them against any
+ * `Agent`, get a console report with pass/fail + cost + tokens. Same
+ * `Agent` instances as your app code — one source of truth.
+ *
+ * @example
+ * ```ts
+ * // evals/support-agent.eval.ts
+ * import { evalSuite, llmJudge, exactMatch, regex } from '@rudderjs/ai/eval'
+ * import { SupportAgent } from '../app/Agents/SupportAgent.js'
+ *
+ * export default evalSuite('SupportAgent', {
+ *   agent: () => new SupportAgent(),
+ *   cases: [
+ *     { name: 'password reset', input: 'How do I reset my password?',
+ *       assert: llmJudge('mentions a password reset link') },
+ *     { name: 'price', input: 'How much?', assert: exactMatch('$99/month') },
+ *     { name: 'support email', input: 'Contact?', assert: regex(/support@/) },
+ *   ],
+ * })
+ * ```
+ *
+ * Run programmatically via `runSuite(suite)` from this entry, or via
+ * `pnpm rudder ai:eval` once Phase 2 lands.
+ *
+ * Built-in metrics: `exactMatch`, `regex`, `llmJudge`, `jsonShape`,
+ * `semanticMatch`, `tokenCost`. Compose multiple via `compose(...)`.
+ * User-defined metrics work today — any `(response, ctx) =>
+ * MetricResult` qualifies.
+ */
+import type { Agent } from '../agent.js';
+import type { AgentResponse } from '../types.js';
+import { z } from 'zod';
+export { reportJson } from './json-reporter.js';
+export type { SuiteJson, SuiteJsonCase } from './json-reporter.js';
+export { stepsFromResponse } from './fixtures.js';
+export type { EvalFixture } from './fixtures.js';
+export { reportHtml } from './html-reporter.js';
+export type { HtmlReportOptions } from './html-reporter.js';
+/**
+ * Result of a single assertion. `pass` is the only required field;
+ * `score` (0..1) and `reason` are surfaced in reports.
+ */
+export interface MetricResult {
+    pass: boolean;
+    score?: number;
+    reason?: string;
+}
+/**
+ * Assertion signature. Sync or async; the runner awaits both.
+ *
+ * `ctx` carries the case context so user metrics can opt into the
+ * input/case-name (e.g. for logging). The built-ins ignore it.
+ */
+export type Metric = (response: AgentResponse, ctx: MetricContext) => MetricResult | Promise<MetricResult>;
+export interface MetricContext {
+    /** The case's input string (the same passed to `agent.prompt`). */
+    input: string;
+    /** Optional case `name` if set on the spec. */
+    caseName: string;
+}
+/** A single eval case. */
+export interface EvalCase {
+    /** Stable identifier used in reports. Defaults to `case-<index>`. */
+    name?: string;
+    /** Input passed to `agent.prompt(input)`. */
+    input: string;
+    /** The assertion. Pass-fail + optional score/reason. */
+    assert: Metric;
+    /**
+     * Per-case agent override. When set, replaces the suite-level
+     * `agent` factory for this case (e.g. swap models for a stress
+     * test).
+     */
+    agent?: () => Agent;
+    /**
+     * Per-case timeout in ms. Defaults to the suite-level timeout
+     * (or no timeout if neither is set).
+     */
+    timeout?: number;
+    /**
+     * Skip this case. Pass `true` to silently skip, or a string for
+     * a reason that surfaces in the report.
+     */
+    skip?: boolean | string;
+}
+export interface EvalSuiteSpec {
+    /** Factory for the agent under test. Called once per case. */
+    agent: () => Agent;
+    /** The cases to run. */
+    cases: EvalCase[];
+    /**
+     * Suite-wide timeout in ms applied to every case unless the case
+     * overrides. Throws cause `pass: false` with the timeout message.
+     */
+    timeout?: number;
+    /**
+     * Optional ownership / context surfaced in the HTML report (#A5
+     * Phase 5). Well-known keys (`owner`, `lastReviewed`, `ticket`)
+     * get formatted headings; any extra string keys render as a
+     * generic key/value row so teams can attach their own metadata.
+     */
+    metadata?: EvalMetadata;
+}
+export interface EvalMetadata {
+    owner?: string;
+    lastReviewed?: string;
+    ticket?: string;
+    [key: string]: string | undefined;
+}
+export interface EvalSuite {
+    name: string;
+    spec: EvalSuiteSpec;
+}
+/** Per-case run record collected by {@link runSuite}. */
+export interface CaseResult {
+    name: string;
+    /** Final result; `'skipped'` skips assertion + cost. */
+    status: 'passed' | 'failed' | 'skipped';
+    metric?: MetricResult;
+    /** Skip reason (when `status === 'skipped'`). */
+    reason?: string;
+    /** Wall-clock ms for the agent call + assertion. */
+    duration: number;
+    /**
+     * Token usage from the agent's `prompt()` (zero on skip / failure
+     * before the call). Includes BOTH the agent under test AND any
+     * judge-model calls the assertion made.
+     */
+    tokens: number;
+    /** USD estimate (see {@link estimateCost}; zero on skip). */
+    cost: number;
+    /**
+     * The case's input string, copied through from `EvalCase.input`
+     * for reporters that want to render the prompt alongside the
+     * response (#A5 Phase 5 HTML report). Always present — runners
+     * always know the input.
+     */
+    input: string;
+    /**
+     * The agent's final assistant text. Absent when the case skipped
+     * or the agent threw before producing a response. The HTML
+     * reporter renders `<no response>` in that case.
+     */
+    responseText?: string;
+}
+/** Full report returned by {@link runSuite}. */
+export interface SuiteReport {
+    suite: string;
+    cases: CaseResult[];
+    passed: number;
+    failed: number;
+    skipped: number;
+    duration: number;
+    cost: number;
+    tokens: number;
+    /** Suite-level metadata (#A5 Phase 5), copied through from the spec. */
+    metadata?: EvalMetadata;
+}
+/**
+ * Define an eval suite. Returns a frozen `EvalSuite` ready to pass
+ * into {@link runSuite} or to default-export from an `evals/*.eval.ts`
+ * file (Phase 2's CLI auto-discovers those).
+ *
+ * The shape is deliberately a function rather than a class — keeps the
+ * file's default export trivially serializable (Phase 2 needs to load
+ * suites via dynamic import) and avoids the "did you forget `new`?"
+ * footgun.
+ */
+export declare function evalSuite(name: string, spec: EvalSuiteSpec): EvalSuite;
+/** Exact string equality against `response.text`. */
+export declare function exactMatch(expected: string): Metric;
+/** Pattern match against `response.text`. */
+export declare function regex(pattern: RegExp): Metric;
+/**
+ * LLM-as-judge: ask a small model whether the response satisfies a
+ * natural-language criterion. Returns the judge's reasoning in
+ * `reason` so failures are debuggable.
+ *
+ * Design: the judge runs as a one-shot anonymous agent (no recursion
+ * concern — default `remembers()` is `false`). Output is shaped via
+ * `Output.object({ schema })` for deterministic parsing. Failures
+ * (network, parse, unhandled judge error) bubble as `pass: false`
+ * with the error in `reason` — a broken judge is not a passing case.
+ *
+ * Pitfall: the judge model has the same biases as any LLM. Use it
+ * for fuzzy "did the answer mention X?" assertions; for exact
+ * structural checks prefer `jsonShape` (Phase 3) or `regex`.
+ */
+export declare function llmJudge(criterion: string, opts?: {
+    model?: string;
+}): Metric;
+/**
+ * Strict structural assertion: parse `response.text` as JSON
+ * (stripping ```json fences) and run it through a zod schema.
+ *
+ * Pairs naturally with `Output.object({ schema })` on the agent —
+ * if the agent declares the same schema, this metric verifies the
+ * output actually conforms. Failures surface the zod issue path
+ * (e.g. `customer.email`) so debugging doesn't require a separate
+ * console log.
+ */
+export declare function jsonShape<T>(schema: z.ZodType<T>): Metric;
+/**
+ * Embedding-based fuzzy match. Embeds both `reference` and
+ * `response.text` via `AI.embed()`, computes cosine similarity,
+ * passes when >= `threshold` (default `0.85` — tighter than
+ * `EmbeddingUserMemory`'s 0.5 retrieval-rank floor since this is
+ * an assertion, not a ranking).
+ *
+ * Uses ≤ 2 embedding calls per case; embed tokens roll into the
+ * case's cost rollup via the same side-channel `llmJudge` uses.
+ *
+ * Pitfall: requires a provider that implements `createEmbedding()`
+ * (openai / google / mistral / cohere / jina). Failures (no
+ * provider, network, etc.) surface as `pass: false` with the
+ * error in `reason` — a broken embed is not a passing case.
+ */
+export declare function semanticMatch(reference: string, opts?: {
+    threshold?: number;
+    model?: string;
+}): Metric;
+/**
+ * Token budget guard. Passes when `response.usage.totalTokens
+ * <= threshold`. Pair with cost-conscious agents to detect prompt-
+ * size regressions before they show up as a billing surprise.
+ *
+ * `response.usage` is the multi-step rollup, so it's meaningful
+ * even when the agent runs tools across several provider calls.
+ */
+export declare function tokenCost(threshold: number): Metric;
+/**
+ * Compose multiple metrics into one assertion. Runs them in order
+ * and short-circuits on the first failure — failure `reason` is
+ * surfaced; success returns `{ pass: true, score: 1 }`.
+ *
+ * @example
+ *   { input: '…',
+ *     assert: compose(
+ *       jsonShape(SummarySchema),
+ *       tokenCost(800),
+ *     ),
+ *   }
+ */
+export declare function compose(...metrics: Metric[]): Metric;
+/**
+ * Run every case in the suite, in declaration order. Returns the
+ * full report; never throws (assertion errors become `failed` cases,
+ * not exceptions).
+ *
+ * Phase 1 runs serially. Parallel execution lands in a follow-up
+ * once we understand the rate-limit shape of real-world judge
+ * models — sequential is correct under any rate limit.
+ */
+export declare function runSuite(suite: EvalSuite): Promise<SuiteReport>;
+export { estimateCost, ModelPricing } from '../budget/pricing.js';
+export type { ModelPriceEntry } from '../budget/pricing.js';
+/**
+ * Default reporter — prints a colorless ANSI-aware table to a
+ * caller-supplied `console`-like sink. Uses Unicode pass/fail glyphs
+ * for visual scanning. JSON / HTML reporters land in Phase 2 / 5.
+ *
+ * Returns the report unchanged so chains compose: `await
+ * reportConsole(await runSuite(suite))`.
+ */
+export declare function reportConsole(report: SuiteReport, sink?: {
+    log: (s: string) => void;
+}): SuiteReport;
+//# sourceMappingURL=index.d.ts.map

package/dist/eval/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAGH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAKhD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAA;AAC/C,YAAY,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAClE,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAA;AACjD,YAAY,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAA;AAC/C,YAAY,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AAI3D;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAK,OAAO,CAAA;IAChB,KAAK,CAAC,EAAG,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED;;;;;GAKG;AACH,MAAM,MAAM,MAAM,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,GAAG,EAAE,aAAa,KAAK,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;AAE1G,MAAM,WAAW,aAAa;IAC5B,mEAAmE;IACnE,KAAK,EAAK,MAAM,CAAA;IAChB,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,0BAA0B;AAC1B,MAAM,WAAW,QAAQ;IACvB,qEAAqE;IACrE,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAA;IACb,wDAAwD;IACxD,MAAM,EAAE,MAAM,CAAA;IACd;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,KAAK,CAAA;IACnB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;;OAGG;IACH,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,CAAA;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,8DAA8D;IAC9D,KAAK,EAAE,MAAM,KAAK,CAAA;IAClB,wBAAwB;IACxB,KAAK,EAAE,QAAQ,EAAE,CAAA;IACjB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,YAAY,CAAA;CACxB;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,CAAC,EAAS,MAAM,CAAA;IACrB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,MAAM,CAAC,EAAQ,MAAM,CAAA;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAA;CAClC;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,aAAa,CAAA;CACpB;AAED,yDAAyD;AACzD,MAAM,WAAW,UAAU;IACzB,IAAI,EAAK,MAAM,CAAA;IACf,wDAAwD;IACxD,MAAM,EAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAA;IACxC,MAAM,CAAC,EAAE,YAAY,CAAA;IACrB,iDAAiD;IACjD,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,oDAAoD;IACpD,QAAQ,EAAE,MAAM,CAAA;IAChB;;;;OAIG;IACH,MAAM,EAAG,MAAM,CAAA;IACf,6DAA6D;IAC7D,IAAI,EAAK,MAAM,CAAA;IACf;;;;;OAKG;IACH,KAAK,EAAI,MAAM,CAAA;IACf;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED,gDAAgD;AAChD,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAK,MAAM,CAAA;IAChB,KAAK,EAAK,UAAU,EAAE,CAAA;IACtB,MAAM,EAAI,MAAM,CAAA;IAChB,MAAM,EAAI,MAAM,CAAA;IAChB,OAAO,EAAG,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,IAAI,EAAM,MAAM,CAAA;IAChB,MAAM,EAAI,MAAM,CAAA;IAChB,wEAAwE;IACxE,QAAQ,CAAC,EAAE,YAAY,CAAA;CACxB;AAID;;;;;;;;;GASG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,GAAG,SAAS,CAStE;AAID,qDAAqD;AACrD,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAUnD;AAED,6CAA6C;AAC7C,wBAAgB,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAS7C;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,MAAM,CA6CjF;AAQD;;;;;;;;;GASG;AACH,wBAAgB,SAAS,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,CAuBzD;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,aAAa,CAC3B,SAAS,EAAE,MAAM,EACjB,IAAI,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAChD,MAAM,CA+BR;AAED;;;;;;;GAOG;AACH,wBAAgB,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAYnD;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,OAAO,CAAC,GAAG,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,CAQpD;AA4BD;;;;;;;;GAQG;AACH,wBAAsB,QAAQ,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,CAgDrE;AA0FD,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAA;AACjE,YAAY,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAI3D;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,WAAW,EAAE,IAAI,GAAE;IAAE,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,IAAI,CAAA;CAAY,GAAG,WAAW,CAyB5G"}