@rudderjs/ai 1.17.3 → 1.18.0

package/dist/eval/index.d.ts

@@ -1,271 +1,2 @@
-/**
- * `@rudderjs/ai/eval` — built-in eval framework for #A5 Phase 1.
- *
- * Define a suite of input cases + assertions, run them against any
- * `Agent`, get a console report with pass/fail + cost + tokens. Same
- * `Agent` instances as your app code — one source of truth.
- *
- * @example
- * ```ts
- * // evals/support-agent.eval.ts
- * import { evalSuite, llmJudge, exactMatch, regex } from '@rudderjs/ai/eval'
- * import { SupportAgent } from '../app/Agents/SupportAgent.js'
- *
- * export default evalSuite('SupportAgent', {
- *   agent: () => new SupportAgent(),
- *   cases: [
- *     { name: 'password reset', input: 'How do I reset my password?',
- *       assert: llmJudge('mentions a password reset link') },
- *     { name: 'price', input: 'How much?', assert: exactMatch('$99/month') },
- *     { name: 'support email', input: 'Contact?', assert: regex(/support@/) },
- *   ],
- * })
- * ```
- *
- * Run programmatically via `runSuite(suite)` from this entry, or via
- * `pnpm rudder ai:eval` once Phase 2 lands.
- *
- * Built-in metrics: `exactMatch`, `regex`, `llmJudge`, `jsonShape`,
- * `semanticMatch`, `tokenCost`. Compose multiple via `compose(...)`.
- * User-defined metrics work today — any `(response, ctx) =>
- * MetricResult` qualifies.
- */
-import type { Agent } from '../agent.js';
-import type { AgentResponse } from '../types.js';
-import { z } from 'zod';
-export { reportJson } from './json-reporter.js';
-export type { SuiteJson, SuiteJsonCase } from './json-reporter.js';
-export { stepsFromResponse } from './fixtures.js';
-export type { EvalFixture } from './fixtures.js';
-export { reportHtml } from './html-reporter.js';
-export type { HtmlReportOptions } from './html-reporter.js';
-/**
- * Result of a single assertion. `pass` is the only required field;
- * `score` (0..1) and `reason` are surfaced in reports.
- */
-export interface MetricResult {
-    pass: boolean;
-    score?: number;
-    reason?: string;
-}
-/**
- * Assertion signature. Sync or async; the runner awaits both.
- *
- * `ctx` carries the case context so user metrics can opt into the
- * input/case-name (e.g. for logging). The built-ins ignore it.
- */
-export type Metric = (response: AgentResponse, ctx: MetricContext) => MetricResult | Promise<MetricResult>;
-export interface MetricContext {
-    /** The case's input string (the same passed to `agent.prompt`). */
-    input: string;
-    /** Optional case `name` if set on the spec. */
-    caseName: string;
-}
-/** A single eval case. */
-export interface EvalCase {
-    /** Stable identifier used in reports. Defaults to `case-<index>`. */
-    name?: string;
-    /** Input passed to `agent.prompt(input)`. */
-    input: string;
-    /** The assertion. Pass-fail + optional score/reason. */
-    assert: Metric;
-    /**
-     * Per-case agent override. When set, replaces the suite-level
-     * `agent` factory for this case (e.g. swap models for a stress
-     * test).
-     */
-    agent?: () => Agent;
-    /**
-     * Per-case timeout in ms. Defaults to the suite-level timeout
-     * (or no timeout if neither is set).
-     */
-    timeout?: number;
-    /**
-     * Skip this case. Pass `true` to silently skip, or a string for
-     * a reason that surfaces in the report.
-     */
-    skip?: boolean | string;
-}
-export interface EvalSuiteSpec {
-    /** Factory for the agent under test. Called once per case. */
-    agent: () => Agent;
-    /** The cases to run. */
-    cases: EvalCase[];
-    /**
-     * Suite-wide timeout in ms applied to every case unless the case
-     * overrides. Throws cause `pass: false` with the timeout message.
-     */
-    timeout?: number;
-    /**
-     * Optional ownership / context surfaced in the HTML report (#A5
-     * Phase 5). Well-known keys (`owner`, `lastReviewed`, `ticket`)
-     * get formatted headings; any extra string keys render as a
-     * generic key/value row so teams can attach their own metadata.
-     */
-    metadata?: EvalMetadata;
-}
-export interface EvalMetadata {
-    owner?: string;
-    lastReviewed?: string;
-    ticket?: string;
-    [key: string]: string | undefined;
-}
-export interface EvalSuite {
-    name: string;
-    spec: EvalSuiteSpec;
-}
-/** Per-case run record collected by {@link runSuite}. */
-export interface CaseResult {
-    name: string;
-    /** Final result; `'skipped'` skips assertion + cost. */
-    status: 'passed' | 'failed' | 'skipped';
-    metric?: MetricResult;
-    /** Skip reason (when `status === 'skipped'`). */
-    reason?: string;
-    /** Wall-clock ms for the agent call + assertion. */
-    duration: number;
-    /**
-     * Token usage from the agent's `prompt()` (zero on skip / failure
-     * before the call). Includes BOTH the agent under test AND any
-     * judge-model calls the assertion made.
-     */
-    tokens: number;
-    /** USD estimate (see {@link estimateCost}; zero on skip). */
-    cost: number;
-    /**
-     * The case's input string, copied through from `EvalCase.input`
-     * for reporters that want to render the prompt alongside the
-     * response (#A5 Phase 5 HTML report). Always present — runners
-     * always know the input.
-     */
-    input: string;
-    /**
-     * The agent's final assistant text. Absent when the case skipped
-     * or the agent threw before producing a response. The HTML
-     * reporter renders `<no response>` in that case.
-     */
-    responseText?: string;
-}
-/** Full report returned by {@link runSuite}. */
-export interface SuiteReport {
-    suite: string;
-    cases: CaseResult[];
-    passed: number;
-    failed: number;
-    skipped: number;
-    duration: number;
-    cost: number;
-    tokens: number;
-    /** Suite-level metadata (#A5 Phase 5), copied through from the spec. */
-    metadata?: EvalMetadata;
-}
-/**
- * Define an eval suite. Returns a frozen `EvalSuite` ready to pass
- * into {@link runSuite} or to default-export from an `evals/*.eval.ts`
- * file (Phase 2's CLI auto-discovers those).
- *
- * The shape is deliberately a function rather than a class — keeps the
- * file's default export trivially serializable (Phase 2 needs to load
- * suites via dynamic import) and avoids the "did you forget `new`?"
- * footgun.
- */
-export declare function evalSuite(name: string, spec: EvalSuiteSpec): EvalSuite;
-/** Exact string equality against `response.text`. */
-export declare function exactMatch(expected: string): Metric;
-/** Pattern match against `response.text`. */
-export declare function regex(pattern: RegExp): Metric;
-/**
- * LLM-as-judge: ask a small model whether the response satisfies a
- * natural-language criterion. Returns the judge's reasoning in
- * `reason` so failures are debuggable.
- *
- * Design: the judge runs as a one-shot anonymous agent (no recursion
- * concern — default `remembers()` is `false`). Output is shaped via
- * `Output.object({ schema })` for deterministic parsing. Failures
- * (network, parse, unhandled judge error) bubble as `pass: false`
- * with the error in `reason` — a broken judge is not a passing case.
- *
- * Pitfall: the judge model has the same biases as any LLM. Use it
- * for fuzzy "did the answer mention X?" assertions; for exact
- * structural checks prefer `jsonShape` (Phase 3) or `regex`.
- */
-export declare function llmJudge(criterion: string, opts?: {
-    model?: string;
-}): Metric;
-/**
- * Strict structural assertion: parse `response.text` as JSON
- * (stripping ```json fences) and run it through a zod schema.
- *
- * Pairs naturally with `Output.object({ schema })` on the agent —
- * if the agent declares the same schema, this metric verifies the
- * output actually conforms. Failures surface the zod issue path
- * (e.g. `customer.email`) so debugging doesn't require a separate
- * console log.
- */
-export declare function jsonShape<T>(schema: z.ZodType<T>): Metric;
-/**
- * Embedding-based fuzzy match. Embeds both `reference` and
- * `response.text` via `AI.embed()`, computes cosine similarity,
- * passes when >= `threshold` (default `0.85` — tighter than
- * `EmbeddingUserMemory`'s 0.5 retrieval-rank floor since this is
- * an assertion, not a ranking).
- *
- * Uses ≤ 2 embedding calls per case; embed tokens roll into the
- * case's cost rollup via the same side-channel `llmJudge` uses.
- *
- * Pitfall: requires a provider that implements `createEmbedding()`
- * (openai / google / mistral / cohere / jina). Failures (no
- * provider, network, etc.) surface as `pass: false` with the
- * error in `reason` — a broken embed is not a passing case.
- */
-export declare function semanticMatch(reference: string, opts?: {
-    threshold?: number;
-    model?: string;
-}): Metric;
-/**
- * Token budget guard. Passes when `response.usage.totalTokens
- * <= threshold`. Pair with cost-conscious agents to detect prompt-
- * size regressions before they show up as a billing surprise.
- *
- * `response.usage` is the multi-step rollup, so it's meaningful
- * even when the agent runs tools across several provider calls.
- */
-export declare function tokenCost(threshold: number): Metric;
-/**
- * Compose multiple metrics into one assertion. Runs them in order
- * and short-circuits on the first failure — failure `reason` is
- * surfaced; success returns `{ pass: true, score: 1 }`.
- *
- * @example
- *   { input: '…',
- *     assert: compose(
- *       jsonShape(SummarySchema),
- *       tokenCost(800),
- *     ),
- *   }
- */
-export declare function compose(...metrics: Metric[]): Metric;
-/**
- * Run every case in the suite, in declaration order. Returns the
- * full report; never throws (assertion errors become `failed` cases,
- * not exceptions).
- *
- * Phase 1 runs serially. Parallel execution lands in a follow-up
- * once we understand the rate-limit shape of real-world judge
- * models — sequential is correct under any rate limit.
- */
-export declare function runSuite(suite: EvalSuite): Promise<SuiteReport>;
-export { estimateCost, ModelPricing } from '../budget/pricing.js';
-export type { ModelPriceEntry } from '../budget/pricing.js';
-/**
- * Default reporter — prints a colorless ANSI-aware table to a
- * caller-supplied `console`-like sink. Uses Unicode pass/fail glyphs
- * for visual scanning. JSON / HTML reporters land in Phase 2 / 5.
- *
- * Returns the report unchanged so chains compose: `await
- * reportConsole(await runSuite(suite))`.
- */
-export declare function reportConsole(report: SuiteReport, sink?: {
-    log: (s: string) => void;
-}): SuiteReport;
+export * from '@gemstack/ai-sdk/eval';
 //# sourceMappingURL=index.d.ts.map

package/dist/eval/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAGH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAKhD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAA;AAC/C,YAAY,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAClE,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAA;AACjD,YAAY,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAA;AAC/C,YAAY,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AAI3D;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAK,OAAO,CAAA;IAChB,KAAK,CAAC,EAAG,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED;;;;;GAKG;AACH,MAAM,MAAM,MAAM,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,GAAG,EAAE,aAAa,KAAK,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;AAE1G,MAAM,WAAW,aAAa;IAC5B,mEAAmE;IACnE,KAAK,EAAK,MAAM,CAAA;IAChB,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,0BAA0B;AAC1B,MAAM,WAAW,QAAQ;IACvB,qEAAqE;IACrE,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAA;IACb,wDAAwD;IACxD,MAAM,EAAE,MAAM,CAAA;IACd;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,KAAK,CAAA;IACnB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;;OAGG;IACH,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,CAAA;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,8DAA8D;IAC9D,KAAK,EAAE,MAAM,KAAK,CAAA;IAClB,wBAAwB;IACxB,KAAK,EAAE,QAAQ,EAAE,CAAA;IACjB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,YAAY,CAAA;CACxB;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,CAAC,EAAS,MAAM,CAAA;IACrB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,MAAM,CAAC,EAAQ,MAAM,CAAA;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAA;CAClC;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,aAAa,CAAA;CACpB;AAED,yDAAyD;AACzD,MAAM,WAAW,UAAU;IACzB,IAAI,EAAK,MAAM,CAAA;IACf,wDAAwD;IACxD,MAAM,EAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAA;IACxC,MAAM,CAAC,EAAE,YAAY,CAAA;IACrB,iDAAiD;IACjD,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,oDAAoD;IACpD,QAAQ,EAAE,MAAM,CAAA;IAChB;;;;OAIG;IACH,MAAM,EAAG,MAAM,CAAA;IACf,6DAA6D;IAC7D,IAAI,EAAK,MAAM,CAAA;IACf;;;;;OAKG;IACH,KAAK,EAAI,MAAM,CAAA;IACf;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED,gDAAgD;AAChD,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAK,MAAM,CAAA;IAChB,KAAK,EAAK,UAAU,EAAE,CAAA;IACtB,MAAM,EAAI,MAAM,CAAA;IAChB,MAAM,EAAI,MAAM,CAAA;IAChB,OAAO,EAAG,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,IAAI,EAAM,MAAM,CAAA;IAChB,MAAM,EAAI,MAAM,CAAA;IAChB,wEAAwE;IACxE,QAAQ,CAAC,EAAE,YAAY,CAAA;CACxB;AAID;;;;;;;;;GASG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,GAAG,SAAS,CAStE;AAID,qDAAqD;AACrD,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAUnD;AAED,6CAA6C;AAC7C,wBAAgB,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAS7C;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,MAAM,CA6CjF;AAQD;;;;;;;;;GASG;AACH,wBAAgB,SAAS,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,CAuBzD;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,aAAa,CAC3B,SAAS,EAAE,MAAM,EACjB,IAAI,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAChD,MAAM,CA+BR;AAED;;;;;;;GAOG;AACH,wBAAgB,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAYnD;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,OAAO,CAAC,GAAG,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,CAQpD;AA4BD;;;;;;;;GAQG;AACH,wBAAsB,QAAQ,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,CAgDrE;AA0FD,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAA;AACjE,YAAY,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAI3D;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,WAAW,EAAE,IAAI,GAAE;IAAE,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,IAAI,CAAA;CAAY,GAAG,WAAW,CAyB5G"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAGA,cAAc,uBAAuB,CAAA"}