@fabricorg/experiments-evals 0.1.0

package/dist/index.d.cts

@@ -0,0 +1,257 @@
+import { E as EvaluatorKind, M as ModelClient, a as EvalScore, C as ChatMessage } from './model-client-B7yk_i1s.cjs';
+export { b as EvalRun, c as EvalRunSpec, d as EvalRunStatus, e as EvalRunTarget, G as GenerationSignal, f as GenerationSignalItem, g as ModelCompletionOptions, h as ModelResponse, S as StaticModelClient, T as TokenUsage, m as maxCompletionTokenUsage } from './model-client-B7yk_i1s.cjs';
+import 'zod';
+
+/** What an evaluator sees: the example under test plus the produced output. */
+interface EvalInput {
+    input: unknown;
+    output: unknown;
+    expectedOutput?: unknown;
+    /** Retrieved context for RAG-style judges (documents, tool results). */
+    context?: unknown;
+    metadata?: Record<string, unknown>;
+}
+interface EvalContext {
+    model: ModelClient;
+    nowMs: () => number;
+}
+interface Evaluator {
+    readonly kind: EvaluatorKind;
+    readonly name: string;
+    readonly version: string;
+    /** Worst-case judge usage reserved before IO; absent for token-free evaluators. */
+    maxJudgeTokens?(input: EvalInput): number;
+    /**
+     * The judge's prompt template, when the evaluator is an LLM judge. Exposed
+     * so budget derivation (deriveReserveTokensPerCall) can size worst-case
+     * per-call token reservations from the actual prompt text.
+     */
+    readonly promptTemplate?: Pick<JudgeTemplate, 'system' | 'user'>;
+    evaluate(input: EvalInput, ctx: EvalContext): Promise<EvalScore>;
+}
+interface JudgeTemplate {
+    name: string;
+    version: string;
+    /** Labels the judge may answer with, mapped to normalized scores. */
+    labels: Record<string, number>;
+    system: string;
+    /** Rendered with {{input}}, {{output}}, {{expectedOutput}}, {{context}}. */
+    user: string;
+    /** Deterministic provider-enforced response ceiling. Default: 256. */
+    maxOutputTokens?: number;
+}
+/**
+ * Build an LLM-judge evaluator from a template. Response contract: the model
+ * answers `LABEL: <label>` on the first line, free-form explanation after —
+ * a portable explanation-bearing format that remains easy to import/export.
+ */
+declare function llmJudge(template: JudgeTemplate): Evaluator;
+/** Build a deterministic code evaluator from a pure function. */
+declare function codeEvaluator(name: string, version: string, fn: (input: EvalInput) => {
+    label?: string;
+    score: number | null;
+    explanation?: string;
+}): Evaluator;
+declare function render(template: string, input: EvalInput): string;
+declare function parseJudgeResponse(text: string, labels: Record<string, number>): {
+    label?: string;
+    score: number | null;
+    explanation?: string;
+};
+
+/**
+ * Built-in judge library. Prompts use a portable binary-label + explanation
+ * contract and are versioned independently — bump the version on ANY prompt
+ * change because evaluator provenance depends on it.
+ */
+declare const hallucinationJudge: Evaluator;
+declare const qaCorrectnessJudge: Evaluator;
+declare const relevanceJudge: Evaluator;
+declare const toxicityJudge: Evaluator;
+declare const summarizationJudge: Evaluator;
+/** Rendered in place of the reference answer when the example carries none. */
+declare const NO_REFERENCE_MARKER = "(no reference provided)";
+/**
+ * Reference-grounded correctness. Examples without an expectedOutput render
+ * the explicit no-reference marker and come back unscored (null score) rather
+ * than being silently graded referenceless.
+ */
+declare const qaCorrectnessReferenceJudge: Evaluator;
+declare const trajectoryCoherenceJudge: Evaluator;
+declare const BUILT_IN_EVALUATORS: readonly Evaluator[];
+declare function getEvaluator(name: string): Evaluator | undefined;
+
+interface EvaluatorAggregate {
+    evaluatorName: string;
+    evaluatorVersion: string;
+    scored: number;
+    /** Scores the judge could not produce (null score — parse failures etc.). */
+    unscored: number;
+    meanScore: number | null;
+    /** Wilson 95% CI over pass-rate; only for binary judges (all scores 0/1). */
+    passRate: {
+        rate: number;
+        low: number;
+        high: number;
+    } | null;
+    labelCounts: Record<string, number>;
+    totalTokens: number;
+    meanLatencyMs: number | null;
+}
+/** Pure aggregation over a run's scores, grouped by evaluator name+version. */
+declare function aggregateScores(scores: readonly EvalScore[]): EvaluatorAggregate[];
+
+interface RunItem {
+    /** Dataset exampleId or span id — opaque to the runner. */
+    itemId: string;
+    input: EvalInput;
+}
+interface ScoredItem {
+    itemId: string;
+    score: EvalScore;
+}
+interface ExecuteOptions {
+    evaluators: readonly Evaluator[];
+    items: readonly RunItem[];
+    ctx: EvalContext;
+    /**
+     * Hard token budget (governance: eval-cost-controls). Enforced by
+     * reservation: a judge call only starts if `spent + reserved +
+     * reserveTokensPerCall` still fits, so concurrent workers can never
+     * collectively overshoot by racing the check.
+     */
+    maxJudgeTokens?: number;
+    /**
+     * Optional minimum reservation per judge call. The runner always takes the
+     * larger of this value and the evaluator's input-specific worst-case bound.
+     * Code and heuristic evaluators reserve zero tokens and continue after a
+     * judge budget is exhausted.
+     */
+    reserveTokensPerCall?: number;
+    concurrency?: number;
+    /** Called after each item completes — the agent checkpoints here. */
+    onProgress?: (done: number, total: number, tokensSpent: number) => void;
+}
+interface ExecuteResult {
+    scores: ScoredItem[];
+    tokensSpent: number;
+    /** Items fully or partially skipped because the token budget was exhausted. */
+    skippedForBudget: number;
+    /** Individual judge evaluations skipped because their reservation did not fit. */
+    skippedEvaluationsForBudget: number;
+    /** Per-item evaluator failures (item kept; other evaluators still ran). */
+    failures: Array<{
+        itemId: string;
+        evaluatorName: string;
+        message: string;
+    }>;
+}
+/**
+ * Completion cap assumed per judge call when sizing reservations. Judges
+ * answer `LABEL: <label>` plus a brief explanation — 512 tokens is a generous
+ * ceiling for that contract.
+ */
+declare const JUDGE_COMPLETION_TOKEN_CAP = 512;
+/**
+ * Prompt-token allowance for the item content rendered into the template
+ * ({{input}}/{{output}}/{{expectedOutput}}/{{context}}). A worst-case bound
+ * for typical dataset examples; oversized examples only weaken the guarantee
+ * to "caps the number of calls started" (see reserveTokensPerCall docs).
+ */
+declare const ITEM_CONTENT_TOKEN_ALLOWANCE = 1024;
+/**
+ * Worst-case tokens one judge call may consume, derived from the largest
+ * prompt template among the run's evaluators (chars/4 heuristic) plus the
+ * item-content allowance and the completion cap. Threaded from run creation
+ * into ExecuteOptions.reserveTokensPerCall so the hard budget is enforced by
+ * reservation rather than after-the-fact accounting.
+ */
+declare function deriveReserveTokensPerCall(evaluators: readonly Evaluator[]): number;
+/**
+ * Pure orchestration: evaluate every item with every evaluator under a hard
+ * token budget, bounded concurrency, and per-evaluator error isolation. IO
+ * happens only through the injected ModelClient; persistence is the caller's
+ * job (the eval-runner agent writes to the warehouse between batches).
+ */
+declare function executeEvalRun(opts: ExecuteOptions): Promise<ExecuteResult>;
+
+/**
+ * Generation phase for playground-replay eval runs (review fix: replay must
+ * not call the model inside the API request lifecycle). The run executor —
+ * in-process drain and the Temporal eval-runner alike — renders a prompt
+ * version over dataset examples up front (pure, no IO), then this phase
+ * completes each rendered conversation through the ModelClient to produce
+ * the system-under-test `output` that the judges score.
+ *
+ * Budget discipline: generation tokens count INSIDE the run's shared token
+ * budget (review finding: "playground generation occurs outside its
+ * budget"). The same reservation scheme as `executeEvalRun` applies — a
+ * completion only starts if `spent + reserveTokensPerCall` still fits, so a
+ * tiny budget starves generation into skips instead of overshooting.
+ */
+/** One example to generate an output for: rendered messages + judge context. */
+interface GenerationItem {
+    itemId: string;
+    /** Prompt version rendered against the example's variables. */
+    messages: ChatMessage[];
+    /**
+     * Judge-facing input for the item; `output` is overwritten with the model
+     * completion when generation succeeds.
+     */
+    input: EvalInput;
+}
+/**
+ * Completion cap assumed per generation call when sizing reservations —
+ * generations are real task outputs, not judge verdicts, so the ceiling is
+ * higher than JUDGE_COMPLETION_TOKEN_CAP. A template that pins
+ * `params.maxTokens` overrides this.
+ */
+declare const GENERATION_COMPLETION_TOKEN_CAP = 1024;
+/**
+ * Worst-case tokens one generation call may consume: template size
+ * (chars/4 heuristic) + rendered-variable allowance + the completion cap
+ * (or the template's own maxTokens pin). Mirrors deriveReserveTokensPerCall.
+ */
+declare function deriveReserveTokensPerGeneration(templateChars: number, maxCompletionTokens?: number): number;
+interface GenerationPhaseOptions {
+    items: readonly GenerationItem[];
+    model: ModelClient;
+    /** Shared run budget still available; undefined = unbounded. */
+    maxTokens?: number;
+    reserveTokensPerCall: number;
+    /** Provider-enforced completion ceiling. Default: 1024. */
+    maxOutputTokens?: number;
+    /** Generation sampling temperature. Default: 0 for reproducible replays. */
+    temperature?: number;
+    /**
+     * Polled before each completion; return true to stop early (cancellation).
+     * Items not yet generated when the phase stops are neither generated nor
+     * counted as budget skips.
+     */
+    shouldStop?: () => boolean | Promise<boolean>;
+}
+interface GenerationPhaseResult {
+    /** Judgeable run items — each carries its model completion as `output`. */
+    items: RunItem[];
+    tokensSpent: number;
+    inputTokens: number;
+    outputTokens: number;
+    /** Items skipped because the shared budget could not cover a reservation. */
+    skippedForBudget: number;
+    /** Per-item generation failures (item excluded from judging). */
+    failures: Array<{
+        itemId: string;
+        message: string;
+    }>;
+    /** True when shouldStop() ended the phase early. */
+    stopped: boolean;
+    modelEndpoint: string;
+}
+/**
+ * Sequential on purpose: generation runs inside durable executors (Temporal
+ * strict determinism) and the per-run example cap is small (maxExamples ≤
+ * 200), so bounded concurrency buys little and costs replay determinism.
+ */
+declare function executeGenerationPhase(opts: GenerationPhaseOptions): Promise<GenerationPhaseResult>;
+
+export { BUILT_IN_EVALUATORS, ChatMessage, type EvalContext, type EvalInput, EvalScore, type Evaluator, type EvaluatorAggregate, EvaluatorKind, type ExecuteOptions, type ExecuteResult, GENERATION_COMPLETION_TOKEN_CAP, type GenerationItem, type GenerationPhaseOptions, type GenerationPhaseResult, ITEM_CONTENT_TOKEN_ALLOWANCE, JUDGE_COMPLETION_TOKEN_CAP, type JudgeTemplate, ModelClient, NO_REFERENCE_MARKER, type RunItem, type ScoredItem, aggregateScores, codeEvaluator, deriveReserveTokensPerCall, deriveReserveTokensPerGeneration, executeEvalRun, executeGenerationPhase, getEvaluator, hallucinationJudge, llmJudge, parseJudgeResponse, qaCorrectnessJudge, qaCorrectnessReferenceJudge, relevanceJudge, render, summarizationJudge, toxicityJudge, trajectoryCoherenceJudge };

package/dist/index.d.ts

@@ -0,0 +1,257 @@
+import { E as EvaluatorKind, M as ModelClient, a as EvalScore, C as ChatMessage } from './model-client-B7yk_i1s.js';
+export { b as EvalRun, c as EvalRunSpec, d as EvalRunStatus, e as EvalRunTarget, G as GenerationSignal, f as GenerationSignalItem, g as ModelCompletionOptions, h as ModelResponse, S as StaticModelClient, T as TokenUsage, m as maxCompletionTokenUsage } from './model-client-B7yk_i1s.js';
+import 'zod';
+
+/** What an evaluator sees: the example under test plus the produced output. */
+interface EvalInput {
+    input: unknown;
+    output: unknown;
+    expectedOutput?: unknown;
+    /** Retrieved context for RAG-style judges (documents, tool results). */
+    context?: unknown;
+    metadata?: Record<string, unknown>;
+}
+interface EvalContext {
+    model: ModelClient;
+    nowMs: () => number;
+}
+interface Evaluator {
+    readonly kind: EvaluatorKind;
+    readonly name: string;
+    readonly version: string;
+    /** Worst-case judge usage reserved before IO; absent for token-free evaluators. */
+    maxJudgeTokens?(input: EvalInput): number;
+    /**
+     * The judge's prompt template, when the evaluator is an LLM judge. Exposed
+     * so budget derivation (deriveReserveTokensPerCall) can size worst-case
+     * per-call token reservations from the actual prompt text.
+     */
+    readonly promptTemplate?: Pick<JudgeTemplate, 'system' | 'user'>;
+    evaluate(input: EvalInput, ctx: EvalContext): Promise<EvalScore>;
+}
+interface JudgeTemplate {
+    name: string;
+    version: string;
+    /** Labels the judge may answer with, mapped to normalized scores. */
+    labels: Record<string, number>;
+    system: string;
+    /** Rendered with {{input}}, {{output}}, {{expectedOutput}}, {{context}}. */
+    user: string;
+    /** Deterministic provider-enforced response ceiling. Default: 256. */
+    maxOutputTokens?: number;
+}
+/**
+ * Build an LLM-judge evaluator from a template. Response contract: the model
+ * answers `LABEL: <label>` on the first line, free-form explanation after —
+ * a portable explanation-bearing format that remains easy to import/export.
+ */
+declare function llmJudge(template: JudgeTemplate): Evaluator;
+/** Build a deterministic code evaluator from a pure function. */
+declare function codeEvaluator(name: string, version: string, fn: (input: EvalInput) => {
+    label?: string;
+    score: number | null;
+    explanation?: string;
+}): Evaluator;
+declare function render(template: string, input: EvalInput): string;
+declare function parseJudgeResponse(text: string, labels: Record<string, number>): {
+    label?: string;
+    score: number | null;
+    explanation?: string;
+};
+
+/**
+ * Built-in judge library. Prompts use a portable binary-label + explanation
+ * contract and are versioned independently — bump the version on ANY prompt
+ * change because evaluator provenance depends on it.
+ */
+declare const hallucinationJudge: Evaluator;
+declare const qaCorrectnessJudge: Evaluator;
+declare const relevanceJudge: Evaluator;
+declare const toxicityJudge: Evaluator;
+declare const summarizationJudge: Evaluator;
+/** Rendered in place of the reference answer when the example carries none. */
+declare const NO_REFERENCE_MARKER = "(no reference provided)";
+/**
+ * Reference-grounded correctness. Examples without an expectedOutput render
+ * the explicit no-reference marker and come back unscored (null score) rather
+ * than being silently graded referenceless.
+ */
+declare const qaCorrectnessReferenceJudge: Evaluator;
+declare const trajectoryCoherenceJudge: Evaluator;
+declare const BUILT_IN_EVALUATORS: readonly Evaluator[];
+declare function getEvaluator(name: string): Evaluator | undefined;
+
+interface EvaluatorAggregate {
+    evaluatorName: string;
+    evaluatorVersion: string;
+    scored: number;
+    /** Scores the judge could not produce (null score — parse failures etc.). */
+    unscored: number;
+    meanScore: number | null;
+    /** Wilson 95% CI over pass-rate; only for binary judges (all scores 0/1). */
+    passRate: {
+        rate: number;
+        low: number;
+        high: number;
+    } | null;
+    labelCounts: Record<string, number>;
+    totalTokens: number;
+    meanLatencyMs: number | null;
+}
+/** Pure aggregation over a run's scores, grouped by evaluator name+version. */
+declare function aggregateScores(scores: readonly EvalScore[]): EvaluatorAggregate[];
+
+interface RunItem {
+    /** Dataset exampleId or span id — opaque to the runner. */
+    itemId: string;
+    input: EvalInput;
+}
+interface ScoredItem {
+    itemId: string;
+    score: EvalScore;
+}
+interface ExecuteOptions {
+    evaluators: readonly Evaluator[];
+    items: readonly RunItem[];
+    ctx: EvalContext;
+    /**
+     * Hard token budget (governance: eval-cost-controls). Enforced by
+     * reservation: a judge call only starts if `spent + reserved +
+     * reserveTokensPerCall` still fits, so concurrent workers can never
+     * collectively overshoot by racing the check.
+     */
+    maxJudgeTokens?: number;
+    /**
+     * Optional minimum reservation per judge call. The runner always takes the
+     * larger of this value and the evaluator's input-specific worst-case bound.
+     * Code and heuristic evaluators reserve zero tokens and continue after a
+     * judge budget is exhausted.
+     */
+    reserveTokensPerCall?: number;
+    concurrency?: number;
+    /** Called after each item completes — the agent checkpoints here. */
+    onProgress?: (done: number, total: number, tokensSpent: number) => void;
+}
+interface ExecuteResult {
+    scores: ScoredItem[];
+    tokensSpent: number;
+    /** Items fully or partially skipped because the token budget was exhausted. */
+    skippedForBudget: number;
+    /** Individual judge evaluations skipped because their reservation did not fit. */
+    skippedEvaluationsForBudget: number;
+    /** Per-item evaluator failures (item kept; other evaluators still ran). */
+    failures: Array<{
+        itemId: string;
+        evaluatorName: string;
+        message: string;
+    }>;
+}
+/**
+ * Completion cap assumed per judge call when sizing reservations. Judges
+ * answer `LABEL: <label>` plus a brief explanation — 512 tokens is a generous
+ * ceiling for that contract.
+ */
+declare const JUDGE_COMPLETION_TOKEN_CAP = 512;
+/**
+ * Prompt-token allowance for the item content rendered into the template
+ * ({{input}}/{{output}}/{{expectedOutput}}/{{context}}). A worst-case bound
+ * for typical dataset examples; oversized examples only weaken the guarantee
+ * to "caps the number of calls started" (see reserveTokensPerCall docs).
+ */
+declare const ITEM_CONTENT_TOKEN_ALLOWANCE = 1024;
+/**
+ * Worst-case tokens one judge call may consume, derived from the largest
+ * prompt template among the run's evaluators (chars/4 heuristic) plus the
+ * item-content allowance and the completion cap. Threaded from run creation
+ * into ExecuteOptions.reserveTokensPerCall so the hard budget is enforced by
+ * reservation rather than after-the-fact accounting.
+ */
+declare function deriveReserveTokensPerCall(evaluators: readonly Evaluator[]): number;
+/**
+ * Pure orchestration: evaluate every item with every evaluator under a hard
+ * token budget, bounded concurrency, and per-evaluator error isolation. IO
+ * happens only through the injected ModelClient; persistence is the caller's
+ * job (the eval-runner agent writes to the warehouse between batches).
+ */
+declare function executeEvalRun(opts: ExecuteOptions): Promise<ExecuteResult>;
+
+/**
+ * Generation phase for playground-replay eval runs (review fix: replay must
+ * not call the model inside the API request lifecycle). The run executor —
+ * in-process drain and the Temporal eval-runner alike — renders a prompt
+ * version over dataset examples up front (pure, no IO), then this phase
+ * completes each rendered conversation through the ModelClient to produce
+ * the system-under-test `output` that the judges score.
+ *
+ * Budget discipline: generation tokens count INSIDE the run's shared token
+ * budget (review finding: "playground generation occurs outside its
+ * budget"). The same reservation scheme as `executeEvalRun` applies — a
+ * completion only starts if `spent + reserveTokensPerCall` still fits, so a
+ * tiny budget starves generation into skips instead of overshooting.
+ */
+/** One example to generate an output for: rendered messages + judge context. */
+interface GenerationItem {
+    itemId: string;
+    /** Prompt version rendered against the example's variables. */
+    messages: ChatMessage[];
+    /**
+     * Judge-facing input for the item; `output` is overwritten with the model
+     * completion when generation succeeds.
+     */
+    input: EvalInput;
+}
+/**
+ * Completion cap assumed per generation call when sizing reservations —
+ * generations are real task outputs, not judge verdicts, so the ceiling is
+ * higher than JUDGE_COMPLETION_TOKEN_CAP. A template that pins
+ * `params.maxTokens` overrides this.
+ */
+declare const GENERATION_COMPLETION_TOKEN_CAP = 1024;
+/**
+ * Worst-case tokens one generation call may consume: template size
+ * (chars/4 heuristic) + rendered-variable allowance + the completion cap
+ * (or the template's own maxTokens pin). Mirrors deriveReserveTokensPerCall.
+ */
+declare function deriveReserveTokensPerGeneration(templateChars: number, maxCompletionTokens?: number): number;
+interface GenerationPhaseOptions {
+    items: readonly GenerationItem[];
+    model: ModelClient;
+    /** Shared run budget still available; undefined = unbounded. */
+    maxTokens?: number;
+    reserveTokensPerCall: number;
+    /** Provider-enforced completion ceiling. Default: 1024. */
+    maxOutputTokens?: number;
+    /** Generation sampling temperature. Default: 0 for reproducible replays. */
+    temperature?: number;
+    /**
+     * Polled before each completion; return true to stop early (cancellation).
+     * Items not yet generated when the phase stops are neither generated nor
+     * counted as budget skips.
+     */
+    shouldStop?: () => boolean | Promise<boolean>;
+}
+interface GenerationPhaseResult {
+    /** Judgeable run items — each carries its model completion as `output`. */
+    items: RunItem[];
+    tokensSpent: number;
+    inputTokens: number;
+    outputTokens: number;
+    /** Items skipped because the shared budget could not cover a reservation. */
+    skippedForBudget: number;
+    /** Per-item generation failures (item excluded from judging). */
+    failures: Array<{
+        itemId: string;
+        message: string;
+    }>;
+    /** True when shouldStop() ended the phase early. */
+    stopped: boolean;
+    modelEndpoint: string;
+}
+/**
+ * Sequential on purpose: generation runs inside durable executors (Temporal
+ * strict determinism) and the per-run example cap is small (maxExamples ≤
+ * 200), so bounded concurrency buys little and costs replay determinism.
+ */
+declare function executeGenerationPhase(opts: GenerationPhaseOptions): Promise<GenerationPhaseResult>;
+
+export { BUILT_IN_EVALUATORS, ChatMessage, type EvalContext, type EvalInput, EvalScore, type Evaluator, type EvaluatorAggregate, EvaluatorKind, type ExecuteOptions, type ExecuteResult, GENERATION_COMPLETION_TOKEN_CAP, type GenerationItem, type GenerationPhaseOptions, type GenerationPhaseResult, ITEM_CONTENT_TOKEN_ALLOWANCE, JUDGE_COMPLETION_TOKEN_CAP, type JudgeTemplate, ModelClient, NO_REFERENCE_MARKER, type RunItem, type ScoredItem, aggregateScores, codeEvaluator, deriveReserveTokensPerCall, deriveReserveTokensPerGeneration, executeEvalRun, executeGenerationPhase, getEvaluator, hallucinationJudge, llmJudge, parseJudgeResponse, qaCorrectnessJudge, qaCorrectnessReferenceJudge, relevanceJudge, render, summarizationJudge, toxicityJudge, trajectoryCoherenceJudge };