@iris-eval/mcp-server 0.3.1 → 0.4.0

package/dist/eval/citation-verify/verifier.js

@@ -0,0 +1,203 @@
+import { callLLMJudge, LLMJudgeError } from '../llm-judge/client.js';
+import { estimateCostUsd, findPricing } from '../llm-judge/pricing.js';
+import { extractCitations } from './extract.js';
+import { resolveSource } from './resolve.js';
+const SYSTEM = `You are a citation verification evaluator. Given a claim extracted from AI-generated output and the text of a cited source, decide whether the source supports the claim.
+
+Score 0.00 means the source contradicts the claim or does not mention it.
+Score 1.00 means the source clearly supports the claim.
+Be strict: do not rate as supported unless the source actually contains the assertion. Do NOT penalize paraphrasing; DO penalize invented specifics not in the source.
+
+Respond with a single JSON object — no markdown, no prose:
+{
+  "supported": <boolean>,
+  "confidence": <number 0.00..1.00>,
+  "rationale": "<1-2 sentences — quote 5-15 words from the source if you found support>"
+}`;
+function buildUser(claim, sourceText) {
+    // Truncate huge sources so we stay within reasonable tokens.
+    const maxSourceChars = 12_000; // ~3k tokens
+    const trimmed = sourceText.length > maxSourceChars
+        ? sourceText.slice(0, maxSourceChars) + '\n\n[…source truncated…]'
+        : sourceText;
+    return `CLAIM:\n${claim}\n\nSOURCE TEXT:\n${trimmed}`;
+}
+function parseJudgeResult(raw) {
+    const trimmed = raw
+        .trim()
+        .replace(/^```json\s*/i, '')
+        .replace(/^```\s*/i, '')
+        .replace(/```$/i, '')
+        .trim();
+    const first = trimmed.indexOf('{');
+    const last = trimmed.lastIndexOf('}');
+    if (first < 0 || last <= first) {
+        throw new LLMJudgeError(`Citation judge did not emit JSON: ${raw.slice(0, 200)}`, 'malformed_response');
+    }
+    const obj = JSON.parse(trimmed.slice(first, last + 1));
+    const supported = obj.supported === true;
+    const confRaw = obj.confidence;
+    const confidence = typeof confRaw === 'number' ? confRaw : Number(confRaw);
+    if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1) {
+        throw new LLMJudgeError(`Citation judge confidence out of [0..1]: ${String(confRaw)}`, 'malformed_response');
+    }
+    const rationale = typeof obj.rationale === 'string' ? obj.rationale : '';
+    return { supported, confidence: Math.round(confidence * 100) / 100, rationale };
+}
+export async function verifyCitations(params) {
+    const citations = extractCitations(params.output);
+    const maxCitations = params.maxCitations ?? 20;
+    const selected = citations.slice(0, maxCitations);
+    const totalFound = citations.length;
+    if (!findPricing(params.model)) {
+        throw new Error(`Unknown model "${params.model}". Add pricing to src/eval/llm-judge/pricing.ts first.`);
+    }
+    const maxCostTotal = params.maxCostUsdTotal ?? 1.0;
+    const out = [];
+    let totalCost = 0;
+    let totalResolved = 0;
+    let totalSupported = 0;
+    for (const citation of selected) {
+        // Only URL/DOI can be resolved. Numbered citations without
+        // footnote definitions + author-year refs without a bibliography
+        // are flagged as unresolvable — the output claims them but we have
+        // nothing to compare against.
+        if (citation.kind !== 'url' && citation.kind !== 'doi') {
+            out.push({ citation, resolveStatus: 'skipped', resolveError: { kind: 'unresolvable_kind', message: `No source to fetch for ${citation.kind} citation` } });
+            continue;
+        }
+        let source;
+        try {
+            source = await resolveSource(citation.identifier, {
+                allowFetch: params.allowFetch,
+                timeoutMs: params.perSourceTimeoutMs,
+                maxBytes: params.perSourceMaxBytes,
+                domainAllowlist: params.domainAllowlist,
+            });
+            totalResolved++;
+        }
+        catch (err) {
+            const e = err;
+            out.push({
+                citation,
+                resolveStatus: 'error',
+                resolveError: { kind: e.kind ?? 'unknown', message: e.message },
+            });
+            continue;
+        }
+        // Before calling the judge: would this blow our total cost?
+        // Use the same pessimistic estimate as the main LLM judge evaluator.
+        const contextLen = citation.contextWindow.length + source.text.length;
+        const pessimistic = estimateCostUsd(params.model, Math.ceil(contextLen / 4), 512) ?? 0;
+        if (totalCost + pessimistic > maxCostTotal) {
+            out.push({
+                citation,
+                resolveStatus: 'ok',
+                source: {
+                    url: source.url,
+                    status: source.status,
+                    contentType: source.contentType,
+                    bytesFetched: source.bytesFetched,
+                    truncated: source.truncated,
+                },
+                resolveError: {
+                    kind: 'cost_cap_reached',
+                    message: `Total cost cap $${maxCostTotal.toFixed(2)} would be exceeded by next judge call`,
+                },
+            });
+            break; // No point continuing — subsequent calls will also exceed.
+        }
+        let judgeResponse;
+        try {
+            judgeResponse = await callLLMJudge({
+                provider: params.provider,
+                model: params.model,
+                systemPrompt: SYSTEM,
+                userPrompt: buildUser(citation.contextWindow, source.text),
+                maxOutputTokens: 256,
+                temperature: 0,
+                apiKey: params.apiKey,
+            });
+        }
+        catch (err) {
+            const e = err;
+            out.push({
+                citation,
+                resolveStatus: 'ok',
+                source: {
+                    url: source.url,
+                    status: source.status,
+                    contentType: source.contentType,
+                    bytesFetched: source.bytesFetched,
+                    truncated: source.truncated,
+                },
+                resolveError: {
+                    kind: err instanceof LLMJudgeError ? err.kind : 'llm_judge_error',
+                    message: e.message,
+                },
+            });
+            continue;
+        }
+        const cost = estimateCostUsd(params.model, judgeResponse.inputTokens, judgeResponse.outputTokens);
+        totalCost += cost ?? 0;
+        let parsed;
+        try {
+            parsed = parseJudgeResult(judgeResponse.content);
+        }
+        catch (err) {
+            const e = err;
+            out.push({
+                citation,
+                resolveStatus: 'ok',
+                source: {
+                    url: source.url,
+                    status: source.status,
+                    contentType: source.contentType,
+                    bytesFetched: source.bytesFetched,
+                    truncated: source.truncated,
+                },
+                resolveError: {
+                    kind: 'malformed_judge_response',
+                    message: e.message,
+                },
+            });
+            continue;
+        }
+        if (parsed.supported)
+            totalSupported++;
+        out.push({
+            citation,
+            resolveStatus: 'ok',
+            source: {
+                url: source.url,
+                status: source.status,
+                contentType: source.contentType,
+                bytesFetched: source.bytesFetched,
+                truncated: source.truncated,
+            },
+            judge: {
+                supported: parsed.supported,
+                confidence: parsed.confidence,
+                rationale: parsed.rationale,
+                inputTokens: judgeResponse.inputTokens,
+                outputTokens: judgeResponse.outputTokens,
+                costUsd: cost,
+                latencyMs: judgeResponse.latencyMs,
+            },
+        });
+    }
+    const overallScore = totalResolved > 0 ? Math.round((totalSupported / totalResolved) * 100) / 100 : null;
+    // Fail if >= 50% of resolved sources don't support the claim. When
+    // no citations or none resolved, we don't fail — there's nothing to
+    // score, we just report that.
+    const passed = overallScore === null ? true : overallScore >= 0.5;
+    return {
+        overallScore,
+        passed,
+        citations: out,
+        totalCostUsd: Math.round(totalCost * 1_000_000) / 1_000_000,
+        totalCitationsFound: totalFound,
+        totalResolved,
+        totalSupported,
+    };
+}

package/dist/eval/decision-moment.d.ts

@@ -0,0 +1,12 @@
+import type { Trace } from '../types/trace.js';
+import type { EvalResult } from '../types/eval.js';
+import type { DecisionMoment, DecisionMomentDetail } from '../types/decision-moment.js';
+export declare function deriveMoment(trace: Trace, evals: EvalResult[]): DecisionMoment;
+export declare function deriveMomentDetail(trace: Trace, evals: EvalResult[], spans?: Array<{
+    span_id: string;
+    parent_span_id?: string;
+    name: string;
+    kind: string;
+    start_time: string;
+    end_time?: string;
+}>): DecisionMomentDetail;

package/dist/eval/decision-moment.js

@@ -0,0 +1,181 @@
+/*
+ * decision-moment — derive Decision Moments from trace + eval data.
+ *
+ * A Decision Moment aggregates one trace and all eval results recorded for
+ * it. The significance classifier runs over the aggregated view to decide
+ * whether this trace is moment-worthy (safety violation, cost spike, etc.)
+ * or normal operational data.
+ *
+ * The classifier is intentionally simple in v0.4: rules-based, deterministic,
+ * no learned baselines. The "first-failure" and "novel-pattern" classes need
+ * agent-history context that we add in v0.4.1 — for now they fall through to
+ * the simpler categories.
+ */
+/* Cost-spike threshold in USD per single trace. Crossing this triggers
+ * cost-spike classification regardless of agent baseline. The bound was
+ * picked to flag any single trace that costs more than a typical
+ * developer-tier monthly budget would absorb at scale (1000 traces/day). */
+const COST_SPIKE_USD_THRESHOLD = 0.10;
+/* Rule names that, if failed, escalate the moment to safety-violation
+ * regardless of the rest of the verdict. Keeps in sync with v0.3.1's
+ * safety category. */
+const SAFETY_RULE_NAMES = new Set([
+    'no_pii',
+    'no_blocklist_words',
+    'no_injection_patterns',
+    'no_stub_output',
+]);
+export function deriveMoment(trace, evals) {
+    const ruleSnapshot = computeRuleSnapshot(evals);
+    const verdict = computeVerdict(evals, ruleSnapshot);
+    const overallScore = computeOverallScore(evals);
+    const significance = classifySignificance({
+        trace,
+        evals,
+        ruleSnapshot,
+        verdict,
+    });
+    return {
+        id: trace.trace_id,
+        traceId: trace.trace_id,
+        agentName: trace.agent_name,
+        timestamp: trace.timestamp,
+        input: trace.input,
+        output: trace.output,
+        costUsd: trace.cost_usd,
+        latencyMs: trace.latency_ms,
+        verdict,
+        overallScore,
+        evalCount: evals.length,
+        ruleSnapshot,
+        significance,
+    };
+}
+export function deriveMomentDetail(trace, evals, spans) {
+    const moment = deriveMoment(trace, evals);
+    return {
+        ...moment,
+        evals: evals.map((e) => ({
+            id: e.id,
+            evalType: e.eval_type,
+            score: e.score,
+            passed: e.passed,
+            ruleResults: e.rule_results.map((r) => ({
+                ruleName: r.ruleName,
+                passed: r.passed,
+                score: r.score,
+                message: r.message,
+                skipped: r.skipped,
+                skipReason: r.skipReason,
+            })),
+            suggestions: e.suggestions ?? [],
+            createdAt: e.created_at,
+        })),
+        toolCalls: trace.tool_calls,
+        spans,
+    };
+}
+function computeRuleSnapshot(evals) {
+    const failed = [];
+    const skipped = [];
+    let passedCount = 0;
+    let totalCount = 0;
+    for (const e of evals) {
+        for (const r of e.rule_results) {
+            totalCount++;
+            if (r.skipped) {
+                skipped.push(r.ruleName);
+            }
+            else if (r.passed) {
+                passedCount++;
+            }
+            else {
+                failed.push(r.ruleName);
+            }
+        }
+    }
+    return { failed, skipped, passedCount, totalCount };
+}
+function computeVerdict(evals, snapshot) {
+    if (evals.length === 0)
+        return 'unevaluated';
+    if (snapshot.totalCount - snapshot.skipped.length === 0)
+        return 'unevaluated';
+    if (snapshot.failed.length === 0)
+        return 'pass';
+    if (snapshot.passedCount === 0)
+        return 'fail';
+    return 'partial';
+}
+function computeOverallScore(evals) {
+    if (evals.length === 0)
+        return 0;
+    const sum = evals.reduce((a, e) => a + e.score, 0);
+    return sum / evals.length;
+}
+function classifySignificance({ trace, evals, ruleSnapshot, verdict, }) {
+    // 1. Safety violation — any safety rule failed → top priority.
+    const safetyFailed = ruleSnapshot.failed.filter((name) => SAFETY_RULE_NAMES.has(name));
+    if (safetyFailed.length > 0) {
+        return {
+            kind: 'safety-violation',
+            score: 1.0,
+            label: `Safety: ${safetyFailed.join(', ')}`,
+            reason: `${safetyFailed.length} safety rule(s) failed: ${safetyFailed.join(', ')}. Output may contain PII, prompt injection compliance, blocklisted content, or stub markers — review before this pattern becomes load-bearing.`,
+        };
+    }
+    // 2. Cost spike — trace cost over absolute threshold.
+    if (trace.cost_usd !== undefined && trace.cost_usd >= COST_SPIKE_USD_THRESHOLD) {
+        return {
+            kind: 'cost-spike',
+            score: 0.9,
+            label: `Cost: $${trace.cost_usd.toFixed(4)}`,
+            reason: `Trace cost ($${trace.cost_usd.toFixed(4)}) crossed the $${COST_SPIKE_USD_THRESHOLD} per-trace threshold. Investigate prompt size, token efficiency, or model-tier choice.`,
+        };
+    }
+    // 3. Rule collision — failures spanning multiple eval_types simultaneously.
+    if (ruleSnapshot.failed.length > 0) {
+        const failedEvalTypes = new Set(evals.filter((e) => !e.passed).map((e) => e.eval_type));
+        if (failedEvalTypes.size >= 2) {
+            return {
+                kind: 'rule-collision',
+                score: 0.7,
+                label: `Multi-category fail (${failedEvalTypes.size})`,
+                reason: `Failures across ${failedEvalTypes.size} eval categories: ${[...failedEvalTypes].join(', ')}. Failed rules: ${ruleSnapshot.failed.join(', ')}.`,
+            };
+        }
+    }
+    // 4. Generic fail.
+    if (verdict === 'fail') {
+        return {
+            kind: 'normal-fail',
+            score: 0.5,
+            label: `Fail: ${ruleSnapshot.failed.join(', ')}`,
+            reason: `Eval verdict fail. Failed rules: ${ruleSnapshot.failed.join(', ')}.`,
+        };
+    }
+    if (verdict === 'partial') {
+        return {
+            kind: 'normal-fail',
+            score: 0.4,
+            label: `Partial: ${ruleSnapshot.failed.length} failed`,
+            reason: `Partial fail — ${ruleSnapshot.failed.length} of ${ruleSnapshot.totalCount - ruleSnapshot.skipped.length} fired rules failed: ${ruleSnapshot.failed.join(', ')}.`,
+        };
+    }
+    // 5. Unevaluated trace (no eval recorded).
+    if (verdict === 'unevaluated') {
+        return {
+            kind: 'normal-pass',
+            score: 0.1,
+            label: 'No eval recorded',
+            reason: 'No eval was recorded for this trace. The agent ran but no rules fired.',
+        };
+    }
+    // 6. Happy path — clean pass.
+    return {
+        kind: 'normal-pass',
+        score: 0.05,
+        label: 'Pass',
+        reason: `All ${ruleSnapshot.passedCount} fired rules passed.`,
+    };
+}

package/dist/eval/llm-judge/client.d.ts

@@ -0,0 +1,28 @@
+export type LLMProvider = 'anthropic' | 'openai';
+export interface LLMJudgeRequest {
+    provider: LLMProvider;
+    model: string;
+    systemPrompt: string;
+    userPrompt: string;
+    maxOutputTokens: number;
+    temperature: number;
+    apiKey: string;
+    timeoutMs?: number;
+    maxInputTokensEstimate?: number;
+}
+export interface LLMJudgeResponse {
+    content: string;
+    inputTokens: number;
+    outputTokens: number;
+    stopReason: string;
+    latencyMs: number;
+    rawProviderResponseId?: string;
+}
+export declare class LLMJudgeError extends Error {
+    readonly kind: 'auth' | 'rate_limit' | 'bad_request' | 'server_error' | 'timeout' | 'malformed_response' | 'unknown';
+    readonly statusCode?: number | undefined;
+    readonly retryAfterSeconds?: number | undefined;
+    constructor(message: string, kind: 'auth' | 'rate_limit' | 'bad_request' | 'server_error' | 'timeout' | 'malformed_response' | 'unknown', statusCode?: number | undefined, retryAfterSeconds?: number | undefined);
+}
+export declare function estimateInputTokens(systemPrompt: string, userPrompt: string): number;
+export declare function callLLMJudge(req: LLMJudgeRequest): Promise<LLMJudgeResponse>;

package/dist/eval/llm-judge/client.js

@@ -0,0 +1,183 @@
+// LLM client abstraction for the judge eval path. Two providers are
+// implemented (Anthropic + OpenAI); both are thin wrappers around the
+// respective completion endpoints. Iris never imports the vendor SDKs at
+// runtime — the wire format is simple enough that a fetch() call + a
+// small amount of response parsing keeps the dependency surface minimal
+// and the supply-chain footprint smaller.
+//
+// Guarantees:
+//   - AbortSignal-respecting timeouts
+//   - Retry on 429 (single retry w/ RateLimit-* header or Retry-After)
+//   - Structured error types (LLMJudgeError) — callers distinguish a
+//     transient rate-limit from a permanent auth failure
+//   - Usage fields (input_tokens / output_tokens) surfaced so the
+//     evaluator can compute cost without a second API call
+export class LLMJudgeError extends Error {
+    kind;
+    statusCode;
+    retryAfterSeconds;
+    constructor(message, kind, statusCode, retryAfterSeconds) {
+        super(message);
+        this.kind = kind;
+        this.statusCode = statusCode;
+        this.retryAfterSeconds = retryAfterSeconds;
+        this.name = 'LLMJudgeError';
+    }
+}
+// Very rough token estimate — good enough to reject obviously-too-big
+// prompts before round-tripping. Real tokenization differs per model;
+// both Anthropic and OpenAI hover near 4 chars/token for English text.
+export function estimateInputTokens(systemPrompt, userPrompt) {
+    const chars = systemPrompt.length + userPrompt.length;
+    return Math.ceil(chars / 4);
+}
+async function fetchWithTimeout(url, init, timeoutMs) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        return await fetch(url, { ...init, signal: controller.signal });
+    }
+    catch (err) {
+        if (err instanceof Error && err.name === 'AbortError') {
+            throw new LLMJudgeError(`Request timed out after ${timeoutMs}ms`, 'timeout');
+        }
+        throw err;
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+function parseRetryAfter(headers) {
+    const raw = headers.get('retry-after') ?? headers.get('anthropic-ratelimit-requests-reset');
+    if (!raw)
+        return undefined;
+    const asSeconds = Number(raw);
+    if (Number.isFinite(asSeconds) && asSeconds > 0)
+        return asSeconds;
+    const asDate = Date.parse(raw);
+    if (Number.isFinite(asDate)) {
+        return Math.max(1, Math.ceil((asDate - Date.now()) / 1000));
+    }
+    return undefined;
+}
+async function callAnthropic(req) {
+    const timeoutMs = req.timeoutMs ?? 60_000;
+    const started = Date.now();
+    const res = await fetchWithTimeout('https://api.anthropic.com/v1/messages', {
+        method: 'POST',
+        headers: {
+            'content-type': 'application/json',
+            'x-api-key': req.apiKey,
+            'anthropic-version': '2023-06-01',
+        },
+        body: JSON.stringify({
+            model: req.model,
+            max_tokens: req.maxOutputTokens,
+            temperature: req.temperature,
+            system: req.systemPrompt,
+            messages: [{ role: 'user', content: req.userPrompt }],
+        }),
+    }, timeoutMs);
+    const latencyMs = Date.now() - started;
+    if (!res.ok) {
+        const body = await res.text().catch(() => '');
+        const retryAfterSeconds = parseRetryAfter(res.headers);
+        if (res.status === 401 || res.status === 403) {
+            throw new LLMJudgeError(`Anthropic auth failed (${res.status}): ${body}`, 'auth', res.status);
+        }
+        if (res.status === 429) {
+            throw new LLMJudgeError(`Anthropic rate-limited (${res.status})`, 'rate_limit', res.status, retryAfterSeconds);
+        }
+        if (res.status >= 500) {
+            throw new LLMJudgeError(`Anthropic server error (${res.status}): ${body}`, 'server_error', res.status);
+        }
+        throw new LLMJudgeError(`Anthropic bad request (${res.status}): ${body}`, 'bad_request', res.status);
+    }
+    const json = (await res.json().catch(() => {
+        throw new LLMJudgeError('Anthropic response was not valid JSON', 'malformed_response');
+    }));
+    const firstText = (json.content ?? []).find((c) => c.type === 'text')?.text;
+    if (!firstText) {
+        throw new LLMJudgeError('Anthropic response had no text content block', 'malformed_response');
+    }
+    return {
+        content: firstText,
+        inputTokens: json.usage?.input_tokens ?? 0,
+        outputTokens: json.usage?.output_tokens ?? 0,
+        stopReason: json.stop_reason ?? 'unknown',
+        latencyMs,
+        rawProviderResponseId: json.id,
+    };
+}
+async function callOpenAI(req) {
+    const timeoutMs = req.timeoutMs ?? 60_000;
+    const started = Date.now();
+    const res = await fetchWithTimeout('https://api.openai.com/v1/chat/completions', {
+        method: 'POST',
+        headers: {
+            'content-type': 'application/json',
+            authorization: `Bearer ${req.apiKey}`,
+        },
+        body: JSON.stringify({
+            model: req.model,
+            max_tokens: req.maxOutputTokens,
+            temperature: req.temperature,
+            messages: [
+                { role: 'system', content: req.systemPrompt },
+                { role: 'user', content: req.userPrompt },
+            ],
+        }),
+    }, timeoutMs);
+    const latencyMs = Date.now() - started;
+    if (!res.ok) {
+        const body = await res.text().catch(() => '');
+        const retryAfterSeconds = parseRetryAfter(res.headers);
+        if (res.status === 401 || res.status === 403) {
+            throw new LLMJudgeError(`OpenAI auth failed (${res.status}): ${body}`, 'auth', res.status);
+        }
+        if (res.status === 429) {
+            throw new LLMJudgeError(`OpenAI rate-limited (${res.status})`, 'rate_limit', res.status, retryAfterSeconds);
+        }
+        if (res.status >= 500) {
+            throw new LLMJudgeError(`OpenAI server error (${res.status}): ${body}`, 'server_error', res.status);
+        }
+        throw new LLMJudgeError(`OpenAI bad request (${res.status}): ${body}`, 'bad_request', res.status);
+    }
+    const json = (await res.json().catch(() => {
+        throw new LLMJudgeError('OpenAI response was not valid JSON', 'malformed_response');
+    }));
+    const firstText = json.choices?.[0]?.message?.content;
+    if (!firstText) {
+        throw new LLMJudgeError('OpenAI response had no message content', 'malformed_response');
+    }
+    return {
+        content: firstText,
+        inputTokens: json.usage?.prompt_tokens ?? 0,
+        outputTokens: json.usage?.completion_tokens ?? 0,
+        stopReason: json.choices?.[0]?.finish_reason ?? 'unknown',
+        latencyMs,
+        rawProviderResponseId: json.id,
+    };
+}
+// Single-retry-on-429 wrapper. Reads Retry-After/RateLimit-Reset header
+// when present; otherwise sleeps 2s. Deliberately one retry — repeated
+// retries burn cost on what is usually a provisioning problem, not a
+// transient spike. Callers that want more aggressive retry can compose.
+export async function callLLMJudge(req) {
+    const estimatedInput = estimateInputTokens(req.systemPrompt, req.userPrompt);
+    if (req.maxInputTokensEstimate && estimatedInput > req.maxInputTokensEstimate) {
+        throw new LLMJudgeError(`Estimated input tokens (${estimatedInput}) exceed cap (${req.maxInputTokensEstimate}) — refusing to call`, 'bad_request');
+    }
+    const call = req.provider === 'anthropic' ? callAnthropic : callOpenAI;
+    try {
+        return await call(req);
+    }
+    catch (err) {
+        if (err instanceof LLMJudgeError && err.kind === 'rate_limit') {
+            const waitSeconds = err.retryAfterSeconds ?? 2;
+            await new Promise((r) => setTimeout(r, waitSeconds * 1000));
+            return await call(req);
+        }
+        throw err;
+    }
+}

package/dist/eval/llm-judge/evaluator.d.ts

@@ -0,0 +1,32 @@
+import { type LLMProvider } from './client.js';
+import { type TemplateName } from './templates/index.js';
+export interface LLMJudgeEvaluateParams {
+    output: string;
+    template: TemplateName;
+    provider: LLMProvider;
+    model: string;
+    apiKey: string;
+    input?: string;
+    expected?: string;
+    sourceMaterial?: string;
+    maxCostUsdPerEval?: number;
+    maxOutputTokens?: number;
+    timeoutMs?: number;
+    temperature?: number;
+    maxInputTokensEstimate?: number;
+}
+export interface LLMJudgeEvaluationResult {
+    passed: boolean;
+    score: number;
+    rationale: string;
+    dimensions: Record<string, number>;
+    model: string;
+    provider: LLMProvider;
+    template: TemplateName;
+    inputTokens: number;
+    outputTokens: number;
+    costUsd: number | null;
+    latencyMs: number;
+    rawResponseId?: string;
+}
+export declare function evaluateWithLLMJudge(params: LLMJudgeEvaluateParams): Promise<LLMJudgeEvaluationResult>;