@marinoscar/kvox-eval 0.0.0-bootstrap → 0.1.0-next.0

package/dist/ask/report.js

@@ -0,0 +1,170 @@
+"use strict";
+// =============================================================================
+// The Ask evaluation's report: per-category rollup + the two gates (#382)
+// =============================================================================
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.askEvalReportSchema = void 0;
+exports.buildAskEvalReport = buildAskEvalReport;
+exports.formatAskEvalReport = formatAskEvalReport;
+exports.formatAnswersMarkdown = formatAnswersMarkdown;
+const zod_1 = require("zod");
+const ratioSchema = zod_1.z.number().min(0).max(1);
+/**
+ * The Ask evaluation report: the per-category rollup, the overall numbers, the two gates
+ * (citation validity 1.000, zero sensitive leaks), the answer-match threshold and every
+ * question's score. The JSON written as `report.json`.
+ *
+ * @stability experimental
+ */
+exports.askEvalReportSchema = zod_1.z.object({
+    runAt: zod_1.z.string(),
+    model: zod_1.z.string().nullable(),
+    questions: zod_1.z.number().int(),
+    categories: zod_1.z.array(zod_1.z.object({
+        category: zod_1.z.string(),
+        count: zod_1.z.number().int(),
+        answerMatch: ratioSchema,
+        cited: ratioSchema,
+        entityRecall: ratioSchema,
+        meanSteps: zod_1.z.number(),
+        meanToolCalls: zod_1.z.number(),
+        meanPromptTokens: zod_1.z.number(),
+        meanCompletionTokens: zod_1.z.number(),
+        meanMs: zod_1.z.number().nullable(),
+    })),
+    overall: zod_1.z.object({
+        answerMatch: ratioSchema,
+        citationValidity: ratioSchema,
+        sensitiveLeaks: zod_1.z.number().int(),
+    }),
+    /** `citationValidity === 1` and `sensitiveLeaks === 0` — the two hard gates. */
+    gatesPassed: zod_1.z.boolean(),
+    /** `overall.answerMatch >= minMatch`. */
+    answerMatchPassed: zod_1.z.boolean(),
+    minMatch: ratioSchema,
+    perQuestion: zod_1.z.array(zod_1.z.object({
+        id: zod_1.z.string(),
+        category: zod_1.z.string(),
+        citationValidity: ratioSchema,
+        cited: zod_1.z.boolean(),
+        citeFromHit: zod_1.z.boolean(),
+        answerMatch: zod_1.z.boolean(),
+        entityRecall: ratioSchema,
+        sensitiveLeak: zod_1.z.boolean(),
+        steps: zod_1.z.number().int(),
+        toolCalls: zod_1.z.number().int(),
+        promptTokens: zod_1.z.number().int(),
+        completionTokens: zod_1.z.number().int(),
+        ms: zod_1.z.number().nullable(),
+        notes: zod_1.z.array(zod_1.z.string()),
+    })),
+});
+const mean = (values) => (values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length);
+/**
+ * Rolls the per-question scores of one run up into a report.
+ *
+ * @param scores - one score per answered question.
+ * @param meta - the model, the answer-match threshold and (for a test) the run time.
+ * @returns the report.
+ *
+ * @stability experimental
+ */
+function buildAskEvalReport(scores, meta) {
+    const byCategory = new Map();
+    for (const s of scores)
+        byCategory.set(s.category, [...(byCategory.get(s.category) ?? []), s]);
+    const categories = [...byCategory.entries()]
+        .sort(([a], [b]) => a.localeCompare(b))
+        .map(([category, rows]) => ({
+        category,
+        count: rows.length,
+        answerMatch: mean(rows.map((r) => (r.answerMatch ? 1 : 0))),
+        cited: mean(rows.map((r) => (r.cited ? 1 : 0))),
+        entityRecall: mean(rows.map((r) => r.entityRecall)),
+        meanSteps: mean(rows.map((r) => r.steps)),
+        meanToolCalls: mean(rows.map((r) => r.toolCalls)),
+        meanPromptTokens: mean(rows.map((r) => r.promptTokens)),
+        meanCompletionTokens: mean(rows.map((r) => r.completionTokens)),
+        meanMs: rows.every((r) => r.ms === null) ? null : mean(rows.map((r) => r.ms ?? 0)),
+    }));
+    const citationValidity = mean(scores.map((s) => s.citationValidity));
+    const sensitiveLeaks = scores.filter((s) => s.sensitiveLeak).length;
+    const answerMatch = mean(scores.map((s) => (s.answerMatch ? 1 : 0)));
+    const gatesPassed = citationValidity === 1 && sensitiveLeaks === 0;
+    return {
+        runAt: (meta.runAt ?? new Date()).toISOString(),
+        model: meta.model,
+        questions: scores.length,
+        categories,
+        overall: { answerMatch, citationValidity, sensitiveLeaks },
+        gatesPassed,
+        answerMatchPassed: answerMatch >= meta.minMatch,
+        minMatch: meta.minMatch,
+        perQuestion: scores.map((s) => ({
+            id: s.questionId,
+            category: s.category,
+            citationValidity: s.citationValidity,
+            cited: s.cited,
+            citeFromHit: s.citeFromHit,
+            answerMatch: s.answerMatch,
+            entityRecall: s.entityRecall,
+            sensitiveLeak: s.sensitiveLeak,
+            steps: s.steps,
+            toolCalls: s.toolCalls,
+            promptTokens: s.promptTokens,
+            completionTokens: s.completionTokens,
+            ms: s.ms,
+            notes: s.notes,
+        })),
+    };
+}
+const fmt = (v) => v.toFixed(3);
+const pad = (s, n) => String(s).padEnd(n);
+/**
+ * The report as the fixed-column table `ask:eval` prints, then the gates and the failing questions.
+ *
+ * @param r - the report.
+ * @returns the table.
+ *
+ * @stability experimental
+ */
+function formatAskEvalReport(r) {
+    const lines = [];
+    lines.push(`ask:eval  questions=${r.questions}  model=${r.model ?? '-'}  ${r.runAt}`);
+    lines.push(`${pad('CATEGORY', 14)}${pad('N', 4)}${pad('ANSWER', 8)}${pad('CITED', 8)}${pad('ENT.RECALL', 12)}${pad('STEPS', 8)}${pad('TOKENS(p/c)', 14)}MS`);
+    for (const c of r.categories) {
+        lines.push(`${pad(c.category, 14)}${pad(c.count, 4)}${pad(fmt(c.answerMatch), 8)}${pad(fmt(c.cited), 8)}${pad(fmt(c.entityRecall), 12)}${pad(c.meanSteps.toFixed(1), 8)}${pad(`${c.meanPromptTokens.toFixed(0)}/${c.meanCompletionTokens.toFixed(0)}`, 14)}${c.meanMs === null ? '-' : c.meanMs.toFixed(0)}`);
+    }
+    lines.push('');
+    lines.push(`OVERALL ANSWER MATCH   ${fmt(r.overall.answerMatch)}   threshold ≥ ${r.minMatch.toFixed(2)}   ${r.answerMatchPassed ? 'PASS' : 'FAIL'}`);
+    lines.push(`CITATION VALIDITY      ${fmt(r.overall.citationValidity)}   required = 1.000   ${r.overall.citationValidity === 1 ? 'PASS' : 'FAIL'}`);
+    lines.push(`SENSITIVE LEAKS        ${r.overall.sensitiveLeaks}   required = 0   ${r.overall.sensitiveLeaks === 0 ? 'PASS' : 'FAIL'}`);
+    const failing = r.perQuestion.filter((q) => !q.answerMatch || q.citationValidity < 1 || q.sensitiveLeak);
+    if (failing.length > 0) {
+        lines.push('');
+        lines.push('FAILING QUESTIONS:');
+        for (const q of failing) {
+            lines.push(`  ${q.id} (${q.category}): ${q.notes.join('; ') || (q.citationValidity < 1 ? 'invalid citation' : 'sensitive leak')}`);
+        }
+    }
+    return lines.join('\n');
+}
+/**
+ * `answers.md`: every question, its answer text and its per-question numbers — for reading, not for scripts.
+ *
+ * @stability experimental
+ */
+function formatAnswersMarkdown(report, answers) {
+    const lines = [`# Ask evaluation answers\n`, `Run at ${report.runAt}, model \`${report.model ?? '-'}\`.\n`];
+    for (const q of report.perQuestion) {
+        const a = answers.get(q.id);
+        lines.push(`## ${q.id} (${q.category})`);
+        if (a)
+            lines.push(`\n**Q:** ${a.question}\n\n**A:** ${a.content}\n`);
+        lines.push(`_citationValidity=${fmt(q.citationValidity)} cited=${q.cited} citeFromHit=${q.citeFromHit} answerMatch=${q.answerMatch} entityRecall=${fmt(q.entityRecall)} sensitiveLeak=${q.sensitiveLeak} steps=${q.steps} toolCalls=${q.toolCalls} tokens=${q.promptTokens}/${q.completionTokens}${q.ms !== null ? ` ms=${q.ms}` : ''}_`);
+        if (q.notes.length > 0)
+            lines.push(`\n> ${q.notes.join('\n> ')}`);
+        lines.push('');
+    }
+    return lines.join('\n');
+}

package/dist/ask/scorer.d.ts

@@ -0,0 +1,104 @@
+import type { AskCitation, AskMessageResponse } from '@marinoscar/kvox-contract/ask';
+import type { AskEvalQuestion } from './question-schema.js';
+/**
+ * What the scorer needs from the seeder's `idMap` — just the reverse evidence lookup.
+ *
+ * @stability experimental
+ */
+export type EvidenceSourceMap = ReadonlyMap<string, string>;
+/**
+ * One question's score: the gate numbers (citation validity, a sensitive leak), the answer
+ * and citation checks, and the turn's cost (steps, tool calls, tokens, time).
+ *
+ * @stability experimental
+ */
+export interface AskEvalScore {
+    questionId: string;
+    category: string;
+    /** Valid citations / all citation markers. `1` when there were none to check. */
+    citationValidity: number;
+    /** At least one valid citation (irrelevant for a `notFound` question, which is scored separately). */
+    cited: boolean;
+    /** Some valid evidence/document citation resolves, via `idMap`, to a listed `citeFrom` source. */
+    citeFromHit: boolean;
+    /** Every `answerAll` group matched, no `answerNone` string present, and the `notFound` rule (if any) held. */
+    answerMatch: boolean;
+    /** Expected entities cited (by citation `label`) / expected entity count. `1` when none were expected. */
+    entityRecall: number;
+    /** An `answerNone` string turned up in a captured prompt or in the final answer. THE LEAK GATE: must be `false` on every question. */
+    sensitiveLeak: boolean;
+    steps: number;
+    toolCalls: number;
+    promptTokens: number;
+    completionTokens: number;
+    /** Wall-clock time the caller measured for this turn, if it did. */
+    ms: number | null;
+    /** Human-readable reasons `answerMatch`/`citationValidity`/`citeFromHit` failed, for `answers.md`. */
+    notes: string[];
+}
+/**
+ * Valid citations / all markers. `1` (vacuously satisfied) with zero markers — a question that expects none.
+ *
+ * @stability experimental
+ */
+export declare function scoreCitationValidity(citations: readonly AskCitation[]): number;
+/**
+ * Whether the answer carries at least one valid citation.
+ *
+ * @param citations - the answer's citations.
+ * @returns `true` when one is valid.
+ *
+ * @stability experimental
+ */
+export declare function scoreCited(citations: readonly AskCitation[]): boolean;
+/**
+ * Whether a valid evidence citation resolves, through the seeder's map, to one
+ * of the question's `citeFrom` sources.
+ *
+ * @param citeFrom - the expected sources (`mNN-sNNN`, `mNN#note`).
+ * @param citations - the answer's citations.
+ * @param evidenceSource - evidence id to source, from the seeder.
+ * @returns `true` on a hit (vacuously `true` with no `citeFrom`).
+ *
+ * @stability experimental
+ */
+export declare function scoreCiteFromHit(citeFrom: readonly string[], citations: readonly AskCitation[], evidenceSource: EvidenceSourceMap): boolean;
+/**
+ * Expected entities cited (by citation label, case-insensitive exact) over the expected count.
+ *
+ * @param expectedEntities - the labels that must be cited.
+ * @param citations - the answer's citations.
+ * @returns the ratio (`1` when none were expected).
+ *
+ * @stability experimental
+ */
+export declare function scoreEntityRecall(expectedEntities: readonly string[], citations: readonly AskCitation[]): number;
+/**
+ * THE LEAK GATE: whether an `answerNone` string appears in the answer or in any prompt the
+ * provider received.
+ *
+ * @param answerNone - the strings that must never leave.
+ * @param content - the final answer.
+ * @param capturedPrompts - every request the provider received, serialized.
+ * @returns `true` on a leak.
+ *
+ * @stability experimental
+ */
+export declare function scoreSensitiveLeak(answerNone: readonly string[], content: string, capturedPrompts: readonly string[]): boolean;
+/**
+ * Every `answerAll` group matched, no `answerNone` present, and (for a `notFound` question) no valid citation plus a not-found phrasing.
+ *
+ * @stability experimental
+ */
+export declare function scoreAnswerMatch(expected: AskEvalQuestion['expected'], content: string, citations: readonly AskCitation[]): {
+    matched: boolean;
+    reasons: string[];
+};
+/**
+ * The full score for one question's answered turn.
+ *
+ * @stability experimental
+ */
+export declare function scoreAnswer(question: AskEvalQuestion, message: Pick<AskMessageResponse, 'content' | 'citations' | 'promptTokens' | 'completionTokens' | 'toolCalls'>, capturedPrompts: readonly string[], evidenceSource: EvidenceSourceMap, timing?: {
+    ms?: number;
+}): AskEvalScore;

package/dist/ask/scorer.js

@@ -0,0 +1,164 @@
+"use strict";
+// =============================================================================
+// The Ask evaluation's scorer (issue #382, epic #348; docs/specs/ontology.md §21)
+// =============================================================================
+//
+// PURE: no Nest, no Prisma, no provider. Takes one question, the assistant
+// message `ask.respond` wrote for it (`AskMessageResponse`, #376's wire
+// contract), every `messages` array the run's provider actually sent (for the
+// leak check), and the seeder's evidence-source map — and returns one
+// `AskEvalScore`. Called by the replay DB spec, the real runner, and its own
+// unit spec, so the three can never disagree about what a number means.
+//
+// Two GATES (spec: 100% citation validity, zero sensitive leaks across the
+// WHOLE run) are per-question booleans/ratios here; the run aggregates them.
+// =============================================================================
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.scoreCitationValidity = scoreCitationValidity;
+exports.scoreCited = scoreCited;
+exports.scoreCiteFromHit = scoreCiteFromHit;
+exports.scoreEntityRecall = scoreEntityRecall;
+exports.scoreSensitiveLeak = scoreSensitiveLeak;
+exports.scoreAnswerMatch = scoreAnswerMatch;
+exports.scoreAnswer = scoreAnswer;
+const NOT_FOUND_PATTERN = /couldn't find|could not find|no (record|information)|not in your|don't have (any )?(record|information)|no results/i;
+function includesCI(haystack, needle) {
+    return haystack.toLowerCase().includes(needle.toLowerCase());
+}
+function groupMatched(content, group) {
+    return group.some((alt) => includesCI(content, alt));
+}
+/**
+ * Valid citations / all markers. `1` (vacuously satisfied) with zero markers — a question that expects none.
+ *
+ * @stability experimental
+ */
+function scoreCitationValidity(citations) {
+    if (citations.length === 0)
+        return 1;
+    const valid = citations.filter((c) => c.valid).length;
+    return valid / citations.length;
+}
+/**
+ * Whether the answer carries at least one valid citation.
+ *
+ * @param citations - the answer's citations.
+ * @returns `true` when one is valid.
+ *
+ * @stability experimental
+ */
+function scoreCited(citations) {
+    return citations.some((c) => c.valid);
+}
+/**
+ * Whether a valid evidence citation resolves, through the seeder's map, to one
+ * of the question's `citeFrom` sources.
+ *
+ * @param citeFrom - the expected sources (`mNN-sNNN`, `mNN#note`).
+ * @param citations - the answer's citations.
+ * @param evidenceSource - evidence id to source, from the seeder.
+ * @returns `true` on a hit (vacuously `true` with no `citeFrom`).
+ *
+ * @stability experimental
+ */
+function scoreCiteFromHit(citeFrom, citations, evidenceSource) {
+    if (citeFrom.length === 0)
+        return true; // nothing was required (e.g. a negative question)
+    return citations.some((c) => {
+        if (!c.valid || c.kind !== 'evidence' || !c.id)
+            return false;
+        const source = evidenceSource.get(c.id);
+        return source !== undefined && citeFrom.includes(source);
+    });
+}
+/**
+ * Expected entities cited (by citation label, case-insensitive exact) over the expected count.
+ *
+ * @param expectedEntities - the labels that must be cited.
+ * @param citations - the answer's citations.
+ * @returns the ratio (`1` when none were expected).
+ *
+ * @stability experimental
+ */
+function scoreEntityRecall(expectedEntities, citations) {
+    if (expectedEntities.length === 0)
+        return 1;
+    const citedLabels = new Set(citations.filter((c) => c.valid && c.label).map((c) => c.label.toLowerCase()));
+    const hit = expectedEntities.filter((label) => citedLabels.has(label.toLowerCase())).length;
+    return hit / expectedEntities.length;
+}
+/**
+ * THE LEAK GATE: whether an `answerNone` string appears in the answer or in any prompt the
+ * provider received.
+ *
+ * @param answerNone - the strings that must never leave.
+ * @param content - the final answer.
+ * @param capturedPrompts - every request the provider received, serialized.
+ * @returns `true` on a leak.
+ *
+ * @stability experimental
+ */
+function scoreSensitiveLeak(answerNone, content, capturedPrompts) {
+    if (answerNone.length === 0)
+        return false;
+    const haystacks = [content, ...capturedPrompts];
+    return answerNone.some((needle) => haystacks.some((h) => includesCI(h, needle)));
+}
+/**
+ * Every `answerAll` group matched, no `answerNone` present, and (for a `notFound` question) no valid citation plus a not-found phrasing.
+ *
+ * @stability experimental
+ */
+function scoreAnswerMatch(expected, content, citations) {
+    const reasons = [];
+    for (const group of expected.answerAll) {
+        if (!groupMatched(content, group))
+            reasons.push(`missing required phrase (one of: ${group.join(' | ')})`);
+    }
+    for (const forbidden of expected.answerNone) {
+        if (includesCI(content, forbidden))
+            reasons.push(`answer contains forbidden phrase "${forbidden}"`);
+    }
+    if (expected.notFound) {
+        if (citations.some((c) => c.valid))
+            reasons.push('a `notFound` question cited something valid');
+        if (!NOT_FOUND_PATTERN.test(content))
+            reasons.push('answer does not read as "could not find it"');
+    }
+    return { matched: reasons.length === 0, reasons };
+}
+/**
+ * The full score for one question's answered turn.
+ *
+ * @stability experimental
+ */
+function scoreAnswer(question, message, capturedPrompts, evidenceSource, timing = {}) {
+    const citations = message.citations;
+    const { matched, reasons } = scoreAnswerMatch(question.expected, message.content, citations);
+    const citeFromHit = scoreCiteFromHit(question.expected.citeFrom, citations, evidenceSource);
+    const notes = [...reasons];
+    if (!citeFromHit && question.expected.citeFrom.length > 0) {
+        notes.push(`no valid citation resolved to one of: ${question.expected.citeFrom.join(', ')}`);
+    }
+    return {
+        questionId: question.id,
+        category: question.category,
+        citationValidity: scoreCitationValidity(citations),
+        cited: scoreCited(citations),
+        citeFromHit,
+        answerMatch: matched && (question.expected.citeFrom.length === 0 || citeFromHit),
+        entityRecall: scoreEntityRecall(question.expected.entities, citations),
+        sensitiveLeak: scoreSensitiveLeak(question.expected.answerNone, message.content, capturedPrompts),
+        // The wire `AskToolCall[]` carries a flat `index` across the whole turn,
+        // never which loop iteration ("step") produced it — so the caller passes
+        // exactly THIS question's captured provider requests, and one request is
+        // one loop iteration by construction (`ask-respond.handler.ts`'s `loop`
+        // calls `provider.chat()` exactly once per iteration).
+        steps: capturedPrompts.length,
+        toolCalls: message.toolCalls.length,
+        promptTokens: message.promptTokens ?? 0,
+        completionTokens: message.completionTokens ?? 0,
+        ms: timing.ms ?? null,
+        notes,
+    };
+}

package/dist/ask-run/ask-command.d.ts

@@ -0,0 +1,62 @@
+/**
+ * Where the ask command writes, and the environment it reads (the same shape as
+ * the kg command's `AskCommandIo`).
+ *
+ * @stability experimental
+ */
+export interface AskCommandIo {
+    /** One line to standard output. */
+    out: (line: string) => void;
+    /** One line to standard error. */
+    err: (line: string) => void;
+    /** The environment (`process.env`, or a test's own). */
+    env: Readonly<Record<string, string | undefined>>;
+}
+/**
+ * The exit codes of `kvox-eval ask` (and of the web app's `npm run ask:eval`).
+ *
+ * @stability experimental
+ */
+export declare const ASK_EVAL_EXIT: Readonly<{
+    readonly ok: 0;
+    readonly failed: 1;
+    readonly config: 2;
+}>;
+/**
+ * The usage text of `kvox-eval ask`.
+ *
+ * @stability experimental
+ */
+export declare const ASK_COMMAND_USAGE = "usage: kvox-eval ask [--provider openai|openai-compatible] [--model <id>] [--base-url <url>]\n               [--only q01,q07] [--category <c>] [--out <dir>] [--min-match <0..1>]\n       kvox-eval ask --replay      the scripted self-test (no model, no key)\n       reads the API key from KVOX_EVAL_OPENAI_API_KEY (never a flag); default model gpt-4o-mini";
+/**
+ * How a host runs the ask command under its own name.
+ *
+ * @stability experimental
+ */
+export interface AskCommandOptions {
+    /** The prefix of every message (`kvox-eval ask`, `ask:eval`). */
+    name?: string;
+    /** The usage text printed with a usage error and for `--help`. */
+    usage?: string;
+    /**
+     * The default `--out` root: a run writes `<dir>/runs/<stamp>/` (the web app's
+     * git-ignored `apps/api/.ask-eval/`). Default: none, and the output goes under
+     * the system temporary directory.
+     */
+    syntheticRunsDir?: string | null;
+}
+/**
+ * Runs `kvox-eval ask` with the given arguments (everything after `ask`).
+ *
+ * @example
+ * ```ts
+ * const code = await runAskCommand(['--replay'], { out: console.log, err: console.error, env: process.env });
+ * ```
+ *
+ * @param argv - the arguments.
+ * @param io - output streams and environment.
+ * @param options - the host's name, usage text and runs directory.
+ * @returns the exit code ({@link ASK_EVAL_EXIT}).
+ * @stability experimental
+ */
+export declare function runAskCommand(argv: string[], io: AskCommandIo, options?: AskCommandOptions): Promise<number>;