@marinoscar/kvox-eval 0.0.0-bootstrap → 0.1.0-next.0

package/dist/ask-run/host/ports.js

@@ -0,0 +1,113 @@
+"use strict";
+// =============================================================================
+// The ports the headless Ask evaluation binds (issue #608)
+// =============================================================================
+//
+// Ten ports, most of them the in-memory fakes of `@marinoscar/kvox-api/testing`.
+// The four that are not:
+//
+//   prisma    the PGlite client (`db.ts`).
+//   ai        the port under test (live: `createLiveKvoxAi`; replay: a scripted
+//             runtime under `createKvoxAi`), handed in by the caller.
+//   settings  in-memory, with the system `ai` namespace REGISTERED here: the
+//             graph and Ask slices read it and none of them declares it (in the
+//             web edition it is the platform AI slice's; the bare host declares
+//             it in `bootstrap.ts`). The graph switch is ON, the caps are null
+//             (the model's own limits govern), no reasoning parameter.
+//   jobs      the fake queue, except `enqueueWithin`: `ask_messages.job_id` is a
+//             foreign key to `jobs`, so the turn's job must be a REAL row written
+//             in the caller's transaction (never on the pool: one connection).
+//             Nothing claims it; the runner calls the handler itself.
+//   identity  one principal, the evaluation user, holding the three read
+//             permissions the Ask handler probes (graph, transcripts, notes).
+// =============================================================================
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ASK_EVAL_PERMISSIONS = void 0;
+exports.createAskEvalSettings = createAskEvalSettings;
+exports.createAskEvalJobs = createAskEvalJobs;
+exports.createAskEvalIdentity = createAskEvalIdentity;
+const zod_1 = require("zod");
+const graph_1 = require("@marinoscar/kvox-api/graph");
+const notes_1 = require("@marinoscar/kvox-api/notes");
+const testing_1 = require("@marinoscar/kvox-api/testing");
+const transcripts_1 = require("@marinoscar/kvox-api/transcripts");
+/**
+ * The permissions the evaluation user holds: what `ask.respond` probes for its
+ * actor. Read-only on purpose; the agent writes nothing.
+ *
+ * @stability experimental
+ */
+exports.ASK_EVAL_PERMISSIONS = Object.freeze([
+    graph_1.GRAPH_READ_PERMISSION,
+    transcripts_1.TRANSCRIPTS_READ_PERMISSION,
+    notes_1.NOTES_READ_PERMISSION,
+]);
+/** The `ai` system namespace as the slices read it (the bare host declares the same shape). */
+const aiNamespaceSchema = zod_1.z.object({
+    maxInputTokens: zod_1.z.number().int().positive().nullable().default(null),
+    maxOutputTokens: zod_1.z.number().int().positive().nullable().default(null),
+    maxDocumentBytes: zod_1.z.number().int().positive().default(10 * 1024 * 1024),
+    graphEnabled: zod_1.z.boolean().default(true),
+    reasoningEffort: zod_1.z.enum(['none', 'low', 'medium', 'high', 'xhigh']).default('none'),
+});
+/**
+ * In-memory settings with the system `ai` namespace declared and the graph switch on.
+ *
+ * @returns the settings port.
+ * @stability experimental
+ */
+function createAskEvalSettings() {
+    const settings = (0, testing_1.createInMemoryKvoxSettings)();
+    settings.registerNamespace({
+        scope: 'system',
+        name: 'ai',
+        schema: aiNamespaceSchema,
+        defaults: {
+            maxInputTokens: null,
+            maxOutputTokens: null,
+            maxDocumentBytes: 10 * 1024 * 1024,
+            graphEnabled: true,
+            reasoningEffort: 'none',
+        },
+    });
+    return settings;
+}
+/**
+ * The fake queue, with `enqueueWithin` writing a real `jobs` row in the caller's
+ * transaction (the turn's `ask_messages.job_id` references it). Nothing claims
+ * the row: the runner hands it to the handler directly.
+ *
+ * @param orgId - the organization every job row is stamped with (the default one).
+ * @returns the jobs port.
+ * @stability experimental
+ */
+function createAskEvalJobs(orgId) {
+    const jobs = (0, testing_1.createFakeKvoxJobs)();
+    // Replaced in place: the fake's getters (`jobs`, `handlers`) stay live.
+    jobs.enqueueWithin = async (tx, input) => {
+        const row = await tx.job.create({
+            data: {
+                type: input.type,
+                reason: input.reason ?? 'upload',
+                subjectType: input.subjectType ?? null,
+                subjectId: input.subjectId ?? null,
+                priority: input.priority ?? 0,
+                payload: input.payload,
+                orgId,
+            },
+        });
+        return { id: row.id, deduplicated: false };
+    };
+    return jobs;
+}
+/**
+ * One principal: the evaluation user with {@link ASK_EVAL_PERMISSIONS}.
+ *
+ * @param userId - the evaluation user's id.
+ * @param orgId - the default organization's id.
+ * @returns the identity port.
+ * @stability experimental
+ */
+function createAskEvalIdentity(userId, orgId) {
+    return (0, testing_1.createFakeKvoxIdentity)({ principal: { userId, orgId, permissions: exports.ASK_EVAL_PERMISSIONS } });
+}

package/dist/ask-run/index.d.ts

@@ -0,0 +1,13 @@
+export { ASK_EVAL_USER_EMAIL, startAskEvalHost } from './host/app.js';
+export type { AskEvalHost, StartAskEvalHostOptions } from './host/app.js';
+export { ASK_EVAL_HOST_DIR, startAskEvalDatabase } from './host/db.js';
+export type { AskEvalDatabase, AskEvalPrismaClient } from './host/db.js';
+export { ASK_EVAL_PERMISSIONS, createAskEvalIdentity, createAskEvalJobs, createAskEvalSettings } from './host/ports.js';
+export { seedGoldenGraph } from './seed-golden-graph.js';
+export type { AskEvalIdMap, AskEvalSeedRefusal, SeededGoldenGraph } from './seed-golden-graph.js';
+export { DEFAULT_ASK_EVAL_MODEL, runAskEval, startAskEvalSession } from './run-ask-eval.js';
+export type { AskEvalRunResult, AskEvalSession, AskQuestionRunResult, RunAskEvalOptions } from './run-ask-eval.js';
+export { ASK_REPLAY_MODEL, ASK_REPLAY_SCENARIOS, ASK_REPLAY_SENSITIVE_TEXT, askReplayStepEvents, runAskReplay, } from './replay.js';
+export type { AskReplayResult, AskReplayRun, AskReplayScenario, AskReplayStep } from './replay.js';
+export { ASK_COMMAND_USAGE, ASK_EVAL_EXIT, runAskCommand } from './ask-command.js';
+export type { AskCommandIo, AskCommandOptions } from './ask-command.js';

package/dist/ask-run/index.js

@@ -0,0 +1,35 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runAskCommand = exports.ASK_EVAL_EXIT = exports.ASK_COMMAND_USAGE = exports.runAskReplay = exports.askReplayStepEvents = exports.ASK_REPLAY_SENSITIVE_TEXT = exports.ASK_REPLAY_SCENARIOS = exports.ASK_REPLAY_MODEL = exports.startAskEvalSession = exports.runAskEval = exports.DEFAULT_ASK_EVAL_MODEL = exports.seedGoldenGraph = exports.createAskEvalSettings = exports.createAskEvalJobs = exports.createAskEvalIdentity = exports.ASK_EVAL_PERMISSIONS = exports.startAskEvalDatabase = exports.ASK_EVAL_HOST_DIR = exports.startAskEvalHost = exports.ASK_EVAL_USER_EMAIL = void 0;
+// The ask-run slice's public surface: the `@marinoscar/kvox-eval/ask-run` entry
+// (kept off the package root, which stays light for the pure scorer): the
+// headless Ask evaluation. An in-memory PostgreSQL (PGlite) migrated with the
+// platform and domain history, the golden graph seeded through the graph write
+// service, the real `ask.respond` handler on the Ask slice, over any model port.
+var app_js_1 = require("./host/app.js");
+Object.defineProperty(exports, "ASK_EVAL_USER_EMAIL", { enumerable: true, get: function () { return app_js_1.ASK_EVAL_USER_EMAIL; } });
+Object.defineProperty(exports, "startAskEvalHost", { enumerable: true, get: function () { return app_js_1.startAskEvalHost; } });
+var db_js_1 = require("./host/db.js");
+Object.defineProperty(exports, "ASK_EVAL_HOST_DIR", { enumerable: true, get: function () { return db_js_1.ASK_EVAL_HOST_DIR; } });
+Object.defineProperty(exports, "startAskEvalDatabase", { enumerable: true, get: function () { return db_js_1.startAskEvalDatabase; } });
+var ports_js_1 = require("./host/ports.js");
+Object.defineProperty(exports, "ASK_EVAL_PERMISSIONS", { enumerable: true, get: function () { return ports_js_1.ASK_EVAL_PERMISSIONS; } });
+Object.defineProperty(exports, "createAskEvalIdentity", { enumerable: true, get: function () { return ports_js_1.createAskEvalIdentity; } });
+Object.defineProperty(exports, "createAskEvalJobs", { enumerable: true, get: function () { return ports_js_1.createAskEvalJobs; } });
+Object.defineProperty(exports, "createAskEvalSettings", { enumerable: true, get: function () { return ports_js_1.createAskEvalSettings; } });
+var seed_golden_graph_js_1 = require("./seed-golden-graph.js");
+Object.defineProperty(exports, "seedGoldenGraph", { enumerable: true, get: function () { return seed_golden_graph_js_1.seedGoldenGraph; } });
+var run_ask_eval_js_1 = require("./run-ask-eval.js");
+Object.defineProperty(exports, "DEFAULT_ASK_EVAL_MODEL", { enumerable: true, get: function () { return run_ask_eval_js_1.DEFAULT_ASK_EVAL_MODEL; } });
+Object.defineProperty(exports, "runAskEval", { enumerable: true, get: function () { return run_ask_eval_js_1.runAskEval; } });
+Object.defineProperty(exports, "startAskEvalSession", { enumerable: true, get: function () { return run_ask_eval_js_1.startAskEvalSession; } });
+var replay_js_1 = require("./replay.js");
+Object.defineProperty(exports, "ASK_REPLAY_MODEL", { enumerable: true, get: function () { return replay_js_1.ASK_REPLAY_MODEL; } });
+Object.defineProperty(exports, "ASK_REPLAY_SCENARIOS", { enumerable: true, get: function () { return replay_js_1.ASK_REPLAY_SCENARIOS; } });
+Object.defineProperty(exports, "ASK_REPLAY_SENSITIVE_TEXT", { enumerable: true, get: function () { return replay_js_1.ASK_REPLAY_SENSITIVE_TEXT; } });
+Object.defineProperty(exports, "askReplayStepEvents", { enumerable: true, get: function () { return replay_js_1.askReplayStepEvents; } });
+Object.defineProperty(exports, "runAskReplay", { enumerable: true, get: function () { return replay_js_1.runAskReplay; } });
+var ask_command_js_1 = require("./ask-command.js");
+Object.defineProperty(exports, "ASK_COMMAND_USAGE", { enumerable: true, get: function () { return ask_command_js_1.ASK_COMMAND_USAGE; } });
+Object.defineProperty(exports, "ASK_EVAL_EXIT", { enumerable: true, get: function () { return ask_command_js_1.ASK_EVAL_EXIT; } });
+Object.defineProperty(exports, "runAskCommand", { enumerable: true, get: function () { return ask_command_js_1.runAskCommand; } });

package/dist/ask-run/replay.d.ts

@@ -0,0 +1,128 @@
+import type { AskMessageResponse } from '@marinoscar/kvox-contract/ask';
+import type { AiStreamEvent } from '@marinoscar/platform-api/ai';
+/**
+ * The provider and model a replay names (the scripted runtime answers for them).
+ *
+ * @stability experimental
+ */
+export declare const ASK_REPLAY_MODEL: Readonly<{
+    readonly provider: "openai";
+    readonly model: "gpt-replay";
+}>;
+/**
+ * The `sensitive` PersonFact text (m13) the leak gate looks for: it must never
+ * appear in a captured request or an answer.
+ *
+ * @stability experimental
+ */
+export declare const ASK_REPLAY_SENSITIVE_TEXT = "shellfish allergy";
+/**
+ * One scripted model call: tool calls (the model asking for tools) or the final answer.
+ *
+ * @stability experimental
+ */
+export type AskReplayStep = {
+    /** The calls, in order: id, tool name, arguments as the JSON text the provider streams. */
+    toolCalls: readonly {
+        id: string;
+        name: string;
+        arguments: string;
+    }[];
+} | {
+    /** The answer text, citation markers included. */
+    answer: string;
+};
+/**
+ * One scripted question and what its answer must show.
+ *
+ * @stability experimental
+ */
+export interface AskReplayScenario {
+    /** A short id (`r1` .. `r6`). */
+    id: string;
+    /** What the test case proves. */
+    title: string;
+    /** The question. */
+    question: string;
+    /** The model calls, in order. */
+    steps: readonly AskReplayStep[];
+    /** What the finished message must show. */
+    expected: {
+        /** Every citation valid (citation validity 1.0), or at least one invalid (below 1). */
+        allValid: boolean;
+        /** Markers that must be present and invalid (with no resolved id). */
+        invalidMarkers: readonly string[];
+        /** The exact number of citations, when the scenario pins it. */
+        citationCount?: number;
+        /** A valid citation that must be present, by marker and kind. */
+        validCitation?: {
+            marker: string;
+            kind: string;
+        };
+    };
+}
+/**
+ * The six scripted questions of the replay, as data.
+ *
+ * @stability experimental
+ */
+export declare const ASK_REPLAY_SCENARIOS: readonly AskReplayScenario[];
+/**
+ * The provider events of one scripted model call: the function calls (finish
+ * reason `tool_calls`) or the answer's text (finish reason `stop`), then the
+ * completed response with fixed usage.
+ *
+ * @param step - the step.
+ * @returns the events, in order.
+ * @stability experimental
+ */
+export declare function askReplayStepEvents(step: AskReplayStep): AiStreamEvent[];
+/**
+ * One replayed question: the finished message and what it was checked for.
+ *
+ * @stability experimental
+ */
+export interface AskReplayResult {
+    /** The scenario. */
+    scenario: AskReplayScenario;
+    /** The finished assistant message. */
+    message: AskMessageResponse;
+    /** Every `chat()` request of this turn, serialized. */
+    capturedPrompts: string[];
+    /** Citation validity (`scoreCitationValidity`). */
+    citationValidity: number;
+    /** Why the scenario failed its expectation; empty when it held. */
+    failures: string[];
+}
+/**
+ * The outcome of {@link runAskReplay}.
+ *
+ * @stability experimental
+ */
+export interface AskReplayRun {
+    /** One per scenario, in order. */
+    results: AskReplayResult[];
+    /** The leak gate: no captured request and no answer contains {@link ASK_REPLAY_SENSITIVE_TEXT}. */
+    leakFree: boolean;
+    /** Every scenario held and the leak gate passed. */
+    passed: boolean;
+}
+/**
+ * Runs the replay: the whole golden graph seeded, the six scripted questions
+ * through the real handler, each checked, and the leak gate over every request
+ * and answer.
+ *
+ * @example
+ * ```ts
+ * const run = await runAskReplay();
+ * if (!run.passed) process.exitCode = 1;
+ * ```
+ *
+ * @param options - the scenarios (default {@link ASK_REPLAY_SCENARIOS}) and a progress sink.
+ * @returns the results and the gates.
+ * @stability experimental
+ */
+export declare function runAskReplay(options?: {
+    scenarios?: readonly AskReplayScenario[];
+    log?: (line: string) => void;
+}): Promise<AskReplayRun>;

package/dist/ask-run/replay.js

@@ -0,0 +1,212 @@
+"use strict";
+// =============================================================================
+// The Ask evaluation's replay: the harness's self-test, no model, no key (#608)
+// =============================================================================
+//
+// Seeds the ENTIRE golden graph (exactly as a live run does), then drives the
+// REAL `AskRespondHandler` (real toolset, real database, real handle registry
+// and citation mapper, and the real `createKvoxAi` event mapping) through six
+// representative questions with a SCRIPTED runtime (`createScriptedRuntime` of
+// `@marinoscar/kvox-ai/testing`): a fixed sequence of tool calls per question,
+// using handles the real tools actually returned, then a canned answer that
+// cites them, plus one answer that cites an unissued `[^ev99]`.
+//
+// It is the same role `kvox-eval kg --predictions gold` plays for extraction:
+// it proves the seeder, the tools, the handler and the scorer agree end to end
+// WITHOUT a real model or a paid key. The scripts are the ones of the web app's
+// `apps/api/test/ask/ask-eval.replay.db.spec.ts` (same tool calls, same
+// arguments, same answers), exported as DATA so a test and `kvox-eval ask
+// --replay` share them.
+//
+// What it checks: citation validity < 1 for the planted answer (with `ev99`
+// invalid) and 1.0 for the other five, every turn `complete`, and the leak
+// gate: the graph's `sensitive` PersonFact (Marcus Webb's shellfish allergy)
+// never reaches a captured request or an answer.
+// =============================================================================
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ASK_REPLAY_SCENARIOS = exports.ASK_REPLAY_SENSITIVE_TEXT = exports.ASK_REPLAY_MODEL = void 0;
+exports.askReplayStepEvents = askReplayStepEvents;
+exports.runAskReplay = runAskReplay;
+const kvox_ai_1 = require("@marinoscar/kvox-ai");
+const testing_1 = require("@marinoscar/kvox-ai/testing");
+const index_js_1 = require("../ask/index.js");
+const index_js_2 = require("../live/index.js");
+const run_ask_eval_js_1 = require("./run-ask-eval.js");
+/**
+ * The provider and model a replay names (the scripted runtime answers for them).
+ *
+ * @stability experimental
+ */
+exports.ASK_REPLAY_MODEL = Object.freeze({ provider: 'openai', model: 'gpt-replay' });
+/**
+ * The `sensitive` PersonFact text (m13) the leak gate looks for: it must never
+ * appear in a captured request or an answer.
+ *
+ * @stability experimental
+ */
+exports.ASK_REPLAY_SENSITIVE_TEXT = 'shellfish allergy';
+const call = (id, name, args) => ({ id, name, arguments: JSON.stringify(args) });
+/**
+ * The six scripted questions of the replay, as data.
+ *
+ * @stability experimental
+ */
+exports.ASK_REPLAY_SCENARIOS = Object.freeze([
+    {
+        id: 'r1',
+        title: 'search → get_entity → cites [^ent1]: valid, citation validity 1.0',
+        question: 'Who does Sarah Chen work for?',
+        steps: [
+            { toolCalls: [call('c1', 'search', { query: 'Sarah Chen', scope: 'entities', limit: null })] },
+            { toolCalls: [call('c2', 'get_entity', { entity: 'ent1' })] },
+            { answer: 'Sarah Chen works for Northwind Robotics.[^ent1]' },
+        ],
+        expected: { allValid: true, invalidMarkers: [] },
+    },
+    {
+        id: 'r2',
+        title: 'search → evidence(ent1) → cites [^ev1]: valid, citation validity 1.0',
+        question: 'Where is the pick-path pilot?',
+        steps: [
+            { toolCalls: [call('c1', 'search', { query: 'pick-path pilot', scope: 'entities', limit: null })] },
+            { toolCalls: [call('c2', 'evidence', { subject: 'ent1', limit: 5 })] },
+            { answer: 'Here is what was said about it.[^ev1]' },
+        ],
+        expected: { allValid: true, invalidMarkers: [], citationCount: 1, validCitation: { marker: 'ev1', kind: 'evidence' } },
+    },
+    {
+        id: 'r3',
+        title: 'search → timeline → evidence(itm1) → cites [^ev1]: valid, citation validity 1.0',
+        question: 'What has happened with Halden Freight?',
+        steps: [
+            { toolCalls: [call('c1', 'search', { query: 'Halden Freight', scope: 'entities', limit: null })] },
+            { toolCalls: [call('c2', 'timeline', { entity: 'ent1', asOf: null, kinds: null, limit: 5 })] },
+            { toolCalls: [call('c3', 'evidence', { subject: 'itm1', limit: 3 })] },
+            { answer: 'Several things happened.[^ev1]' },
+        ],
+        expected: { allValid: true, invalidMarkers: [] },
+    },
+    {
+        id: 'r4',
+        title: 'a not-found question: search returns nothing, the answer cites nothing (citation validity vacuously 1.0)',
+        question: 'What did Jamie Fontaine decide about the Meridian project?',
+        steps: [
+            { toolCalls: [call('c1', 'search', { query: 'Jamie Fontaine', scope: 'all', limit: null })] },
+            { answer: "I couldn't find Jamie Fontaine or a Meridian project in your graph." },
+        ],
+        expected: { allValid: true, invalidMarkers: [], citationCount: 0 },
+    },
+    {
+        id: 'r5',
+        title: 'cites entity + evidence together, all valid: citation validity 1.0',
+        question: 'Who runs the northern depots cutover, and what is the evidence?',
+        steps: [
+            { toolCalls: [call('c1', 'search', { query: 'Marcus Webb', scope: 'entities', limit: null })] },
+            { toolCalls: [call('c2', 'get_entity', { entity: 'ent1' })] },
+            { toolCalls: [call('c3', 'evidence', { subject: 'ent1', limit: 3 })] },
+            { answer: 'Marcus Webb is involved.[^ent1] See here.[^ev1]' },
+        ],
+        expected: { allValid: true, invalidMarkers: [] },
+    },
+    {
+        id: 'r6',
+        title: 'THE PLANTED BAD ANSWER: cites an unissued [^ev99] alongside a real one: citation validity < 1',
+        question: 'What is going on with the Corvid Health project?',
+        steps: [
+            { toolCalls: [call('c1', 'search', { query: 'Corvid Health', scope: 'entities', limit: null })] },
+            { toolCalls: [call('c2', 'get_entity', { entity: 'ent1' })] },
+            { answer: 'Corvid Health is a client.[^ent1] Also this, which nobody ever returned.[^ev99]' },
+        ],
+        expected: { allValid: false, invalidMarkers: ['ev99'] },
+    },
+]);
+/**
+ * The provider events of one scripted model call: the function calls (finish
+ * reason `tool_calls`) or the answer's text (finish reason `stop`), then the
+ * completed response with fixed usage.
+ *
+ * @param step - the step.
+ * @returns the events, in order.
+ * @stability experimental
+ */
+function askReplayStepEvents(step) {
+    const completed = (finishReason) => (0, testing_1.completedEvent)({
+        id: 'resp_replay',
+        provider: exports.ASK_REPLAY_MODEL.provider,
+        model: exports.ASK_REPLAY_MODEL.model,
+        usage: { inputTokens: 100, outputTokens: 20 },
+        finishReason,
+    });
+    if ('answer' in step)
+        return [(0, testing_1.textDeltaEvent)(step.answer), completed('stop')];
+    return [...step.toolCalls.map((c) => (0, testing_1.functionCallEvent)(c.id, c.name, c.arguments)), completed('tool_calls')];
+}
+/** What one finished message failed of its scenario's expectation. */
+function checkScenario(scenario, message) {
+    const failures = [];
+    const { expected } = scenario;
+    if (message.status !== 'complete')
+        failures.push(`status is '${message.status}', not 'complete'`);
+    const validity = (0, index_js_1.scoreCitationValidity)(message.citations);
+    if (expected.allValid && validity !== 1)
+        failures.push(`citation validity ${validity}, expected 1`);
+    if (!expected.allValid && !(validity < 1))
+        failures.push(`citation validity ${validity}, expected < 1`);
+    for (const marker of expected.invalidMarkers) {
+        const citation = message.citations.find((c) => c.marker === marker);
+        if (!citation || citation.valid || citation.id !== null)
+            failures.push(`[^${marker}] is not present as an invalid citation`);
+    }
+    if (expected.citationCount !== undefined && message.citations.length !== expected.citationCount) {
+        failures.push(`${message.citations.length} citation(s), expected ${expected.citationCount}`);
+    }
+    if (expected.validCitation) {
+        const { marker, kind } = expected.validCitation;
+        if (!message.citations.some((c) => c.marker === marker && c.kind === kind && c.valid)) {
+            failures.push(`no valid ${kind} citation [^${marker}]`);
+        }
+    }
+    return failures;
+}
+/**
+ * Runs the replay: the whole golden graph seeded, the six scripted questions
+ * through the real handler, each checked, and the leak gate over every request
+ * and answer.
+ *
+ * @example
+ * ```ts
+ * const run = await runAskReplay();
+ * if (!run.passed) process.exitCode = 1;
+ * ```
+ *
+ * @param options - the scenarios (default {@link ASK_REPLAY_SCENARIOS}) and a progress sink.
+ * @returns the results and the gates.
+ * @stability experimental
+ */
+async function runAskReplay(options = {}) {
+    const scenarios = options.scenarios ?? exports.ASK_REPLAY_SCENARIOS;
+    const runtime = (0, testing_1.createScriptedRuntime)();
+    const ai = (0, kvox_ai_1.createKvoxAi)({ runtime, taskModels: (0, index_js_2.staticTaskModelSource)(exports.ASK_REPLAY_MODEL.provider, exports.ASK_REPLAY_MODEL.model) });
+    const session = await (0, run_ask_eval_js_1.startAskEvalSession)({ ai, ...(options.log ? { log: options.log } : {}) });
+    try {
+        const results = [];
+        for (const scenario of scenarios) {
+            for (const step of scenario.steps)
+                runtime.queueStream({ events: askReplayStepEvents(step) });
+            const { message, capturedPrompts } = await session.ask(scenario.question, exports.ASK_REPLAY_MODEL.model);
+            const failures = checkScenario(scenario, message);
+            // Every scripted call played, and no more: a leftover would leak into the next question.
+            const played = runtime.calls.filter((c) => c.kind === 'stream').length;
+            if (played !== scenario.steps.length)
+                failures.push(`${played} model call(s), the script has ${scenario.steps.length}`);
+            runtime.reset();
+            results.push({ scenario, message, capturedPrompts, citationValidity: (0, index_js_1.scoreCitationValidity)(message.citations), failures });
+        }
+        const leakFree = results.every((r) => !r.capturedPrompts.some((p) => p.toLowerCase().includes(exports.ASK_REPLAY_SENSITIVE_TEXT)) &&
+            !(0, index_js_1.scoreSensitiveLeak)([exports.ASK_REPLAY_SENSITIVE_TEXT], r.message.content, []));
+        return { results, leakFree, passed: leakFree && results.every((r) => r.failures.length === 0) };
+    }
+    finally {
+        await session.close();
+    }
+}

package/dist/ask-run/run-ask-eval.d.ts

@@ -0,0 +1,125 @@
+import type { AskMessageResponse } from '@marinoscar/kvox-contract/ask';
+import { type KvoxAiPort } from '@marinoscar/kvox-api/ports';
+import { type AskEvalQuestion, type AskEvalReport, type AskEvalScore } from '../ask/index.js';
+import { type GoldenFixture } from '../fixtures/index.js';
+import { type AskEvalHost } from './host/app.js';
+import { type AskEvalIdMap, type SeededGoldenGraph } from './seed-golden-graph.js';
+/**
+ * The model a live run uses when none is named.
+ *
+ * @stability experimental
+ */
+export declare const DEFAULT_ASK_EVAL_MODEL = "gpt-4o-mini";
+/**
+ * One answered question.
+ *
+ * @stability experimental
+ */
+export interface AskQuestionRunResult {
+    /** The question as asked. */
+    question: AskEvalQuestion;
+    /** The finished assistant message, as the API would return it. */
+    message: AskMessageResponse;
+    /** Every `chat()` request the port received while answering THIS question, serialized (signal removed). */
+    capturedPrompts: string[];
+    /** Wall-clock time of the turn, in milliseconds. */
+    ms: number;
+    /**
+     * The model call that failed the turn, by the port's error class (`auth`,
+     * `unavailable`, ...): the handler marked the message `failed` and the run went
+     * on. Absent when the turn ran to its end.
+     */
+    modelError?: {
+        errorClass: string;
+    };
+}
+/**
+ * The scored outcome of a run.
+ *
+ * @stability experimental
+ */
+export interface AskEvalRunResult {
+    /** One per question, in the order asked. */
+    results: AskQuestionRunResult[];
+    /** Fixture ids ↔ row ids (`evidenceSource` is what `citeFrom` resolves against). */
+    idMap: AskEvalIdMap;
+    /** What the seeder reported (refusals, warnings, counts). */
+    seed: Pick<SeededGoldenGraph, 'refusals' | 'warnings' | 'counts'>;
+    /** One score per result. */
+    scores: AskEvalScore[];
+    /** The aggregated report. */
+    report: AskEvalReport;
+}
+/**
+ * What {@link runAskEval} runs.
+ *
+ * @stability experimental
+ */
+export interface RunAskEvalOptions {
+    /** The model port under test. */
+    ai: KvoxAiPort;
+    /** The model id each turn names (and the report records). */
+    model: string;
+    /** The questions, or ids of the committed ones. Default: every committed question. */
+    questions?: readonly AskEvalQuestion[] | readonly string[];
+    /** The golden meetings to seed. Default: `loadGoldenSet()`. */
+    fixtures?: readonly GoldenFixture[];
+    /** The answer-match gate of the report (0..1). Default 0.8. */
+    minMatch?: number;
+    /** Receives progress lines (steps, counts, question ids; never a question's text or an answer). */
+    log?: (line: string) => void;
+}
+/**
+ * A seeded host that answers one question at a time: what {@link runAskEval}
+ * and the replay drive.
+ *
+ * @stability experimental
+ */
+export interface AskEvalSession {
+    /** The running host. */
+    readonly host: AskEvalHost;
+    /** What the seeder wrote. */
+    readonly seeded: SeededGoldenGraph;
+    /**
+     * Asks one question in a new conversation and runs its `ask.respond` job to the end.
+     *
+     * @param question - the question text.
+     * @param model - the model id the turn names.
+     * @returns the finished message and this turn's captured requests.
+     */
+    ask(question: string, model: string): Promise<Omit<AskQuestionRunResult, 'question'>>;
+    /** Restores the port's `chat()` and closes the host. Safe to call twice. */
+    close(): Promise<void>;
+}
+/**
+ * Starts the host on `ai`, seeds the golden graph and returns a session that asks
+ * questions through the real Ask services and handler.
+ *
+ * @param options - the port, the fixtures (default: the whole golden set) and a progress sink.
+ * @returns the session; call `close()` when done (in a `finally`).
+ * @stability experimental
+ */
+export declare function startAskEvalSession(options: {
+    ai: KvoxAiPort;
+    fixtures?: readonly GoldenFixture[];
+    log?: (line: string) => void;
+}): Promise<AskEvalSession>;
+/**
+ * Runs the Ask evaluation headless: an in-memory PostgreSQL, the golden graph
+ * seeded through the graph write service, every question through the real
+ * `ask.respond` handler on `ai`, each answer scored.
+ *
+ * @example
+ * ```ts
+ * const key = readEvalApiKey(process.env);
+ * const { ai } = createLiveKvoxAi({ provider: 'openai', model: 'gpt-4o-mini', apiKey: key?.apiKey ?? null });
+ * const run = await runAskEval({ ai, model: 'gpt-4o-mini', questions: ['q01', 'q07'] });
+ * console.log(formatAskEvalReport(run.report));
+ * ```
+ *
+ * @param options - the port, the model, the questions and the gate.
+ * @returns the answers, the id map, the scores and the report.
+ * @throws `AskEvalQuestionLoadError` for an unknown question id; `KgEvalLoadError` for a malformed fixture.
+ * @stability experimental
+ */
+export declare function runAskEval(options: RunAskEvalOptions): Promise<AskEvalRunResult>;