@marinoscar/kvox-eval 0.0.0-bootstrap → 0.1.0-next.0

package/dist/ask-run/run-ask-eval.js

@@ -0,0 +1,193 @@
+"use strict";
+// =============================================================================
+// runAskEval: the Ask evaluation, headless (issue #608; the web app's runner #382)
+// =============================================================================
+//
+// Starts the in-memory host (`host/app.ts`), seeds the golden graph under the
+// evaluation user (`seed-golden-graph.ts`), then for every selected question:
+//
+//   create a conversation (`AskConversationsService`) → post the question
+//   through the REAL `AskMessagesService` (checks, transaction, the real
+//   `ask.respond` job row) → invoke `AskRespondHandler.process(job)` DIRECTLY
+//   (nothing else claims the job) → read the finished assistant message back.
+//
+// The turn runs on the model port the caller hands in, exactly as a production
+// turn runs on the host's binding: live, `createLiveKvoxAi` (`createKvoxAi`
+// over the direct runtime and the provider adapters); replay, `createKvoxAi`
+// over a scripted runtime (`replay.ts`). Every request the port receives is
+// captured through a thin wrapper around its own `chat()` (never a second
+// implementation), so the scorer's leak check sees exactly what left the run.
+//
+// EVERYTHING DIES WITH THE RUN: the database is PGlite in memory and `close()`
+// runs in a `finally`, so there is nothing to tear down and no deployment
+// setting to restore (the web app's runner had both).
+// =============================================================================
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_ASK_EVAL_MODEL = void 0;
+exports.startAskEvalSession = startAskEvalSession;
+exports.runAskEval = runAskEval;
+const ask_1 = require("@marinoscar/kvox-api/ask");
+const testing_1 = require("@marinoscar/kvox-api/ask/testing");
+const ports_1 = require("@marinoscar/kvox-api/ports");
+const index_js_1 = require("../ask/index.js");
+const index_js_2 = require("../fixtures/index.js");
+const app_js_1 = require("./host/app.js");
+const seed_golden_graph_js_1 = require("./seed-golden-graph.js");
+/**
+ * The model a live run uses when none is named.
+ *
+ * @stability experimental
+ */
+exports.DEFAULT_ASK_EVAL_MODEL = 'gpt-4o-mini';
+/**
+ * Wraps the port's own `chat()` so every request is recorded. Mutates the port in
+ * place (the handler holds the same object); returns the restorer.
+ */
+function wrapChatForCapture(port, onRequest) {
+    const original = port.chat;
+    const bound = original.bind(port);
+    port.chat = (ctx, request) => {
+        const { signal: _signal, ...rest } = ctx;
+        onRequest({ ctx: rest, request });
+        return bound(ctx, request);
+    };
+    return () => {
+        port.chat = original;
+    };
+}
+/**
+ * Starts the host on `ai`, seeds the golden graph and returns a session that asks
+ * questions through the real Ask services and handler.
+ *
+ * @param options - the port, the fixtures (default: the whole golden set) and a progress sink.
+ * @returns the session; call `close()` when done (in a `finally`).
+ * @stability experimental
+ */
+async function startAskEvalSession(options) {
+    const log = options.log ?? (() => { });
+    const fixtures = options.fixtures ?? (0, index_js_2.loadGoldenSet)();
+    const host = await (0, app_js_1.startAskEvalHost)({ ai: options.ai, log });
+    let bucket = [];
+    let unwrap = null;
+    try {
+        const seeded = await (0, seed_golden_graph_js_1.seedGoldenGraph)(host, fixtures);
+        log(`seed: ${fixtures.length} meetings, ${seeded.counts.entities} entities, ${seeded.counts.relations} relations, ` +
+            `${seeded.counts.items} items, ${seeded.counts.evidence} citations` +
+            (seeded.refusals.length ? `, ${seeded.refusals.length} refused` : '') +
+            (seeded.warnings.length ? `, ${seeded.warnings.length} warning(s)` : ''));
+        for (const r of seeded.refusals)
+            log(`seed refused: ${r.fixtureId} ${r.row}: ${r.reason}`);
+        for (const w of seeded.warnings)
+            log(`seed warning: ${w}`);
+        const conversations = host.resolve(ask_1.AskConversationsService);
+        const messages = host.resolve(testing_1.internals.AskMessagesService);
+        const handler = host.resolve(testing_1.internals.AskRespondHandler);
+        unwrap = wrapChatForCapture(host.ai, (captured) => bucket.push(captured));
+        let closed = false;
+        return {
+            host,
+            seeded,
+            async ask(question, model) {
+                bucket = [];
+                const started = Date.now();
+                const conversation = await conversations.create(host.userId, {});
+                const posted = await messages.post(host.userId, conversation.id, { content: question, model });
+                const row = await host.prisma.askMessage.findUniqueOrThrow({ where: { id: posted.assistantMessage.id } });
+                const job = await host.prisma.job.findUniqueOrThrow({ where: { id: row.jobId } });
+                let modelError;
+                try {
+                    await handler.process({
+                        id: job.id,
+                        type: job.type,
+                        subjectType: job.subjectType,
+                        subjectId: job.subjectId,
+                        payload: job.payload,
+                        attempts: job.attempts,
+                        orgId: job.orgId,
+                    });
+                }
+                catch (error) {
+                    // A failed model call: the handler has marked the turn `failed` and
+                    // rethrown for the queue (which would record the attempt). This run is
+                    // the queue; it records the class and goes on. A rate limit before any
+                    // text leaves the turn `pending` for a re-run, which a run cannot wait
+                    // for, so it (and anything else) stops the run.
+                    if (!(error instanceof ports_1.KvoxAiError))
+                        throw error;
+                    const settled = await host.prisma.askMessage.findUniqueOrThrow({ where: { id: posted.assistantMessage.id }, select: { status: true } });
+                    if (settled.status !== 'failed')
+                        throw error;
+                    modelError = { errorClass: error.errorClass };
+                }
+                const finished = await host.prisma.askMessage.findUniqueOrThrow({ where: { id: posted.assistantMessage.id } });
+                return {
+                    message: testing_1.internals.toAskMessage(finished),
+                    capturedPrompts: bucket.map((r) => JSON.stringify(r)),
+                    ms: Date.now() - started,
+                    ...(modelError ? { modelError } : {}),
+                };
+            },
+            async close() {
+                if (closed)
+                    return;
+                closed = true;
+                unwrap?.();
+                await host.close();
+            },
+        };
+    }
+    catch (error) {
+        unwrap?.();
+        await host.close();
+        throw error;
+    }
+}
+/** The committed questions, narrowed to `ids`. */
+function resolveQuestions(questions) {
+    if (questions === undefined)
+        return (0, index_js_2.loadGoldenAskQuestions)();
+    if (questions.every((q) => typeof q === 'string')) {
+        return (0, index_js_1.selectAskEvalQuestions)((0, index_js_2.loadGoldenAskQuestions)(), [...questions], null);
+    }
+    return [...questions];
+}
+/**
+ * Runs the Ask evaluation headless: an in-memory PostgreSQL, the golden graph
+ * seeded through the graph write service, every question through the real
+ * `ask.respond` handler on `ai`, each answer scored.
+ *
+ * @example
+ * ```ts
+ * const key = readEvalApiKey(process.env);
+ * const { ai } = createLiveKvoxAi({ provider: 'openai', model: 'gpt-4o-mini', apiKey: key?.apiKey ?? null });
+ * const run = await runAskEval({ ai, model: 'gpt-4o-mini', questions: ['q01', 'q07'] });
+ * console.log(formatAskEvalReport(run.report));
+ * ```
+ *
+ * @param options - the port, the model, the questions and the gate.
+ * @returns the answers, the id map, the scores and the report.
+ * @throws `AskEvalQuestionLoadError` for an unknown question id; `KgEvalLoadError` for a malformed fixture.
+ * @stability experimental
+ */
+async function runAskEval(options) {
+    const log = options.log ?? (() => { });
+    const questions = resolveQuestions(options.questions);
+    const session = await startAskEvalSession({ ai: options.ai, ...(options.fixtures ? { fixtures: options.fixtures } : {}), log });
+    try {
+        const results = [];
+        for (const question of questions) {
+            const answered = await session.ask(question.question, options.model);
+            log(`ask: ${question.id} ${answered.message.status} in ${answered.ms} ms` +
+                (answered.modelError ? ` (model call failed: ${answered.modelError.errorClass})` : ''));
+            results.push({ question, ...answered });
+        }
+        const idMap = session.seeded.idMap;
+        const scores = results.map((r) => (0, index_js_1.scoreAnswer)(r.question, r.message, r.capturedPrompts, idMap.evidenceSource, { ms: r.ms }));
+        const report = (0, index_js_1.buildAskEvalReport)(scores, { model: options.model, minMatch: options.minMatch ?? 0.8 });
+        const { refusals, warnings, counts } = session.seeded;
+        return { results, idMap, seed: { refusals, warnings, counts }, scores, report };
+    }
+    finally {
+        await session.close();
+    }
+}

package/dist/ask-run/seed-golden-graph.d.ts

@@ -0,0 +1,64 @@
+import { type GoldenFixture } from '../fixtures/index.js';
+import type { AskEvalHost } from './host/app.js';
+/**
+ * Fixture ids and labels ↔ the database rows the seeder wrote for them.
+ *
+ * @stability experimental
+ */
+export interface AskEvalIdMap {
+    /** `"m01-s009"` → the real `transcript_segments.id`. */
+    segments: Map<string, string>;
+    /** Fixture id → that meeting's `notes.id`. */
+    notes: Map<string, string>;
+    /** A global (`knownEntities`) id → `kg_entities.id`. Mostly for debugging. */
+    entities: Map<string, string>;
+    /** `kg_evidence.id` → `"m01-s009"` or `"m01#note"`: what the scorer's `citeFrom` checks resolve against. */
+    evidenceSource: Map<string, string>;
+}
+/**
+ * One row the write service refused: which fixture and row, and why (the
+ * service's message and issue paths; never a label, statement or quote).
+ *
+ * @stability experimental
+ */
+export interface AskEvalSeedRefusal {
+    /** The fixture id (`"m07"`). */
+    fixtureId: string;
+    /** The row, by kind, position and type key (`"relation #2 (WORKS_AT)"`). */
+    row: string;
+    /** The service's reason. */
+    reason: string;
+}
+/**
+ * What {@link seedGoldenGraph} wrote.
+ *
+ * @stability experimental
+ */
+export interface SeededGoldenGraph {
+    /** The owner of every row (the host's evaluation user). */
+    userId: string;
+    /** Fixture ids ↔ row ids. */
+    idMap: AskEvalIdMap;
+    /** Rows the write service refused (skipped, reported). */
+    refusals: AskEvalSeedRefusal[];
+    /** Fixture data that could not be resolved (an unknown segment or entity reference). */
+    warnings: string[];
+    /** Row counts, for the log. */
+    counts: {
+        entities: number;
+        relations: number;
+        items: number;
+        evidence: number;
+    };
+}
+/**
+ * Seeds the evaluation user's entire gold graph from EVERY fixture in
+ * `fixtures` (pass the full loaded set: `loadGoldenSet()`), one transaction per
+ * fixture, in fixture id order, through `GraphWriteService`.
+ *
+ * @param host - the running host (its client, user and organization, and the write service).
+ * @param fixtures - the golden meetings.
+ * @returns the id map the scorer needs, the refusals and the warnings.
+ * @stability experimental
+ */
+export declare function seedGoldenGraph(host: AskEvalHost, fixtures: readonly GoldenFixture[]): Promise<SeededGoldenGraph>;