@marinoscar/kvox-eval 0.0.0-bootstrap → 0.1.0-next.0

package/dist/ask-run/ask-command.js

@@ -0,0 +1,254 @@
+"use strict";
+// =============================================================================
+// `kvox-eval ask` — the Ask evaluation, headless (issue #608; the web app's
+// `npm run ask:eval`, #382)
+// =============================================================================
+//
+//   kvox-eval ask --model gpt-4o-mini
+//   kvox-eval ask --model <id> --only q01,q22
+//   kvox-eval ask --model <id> --category negative --min-match 0.7
+//   kvox-eval ask --provider openai-compatible --base-url <url> --model <id>
+//   kvox-eval ask --replay
+//
+// NO web app and NO external database: the run starts PGlite in memory, applies
+// the platform and domain migrations, seeds the golden graph through the graph
+// write service, and runs every selected question through the real
+// `ask.respond` handler of `@marinoscar/kvox-api/ask` on the live model port
+// (`createLiveKvoxAi`). Everything dies with the process. It prints the table
+// and writes `report.json` and `answers.md` under `--out` (default: a stamped
+// folder under the system temporary directory, or under the host's own
+// git-ignored runs directory).
+//
+// `--replay` runs the harness's self-test instead: six scripted questions on a
+// scripted runtime (no model, no key, no network), checked for citation
+// validity, every turn complete and the sensitive-fact leak gate; it prints one
+// line per question and writes `replay.json` (the same `--out` default).
+//
+// The API key comes from KVOX_EVAL_OPENAI_API_KEY only (ASK_EVAL_OPENAI_API_KEY
+// for one more release), never a flag, and is never printed or written. The key
+// gate runs FIRST, before the database starts. A failed provider round trip is
+// reported by its error code and HTTP status only.
+//
+// Exit codes: 0 pass · 1 a gate failed (citation validity < 1, a sensitive
+// leak) or answer match fell below `--min-match`, or a replay expectation
+// failed · 2 configuration error (no key, a refused provider, a bad flag, an
+// unknown question id, a malformed fixture, every model call failed, or a rate
+// limit the run cannot wait out). A turn whose model call failed is otherwise
+// scored as a failed answer and the run goes on.
+// =============================================================================
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ASK_COMMAND_USAGE = exports.ASK_EVAL_EXIT = void 0;
+exports.runAskCommand = runAskCommand;
+const fs_1 = require("fs");
+const os_1 = require("os");
+const path_1 = require("path");
+const util_1 = require("util");
+const ports_1 = require("@marinoscar/kvox-api/ports");
+const index_js_1 = require("../ask/index.js");
+const index_js_2 = require("../fixtures/index.js");
+const index_js_3 = require("../live/index.js");
+const index_js_4 = require("../live/index.js");
+const replay_js_1 = require("./replay.js");
+const run_ask_eval_js_1 = require("./run-ask-eval.js");
+/**
+ * The exit codes of `kvox-eval ask` (and of the web app's `npm run ask:eval`).
+ *
+ * @stability experimental
+ */
+exports.ASK_EVAL_EXIT = Object.freeze({ ok: 0, failed: 1, config: 2 });
+/**
+ * The usage text of `kvox-eval ask`.
+ *
+ * @stability experimental
+ */
+exports.ASK_COMMAND_USAGE = `usage: kvox-eval ask [--provider openai|openai-compatible] [--model <id>] [--base-url <url>]
+               [--only q01,q07] [--category <c>] [--out <dir>] [--min-match <0..1>]
+       kvox-eval ask --replay      the scripted self-test (no model, no key)
+       reads the API key from ${index_js_3.KVOX_EVAL_API_KEY_ENV} (never a flag); default model ${run_ask_eval_js_1.DEFAULT_ASK_EVAL_MODEL}`;
+/** A command line that cannot run; printed with the usage text. */
+class UsageError extends Error {
+}
+/** Flags `--replay` does not take: the replay's questions, model and gate are fixed. */
+const LIVE_ONLY_FLAGS = ['provider', 'model', 'base-url', 'only', 'category', 'min-match'];
+/**
+ * Runs `kvox-eval ask` with the given arguments (everything after `ask`).
+ *
+ * @example
+ * ```ts
+ * const code = await runAskCommand(['--replay'], { out: console.log, err: console.error, env: process.env });
+ * ```
+ *
+ * @param argv - the arguments.
+ * @param io - output streams and environment.
+ * @param options - the host's name, usage text and runs directory.
+ * @returns the exit code ({@link ASK_EVAL_EXIT}).
+ * @stability experimental
+ */
+async function runAskCommand(argv, io, options = {}) {
+    const name = options.name ?? 'kvox-eval ask';
+    const usage = options.usage ?? exports.ASK_COMMAND_USAGE;
+    try {
+        return await run(argv, io, options, name, usage);
+    }
+    catch (err) {
+        if (err instanceof UsageError) {
+            io.err(`${name}: ${err.message}`);
+            io.err(usage);
+            return exports.ASK_EVAL_EXIT.config;
+        }
+        if (err instanceof index_js_1.AskEvalQuestionLoadError || err instanceof index_js_2.KgEvalLoadError || err instanceof index_js_4.LiveProviderError) {
+            io.err(`${name}: ${err.message}`);
+            return exports.ASK_EVAL_EXIT.config;
+        }
+        throw err;
+    }
+}
+async function run(argv, io, options, name, usage) {
+    let values;
+    try {
+        ({ values } = (0, util_1.parseArgs)({
+            args: argv,
+            strict: true,
+            allowPositionals: false,
+            options: {
+                provider: { type: 'string' },
+                model: { type: 'string' },
+                'base-url': { type: 'string' },
+                only: { type: 'string' },
+                category: { type: 'string' },
+                out: { type: 'string' },
+                'min-match': { type: 'string' },
+                replay: { type: 'boolean' },
+                help: { type: 'boolean' },
+            },
+        }));
+    }
+    catch (err) {
+        throw new UsageError(err.message);
+    }
+    const str = (k) => (typeof values[k] === 'string' ? values[k] : undefined);
+    if (values.help) {
+        io.out(usage);
+        return exports.ASK_EVAL_EXIT.ok;
+    }
+    const log = (line) => io.err(`${name}: ${line}`);
+    if (values.replay) {
+        const extra = LIVE_ONLY_FLAGS.filter((flag) => values[flag] !== undefined);
+        if (extra.length > 0)
+            throw new UsageError(`--replay takes no ${extra.map((f) => `--${f}`).join(', ')}`);
+        return replay(io, log, outDirFor(str('out'), options));
+    }
+    const minMatchArg = str('min-match');
+    const minMatch = minMatchArg !== undefined ? Number(minMatchArg) : 0.8;
+    if (!Number.isFinite(minMatch) || minMatch < 0 || minMatch > 1)
+        throw new UsageError('--min-match must be a number between 0 and 1');
+    // The provider is refused (copilot, an unknown id) before the key is looked for,
+    // and the key gate runs before anything is loaded or started: a run with no key
+    // is a configuration error, not a run that seeds a graph and then fails.
+    const provider = str('provider') ?? 'openai';
+    (0, index_js_4.assertLiveProvider)(provider);
+    const model = str('model') ?? run_ask_eval_js_1.DEFAULT_ASK_EVAL_MODEL;
+    const key = (0, index_js_3.readEvalApiKey)(io.env, io.err);
+    if (key === null && provider === 'openai') {
+        io.err(`${name}: reads its API key from ${index_js_3.KVOX_EVAL_API_KEY_ENV}; it is not set`);
+        return exports.ASK_EVAL_EXIT.config;
+    }
+    const baseUrl = str('base-url');
+    const { ai, failures } = (0, index_js_4.createLiveKvoxAi)({
+        provider,
+        model,
+        apiKey: key?.apiKey ?? null,
+        ...(baseUrl !== undefined ? { baseUrl } : {}),
+    });
+    const only = str('only')
+        ?.split(',')
+        .map((s) => s.trim())
+        .filter(Boolean);
+    const questions = (0, index_js_1.selectAskEvalQuestions)((0, index_js_2.loadGoldenAskQuestions)(), only ?? null, str('category') ?? null);
+    if (questions.length === 0) {
+        io.err(`${name}: no questions selected`);
+        return exports.ASK_EVAL_EXIT.config;
+    }
+    // A malformed fixture is a configuration error (`KgEvalLoadError`), and must not seed a partial graph.
+    const fixtures = (0, index_js_2.loadGoldenSet)();
+    const outDir = outDirFor(str('out'), options);
+    let result;
+    try {
+        result = await (0, run_ask_eval_js_1.runAskEval)({ ai, model, questions, fixtures, minMatch, log });
+    }
+    catch (err) {
+        // A failure the run cannot go on from (a rate limit before any text, which
+        // a queue would re-run later): reported by class, code and status only.
+        if (err instanceof ports_1.KvoxAiError || err instanceof ports_1.KvoxRateLimitError) {
+            reportFailures(failures, log);
+            io.err(`${name}: the live call failed (${err instanceof ports_1.KvoxAiError ? err.errorClass : 'rate_limit'}); the run stopped`);
+            return exports.ASK_EVAL_EXIT.config;
+        }
+        throw err;
+    }
+    reportFailures(failures, log);
+    io.out((0, index_js_1.formatAskEvalReport)(result.report));
+    (0, fs_1.mkdirSync)(outDir, { recursive: true });
+    (0, fs_1.writeFileSync)((0, path_1.join)(outDir, 'report.json'), `${JSON.stringify(result.report, null, 2)}\n`);
+    const answers = new Map(result.results.map((r) => [r.question.id, { question: r.question.question, content: r.message.content }]));
+    (0, fs_1.writeFileSync)((0, path_1.join)(outDir, 'answers.md'), (0, index_js_1.formatAnswersMarkdown)(result.report, answers));
+    log(`report written to ${outDir}`);
+    // Not one turn reached the model: a configuration problem (a wrong key, model
+    // or base URL), not a measurement.
+    if (result.results.every((r) => r.modelError !== undefined)) {
+        io.err(`${name}: every model call failed (${[...new Set(result.results.map((r) => r.modelError?.errorClass))].join(', ')}); check the key, model and base URL`);
+        return exports.ASK_EVAL_EXIT.config;
+    }
+    if (!result.report.gatesPassed || !result.report.answerMatchPassed)
+        return exports.ASK_EVAL_EXIT.failed;
+    return exports.ASK_EVAL_EXIT.ok;
+}
+/** `--out`, else `<runs dir>/runs/<stamp>`, else the system temporary directory. */
+function outDirFor(out, options) {
+    if (out)
+        return (0, path_1.resolve)(out);
+    const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+    const root = options.syntheticRunsDir ?? (0, path_1.join)((0, os_1.tmpdir)(), 'kvox-eval', 'ask');
+    return (0, path_1.join)(root, 'runs', stamp);
+}
+/** How many provider round trips failed, by code and status (never a message body). */
+function reportFailures(failures, log) {
+    if (failures.length === 0)
+        return;
+    const counts = new Map();
+    for (const f of failures) {
+        const label = `${f.code}${f.status !== null ? `, HTTP ${f.status}` : ''}`;
+        counts.set(label, (counts.get(label) ?? 0) + 1);
+    }
+    log(`${failures.length} provider round trip(s) failed: ${[...counts].map(([l, n]) => `${n} x (${l})`).join('; ')}`);
+}
+/** `--replay`: the six scripted questions, one line each, then the gates; `replay.json` under `--out`. */
+async function replay(io, log, outDir) {
+    const run = await (0, replay_js_1.runAskReplay)({ log });
+    io.out('Ask evaluation replay (scripted runtime, no model)');
+    for (const r of run.results) {
+        const tools = r.message.toolCalls.map((c) => c.name).join(' > ') || '-';
+        const cites = r.message.citations.map((c) => `[^${c.marker}] ${c.valid ? 'valid' : 'INVALID'}`).join(', ') || 'none';
+        const verdict = r.failures.length === 0 ? 'ok' : `FAILED: ${r.failures.join('; ')}`;
+        io.out(`  ${r.scenario.id}  ${r.message.status}  tools: ${tools}  citations: ${cites}  validity ${r.citationValidity.toFixed(2)}  ${verdict}`);
+    }
+    io.out(`  leak gate: ${run.leakFree ? 'ok (the sensitive fact reached no request and no answer)' : 'FAILED'}`);
+    io.out(run.passed ? 'replay passed' : 'replay FAILED');
+    (0, fs_1.mkdirSync)(outDir, { recursive: true });
+    const summary = {
+        passed: run.passed,
+        leakFree: run.leakFree,
+        scenarios: run.results.map((r) => ({
+            id: r.scenario.id,
+            title: r.scenario.title,
+            status: r.message.status,
+            toolCalls: r.message.toolCalls.map((c) => c.name),
+            citations: r.message.citations.map((c) => ({ marker: c.marker, kind: c.kind, valid: c.valid })),
+            citationValidity: r.citationValidity,
+            failures: r.failures,
+        })),
+    };
+    (0, fs_1.writeFileSync)((0, path_1.join)(outDir, 'replay.json'), `${JSON.stringify(summary, null, 2)}\n`);
+    log(`replay summary written to ${outDir}`);
+    return run.passed ? exports.ASK_EVAL_EXIT.ok : exports.ASK_EVAL_EXIT.failed;
+}

package/dist/ask-run/host/app.d.ts

@@ -0,0 +1,65 @@
+import 'reflect-metadata';
+import type { INestApplicationContext, Type } from '@nestjs/common';
+import type { KvoxAiPort, KvoxSettingsPort } from '@marinoscar/kvox-api/ports';
+import { type AskEvalPrismaClient } from './db.js';
+/**
+ * The evaluation user's email. A reserved domain (`.invalid`): the account exists
+ * only in the run's in-memory database.
+ *
+ * @stability experimental
+ */
+export declare const ASK_EVAL_USER_EMAIL = "ask-eval@kvox-eval.invalid";
+/**
+ * What {@link startAskEvalHost} is built from.
+ *
+ * @stability experimental
+ */
+export interface StartAskEvalHostOptions {
+    /** The model port every turn runs on (the port under test). */
+    ai: KvoxAiPort;
+    /** Receives progress lines (steps and counts, never data). */
+    log?: (line: string) => void;
+}
+/**
+ * A running evaluation host.
+ *
+ * @stability experimental
+ */
+export interface AskEvalHost {
+    /** The Nest application context the slices run in. */
+    readonly app: INestApplicationContext;
+    /** The client on the in-memory database. */
+    readonly prisma: AskEvalPrismaClient;
+    /** The model port the slices were given. */
+    readonly ai: KvoxAiPort;
+    /** The settings port the slices were given (in memory; the `ai` namespace declared). */
+    readonly settings: KvoxSettingsPort;
+    /** The evaluation user (owner of every seeded row). */
+    readonly userId: string;
+    /** The default organization (adopted from the platform migrations). */
+    readonly orgId: string;
+    /** A provider from anywhere in the container (`app.get(type, { strict: false })`). */
+    resolve<T>(type: Type<T>): T;
+    /** Closes the application context, the client and the database. Safe to call twice. */
+    close(): Promise<void>;
+}
+/**
+ * Starts the in-memory database, adopts its default organization, creates the
+ * evaluation user and boots the domain slices over the evaluation's ports.
+ *
+ * @example
+ * ```ts
+ * const host = await startAskEvalHost({ ai });
+ * try {
+ *   const seeded = await seedGoldenGraph(host, loadGoldenSet());
+ *   // ... post questions, run the handler
+ * } finally {
+ *   await host.close();
+ * }
+ * ```
+ *
+ * @param options - the model port and a progress sink.
+ * @returns the host; call `close()` when done.
+ * @stability experimental
+ */
+export declare function startAskEvalHost(options: StartAskEvalHostOptions): Promise<AskEvalHost>;

package/dist/ask-run/host/app.js

@@ -0,0 +1,146 @@
+"use strict";
+// =============================================================================
+// The headless Ask evaluation's host: PGlite + the domain slices on Nest (#608)
+// =============================================================================
+//
+// What `ask.respond` needs and nothing more, as a Nest APPLICATION CONTEXT (no
+// HTTP server): the ports module, then the transcription, search, transcripts,
+// notes, graph and Ask slices, every optional feature switched off the way the
+// bare host (examples/bare-host) switches them off:
+//
+//   schedules: false        no `@nestjs/schedule` timers; nothing housekeeps a
+//                           database that lives for one run
+//   media.transcode: false  no ffmpeg
+//   exporters / documentSources pdf, docx: false   no pdfkit, docx, unpdf
+//   providers.assemblyai: false   no transcription provider at all: the seeded
+//                           transcripts are already `ready`
+//   semantic search off     the seeded rows carry no embeddings, and a query
+//                           embedding would spend the key under test
+//
+// NOT graph-extraction and NOT graph-rdf: the golden graph is seeded from the
+// fixtures' HAND LABELS through `GraphWriteService`, so the evaluation isolates
+// the agent (extraction quality is `kvox-eval kg`'s question).
+//
+// THE `peers` OPTION passes the very module objects this module imports, so the
+// container holds one module per slice and each job handler registers once.
+//
+// The host module is a plain class: a dynamic module needs no decorator, and the
+// package compiles without decorator metadata (the slices carry their own).
+// =============================================================================
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ASK_EVAL_USER_EMAIL = void 0;
+exports.startAskEvalHost = startAskEvalHost;
+require("reflect-metadata");
+const core_1 = require("@nestjs/core");
+const ask_1 = require("@marinoscar/kvox-api/ask");
+const graph_1 = require("@marinoscar/kvox-api/graph");
+const notes_1 = require("@marinoscar/kvox-api/notes");
+const search_1 = require("@marinoscar/kvox-api/search");
+const testing_1 = require("@marinoscar/kvox-api/testing");
+const transcription_1 = require("@marinoscar/kvox-api/transcription");
+const transcripts_1 = require("@marinoscar/kvox-api/transcripts");
+const db_js_1 = require("./db.js");
+const ports_js_1 = require("./ports.js");
+/**
+ * The evaluation user's email. A reserved domain (`.invalid`): the account exists
+ * only in the run's in-memory database.
+ *
+ * @stability experimental
+ */
+exports.ASK_EVAL_USER_EMAIL = 'ask-eval@kvox-eval.invalid';
+/** The domain slices `ask.respond` needs, wired like the bare host's, minus extraction and RDF. */
+function createAskEvalModule(ports) {
+    const transcription = transcription_1.TranscriptionModule.forRoot({ providers: { assemblyai: false } });
+    const searchIndexing = search_1.SearchIndexingModule.forRoot();
+    const transcripts = transcripts_1.TranscriptsModule.forRoot({
+        schedules: false,
+        media: { transcode: false },
+        exporters: { pdf: false },
+        peers: { transcription, searchIndexing },
+    });
+    const notes = notes_1.NotesModule.forRoot({
+        schedules: false,
+        exporters: { pdf: false, docx: false },
+        documentSources: { pdf: false },
+        peers: { transcripts, searchIndexing },
+    });
+    const search = search_1.SearchModule.forRoot({ semantic: { enabled: false } });
+    const graph = graph_1.GraphModule.forRoot({ schedules: false, peers: { transcripts, notes, search } });
+    const ask = ask_1.AskModule.forRoot({ peers: { graph, notes, search } });
+    class AskEvalHostModule {
+    }
+    return {
+        module: AskEvalHostModule,
+        imports: [(0, testing_1.createKvoxTestingModule)(ports), transcription, search, searchIndexing, transcripts, notes, graph, ask],
+    };
+}
+/**
+ * Starts the in-memory database, adopts its default organization, creates the
+ * evaluation user and boots the domain slices over the evaluation's ports.
+ *
+ * @example
+ * ```ts
+ * const host = await startAskEvalHost({ ai });
+ * try {
+ *   const seeded = await seedGoldenGraph(host, loadGoldenSet());
+ *   // ... post questions, run the handler
+ * } finally {
+ *   await host.close();
+ * }
+ * ```
+ *
+ * @param options - the model port and a progress sink.
+ * @returns the host; call `close()` when done.
+ * @stability experimental
+ */
+async function startAskEvalHost(options) {
+    const log = options.log ?? (() => { });
+    const database = await (0, db_js_1.startAskEvalDatabase)(log);
+    const { prisma } = database;
+    let app = null;
+    try {
+        // The platform's own migrations create the default organization (its partial
+        // unique index allows exactly one): adopt it, as the bare host does.
+        const org = await prisma.organization.findFirst({ where: { isDefault: true }, select: { id: true } });
+        if (!org)
+            throw new Error('The platform migrations created no default organization.');
+        const user = await prisma.user.create({
+            data: { email: exports.ASK_EVAL_USER_EMAIL, displayName: 'Ask evaluation', isActive: true },
+            select: { id: true },
+        });
+        const settings = (0, ports_js_1.createAskEvalSettings)();
+        app = await core_1.NestFactory.createApplicationContext(createAskEvalModule({
+            overrides: {
+                ai: options.ai,
+                prisma: prisma,
+                settings,
+                jobs: (0, ports_js_1.createAskEvalJobs)(org.id),
+                identity: (0, ports_js_1.createAskEvalIdentity)(user.id, org.id),
+            },
+        }), { logger: ['error', 'warn'] });
+        log('host: slices started');
+        const context = app;
+        let closed = false;
+        return {
+            app: context,
+            prisma,
+            ai: options.ai,
+            settings,
+            userId: user.id,
+            orgId: org.id,
+            resolve: (type) => context.get(type, { strict: false }),
+            async close() {
+                if (closed)
+                    return;
+                closed = true;
+                await context.close().catch(() => undefined);
+                await database.close();
+            },
+        };
+    }
+    catch (error) {
+        await app?.close().catch(() => undefined);
+        await database.close();
+        throw error;
+    }
+}

package/dist/ask-run/host/db.d.ts

@@ -0,0 +1,38 @@
+import { PrismaClient } from '../../../host/generated/index.js';
+/**
+ * The package's `host/` folder: `prisma/migrations/` and `generated/` (built by
+ * `scripts/prepare-host.mjs`). `dist/ask-run/host/db.js` and `src/ask-run/host/db.ts`
+ * are both three levels below the package root.
+ *
+ * @stability experimental
+ */
+export declare const ASK_EVAL_HOST_DIR: string;
+/**
+ * The Prisma client of the evaluation's database: the platform and domain schema,
+ * generated into `host/generated/` at build time.
+ *
+ * @stability experimental
+ */
+export type AskEvalPrismaClient = PrismaClient;
+/**
+ * A started, fully migrated in-memory database and the client on it.
+ *
+ * @stability experimental
+ */
+export interface AskEvalDatabase {
+    /** The client (one pooled connection). */
+    readonly prisma: AskEvalPrismaClient;
+    /** How many migration folders were applied. */
+    readonly migrations: number;
+    /** Disconnects the client, stops the socket and closes PGlite. Safe to call twice. */
+    close(): Promise<void>;
+}
+/**
+ * Starts PGlite in memory, applies every shipped migration, serves it on a
+ * loopback socket and connects a Prisma client to it.
+ *
+ * @param log - receives one line per step (never data).
+ * @returns the database; call `close()` when done.
+ * @stability experimental
+ */
+export declare function startAskEvalDatabase(log?: (line: string) => void): Promise<AskEvalDatabase>;

package/dist/ask-run/host/db.js

@@ -0,0 +1,126 @@
+"use strict";
+// =============================================================================
+// The in-memory database of the headless Ask evaluation (issue #608)
+// =============================================================================
+//
+// PGlite (PostgreSQL compiled to WebAssembly) in THIS process, with `vector`,
+// `pg_trgm` and `uuid-ossp`, served on a loopback socket so the Prisma client's
+// `pg` driver adapter can reach it. The data lives in memory and dies with the
+// run: nothing an evaluation does ever touches a real database, so there is no
+// safe-database guard to pass any more.
+//
+// THE SCHEMA. Extensions first (the bootstrap superuser's job, never a
+// migration's), then every migration file the package ships under
+// `host/prisma/migrations/` (the platform history stamped 2024-01-01, then the
+// domain's under stable names, so the folder names sort in apply order), each
+// executed as one simple-protocol script, exactly what `prisma migrate deploy`
+// sends. No Prisma CLI runs at evaluation time; `scripts/prepare-host.mjs`
+// composed the schema and generated the client at build time.
+//
+// ONE SESSION. PGlite serialises every query onto one backend, so the client
+// uses a pool of ONE connection: an interactive transaction holds the only
+// connection and anything outside it waits. A port implementation must never
+// await a query on the pool while holding a transaction open (on PGlite that is
+// a deadlock); the fake jobs port of `ports.ts` writes through the caller's tx.
+// =============================================================================
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ASK_EVAL_HOST_DIR = void 0;
+exports.startAskEvalDatabase = startAskEvalDatabase;
+const node_fs_1 = require("node:fs");
+const node_net_1 = require("node:net");
+const node_path_1 = require("node:path");
+const pglite_1 = require("@electric-sql/pglite");
+const pg_trgm_1 = require("@electric-sql/pglite/contrib/pg_trgm");
+const uuid_ossp_1 = require("@electric-sql/pglite/contrib/uuid_ossp");
+const pglite_pgvector_1 = require("@electric-sql/pglite-pgvector");
+const pglite_socket_1 = require("@electric-sql/pglite-socket");
+const adapter_pg_1 = require("@prisma/adapter-pg");
+const index_js_1 = require("../../../host/generated/index.js");
+/**
+ * The package's `host/` folder: `prisma/migrations/` and `generated/` (built by
+ * `scripts/prepare-host.mjs`). `dist/ask-run/host/db.js` and `src/ask-run/host/db.ts`
+ * are both three levels below the package root.
+ *
+ * @stability experimental
+ */
+exports.ASK_EVAL_HOST_DIR = (0, node_path_1.join)(__dirname, '..', '..', '..', 'host');
+/** A free loopback port (bound and released). */
+function freePort() {
+    return new Promise((resolve, reject) => {
+        const probe = (0, node_net_1.createServer)();
+        probe.on('error', reject);
+        probe.listen(0, '127.0.0.1', () => {
+            const address = probe.address();
+            const port = typeof address === 'object' && address ? address.port : 0;
+            probe.close(() => resolve(port));
+        });
+    });
+}
+/** The shipped migration folders, in apply order. */
+function migrationFolders() {
+    const dir = (0, node_path_1.join)(exports.ASK_EVAL_HOST_DIR, 'prisma', 'migrations');
+    let names;
+    try {
+        names = (0, node_fs_1.readdirSync)(dir);
+    }
+    catch {
+        throw new Error(`The evaluation's database host is missing (${dir}); build the package first (node scripts/prepare-host.mjs).`);
+    }
+    return names
+        .filter((name) => (0, node_fs_1.statSync)((0, node_path_1.join)(dir, name)).isDirectory())
+        .sort()
+        .map((name) => (0, node_path_1.join)(dir, name, 'migration.sql'));
+}
+/**
+ * Starts PGlite in memory, applies every shipped migration, serves it on a
+ * loopback socket and connects a Prisma client to it.
+ *
+ * @param log - receives one line per step (never data).
+ * @returns the database; call `close()` when done.
+ * @stability experimental
+ */
+async function startAskEvalDatabase(log = () => { }) {
+    const files = migrationFolders();
+    const db = await pglite_1.PGlite.create({ extensions: { vector: pglite_pgvector_1.vector, pg_trgm: pg_trgm_1.pg_trgm, uuid_ossp: uuid_ossp_1.uuid_ossp } });
+    let server = null;
+    let prisma = null;
+    const close = async () => {
+        const [p, s] = [prisma, server];
+        prisma = null;
+        server = null;
+        await p?.$disconnect().catch(() => undefined);
+        await s?.stop().catch(() => undefined);
+        if (!db.closed)
+            await db.close().catch(() => undefined);
+    };
+    try {
+        // What the application's bootstrap superuser does before the first migration.
+        await db.exec('CREATE EXTENSION IF NOT EXISTS vector; CREATE EXTENSION IF NOT EXISTS pg_trgm;');
+        log(`database: applying ${files.length} migrations in memory`);
+        for (const file of files)
+            await db.exec((0, node_fs_1.readFileSync)(file, 'utf8'));
+        const port = await freePort();
+        server = new pglite_socket_1.PGLiteSocketServer({ db, port, host: '127.0.0.1', maxConnections: 1 });
+        await server.start();
+        const url = `postgresql://postgres@127.0.0.1:${port}/postgres?sslmode=disable`;
+        prisma = new index_js_1.PrismaClient({ adapter: new adapter_pg_1.PrismaPg({ connectionString: url, max: 1 }) });
+        await prisma.$connect();
+        log('database: ready');
+    }
+    catch (error) {
+        await close();
+        throw error;
+    }
+    const client = prisma;
+    let closed = false;
+    return {
+        prisma: client,
+        migrations: files.length,
+        async close() {
+            if (closed)
+                return;
+            closed = true;
+            await close();
+        },
+    };
+}

package/dist/ask-run/host/ports.d.ts

@@ -0,0 +1,35 @@
+import type { KvoxJobsPort, KvoxSettingsPort } from '@marinoscar/kvox-api/ports';
+import { type FakeKvoxIdentity } from '@marinoscar/kvox-api/testing';
+/**
+ * The permissions the evaluation user holds: what `ask.respond` probes for its
+ * actor. Read-only on purpose; the agent writes nothing.
+ *
+ * @stability experimental
+ */
+export declare const ASK_EVAL_PERMISSIONS: readonly string[];
+/**
+ * In-memory settings with the system `ai` namespace declared and the graph switch on.
+ *
+ * @returns the settings port.
+ * @stability experimental
+ */
+export declare function createAskEvalSettings(): KvoxSettingsPort;
+/**
+ * The fake queue, with `enqueueWithin` writing a real `jobs` row in the caller's
+ * transaction (the turn's `ask_messages.job_id` references it). Nothing claims
+ * the row: the runner hands it to the handler directly.
+ *
+ * @param orgId - the organization every job row is stamped with (the default one).
+ * @returns the jobs port.
+ * @stability experimental
+ */
+export declare function createAskEvalJobs(orgId: string): KvoxJobsPort;
+/**
+ * One principal: the evaluation user with {@link ASK_EVAL_PERMISSIONS}.
+ *
+ * @param userId - the evaluation user's id.
+ * @param orgId - the default organization's id.
+ * @returns the identity port.
+ * @stability experimental
+ */
+export declare function createAskEvalIdentity(userId: string, orgId: string): FakeKvoxIdentity;