@marinoscar/kvox-eval 0.0.0-bootstrap → 0.1.0-next.0

package/dist/cli/kg-command.js

@@ -0,0 +1,238 @@
+"use strict";
+// =============================================================================
+// `kvox-eval kg` — score knowledge-graph extraction against the golden set
+// (issue #362; the package command since #580)
+// =============================================================================
+//
+//   kvox-eval kg --predictions gold
+//   kvox-eval kg --predictions <dir> [--only m01,m07] [--tag promotion]
+//   kvox-eval kg --runner extract --provider openai --model <id> [--out <dir>]
+//   kvox-eval kg --runner extract --provider openai-compatible --base-url <url> --model <id>
+//   kvox-eval kg --real-dir <dir outside the repo> --predictions <dir>
+//
+// Flags: --json <file> also writes the report as JSON; --enforce exits 1 when a
+// §6 target (docs/specs/ontology.md) fails — for local use, CI reports only.
+// `--run` is accepted for `--runner` (the web app's `npm run kg:eval` spelling).
+//
+// Exit codes: 0 ok · 1 --enforce and a target failed · 2 usage/runner/load
+// error (a refused provider and a failed live call included) · 3 a real-data
+// path inside the repository (refused).
+//
+// The API key for a live runner comes from KVOX_EVAL_OPENAI_API_KEY only
+// (KG_EVAL_OPENAI_API_KEY and ASK_EVAL_OPENAI_API_KEY for one more release),
+// never a flag, and is never printed or written anywhere. A failed live call
+// is reported by its error class, code and HTTP status only.
+//
+// `runKgCommand` takes its output streams and environment as arguments, so a
+// test (and the web app's thin `scripts/kg-eval.ts` wrapper) drives it in
+// process.
+// =============================================================================
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KG_COMMAND_USAGE = exports.KG_EVAL_EXIT = void 0;
+exports.runKgCommand = runKgCommand;
+const fs_1 = require("fs");
+const os_1 = require("os");
+const path_1 = require("path");
+const util_1 = require("util");
+const ports_1 = require("@marinoscar/kvox-api/ports");
+const index_js_1 = require("../fixtures/index.js");
+const index_js_2 = require("../kg/index.js");
+const index_js_3 = require("../kg/index.js");
+const index_js_4 = require("../kg/index.js");
+const index_js_5 = require("../live/index.js");
+const index_js_6 = require("../live/index.js");
+const index_js_7 = require("../real-data/index.js");
+/**
+ * The exit codes of `kvox-eval kg` (and of the web app's `npm run kg:eval`).
+ *
+ * @stability experimental
+ */
+exports.KG_EVAL_EXIT = Object.freeze({ ok: 0, targetsFailed: 1, usage: 2, pathRefused: 3 });
+/**
+ * The usage text of `kvox-eval kg`.
+ *
+ * @stability experimental
+ */
+exports.KG_COMMAND_USAGE = `usage: kvox-eval kg (--predictions <dir|gold> | --runner extract|extract-resolve --model <id>)
+               [--provider openai|openai-compatible|copilot] [--base-url <url>] [--out <dir>]
+               [--only m01,m07] [--tag <tag>] [--json <file>] [--enforce] [--real-dir <dir>]
+       reads a live runner's API key from ${index_js_5.KVOX_EVAL_API_KEY_ENV} (never a flag)`;
+/** A command line that cannot run; printed with the usage text. */
+class UsageError extends Error {
+}
+/** A live call that failed; printed by class, code and status only. */
+class LiveCallError extends Error {
+}
+/**
+ * Runs `kvox-eval kg` with the given arguments (everything after `kg`).
+ *
+ * @example
+ * ```ts
+ * const code = await runKgCommand(['--predictions', 'gold'], { out: console.log, err: console.error, env: process.env });
+ * ```
+ *
+ * @param argv - the arguments.
+ * @param io - output streams and environment.
+ * @param options - the host's name, usage text, scratch directory and adapters.
+ * @returns the exit code ({@link KG_EVAL_EXIT}).
+ * @stability experimental
+ */
+async function runKgCommand(argv, io, options = {}) {
+    const name = options.name ?? 'kvox-eval kg';
+    const usage = options.usage ?? exports.KG_COMMAND_USAGE;
+    try {
+        return await run(argv, io, options, name, usage);
+    }
+    catch (err) {
+        if (err instanceof index_js_7.RealDataPathError) {
+            io.err(`${name}: ${err.message}`);
+            return exports.KG_EVAL_EXIT.pathRefused;
+        }
+        if (err instanceof UsageError || err instanceof index_js_3.KgEvalUsageError) {
+            io.err(`${name}: ${err.message}`);
+            io.err(usage);
+            return exports.KG_EVAL_EXIT.usage;
+        }
+        if (err instanceof index_js_1.KgEvalLoadError || err instanceof index_js_6.LiveProviderError || err instanceof LiveCallError) {
+            io.err(`${name}: ${err.message}`);
+            return exports.KG_EVAL_EXIT.usage;
+        }
+        throw err;
+    }
+}
+async function run(argv, io, options, name, usage) {
+    let values;
+    try {
+        ({ values } = (0, util_1.parseArgs)({
+            args: argv,
+            strict: true,
+            allowPositionals: false,
+            options: {
+                predictions: { type: 'string' },
+                runner: { type: 'string' },
+                run: { type: 'string' },
+                provider: { type: 'string' },
+                model: { type: 'string' },
+                'base-url': { type: 'string' },
+                out: { type: 'string' },
+                only: { type: 'string' },
+                tag: { type: 'string' },
+                json: { type: 'string' },
+                enforce: { type: 'boolean' },
+                'real-dir': { type: 'string' },
+                help: { type: 'boolean' },
+            },
+        }));
+    }
+    catch (err) {
+        throw new UsageError(err.message);
+    }
+    const str = (k) => (typeof values[k] === 'string' ? values[k] : undefined);
+    if (values.help) {
+        io.out(usage);
+        return exports.KG_EVAL_EXIT.ok;
+    }
+    // Real-data mode: the path guard runs before anything is read or written.
+    const realDirArg = str('real-dir') ?? (io.env.KG_EVAL_REAL_DIR || undefined);
+    const realDir = realDirArg ? (0, index_js_7.assertOutsideRepo)(realDirArg) : null;
+    if (str('runner') !== undefined && str('run') !== undefined)
+        throw new UsageError('pass --runner or --run, not both');
+    const runnerFlag = str('runner') !== undefined ? '--runner' : '--run';
+    const runnerName = str('runner') ?? str('run');
+    const predictionsArg = str('predictions');
+    if ((predictionsArg === undefined) === (runnerName === undefined)) {
+        throw new UsageError(`pass exactly one of --predictions <dir|gold> or ${runnerFlag} <runner>`);
+    }
+    // An unregistered runner is reported before anything is loaded or written.
+    const live = runnerName !== undefined ? (0, index_js_4.findKgEvalRunner)(runnerName) : null;
+    if (runnerName !== undefined && live === null) {
+        io.err(`${name}: no runner '${runnerName}' registered (known: ${Object.keys(index_js_4.KG_EVAL_RUNNERS).join(', ')})`);
+        return exports.KG_EVAL_EXIT.usage;
+    }
+    // The output guard runs before anything is loaded: a run over real data may
+    // never write inside the repository, not even into the git-ignored scratch dir.
+    let outDir = null;
+    if (live !== null) {
+        const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+        const synthetic = options.syntheticRunsDir ?? null;
+        const outArg = str('out') ??
+            (realDir ? (0, path_1.join)(realDir, 'runs', stamp) : synthetic ? (0, path_1.join)(synthetic, 'runs', stamp) : (0, path_1.join)((0, os_1.tmpdir)(), 'kvox-eval', 'runs', stamp));
+        outDir = (0, index_js_7.assertOutsideRepo)(outArg, { allowInside: realDir === null ? synthetic : null });
+    }
+    const only = str('only')
+        ?.split(',')
+        .map((s) => s.trim())
+        .filter(Boolean);
+    const selection = {
+        ...(only ? { fixtures: only } : {}),
+        ...(str('tag') !== undefined ? { tag: str('tag') } : {}),
+        ...(realDir ? { fixturesDir: realDir } : {}),
+    };
+    let report;
+    if (live !== null) {
+        // The provider is refused (copilot, an unknown id) before the key is looked for.
+        const provider = str('provider') ?? 'openai';
+        (0, index_js_6.assertLiveProvider)(provider);
+        const model = str('model');
+        if (!model)
+            throw new UsageError(`${runnerFlag} requires --model <id>`);
+        const key = (0, index_js_5.readEvalApiKey)(io.env, io.err);
+        if (key === null && provider === 'openai') {
+            throw new UsageError(`${runnerFlag} reads its API key from ${index_js_5.KVOX_EVAL_API_KEY_ENV}; it is not set`);
+        }
+        const baseUrl = str('base-url');
+        const { ai, failures } = (0, index_js_6.createLiveKvoxAi)({
+            provider,
+            model,
+            apiKey: key?.apiKey ?? null,
+            ...(baseUrl !== undefined ? { baseUrl } : {}),
+            ...(options.adapters !== undefined ? { adapters: options.adapters } : {}),
+        });
+        const dir = outDir;
+        (0, fs_1.mkdirSync)(dir, { recursive: true });
+        try {
+            report = await (0, index_js_3.runKgEval)({
+                runner: live.name,
+                ai,
+                model,
+                provider,
+                ...selection,
+                onPrediction: (fixtureId, pred) => (0, fs_1.writeFileSync)((0, path_1.join)(dir, `${fixtureId}.json`), `${JSON.stringify(pred, null, 2)}\n`),
+            });
+        }
+        catch (err) {
+            if (err instanceof ports_1.KvoxAiError)
+                throw new LiveCallError(describeLiveFailure(err, failures));
+            throw err;
+        }
+        io.err(`${name}: predictions written to ${dir}`);
+    }
+    else if (predictionsArg === 'gold') {
+        report = await (0, index_js_3.runKgEval)({ runner: 'gold', ...selection });
+    }
+    else {
+        report = await (0, index_js_3.runKgEval)({
+            predictionsDir: (0, path_1.resolve)(predictionsArg),
+            ...selection,
+            onMissingPrediction: (fixtureId, dir) => io.err(`${name}: no prediction for ${fixtureId} in ${dir} (scored as empty)`),
+        });
+    }
+    io.out((0, index_js_2.formatReport)(report));
+    const jsonPath = str('json');
+    if (jsonPath) {
+        const abs = (0, path_1.resolve)(jsonPath);
+        (0, fs_1.mkdirSync)((0, path_1.dirname)(abs), { recursive: true });
+        (0, fs_1.writeFileSync)(abs, `${JSON.stringify(report, null, 2)}\n`);
+    }
+    if (values.enforce && !report.targetsMet) {
+        io.err(`${name}: one or more §6 targets failed (--enforce)`);
+        return exports.KG_EVAL_EXIT.targetsFailed;
+    }
+    return exports.KG_EVAL_EXIT.ok;
+}
+/** `the live call failed (auth; AI_KEY_INVALID, HTTP 400)`: class, code, status. Never a message body. */
+function describeLiveFailure(err, failures) {
+    const last = failures[failures.length - 1];
+    const detail = last ? `; ${last.code}${last.status !== null ? `, HTTP ${last.status}` : ''}` : '';
+    return `the live call failed (${err.errorClass}${detail})`;
+}

package/dist/cli/main.d.ts

@@ -0,0 +1,18 @@
+#!/usr/bin/env node
+import { type EvalCommandIo, type KgCommandOptions } from './kg-command.js';
+/**
+ * The usage text of `kvox-eval`.
+ *
+ * @stability experimental
+ */
+export declare const KVOX_EVAL_USAGE = "usage: kvox-eval kg [flags]    score knowledge-graph extraction against the golden set\n       kvox-eval ask [flags]   run the Ask evaluation headless (in-memory database, live model)\n       kvox-eval ask --replay  the Ask harness's scripted self-test (no model, no key)\n       kvox-eval kg --help     the kg flags\n       kvox-eval ask --help    the ask flags";
+/**
+ * Runs `kvox-eval` with the given arguments (everything after the command name).
+ *
+ * @param argv - the arguments, the subcommand first.
+ * @param io - output streams and environment.
+ * @param options - passed to the kg command (the ask command takes its defaults).
+ * @returns the exit code.
+ * @stability experimental
+ */
+export declare function main(argv: string[], io?: EvalCommandIo, options?: KgCommandOptions): Promise<number>;

package/dist/cli/main.js

@@ -0,0 +1,69 @@
+#!/usr/bin/env node
+"use strict";
+// =============================================================================
+// The `kvox-eval` command (issue #580)
+// =============================================================================
+//
+//   kvox-eval kg  ...   score knowledge-graph extraction (see kg-command.ts)
+//   kvox-eval ask ...   the Ask evaluation, headless (see ask-command.ts): an
+//                       in-memory PostgreSQL, the golden graph, the real
+//                       `ask.respond` handler; `--replay` needs no model or key
+// =============================================================================
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KVOX_EVAL_USAGE = void 0;
+exports.main = main;
+const kg_command_js_1 = require("./kg-command.js");
+/**
+ * The usage text of `kvox-eval`.
+ *
+ * @stability experimental
+ */
+exports.KVOX_EVAL_USAGE = `usage: kvox-eval kg [flags]    score knowledge-graph extraction against the golden set
+       kvox-eval ask [flags]   run the Ask evaluation headless (in-memory database, live model)
+       kvox-eval ask --replay  the Ask harness's scripted self-test (no model, no key)
+       kvox-eval kg --help     the kg flags
+       kvox-eval ask --help    the ask flags`;
+/**
+ * Runs `kvox-eval` with the given arguments (everything after the command name).
+ *
+ * @param argv - the arguments, the subcommand first.
+ * @param io - output streams and environment.
+ * @param options - passed to the kg command (the ask command takes its defaults).
+ * @returns the exit code.
+ * @stability experimental
+ */
+async function main(argv, io = defaultIo(), options = {}) {
+    const [command, ...rest] = argv;
+    if (command === 'kg')
+        return (0, kg_command_js_1.runKgCommand)(rest, io, options);
+    if (command === 'ask') {
+        // Loaded only here: the ask command brings Nest, the domain slices, PGlite and
+        // the generated Prisma client, which `kvox-eval kg` and `--help` never need.
+        const { runAskCommand } = require('../ask-run/index.js');
+        return runAskCommand(rest, io);
+    }
+    if (command === '--help' || command === '-h' || command === 'help') {
+        io.out(exports.KVOX_EVAL_USAGE);
+        return kg_command_js_1.KG_EVAL_EXIT.ok;
+    }
+    io.err(command === undefined ? 'kvox-eval: no command given' : `kvox-eval: unknown command '${command}'`);
+    io.err(exports.KVOX_EVAL_USAGE);
+    return kg_command_js_1.KG_EVAL_EXIT.usage;
+}
+function defaultIo() {
+    return {
+        out: (s) => process.stdout.write(`${s}\n`),
+        err: (s) => process.stderr.write(`${s}\n`),
+        env: process.env,
+    };
+}
+if (require.main === module) {
+    main(process.argv.slice(2)).then((code) => {
+        process.exitCode = code;
+    }, (err) => {
+        // A KvoxAiError never reaches here (the kg command reports it by class);
+        // anything else is a bug, and its stack is the useful part.
+        process.stderr.write(`kvox-eval: ${err.stack ?? String(err)}\n`);
+        process.exitCode = kg_command_js_1.KG_EVAL_EXIT.usage;
+    });
+}

package/dist/fixtures/domains.d.ts

@@ -0,0 +1,16 @@
+import type { DomainKey } from '@marinoscar/kvox-contract/ontology';
+import type { GoldenFixture } from './fixture-schema.js';
+/**
+ * The tag that marks a fixture as written for a user with `personal` enabled.
+ *
+ * @stability experimental
+ */
+export declare const PERSONAL_FIXTURE_TAG = "personal";
+/**
+ * The ontology domains a fixture is extracted and validated against.
+ *
+ * @param fixture - the fixture (its tags).
+ * @returns `core ∪ work`, plus `personal` for a fixture tagged {@link PERSONAL_FIXTURE_TAG}.
+ * @stability experimental
+ */
+export declare function fixtureDomains(fixture: Pick<GoldenFixture, 'tags'>): DomainKey[];

package/dist/fixtures/domains.js

@@ -0,0 +1,33 @@
+"use strict";
+// =============================================================================
+// Which ontology domains a golden fixture is extracted and validated against
+// (issue #383). PURE.
+// =============================================================================
+//
+// Every fixture is `core ∪ work` — the default a user who never touched the
+// setting has. A fixture tagged `personal` is the story of a user who turned
+// the `personal` domain on (docs/specs/ontology.md §17.2, §16 P6), so it is
+// read against `core ∪ work ∪ personal`: its labels may use Interest, Trip,
+// Milestone and the personal relations, and an extraction run over it is
+// offered them. Keeping the default fixtures on `core ∪ work` is what proves a
+// user who never enables `personal` sees no personal type anywhere.
+// =============================================================================
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PERSONAL_FIXTURE_TAG = void 0;
+exports.fixtureDomains = fixtureDomains;
+/**
+ * The tag that marks a fixture as written for a user with `personal` enabled.
+ *
+ * @stability experimental
+ */
+exports.PERSONAL_FIXTURE_TAG = 'personal';
+/**
+ * The ontology domains a fixture is extracted and validated against.
+ *
+ * @param fixture - the fixture (its tags).
+ * @returns `core ∪ work`, plus `personal` for a fixture tagged {@link PERSONAL_FIXTURE_TAG}.
+ * @stability experimental
+ */
+function fixtureDomains(fixture) {
+    return fixture.tags.includes(exports.PERSONAL_FIXTURE_TAG) ? ['core', 'work', 'personal'] : ['core', 'work'];
+}