@marinoscar/kvox-eval 0.0.0-bootstrap → 0.1.0-next.0

package/dist/kg/runners/extract-runner.js

@@ -0,0 +1,243 @@
+"use strict";
+// =============================================================================
+// The `extract` kg:eval runner (#363; docs/specs/ontology.md §6)
+// =============================================================================
+//
+// Runs `kg.extract`'s own pure pipeline against one golden fixture, with NO
+// database: fixture → `ExtractionInput` → `buildExtractionContext` →
+// `assembleExtractionPrompt` + `buildExtractionOutputSchema` → ONE
+// `KvoxAiPort.generateStructured` call (`createKvoxAi` over the provider
+// adapters for a live run, a fake in a test) → `validateExtraction` → the deterministic Meeting
+// and ATTENDED rows → `applyPrecheck` (default preferences) → a prediction.
+//
+// It is the production code path minus the loader and the writer, so a score
+// measures the prompt, the schema and the validator the product ships.
+//
+// A fixture's speaker is treated as IDENTIFIED_AS the known Person whose label
+// or alias is its display name — what #356's speaker naming produces — so the
+// known-entity priority and the ATTENDED rows behave as they would in the app.
+// The golden set labels its own Meeting entity (key `meeting`); the runner
+// emits the deterministic Meeting row under the same ref.
+//
+// ⚠ The runner never sees a key: the port it is handed owns it (#580). A
+// prediction carries `model` alone.
+// =============================================================================
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EVAL_USER_ID = exports.EVAL_CONTEXT_WINDOW_TOKENS = exports.EVAL_TIMEOUT_MS = exports.EVAL_MAX_OUTPUT_TOKENS = void 0;
+exports.fixtureToInput = fixtureToInput;
+exports.rowsToPrediction = rowsToPrediction;
+exports.runExtraction = runExtraction;
+exports.precheckToPrediction = precheckToPrediction;
+exports.createExtractRunner = createExtractRunner;
+const ontology_1 = require("@marinoscar/kvox-contract/ontology");
+const zod_1 = require("zod");
+const internals_js_1 = require("../internals.js");
+const graph_extraction_1 = require("@marinoscar/kvox-api/graph-extraction");
+const graph_1 = require("@marinoscar/kvox-api/graph");
+const index_js_1 = require("../../fixtures/index.js");
+/**
+ * The completion ceiling an eval run uses (a production run takes it from policy).
+ *
+ * @stability experimental
+ */
+exports.EVAL_MAX_OUTPUT_TOKENS = 16_000;
+/**
+ * How long one structured call may take before the run gives up on it.
+ *
+ * @stability experimental
+ */
+exports.EVAL_TIMEOUT_MS = 180_000;
+/**
+ * The context window an eval target reports (the port's target carries one; a direct runtime never reads it).
+ *
+ * @stability experimental
+ */
+exports.EVAL_CONTEXT_WINDOW_TOKENS = 128_000;
+/**
+ * The user id an eval call carries (there is no signed-in user).
+ *
+ * @stability experimental
+ */
+exports.EVAL_USER_ID = 'kg-eval';
+/**
+ * The one fixture → input mapping. Pure.
+ *
+ * @stability experimental
+ */
+function fixtureToInput(fixture) {
+    const recordedAt = new Date(fixture.recordedAt);
+    const known = fixture.knownEntities.map((k) => ({
+        id: k.id,
+        type: k.type,
+        label: k.label,
+        aliases: [...k.aliases],
+        reviewStatus: 'accepted',
+        mergedIntoId: null,
+        orgLabel: null,
+    }));
+    const personFor = (name) => {
+        if (!name)
+            return null;
+        const n = (0, internals_js_1.normalizeForMatch)(name);
+        const hit = known.find((k) => k.type === 'Person' && [k.label, ...k.aliases].some((a) => (0, internals_js_1.normalizeForMatch)(a) === n));
+        return hit?.id ?? null;
+    };
+    const speakers = fixture.hasTranscript
+        ? fixture.speakers.map((s) => ({ id: s.id, label: s.label, displayName: s.displayName, personEntityId: personFor(s.displayName) }))
+        : [];
+    const linked = new Set(speakers.map((s) => s.personEntityId).filter((id) => id !== null));
+    return {
+        note: {
+            id: `${fixture.id}-note`,
+            title: fixture.title,
+            bodyAtVersion: fixture.note.body,
+            version: fixture.note.version,
+            contextText: fixture.contextText,
+            createdAt: recordedAt,
+        },
+        transcript: fixture.hasTranscript
+            ? { id: `${fixture.id}-transcript`, title: fixture.title, recordedAt, createdAt: recordedAt }
+            : null,
+        segments: fixture.hasTranscript
+            ? fixture.segments.map((s) => ({ id: s.id, rev: s.rev, startMs: s.startMs, endMs: s.endMs, speakerId: s.speakerId, text: s.text }))
+            : [],
+        speakers,
+        effectiveSchema: (0, ontology_1.computeEffectiveSchema)({ enabledDomains: (0, index_js_1.fixtureDomains)(fixture), userAttributes: [] }),
+        guidance: null,
+        knownEntityCandidates: {
+            pinned: [],
+            speakerPersons: known.filter((k) => linked.has(k.id)),
+            organizations: [],
+            contextPool: known,
+            recentlyMentioned: known,
+        },
+        existingMeeting: null,
+    };
+}
+const endpoint = (ref) => (ref === null ? null : 'entityId' in ref ? ref.entityId : ref.ref);
+const evidenceOf = (row) => row.evidence.map((e) => e.source === 'segment'
+    ? { source: 'segment', segmentId: e.segmentId, quote: e.quote }
+    : { source: 'note', segmentId: null, quote: e.quote });
+/**
+ * Proposal rows (with their pre-check decisions) → a kg:eval prediction. Pure.
+ *
+ * @stability experimental
+ */
+function rowsToPrediction(fixtureId, model, rows, decisions, stats) {
+    const prediction = { fixtureId, model, entities: [], relations: [], items: [], stats };
+    rows.forEach((row, i) => {
+        const prechecked = decisions[i] === 'accept';
+        if (row.kind === 'entity') {
+            const linked = row.resolution?.ref ?? null;
+            prediction.entities.push({
+                ref: row.payload.ref,
+                type: row.payload.type,
+                label: row.payload.label,
+                aliases: [...row.payload.aliases],
+                resolution: linked
+                    ? { outcome: 'linked', entityId: linked, score: row.resolution?.score ?? null, prechecked }
+                    : { outcome: 'new', entityId: null, score: null, prechecked },
+                evidence: evidenceOf(row),
+            });
+        }
+        else if (row.kind === 'relation') {
+            prediction.relations.push({
+                type: row.payload.type,
+                from: endpoint(row.payload.from),
+                to: endpoint(row.payload.to),
+                validFrom: row.payload.validFrom,
+                validTo: row.payload.validTo,
+                precision: row.payload.precision,
+                evidence: evidenceOf(row),
+            });
+        }
+        else {
+            prediction.items.push({
+                kind: row.payload.kind,
+                subject: endpoint(row.payload.subject) ?? '',
+                owner: endpoint(row.payload.owner),
+                counterparty: endpoint(row.payload.counterparty),
+                title: row.payload.title,
+                statement: row.payload.statement,
+                occurredAt: row.payload.occurredAt,
+                dueAt: row.payload.dueAt,
+                sensitivity: row.payload.sensitivity,
+                evidence: evidenceOf(row),
+            });
+        }
+    });
+    return prediction;
+}
+/**
+ * The shared half of every runner: one `generateStructured` call and the
+ * validator, returning the proposal rows before any pre-check. `null` when the
+ * model's answer fails validation (`invalidOutput`). #364's `extract+resolve`
+ * runner resolves these rows before the pre-check runs.
+  *
+  * @stability experimental
+ */
+async function runExtraction(ai, fixture, opts) {
+    const ctx = (0, internals_js_1.buildExtractionContext)(fixtureToInput(fixture));
+    // The row caps a production run at this ceiling would send (#435).
+    const caps = (0, internals_js_1.extractionRowCaps)(exports.EVAL_MAX_OUTPUT_TOKENS, 'none');
+    const prompt = (0, internals_js_1.assembleExtractionPrompt)(ctx, caps);
+    const schema = (0, internals_js_1.buildExtractionOutputSchema)(ctx, caps);
+    // `schema` is the pass-through validator: `validateExtraction` below is the
+    // product's own validation, exactly as in the `kg.extract` handler.
+    const result = await ai.generateStructured({ userId: exports.EVAL_USER_ID, orgId: null, feature: 'graph.extract', signal: AbortSignal.timeout(exports.EVAL_TIMEOUT_MS) }, {
+        target: {
+            provider: opts.provider ?? 'openai',
+            model: opts.model,
+            contextWindowTokens: exports.EVAL_CONTEXT_WINDOW_TOKENS,
+            maxOutputTokens: exports.EVAL_MAX_OUTPUT_TOKENS,
+            source: 'requested',
+        },
+        system: prompt.systemPrompt,
+        user: prompt.userContent,
+        schemaName: internals_js_1.EXTRACTION_SCHEMA_NAME,
+        schema: zod_1.z.unknown(),
+        jsonSchema: schema,
+        maxOutputTokens: exports.EVAL_MAX_OUTPUT_TOKENS,
+    });
+    const validated = (0, internals_js_1.validateExtraction)(result.value, ctx);
+    if (!validated.ok)
+        return null;
+    const final = (0, internals_js_1.addDeterministicRows)(ctx, validated);
+    return { rows: final.rows, stats: { ...final.stats.dropped, quoteNotLocated: final.stats.quoteNotLocated } };
+}
+const ALL_DOMAINS_REVIEW_ONLY = (0, graph_extraction_1.reviewOnlyTypes)((0, ontology_1.computeEffectiveSchema)({ enabledDomains: ['core', 'work', 'personal'], userAttributes: [] }));
+/**
+ * Pre-check the rows with default preferences and turn them into a prediction.
+ *
+ * @stability experimental
+ */
+function precheckToPrediction(fixtureId, model, rows, stats) {
+    const precheck = rows.map((row) => ({
+        kind: row.kind,
+        payload: row.payload,
+        resolution: row.resolution,
+        flags: row.flags,
+        decision: 'pending',
+    }));
+    // Every domain's review-only types: a type a fixture's domains do not
+    // include can never have been proposed, so listing it changes nothing.
+    (0, graph_extraction_1.applyPrecheck)(precheck, graph_1.GRAPH_PREFERENCE_DEFAULTS, { reviewOnlyTypes: ALL_DOMAINS_REVIEW_ONLY });
+    return rowsToPrediction(fixtureId, model, rows, precheck.map((p) => p.decision), stats);
+}
+/**
+ * The `extract` runner over `deps.ai`.
+ *
+ * @stability experimental
+ */
+function createExtractRunner(deps) {
+    return {
+        name: 'extract',
+        async run(fixture, opts) {
+            const extracted = await runExtraction(deps.ai, fixture, opts);
+            if (!extracted) {
+                return { fixtureId: fixture.id, model: opts.model, entities: [], relations: [], items: [], stats: { invalidOutput: 1 } };
+            }
+            return precheckToPrediction(fixture.id, opts.model, extracted.rows, extracted.stats);
+        },
+    };
+}

package/dist/kg/scorer.d.ts

@@ -0,0 +1,226 @@
+import type { GoldenFixture, ItemLabelKind, ValidPrecisionLabel } from '../fixtures/index.js';
+import type { KgEvalPrediction } from './prediction-schema.js';
+export { normalizeName } from './text.js';
+/**
+ * The token-set Jaccard a predicted label needs against a gold name to be a match candidate.
+ *
+ * @stability experimental
+ */
+export declare const ENTITY_MATCH_JACCARD = 0.8;
+/**
+ * The token-set F1 a predicted item statement needs against the gold statement.
+ *
+ * @stability experimental
+ */
+export declare const ITEM_STATEMENT_F1 = 0.5;
+/**
+ * Report rows, fixed order. Relation rows (`relation:<TYPE>`) follow, sorted.
+ *
+ * @stability experimental
+ */
+export declare const ENTITY_REPORT_TYPES: readonly ["Person", "Organization", "Project", "Meeting", "Interest", "Trip", "Milestone"];
+/**
+ * The report row of each item kind.
+ *
+ * @stability experimental
+ */
+export declare const ITEM_REPORT_TYPES: Record<ItemLabelKind, string>;
+/**
+ * Every fixed report row: the entity types, then the item kinds.
+ *
+ * @stability experimental
+ */
+export declare const BASE_REPORT_TYPES: string[];
+/**
+ * True positives, false positives and false negatives of one report row.
+ *
+ * @stability experimental
+ */
+export interface Counts {
+    tp: number;
+    fp: number;
+    fn: number;
+}
+/**
+ * One fixture's score: counts per type, entity coverage, auto-link precision, evidence
+ * validity, temporal accuracy and the prediction's dropped counts.
+ *
+ * @stability experimental
+ */
+export interface FixtureScore {
+    fixtureId: string;
+    types: Record<string, Counts>;
+    coverage: {
+        matched: number;
+        total: number;
+    };
+    /** `anyResolution`: at least one prediction carried a resolution at all. */
+    autoLink: {
+        correct: number;
+        total: number;
+        anyResolution: boolean;
+    };
+    evidence: {
+        valid: number;
+        total: number;
+    };
+    temporal: {
+        accurate: number;
+        total: number;
+    };
+    stats: Record<string, number>;
+}
+/**
+ * A run's score: the fixtures' scores summed, plus each fixture's own.
+ *
+ * @stability experimental
+ */
+export interface RunScore {
+    fixtures: number;
+    types: Record<string, Counts>;
+    coverage: {
+        matched: number;
+        total: number;
+    };
+    autoLink: {
+        correct: number;
+        total: number;
+        anyResolution: boolean;
+    };
+    evidence: {
+        valid: number;
+        total: number;
+    };
+    temporal: {
+        accurate: number;
+        total: number;
+    };
+    stats: Record<string, number>;
+    perFixture: FixtureScore[];
+}
+interface Named {
+    type: string;
+    names: Set<string>;
+    tokenSets: Set<string>[];
+}
+/**
+ * Similarity of one predicted label to one gold name set; 0 when not a candidate.
+ *
+ * @stability experimental
+ */
+export declare function nameSimilarity(predLabel: string, gold: Pick<Named, 'names' | 'tokenSets'>): number;
+/**
+ * The one-to-one assignment of predicted to gold entities (greedy by descending similarity).
+ *
+ * @param gold - the gold entities (label and aliases).
+ * @param preds - the predicted entities.
+ * @param opts - `typed`: candidates must share a type (entity coverage drops the check).
+ * @returns predicted index to gold index.
+ *
+ * @stability experimental
+ */
+export declare function matchEntities(gold: ReadonlyArray<{
+    type: string;
+    label: string;
+    aliases: readonly string[];
+}>, preds: ReadonlyArray<{
+    type: string;
+    label: string;
+}>, opts: {
+    typed: boolean;
+}): Map<number, number>;
+/**
+ * Whether two temporal ranges are equal at the gold precision (`unknown` matches only `unknown`).
+ *
+ * @param gold - the gold bounds and precision.
+ * @param pred - the predicted bounds and precision.
+ * @returns `true` when equal.
+ *
+ * @stability experimental
+ */
+export declare function temporalEqual(gold: {
+    validFrom: string | null;
+    validTo: string | null;
+    precision: ValidPrecisionLabel;
+}, pred: {
+    validFrom: string | null;
+    validTo: string | null;
+    precision: ValidPrecisionLabel;
+}): boolean;
+/**
+ * Scores one fixture's prediction. Pure and deterministic.
+ *
+ * @param fixture - the fixture and its labels.
+ * @param prediction - what a run predicted for it.
+ * @returns the fixture's score.
+ *
+ * @stability experimental
+ */
+export declare function scoreFixture(fixture: GoldenFixture, prediction: KgEvalPrediction): FixtureScore;
+/**
+ * Scores every fixture of a run (a fixture with no prediction scores as empty).
+ *
+ * @param fixtures - the fixtures.
+ * @param predictions - the predictions by fixture id.
+ * @returns the run's score.
+ *
+ * @stability experimental
+ */
+export declare function scoreRun(fixtures: readonly GoldenFixture[], predictions: ReadonlyMap<string, KgEvalPrediction>): RunScore;
+/**
+ * `num / den`, or `null` (printed `n/a`) when `den` is 0.
+ *
+ * @stability experimental
+ */
+export declare const ratio: (num: number, den: number) => number | null;
+/**
+ * Precision of one row, or `null` with nothing predicted.
+ *
+ * @param c - the row's counts.
+ * @returns the ratio or `null`.
+ *
+ * @stability experimental
+ */
+export declare function precisionOf(c: Counts): number | null;
+/**
+ * Recall of one row, or `null` with nothing labelled.
+ *
+ * @param c - the row's counts.
+ * @returns the ratio or `null`.
+ *
+ * @stability experimental
+ */
+export declare function recallOf(c: Counts): number | null;
+/**
+ * F1 of one row, or `null` when precision or recall is.
+ *
+ * @param c - the row's counts.
+ * @returns the ratio or `null`.
+ *
+ * @stability experimental
+ */
+export declare function f1Of(c: Counts): number | null;
+/**
+ * Gold entities matched by any prediction, type ignored, over all gold entities (the §6 entity-coverage target).
+ *
+ * @param run - the run's coverage counts.
+ * @returns the ratio or `null`.
+ *
+ * @stability experimental
+ */
+export declare function entityCoverage(run: Pick<RunScore, 'coverage'>): number | null;
+/**
+ * Recall of the Commitment row (the §6 commitment-recall target).
+ *
+ * @param run - the run's per-type counts.
+ * @returns the ratio or `null`.
+ *
+ * @stability experimental
+ */
+export declare function commitmentRecall(run: Pick<RunScore, 'types'>): number | null;
+/**
+ * `null` (printed `n/a`) when no prediction carries a resolution, or none was a prechecked link.
+ *
+ * @stability experimental
+ */
+export declare function autoLinkPrecision(run: Pick<RunScore, 'autoLink'>): number | null;