daftari 1.14.0 → 1.16.0

package/dist/curation/vault-docs.js

@@ -0,0 +1,90 @@
+// Shared vault-document loading and in-vault link helpers.
+//
+// Extracted from lint.ts so the same loader and link-resolution machinery can
+// back any curation surface that needs the full set of vault docs plus a
+// reverse-link view (lint, tension blast radius). Keeps the link-extraction
+// regexes and the path-normalisation rules in one place so the two callers
+// can't drift apart.
+import { posix } from "node:path";
+import { parseDocument } from "../frontmatter/parser.js";
+import { ok } from "../frontmatter/types.js";
+import { listFiles, readFile, resolveVaultPath } from "../storage/local.js";
+// Loads every markdown file under the vault root, returning frontmatter +
+// body for each. Files that fail to parse or to read are silently skipped —
+// the curation surface should never crash because a single file is malformed.
+export async function loadDocuments(vaultRoot) {
+    const list = await listFiles(vaultRoot);
+    if (!list.ok)
+        return list;
+    const docs = [];
+    for (const relPath of list.value) {
+        const resolved = resolveVaultPath(vaultRoot, relPath);
+        if (!resolved.ok)
+            continue;
+        const file = await readFile(resolved.value);
+        if (!file.ok)
+            continue;
+        const parsed = parseDocument(file.value);
+        if (!parsed.ok)
+            continue;
+        docs.push({
+            path: relPath,
+            frontmatter: parsed.value.frontmatter,
+            content: parsed.value.content,
+        });
+    }
+    return ok(docs);
+}
+// Pulls every internal link target out of a markdown body: both [[wikilinks]]
+// and [text](target) markdown links. External URLs and anchors are dropped.
+export function extractLinks(content) {
+    const targets = [];
+    for (const m of content.matchAll(/\[\[([^\]]+)\]\]/g)) {
+        // A wikilink may carry a |display alias and/or a #heading anchor.
+        const raw = m[1].split("|")[0]?.split("#")[0]?.trim();
+        if (raw)
+            targets.push(raw);
+    }
+    for (const m of content.matchAll(/\[[^\]]*\]\(([^)\s]+)\)/g)) {
+        const raw = m[1].split("#")[0]?.trim();
+        if (!raw)
+            continue;
+        if (/^(https?:|mailto:|#)/i.test(raw))
+            continue;
+        targets.push(raw);
+    }
+    return targets;
+}
+// Resolves a raw link target to a vault-relative path, or null if it points
+// nowhere. Tries, in order: the target as-is, with a .md suffix, resolved
+// relative to the linking file's directory, then a bare basename match (the
+// common [[note-name]] wikilink form).
+export function resolveLink(rawTarget, fromPath, byPath, byBasename) {
+    const withMd = (p) => (p.endsWith(".md") ? p : `${p}.md`);
+    if (byPath.has(rawTarget))
+        return rawTarget;
+    if (byPath.has(withMd(rawTarget)))
+        return withMd(rawTarget);
+    const relual = posix.normalize(posix.join(posix.dirname(fromPath), rawTarget));
+    if (byPath.has(relual))
+        return relual;
+    if (byPath.has(withMd(relual)))
+        return withMd(relual);
+    const base = posix.basename(rawTarget).replace(/\.md$/, "");
+    return byBasename.get(base) ?? null;
+}
+// Precomputes the two indexes resolveLink consults: the set of every known
+// vault-relative path, and the basename → path map used for bare-name
+// wikilinks. First write wins on basename collisions so the mapping is
+// deterministic across runs.
+export function buildPathIndexes(docs) {
+    const byPath = new Set(docs.map((d) => d.path));
+    const byBasename = new Map();
+    for (const d of docs) {
+        const base = posix.basename(d.path).replace(/\.md$/, "");
+        if (!byBasename.has(base))
+            byBasename.set(base, d.path);
+    }
+    return { byPath, byBasename };
+}
+//# sourceMappingURL=vault-docs.js.map

package/dist/curation/vault-docs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vault-docs.js","sourceRoot":"","sources":["../../src/curation/vault-docs.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,EAAE;AACF,8EAA8E;AAC9E,yEAAyE;AACzE,4EAA4E;AAC5E,2EAA2E;AAC3E,qBAAqB;AAErB,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAoB,EAAE,EAAe,MAAM,yBAAyB,CAAC;AAC5E,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAQ5E,0EAA0E;AAC1E,4EAA4E;AAC5E,8EAA8E;AAC9E,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,SAAiB;IACnD,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,CAAC;IACxC,IAAI,CAAC,IAAI,CAAC,EAAE;QAAE,OAAO,IAAI,CAAC;IAE1B,MAAM,IAAI,GAAgB,EAAE,CAAC;IAC7B,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,gBAAgB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACtD,IAAI,CAAC,QAAQ,CAAC,EAAE;YAAE,SAAS;QAC3B,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC5C,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,SAAS;QACvB,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,CAAC,EAAE;YAAE,SAAS;QACzB,IAAI,CAAC,IAAI,CAAC;YACR,IAAI,EAAE,OAAO;YACb,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW;YACrC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO;SAC9B,CAAC,CAAC;IACL,CAAC;IACD,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC;AAClB,CAAC;AAED,8EAA8E;AAC9E,4EAA4E;AAC5E,MAAM,UAAU,YAAY,CAAC,OAAe;IAC1C,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;QACtD,kEAAkE;QAClE,MAAM,GAAG,GAAI,CAAC,CAAC,CAAC,CAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QAClE,IAAI,GAAG;YAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,0BAA0B,CAAC,EAAE,CAAC;QAC7D,MAAM,GAAG,GAAI,CAAC,CAAC,CAAC,CAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QACnD,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,IAAI,uBAAuB,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAS;QAChD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpB,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,4EAA4E;AAC5E,0EAA0E;AAC1E,4EAA4E;AAC5E,uCAAuC;AACvC,MAAM,UAAU,WAAW,CACzB,SAAiB,EACjB,QAAgB,EAChB,MAAmB,EACnB,UAA+B;IAE/B,MAAM,MAAM,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAElE,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC;QAAE,OAAO,SAAS,CAAC;IAC5C,IAAI,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAAE,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC;IAE5D,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;IAC/E,IAAI,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;QAAE,OAAO,MAAM,CAAC;IACtC,IAAI,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAAE,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC;IAEtD,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAC5D,OAAO,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;AACtC,CAAC;AAED,2EAA2E;AAC3E,sEAAsE;AACtE,uEAAuE;AACvE,6BAA6B;AAC7B,MAAM,UAAU,gBAAgB,CAAC,IAAiB;IAIhD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAChD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACzD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;AAChC,CAAC"}

package/dist/eval/generate.d.ts

@@ -0,0 +1,12 @@
+import { type Result } from "../frontmatter/types.js";
+import type { LlmClient } from "./llm.js";
+import type { Subgraph } from "./subgraph.js";
+import { type CortexEvalError, type QuestionSet } from "./types.js";
+export interface GenerateOptions {
+    n: number;
+    model: string;
+    vaultHash?: string;
+    seed?: string;
+}
+export declare function generateQuestions(subgraph: Subgraph, llm: LlmClient, opts: GenerateOptions): Promise<Result<QuestionSet, CortexEvalError>>;
+//# sourceMappingURL=generate.d.ts.map

package/dist/eval/generate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../src/eval/generate.ts"],"names":[],"mappings":"AAUA,OAAO,EAAW,KAAK,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAE1C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EACL,KAAK,eAAe,EAEpB,KAAK,WAAW,EAIjB,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,eAAe;IAC9B,CAAC,EAAE,MAAM,CAAC;IACV,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,QAAQ,EAClB,GAAG,EAAE,SAAS,EACd,IAAI,EAAE,eAAe,GACpB,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC,CA0E/C"}

package/dist/eval/generate.js

@@ -0,0 +1,221 @@
+// Question-set generation for the cortex quality metric.
+//
+// Given a sampled subgraph, ask the generator LLM for multi-hop questions
+// across three tiers, then validate/filter the output: drop questions whose
+// expected_sources escape the subgraph, and drop trivial yes/no answers. If the
+// generator under-produces any tier, make ONE focused top-up call (spec §5.3) —
+// remaining imbalance is accepted and recorded, not treated as an error.
+// Finally, augment the contradiction tier from logged tension edges.
+import { createHash } from "node:crypto";
+import { err, ok } from "../frontmatter/types.js";
+import { GENERATOR_PROMPT, PROMPT_VERSION } from "./prompts.js";
+import { QuestionSetSchema, TIERS, } from "./types.js";
+export async function generateQuestions(subgraph, llm, opts) {
+    const perTier = Math.floor(opts.n / TIERS.length);
+    const tierCountsRequested = {
+        retrieval: perTier,
+        cross_reference: perTier,
+        contradiction: perTier,
+    };
+    const validNodes = new Set(subgraph.nodes.map((n) => n.path));
+    // [FIRST GENERATION] full per-tier counts. A failure here is fatal.
+    const firstRes = await llm.completeJson({
+        model: opts.model,
+        system: GENERATOR_PROMPT,
+        user: renderUserPrompt(subgraph, tierCountsRequested),
+        schema: QuestionSetSchema,
+    });
+    if (!firstRes.ok)
+        return firstRes;
+    const firstRaw = extractQuestions(firstRes.value.parsed);
+    if (firstRaw === null) {
+        return err({
+            kind: "llm",
+            message: "generator returned non-conforming JSON",
+            retryable: false,
+        });
+    }
+    const generated = validateAndMap(firstRaw, validNodes);
+    // [TOP-UP] Shortfall is measured on GENERATOR output only (pre-augmentation).
+    const shortfall = computeShortfall(tierCountsRequested, generated);
+    let merged = generated;
+    if (hasShortfall(shortfall)) {
+        // Exactly ONE top-up call. A failure or non-conforming output is non-fatal:
+        // we swallow it (zero additional questions) and accept the imbalance.
+        const topUpRes = await llm.completeJson({
+            model: opts.model,
+            system: GENERATOR_PROMPT,
+            user: renderTopUpPrompt(subgraph, shortfall),
+            schema: QuestionSetSchema,
+        });
+        if (topUpRes.ok) {
+            const topUpRaw = extractQuestions(topUpRes.value.parsed);
+            if (topUpRaw !== null) {
+                const topUp = validateAndMap(topUpRaw, validNodes);
+                merged = dedupeById(generated, topUp);
+            }
+        }
+    }
+    // [AUGMENT] Additive contradiction questions from tension edges, deduped by id.
+    const augmented = augmentFromTensions(subgraph, tierCountsRequested.contradiction);
+    const all = dedupeById(merged, augmented);
+    const tierCountsProduced = countByTier(all);
+    const ts = "2026-01-01T00:00:00Z"; // placeholder; the caller overwrites this.
+    const vaultHash = opts.vaultHash ?? "";
+    const seed = opts.seed ?? "";
+    return ok({
+        id: `${vaultHash}-${seed}-${ts}`,
+        vault_hash: vaultHash,
+        seed,
+        timestamp: ts,
+        subgraph: {
+            seed_doc: subgraph.seed_doc,
+            nodes: subgraph.nodes.map((n) => n.path),
+            edges: subgraph.edges,
+        },
+        questions: all,
+        generator_model: opts.model,
+        prompt_version: PROMPT_VERSION,
+        tier_counts_requested: tierCountsRequested,
+        tier_counts_produced: tierCountsProduced,
+    });
+}
+// Pulls the `questions` array out of parsed generator JSON, or null if the
+// shape is non-conforming (missing/!array). The element shapes are validated
+// later by validateAndMap.
+function extractQuestions(parsed) {
+    if (typeof parsed !== "object" || parsed === null)
+        return null;
+    // biome-ignore lint/suspicious/noExplicitAny: structural access to parsed JSON
+    const questions = parsed.questions;
+    if (!Array.isArray(questions))
+        return null;
+    return questions;
+}
+// Validates each raw question and maps survivors to `Question` records with a
+// stable id and origin "generated". Reused for both the first call and the
+// top-up call. Rejects: bad tier, empty question/answer, empty or
+// out-of-subgraph expected_sources, and trivial yes/no answers.
+function validateAndMap(rawQuestions, validNodes) {
+    const out = [];
+    for (const raw of rawQuestions) {
+        if (typeof raw !== "object" || raw === null)
+            continue;
+        // biome-ignore lint/suspicious/noExplicitAny: structural access to parsed JSON
+        const q = raw;
+        const tier = q.tier;
+        if (!isTier(tier))
+            continue;
+        const question = q.question;
+        const answer = q.expected_answer;
+        if (typeof question !== "string" || question.trim().length === 0)
+            continue;
+        if (typeof answer !== "string" || answer.trim().length === 0)
+            continue;
+        const sources = q.expected_sources;
+        if (!Array.isArray(sources) || sources.length === 0)
+            continue;
+        if (!sources.every((s) => typeof s === "string" && validNodes.has(s)))
+            continue;
+        if (isTrivial(question, answer))
+            continue;
+        const expectedSources = sources;
+        out.push({
+            id: questionId(tier, question, expectedSources),
+            tier,
+            question,
+            expected_answer: answer,
+            expected_sources: expectedSources,
+            origin: "generated",
+        });
+    }
+    return out;
+}
+function isTier(value) {
+    return typeof value === "string" && TIERS.includes(value);
+}
+// Per-tier shortfall against the requested counts, measured on the supplied
+// (generator-only) question set. Never negative.
+function computeShortfall(requested, produced) {
+    const counts = countByTier(produced);
+    const shortfall = {
+        retrieval: 0,
+        cross_reference: 0,
+        contradiction: 0,
+    };
+    for (const tier of TIERS) {
+        shortfall[tier] = Math.max(0, requested[tier] - counts[tier]);
+    }
+    return shortfall;
+}
+function hasShortfall(shortfall) {
+    return TIERS.some((tier) => shortfall[tier] > 0);
+}
+function countByTier(questions) {
+    const counts = {
+        retrieval: 0,
+        cross_reference: 0,
+        contradiction: 0,
+    };
+    for (const q of questions)
+        counts[q.tier] += 1;
+    return counts;
+}
+// Merges `base` with `extra`, dropping any `extra` whose id already appears in
+// `base` (or earlier in `extra`). Identical-text/source questions within a tier
+// hash to the same id and collapse to one.
+function dedupeById(base, extra) {
+    const seen = new Set(base.map((q) => q.id));
+    const out = [...base];
+    for (const q of extra) {
+        if (seen.has(q.id))
+            continue;
+        seen.add(q.id);
+        out.push(q);
+    }
+    return out;
+}
+function renderUserPrompt(sg, counts) {
+    const docs = sg.nodes.map((n) => `## ${n.path}\n\n${n.body}\n`).join("\n");
+    return `Subgraph docs:\n\n${docs}\n\nProduce exactly ${counts.retrieval} retrieval, ${counts.cross_reference} cross_reference, and ${counts.contradiction} contradiction questions.`;
+}
+// Top-up prompt: lists ONLY the under-produced tiers and their shortfall counts.
+// Tiers with shortfall 0 are omitted entirely.
+function renderTopUpPrompt(sg, shortfall) {
+    const docs = sg.nodes.map((n) => `## ${n.path}\n\n${n.body}\n`).join("\n");
+    const wanted = TIERS.filter((tier) => shortfall[tier] > 0)
+        .map((tier) => `${shortfall[tier]} more ${tier}`)
+        .join(" and ");
+    return `Subgraph docs:\n\n${docs}\n\nYou previously produced too few of some tiers. Produce exactly ${wanted} questions, same rules. Do not produce any other tiers.`;
+}
+function isTrivial(_question, answer) {
+    const a = answer.trim().toLowerCase();
+    return a === "yes" || a === "no" || a.length < 3;
+}
+function questionId(tier, question, sources) {
+    const h = createHash("sha256");
+    h.update(`${tier}\x00${question}\x00${[...sources].sort().join("\x00")}`);
+    return h.digest("hex").slice(0, 16);
+}
+// Additive contradiction questions seeded from logged tension edges in the
+// subgraph: max(1, floor(0.2 × contradictionBudget)) of them, capped by how many
+// tension edges exist. Origin "augmented" (no generator LLM involved).
+function augmentFromTensions(sg, contradictionBudget) {
+    const tensionEdges = sg.edges.filter((e) => e.kind === "tension");
+    if (tensionEdges.length === 0)
+        return [];
+    const count = Math.max(1, Math.floor(0.2 * contradictionBudget));
+    return tensionEdges.slice(0, count).map((e) => {
+        const q = `${e.from} and ${e.to} appear to disagree on a specific point. Read both docs, identify the disagreement, and cite the position each takes. Cite both docs in your answer.`;
+        const sources = [e.from, e.to];
+        return {
+            id: questionId("contradiction", q, sources),
+            tier: "contradiction",
+            question: q,
+            expected_answer: `A correct answer identifies the substantive contradiction between ${e.from} and ${e.to} and cites both source paths.`,
+            expected_sources: sources,
+            origin: "augmented",
+        };
+    });
+}
+//# sourceMappingURL=generate.js.map

package/dist/eval/generate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate.js","sourceRoot":"","sources":["../../src/eval/generate.ts"],"names":[],"mappings":"AAAA,yDAAyD;AACzD,EAAE;AACF,0EAA0E;AAC1E,4EAA4E;AAC5E,gFAAgF;AAChF,gFAAgF;AAChF,yEAAyE;AACzE,qEAAqE;AAErE,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,GAAG,EAAE,EAAE,EAAe,MAAM,yBAAyB,CAAC;AAE/D,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEhE,OAAO,EAIL,iBAAiB,EACjB,KAAK,GAEN,MAAM,YAAY,CAAC;AASpB,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,QAAkB,EAClB,GAAc,EACd,IAAqB;IAErB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IAClD,MAAM,mBAAmB,GAAyB;QAChD,SAAS,EAAE,OAAO;QAClB,eAAe,EAAE,OAAO;QACxB,aAAa,EAAE,OAAO;KACvB,CAAC;IAEF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAE9D,oEAAoE;IACpE,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,YAAY,CAAC;QACtC,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,MAAM,EAAE,gBAAgB;QACxB,IAAI,EAAE,gBAAgB,CAAC,QAAQ,EAAE,mBAAmB,CAAC;QACrD,MAAM,EAAE,iBAAiB;KAC1B,CAAC,CAAC;IACH,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,OAAO,QAAQ,CAAC;IAClC,MAAM,QAAQ,GAAG,gBAAgB,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACzD,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QACtB,OAAO,GAAG,CAAC;YACT,IAAI,EAAE,KAAK;YACX,OAAO,EAAE,wCAAwC;YACjD,SAAS,EAAE,KAAK;SACjB,CAAC,CAAC;IACL,CAAC;IACD,MAAM,SAAS,GAAG,cAAc,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAEvD,8EAA8E;IAC9E,MAAM,SAAS,GAAG,gBAAgB,CAAC,mBAAmB,EAAE,SAAS,CAAC,CAAC;IACnE,IAAI,MAAM,GAAG,SAAS,CAAC;IACvB,IAAI,YAAY,CAAC,SAAS,CAAC,EAAE,CAAC;QAC5B,4EAA4E;QAC5E,sEAAsE;QACtE,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,YAAY,CAAC;YACtC,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,gBAAgB;YACxB,IAAI,EAAE,iBAAiB,CAAC,QAAQ,EAAE,SAAS,CAAC;YAC5C,MAAM,EAAE,iBAAiB;SAC1B,CAAC,CAAC;QACH,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;YAChB,MAAM,QAAQ,GAAG,gBAAgB,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACzD,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;gBACnD,MAAM,GAAG,UAAU,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;IACH,CAAC;IAED,gFAAgF;IAChF,MAAM,SAAS,GAAG,mBAAmB,CAAC,QAAQ,EAAE,mBAAmB,CAAC,aAAa,CAAC,CAAC;IACnF,MAAM,GAAG,GAAG,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAE1C,MAAM,kBAAkB,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;IAC5C,MAAM,EAAE,GAAG,sBAAsB,CAAC,CAAC,2CAA2C;IAC9E,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;IACvC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;IAE7B,OAAO,EAAE,CAAC;QACR,EAAE,EAAE,GAAG,SAAS,IAAI,IAAI,IAAI,EAAE,EAAE;QAChC,UAAU,EAAE,SAAS;QACrB,IAAI;QACJ,SAAS,EAAE,EAAE;QACb,QAAQ,EAAE;YACR,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;YACxC,KAAK,EAAE,QAAQ,CAAC,KAAK;SACtB;QACD,SAAS,EAAE,GAAG;QACd,eAAe,EAAE,IAAI,CAAC,KAAK;QAC3B,cAAc,EAAE,cAAc;QAC9B,qBAAqB,EAAE,mBAAmB;QAC1C,oBAAoB,EAAE,kBAAkB;KACzC,CAAC,CAAC;AACL,CAAC;AAED,2EAA2E;AAC3E,6EAA6E;AAC7E,2BAA2B;AAC3B,SAAS,gBAAgB,CAAC,MAAe;IACvC,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAC/D,+EAA+E;IAC/E,MAAM,SAAS,GAAI,MAAc,CAAC,SAAS,CAAC;IAC5C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3C,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,8EAA8E;AAC9E,2EAA2E;AAC3E,kEAAkE;AAClE,gEAAgE;AAChE,SAAS,cAAc,CAAC,YAAuB,EAAE,UAAuB;IACtE,MAAM,GAAG,GAAe,EAAE,CAAC;IAC3B,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;QAC/B,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI;YAAE,SAAS;QACtD,+EAA+E;QAC/E,MAAM,CAAC,GAAG,GAAU,CAAC;QACrB,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;QACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YAAE,SAAS;QAC5B,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC;QAC5B,MAAM,MAAM,GAAG,CAAC,CAAC,eAAe,CAAC;QACjC,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAC3E,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QACvE,MAAM,OAAO,GAAG,CAAC,CAAC,gBAAgB,CAAC;QACnC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAC9D,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAAE,SAAS;QAChF,IAAI,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC;YAAE,SAAS;QAC1C,MAAM,eAAe,GAAG,OAAmB,CAAC;QAC5C,GAAG,CAAC,IAAI,CAAC;YACP,EAAE,EAAE,UAAU,CAAC,IAAI,EAAE,QAAQ,EAAE,eAAe,CAAC;YAC/C,IAAI;YACJ,QAAQ;YACR,eAAe,EAAE,MAAM;YACvB,gBAAgB,EAAE,eAAe;YACjC,MAAM,EAAE,WAAW;SACpB,CAAC,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,MAAM,CAAC,KAAc;IAC5B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAK,KAA2B,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;AACnF,CAAC;AAED,4EAA4E;AAC5E,iDAAiD;AACjD,SAAS,gBAAgB,CACvB,SAA+B,EAC/B,QAAoB;IAEpB,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;IACrC,MAAM,SAAS,GAAyB;QACtC,SAAS,EAAE,CAAC;QACZ,eAAe,EAAE,CAAC;QAClB,aAAa,EAAE,CAAC;KACjB,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,SAAS,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,YAAY,CAAC,SAA+B;IACnD,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,WAAW,CAAC,SAAqB;IACxC,MAAM,MAAM,GAAyB;QACnC,SAAS,EAAE,CAAC;QACZ,eAAe,EAAE,CAAC;QAClB,aAAa,EAAE,CAAC;KACjB,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,SAAS;QAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/C,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,+EAA+E;AAC/E,gFAAgF;AAChF,2CAA2C;AAC3C,SAAS,UAAU,CAAC,IAAgB,EAAE,KAAiB;IACrD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5C,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACtB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAAE,SAAS;QAC7B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACf,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACd,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,gBAAgB,CAAC,EAAY,EAAE,MAA4B;IAClE,MAAM,IAAI,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3E,OAAO,qBAAqB,IAAI,uBAAuB,MAAM,CAAC,SAAS,eAAe,MAAM,CAAC,eAAe,yBAAyB,MAAM,CAAC,aAAa,2BAA2B,CAAC;AACvL,CAAC;AAED,iFAAiF;AACjF,+CAA+C;AAC/C,SAAS,iBAAiB,CAAC,EAAY,EAAE,SAA+B;IACtE,MAAM,IAAI,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3E,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SACvD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;SAChD,IAAI,CAAC,OAAO,CAAC,CAAC;IACjB,OAAO,qBAAqB,IAAI,sEAAsE,MAAM,yDAAyD,CAAC;AACxK,CAAC;AAED,SAAS,SAAS,CAAC,SAAiB,EAAE,MAAc;IAClD,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACtC,OAAO,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,UAAU,CAAC,IAAY,EAAE,QAAgB,EAAE,OAAiB;IACnE,MAAM,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC,CAAC,MAAM,CAAC,GAAG,IAAI,OAAO,QAAQ,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC1E,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACtC,CAAC;AAED,2EAA2E;AAC3E,iFAAiF;AACjF,uEAAuE;AACvE,SAAS,mBAAmB,CAAC,EAAY,EAAE,mBAA2B;IACpE,MAAM,YAAY,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;IAClE,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,mBAAmB,CAAC,CAAC,CAAC;IACjE,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC5C,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,EAAE,sJAAsJ,CAAC;QACtL,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QAC/B,OAAO;YACL,EAAE,EAAE,UAAU,CAAC,eAAe,EAAE,CAAC,EAAE,OAAO,CAAC;YAC3C,IAAI,EAAE,eAAwB;YAC9B,QAAQ,EAAE,CAAC;YACX,eAAe,EAAE,qEAAqE,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,EAAE,+BAA+B;YACvI,gBAAgB,EAAE,OAAO;YACzB,MAAM,EAAE,WAAoB;SAC7B,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/eval/index.d.ts

@@ -0,0 +1,2 @@
+export declare function runEval(argv: string[]): Promise<number>;
+//# sourceMappingURL=index.d.ts.map

package/dist/eval/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AA6DA,wBAAsB,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAuB7D"}

package/dist/eval/index.js

@@ -0,0 +1,311 @@
+// src/eval/index.ts
+// Top-level CLI dispatcher for `daftari eval`. Parses flags, routes to
+// generate/run/score/top-level, translates Result<T, CortexEvalError> to exit
+// codes (2 = config, 3 = runtime/llm).
+import { createHash } from "node:crypto";
+import { resolve } from "node:path";
+import { generateQuestions } from "./generate.js";
+import { createAnthropicClient } from "./llm.js";
+import { PROMPT_VERSION } from "./prompts.js";
+import { runAnswerer } from "./run.js";
+import { aggregateScore, gradeAnswer } from "./score.js";
+import { appendHistory, readQuestionSet, readResults, writeQuestionSet, writeResults, writeScore, } from "./storage.js";
+import { sampleSubgraph } from "./subgraph.js";
+import { SPEC_VERSION, TIERS, } from "./types.js";
+const HELP = `daftari eval — cortex quality metric.
+
+Usage:
+  daftari eval [--vault <path>] [--n <count>] [--k <count>] [--seed <str>]
+  daftari eval generate [--vault <path>] [--n <count>] [--seed <str>]
+  daftari eval run      [--questions <id>] [--vault <path>] [--model <id>] [--k <count>] [--resume <results-id>]
+  daftari eval score    [--results <id>] [--vault <path>] [--grader-model <id>]
+
+  (--questions and --results take the artifact id printed by a prior stage,
+   not a file path; artifacts live under .daftari/eval/.)
+
+Defaults:
+  --n 15      total questions across three tiers (5 each)
+  --k 2       runs per question for variance estimation
+  --model     claude-sonnet-4-6 (DEFAULT_MODEL in src/eval/index.ts)
+  --vault     current working directory
+
+Environment:
+  ANTHROPIC_API_KEY   required for any LLM-mediated stage
+
+Disk usage:
+  .daftari/eval/results/ and scores/ grow without bound across runs. v1
+  recovery is a manual rm -rf .daftari/eval/results/; rerunning regenerates
+  what's needed. A daftari eval prune command is the planned v2 follow-up.
+
+Exit codes:
+  0 — eval completed
+  2 — config error (missing API key, bad flags, no vault)
+  3 — runtime/LLM error (retries exhausted, vault I/O failure)
+`;
+export async function runEval(argv) {
+    if (argv.includes("--help") || argv.includes("-h")) {
+        process.stdout.write(HELP);
+        return 0;
+    }
+    // runEval is the CLI boundary and must not throw: a bad integer flag
+    // (intFlag) or any unexpected error becomes a config-error exit code (2).
+    try {
+        const [mode, ...rest] = argv;
+        switch (mode) {
+            case "generate":
+                return await runGenerate(rest);
+            case "run":
+                return await runRun(rest);
+            case "score":
+                return await runScore(rest);
+            default:
+                return await runTopLevel(argv);
+        }
+    }
+    catch (e) {
+        process.stderr.write(`${e instanceof Error ? e.message : String(e)}\n`);
+        return 2;
+    }
+}
+function flag(argv, name) {
+    const i = argv.indexOf(`--${name}`);
+    if (i < 0 || i + 1 >= argv.length)
+        return undefined;
+    return argv[i + 1];
+}
+function intFlag(argv, name, def) {
+    const v = flag(argv, name);
+    if (v === undefined)
+        return def;
+    const n = parseInt(v, 10);
+    if (Number.isNaN(n))
+        throw new Error(`--${name} must be an integer`);
+    return n;
+}
+function vaultHash(vault) {
+    return createHash("sha256").update(resolve(vault)).digest("hex").slice(0, 12);
+}
+function defaultSeed(vault) {
+    const today = new Date().toISOString().slice(0, 10);
+    return `${vaultHash(vault)}-${today}`;
+}
+const DEFAULT_MODEL = "claude-sonnet-4-6";
+async function runGenerate(argv) {
+    if (!process.env.ANTHROPIC_API_KEY) {
+        process.stderr.write("ANTHROPIC_API_KEY required\n");
+        return 2;
+    }
+    const vault = flag(argv, "vault") ?? process.cwd();
+    const n = intFlag(argv, "n", 15);
+    const seed = flag(argv, "seed") ?? defaultSeed(vault);
+    const sg = await sampleSubgraph(vault, seed, { maxNodes: 5 });
+    if (!sg.ok) {
+        process.stderr.write(`${sg.error.message}\n`);
+        return 3;
+    }
+    const client = createAnthropicClient();
+    const qs = await generateQuestions(sg.value, client, {
+        n,
+        model: DEFAULT_MODEL,
+        vaultHash: vaultHash(vault),
+        seed,
+    });
+    if (!qs.ok) {
+        process.stderr.write(`${qs.error.message}\n`);
+        return 3;
+    }
+    qs.value.timestamp = new Date().toISOString();
+    qs.value.id = `${qs.value.vault_hash}-${qs.value.seed}-${qs.value.timestamp}`;
+    await writeQuestionSet(vault, qs.value);
+    process.stdout.write(`wrote question set ${qs.value.id} (${qs.value.questions.length} questions)\n`);
+    return 0;
+}
+async function runRun(argv) {
+    if (!process.env.ANTHROPIC_API_KEY) {
+        process.stderr.write("ANTHROPIC_API_KEY required\n");
+        return 2;
+    }
+    const vault = flag(argv, "vault") ?? process.cwd();
+    const questionsId = flag(argv, "questions");
+    if (!questionsId) {
+        process.stderr.write("--questions required\n");
+        return 2;
+    }
+    const k = intFlag(argv, "k", 2);
+    const model = flag(argv, "model") ?? DEFAULT_MODEL;
+    const qsRead = await readQuestionSet(vault, questionsId);
+    if (!qsRead.ok) {
+        process.stderr.write(`${qsRead.error.message}\n`);
+        return 3;
+    }
+    let resumeFrom;
+    const resumeId = flag(argv, "resume");
+    if (resumeId) {
+        const r = await readResults(vault, resumeId);
+        if (r.ok)
+            resumeFrom = r.value;
+    }
+    // Mint the stable id + timestamp up front so the on-disk file path is stable
+    // across the run and any later --resume; persist incrementally so a mid-run
+    // failure leaves a resumable partial file.
+    const timestamp = new Date().toISOString();
+    const runId = resumeFrom ? resumeFrom.id : `${qsRead.value.id}-${model}-${timestamp}`;
+    const client = createAnthropicClient();
+    const run = await runAnswerer(qsRead.value, vault, client, {
+        k,
+        model,
+        resumeFrom,
+        runId,
+        timestamp,
+        persist: (r) => writeResults(vault, r),
+    });
+    if (!run.ok) {
+        process.stderr.write(`${run.error.message}\n`);
+        process.stderr.write(`partial results saved as ${runId}; resume with: daftari eval run --questions ${questionsId} --resume ${runId}\n`);
+        return 3;
+    }
+    await writeResults(vault, run.value); // final write (covers the zero-question edge where persist never fired)
+    process.stdout.write(`wrote results ${run.value.id}\n`);
+    return 0;
+}
+async function runScore(argv) {
+    const vault = flag(argv, "vault") ?? process.cwd();
+    const resultsId = flag(argv, "results");
+    if (!resultsId) {
+        process.stderr.write("--results required\n");
+        return 2;
+    }
+    const graderModel = flag(argv, "grader-model") ?? DEFAULT_MODEL;
+    if (!process.env.ANTHROPIC_API_KEY) {
+        process.stderr.write("ANTHROPIC_API_KEY required\n");
+        return 2;
+    }
+    const runRead = await readResults(vault, resultsId);
+    if (!runRead.ok) {
+        process.stderr.write(`${runRead.error.message}\n`);
+        return 3;
+    }
+    const run = runRead.value;
+    const qsRead = await readQuestionSet(vault, run.questions_id);
+    if (!qsRead.ok) {
+        process.stderr.write(`${qsRead.error.message}\n`);
+        return 3;
+    }
+    const qs = qsRead.value;
+    const grader = createAnthropicClient();
+    const grades = [];
+    const traces = new Map();
+    for (const [, pr] of Object.entries(run.runs)) {
+        if (pr.status !== "complete" || !pr.trace)
+            continue;
+        const q = qs.questions[pr.question_index];
+        if (!q)
+            continue;
+        const g = await gradeAnswer(q, pr.question_index, pr.k_index, pr.trace, grader, {
+            model: graderModel,
+        });
+        if (g.ok) {
+            grades.push(g.value);
+            traces.set(`${q.id}:${pr.k_index}`, pr.trace);
+        }
+    }
+    const score = aggregateScore(grades, qs.questions, { traces });
+    score.models = {
+        generator: qs.generator_model,
+        answerer: run.answerer_model,
+        grader: graderModel,
+    };
+    score.prompt_version = PROMPT_VERSION;
+    score.spec_version = SPEC_VERSION;
+    score.questions_id = qs.id;
+    score.results_id = run.id;
+    score.vault_hash = qs.vault_hash;
+    score.k = run.k;
+    score.n = qs.questions.length;
+    score.timestamp = new Date().toISOString();
+    await writeScore(vault, score);
+    const histEntry = {
+        score_id: score.results_id,
+        score: score.score,
+        score_std: score.score_std,
+        by_tier: {
+            retrieval: score.by_tier.retrieval.mean,
+            cross_reference: score.by_tier.cross_reference.mean,
+            contradiction: score.by_tier.contradiction.mean,
+        },
+        vault_hash: score.vault_hash,
+        timestamp: score.timestamp,
+        n: score.n,
+        k: score.k,
+        models: score.models,
+        prompt_version: score.prompt_version,
+        spec_version: score.spec_version,
+    };
+    await appendHistory(vault, histEntry);
+    // Pretty-print headline + per-tier means.
+    process.stdout.write(`score: ${score.score.toFixed(3)} ± ${score.score_std.toFixed(3)}\n`);
+    for (const t of TIERS) {
+        const ts = score.by_tier[t];
+        process.stdout.write(`  ${t.padEnd(16)}: ${ts.mean.toFixed(3)} (n=${ts.n}, efficiency=${ts.trace_efficiency.toFixed(1)} calls)\n`);
+    }
+    return 0;
+}
+async function runTopLevel(argv) {
+    // Spec §3 "Top-level convenience": runs generate → run → score in one shot.
+    // We thread the IDs in-memory rather than re-reading from disk, so a
+    // failure mid-pipeline still leaves the on-disk artifacts that did
+    // succeed for forensic / resume use.
+    if (!process.env.ANTHROPIC_API_KEY) {
+        process.stderr.write("ANTHROPIC_API_KEY required\n");
+        return 2;
+    }
+    const vault = flag(argv, "vault") ?? process.cwd();
+    const n = intFlag(argv, "n", 15);
+    const k = intFlag(argv, "k", 2);
+    const seed = flag(argv, "seed") ?? defaultSeed(vault);
+    const model = flag(argv, "model") ?? DEFAULT_MODEL;
+    // 1. Generate
+    const sg = await sampleSubgraph(vault, seed, { maxNodes: 5 });
+    if (!sg.ok) {
+        process.stderr.write(`${sg.error.message}\n`);
+        return 3;
+    }
+    const apiClient = createAnthropicClient();
+    const qsRes = await generateQuestions(sg.value, apiClient, {
+        n,
+        model,
+        vaultHash: vaultHash(vault),
+        seed,
+    });
+    if (!qsRes.ok) {
+        process.stderr.write(`${qsRes.error.message}\n`);
+        return 3;
+    }
+    const qs = qsRes.value;
+    qs.timestamp = new Date().toISOString();
+    qs.id = `${qs.vault_hash}-${qs.seed}-${qs.timestamp}`;
+    await writeQuestionSet(vault, qs);
+    process.stdout.write(`generated ${qs.questions.length} questions (id=${qs.id})\n`);
+    // 2. Run — mint the stable id + timestamp up front and persist incrementally
+    // so a mid-run failure leaves a resumable partial file.
+    const runTimestamp = new Date().toISOString();
+    const runId = `${qs.id}-${model}-${runTimestamp}`;
+    const runRes = await runAnswerer(qs, vault, apiClient, {
+        k,
+        model,
+        runId,
+        timestamp: runTimestamp,
+        persist: (r) => writeResults(vault, r),
+    });
+    if (!runRes.ok) {
+        process.stderr.write(`${runRes.error.message}\n`);
+        process.stderr.write(`partial results saved as ${runId}; resume with: daftari eval run --questions ${qs.id} --resume ${runId}\n`);
+        return 3;
+    }
+    const run = runRes.value;
+    await writeResults(vault, run); // final write (covers the zero-question edge where persist never fired)
+    process.stdout.write(`ran ${Object.keys(run.runs).length} answerer invocations (id=${run.id})\n`);
+    // 3. Score — invoke the same grading logic runScore uses, in-process.
+    return await runScore(["--vault", vault, "--results", run.id, "--grader-model", model]);
+}
+//# sourceMappingURL=index.js.map