daftari 1.15.0 → 1.16.0

package/dist/eval/index.js

@@ -0,0 +1,311 @@
+// src/eval/index.ts
+// Top-level CLI dispatcher for `daftari eval`. Parses flags, routes to
+// generate/run/score/top-level, translates Result<T, CortexEvalError> to exit
+// codes (2 = config, 3 = runtime/llm).
+import { createHash } from "node:crypto";
+import { resolve } from "node:path";
+import { generateQuestions } from "./generate.js";
+import { createAnthropicClient } from "./llm.js";
+import { PROMPT_VERSION } from "./prompts.js";
+import { runAnswerer } from "./run.js";
+import { aggregateScore, gradeAnswer } from "./score.js";
+import { appendHistory, readQuestionSet, readResults, writeQuestionSet, writeResults, writeScore, } from "./storage.js";
+import { sampleSubgraph } from "./subgraph.js";
+import { SPEC_VERSION, TIERS, } from "./types.js";
+const HELP = `daftari eval — cortex quality metric.
+
+Usage:
+  daftari eval [--vault <path>] [--n <count>] [--k <count>] [--seed <str>]
+  daftari eval generate [--vault <path>] [--n <count>] [--seed <str>]
+  daftari eval run      [--questions <id>] [--vault <path>] [--model <id>] [--k <count>] [--resume <results-id>]
+  daftari eval score    [--results <id>] [--vault <path>] [--grader-model <id>]
+
+  (--questions and --results take the artifact id printed by a prior stage,
+   not a file path; artifacts live under .daftari/eval/.)
+
+Defaults:
+  --n 15      total questions across three tiers (5 each)
+  --k 2       runs per question for variance estimation
+  --model     claude-sonnet-4-6 (DEFAULT_MODEL in src/eval/index.ts)
+  --vault     current working directory
+
+Environment:
+  ANTHROPIC_API_KEY   required for any LLM-mediated stage
+
+Disk usage:
+  .daftari/eval/results/ and scores/ grow without bound across runs. v1
+  recovery is a manual rm -rf .daftari/eval/results/; rerunning regenerates
+  what's needed. A daftari eval prune command is the planned v2 follow-up.
+
+Exit codes:
+  0 — eval completed
+  2 — config error (missing API key, bad flags, no vault)
+  3 — runtime/LLM error (retries exhausted, vault I/O failure)
+`;
+export async function runEval(argv) {
+    if (argv.includes("--help") || argv.includes("-h")) {
+        process.stdout.write(HELP);
+        return 0;
+    }
+    // runEval is the CLI boundary and must not throw: a bad integer flag
+    // (intFlag) or any unexpected error becomes a config-error exit code (2).
+    try {
+        const [mode, ...rest] = argv;
+        switch (mode) {
+            case "generate":
+                return await runGenerate(rest);
+            case "run":
+                return await runRun(rest);
+            case "score":
+                return await runScore(rest);
+            default:
+                return await runTopLevel(argv);
+        }
+    }
+    catch (e) {
+        process.stderr.write(`${e instanceof Error ? e.message : String(e)}\n`);
+        return 2;
+    }
+}
+function flag(argv, name) {
+    const i = argv.indexOf(`--${name}`);
+    if (i < 0 || i + 1 >= argv.length)
+        return undefined;
+    return argv[i + 1];
+}
+function intFlag(argv, name, def) {
+    const v = flag(argv, name);
+    if (v === undefined)
+        return def;
+    const n = parseInt(v, 10);
+    if (Number.isNaN(n))
+        throw new Error(`--${name} must be an integer`);
+    return n;
+}
+function vaultHash(vault) {
+    return createHash("sha256").update(resolve(vault)).digest("hex").slice(0, 12);
+}
+function defaultSeed(vault) {
+    const today = new Date().toISOString().slice(0, 10);
+    return `${vaultHash(vault)}-${today}`;
+}
+const DEFAULT_MODEL = "claude-sonnet-4-6";
+async function runGenerate(argv) {
+    if (!process.env.ANTHROPIC_API_KEY) {
+        process.stderr.write("ANTHROPIC_API_KEY required\n");
+        return 2;
+    }
+    const vault = flag(argv, "vault") ?? process.cwd();
+    const n = intFlag(argv, "n", 15);
+    const seed = flag(argv, "seed") ?? defaultSeed(vault);
+    const sg = await sampleSubgraph(vault, seed, { maxNodes: 5 });
+    if (!sg.ok) {
+        process.stderr.write(`${sg.error.message}\n`);
+        return 3;
+    }
+    const client = createAnthropicClient();
+    const qs = await generateQuestions(sg.value, client, {
+        n,
+        model: DEFAULT_MODEL,
+        vaultHash: vaultHash(vault),
+        seed,
+    });
+    if (!qs.ok) {
+        process.stderr.write(`${qs.error.message}\n`);
+        return 3;
+    }
+    qs.value.timestamp = new Date().toISOString();
+    qs.value.id = `${qs.value.vault_hash}-${qs.value.seed}-${qs.value.timestamp}`;
+    await writeQuestionSet(vault, qs.value);
+    process.stdout.write(`wrote question set ${qs.value.id} (${qs.value.questions.length} questions)\n`);
+    return 0;
+}
+async function runRun(argv) {
+    if (!process.env.ANTHROPIC_API_KEY) {
+        process.stderr.write("ANTHROPIC_API_KEY required\n");
+        return 2;
+    }
+    const vault = flag(argv, "vault") ?? process.cwd();
+    const questionsId = flag(argv, "questions");
+    if (!questionsId) {
+        process.stderr.write("--questions required\n");
+        return 2;
+    }
+    const k = intFlag(argv, "k", 2);
+    const model = flag(argv, "model") ?? DEFAULT_MODEL;
+    const qsRead = await readQuestionSet(vault, questionsId);
+    if (!qsRead.ok) {
+        process.stderr.write(`${qsRead.error.message}\n`);
+        return 3;
+    }
+    let resumeFrom;
+    const resumeId = flag(argv, "resume");
+    if (resumeId) {
+        const r = await readResults(vault, resumeId);
+        if (r.ok)
+            resumeFrom = r.value;
+    }
+    // Mint the stable id + timestamp up front so the on-disk file path is stable
+    // across the run and any later --resume; persist incrementally so a mid-run
+    // failure leaves a resumable partial file.
+    const timestamp = new Date().toISOString();
+    const runId = resumeFrom ? resumeFrom.id : `${qsRead.value.id}-${model}-${timestamp}`;
+    const client = createAnthropicClient();
+    const run = await runAnswerer(qsRead.value, vault, client, {
+        k,
+        model,
+        resumeFrom,
+        runId,
+        timestamp,
+        persist: (r) => writeResults(vault, r),
+    });
+    if (!run.ok) {
+        process.stderr.write(`${run.error.message}\n`);
+        process.stderr.write(`partial results saved as ${runId}; resume with: daftari eval run --questions ${questionsId} --resume ${runId}\n`);
+        return 3;
+    }
+    await writeResults(vault, run.value); // final write (covers the zero-question edge where persist never fired)
+    process.stdout.write(`wrote results ${run.value.id}\n`);
+    return 0;
+}
+async function runScore(argv) {
+    const vault = flag(argv, "vault") ?? process.cwd();
+    const resultsId = flag(argv, "results");
+    if (!resultsId) {
+        process.stderr.write("--results required\n");
+        return 2;
+    }
+    const graderModel = flag(argv, "grader-model") ?? DEFAULT_MODEL;
+    if (!process.env.ANTHROPIC_API_KEY) {
+        process.stderr.write("ANTHROPIC_API_KEY required\n");
+        return 2;
+    }
+    const runRead = await readResults(vault, resultsId);
+    if (!runRead.ok) {
+        process.stderr.write(`${runRead.error.message}\n`);
+        return 3;
+    }
+    const run = runRead.value;
+    const qsRead = await readQuestionSet(vault, run.questions_id);
+    if (!qsRead.ok) {
+        process.stderr.write(`${qsRead.error.message}\n`);
+        return 3;
+    }
+    const qs = qsRead.value;
+    const grader = createAnthropicClient();
+    const grades = [];
+    const traces = new Map();
+    for (const [, pr] of Object.entries(run.runs)) {
+        if (pr.status !== "complete" || !pr.trace)
+            continue;
+        const q = qs.questions[pr.question_index];
+        if (!q)
+            continue;
+        const g = await gradeAnswer(q, pr.question_index, pr.k_index, pr.trace, grader, {
+            model: graderModel,
+        });
+        if (g.ok) {
+            grades.push(g.value);
+            traces.set(`${q.id}:${pr.k_index}`, pr.trace);
+        }
+    }
+    const score = aggregateScore(grades, qs.questions, { traces });
+    score.models = {
+        generator: qs.generator_model,
+        answerer: run.answerer_model,
+        grader: graderModel,
+    };
+    score.prompt_version = PROMPT_VERSION;
+    score.spec_version = SPEC_VERSION;
+    score.questions_id = qs.id;
+    score.results_id = run.id;
+    score.vault_hash = qs.vault_hash;
+    score.k = run.k;
+    score.n = qs.questions.length;
+    score.timestamp = new Date().toISOString();
+    await writeScore(vault, score);
+    const histEntry = {
+        score_id: score.results_id,
+        score: score.score,
+        score_std: score.score_std,
+        by_tier: {
+            retrieval: score.by_tier.retrieval.mean,
+            cross_reference: score.by_tier.cross_reference.mean,
+            contradiction: score.by_tier.contradiction.mean,
+        },
+        vault_hash: score.vault_hash,
+        timestamp: score.timestamp,
+        n: score.n,
+        k: score.k,
+        models: score.models,
+        prompt_version: score.prompt_version,
+        spec_version: score.spec_version,
+    };
+    await appendHistory(vault, histEntry);
+    // Pretty-print headline + per-tier means.
+    process.stdout.write(`score: ${score.score.toFixed(3)} ± ${score.score_std.toFixed(3)}\n`);
+    for (const t of TIERS) {
+        const ts = score.by_tier[t];
+        process.stdout.write(`  ${t.padEnd(16)}: ${ts.mean.toFixed(3)} (n=${ts.n}, efficiency=${ts.trace_efficiency.toFixed(1)} calls)\n`);
+    }
+    return 0;
+}
+async function runTopLevel(argv) {
+    // Spec §3 "Top-level convenience": runs generate → run → score in one shot.
+    // We thread the IDs in-memory rather than re-reading from disk, so a
+    // failure mid-pipeline still leaves the on-disk artifacts that did
+    // succeed for forensic / resume use.
+    if (!process.env.ANTHROPIC_API_KEY) {
+        process.stderr.write("ANTHROPIC_API_KEY required\n");
+        return 2;
+    }
+    const vault = flag(argv, "vault") ?? process.cwd();
+    const n = intFlag(argv, "n", 15);
+    const k = intFlag(argv, "k", 2);
+    const seed = flag(argv, "seed") ?? defaultSeed(vault);
+    const model = flag(argv, "model") ?? DEFAULT_MODEL;
+    // 1. Generate
+    const sg = await sampleSubgraph(vault, seed, { maxNodes: 5 });
+    if (!sg.ok) {
+        process.stderr.write(`${sg.error.message}\n`);
+        return 3;
+    }
+    const apiClient = createAnthropicClient();
+    const qsRes = await generateQuestions(sg.value, apiClient, {
+        n,
+        model,
+        vaultHash: vaultHash(vault),
+        seed,
+    });
+    if (!qsRes.ok) {
+        process.stderr.write(`${qsRes.error.message}\n`);
+        return 3;
+    }
+    const qs = qsRes.value;
+    qs.timestamp = new Date().toISOString();
+    qs.id = `${qs.vault_hash}-${qs.seed}-${qs.timestamp}`;
+    await writeQuestionSet(vault, qs);
+    process.stdout.write(`generated ${qs.questions.length} questions (id=${qs.id})\n`);
+    // 2. Run — mint the stable id + timestamp up front and persist incrementally
+    // so a mid-run failure leaves a resumable partial file.
+    const runTimestamp = new Date().toISOString();
+    const runId = `${qs.id}-${model}-${runTimestamp}`;
+    const runRes = await runAnswerer(qs, vault, apiClient, {
+        k,
+        model,
+        runId,
+        timestamp: runTimestamp,
+        persist: (r) => writeResults(vault, r),
+    });
+    if (!runRes.ok) {
+        process.stderr.write(`${runRes.error.message}\n`);
+        process.stderr.write(`partial results saved as ${runId}; resume with: daftari eval run --questions ${qs.id} --resume ${runId}\n`);
+        return 3;
+    }
+    const run = runRes.value;
+    await writeResults(vault, run); // final write (covers the zero-question edge where persist never fired)
+    process.stdout.write(`ran ${Object.keys(run.runs).length} answerer invocations (id=${run.id})\n`);
+    // 3. Score — invoke the same grading logic runScore uses, in-process.
+    return await runScore(["--vault", vault, "--results", run.id, "--grader-model", model]);
+}
+//# sourceMappingURL=index.js.map

package/dist/eval/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,oBAAoB;AACpB,uEAAuE;AACvE,8EAA8E;AAC9E,uCAAuC;AAEvC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzD,OAAO,EACL,aAAa,EACb,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,YAAY,EACZ,UAAU,GACX,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,EAIL,YAAY,EACZ,KAAK,GAEN,MAAM,YAAY,CAAC;AAEpB,MAAM,IAAI,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6BZ,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAAc;IAC1C,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3B,OAAO,CAAC,CAAC;IACX,CAAC;IACD,qEAAqE;IACrE,0EAA0E;IAC1E,IAAI,CAAC;QACH,MAAM,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;QAC7B,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,UAAU;gBACb,OAAO,MAAM,WAAW,CAAC,IAAI,CAAC,CAAC;YACjC,KAAK,KAAK;gBACR,OAAO,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;YAC5B,KAAK,OAAO;gBACV,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC9B;gBACE,OAAO,MAAM,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACxE,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAED,SAAS,IAAI,CAAC,IAAc,EAAE,IAAY;IACxC,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IACpC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM;QAAE,OAAO,SAAS,CAAC;IACpD,OAAO,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AACrB,CAAC;AACD,SAAS,OAAO,CAAC,IAAc,EAAE,IAAY,EAAE,GAAW;IACxD,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC3B,IAAI,CAAC,KAAK,SAAS;QAAE,OAAO,GAAG,CAAC;IAChC,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC1B,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,KAAK,IAAI,qBAAqB,CAAC,CAAC;IACrE,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,WAAW,CAAC,KAAa;IAChC,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACpD,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,EAAE,CAAC;AACxC,CAAC;AAED,MAAM,aAAa,GAAG,mBAAmB,CAAC;AAE1C,KAAK,UAAU,WAAW,CAAC,IAAc;IACvC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;QACnC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QACrD,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACnD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;IACjC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC;IAEtD,MAAM,EAAE,GAAG,MAAM,cAAc,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;IAC9D,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACX,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;QAC9C,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,MAAM,GAAG,qBAAqB,EAAE,CAAC;IACvC,MAAM,EAAE,GAAG,MAAM,iBAAiB,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE;QACnD,CAAC;QACD,KAAK,EAAE,aAAa;QACpB,SAAS,EAAE,SAAS,CAAC,KAAK,CAAC;QAC3B,IAAI;KACL,CAAC,CAAC;IACH,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACX,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;QAC9C,OAAO,CAAC,CAAC;IACX,CAAC;IACD,EAAE,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC9C,EAAE,CAAC,KAAK,CAAC,EAAE,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,UAAU,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;IAC9E,MAAM,gBAAgB,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IACxC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,sBAAsB,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,eAAe,CAC/E,CAAC;IACF,OAAO,CAAC,CAAC;AACX,CAAC;AAED,KAAK,UAAU,MAAM,CAAC,IAAc;IAClC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;QACnC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QACrD,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACnD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IAC5C,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC/C,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,aAAa,CAAC;IAEnD,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IACzD,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;QAClD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,UAA+B,CAAC;IACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACtC,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,CAAC,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC7C,IAAI,CAAC,CAAC,EAAE;YAAE,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC;IACjC,CAAC;IAED,6EAA6E;IAC7E,4EAA4E;IAC5E,2CAA2C;IAC3C,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;IACtF,MAAM,MAAM,GAAG,qBAAqB,EAAE,CAAC;IACvC,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE;QACzD,CAAC;QACD,KAAK;QACL,UAAU;QACV,KAAK;QACL,SAAS;QACT,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC,CAAC;KACvC,CAAC,CAAC;IACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;QAC/C,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,4BAA4B,KAAK,+CAA+C,WAAW,aAAa,KAAK,IAAI,CAClH,CAAC;QACF,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,wEAAwE;IAC9G,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC;IACxD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,IAAc;IACpC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACnD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IACxC,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC7C,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,cAAc,CAAC,IAAI,aAAa,CAAC;IAChE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;QACnC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QACrD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IACpD,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;QAChB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;QACnD,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC;IAC1B,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,KAAK,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC;IAC9D,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;QAClD,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC;IAExB,MAAM,MAAM,GAAG,qBAAqB,EAAE,CAAC;IACvC,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAiB,CAAC;IACxC,KAAK,MAAM,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9C,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU,IAAI,CAAC,EAAE,CAAC,KAAK;YAAE,SAAS;QACpD,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,cAAc,CAAC,CAAC;QAC1C,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,MAAM,CAAC,GAAG,MAAM,WAAW,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,EAAE,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE;YAC9E,KAAK,EAAE,WAAW;SACnB,CAAC,CAAC;QACH,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;YACT,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACrB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IACD,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IAC/D,KAAK,CAAC,MAAM,GAAG;QACb,SAAS,EAAE,EAAE,CAAC,eAAe;QAC7B,QAAQ,EAAE,GAAG,CAAC,cAAc;QAC5B,MAAM,EAAE,WAAW;KACpB,CAAC;IACF,KAAK,CAAC,cAAc,GAAG,cAAc,CAAC;IACtC,KAAK,CAAC,YAAY,GAAG,YAAY,CAAC;IAClC,KAAK,CAAC,YAAY,GAAG,EAAE,CAAC,EAAE,CAAC;IAC3B,KAAK,CAAC,UAAU,GAAG,GAAG,CAAC,EAAE,CAAC;IAC1B,KAAK,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC;IACjC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;IAChB,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC;IAC9B,KAAK,CAAC,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,UAAU,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAE/B,MAAM,SAAS,GAAiB;QAC9B,QAAQ,EAAE,KAAK,CAAC,UAAU;QAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,OAAO,EAAE;YACP,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI;YACvC,eAAe,EAAE,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI;YACnD,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI;SAChD;QACD,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,CAAC,EAAE,KAAK,CAAC,CAAC;QACV,CAAC,EAAE,KAAK,CAAC,CAAC;QACV,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,YAAY,EAAE,KAAK,CAAC,YAAY;KACjC,CAAC;IACF,MAAM,aAAa,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAEtC,0CAA0C;IAC1C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC3F,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,KAAK,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,gBAAgB,EAAE,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAC7G,CAAC;IACJ,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,IAAc;IACvC,4EAA4E;IAC5E,qEAAqE;IACrE,mEAAmE;IACnE,qCAAqC;IACrC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;QACnC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QACrD,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACnD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;IACjC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;IAChC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC;IACtD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,aAAa,CAAC;IAEnD,cAAc;IACd,MAAM,EAAE,GAAG,MAAM,cAAc,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;IAC9D,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACX,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;QAC9C,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,SAAS,GAAG,qBAAqB,EAAE,CAAC;IAC1C,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE;QACzD,CAAC;QACD,KAAK;QACL,SAAS,EAAE,SAAS,CAAC,KAAK,CAAC;QAC3B,IAAI;KACL,CAAC,CAAC;IACH,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QACd,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;QACjD,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC;IACvB,EAAE,CAAC,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACxC,EAAE,CAAC,EAAE,GAAG,GAAG,EAAE,CAAC,UAAU,IAAI,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,SAAS,EAAE,CAAC;IACtD,MAAM,gBAAgB,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAClC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC,SAAS,CAAC,MAAM,kBAAkB,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;IAEnF,6EAA6E;IAC7E,wDAAwD;IACxD,MAAM,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC9C,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,EAAE,IAAI,KAAK,IAAI,YAAY,EAAE,CAAC;IAClD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE;QACrD,CAAC;QACD,KAAK;QACL,KAAK;QACL,SAAS,EAAE,YAAY;QACvB,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC,CAAC;KACvC,CAAC,CAAC;IACH,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;QAClD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,4BAA4B,KAAK,+CAA+C,EAAE,CAAC,EAAE,aAAa,KAAK,IAAI,CAC5G,CAAC;QACF,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC;IACzB,MAAM,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,wEAAwE;IACxG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,6BAA6B,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;IAElG,sEAAsE;IACtE,OAAO,MAAM,QAAQ,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,CAAC,EAAE,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC,CAAC;AAC1F,CAAC"}

package/dist/eval/llm.d.ts

@@ -0,0 +1,47 @@
+import { type Result } from "../frontmatter/types.js";
+import type { CortexEvalError } from "./types.js";
+export interface CompleteOpts {
+    model: string;
+    system: string;
+    user: string;
+    maxTokens?: number;
+}
+export interface CompleteJsonOpts extends CompleteOpts {
+    schema: any;
+}
+export interface ToolDef {
+    name: string;
+    description: string;
+    input_schema: any;
+}
+export interface CompleteWithToolsOpts extends CompleteOpts {
+    tools: ToolDef[];
+    toolHandler: (name: string, input: unknown) => Promise<unknown>;
+    maxRounds?: number;
+}
+export interface CompleteResult {
+    text: string;
+    input_tokens: number;
+    output_tokens: number;
+    stop_reason: string;
+}
+export interface CompleteJsonResult extends CompleteResult {
+    parsed: unknown;
+}
+export interface CompleteWithToolsResult extends CompleteResult {
+    tool_calls: {
+        tool: string;
+        input: unknown;
+        output: unknown;
+        latency_ms: number;
+    }[];
+}
+export interface LlmClient {
+    complete(opts: CompleteOpts): Promise<Result<CompleteResult, CortexEvalError>>;
+    completeJson(opts: CompleteJsonOpts): Promise<Result<CompleteJsonResult, CortexEvalError>>;
+    completeWithTools(opts: CompleteWithToolsOpts): Promise<Result<CompleteWithToolsResult, CortexEvalError>>;
+}
+export declare function createAnthropicClient(): LlmClient;
+export declare function retry<T>(fn: () => Promise<Result<T, CortexEvalError>>): Promise<Result<T, CortexEvalError>>;
+export declare function stripCodeFence(s: string): string;
+//# sourceMappingURL=llm.d.ts.map

package/dist/eval/llm.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../src/eval/llm.ts"],"names":[],"mappings":"AAMA,OAAO,EAAW,KAAK,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAiB,SAAQ,YAAY;IAEpD,MAAM,EAAE,GAAG,CAAC;CACb;AAED,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IAEpB,YAAY,EAAE,GAAG,CAAC;CACnB;AAED,MAAM,WAAW,qBAAsB,SAAQ,YAAY;IACzD,KAAK,EAAE,OAAO,EAAE,CAAC;IACjB,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,kBAAmB,SAAQ,cAAc;IACxD,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,uBAAwB,SAAQ,cAAc;IAC7D,UAAU,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACrF;AAED,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC,CAAC;IAC/E,YAAY,CAAC,IAAI,EAAE,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAAC,CAAC;IAC3F,iBAAiB,CACf,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,MAAM,CAAC,uBAAuB,EAAE,eAAe,CAAC,CAAC,CAAC;CAC9D;AAED,wBAAgB,qBAAqB,IAAI,SAAS,CAiIjD;AAWD,wBAAsB,KAAK,CAAC,CAAC,EAC3B,EAAE,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC,GAC5C,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC,CAuBrC;AAED,wBAAgB,cAAc,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAGhD"}

package/dist/eval/llm.js

@@ -0,0 +1,165 @@
+// src/eval/llm.ts
+// Single-point wrapper around @anthropic-ai/sdk. Other eval modules depend
+// on the LlmClient interface, not the SDK, so they can be unit-tested with
+// hand-rolled mocks.
+import Anthropic from "@anthropic-ai/sdk";
+import { err, ok } from "../frontmatter/types.js";
+export function createAnthropicClient() {
+    const apiKey = process.env.ANTHROPIC_API_KEY;
+    if (!apiKey)
+        throw new Error("ANTHROPIC_API_KEY env var is required for daftari eval");
+    const client = new Anthropic({ apiKey });
+    const complete = async (opts) => {
+        return retry(async () => {
+            const res = await client.messages.create({
+                model: opts.model,
+                max_tokens: opts.maxTokens ?? 4096,
+                system: opts.system,
+                messages: [{ role: "user", content: opts.user }],
+            });
+            const text = res.content
+                .filter((b) => b.type === "text")
+                .map((b) => b.text)
+                .join("");
+            return ok({
+                text,
+                input_tokens: res.usage.input_tokens,
+                output_tokens: res.usage.output_tokens,
+                stop_reason: res.stop_reason ?? "unknown",
+            });
+        });
+    };
+    const completeJson = async (opts) => {
+        // The schema is embedded in the system prompt as a hint to the LLM, then
+        // the response goes through JSON.parse + a manual shape check by the
+        // caller (see generate.ts and score.ts). This is NOT strict JSON Schema
+        // validation — there is no schema validator dep in v1. Callers must
+        // verify required fields exist after parse. If we ever need strict
+        // validation, add `ajv` and validate `parsed` here.
+        const sysWithSchema = `${opts.system}\n\nReturn JSON matching:\n${JSON.stringify(opts.schema, null, 2)}\nReturn ONLY JSON, no prose.`;
+        const r = await complete({ ...opts, system: sysWithSchema });
+        if (!r.ok)
+            return r;
+        try {
+            const parsed = JSON.parse(stripCodeFence(r.value.text));
+            return ok({ ...r.value, parsed });
+        }
+        catch (e) {
+            const msg = e instanceof Error ? e.message : String(e);
+            return err({
+                kind: "llm",
+                message: `JSON parse: ${msg} — output was: ${r.value.text.slice(0, 200)}`,
+                retryable: false,
+            });
+        }
+    };
+    const completeWithTools = async (opts) => {
+        const maxRounds = opts.maxRounds ?? 12;
+        const toolCalls = [];
+        const messages = [
+            { role: "user", content: opts.user },
+        ];
+        let totalIn = 0;
+        let totalOut = 0;
+        let lastStop = "unknown";
+        for (let round = 0; round < maxRounds; round++) {
+            const res = await retry(async () => ok(await client.messages.create({
+                model: opts.model,
+                max_tokens: opts.maxTokens ?? 4096,
+                system: opts.system,
+                // biome-ignore lint/suspicious/noExplicitAny: SDK types
+                tools: opts.tools,
+                // biome-ignore lint/suspicious/noExplicitAny: SDK types
+                messages: messages,
+            })));
+            if (!res.ok)
+                return res;
+            const message = res.value;
+            totalIn += message.usage.input_tokens;
+            totalOut += message.usage.output_tokens;
+            lastStop = message.stop_reason ?? "unknown";
+            // biome-ignore lint/suspicious/noExplicitAny: SDK content union
+            const blocks = message.content;
+            const toolUses = blocks.filter((b) => b.type === "tool_use");
+            if (toolUses.length === 0) {
+                const text = blocks
+                    .filter((b) => b.type === "text")
+                    .map((b) => b.text)
+                    .join("");
+                return ok({
+                    text,
+                    input_tokens: totalIn,
+                    output_tokens: totalOut,
+                    stop_reason: lastStop,
+                    tool_calls: toolCalls,
+                });
+            }
+            messages.push({ role: "assistant", content: blocks });
+            const toolResults = [];
+            for (const tu of toolUses) {
+                const t0 = Date.now();
+                let output;
+                try {
+                    output = await opts.toolHandler(tu.name, tu.input);
+                }
+                catch (e) {
+                    output = { tool_error: e instanceof Error ? e.message : String(e) };
+                }
+                const latency = Date.now() - t0;
+                toolCalls.push({ tool: tu.name, input: tu.input, output, latency_ms: latency });
+                toolResults.push({
+                    type: "tool_result",
+                    tool_use_id: tu.id,
+                    content: typeof output === "string" ? output : JSON.stringify(output),
+                });
+            }
+            messages.push({ role: "user", content: toolResults });
+        }
+        return err({
+            kind: "llm",
+            message: `exceeded maxRounds (${maxRounds}) without final answer`,
+            retryable: false,
+        });
+    };
+    return { complete, completeJson, completeWithTools };
+}
+// --- helpers ---
+const MAX_RETRIES = 5;
+const BASE_BACKOFF_MS = 500;
+const MAX_BACKOFF_MS = 60_000;
+// Exported for unit testing — these two pure helpers carry the trickiest logic
+// in this module (retry arithmetic/predicate, fence stripping) and would
+// otherwise be unreachable, since createAnthropicClient news up the SDK.
+export async function retry(fn) {
+    let lastErr = null;
+    for (let i = 0; i < MAX_RETRIES; i++) {
+        try {
+            const r = await fn();
+            if (r.ok)
+                return r;
+            if (!r.error || r.error.kind !== "llm" || !r.error.retryable)
+                return r;
+            lastErr = r.error;
+        }
+        catch (e) {
+            const msg = e instanceof Error ? e.message : String(e);
+            const status = e?.status;
+            const retryable = status === 429 || (typeof status === "number" && status >= 500);
+            if (!retryable)
+                return err({ kind: "llm", message: msg, retryable: false });
+            lastErr = { kind: "llm", message: msg, retryable: true };
+        }
+        // Don't sleep after the final attempt — the loop is about to exit and
+        // surface the error; a trailing backoff would just delay the failure.
+        if (i < MAX_RETRIES - 1) {
+            const backoff = Math.min(BASE_BACKOFF_MS * 2 ** i, MAX_BACKOFF_MS);
+            await new Promise((res) => setTimeout(res, backoff));
+        }
+    }
+    return err(lastErr ?? { kind: "llm", message: "retries exhausted", retryable: false });
+}
+export function stripCodeFence(s) {
+    const m = s.match(/^```(?:json)?\n([\s\S]*?)\n```\s*$/);
+    return m ? m[1] : s;
+}
+//# sourceMappingURL=llm.js.map

package/dist/eval/llm.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/eval/llm.ts"],"names":[],"mappings":"AAAA,kBAAkB;AAClB,2EAA2E;AAC3E,2EAA2E;AAC3E,qBAAqB;AAErB,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,GAAG,EAAE,EAAE,EAAe,MAAM,yBAAyB,CAAC;AAmD/D,MAAM,UAAU,qBAAqB;IACnC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAC7C,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;IACvF,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAEzC,MAAM,QAAQ,GAAG,KAAK,EAAE,IAAkB,EAAoD,EAAE;QAC9F,OAAO,KAAK,CAAC,KAAK,IAAI,EAAE;YACtB,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;gBACvC,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,UAAU,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI;gBAClC,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;aACjD,CAAC,CAAC;YACH,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO;iBACrB,MAAM,CAAC,CAAC,CAAC,EAAwD,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;iBACtF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;iBAClB,IAAI,CAAC,EAAE,CAAC,CAAC;YACZ,OAAO,EAAE,CAAC;gBACR,IAAI;gBACJ,YAAY,EAAE,GAAG,CAAC,KAAK,CAAC,YAAY;gBACpC,aAAa,EAAE,GAAG,CAAC,KAAK,CAAC,aAAa;gBACtC,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,SAAS;aAC1C,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,KAAK,EACxB,IAAsB,EACgC,EAAE;QACxD,yEAAyE;QACzE,qEAAqE;QACrE,wEAAwE;QACxE,oEAAoE;QACpE,mEAAmE;QACnE,oDAAoD;QACpD,MAAM,aAAa,GAAG,GAAG,IAAI,CAAC,MAAM,8BAA8B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,+BAA+B,CAAC;QACtI,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAC7D,IAAI,CAAC,CAAC,CAAC,EAAE;YAAE,OAAO,CAAC,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YACxD,OAAO,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QACpC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACvD,OAAO,GAAG,CAAC;gBACT,IAAI,EAAE,KAAK;gBACX,OAAO,EAAE,eAAe,GAAG,kBAAkB,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;gBACzE,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,iBAAiB,GAAG,KAAK,EAC7B,IAA2B,EACgC,EAAE;QAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;QACvC,MAAM,SAAS,GAA0C,EAAE,CAAC;QAC5D,MAAM,QAAQ,GAA4D;YACxE,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,EAAE;SACrC,CAAC;QACF,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,QAAQ,GAAG,SAAS,CAAC;QAEzB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC;YAC/C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,KAAK,IAAI,EAAE,CACjC,EAAE,CACA,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC3B,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,UAAU,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI;gBAClC,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,wDAAwD;gBACxD,KAAK,EAAE,IAAI,CAAC,KAAY;gBACxB,wDAAwD;gBACxD,QAAQ,EAAE,QAAe;aAC1B,CAAC,CACH,CACF,CAAC;YACF,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,OAAO,GAAG,CAAC;YACxB,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC;YAC1B,OAAO,IAAI,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC;YACtC,QAAQ,IAAI,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC;YACxC,QAAQ,GAAG,OAAO,CAAC,WAAW,IAAI,SAAS,CAAC;YAE5C,gEAAgE;YAChE,MAAM,MAAM,GAAG,OAAO,CAAC,OAAgB,CAAC;YACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;YAC7D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,GAAG,MAAM;qBAChB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;qBAChC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;qBAClB,IAAI,CAAC,EAAE,CAAC,CAAC;gBACZ,OAAO,EAAE,CAAC;oBACR,IAAI;oBACJ,YAAY,EAAE,OAAO;oBACrB,aAAa,EAAE,QAAQ;oBACvB,WAAW,EAAE,QAAQ;oBACrB,UAAU,EAAE,SAAS;iBACtB,CAAC,CAAC;YACL,CAAC;YAED,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;YAEtD,MAAM,WAAW,GAAc,EAAE,CAAC;YAClC,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;gBAC1B,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACtB,IAAI,MAAe,CAAC;gBACpB,IAAI,CAAC;oBACH,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;gBACrD,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,MAAM,GAAG,EAAE,UAAU,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;gBACtE,CAAC;gBACD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;gBAChC,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC;gBAChF,WAAW,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,aAAa;oBACnB,WAAW,EAAE,EAAE,CAAC,EAAE;oBAClB,OAAO,EAAE,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;iBACtE,CAAC,CAAC;YACL,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC;QACxD,CAAC;QACD,OAAO,GAAG,CAAC;YACT,IAAI,EAAE,KAAK;YACX,OAAO,EAAE,uBAAuB,SAAS,wBAAwB;YACjE,SAAS,EAAE,KAAK;SACjB,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,iBAAiB,EAAE,CAAC;AACvD,CAAC;AAED,kBAAkB;AAElB,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,eAAe,GAAG,GAAG,CAAC;AAC5B,MAAM,cAAc,GAAG,MAAM,CAAC;AAE9B,+EAA+E;AAC/E,yEAAyE;AACzE,yEAAyE;AACzE,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,EAA6C;IAE7C,IAAI,OAAO,GAA2B,IAAI,CAAC;IAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,MAAM,EAAE,EAAE,CAAC;YACrB,IAAI,CAAC,CAAC,EAAE;gBAAE,OAAO,CAAC,CAAC;YACnB,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS;gBAAE,OAAO,CAAC,CAAC;YACvE,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC;QACpB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACvD,MAAM,MAAM,GAAI,CAAyB,EAAE,MAAM,CAAC;YAClD,MAAM,SAAS,GAAG,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,IAAI,GAAG,CAAC,CAAC;YAClF,IAAI,CAAC,SAAS;gBAAE,OAAO,GAAG,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;YAC5E,OAAO,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QAC3D,CAAC;QACD,sEAAsE;QACtE,sEAAsE;QACtE,IAAI,CAAC,GAAG,WAAW,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,GAAG,CAAC,IAAI,CAAC,EAAE,cAAc,CAAC,CAAC;YACnE,MAAM,IAAI,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC,OAAO,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;AACzF,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,CAAS;IACtC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACxD,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACtB,CAAC"}

package/dist/eval/prompts.d.ts

@@ -0,0 +1,5 @@
+export declare const PROMPT_VERSION = 1;
+export declare const GENERATOR_PROMPT = "You will read a connected subgraph of a Markdown knowledge vault and produce\nmulti-hop questions across three tiers. The questions must be answerable using\nONLY the docs provided. For each question, supply: question text, tier,\nexpected answer, source paths (must be a subset of the supplied docs).\n\nTiers:\n  retrieval        \u2014 single-doc lookup, 1-hop reasoning\n  cross_reference  \u2014 requires combining 2\u20133 docs\n  contradiction    \u2014 surfaces a tension or conflict across docs (use the\n                     tension log entries in the subgraph as seed material\n                     where present)\n\nReturn JSON matching the QuestionSetSchema declared in src/eval/types.ts.\nDo not include questions whose expected_sources are not in the supplied docs.\nDo not generate trivial yes/no questions.";
+export declare const ANSWERER_SYSTEM_PROMPT = "You will answer a question about a Markdown knowledge vault using ONLY the\nprovided Daftari tools. Do not use training knowledge. Do not guess. If the\nvault does not contain the answer, say \"Vault does not contain the answer.\"\nCite source paths in your final answer using the format [path/to/doc.md].";
+export declare const GRADER_PROMPT = "You are grading an answer to a question about a Markdown knowledge vault.\n\nQuestion:           {{QUESTION}}\nExpected answer:    {{EXPECTED_ANSWER}}\nExpected sources:   {{EXPECTED_SOURCES}}\nClaimed answer:     {{CLAIMED_ANSWER}}\nCited sources:      {{CITED_SOURCES}}\n\nReturn JSON: {\"correct\": \"yes\" | \"partial\" | \"no\", \"reasoning\": \"<string>\"}\n\nDefinitions:\n  yes     \u2014 claimed answer is substantively correct and cites at least one\n            expected source\n  partial \u2014 claimed answer is partially correct OR cites the right sources\n            but misses key content OR the answerer correctly said \"Vault\n            does not contain the answer\" when the expected answer disagrees\n            (records a question-set quality issue, not a cortex failure)\n  no      \u2014 claimed answer is wrong, hallucinated, or cites no expected\n            sources";
+//# sourceMappingURL=prompts.d.ts.map

package/dist/eval/prompts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompts.d.ts","sourceRoot":"","sources":["../../src/eval/prompts.ts"],"names":[],"mappings":"AAKA,eAAO,MAAM,cAAc,IAAI,CAAC;AAEhC,eAAO,MAAM,gBAAgB,4zBAca,CAAC;AAE3C,eAAO,MAAM,sBAAsB,sTAGuC,CAAC;AAE3E,eAAO,MAAM,aAAa,o4BAkBN,CAAC"}

package/dist/eval/prompts.js

@@ -0,0 +1,44 @@
+// src/eval/prompts.ts
+// Frozen prompts for the three eval LLM roles. Bumping any prompt requires
+// bumping PROMPT_VERSION in the same commit. PROMPT_VERSION is recorded in
+// every output file for forensics and cross-version comparison gates.
+export const PROMPT_VERSION = 1;
+export const GENERATOR_PROMPT = `You will read a connected subgraph of a Markdown knowledge vault and produce
+multi-hop questions across three tiers. The questions must be answerable using
+ONLY the docs provided. For each question, supply: question text, tier,
+expected answer, source paths (must be a subset of the supplied docs).
+
+Tiers:
+  retrieval        — single-doc lookup, 1-hop reasoning
+  cross_reference  — requires combining 2–3 docs
+  contradiction    — surfaces a tension or conflict across docs (use the
+                     tension log entries in the subgraph as seed material
+                     where present)
+
+Return JSON matching the QuestionSetSchema declared in src/eval/types.ts.
+Do not include questions whose expected_sources are not in the supplied docs.
+Do not generate trivial yes/no questions.`;
+export const ANSWERER_SYSTEM_PROMPT = `You will answer a question about a Markdown knowledge vault using ONLY the
+provided Daftari tools. Do not use training knowledge. Do not guess. If the
+vault does not contain the answer, say "Vault does not contain the answer."
+Cite source paths in your final answer using the format [path/to/doc.md].`;
+export const GRADER_PROMPT = `You are grading an answer to a question about a Markdown knowledge vault.
+
+Question:           {{QUESTION}}
+Expected answer:    {{EXPECTED_ANSWER}}
+Expected sources:   {{EXPECTED_SOURCES}}
+Claimed answer:     {{CLAIMED_ANSWER}}
+Cited sources:      {{CITED_SOURCES}}
+
+Return JSON: {"correct": "yes" | "partial" | "no", "reasoning": "<string>"}
+
+Definitions:
+  yes     — claimed answer is substantively correct and cites at least one
+            expected source
+  partial — claimed answer is partially correct OR cites the right sources
+            but misses key content OR the answerer correctly said "Vault
+            does not contain the answer" when the expected answer disagrees
+            (records a question-set quality issue, not a cortex failure)
+  no      — claimed answer is wrong, hallucinated, or cites no expected
+            sources`;
+//# sourceMappingURL=prompts.js.map

package/dist/eval/prompts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompts.js","sourceRoot":"","sources":["../../src/eval/prompts.ts"],"names":[],"mappings":"AAAA,sBAAsB;AACtB,2EAA2E;AAC3E,2EAA2E;AAC3E,sEAAsE;AAEtE,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC;AAEhC,MAAM,CAAC,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;0CAcU,CAAC;AAE3C,MAAM,CAAC,MAAM,sBAAsB,GAAG;;;0EAGoC,CAAC;AAE3E,MAAM,CAAC,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;oBAkBT,CAAC"}

package/dist/eval/run.d.ts

@@ -0,0 +1,13 @@
+import { type Result } from "../frontmatter/types.js";
+import type { LlmClient } from "./llm.js";
+import type { CortexEvalError, EvalRun, QuestionSet } from "./types.js";
+export interface RunOptions {
+    k: number;
+    model: string;
+    resumeFrom?: EvalRun;
+    runId?: string;
+    timestamp?: string;
+    persist?: (run: EvalRun) => Promise<void>;
+}
+export declare function runAnswerer(questions: QuestionSet, vaultRoot: string, llm: LlmClient, opts: RunOptions): Promise<Result<EvalRun, CortexEvalError>>;
+//# sourceMappingURL=run.d.ts.map

package/dist/eval/run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/eval/run.ts"],"names":[],"mappings":"AAMA,OAAO,EAAW,KAAK,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,KAAK,EAAE,SAAS,EAAW,MAAM,UAAU,CAAC;AAGnD,OAAO,KAAK,EAAE,eAAe,EAAE,OAAO,EAAgB,WAAW,EAAS,MAAM,YAAY,CAAC;AAE7F,MAAM,WAAW,UAAU;IACzB,CAAC,EAAE,MAAM,CAAC;IACV,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3C;AAED,wBAAsB,WAAW,CAC/B,SAAS,EAAE,WAAW,EACtB,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,SAAS,EACd,IAAI,EAAE,UAAU,GACf,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,CAyE3C"}