@justyork/repo-mind 0.4.0 → 0.7.0

package/README.md

@@ -4,7 +4,7 @@
 
 ## Status
 
-**v0.3.0** — keyboard nav, image upload, domain labels, yaml/json reader, `ab-demo` harness. Confluence-style UI, wikilinks, deep links (`?slug=`), prepare/sync-links.
+**v0.6.1** — live A/B eval (`repo-mind ab-eval`) for kill-switch proof on real `docs/`. Also: `create_draft` MCP, `publish --pr` (0.6.0). Prior: WYSIWYG editor, keyboard nav, domain labels.
 
 | Artifact | Location |
 |----------|----------|
@@ -46,9 +46,11 @@ Then ask your agent a project question — it should call `search_docs` / `get_d
 | `repo-mind prepare` | Add frontmatter to legacy markdown (`--all` for batch) |
 | `repo-mind sync-links` | Convert markdown links to wikilinks; sync `related:` |
 | `repo-mind export` | Write `agents.md` to repo root |
+| `repo-mind publish` | Publish active drafts to `docs/`; `--pr` opens a GitHub pull request |
 | `repo-mind mcp` | Start the MCP stdio server |
 | `repo-mind ui` | Confluence-style workspace over `docs/` (127.0.0.1:3847) |
 | `npm run ab-demo` | Validate A/B demo fixture (repo checkout only) |
+| `repo-mind ab-eval` | Live A/B eval on project `docs/` (skyforge dogfood gate) |
 
 ## Cursor skill
 
@@ -80,6 +82,7 @@ Binds **127.0.0.1** only. MCP reads published files in `docs/` only (not SQLite
 - `get_doc` — fetch one doc by slug
 - `get_glossary_term` — resolve glossary entries
 - `explore_graph` — BFS over `related:` links
+- `create_draft` — create a SQLite draft for human review (gated on kill-switch pass)
 
 ## CI
 
@@ -103,7 +106,7 @@ This repo is developed with [gstack](https://github.com/garrytan/gstack). Projec
 
 ## Roadmap
 
-See [`docs/product/wiki/roadmap.md`](docs/product/wiki/roadmap.md) for v4.0–v4.2 phases. Next: v4.1 keyboard nav and image upload; v4.2 agent write (gated on kill-switch).
+See [`docs/product/wiki/roadmap.md`](docs/product/wiki/roadmap.md) for v4.0–v4.2 phases. v4.2 agent write shipped in 0.6.0 (gated); next: v4.0 prove closure (dogfood + hallucination scoring).
 
 ## License
 

package/dist/ab-demo/compute-pass.d.ts

@@ -0,0 +1,24 @@
+import type { AbArmSummary, AbQuestionComparison, HumanScoreEntry } from './types.js';
+export declare function summarizeArmTokens(arm: 'baseline' | 'repomind', comparisons: AbQuestionComparison[]): AbArmSummary;
+export declare function computeTokenPass(comparisons: AbQuestionComparison[], baseline: AbArmSummary, repomind: AbArmSummary): {
+    tokenPass: boolean;
+    repomindWins: number;
+    passThreshold: number;
+};
+export type EvalCategory = 'factual' | 'synthesis' | 'glossary-adr';
+export declare function categoryFromTags(tags: string[] | undefined): EvalCategory;
+export interface CategoryWin {
+    category: EvalCategory;
+    questionIds: string[];
+    baselineHallucinationTotal: number;
+    repomindHallucinationTotal: number;
+    baselineMedianTokens: number;
+    repomindMedianTokens: number;
+    repomindWins: boolean;
+}
+/** RepoMind wins a category when hallucination total is lower and median tokens are lower or equal. */
+export declare function computeCategoryWins(scores: HumanScoreEntry[], comparisons: AbQuestionComparison[], questionsById: Map<string, {
+    tags?: string[];
+}>): CategoryWin[];
+export declare function computeHallucinationPass(categoryWins: CategoryWin[]): boolean;
+export declare function mergePassFlags(tokenPass: boolean, hallucinationPass: boolean | null): boolean | null;

package/dist/ab-demo/compute-pass.js

@@ -0,0 +1,80 @@
+function median(values) {
+    if (values.length === 0) {
+        return 0;
+    }
+    const sorted = [...values].sort((a, b) => a - b);
+    const mid = Math.floor(sorted.length / 2);
+    if (sorted.length % 2 === 0) {
+        return Math.round((sorted[mid - 1] + sorted[mid]) / 2);
+    }
+    return sorted[mid];
+}
+export function summarizeArmTokens(arm, comparisons) {
+    const tokens = comparisons.map((row) => arm === 'baseline' ? row.baseline.tokens : row.repomind.tokens);
+    return {
+        arm,
+        medianTokens: median(tokens),
+        perQuestionTokens: tokens,
+    };
+}
+export function computeTokenPass(comparisons, baseline, repomind) {
+    const n = comparisons.length;
+    const repomindWins = comparisons.filter((row) => row.tokenWinner === 'repomind').length;
+    const passThreshold = Math.ceil((n * 2) / 3);
+    const tokenPass = repomind.medianTokens < baseline.medianTokens && repomindWins >= passThreshold;
+    return { tokenPass, repomindWins, passThreshold };
+}
+export function categoryFromTags(tags) {
+    if (!tags || tags.length === 0) {
+        return 'factual';
+    }
+    if (tags.some((t) => t === 'synthesis' || t === 'open-question')) {
+        return 'synthesis';
+    }
+    if (tags.some((t) => t === 'glossary' || t === 'adr')) {
+        return 'glossary-adr';
+    }
+    return 'factual';
+}
+/** RepoMind wins a category when hallucination total is lower and median tokens are lower or equal. */
+export function computeCategoryWins(scores, comparisons, questionsById) {
+    const byCategory = new Map();
+    for (const score of scores) {
+        const question = questionsById.get(score.questionId);
+        const category = categoryFromTags(question?.tags);
+        const list = byCategory.get(category) ?? [];
+        list.push(score);
+        byCategory.set(category, list);
+    }
+    const results = [];
+    for (const [category, categoryScores] of byCategory) {
+        const questionIds = categoryScores.map((s) => s.questionId);
+        const baselineHallucinationTotal = categoryScores.reduce((sum, s) => sum + s.baseline, 0);
+        const repomindHallucinationTotal = categoryScores.reduce((sum, s) => sum + s.repomind, 0);
+        const compRows = comparisons.filter((c) => questionIds.includes(c.questionId));
+        const baselineTokens = compRows.map((c) => c.baseline.tokens);
+        const repomindTokens = compRows.map((c) => c.repomind.tokens);
+        const repomindWins = repomindHallucinationTotal < baselineHallucinationTotal &&
+            median(repomindTokens) <= median(baselineTokens);
+        results.push({
+            category,
+            questionIds,
+            baselineHallucinationTotal,
+            repomindHallucinationTotal,
+            baselineMedianTokens: median(baselineTokens),
+            repomindMedianTokens: median(repomindTokens),
+            repomindWins,
+        });
+    }
+    return results;
+}
+export function computeHallucinationPass(categoryWins) {
+    const wins = categoryWins.filter((row) => row.repomindWins).length;
+    return wins >= 2 && categoryWins.length >= 2;
+}
+export function mergePassFlags(tokenPass, hallucinationPass) {
+    if (hallucinationPass === null) {
+        return tokenPass ? null : false;
+    }
+    return tokenPass && hallucinationPass;
+}

package/dist/ab-demo/live-answer.d.ts

@@ -0,0 +1,6 @@
+import type { DocIndex } from '../index/doc-index.js';
+import type { AbQuestion, LiveArmAnswer } from './types.js';
+/** Build a grounded answer excerpt from baseline grep-then-read retrieval. */
+export declare function buildBaselineAnswer(index: DocIndex, question: AbQuestion): LiveArmAnswer;
+/** Build a grounded answer excerpt from RepoMind search_docs + get_doc retrieval. */
+export declare function buildRepomindAnswer(index: DocIndex, question: AbQuestion): LiveArmAnswer;

package/dist/ab-demo/live-answer.js

@@ -0,0 +1,89 @@
+import { getDoc } from '../tools/get-doc.js';
+import { searchDocs } from '../tools/search-docs.js';
+import { runArmBaseline } from './arm-baseline.js';
+import { runArmRepomind } from './arm-repomind.js';
+import { BASELINE_CLAUDE_SNIPPET, mcpToolSchemaTokenEstimate, } from './session-overhead.js';
+import { estimateTokens } from './estimate-tokens.js';
+const SNIPPET_CHARS = 1200;
+const GET_DOC_LIMIT = 3;
+function excerpt(body, maxChars = SNIPPET_CHARS) {
+    const trimmed = body.trim();
+    if (trimmed.length <= maxChars) {
+        return trimmed;
+    }
+    return `${trimmed.slice(0, maxChars)}…`;
+}
+function queryTerms(prompt) {
+    return prompt
+        .toLowerCase()
+        .split(/[^a-z0-9]+/)
+        .filter((term) => term.length >= 3);
+}
+function docMatchesTerms(doc, terms) {
+    if (terms.length === 0) {
+        return true;
+    }
+    const haystack = `${doc.title}\n${doc.tags.join(' ')}\n${doc.body}`.toLowerCase();
+    return terms.some((term) => haystack.includes(term));
+}
+/** Build a grounded answer excerpt from baseline grep-then-read retrieval. */
+export function buildBaselineAnswer(index, question) {
+    const docs = index.refresh();
+    const terms = queryTerms(question.prompt);
+    let matched = docs.filter((doc) => docMatchesTerms(doc, terms));
+    let strategy = 'grep-then-read';
+    if (matched.length === 0) {
+        matched = docs.slice(0, 5);
+        strategy = 'read-all';
+    }
+    else {
+        matched = matched.slice(0, 5);
+    }
+    const listing = docs.map((doc) => doc.relativePath).join('\n');
+    const sessionOverhead = estimateTokens(BASELINE_CLAUDE_SNIPPET) + estimateTokens(listing);
+    const perQuestionOverhead = Math.ceil(sessionOverhead / Math.max(docs.length, 1));
+    const simulated = runArmBaseline(index, question, perQuestionOverhead);
+    const parts = matched.map((doc) => {
+        const title = doc.title || doc.slug;
+        return `## ${title} (${doc.slug})\n\n${excerpt(doc.body)}`;
+    });
+    return {
+        arm: 'baseline',
+        questionId: question.id,
+        answer: parts.join('\n\n---\n\n'),
+        retrievedSlugs: matched.map((doc) => doc.slug),
+        tokens: simulated.tokens,
+        filesRead: matched.length,
+        strategy,
+        searchHits: matched.length,
+        docsFetched: matched.length,
+    };
+}
+/** Build a grounded answer excerpt from RepoMind search_docs + get_doc retrieval. */
+export function buildRepomindAnswer(index, question) {
+    const docs = index.refresh();
+    const sessionOverhead = mcpToolSchemaTokenEstimate();
+    const perQuestionOverhead = Math.ceil(sessionOverhead / Math.max(docs.length, 1));
+    const simulated = runArmRepomind(index, question, perQuestionOverhead);
+    const hits = searchDocs(index, { query: question.prompt });
+    const slugsToFetch = hits.slice(0, GET_DOC_LIMIT).map((hit) => hit.slug);
+    const parts = [];
+    for (const slug of slugsToFetch) {
+        const doc = getDoc(index, slug);
+        if (doc.found && doc.body) {
+            const title = typeof doc.frontmatter?.title === 'string' ? doc.frontmatter.title : doc.slug;
+            parts.push(`## ${title} (${doc.slug})\n\n${excerpt(doc.body)}`);
+        }
+    }
+    return {
+        arm: 'repomind',
+        questionId: question.id,
+        answer: parts.length > 0 ? parts.join('\n\n---\n\n') : '_No matching documents retrieved._',
+        retrievedSlugs: slugsToFetch,
+        tokens: simulated.tokens,
+        filesRead: slugsToFetch.length,
+        strategy: 'search-then-get_doc',
+        searchHits: hits.length,
+        docsFetched: parts.length,
+    };
+}

package/dist/ab-demo/live-eval.d.ts

@@ -0,0 +1,19 @@
+import { DocIndex } from '../index/doc-index.js';
+import type { HumanScoreEntry, LiveEvalResult, LiveQuestionResult } from './types.js';
+import type { AbQuestion } from './types.js';
+export interface RunLiveEvalOptions {
+    cwd: string;
+    questionsFile: string;
+    outputPath: string;
+    dryRun?: boolean;
+    baselineTranscript?: string;
+    repomindTranscript?: string;
+}
+export declare function runLiveEvalQuestions(index: DocIndex, questions: AbQuestion[], options: Pick<RunLiveEvalOptions, 'baselineTranscript' | 'repomindTranscript'>): LiveQuestionResult[];
+export declare function exportBlindPack(live: LiveQuestionResult[], outputPath: string, meta: {
+    corpusCwd: string;
+    runAt: string;
+}): void;
+export declare function buildLiveEvalResult(options: RunLiveEvalOptions, questionsFile: AbQuestion[], questionsVersion: number, live: LiveQuestionResult[], humanScores?: HumanScoreEntry[] | null): LiveEvalResult;
+export declare function runLiveEval(options: RunLiveEvalOptions): number;
+export declare function mergeHumanScores(resultsPath: string, scores: HumanScoreEntry[], outputPath?: string): LiveEvalResult;

package/dist/ab-demo/live-eval.js

@@ -0,0 +1,192 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { DocIndex } from '../index/doc-index.js';
+import { computeCategoryWins, computeHallucinationPass, computeTokenPass, mergePassFlags, summarizeArmTokens, } from './compute-pass.js';
+import { loadQuestions } from './load-questions.js';
+import { buildBaselineAnswer, buildRepomindAnswer } from './live-answer.js';
+import { parseTranscriptTokensSync } from './record-transcript.js';
+import { validateQuestionsAgainstIndex } from './validate-questions.js';
+function tokenWinner(baseline, repomind) {
+    if (repomind < baseline) {
+        return 'repomind';
+    }
+    if (repomind > baseline) {
+        return 'baseline';
+    }
+    return 'tie';
+}
+function assignBlindLabels(seed) {
+    return seed % 2 === 0
+        ? { baseline: 'A', repomind: 'B' }
+        : { baseline: 'B', repomind: 'A' };
+}
+function applyTranscriptTokens(answer, transcriptPath) {
+    if (!transcriptPath) {
+        answer.transcriptSource = 'simulated';
+        return;
+    }
+    const usage = parseTranscriptTokensSync(transcriptPath);
+    if (usage.source === 'transcript' && usage.totalTokens > 0) {
+        answer.tokens = usage.totalTokens;
+        answer.transcriptSource = 'transcript';
+    }
+    else {
+        answer.transcriptSource = 'simulated';
+    }
+}
+function toComparison(row) {
+    return {
+        questionId: row.questionId,
+        prompt: row.prompt,
+        anchorSlugs: row.anchorSlugs,
+        baseline: {
+            arm: 'baseline',
+            questionId: row.questionId,
+            tokens: row.baseline.tokens,
+            filesRead: row.baseline.filesRead,
+            strategy: row.baseline.strategy,
+        },
+        repomind: {
+            arm: 'repomind',
+            questionId: row.questionId,
+            tokens: row.repomind.tokens,
+            searchHits: row.repomind.searchHits,
+            docsFetched: row.repomind.docsFetched,
+        },
+        tokenWinner: row.tokenWinner,
+    };
+}
+export function runLiveEvalQuestions(index, questions, options) {
+    return questions.map((question, questionIndex) => {
+        const baseline = buildBaselineAnswer(index, question);
+        const repomind = buildRepomindAnswer(index, question);
+        applyTranscriptTokens(baseline, options.baselineTranscript);
+        applyTranscriptTokens(repomind, options.repomindTranscript);
+        const winner = tokenWinner(baseline.tokens, repomind.tokens);
+        return {
+            questionId: question.id,
+            prompt: question.prompt,
+            anchorSlugs: question.anchorSlugs,
+            tags: question.tags,
+            baseline,
+            repomind,
+            tokenWinner: winner,
+            blindLabels: assignBlindLabels(questionIndex),
+        };
+    });
+}
+export function exportBlindPack(live, outputPath, meta) {
+    const lines = [
+        '# RepoMind live eval — blind review pack',
+        '',
+        `Generated: ${meta.runAt}`,
+        `Corpus: ${meta.corpusCwd}`,
+        '',
+        'Score each answer 0–3 using `ab-demo/score-hallucination.md`.',
+        'Do not look up slugs until scoring is complete.',
+        '',
+    ];
+    for (const row of live) {
+        const labels = row.blindLabels ?? { baseline: 'A', repomind: 'B' };
+        const answerA = labels.baseline === 'A' ? row.baseline.answer : row.repomind.answer;
+        const answerB = labels.baseline === 'B' ? row.baseline.answer : row.repomind.answer;
+        lines.push(`## ${row.questionId}`, '', `**Prompt:** ${row.prompt}`, '');
+        lines.push(`### Answer A`, '', answerA, '');
+        lines.push(`### Answer B`, '', answerB, '');
+        lines.push('---', '');
+    }
+    fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+    fs.writeFileSync(outputPath, lines.join('\n'), 'utf8');
+}
+export function buildLiveEvalResult(options, questionsFile, questionsVersion, live, humanScores = null) {
+    const comparisons = live.map(toComparison);
+    const baseline = summarizeArmTokens('baseline', comparisons);
+    const repomind = summarizeArmTokens('repomind', comparisons);
+    const { tokenPass, repomindWins, passThreshold } = computeTokenPass(comparisons, baseline, repomind);
+    const questionsById = new Map(questionsFile.map((q) => [q.id, q]));
+    let hallucinationPass = null;
+    let categoryWins;
+    if (humanScores && humanScores.length > 0) {
+        const wins = computeCategoryWins(humanScores, comparisons, questionsById);
+        categoryWins = wins.map((row) => ({
+            category: row.category,
+            repomindWins: row.repomindWins,
+            baselineHallucinationTotal: row.baselineHallucinationTotal,
+            repomindHallucinationTotal: row.repomindHallucinationTotal,
+        }));
+        hallucinationPass = computeHallucinationPass(wins);
+    }
+    const pass = mergePassFlags(tokenPass, hallucinationPass);
+    return {
+        runAt: new Date().toISOString(),
+        evalKind: 'live',
+        corpusCwd: path.resolve(options.cwd),
+        questionsFile: options.questionsFile,
+        questionsVersion,
+        comparisons,
+        live,
+        baseline,
+        repomind,
+        repomindTokenWins: repomindWins,
+        passThreshold,
+        tokenPass,
+        humanScores,
+        hallucinationPass,
+        categoryWins,
+        pass,
+        note: humanScores === null
+            ? 'Token comparison complete. Add humanScores and re-run with --record-scores to compute hallucinationPass.'
+            : 'Token and hallucination gates evaluated.',
+    };
+}
+export function runLiveEval(options) {
+    const cwd = path.resolve(options.cwd);
+    const questionsPayload = loadQuestions(options.questionsFile);
+    const index = new DocIndex(cwd);
+    const report = validateQuestionsAgainstIndex(index, questionsPayload.questions, cwd);
+    if (report.missingAnchors.length > 0) {
+        for (const missing of report.missingAnchors) {
+            console.error(`missing anchor: question=${missing.questionId} slug=${missing.slug}`);
+        }
+        return 1;
+    }
+    console.log(`corpus ok: ${report.docCount} docs, ${report.questionCount} questions (${cwd})`);
+    if (options.dryRun) {
+        console.log('dry-run complete — eval arms not executed');
+        return 0;
+    }
+    const live = runLiveEvalQuestions(index, questionsPayload.questions, options);
+    const result = buildLiveEvalResult(options, questionsPayload.questions, questionsPayload.version, live, null);
+    const blindPath = options.outputPath.replace(/\.json$/i, '-blind.md');
+    exportBlindPack(live, blindPath, { corpusCwd: cwd, runAt: result.runAt });
+    result.blindPackPath = blindPath;
+    fs.mkdirSync(path.dirname(options.outputPath), { recursive: true });
+    fs.writeFileSync(options.outputPath, `${JSON.stringify(result, null, 2)}\n`, 'utf8');
+    console.log(`wrote ${options.outputPath}`);
+    console.log(`wrote ${blindPath}`);
+    console.log(`tokenPass: ${result.tokenPass} (repomind wins ${result.repomindTokenWins}/${result.comparisons.length}, median ${result.repomind.medianTokens} vs ${result.baseline.medianTokens})`);
+    console.log('Next: blind-score answers, then `repo-mind ab-eval --record-scores <results.json>`');
+    return 0;
+}
+export function mergeHumanScores(resultsPath, scores, outputPath) {
+    const raw = JSON.parse(fs.readFileSync(resultsPath, 'utf8'));
+    if (!raw.live || raw.evalKind !== 'live') {
+        throw new Error('results file is not a live eval output');
+    }
+    const questions = raw.live.map((row) => ({
+        id: row.questionId,
+        prompt: row.prompt,
+        anchorSlugs: row.anchorSlugs,
+        tags: row.tags,
+    }));
+    const merged = buildLiveEvalResult({
+        cwd: raw.corpusCwd,
+        questionsFile: raw.questionsFile,
+        outputPath: outputPath ?? resultsPath,
+    }, questions, raw.questionsVersion, raw.live, scores);
+    merged.blindPackPath = raw.blindPackPath;
+    merged.runAt = raw.runAt;
+    const dest = outputPath ?? resultsPath;
+    fs.writeFileSync(dest, `${JSON.stringify(merged, null, 2)}\n`, 'utf8');
+    return merged;
+}

package/dist/ab-demo/paths.d.ts

@@ -3,3 +3,4 @@ export declare function corpusPath(abDemoRoot: string): string;
 export declare function questionsPath(abDemoRoot: string): string;
 export declare function resultsDir(abDemoRoot: string): string;
 export declare function scoreRubricPath(abDemoRoot: string): string;
+export declare function skyforgeQuestionsPath(abDemoRoot: string): string;

package/dist/ab-demo/paths.js

@@ -29,3 +29,6 @@ export function resultsDir(abDemoRoot) {
 export function scoreRubricPath(abDemoRoot) {
     return path.join(abDemoRoot, 'score-hallucination.md');
 }
+export function skyforgeQuestionsPath(abDemoRoot) {
+    return path.join(abDemoRoot, 'skyforge-questions.json');
+}

package/dist/ab-demo/record-transcript.d.ts

@@ -0,0 +1,9 @@
+export interface TranscriptTokenUsage {
+    inputTokens: number;
+    outputTokens: number;
+    totalTokens: number;
+    source: 'transcript' | 'unavailable';
+}
+/** Sum token usage from agent transcript JSONL (best-effort). */
+export declare function parseTranscriptTokens(filePath: string): Promise<TranscriptTokenUsage>;
+export declare function parseTranscriptTokensSync(filePath: string): TranscriptTokenUsage;

package/dist/ab-demo/record-transcript.js

@@ -0,0 +1,97 @@
+import fs from 'node:fs';
+import readline from 'node:readline';
+function addUsage(target, usage) {
+    const input = typeof usage.input_tokens === 'number'
+        ? usage.input_tokens
+        : typeof usage.prompt_tokens === 'number'
+            ? usage.prompt_tokens
+            : 0;
+    const output = typeof usage.output_tokens === 'number'
+        ? usage.output_tokens
+        : typeof usage.completion_tokens === 'number'
+            ? usage.completion_tokens
+            : 0;
+    target.inputTokens += input;
+    target.outputTokens += output;
+    target.totalTokens += input + output;
+}
+function extractUsageFromObject(value, target) {
+    if (!value || typeof value !== 'object') {
+        return;
+    }
+    const record = value;
+    if (record.usage && typeof record.usage === 'object') {
+        addUsage(target, record.usage);
+    }
+    if (record.message && typeof record.message === 'object') {
+        const message = record.message;
+        if (message.usage && typeof message.usage === 'object') {
+            addUsage(target, message.usage);
+        }
+    }
+    if (record.metadata && typeof record.metadata === 'object') {
+        const metadata = record.metadata;
+        if (metadata.usage && typeof metadata.usage === 'object') {
+            addUsage(target, metadata.usage);
+        }
+    }
+}
+/** Sum token usage from agent transcript JSONL (best-effort). */
+export async function parseTranscriptTokens(filePath) {
+    if (!fs.existsSync(filePath)) {
+        return { inputTokens: 0, outputTokens: 0, totalTokens: 0, source: 'unavailable' };
+    }
+    const usage = {
+        inputTokens: 0,
+        outputTokens: 0,
+        totalTokens: 0,
+        source: 'transcript',
+    };
+    const stream = fs.createReadStream(filePath, { encoding: 'utf8' });
+    const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+    for await (const line of rl) {
+        const trimmed = line.trim();
+        if (!trimmed) {
+            continue;
+        }
+        try {
+            const parsed = JSON.parse(trimmed);
+            extractUsageFromObject(parsed, usage);
+        }
+        catch {
+            // skip non-JSON lines
+        }
+    }
+    if (usage.totalTokens === 0) {
+        usage.source = 'unavailable';
+    }
+    return usage;
+}
+export function parseTranscriptTokensSync(filePath) {
+    if (!fs.existsSync(filePath)) {
+        return { inputTokens: 0, outputTokens: 0, totalTokens: 0, source: 'unavailable' };
+    }
+    const usage = {
+        inputTokens: 0,
+        outputTokens: 0,
+        totalTokens: 0,
+        source: 'transcript',
+    };
+    const content = fs.readFileSync(filePath, 'utf8');
+    for (const line of content.split('\n')) {
+        const trimmed = line.trim();
+        if (!trimmed) {
+            continue;
+        }
+        try {
+            extractUsageFromObject(JSON.parse(trimmed), usage);
+        }
+        catch {
+            // skip
+        }
+    }
+    if (usage.totalTokens === 0) {
+        usage.source = 'unavailable';
+    }
+    return usage;
+}

package/dist/ab-demo/types.d.ts

@@ -63,3 +63,60 @@ export interface AbRunResult {
     arms: AbArmsRunResult;
     pass: boolean | null;
 }
+export interface HumanScoreEntry {
+    questionId: string;
+    baseline: number;
+    repomind: number;
+    reviewer?: string;
+    notes?: string;
+}
+export interface LiveArmAnswer {
+    arm: 'baseline' | 'repomind';
+    questionId: string;
+    answer: string;
+    retrievedSlugs: string[];
+    tokens: number;
+    filesRead: number;
+    strategy: string;
+    searchHits: number;
+    docsFetched: number;
+    transcriptSource?: 'transcript' | 'simulated';
+}
+export interface LiveQuestionResult {
+    questionId: string;
+    prompt: string;
+    anchorSlugs: string[];
+    tags?: string[];
+    baseline: LiveArmAnswer;
+    repomind: LiveArmAnswer;
+    tokenWinner: 'baseline' | 'repomind' | 'tie';
+    blindLabels?: {
+        baseline: 'A' | 'B';
+        repomind: 'A' | 'B';
+    };
+}
+export interface LiveEvalResult {
+    runAt: string;
+    evalKind: 'live';
+    corpusCwd: string;
+    questionsFile: string;
+    questionsVersion: number;
+    comparisons: AbQuestionComparison[];
+    live: LiveQuestionResult[];
+    baseline: AbArmSummary;
+    repomind: AbArmSummary;
+    repomindTokenWins: number;
+    passThreshold: number;
+    tokenPass: boolean;
+    humanScores: HumanScoreEntry[] | null;
+    hallucinationPass: boolean | null;
+    categoryWins?: Array<{
+        category: string;
+        repomindWins: boolean;
+        baselineHallucinationTotal: number;
+        repomindHallucinationTotal: number;
+    }>;
+    pass: boolean | null;
+    blindPackPath?: string;
+    note: string;
+}

package/dist/ab-demo/validate-questions.d.ts

@@ -0,0 +1,5 @@
+import type { DocIndex } from '../index/doc-index.js';
+import type { AbDryRunReport, AbQuestion } from './types.js';
+export declare function validateQuestionsAgainstIndex(index: DocIndex, questions: AbQuestion[], label: string): AbDryRunReport;
+export declare function defaultSkyforgeQuestionsPath(abDemoRoot: string): string;
+export declare function assertQuestionsFile(filePath: string): void;

package/dist/ab-demo/validate-questions.js

@@ -0,0 +1,32 @@
+import fs from 'node:fs';
+import path from 'node:path';
+export function validateQuestionsAgainstIndex(index, questions, label) {
+    const knowledgeRoot = index.getKnowledgeRoot();
+    if (!knowledgeRoot) {
+        throw new Error(`no docs/ found in ${label}`);
+    }
+    const docs = index.refresh();
+    const slugSet = new Set(docs.map((doc) => doc.slug));
+    const missingAnchors = [];
+    for (const question of questions) {
+        for (const slug of question.anchorSlugs) {
+            if (!slugSet.has(slug)) {
+                missingAnchors.push({ questionId: question.id, slug });
+            }
+        }
+    }
+    return {
+        corpusPath: knowledgeRoot,
+        docCount: docs.length,
+        questionCount: questions.length,
+        missingAnchors,
+    };
+}
+export function defaultSkyforgeQuestionsPath(abDemoRoot) {
+    return path.join(abDemoRoot, 'skyforge-questions.json');
+}
+export function assertQuestionsFile(filePath) {
+    if (!fs.existsSync(filePath)) {
+        throw new Error(`questions file not found: ${filePath}`);
+    }
+}