@aperdomoll90/ledger-ai 1.3.0 → 1.4.2

package/dist/lib/eval/eval-judge-session.js

@@ -0,0 +1,233 @@
+// eval-judge-session.ts
+// Session state, input parsing, progress rendering, and durable writes for
+// the `ledger eval:judge` rejudging walkthrough.
+import { createInterface } from 'node:readline';
+import { searchHybrid } from '../search/ai-search.js';
+import { CURRENT_SEARCH_CONFIG } from './eval-store.js';
+// =============================================================================
+// Pure helpers (unit-tested)
+// =============================================================================
+export function parseGradeInput(rawInput) {
+    const input = rawInput.trim();
+    if (input === '0' || input === '1' || input === '2' || input === '3') {
+        return { kind: 'grade', value: parseInt(input, 10) };
+    }
+    if (input === 's')
+        return { kind: 'skip' };
+    if (input === 'b')
+        return { kind: 'back' };
+    if (input === 'n')
+        return { kind: 'note' };
+    if (input === '?')
+        return { kind: 'rubric' };
+    if (input === 'q')
+        return { kind: 'quit' };
+    return { kind: 'invalid', raw: input };
+}
+export function pickNextUngraded(candidates) {
+    for (const candidate of candidates) {
+        if (!candidate.graded)
+            return candidate;
+    }
+    return null;
+}
+export function formatProgressLine(progress) {
+    const percentage = progress.queriesTotal > 0
+        ? Math.round((progress.queriesComplete / progress.queriesTotal) * 100)
+        : 0;
+    return `Progress: ${progress.queriesComplete} / ${progress.queriesTotal} queries complete (${percentage}%). Judgments: ${progress.judgmentsTotal}.`;
+}
+// =============================================================================
+// Rubric
+// =============================================================================
+const RUBRIC_TEXT = `
+TREC 4-level grading rubric:
+
+  0  NOT RELEVANT    No useful info for this query. Wrong topic.
+  1  RELATED         Touches the topic but doesn't answer.
+  2  RELEVANT        Answers the query, but not the ideal/canonical source.
+  3  HIGHLY RELEVANT The canonical, complete answer.
+
+Boundary heuristics:
+  1 vs 2: "Would a user be happy if this was the top result?" Yes = 2, No = 1.
+  2 vs 3: "Is there a better doc for this query that I know exists?" Yes = 2, No = 3.
+`;
+// =============================================================================
+// Database I/O
+// =============================================================================
+async function loadNextGolden(supabase, startId = 0) {
+    const { data, error } = await supabase
+        .from('eval_golden_dataset')
+        .select('id, query, tags, judgments:eval_golden_judgments(document_id, grade)')
+        .gte('id', startId)
+        .order('id', { ascending: true });
+    if (error) {
+        process.stderr.write(`[ledger] loadNextGolden failed: ${error.message}\n`);
+        return null;
+    }
+    if (!data)
+        return null;
+    for (const row of data) {
+        const gradedMap = new Map();
+        for (const judgment of row.judgments ?? []) {
+            gradedMap.set(judgment.document_id, judgment.grade);
+        }
+        return {
+            id: row.id,
+            query: row.query,
+            tags: row.tags ?? [],
+            existing_grades: gradedMap,
+        };
+    }
+    return null;
+}
+async function fetchProgress(supabase) {
+    const { count: queriesTotal } = await supabase
+        .from('eval_golden_dataset')
+        .select('*', { count: 'exact', head: true });
+    const { count: judgmentsTotal } = await supabase
+        .from('eval_golden_judgments')
+        .select('*', { count: 'exact', head: true });
+    const { data: rpcData } = await supabase
+        .rpc('count_golden_with_min_judgments', { p_min: 10 });
+    const queriesComplete = typeof rpcData === 'number' ? rpcData : 0;
+    return {
+        queriesTotal: queriesTotal ?? 0,
+        queriesComplete,
+        judgmentsTotal: judgmentsTotal ?? 0,
+    };
+}
+// =============================================================================
+// Prompt helper
+// =============================================================================
+function promptUser(readline, question) {
+    return new Promise(resolve => {
+        readline.question(question, (answer) => resolve(answer));
+    });
+}
+function snippet(content, maxChars = 200) {
+    if (!content)
+        return '';
+    return content.replace(/\s+/g, ' ').slice(0, maxChars);
+}
+// =============================================================================
+// Interactive session
+// =============================================================================
+export async function runJudgeSession(clients, startGoldenId) {
+    const supabase = clients.supabase;
+    const readline = createInterface({ input: process.stdin, output: process.stdout });
+    try {
+        let currentId = startGoldenId ?? 0;
+        while (true) {
+            const progress = await fetchProgress(supabase);
+            console.log('');
+            console.log(formatProgressLine(progress));
+            console.log('');
+            const golden = await loadNextGolden(supabase, currentId);
+            if (!golden) {
+                console.log('No more queries to judge. Done.');
+                return;
+            }
+            // Run search for this query
+            const searchResults = await searchHybrid(clients, {
+                query: golden.query,
+                limit: CURRENT_SEARCH_CONFIG.limit,
+                reranker: CURRENT_SEARCH_CONFIG.reranker,
+            });
+            // Build candidate list from top 10
+            const candidates = [];
+            for (const result of searchResults.slice(0, 10)) {
+                candidates.push({
+                    id: result.id,
+                    name: result.name ?? '<unknown>',
+                    score: result.score ?? result.similarity ?? 0,
+                    content: snippet(result.content),
+                    graded: golden.existing_grades.has(result.id),
+                });
+            }
+            const ungradedList = candidates.filter(candidate => !candidate.graded);
+            if (ungradedList.length === 0) {
+                // This query is fully graded. Advance.
+                currentId = golden.id + 1;
+                continue;
+            }
+            // Print header
+            console.log('='.repeat(60));
+            console.log(`Query #${golden.id}: "${golden.query}"`);
+            if (golden.tags.length > 0)
+                console.log(`Tags: ${golden.tags.join(', ')}`);
+            if (golden.existing_grades.size > 0) {
+                console.log('Already graded:');
+                for (const [documentId, grade] of golden.existing_grades.entries()) {
+                    console.log(`  #${documentId} -> ${grade}`);
+                }
+            }
+            console.log('='.repeat(60));
+            let pendingNote = null;
+            let candidateIndex = 0;
+            while (candidateIndex < ungradedList.length) {
+                const candidate = ungradedList[candidateIndex];
+                console.log('');
+                console.log(`[${candidateIndex + 1}/${ungradedList.length}]  #${candidate.id} ${candidate.name}  (score ${candidate.score.toFixed(3)})`);
+                console.log(`"${candidate.content}..."`);
+                const answer = await promptUser(readline, 'Grade [0/1/2/3]  s=skip  b=back  n=notes  ?=rubric  q=save & quit: ');
+                const parsed = parseGradeInput(answer);
+                if (parsed.kind === 'invalid') {
+                    console.log(`(unrecognized input "${parsed.raw}". Press ? for rubric.)`);
+                    continue;
+                }
+                if (parsed.kind === 'rubric') {
+                    console.log(RUBRIC_TEXT);
+                    continue;
+                }
+                if (parsed.kind === 'note') {
+                    pendingNote = await promptUser(readline, 'Note: ');
+                    continue;
+                }
+                if (parsed.kind === 'quit') {
+                    console.log('Saving and exiting.');
+                    return;
+                }
+                if (parsed.kind === 'skip') {
+                    candidateIndex++;
+                    pendingNote = null;
+                    continue;
+                }
+                if (parsed.kind === 'back') {
+                    if (candidateIndex > 0)
+                        candidateIndex--;
+                    pendingNote = null;
+                    continue;
+                }
+                // Grade: durable write via RPC
+                const { error: rpcError } = await supabase.rpc('judgment_create', {
+                    p_golden_id: golden.id,
+                    p_document_id: candidate.id,
+                    p_grade: parsed.value,
+                    p_judged_by: 'adrian',
+                    p_notes: pendingNote,
+                });
+                if (rpcError) {
+                    // Duplicate? Try update instead.
+                    const { error: updateError } = await supabase.rpc('judgment_update', {
+                        p_golden_id: golden.id,
+                        p_document_id: candidate.id,
+                        p_grade: parsed.value,
+                        p_notes: pendingNote,
+                    });
+                    if (updateError) {
+                        console.error(`  [ERR] Could not save grade: ${updateError.message}`);
+                        continue;
+                    }
+                }
+                pendingNote = null;
+                candidateIndex++;
+            }
+            console.log(`Query #${golden.id} complete.`);
+            currentId = golden.id + 1;
+        }
+    }
+    finally {
+        readline.close();
+    }
+}

package/dist/lib/eval/eval-store.js

@@ -0,0 +1,105 @@
+// eval-store.ts
+// Persistence layer for eval runs — save results to eval_runs table and load previous runs.
+/**
+ * Current search configuration. Saved with each eval run for reproducibility.
+ * Single source of truth — used by both the eval script and CLI command.
+ * Update this when search parameters change (threshold, model, reranker, etc.).
+ */
+export const CURRENT_SEARCH_CONFIG = {
+    threshold: 0.38,
+    reciprocalRankFusionK: 60,
+    embedding_model: 'openai/text-embedding-3-small',
+    limit: 10,
+    chunking: 'recursive',
+    chunk_max_size: 1000,
+    chunk_overlap: 200,
+    context_enrichment: true,
+    context_enrichment_model: 'gpt-4o-mini',
+    reranker: 'none',
+    hit_threshold: 2,
+    ndcg_gain_formula: '2^g - 1',
+};
+// =============================================================================
+// Save
+// =============================================================================
+export async function saveEvalRun(supabase, props) {
+    const { metrics, config, results, confidenceIntervals, scoreCalibration, coverageAnalysis } = props;
+    const row = {
+        config,
+        test_case_count: metrics.totalCases,
+        hit_rate: metrics.hitRate,
+        first_result_accuracy: metrics.firstResultAccuracy,
+        recall: metrics.recall,
+        zero_result_rate: metrics.zeroResultRate,
+        avg_response_time_ms: metrics.avgResponseTimeMs,
+        mean_reciprocal_rank: metrics.meanReciprocalRank,
+        normalized_discounted_cumulative_gain: metrics.normalizedDiscountedCumulativeGain,
+        confidence_intervals: confidenceIntervals ?? null,
+        score_calibration: scoreCalibration ?? null,
+        coverage_analysis: coverageAnalysis ?? null,
+        results_by_tag: metrics.tagStats,
+        missed_queries: metrics.missed.map(missedResult => ({
+            query: missedResult.testCase.query,
+            tags: missedResult.testCase.tags,
+            judgments: missedResult.testCase.judgments,
+            got: missedResult.returnedIds,
+            gotScores: missedResult.returnedScores,
+        })),
+        per_query_results: results.map(testResult => ({
+            query: testResult.testCase.query,
+            tags: testResult.testCase.tags,
+            judgments: testResult.testCase.judgments,
+            hit: testResult.hit,
+            firstResultHit: testResult.firstResultHit,
+            position: testResult.position,
+            expectedFound: testResult.expectedFound,
+            expectedTotal: testResult.expectedTotal,
+            responseTimeMs: testResult.responseTimeMs,
+            reciprocalRank: testResult.reciprocalRank,
+            normalizedDiscountedCumulativeGain: testResult.normalizedDiscountedCumulativeGain,
+            returnedIds: testResult.returnedIds,
+            returnedScores: testResult.returnedScores,
+        })),
+    };
+    const { data, error } = await supabase
+        .from('eval_runs')
+        .insert(row)
+        .select('id')
+        .single();
+    if (error) {
+        throw new Error(`Failed to save eval run (${props.metrics.totalCases} cases): ${error.message}`);
+    }
+    return data.id;
+}
+// =============================================================================
+// Load
+// =============================================================================
+export async function loadPreviousRun(supabase) {
+    const { data, error } = await supabase
+        .from('eval_runs')
+        .select('*')
+        .order('run_date', { ascending: false })
+        .limit(1)
+        .single();
+    if (error) {
+        if (error.code !== 'PGRST116') {
+            process.stderr.write(`[ledger] loadPreviousRun failed: ${error.message}\n`);
+        }
+        return null;
+    }
+    return data;
+}
+export async function loadEvalRun(supabase, runId) {
+    const { data, error } = await supabase
+        .from('eval_runs')
+        .select('*')
+        .eq('id', runId)
+        .single();
+    if (error) {
+        if (error.code !== 'PGRST116') {
+            process.stderr.write(`[ledger] loadEvalRun(${runId}) failed: ${error.message}\n`);
+        }
+        return null;
+    }
+    return data;
+}

package/dist/lib/eval/eval.js

@@ -0,0 +1,303 @@
+// eval.ts
+// Types and metric computation for search evaluation.
+// Pure functions — no I/O, no database calls.
+/**
+ * Rate metrics (hit rate, first-result accuracy, recall, MRR) count a result
+ * as "good" when its grade is at or above this threshold. Only NDCG uses the
+ * full 2^g - 1 gain function across all grades.
+ */
+export const HIT_THRESHOLD = 2;
+// =============================================================================
+// NDCG@k helper — graded (2^g - 1 gain, TREC 4-level)
+// =============================================================================
+function gradeGain(grade) {
+    return Math.pow(2, grade) - 1;
+}
+function computeNormalizedDiscountedCumulativeGain(returnedIds, gradeByDocId) {
+    // Collect all non-zero grades (any doc that contributes to ideal ranking).
+    const relevantGrades = [];
+    for (const grade of gradeByDocId.values()) {
+        if (grade >= 1)
+            relevantGrades.push(grade);
+    }
+    if (relevantGrades.length === 0)
+        return 0;
+    // DCG against the returned order
+    let discountedCumulativeGain = 0;
+    for (let position = 0; position < returnedIds.length; position++) {
+        const grade = gradeByDocId.get(returnedIds[position]) ?? 0;
+        discountedCumulativeGain += gradeGain(grade) / Math.log2(position + 2);
+    }
+    // IDCG: ideal ordering is grades sorted descending
+    const idealGrades = relevantGrades.slice().sort((gradeA, gradeB) => gradeB - gradeA);
+    let idealDiscountedCumulativeGain = 0;
+    for (let position = 0; position < idealGrades.length; position++) {
+        idealDiscountedCumulativeGain += gradeGain(idealGrades[position]) / Math.log2(position + 2);
+    }
+    if (idealDiscountedCumulativeGain === 0)
+        return 0;
+    return discountedCumulativeGain / idealDiscountedCumulativeGain;
+}
+// =============================================================================
+// Scoring a single test case
+// =============================================================================
+export function scoreTestCase(testCase, searchResults, responseTimeMs) {
+    const returnedIds = searchResults.map(result => result.id);
+    const returnedScores = searchResults.map(result => result.score ?? result.similarity ?? 0);
+    // Build grade lookup. Missing doc defaults to grade 0 (treated as irrelevant).
+    const gradeByDocId = new Map();
+    for (const judgment of testCase.judgments) {
+        gradeByDocId.set(judgment.document_id, judgment.grade);
+    }
+    const relevantJudgments = testCase.judgments.filter(judgment => judgment.grade >= HIT_THRESHOLD);
+    const isOutOfScope = relevantJudgments.length === 0;
+    if (isOutOfScope) {
+        return {
+            testCase,
+            returnedIds,
+            returnedScores,
+            hit: searchResults.length === 0,
+            firstResultHit: searchResults.length === 0,
+            expectedFound: 0,
+            expectedTotal: 0,
+            position: null,
+            responseTimeMs,
+            reciprocalRank: 0,
+            normalizedDiscountedCumulativeGain: 0,
+        };
+    }
+    // First returned doc with grade >= HIT_THRESHOLD
+    let firstHitPosition = null;
+    for (let position = 0; position < returnedIds.length; position++) {
+        const grade = gradeByDocId.get(returnedIds[position]) ?? 0;
+        if (grade >= HIT_THRESHOLD) {
+            firstHitPosition = position;
+            break;
+        }
+    }
+    const topGrade = returnedIds.length > 0 ? (gradeByDocId.get(returnedIds[0]) ?? 0) : 0;
+    const foundCount = returnedIds.filter(docId => (gradeByDocId.get(docId) ?? 0) >= HIT_THRESHOLD).length;
+    return {
+        testCase,
+        returnedIds,
+        returnedScores,
+        hit: firstHitPosition !== null,
+        firstResultHit: topGrade >= HIT_THRESHOLD,
+        expectedFound: foundCount,
+        expectedTotal: relevantJudgments.length,
+        position: firstHitPosition,
+        responseTimeMs,
+        reciprocalRank: firstHitPosition !== null ? 1 / (firstHitPosition + 1) : 0,
+        normalizedDiscountedCumulativeGain: computeNormalizedDiscountedCumulativeGain(returnedIds, gradeByDocId),
+    };
+}
+// =============================================================================
+// Aggregate metrics from scored results
+// =============================================================================
+const hasRelevantJudgment = (result) => result.testCase.judgments.some(judgment => judgment.grade >= HIT_THRESHOLD);
+export function computeMetrics(results) {
+    const normalResults = results.filter(hasRelevantJudgment);
+    const outOfScopeResults = results.filter(result => !hasRelevantJudgment(result));
+    const totalNormal = normalResults.length;
+    const hits = normalResults.filter(result => result.hit).length;
+    const firstResultHits = normalResults.filter(result => result.firstResultHit).length;
+    const totalExpected = normalResults.reduce((sum, result) => sum + result.expectedTotal, 0);
+    const totalFound = normalResults.reduce((sum, result) => sum + result.expectedFound, 0);
+    const zeroResults = normalResults.filter(result => result.returnedIds.length === 0).length;
+    const outOfScopeCorrect = outOfScopeResults.filter(result => result.hit).length;
+    const avgResponseTimeMs = results.length > 0
+        ? results.reduce((sum, result) => sum + result.responseTimeMs, 0) / results.length
+        : 0;
+    const tagStats = {};
+    for (const result of normalResults) {
+        for (const tag of result.testCase.tags) {
+            if (!tagStats[tag])
+                tagStats[tag] = { total: 0, hits: 0, firstHits: 0 };
+            tagStats[tag].total++;
+            if (result.hit)
+                tagStats[tag].hits++;
+            if (result.firstResultHit)
+                tagStats[tag].firstHits++;
+        }
+    }
+    return {
+        totalCases: results.length,
+        normalCases: totalNormal,
+        outOfScopeCases: outOfScopeResults.length,
+        hits,
+        firstResultHits,
+        totalExpected,
+        totalFound,
+        zeroResults,
+        outOfScopeCorrect,
+        avgResponseTimeMs,
+        hitRate: totalNormal > 0 ? (hits / totalNormal) * 100 : 0,
+        firstResultAccuracy: totalNormal > 0 ? (firstResultHits / totalNormal) * 100 : 0,
+        recall: totalExpected > 0 ? (totalFound / totalExpected) * 100 : 0,
+        zeroResultRate: totalNormal > 0 ? (zeroResults / totalNormal) * 100 : 0,
+        outOfScopeAccuracy: outOfScopeResults.length > 0 ? (outOfScopeCorrect / outOfScopeResults.length) * 100 : 0,
+        meanReciprocalRank: totalNormal > 0
+            ? normalResults.reduce((sum, result) => sum + result.reciprocalRank, 0) / totalNormal
+            : 0,
+        normalizedDiscountedCumulativeGain: totalNormal > 0
+            ? normalResults.reduce((sum, result) => sum + result.normalizedDiscountedCumulativeGain, 0) / totalNormal
+            : 0,
+        tagStats,
+        missed: normalResults.filter(result => !result.hit),
+    };
+}
+// =============================================================================
+// Format report as string (no console.log — caller decides output)
+// =============================================================================
+export function formatReport(metrics) {
+    const lines = [];
+    lines.push('='.repeat(60));
+    lines.push(`Results — hit_threshold=${HIT_THRESHOLD}, ndcg_gain=2^g-1`);
+    lines.push('='.repeat(60));
+    lines.push('');
+    lines.push(`Test cases:          ${metrics.totalCases} total (${metrics.normalCases} normal, ${metrics.outOfScopeCases} out-of-scope)`);
+    lines.push('');
+    lines.push('METRICS:');
+    lines.push(`  Hit rate:              ${metrics.hitRate.toFixed(1)}% (${metrics.hits}/${metrics.normalCases} queries found at least one expected doc)`);
+    lines.push(`  First-result accuracy: ${metrics.firstResultAccuracy.toFixed(1)}% (${metrics.firstResultHits}/${metrics.normalCases} queries had correct #1 result)`);
+    lines.push(`  Recall:                ${metrics.recall.toFixed(1)}% (${metrics.totalFound}/${metrics.totalExpected} expected docs found across all queries)`);
+    lines.push(`  Zero-result rate:      ${metrics.zeroResultRate.toFixed(1)}% (${metrics.zeroResults}/${metrics.normalCases} queries returned nothing)`);
+    lines.push(`  Out-of-scope accuracy: ${metrics.outOfScopeAccuracy.toFixed(1)}% (${metrics.outOfScopeCorrect}/${metrics.outOfScopeCases} correctly returned nothing)`);
+    lines.push(`  Avg response time:     ${metrics.avgResponseTimeMs.toFixed(0)}ms`);
+    lines.push(`  MRR:                   ${metrics.meanReciprocalRank.toFixed(3)} (1.0 = perfect ranking, 0.5 = avg position 2)`);
+    lines.push(`  NDCG@k:                ${metrics.normalizedDiscountedCumulativeGain.toFixed(3)} (1.0 = perfect ranking of all relevant docs)`);
+    lines.push('');
+    if (metrics.missed.length > 0) {
+        lines.push(`MISSED QUERIES (no returned doc at grade >= ${HIT_THRESHOLD}):`);
+        for (const miss of metrics.missed) {
+            const relevantDocs = miss.testCase.judgments
+                .filter(judgment => judgment.grade >= HIT_THRESHOLD)
+                .map(judgment => `${judgment.document_id}(g${judgment.grade})`);
+            lines.push(`  "${miss.testCase.query}" — relevant [${relevantDocs.join(', ')}], got [${miss.returnedIds.slice(0, 5).join(', ')}]`);
+        }
+        lines.push('');
+    }
+    lines.push('BY TAG:');
+    const sortedTags = Object.entries(metrics.tagStats).sort((entryA, entryB) => entryB[1].total - entryA[1].total);
+    for (const [tag, stats] of sortedTags) {
+        const hitPercentage = ((stats.hits / stats.total) * 100).toFixed(0);
+        const firstResultPercentage = ((stats.firstHits / stats.total) * 100).toFixed(0);
+        lines.push(`  ${tag}: ${hitPercentage}% hit rate, ${firstResultPercentage}% first-result (${stats.total} queries)`);
+    }
+    lines.push('');
+    lines.push('='.repeat(60));
+    return lines.join('\n');
+}
+// =============================================================================
+// compareRuns — diff two eval runs and detect regressions
+// =============================================================================
+const INVERTED_METRICS = new Set(['zeroResultRate', 'avgResponseTimeMs']);
+const UNCHANGED_THRESHOLD = 0.01;
+export function compareRuns(current, previous) {
+    const metricKeys = [
+        'hitRate',
+        'firstResultAccuracy',
+        'recall',
+        'zeroResultRate',
+        'meanReciprocalRank',
+        'normalizedDiscountedCumulativeGain',
+        'avgResponseTimeMs',
+    ];
+    const improvements = [];
+    const regressions = [];
+    const unchanged = [];
+    for (const metricKey of metricKeys) {
+        const currentValue = current[metricKey];
+        const previousValue = previous[metricKey];
+        const diff = currentValue - previousValue;
+        const metricDiff = {
+            metric: metricKey,
+            current: currentValue,
+            previous: previousValue,
+            diff,
+        };
+        if (Math.abs(diff) < UNCHANGED_THRESHOLD) {
+            unchanged.push(metricDiff);
+            continue;
+        }
+        const isInverted = INVERTED_METRICS.has(metricKey);
+        // For normal metrics: positive diff = improvement. For inverted: negative diff = improvement.
+        const isImprovement = isInverted ? diff < 0 : diff > 0;
+        if (isImprovement) {
+            improvements.push(metricDiff);
+        }
+        else {
+            regressions.push(metricDiff);
+        }
+    }
+    const severity = determineSeverity(current, regressions);
+    return { improvements, regressions, unchanged, severity };
+}
+function determineSeverity(current, regressions) {
+    if (current.hitRate < 80 || current.zeroResultRate > 10) {
+        return 'critical';
+    }
+    if (regressions.length === 0) {
+        return 'ok';
+    }
+    // Worst regression drop: for normal metrics use |diff| (diff is negative for regressions),
+    // for inverted metrics use |diff| (diff is positive for regressions).
+    // In both cases Math.abs(diff) gives the magnitude of the drop.
+    const worstDrop = regressions.reduce((maxDrop, regression) => {
+        const dropMagnitude = Math.abs(regression.diff);
+        return dropMagnitude > maxDrop ? dropMagnitude : maxDrop;
+    }, 0);
+    if (worstDrop > 5)
+        return 'block';
+    if (worstDrop > 2)
+        return 'warning';
+    return 'ok';
+}
+// =============================================================================
+// formatComparison — human-readable comparison report
+// =============================================================================
+const PERCENTAGE_METRICS = new Set(['hitRate', 'firstResultAccuracy', 'recall', 'zeroResultRate']);
+function formatMetricValue(metricKey, value) {
+    if (metricKey === 'meanReciprocalRank' || metricKey === 'normalizedDiscountedCumulativeGain')
+        return value.toFixed(3);
+    if (PERCENTAGE_METRICS.has(metricKey))
+        return `${value.toFixed(1)}%`;
+    return value.toFixed(1);
+}
+export function formatComparison(comparison) {
+    const lines = [];
+    lines.push('='.repeat(60));
+    lines.push(`Run Comparison — severity: ${comparison.severity}`);
+    lines.push('='.repeat(60));
+    lines.push('');
+    if (comparison.improvements.length > 0) {
+        lines.push('IMPROVEMENTS:');
+        for (const metricDiff of comparison.improvements) {
+            const previousFormatted = formatMetricValue(metricDiff.metric, metricDiff.previous);
+            const currentFormatted = formatMetricValue(metricDiff.metric, metricDiff.current);
+            const diffFormatted = formatMetricValue(metricDiff.metric, Math.abs(metricDiff.diff));
+            lines.push(`  ${metricDiff.metric}: ${previousFormatted} → ${currentFormatted} (+${diffFormatted})`);
+        }
+        lines.push('');
+    }
+    if (comparison.regressions.length > 0) {
+        lines.push('REGRESSIONS:');
+        for (const metricDiff of comparison.regressions) {
+            const previousFormatted = formatMetricValue(metricDiff.metric, metricDiff.previous);
+            const currentFormatted = formatMetricValue(metricDiff.metric, metricDiff.current);
+            const diffFormatted = formatMetricValue(metricDiff.metric, Math.abs(metricDiff.diff));
+            lines.push(`  ${metricDiff.metric}: ${previousFormatted} → ${currentFormatted} (-${diffFormatted})`);
+        }
+        lines.push('');
+    }
+    if (comparison.unchanged.length > 0) {
+        lines.push('UNCHANGED:');
+        for (const metricDiff of comparison.unchanged) {
+            const currentFormatted = formatMetricValue(metricDiff.metric, metricDiff.current);
+            lines.push(`  ${metricDiff.metric}: ${currentFormatted}`);
+        }
+        lines.push('');
+    }
+    lines.push('='.repeat(60));
+    return lines.join('\n');
+}

package/dist/lib/file-writer.js

@@ -0,0 +1,23 @@
+import { writeFileSync, existsSync, readFileSync, mkdirSync, chmodSync } from 'fs';
+import { dirname } from 'path';
+/**
+ * Write a note's content to disk at the specified file_path.
+ * Creates parent directories if needed. Sets Unix permissions.
+ * Skips write if content matches existing file (idempotent).
+ */
+export function writeNoteFile(content, filePath, permissions) {
+    const dir = dirname(filePath);
+    const mode = permissions ? parseInt(permissions, 8) : 0o644;
+    if (!existsSync(dir)) {
+        mkdirSync(dir, { recursive: true });
+    }
+    if (existsSync(filePath)) {
+        const existing = readFileSync(filePath, 'utf-8');
+        if (existing === content) {
+            return { status: 'skipped', path: filePath };
+        }
+    }
+    writeFileSync(filePath, content, { mode });
+    chmodSync(filePath, mode);
+    return { status: 'written', path: filePath };
+}