@aperdomoll90/ledger-ai 1.4.0 → 1.4.2

package/dist/scripts/diagnose-first-result.js

@@ -0,0 +1,333 @@
+// diagnose-first-result.ts
+// Diagnostic analysis of first-result accuracy failures in eval run 16.
+// Classifies every query by failure mode, analyzes score distributions,
+// and writes a structured markdown report.
+//
+// Run: npx tsx src/scripts/diagnose-first-result.ts
+// Custom run: npx tsx src/scripts/diagnose-first-result.ts --run 14
+import 'dotenv/config';
+import { resolve } from 'path';
+import { config } from 'dotenv';
+import { createClient } from '@supabase/supabase-js';
+import { loadEvalRun } from '../lib/eval/eval-store.js';
+import { writeFileSync, mkdirSync } from 'fs';
+// Load .env from ~/.ledger/.env (same as other scripts)
+config({ path: resolve(process.env.HOME ?? '', '.ledger', '.env') });
+const supabaseUrl = process.env.SUPABASE_URL;
+const supabaseKey = process.env.SUPABASE_SERVICE_ROLE_KEY;
+if (!supabaseUrl || !supabaseKey) {
+    console.error('Missing SUPABASE_URL or SUPABASE_SERVICE_ROLE_KEY');
+    process.exit(1);
+}
+const supabase = createClient(supabaseUrl, supabaseKey);
+// Allow overriding run ID via --run flag, default to 16
+const runIdArg = process.argv.indexOf('--run');
+const RUN_ID = runIdArg !== -1 ? parseInt(process.argv[runIdArg + 1], 10) : 16;
+// =============================================================================
+// Classification (priority order per spec)
+// =============================================================================
+const HIT_THRESHOLD = 2;
+function classify(result) {
+    const topDocId = result.returnedIds.length > 0 ? result.returnedIds[0] : null;
+    const judgmentMap = new Map(result.judgments.map(j => [j.document_id, j.grade]));
+    const topDocGrade = topDocId !== null ? (judgmentMap.get(topDocId) ?? null) : null;
+    let category;
+    // Priority 1: out-of-scope
+    if (result.tags.includes('out-of-scope')) {
+        category = 'out-of-scope';
+    }
+    // Priority 2: top-1 correct
+    else if (result.firstResultHit) {
+        category = 'top-1-correct';
+    }
+    // Priority 3: unjudged winner (top doc has no judgment at all)
+    else if (topDocId !== null && topDocGrade === null) {
+        category = 'unjudged-winner';
+    }
+    // Priority 4: near miss (first correct at position 2-3, i.e., 0-indexed 1-2)
+    else if (result.position !== null && result.position >= 1 && result.position <= 2) {
+        category = 'near-miss';
+    }
+    // Priority 5: buried (first correct at position 4-10, i.e., 0-indexed 3-9)
+    else if (result.position !== null && result.position >= 3 && result.position <= 9) {
+        category = 'buried';
+    }
+    // Priority 6: absent (no grade >= 2 in top 10)
+    else if (!result.hit) {
+        category = 'absent';
+    }
+    // Fallback: position is set but doesn't match ranges (shouldn't happen)
+    else {
+        category = 'buried';
+    }
+    return {
+        query: result.query,
+        tags: result.tags,
+        category,
+        position: result.position,
+        returnedIds: result.returnedIds,
+        returnedScores: result.returnedScores,
+        judgments: result.judgments,
+        topDocId,
+        topDocGrade,
+        ndcg: result.normalizedDiscountedCumulativeGain,
+    };
+}
+function classifyAll(results) {
+    return results.map(classify);
+}
+function computeStats(values) {
+    if (values.length === 0)
+        return { min: 0, median: 0, max: 0, mean: 0 };
+    const sorted = [...values].sort((a, b) => a - b);
+    const mid = Math.floor(sorted.length / 2);
+    return {
+        min: sorted[0],
+        median: sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid],
+        max: sorted[sorted.length - 1],
+        mean: sorted.reduce((sum, v) => sum + v, 0) / sorted.length,
+    };
+}
+function analyzeScores(classified) {
+    const categories = ['near-miss', 'buried', 'absent', 'unjudged-winner'];
+    const stats = [];
+    for (const category of categories) {
+        const queries = classified.filter(q => q.category === category);
+        if (queries.length === 0)
+            continue;
+        const scoreAtPos1 = queries
+            .filter(q => q.returnedScores.length > 0)
+            .map(q => q.returnedScores[0]);
+        const scoreGapPos1Pos2 = queries
+            .filter(q => q.returnedScores.length >= 2)
+            .map(q => q.returnedScores[0] - q.returnedScores[1]);
+        const queriesWithCorrect = queries.filter(q => q.position !== null);
+        const scoreOfFirstCorrect = queriesWithCorrect.map(q => q.returnedScores[q.position]);
+        const scoreGapToCorrect = queriesWithCorrect.map(q => q.returnedScores[0] - q.returnedScores[q.position]);
+        stats.push({
+            category,
+            count: queries.length,
+            scoreAtPos1: computeStats(scoreAtPos1),
+            scoreGapPos1Pos2: computeStats(scoreGapPos1Pos2),
+            scoreOfFirstCorrect: queriesWithCorrect.length > 0 ? computeStats(scoreOfFirstCorrect) : null,
+            scoreGapToCorrect: queriesWithCorrect.length > 0 ? computeStats(scoreGapToCorrect) : null,
+            avgNdcg: queries.reduce((sum, q) => sum + q.ndcg, 0) / queries.length,
+        });
+    }
+    return stats;
+}
+function analyzeByTag(classified) {
+    const tagMap = new Map();
+    for (const query of classified) {
+        if (query.category === 'out-of-scope')
+            continue;
+        for (const tag of query.tags) {
+            if (!tagMap.has(tag))
+                tagMap.set(tag, []);
+            tagMap.get(tag).push(query);
+        }
+    }
+    const breakdowns = [];
+    for (const [tag, queries] of tagMap) {
+        const correct = queries.filter(q => q.category === 'top-1-correct').length;
+        const nearMiss = queries.filter(q => q.category === 'near-miss').length;
+        const buried = queries.filter(q => q.category === 'buried').length;
+        const absent = queries.filter(q => q.category === 'absent').length;
+        const unjudged = queries.filter(q => q.category === 'unjudged-winner').length;
+        const gaps = queries
+            .filter(q => q.returnedScores.length >= 2)
+            .map(q => q.returnedScores[0] - q.returnedScores[1]);
+        breakdowns.push({
+            tag,
+            total: queries.length,
+            correct,
+            nearMiss,
+            buried,
+            absent,
+            unjudged,
+            firstResultAccuracy: queries.length > 0 ? (correct / queries.length) * 100 : 0,
+            avgScoreGap: gaps.length > 0 ? gaps.reduce((s, v) => s + v, 0) / gaps.length : 0,
+        });
+    }
+    return breakdowns.sort((a, b) => a.firstResultAccuracy - b.firstResultAccuracy);
+}
+// =============================================================================
+// Report generation
+// =============================================================================
+function generateReport(classified, run) {
+    const scoreStats = analyzeScores(classified);
+    const tagBreakdowns = analyzeByTag(classified);
+    const nonScope = classified.filter(q => q.category !== 'out-of-scope');
+    const correct = classified.filter(q => q.category === 'top-1-correct');
+    const nearMiss = classified.filter(q => q.category === 'near-miss');
+    const buried = classified.filter(q => q.category === 'buried');
+    const absent = classified.filter(q => q.category === 'absent');
+    const unjudged = classified.filter(q => q.category === 'unjudged-winner');
+    const outOfScope = classified.filter(q => q.category === 'out-of-scope');
+    const lines = [];
+    // --- Header ---
+    lines.push(`# First-Result Accuracy Diagnostic: Run ${run.id}`);
+    lines.push('');
+    lines.push(`> Generated: ${new Date().toISOString().slice(0, 10)}`);
+    lines.push(`> Run date: ${run.run_date}`);
+    lines.push(`> Queries: ${classified.length} (${nonScope.length} scored, ${outOfScope.length} out-of-scope)`);
+    lines.push(`> First-result accuracy: ${run.first_result_accuracy.toFixed(1)}%`);
+    lines.push('');
+    // --- Executive summary ---
+    lines.push('## Executive Summary');
+    lines.push('');
+    const failCount = nonScope.length - correct.length;
+    lines.push(`${failCount} of ${nonScope.length} queries (${((failCount / nonScope.length) * 100).toFixed(1)}%) do not have the best result at position 1. `
+        + `The dominant failure mode is **${getDominantCategory(nearMiss.length, buried.length, absent.length, unjudged.length)}**. `
+        + `${unjudged.length > 0 ? `${unjudged.length} queries have unjudged documents at position 1 and need manual review before the classification is trustworthy.` : 'All top-1 documents have been judged.'}`);
+    lines.push('');
+    // --- Failure taxonomy ---
+    lines.push('## Failure Taxonomy');
+    lines.push('');
+    lines.push('| Category        | Count | %     | Description                             |');
+    lines.push('|-----------------|-------|-------|-----------------------------------------|');
+    lines.push(`| Top-1 correct   | ${pad(correct.length)}  | ${pad(pct(correct.length, nonScope.length))} | Position 1 has grade >= 2               |`);
+    lines.push(`| Near miss       | ${pad(nearMiss.length)}  | ${pad(pct(nearMiss.length, nonScope.length))} | First correct at position 2-3           |`);
+    lines.push(`| Buried          | ${pad(buried.length)}  | ${pad(pct(buried.length, nonScope.length))} | First correct at position 4-10          |`);
+    lines.push(`| Unjudged winner | ${pad(unjudged.length)}  | ${pad(pct(unjudged.length, nonScope.length))} | Top doc never graded (judgment gap)     |`);
+    lines.push(`| Absent          | ${pad(absent.length)}  | ${pad(pct(absent.length, nonScope.length))} | No grade >= 2 doc in top 10             |`);
+    lines.push(`| Out-of-scope    | ${pad(outOfScope.length)}  | n/a   | Excluded from metric                    |`);
+    lines.push('');
+    // --- Tag heatmap ---
+    lines.push('## Tag Breakdown');
+    lines.push('');
+    lines.push('Sorted by first-result accuracy (worst first).');
+    lines.push('');
+    lines.push('| Tag                | Total | Correct | Near Miss | Buried | Absent | Unjudged | 1st-Result% | Avg Gap  |');
+    lines.push('|--------------------|-------|---------|-----------|--------|--------|----------|-------------|----------|');
+    for (const t of tagBreakdowns) {
+        lines.push(`| ${padR(t.tag, 18)} | ${pad(t.total)}   | ${pad(t.correct)}     | ${pad(t.nearMiss)}       | ${pad(t.buried)}    | ${pad(t.absent)}    | ${pad(t.unjudged)}      | ${padR(t.firstResultAccuracy.toFixed(1) + '%', 11)} | ${t.avgScoreGap.toFixed(4)} |`);
+    }
+    lines.push('');
+    // --- Score analysis ---
+    lines.push('## Score Distribution by Failure Category');
+    lines.push('');
+    for (const stat of scoreStats) {
+        lines.push(`### ${stat.category} (${stat.count} queries, avg NDCG: ${stat.avgNdcg.toFixed(3)})`);
+        lines.push('');
+        lines.push('| Metric                  | Min    | Median | Max    | Mean   |');
+        lines.push('|-------------------------|--------|--------|--------|--------|');
+        lines.push(`| Score at position 1     | ${fmtRow(stat.scoreAtPos1)} |`);
+        lines.push(`| Gap (pos 1 vs pos 2)    | ${fmtRow(stat.scoreGapPos1Pos2)} |`);
+        if (stat.scoreOfFirstCorrect) {
+            lines.push(`| Score of first correct  | ${fmtRow(stat.scoreOfFirstCorrect)} |`);
+        }
+        if (stat.scoreGapToCorrect) {
+            lines.push(`| Gap to correct          | ${fmtRow(stat.scoreGapToCorrect)} |`);
+        }
+        lines.push('');
+    }
+    // --- Unjudged audit ---
+    if (unjudged.length > 0) {
+        lines.push('## Unjudged Audit');
+        lines.push('');
+        lines.push('These queries have a document at position 1 with no judgment. Grade them');
+        lines.push('via `ledger eval:judge` before trusting the failure classification.');
+        lines.push('');
+        lines.push('| Query | Top Doc ID | Tags |');
+        lines.push('|-------|------------|------|');
+        for (const q of unjudged) {
+            lines.push(`| ${q.query} | ${q.topDocId} | ${q.tags.join(', ')} |`);
+        }
+        lines.push('');
+    }
+    // --- Worst offenders ---
+    lines.push('## Worst Offenders');
+    lines.push('');
+    lines.push('Queries with the largest gap between expected and actual ranking (excluding absent and out-of-scope).');
+    lines.push('');
+    const offenders = classified
+        .filter(q => q.category === 'near-miss' || q.category === 'buried')
+        .sort((a, b) => {
+        const gapA = a.position !== null && a.returnedScores.length > 0
+            ? a.returnedScores[0] - a.returnedScores[a.position]
+            : 0;
+        const gapB = b.position !== null && b.returnedScores.length > 0
+            ? b.returnedScores[0] - b.returnedScores[b.position]
+            : 0;
+        return gapB - gapA;
+    })
+        .slice(0, 10);
+    lines.push('| Query | Category | Position | Top Doc (grade) | Score Gap |');
+    lines.push('|-------|----------|----------|-----------------|-----------|');
+    for (const q of offenders) {
+        const gradeLabel = q.topDocGrade !== null ? `g${q.topDocGrade}` : 'unjudged';
+        const gap = q.position !== null ? (q.returnedScores[0] - q.returnedScores[q.position]).toFixed(4) : 'n/a';
+        lines.push(`| ${q.query} | ${q.category} | ${q.position !== null ? q.position + 1 : 'n/a'} | ${q.topDocId} (${gradeLabel}) | ${gap} |`);
+    }
+    lines.push('');
+    // --- Hypotheses ---
+    lines.push('## Hypotheses');
+    lines.push('');
+    lines.push('Ranked by estimated impact (number of queries affected). These are generated');
+    lines.push('from the data patterns above and need human review before acting on them.');
+    lines.push('');
+    lines.push('*Hypotheses will be written manually after reviewing the data above.*');
+    lines.push('*The script produces the data; Adrian and Charlie interpret it together.*');
+    return lines.join('\n');
+}
+// --- Formatting helpers ---
+function pad(value, width = 5) {
+    return String(value).padStart(width);
+}
+function padR(value, width) {
+    return value.padEnd(width);
+}
+function pct(count, total) {
+    if (total === 0)
+        return '0.0%';
+    return ((count / total) * 100).toFixed(1) + '%';
+}
+function fmtRow(s) {
+    return `${s.min.toFixed(4)} | ${s.median.toFixed(4)} | ${s.max.toFixed(4)} | ${s.mean.toFixed(4)}`;
+}
+function getDominantCategory(nearMiss, buried, absent, unjudged) {
+    const max = Math.max(nearMiss, buried, absent, unjudged);
+    if (max === nearMiss)
+        return 'near miss (position 2-3)';
+    if (max === buried)
+        return 'buried (position 4-10)';
+    if (max === unjudged)
+        return 'unjudged winner (judgment gap)';
+    return 'absent (retrieval failure)';
+}
+// =============================================================================
+// Write report to disk
+// =============================================================================
+function writeReport(report) {
+    const scriptDir = new URL('.', import.meta.url).pathname;
+    const dir = resolve(scriptDir, '../../docs/analysis');
+    mkdirSync(dir, { recursive: true });
+    const path = resolve(dir, `first-result-accuracy-run${RUN_ID}.md`);
+    writeFileSync(path, report, 'utf-8');
+    console.error(`Report written to ${path}`);
+}
+// =============================================================================
+// Main
+// =============================================================================
+async function main() {
+    console.error(`Loading eval run ${RUN_ID}...`);
+    const run = await loadEvalRun(supabase, RUN_ID);
+    if (!run) {
+        console.error(`Run ${RUN_ID} not found.`);
+        process.exit(1);
+    }
+    const results = run.per_query_results;
+    if (!results || results.length === 0) {
+        console.error(`Run ${RUN_ID} has no per_query_results.`);
+        process.exit(1);
+    }
+    console.error(`Loaded ${results.length} queries from run ${RUN_ID}.`);
+    const classified = classifyAll(results);
+    const report = generateReport(classified, run);
+    writeReport(report);
+}
+main().catch(err => {
+    console.error(err);
+    process.exit(1);
+});

package/dist/scripts/drop-golden-query.js

@@ -0,0 +1,53 @@
+// drop-golden-query.ts
+// One-off: remove the "all system rules and sync rules" enumeration query
+// from eval_golden_dataset. It's a listing query, not a retrieval test.
+//
+// Run: npx tsx src/scripts/drop-golden-query.ts
+import 'dotenv/config';
+import { createClient } from '@supabase/supabase-js';
+const TARGET_QUERY = 'all system rules and sync rules';
+const supabaseUrl = process.env.SUPABASE_URL;
+const supabaseKey = process.env.SUPABASE_SERVICE_ROLE_KEY;
+if (!supabaseUrl || !supabaseKey) {
+    console.error('Missing SUPABASE_URL or SUPABASE_SERVICE_ROLE_KEY');
+    process.exit(1);
+}
+const supabase = createClient(supabaseUrl, supabaseKey);
+async function main() {
+    const { data: matches, error: findError } = await supabase
+        .from('eval_golden_dataset')
+        .select('id, query, expected_doc_ids, tags')
+        .eq('query', TARGET_QUERY);
+    if (findError) {
+        console.error('Lookup failed:', findError.message);
+        process.exit(1);
+    }
+    if (!matches || matches.length === 0) {
+        console.log(`No row found with query: "${TARGET_QUERY}"`);
+        process.exit(0);
+    }
+    if (matches.length > 1) {
+        console.error(`Refusing to delete: found ${matches.length} matching rows. Inspect manually.`);
+        console.error(matches);
+        process.exit(1);
+    }
+    const targetRow = matches[0];
+    console.log('Found row:');
+    console.log(`  id:               ${targetRow.id}`);
+    console.log(`  query:            "${targetRow.query}"`);
+    console.log(`  expected_doc_ids: ${JSON.stringify(targetRow.expected_doc_ids)}`);
+    console.log(`  tags:             ${JSON.stringify(targetRow.tags)}`);
+    const { error: deleteError } = await supabase
+        .from('eval_golden_dataset')
+        .delete()
+        .eq('id', targetRow.id);
+    if (deleteError) {
+        console.error('Delete failed:', deleteError.message);
+        process.exit(1);
+    }
+    console.log(`\nDeleted row id ${targetRow.id}.`);
+}
+main().catch((error) => {
+    console.error(error);
+    process.exit(1);
+});

package/dist/scripts/eval-search.js

@@ -0,0 +1,115 @@
+// eval-search.ts
+// Run the golden dataset through search, compute metrics, print report.
+//
+// Run: npx tsx src/scripts/eval-search.ts
+// This gives us a measurable score for search quality.
+// Every future change gets compared against this baseline.
+import 'dotenv/config';
+import { createClient } from '@supabase/supabase-js';
+import OpenAI from 'openai';
+import { searchHybrid } from '../lib/search/ai-search.js';
+import { scoreTestCase, computeMetrics, formatReport, compareRuns, formatComparison } from '../lib/eval/eval.js';
+import { saveEvalRun, loadPreviousRun, CURRENT_SEARCH_CONFIG } from '../lib/eval/eval-store.js';
+import { computeConfidenceIntervals, computeScoreCalibration, computeCoverageAnalysis, formatAdvancedReport } from '../lib/eval/eval-advanced.js';
+// =============================================================================
+// Setup
+// =============================================================================
+const supabaseUrl = process.env.SUPABASE_URL;
+const supabaseKey = process.env.SUPABASE_SERVICE_ROLE_KEY;
+const openaiKey = process.env.OPENAI_API_KEY;
+if (!supabaseUrl || !supabaseKey || !openaiKey) {
+    console.error('Missing SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY, or OPENAI_API_KEY');
+    process.exit(1);
+}
+const clients = {
+    supabase: createClient(supabaseUrl, supabaseKey),
+    openai: new OpenAI({ apiKey: openaiKey }),
+    cohereApiKey: process.env.COHERE_API_KEY || undefined,
+};
+// Search config imported from eval-store.ts (single source of truth)
+// =============================================================================
+// Run eval
+// =============================================================================
+async function runEval() {
+    console.log('\n' + '='.repeat(60));
+    console.log('Ledger Search Evaluation');
+    console.log('='.repeat(60) + '\n');
+    const previousRun = await loadPreviousRun(clients.supabase);
+    if (previousRun) {
+        console.log(`Previous run: ${previousRun.run_date} (id: ${previousRun.id})\n`);
+    }
+    else {
+        console.log('No previous run found — this will be the first stored run.\n');
+    }
+    const { data: testCases, error } = await clients.supabase
+        .from('eval_golden_dataset')
+        .select('id, query, expected_doc_ids, tags')
+        .order('id');
+    if (error || !testCases) {
+        console.error('Failed to load golden dataset:', error?.message);
+        process.exit(1);
+    }
+    console.log(`Loaded ${testCases.length} test cases.\n`);
+    const results = [];
+    for (const testCase of testCases) {
+        const startTime = Date.now();
+        const searchResults = await searchHybrid(clients, {
+            query: testCase.query,
+            limit: CURRENT_SEARCH_CONFIG.limit,
+            reranker: CURRENT_SEARCH_CONFIG.reranker,
+        });
+        const result = scoreTestCase(testCase, searchResults, Date.now() - startTime);
+        results.push(result);
+        // Live progress
+        const isOutOfScope = testCase.expected_doc_ids.length === 0;
+        if (isOutOfScope) {
+            const status = result.hit ? 'PASS' : `NOISE (${result.returnedIds.length} results)`;
+            console.log(`  [${status}] "${testCase.query}" (out-of-scope)`);
+        }
+        else {
+            const status = result.firstResultHit ? 'TOP' : result.hit ? 'HIT' : 'MISS';
+            const positionInfo = result.position !== null ? `@${result.position + 1}` : '';
+            console.log(`  [${status}${positionInfo}] "${testCase.query}" → found ${result.expectedFound}/${result.expectedTotal}`);
+        }
+    }
+    const metrics = computeMetrics(results);
+    console.log('\n' + formatReport(metrics));
+    // Compute advanced analysis before saving so everything is persisted
+    const confidenceIntervals = computeConfidenceIntervals(results);
+    const scoreCalibration = computeScoreCalibration(results);
+    const coverageAnalysis = computeCoverageAnalysis(results);
+    const runId = await saveEvalRun(clients.supabase, {
+        metrics,
+        config: CURRENT_SEARCH_CONFIG,
+        results,
+        confidenceIntervals,
+        scoreCalibration,
+        coverageAnalysis,
+    });
+    console.log(`\nRun saved to eval_runs (id: ${runId})`);
+    if (previousRun) {
+        const comparison = compareRuns({
+            hitRate: metrics.hitRate,
+            firstResultAccuracy: metrics.firstResultAccuracy,
+            recall: metrics.recall,
+            zeroResultRate: metrics.zeroResultRate,
+            meanReciprocalRank: metrics.meanReciprocalRank,
+            normalizedDiscountedCumulativeGain: metrics.normalizedDiscountedCumulativeGain,
+            avgResponseTimeMs: metrics.avgResponseTimeMs,
+        }, {
+            hitRate: previousRun.hit_rate,
+            firstResultAccuracy: previousRun.first_result_accuracy,
+            recall: previousRun.recall,
+            zeroResultRate: previousRun.zero_result_rate,
+            meanReciprocalRank: previousRun.mean_reciprocal_rank ?? 0,
+            normalizedDiscountedCumulativeGain: previousRun.normalized_discounted_cumulative_gain ?? 0,
+            avgResponseTimeMs: previousRun.avg_response_time_ms,
+        });
+        console.log('\n' + formatComparison(comparison));
+    }
+    console.log('\n' + formatAdvancedReport(confidenceIntervals, scoreCalibration, coverageAnalysis));
+}
+runEval().catch((error) => {
+    console.error('Eval crashed:', error);
+    process.exit(1);
+});

package/dist/scripts/grade-unjudged-top1.js

@@ -0,0 +1,138 @@
+// grade-unjudged-top1.ts
+// Targeted grading for queries where the top-1 document has no judgment.
+// Shows the query, document name, and a content preview. You type 0-3.
+//
+// Run: npx tsx src/scripts/grade-unjudged-top1.ts
+// Dry run: npx tsx src/scripts/grade-unjudged-top1.ts --dry-run
+import 'dotenv/config';
+import { resolve } from 'path';
+import { config } from 'dotenv';
+import { createClient } from '@supabase/supabase-js';
+import { createInterface } from 'node:readline';
+config({ path: resolve(process.env.HOME ?? '', '.ledger', '.env') });
+const supabaseUrl = process.env.SUPABASE_URL;
+const supabaseKey = process.env.SUPABASE_SERVICE_ROLE_KEY;
+if (!supabaseUrl || !supabaseKey) {
+    console.error('Missing SUPABASE_URL or SUPABASE_SERVICE_ROLE_KEY');
+    process.exit(1);
+}
+const supabase = createClient(supabaseUrl, supabaseKey);
+const dryRun = process.argv.includes('--dry-run');
+// The 11 unjudged pairs from run 17 diagnostic (query text + top doc ID)
+const UNJUDGED_PAIRS = [
+    { query: 'how does Ledger process a search query end to end', docId: 152 },
+    { query: 'how does chunking work for embeddings', docId: 160 },
+    { query: 'how does Ledger\'s audit log track document changes', docId: 152 },
+    { query: 'what is Ledger\'s query cache and how does it work', docId: 152 },
+    { query: 'how does Ledger protect sensitive documents with access control', docId: 144 },
+    { query: 'document_create RPC', docId: 152 },
+    { query: 'Ledger text-embedding-3-small embedding model', docId: 152 },
+    { query: 'Ledger keyword search websearch_to_tsquery GIN index', docId: 152 },
+    { query: 'soft delete deleted_at document_purge', docId: 162 },
+    { query: 'ledger architecture all sections', docId: 144 },
+    { query: 'what is the right way to write a custom skill', docId: 163 },
+];
+const RUBRIC = `
+  0 = NOT RELEVANT    Wrong topic, no useful info
+  1 = RELATED         Touches the topic but doesn't answer
+  2 = RELEVANT        Answers the query (user would be happy with this result)
+  3 = HIGHLY RELEVANT The canonical, best-possible answer
+`;
+async function main() {
+    const readline = createInterface({ input: process.stdin, output: process.stdout });
+    const ask = (q) => new Promise(resolve => readline.question(q, resolve));
+    console.log('='.repeat(60));
+    console.log('Targeted grading: 12 unjudged top-1 documents');
+    console.log(dryRun ? '(DRY RUN: no writes)' : '(LIVE: writes to eval_golden_judgments)');
+    console.log('='.repeat(60));
+    console.log(RUBRIC);
+    let graded = 0;
+    let skipped = 0;
+    for (let i = 0; i < UNJUDGED_PAIRS.length; i++) {
+        const pair = UNJUDGED_PAIRS[i];
+        // Look up golden dataset ID for this query
+        const { data: goldenRows } = await supabase
+            .from('eval_golden_dataset')
+            .select('id')
+            .eq('query', pair.query)
+            .limit(1);
+        const goldenId = goldenRows?.[0]?.id;
+        if (!goldenId) {
+            console.log(`\n[${i + 1}/12] SKIP: query not found in golden dataset: "${pair.query}"`);
+            skipped++;
+            continue;
+        }
+        // Fetch document name and content preview
+        const { data: docRow } = await supabase
+            .from('documents')
+            .select('name, content')
+            .eq('id', pair.docId)
+            .single();
+        const docName = docRow?.name ?? '(unknown)';
+        const preview = docRow?.content
+            ? docRow.content.slice(0, 300).replace(/\n/g, ' ')
+            : '(no content)';
+        console.log('\n' + '-'.repeat(60));
+        console.log(`[${i + 1}/12]  Query: "${pair.query}"`);
+        console.log(`  Doc #${pair.docId}: ${docName}`);
+        console.log(`  Preview: ${preview}...`);
+        console.log('-'.repeat(60));
+        let grade = null;
+        while (grade === null) {
+            const input = await ask('Grade (0/1/2/3) or s=skip, q=quit: ');
+            const trimmed = input.trim();
+            if (trimmed === 'q') {
+                console.log(`\nQuit. Graded ${graded}, skipped ${skipped + (UNJUDGED_PAIRS.length - i)}.`);
+                readline.close();
+                return;
+            }
+            if (trimmed === 's') {
+                skipped++;
+                break;
+            }
+            if (['0', '1', '2', '3'].includes(trimmed)) {
+                grade = parseInt(trimmed, 10);
+            }
+            else {
+                console.log('  Invalid. Type 0, 1, 2, 3, s, or q.');
+            }
+        }
+        if (grade === null)
+            continue; // skipped
+        if (dryRun) {
+            console.log(`  [DRY RUN] Would save: golden_id=${goldenId}, doc_id=${pair.docId}, grade=${grade}`);
+        }
+        else {
+            const { error } = await supabase.rpc('judgment_create', {
+                p_golden_id: goldenId,
+                p_document_id: pair.docId,
+                p_grade: grade,
+                p_judged_by: 'adrian',
+                p_notes: 'unjudged top-1 from run-16 diagnostic',
+            });
+            if (error) {
+                // Might already exist (race condition), try update
+                const { error: updateError } = await supabase.rpc('judgment_update', {
+                    p_golden_id: goldenId,
+                    p_document_id: pair.docId,
+                    p_grade: grade,
+                    p_notes: 'unjudged top-1 from run-16 diagnostic',
+                });
+                if (updateError) {
+                    console.log(`  [ERR] Failed to save: ${updateError.message}`);
+                    continue;
+                }
+            }
+            console.log(`  Saved: grade ${grade}`);
+        }
+        graded++;
+    }
+    console.log('\n' + '='.repeat(60));
+    console.log(`Done. Graded: ${graded}, Skipped: ${skipped}`);
+    console.log('='.repeat(60));
+    readline.close();
+}
+main().catch(err => {
+    console.error(err);
+    process.exit(1);
+});

package/dist/scripts/hunter-analytics.js

@@ -0,0 +1,38 @@
+import { loadConfig, loadConfigFile } from '../lib/config.js';
+import { getOpportunitiesForPeriod, insertAnalytics, buildAnalyticsRow } from '../lib/hunter-db.js';
+import { DEFAULT_HUNTER_CONFIG } from '../lib/hunter-types.js';
+export async function runAnalytics(periodDays) {
+    const config = loadConfig();
+    const fileConfig = loadConfigFile();
+    const hunterConfig = {
+        ...DEFAULT_HUNTER_CONFIG,
+        ...fileConfig.hunter,
+    };
+    const days = periodDays ?? hunterConfig.analytics_interval_days;
+    const end = new Date();
+    const start = new Date();
+    start.setDate(start.getDate() - days);
+    const periodStart = start.toISOString().split('T')[0];
+    const periodEnd = end.toISOString().split('T')[0];
+    console.error(`[hunter-analytics] Analyzing ${periodStart} to ${periodEnd}...`);
+    const opportunities = await getOpportunitiesForPeriod(config.supabase, start.toISOString(), end.toISOString());
+    if (opportunities.length === 0) {
+        console.error('[hunter-analytics] No opportunities found in period.');
+        return;
+    }
+    const row = buildAnalyticsRow(opportunities, periodStart, periodEnd);
+    await insertAnalytics(config.supabase, row);
+    console.error(`[hunter-analytics] Saved. ${row.total_found} opportunities analyzed.`);
+    console.error(`  Freelance: ${row.freelance_count} | Employment: ${row.employment_count}`);
+    console.error(`  80+: ${row.score_80_plus} | 60-79: ${row.score_60_79} | 40-59: ${row.score_40_59} | <40: ${row.score_below_40}`);
+    console.error(`  Applied: ${row.applied_count} | Won: ${row.won_count} | Lost: ${row.lost_count}`);
+}
+const isDirectExecution = process.argv[1]?.endsWith('hunter-analytics.js')
+    || process.argv[1]?.endsWith('hunter-analytics.ts');
+if (isDirectExecution) {
+    const days = process.argv[2] ? parseInt(process.argv[2], 10) : undefined;
+    runAnalytics(days).catch(err => {
+        console.error(`[hunter-analytics] Error: ${err.message}`);
+        process.exit(1);
+    });
+}