@claudetools/tools 0.8.2 → 0.8.3

package/dist/evaluation/build-dataset.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/evaluation/build-dataset.js

@@ -0,0 +1,135 @@
+// =============================================================================
+// Test Dataset Builder
+// =============================================================================
+// Interactive helper to build test dataset with ground truth judgments
+// =============================================================================
+import { getMemoryIndex } from '../helpers/api-client.js';
+import { DEFAULT_USER_ID } from '../helpers/config.js';
+import * as readline from 'readline/promises';
+/**
+ * Interactive CLI to build test dataset.
+ *
+ * For each query:
+ * 1. Shows all facts with relevance scores
+ * 2. User marks which are actually relevant (ground truth)
+ * 3. Saves to JSON file
+ */
+async function buildDatasetInteractive(projectId, userId = DEFAULT_USER_ID) {
+    const rl = readline.createInterface({
+        input: process.stdin,
+        output: process.stdout,
+    });
+    const dataset = [];
+    console.log('📝 Interactive Test Dataset Builder\n');
+    console.log('This tool helps you create ground truth judgments for threshold evaluation.\n');
+    console.log('For each query, you\'ll see all matching facts and mark which are truly relevant.\n');
+    let addMore = true;
+    while (addMore) {
+        // Get query
+        const query = await rl.question('\n🔍 Enter test query (or "done" to finish): ');
+        if (query.toLowerCase() === 'done') {
+            break;
+        }
+        if (!query.trim()) {
+            console.log('⚠️  Query cannot be empty');
+            continue;
+        }
+        // Fetch all facts (no relevance filter)
+        console.log('\n⏳ Fetching facts...\n');
+        const result = await getMemoryIndex(projectId, {
+            query,
+            limit: 50,
+            min_relevance: 0, // Get everything
+        }, userId);
+        if (result.index.length === 0) {
+            console.log('❌ No facts found for this query. Try a different query.\n');
+            continue;
+        }
+        // Display facts
+        console.log(`\n📊 Found ${result.index.length} facts:\n`);
+        console.log('─'.repeat(80));
+        result.index.forEach((entry, i) => {
+            const relevanceBar = '█'.repeat(Math.round(entry.relevance * 20));
+            console.log(`\n[${i + 1}] ID: ${entry.id.slice(0, 12)}...`);
+            console.log(`    Relevance: ${(entry.relevance * 100).toFixed(0)}% ${relevanceBar}`);
+            console.log(`    Category: ${entry.category}`);
+            console.log(`    Summary: ${entry.summary}`);
+        });
+        console.log('\n' + '─'.repeat(80));
+        // Get ground truth judgments
+        const relevantIndices = await rl.question('\n✅ Which facts are TRULY RELEVANT? (comma-separated numbers, e.g., "1,3,5"): ');
+        if (!relevantIndices.trim()) {
+            console.log('⚠️  No facts marked as relevant. Skipping this query.');
+            continue;
+        }
+        // Parse indices
+        const indices = relevantIndices
+            .split(',')
+            .map(s => parseInt(s.trim()) - 1)
+            .filter(i => i >= 0 && i < result.index.length);
+        if (indices.length === 0) {
+            console.log('⚠️  Invalid indices. Skipping this query.');
+            continue;
+        }
+        const relevantFactIds = indices.map(i => result.index[i].id);
+        console.log(`\n✓ Marked ${relevantFactIds.length} facts as relevant`);
+        // Get metadata
+        const category = await rl.question('📂 Category (architecture/pattern/decision/preference/fact): ');
+        const description = await rl.question('📝 Description (what is this query testing?): ');
+        // Add to dataset
+        dataset.push({
+            query,
+            relevantFactIds,
+            category: category.trim() || 'fact',
+            description: description.trim() || query,
+        });
+        console.log(`\n✅ Added test case ${dataset.length}`);
+    }
+    rl.close();
+    // Save dataset
+    if (dataset.length > 0) {
+        const fs = await import('fs/promises');
+        const outputPath = `/Users/oweninnes/Projects/memory/docs/evaluation/dataset-${Date.now()}.json`;
+        await fs.mkdir('/Users/oweninnes/Projects/memory/docs/evaluation', { recursive: true });
+        await fs.writeFile(outputPath, JSON.stringify(dataset, null, 2));
+        console.log('\n' + '='.repeat(80));
+        console.log(`\n✅ Dataset saved to: ${outputPath}`);
+        console.log(`\n📊 Total test cases: ${dataset.length}`);
+        console.log(`📋 Total ground truth facts: ${dataset.reduce((sum, d) => sum + d.relevantFactIds.length, 0)}`);
+        // Show code to use in threshold-eval.ts
+        console.log('\n📝 Copy this into threshold-eval.ts buildTestDataset():\n');
+        console.log('```typescript');
+        console.log('export function buildTestDataset(): QueryJudgment[] {');
+        console.log('  return ' + JSON.stringify(dataset, null, 2) + ';');
+        console.log('}');
+        console.log('```\n');
+    }
+    else {
+        console.log('\n⚠️  No test cases created.');
+    }
+}
+/**
+ * CLI entry point.
+ */
+async function main() {
+    const projectId = process.argv[2];
+    if (!projectId) {
+        console.error('Usage: tsx src/evaluation/build-dataset.ts <project-id>');
+        process.exit(1);
+    }
+    try {
+        await buildDatasetInteractive(projectId);
+    }
+    catch (error) {
+        console.error('\n❌ Error:', error);
+        process.exit(1);
+    }
+}
+// Run if called directly (ES module check)
+const isMainModule = process.argv[1] && process.argv[1].endsWith('build-dataset.ts');
+if (isMainModule) {
+    main().catch(err => {
+        console.error(err);
+        process.exit(1);
+    });
+}

package/dist/evaluation/threshold-eval.d.ts

@@ -0,0 +1,63 @@
+export interface QueryJudgment {
+    query: string;
+    relevantFactIds: string[];
+    category: 'architecture' | 'pattern' | 'decision' | 'preference' | 'fact';
+    description: string;
+}
+export interface ThresholdResult {
+    threshold: number;
+    precision: number;
+    recall: number;
+    f1: number;
+    truePositives: number;
+    falsePositives: number;
+    falseNegatives: number;
+    avgRelevanceReturned: number;
+    totalReturned: number;
+}
+export interface EvaluationReport {
+    projectId: string;
+    testDate: string;
+    results: ThresholdResult[];
+    bestThreshold: {
+        byF1: number;
+        byPrecision: number;
+        byRecall: number;
+    };
+    dataset: QueryJudgment[];
+}
+/**
+ * Build test dataset with query-result relevance judgments.
+ *
+ * To use this framework:
+ * 1. Add real queries from your project
+ * 2. Run memory_index with no filter to see all available facts
+ * 3. Manually judge which fact IDs are relevant to each query
+ * 4. Add judgments to this dataset
+ *
+ * Example workflow:
+ * - Query: "authentication patterns"
+ * - Run: memory_index(query="authentication patterns", min_relevance=0)
+ * - Review returned facts, note IDs of truly relevant ones
+ * - Add to dataset with those fact IDs as ground truth
+ */
+export declare function buildTestDataset(): QueryJudgment[];
+/**
+ * Evaluate a single threshold value against the test dataset.
+ */
+export declare function evaluateThreshold(projectId: string, threshold: number, dataset: QueryJudgment[], userId?: string): Promise<ThresholdResult>;
+/**
+ * Run full evaluation across multiple threshold values.
+ */
+export declare function runThresholdEvaluation(projectId: string, thresholds?: number[], userId?: string): Promise<EvaluationReport>;
+/**
+ * Format evaluation report as markdown.
+ */
+export declare function formatReport(report: EvaluationReport): string;
+/**
+ * Run evaluation from command line.
+ *
+ * Usage:
+ *   tsx src/evaluation/threshold-eval.ts <project-id>
+ */
+export declare function main(): Promise<void>;

package/dist/evaluation/threshold-eval.js

@@ -0,0 +1,250 @@
+// =============================================================================
+// Threshold Evaluation Framework
+// =============================================================================
+// Empirical evaluation of relevance threshold values
+// Tests 0.2, 0.3, 0.4, 0.5 to find optimal balance of precision vs recall
+// =============================================================================
+import { getMemoryIndex } from '../helpers/api-client.js';
+import { DEFAULT_USER_ID } from '../helpers/config.js';
+// -----------------------------------------------------------------------------
+// Test Dataset
+// -----------------------------------------------------------------------------
+/**
+ * Build test dataset with query-result relevance judgments.
+ *
+ * To use this framework:
+ * 1. Add real queries from your project
+ * 2. Run memory_index with no filter to see all available facts
+ * 3. Manually judge which fact IDs are relevant to each query
+ * 4. Add judgments to this dataset
+ *
+ * Example workflow:
+ * - Query: "authentication patterns"
+ * - Run: memory_index(query="authentication patterns", min_relevance=0)
+ * - Review returned facts, note IDs of truly relevant ones
+ * - Add to dataset with those fact IDs as ground truth
+ */
+export function buildTestDataset() {
+    return [
+        // Example 1: Architecture query
+        {
+            query: 'context injection system',
+            relevantFactIds: [
+            // TODO: Fill with actual fact IDs from your project
+            // These should be manually verified as relevant to this query
+            ],
+            category: 'architecture',
+            description: 'How context injection works in the system',
+        },
+        // Example 2: Pattern query
+        {
+            query: 'task management workflow',
+            relevantFactIds: [
+            // TODO: Fill with actual fact IDs
+            ],
+            category: 'pattern',
+            description: 'Patterns for task creation and execution',
+        },
+        // Example 3: Decision query
+        {
+            query: 'why use BM25 instead of TF-IDF',
+            relevantFactIds: [
+            // TODO: Fill with actual fact IDs
+            ],
+            category: 'decision',
+            description: 'Architectural decision rationale',
+        },
+        // TODO: Add more test cases
+        // Aim for 15-20 diverse queries covering different:
+        // - Query types (architecture, pattern, decision, preference, fact)
+        // - Query lengths (short vs long)
+        // - Query specificity (broad vs narrow)
+        // - Expected result counts (1-2 facts vs 10+ facts)
+    ];
+}
+// -----------------------------------------------------------------------------
+// Evaluation Logic
+// -----------------------------------------------------------------------------
+/**
+ * Evaluate a single threshold value against the test dataset.
+ */
+export async function evaluateThreshold(projectId, threshold, dataset, userId = DEFAULT_USER_ID) {
+    let totalTP = 0;
+    let totalFP = 0;
+    let totalFN = 0;
+    let totalRelevanceScore = 0;
+    let totalReturned = 0;
+    for (const judgment of dataset) {
+        try {
+            // Query with this threshold
+            const result = await getMemoryIndex(projectId, {
+                query: judgment.query,
+                limit: 50,
+                min_relevance: threshold,
+            }, userId);
+            const returnedIds = new Set(result.index.map(entry => entry.id));
+            const relevantIds = new Set(judgment.relevantFactIds);
+            // Calculate metrics for this query
+            const tp = Array.from(returnedIds).filter(id => relevantIds.has(id)).length;
+            const fp = Array.from(returnedIds).filter(id => !relevantIds.has(id)).length;
+            const fn = Array.from(relevantIds).filter(id => !returnedIds.has(id)).length;
+            totalTP += tp;
+            totalFP += fp;
+            totalFN += fn;
+            // Track average relevance of returned results
+            const relevanceScores = result.index.map(e => e.relevance);
+            totalRelevanceScore += relevanceScores.reduce((sum, r) => sum + r, 0);
+            totalReturned += result.index.length;
+        }
+        catch (error) {
+            console.error(`Error evaluating query "${judgment.query}":`, error);
+        }
+    }
+    // Calculate aggregate metrics
+    const precision = totalTP + totalFP > 0 ? totalTP / (totalTP + totalFP) : 0;
+    const recall = totalTP + totalFN > 0 ? totalTP / (totalTP + totalFN) : 0;
+    const f1 = precision + recall > 0 ? (2 * precision * recall) / (precision + recall) : 0;
+    const avgRelevance = totalReturned > 0 ? totalRelevanceScore / totalReturned : 0;
+    return {
+        threshold,
+        precision,
+        recall,
+        f1,
+        truePositives: totalTP,
+        falsePositives: totalFP,
+        falseNegatives: totalFN,
+        avgRelevanceReturned: avgRelevance,
+        totalReturned,
+    };
+}
+/**
+ * Run full evaluation across multiple threshold values.
+ */
+export async function runThresholdEvaluation(projectId, thresholds = [0.2, 0.3, 0.4, 0.5], userId = DEFAULT_USER_ID) {
+    const dataset = buildTestDataset();
+    if (dataset.length === 0 || dataset.every(j => j.relevantFactIds.length === 0)) {
+        throw new Error('Test dataset is empty or has no ground truth judgments. ' +
+            'Please fill buildTestDataset() with real queries and fact IDs.');
+    }
+    console.log(`\n📊 Evaluating ${thresholds.length} thresholds on ${dataset.length} queries...\n`);
+    const results = [];
+    for (const threshold of thresholds) {
+        console.log(`Testing threshold ${threshold}...`);
+        const result = await evaluateThreshold(projectId, threshold, dataset, userId);
+        results.push(result);
+        console.log(`  P=${result.precision.toFixed(3)} R=${result.recall.toFixed(3)} F1=${result.f1.toFixed(3)}`);
+    }
+    // Find best thresholds by different metrics
+    const bestF1 = results.reduce((best, curr) => curr.f1 > best.f1 ? curr : best);
+    const bestPrecision = results.reduce((best, curr) => curr.precision > best.precision ? curr : best);
+    const bestRecall = results.reduce((best, curr) => curr.recall > best.recall ? curr : best);
+    return {
+        projectId,
+        testDate: new Date().toISOString(),
+        results,
+        bestThreshold: {
+            byF1: bestF1.threshold,
+            byPrecision: bestPrecision.threshold,
+            byRecall: bestRecall.threshold,
+        },
+        dataset,
+    };
+}
+// -----------------------------------------------------------------------------
+// Reporting
+// -----------------------------------------------------------------------------
+/**
+ * Format evaluation report as markdown.
+ */
+export function formatReport(report) {
+    let md = `# Relevance Threshold Evaluation Report\n\n`;
+    md += `**Project:** ${report.projectId}\n`;
+    md += `**Date:** ${report.testDate}\n`;
+    md += `**Test Queries:** ${report.dataset.length}\n\n`;
+    md += `## Results Summary\n\n`;
+    md += `| Threshold | Precision | Recall | F1 Score | TP | FP | FN | Avg Relevance | Total Returned |\n`;
+    md += `|-----------|-----------|--------|----------|----|----|----|--------------:|---------------:|\n`;
+    for (const r of report.results) {
+        md += `| ${r.threshold.toFixed(1)} `;
+        md += `| ${(r.precision * 100).toFixed(1)}% `;
+        md += `| ${(r.recall * 100).toFixed(1)}% `;
+        md += `| ${(r.f1 * 100).toFixed(1)}% `;
+        md += `| ${r.truePositives} `;
+        md += `| ${r.falsePositives} `;
+        md += `| ${r.falseNegatives} `;
+        md += `| ${(r.avgRelevanceReturned * 100).toFixed(1)}% `;
+        md += `| ${r.totalReturned} |\n`;
+    }
+    md += `\n## Recommendations\n\n`;
+    md += `- **Best F1 Score:** ${report.bestThreshold.byF1} (balanced precision/recall)\n`;
+    md += `- **Best Precision:** ${report.bestThreshold.byPrecision} (fewer false positives)\n`;
+    md += `- **Best Recall:** ${report.bestThreshold.byRecall} (fewer false negatives)\n\n`;
+    md += `## Analysis\n\n`;
+    const current = report.results.find(r => r.threshold === 0.3);
+    const best = report.results.find(r => r.threshold === report.bestThreshold.byF1);
+    if (current && best && current.threshold !== best.threshold) {
+        const precisionChange = ((best.precision - current.precision) / current.precision) * 100;
+        const recallChange = ((best.recall - current.recall) / current.recall) * 100;
+        const f1Change = ((best.f1 - current.f1) / current.f1) * 100;
+        md += `Current threshold (0.3) vs Best threshold (${best.threshold}):\n`;
+        md += `- Precision: ${precisionChange > 0 ? '+' : ''}${precisionChange.toFixed(1)}%\n`;
+        md += `- Recall: ${recallChange > 0 ? '+' : ''}${recallChange.toFixed(1)}%\n`;
+        md += `- F1 Score: ${f1Change > 0 ? '+' : ''}${f1Change.toFixed(1)}%\n\n`;
+        if (f1Change > 5) {
+            md += `**Recommendation:** Change default threshold from 0.3 to ${best.threshold}\n`;
+        }
+        else {
+            md += `**Recommendation:** Keep current threshold (0.3) - minimal improvement from change\n`;
+        }
+    }
+    else if (current && best && current.threshold === best.threshold) {
+        md += `**Recommendation:** Keep current threshold (0.3) - already optimal\n`;
+    }
+    md += `\n## Test Dataset\n\n`;
+    for (const judgment of report.dataset) {
+        md += `### ${judgment.query}\n`;
+        md += `- **Category:** ${judgment.category}\n`;
+        md += `- **Description:** ${judgment.description}\n`;
+        md += `- **Ground Truth Facts:** ${judgment.relevantFactIds.length}\n\n`;
+    }
+    return md;
+}
+// -----------------------------------------------------------------------------
+// CLI Interface
+// -----------------------------------------------------------------------------
+/**
+ * Run evaluation from command line.
+ *
+ * Usage:
+ *   tsx src/evaluation/threshold-eval.ts <project-id>
+ */
+export async function main() {
+    const projectId = process.argv[2];
+    if (!projectId) {
+        console.error('Usage: tsx src/evaluation/threshold-eval.ts <project-id>');
+        process.exit(1);
+    }
+    try {
+        const report = await runThresholdEvaluation(projectId);
+        const markdown = formatReport(report);
+        console.log('\n' + markdown);
+        // Optionally save to file
+        const fs = await import('fs/promises');
+        const outputPath = `/Users/oweninnes/Projects/memory/docs/evaluation/threshold-eval-${Date.now()}.md`;
+        await fs.mkdir('/Users/oweninnes/Projects/memory/docs/evaluation', { recursive: true });
+        await fs.writeFile(outputPath, markdown);
+        console.log(`\n📄 Report saved to: ${outputPath}`);
+    }
+    catch (error) {
+        console.error('Evaluation failed:', error);
+        process.exit(1);
+    }
+}
+// Run if called directly (ES module check)
+const isMainModule = process.argv[1] && process.argv[1].endsWith('threshold-eval.ts');
+if (isMainModule) {
+    main().catch(err => {
+        console.error(err);
+        process.exit(1);
+    });
+}

package/dist/handlers/codedna-handlers.d.ts

@@ -213,7 +213,7 @@ export declare function handleValidateSpec(args: any): Promise<{
         fields: {
             name: string;
             type: import("../codedna/parser.js").FieldType;
-            constraints: ("default" | "length" | "email" | "min" | "max" | "url" | "required" | "pattern" | "nullable" | "unique" | "hashed" | "index" | "immutable" | "textarea" | "switch" | "radio")[];
+            constraints: ("default" | "length" | "email" | "min" | "max" | "url" | "pattern" | "required" | "nullable" | "unique" | "hashed" | "index" | "immutable" | "textarea" | "switch" | "radio")[];
         }[];
     };
     summary: {