@claudetools/tools 0.8.2 → 0.8.4

package/dist/evaluation/threshold-eval.js

@@ -0,0 +1,250 @@
+// =============================================================================
+// Threshold Evaluation Framework
+// =============================================================================
+// Empirical evaluation of relevance threshold values
+// Tests 0.2, 0.3, 0.4, 0.5 to find optimal balance of precision vs recall
+// =============================================================================
+import { getMemoryIndex } from '../helpers/api-client.js';
+import { DEFAULT_USER_ID } from '../helpers/config.js';
+// -----------------------------------------------------------------------------
+// Test Dataset
+// -----------------------------------------------------------------------------
+/**
+ * Build test dataset with query-result relevance judgments.
+ *
+ * To use this framework:
+ * 1. Add real queries from your project
+ * 2. Run memory_index with no filter to see all available facts
+ * 3. Manually judge which fact IDs are relevant to each query
+ * 4. Add judgments to this dataset
+ *
+ * Example workflow:
+ * - Query: "authentication patterns"
+ * - Run: memory_index(query="authentication patterns", min_relevance=0)
+ * - Review returned facts, note IDs of truly relevant ones
+ * - Add to dataset with those fact IDs as ground truth
+ */
+export function buildTestDataset() {
+    return [
+        // Example 1: Architecture query
+        {
+            query: 'context injection system',
+            relevantFactIds: [
+            // TODO: Fill with actual fact IDs from your project
+            // These should be manually verified as relevant to this query
+            ],
+            category: 'architecture',
+            description: 'How context injection works in the system',
+        },
+        // Example 2: Pattern query
+        {
+            query: 'task management workflow',
+            relevantFactIds: [
+            // TODO: Fill with actual fact IDs
+            ],
+            category: 'pattern',
+            description: 'Patterns for task creation and execution',
+        },
+        // Example 3: Decision query
+        {
+            query: 'why use BM25 instead of TF-IDF',
+            relevantFactIds: [
+            // TODO: Fill with actual fact IDs
+            ],
+            category: 'decision',
+            description: 'Architectural decision rationale',
+        },
+        // TODO: Add more test cases
+        // Aim for 15-20 diverse queries covering different:
+        // - Query types (architecture, pattern, decision, preference, fact)
+        // - Query lengths (short vs long)
+        // - Query specificity (broad vs narrow)
+        // - Expected result counts (1-2 facts vs 10+ facts)
+    ];
+}
+// -----------------------------------------------------------------------------
+// Evaluation Logic
+// -----------------------------------------------------------------------------
+/**
+ * Evaluate a single threshold value against the test dataset.
+ */
+export async function evaluateThreshold(projectId, threshold, dataset, userId = DEFAULT_USER_ID) {
+    let totalTP = 0;
+    let totalFP = 0;
+    let totalFN = 0;
+    let totalRelevanceScore = 0;
+    let totalReturned = 0;
+    for (const judgment of dataset) {
+        try {
+            // Query with this threshold
+            const result = await getMemoryIndex(projectId, {
+                query: judgment.query,
+                limit: 50,
+                min_relevance: threshold,
+            }, userId);
+            const returnedIds = new Set(result.index.map(entry => entry.id));
+            const relevantIds = new Set(judgment.relevantFactIds);
+            // Calculate metrics for this query
+            const tp = Array.from(returnedIds).filter(id => relevantIds.has(id)).length;
+            const fp = Array.from(returnedIds).filter(id => !relevantIds.has(id)).length;
+            const fn = Array.from(relevantIds).filter(id => !returnedIds.has(id)).length;
+            totalTP += tp;
+            totalFP += fp;
+            totalFN += fn;
+            // Track average relevance of returned results
+            const relevanceScores = result.index.map(e => e.relevance);
+            totalRelevanceScore += relevanceScores.reduce((sum, r) => sum + r, 0);
+            totalReturned += result.index.length;
+        }
+        catch (error) {
+            console.error(`Error evaluating query "${judgment.query}":`, error);
+        }
+    }
+    // Calculate aggregate metrics
+    const precision = totalTP + totalFP > 0 ? totalTP / (totalTP + totalFP) : 0;
+    const recall = totalTP + totalFN > 0 ? totalTP / (totalTP + totalFN) : 0;
+    const f1 = precision + recall > 0 ? (2 * precision * recall) / (precision + recall) : 0;
+    const avgRelevance = totalReturned > 0 ? totalRelevanceScore / totalReturned : 0;
+    return {
+        threshold,
+        precision,
+        recall,
+        f1,
+        truePositives: totalTP,
+        falsePositives: totalFP,
+        falseNegatives: totalFN,
+        avgRelevanceReturned: avgRelevance,
+        totalReturned,
+    };
+}
+/**
+ * Run full evaluation across multiple threshold values.
+ */
+export async function runThresholdEvaluation(projectId, thresholds = [0.2, 0.3, 0.4, 0.5], userId = DEFAULT_USER_ID) {
+    const dataset = buildTestDataset();
+    if (dataset.length === 0 || dataset.every(j => j.relevantFactIds.length === 0)) {
+        throw new Error('Test dataset is empty or has no ground truth judgments. ' +
+            'Please fill buildTestDataset() with real queries and fact IDs.');
+    }
+    console.log(`\n📊 Evaluating ${thresholds.length} thresholds on ${dataset.length} queries...\n`);
+    const results = [];
+    for (const threshold of thresholds) {
+        console.log(`Testing threshold ${threshold}...`);
+        const result = await evaluateThreshold(projectId, threshold, dataset, userId);
+        results.push(result);
+        console.log(`  P=${result.precision.toFixed(3)} R=${result.recall.toFixed(3)} F1=${result.f1.toFixed(3)}`);
+    }
+    // Find best thresholds by different metrics
+    const bestF1 = results.reduce((best, curr) => curr.f1 > best.f1 ? curr : best);
+    const bestPrecision = results.reduce((best, curr) => curr.precision > best.precision ? curr : best);
+    const bestRecall = results.reduce((best, curr) => curr.recall > best.recall ? curr : best);
+    return {
+        projectId,
+        testDate: new Date().toISOString(),
+        results,
+        bestThreshold: {
+            byF1: bestF1.threshold,
+            byPrecision: bestPrecision.threshold,
+            byRecall: bestRecall.threshold,
+        },
+        dataset,
+    };
+}
+// -----------------------------------------------------------------------------
+// Reporting
+// -----------------------------------------------------------------------------
+/**
+ * Format evaluation report as markdown.
+ */
+export function formatReport(report) {
+    let md = `# Relevance Threshold Evaluation Report\n\n`;
+    md += `**Project:** ${report.projectId}\n`;
+    md += `**Date:** ${report.testDate}\n`;
+    md += `**Test Queries:** ${report.dataset.length}\n\n`;
+    md += `## Results Summary\n\n`;
+    md += `| Threshold | Precision | Recall | F1 Score | TP | FP | FN | Avg Relevance | Total Returned |\n`;
+    md += `|-----------|-----------|--------|----------|----|----|----|--------------:|---------------:|\n`;
+    for (const r of report.results) {
+        md += `| ${r.threshold.toFixed(1)} `;
+        md += `| ${(r.precision * 100).toFixed(1)}% `;
+        md += `| ${(r.recall * 100).toFixed(1)}% `;
+        md += `| ${(r.f1 * 100).toFixed(1)}% `;
+        md += `| ${r.truePositives} `;
+        md += `| ${r.falsePositives} `;
+        md += `| ${r.falseNegatives} `;
+        md += `| ${(r.avgRelevanceReturned * 100).toFixed(1)}% `;
+        md += `| ${r.totalReturned} |\n`;
+    }
+    md += `\n## Recommendations\n\n`;
+    md += `- **Best F1 Score:** ${report.bestThreshold.byF1} (balanced precision/recall)\n`;
+    md += `- **Best Precision:** ${report.bestThreshold.byPrecision} (fewer false positives)\n`;
+    md += `- **Best Recall:** ${report.bestThreshold.byRecall} (fewer false negatives)\n\n`;
+    md += `## Analysis\n\n`;
+    const current = report.results.find(r => r.threshold === 0.3);
+    const best = report.results.find(r => r.threshold === report.bestThreshold.byF1);
+    if (current && best && current.threshold !== best.threshold) {
+        const precisionChange = ((best.precision - current.precision) / current.precision) * 100;
+        const recallChange = ((best.recall - current.recall) / current.recall) * 100;
+        const f1Change = ((best.f1 - current.f1) / current.f1) * 100;
+        md += `Current threshold (0.3) vs Best threshold (${best.threshold}):\n`;
+        md += `- Precision: ${precisionChange > 0 ? '+' : ''}${precisionChange.toFixed(1)}%\n`;
+        md += `- Recall: ${recallChange > 0 ? '+' : ''}${recallChange.toFixed(1)}%\n`;
+        md += `- F1 Score: ${f1Change > 0 ? '+' : ''}${f1Change.toFixed(1)}%\n\n`;
+        if (f1Change > 5) {
+            md += `**Recommendation:** Change default threshold from 0.3 to ${best.threshold}\n`;
+        }
+        else {
+            md += `**Recommendation:** Keep current threshold (0.3) - minimal improvement from change\n`;
+        }
+    }
+    else if (current && best && current.threshold === best.threshold) {
+        md += `**Recommendation:** Keep current threshold (0.3) - already optimal\n`;
+    }
+    md += `\n## Test Dataset\n\n`;
+    for (const judgment of report.dataset) {
+        md += `### ${judgment.query}\n`;
+        md += `- **Category:** ${judgment.category}\n`;
+        md += `- **Description:** ${judgment.description}\n`;
+        md += `- **Ground Truth Facts:** ${judgment.relevantFactIds.length}\n\n`;
+    }
+    return md;
+}
+// -----------------------------------------------------------------------------
+// CLI Interface
+// -----------------------------------------------------------------------------
+/**
+ * Run evaluation from command line.
+ *
+ * Usage:
+ *   tsx src/evaluation/threshold-eval.ts <project-id>
+ */
+export async function main() {
+    const projectId = process.argv[2];
+    if (!projectId) {
+        console.error('Usage: tsx src/evaluation/threshold-eval.ts <project-id>');
+        process.exit(1);
+    }
+    try {
+        const report = await runThresholdEvaluation(projectId);
+        const markdown = formatReport(report);
+        console.log('\n' + markdown);
+        // Optionally save to file
+        const fs = await import('fs/promises');
+        const outputPath = `/Users/oweninnes/Projects/memory/docs/evaluation/threshold-eval-${Date.now()}.md`;
+        await fs.mkdir('/Users/oweninnes/Projects/memory/docs/evaluation', { recursive: true });
+        await fs.writeFile(outputPath, markdown);
+        console.log(`\n📄 Report saved to: ${outputPath}`);
+    }
+    catch (error) {
+        console.error('Evaluation failed:', error);
+        process.exit(1);
+    }
+}
+// Run if called directly (ES module check)
+const isMainModule = process.argv[1] && process.argv[1].endsWith('threshold-eval.ts');
+if (isMainModule) {
+    main().catch(err => {
+        console.error(err);
+        process.exit(1);
+    });
+}

package/dist/handlers/codedna-handlers.d.ts

@@ -213,7 +213,7 @@ export declare function handleValidateSpec(args: any): Promise<{
         fields: {
             name: string;
             type: import("../codedna/parser.js").FieldType;
-            constraints: ("default" | "length" | "email" | "min" | "max" | "url" | "required" | "pattern" | "nullable" | "unique" | "hashed" | "index" | "immutable" | "textarea" | "switch" | "radio")[];
+            constraints: ("default" | "length" | "email" | "min" | "max" | "url" | "pattern" | "required" | "nullable" | "unique" | "hashed" | "index" | "immutable" | "textarea" | "switch" | "radio")[];
         }[];
     };
     summary: {
@@ -369,7 +369,7 @@ export declare function handleInitProject(args: {
     patterns: {
         pattern_id: string;
         name: string;
-        category: "hooks" | "components" | "forms" | "state" | "validation" | "styling" | "anti-patterns";
+        category: "hooks" | "state" | "components" | "forms" | "validation" | "styling" | "anti-patterns";
         description: string;
     }[];
     summary: {