@claudetools/tools 0.8.2 → 0.8.4

package/dist/context-cli.js

@@ -0,0 +1,309 @@
+// =============================================================================
+// Context Management CLI Commands
+// =============================================================================
+// CLI interface for context window management: status, evict, summarise, reset
+import chalk from 'chalk';
+import ora from 'ora';
+import prompts from 'prompts';
+import { getSessionStore } from './context/session-store.js';
+import { createEvictionEngine } from './context/eviction-engine.js';
+import { createExchangeSummariser } from './context/exchange-summariser.js';
+// -----------------------------------------------------------------------------
+// Utility Functions
+// -----------------------------------------------------------------------------
+function success(msg) {
+    console.log(chalk.green('✓ ') + msg);
+}
+function error(msg) {
+    console.log(chalk.red('✗ ') + msg);
+}
+function info(msg) {
+    console.log(chalk.blue('ℹ ') + msg);
+}
+function warn(msg) {
+    console.log(chalk.yellow('⚠ ') + msg);
+}
+function header(title) {
+    console.log('\n' + chalk.cyan('━'.repeat(50)));
+    console.log(chalk.cyan.bold(title));
+    console.log(chalk.cyan('━'.repeat(50)) + '\n');
+}
+/**
+ * Get session ID from args or prompt user to select
+ */
+async function getSessionId(args) {
+    // Check if session ID provided as argument
+    const sessionIdArg = args.find((arg) => arg.startsWith('--session='));
+    if (sessionIdArg) {
+        return sessionIdArg.split('=')[1];
+    }
+    // List available sessions
+    const store = getSessionStore();
+    const sessions = await store.listSessions();
+    if (sessions.length === 0) {
+        error('No active sessions found');
+        return null;
+    }
+    // If only one session, use it
+    if (sessions.length === 1) {
+        return sessions[0].session_id;
+    }
+    // Prompt user to select
+    const choices = sessions.map((s) => ({
+        title: `${s.session_id} (${s.model}, started ${s.started_at.toLocaleString()})`,
+        value: s.session_id,
+    }));
+    const response = await prompts({
+        type: 'select',
+        name: 'sessionId',
+        message: 'Select a session:',
+        choices,
+    });
+    return response.sessionId || null;
+}
+/**
+ * Format fill percentage with colour coding
+ */
+function formatFill(fill) {
+    const percentage = (fill * 100).toFixed(1);
+    if (fill < 0.5) {
+        return chalk.green(`${percentage}%`);
+    }
+    else if (fill < 0.7) {
+        return chalk.yellow(`${percentage}%`);
+    }
+    else {
+        return chalk.red(`${percentage}%`);
+    }
+}
+/**
+ * Format session summary
+ */
+function formatSessionSummary(session) {
+    console.log(chalk.bold('Session ID:'), session.session_id);
+    console.log(chalk.bold('Model:'), session.model);
+    console.log(chalk.bold('Started:'), session.started_at.toLocaleString());
+    console.log(chalk.bold('Context Limit:'), session.context_limit.toLocaleString(), 'tokens');
+    console.log(chalk.bold('Estimated Fill:'), formatFill(session.estimated_fill));
+    console.log(chalk.bold('Used Tokens:'), Math.round(session.estimated_fill * session.context_limit).toLocaleString());
+    console.log(chalk.bold('Injected Facts:'), session.injected_facts.length);
+    console.log(chalk.bold('Exchanges:'), session.exchanges.length);
+    // Show exchange summary
+    const summarisedCount = session.exchanges.filter((ex) => ex.summarised_at).length;
+    if (summarisedCount > 0) {
+        console.log(chalk.bold('Summarised Exchanges:'), summarisedCount);
+    }
+    console.log(chalk.bold('Last Updated:'), session.last_updated.toLocaleString());
+}
+// -----------------------------------------------------------------------------
+// Commands
+// -----------------------------------------------------------------------------
+/**
+ * claudetools context status - Show current session context usage
+ */
+export async function contextStatus(args) {
+    header('Context Status');
+    const sessionId = await getSessionId(args);
+    if (!sessionId) {
+        process.exit(1);
+    }
+    const spinner = ora('Loading session state...').start();
+    try {
+        const store = getSessionStore();
+        const session = await store.getSession(sessionId);
+        if (!session) {
+            spinner.fail('Session not found');
+            process.exit(1);
+        }
+        spinner.succeed('Session loaded');
+        console.log();
+        formatSessionSummary(session);
+        // Show warnings
+        console.log();
+        if (session.estimated_fill > 0.85) {
+            warn('Critical: Context window is near full (>85%). Emergency eviction may be triggered.');
+        }
+        else if (session.estimated_fill > 0.6) {
+            warn('Warning: Context window is filling up (>60%). Consider running eviction.');
+        }
+        else {
+            success('Context window is healthy.');
+        }
+        // Show eviction eligibility
+        const engine = createEvictionEngine();
+        if (engine.shouldEvict(session)) {
+            console.log();
+            info('Automatic eviction will be triggered at next opportunity.');
+        }
+    }
+    catch (err) {
+        spinner.fail('Failed to load session');
+        throw err;
+    }
+}
+/**
+ * claudetools context evict - Manually trigger eviction cycle
+ */
+export async function contextEvict(args) {
+    header('Context Eviction');
+    const sessionId = await getSessionId(args);
+    if (!sessionId) {
+        process.exit(1);
+    }
+    const spinner = ora('Loading session state...').start();
+    try {
+        const store = getSessionStore();
+        const session = await store.getSession(sessionId);
+        if (!session) {
+            spinner.fail('Session not found');
+            process.exit(1);
+        }
+        spinner.text = 'Analysing session...';
+        // Check if eviction is needed
+        const engine = createEvictionEngine();
+        if (!engine.shouldEvict(session)) {
+            spinner.info(`No eviction needed (fill: ${formatFill(session.estimated_fill)})`);
+            return;
+        }
+        // Get eviction plan
+        const plan = engine.getEvictionPlan(session);
+        spinner.stop();
+        console.log();
+        console.log(chalk.bold('Eviction Plan:'));
+        console.log(chalk.bold('Facts to evict:'), plan.factsToEvict.length);
+        console.log(chalk.bold('Current fill:'), formatFill(session.estimated_fill));
+        console.log(chalk.bold('Expected fill after:'), formatFill(plan.expectedFillAfter));
+        if (plan.includesCritical) {
+            warn('Plan includes evicting CRITICAL facts (fill >85%)');
+        }
+        // Confirm before proceeding
+        const confirm = await prompts({
+            type: 'confirm',
+            name: 'proceed',
+            message: 'Proceed with eviction?',
+            initial: true,
+        });
+        if (!confirm.proceed) {
+            info('Eviction cancelled');
+            return;
+        }
+        spinner.start('Evicting facts...');
+        // Execute eviction
+        const result = await engine.runEviction(session);
+        spinner.succeed('Eviction complete');
+        console.log();
+        console.log(chalk.bold('Evicted:'), result.evictedCount, 'facts');
+        console.log(chalk.bold('New fill:'), formatFill(result.newEstimatedFill));
+        // Update session in store
+        await store.updateSession(sessionId, { estimated_fill: result.newEstimatedFill });
+        success(`Context window reduced from ${formatFill(session.estimated_fill)} to ${formatFill(result.newEstimatedFill)}`);
+    }
+    catch (err) {
+        spinner.fail('Eviction failed');
+        throw err;
+    }
+}
+/**
+ * claudetools context summarise - Summarise and compress exchanges
+ */
+export async function contextSummarise(args) {
+    header('Exchange Summarisation');
+    const sessionId = await getSessionId(args);
+    if (!sessionId) {
+        process.exit(1);
+    }
+    const spinner = ora('Loading session state...').start();
+    try {
+        const store = getSessionStore();
+        const session = await store.getSession(sessionId);
+        if (!session) {
+            spinner.fail('Session not found');
+            process.exit(1);
+        }
+        spinner.text = 'Analysing exchanges...';
+        // Check if summarisation is needed
+        const summariser = createExchangeSummariser();
+        if (!summariser.shouldSummarise(session)) {
+            spinner.info('No exchanges need summarisation (< 10 unsummarised)');
+            return;
+        }
+        spinner.stop();
+        console.log();
+        const unsummarised = session.exchanges.filter((ex) => !ex.summarised_at);
+        console.log(chalk.bold('Unsummarised exchanges:'), unsummarised.length);
+        console.log(chalk.bold('Total exchanges:'), session.exchanges.length);
+        // Confirm before proceeding
+        const confirm = await prompts({
+            type: 'confirm',
+            name: 'proceed',
+            message: 'Proceed with summarisation?',
+            initial: true,
+        });
+        if (!confirm.proceed) {
+            info('Summarisation cancelled');
+            return;
+        }
+        spinner.start('Summarising exchanges...');
+        // Execute summarisation
+        const result = await summariser.summariseOldExchanges(session);
+        spinner.succeed('Summarisation complete');
+        console.log();
+        console.log(chalk.bold('Summarised:'), result.summarisedCount, 'exchanges');
+        console.log(chalk.bold('Tokens saved:'), result.tokensSaved.toLocaleString());
+        // Mark exchanges as summarised in store
+        for (let i = 0; i < result.summarisedCount; i++) {
+            await store.markExchangeSummarised(sessionId, i);
+        }
+        success(`Compressed ${result.summarisedCount} exchanges, saved ~${result.tokensSaved} tokens`);
+    }
+    catch (err) {
+        spinner.fail('Summarisation failed');
+        throw err;
+    }
+}
+/**
+ * claudetools context reset - Clear session state
+ */
+export async function contextReset(args) {
+    header('Context Reset');
+    const sessionId = await getSessionId(args);
+    if (!sessionId) {
+        process.exit(1);
+    }
+    const spinner = ora('Loading session state...').start();
+    try {
+        const store = getSessionStore();
+        const session = await store.getSession(sessionId);
+        if (!session) {
+            spinner.fail('Session not found');
+            process.exit(1);
+        }
+        spinner.stop();
+        console.log();
+        formatSessionSummary(session);
+        console.log();
+        warn('This will permanently delete all session data including:');
+        console.log('  - Injected facts');
+        console.log('  - Exchange history');
+        console.log('  - Token usage estimates');
+        // Confirm before proceeding
+        const confirm = await prompts({
+            type: 'confirm',
+            name: 'proceed',
+            message: chalk.red('Are you sure you want to delete this session?'),
+            initial: false,
+        });
+        if (!confirm.proceed) {
+            info('Reset cancelled');
+            return;
+        }
+        spinner.start('Deleting session...');
+        await store.deleteSession(sessionId);
+        spinner.succeed('Session deleted');
+        success('Context state cleared');
+    }
+    catch (err) {
+        spinner.fail('Reset failed');
+        throw err;
+    }
+}

package/dist/evaluation/build-dataset.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/evaluation/build-dataset.js

@@ -0,0 +1,135 @@
+// =============================================================================
+// Test Dataset Builder
+// =============================================================================
+// Interactive helper to build test dataset with ground truth judgments
+// =============================================================================
+import { getMemoryIndex } from '../helpers/api-client.js';
+import { DEFAULT_USER_ID } from '../helpers/config.js';
+import * as readline from 'readline/promises';
+/**
+ * Interactive CLI to build test dataset.
+ *
+ * For each query:
+ * 1. Shows all facts with relevance scores
+ * 2. User marks which are actually relevant (ground truth)
+ * 3. Saves to JSON file
+ */
+async function buildDatasetInteractive(projectId, userId = DEFAULT_USER_ID) {
+    const rl = readline.createInterface({
+        input: process.stdin,
+        output: process.stdout,
+    });
+    const dataset = [];
+    console.log('📝 Interactive Test Dataset Builder\n');
+    console.log('This tool helps you create ground truth judgments for threshold evaluation.\n');
+    console.log('For each query, you\'ll see all matching facts and mark which are truly relevant.\n');
+    let addMore = true;
+    while (addMore) {
+        // Get query
+        const query = await rl.question('\n🔍 Enter test query (or "done" to finish): ');
+        if (query.toLowerCase() === 'done') {
+            break;
+        }
+        if (!query.trim()) {
+            console.log('⚠️  Query cannot be empty');
+            continue;
+        }
+        // Fetch all facts (no relevance filter)
+        console.log('\n⏳ Fetching facts...\n');
+        const result = await getMemoryIndex(projectId, {
+            query,
+            limit: 50,
+            min_relevance: 0, // Get everything
+        }, userId);
+        if (result.index.length === 0) {
+            console.log('❌ No facts found for this query. Try a different query.\n');
+            continue;
+        }
+        // Display facts
+        console.log(`\n📊 Found ${result.index.length} facts:\n`);
+        console.log('─'.repeat(80));
+        result.index.forEach((entry, i) => {
+            const relevanceBar = '█'.repeat(Math.round(entry.relevance * 20));
+            console.log(`\n[${i + 1}] ID: ${entry.id.slice(0, 12)}...`);
+            console.log(`    Relevance: ${(entry.relevance * 100).toFixed(0)}% ${relevanceBar}`);
+            console.log(`    Category: ${entry.category}`);
+            console.log(`    Summary: ${entry.summary}`);
+        });
+        console.log('\n' + '─'.repeat(80));
+        // Get ground truth judgments
+        const relevantIndices = await rl.question('\n✅ Which facts are TRULY RELEVANT? (comma-separated numbers, e.g., "1,3,5"): ');
+        if (!relevantIndices.trim()) {
+            console.log('⚠️  No facts marked as relevant. Skipping this query.');
+            continue;
+        }
+        // Parse indices
+        const indices = relevantIndices
+            .split(',')
+            .map(s => parseInt(s.trim()) - 1)
+            .filter(i => i >= 0 && i < result.index.length);
+        if (indices.length === 0) {
+            console.log('⚠️  Invalid indices. Skipping this query.');
+            continue;
+        }
+        const relevantFactIds = indices.map(i => result.index[i].id);
+        console.log(`\n✓ Marked ${relevantFactIds.length} facts as relevant`);
+        // Get metadata
+        const category = await rl.question('📂 Category (architecture/pattern/decision/preference/fact): ');
+        const description = await rl.question('📝 Description (what is this query testing?): ');
+        // Add to dataset
+        dataset.push({
+            query,
+            relevantFactIds,
+            category: category.trim() || 'fact',
+            description: description.trim() || query,
+        });
+        console.log(`\n✅ Added test case ${dataset.length}`);
+    }
+    rl.close();
+    // Save dataset
+    if (dataset.length > 0) {
+        const fs = await import('fs/promises');
+        const outputPath = `/Users/oweninnes/Projects/memory/docs/evaluation/dataset-${Date.now()}.json`;
+        await fs.mkdir('/Users/oweninnes/Projects/memory/docs/evaluation', { recursive: true });
+        await fs.writeFile(outputPath, JSON.stringify(dataset, null, 2));
+        console.log('\n' + '='.repeat(80));
+        console.log(`\n✅ Dataset saved to: ${outputPath}`);
+        console.log(`\n📊 Total test cases: ${dataset.length}`);
+        console.log(`📋 Total ground truth facts: ${dataset.reduce((sum, d) => sum + d.relevantFactIds.length, 0)}`);
+        // Show code to use in threshold-eval.ts
+        console.log('\n📝 Copy this into threshold-eval.ts buildTestDataset():\n');
+        console.log('```typescript');
+        console.log('export function buildTestDataset(): QueryJudgment[] {');
+        console.log('  return ' + JSON.stringify(dataset, null, 2) + ';');
+        console.log('}');
+        console.log('```\n');
+    }
+    else {
+        console.log('\n⚠️  No test cases created.');
+    }
+}
+/**
+ * CLI entry point.
+ */
+async function main() {
+    const projectId = process.argv[2];
+    if (!projectId) {
+        console.error('Usage: tsx src/evaluation/build-dataset.ts <project-id>');
+        process.exit(1);
+    }
+    try {
+        await buildDatasetInteractive(projectId);
+    }
+    catch (error) {
+        console.error('\n❌ Error:', error);
+        process.exit(1);
+    }
+}
+// Run if called directly (ES module check)
+const isMainModule = process.argv[1] && process.argv[1].endsWith('build-dataset.ts');
+if (isMainModule) {
+    main().catch(err => {
+        console.error(err);
+        process.exit(1);
+    });
+}

package/dist/evaluation/threshold-eval.d.ts

@@ -0,0 +1,63 @@
+export interface QueryJudgment {
+    query: string;
+    relevantFactIds: string[];
+    category: 'architecture' | 'pattern' | 'decision' | 'preference' | 'fact';
+    description: string;
+}
+export interface ThresholdResult {
+    threshold: number;
+    precision: number;
+    recall: number;
+    f1: number;
+    truePositives: number;
+    falsePositives: number;
+    falseNegatives: number;
+    avgRelevanceReturned: number;
+    totalReturned: number;
+}
+export interface EvaluationReport {
+    projectId: string;
+    testDate: string;
+    results: ThresholdResult[];
+    bestThreshold: {
+        byF1: number;
+        byPrecision: number;
+        byRecall: number;
+    };
+    dataset: QueryJudgment[];
+}
+/**
+ * Build test dataset with query-result relevance judgments.
+ *
+ * To use this framework:
+ * 1. Add real queries from your project
+ * 2. Run memory_index with no filter to see all available facts
+ * 3. Manually judge which fact IDs are relevant to each query
+ * 4. Add judgments to this dataset
+ *
+ * Example workflow:
+ * - Query: "authentication patterns"
+ * - Run: memory_index(query="authentication patterns", min_relevance=0)
+ * - Review returned facts, note IDs of truly relevant ones
+ * - Add to dataset with those fact IDs as ground truth
+ */
+export declare function buildTestDataset(): QueryJudgment[];
+/**
+ * Evaluate a single threshold value against the test dataset.
+ */
+export declare function evaluateThreshold(projectId: string, threshold: number, dataset: QueryJudgment[], userId?: string): Promise<ThresholdResult>;
+/**
+ * Run full evaluation across multiple threshold values.
+ */
+export declare function runThresholdEvaluation(projectId: string, thresholds?: number[], userId?: string): Promise<EvaluationReport>;
+/**
+ * Format evaluation report as markdown.
+ */
+export declare function formatReport(report: EvaluationReport): string;
+/**
+ * Run evaluation from command line.
+ *
+ * Usage:
+ *   tsx src/evaluation/threshold-eval.ts <project-id>
+ */
+export declare function main(): Promise<void>;