ctxman 3.0.0

package/bin/cm-ask.js

@@ -0,0 +1,95 @@
+#!/usr/bin/env node
+import { VectorStoreFactory } from '../lib/rag/VectorStoreFactory.js';
+import { EmbeddingProvider, TransformersEmbeddingProvider } from '../lib/rag/EmbeddingProvider.js';
+import { Indexer } from '../lib/rag/Indexer.js';
+import { FileDataSource } from '../lib/rag/sources/FileDataSource.js';
+import path from 'path';
+import fs from 'fs/promises';
+import { TokenizerManager } from '../lib/utils/tokenizer-adapter.js';
+
+async function main() {
+  const args = process.argv.slice(2);
+  const query = args.join(' ');
+
+  if (!query) {
+    console.error('Usage: ctxman ask "Your query here"');
+    process.exit(1);
+  }
+
+  // 1. Initialize RAG Components
+  let provider;
+  try {
+    // Try to load local transformers
+    provider = new TransformersEmbeddingProvider();
+    // Lazy init will handle model download on first use
+  } catch (e) {
+    console.error('⚠️ Failed to initialize embedding provider:', e.message);
+    process.exit(1);
+  }
+
+  // Use LanceDB in .ctxman/self-knowledge
+  const storePath = path.join(process.cwd(), '.ctxman', 'self-knowledge');
+  const store = VectorStoreFactory.create('lancedb', provider, { path: storePath });
+
+  // 2. Check if indexed
+  const isIndexed = await checkIndexExists(storePath);
+  
+  if (!isIndexed) {
+    console.log('📚 Index not found. Auto-indexing current project...');
+    await indexProject(store);
+  }
+
+  // 3. Search
+  console.log(`🔍 Searching for: "${query}"...`);
+  const results = await store.search(query, 5); // Get top 5
+
+  if (results.length === 0) {
+    console.log('❌ No relevant context found.');
+    return;
+  }
+
+  // 4. Display Results (or feed to LLM if we had one configured)
+  console.log(`\nFound ${results.length} relevant snippets:\n`);
+  
+  results.forEach((r, i) => {
+    console.log(`[${i+1}] ${r.metadata.filename} (Score: ${r.score?.toFixed(4)})`);
+    console.log('─'.repeat(40));
+    // Show a small snippet around the match if possible, but we stored full text.
+    // Let's just show the first 200 chars or so.
+    console.log(r.text.substring(0, 300).replace(/\n/g, ' ') + '...');
+    console.log('\n');
+  });
+  
+  // Future: Pass `results` to an LLM API to generate a real answer.
+  // For now, this validates the retrieval pipeline works from CLI.
+}
+
+async function checkIndexExists(storePath) {
+  try {
+    await fs.access(storePath);
+    // Also check if it has tables (LanceDB creates folders)
+    const files = await fs.readdir(storePath);
+    return files.length > 0;
+  } catch {
+    return false;
+  }
+}
+
+async function indexProject(store) {
+  const indexer = new Indexer(store);
+  const fileSource = new FileDataSource();
+  indexer.registerSource(fileSource);
+
+  console.log('⏳ Indexing files (this may take a moment)...');
+  await indexer.index('local-files', {
+    targetDir: process.cwd(),
+    include: ['.js', '.md', '.ts', '.json'], // Configurable?
+    exclude: ['node_modules', '.git', 'dist', 'coverage', '.ctxman']
+  });
+  console.log('✅ Indexing complete.');
+}
+
+main().catch(err => {
+  console.error('❌ Error:', err);
+  process.exit(1);
+});

package/bin/cm-gitingest.js

@@ -0,0 +1,134 @@
+#!/usr/bin/env node
+/**
+ * GitHub GitIngest Command
+ * Generate GitIngest digest from GitHub repositories
+ */
+
+const GitUtils = require('../lib/utils/git-utils');
+const { getLogger } = require('../lib/utils/logger');
+
+const logger = getLogger({ level: 'info' });
+
+async function main() {
+    const args = process.argv.slice(2);
+
+    if (args.length === 0 || args.includes('--help') || args.includes('-h')) {
+        printHelp();
+        return;
+    }
+
+    const url = args[0];
+    const verbose = args.includes('--verbose') || args.includes('-v');
+    const keepClone = args.includes('--keep-clone');
+    const shallow = !args.includes('--full-clone');
+
+    // Parse options
+    const outputIndex = args.findIndex(arg => arg === '--output' || arg === '-o');
+    const outputFile = outputIndex !== -1 ? args[outputIndex + 1] : null;
+
+    const branchIndex = args.findIndex(arg => arg === '--branch' || arg === '-b');
+    const branch = branchIndex !== -1 ? args[branchIndex + 1] : 'main';
+
+    const chunkSize = args.includes('--chunk-size')
+        ? parseInt(args[args.findIndex(arg => arg === '--chunk-size') + 1])
+        : null;
+
+    try {
+        console.log('╔════════════════════════════════════════════════════════╗');
+        console.log('║        GitHub GitIngest Generator v2.3.6              ║');
+        console.log('╚════════════════════════════════════════════════════════╝\n');
+
+        const gitUtils = new GitUtils({
+            verbose,
+            outputDir: 'docs'
+        });
+
+        // Get repository info first
+        console.log('📋 Fetching repository information...');
+        const repoInfo = gitUtils.parseGitHubURL(url);
+        repoInfo.branch = branch; // Override branch if specified
+
+        const repoData = await gitUtils.getRepositoryInfo(repoInfo);
+        console.log(`\n📊 Repository Info:`);
+        console.log(`   Name: ${repoData.fullName}`);
+        console.log(`   Description: ${repoData.description || 'N/A'}`);
+        console.log(`   Stars: ⭐ ${repoData.stars.toLocaleString()}`);
+        console.log(`   Language: ${repoData.language || 'Multiple'}`);
+        console.log(`   Default Branch: ${repoData.defaultBranch}`);
+
+        // Generate GitIngest
+        const options = {
+            outputFile,
+            cleanup: !keepClone,
+            shallow,
+            formatterOptions: chunkSize ? {
+                chunking: {
+                    enabled: true,
+                    maxTokensPerChunk: chunkSize
+                }
+            } : {}
+        };
+
+        const stats = await gitUtils.generateFromGitHub(url, options);
+
+        // Log to file
+        logger.info('GitHub GitIngest generated', {
+            repository: stats.repository,
+            outputFile: stats.outputFile,
+            files: stats.files,
+            tokens: stats.tokens
+        });
+
+        console.log('\n✅ Complete!\n');
+
+        // Show next steps
+        console.log('📝 Next steps:');
+        console.log(`   1. View digest: cat "${stats.outputFile}"`);
+        console.log(`   2. Use in LLM: Copy to clipboard`);
+        console.log(`   3. Cleanup: npm run clean\n`);
+
+    } catch (error) {
+        console.error('\n❌ Error:', error.message);
+        logger.error('GitHub GitIngest failed', { error: error.message });
+        process.exit(1);
+    }
+}
+
+function printHelp() {
+    console.log('GitHub GitIngest Generator - Generate digest from GitHub repositories\n');
+    console.log('Usage: ctxman github <url> [options]\n');
+    console.log('Arguments:');
+    console.log('  url                      GitHub repository URL or owner/repo\n');
+    console.log('Options:');
+    console.log('  -o, --output FILE        Output file path (default: docs/{repo}-gitingest-{hash}.txt)');
+    console.log('  -b, --branch BRANCH      Branch to clone (default: main)');
+    console.log('  -v, --verbose            Verbose output');
+    console.log('  --keep-clone             Keep cloned repository (don\'t cleanup)');
+    console.log('  --full-clone             Full clone (default: shallow)');
+    console.log('  --chunk-size TOKENS      Enable chunking with max tokens per chunk');
+    console.log('  -h, --help               Show this help\n');
+    console.log('URL Formats:');
+    console.log('  https://github.com/owner/repo');
+    console.log('  https://github.com/owner/repo.git');
+    console.log('  git@github.com:owner/repo.git');
+    console.log('  github.com/owner/repo');
+    console.log('  owner/repo\n');
+    console.log('Examples:');
+    console.log('  # Basic usage');
+    console.log('  ctxman github https://github.com/facebook/react\n');
+    console.log('  # Short format');
+    console.log('  ctxman github facebook/react\n');
+    console.log('  # Specific branch');
+    console.log('  ctxman github vercel/next.js --branch canary\n');
+    console.log('  # Custom output location');
+    console.log('  ctxman github angular/angular -o docs/angular-digest.txt\n');
+    console.log('  # With chunking');
+    console.log('  ctxman github microsoft/vscode --chunk-size 100000\n');
+    console.log('  # Keep clone for inspection');
+    console.log('  ctxman github nodejs/node --keep-clone --verbose\n');
+    console.log('Output:');
+    console.log('  Generated digest file in docs/ directory');
+    console.log('  Automatic cleanup of temporary clone (unless --keep-clone)\n');
+}
+
+main();

package/bin/cm-update.js

@@ -0,0 +1,175 @@
+#!/usr/bin/env node
+/**
+ * Ctxman - Update Command
+ * Handles version updates and channel switching
+ */
+
+const Updater = require('../lib/utils/updater');
+const { getLogger } = require('../lib/utils/logger');
+
+const logger = getLogger({ level: 'info' });
+
+async function main() {
+    const args = process.argv.slice(2);
+
+    // Parse command
+    const command = args[0];
+
+    const updater = new Updater();
+
+    switch (command) {
+        case 'check':
+            await checkForUpdates(updater);
+            break;
+
+        case 'install':
+            await installUpdate(updater);
+            break;
+
+        case 'rollback':
+            await rollbackVersion(updater);
+            break;
+
+        case 'channel':
+            await switchChannel(updater, args[1]);
+            break;
+
+        case 'info':
+            showUpdateInfo(updater);
+            break;
+
+        default:
+            // Default: check and optionally install
+            await checkAndUpdate(updater);
+            break;
+    }
+}
+
+async function checkForUpdates(updater) {
+    console.log('🔍 Checking for updates...\n');
+
+    const updateInfo = await updater.checkForUpdates();
+
+    if (updateInfo.error) {
+        logger.error('Failed to check for updates', { error: updateInfo.error });
+        console.log(`\n❌ ${updateInfo.message}`);
+        return;
+    }
+
+    if (updateInfo.available) {
+        console.log('✨ Update available!\n');
+        console.log(`   Current: ${updateInfo.currentVersion}`);
+        console.log(`   Latest:  ${updateInfo.latestVersion}`);
+        console.log(`   Channel: ${updateInfo.channel}\n`);
+
+        if (updateInfo.releaseNotes) {
+            console.log('📝 Release Notes:');
+            console.log('   ' + updateInfo.releaseNotes.split('\n').join('\n   '));
+            console.log('');
+        }
+
+        console.log('To install: ctxman update install\n');
+    } else {
+        console.log('✓ You are using the latest version\n');
+        console.log(`   Version: ${updateInfo.currentVersion}`);
+        console.log(`   Channel: ${updater.channel}\n`);
+    }
+}
+
+async function installUpdate(updater) {
+    console.log('📦 Installing update...\n');
+
+    const updateInfo = await updater.checkForUpdates();
+
+    if (!updateInfo.available) {
+        console.log('✓ Already using the latest version\n');
+        return;
+    }
+
+    const result = await updater.installUpdate(updateInfo);
+
+    if (result.success) {
+        console.log('\n✅ Update installed successfully!\n');
+        logger.info('Update installed', {
+            from: updateInfo.currentVersion,
+            to: updateInfo.latestVersion
+        });
+    } else if (result.manualUpdateRequired) {
+        logger.warn('Manual update required');
+    } else {
+        console.log('\n❌ Update failed\n');
+        logger.error('Update failed', { error: result.error });
+    }
+}
+
+async function switchChannel(updater, newChannel) {
+    if (!newChannel) {
+        console.log('❌ Please specify a channel: stable or insider\n');
+        console.log('Usage: ctxman update channel <stable|insider>\n');
+        return;
+    }
+
+    try {
+        console.log(`🔄 Switching to ${newChannel} channel...\n`);
+
+        const updateInfo = await updater.switchChannel(newChannel);
+
+        logger.info('Channel switched', { channel: newChannel });
+
+        if (updateInfo.available) {
+            console.log(`\n✨ Update available on ${newChannel} channel!\n`);
+            console.log(`   Latest: ${updateInfo.latestVersion}\n`);
+            console.log('To install: ctxman update install\n');
+        }
+    } catch (error) {
+        console.log(`\n❌ ${error.message}\n`);
+        logger.error('Failed to switch channel', { error: error.message });
+    }
+}
+
+async function rollbackVersion(updater) {
+    console.log('🔄 Rolling back to previous version...\n');
+
+    const result = await updater.rollback();
+
+    if (result.success) {
+        console.log(`\n✅ Successfully rolled back to v${result.restoredVersion}\n`);
+        logger.info('Version rolled back', {
+            restoredVersion: result.restoredVersion
+        });
+    } else {
+        console.log(`\n❌ Rollback failed: ${result.error}\n`);
+        logger.error('Rollback failed', { error: result.error });
+    }
+}
+
+function showUpdateInfo(updater) {
+    console.log('ℹ️  Update Configuration\n');
+    console.log(`   Current version: ${updater.currentVersion}`);
+    console.log(`   Update channel:  ${updater.channel}`);
+    console.log(`   Auto-update:     ${updater.autoUpdate ? 'Enabled' : 'Disabled'}\n`);
+
+    console.log('Available commands:\n');
+    console.log('   ctxman update check    - Check for updates');
+    console.log('   ctxman update install  - Install latest update');
+    console.log('   ctxman update rollback - Rollback to previous version');
+    console.log('   ctxman update channel  - Switch update channel');
+    console.log('   ctxman update info     - Show this information\n');
+
+    console.log('Channels:\n');
+    console.log('   stable  - Stable releases (recommended)');
+    console.log('   insider - Pre-release builds (bleeding edge)\n');
+}
+
+async function checkAndUpdate(updater) {
+    const updateInfo = await updater.checkAndNotify();
+
+    if (updateInfo && updateInfo.available) {
+        console.log('Run "ctxman update install" to update\n');
+    }
+}
+
+main().catch(error => {
+    console.error('Update command failed:', error.message);
+    process.exit(1);
+});

package/bin/mcp-server.js

@@ -0,0 +1,31 @@
+#!/usr/bin/env node
+
+import MCPServer from "../lib/api/mcp/server.js";
+import { HttpSseTransport } from "../lib/api/mcp/transports/HttpSseTransport.js";
+
+const args = process.argv.slice(2);
+const transportType = args.includes("--transport=sse") ? "sse" : "stdio";
+const port = args.find(arg => arg.startsWith("--port="))?.split("=")[1] || 3000;
+const apiKey = args.find(arg => arg.startsWith("--api-key="))?.split("=")[1];
+
+const server = new MCPServer();
+
+async function main() {
+    try {
+        if (transportType === "sse") {
+            const transport = new HttpSseTransport({
+                port: parseInt(port),
+                auth: apiKey ? { apiKey } : undefined
+            });
+            await transport.start();
+            await server.start(transport.transport);
+        } else {
+            await server.start();
+        }
+    } catch (error) {
+        console.error("Fatal error running MCP server:", error);
+        process.exit(1);
+    }
+}
+
+main();

package/ctxman.js

@@ -0,0 +1,236 @@
+#!/usr/bin/env node
+
+/**
+ * Ctxman - Main Orchestrator
+ * LLM context optimization with method-level filtering and GitIngest support
+ */
+
+import fs from 'fs';
+import path from 'path';
+import TokenCalculator from './lib/analyzers/token-calculator.js';
+import GitIngestFormatter from './lib/formatters/gitingest-formatter.js';
+import TokenUtils from './lib/utils/token-utils.js';
+
+/**
+ * Generate GitIngest digest from token-analysis-report.json
+ */
+function generateDigestFromReport(reportPath) {
+    console.log('📄 Loading token analysis report...');
+
+    if (!fs.existsSync(reportPath)) {
+        console.error(`❌ Error: Report file not found: ${reportPath}`);
+        process.exit(1);
+    }
+
+    try {
+        const report = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
+
+        if (!report.metadata || !report.summary || !report.files) {
+            console.error('❌ Error: Invalid report format. Missing required fields.');
+            process.exit(1);
+        }
+
+        const projectRoot = report.metadata.projectRoot || process.cwd();
+
+        console.log(`✅ Report loaded: ${report.files.length} files`);
+        console.log('📄 Generating GitIngest digest from report...\n');
+
+        const formatter = new GitIngestFormatter(
+            projectRoot,
+            report.summary,
+            report.files
+        );
+
+        const digestPath = path.join(process.cwd(), 'digest.txt');
+        const digestSize = formatter.saveToFile(digestPath);
+
+        console.log(`💾 GitIngest digest saved to: digest.txt`);
+        console.log(`📊 Digest size: ${(digestSize / 1024).toFixed(1)} KB`);
+        console.log(`✨ Generated from: ${path.basename(reportPath)}`);
+
+    } catch (error) {
+        console.error(`❌ Error processing report: ${error.message}`);
+        process.exit(1);
+    }
+}
+
+/**
+ * Generate GitIngest digest from llm-context.json
+ */
+function generateDigestFromContext(contextPath) {
+    console.log('📄 Loading LLM context file...');
+
+    if (!fs.existsSync(contextPath)) {
+        console.error(`❌ Error: Context file not found: ${contextPath}`);
+        process.exit(1);
+    }
+
+    try {
+        const context = JSON.parse(fs.readFileSync(contextPath, 'utf8'));
+
+        if (!context.project || !context.paths) {
+            console.error('❌ Error: Invalid context format. Missing required fields.');
+            process.exit(1);
+        }
+
+        console.log(`✅ Context loaded: ${context.project.totalFiles} files`);
+        console.log('📄 Generating GitIngest digest from context...\n');
+
+        // Extract files from context.paths structure
+        const files = [];
+        for (const [dirPath, fileNames] of Object.entries(context.paths)) {
+            for (const fileName of fileNames) {
+                const relativePath = dirPath === '/'
+                    ? fileName
+                    : `${dirPath}${fileName}`;
+
+                const fullPath = path.join(process.cwd(), relativePath);
+
+                if (fs.existsSync(fullPath)) {
+                    const stats = fs.statSync(fullPath);
+                    files.push({
+                        path: fullPath,
+                        relativePath: relativePath,
+                        sizeBytes: stats.size,
+                        tokens: 0,
+                        lines: 0
+                    });
+                }
+            }
+        }
+
+        console.log(`📁 Found ${files.length} accessible files`);
+
+        const stats = {
+            totalFiles: files.length,
+            totalTokens: context.project.totalTokens,
+            totalBytes: files.reduce((sum, f) => sum + f.sizeBytes, 0),
+            totalLines: 0
+        };
+
+        const formatter = new GitIngestFormatter(
+            process.cwd(),
+            stats,
+            files
+        );
+
+        const digestPath = path.join(process.cwd(), 'digest.txt');
+        const digestSize = formatter.saveToFile(digestPath);
+
+        console.log(`💾 GitIngest digest saved to: digest.txt`);
+        console.log(`📊 Digest size: ${(digestSize / 1024).toFixed(1)} KB`);
+        console.log(`✨ Generated from: ${path.basename(contextPath)}`);
+
+    } catch (error) {
+        console.error(`❌ Error processing context: ${error.message}`);
+        process.exit(1);
+    }
+}
+
+/**
+ * Main execution
+ */
+async function main(args = process.argv.slice(2)) {
+    if (args.includes('--help') || args.includes('-h')) {
+        printHelp();
+        return;
+    }
+
+    // Handle --gitingest-from-report
+    const reportFlagIndex = args.indexOf('--gitingest-from-report');
+    if (reportFlagIndex !== -1) {
+        const reportPath = args[reportFlagIndex + 1] || 'token-analysis-report.json';
+        generateDigestFromReport(reportPath);
+        return;
+    }
+
+    // Handle --gitingest-from-context
+    const contextFlagIndex = args.indexOf('--gitingest-from-context');
+    if (contextFlagIndex !== -1) {
+        const contextPath = args[contextFlagIndex + 1] || 'llm-context.json';
+        generateDigestFromContext(contextPath);
+        return;
+    }
+
+    const options = {
+        saveReport: args.includes('--save-report') || args.includes('-s'),
+        verbose: args.includes('--verbose') || args.includes('-v'),
+        contextExport: args.includes('--context-export'),
+        contextToClipboard: args.includes('--context-clipboard'),
+        methodLevel: args.includes('--method-level') || args.includes('-m'),
+        gitingest: args.includes('--gitingest') || args.includes('-g')
+    };
+
+    printStartupInfo();
+
+    const calculator = new TokenCalculator(process.cwd(), options);
+    await calculator.run();
+}
+
+function printStartupInfo() {
+    console.log('🚀 Code Analyzer by Hakkı Sağdıç');
+    console.log('='.repeat(50));
+    console.log('📋 Available options:');
+    console.log('  --save-report, -s     Save detailed JSON report');
+    console.log('  --verbose, -v         Show included files and directories');
+    console.log('  --context-export      Generate LLM context file list');
+    console.log('  --context-clipboard   Copy context to clipboard');
+    console.log('  --method-level, -m    Enable method-level analysis');
+    console.log('  --gitingest, -g       Generate GitIngest-style digest');
+    console.log('  --help, -h           Show this help message');
+
+    if (!TokenUtils.hasExactCounting()) {
+        console.log('\n💡 For exact token counts, install tiktoken: npm install tiktoken');
+    }
+    console.log();
+}
+
+function printHelp() {
+    console.log('Ctxman - LLM context optimization with method-level filtering');
+    console.log();
+    console.log('Usage: ctxman [options]');
+    console.log('       node ctxman.js [options]  # Direct usage');
+    console.log();
+    console.log('Options:');
+    console.log('  -s, --save-report                    Save detailed JSON report');
+    console.log('  -v, --verbose                        Show all included files');
+    console.log('  --context-export                     Generate LLM context file');
+    console.log('  --context-clipboard                  Copy context to clipboard');
+    console.log('  -m, --method-level                   Enable method-level analysis');
+    console.log('  -g, --gitingest                      Generate GitIngest-style digest');
+    console.log('  --gitingest-from-report <file>       Generate digest from report JSON');
+    console.log('  --gitingest-from-context <file>      Generate digest from context JSON');
+    console.log('  -h, --help                           Show this help');
+    console.log();
+    console.log('Method-level Configuration:');
+    console.log('  .methodinclude                       Include only specified methods');
+    console.log('  .methodignore                        Exclude specified methods');
+    console.log();
+    console.log('Examples:');
+    console.log('  # Standard workflow');
+    console.log('  ctxman                      # Interactive export selection');
+    console.log('  ctxman --save-report        # Save JSON report');
+    console.log('  ctxman --gitingest          # Generate digest.txt');
+    console.log('  ctxman -g -s                # Both digest + report');
+    console.log();
+    console.log('  # Generate digest from existing JSON files (fast, no re-scan)');
+    console.log('  ctxman --gitingest-from-report token-analysis-report.json');
+    console.log('  ctxman --gitingest-from-context llm-context.json');
+    console.log();
+    console.log('  # Two-step workflow');
+    console.log('  ctxman -s                   # Step 1: Analyze and save report');
+    console.log('  ctxman --gitingest-from-report token-analysis-report.json');
+    console.log('                                       # Step 2: Generate digest (instant)');
+}
+
+// ESM entry point check
+if (import.meta.url === `file://${process.argv[1]}`) {
+    main();
+}
+
+export {
+    TokenCalculator,
+    generateDigestFromReport,
+    generateDigestFromContext,
+    main
+};