ctxman 3.0.0 → 3.1.0

package/lib/lsp/server.js

@@ -0,0 +1,311 @@
+/**
+ * Ctxman Language Server Protocol (LSP) Server
+ *
+ * This module provides LSP server functionality for IDE integrations.
+ * It enables real-time token counting and context generation features.
+ *
+ * This is a stub implementation. The full LSP server requires the
+ * vscode-languageserver package to be installed. When available, the
+ * server provides real-time token counting and context generation.
+ *
+ * @version 1.0.0
+ * @module lib/lsp/server
+ */
+
+// Characters per token by model family, for the editor's live estimate; exact counts
+// come from the CLI (`ctxman count --model <id>`). Claude's tokenizer yields more tokens
+// for the same text than OpenAI's, hence the lower ratio.
+const MODEL_TOKEN_RATIOS = {
+  claude: 3.5,
+  gpt: 4,
+  gemini: 4,
+  deepseek: 4,
+  mistral: 3.5,
+  codestral: 3.5,
+  grok: 4,
+  qwen: 4,
+  llama: 4,
+};
+
+let currentModel = 'claude-sonnet-5-5';
+
+/**
+ * Estimate token count for text
+ * @param {string} text - Text to estimate
+ * @param {string} model - Target model id (e.g. claude-sonnet-5-5, gpt-6.1-sol)
+ * @returns {number} Estimated token count
+ */
+function estimateTokens(text, model = currentModel) {
+  if (!text) return 0;
+  const family = Object.keys(MODEL_TOKEN_RATIOS).find((key) =>
+    String(model).toLowerCase().startsWith(key)
+  );
+  const ratio = MODEL_TOKEN_RATIOS[family] || 4;
+  return Math.ceil(text.length / ratio);
+}
+
+// Stub connection and documents for when vscode-languageserver is not available
+let connection = null;
+let documents = null;
+
+// Try to import vscode-languageserver if available
+try {
+  const vscodeLsp = await import('vscode-languageserver/node.js');
+  const vscodeTextDocument = await import('vscode-languageserver-textdocument');
+
+  const { createConnection, TextDocuments, ProposedFeatures, TextDocumentSyncKind } = vscodeLsp;
+  const { TextDocument } = vscodeTextDocument;
+
+  // Create a connection for the server
+  connection = createConnection(ProposedFeatures.all);
+
+  // Create a simple text document manager
+  documents = new TextDocuments(TextDocument);
+
+  // Token estimation cache
+  const tokenCache = new Map();
+
+  // Server configuration
+  let hasConfigurationCapability = false;
+  let hasWorkspaceFolderCapability = false;
+  let _hasDiagnosticRelatedInformationCapability = false;
+
+  /**
+   * Handle initialize request
+   */
+  connection.onInitialize((params) => {
+    const capabilities = params.capabilities;
+
+    // Does the client support the `workspace/configuration` request?
+    hasConfigurationCapability = !!(
+      capabilities.workspace && !!capabilities.workspace.configuration
+    );
+    hasWorkspaceFolderCapability = !!(
+      capabilities.workspace && !!capabilities.workspace.workspaceFolders
+    );
+    _hasDiagnosticRelatedInformationCapability = !!(
+      capabilities.textDocument &&
+      capabilities.textDocument.publishDiagnostics &&
+      capabilities.textDocument.publishDiagnostics.relatedInformation
+    );
+
+    const result = {
+      capabilities: {
+        textDocumentSync: TextDocumentSyncKind.Incremental,
+        // Tell the client that this server supports code completion
+        completionProvider: {
+          resolveProvider: true,
+        },
+        // Custom commands
+        executeCommandProvider: {
+          commands: ['ctxman.generateContext', 'ctxman.analyzeFile', 'ctxman.getTokenCount'],
+        },
+      },
+    };
+
+    if (hasWorkspaceFolderCapability) {
+      result.capabilities.workspace = {
+        workspaceFolders: {
+          supported: true,
+        },
+      };
+    }
+
+    return result;
+  });
+
+  /**
+   * Handle initialized notification
+   */
+  connection.onInitialized(() => {
+    if (hasConfigurationCapability) {
+      // Register for all configuration changes
+      connection.client.register(vscodeLsp.DidChangeConfigurationNotification.type, undefined);
+    }
+
+    connection.console.log('Ctxman LSP Server initialized');
+  });
+
+  /**
+   * Handle configuration changes
+   */
+  connection.onDidChangeConfiguration((_change) => {
+    if (hasConfigurationCapability) {
+      // Reset all cached document settings
+      tokenCache.clear();
+    }
+
+    // Revalidate all open text documents
+    documents.all().forEach(validateTextDocument);
+  });
+
+  /**
+   * Validate text document and compute token count
+   * @param {TextDocument} textDocument
+   */
+  async function validateTextDocument(textDocument) {
+    const text = textDocument.getText();
+    const tokens = estimateTokens(text);
+
+    // Cache the token count
+    tokenCache.set(textDocument.uri, {
+      tokens,
+      timestamp: Date.now(),
+    });
+
+    // Send token count notification
+    connection.sendNotification('ctxman/tokenCount', {
+      uri: textDocument.uri,
+      tokens,
+      model: currentModel,
+    });
+  }
+
+  /**
+   * Handle document open
+   */
+  connection.onDidOpenTextDocument((params) => {
+    const document = documents.get(params.textDocument.uri);
+    if (document) {
+      validateTextDocument(document);
+    }
+  });
+
+  /**
+   * Handle document change
+   */
+  connection.onDidChangeTextDocument((params) => {
+    const document = documents.get(params.textDocument.uri);
+    if (document) {
+      validateTextDocument(document);
+    }
+  });
+
+  /**
+   * Handle document save
+   */
+  connection.onDidSaveTextDocument((params) => {
+    const document = documents.get(params.textDocument.uri);
+    if (document) {
+      validateTextDocument(document);
+    }
+  });
+
+  /**
+   * Handle custom command: generateContext
+   */
+  connection.onExecuteCommand((params) => {
+    if (params.command === 'ctxman.generateContext') {
+      const [uri, options = {}] = params.arguments || [];
+
+      // TODO: Integrate with ctxman CLI
+      // For now, return stub response
+      return {
+        success: true,
+        message: 'Context generation is a stub. Integration with ctxman CLI coming soon.',
+        uri,
+        options,
+      };
+    }
+
+    if (params.command === 'ctxman.getTokenCount') {
+      const [uri] = params.arguments || [];
+      const cached = tokenCache.get(uri);
+
+      if (cached) {
+        return {
+          uri,
+          tokens: cached.tokens,
+          model: currentModel,
+          timestamp: cached.timestamp,
+        };
+      }
+
+      const document = documents.get(uri);
+      if (document) {
+        const tokens = estimateTokens(document.getText());
+        return {
+          uri,
+          tokens,
+          model: currentModel,
+        };
+      }
+
+      return null;
+    }
+
+    if (params.command === 'ctxman.analyzeFile') {
+      const [uri] = params.arguments || [];
+      const document = documents.get(uri);
+
+      if (document) {
+        const text = document.getText();
+        const tokens = estimateTokens(text);
+        const lines = text.split('\n').length;
+        const chars = text.length;
+
+        return {
+          uri,
+          tokens,
+          lines,
+          characters: chars,
+          tokensPerLine: Math.round(tokens / lines),
+          model: currentModel,
+        };
+      }
+
+      return null;
+    }
+
+    return null;
+  });
+
+  /**
+   * Handle completion request
+   */
+  connection.onCompletion((textDocumentPosition) => {
+    // The pass parameter contains the position of the text document in
+    // which code complete got requested
+    const document = documents.get(textDocumentPosition.textDocument.uri);
+    if (!document) {
+      return [];
+    }
+
+    // Return stub completion items
+    // Future: provide context-aware suggestions
+    return [];
+  });
+
+  /**
+   * Handle completion resolve
+   */
+  connection.onCompletionResolve((item) => {
+    return item;
+  });
+
+  // Make the text document manager listen on the connection
+  // for open, change and close text document events
+  documents.listen(connection);
+
+  // Listen on the connection
+  connection.listen();
+} catch (_error) {
+  // vscode-languageserver not available - running in stub mode
+  console.log('Ctxman LSP: Running in stub mode (vscode-languageserver not available)');
+  connection = {
+    onInitialize: () => {},
+    onInitialized: () => {},
+    listen: () => {},
+    console: { log: () => {} },
+  };
+  documents = {
+    get: () => null,
+    all: () => [],
+    listen: () => {},
+  };
+}
+
+/**
+ * Export for testing
+ */
+export { estimateTokens, MODEL_TOKEN_RATIOS, connection, documents };

package/lib/map/graph.js

@@ -0,0 +1,299 @@
+/**
+ * Repository graph for `ctxman map`
+ *
+ * One node per source file the scanner returns (so .gitignore, .contextignore and
+ * .contextinclude apply), one edge per resolved import, plus what ranking needs: the
+ * functions of each file, how often other files mention them, and how often git
+ * changed each file recently.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { execFileSync } from 'child_process';
+import { Scanner } from '../core/Scanner.js';
+import MethodAnalyzer from '../analyzers/method-analyzer.js';
+import { redactSecrets } from '../utils/secret-redactor.js';
+import { languageOf, extractImports, extractDeclarations } from './imports.js';
+import { ImportResolver } from './resolver.js';
+
+/** Commits of git history mined by default */
+export const DEFAULT_HISTORY = 300;
+/** Larger files are left out: they are generated or vendored far more often than written */
+export const DEFAULT_MAX_FILE_BYTES = 512 * 1024;
+/** Average characters per line above which a file counts as minified */
+const MINIFIED_LINE_LENGTH = 300;
+/** Longest signature kept on one map line */
+const MAX_SIGNATURE_LENGTH = 120;
+const IDENTIFIER = /[A-Za-z_$][\w$]*/g;
+
+/**
+ * @typedef {Object} MapMethod
+ * @property {string} name - Function or method name
+ * @property {number} line - 1-based line of the declaration
+ * @property {string} signature - Declaration text on one line (secrets redacted)
+ */
+
+/**
+ * @typedef {Object} MapNode
+ * @property {string} path - Project-relative path with forward slashes
+ * @property {string} language
+ * @property {number} bytes
+ * @property {MapMethod[]} methods
+ * @property {number} symbolRefs - How much other files mention this file's function names
+ * @property {number} unresolvedImports - References that named no file of the project
+ */
+
+/**
+ * @typedef {Object} RepoGraph
+ * @property {string} root - Absolute project root
+ * @property {MapNode[]} nodes - Sorted by path
+ * @property {Array<{from: string, to: string, weight: number}>} edges - Importer → imported
+ * @property {{available: boolean, requested: number, commits: number,
+ *   counts: Object<string, number>}} history - Commits touching each file
+ * @property {Object} stats
+ */
+
+/**
+ * Scan a project and build its file dependency graph
+ * @param {string} root - Project directory
+ * @param {Object} [options]
+ * @param {number} [options.history=300] - Commits to mine for change frequency (0 disables)
+ * @param {boolean} [options.redact=true] - Mask secrets in signatures
+ * @param {number} [options.maxFileBytes=524288] - Skip larger files
+ * @returns {RepoGraph}
+ */
+export function buildRepoGraph(root, options = {}) {
+  const history = options.history ?? DEFAULT_HISTORY;
+  const maxFileBytes = options.maxFileBytes ?? DEFAULT_MAX_FILE_BYTES;
+  const redact = options.redact !== false;
+  if (!Number.isInteger(history) || history < 0) {
+    throw new Error(`history must be a non-negative integer, got ${JSON.stringify(history)}`);
+  }
+
+  const absoluteRoot = path.resolve(root);
+  if (!fs.statSync(absoluteRoot, { throwIfNoEntry: false })?.isDirectory()) {
+    throw new Error(`Not a directory: ${root}`);
+  }
+
+  const timings = {};
+  let started = performance.now();
+  const scanned = new Scanner(absoluteRoot).scan();
+  timings.scan = performance.now() - started;
+
+  started = performance.now();
+  const analyzer = new MethodAnalyzer();
+  const redactions = {};
+  const skipped = [];
+  const nodes = [];
+  const identifiers = [];
+  const candidates = scanned
+    .map((file) => ({ file, relative: file.relativePath.split(path.sep).join('/') }))
+    .filter(({ relative }) => languageOf(relative))
+    .sort((a, b) => compare(a.relative, b.relative));
+
+  for (const { file, relative } of candidates) {
+    if (file.size > maxFileBytes) {
+      skipped.push({ path: relative, reason: 'too large' });
+      continue;
+    }
+    let content;
+    try {
+      content = fs.readFileSync(file.path, 'utf-8');
+    } catch (_error) {
+      skipped.push({ path: relative, reason: 'unreadable' });
+      continue;
+    }
+    const lines = content.split('\n');
+    if (content.length > 4096 && content.length / lines.length > MINIFIED_LINE_LENGTH) {
+      skipped.push({ path: relative, reason: 'minified' });
+      continue;
+    }
+
+    const language = languageOf(relative);
+    nodes.push({
+      path: relative,
+      language,
+      bytes: file.size,
+      methods: extractMethods(analyzer, content, lines, file.path, redact ? redactions : null),
+      symbolRefs: 0,
+      unresolvedImports: 0,
+      imports: extractImports(language, content),
+      declarations: extractDeclarations(language, content),
+    });
+    identifiers.push(new Set(content.match(IDENTIFIER)));
+  }
+  timings.parse = performance.now() - started;
+
+  started = performance.now();
+  const resolver = new ImportResolver(absoluteRoot, nodes);
+  const edges = [];
+  for (const node of nodes) {
+    const { edges: targets, unresolved } = resolver.resolve(node);
+    node.unresolvedImports = unresolved;
+    for (const { to, weight } of targets) edges.push({ from: node.path, to, weight });
+  }
+  edges.sort((a, b) => compare(a.from, b.from) || compare(a.to, b.to));
+  for (const node of nodes) {
+    delete node.imports;
+    delete node.declarations;
+  }
+  countSymbolReferences(nodes, identifiers);
+  timings.resolve = performance.now() - started;
+
+  started = performance.now();
+  const mined = mineHistory(absoluteRoot, history);
+  const known = new Set(nodes.map((node) => node.path));
+  mined.counts = Object.fromEntries(
+    Object.entries(mined.counts).filter(([file]) => known.has(file))
+  );
+  timings.history = performance.now() - started;
+
+  return {
+    root: absoluteRoot,
+    nodes,
+    edges,
+    history: mined,
+    stats: {
+      scannedFiles: scanned.length,
+      files: nodes.length,
+      edges: edges.length,
+      unresolvedImports: nodes.reduce((sum, node) => sum + node.unresolvedImports, 0),
+      skipped,
+      redactions,
+      timings,
+    },
+  };
+}
+
+/** Byte-order comparison, the same on every platform and locale */
+function compare(a, b) {
+  return a < b ? -1 : a > b ? 1 : 0;
+}
+
+/**
+ * Functions of a file with a one-line signature each
+ * @returns {MapMethod[]}
+ */
+function extractMethods(analyzer, content, lines, filePath, redactions) {
+  const found = analyzer
+    .extractMethods(content, filePath)
+    .sort((a, b) => a.line - b.line || compare(a.name, b.name));
+  return found.map((method) => {
+    let signature = signatureAt(lines, method.line);
+    if (redactions) {
+      const result = redactSecrets(signature);
+      signature = result.text;
+      for (const type of result.findings) redactions[type] = (redactions[type] || 0) + 1;
+    }
+    return { name: method.name, line: method.line, signature };
+  });
+}
+
+/**
+ * Declaration starting at a line, joined onto one line until its parameter list closes,
+ * without the body
+ * @param {string[]} lines
+ * @param {number} lineNumber - 1-based
+ * @returns {string}
+ */
+export function signatureAt(lines, lineNumber) {
+  let text = (lines[lineNumber - 1] || '').trim();
+  const depth = (value) => (value.match(/\(/g) || []).length - (value.match(/\)/g) || []).length;
+  for (let i = lineNumber; depth(text) > 0 && i < lineNumber + 6 && i < lines.length; i++) {
+    const next = lines[i].trim();
+    text += text.endsWith('(') || next.startsWith(')') ? next : ` ${next}`;
+  }
+  text = text.replace(/\s+/g, ' ');
+  // A one-line body (`f() { return 1; }`) is cut after the parameter list
+  const body = text.search(/\)\s*\{/);
+  if (body !== -1) text = text.slice(0, body + 1);
+  text = text.replace(/\s*\{\s*\}?\s*$/, '').trim();
+  if (text.length > MAX_SIGNATURE_LENGTH) text = `${text.slice(0, MAX_SIGNATURE_LENGTH - 1)}…`;
+  return text;
+}
+
+/**
+ * symbolRefs of each node: for each function name it defines, the number of other files
+ * that mention the name, shared between the files that define the same name (so a
+ * `constructor` or `render` defined everywhere earns little)
+ */
+function countSymbolReferences(nodes, identifiers) {
+  const definedNames = nodes.map(
+    (node) =>
+      new Set(
+        node.methods.map((method) => method.name.split(' ').pop()).filter((name) => name.length > 2)
+      )
+  );
+  const definers = new Map();
+  for (const names of definedNames) {
+    for (const name of names) definers.set(name, (definers.get(name) || 0) + 1);
+  }
+  const mentions = new Map();
+  for (const ids of identifiers) {
+    for (const id of ids) {
+      if (definers.has(id)) mentions.set(id, (mentions.get(id) || 0) + 1);
+    }
+  }
+  nodes.forEach((node, i) => {
+    let score = 0;
+    for (const name of definedNames[i]) {
+      const defining = definers.get(name);
+      score += Math.max(0, (mentions.get(name) || 0) - defining) / defining;
+    }
+    node.symbolRefs = score;
+  });
+}
+
+/**
+ * How many of the last `commits` commits (merges excluded) touched each file
+ * @param {string} root - Absolute project root; paths come back relative to it
+ * @param {number} commits - 0 disables
+ * @returns {{available: boolean, requested: number, commits: number,
+ *   counts: Object<string, number>, reason?: string}}
+ */
+export function mineHistory(root, commits) {
+  const result = { available: false, requested: commits, commits: 0, counts: {} };
+  if (!commits) {
+    result.reason = 'disabled';
+    return result;
+  }
+  let output;
+  try {
+    output = execFileSync(
+      'git',
+      [
+        '-c',
+        'core.quotePath=false',
+        'log',
+        `--max-count=${commits}`,
+        '--no-merges',
+        '--name-only',
+        '--relative',
+        '--format=%x00',
+      ],
+      {
+        cwd: root,
+        encoding: 'utf-8',
+        maxBuffer: 256 * 1024 * 1024,
+        stdio: ['ignore', 'pipe', 'ignore'],
+      }
+    );
+  } catch (_error) {
+    result.reason = 'not a git repository';
+    return result;
+  }
+
+  const chunks = output.split('\0').slice(1);
+  for (const chunk of chunks) {
+    const files = new Set(
+      chunk
+        .split('\n')
+        .map((line) => line.trim())
+        .filter(Boolean)
+    );
+    for (const file of files) result.counts[file] = (result.counts[file] || 0) + 1;
+  }
+  result.available = true;
+  result.commits = chunks.length;
+  return result;
+}