ctxman 3.1.1 → 3.2.8

package/lib/cache/CacheManager.js

@@ -4,8 +4,8 @@
  *
  * Responsibilities:
  * - Cache analysis results
- * - Invalidate on file changes
- * - Support multiple storage backends
+ * - Invalidate on file changes (mtimeMs, byte size, analysis options, tokenizer encoding)
+ * - Support multiple storage backends (memory, disk)
  * - TTL-based expiration
  */
 
@@ -16,6 +16,22 @@ import { getLogger } from '../utils/logger.js';
 
 const logger = getLogger('CacheManager');
 
+/**
+ * Normalize a Date or numeric timestamp to milliseconds, or null when missing
+ * @param {Date|number|undefined|null} value
+ * @returns {number|null}
+ */
+function normalizeMtime(value) {
+  if (value instanceof Date) {
+    const ms = value.getTime();
+    return Number.isFinite(ms) ? ms : null;
+  }
+  if (typeof value === 'number' && Number.isFinite(value)) {
+    return value;
+  }
+  return null;
+}
+
 export class CacheManager {
   constructor(options = {}) {
     this.options = {
@@ -23,6 +39,7 @@ export class CacheManager {
       strategy: 'disk', // disk, memory
       ttl: 3600, // 1 hour in seconds
       path: '.context-cache',
+      maxEntries: 5000,
       maxSize: 100 * 1024 * 1024, // 100MB
       ...options,
     };
@@ -36,8 +53,8 @@ export class CacheManager {
       errors: 0,
     };
 
-    // Initialize cache directory
-    if (this.options.strategy === 'disk') {
+    // Initialize cache directory when enabled on disk
+    if (this.options.enabled && this.options.strategy === 'disk') {
       this.initCacheDirectory();
     }
   }
@@ -62,22 +79,29 @@ export class CacheManager {
   /**
    * Get cached analysis for file
    * @param {string} filePath
-   * @param {number} modifiedTime - File modification timestamp
-   * @returns {FileAnalysis|null}
+   * @param {number|Date} modifiedTime - File modification timestamp
+   * @param {object} [meta] - Additional cache key/validation metadata
+   * @param {number} [meta.size] - Current file byte size
+   * @param {string} [meta.namespace] - Caller namespace ('analyzer', 'token-calculator', ...)
+   * @param {string} [meta.encoding] - Tokenizer encoding ('cl100k_base', 'o200k_base', ...)
+   * @param {boolean} [meta.methodLevel] - Whether method-level analysis is included
+   * @param {string} [meta.extraKey] - Extra discriminator (e.g. method filter signature)
+   * @returns {object|null}
    */
-  get(filePath, modifiedTime) {
+  get(filePath, modifiedTime, meta = {}) {
     if (!this.options.enabled) {
       return null;
     }
 
-    const key = this.getCacheKey(filePath);
+    const key = this.getCacheKey(filePath, meta);
+    const mtimeMs = normalizeMtime(modifiedTime);
+    const size = typeof meta?.size === 'number' ? meta.size : null;
 
     try {
       if (this.options.strategy === 'memory') {
-        return this.getFromMemory(key, modifiedTime);
-      } else {
-        return this.getFromDisk(key, modifiedTime);
+        return this.getFromMemory(key, mtimeMs, size);
       }
+      return this.getFromDisk(key, mtimeMs, size);
     } catch (error) {
       logger.error(`Cache get error: ${error.message}`);
       this.stats.errors++;
@@ -88,10 +112,11 @@ export class CacheManager {
   /**
    * Get from memory cache
    * @param {string} key
-   * @param {number} modifiedTime
-   * @returns {FileAnalysis|null}
+   * @param {number|null} modifiedTime
+   * @param {number|null} size
+   * @returns {object|null}
    */
-  getFromMemory(key, modifiedTime) {
+  getFromMemory(key, modifiedTime, size = null) {
     const cached = this.memoryCache.get(key);
 
     if (!cached) {
@@ -99,25 +124,30 @@ export class CacheManager {
       return null;
     }
 
-    // Check if expired
-    if (this.isExpired(cached.timestamp, cached.modifiedTime, modifiedTime)) {
+    // Check if expired or invalidated by mtime/size change
+    if (this.isExpired(cached.timestamp, cached.modifiedTime, modifiedTime, cached.size, size)) {
       this.memoryCache.delete(key);
       this.stats.misses++;
       return null;
     }
 
+    // Refresh insertion order for LRU eviction
+    this.memoryCache.delete(key);
+    this.memoryCache.set(key, cached);
+
     this.stats.hits++;
     logger.debug(`Cache hit (memory): ${key}`);
-    return cached.data;
+    return structuredClone(cached.data);
   }
 
   /**
    * Get from disk cache
    * @param {string} key
-   * @param {number} modifiedTime
-   * @returns {FileAnalysis|null}
+   * @param {number|null} modifiedTime
+   * @param {number|null} size
+   * @returns {object|null}
    */
-  getFromDisk(key, modifiedTime) {
+  getFromDisk(key, modifiedTime, size = null) {
     const cacheFile = path.join(this.options.path, `${key}.json`);
 
     if (!fs.existsSync(cacheFile)) {
@@ -129,9 +159,13 @@ export class CacheManager {
       const content = fs.readFileSync(cacheFile, 'utf-8');
       const cached = JSON.parse(content);
 
-      // Check if expired
-      if (this.isExpired(cached.timestamp, cached.modifiedTime, modifiedTime)) {
-        fs.unlinkSync(cacheFile);
+      // Check if expired or invalidated by mtime/size change
+      if (this.isExpired(cached.timestamp, cached.modifiedTime, modifiedTime, cached.size, size)) {
+        try {
+          fs.unlinkSync(cacheFile);
+        } catch {
+          // Ignore race if already removed
+        }
         this.stats.misses++;
         return null;
       }
@@ -148,19 +182,21 @@ export class CacheManager {
   /**
    * Set cache entry
    * @param {string} filePath
-   * @param {FileAnalysis} analysis
-   * @param {number} modifiedTime
+   * @param {object} analysis
+   * @param {number|Date} modifiedTime
+   * @param {object} [meta]
    */
-  set(filePath, analysis, modifiedTime) {
+  set(filePath, analysis, modifiedTime, meta = {}) {
     if (!this.options.enabled) {
       return;
     }
 
-    const key = this.getCacheKey(filePath);
+    const key = this.getCacheKey(filePath, meta);
     const cacheEntry = {
-      data: analysis,
+      data: this.options.strategy === 'memory' ? structuredClone(analysis) : analysis,
       timestamp: Date.now(),
-      modifiedTime: modifiedTime,
+      modifiedTime: normalizeMtime(modifiedTime),
+      size: typeof meta?.size === 'number' ? meta.size : null,
     };
 
     try {
@@ -184,10 +220,14 @@ export class CacheManager {
    * @param {object} cacheEntry
    */
   setInMemory(key, cacheEntry) {
+    if (this.memoryCache.has(key)) {
+      this.memoryCache.delete(key);
+    }
     this.memoryCache.set(key, cacheEntry);
 
     // Check memory limit
-    if (this.memoryCache.size > 1000) {
+    const limit = Number.isInteger(this.options.maxEntries) ? this.options.maxEntries : 1000;
+    while (this.memoryCache.size > limit) {
       this.evictOldest();
     }
   }
@@ -198,26 +238,47 @@ export class CacheManager {
    * @param {object} cacheEntry
    */
   setOnDisk(key, cacheEntry) {
-    const cacheFile = path.join(this.options.path, `${key}.json`);
+    const cacheDir = path.resolve(this.options.path);
+    if (!fs.existsSync(cacheDir)) {
+      fs.mkdirSync(cacheDir, { recursive: true });
+    }
+    const cacheFile = path.join(cacheDir, `${key}.json`);
     fs.writeFileSync(cacheFile, JSON.stringify(cacheEntry), 'utf-8');
   }
 
   /**
-   * Check if cache entry is expired
+   * Check if cache entry is expired or invalidated
    * @param {number} cacheTimestamp - When cache was created
-   * @param {number} cacheModifiedTime - File modified time when cached
-   * @param {number} currentModifiedTime - Current file modified time
+   * @param {number|null} cacheModifiedTime - File modified time when cached
+   * @param {number|Date|null} currentModifiedTime - Current file modified time
+   * @param {number|null} [cachedSize] - File size when cached
+   * @param {number|null} [currentSize] - Current file size
    * @returns {boolean}
    */
-  isExpired(cacheTimestamp, cacheModifiedTime, currentModifiedTime) {
-    // Check TTL
-    const age = (Date.now() - cacheTimestamp) / 1000; // seconds
-    if (age > this.options.ttl) {
+  isExpired(
+    cacheTimestamp,
+    cacheModifiedTime,
+    currentModifiedTime,
+    cachedSize = null,
+    currentSize = null
+  ) {
+    // Check TTL when positive
+    if (this.options.ttl > 0) {
+      const age = (Date.now() - cacheTimestamp) / 1000; // seconds
+      if (age > this.options.ttl) {
+        return true;
+      }
+    }
+
+    // Any modification time mismatch (newer OR reverted to an older commit) invalidates
+    const currentMs = normalizeMtime(currentModifiedTime);
+    const cachedMs = normalizeMtime(cacheModifiedTime);
+    if (currentMs !== null && cachedMs !== null && currentMs !== cachedMs) {
       return true;
     }
 
-    // Check if file was modified
-    if (currentModifiedTime > cacheModifiedTime) {
+    // Byte size mismatch invalidates even within identical mtime granularity
+    if (currentSize !== null && cachedSize !== null && currentSize !== cachedSize) {
       return true;
     }
 
@@ -225,12 +286,30 @@ export class CacheManager {
   }
 
   /**
-   * Generate cache key from file path
+   * Generate cache key from file path and analysis discriminator metadata
    * @param {string} filePath
+   * @param {object} [meta]
    * @returns {string}
    */
-  getCacheKey(filePath) {
-    return crypto.createHash('md5').update(filePath).digest('hex');
+  getCacheKey(filePath, meta = {}) {
+    const hasMeta =
+      meta &&
+      (meta.namespace !== undefined ||
+        meta.encoding !== undefined ||
+        meta.methodLevel !== undefined ||
+        meta.extraKey !== undefined);
+
+    const payload = hasMeta
+      ? JSON.stringify([
+          path.resolve(filePath),
+          meta.namespace || '',
+          meta.encoding || '',
+          Boolean(meta.methodLevel),
+          meta.extraKey || '',
+        ])
+      : String(filePath);
+
+    return crypto.createHash('sha256').update(payload).digest('hex');
   }
 
   /**
@@ -256,7 +335,9 @@ export class CacheManager {
       if (fs.existsSync(cachePath)) {
         const files = fs.readdirSync(cachePath);
         for (const file of files) {
-          fs.unlinkSync(path.join(cachePath, file));
+          if (file.endsWith('.json')) {
+            fs.unlinkSync(path.join(cachePath, file));
+          }
         }
       }
     } catch (error) {
@@ -278,7 +359,7 @@ export class CacheManager {
       }
     }
 
-    if (oldestKey) {
+    if (oldestKey !== null) {
       this.memoryCache.delete(oldestKey);
       this.stats.evictions++;
       logger.debug(`Evicted oldest entry: ${oldestKey}`);
@@ -291,7 +372,7 @@ export class CacheManager {
    */
   getStats() {
     const totalRequests = this.stats.hits + this.stats.misses;
-    const hitRate = totalRequests > 0 ? ((this.stats.hits / totalRequests) * 100).toFixed(1) : 0;
+    const hitRate = totalRequests > 0 ? ((this.stats.hits / totalRequests) * 100).toFixed(1) : '0';
 
     return {
       ...this.stats,
@@ -311,7 +392,7 @@ export class CacheManager {
       const cachePath = path.resolve(this.options.path);
       if (!fs.existsSync(cachePath)) return 0;
 
-      const files = fs.readdirSync(cachePath);
+      const files = fs.readdirSync(cachePath).filter((f) => f.endsWith('.json'));
       return files.length;
     } catch (_error) {
       return 0;
@@ -335,6 +416,7 @@ export class CacheManager {
    * Prune memory cache
    */
   pruneMemoryCache() {
+    if (this.options.ttl <= 0) return;
     const now = Date.now();
     const ttlMs = this.options.ttl * 1000;
     let pruned = 0;
@@ -353,14 +435,17 @@ export class CacheManager {
    * Prune disk cache
    */
   pruneDiskCache() {
+    if (this.options.ttl <= 0) return;
     try {
       const cachePath = path.resolve(this.options.path);
+      if (!fs.existsSync(cachePath)) return;
       const files = fs.readdirSync(cachePath);
       const now = Date.now();
       const ttlMs = this.options.ttl * 1000;
       let pruned = 0;
 
       for (const file of files) {
+        if (!file.endsWith('.json')) continue;
         const filePath = path.join(cachePath, file);
         const stats = fs.statSync(filePath);
 
@@ -377,4 +462,33 @@ export class CacheManager {
   }
 }
 
+/**
+ * Resolve a `cache` option value (`false`, `true`, `'memory'`, `'disk'`, options object,
+ * or a `CacheManager` instance) into a `CacheManager` or `null`.
+ * @param {boolean|string|object|CacheManager|null|undefined} cacheOption
+ * @param {object} [defaults]
+ * @returns {CacheManager|null}
+ */
+export function resolveCacheManager(cacheOption, defaults = {}) {
+  if (!cacheOption) return null;
+  if (
+    cacheOption instanceof CacheManager ||
+    (typeof cacheOption === 'object' &&
+      typeof cacheOption.get === 'function' &&
+      typeof cacheOption.set === 'function')
+  ) {
+    return cacheOption;
+  }
+  if (cacheOption === true || cacheOption === 'memory') {
+    return new CacheManager({ strategy: 'memory', ...defaults });
+  }
+  if (cacheOption === 'disk') {
+    return new CacheManager({ strategy: 'disk', ...defaults });
+  }
+  if (typeof cacheOption === 'object') {
+    return new CacheManager({ ...defaults, ...cacheOption });
+  }
+  return null;
+}
+
 export default CacheManager;

package/lib/core/Analyzer.js

@@ -7,6 +7,7 @@ import fs from 'fs';
 import TokenUtils from '../utils/token-utils.js';
 import FileUtils from '../utils/file-utils.js';
 import MethodAnalyzer from '../analyzers/method-analyzer.js';
+import { resolveCacheManager } from '../cache/CacheManager.js';
 import { getLogger } from '../utils/logger.js';
 
 const logger = getLogger('Analyzer');
@@ -17,15 +18,21 @@ export class Analyzer {
       methodLevel: false,
       parallel: false,
       maxWorkers: 4,
+      targetModel: null,
+      cache: false,
       ...options,
     };
 
+    this.encoding = options.encoding || TokenUtils.encodingFor(this.options.targetModel);
+    this.cache = resolveCacheManager(this.options.cache, {
+      ...(this.options.cachePath ? { path: this.options.cachePath } : {}),
+    });
     this.methodAnalyzer = new MethodAnalyzer();
     this.stats = this.initStats();
   }
 
   initStats() {
-    return {
+    const base = {
       totalFiles: 0,
       totalTokens: 0,
       totalSize: 0,
@@ -35,6 +42,11 @@ export class Analyzer {
       largestFiles: [],
       analysisTime: 0,
     };
+    if (this.cache) {
+      base.cacheHits = 0;
+      base.cacheMisses = 0;
+    }
+    return base;
   }
 
   async analyze(files) {
@@ -75,16 +87,56 @@ export class Analyzer {
         return null;
       }
 
+      let size = fileInfo.size;
+      let modified = fileInfo.modified;
+      let mtimeMs =
+        modified instanceof Date
+          ? modified.getTime()
+          : typeof modified === 'number'
+            ? modified
+            : null;
+
+      if (this.cache && (mtimeMs === null || typeof size !== 'number')) {
+        const stat = fs.statSync(fileInfo.path);
+        mtimeMs = stat.mtimeMs;
+        size = stat.size;
+        if (modified === undefined) modified = stat.mtime;
+      }
+
+      const cacheMeta = {
+        namespace: 'analyzer',
+        size,
+        encoding: this.encoding,
+        methodLevel: Boolean(this.options.methodLevel),
+      };
+
+      if (this.cache) {
+        const cached = this.cache.get(fileInfo.path, mtimeMs, cacheMeta);
+        if (cached) {
+          this.stats.cacheHits++;
+          return {
+            ...cached,
+            path: fileInfo.path,
+            relativePath: fileInfo.relativePath,
+            name: fileInfo.name ?? cached.name,
+            extension: fileInfo.extension ?? cached.extension,
+            size: size ?? cached.size,
+            modified: modified ?? cached.modified,
+          };
+        }
+        this.stats.cacheMisses++;
+      }
+
       const content = fs.readFileSync(fileInfo.path, 'utf-8');
-      const tokens = TokenUtils.calculate(content, fileInfo.path);
+      const tokens = TokenUtils.calculate(content, fileInfo.path, this.encoding);
 
       const analysis = {
         path: fileInfo.path,
         relativePath: fileInfo.relativePath,
         name: fileInfo.name,
         extension: fileInfo.extension,
-        size: fileInfo.size,
-        modified: fileInfo.modified,
+        size: size,
+        modified: modified,
         tokens: tokens,
         lines: content.split('\n').length,
         language: this.detectLanguage(fileInfo.extension),
@@ -95,11 +147,15 @@ export class Analyzer {
         const sources = this.methodAnalyzer.extractMethodSources(content, methods, fileInfo.path);
         analysis.methods = methods.map((method, i) => ({
           ...method,
-          tokens: TokenUtils.calculate(sources[i], fileInfo.path),
+          tokens: TokenUtils.calculate(sources[i], fileInfo.path, this.encoding),
         }));
         analysis.methodCount = methods.length;
       }
 
+      if (this.cache) {
+        this.cache.set(fileInfo.path, analysis, mtimeMs, cacheMeta);
+      }
+
       return analysis;
     } catch (error) {
       logger.error(`Error analyzing file ${fileInfo.relativePath}: ${error.message}`);

package/lib/count/index.js

@@ -1,40 +1,73 @@
 /**
  * Token count for a target model
- * Counts a project (or a piped text) with the target model's tokenizer where one is
- * available offline (tiktoken for OpenAI models), and, on request, exactly through
- * Anthropic's count_tokens API for Claude models.
+ * Counts a project, multi-repository workspace, or piped text with the target model's
+ * tokenizer where one is available offline (tiktoken for OpenAI models), and, on request,
+ * exactly through Anthropic's count_tokens API for Claude models.
  */
 
 import fs from 'fs';
+import path from 'path';
 import Scanner from '../core/Scanner.js';
 import TokenUtils from '../utils/token-utils.js';
 import { LLMDetector } from '../utils/llm-detector.js';
 import { ClaudeTokenCounter } from '../utils/claude-token-counter.js';
 import { redactSecrets } from '../utils/secret-redactor.js';
+import { resolveWorkspaceRepos } from '../workspace/index.js';
 
 /**
- * Read the documents to count
+ * Read the documents to count from a single project or multi-repository workspace
  * @param {string} root - Project root (ignored when `text` is given)
  * @param {string} [text] - A single piped text instead of the project files
- * @returns {{path: string, content: string}[]}
+ * @param {Array<string|{root: string, alias?: string}>} [repos] - Additional repository roots
+ * @returns {{documents: {path: string, content: string, repo?: string}[], resolvedRepos: {root: string, alias: string}[] | null}}
  */
-function readDocuments(root, text) {
-  if (text !== undefined) return [{ path: '<stdin>', content: text }];
-  return new Scanner(root).scan().flatMap((file) => {
+function readDocuments(root, text, repos) {
+  if (text !== undefined) {
+    return {
+      documents: [{ path: '<stdin>', content: text }],
+      resolvedRepos: null,
+    };
+  }
+
+  const extraRepos = Array.isArray(repos) ? repos.filter(Boolean) : [];
+  if (extraRepos.length > 0) {
+    const resolvedRepos = resolveWorkspaceRepos([root, ...extraRepos]);
+    const documents = [];
+    for (const repo of resolvedRepos) {
+      const scanned = new Scanner(repo.root, { policy: repo.policy }).scan();
+      for (const file of scanned) {
+        try {
+          const relPosix = file.relativePath.split(path.sep).join('/');
+          documents.push({
+            path: `${repo.alias}/${relPosix}`,
+            repo: repo.alias,
+            content: fs.readFileSync(file.path, 'utf8'),
+          });
+        } catch (_error) {
+          // Skip unreadable files
+        }
+      }
+    }
+    return { documents, resolvedRepos };
+  }
+
+  const documents = new Scanner(root).scan().flatMap((file) => {
     try {
       return [{ path: file.relativePath, content: fs.readFileSync(file.path, 'utf8') }];
     } catch (_error) {
       return [];
     }
   });
+  return { documents, resolvedRepos: null };
 }
 
 /**
- * Count tokens for a target model
+ * Count tokens for a target model across a project, multi-repo workspace, or piped text
  * @param {string} root - Project root
  * @param {object} [options]
  * @param {string} [options.model] - Target model id or alias (default: DEFAULT_TARGET_MODEL)
  * @param {string} [options.text] - Count this text instead of the project
+ * @param {Array<string|{root: string, alias?: string}>} [options.repos] - Additional workspace repository roots
  * @param {boolean} [options.api=false] - Count exactly with the Anthropic API (Claude only)
  * @param {object} [options.client] - Anthropic client stand-in (tests)
  * @param {(done: number, total: number) => void} [options.onProgress]
@@ -45,12 +78,13 @@ export async function countForModel(root, options = {}) {
   const profile = LLMDetector.getProfile(requested);
   const modelId = profile.id || requested;
   const encoding = TokenUtils.encodingFor(modelId);
-  const documents = readDocuments(root, options.text);
+  const { documents, resolvedRepos } = readDocuments(root, options.text, options.repos);
 
   const files = documents
     .map((doc) => ({
       path: doc.path,
       tokens: TokenUtils.calculate(doc.content, doc.path, encoding),
+      ...(doc.repo ? { repo: doc.repo } : {}),
     }))
     .sort((a, b) => b.tokens - a.tokens);
   const localTokens = files.reduce((sum, file) => sum + file.tokens, 0);
@@ -67,7 +101,7 @@ export async function countForModel(root, options = {}) {
     // tiktoken matches OpenAI models whose encoding is published; others are approximations
     exact: TokenUtils.isExact() && profile.tokenizer?.exact === true,
     contextWindow: profile.contextWindow,
-    largestFiles: files,
+    largestFiles: files.map(({ path: filePath, tokens }) => ({ path: filePath, tokens })),
   };
 
   if (options.api) {
@@ -89,11 +123,37 @@ export async function countForModel(root, options = {}) {
     result.method = 'anthropic-api';
     result.exact = true;
     result.requests = requests;
-    result.ratio = localTokens > 0 ? total / localTokens : null;
+    result.ratio = localTokens > 0 ? total / localTokens : 1;
+    // The API counts batches, not single files; scale each file's local count by the
+    // observed ratio so the per-file breakdown matches the model too
+    result.largestFiles = files.map(({ path: filePath, tokens }) => ({
+      path: filePath,
+      tokens: Math.round(tokens * result.ratio),
+    }));
+  }
+
+  const window = result.contextWindow || 0;
+  result.fits = window > 0 ? result.tokens <= window : true;
+  result.percentOfWindow = window > 0 ? (result.tokens / window) * 100 : 0;
+
+  if (resolvedRepos) {
+    const ratio = result.method === 'anthropic-api' && result.ratio ? result.ratio : 1;
+    result.repos = resolvedRepos.map((repo) => {
+      const repoFiles = files.filter((f) => f.repo === repo.alias);
+      const repoLocalTokens = repoFiles.reduce((sum, f) => sum + f.tokens, 0);
+      const repoTokens =
+        result.method === 'anthropic-api' ? Math.round(repoLocalTokens * ratio) : repoLocalTokens;
+      return {
+        alias: repo.alias,
+        root: repo.root,
+        files: repoFiles.length,
+        localTokens: repoLocalTokens,
+        tokens: repoTokens,
+        percentOfWindow: window > 0 ? (repoTokens / window) * 100 : 0,
+      };
+    });
   }
 
-  result.fits = result.tokens <= result.contextWindow;
-  result.percentOfWindow = (result.tokens / result.contextWindow) * 100;
   return result;
 }