ctxman 3.0.0 → 3.1.0

package/lib/core/Analyzer.js

@@ -17,7 +17,7 @@ export class Analyzer {
       methodLevel: false,
       parallel: false,
       maxWorkers: 4,
-      ...options
+      ...options,
     };
 
     this.methodAnalyzer = new MethodAnalyzer();
@@ -30,9 +30,10 @@ export class Analyzer {
       totalTokens: 0,
       totalSize: 0,
       totalMethods: 0,
+      totalLines: 0,
       byLanguage: {},
       largestFiles: [],
-      analysisTime: 0
+      analysisTime: 0,
     };
   }
 
@@ -49,7 +50,7 @@ export class Analyzer {
 
     return {
       files: results,
-      stats: this.getStats()
+      stats: this.getStats(),
     };
   }
 
@@ -83,19 +84,23 @@ export class Analyzer {
         name: fileInfo.name,
         extension: fileInfo.extension,
         size: fileInfo.size,
+        modified: fileInfo.modified,
         tokens: tokens,
         lines: content.split('\n').length,
-        language: this.detectLanguage(fileInfo.extension)
+        language: this.detectLanguage(fileInfo.extension),
       };
 
       if (this.options.methodLevel) {
         const methods = this.methodAnalyzer.extractMethods(content, fileInfo.path);
-        analysis.methods = methods;
+        const sources = this.methodAnalyzer.extractMethodSources(content, methods, fileInfo.path);
+        analysis.methods = methods.map((method, i) => ({
+          ...method,
+          tokens: TokenUtils.calculate(sources[i], fileInfo.path),
+        }));
         analysis.methodCount = methods.length;
       }
 
       return analysis;
-
     } catch (error) {
       logger.error(`Error analyzing file ${fileInfo.relativePath}: ${error.message}`);
       return null;
@@ -106,12 +111,13 @@ export class Analyzer {
     this.stats.totalFiles++;
     this.stats.totalTokens += analysis.tokens;
     this.stats.totalSize += analysis.size;
+    this.stats.totalLines += analysis.lines || 0;
 
     if (!this.stats.byLanguage[analysis.language]) {
       this.stats.byLanguage[analysis.language] = {
         files: 0,
         tokens: 0,
-        size: 0
+        size: 0,
       };
     }
 
@@ -122,7 +128,7 @@ export class Analyzer {
     this.stats.largestFiles.push({
       path: analysis.relativePath,
       tokens: analysis.tokens,
-      size: analysis.size
+      size: analysis.size,
     });
 
     this.stats.largestFiles.sort((a, b) => b.tokens - a.tokens);
@@ -152,7 +158,7 @@ export class Analyzer {
       '.cpp': 'C++',
       '.h': 'C/C++',
       '.hpp': 'C++',
-      '.scala': 'Scala'
+      '.scala': 'Scala',
     };
 
     return languageMap[extension] || 'Other';
@@ -171,7 +177,7 @@ export class Analyzer {
       .map(([language, stats]) => ({
         language,
         ...stats,
-        percentage: (stats.tokens / this.stats.totalTokens) * 100
+        percentage: (stats.tokens / this.stats.totalTokens) * 100,
       }))
       .sort((a, b) => b.tokens - a.tokens);
   }

package/lib/core/ContextBuilder.js

@@ -24,7 +24,7 @@ export class ContextBuilder {
       includeTests: true,
       includeDocumentation: true,
       priorityStrategy: 'balanced', // balanced, changed-first, core-first
-      ...options
+      ...options,
     };
   }
 
@@ -49,7 +49,7 @@ export class ContextBuilder {
       metadata: this.buildMetadata(stats),
       files: this.buildFileList(filteredFiles),
       methods: this.options.methodLevel ? this.buildMethodList(filteredFiles) : null,
-      statistics: stats
+      statistics: stats,
     };
 
     // Optimize for target LLM if specified
@@ -76,8 +76,8 @@ export class ContextBuilder {
       configuration: {
         includeTests: this.options.includeTests,
         includeDocumentation: this.options.includeDocumentation,
-        methodLevel: this.options.methodLevel
-      }
+        methodLevel: this.options.methodLevel,
+      },
     };
   }
 
@@ -103,7 +103,7 @@ export class ContextBuilder {
         path: file.relativePath,
         tokens: file.tokens,
         size: file.size,
-        language: file.language
+        language: file.language,
       });
     }
 
@@ -120,11 +120,11 @@ export class ContextBuilder {
 
     for (const file of files) {
       if (file.methods && file.methods.length > 0) {
-        methods[file.relativePath] = file.methods.map(method => ({
+        methods[file.relativePath] = file.methods.map((method) => ({
           name: method.name,
           line: method.line,
           tokens: method.tokens || 0,
-          type: method.type || 'function'
+          type: method.type || 'function',
         }));
       }
     }
@@ -149,26 +149,15 @@ export class ContextBuilder {
       return files;
     }
 
-    // Apply priority strategy
-    const prioritized = this.prioritizeFiles(files);
-
-    // Select files until token budget reached
+    // Pack by priority. A file that does not fit is skipped whole (nothing truncates file
+    // content), and smaller files after it can still use the remaining budget.
     const selected = [];
     let runningTotal = 0;
 
-    for (const file of prioritized) {
+    for (const file of this.prioritizeFiles(files)) {
       if (runningTotal + file.tokens <= targetTokens) {
         selected.push(file);
         runningTotal += file.tokens;
-      } else {
-        // Check if we can include part of the file
-        const remainingBudget = targetTokens - runningTotal;
-        if (remainingBudget > 1000) { // Minimum 1k tokens
-          logger.info(`Partial file inclusion: ${file.relativePath} (${remainingBudget} tokens)`);
-          // Note: Actual implementation would truncate file content
-          selected.push(file);
-        }
-        break;
       }
     }
 
@@ -183,11 +172,13 @@ export class ContextBuilder {
    */
   prioritizeFiles(files) {
     const strategy = this.options.priorityStrategy;
+    // Sort a copy: the caller's analysis result keeps its order
+    const sorted = [...files];
 
     switch (strategy) {
       case 'changed-first':
-        // Prioritize recently changed files (requires git metadata)
-        return files.sort((a, b) => {
+        // Most recently modified files first (mtime from the scanner)
+        return sorted.sort((a, b) => {
           const aModified = a.modified || 0;
           const bModified = b.modified || 0;
           return bModified - aModified;
@@ -195,7 +186,7 @@ export class ContextBuilder {
 
       case 'core-first':
         // Prioritize core application files (heuristic-based)
-        return files.sort((a, b) => {
+        return sorted.sort((a, b) => {
           const aCoreScore = this.calculateCoreScore(a);
           const bCoreScore = this.calculateCoreScore(b);
           return bCoreScore - aCoreScore;
@@ -204,7 +195,7 @@ export class ContextBuilder {
       case 'balanced':
       default:
         // Balance between size and importance
-        return files.sort((a, b) => {
+        return sorted.sort((a, b) => {
           const aScore = this.calculateBalancedScore(a);
           const bScore = this.calculateBalancedScore(b);
           return bScore - aScore;
@@ -277,7 +268,7 @@ export class ContextBuilder {
       name: profile.name,
       vendor: profile.vendor,
       contextWindow: profile.contextWindow,
-      recommendedInput: profile.maxRecommendedInput
+      recommendedInput: profile.maxRecommendedInput,
     };
 
     return optimized;
@@ -292,7 +283,7 @@ export class ContextBuilder {
     const compact = {};
 
     for (const [dir, files] of Object.entries(context.files)) {
-      const fileNames = files.map(f => f.name);
+      const fileNames = files.map((f) => f.name);
       compact[dir] = fileNames;
     }
 
@@ -306,7 +297,7 @@ export class ContextBuilder {
    */
   getSummary(context) {
     const meta = context.metadata;
-    const stats = context.statistics;
+    const _stats = context.statistics;
 
     let summary = `Context Summary:\n`;
     summary += `  Files: ${meta.totalFiles}\n`;

package/lib/core/Reporter.js

@@ -24,7 +24,7 @@ export class Reporter {
       outputPath: null,
       clipboard: false,
       verbose: true,
-      ...options
+      ...options,
     };
 
     this.formatRegistry = new FormatRegistry();
@@ -79,7 +79,7 @@ export class Reporter {
         .slice(0, 5);
 
       for (const [lang, langStats] of languages) {
-        const percentage = (langStats.tokens / stats.totalTokens * 100).toFixed(1);
+        const percentage = ((langStats.tokens / stats.totalTokens) * 100).toFixed(1);
         const bar = '▓'.repeat(Math.round(percentage / 2));
         console.log(`   ${lang.padEnd(15)} ${percentage.padStart(5)}%  ${bar}`);
       }
@@ -91,8 +91,10 @@ export class Reporter {
       console.log('🏆 LARGEST FILES (by tokens):');
       for (let i = 0; i < Math.min(5, stats.largestFiles.length); i++) {
         const file = stats.largestFiles[i];
-        const percentage = (file.tokens / stats.totalTokens * 100).toFixed(1);
-        console.log(`   ${(i + 1)}. ${file.path.padEnd(40)} ${file.tokens.toLocaleString().padStart(8)} tokens (${percentage}%)`);
+        const percentage = ((file.tokens / stats.totalTokens) * 100).toFixed(1);
+        console.log(
+          `   ${i + 1}. ${file.path.padEnd(40)} ${file.tokens.toLocaleString().padStart(8)} tokens (${percentage}%)`
+        );
       }
       console.log();
     }
@@ -114,7 +116,7 @@ export class Reporter {
       console.log(`   Analysis Time: ${stats.analysisTime}ms`);
       if (stats.cacheHits !== undefined) {
         const totalCache = stats.cacheHits + stats.cacheMisses;
-        const hitRate = totalCache > 0 ? (stats.cacheHits / totalCache * 100).toFixed(1) : 0;
+        const hitRate = totalCache > 0 ? ((stats.cacheHits / totalCache) * 100).toFixed(1) : 0;
         console.log(`   Cache Hit Rate: ${hitRate}% (${stats.cacheHits}/${totalCache})`);
       }
       console.log();
@@ -129,7 +131,10 @@ export class Reporter {
   async exportToFile(context, outputPath) {
     try {
       const format = this.detectFormatFromPath(outputPath);
-      const content = this.formatRegistry.encode(format, context);
+      const content =
+        format === 'summary'
+          ? this.generateSummary(context)
+          : this.formatRegistry.encode(format, context);
 
       fs.writeFileSync(outputPath, content, 'utf-8');
 
@@ -139,7 +144,6 @@ export class Reporter {
       console.log(`💾 Report saved: ${outputPath}`);
       console.log(`   Format: ${format.toUpperCase()}`);
       console.log(`   Size: ${size} KB`);
-
     } catch (error) {
       logger.error(`Failed to export to file: ${error.message}`);
       throw error;
@@ -169,7 +173,6 @@ export class Reporter {
         const fallbackPath = path.join(process.cwd(), `context-${Date.now()}.${format}`);
         await this.exportToFile(context, fallbackPath);
       }
-
     } catch (error) {
       logger.error(`Failed to export to clipboard: ${error.message}`);
       throw error;
@@ -192,7 +195,8 @@ export class Reporter {
       '.md': 'markdown',
       '.csv': 'csv',
       '.xml': 'xml',
-      '.txt': 'gitingest'
+      // A GitIngest digest needs file contents, which a Context does not carry
+      '.txt': 'summary',
     };
 
     return formatMap[ext] || this.options.format || 'toon';
@@ -216,7 +220,7 @@ export class Reporter {
    */
   generateSummary(context) {
     const meta = context.metadata;
-    const stats = context.statistics;
+    const _stats = context.statistics;
 
     let summary = 'Ctxman Analysis Summary\n';
     summary += '='.repeat(50) + '\n\n';
@@ -250,10 +254,7 @@ export class Reporter {
     const results = [];
 
     for (const format of formats) {
-      const outputPath = path.join(
-        process.cwd(),
-        `context-${Date.now()}.${format}`
-      );
+      const outputPath = path.join(process.cwd(), `context-${Date.now()}.${format}`);
 
       try {
         await this.exportToFile(context, outputPath);

package/lib/core/Scanner.js

@@ -8,17 +8,29 @@ import path from 'path';
 import GitIgnoreParser from '../parsers/gitignore-parser.js';
 import FileUtils from '../utils/file-utils.js';
 import { getLogger } from '../utils/logger.js';
+import { loadPolicy } from '../policy/policy.js';
+import { loadIgnoreFileRules, resolveIgnoreFileNames } from '../policy/ignore-files.js';
 
 const logger = getLogger('Scanner');
 
 export class Scanner {
+  /**
+   * @param {string} rootPath - Project root
+   * @param {object} [options]
+   * @param {import('../policy/policy.js').Policy|false|null} [options.policy] - Export policy
+   *   whose deny rules drop files; loaded from the project (.ctxman/policy.json and
+   *   CTXMAN_POLICY) when not given, and turned off with false
+   * @param {string|string[]} [options.respectIgnoreFiles] - AI-tool ignore files to honour
+   *   ("all", ".cursorignore,.aiexclude" or an array), in addition to the policy's list
+   * @param {object} [options.env=process.env] - Environment for loading the policy
+   */
   constructor(rootPath, options = {}) {
     this.rootPath = rootPath;
     this.options = {
       respectGitignore: true,
       followSymlinks: false,
       maxDepth: Infinity,
-      ...options
+      ...options,
     };
 
     // Initialize GitIgnoreParser with correct file paths
@@ -31,14 +43,23 @@ export class Scanner {
       filesScanned: 0,
       directoriesTraversed: 0,
       filesIgnored: 0,
-      errors: 0
+      filesDenied: 0,
+      errors: 0,
     };
+    // Loaded on first use, so constructing a Scanner reads no policy files
+    this._policy = undefined;
+    this._ignoreFileRules = undefined;
   }
 
   scan() {
     logger.info(`Starting scan: ${this.rootPath}`);
     const startTime = Date.now();
 
+    // Load here, outside scanDirectory's error handling: an invalid policy must fail the
+    // scan instead of yielding an empty file list
+    this.getPolicy();
+    this.getIgnoreFileRules();
+
     const files = this.scanDirectory(this.rootPath, 0);
 
     const elapsed = Date.now() - startTime;
@@ -62,25 +83,32 @@ export class Scanner {
         const fullPath = path.join(dirPath, entry.name);
         const relativePath = path.relative(this.rootPath, fullPath);
 
-        if (this.shouldIgnore(relativePath)) {
+        if (this.shouldIgnore(relativePath, fullPath)) {
           this.stats.filesIgnored++;
           continue;
         }
 
         if (entry.isDirectory()) {
+          if (FileUtils.isSkippedDirectory(entry.name)) continue;
           const subFiles = this.scanDirectory(fullPath, depth + 1);
           files.push(...subFiles);
         } else if (entry.isFile()) {
           // Check if it's a text file (not binary)
           try {
-            if (FileUtils.isText(fullPath)) {
-              const fileInfo = this.getFileInfo(fullPath, relativePath);
-              files.push(fileInfo);
-              this.stats.filesScanned++;
+            if (this.getPolicy()?.isAuditLog(relativePath)) {
+              this.stats.filesIgnored++;
+            } else if (FileUtils.isText(fullPath)) {
+              if (this.getPolicy()?.isDenied(relativePath)) {
+                this.stats.filesDenied++;
+              } else {
+                const fileInfo = this.getFileInfo(fullPath, relativePath);
+                files.push(fileInfo);
+                this.stats.filesScanned++;
+              }
             } else {
               this.stats.filesIgnored++;
             }
-          } catch (error) {
+          } catch (_error) {
             // Skip files that can't be processed
             this.stats.filesIgnored++;
           }
@@ -104,12 +132,52 @@ export class Scanner {
       extension: path.extname(fullPath),
       size: stats.size,
       modified: stats.mtime,
-      created: stats.birthtime
+      created: stats.birthtime,
     };
   }
 
-  shouldIgnore(relativePath) {
-    return this.gitIgnore.isIgnored(null, relativePath);
+  /**
+   * Whether the export policy denies a project-relative path
+   * @param {string} relativePath
+   * @returns {boolean}
+   */
+  isDenied(relativePath) {
+    return Boolean(this.getPolicy()?.isDenied(relativePath));
+  }
+
+  shouldIgnore(relativePath, fullPath = path.join(this.rootPath, relativePath)) {
+    if (this.gitIgnore.isIgnored(fullPath, relativePath)) return true;
+    return Boolean(this.getIgnoreFileRules()?.isIgnored(relativePath));
+  }
+
+  /**
+   * Effective export policy, or null when none applies
+   * @returns {import('../policy/policy.js').Policy|null}
+   */
+  getPolicy() {
+    if (this._policy === undefined) {
+      const { policy, env } = this.options;
+      const loaded = policy === undefined ? loadPolicy(this.rootPath, { env }) : policy;
+      this._policy = loaded?.active ? loaded : null;
+    }
+    return this._policy;
+  }
+
+  /**
+   * AI-tool ignore files from options.respectIgnoreFiles and the policy's ignoreFiles
+   * @returns {import('../policy/ignore-files.js').IgnoreFileRules|null}
+   */
+  getIgnoreFileRules() {
+    if (this._ignoreFileRules === undefined) {
+      const names = [
+        ...new Set([
+          ...resolveIgnoreFileNames(this.options.respectIgnoreFiles),
+          ...(this.getPolicy()?.ignoreFiles ?? []),
+        ]),
+      ];
+      this._ignoreFileRules = names.length > 0 ? loadIgnoreFileRules(this.rootPath, names) : null;
+    }
+    return this._ignoreFileRules;
   }
 
   getStats() {
@@ -121,7 +189,8 @@ export class Scanner {
       filesScanned: 0,
       directoriesTraversed: 0,
       filesIgnored: 0,
-      errors: 0
+      filesDenied: 0,
+      errors: 0,
     };
   }
 }

package/lib/count/index.js

@@ -0,0 +1,100 @@
+/**
+ * Token count for a target model
+ * Counts a project (or a piped text) with the target model's tokenizer where one is
+ * available offline (tiktoken for OpenAI models), and, on request, exactly through
+ * Anthropic's count_tokens API for Claude models.
+ */
+
+import fs from 'fs';
+import Scanner from '../core/Scanner.js';
+import TokenUtils from '../utils/token-utils.js';
+import { LLMDetector } from '../utils/llm-detector.js';
+import { ClaudeTokenCounter } from '../utils/claude-token-counter.js';
+import { redactSecrets } from '../utils/secret-redactor.js';
+
+/**
+ * Read the documents to count
+ * @param {string} root - Project root (ignored when `text` is given)
+ * @param {string} [text] - A single piped text instead of the project files
+ * @returns {{path: string, content: string}[]}
+ */
+function readDocuments(root, text) {
+  if (text !== undefined) return [{ path: '<stdin>', content: text }];
+  return new Scanner(root).scan().flatMap((file) => {
+    try {
+      return [{ path: file.relativePath, content: fs.readFileSync(file.path, 'utf8') }];
+    } catch (_error) {
+      return [];
+    }
+  });
+}
+
+/**
+ * Count tokens for a target model
+ * @param {string} root - Project root
+ * @param {object} [options]
+ * @param {string} [options.model] - Target model id or alias (default: DEFAULT_TARGET_MODEL)
+ * @param {string} [options.text] - Count this text instead of the project
+ * @param {boolean} [options.api=false] - Count exactly with the Anthropic API (Claude only)
+ * @param {object} [options.client] - Anthropic client stand-in (tests)
+ * @param {(done: number, total: number) => void} [options.onProgress]
+ * @returns {Promise<object>} Count result
+ */
+export async function countForModel(root, options = {}) {
+  const requested = options.model || LLMDetector.getDefaultModelId();
+  const profile = LLMDetector.getProfile(requested);
+  const modelId = profile.id || requested;
+  const encoding = TokenUtils.encodingFor(modelId);
+  const documents = readDocuments(root, options.text);
+
+  const files = documents
+    .map((doc) => ({
+      path: doc.path,
+      tokens: TokenUtils.calculate(doc.content, doc.path, encoding),
+    }))
+    .sort((a, b) => b.tokens - a.tokens);
+  const localTokens = files.reduce((sum, file) => sum + file.tokens, 0);
+
+  const result = {
+    model: modelId,
+    modelName: profile.name,
+    vendor: profile.vendor,
+    files: files.length,
+    encoding: TokenUtils.isExact() ? encoding : 'estimate',
+    localTokens,
+    tokens: localTokens,
+    method: 'local',
+    // tiktoken matches OpenAI models whose encoding is published; others are approximations
+    exact: TokenUtils.isExact() && profile.tokenizer?.exact === true,
+    contextWindow: profile.contextWindow,
+    largestFiles: files,
+  };
+
+  if (options.api) {
+    if (profile.vendor !== 'Anthropic') {
+      throw new Error(
+        `--api counts with Anthropic's API, so it needs a Claude model (got ${requested})`
+      );
+    }
+    const counter = await ClaudeTokenCounter.create({ model: modelId, client: options.client });
+    // Secrets never leave the machine, even for counting
+    const redacted = documents.map((doc) => ({
+      path: doc.path,
+      content: redactSecrets(doc.content).text,
+    }));
+    const { total, requests } = await counter.countDocuments(redacted, {
+      onProgress: options.onProgress,
+    });
+    result.tokens = total;
+    result.method = 'anthropic-api';
+    result.exact = true;
+    result.requests = requests;
+    result.ratio = localTokens > 0 ? total / localTokens : null;
+  }
+
+  result.fits = result.tokens <= result.contextWindow;
+  result.percentOfWindow = (result.tokens / result.contextWindow) * 100;
+  return result;
+}
+
+export default countForModel;