ctxman 3.0.0 → 3.1.0

package/bin/cm-ask.js

@@ -1,11 +1,10 @@
 #!/usr/bin/env node
 import { VectorStoreFactory } from '../lib/rag/VectorStoreFactory.js';
-import { EmbeddingProvider, TransformersEmbeddingProvider } from '../lib/rag/EmbeddingProvider.js';
+import { TransformersEmbeddingProvider } from '../lib/rag/EmbeddingProvider.js';
 import { Indexer } from '../lib/rag/Indexer.js';
 import { FileDataSource } from '../lib/rag/sources/FileDataSource.js';
 import path from 'path';
 import fs from 'fs/promises';
-import { TokenizerManager } from '../lib/utils/tokenizer-adapter.js';
 
 async function main() {
   const args = process.argv.slice(2);
@@ -33,7 +32,7 @@ async function main() {
 
   // 2. Check if indexed
   const isIndexed = await checkIndexExists(storePath);
-  
+
   if (!isIndexed) {
     console.log('📚 Index not found. Auto-indexing current project...');
     await indexProject(store);
@@ -50,16 +49,16 @@ async function main() {
 
   // 4. Display Results (or feed to LLM if we had one configured)
   console.log(`\nFound ${results.length} relevant snippets:\n`);
-  
+
   results.forEach((r, i) => {
-    console.log(`[${i+1}] ${r.metadata.filename} (Score: ${r.score?.toFixed(4)})`);
+    console.log(`[${i + 1}] ${r.metadata.filename} (Score: ${r.score?.toFixed(4)})`);
     console.log('─'.repeat(40));
     // Show a small snippet around the match if possible, but we stored full text.
     // Let's just show the first 200 chars or so.
     console.log(r.text.substring(0, 300).replace(/\n/g, ' ') + '...');
     console.log('\n');
   });
-  
+
   // Future: Pass `results` to an LLM API to generate a real answer.
   // For now, this validates the retrieval pipeline works from CLI.
 }
@@ -81,15 +80,21 @@ async function indexProject(store) {
   indexer.registerSource(fileSource);
 
   console.log('⏳ Indexing files (this may take a moment)...');
-  await indexer.index('local-files', {
-    targetDir: process.cwd(),
-    include: ['.js', '.md', '.ts', '.json'], // Configurable?
-    exclude: ['node_modules', '.git', 'dist', 'coverage', '.ctxman']
-  });
+  try {
+    await indexer.index('local-files', {
+      targetDir: process.cwd(),
+      include: ['.js', '.md', '.ts', '.json'], // Configurable?
+      exclude: ['node_modules', '.git', 'dist', 'coverage', '.ctxman'],
+    });
+  } catch (error) {
+    console.error(`❌ ${error.message}`);
+    process.exit(1);
+  }
   console.log('✅ Indexing complete.');
 }
 
-main().catch(err => {
-  console.error('❌ Error:', err);
+main().catch((err) => {
+  // Message only: missing optional dependencies surface here with an install hint
+  console.error('❌ Error:', err.message);
   process.exit(1);
 });

package/bin/cm-audit.js

@@ -0,0 +1,124 @@
+#!/usr/bin/env node
+/**
+ * Audit Command
+ * Print the export audit log written while a policy enables `audit`
+ */
+
+import path from 'path';
+import {
+  loadPolicy,
+  readAuditLog,
+  filterAuditEvents,
+  formatAuditTable,
+  parseSince,
+  DEFAULT_AUDIT_PATH,
+  POLICY_FILE,
+} from '../lib/policy/index.js';
+
+const DEFAULT_TABLE_LIMIT = 50;
+
+function main() {
+  const args = process.argv.slice(2);
+
+  if (args.includes('--help') || args.includes('-h')) {
+    printHelp();
+    return;
+  }
+
+  const json = args.includes('--json');
+  let since = null;
+  let limit = json ? 0 : DEFAULT_TABLE_LIMIT;
+  let file = null;
+
+  try {
+    const sinceValue = valueOf(args, '--since');
+    if (sinceValue !== undefined) since = parseSince(sinceValue);
+
+    const limitValue = valueOf(args, '--limit');
+    if (limitValue !== undefined) {
+      if (!/^\d+$/.test(limitValue)) {
+        throw new Error(`Invalid --limit value ${JSON.stringify(limitValue)}: expected 0 or more`);
+      }
+      limit = Number(limitValue);
+    }
+
+    file = valueOf(args, '--file') ?? null;
+  } catch (error) {
+    console.error(`❌ ${error.message}`);
+    process.exitCode = 2;
+    return;
+  }
+
+  const files = file ? [path.resolve(file)] : auditFiles();
+  const { events, skipped, missing } = readAuditLog(files);
+  const selected = filterAuditEvents(events, { since, limit });
+
+  if (json) {
+    console.log(JSON.stringify(selected, null, 2));
+  } else if (missing.length === files.length) {
+    console.log(`No audit log found (${files.join(', ')}).`);
+    console.log(
+      `Exports are logged while ${POLICY_FILE} has "audit": {"enabled": true} (ctxman policy init).`
+    );
+  } else if (selected.length === 0) {
+    console.log(
+      since ? `No exports recorded since ${since.toISOString()}.` : 'No exports recorded.'
+    );
+  } else {
+    console.log(formatAuditTable(selected));
+    console.log();
+    const matching = filterAuditEvents(events, { since }).length;
+    const shown =
+      selected.length < matching
+        ? `Showing the last ${selected.length} of ${matching} events (--limit 0 shows all)`
+        : `${selected.length} event${selected.length === 1 ? '' : 's'}`;
+    console.log(`${shown} from ${files.filter((f) => !missing.includes(f)).join(', ')}`);
+  }
+
+  if (skipped > 0) {
+    console.error(
+      `⚠️  Skipped ${skipped} line${skipped === 1 ? '' : 's'} that are not JSON events`
+    );
+  }
+}
+
+// The policy's audit logs; the default location when no policy names one
+function auditFiles() {
+  const root = process.cwd();
+  try {
+    const policy = loadPolicy(root);
+    if (policy.audit.paths.length > 0) return policy.audit.paths;
+  } catch (error) {
+    console.error(`⚠️  ${error.message}`);
+    console.error(`   Reading the default log ${DEFAULT_AUDIT_PATH} instead.`);
+  }
+  return [path.join(root, DEFAULT_AUDIT_PATH)];
+}
+
+function valueOf(args, flag) {
+  const index = args.indexOf(flag);
+  if (index === -1) return undefined;
+  const value = args[index + 1];
+  if (value === undefined || value.startsWith('--')) {
+    throw new Error(`${flag} needs a value`);
+  }
+  return value;
+}
+
+function printHelp() {
+  console.log('ctxman audit - show the export audit log\n');
+  console.log('Usage: ctxman audit [options]\n');
+  console.log('Options:');
+  console.log('  --since DATE         Only exports at or after DATE: a date (2026-10-01), an ISO');
+  console.log('                       time, or an age such as 30m, 12h, 7d, 2w');
+  console.log(`  --limit N            Show the last N exports (default: ${DEFAULT_TABLE_LIMIT};`);
+  console.log('                       with --json: all; 0 = all)');
+  console.log('  --json               Print the events as a JSON array');
+  console.log("  --file PATH          Read this log instead of the policy's audit log(s)");
+  console.log('  -h, --help           Show this help\n');
+  console.log(`The log is written while ${POLICY_FILE} (or $CTXMAN_POLICY) enables audit;`);
+  console.log(`default location: ${DEFAULT_AUDIT_PATH}. One JSON object per line; each export`);
+  console.log('records who, when, where to, which files and a SHA-256 of the exported text.');
+}
+
+main();

package/bin/cm-count.js

@@ -0,0 +1,151 @@
+#!/usr/bin/env node
+/**
+ * ctxman count — token count for a target model, exact for Claude through the API
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { countForModel } from '../lib/count/index.js';
+import { LLMDetector } from '../lib/utils/llm-detector.js';
+import { getLogger } from '../lib/utils/logger.js';
+import { exitOnBrokenPipe } from '../lib/utils/cli-output.js';
+
+// `ctxman ... | head` closes stdout early; stop quietly instead of throwing EPIPE
+exitOnBrokenPipe();
+
+// stdout carries only the result (it may be JSON); progress, notices and warnings go to stderr
+const print = (line = '') => process.stdout.write(`${line}\n`);
+console.log = console.error;
+getLogger().level = 'warn';
+
+const HELP = `Usage: ctxman count [path | -] [options]
+
+Count the tokens of a project (or of text piped to stdin with "-") for a target model,
+and check them against the model's context window.
+
+Options:
+  --model, -m ID     Target model (default: ${LLMDetector.getDefaultModelId()}; see ctxman --list-llms)
+  --api              Claude models: count exactly with Anthropic's count_tokens API.
+                     Sends the (secret-redacted) text to Anthropic; needs ANTHROPIC_API_KEY
+                     and the @anthropic-ai/sdk package. Counting is free and generates nothing.
+  --top N            Show the N largest files (default 5, 0 to hide)
+  --json             Print the result as JSON
+  -h, --help         Show this help
+
+Without --api, OpenAI models are counted exactly with their tiktoken encoding
+(o200k_base or cl100k_base); other vendors' models use their own tokenizers, so
+their counts are approximate.
+
+Examples:
+  ctxman count --model gpt-6.1-sol
+  ctxman count --model claude-opus-5-5 --api
+  ctxman pack | ctxman count - --model claude-sonnet-5-5 --api`;
+
+function parseArgs(argv) {
+  const options = { top: 5 };
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    if (arg === '--model' || arg === '-m') options.model = argv[++i];
+    else if (arg === '--api') options.api = true;
+    else if (arg === '--json') options.json = true;
+    else if (arg === '--top') options.top = Number.parseInt(argv[++i], 10);
+    else if (arg === '--help' || arg === '-h') options.help = true;
+    else if (arg === '-' || !arg.startsWith('-')) options.target = arg;
+    else throw new Error(`Unknown option: ${arg}`);
+  }
+  if (options.model === undefined && argv.includes('--model')) {
+    throw new Error('--model needs a model id');
+  }
+  if (!Number.isInteger(options.top) || options.top < 0) {
+    throw new Error('--top needs a number of files');
+  }
+  return options;
+}
+
+const n = (value) => value.toLocaleString('en-US');
+
+function printResult(result, top) {
+  const label =
+    result.method === 'anthropic-api'
+      ? `exact, Anthropic count_tokens API (${result.requests} request${result.requests === 1 ? '' : 's'})`
+      : result.encoding === 'estimate'
+        ? 'estimate (install tiktoken for exact counts)'
+        : result.exact
+          ? `exact, tiktoken ${result.encoding}`
+          : `approximate, tiktoken ${result.encoding}; ${result.vendor} models use their own tokenizer`;
+
+  print(`Model:   ${result.modelName} (${result.model})`);
+  print(`Files:   ${n(result.files)}`);
+  print(`Tokens:  ${n(result.tokens)}  [${label}]`);
+  if (result.method === 'anthropic-api' && result.ratio) {
+    print(
+      `         ${n(result.localTokens)} with cl100k_base; Claude counts ${result.ratio.toFixed(2)}x as many`
+    );
+  }
+  print(
+    `Window:  ${n(result.contextWindow)} tokens — ${result.percentOfWindow.toFixed(1)}% used, ${result.fits ? 'fits' : "doesn't fit"}`
+  );
+  if (result.method === 'local' && !result.exact && result.vendor === 'Anthropic') {
+    print(`         For an exact count: ctxman count --model ${result.model} --api`);
+  }
+
+  const largest = result.largestFiles.slice(0, top);
+  if (largest.length > 1) {
+    const scale = result.method === 'anthropic-api' && result.ratio ? result.ratio : 1;
+    print(`\nLargest files${scale !== 1 ? ' (≈, scaled to the API count)' : ''}:`);
+    for (const file of largest) {
+      print(`  ${n(Math.round(file.tokens * scale)).padStart(9)}  ${file.path}`);
+    }
+  }
+}
+
+async function main() {
+  let options;
+  try {
+    options = parseArgs(process.argv.slice(2));
+  } catch (error) {
+    console.error(`❌ ${error.message}\n\n${HELP}`);
+    process.exit(1);
+  }
+  if (options.help) {
+    print(HELP);
+    return;
+  }
+
+  const root = path.resolve(options.target && options.target !== '-' ? options.target : '.');
+  const text = options.target === '-' ? fs.readFileSync(0, 'utf8') : undefined;
+  if (text === undefined && !fs.statSync(root, { throwIfNoEntry: false })?.isDirectory()) {
+    console.error(`❌ Not a directory: ${root}`);
+    process.exit(1);
+  }
+
+  const model = options.model || LLMDetector.getDefaultModelId();
+  if (options.api && LLMDetector.getProfile(model).vendor === 'Anthropic') {
+    console.error('Sending the redacted text to the Anthropic API to count tokens...');
+  }
+
+  try {
+    const result = await countForModel(root, {
+      model,
+      api: options.api,
+      text,
+      onProgress: (done, total) => {
+        if (total > 1) process.stderr.write(`\r  request ${done}/${total}`);
+        if (total > 1 && done === total) process.stderr.write('\n');
+      },
+    });
+    if (options.json) {
+      const { largestFiles, ...summary } = result;
+      print(
+        JSON.stringify({ ...summary, largestFiles: largestFiles.slice(0, options.top) }, null, 2)
+      );
+    } else {
+      printResult(result, options.top);
+    }
+  } catch (error) {
+    console.error(`❌ ${error.message}`);
+    process.exit(1);
+  }
+}
+
+main();

package/bin/cm-gitingest.js

@@ -4,131 +4,144 @@
  * Generate GitIngest digest from GitHub repositories
  */
 
-const GitUtils = require('../lib/utils/git-utils');
-const { getLogger } = require('../lib/utils/logger');
+import GitUtils from '../lib/utils/git-utils.js';
+import { getLogger } from '../lib/utils/logger.js';
+import { loadPolicy } from '../lib/policy/policy.js';
 
 const logger = getLogger({ level: 'info' });
 
 async function main() {
-    const args = process.argv.slice(2);
-
-    if (args.length === 0 || args.includes('--help') || args.includes('-h')) {
-        printHelp();
-        return;
-    }
-
-    const url = args[0];
-    const verbose = args.includes('--verbose') || args.includes('-v');
-    const keepClone = args.includes('--keep-clone');
-    const shallow = !args.includes('--full-clone');
-
-    // Parse options
-    const outputIndex = args.findIndex(arg => arg === '--output' || arg === '-o');
-    const outputFile = outputIndex !== -1 ? args[outputIndex + 1] : null;
-
-    const branchIndex = args.findIndex(arg => arg === '--branch' || arg === '-b');
-    const branch = branchIndex !== -1 ? args[branchIndex + 1] : 'main';
-
-    const chunkSize = args.includes('--chunk-size')
-        ? parseInt(args[args.findIndex(arg => arg === '--chunk-size') + 1])
-        : null;
-
+  const args = process.argv.slice(2);
+
+  if (args.length === 0 || args.includes('--help') || args.includes('-h')) {
+    printHelp();
+    return;
+  }
+
+  const url = args[0];
+  const verbose = args.includes('--verbose') || args.includes('-v');
+  const keepClone = args.includes('--keep-clone');
+  const shallow = !args.includes('--full-clone');
+
+  // Parse options
+  const outputIndex = args.findIndex((arg) => arg === '--output' || arg === '-o');
+  const outputFile = outputIndex !== -1 ? args[outputIndex + 1] : null;
+
+  const branchIndex = args.findIndex((arg) => arg === '--branch' || arg === '-b');
+  const branch = branchIndex !== -1 ? args[branchIndex + 1] : null;
+
+  const chunkSize = args.includes('--chunk-size')
+    ? parseInt(args[args.findIndex((arg) => arg === '--chunk-size') + 1])
+    : null;
+
+  try {
+    console.log('╔════════════════════════════════════════════════════════╗');
+    console.log('║        GitHub GitIngest Generator v2.3.6              ║');
+    console.log('╚════════════════════════════════════════════════════════╝\n');
+
+    const gitUtils = new GitUtils({
+      verbose,
+      outputDir: 'docs',
+    });
+
+    // Repository info is informational: an API error (rate limit, renamed repo) must not
+    // stop the clone
+    console.log('📋 Fetching repository information...');
+    const repoInfo = gitUtils.parseGitHubURL(url);
     try {
-        console.log('╔════════════════════════════════════════════════════════╗');
-        console.log('║        GitHub GitIngest Generator v2.3.6              ║');
-        console.log('╚════════════════════════════════════════════════════════╝\n');
-
-        const gitUtils = new GitUtils({
-            verbose,
-            outputDir: 'docs'
-        });
-
-        // Get repository info first
-        console.log('📋 Fetching repository information...');
-        const repoInfo = gitUtils.parseGitHubURL(url);
-        repoInfo.branch = branch; // Override branch if specified
-
-        const repoData = await gitUtils.getRepositoryInfo(repoInfo);
-        console.log(`\n📊 Repository Info:`);
-        console.log(`   Name: ${repoData.fullName}`);
-        console.log(`   Description: ${repoData.description || 'N/A'}`);
-        console.log(`   Stars: ⭐ ${repoData.stars.toLocaleString()}`);
-        console.log(`   Language: ${repoData.language || 'Multiple'}`);
-        console.log(`   Default Branch: ${repoData.defaultBranch}`);
-
-        // Generate GitIngest
-        const options = {
-            outputFile,
-            cleanup: !keepClone,
-            shallow,
-            formatterOptions: chunkSize ? {
-                chunking: {
-                    enabled: true,
-                    maxTokensPerChunk: chunkSize
-                }
-            } : {}
-        };
-
-        const stats = await gitUtils.generateFromGitHub(url, options);
-
-        // Log to file
-        logger.info('GitHub GitIngest generated', {
-            repository: stats.repository,
-            outputFile: stats.outputFile,
-            files: stats.files,
-            tokens: stats.tokens
-        });
-
-        console.log('\n✅ Complete!\n');
-
-        // Show next steps
-        console.log('📝 Next steps:');
-        console.log(`   1. View digest: cat "${stats.outputFile}"`);
-        console.log(`   2. Use in LLM: Copy to clipboard`);
-        console.log(`   3. Cleanup: npm run clean\n`);
-
+      const repoData = await gitUtils.getRepositoryInfo(repoInfo);
+      console.log(`\n📊 Repository Info:`);
+      console.log(`   Name: ${repoData.fullName}`);
+      console.log(`   Description: ${repoData.description || 'N/A'}`);
+      console.log(`   Stars: ⭐ ${(repoData.stars ?? 0).toLocaleString()}`);
+      console.log(`   Language: ${repoData.language || 'Multiple'}`);
+      console.log(`   Default Branch: ${repoData.defaultBranch}`);
     } catch (error) {
-        console.error('\n❌ Error:', error.message);
-        logger.error('GitHub GitIngest failed', { error: error.message });
-        process.exit(1);
+      console.log(`⚠️  Could not fetch repository info (${error.message}); continuing`);
     }
+
+    // Generate GitIngest
+    const options = {
+      outputFile,
+      branch,
+      cleanup: !keepClone,
+      shallow,
+      // Policy of the current directory and CTXMAN_POLICY (deny rules apply to the clone)
+      policy: loadPolicy(process.cwd()),
+      formatterOptions: chunkSize
+        ? {
+            chunking: {
+              enabled: true,
+              maxTokensPerChunk: chunkSize,
+            },
+          }
+        : {},
+    };
+
+    const stats = await gitUtils.generateFromGitHub(url, options);
+
+    // Log to file
+    logger.info('GitHub GitIngest generated', {
+      repository: stats.repository,
+      outputFile: stats.outputFile,
+      files: stats.files,
+      tokens: stats.tokens,
+    });
+
+    console.log('\n✅ Complete!\n');
+
+    // Show next steps
+    console.log('📝 Next steps:');
+    console.log(`   1. View digest: cat "${stats.outputFile}"`);
+    console.log(`   2. Use in LLM: Copy to clipboard`);
+    console.log(`   3. Cleanup: npm run clean\n`);
+  } catch (error) {
+    console.error('\n❌ Error:', error.message);
+    logger.error('GitHub GitIngest failed', { error: error.message });
+    // 2: invalid policy file, 3: export refused by the policy
+    process.exit(error.exitCode ?? 1);
+  }
 }
 
 function printHelp() {
-    console.log('GitHub GitIngest Generator - Generate digest from GitHub repositories\n');
-    console.log('Usage: ctxman github <url> [options]\n');
-    console.log('Arguments:');
-    console.log('  url                      GitHub repository URL or owner/repo\n');
-    console.log('Options:');
-    console.log('  -o, --output FILE        Output file path (default: docs/{repo}-gitingest-{hash}.txt)');
-    console.log('  -b, --branch BRANCH      Branch to clone (default: main)');
-    console.log('  -v, --verbose            Verbose output');
-    console.log('  --keep-clone             Keep cloned repository (don\'t cleanup)');
-    console.log('  --full-clone             Full clone (default: shallow)');
-    console.log('  --chunk-size TOKENS      Enable chunking with max tokens per chunk');
-    console.log('  -h, --help               Show this help\n');
-    console.log('URL Formats:');
-    console.log('  https://github.com/owner/repo');
-    console.log('  https://github.com/owner/repo.git');
-    console.log('  git@github.com:owner/repo.git');
-    console.log('  github.com/owner/repo');
-    console.log('  owner/repo\n');
-    console.log('Examples:');
-    console.log('  # Basic usage');
-    console.log('  ctxman github https://github.com/facebook/react\n');
-    console.log('  # Short format');
-    console.log('  ctxman github facebook/react\n');
-    console.log('  # Specific branch');
-    console.log('  ctxman github vercel/next.js --branch canary\n');
-    console.log('  # Custom output location');
-    console.log('  ctxman github angular/angular -o docs/angular-digest.txt\n');
-    console.log('  # With chunking');
-    console.log('  ctxman github microsoft/vscode --chunk-size 100000\n');
-    console.log('  # Keep clone for inspection');
-    console.log('  ctxman github nodejs/node --keep-clone --verbose\n');
-    console.log('Output:');
-    console.log('  Generated digest file in docs/ directory');
-    console.log('  Automatic cleanup of temporary clone (unless --keep-clone)\n');
+  console.log('GitHub GitIngest Generator - Generate digest from GitHub repositories\n');
+  console.log('Usage: ctxman github <url> [options]\n');
+  console.log('Arguments:');
+  console.log('  url                      GitHub repository URL or owner/repo\n');
+  console.log('Options:');
+  console.log(
+    '  -o, --output FILE        Output file path (default: docs/{repo}-gitingest-{hash}.txt)'
+  );
+  console.log(
+    "  -b, --branch BRANCH      Branch to clone (default: the repository's default branch)"
+  );
+  console.log('  -v, --verbose            Verbose output');
+  console.log("  --keep-clone             Keep cloned repository (don't cleanup)");
+  console.log('  --full-clone             Full clone (default: shallow)');
+  console.log('  --chunk-size TOKENS      Enable chunking with max tokens per chunk');
+  console.log('  -h, --help               Show this help\n');
+  console.log('URL Formats:');
+  console.log('  https://github.com/owner/repo');
+  console.log('  https://github.com/owner/repo.git');
+  console.log('  git@github.com:owner/repo.git');
+  console.log('  github.com/owner/repo');
+  console.log('  owner/repo\n');
+  console.log('Examples:');
+  console.log('  # Basic usage');
+  console.log('  ctxman github https://github.com/facebook/react\n');
+  console.log('  # Short format');
+  console.log('  ctxman github facebook/react\n');
+  console.log('  # Specific branch');
+  console.log('  ctxman github vercel/next.js --branch canary\n');
+  console.log('  # Custom output location');
+  console.log('  ctxman github angular/angular -o docs/angular-digest.txt\n');
+  console.log('  # With chunking');
+  console.log('  ctxman github microsoft/vscode --chunk-size 100000\n');
+  console.log('  # Keep clone for inspection');
+  console.log('  ctxman github nodejs/node --keep-clone --verbose\n');
+  console.log('Output:');
+  console.log('  Generated digest file in docs/ directory');
+  console.log('  Automatic cleanup of temporary clone (unless --keep-clone)\n');
 }
 
 main();