ctxman 3.1.0 → 3.1.1

package/README.md

@@ -322,7 +322,8 @@ ctxman mcp                                   # run the server (stdio)
 The installer merges into the client's configuration and keeps a `.bak` of the previous file.
 `--root` confines every tool to one directory. Tools: `analyze_codebase`, `generate_context`,
 `git_diff`, `search_code`, `list_methods`, `repo_map`, `pr_context`, `get_symbol`,
-`count_tokens`, `explain_history`, `pack_context`. Details:
+`count_tokens`, `explain_history`, `pack_context`, `check_policy`, `audit_log`, `ask_codebase`.
+Details:
 [docs/commands/mcp.md](docs/commands/mcp.md).
 
 ### 🛡️ Export Policy and Audit Log

package/bin/cm-ask.js

@@ -1,10 +1,5 @@
 #!/usr/bin/env node
-import { VectorStoreFactory } from '../lib/rag/VectorStoreFactory.js';
-import { TransformersEmbeddingProvider } from '../lib/rag/EmbeddingProvider.js';
-import { Indexer } from '../lib/rag/Indexer.js';
-import { FileDataSource } from '../lib/rag/sources/FileDataSource.js';
-import path from 'path';
-import fs from 'fs/promises';
+import { askCodebase } from '../lib/rag/ask.js';
 
 async function main() {
   const args = process.argv.slice(2);
@@ -15,86 +10,37 @@ async function main() {
     process.exit(1);
   }
 
-  // 1. Initialize RAG Components
-  let provider;
-  try {
-    // Try to load local transformers
-    provider = new TransformersEmbeddingProvider();
-    // Lazy init will handle model download on first use
-  } catch (e) {
-    console.error('⚠️ Failed to initialize embedding provider:', e.message);
-    process.exit(1);
-  }
-
-  // Use LanceDB in .ctxman/self-knowledge
-  const storePath = path.join(process.cwd(), '.ctxman', 'self-knowledge');
-  const store = VectorStoreFactory.create('lancedb', provider, { path: storePath });
-
-  // 2. Check if indexed
-  const isIndexed = await checkIndexExists(storePath);
-
-  if (!isIndexed) {
-    console.log('📚 Index not found. Auto-indexing current project...');
-    await indexProject(store);
-  }
-
-  // 3. Search
   console.log(`🔍 Searching for: "${query}"...`);
-  const results = await store.search(query, 5); // Get top 5
+  const { results, removedByPolicy } = await askCodebase(process.cwd(), query, {
+    onProgress: (event) => {
+      if (event === 'indexing') {
+        console.log('📚 Index not found. Auto-indexing current project...');
+        console.log('⏳ Indexing files (this may take a moment)...');
+      } else if (event === 'indexed') {
+        console.log('✅ Indexing complete.');
+      }
+    },
+  });
 
+  if (removedByPolicy > 0) {
+    console.log(`🛡️  Removed ${removedByPolicy} results denied by the export policy`);
+  }
   if (results.length === 0) {
     console.log('❌ No relevant context found.');
     return;
   }
 
-  // 4. Display Results (or feed to LLM if we had one configured)
   console.log(`\nFound ${results.length} relevant snippets:\n`);
-
   results.forEach((r, i) => {
-    console.log(`[${i + 1}] ${r.metadata.filename} (Score: ${r.score?.toFixed(4)})`);
+    console.log(`[${i + 1}] ${r.file} (Score: ${r.score?.toFixed(4)})`);
     console.log('─'.repeat(40));
-    // Show a small snippet around the match if possible, but we stored full text.
-    // Let's just show the first 200 chars or so.
     console.log(r.text.substring(0, 300).replace(/\n/g, ' ') + '...');
     console.log('\n');
   });
-
-  // Future: Pass `results` to an LLM API to generate a real answer.
-  // For now, this validates the retrieval pipeline works from CLI.
-}
-
-async function checkIndexExists(storePath) {
-  try {
-    await fs.access(storePath);
-    // Also check if it has tables (LanceDB creates folders)
-    const files = await fs.readdir(storePath);
-    return files.length > 0;
-  } catch {
-    return false;
-  }
-}
-
-async function indexProject(store) {
-  const indexer = new Indexer(store);
-  const fileSource = new FileDataSource();
-  indexer.registerSource(fileSource);
-
-  console.log('⏳ Indexing files (this may take a moment)...');
-  try {
-    await indexer.index('local-files', {
-      targetDir: process.cwd(),
-      include: ['.js', '.md', '.ts', '.json'], // Configurable?
-      exclude: ['node_modules', '.git', 'dist', 'coverage', '.ctxman'],
-    });
-  } catch (error) {
-    console.error(`❌ ${error.message}`);
-    process.exit(1);
-  }
-  console.log('✅ Indexing complete.');
 }
 
 main().catch((err) => {
-  // Message only: missing optional dependencies surface here with an install hint
+  // Message only: missing optional dependencies and indexing failures surface here
   console.error('❌ Error:', err.message);
   process.exit(1);
 });

package/bin/cm-audit.js

@@ -6,7 +6,7 @@
 
 import path from 'path';
 import {
-  loadPolicy,
+  auditLogFiles,
   readAuditLog,
   filterAuditEvents,
   formatAuditTable,
@@ -84,15 +84,12 @@ function main() {
 
 // The policy's audit logs; the default location when no policy names one
 function auditFiles() {
-  const root = process.cwd();
-  try {
-    const policy = loadPolicy(root);
-    if (policy.audit.paths.length > 0) return policy.audit.paths;
-  } catch (error) {
+  const { files, error } = auditLogFiles(process.cwd());
+  if (error) {
     console.error(`⚠️  ${error.message}`);
     console.error(`   Reading the default log ${DEFAULT_AUDIT_PATH} instead.`);
   }
-  return [path.join(root, DEFAULT_AUDIT_PATH)];
+  return files;
 }
 
 function valueOf(args, flag) {

package/examples/github-actions/ctxman-check.yml

@@ -24,7 +24,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           persist-credentials: false
           # changed-since needs the base commit of the pull request in the checkout

package/lib/analyzers/performance-dashboard.js

@@ -96,9 +96,13 @@ export class PerformanceDashboard {
       }
 
       const metricsFile = path.join(this.storagePath, 'metrics.json');
-      const data = fs.existsSync(metricsFile)
-        ? JSON.parse(fs.readFileSync(metricsFile, 'utf8'))
-        : [];
+      let data = [];
+      try {
+        data = JSON.parse(fs.readFileSync(metricsFile, 'utf8'));
+      } catch (error) {
+        // No metrics yet; an unreadable file stops the save, as before
+        if (error.code !== 'ENOENT') throw error;
+      }
 
       data.push(...this.metrics);
       fs.writeFileSync(metricsFile, JSON.stringify(data, null, 2));

package/lib/api/mcp/server.js

@@ -23,7 +23,16 @@ import { getSymbol } from '../../search/symbols.js';
 import { countForModel } from '../../count/index.js';
 import { explainHistory } from '../../why/index.js';
 import { packContext, describePack, TOOL_DEFAULT_BUDGET } from '../../pack/index.js';
-import { loadPolicy, guardExport } from '../../policy/index.js';
+import {
+  loadPolicy,
+  guardExport,
+  PolicyConfigError,
+  auditLogFiles,
+  readAuditLog,
+  filterAuditEvents,
+  parseSince,
+} from '../../policy/index.js';
+import { askCodebase, DEFAULT_ASK_LIMIT } from '../../rag/ask.js';
 import fs from 'fs';
 import path from 'path';
 import { createRequire } from 'module';
@@ -33,6 +42,10 @@ const { version } = createRequire(import.meta.url)('../../../package.json');
 // Tools whose `path` may also name a single file
 const FILE_PATH_TOOLS = new Set(['list_methods']);
 
+const DEFAULT_AUDIT_LIMIT = 50;
+// Characters of each ask_codebase snippet (the index stores whole files)
+const ASK_SNIPPET_CHARS = 2000;
+
 class MCPServer {
   /**
    * @param {Object} [options]
@@ -340,6 +353,56 @@ class MCPServer {
               required: ['path'],
             },
           },
+          {
+            name: 'check_policy',
+            description:
+              'The effective export policy of a project (.ctxman/policy.json plus the organisation policy named by CTXMAN_POLICY): denied paths, redaction and token limits, allowed target models, AI-tool ignore files and audit logging. Reports whether the policy files are valid and, if not, every error.',
+            inputSchema: {
+              type: 'object',
+              properties: {
+                path: { type: 'string', description: 'Project root' },
+              },
+              required: ['path'],
+            },
+          },
+          {
+            name: 'audit_log',
+            description:
+              "Exports recorded in the project's audit log while the export policy enables audit: when, by whom, which command, where to, which files, token count and a SHA-256 of the exported text. Newest last.",
+            inputSchema: {
+              type: 'object',
+              properties: {
+                path: { type: 'string', description: 'Project root' },
+                since: {
+                  type: 'string',
+                  description:
+                    'Only exports at or after this date (2026-10-01), ISO time or age (30m, 12h, 7d, 2w)',
+                },
+                limit: {
+                  type: 'number',
+                  description: `The last N events (default ${DEFAULT_AUDIT_LIMIT}; 0 = all)`,
+                },
+              },
+              required: ['path'],
+            },
+          },
+          {
+            name: 'ask_codebase',
+            description:
+              'Semantic search: the files whose content is closest in meaning to a question, from a local embedding index (.ctxman/self-knowledge, built on first use; the first call downloads the embedding model and can take minutes). Needs the optional @xenova/transformers and @lancedb/lancedb packages. For exact names or keywords, search_code is faster.',
+            inputSchema: {
+              type: 'object',
+              properties: {
+                path: { type: 'string', description: 'Project root' },
+                query: { type: 'string', description: 'The question, in natural language' },
+                limit: {
+                  type: 'number',
+                  description: `How many files (default ${DEFAULT_ASK_LIMIT})`,
+                },
+              },
+              required: ['path', 'query'],
+            },
+          },
         ],
       };
     });
@@ -666,6 +729,40 @@ class MCPServer {
               ],
             };
           }
+          case 'check_policy': {
+            let result;
+            try {
+              result = { valid: true, ...loadPolicy(args.path).toJSON() };
+            } catch (error) {
+              if (!(error instanceof PolicyConfigError)) throw error;
+              result = { valid: false, file: error.file, errors: error.errors };
+            }
+            return { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }] };
+          }
+          case 'audit_log': {
+            const since = args.since === undefined ? null : parseSince(String(args.since));
+            const limit =
+              Number.isInteger(args.limit) && args.limit >= 0 ? args.limit : DEFAULT_AUDIT_LIMIT;
+            // Only the policy's own logs: a tool argument never names a file to read
+            const { files, error } = auditLogFiles(args.path);
+            const { events, skipped, missing } = readAuditLog(files);
+            const matching = filterAuditEvents(events, { since });
+            const result = {
+              files: files.filter((file) => !missing.includes(file)),
+              total: matching.length,
+              events: filterAuditEvents(matching, { limit }),
+              skippedLines: skipped,
+            };
+            if (error) result.policyError = error.message;
+            return { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }] };
+          }
+          case 'ask_codebase': {
+            const result = await askCodebase(args.path, args.query, {
+              limit: Number.isInteger(args.limit) ? args.limit : undefined,
+              maxChars: ASK_SNIPPET_CHARS,
+            });
+            return { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }] };
+          }
           default:
             throw new Error(`Unknown tool: ${name}`);
         }

package/lib/parsers/gitignore-parser.js

@@ -43,7 +43,10 @@ class GitIgnoreParser {
     } catch (error) {
       // A directory, or a file that doesn't exist or can't be read
       if (error.code === 'EISDIR') {
-        console.warn(`Warning: ${filePath} is a directory, expected a file`);
+        // Quoted and escaped: a line break in the path cannot start a line of its own
+        console.warn(
+          `Warning: ${JSON.stringify(String(filePath))} is a directory, expected a file`
+        );
       }
       return [];
     }

package/lib/policy/audit.js

@@ -12,7 +12,7 @@ import os from 'os';
 import path from 'path';
 import { execFileSync } from 'child_process';
 import { createRequire } from 'module';
-import { relativePathOf } from './policy.js';
+import { relativePathOf, loadPolicy, DEFAULT_AUDIT_PATH } from './policy.js';
 
 const { version: CTXMAN_VERSION } = createRequire(import.meta.url)('../../package.json');
 
@@ -167,6 +167,24 @@ export function recordExport(policy, event = {}, { warn = defaultWarn } = {}) {
   return record;
 }
 
+/**
+ * The audit logs of a project's policy, or the default location when the policy names none
+ * @param {string} root - Project root
+ * @param {Object} [options]
+ * @param {object} [options.env=process.env] - Environment for loading the policy
+ * @returns {{files: string[], error: Error|null}} `error` when the policy could not be
+ *   loaded (the default location is returned then)
+ */
+export function auditLogFiles(root, { env = process.env } = {}) {
+  try {
+    const policy = loadPolicy(root, { env });
+    if (policy.audit.paths.length > 0) return { files: policy.audit.paths, error: null };
+    return { files: [path.join(root, DEFAULT_AUDIT_PATH)], error: null };
+  } catch (error) {
+    return { files: [path.join(root, DEFAULT_AUDIT_PATH)], error };
+  }
+}
+
 /**
  * Read audit logs. Lines that are not JSON objects are skipped and counted; events that
  * appear in several logs (organisation and project) are listed once.

package/lib/policy/index.js

@@ -38,6 +38,7 @@ export {
   recordExport,
   buildAuditRecord,
   readAuditLog,
+  auditLogFiles,
   filterAuditEvents,
   formatAuditTable,
   parseSince,

package/lib/rag/ask.js

@@ -0,0 +1,122 @@
+/**
+ * Semantic search over a project (`ctxman ask` and the MCP `ask_codebase` tool): an
+ * embedding index under .ctxman/self-knowledge, built on first use, then the snippets
+ * closest to the question. Needs the optional @xenova/transformers and @lancedb/lancedb
+ * packages; the embedding model is downloaded on first use.
+ *
+ * Files left out by .gitignore, .contextignore, .contextinclude or the export policy are
+ * not indexed, and results from an older index are checked against the policy again.
+ */
+
+import fs from 'fs/promises';
+import path from 'path';
+import { Scanner } from '../core/Scanner.js';
+import { redactSecrets } from '../utils/secret-redactor.js';
+import { VectorStoreFactory } from './VectorStoreFactory.js';
+import { TransformersEmbeddingProvider } from './EmbeddingProvider.js';
+import { Indexer } from './Indexer.js';
+import { FileDataSource } from './sources/FileDataSource.js';
+
+export const ASK_STORE_DIR = path.join('.ctxman', 'self-knowledge');
+export const DEFAULT_ASK_LIMIT = 5;
+
+const INCLUDE = ['.js', '.md', '.ts', '.json'];
+const EXCLUDE = ['node_modules', '.git', 'dist', 'coverage', '.ctxman'];
+// Extra rows asked of the store, so results the policy removes still leave `limit`
+const OVERFETCH = 10;
+
+async function hasIndex(storePath) {
+  try {
+    return (await fs.readdir(storePath)).length > 0;
+  } catch {
+    return false;
+  }
+}
+
+const toPosix = (p) => p.split(path.sep).join('/');
+
+/**
+ * @param {string} root - Project directory
+ * @param {string} query - The question
+ * @param {Object} [options]
+ * @param {number} [options.limit=5] - Snippets to return
+ * @param {boolean} [options.redact=true] - Redact secrets in the snippets
+ * @param {number} [options.maxChars] - Cut each snippet to this many characters
+ * @param {(event: 'indexing'|'indexed') => void} [options.onProgress]
+ * @param {(storePath: string) => object} [options.createStore] - Vector store factory
+ *   (tests); default: LanceDB with the local embedding model
+ * @returns {Promise<{query: string, indexed: boolean, results: Array<{file: string,
+ *   score: number|null, text: string, truncated: boolean}>, removedByPolicy: number,
+ *   redactions: number}>}
+ * @throws {Error} When the question is empty, the policy is invalid, the optional packages
+ *   are missing or indexing fails
+ */
+export async function askCodebase(root, query, options = {}) {
+  const question = typeof query === 'string' ? query.trim() : '';
+  if (!question) throw new Error('A question is required');
+  const limit =
+    Number.isInteger(options.limit) && options.limit > 0 ? options.limit : DEFAULT_ASK_LIMIT;
+  const projectRoot = path.resolve(root);
+  const storePath = path.join(projectRoot, ASK_STORE_DIR);
+  const onProgress = options.onProgress || (() => {});
+
+  const scanner = new Scanner(projectRoot);
+  // Load now: an invalid policy fails the command instead of being skipped
+  scanner.getPolicy();
+  const relative = (fullPath) => toPosix(path.relative(projectRoot, fullPath));
+  const left = (fullPath) => {
+    const rel = relative(fullPath);
+    return rel.startsWith('../') || path.isAbsolute(rel);
+  };
+
+  const store = options.createStore
+    ? options.createStore(storePath)
+    : VectorStoreFactory.create('lancedb', new TransformersEmbeddingProvider(), {
+        path: storePath,
+      });
+
+  let indexed = false;
+  if (!(await hasIndex(storePath))) {
+    onProgress('indexing');
+    const indexer = new Indexer(store);
+    indexer.registerSource(new FileDataSource());
+    await indexer.index('local-files', {
+      targetDir: projectRoot,
+      include: INCLUDE,
+      exclude: EXCLUDE,
+      skip: (fullPath) =>
+        scanner.shouldIgnore(path.relative(projectRoot, fullPath), fullPath) ||
+        scanner.isDenied(relative(fullPath)),
+    });
+    indexed = true;
+    onProgress('indexed');
+  }
+
+  const rows = await store.search(question, limit + OVERFETCH);
+  const results = [];
+  let removedByPolicy = 0;
+  let redactions = 0;
+  for (const row of rows) {
+    const fullPath = row.metadata?.path;
+    if (typeof fullPath !== 'string' || left(fullPath)) continue;
+    const file = relative(fullPath);
+    if (scanner.isDenied(file)) {
+      removedByPolicy++;
+      continue;
+    }
+    if (results.length === limit) continue;
+    let text = String(row.text ?? '');
+    if (options.redact !== false) {
+      const redacted = redactSecrets(text);
+      text = redacted.text;
+      redactions += redacted.findings.length;
+    }
+    const truncated = Boolean(options.maxChars) && text.length > options.maxChars;
+    if (truncated) text = text.slice(0, options.maxChars);
+    results.push({ file, score: row.score ?? null, text, truncated });
+  }
+
+  return { query: question, indexed, results, removedByPolicy, redactions };
+}
+
+export default askCodebase;

package/lib/rag/sources/FileDataSource.js

@@ -19,11 +19,13 @@ export class FileDataSource extends DataSourcePlugin {
     const maxDepth = options.maxDepth || 5;
     const include = options.include || ['.js', '.md', '.ts', '.json'];
     const exclude = options.exclude || ['node_modules', '.git', 'dist', 'coverage'];
+    // (fullPath, isDirectory) => true leaves a file or directory out
+    const skip = options.skip || (() => false);
 
-    yield* this.walkDirectory(targetDir, maxDepth, include, exclude);
+    yield* this.walkDirectory(targetDir, maxDepth, include, exclude, skip);
   }
 
-  async *walkDirectory(dir, depth, include, exclude) {
+  async *walkDirectory(dir, depth, include, exclude, skip = () => false) {
     if (depth < 0) return;
 
     let entries;
@@ -38,9 +40,10 @@ export class FileDataSource extends DataSourcePlugin {
       const fullPath = path.join(dir, entry.name);
 
       if (exclude.some((ex) => fullPath.includes(ex))) continue;
+      if (skip(fullPath, entry.isDirectory())) continue;
 
       if (entry.isDirectory()) {
-        yield* this.walkDirectory(fullPath, depth - 1, include, exclude);
+        yield* this.walkDirectory(fullPath, depth - 1, include, exclude, skip);
       } else if (entry.isFile()) {
         const ext = path.extname(entry.name);
         if (include.length > 0 && !include.includes(ext)) continue;

package/lib/utils/logger.js

@@ -94,13 +94,16 @@ class Logger {
   formatMessage(level, message, meta = {}) {
     const timestamp = this.getTimestamp();
     const icon = this.icons[level];
+    // One entry per line: line breaks in the message (a path, an error text) are written
+    // escaped, so the message cannot add entries of its own
+    const text = String(message).replace(/\r\n|\r|\n/g, '\\n');
 
     // Console format (with colors)
-    const consoleMessage = `${this.colors[level]}${icon} [${level.toUpperCase()}]${this.colors.reset} ${message}`;
+    const consoleMessage = `${this.colors[level]}${icon} [${level.toUpperCase()}]${this.colors.reset} ${text}`;
 
     // File format (plain text with metadata)
     const metaString = Object.keys(meta).length > 0 ? ` | ${JSON.stringify(meta)}` : '';
-    const fileMessage = `[${timestamp}] [${level.toUpperCase()}] ${message}${metaString}`;
+    const fileMessage = `[${timestamp}] [${level.toUpperCase()}] ${text}${metaString}`;
 
     return { consoleMessage, fileMessage };
   }

package/lib/wizards/config-generator.js

@@ -81,37 +81,39 @@ class ConfigGenerator {
       }
 
       const filePath = path.join(this.projectRoot, filename);
-      const exists = fs.existsSync(filePath);
-
-      // Check if should skip
-      if (exists && !force) {
+      const skip = () =>
         result.skipped.push({
           type,
           filename,
           reason: 'File already exists (use --force to overwrite)',
         });
-        continue;
-      }
-
-      // Generate file content
-      const content = this.generateFileContent(type, patterns);
 
       if (dryRun) {
-        if (exists) {
-          result.overwritten.push({ type, filename, content });
-        } else {
-          result.created.push({ type, filename, content });
+        const exists = fs.existsSync(filePath);
+        if (exists && !force) {
+          skip();
+          continue;
         }
+        const content = this.generateFileContent(type, patterns);
+        (exists ? result.overwritten : result.created).push({ type, filename, content });
         continue;
       }
 
-      // Write file
+      const content = this.generateFileContent(type, patterns);
+      // Create exclusively first, so an existing file is found by the write itself rather
+      // than by a separate check that could be out of date
       try {
-        fs.writeFileSync(filePath, content, 'utf-8');
-        if (exists) {
-          result.overwritten.push({ type, filename });
-        } else {
+        try {
+          fs.writeFileSync(filePath, content, { encoding: 'utf-8', flag: 'wx' });
           result.created.push({ type, filename });
+        } catch (error) {
+          if (error.code !== 'EEXIST') throw error;
+          if (!force) {
+            skip();
+            continue;
+          }
+          fs.writeFileSync(filePath, content, 'utf-8');
+          result.overwritten.push({ type, filename });
         }
       } catch (error) {
         result.errors.push({

package/lib/wizards/project-detector.js

@@ -514,7 +514,8 @@ class ProjectDetector {
    * @returns {boolean} True if matches
    */
   matchesGlob(filename, pattern) {
-    const regex = pattern.replace(/\./g, '\\.').replace(/\*/g, '.*');
+    // Every regex character is literal except `*`, which matches any run of characters
+    const regex = pattern.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*');
     return new RegExp(`^${regex}$`).test(filename);
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ctxman",
-  "version": "3.1.0",
+  "version": "3.1.1",
   "description": "AI Development Platform with plugin architecture, Git integration, REST API, watch mode, LLM optimization, and support for 14+ languages. Features include TOON format (40-50% token reduction), real-time analysis, and comprehensive context generation.",
   "type": "module",
   "main": "index.js",
@@ -83,7 +83,7 @@
   ],
   "author": "Hakkı Sağdıç",
   "license": "MIT",
-  "homepage": "https://github.com/hakkisagdic/ctxman#readme",
+  "homepage": "https://hakkisagdic.github.io/ctxman/",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/hakkisagdic/ctxman.git"