ctxman 3.1.0 → 3.2.7

package/lib/pr/pr-renderer.js

@@ -140,6 +140,9 @@ const linesLabel = (item) =>
 export function summaryFacts(model) {
   const { stats } = model;
   const facts = [];
+  if (Array.isArray(model.repos) && model.repos.length > 0) {
+    facts.push(['Repositories', model.repos.map((r) => r.alias).join(', ')]);
+  }
   if (model.mode === 'range') {
     facts.push(['Base', `${model.base.ref} (${model.base.short})`]);
     facts.push(['Head', `${model.head.ref} (${model.head.short})`]);
@@ -206,7 +209,8 @@ const markdown = {
   },
 
   commit(commit, model) {
-    let text = `### ${code(commit.short)} ${commit.subject}\n${commit.author}, ${commit.date}`;
+    const repoBadge = commit.repo ? `[${commit.repo}] ` : '';
+    let text = `### ${code(commit.short)} ${repoBadge}${commit.subject}\n${commit.author}, ${commit.date}`;
     if (commit.body && !model.logBodiesDropped) {
       text += `\n\n${commit.body
         .split('\n')
@@ -229,7 +233,7 @@ const markdown = {
     const rows = model.files.map((file) => {
       const name =
         file.oldPath && file.oldPath !== file.path && (file.status === 'R' || file.status === 'C')
-          ? `${code(file.oldPath)} → ${code(file.path)}`
+          ? `${code(file.oldPath)} \u2192 ${code(file.path)}`
           : code(file.path);
       const note = file.excluded ? ` (content excluded by ${file.excluded})` : '';
       const added = file.binary ? 'binary' : file.additions;
@@ -344,8 +348,9 @@ function createXml() {
     },
 
     commit(commit, model) {
+      const repoAttr = commit.repo ? ` repo="${escapeAttr(commit.repo)}"` : '';
       const body = commit.body && !model.logBodiesDropped ? `\n\n${commit.body}` : '';
-      return `<commit sha="${commit.short}" author="${escapeAttr(commit.author)}" date="${commit.date}">\n${commit.subject}${body}\n</commit>`;
+      return `<commit sha="${commit.short}"${repoAttr} author="${escapeAttr(commit.author)}" date="${commit.date}">\n${commit.subject}${body}\n</commit>`;
     },
 
     commits(model) {
@@ -468,6 +473,7 @@ function createXml() {
 const json = {
   summary(model) {
     const summary = {
+      ...(Array.isArray(model.repos) && model.repos.length > 0 ? { repos: model.repos } : {}),
       mode: model.mode,
       base: model.base,
       head: model.head,
@@ -483,10 +489,11 @@ const json = {
   },
 
   commit(commit, model) {
-    const { sha, short, author, date, subject, body } = commit;
-    return model.logBodiesDropped
-      ? { sha, short, author, date, subject }
-      : { sha, short, author, date, subject, body };
+    const { sha, short, author, date, subject, body, repo } = commit;
+    const base = repo
+      ? { repo, sha, short, author, date, subject }
+      : { sha, short, author, date, subject };
+    return model.logBodiesDropped ? base : { ...base, body };
   },
 
   commits(model) {
@@ -497,6 +504,7 @@ const json = {
   files(model) {
     if (model.filesDropped) return null;
     return model.files.map((file) => ({
+      ...(file.repo ? { repo: file.repo } : {}),
       status: file.status,
       path: file.path,
       oldPath: file.oldPath,
@@ -510,8 +518,9 @@ const json = {
   },
 
   method(method) {
-    const { file, name, start, end, change, changedLines } = method;
+    const { file, name, start, end, change, changedLines, repo } = method;
     return {
+      ...(repo ? { repo } : {}),
       file,
       name,
       start,
@@ -527,7 +536,8 @@ const json = {
     if (model.methods.length === 0 && model.deletedMethods.length === 0) return null;
     return {
       changed: model.methods.map(json.method),
-      deleted: model.deletedMethods.map(({ file, name, start, end }) => ({
+      deleted: model.deletedMethods.map(({ file, name, start, end, repo }) => ({
+        ...(repo ? { repo } : {}),
         file,
         name,
         start,
@@ -548,7 +558,12 @@ const json = {
   },
 
   testLine(test) {
-    const entry = { path: test.path, reasons: test.reasons, changed: test.changed };
+    const entry = {
+      ...(test.repo ? { repo: test.repo } : {}),
+      path: test.path,
+      reasons: test.reasons,
+      changed: test.changed,
+    };
     if (test.excluded) entry.contentExcludedBy = test.excluded;
     return entry;
   },

package/lib/rag/EmbeddingProvider.js

@@ -5,13 +5,103 @@ export class EmbeddingProvider {
   /**
    * Generates an embedding for the given text.
    * @param {string} text
-   * @returns {Promise<number[]>}
-   */
+   * @returns {Promise<number[]>}\n   */
   async embed(_text) {
     throw new Error('Not implemented');
   }
 }
 
+/**
+ * Deterministic, zero-dependency local embedding provider (256 dimensions).
+ * Uses feature-hashing over identifier tokens and character trigrams with L2 normalization.
+ * Requires zero external packages and works 100% offline.
+ */
+export class LocalHashEmbeddingProvider extends EmbeddingProvider {
+  constructor(dimension = 256) {
+    super();
+    this.dimension = dimension;
+  }
+
+  /**
+   * 32-bit FNV-1a hash function with optional seed.
+   * @private
+   */
+  _hash(str, seed = 0x811c9dc5) {
+    let h = seed >>> 0;
+    for (let i = 0; i < str.length; i++) {
+      h ^= str.charCodeAt(i);
+      h = Math.imul(h, 0x01000193) >>> 0;
+    }
+    return h >>> 0;
+  }
+
+  /**
+   * Generates a deterministic 256-dim L2-normalized embedding for the given text.
+   * @param {string} text
+   * @returns {Promise<number[]>}
+   */
+  async embed(text) {
+    const raw = String(text ?? '');
+    const vector = new Float64Array(this.dimension);
+
+    // 1. Extract identifier tokens
+    const tokens = raw.match(/[a-zA-Z0-9_$]+/g) || [];
+    for (const token of tokens) {
+      const lower = token.toLowerCase();
+      // Hash whole token
+      const h1 = this._hash(lower, 0x12345678);
+      const idx1 = h1 % this.dimension;
+      const sign1 = (this._hash(lower, 0x87654321) & 1) === 0 ? 1 : -1;
+      vector[idx1] += sign1 * 1.5;
+
+      // Split camelCase/snake_case sub-tokens if composite
+      const subTokens = token.replace(/([a-z])([A-Z])/g, '$1 $2').split(/[_\s]+/);
+      if (subTokens.length > 1) {
+        for (const sub of subTokens) {
+          if (sub.length > 1) {
+            const subLower = sub.toLowerCase();
+            const hSub = this._hash(subLower, 0x2468ace0);
+            const idxSub = hSub % this.dimension;
+            const signSub = (this._hash(subLower, 0x0eca8642) & 1) === 0 ? 1 : -1;
+            vector[idxSub] += signSub * 1.0;
+          }
+        }
+      }
+    }
+
+    // 2. Extract character trigrams
+    const normalized = raw.toLowerCase().replace(/\s+/g, ' ');
+    if (normalized.length >= 3) {
+      for (let i = 0; i <= normalized.length - 3; i++) {
+        const trigram = normalized.slice(i, i + 3);
+        const hTri = this._hash(trigram, 0x13579bdf);
+        const idxTri = hTri % this.dimension;
+        const signTri = (this._hash(trigram, 0xfdb97531) & 1) === 0 ? 1 : -1;
+        vector[idxTri] += signTri * 1.0;
+      }
+    } else if (normalized.length > 0) {
+      const hShort = this._hash(normalized, 0x99887766);
+      const idxShort = hShort % this.dimension;
+      const signShort = (this._hash(normalized, 0x66778899) & 1) === 0 ? 1 : -1;
+      vector[idxShort] += signShort * 1.0;
+    }
+
+    // 3. L2 normalize
+    let sumSq = 0;
+    for (let i = 0; i < this.dimension; i++) {
+      sumSq += vector[i] * vector[i];
+    }
+    const norm = Math.sqrt(sumSq);
+    if (norm > 0) {
+      for (let i = 0; i < this.dimension; i++) {
+        vector[i] /= norm;
+      }
+    }
+
+    return Array.from(vector);
+  }
+}
+
 /**
  * Uses @xenova/transformers to generate embeddings locally using ONNX models.
  * Default model: Xenova/all-MiniLM-L6-v2 (384 dimensions)
@@ -53,6 +143,53 @@ export class TransformersEmbeddingProvider extends EmbeddingProvider {
   }
 }
 
+/**
+ * Tries TransformersEmbeddingProvider if @xenova/transformers is installed
+ * and CTXMAN_EMBEDDER !== 'local'; otherwise seamlessly falls back to
+ * LocalHashEmbeddingProvider without network or model loading failure.
+ */
+export class AutoEmbeddingProvider extends EmbeddingProvider {
+  /**
+   * @param {object} [options]
+   * @param {string} [options.modelName]
+   */
+  constructor(options = {}) {
+    super();
+    this.modelName = options.modelName;
+    this.activeProvider = null;
+  }
+
+  async init() {
+    if (this.activeProvider) return;
+
+    if (process.env.CTXMAN_EMBEDDER === 'local') {
+      this.activeProvider = new LocalHashEmbeddingProvider();
+      return;
+    }
+
+    try {
+      await import('@xenova/transformers');
+    } catch (err) {
+      if (err.code === 'ERR_MODULE_NOT_FOUND') {
+        this.activeProvider = new LocalHashEmbeddingProvider();
+        return;
+      }
+      throw err;
+    }
+
+    const provider = new TransformersEmbeddingProvider(this.modelName);
+    await provider.init();
+    this.activeProvider = provider;
+  }
+
+  async embed(text) {
+    if (!this.activeProvider) {
+      await this.init();
+    }
+    return this.activeProvider.embed(text);
+  }
+}
+
 /**
  * Mock provider for testing without loading heavy models.
  */

package/lib/rag/VectorStoreFactory.js

@@ -1,15 +1,19 @@
+import { SqliteVectorStore } from './stores/SqliteVectorStore.js';
 import { LanceDBStore } from './stores/LanceDBStore.js';
 import { ChromaDBStore } from './stores/ChromaDBStore.js';
 import { PineconeStore } from './stores/PineconeStore.js';
 
 export class VectorStoreFactory {
   /**
-   * @param {string} type
+   * @param {string} [type='sqlite']
    * @param {import('./EmbeddingProvider').EmbeddingProvider} embeddingProvider
-   * @param {Object} config
+   * @param {Object} [config={}]
    */
-  static create(type, embeddingProvider, config = {}) {
-    switch (type.toLowerCase()) {
+  static create(type = 'sqlite', embeddingProvider, config = {}) {
+    const storeType = (type || 'sqlite').toLowerCase();
+    switch (storeType) {
+      case 'sqlite':
+        return new SqliteVectorStore(embeddingProvider, config.path || config.storagePath);
       case 'lancedb':
         return new LanceDBStore(embeddingProvider, config.path);
       case 'chroma':
@@ -22,3 +26,5 @@ export class VectorStoreFactory {
     }
   }
 }
+
+export default VectorStoreFactory;

package/lib/rag/ask.js

@@ -0,0 +1,291 @@
+/**
+ * Semantic search over a project (`ctxman ask` and the MCP `ask_codebase` tool): an
+ * embedding index under .ctxman/self-knowledge, built on first use, then the snippets
+ * closest to the question. By default uses the built-in zero-native-binary SqliteVectorStore
+ * and AutoEmbeddingProvider (100% offline, zero optional dependencies). Optional LanceDB
+ * and @xenova/transformers can also be used if configured.
+ *
+ * Files left out by .gitignore, .contextignore, .contextinclude or the export policy are
+ * not indexed, and results from an older index are checked against the policy again.
+ */
+
+import fsSync from 'fs';
+import fs from 'fs/promises';
+import path from 'path';
+import { Scanner } from '../core/Scanner.js';
+import { redactSecrets } from '../utils/secret-redactor.js';
+import { VectorStoreFactory } from './VectorStoreFactory.js';
+import { AutoEmbeddingProvider } from './EmbeddingProvider.js';
+import { Indexer } from './Indexer.js';
+import { FileDataSource } from './sources/FileDataSource.js';
+import { resolveWorkspaceRepos, enforceWorkspacePolicies } from '../workspace/index.js';
+
+export const ASK_STORE_DIR = path.join('.ctxman', 'self-knowledge');
+export const DEFAULT_ASK_LIMIT = 5;
+
+const INCLUDE = ['.js', '.md', '.ts', '.json'];
+const EXCLUDE = ['node_modules', '.git', 'dist', 'coverage', '.ctxman'];
+// Extra rows asked of the store, so results the policy removes still leave `limit`
+const OVERFETCH = 10;
+
+async function hasIndex(storePath, store) {
+  try {
+    if ((await fs.readdir(storePath)).length === 0) return false;
+    if (store && typeof store.count === 'function') {
+      if (typeof store.init === 'function' && !store.initialized) {
+        await store.init();
+      }
+      return store.count() > 0;
+    }
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+const toPosix = (p) => p.split(path.sep).join('/');
+
+function buildWorkspaceRepoInputs(primaryRoot, extraRepos) {
+  const realPrimary = fsSync.existsSync(primaryRoot)
+    ? fsSync.realpathSync(primaryRoot)
+    : path.resolve(primaryRoot);
+  const isContainerOfExtra = extraRepos.some((item) => {
+    const raw = typeof item === 'string' ? item : item?.path;
+    if (typeof raw !== 'string' || !raw.trim()) return false;
+    const resolved = path.resolve(raw);
+    const real = fsSync.existsSync(resolved) ? fsSync.realpathSync(resolved) : resolved;
+    return real !== realPrimary && real.startsWith(realPrimary + path.sep);
+  });
+  return isContainerOfExtra ? [...extraRepos] : [primaryRoot, ...extraRepos];
+}
+
+/**
+ * @param {string} root - Project directory
+ * @param {string} query - The question
+ * @param {Object} [options]
+ * @param {number} [options.limit=5] - Snippets to return
+ * @param {boolean} [options.redact=true] - Redact secrets in the snippets
+ * @param {number} [options.maxChars] - Cut each snippet to this many characters
+ * @param {Array<string|{path: string, alias?: string}>} [options.repos] - Additional repository roots for multi-repo semantic search
+ * @param {(event: 'indexing'|'indexed') => void} [options.onProgress]
+ * @param {(storePath: string, repo?: object) => object} [options.createStore] - Vector store factory
+ *   (tests); default: SqliteVectorStore with AutoEmbeddingProvider
+ * @returns {Promise<{query: string, indexed: boolean, repos?: Array<{alias: string, root: string}>,
+ *   results: Array<{repo?: string, file: string, score: number|null, text: string, truncated: boolean}>,
+ *   removedByPolicy: number, redactions: number}>}
+ * @throws {Error} When the question is empty, the policy is invalid, or indexing fails
+ */
+export async function askCodebase(root, query, options = {}) {
+  const question = typeof query === 'string' ? query.trim() : '';
+  if (!question) throw new Error('A question is required');
+  const limit =
+    Number.isInteger(options.limit) && options.limit > 0 ? options.limit : DEFAULT_ASK_LIMIT;
+  const projectRoot = path.resolve(root);
+  const onProgress = options.onProgress || (() => {});
+  const isWorkspace = Array.isArray(options.repos) && options.repos.length > 0;
+
+  if (isWorkspace) {
+    const resolvedRepos = resolveWorkspaceRepos(
+      buildWorkspaceRepoInputs(projectRoot, options.repos)
+    );
+    enforceWorkspacePolicies(resolvedRepos, {
+      filesByRepo: new Map(),
+      tokens: 0,
+      redact: options.redact !== false,
+    });
+
+    const scannersByAlias = new Map();
+    const repoByAlias = new Map();
+    for (const repo of resolvedRepos) {
+      const scanner = new Scanner(repo.root);
+      scanner.getPolicy();
+      scannersByAlias.set(repo.alias, scanner);
+      repoByAlias.set(repo.alias, repo);
+    }
+
+    let indexedAny = false;
+    let startedIndexing = false;
+    let removedByPolicy = 0;
+    const candidatesByFile = new Map();
+
+    for (const repo of resolvedRepos) {
+      const repoStorePath = path.join(repo.root, ASK_STORE_DIR);
+      const repoScanner = scannersByAlias.get(repo.alias);
+      const relativeToRepo = (targetRepo, fullPath) =>
+        toPosix(path.relative(targetRepo.root, fullPath));
+      const leftOfRepo = (targetRepo, fullPath) => {
+        const rel = relativeToRepo(targetRepo, fullPath);
+        return rel.startsWith('../') || path.isAbsolute(rel);
+      };
+
+      const store = options.createStore
+        ? options.createStore(repoStorePath, repo)
+        : VectorStoreFactory.create('sqlite', new AutoEmbeddingProvider(), {
+            path: repoStorePath,
+          });
+
+      if (!(await hasIndex(repoStorePath, store))) {
+        if (!startedIndexing) {
+          onProgress('indexing');
+          startedIndexing = true;
+        }
+        const indexer = new Indexer(store);
+        indexer.registerSource(new FileDataSource());
+        await indexer.index('local-files', {
+          targetDir: repo.root,
+          include: INCLUDE,
+          exclude: EXCLUDE,
+          skip: (fullPath) =>
+            repoScanner.shouldIgnore(path.relative(repo.root, fullPath), fullPath) ||
+            repoScanner.isDenied(relativeToRepo(repo, fullPath)),
+        });
+        indexedAny = true;
+      }
+
+      const rows = await store.search(question, limit + OVERFETCH);
+      for (const row of rows) {
+        const rawPath = row.metadata?.path ?? row.metadata?.relativePath;
+        if (typeof rawPath !== 'string') continue;
+
+        let targetRepo = repo;
+        let resolvedFullPath;
+        const posixRaw = toPosix(rawPath);
+        const slashIdx = posixRaw.indexOf('/');
+        const maybeAlias =
+          row.metadata?.repo || (slashIdx > 0 ? posixRaw.slice(0, slashIdx) : null);
+
+        if (
+          maybeAlias &&
+          repoByAlias.has(maybeAlias) &&
+          !path.isAbsolute(rawPath) &&
+          posixRaw.startsWith(`${maybeAlias}/`)
+        ) {
+          targetRepo = repoByAlias.get(maybeAlias);
+          resolvedFullPath = path.resolve(targetRepo.root, posixRaw.slice(maybeAlias.length + 1));
+        } else {
+          resolvedFullPath = path.isAbsolute(rawPath) ? rawPath : path.resolve(repo.root, rawPath);
+        }
+
+        if (leftOfRepo(targetRepo, resolvedFullPath)) continue;
+        const relFile = relativeToRepo(targetRepo, resolvedFullPath);
+        const targetScanner = scannersByAlias.get(targetRepo.alias);
+        if (targetScanner.isDenied(relFile)) {
+          removedByPolicy++;
+          continue;
+        }
+
+        const prefixedFile = `${targetRepo.alias}/${relFile}`;
+        const existing = candidatesByFile.get(prefixedFile);
+        const score = row.score ?? null;
+        if (!existing || (score ?? -Infinity) > (existing.score ?? -Infinity)) {
+          candidatesByFile.set(prefixedFile, {
+            repo: targetRepo.alias,
+            file: prefixedFile,
+            score,
+            rawText: String(row.text ?? ''),
+          });
+        }
+      }
+    }
+
+    if (startedIndexing) {
+      onProgress('indexed');
+    }
+
+    const sortedCandidates = [...candidatesByFile.values()].sort(
+      (a, b) => (b.score ?? -Infinity) - (a.score ?? -Infinity) || a.file.localeCompare(b.file)
+    );
+
+    const results = [];
+    let redactions = 0;
+    for (const cand of sortedCandidates.slice(0, limit)) {
+      let text = cand.rawText;
+      if (options.redact !== false) {
+        const redacted = redactSecrets(text);
+        text = redacted.text;
+        redactions += redacted.findings.length;
+      }
+      const truncated = Boolean(options.maxChars) && text.length > options.maxChars;
+      if (truncated) text = text.slice(0, options.maxChars);
+      results.push({
+        repo: cand.repo,
+        file: cand.file,
+        score: cand.score,
+        text,
+        truncated,
+      });
+    }
+
+    return {
+      query: question,
+      indexed: indexedAny,
+      repos: resolvedRepos.map(({ alias, root: r }) => ({ alias, root: r })),
+      results,
+      removedByPolicy,
+      redactions,
+    };
+  }
+
+  const storePath = path.join(projectRoot, ASK_STORE_DIR);
+  const scanner = new Scanner(projectRoot);
+  // Load now: an invalid policy fails the command instead of being skipped
+  scanner.getPolicy();
+  const relative = (fullPath) => toPosix(path.relative(projectRoot, fullPath));
+  const left = (fullPath) => {
+    const rel = relative(fullPath);
+    return rel.startsWith('../') || path.isAbsolute(rel);
+  };
+
+  const store = options.createStore
+    ? options.createStore(storePath)
+    : VectorStoreFactory.create('sqlite', new AutoEmbeddingProvider(), {
+        path: storePath,
+      });
+
+  let indexed = false;
+  if (!(await hasIndex(storePath, store))) {
+    onProgress('indexing');
+    const indexer = new Indexer(store);
+    indexer.registerSource(new FileDataSource());
+    await indexer.index('local-files', {
+      targetDir: projectRoot,
+      include: INCLUDE,
+      exclude: EXCLUDE,
+      skip: (fullPath) =>
+        scanner.shouldIgnore(path.relative(projectRoot, fullPath), fullPath) ||
+        scanner.isDenied(relative(fullPath)),
+    });
+    indexed = true;
+    onProgress('indexed');
+  }
+
+  const rows = await store.search(question, limit + OVERFETCH);
+  const results = [];
+  let removedByPolicy = 0;
+  let redactions = 0;
+  for (const row of rows) {
+    const rawPath = row.metadata?.path ?? row.metadata?.relativePath;
+    if (typeof rawPath !== 'string') continue;
+    const fullPath = path.isAbsolute(rawPath) ? rawPath : path.resolve(projectRoot, rawPath);
+    if (left(fullPath)) continue;
+    const file = relative(fullPath);
+    if (scanner.isDenied(file)) {
+      removedByPolicy++;
+      continue;
+    }
+    if (results.length === limit) continue;
+    let text = String(row.text ?? '');
+    if (options.redact !== false) {
+      const redacted = redactSecrets(text);
+      text = redacted.text;
+      redactions += redacted.findings.length;
+    }
+    const truncated = Boolean(options.maxChars) && text.length > options.maxChars;
+    if (truncated) text = text.slice(0, options.maxChars);
+    results.push({ file, score: row.score ?? null, text, truncated });
+  }
+
+  return { query: question, indexed, results, removedByPolicy, redactions };
+}
+
+export default askCodebase;

package/lib/rag/sources/FileDataSource.js

@@ -19,11 +19,13 @@ export class FileDataSource extends DataSourcePlugin {
     const maxDepth = options.maxDepth || 5;
     const include = options.include || ['.js', '.md', '.ts', '.json'];
     const exclude = options.exclude || ['node_modules', '.git', 'dist', 'coverage'];
+    // (fullPath, isDirectory) => true leaves a file or directory out
+    const skip = options.skip || (() => false);
 
-    yield* this.walkDirectory(targetDir, maxDepth, include, exclude);
+    yield* this.walkDirectory(targetDir, maxDepth, include, exclude, skip);
   }
 
-  async *walkDirectory(dir, depth, include, exclude) {
+  async *walkDirectory(dir, depth, include, exclude, skip = () => false) {
     if (depth < 0) return;
 
     let entries;
@@ -38,9 +40,10 @@ export class FileDataSource extends DataSourcePlugin {
       const fullPath = path.join(dir, entry.name);
 
       if (exclude.some((ex) => fullPath.includes(ex))) continue;
+      if (skip(fullPath, entry.isDirectory())) continue;
 
       if (entry.isDirectory()) {
-        yield* this.walkDirectory(fullPath, depth - 1, include, exclude);
+        yield* this.walkDirectory(fullPath, depth - 1, include, exclude, skip);
       } else if (entry.isFile()) {
         const ext = path.extname(entry.name);
         if (include.length > 0 && !include.includes(ext)) continue;