@tai-io/codesearch 2026.313.1614

package/dist/core/preview.js

@@ -0,0 +1,58 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { scanFiles } from './sync.js';
+import { normalizePath } from '../paths.js';
+export async function previewCodebase(rootPath, customExtensions, customIgnorePatterns) {
+    const normalizedPath = normalizePath(rootPath);
+    const filePaths = await scanFiles(normalizedPath, customExtensions, customIgnorePatterns);
+    const byExtension = {};
+    for (const f of filePaths) {
+        const ext = path.extname(f).toLowerCase() || '(no ext)';
+        byExtension[ext] = (byExtension[ext] ?? 0) + 1;
+    }
+    const dirCounts = {};
+    for (const f of filePaths) {
+        const firstSeg = f.split(/[/\\]/)[0];
+        const dir = f.includes('/') || f.includes('\\') ? firstSeg : '(root)';
+        dirCounts[dir] = (dirCounts[dir] ?? 0) + 1;
+    }
+    const topDirectories = Object.entries(dirCounts)
+        .map(([dir, count]) => ({ dir, count }))
+        .sort((a, b) => b.count - a.count)
+        .slice(0, 10);
+    let totalBytes = 0;
+    for (const f of filePaths) {
+        try {
+            const stat = fs.statSync(path.join(normalizedPath, f));
+            totalBytes += stat.size;
+        }
+        catch {
+            // file may have disappeared between scan and stat
+        }
+    }
+    const estimatedTokens = Math.ceil(totalBytes / 3);
+    const estimatedCostUsd = (estimatedTokens / 1_000_000) * 0.02;
+    const warnings = [];
+    if (filePaths.length === 0) {
+        warnings.push('No indexable files found. Check file extension filters and ignore patterns.');
+    }
+    if (filePaths.length > 5000) {
+        warnings.push(`Found ${filePaths.length.toLocaleString()} files. Most codebases have 100-5,000 source files. Consider adding ignore patterns.`);
+    }
+    if (topDirectories.length > 0 && filePaths.length > 0) {
+        const topDir = topDirectories[0];
+        const pct = Math.round((topDir.count / filePaths.length) * 100);
+        if (pct > 50 && topDir.dir !== '(root)') {
+            warnings.push(`Directory '${topDir.dir}/' contains ${pct}% of files -- consider ignoring if it contains build artifacts or dependencies.`);
+        }
+    }
+    return {
+        totalFiles: filePaths.length,
+        byExtension,
+        topDirectories,
+        estimatedTokens,
+        estimatedCostUsd,
+        warnings,
+    };
+}
+//# sourceMappingURL=preview.js.map

package/dist/core/repo-map.d.ts

@@ -0,0 +1,33 @@
+import type { VectorDB, SymbolEntry } from '../vectordb/types.js';
+export interface RepoMapOptions {
+    pathFilter?: string;
+    kindFilter?: string;
+    maxTokens?: number;
+}
+export interface ListSymbolsOptions {
+    pathFilter?: string;
+    kindFilter?: string;
+    nameFilter?: string;
+}
+export interface SymbolSource {
+    getSymbols(collectionName: string, options?: RepoMapOptions | ListSymbolsOptions): Promise<SymbolEntry[]>;
+}
+export declare class VectorDBSymbolSource implements SymbolSource {
+    private vectordb;
+    constructor(vectordb: VectorDB);
+    getSymbols(collectionName: string, options?: RepoMapOptions | ListSymbolsOptions): Promise<SymbolEntry[]>;
+}
+/**
+ * Convert a glob-like pattern to a regex for path filtering.
+ * Supports * (non-separator) and ** (any path segment).
+ */
+export declare function matchesPathFilter(relativePath: string, pattern: string): boolean;
+/**
+ * Generate a structured repo map grouped by file, with methods nested under classes.
+ */
+export declare function generateRepoMap(rootPath: string, source: SymbolSource, options?: RepoMapOptions): Promise<string>;
+/**
+ * List symbols as a compact Name|Kind|Location table.
+ */
+export declare function listSymbolsTable(rootPath: string, source: SymbolSource, options?: ListSymbolsOptions): Promise<string>;
+//# sourceMappingURL=repo-map.d.ts.map

package/dist/core/repo-map.js

@@ -0,0 +1,144 @@
+import { pathToCollectionName } from '../paths.js';
+export class VectorDBSymbolSource {
+    vectordb;
+    constructor(vectordb) {
+        this.vectordb = vectordb;
+    }
+    async getSymbols(collectionName, options) {
+        const all = await this.vectordb.listSymbols(collectionName);
+        let result = all;
+        const pathFilter = options?.pathFilter;
+        if (pathFilter) {
+            result = result.filter((s) => matchesPathFilter(s.relativePath, pathFilter));
+        }
+        const kindFilter = options?.kindFilter;
+        if (kindFilter) {
+            const kind = kindFilter.toLowerCase();
+            result = result.filter((s) => s.kind.toLowerCase() === kind);
+        }
+        const nameFilter = options?.nameFilter;
+        if (nameFilter) {
+            const lower = nameFilter.toLowerCase();
+            result = result.filter((s) => s.name.toLowerCase().includes(lower));
+        }
+        return result;
+    }
+}
+/**
+ * Convert a glob-like pattern to a regex for path filtering.
+ * Supports * (non-separator) and ** (any path segment).
+ */
+export function matchesPathFilter(relativePath, pattern) {
+    // Escape regex special chars except * which we handle
+    const escaped = pattern
+        .replace(/[.+^${}()|[\]\\]/g, '\\$&')
+        .replace(/\*\*/g, '\uFFFD') // placeholder for **
+        .replace(/\*/g, '[^/]*')
+        .replace(/\uFFFD/g, '.*');
+    const regex = new RegExp(`^${escaped}$`);
+    return regex.test(relativePath);
+}
+/**
+ * Deduplicate symbol entries: prefer those with a signature, then by first occurrence.
+ */
+function deduplicateSymbols(symbols) {
+    const seen = new Map();
+    for (const sym of symbols) {
+        const key = `${sym.relativePath}:${sym.name}:${sym.kind}`;
+        const existing = seen.get(key);
+        if (!existing || (!existing.signature && sym.signature)) {
+            seen.set(key, sym);
+        }
+    }
+    return [...seen.values()];
+}
+/**
+ * Generate a structured repo map grouped by file, with methods nested under classes.
+ */
+export async function generateRepoMap(rootPath, source, options) {
+    const collectionName = pathToCollectionName(rootPath);
+    const maxTokens = options?.maxTokens ?? 4000;
+    const maxChars = maxTokens * 4;
+    const symbols = await source.getSymbols(collectionName, options);
+    if (symbols.length === 0) {
+        return '(no symbols found — codebase may not be indexed yet)';
+    }
+    const deduped = deduplicateSymbols(symbols);
+    // Group by file
+    const byFile = new Map();
+    for (const sym of deduped) {
+        const list = byFile.get(sym.relativePath) ?? [];
+        list.push(sym);
+        byFile.set(sym.relativePath, list);
+    }
+    // Sort files
+    const files = [...byFile.keys()].sort();
+    const lines = [];
+    let totalChars = 0;
+    for (const file of files) {
+        const fileSymbols = byFile.get(file) ?? [];
+        // Separate top-level from methods (those with a parentName)
+        const topLevel = fileSymbols.filter((s) => !s.parentName);
+        const methods = fileSymbols.filter((s) => s.parentName !== undefined);
+        // Build method lookup by parent
+        const methodsByParent = new Map();
+        for (const m of methods) {
+            const list = methodsByParent.get(m.parentName) ?? [];
+            list.push(m);
+            methodsByParent.set(m.parentName, list);
+        }
+        const fileHeader = `${file}:`;
+        if (totalChars + fileHeader.length > maxChars)
+            break;
+        lines.push(fileHeader);
+        totalChars += fileHeader.length + 1;
+        for (const sym of topLevel) {
+            const sig = sym.signature ? ` ${sym.signature.trim()}` : ` ${sym.name}`;
+            const line = `  [${sym.kind}]${sig}`;
+            if (totalChars + line.length > maxChars) {
+                lines.push('  ...(truncated)');
+                return lines.join('\n');
+            }
+            lines.push(line);
+            totalChars += line.length + 1;
+            // Nest methods under this symbol if it's a container
+            const children = methodsByParent.get(sym.name) ?? [];
+            for (const child of children) {
+                const childSig = child.signature ? ` ${child.signature.trim()}` : ` ${child.name}`;
+                const childLine = `    [${child.kind}]${childSig}`;
+                if (totalChars + childLine.length > maxChars) {
+                    lines.push('    ...(truncated)');
+                    return lines.join('\n');
+                }
+                lines.push(childLine);
+                totalChars += childLine.length + 1;
+            }
+        }
+    }
+    return lines.join('\n');
+}
+/**
+ * List symbols as a compact Name|Kind|Location table.
+ */
+export async function listSymbolsTable(rootPath, source, options) {
+    const collectionName = pathToCollectionName(rootPath);
+    const symbols = await source.getSymbols(collectionName, options);
+    if (symbols.length === 0) {
+        return '(no symbols found)';
+    }
+    const deduped = deduplicateSymbols(symbols);
+    deduped.sort((a, b) => {
+        const pathCmp = a.relativePath.localeCompare(b.relativePath);
+        if (pathCmp !== 0)
+            return pathCmp;
+        return a.startLine - b.startLine;
+    });
+    const header = 'Name | Kind | Location';
+    const sep = '-----|------|--------';
+    const rows = deduped.map((s) => {
+        const location = `${s.relativePath}:${s.startLine}`;
+        return `${s.name} | ${s.kind} | ${location}`;
+    });
+    return [header, sep, ...rows].join('\n');
+}
+//# sourceMappingURL=repo-map.js.map

package/dist/core/searcher.d.ts

@@ -0,0 +1,12 @@
+import type { Embedding } from '../embedding/types.js';
+import type { VectorDB, SearchResult } from '../vectordb/types.js';
+export interface SearchOptions {
+    limit?: number;
+    extensionFilter?: string[];
+}
+export declare function searchCode(rootPath: string, query: string, embedding: Embedding, vectordb: VectorDB, options?: SearchOptions): Promise<SearchResult[]>;
+export declare function applyCategoryBoost(results: SearchResult[]): SearchResult[];
+export declare function deduplicateResults(results: SearchResult[], limit: number): SearchResult[];
+export declare function formatCompactResults(results: SearchResult[], query: string, rootPath: string): string;
+export declare function formatSearchResults(results: SearchResult[], query: string, rootPath: string): string;
+//# sourceMappingURL=searcher.d.ts.map

package/dist/core/searcher.js

@@ -0,0 +1,97 @@
+import { normalizePath, pathToCollectionName } from '../paths.js';
+import { SearchError } from '../errors.js';
+const DEFAULT_LIMIT = 10;
+const MAX_LIMIT = 50;
+export async function searchCode(rootPath, query, embedding, vectordb, options = {}) {
+    const normalizedPath = normalizePath(rootPath);
+    const collectionName = pathToCollectionName(normalizedPath);
+    const exists = await vectordb.hasCollection(collectionName);
+    if (!exists) {
+        throw new SearchError(`Codebase at "${normalizedPath}" is not indexed. ` +
+            `Use the index tool to index it first.`);
+    }
+    const limit = Math.min(Math.max(1, options.limit ?? DEFAULT_LIMIT), MAX_LIMIT);
+    const queryVector = await embedding.embed(query);
+    const overFetchLimit = Math.min(limit * 5, MAX_LIMIT);
+    const results = await vectordb.search(collectionName, {
+        queryVector,
+        queryText: query,
+        limit: overFetchLimit,
+        extensionFilter: options.extensionFilter,
+    });
+    return deduplicateResults(applyCategoryBoost(results), limit);
+}
+const CATEGORY_BOOST = {
+    source: 1.0,
+    test: 0.75,
+    doc: 0.65,
+    config: 0.7,
+    generated: 0.6,
+};
+const DEFAULT_BOOST = 1.0; // legacy points without fileCategory get no penalty
+export function applyCategoryBoost(results) {
+    return results
+        .map((r) => ({
+        ...r,
+        score: r.score * (CATEGORY_BOOST[r.fileCategory ?? ''] ?? DEFAULT_BOOST),
+    }))
+        .sort((a, b) => b.score - a.score);
+}
+export function deduplicateResults(results, limit) {
+    const accepted = [];
+    // Track accepted line ranges per file: relativePath -> [startLine, endLine][]
+    const fileRanges = new Map();
+    for (const r of results) {
+        if (accepted.length >= limit)
+            break;
+        const ranges = fileRanges.get(r.relativePath);
+        if (ranges?.some(([s, e]) => r.startLine <= e && r.endLine >= s)) {
+            continue; // overlaps with an already-accepted chunk from same file
+        }
+        accepted.push(r);
+        if (!ranges) {
+            fileRanges.set(r.relativePath, [[r.startLine, r.endLine]]);
+        }
+        else {
+            ranges.push([r.startLine, r.endLine]);
+        }
+    }
+    return accepted;
+}
+export function formatCompactResults(results, query, rootPath) {
+    if (results.length === 0) {
+        return `No results found for "${query}" in ${rootPath}.`;
+    }
+    const lines = [
+        `Found ${results.length} result(s) for "${query}" in ${rootPath}:\n`,
+        '| # | File | Lines | Score | ~Tokens |',
+        '|---|------|-------|-------|---------|',
+    ];
+    for (let i = 0; i < results.length; i++) {
+        const r = results[i];
+        const tokens = Math.ceil(r.content.length / 4);
+        lines.push(`| ${i + 1} | \`${r.relativePath}\` | ${r.startLine}-${r.endLine} | ${r.score.toFixed(2)} | ~${tokens} |`);
+    }
+    lines.push('');
+    lines.push('Use the Read tool to view full code for specific results.');
+    return lines.join('\n');
+}
+export function formatSearchResults(results, query, rootPath) {
+    if (results.length === 0) {
+        return `No results found for "${query}" in ${rootPath}.`;
+    }
+    const lines = [`Found ${results.length} result(s) for "${query}" in ${rootPath}:\n`];
+    for (let i = 0; i < results.length; i++) {
+        const r = results[i];
+        lines.push(`### Result ${i + 1} of ${results.length}`);
+        lines.push(`**File:** \`${r.relativePath}\` (lines ${r.startLine}-${r.endLine})`);
+        lines.push(`**Language:** ${r.language} | **Score:** ${r.score.toFixed(4)}`);
+        const safeLang = r.language.replace(/[^a-zA-Z0-9_+-]/g, '');
+        lines.push('```' + safeLang);
+        lines.push(r.content);
+        lines.push('```');
+        lines.push('');
+    }
+    return lines.join('\n');
+}
+//# sourceMappingURL=searcher.js.map

package/dist/core/sync.d.ts

@@ -0,0 +1,15 @@
+export type FileSnapshot = Record<string, {
+    contentHash: string;
+}>;
+interface SyncResult {
+    added: string[];
+    modified: string[];
+    removed: string[];
+}
+export declare function scanFiles(rootPath: string, customExtensions?: string[], customIgnore?: string[]): Promise<string[]>;
+export declare function buildSnapshot(rootPath: string, relativePaths: string[]): FileSnapshot;
+export declare function diffSnapshots(previous: FileSnapshot, current: FileSnapshot): SyncResult;
+export declare function parseGitignorePatterns(content: string): string[];
+export declare function extensionToLanguage(ext: string): string;
+export {};
+//# sourceMappingURL=sync.d.ts.map

package/dist/core/sync.js

@@ -0,0 +1,212 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { createHash } from 'node:crypto';
+import { glob } from 'glob';
+const DEFAULT_EXTENSIONS = new Set([
+    '.ts',
+    '.tsx',
+    '.js',
+    '.jsx',
+    '.mjs',
+    '.cjs',
+    '.py',
+    '.pyi',
+    '.go',
+    '.java',
+    '.rs',
+    '.cpp',
+    '.cc',
+    '.cxx',
+    '.c',
+    '.h',
+    '.hpp',
+    '.cs',
+    '.scala',
+    '.rb',
+    '.php',
+    '.swift',
+    '.kt',
+    '.kts',
+    '.lua',
+    '.sh',
+    '.bash',
+    '.zsh',
+    '.sql',
+    '.r',
+    '.R',
+    '.m',
+    '.mm', // Objective-C
+    '.dart',
+    '.ex',
+    '.exs', // Elixir
+    '.erl',
+    '.hrl', // Erlang
+    '.hs', // Haskell
+    '.ml',
+    '.mli', // OCaml
+    '.vue',
+    '.svelte',
+    '.astro',
+    '.yaml',
+    '.yml',
+    '.toml',
+    '.json',
+    '.md',
+    '.mdx',
+    '.html',
+    '.css',
+    '.scss',
+    '.less',
+]);
+const DEFAULT_IGNORE = [
+    '**/node_modules/**',
+    '**/.git/**',
+    '**/dist/**',
+    '**/build/**',
+    '**/.next/**',
+    '**/target/**',
+    '**/__pycache__/**',
+    '**/.venv/**',
+    '**/venv/**',
+    '**/vendor/**',
+    '**/.cache/**',
+    '**/coverage/**',
+    '**/*.min.js',
+    '**/*.min.css',
+    '**/package-lock.json',
+    '**/pnpm-lock.yaml',
+    '**/yarn.lock',
+];
+export async function scanFiles(rootPath, customExtensions = [], customIgnore = []) {
+    const extensions = new Set([...DEFAULT_EXTENSIONS, ...customExtensions]);
+    const gitignorePatterns = readGitignore(rootPath);
+    const allIgnore = [...DEFAULT_IGNORE, ...gitignorePatterns, ...customIgnore];
+    const files = await glob('**/*', {
+        cwd: rootPath,
+        nodir: true,
+        dot: false,
+        ignore: allIgnore,
+        absolute: false,
+    });
+    return files.filter((f) => extensions.has(path.extname(f).toLowerCase())).sort();
+}
+function hashFileContent(fullPath) {
+    const content = fs.readFileSync(fullPath);
+    return createHash('sha256').update(content).digest('hex').slice(0, 16);
+}
+export function buildSnapshot(rootPath, relativePaths) {
+    const snapshot = {};
+    for (const rel of relativePaths) {
+        const fullPath = path.join(rootPath, rel);
+        try {
+            const contentHash = hashFileContent(fullPath);
+            snapshot[rel] = { contentHash };
+        }
+        catch (err) {
+            process.stderr.write(`Skipping "${rel}": ${String(err)}\n`);
+        }
+    }
+    return snapshot;
+}
+export function diffSnapshots(previous, current) {
+    const added = [];
+    const modified = [];
+    const removed = [];
+    for (const [rel, cur] of Object.entries(current)) {
+        if (!Object.hasOwn(previous, rel)) {
+            added.push(rel);
+        }
+        else if (previous[rel].contentHash !== cur.contentHash) {
+            modified.push(rel);
+        }
+    }
+    for (const rel of Object.keys(previous)) {
+        if (!(rel in current)) {
+            removed.push(rel);
+        }
+    }
+    return { added, modified, removed };
+}
+export function parseGitignorePatterns(content) {
+    return content
+        .split('\n')
+        .map((line) => line.trim())
+        .filter((line) => line && !line.startsWith('#') && !line.startsWith('!'))
+        .map((pattern) => {
+        pattern = pattern.replace(/\s+$/, '');
+        if (pattern.endsWith('/')) {
+            pattern = pattern.slice(0, -1);
+        }
+        if (pattern.startsWith('/'))
+            return pattern.slice(1);
+        if (!pattern.includes('/'))
+            return `**/${pattern}`;
+        return pattern;
+    })
+        .filter((p) => p.length > 0);
+}
+function readGitignore(rootPath) {
+    const gitignorePath = path.join(rootPath, '.gitignore');
+    try {
+        const content = fs.readFileSync(gitignorePath, 'utf-8');
+        return parseGitignorePatterns(content);
+    }
+    catch {
+        return [];
+    }
+}
+export function extensionToLanguage(ext) {
+    const map = {
+        '.ts': 'typescript',
+        '.tsx': 'tsx',
+        '.js': 'javascript',
+        '.jsx': 'javascript',
+        '.mjs': 'javascript',
+        '.cjs': 'javascript',
+        '.py': 'python',
+        '.pyi': 'python',
+        '.go': 'go',
+        '.java': 'java',
+        '.rs': 'rust',
+        '.cpp': 'cpp',
+        '.cc': 'cpp',
+        '.cxx': 'cpp',
+        '.c': 'c',
+        '.h': 'cpp',
+        '.hpp': 'cpp',
+        '.cs': 'csharp',
+        '.scala': 'scala',
+        '.rb': 'ruby',
+        '.php': 'php',
+        '.swift': 'swift',
+        '.kt': 'kotlin',
+        '.kts': 'kotlin',
+        '.lua': 'lua',
+        '.sh': 'bash',
+        '.bash': 'bash',
+        '.zsh': 'bash',
+        '.sql': 'sql',
+        '.r': 'r',
+        '.R': 'r',
+        '.dart': 'dart',
+        '.ex': 'elixir',
+        '.exs': 'elixir',
+        '.hs': 'haskell',
+        '.ml': 'ocaml',
+        '.vue': 'vue',
+        '.svelte': 'svelte',
+        '.astro': 'astro',
+        '.yaml': 'yaml',
+        '.yml': 'yaml',
+        '.toml': 'toml',
+        '.json': 'json',
+        '.md': 'markdown',
+        '.mdx': 'markdown',
+        '.html': 'html',
+        '.css': 'css',
+        '.scss': 'scss',
+        '.less': 'less',
+    };
+    return map[ext.toLowerCase()] ?? 'unknown';
+}
+//# sourceMappingURL=sync.js.map

package/dist/core/targeted-indexer.d.ts

@@ -0,0 +1,19 @@
+import type { Embedding } from '../embedding/types.js';
+import type { VectorDB } from '../vectordb/types.js';
+export interface TargetedIndexResult {
+    processedFiles: number;
+    totalChunks: number;
+    skippedFiles: number;
+    durationMs: number;
+}
+/**
+ * Re-index a specific set of files within a project.
+ * For each file: delete stale vectors, re-split, re-embed, re-insert, update snapshot.
+ *
+ * @param rootPath       Absolute path to the project root
+ * @param relativePaths  Relative paths (from rootPath) of files to re-index
+ * @param embedding      Embedding provider
+ * @param vectordb       Vector DB provider
+ */
+export declare function indexFiles(rootPath: string, relativePaths: string[], embedding: Embedding, vectordb: VectorDB): Promise<TargetedIndexResult>;
+//# sourceMappingURL=targeted-indexer.d.ts.map