edhindex 1.0.0

package/src/embeddings/transformers.ts

@@ -0,0 +1,130 @@
+import { EmbeddingProvider } from './provider.js';
+import { MODEL_CONFIGS, RERANKER_MODEL, ModelTier } from '../config/index.js';
+import { logger } from '../logging.js';
+
+export class TransformersEmbeddingProvider implements EmbeddingProvider {
+  private pipeline: any = null;
+  private modelId: string;
+  private dims: number;
+  private modelTier: ModelTier;
+  private loaded = false;
+
+  constructor(tier: ModelTier) {
+    this.modelTier = tier;
+    const cfg = MODEL_CONFIGS[tier];
+    this.modelId = cfg.modelId;
+    this.dims = cfg.dimensions;
+  }
+
+  async load(): Promise<void> {
+    if (this.loaded) return;
+    try {
+      logger.info(`Loading embedding model: ${this.modelId}`);
+      const mod = await import('@huggingface/transformers');
+      this.pipeline = await mod.pipeline('feature-extraction', this.modelId, {
+        progress_callback: (progress: any) => {
+          if (progress?.status === 'progress' && progress?.percent !== undefined) {
+            logger.verbose(`Model download: ${Math.round(progress.percent)}%`);
+          }
+        },
+      });
+      this.loaded = true;
+      logger.info(`Embedding model loaded: ${this.modelId}`);
+    } catch (e) {
+      logger.error('Failed to load embedding model:', e);
+      throw e;
+    }
+  }
+
+  async embed(texts: string[]): Promise<number[][]> {
+    if (!this.loaded) {
+      await this.load();
+    }
+
+    const results: number[][] = [];
+    for (const text of texts) {
+      try {
+        const output = await this.pipeline(text, {
+          pooling: 'mean',
+          normalize: true,
+        });
+        const embedding = output.tolist ? output.tolist() : output.data || output;
+        results.push(Array.isArray(embedding[0]) ? embedding[0] : embedding);
+      } catch (e) {
+        logger.debug('Embedding failed for text, using zero vector:', e);
+        results.push(new Array(this.dims).fill(0));
+      }
+    }
+    return results;
+  }
+
+  dimensions(): number {
+    return this.dims;
+  }
+
+  name(): string {
+    return this.modelTier;
+  }
+
+  dispose(): void {
+    this.pipeline = null;
+    this.loaded = false;
+  }
+}
+
+export class TransformersReranker {
+  private pipeline: any = null;
+  private loaded = false;
+
+  async load(): Promise<void> {
+    if (this.loaded) return;
+    try {
+      logger.info('Loading reranker model');
+      const mod = await import('@huggingface/transformers');
+      this.pipeline = await mod.pipeline('text-classification', RERANKER_MODEL, {
+        progress_callback: (progress: any) => {
+          if (progress?.status === 'progress' && progress?.percent !== undefined) {
+            logger.verbose(`Reranker download: ${Math.round(progress.percent)}%`);
+          }
+        },
+      });
+      this.loaded = true;
+      logger.info('Reranker model loaded');
+    } catch (e) {
+      logger.error('Failed to load reranker model:', e);
+      throw e;
+    }
+  }
+
+  async rerank(query: string, texts: string[]): Promise<{ index: number; score: number }[]> {
+    if (!this.loaded) {
+      await this.load();
+    }
+
+    const pairs = texts.map(text => ({ text: query, text_pair: text }));
+
+    try {
+      const results = await this.pipeline(pairs);
+      const scores: { index: number; score: number }[] = [];
+
+      for (let i = 0; i < results.length; i++) {
+        const result = results[i];
+        const score = result.score !== undefined
+          ? (result.label === 'LABEL_1' ? result.score : 1 - result.score)
+          : 0;
+        scores.push({ index: i, score });
+      }
+
+      scores.sort((a, b) => b.score - a.score);
+      return scores;
+    } catch (e) {
+      logger.debug('Reranking failed:', e);
+      return texts.map((_, i) => ({ index: i, score: 0 }));
+    }
+  }
+
+  dispose(): void {
+    this.pipeline = null;
+    this.loaded = false;
+  }
+}

package/src/indexer/chunker.ts

@@ -0,0 +1,198 @@
+import { createHash } from 'node:crypto';
+import { SymbolInfo } from './parser.js';
+import { logger } from '../logging.js';
+
+export interface Chunk {
+  id: string;
+  file: string;
+  language: string;
+  symbol: string;
+  kind: string | null;
+  parent: string | null;
+  hash: string;
+  startLine: number;
+  endLine: number;
+  content: string;
+  imports: string[];
+  exports: string[];
+}
+
+const MAX_TOKENS_PER_CHUNK = 512;
+
+function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 4);
+}
+
+function computeHash(content: string): string {
+  return createHash('sha256').update(content, 'utf-8').digest('hex');
+}
+
+export async function chunkFile(
+  filePath: string,
+  relativePath: string,
+  language: string,
+  symbols: SymbolInfo[],
+  content: string,
+  lines: string[],
+  imports: string[],
+  exports: string[],
+): Promise<Chunk[]> {
+  const chunks: Chunk[] = [];
+  const priorityOrder: Record<string, number> = {
+    function: 0,
+    method: 1,
+    class: 2,
+    interface: 3,
+    enum: 4,
+    module: 5,
+  };
+
+  const sorted = [...symbols].sort((a, b) => {
+    const aP = priorityOrder[a.kind] ?? 99;
+    const bP = priorityOrder[b.kind] ?? 99;
+    if (aP !== bP) return aP - bP;
+    return a.startLine - b.startLine;
+  });
+
+  const covered = new Set<number>();
+
+  for (const sym of sorted) {
+    if (isCovered(covered, sym.startLine, sym.endLine)) continue;
+    markCovered(covered, sym.startLine, sym.endLine);
+
+    const chunkContent = sym.text || content.slice(
+      content.split('\n').slice(0, sym.startLine - 1).join('\n').length + (sym.startLine > 1 ? 1 : 0),
+      content.split('\n').slice(0, sym.endLine).join('\n').length
+    );
+
+    if (estimateTokens(chunkContent) <= MAX_TOKENS_PER_CHUNK) {
+      const hash = computeHash(chunkContent);
+      chunks.push({
+        id: `${relativePath}:${sym.startLine}-${sym.endLine}`,
+        file: relativePath,
+        language,
+        symbol: sym.name,
+        kind: sym.kind,
+        parent: null,
+        hash,
+        startLine: sym.startLine,
+        endLine: sym.endLine,
+        content: chunkContent,
+        imports,
+        exports,
+      });
+    } else {
+      const childChunks = splitRecursive(
+        chunkContent, lines, sym.startLine, sym.name, sym.kind,
+        relativePath, language, imports, exports
+      );
+      chunks.push(...childChunks);
+    }
+  }
+
+  const fileContent = content;
+  if (estimateTokens(fileContent) <= MAX_TOKENS_PER_CHUNK * 2 && covered.size < lines.length) {
+    const uncoveredLines = getUncoveredLines(lines, covered);
+    if (uncoveredLines.length > 0 && estimateTokens(uncoveredLines.join('\n')) <= MAX_TOKENS_PER_CHUNK) {
+      const startLine = uncoveredLines[0].line;
+      const endLine = uncoveredLines[uncoveredLines.length - 1].line;
+      const hash = computeHash(uncoveredLines.map(l => l.text).join('\n'));
+      chunks.push({
+        id: `${relativePath}:${startLine}-${endLine}`,
+        file: relativePath,
+        language,
+        symbol: `file_${relativePath.replace(/[^a-zA-Z0-9]/g, '_')}`,
+        kind: 'module',
+        parent: null,
+        hash,
+        startLine,
+        endLine,
+        content: uncoveredLines.map(l => l.text).join('\n'),
+        imports,
+        exports,
+      });
+    }
+  }
+
+  return chunks;
+}
+
+function splitRecursive(
+  text: string,
+  allLines: string[],
+  baseLine: number,
+  symbolName: string,
+  kind: string | null,
+  relativePath: string,
+  language: string,
+  imports: string[],
+  exports: string[],
+): Chunk[] {
+  const chunks: Chunk[] = [];
+  const textLines = text.split('\n');
+  const lineCount = textLines.length;
+
+  let currentStart = 0;
+  while (currentStart < lineCount) {
+    const remaining = lineCount - currentStart;
+    let end = currentStart + Math.min(remaining, Math.floor(MAX_TOKENS_PER_CHUNK * 2));
+
+    let chunkText = textLines.slice(currentStart, end).join('\n');
+    while (estimateTokens(chunkText) > MAX_TOKENS_PER_CHUNK && end > currentStart + 1) {
+      end--;
+      chunkText = textLines.slice(currentStart, end).join('\n');
+    }
+
+    if (end <= currentStart) {
+      chunkText = textLines[currentStart];
+      end = currentStart + 1;
+    }
+
+    const hash = computeHash(chunkText);
+    chunks.push({
+      id: `${relativePath}:${baseLine + currentStart}-${baseLine + end - 1}`,
+      file: relativePath,
+      language,
+      symbol: symbolName,
+      kind,
+      parent: `${relativePath}:${baseLine}-${baseLine + lineCount - 1}`,
+      hash,
+      startLine: baseLine + currentStart,
+      endLine: baseLine + end - 1,
+      content: chunkText,
+      imports,
+      exports,
+    });
+
+    currentStart = end;
+  }
+
+  return chunks;
+}
+
+function isCovered(covered: Set<number>, start: number, end: number): boolean {
+  for (let i = start; i <= end; i++) {
+    if (covered.has(i)) return true;
+  }
+  return false;
+}
+
+function markCovered(covered: Set<number>, start: number, end: number) {
+  for (let i = start; i <= end; i++) {
+    covered.add(i);
+  }
+}
+
+function getUncoveredLines(lines: string[], covered: Set<number>): { line: number; text: string }[] {
+  const result: { line: number; text: string }[] = [];
+  for (let i = 0; i < lines.length; i++) {
+    if (!covered.has(i + 1)) {
+      result.push({ line: i + 1, text: lines[i] });
+    }
+  }
+  return result;
+}
+
+export function chunkHash(content: string): string {
+  return createHash('sha256').update(content, 'utf-8').digest('hex');
+}

package/src/indexer/file-utils.ts

@@ -0,0 +1,139 @@
+import { readdirSync, statSync, readFileSync } from 'node:fs';
+import { join, relative } from 'node:path';
+import { createHash } from 'node:crypto';
+import { checkSymlinkSafety } from '../security.js';
+import { createIgnoreRules, IgnoreRules } from './ignore.js';
+import { logger } from '../logging.js';
+
+const MAX_FILE_SIZE = 10 * 1024 * 1024;
+
+const BINARY_EXTENSIONS = new Set([
+  '.exe', '.dll', '.so', '.dylib', '.bin', '.wasm',
+  '.o', '.a', '.lib',
+  '.pyc', '.class', '.jar', '.war',
+  '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.webp', '.bmp', '.tiff',
+  '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx',
+  '.zip', '.tar', '.gz', '.bz2', '.7z', '.rar',
+  '.mp4', '.avi', '.mov', '.wmv', '.flv', '.mkv',
+  '.mp3', '.wav', '.flac', '.ogg',
+]);
+
+const LANGUAGE_EXTENSIONS: Record<string, string[]> = {
+  ts: ['.ts', '.tsx', '.mts', '.cts'],
+  js: ['.js', '.jsx', '.mjs', '.cjs'],
+  py: ['.py', '.pyw'],
+  go: ['.go'],
+};
+
+const EXTENSION_TO_LANG: Record<string, string> = {};
+for (const [lang, exts] of Object.entries(LANGUAGE_EXTENSIONS)) {
+  for (const ext of exts) {
+    EXTENSION_TO_LANG[ext] = lang;
+  }
+}
+
+export function getLanguageForFile(filePath: string): string | null {
+  const ext = filePath.toLowerCase().split('.').pop();
+  return EXTENSION_TO_LANG[`.${ext}`] || null;
+}
+
+export function languageFromExtension(ext: string): string | null {
+  return EXTENSION_TO_LANG[ext.toLowerCase()] || null;
+}
+
+export function isIndexableFile(filePath: string, stat: { size: number }): boolean {
+  if (stat.size > MAX_FILE_SIZE) return false;
+  if (stat.size === 0) return false;
+  const ext = filePath.toLowerCase().split('.').pop();
+  if (BINARY_EXTENSIONS.has(`.${ext}`)) return false;
+  return true;
+}
+
+export function isGeneratedFile(filePath: string): boolean {
+  const name = filePath.split('/').pop() || filePath;
+  return name.endsWith('.generated.ts') || name.endsWith('.g.ts') ||
+         name === 'tsconfig.json' || name === 'package.json' ||
+         name.endsWith('.min.js') || name.endsWith('.min.css');
+}
+
+export function computeHash(content: string): string {
+  return createHash('sha256').update(content, 'utf-8').digest('hex');
+}
+
+export function computeFileHash(filePath: string): string {
+  const content = readFileSync(filePath, 'utf-8');
+  return computeHash(content);
+}
+
+export interface ScannedFile {
+  path: string;
+  relativePath: string;
+  language: string;
+  hash: string;
+  size: number;
+}
+
+export function scanFiles(rootPath: string, languages: string[]): ScannedFile[] {
+  const ignore = createIgnoreRules(rootPath);
+  const results: ScannedFile[] = [];
+
+  const allowedExts = new Set<string>();
+  for (const lang of languages) {
+    const exts = LANGUAGE_EXTENSIONS[lang];
+    if (exts) for (const ext of exts) allowedExts.add(ext);
+  }
+
+  function walk(dir: string) {
+    let entries: string[];
+    try {
+      entries = readdirSync(dir);
+    } catch {
+      return;
+    }
+
+    for (const entry of entries) {
+      const fullPath = join(dir, entry);
+      const relPath = relative(rootPath, fullPath).replace(/\\/g, '/');
+
+      if (ignore.isIgnored(relPath)) continue;
+
+      try {
+        const stat = statSync(fullPath);
+        if (stat.isDirectory()) {
+          walk(fullPath);
+        } else if (stat.isFile() || stat.isSymbolicLink()) {
+          if (stat.isSymbolicLink()) {
+            if (!checkSymlinkSafety(rootPath, fullPath)) continue;
+          }
+
+          const ext = '.' + (entry.split('.').pop() || '').toLowerCase();
+          if (!allowedExts.has(ext)) continue;
+          if (!isIndexableFile(fullPath, stat)) continue;
+
+          const lang = EXTENSION_TO_LANG[ext];
+          if (!lang) continue;
+
+          try {
+            const hash = computeFileHash(fullPath);
+            results.push({
+              path: fullPath,
+              relativePath: relPath,
+              language: lang,
+              hash,
+              size: stat.size,
+            });
+          } catch {
+            // skip unreadable files
+          }
+        }
+      } catch {
+        // skip inaccessible entries
+      }
+    }
+  }
+
+  walk(rootPath);
+  return results;
+}
+
+

package/src/indexer/ignore.ts

@@ -0,0 +1,100 @@
+import { readFileSync, existsSync } from 'node:fs';
+import { join, relative, isAbsolute, sep } from 'node:path';
+
+const IGNORE_FILE = '.edhindexignore';
+
+const DEFAULT_IGNORE_PATTERNS = [
+  'node_modules',
+  '.git',
+  'dist',
+  'build',
+  'coverage',
+  '.next',
+  'out',
+  'vendor',
+  'bin',
+  'obj',
+  'target',
+  'tmp',
+  '.cache',
+  '.turbo',
+  '.edhindex',
+  '*.exe', '*.dll', '*.so', '*.dylib', '*.bin', '*.wasm',
+  '*.o', '*.a', '*.lib',
+  '*.pyc', '*.class', '*.jar', '*.war',
+  '*.png', '*.jpg', '*.jpeg', '*.gif', '*.svg', '*.ico', '*.webp', '*.bmp', '*.tiff',
+  '*.pdf', '*.doc', '*.docx', '*.xls', '*.xlsx', '*.ppt', '*.pptx',
+  '*.zip', '*.tar', '*.gz', '*.bz2', '*.7z', '*.rar',
+  '*.mp4', '*.avi', '*.mov', '*.wmv', '*.flv', '*.mkv',
+  '*.mp3', '*.wav', '*.flac', '*.ogg',
+  '*.min.js', '*.min.css',
+  'package-lock.json', 'yarn.lock', 'pnpm-lock.yaml',
+];
+
+export interface IgnoreRules {
+  isIgnored(relativePath: string): boolean;
+}
+
+function globToRegex(pattern: string): RegExp {
+  const isNegated = pattern.startsWith('!');
+  const p = isNegated ? pattern.slice(1) : pattern;
+
+  let regexStr = '^';
+  for (let i = 0; i < p.length; i++) {
+    const ch = p[i];
+    if (ch === '*') {
+      if (i + 1 < p.length && p[i + 1] === '*') {
+        regexStr += '.*';
+        i++;
+        if (i + 1 < p.length && (p[i + 1] === '/' || p[i + 1] === '\\\\')) i++;
+      } else {
+        regexStr += '[^/]*';
+      }
+    } else if (ch === '?') {
+      regexStr += '[^/]';
+    } else if (ch === '.') {
+      regexStr += '\\.';
+    } else if (ch === '/') {
+      regexStr += '[/\\\\]';
+    } else if (ch === '\\') {
+      regexStr += '\\\\';
+    } else {
+      regexStr += ch;
+    }
+  }
+  regexStr += '$';
+
+  return new RegExp(regexStr, 'i');
+}
+
+export function createIgnoreRules(rootPath: string): IgnoreRules {
+  const patterns: { regex: RegExp; negated: boolean }[] = [];
+
+  for (const p of DEFAULT_IGNORE_PATTERNS) {
+    patterns.push({ regex: globToRegex(p), negated: false });
+  }
+
+  const ignoreFilePath = join(rootPath, IGNORE_FILE);
+  if (existsSync(ignoreFilePath)) {
+    const content = readFileSync(ignoreFilePath, 'utf-8');
+    for (const line of content.split('\n')) {
+      const trimmed = line.trim();
+      if (!trimmed || trimmed.startsWith('#')) continue;
+      const isNegated = trimmed.startsWith('!');
+      patterns.push({ regex: globToRegex(trimmed), negated: isNegated });
+    }
+  }
+
+  return {
+    isIgnored(relativePath: string): boolean {
+      const normalized = relativePath.replace(/\\/g, '/');
+      let ignored = false;
+      for (const p of patterns) {
+        if (p.regex.test(normalized) || p.regex.test(normalized.split('/').pop() || '')) {
+          ignored = !p.negated;
+        }
+      }
+      return ignored;
+    },
+  };
+}

package/src/indexer/incremental.ts

@@ -0,0 +1,80 @@
+import { logger } from '../logging.js';
+import { ScannedFile } from './file-utils.js';
+
+export interface FileDelta {
+  added: ScannedFile[];
+  modified: ScannedFile[];
+  removed: string[];
+  unchanged: ScannedFile[];
+}
+
+export function computeFileDelta(
+  scanned: Map<string, ScannedFile>,
+  existing: Map<string, string>, // relativePath -> hash
+): FileDelta {
+  const delta: FileDelta = {
+    added: [],
+    modified: [],
+    removed: [],
+    unchanged: [],
+  };
+
+  for (const [relPath, file] of scanned) {
+    const existingHash = existing.get(relPath);
+    if (existingHash === undefined) {
+      delta.added.push(file);
+    } else if (existingHash !== file.hash) {
+      delta.modified.push(file);
+    } else {
+      delta.unchanged.push(file);
+    }
+  }
+
+  const scannedPaths = new Set(scanned.keys());
+  for (const relPath of existing.keys()) {
+    if (!scannedPaths.has(relPath)) {
+      delta.removed.push(relPath);
+    }
+  }
+
+  return delta;
+}
+
+export interface ChunkDelta {
+  added: string[];
+  modified: string[];
+  removed: string[];
+  unchanged: string[];
+}
+
+export function computeChunkDelta(
+  newChunks: Map<string, string>,
+  existingChunks: Map<string, string>,
+): ChunkDelta {
+  const delta: ChunkDelta = {
+    added: [],
+    modified: [],
+    removed: [],
+    unchanged: [],
+  };
+
+  for (const [chunkId, hash] of newChunks) {
+    const existingHash = existingChunks.get(chunkId);
+    if (existingHash === undefined) {
+      delta.added.push(chunkId);
+    } else if (existingHash !== hash) {
+      delta.modified.push(chunkId);
+    } else {
+      delta.unchanged.push(chunkId);
+    }
+  }
+
+  const newIds = new Set(newChunks.keys());
+  for (const chunkId of existingChunks.keys()) {
+    if (!newIds.has(chunkId)) {
+      delta.removed.push(chunkId);
+    }
+  }
+
+  return delta;
+}