claude-eidetic 0.1.0

package/dist/core/sync.js

@@ -0,0 +1,188 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { createHash } from 'node:crypto';
+import { glob } from 'glob';
+const DEFAULT_EXTENSIONS = new Set([
+    '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs',
+    '.py', '.pyi',
+    '.go',
+    '.java',
+    '.rs',
+    '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp',
+    '.cs',
+    '.scala',
+    '.rb',
+    '.php',
+    '.swift',
+    '.kt', '.kts',
+    '.lua',
+    '.sh', '.bash', '.zsh',
+    '.sql',
+    '.r', '.R',
+    '.m', '.mm', // Objective-C
+    '.dart',
+    '.ex', '.exs', // Elixir
+    '.erl', '.hrl', // Erlang
+    '.hs', // Haskell
+    '.ml', '.mli', // OCaml
+    '.vue', '.svelte', '.astro',
+    '.yaml', '.yml',
+    '.toml',
+    '.json',
+    '.md', '.mdx',
+    '.html', '.css', '.scss', '.less',
+]);
+const DEFAULT_IGNORE = [
+    '**/node_modules/**',
+    '**/.git/**',
+    '**/dist/**',
+    '**/build/**',
+    '**/.next/**',
+    '**/target/**',
+    '**/__pycache__/**',
+    '**/.venv/**',
+    '**/venv/**',
+    '**/vendor/**',
+    '**/.cache/**',
+    '**/coverage/**',
+    '**/*.min.js',
+    '**/*.min.css',
+    '**/package-lock.json',
+    '**/pnpm-lock.yaml',
+    '**/yarn.lock',
+];
+/**
+ * Scan a directory and return relative paths of indexable files.
+ * Respects .gitignore if present.
+ */
+export async function scanFiles(rootPath, customExtensions = [], customIgnore = []) {
+    const extensions = new Set([...DEFAULT_EXTENSIONS, ...customExtensions]);
+    // Read .gitignore patterns if present
+    const gitignorePatterns = readGitignore(rootPath);
+    const allIgnore = [...DEFAULT_IGNORE, ...gitignorePatterns, ...customIgnore];
+    const files = await glob('**/*', {
+        cwd: rootPath,
+        nodir: true,
+        dot: false,
+        ignore: allIgnore,
+        absolute: false,
+    });
+    return files
+        .filter(f => extensions.has(path.extname(f).toLowerCase()))
+        .sort();
+}
+/**
+ * Compute a truncated SHA-256 hash of a file's contents.
+ * 16 hex chars (64 bits) is sufficient for change detection — collisions
+ * would only cause a redundant re-index, not data loss.
+ */
+function hashFileContent(fullPath) {
+    const content = fs.readFileSync(fullPath);
+    return createHash('sha256').update(content).digest('hex').slice(0, 16);
+}
+/**
+ * Build a size+contentHash snapshot for a list of files.
+ */
+export function buildSnapshot(rootPath, relativePaths) {
+    const snapshot = {};
+    for (const rel of relativePaths) {
+        const fullPath = path.join(rootPath, rel);
+        try {
+            const contentHash = hashFileContent(fullPath);
+            snapshot[rel] = { contentHash };
+        }
+        catch (err) {
+            console.warn(`Skipping "${rel}": ${err}`);
+        }
+    }
+    return snapshot;
+}
+/**
+ * Compare current snapshot to a previous one. Returns added, modified, and removed files.
+ * Uses content hash as the authoritative change signal — immune to git ops, IDE formatters,
+ * NFS clock skew, and other mtime-only pitfalls.
+ */
+export function diffSnapshots(previous, current) {
+    const added = [];
+    const modified = [];
+    const removed = [];
+    for (const [rel, cur] of Object.entries(current)) {
+        const prev = previous[rel];
+        if (!prev) {
+            added.push(rel);
+        }
+        else if (prev.contentHash !== cur.contentHash) {
+            modified.push(rel);
+        }
+    }
+    for (const rel of Object.keys(previous)) {
+        if (!(rel in current)) {
+            removed.push(rel);
+        }
+    }
+    return { added, modified, removed };
+}
+/**
+ * Parse .gitignore content into glob patterns.
+ * Pure function — no filesystem access.
+ */
+export function parseGitignorePatterns(content) {
+    return content
+        .split('\n')
+        .map(line => line.trim())
+        .filter(line => line && !line.startsWith('#') && !line.startsWith('!'))
+        .map(pattern => {
+        // Strip trailing spaces (gitignore spec)
+        pattern = pattern.replace(/\s+$/, '');
+        // Directory-only patterns: trailing /
+        if (pattern.endsWith('/')) {
+            pattern = pattern.slice(0, -1);
+        }
+        // Rooted patterns: leading /
+        if (pattern.startsWith('/'))
+            return pattern.slice(1);
+        // Unrooted patterns without / match anywhere
+        if (!pattern.includes('/'))
+            return `**/${pattern}`;
+        return pattern;
+    })
+        .filter(p => p.length > 0);
+}
+function readGitignore(rootPath) {
+    const gitignorePath = path.join(rootPath, '.gitignore');
+    try {
+        const content = fs.readFileSync(gitignorePath, 'utf-8');
+        return parseGitignorePatterns(content);
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Map file extension to language name for the splitter.
+ */
+export function extensionToLanguage(ext) {
+    const map = {
+        '.ts': 'typescript', '.tsx': 'tsx',
+        '.js': 'javascript', '.jsx': 'javascript', '.mjs': 'javascript', '.cjs': 'javascript',
+        '.py': 'python', '.pyi': 'python',
+        '.go': 'go',
+        '.java': 'java',
+        '.rs': 'rust',
+        '.cpp': 'cpp', '.cc': 'cpp', '.cxx': 'cpp', '.c': 'c', '.h': 'cpp', '.hpp': 'cpp',
+        '.cs': 'csharp',
+        '.scala': 'scala',
+        '.rb': 'ruby', '.php': 'php', '.swift': 'swift',
+        '.kt': 'kotlin', '.kts': 'kotlin',
+        '.lua': 'lua', '.sh': 'bash', '.bash': 'bash', '.zsh': 'bash',
+        '.sql': 'sql', '.r': 'r', '.R': 'r',
+        '.dart': 'dart', '.ex': 'elixir', '.exs': 'elixir',
+        '.hs': 'haskell', '.ml': 'ocaml',
+        '.vue': 'vue', '.svelte': 'svelte', '.astro': 'astro',
+        '.yaml': 'yaml', '.yml': 'yaml', '.toml': 'toml', '.json': 'json',
+        '.md': 'markdown', '.mdx': 'markdown',
+        '.html': 'html', '.css': 'css', '.scss': 'scss', '.less': 'less',
+    };
+    return map[ext.toLowerCase()] ?? 'unknown';
+}
+//# sourceMappingURL=sync.js.map

package/dist/embedding/factory.d.ts

@@ -0,0 +1,17 @@
+import type { Config } from '../config.js';
+import type { Embedding } from './types.js';
+/**
+ * Create an Embedding instance based on the configured provider.
+ *
+ * - 'openai'  Uses the OpenAI API directly (requires OPENAI_API_KEY).
+ * - 'ollama'  Uses Ollama's OpenAI-compatible /v1/embeddings endpoint.
+ *             No API key required; defaults to model "nomic-embed-text".
+ * - 'local'   Uses any OpenAI-compatible server at OPENAI_BASE_URL.
+ *             Useful for LM Studio, vLLM, LocalAI, etc.
+ *
+ * The key insight is that Ollama and most local servers expose an
+ * OpenAI-compatible embeddings API, so we reuse OpenAIEmbedding
+ * with different connection parameters rather than creating separate classes.
+ */
+export declare function createEmbedding(config: Config): Embedding;
+//# sourceMappingURL=factory.d.ts.map

package/dist/embedding/factory.js

@@ -0,0 +1,41 @@
+import { OpenAIEmbedding } from './openai.js';
+/**
+ * Create an Embedding instance based on the configured provider.
+ *
+ * - 'openai'  Uses the OpenAI API directly (requires OPENAI_API_KEY).
+ * - 'ollama'  Uses Ollama's OpenAI-compatible /v1/embeddings endpoint.
+ *             No API key required; defaults to model "nomic-embed-text".
+ * - 'local'   Uses any OpenAI-compatible server at OPENAI_BASE_URL.
+ *             Useful for LM Studio, vLLM, LocalAI, etc.
+ *
+ * The key insight is that Ollama and most local servers expose an
+ * OpenAI-compatible embeddings API, so we reuse OpenAIEmbedding
+ * with different connection parameters rather than creating separate classes.
+ */
+export function createEmbedding(config) {
+    switch (config.embeddingProvider) {
+        case 'openai':
+            return new OpenAIEmbedding();
+        case 'ollama':
+            // Ollama exposes OpenAI-compatible /v1/embeddings endpoint.
+            // It ignores the API key but the OpenAI SDK requires a non-empty string.
+            return new OpenAIEmbedding({
+                apiKey: config.openaiApiKey || 'ollama',
+                baseUrl: config.ollamaBaseUrl,
+                model: config.embeddingModel,
+            });
+        case 'local':
+            // Generic OpenAI-compatible endpoint (LM Studio, vLLM, LocalAI, etc.).
+            // API key is optional (many local servers skip auth).
+            return new OpenAIEmbedding({
+                apiKey: config.openaiApiKey || 'local',
+                baseUrl: config.openaiBaseUrl,
+                model: config.embeddingModel,
+            });
+        default: {
+            const _exhaustive = config.embeddingProvider;
+            throw new Error(`Unknown embedding provider: ${_exhaustive}`);
+        }
+    }
+}
+//# sourceMappingURL=factory.js.map

package/dist/embedding/openai.d.ts

@@ -0,0 +1,45 @@
+import { type Embedding, type EmbeddingVector } from './types.js';
+export declare function contentHash(text: string): string;
+export interface OpenAIEmbeddingOptions {
+    apiKey?: string;
+    baseUrl?: string;
+    model?: string;
+}
+export declare class OpenAIEmbedding implements Embedding {
+    private client;
+    private model;
+    private _dimension;
+    private initialized;
+    private memoryCache;
+    private cacheDir;
+    constructor(options?: OpenAIEmbeddingOptions);
+    get dimension(): number;
+    /**
+     * Validate the API key and detect embedding dimension by embedding a test string.
+     * Must be called once before any other operations.
+     */
+    initialize(): Promise<void>;
+    private ensureInitialized;
+    embed(text: string): Promise<EmbeddingVector>;
+    embedBatch(texts: string[]): Promise<EmbeddingVector[]>;
+    /**
+     * Estimate the token cost for embedding a set of texts.
+     * Rough heuristic: ~4 chars per token for code.
+     * Cost rates are model-specific; local models (Ollama, etc.) are free.
+     */
+    estimateTokens(texts: string[]): {
+        totalChars: number;
+        estimatedTokens: number;
+        estimatedCostUsd: number;
+    };
+    /**
+     * Set a value in the memory cache, evicting the oldest entry if at capacity.
+     */
+    private setMemoryCache;
+    private callWithRetry;
+    private callApi;
+    private getDiskCachePath;
+    private readDiskCache;
+    private writeDiskCache;
+}
+//# sourceMappingURL=openai.d.ts.map

package/dist/embedding/openai.js

@@ -0,0 +1,243 @@
+import { createHash } from 'node:crypto';
+import fsp from 'node:fs/promises';
+import path from 'node:path';
+import OpenAI from 'openai';
+import { EmbeddingError } from '../errors.js';
+import { getConfig } from '../config.js';
+import { getCacheDir } from '../paths.js';
+import { truncateToSafeLength } from './truncate.js';
+const RETRY_DELAYS = [1000, 4000, 16000]; // exponential backoff
+const RETRYABLE_STATUS = new Set([429, 500, 502, 503]);
+const MAX_MEMORY_CACHE_SIZE = 10_000;
+const MAX_RETRY_AFTER_MS = 60_000;
+export function contentHash(text) {
+    return createHash('sha256').update(text).digest('hex').slice(0, 16);
+}
+export class OpenAIEmbedding {
+    client;
+    model;
+    _dimension = 0;
+    initialized = false;
+    memoryCache = new Map();
+    cacheDir;
+    constructor(options) {
+        const config = getConfig();
+        const apiKey = options?.apiKey ?? config.openaiApiKey;
+        const baseUrl = options?.baseUrl ?? config.openaiBaseUrl;
+        this.client = new OpenAI({
+            apiKey,
+            ...(baseUrl && { baseURL: baseUrl }),
+        });
+        this.model = options?.model ?? config.embeddingModel;
+        this.cacheDir = path.join(getCacheDir(), 'embeddings');
+    }
+    get dimension() {
+        return this._dimension;
+    }
+    /**
+     * Validate the API key and detect embedding dimension by embedding a test string.
+     * Must be called once before any other operations.
+     */
+    async initialize() {
+        try {
+            const result = await this.callApi(['dimension probe']);
+            this._dimension = result[0].length;
+            this.initialized = true;
+            console.log(`Embedding model "${this.model}" validated. Dimension: ${this._dimension}`);
+        }
+        catch (err) {
+            throw new EmbeddingError(`Failed to initialize embedding provider. Check your API key, base URL, and model name. ` +
+                `Model: "${this.model}"`, err);
+        }
+    }
+    ensureInitialized() {
+        if (!this.initialized) {
+            throw new EmbeddingError('Embedding provider not initialized. Call initialize() before embed/embedBatch.');
+        }
+    }
+    async embed(text) {
+        this.ensureInitialized();
+        const results = await this.embedBatch([text]);
+        return results[0];
+    }
+    async embedBatch(texts) {
+        this.ensureInitialized();
+        if (texts.length === 0)
+            return [];
+        // Track which indices have empty/whitespace-only text so we return zero vectors for them
+        const emptyIndices = new Set();
+        for (let i = 0; i < texts.length; i++) {
+            if (texts[i].trim().length === 0) {
+                emptyIndices.add(i);
+            }
+        }
+        const results = new Array(texts.length).fill(null);
+        // Fill empty-text slots with zero vectors immediately
+        for (const i of emptyIndices) {
+            results[i] = new Array(this._dimension).fill(0);
+        }
+        // Check caches for non-empty texts
+        for (let i = 0; i < texts.length; i++) {
+            if (emptyIndices.has(i))
+                continue;
+            const hash = contentHash(texts[i]);
+            const memHit = this.memoryCache.get(hash);
+            if (memHit) {
+                results[i] = memHit;
+                continue;
+            }
+            const diskHit = await this.readDiskCache(hash);
+            if (diskHit) {
+                this.setMemoryCache(hash, diskHit);
+                results[i] = diskHit;
+            }
+        }
+        const uncachedIndices = [];
+        const uncachedTexts = [];
+        for (let i = 0; i < texts.length; i++) {
+            if (results[i] === null) {
+                uncachedIndices.push(i);
+                uncachedTexts.push(texts[i]);
+            }
+        }
+        if (uncachedTexts.length === 0) {
+            return results;
+        }
+        const batchSize = getConfig().embeddingBatchSize;
+        const freshEmbeddings = [];
+        for (let offset = 0; offset < uncachedTexts.length; offset += batchSize) {
+            const batch = uncachedTexts.slice(offset, offset + batchSize);
+            const batchResult = await this.callWithRetry(batch);
+            freshEmbeddings.push(...batchResult);
+        }
+        for (let i = 0; i < uncachedIndices.length; i++) {
+            const idx = uncachedIndices[i];
+            const hash = contentHash(texts[idx]);
+            const vec = freshEmbeddings[i];
+            this.setMemoryCache(hash, vec);
+            // Fire-and-forget: don't await the disk write
+            this.writeDiskCache(hash, vec);
+            results[idx] = vec;
+        }
+        // Verify no nulls remain before casting
+        if (results.some(r => r === null)) {
+            throw new EmbeddingError('Missing embeddings: some texts did not receive vectors after cache lookup and API call.');
+        }
+        return results;
+    }
+    /**
+     * Estimate the token cost for embedding a set of texts.
+     * Rough heuristic: ~4 chars per token for code.
+     * Cost rates are model-specific; local models (Ollama, etc.) are free.
+     */
+    estimateTokens(texts) {
+        const totalChars = texts.reduce((sum, t) => sum + t.length, 0);
+        const estimatedTokens = Math.ceil(totalChars / 4);
+        // Per-million-token pricing for known OpenAI models; local models are free
+        const COST_PER_MILLION = {
+            'text-embedding-3-small': 0.02,
+            'text-embedding-3-large': 0.13,
+            'text-embedding-ada-002': 0.10,
+        };
+        const rate = COST_PER_MILLION[this.model] ?? 0;
+        const estimatedCostUsd = (estimatedTokens / 1_000_000) * rate;
+        return { totalChars, estimatedTokens, estimatedCostUsd };
+    }
+    /**
+     * Set a value in the memory cache, evicting the oldest entry if at capacity.
+     */
+    setMemoryCache(hash, vec) {
+        if (this.memoryCache.size >= MAX_MEMORY_CACHE_SIZE && !this.memoryCache.has(hash)) {
+            // Delete the oldest entry (first key from the iterator)
+            const oldest = this.memoryCache.keys().next().value;
+            if (oldest !== undefined) {
+                this.memoryCache.delete(oldest);
+            }
+        }
+        this.memoryCache.set(hash, vec);
+    }
+    async callWithRetry(texts) {
+        let currentBatchSize = texts.length;
+        for (let attempt = 0; attempt <= RETRY_DELAYS.length; attempt++) {
+            try {
+                const allResults = [];
+                for (let offset = 0; offset < texts.length; offset += currentBatchSize) {
+                    const batch = texts.slice(offset, offset + currentBatchSize);
+                    const result = await this.callApi(batch);
+                    allResults.push(...result);
+                }
+                return allResults;
+            }
+            catch (err) {
+                const status = err.status;
+                const isRetryable = status !== undefined && RETRYABLE_STATUS.has(status);
+                if (!isRetryable || attempt >= RETRY_DELAYS.length) {
+                    throw new EmbeddingError(`Embedding API call failed after ${attempt + 1} attempt(s). Status: ${status ?? 'unknown'}`, err);
+                }
+                let delay = RETRY_DELAYS[attempt];
+                if (status === 429) {
+                    const retryAfter = err.headers?.['retry-after'];
+                    if (retryAfter) {
+                        const parsed = parseInt(retryAfter, 10);
+                        if (!isNaN(parsed))
+                            delay = Math.min(parsed * 1000, MAX_RETRY_AFTER_MS);
+                    }
+                    // Halve batch size on rate limit to avoid repeated throttling
+                    currentBatchSize = Math.max(1, Math.floor(currentBatchSize / 2));
+                    console.warn(`Rate limited. Retrying in ${delay}ms with batch size ${currentBatchSize}.`);
+                }
+                else {
+                    console.warn(`Embedding API error (status ${status}). Retrying in ${delay}ms...`);
+                }
+                await sleep(delay);
+            }
+        }
+        // Unreachable but satisfies TypeScript compiler
+        throw new EmbeddingError('Unexpected: exhausted retries');
+    }
+    async callApi(texts) {
+        const response = await this.client.embeddings.create({
+            model: this.model,
+            input: texts.map(truncateToSafeLength),
+        });
+        // Sort by index to guarantee order (API may return out-of-order)
+        const sorted = response.data.sort((a, b) => a.index - b.index);
+        return sorted.map(d => d.embedding);
+    }
+    getDiskCachePath(hash) {
+        // Shard into subdirectories to avoid too many files in one dir
+        const shard = hash.slice(0, 2);
+        return path.join(this.cacheDir, this.model.replace(/[^a-zA-Z0-9-]/g, '_'), shard, `${hash}.json`);
+    }
+    async readDiskCache(hash) {
+        const filepath = this.getDiskCachePath(hash);
+        try {
+            const data = await fsp.readFile(filepath, 'utf-8');
+            return JSON.parse(data);
+        }
+        catch (err) {
+            const code = err.code;
+            if (code === 'ENOENT') {
+                // File doesn't exist -- normal cache miss
+                return null;
+            }
+            // Parse error or other I/O problem: warn and remove corrupted file
+            console.warn(`Corrupted embedding cache file ${filepath}, deleting.`);
+            fsp.unlink(filepath).catch(() => { });
+            return null;
+        }
+    }
+    writeDiskCache(hash, vector) {
+        const filepath = this.getDiskCachePath(hash);
+        // Fire-and-forget async write
+        fsp.mkdir(path.dirname(filepath), { recursive: true })
+            .then(() => fsp.writeFile(filepath, JSON.stringify(vector)))
+            .catch(() => {
+            // Non-fatal: cache write failure doesn't block indexing
+        });
+    }
+}
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+//# sourceMappingURL=openai.js.map

package/dist/embedding/truncate.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Truncate text to stay within the embedding model's token limit.
+ * Cuts at the last newline boundary to avoid splitting mid-line.
+ */
+export declare function truncateToSafeLength(text: string): string;
+//# sourceMappingURL=truncate.d.ts.map

package/dist/embedding/truncate.js

@@ -0,0 +1,14 @@
+// Safety margin below 8,191 token limit (~4 chars/token for code)
+const MAX_EMBED_CHARS = 6000;
+/**
+ * Truncate text to stay within the embedding model's token limit.
+ * Cuts at the last newline boundary to avoid splitting mid-line.
+ */
+export function truncateToSafeLength(text) {
+    if (text.length <= MAX_EMBED_CHARS)
+        return text;
+    const truncated = text.slice(0, MAX_EMBED_CHARS);
+    const lastNewline = truncated.lastIndexOf('\n');
+    return lastNewline > 0 ? truncated.slice(0, lastNewline) : truncated;
+}
+//# sourceMappingURL=truncate.js.map

package/dist/embedding/types.d.ts

@@ -0,0 +1,18 @@
+export type EmbeddingVector = number[];
+export interface TokenEstimate {
+    totalChars: number;
+    estimatedTokens: number;
+    estimatedCostUsd: number;
+}
+export interface Embedding {
+    /**
+     * Validate provider connectivity and detect embedding dimension.
+     * Must be called once before any embed/embedBatch operations.
+     */
+    initialize(): Promise<void>;
+    embed(text: string): Promise<EmbeddingVector>;
+    embedBatch(texts: string[]): Promise<EmbeddingVector[]>;
+    estimateTokens(texts: string[]): TokenEstimate;
+    readonly dimension: number;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/embedding/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/errors.d.ts

@@ -0,0 +1,17 @@
+export declare class EideticError extends Error {
+    readonly cause?: unknown | undefined;
+    constructor(message: string, cause?: unknown | undefined);
+}
+export declare class ConfigError extends EideticError {
+}
+export declare class EmbeddingError extends EideticError {
+}
+export declare class VectorDBError extends EideticError {
+}
+export declare class IndexingError extends EideticError {
+}
+export declare class SearchError extends EideticError {
+}
+export declare class BootstrapError extends EideticError {
+}
+//# sourceMappingURL=errors.d.ts.map

package/dist/errors.js

@@ -0,0 +1,21 @@
+export class EideticError extends Error {
+    cause;
+    constructor(message, cause) {
+        super(message);
+        this.cause = cause;
+        this.name = this.constructor.name;
+    }
+}
+export class ConfigError extends EideticError {
+}
+export class EmbeddingError extends EideticError {
+}
+export class VectorDBError extends EideticError {
+}
+export class IndexingError extends EideticError {
+}
+export class SearchError extends EideticError {
+}
+export class BootstrapError extends EideticError {
+}
+//# sourceMappingURL=errors.js.map

package/dist/format.d.ts

@@ -0,0 +1,12 @@
+import type { PreviewResult, IndexResult } from './core/indexer.js';
+import type { CodebaseState } from './state/snapshot.js';
+export declare function textResult(text: string): {
+    content: {
+        type: "text";
+        text: string;
+    }[];
+};
+export declare function formatIndexResult(result: IndexResult, normalizedPath: string): string;
+export declare function formatPreview(preview: PreviewResult, rootPath: string): string;
+export declare function formatListIndexed(states: CodebaseState[]): string;
+//# sourceMappingURL=format.d.ts.map