grepmind-core 0.1.0-alpha

package/dist/indexer/index.js

@@ -0,0 +1,1096 @@
+// Main Indexer class with dependency injection
+// No singletons - all dependencies passed via constructor
+import * as lancedb from '@lancedb/lancedb';
+import { BooleanQuery, MatchQuery, Occur, PhraseQuery } from '@lancedb/lancedb';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { DEFAULT_RERANKING_CONFIG } from '../config/types.js';
+import { parseNote, generateContentHash } from './parser.js';
+import { chunkNote } from './chunker.js';
+import { createError, isError } from '../types.js';
+import { buildJsonStringArrayContainsAnyFilter, buildPathPrefixFilter, combineFilters, } from '../search/fts.js';
+/**
+ * Indexer class - manages the vector index for notes.
+ * All dependencies are passed via constructor (no singletons).
+ */
+export class Indexer {
+    config;
+    embeddingProvider;
+    rerankerProvider;
+    rerankingConfig;
+    vaultUtils;
+    chunkingConfig;
+    db = null;
+    chunksTable = null;
+    fileIndexTable = null;
+    initialized = false;
+    metaPath;
+    hasVectorIndex = false;
+    // Unified SyncWal (optional - when provided, uses unified recovery)
+    syncWal = null;
+    constructor(deps) {
+        this.config = deps.config;
+        this.embeddingProvider = deps.embeddingProvider;
+        this.rerankerProvider = deps.rerankerProvider ?? null;
+        this.rerankingConfig = deps.rerankingConfig ?? deps.config.reranking ?? DEFAULT_RERANKING_CONFIG;
+        this.vaultUtils = deps.vaultUtils;
+        this.chunkingConfig = deps.config.chunking;
+        const isRemoteDbPath = /^[a-z][a-z0-9+.-]*:\/\//i.test(deps.config.lancedb.dbPath);
+        if (isRemoteDbPath) {
+            const dbHash = crypto.createHash('sha1').update(deps.config.lancedb.dbPath).digest('hex').slice(0, 12);
+            this.metaPath = path.join(process.cwd(), '.lancedb-state', dbHash, 'meta.json');
+        }
+        else {
+            this.metaPath = path.join(deps.config.lancedb.dbPath, 'meta.json');
+        }
+        this.syncWal = deps.syncWal || null;
+    }
+    async loadMeta() {
+        try {
+            const content = await fs.readFile(this.metaPath, 'utf-8');
+            return JSON.parse(content);
+        }
+        catch {
+            return null;
+        }
+    }
+    async saveMeta() {
+        const modelName = this.embeddingProvider.getModelName();
+        const meta = {
+            model: modelName,
+            createdAt: Date.now(),
+            provider: this.config.embedding.provider,
+            dimensions: this.embeddingProvider.getDimensions(),
+        };
+        await fs.mkdir(path.dirname(this.metaPath), { recursive: true });
+        await fs.writeFile(this.metaPath, JSON.stringify(meta, null, 2));
+    }
+    async cleanupLegacyIndex() {
+        for (const vaultPath of this.config.vaultPaths) {
+            const legacyPath = path.join(vaultPath, '.index');
+            try {
+                await fs.access(legacyPath);
+                console.warn('[indexer] Removing legacy Vectra index at', legacyPath);
+                await fs.rm(legacyPath, { recursive: true });
+            }
+            catch {
+                // Directory doesn't exist
+            }
+        }
+    }
+    async initialize() {
+        if (this.initialized)
+            return;
+        await this.cleanupLegacyIndex();
+        // Initialize embedding provider
+        await this.embeddingProvider.initialize();
+        // Ensure local LanceDB directory exists (skip for remote URIs)
+        const isRemoteDbPath = /^[a-z][a-z0-9+.-]*:\/\//i.test(this.config.lancedb.dbPath);
+        if (!isRemoteDbPath) {
+            await fs.mkdir(this.config.lancedb.dbPath, { recursive: true });
+        }
+        // Connect to LanceDB
+        this.db = await lancedb.connect(this.config.lancedb.dbPath, {
+            storageOptions: this.config.lancedb.storageOptions,
+        });
+        const tableNames = await this.db.tableNames();
+        // Check if embedding config changed
+        const meta = await this.loadMeta();
+        const currentModel = this.embeddingProvider.getModelName();
+        const currentDimensions = this.embeddingProvider.getDimensions();
+        const needsRecreate = meta && (meta.model !== currentModel ||
+            meta.provider !== this.config.embedding.provider ||
+            meta.dimensions !== currentDimensions);
+        if (needsRecreate) {
+            console.warn('[indexer] Embedding config changed, recreating index...');
+            console.warn(`  Previous: model=${meta.model}, provider=${meta.provider}, dimensions=${meta.dimensions}`);
+            console.warn(`  Current:  model=${currentModel}, provider=${this.config.embedding.provider}, dimensions=${currentDimensions}`);
+            if (tableNames.includes(this.config.lancedb.chunksTable)) {
+                await this.db.dropTable(this.config.lancedb.chunksTable);
+            }
+            if (tableNames.includes(this.config.lancedb.fileIndexTable)) {
+                await this.db.dropTable(this.config.lancedb.fileIndexTable);
+            }
+        }
+        if (tableNames.includes(this.config.lancedb.chunksTable)) {
+            this.chunksTable = await this.db.openTable(this.config.lancedb.chunksTable);
+        }
+        if (tableNames.includes(this.config.lancedb.fileIndexTable)) {
+            this.fileIndexTable = await this.db.openTable(this.config.lancedb.fileIndexTable);
+        }
+        await this.saveMeta();
+        this.initialized = true;
+        // Auto-create all tables at startup
+        await this.ensureAllTables();
+    }
+    /**
+     * Ensure all notes index tables exist with proper schema.
+     * Safe to call multiple times (idempotent).
+     */
+    async ensureAllTables() {
+        if (!this.initialized) {
+            await this.initialize();
+            return; // initialize() already called ensureAllTables()
+        }
+        await this.ensureChunksTable();
+        await this.ensureFileIndexTable();
+    }
+    async ensureChunksTable() {
+        if (this.chunksTable)
+            return this.chunksTable;
+        const sampleData = [{
+                id: '__init__',
+                vector: new Array(this.embeddingProvider.getDimensions()).fill(0),
+                path: '',
+                section: '',
+                title: '',
+                tags: '[]',
+                links: '[]',
+                modified: 0,
+                content_hash: '',
+                content: '',
+            }];
+        this.chunksTable = await this.db.createTable(this.config.lancedb.chunksTable, sampleData, { mode: 'overwrite' });
+        await this.chunksTable.delete("id = '__init__'");
+        return this.chunksTable;
+    }
+    async ensureFileIndexTable() {
+        if (this.fileIndexTable)
+            return this.fileIndexTable;
+        const sampleData = [{
+                path: '__init__',
+                mtime: 0,
+                content_hash: '',
+                chunk_ids: '[]',
+                indexed_at: 0,
+            }];
+        this.fileIndexTable = await this.db.createTable(this.config.lancedb.fileIndexTable, sampleData, { mode: 'overwrite' });
+        await this.fileIndexTable.delete("path = '__init__'");
+        return this.fileIndexTable;
+    }
+    async getFileRecord(filePath) {
+        const table = await this.ensureFileIndexTable();
+        const results = await table.query()
+            .where(`path = '${filePath.replace(/'/g, "''")}'`)
+            .toArray();
+        if (results.length === 0)
+            return null;
+        const r = results[0];
+        return {
+            path: r.path,
+            mtime: r.mtime,
+            content_hash: r.content_hash,
+            chunk_ids: r.chunk_ids,
+            indexed_at: r.indexed_at,
+        };
+    }
+    async getAllIndexedPaths() {
+        const table = await this.ensureFileIndexTable();
+        const results = await table.query().select(['path']).toArray();
+        return new Set(results.map(r => r.path));
+    }
+    async updateFileIndex(filePath, mtime, contentHash, chunkIds) {
+        const table = await this.ensureFileIndexTable();
+        await table.delete(`path = '${filePath.replace(/'/g, "''")}'`);
+        await table.add([{
+                path: filePath,
+                mtime,
+                content_hash: contentHash,
+                chunk_ids: JSON.stringify(chunkIds),
+                indexed_at: Date.now(),
+            }]);
+    }
+    async removeFromFileIndex(filePath) {
+        const table = await this.ensureFileIndexTable();
+        await table.delete(`path = '${filePath.replace(/'/g, "''")}'`);
+    }
+    async indexChunk(chunk, modified) {
+        const contentHash = generateContentHash(chunk.content);
+        const table = await this.ensureChunksTable();
+        const existingResult = await table.query()
+            .where(`id = '${chunk.id}'`)
+            .toArray();
+        if (existingResult.length > 0 && existingResult[0].content_hash === contentHash) {
+            return chunk.id;
+        }
+        const embedding = await this.embeddingProvider.getEmbedding(chunk.content);
+        if (isError(embedding)) {
+            return embedding;
+        }
+        if (existingResult.length > 0) {
+            await table.delete(`id = '${chunk.id}'`);
+        }
+        await table.add([{
+                id: chunk.id,
+                vector: embedding,
+                path: chunk.path,
+                section: chunk.section,
+                title: chunk.title,
+                tags: JSON.stringify(chunk.tags),
+                links: JSON.stringify(chunk.links),
+                modified,
+                content_hash: contentHash,
+                content: chunk.content,
+            }]);
+        return chunk.id;
+    }
+    async deleteChunksForFile(chunkIds) {
+        if (chunkIds.length === 0)
+            return;
+        const table = await this.ensureChunksTable();
+        for (const id of chunkIds) {
+            await table.delete(`id = '${id}'`);
+        }
+    }
+    async removeChunksForPath(filePath) {
+        const table = await this.ensureChunksTable();
+        await table.delete(`path = '${filePath.replace(/'/g, "''")}'`);
+    }
+    async createOrUpdateVectorIndex() {
+        const table = await this.ensureChunksTable();
+        const count = await table.countRows();
+        if (count < 10) {
+            return;
+        }
+        console.log(`[indexer] Creating/updating IVF_PQ vector index (${count} chunks)...`);
+        try {
+            await table.createIndex('vector', {
+                config: lancedb.Index.ivfPq({
+                    numPartitions: Math.max(4, Math.min(256, Math.floor(count / 10))),
+                    numSubVectors: 16,
+                }),
+            });
+            this.hasVectorIndex = true;
+            console.log('[indexer] Vector index created successfully');
+        }
+        catch (err) {
+            const msg = err.message;
+            if (!msg.includes('already exists')) {
+                console.warn('[indexer] Could not create vector index:', msg);
+            }
+        }
+    }
+    async createOrUpdateFtsIndex() {
+        if (!this.config.notesFts.enabled) {
+            return;
+        }
+        const table = await this.ensureChunksTable();
+        const count = await table.countRows();
+        if (count === 0) {
+            return;
+        }
+        const indexName = 'content_idx';
+        console.log(`[indexer] Creating/updating FTS index (${count} chunks)...`);
+        try {
+            await table.createIndex('content', {
+                config: lancedb.Index.fts({
+                    withPosition: this.config.notesFts.withPosition,
+                    baseTokenizer: this.config.notesFts.baseTokenizer,
+                    language: this.config.notesFts.language,
+                    maxTokenLength: this.config.notesFts.maxTokenLength,
+                    lowercase: this.config.notesFts.lowercase,
+                    stem: this.config.notesFts.stem,
+                    removeStopWords: this.config.notesFts.removeStopWords,
+                    asciiFolding: this.config.notesFts.asciiFolding,
+                }),
+                replace: true,
+            });
+            await table.waitForIndex([indexName], 60);
+            console.log('[indexer] FTS index created successfully');
+        }
+        catch (err) {
+            console.warn('[indexer] Could not create FTS index:', err.message);
+        }
+    }
+    normalizeSearchScores(results) {
+        if (results.length === 0)
+            return [];
+        const values = results.map((r) => r.score);
+        const min = Math.min(...values);
+        const max = Math.max(...values);
+        if (max === min)
+            return values.map(() => 1);
+        return values.map((v) => (v - min) / (max - min));
+    }
+    buildHybridResultKey(result) {
+        return `${result.path}::${result.section ?? ''}::${result.content.slice(0, 120)}`;
+    }
+    async search(query, options = {}) {
+        const { limit = this.config.search.defaultLimit, threshold = this.config.search.defaultThreshold, tags, folder, rerank = true, } = options;
+        await this.initialize();
+        const table = await this.ensureChunksTable();
+        const count = await table.countRows();
+        if (count === 0) {
+            return [];
+        }
+        const queryEmbedding = await this.embeddingProvider.getEmbedding(query, 'query');
+        if (isError(queryEmbedding)) {
+            return queryEmbedding;
+        }
+        const useReranking = Boolean(rerank &&
+            this.rerankerProvider &&
+            this.rerankingConfig.enabled &&
+            this.rerankingConfig.provider !== 'none');
+        const fetchLimit = useReranking
+            ? Math.max(this.rerankingConfig.topK, limit)
+            : limit * 2;
+        const vectorThreshold = useReranking
+            ? this.rerankingConfig.prerankThreshold
+            : threshold;
+        let searchQuery = table.search(queryEmbedding).limit(fetchLimit);
+        const folderFilter = folder
+            ? buildPathPrefixFilter('path', folder, { ensureTrailingSlash: true })
+            : null;
+        if (folderFilter) {
+            searchQuery = searchQuery.where(folderFilter);
+        }
+        const results = await searchQuery.toArray();
+        const processed = [];
+        for (const r of results) {
+            const distance = r._distance;
+            const score = 1 - (distance / 2);
+            if (score < vectorThreshold)
+                continue;
+            const itemTags = JSON.parse(r.tags);
+            const itemLinks = JSON.parse(r.links);
+            if (tags && tags.length > 0) {
+                if (!tags.some((tag) => itemTags.includes(tag))) {
+                    continue;
+                }
+            }
+            processed.push({
+                path: r.path,
+                section: r.section,
+                score,
+                content: r.content,
+                title: r.title,
+                tags: itemTags,
+                links: itemLinks,
+            });
+            if (processed.length >= fetchLimit)
+                break;
+        }
+        if (processed.length === 0) {
+            return [];
+        }
+        if (useReranking && processed.length >= 5 && this.rerankerProvider) {
+            try {
+                if (!this.rerankerProvider.isInitialized()) {
+                    await this.rerankerProvider.initialize();
+                }
+                const documents = processed.map((item) => item.content);
+                const reranked = await this.rerankerProvider.rerank(query, documents, {
+                    topK: limit,
+                });
+                if (!isError(reranked)) {
+                    const rerankedResults = reranked
+                        .filter((rr) => rr.index >= 0 && rr.index < processed.length)
+                        .map((rr) => ({
+                        ...processed[rr.index],
+                        score: rr.score,
+                        _originalScore: processed[rr.index].score,
+                    }))
+                        .filter((result) => result.score >= threshold);
+                    return rerankedResults.slice(0, limit);
+                }
+                console.warn('[indexer] Reranking failed, using vector scores:', reranked.message);
+            }
+            catch (error) {
+                console.warn('[indexer] Reranking error, falling back to vector scores:', error);
+            }
+        }
+        return processed
+            .filter((result) => result.score >= threshold)
+            .slice(0, limit);
+    }
+    async searchFts(options) {
+        await this.initialize();
+        const query = options.query.trim();
+        if (!query) {
+            return createError('INVALID_QUERY', 'Query must not be empty');
+        }
+        const table = await this.ensureChunksTable();
+        const count = await table.countRows();
+        if (count === 0) {
+            return [];
+        }
+        const limit = options.limit ?? this.config.search.defaultLimit;
+        const hasFuzzyOptions = options.fuzziness !== undefined ||
+            options.maxExpansions !== undefined ||
+            options.prefixLength !== undefined;
+        const phraseQuery = options.phraseMatch
+            ? new PhraseQuery(query, 'content', { slop: options.phraseSlop ?? 0 })
+            : null;
+        const fuzzyQuery = hasFuzzyOptions
+            ? new MatchQuery(query, 'content', {
+                fuzziness: options.fuzziness ?? 1,
+                maxExpansions: options.maxExpansions,
+                prefixLength: options.prefixLength,
+            })
+            : null;
+        let ftsQuery = query;
+        if (phraseQuery && fuzzyQuery) {
+            ftsQuery = new BooleanQuery([
+                [Occur.Should, phraseQuery],
+                [Occur.Should, fuzzyQuery],
+            ]);
+        }
+        else if (phraseQuery) {
+            ftsQuery = phraseQuery;
+        }
+        else if (fuzzyQuery) {
+            ftsQuery = fuzzyQuery;
+        }
+        let searchQuery = table.search(ftsQuery, 'fts', 'content').limit(limit);
+        const whereClause = combineFilters([
+            options.folder
+                ? buildPathPrefixFilter('path', options.folder, { ensureTrailingSlash: true })
+                : null,
+            buildJsonStringArrayContainsAnyFilter('tags', options.tags),
+        ]);
+        if (whereClause) {
+            searchQuery = searchQuery.where(whereClause);
+        }
+        const rows = await searchQuery.toArray();
+        const results = rows.map((r) => {
+            let parsedTags = [];
+            let parsedLinks = [];
+            try {
+                parsedTags = JSON.parse(r.tags || '[]');
+            }
+            catch {
+                parsedTags = [];
+            }
+            try {
+                parsedLinks = JSON.parse(r.links || '[]');
+            }
+            catch {
+                parsedLinks = [];
+            }
+            const ftsScore = typeof r._score === 'number' ? r._score : undefined;
+            const fallbackScore = typeof r._distance === 'number'
+                ? 1 - (r._distance / 2)
+                : 0;
+            return {
+                path: r.path,
+                section: r.section,
+                score: ftsScore ?? fallbackScore,
+                content: r.content,
+                title: r.title,
+                tags: parsedTags,
+                links: parsedLinks,
+            };
+        });
+        return results;
+    }
+    async searchHybrid(options) {
+        const query = options.query.trim();
+        if (!query) {
+            return createError('INVALID_QUERY', 'Query must not be empty');
+        }
+        const limit = options.limit ?? this.config.search.defaultLimit;
+        const threshold = options.threshold ?? this.config.search.defaultThreshold;
+        const semanticLimit = options.semanticLimit ?? Math.max(limit * 3, 20);
+        const ftsLimit = options.ftsLimit ?? Math.max(limit * 3, 20);
+        const vectorWeight = options.vectorWeight ?? 0.55;
+        const ftsWeight = options.ftsWeight ?? 0.45;
+        const rerankEnabled = options.rerank ?? true;
+        const [semanticResults, ftsResults] = await Promise.all([
+            this.search(query, {
+                limit: semanticLimit,
+                threshold: 0,
+                tags: options.tags,
+                folder: options.folder,
+                rerank: false,
+            }),
+            this.searchFts({
+                query,
+                limit: ftsLimit,
+                fuzziness: options.fuzziness,
+                maxExpansions: options.maxExpansions,
+                prefixLength: options.prefixLength,
+                phraseMatch: options.phraseMatch,
+                phraseSlop: options.phraseSlop,
+                folder: options.folder,
+                tags: options.tags,
+            }),
+        ]);
+        if (isError(semanticResults))
+            return semanticResults;
+        if (isError(ftsResults))
+            return ftsResults;
+        if (semanticResults.length === 0 && ftsResults.length === 0)
+            return [];
+        const semanticNorm = this.normalizeSearchScores(semanticResults);
+        const ftsNorm = this.normalizeSearchScores(ftsResults);
+        const merged = new Map();
+        for (let i = 0; i < semanticResults.length; i++) {
+            const item = semanticResults[i];
+            const key = this.buildHybridResultKey(item);
+            const existing = merged.get(key);
+            if (existing) {
+                existing.vector = Math.max(existing.vector, semanticNorm[i] ?? 0);
+            }
+            else {
+                merged.set(key, { item, vector: semanticNorm[i] ?? 0, fts: 0 });
+            }
+        }
+        for (let i = 0; i < ftsResults.length; i++) {
+            const item = ftsResults[i];
+            const key = this.buildHybridResultKey(item);
+            const existing = merged.get(key);
+            if (existing) {
+                existing.fts = Math.max(existing.fts, ftsNorm[i] ?? 0);
+            }
+            else {
+                merged.set(key, { item, vector: 0, fts: ftsNorm[i] ?? 0 });
+            }
+        }
+        const fused = Array.from(merged.values())
+            .map(({ item, vector, fts }) => {
+            const overlapBoost = vector > 0 && fts > 0 ? 0.05 : 0;
+            const score = Math.min(1, vectorWeight * vector + ftsWeight * fts + overlapBoost);
+            return { ...item, score };
+        })
+            .sort((a, b) => b.score - a.score);
+        const useReranking = Boolean(rerankEnabled &&
+            this.rerankerProvider &&
+            this.rerankingConfig.enabled &&
+            this.rerankingConfig.provider !== 'none');
+        if (useReranking && fused.length >= 5 && this.rerankerProvider) {
+            try {
+                if (!this.rerankerProvider.isInitialized()) {
+                    await this.rerankerProvider.initialize();
+                }
+                const candidateLimit = Math.max(this.rerankingConfig.topK, limit);
+                const candidates = fused.slice(0, candidateLimit);
+                const reranked = await this.rerankerProvider.rerank(query, candidates.map((c) => c.content), { topK: limit });
+                if (!isError(reranked)) {
+                    return reranked
+                        .filter((rr) => rr.index >= 0 && rr.index < candidates.length)
+                        .map((rr) => ({
+                        ...candidates[rr.index],
+                        score: rr.score,
+                        _originalScore: candidates[rr.index].score,
+                    }))
+                        .filter((result) => result.score >= threshold)
+                        .slice(0, limit);
+                }
+                console.warn('[indexer] Hybrid reranking failed, using fused scores:', reranked.message);
+            }
+            catch (error) {
+                console.warn('[indexer] Hybrid reranking error, using fused scores:', error);
+            }
+        }
+        return fused
+            .filter((result) => result.score >= threshold)
+            .slice(0, limit);
+    }
+    async reindexAll() {
+        await this.initialize();
+        const files = await this.vaultUtils.globVaults('**/*.md');
+        let indexed = 0;
+        let skipped = 0;
+        let deleted = 0;
+        const errors = [];
+        const indexedPaths = await this.getAllIndexedPaths();
+        const currentPaths = new Set();
+        for (const file of files) {
+            const relativePath = this.vaultUtils.toRelativePath(file);
+            currentPaths.add(relativePath);
+            try {
+                const stat = await fs.stat(file);
+                const existingRecord = await this.getFileRecord(relativePath);
+                if (existingRecord && existingRecord.mtime === stat.mtimeMs) {
+                    skipped++;
+                    continue;
+                }
+                const content = await fs.readFile(file, 'utf-8');
+                const fileHash = generateContentHash(content);
+                if (existingRecord && existingRecord.content_hash === fileHash) {
+                    await this.updateFileIndex(relativePath, stat.mtimeMs, fileHash, JSON.parse(existingRecord.chunk_ids));
+                    skipped++;
+                    continue;
+                }
+                const note = parseNote(relativePath, content);
+                const chunks = chunkNote(note, this.chunkingConfig);
+                if (existingRecord) {
+                    const oldChunkIds = JSON.parse(existingRecord.chunk_ids);
+                    await this.deleteChunksForFile(oldChunkIds);
+                }
+                const newChunkIds = [];
+                for (const chunk of chunks) {
+                    const result = await this.indexChunk(chunk, stat.mtimeMs);
+                    if (isError(result)) {
+                        errors.push(result);
+                    }
+                    else {
+                        newChunkIds.push(result);
+                        indexed++;
+                    }
+                }
+                await this.updateFileIndex(relativePath, stat.mtimeMs, fileHash, newChunkIds);
+            }
+            catch (err) {
+                errors.push(createError('INDEX_FILE_ERROR', `Failed to index ${relativePath}`, {
+                    error: err.message,
+                }));
+            }
+        }
+        for (const indexedPath of indexedPaths) {
+            if (!currentPaths.has(indexedPath)) {
+                const record = await this.getFileRecord(indexedPath);
+                if (record) {
+                    const chunkIds = JSON.parse(record.chunk_ids);
+                    await this.deleteChunksForFile(chunkIds);
+                    await this.removeFromFileIndex(indexedPath);
+                    deleted++;
+                }
+            }
+        }
+        await this.createOrUpdateVectorIndex();
+        await this.createOrUpdateFtsIndex();
+        return { indexed, skipped, deleted, errors };
+    }
+    /**
+     * Compute manifest hash from sorted file paths for recovery validation.
+     * Both notes and code use the same hash algorithm.
+     */
+    computeManifestHash(files) {
+        const sortedPaths = files.map(f => this.vaultUtils.toRelativePath(f)).sort();
+        const content = sortedPaths.join('\n');
+        return crypto.createHash('sha256').update(content).digest('hex');
+    }
+    /**
+     * Reindex all notes using unified SyncWal for recovery.
+     * Uses offset-based recovery (deterministic file ordering).
+     *
+     * @param recovery Optional recovery plan from SyncWal
+     * @returns Result with counts
+     */
+    async reindexAllWithSyncWal(recovery) {
+        if (!this.syncWal) {
+            throw new Error('SyncWal not provided - use reindexAll() instead');
+        }
+        await this.initialize();
+        // Get all files sorted deterministically
+        let files = await this.vaultUtils.globVaults('**/*.md');
+        files = files.sort((a, b) => a.localeCompare(b));
+        // Compute manifest hash for recovery validation
+        const manifestHash = this.computeManifestHash(files);
+        // Validate recovery plan
+        let skipCount = 0;
+        if (recovery) {
+            if (recovery.manifestHash === manifestHash) {
+                skipCount = recovery.skipCount;
+                console.log(`[indexer] Resuming from file ${skipCount}/${files.length}`);
+            }
+            else {
+                console.warn('[indexer] Manifest hash mismatch, starting full reindex');
+            }
+        }
+        // Start task (acquires lock, starts heartbeat)
+        const taskId = await this.syncWal.startTask('notes', files.length, manifestHash);
+        let indexed = 0;
+        let skipped = 0;
+        let deleted = 0;
+        const errors = [];
+        const indexedPaths = await this.getAllIndexedPaths();
+        const currentPaths = new Set();
+        try {
+            for (let i = skipCount; i < files.length; i++) {
+                // Check cancellation
+                if (this.syncWal.isCancelled(taskId)) {
+                    await this.syncWal.cancelTask(taskId);
+                    return { indexed, skipped, deleted, errors };
+                }
+                // Check pause - wait until resumed or cancelled
+                const shouldContinue = await this.syncWal.waitWhilePaused(taskId);
+                if (!shouldContinue) {
+                    // Cancelled while paused
+                    await this.syncWal.cancelTask(taskId);
+                    return { indexed, skipped, deleted, errors };
+                }
+                const file = files[i];
+                const relativePath = this.vaultUtils.toRelativePath(file);
+                currentPaths.add(relativePath);
+                try {
+                    const stat = await fs.stat(file);
+                    const existingRecord = await this.getFileRecord(relativePath);
+                    if (existingRecord && existingRecord.mtime === stat.mtimeMs) {
+                        skipped++;
+                        await this.syncWal.fileProcessed(taskId, relativePath, 0, 'updated');
+                        continue;
+                    }
+                    const content = await fs.readFile(file, 'utf-8');
+                    const fileHash = generateContentHash(content);
+                    if (existingRecord && existingRecord.content_hash === fileHash) {
+                        await this.updateFileIndex(relativePath, stat.mtimeMs, fileHash, JSON.parse(existingRecord.chunk_ids));
+                        skipped++;
+                        await this.syncWal.fileProcessed(taskId, relativePath, 0, 'updated');
+                        continue;
+                    }
+                    const note = parseNote(relativePath, content);
+                    const chunks = chunkNote(note, this.chunkingConfig);
+                    if (existingRecord) {
+                        const oldChunkIds = JSON.parse(existingRecord.chunk_ids);
+                        await this.deleteChunksForFile(oldChunkIds);
+                    }
+                    const newChunkIds = [];
+                    for (const chunk of chunks) {
+                        const result = await this.indexChunk(chunk, stat.mtimeMs);
+                        if (isError(result)) {
+                            errors.push(result);
+                        }
+                        else {
+                            newChunkIds.push(result);
+                            indexed++;
+                        }
+                    }
+                    await this.updateFileIndex(relativePath, stat.mtimeMs, fileHash, newChunkIds);
+                    await this.syncWal.fileProcessed(taskId, relativePath, newChunkIds.length, existingRecord ? 'updated' : 'added');
+                }
+                catch (err) {
+                    const error = createError('INDEX_FILE_ERROR', `Failed to index ${relativePath}`, {
+                        error: err.message,
+                    });
+                    errors.push(error);
+                    await this.syncWal.fileFailed(taskId, relativePath, err.message, 0);
+                }
+            }
+            // Handle deleted files
+            this.syncWal.setPhase('deleting');
+            for (const indexedPath of indexedPaths) {
+                if (!currentPaths.has(indexedPath)) {
+                    const record = await this.getFileRecord(indexedPath);
+                    if (record) {
+                        const chunkIds = JSON.parse(record.chunk_ids);
+                        await this.deleteChunksForFile(chunkIds);
+                        await this.removeFromFileIndex(indexedPath);
+                        deleted++;
+                    }
+                }
+            }
+            await this.createOrUpdateVectorIndex();
+            await this.createOrUpdateFtsIndex();
+            // Complete task
+            await this.syncWal.completeTask(taskId);
+        }
+        catch (err) {
+            await this.syncWal.failTask(taskId, err.message);
+            throw err;
+        }
+        return { indexed, skipped, deleted, errors };
+    }
+    async rebuildFtsIndex() {
+        await this.initialize();
+        await this.createOrUpdateFtsIndex();
+    }
+    /**
+     * Set the SyncWal instance for unified recovery.
+     * Should be called when SyncWal is created after Indexer construction.
+     */
+    setSyncWal(syncWal) {
+        this.syncWal = syncWal;
+    }
+    /**
+     * Get the SyncWal instance.
+     */
+    getSyncWal() {
+        return this.syncWal;
+    }
+    /**
+     * Check if SyncWal is configured.
+     */
+    hasSyncWal() {
+        return this.syncWal !== null;
+    }
+    /**
+     * Clear all indexed data (chunks and file index).
+     * Used for force reindex.
+     */
+    async clearIndex() {
+        await this.initialize();
+        if (!this.db)
+            return;
+        const tableNames = await this.db.tableNames();
+        if (tableNames.includes(this.config.lancedb.chunksTable)) {
+            await this.db.dropTable(this.config.lancedb.chunksTable);
+            this.chunksTable = null;
+        }
+        if (tableNames.includes(this.config.lancedb.fileIndexTable)) {
+            await this.db.dropTable(this.config.lancedb.fileIndexTable);
+            this.fileIndexTable = null;
+        }
+    }
+    async indexFile(filePath) {
+        await this.initialize();
+        let absolutePath;
+        if (path.isAbsolute(filePath)) {
+            absolutePath = filePath;
+        }
+        else {
+            const resolved = this.vaultUtils.resolveNotePath(filePath);
+            if (!resolved) {
+                return createError('FILE_NOT_FOUND', `File not found in any vault: ${filePath}`);
+            }
+            absolutePath = resolved;
+        }
+        const relativePath = this.vaultUtils.toRelativePath(absolutePath);
+        try {
+            const content = await fs.readFile(absolutePath, 'utf-8');
+            const stat = await fs.stat(absolutePath);
+            const note = parseNote(relativePath, content);
+            const chunks = chunkNote(note, this.chunkingConfig);
+            const fileHash = generateContentHash(content);
+            await this.removeChunksForPath(relativePath);
+            const chunkIds = [];
+            for (const chunk of chunks) {
+                const result = await this.indexChunk(chunk, stat.mtimeMs);
+                if (isError(result))
+                    return result;
+                chunkIds.push(result);
+            }
+            await this.updateFileIndex(relativePath, stat.mtimeMs, fileHash, chunkIds);
+        }
+        catch (err) {
+            return createError('INDEX_FILE_ERROR', `Failed to index ${relativePath}`, {
+                error: err.message,
+            });
+        }
+    }
+    async removeFile(filePath) {
+        const relativePath = path.isAbsolute(filePath)
+            ? this.vaultUtils.toRelativePath(filePath)
+            : filePath;
+        const record = await this.getFileRecord(relativePath);
+        if (record) {
+            const chunkIds = JSON.parse(record.chunk_ids);
+            await this.deleteChunksForFile(chunkIds);
+            await this.removeFromFileIndex(relativePath);
+        }
+        else {
+            await this.removeChunksForPath(relativePath);
+        }
+    }
+    /**
+     * Reindex all notes with detailed profiling.
+     * Forces full reindex (ignores mtime/content_hash checks) to get accurate timing.
+     */
+    async reindexWithProfiling() {
+        const totalStart = performance.now();
+        await this.initialize();
+        const errors = [];
+        // ============================================================
+        // PHASE 1: PARSING
+        // ============================================================
+        const parsingStart = performance.now();
+        const files = await this.vaultUtils.globVaults('**/*.md');
+        const allChunks = [];
+        let filesProcessed = 0;
+        for (const file of files) {
+            const relativePath = this.vaultUtils.toRelativePath(file);
+            try {
+                const stat = await fs.stat(file);
+                const content = await fs.readFile(file, 'utf-8');
+                const fileHash = generateContentHash(content);
+                const note = parseNote(relativePath, content);
+                const chunks = chunkNote(note, this.chunkingConfig);
+                for (const chunk of chunks) {
+                    allChunks.push({
+                        chunk,
+                        mtime: stat.mtimeMs,
+                        filePath: relativePath,
+                        fileHash,
+                    });
+                }
+                filesProcessed++;
+            }
+            catch (err) {
+                errors.push(createError('INDEX_FILE_ERROR', `Failed to parse ${relativePath}`, {
+                    error: err.message,
+                }));
+            }
+        }
+        const parsingTimeMs = performance.now() - parsingStart;
+        // ============================================================
+        // PHASE 2: EMBEDDING
+        // ============================================================
+        const embeddingStart = performance.now();
+        // Extract all texts for batch embedding
+        const texts = allChunks.map((item) => item.chunk.content);
+        const batchSize = this.config.embedding.provider === 'voyage'
+            ? this.config.embedding.voyageBatchSize
+            : this.config.embedding.provider === 'ollama' || this.config.embedding.provider === 'localai'
+                ? 128
+                : 32;
+        const batchCount = Math.ceil(texts.length / batchSize);
+        let embeddings = [];
+        if (texts.length > 0) {
+            const result = await this.embeddingProvider.getEmbeddingsBatch(texts, batchSize);
+            if (isError(result)) {
+                errors.push(result);
+            }
+            else {
+                embeddings = result;
+            }
+        }
+        const embeddingTimeMs = performance.now() - embeddingStart;
+        const chunksEmbedded = embeddings.length;
+        // ============================================================
+        // PHASE 3: DB WRITE
+        // ============================================================
+        const dbWriteStart = performance.now();
+        // Clear existing data for full reindex
+        const table = await this.ensureChunksTable();
+        const fileTable = await this.ensureFileIndexTable();
+        // Delete all existing records
+        try {
+            await table.delete('id IS NOT NULL');
+        }
+        catch {
+            // Table might be empty
+        }
+        try {
+            await fileTable.delete('path IS NOT NULL');
+        }
+        catch {
+            // Table might be empty
+        }
+        // Prepare records for bulk insert
+        const chunkRecords = [];
+        const fileIndexMap = new Map();
+        for (let i = 0; i < allChunks.length; i++) {
+            const { chunk, mtime, filePath, fileHash } = allChunks[i];
+            const embedding = embeddings[i];
+            if (!embedding)
+                continue; // Skip if embedding failed
+            const contentHash = generateContentHash(chunk.content);
+            chunkRecords.push({
+                id: chunk.id,
+                vector: embedding,
+                path: chunk.path,
+                section: chunk.section,
+                title: chunk.title,
+                tags: JSON.stringify(chunk.tags),
+                links: JSON.stringify(chunk.links),
+                modified: mtime,
+                content_hash: contentHash,
+                content: chunk.content,
+            });
+            // Track for file index
+            if (!fileIndexMap.has(filePath)) {
+                fileIndexMap.set(filePath, { mtime, fileHash, chunkIds: [] });
+            }
+            fileIndexMap.get(filePath).chunkIds.push(chunk.id);
+        }
+        // Bulk insert chunks
+        let chunksWritten = 0;
+        if (chunkRecords.length > 0) {
+            await table.add(chunkRecords);
+            chunksWritten = chunkRecords.length;
+        }
+        // Insert file index records
+        const fileRecords = [];
+        for (const [filePath, data] of fileIndexMap) {
+            fileRecords.push({
+                path: filePath,
+                mtime: data.mtime,
+                content_hash: data.fileHash,
+                chunk_ids: JSON.stringify(data.chunkIds),
+                indexed_at: Date.now(),
+            });
+        }
+        if (fileRecords.length > 0) {
+            await fileTable.add(fileRecords);
+        }
+        const dbWriteTimeMs = performance.now() - dbWriteStart;
+        // ============================================================
+        // PHASE 4: VECTOR INDEX
+        // ============================================================
+        const vectorIndexStart = performance.now();
+        await this.createOrUpdateVectorIndex();
+        await this.createOrUpdateFtsIndex();
+        const vectorIndexTimeMs = performance.now() - vectorIndexStart;
+        const totalTimeMs = performance.now() - totalStart;
+        const stats = {
+            totalTimeMs,
+            phases: {
+                parsing: {
+                    timeMs: parsingTimeMs,
+                    filesProcessed,
+                    chunksGenerated: allChunks.length,
+                },
+                embedding: {
+                    timeMs: embeddingTimeMs,
+                    chunksEmbedded,
+                    batchCount,
+                },
+                dbWrite: {
+                    timeMs: dbWriteTimeMs,
+                    chunksWritten,
+                },
+                vectorIndex: {
+                    timeMs: vectorIndexTimeMs,
+                },
+            },
+        };
+        return { stats, indexed: chunksWritten, errors };
+    }
+    /**
+     * Get index statistics.
+     */
+    async getStats() {
+        await this.initialize();
+        const chunksTable = await this.ensureChunksTable();
+        const fileTable = await this.ensureFileIndexTable();
+        const totalChunks = await chunksTable.countRows();
+        const totalFiles = await fileTable.countRows();
+        return { totalChunks, totalFiles };
+    }
+    /**
+     * Get all indexed files with their metadata.
+     * Returns a Map of relative file path to index info.
+     */
+    async getIndexedFiles() {
+        await this.initialize();
+        const table = await this.ensureFileIndexTable();
+        const results = await table.query().toArray();
+        const map = new Map();
+        for (const r of results) {
+            const path = r.path;
+            const chunk_ids = r.chunk_ids;
+            const chunks = chunk_ids ? JSON.parse(chunk_ids).length : 0;
+            map.set(path, {
+                chunks,
+                indexed_at: r.indexed_at,
+                content_hash: r.content_hash,
+            });
+        }
+        return map;
+    }
+    /**
+     * Get the config for this indexer.
+     */
+    getConfig() {
+        return this.config;
+    }
+    /**
+     * Get the embedding provider for this indexer.
+     */
+    getEmbeddingProvider() {
+        return this.embeddingProvider;
+    }
+    /**
+     * Get the vault utils for this indexer.
+     */
+    getVaultUtils() {
+        return this.vaultUtils;
+    }
+}
+// Re-export everything from submodules
+export { parseNote, extractLinks, extractHashtags, extractTitle, generateContentHash } from './parser.js';
+export { chunkNote } from './chunker.js';
+export { WasmEmbeddingProvider, VoyageEmbeddingProvider, OllamaEmbeddingProvider, createEmbeddingProvider, } from './embeddings.js';
+//# sourceMappingURL=index.js.map