@xfabric/memory 0.1.0

package/dist/memory-manager.js

@@ -0,0 +1,1055 @@
+import { join, relative, resolve, dirname, basename } from "node:path";
+import { readFile, stat, readdir, copyFile, rename, unlink } from "node:fs/promises";
+import { existsSync, mkdirSync } from "node:fs";
+import { hashText } from "./utils/hash.js";
+import { retry } from "./utils/retry.js";
+import { Semaphore } from "./utils/concurrency.js";
+import { createEmbeddingProvider, } from "./providers/index.js";
+import { MemoryStorage } from "./storage/sqlite.js";
+import { loadSqliteVec } from "./storage/sqlite-vec.js";
+import { chunkMarkdown } from "./chunking/markdown.js";
+import { chunkSession } from "./chunking/session.js";
+import { FileWatcher } from "./sync/watcher.js";
+import { searchVector, searchVectorWithSqliteVec } from "./search/vector.js";
+import { searchFts } from "./search/fts.js";
+import { mergeHybridResults } from "./search/hybrid.js";
+import { computeProviderKey } from "./providers/index.js";
+// Cache of MemoryManager instances
+const INDEX_CACHE = new Map();
+// Default batch configuration
+const DEFAULT_BATCH_CONFIG = {
+    enabled: true,
+    maxBatchSize: 100,
+    concurrency: 5,
+    timeoutMs: 60000,
+    maxConsecutiveFailures: 3,
+};
+/**
+ * Memory manager for indexing and searching workspace content
+ */
+export class MemoryManager {
+    config;
+    storage;
+    providerResult = null;
+    watcher = null;
+    syncPromise = null;
+    embeddingSemaphore;
+    closed = false;
+    // Dirty flags for triggering sync
+    memoryDirty = false;
+    sessionsDirty = false;
+    memoryDirtyFiles = new Set();
+    sessionsDirtyFiles = new Set();
+    // Provider fallback tracking
+    consecutiveEmbeddingFailures = 0;
+    fallbackActivated = false;
+    fallbackProvider = null;
+    // Batch processing state
+    batchDisabled = false;
+    batchDisableReason;
+    // sqlite-vec availability
+    sqliteVecLoaded = false;
+    sqliteVecError;
+    // Interval sync timer
+    syncIntervalTimer = null;
+    // Last sync timestamp
+    lastSyncAt;
+    constructor(config, storage) {
+        this.config = config;
+        this.storage = storage;
+        this.embeddingSemaphore = new Semaphore(config.batch.concurrency);
+    }
+    /**
+     * Create or get a cached MemoryManager instance
+     */
+    static async create(config) {
+        const extConfig = config;
+        const normalizedConfig = {
+            ...config,
+            workspaceDir: resolve(config.workspaceDir),
+            provider: config.provider ?? "openai",
+            chunking: {
+                tokens: config.chunking?.tokens ?? 400,
+                overlap: config.chunking?.overlap ?? 80,
+            },
+            query: {
+                maxResults: config.query?.maxResults ?? 10,
+                minScore: config.query?.minScore ?? 0.35,
+                hybrid: {
+                    vectorWeight: config.query?.hybrid?.vectorWeight ?? 0.7,
+                    textWeight: config.query?.hybrid?.textWeight ?? 0.3,
+                },
+            },
+            sync: {
+                watchDebounceMs: config.sync?.watchDebounceMs ?? 1500,
+                enabled: config.sync?.enabled ?? true,
+                intervalMs: extConfig.sync?.intervalMs ?? 0,
+                syncSessions: extConfig.sync?.syncSessions ?? false,
+            },
+            batch: {
+                ...DEFAULT_BATCH_CONFIG,
+                ...extConfig.batch,
+            },
+            sessionsDir: extConfig.sessionsDir,
+            extraPaths: extConfig.extraPaths,
+            onProgress: extConfig.onProgress,
+        };
+        // Generate cache key
+        const cacheKey = `${normalizedConfig.agentId}:${normalizedConfig.workspaceDir}`;
+        // Check cache
+        const cached = INDEX_CACHE.get(cacheKey);
+        if (cached && !cached.closed) {
+            return cached;
+        }
+        // Create storage
+        const dbPath = normalizedConfig.store?.path ??
+            join(normalizedConfig.workspaceDir, ".memory", "index.sqlite");
+        const storage = new MemoryStorage({ path: dbPath });
+        const manager = new MemoryManager(normalizedConfig, storage);
+        // Try to load sqlite-vec
+        manager.tryLoadSqliteVec();
+        // Initialize embedding provider
+        await manager.initProvider();
+        // Initial sync
+        await manager.sync();
+        // Start file watcher if enabled
+        if (normalizedConfig.sync.enabled) {
+            await manager.startWatching();
+        }
+        // Start interval sync if configured
+        if (normalizedConfig.sync.intervalMs > 0) {
+            manager.startIntervalSync();
+        }
+        // Cache instance
+        INDEX_CACHE.set(cacheKey, manager);
+        return manager;
+    }
+    /**
+     * Try to load sqlite-vec extension
+     */
+    tryLoadSqliteVec() {
+        try {
+            this.sqliteVecLoaded = loadSqliteVec(this.storage.database);
+        }
+        catch (err) {
+            this.sqliteVecError = err instanceof Error ? err.message : String(err);
+            this.sqliteVecLoaded = false;
+        }
+    }
+    /**
+     * Initialize the embedding provider
+     */
+    async initProvider() {
+        const apiKey = this.config.remote?.apiKey ?? process.env.OPENAI_API_KEY;
+        this.providerResult = await createEmbeddingProvider(this.config.provider, {
+            apiKey,
+            baseUrl: this.config.remote?.baseUrl,
+            model: this.config.remote?.model,
+            modelPath: this.config.local?.modelPath,
+        }, {
+            geminiApiKey: process.env.GEMINI_API_KEY,
+        });
+    }
+    /**
+     * Get the embedding provider (with fallback support)
+     */
+    get provider() {
+        if (this.fallbackActivated && this.fallbackProvider) {
+            return this.fallbackProvider;
+        }
+        if (!this.providerResult) {
+            throw new Error("Provider not initialized");
+        }
+        return this.providerResult.provider;
+    }
+    /**
+     * Get provider key for cache isolation
+     */
+    get providerKey() {
+        return computeProviderKey(this.provider.id, this.config.remote?.baseUrl, this.config.remote?.model);
+    }
+    /**
+     * Search memory for relevant content
+     */
+    async search(query, options = {}) {
+        const maxResults = options.maxResults ?? this.config.query.maxResults;
+        const minScore = options.minScore ?? this.config.query.minScore;
+        // Get query embedding
+        const queryEmbedding = await this.embedWithFallback(query);
+        // Vector search - use sqlite-vec if available
+        const vectorResults = this.sqliteVecLoaded
+            ? searchVectorWithSqliteVec(this.storage.database, queryEmbedding, {
+                maxResults: maxResults * 2,
+                minScore,
+                source: options.source,
+            })
+            : searchVector(this.storage.database, queryEmbedding, {
+                maxResults: maxResults * 2,
+                minScore,
+                source: options.source,
+            });
+        // FTS search
+        const keywordResults = searchFts(this.storage.database, query, {
+            maxResults: maxResults * 2,
+            source: options.source,
+        });
+        // Merge results
+        const merged = mergeHybridResults(vectorResults, keywordResults, this.config.query.hybrid);
+        // Filter by minimum score and limit
+        return merged.filter((r) => r.score >= minScore).slice(0, maxResults);
+    }
+    /**
+     * Embed text with automatic fallback on provider failures
+     */
+    async embedWithFallback(text) {
+        try {
+            const result = await this.provider.embedQuery(text);
+            // Reset failure counter on success
+            this.consecutiveEmbeddingFailures = 0;
+            return result;
+        }
+        catch (err) {
+            this.consecutiveEmbeddingFailures++;
+            // Attempt fallback if we've hit threshold and haven't already activated
+            if (this.consecutiveEmbeddingFailures >= this.config.batch.maxConsecutiveFailures &&
+                !this.fallbackActivated) {
+                await this.activateFallbackProvider(err instanceof Error ? err.message : String(err));
+                // Retry with fallback
+                if (this.fallbackProvider) {
+                    return this.fallbackProvider.embedQuery(text);
+                }
+            }
+            throw err;
+        }
+    }
+    /**
+     * Activate fallback provider
+     */
+    async activateFallbackProvider(reason) {
+        if (this.fallbackActivated)
+            return;
+        // Determine fallback based on current provider
+        const currentId = this.provider.id;
+        let fallbackType = null;
+        if (currentId === "openai" && process.env.GEMINI_API_KEY) {
+            fallbackType = "gemini";
+        }
+        else if (currentId === "gemini" && (this.config.remote?.apiKey ?? process.env.OPENAI_API_KEY)) {
+            fallbackType = "openai";
+        }
+        if (!fallbackType)
+            return;
+        try {
+            const result = await createEmbeddingProvider(fallbackType, {
+                apiKey: fallbackType === "gemini" ? process.env.GEMINI_API_KEY : this.config.remote?.apiKey,
+                baseUrl: this.config.remote?.baseUrl,
+            }, {});
+            this.fallbackProvider = result.provider;
+            this.fallbackActivated = true;
+            this.providerResult.fallbackFrom = currentId;
+            this.providerResult.fallbackReason = reason;
+        }
+        catch {
+            // Fallback failed, continue with original provider errors
+        }
+    }
+    /**
+     * Read file content with optional line range
+     */
+    async readFile(params) {
+        const absPath = join(this.config.workspaceDir, params.relPath);
+        if (!existsSync(absPath)) {
+            throw new Error(`File not found: ${params.relPath}`);
+        }
+        const content = await readFile(absPath, "utf-8");
+        const allLines = content.split("\n");
+        const startLine = Math.max(1, params.from ?? 1);
+        const endLine = params.lines
+            ? Math.min(startLine + params.lines - 1, allLines.length)
+            : allLines.length;
+        const selectedLines = allLines.slice(startLine - 1, endLine);
+        return {
+            content: selectedLines.join("\n"),
+            startLine,
+            endLine,
+        };
+    }
+    /**
+     * Synchronize memory index with filesystem
+     */
+    async sync() {
+        // Deduplicate concurrent sync calls
+        if (this.syncPromise) {
+            return this.syncPromise;
+        }
+        this.syncPromise = this.performSync();
+        try {
+            await this.syncPromise;
+        }
+        finally {
+            this.syncPromise = null;
+        }
+    }
+    /**
+     * Perform atomic reindex into a temporary database
+     * On success, swap temp DB with main DB
+     * On failure, restore original
+     */
+    async reindex() {
+        const mainDbPath = this.storage.path;
+        if (mainDbPath === ":memory:") {
+            // For in-memory DB, just do a regular sync
+            await this.performFullSync();
+            return;
+        }
+        const tempDbPath = `${mainDbPath}.tmp`;
+        const backupDbPath = `${mainDbPath}.backup`;
+        // Ensure temp directory exists
+        mkdirSync(dirname(tempDbPath), { recursive: true });
+        try {
+            // Create temp storage
+            const tempStorage = new MemoryStorage({ path: tempDbPath });
+            // Try to load sqlite-vec in temp DB
+            try {
+                loadSqliteVec(tempStorage.database);
+            }
+            catch {
+                // Ignore - sqlite-vec optional
+            }
+            // Index all files into temp DB
+            await this.indexAllFilesInto(tempStorage);
+            // Close temp storage
+            tempStorage.close();
+            // Backup current DB
+            await copyFile(mainDbPath, backupDbPath);
+            // Close main storage
+            this.storage.close();
+            // Swap temp to main
+            await rename(tempDbPath, mainDbPath);
+            // Reopen main storage
+            this.storage = new MemoryStorage({ path: mainDbPath });
+            this.tryLoadSqliteVec();
+            // Clean up backup
+            try {
+                await unlink(backupDbPath);
+            }
+            catch {
+                // Ignore cleanup errors
+            }
+            this.lastSyncAt = Date.now();
+            this.memoryDirty = false;
+            this.sessionsDirty = false;
+            this.memoryDirtyFiles.clear();
+            this.sessionsDirtyFiles.clear();
+        }
+        catch (err) {
+            // Restore from backup if it exists
+            if (existsSync(backupDbPath)) {
+                try {
+                    this.storage.close();
+                    await rename(backupDbPath, mainDbPath);
+                    this.storage = new MemoryStorage({ path: mainDbPath });
+                    this.tryLoadSqliteVec();
+                }
+                catch {
+                    // Last resort - reopen original if rename fails
+                    this.storage = new MemoryStorage({ path: mainDbPath });
+                }
+            }
+            // Clean up temp if it exists
+            if (existsSync(tempDbPath)) {
+                try {
+                    await unlink(tempDbPath);
+                }
+                catch {
+                    // Ignore
+                }
+            }
+            throw err;
+        }
+    }
+    /**
+     * Index all files into a specific storage instance
+     */
+    async indexAllFilesInto(storage) {
+        const memoryDir = join(this.config.workspaceDir, "memory");
+        if (existsSync(memoryDir)) {
+            const files = await this.scanMemoryFiles(memoryDir);
+            for (const file of files) {
+                await this.indexFileInto(file, storage, "memory");
+            }
+        }
+        // Index extra paths
+        if (this.config.extraPaths) {
+            for (const extraPath of this.config.extraPaths) {
+                if (existsSync(extraPath)) {
+                    const files = await this.scanMemoryFiles(extraPath);
+                    for (const file of files) {
+                        await this.indexFileInto(file, storage, "memory");
+                    }
+                }
+            }
+        }
+        // Index sessions if configured
+        if (this.config.sync.syncSessions && this.config.sessionsDir) {
+            await this.indexSessionsInto(storage);
+        }
+    }
+    /**
+     * Perform full sync (for reindex operation)
+     */
+    async performFullSync() {
+        // Delete all existing data
+        this.storage.deleteBySource("memory");
+        this.storage.deleteBySource("sessions");
+        // Re-index everything
+        await this.performSync();
+    }
+    /**
+     * Perform the actual sync operation
+     */
+    async performSync() {
+        const progress = {
+            phase: "scanning",
+            source: "memory",
+            totalFiles: 0,
+            processedFiles: 0,
+            totalChunks: 0,
+            embeddedChunks: 0,
+            errors: [],
+        };
+        this.reportProgress(progress);
+        // Sync memory files
+        await this.syncMemoryFiles(progress);
+        // Sync sessions if configured
+        if (this.config.sync.syncSessions && this.config.sessionsDir) {
+            progress.source = "sessions";
+            await this.syncSessionFiles(progress);
+        }
+        progress.phase = "complete";
+        this.reportProgress(progress);
+        this.lastSyncAt = Date.now();
+        this.memoryDirty = false;
+        this.sessionsDirty = false;
+        this.memoryDirtyFiles.clear();
+        this.sessionsDirtyFiles.clear();
+    }
+    /**
+     * Sync memory markdown files
+     */
+    async syncMemoryFiles(progress) {
+        const memoryDir = join(this.config.workspaceDir, "memory");
+        if (!existsSync(memoryDir)) {
+            return;
+        }
+        // Get current files
+        const files = await this.scanMemoryFiles(memoryDir);
+        // Also scan extra paths
+        if (this.config.extraPaths) {
+            for (const extraPath of this.config.extraPaths) {
+                if (existsSync(extraPath)) {
+                    const extraFiles = await this.scanMemoryFiles(extraPath);
+                    files.push(...extraFiles);
+                }
+            }
+        }
+        const storedFiles = this.storage.getAllFiles();
+        const storedMemoryFiles = storedFiles.filter((f) => f.source === "memory");
+        const storedFileMap = new Map(storedMemoryFiles.map((f) => [f.path, f]));
+        // Find files to add, update, or remove
+        const toProcess = [];
+        const currentPaths = new Set();
+        progress.totalFiles = files.length;
+        for (const file of files) {
+            currentPaths.add(file.path);
+            const stored = storedFileMap.get(file.path);
+            if (!stored || stored.hash !== file.hash) {
+                toProcess.push(file);
+            }
+        }
+        // Remove deleted files
+        for (const stored of storedMemoryFiles) {
+            if (!currentPaths.has(stored.path)) {
+                this.storage.deleteFile(stored.path);
+                this.storage.deleteChunksByPath(stored.path);
+            }
+        }
+        // Process changed files
+        progress.phase = "chunking";
+        this.reportProgress(progress);
+        for (const file of toProcess) {
+            try {
+                await this.indexFile(file);
+                progress.processedFiles++;
+                this.reportProgress(progress);
+            }
+            catch (err) {
+                progress.errors.push({
+                    file: file.path,
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            }
+        }
+    }
+    /**
+     * Sync session files
+     */
+    async syncSessionFiles(progress) {
+        if (!this.config.sessionsDir)
+            return;
+        const sessionsDir = this.config.sessionsDir;
+        if (!existsSync(sessionsDir))
+            return;
+        // Find all JSONL files
+        const sessionFiles = await this.scanSessionFiles(sessionsDir);
+        const storedFiles = this.storage.getFilesBySource("sessions");
+        const storedFileMap = new Map(storedFiles.map((f) => [f.path, f]));
+        progress.totalFiles = sessionFiles.length;
+        for (const file of sessionFiles) {
+            const stored = storedFileMap.get(file.path);
+            // Check if file changed (using mtime or hash)
+            if (!stored || stored.hash !== file.hash) {
+                try {
+                    await this.indexSessionFile(file, stored?.byteOffset ?? 0);
+                    progress.processedFiles++;
+                    this.reportProgress(progress);
+                }
+                catch (err) {
+                    progress.errors.push({
+                        file: file.path,
+                        error: err instanceof Error ? err.message : String(err),
+                    });
+                }
+            }
+        }
+    }
+    /**
+     * Scan session directory for JSONL files
+     */
+    async scanSessionFiles(dir) {
+        const files = [];
+        try {
+            const entries = await readdir(dir, { withFileTypes: true });
+            for (const entry of entries) {
+                const absPath = join(dir, entry.name);
+                if (entry.isDirectory()) {
+                    const subFiles = await this.scanSessionFiles(absPath);
+                    files.push(...subFiles);
+                }
+                else if (entry.isFile() && entry.name.endsWith(".jsonl")) {
+                    const fileStat = await stat(absPath);
+                    const content = await readFile(absPath, "utf-8");
+                    const hash = hashText(content);
+                    files.push({
+                        path: absPath, // Use absolute path for sessions
+                        absPath,
+                        mtimeMs: fileStat.mtimeMs,
+                        size: fileStat.size,
+                        hash,
+                    });
+                }
+            }
+        }
+        catch {
+            // Directory might not exist or be accessible
+        }
+        return files;
+    }
+    /**
+     * Index a session JSONL file with delta tracking
+     */
+    async indexSessionFile(file, lastByteOffset) {
+        const content = await readFile(file.absPath, "utf-8");
+        // Only process new content if we have a previous offset
+        const newContent = lastByteOffset > 0 ? content.slice(lastByteOffset) : content;
+        if (!newContent.trim()) {
+            // Just update the file record with new mtime
+            this.storage.upsertFile({
+                path: file.path,
+                source: "sessions",
+                hash: file.hash,
+                mtime: file.mtimeMs,
+                size: file.size,
+            });
+            return;
+        }
+        // Extract session ID from filename
+        const sessionId = basename(file.path, ".jsonl");
+        // Chunk the session content
+        const chunks = chunkSession(newContent, sessionId, {
+            tokens: this.config.chunking.tokens,
+            overlap: this.config.chunking.overlap,
+            startByteOffset: lastByteOffset,
+        });
+        if (chunks.length === 0) {
+            return;
+        }
+        // Generate embeddings
+        const texts = chunks.map((c) => c.text);
+        const embeddings = await this.embedBatch(texts);
+        const now = Date.now();
+        const storedChunks = chunks.map((chunk, i) => ({
+            id: `${file.path}:${chunk.byteOffset}:${chunk.chunkIndex}`,
+            path: file.path,
+            source: "sessions",
+            startLine: chunk.startLine,
+            endLine: chunk.endLine,
+            hash: chunk.hash,
+            model: this.provider.model,
+            text: chunk.text,
+            embedding: embeddings[i],
+            updatedAt: now,
+            sessionId: chunk.sessionId,
+            role: chunk.role,
+            byteOffset: chunk.byteOffset,
+        }));
+        // Store chunks in batch
+        this.storage.upsertChunksBatch(storedChunks);
+        // Update file record with new byte offset
+        this.storage.upsertFile({
+            path: file.path,
+            source: "sessions",
+            hash: file.hash,
+            mtime: file.mtimeMs,
+            size: file.size,
+        });
+        this.storage.updateFileByteOffset(file.path, content.length);
+    }
+    /**
+     * Index sessions into a specific storage
+     */
+    async indexSessionsInto(storage) {
+        if (!this.config.sessionsDir)
+            return;
+        const sessionsDir = this.config.sessionsDir;
+        if (!existsSync(sessionsDir))
+            return;
+        const sessionFiles = await this.scanSessionFiles(sessionsDir);
+        for (const file of sessionFiles) {
+            const content = await readFile(file.absPath, "utf-8");
+            const sessionId = basename(file.path, ".jsonl");
+            const chunks = chunkSession(content, sessionId, {
+                tokens: this.config.chunking.tokens,
+                overlap: this.config.chunking.overlap,
+                startByteOffset: 0,
+            });
+            if (chunks.length === 0)
+                continue;
+            const texts = chunks.map((c) => c.text);
+            const embeddings = await this.embedBatch(texts);
+            const now = Date.now();
+            const storedChunks = chunks.map((chunk, i) => ({
+                id: `${file.path}:${chunk.byteOffset}:${chunk.chunkIndex}`,
+                path: file.path,
+                source: "sessions",
+                startLine: chunk.startLine,
+                endLine: chunk.endLine,
+                hash: chunk.hash,
+                model: this.provider.model,
+                text: chunk.text,
+                embedding: embeddings[i],
+                updatedAt: now,
+                sessionId: chunk.sessionId,
+                role: chunk.role,
+                byteOffset: chunk.byteOffset,
+            }));
+            storage.upsertChunksBatch(storedChunks);
+            storage.upsertFile({
+                path: file.path,
+                source: "sessions",
+                hash: file.hash,
+                mtime: file.mtimeMs,
+                size: file.size,
+            });
+        }
+    }
+    /**
+     * Report sync progress
+     */
+    reportProgress(progress) {
+        if (this.config.onProgress) {
+            this.config.onProgress({ ...progress });
+        }
+    }
+    /**
+     * Scan memory directory for markdown files
+     */
+    async scanMemoryFiles(dir) {
+        const files = [];
+        const entries = await readdir(dir, { withFileTypes: true });
+        for (const entry of entries) {
+            const absPath = join(dir, entry.name);
+            if (entry.isDirectory()) {
+                const subFiles = await this.scanMemoryFiles(absPath);
+                files.push(...subFiles);
+            }
+            else if (entry.isFile() && entry.name.endsWith(".md")) {
+                const fileStat = await stat(absPath);
+                const content = await readFile(absPath, "utf-8");
+                const hash = hashText(content);
+                files.push({
+                    path: relative(this.config.workspaceDir, absPath),
+                    absPath,
+                    mtimeMs: fileStat.mtimeMs,
+                    size: fileStat.size,
+                    hash,
+                });
+            }
+        }
+        return files;
+    }
+    /**
+     * Index a single file
+     */
+    async indexFile(file) {
+        const content = await readFile(file.absPath, "utf-8");
+        // Chunk the content
+        const chunks = chunkMarkdown(content, this.config.chunking);
+        // Delete existing chunks for this file
+        this.storage.deleteChunksByPath(file.path);
+        // Generate embeddings and store chunks
+        const texts = chunks.map((c) => c.text);
+        const embeddings = await this.embedBatch(texts);
+        const now = Date.now();
+        const storedChunks = chunks.map((chunk, i) => ({
+            id: `${file.path}:${chunk.startLine}`,
+            path: file.path,
+            source: "memory",
+            startLine: chunk.startLine,
+            endLine: chunk.endLine,
+            hash: chunk.hash,
+            model: this.provider.model,
+            text: chunk.text,
+            embedding: embeddings[i],
+            updatedAt: now,
+        }));
+        // Store chunks in batch
+        this.storage.upsertChunksBatch(storedChunks);
+        // Update file record
+        this.storage.upsertFile({
+            path: file.path,
+            source: "memory",
+            hash: file.hash,
+            mtime: file.mtimeMs,
+            size: file.size,
+        });
+    }
+    /**
+     * Index a single file into a specific storage
+     */
+    async indexFileInto(file, storage, source) {
+        const content = await readFile(file.absPath, "utf-8");
+        const chunks = chunkMarkdown(content, this.config.chunking);
+        const texts = chunks.map((c) => c.text);
+        const embeddings = await this.embedBatch(texts);
+        const now = Date.now();
+        const storedChunks = chunks.map((chunk, i) => ({
+            id: `${file.path}:${chunk.startLine}`,
+            path: file.path,
+            source,
+            startLine: chunk.startLine,
+            endLine: chunk.endLine,
+            hash: chunk.hash,
+            model: this.provider.model,
+            text: chunk.text,
+            embedding: embeddings[i],
+            updatedAt: now,
+        }));
+        storage.upsertChunksBatch(storedChunks);
+        storage.upsertFile({
+            path: file.path,
+            source,
+            hash: file.hash,
+            mtime: file.mtimeMs,
+            size: file.size,
+        });
+    }
+    /**
+     * Embed a batch of texts with caching and rate limiting
+     */
+    async embedBatch(texts) {
+        if (texts.length === 0) {
+            return [];
+        }
+        const provider = this.provider;
+        const providerKey = this.providerKey;
+        // Check cache for existing embeddings
+        const results = [];
+        const uncachedTexts = [];
+        for (let i = 0; i < texts.length; i++) {
+            const text = texts[i];
+            const hash = hashText(text);
+            const cached = this.storage.getCachedEmbedding(provider.id, provider.model, providerKey, hash);
+            if (cached) {
+                results[i] = cached;
+            }
+            else {
+                results[i] = null;
+                uncachedTexts.push({ index: i, text, hash });
+            }
+        }
+        // Embed uncached texts in batches
+        if (uncachedTexts.length > 0) {
+            const batchSize = this.config.batch.maxBatchSize;
+            for (let i = 0; i < uncachedTexts.length; i += batchSize) {
+                const batch = uncachedTexts.slice(i, i + batchSize);
+                const batchTexts = batch.map((t) => t.text);
+                // Use semaphore for rate limiting
+                const embeddings = await this.embeddingSemaphore.run(() => retry(async () => {
+                    try {
+                        const result = await provider.embedBatch(batchTexts);
+                        this.consecutiveEmbeddingFailures = 0;
+                        return result;
+                    }
+                    catch (err) {
+                        this.consecutiveEmbeddingFailures++;
+                        if (this.consecutiveEmbeddingFailures >= this.config.batch.maxConsecutiveFailures &&
+                            !this.fallbackActivated) {
+                            await this.activateFallbackProvider(err instanceof Error ? err.message : String(err));
+                            if (this.fallbackProvider) {
+                                return this.fallbackProvider.embedBatch(batchTexts);
+                            }
+                        }
+                        throw err;
+                    }
+                }, {
+                    maxAttempts: 3,
+                    baseDelayMs: 1000,
+                }));
+                // Store in cache and results
+                for (let j = 0; j < batch.length; j++) {
+                    const { index, hash } = batch[j];
+                    const embedding = embeddings[j];
+                    results[index] = embedding;
+                    this.storage.cacheEmbedding(provider.id, provider.model, providerKey, hash, embedding);
+                }
+            }
+        }
+        return results;
+    }
+    /**
+     * Start watching for file changes
+     */
+    async startWatching() {
+        if (this.watcher) {
+            return;
+        }
+        const patterns = ["memory/**/*.md"];
+        this.watcher = new FileWatcher(this.config.workspaceDir, {
+            debounceMs: this.config.sync.watchDebounceMs,
+            patterns,
+            onEvent: (event) => {
+                this.memoryDirty = true;
+                this.memoryDirtyFiles.add(event.path);
+                // Trigger sync on file changes
+                this.sync().catch(() => { });
+            },
+        });
+        await this.watcher.start();
+    }
+    /**
+     * Start interval-based sync
+     */
+    startIntervalSync() {
+        if (this.syncIntervalTimer) {
+            return;
+        }
+        this.syncIntervalTimer = setInterval(() => {
+            if (this.memoryDirty || this.sessionsDirty) {
+                this.sync().catch(() => { });
+            }
+        }, this.config.sync.intervalMs);
+    }
+    /**
+     * Stop interval-based sync
+     */
+    stopIntervalSync() {
+        if (this.syncIntervalTimer) {
+            clearInterval(this.syncIntervalTimer);
+            this.syncIntervalTimer = null;
+        }
+    }
+    /**
+     * Stop watching for file changes
+     */
+    async stopWatching() {
+        if (this.watcher) {
+            await this.watcher.stop();
+            this.watcher = null;
+        }
+    }
+    /**
+     * Close the manager and release resources
+     */
+    async close() {
+        this.stopIntervalSync();
+        await this.stopWatching();
+        this.storage.close();
+        this.closed = true;
+        // Remove from cache
+        const cacheKey = `${this.config.agentId}:${this.config.workspaceDir}`;
+        INDEX_CACHE.delete(cacheKey);
+    }
+    /**
+     * Get provider information
+     */
+    getProviderInfo() {
+        if (!this.providerResult) {
+            return { provider: "unknown", model: "unknown" };
+        }
+        return {
+            provider: this.fallbackActivated && this.fallbackProvider
+                ? this.fallbackProvider.id
+                : this.providerResult.provider.id,
+            model: this.fallbackActivated && this.fallbackProvider
+                ? this.fallbackProvider.model
+                : this.providerResult.provider.model,
+            fallbackFrom: this.providerResult.fallbackFrom,
+            fallbackReason: this.providerResult.fallbackReason,
+        };
+    }
+    /**
+     * Get comprehensive status information
+     */
+    getStatus() {
+        const sourceCounts = this.storage.getSourceCounts();
+        const memoryStats = sourceCounts.find((s) => s.source === "memory") ?? { files: 0, chunks: 0 };
+        const sessionStats = sourceCounts.find((s) => s.source === "sessions") ?? { files: 0, chunks: 0 };
+        const provider = this.providerResult?.provider ?? { id: "unknown", model: "unknown", dimensions: 0 };
+        return {
+            database: {
+                path: this.storage.path,
+                sizeBytes: this.storage.getSizeBytes(),
+                schemaVersion: this.storage.schemaStatus.version,
+            },
+            sources: {
+                memory: { files: memoryStats.files, chunks: memoryStats.chunks },
+                sessions: { files: sessionStats.files, chunks: sessionStats.chunks },
+            },
+            fts: {
+                available: this.storage.schemaStatus.ftsAvailable,
+                error: this.storage.schemaStatus.ftsError,
+            },
+            vector: {
+                sqliteVecAvailable: this.sqliteVecLoaded,
+                usingJsFallback: !this.sqliteVecLoaded,
+                error: this.sqliteVecError,
+            },
+            provider: {
+                id: this.fallbackActivated && this.fallbackProvider
+                    ? this.fallbackProvider.id
+                    : provider.id,
+                model: this.fallbackActivated && this.fallbackProvider
+                    ? this.fallbackProvider.model
+                    : provider.model,
+                dimensions: this.fallbackActivated && this.fallbackProvider
+                    ? (this.fallbackProvider.dimensions ?? 0)
+                    : (provider.dimensions ?? 0),
+                fallbackActivated: this.fallbackActivated,
+                fallbackFrom: this.providerResult?.fallbackFrom,
+                fallbackReason: this.providerResult?.fallbackReason,
+                consecutiveFailures: this.consecutiveEmbeddingFailures,
+            },
+            batch: {
+                enabled: this.config.batch.enabled && !this.batchDisabled,
+                pendingJobs: 0, // TODO: track pending batch jobs
+                autoDisabled: this.batchDisabled,
+                disableReason: this.batchDisableReason,
+            },
+            sync: {
+                watching: this.watcher !== null,
+                intervalMs: this.config.sync.intervalMs > 0 ? this.config.sync.intervalMs : undefined,
+                lastSyncAt: this.lastSyncAt,
+                memoryDirty: this.memoryDirty,
+                sessionsDirty: this.sessionsDirty,
+            },
+        };
+    }
+    /**
+     * Probe vector search availability
+     */
+    probeVectorAvailability() {
+        const start = Date.now();
+        if (this.sqliteVecLoaded) {
+            try {
+                // Test a simple vector operation
+                this.storage.database.prepare("SELECT vec_version()").get();
+                return {
+                    available: true,
+                    latencyMs: Date.now() - start,
+                };
+            }
+            catch (err) {
+                return {
+                    available: false,
+                    error: err instanceof Error ? err.message : String(err),
+                    latencyMs: Date.now() - start,
+                };
+            }
+        }
+        // JS fallback is always available
+        return {
+            available: true,
+            latencyMs: Date.now() - start,
+        };
+    }
+    /**
+     * Probe embedding API availability
+     */
+    async probeEmbeddingAvailability() {
+        const start = Date.now();
+        try {
+            await this.provider.embedQuery("test");
+            return {
+                available: true,
+                latencyMs: Date.now() - start,
+            };
+        }
+        catch (err) {
+            return {
+                available: false,
+                error: err instanceof Error ? err.message : String(err),
+                latencyMs: Date.now() - start,
+            };
+        }
+    }
+    /**
+     * Probe FTS5 availability
+     */
+    probeFtsAvailability() {
+        const start = Date.now();
+        try {
+            this.storage.database.prepare("SELECT 1 FROM chunks_fts LIMIT 0").run();
+            return {
+                available: true,
+                latencyMs: Date.now() - start,
+            };
+        }
+        catch (err) {
+            return {
+                available: false,
+                error: err instanceof Error ? err.message : String(err),
+                latencyMs: Date.now() - start,
+            };
+        }
+    }
+    /**
+     * Mark memory files as dirty (for external triggers)
+     */
+    markMemoryDirty(files) {
+        this.memoryDirty = true;
+        if (files) {
+            files.forEach((f) => this.memoryDirtyFiles.add(f));
+        }
+    }
+    /**
+     * Mark session files as dirty (for external triggers)
+     */
+    markSessionsDirty(files) {
+        this.sessionsDirty = true;
+        if (files) {
+            files.forEach((f) => this.sessionsDirtyFiles.add(f));
+        }
+    }
+}
+//# sourceMappingURL=memory-manager.js.map