mono-pilot 0.2.9 → 0.2.12

package/dist/src/memory/index-manager.js

@@ -0,0 +1,552 @@
+import { readFile } from "node:fs/promises";
+import { relative } from "node:path";
+import chokidar from "chokidar";
+import { getAgentMemoryDir } from "../agents-paths.js";
+import { getMemoryIndexPath } from "./paths.js";
+import { getSessionTranscriptsRootDir } from "./session/transcript/paths.js";
+import { memoryLog } from "./log.js";
+import { openSqliteDatabase } from "./store/sqlite.js";
+import { ensureMemoryIndexSchema } from "./store/schema.js";
+import { FILES_TABLE, CHUNKS_TABLE, FTS_TABLE, VECTOR_TABLE } from "./store/schema.js";
+import { buildFileEntry, hashText, listMemoryFiles, resolveExtraPaths } from "./indexing/files.js";
+import { indexMemoryFile } from "./indexing/index-file.js";
+import { searchFts } from "./search/fts.js";
+import { searchVector } from "./search/vector.js";
+import { mergeHybridResults } from "./search/hybrid.js";
+import { loadSqliteVecExtension } from "./store/sqlite.js";
+const SNIPPET_MAX_CHARS = 400;
+export class MemoryIndexManager {
+    agentId;
+    workspaceDir;
+    settings;
+    db;
+    memoryDir;
+    sessionTranscriptsDir;
+    includeMemorySource;
+    includeSessionsSource;
+    ftsAvailable;
+    provider = null;
+    providerKey = null;
+    vectorAvailable = null;
+    vectorDims;
+    dirty = false;
+    watcher = null;
+    watchTimer = null;
+    intervalTimer = null;
+    syncInProgress = false;
+    onAutoSyncEvent;
+    embedFn = null;
+    embedModel = null;
+    constructor(params) {
+        this.agentId = params.agentId;
+        this.workspaceDir = params.workspaceDir;
+        this.settings = params.settings;
+        this.embedFn = params.embedFn ?? null;
+        this.embedModel = params.embedModel ?? null;
+        this.onAutoSyncEvent = params.onAutoSyncEvent;
+        this.memoryDir = getAgentMemoryDir(params.agentId);
+        this.sessionTranscriptsDir = getSessionTranscriptsRootDir(params.agentId);
+        this.includeMemorySource = this.settings.sources.includes("memory");
+        this.includeSessionsSource = this.settings.sources.includes("sessions");
+        this.db = openSqliteDatabase(getMemoryIndexPath(), true);
+        const schema = ensureMemoryIndexSchema({ db: this.db, ftsEnabled: true });
+        this.ftsAvailable = schema.ftsAvailable;
+        if (schema.ftsError) {
+            console.warn(`[memory] FTS unavailable: ${schema.ftsError}`);
+            memoryLog.warn("fts unavailable", { error: schema.ftsError, agentId: this.agentId });
+        }
+        this.dirty = true;
+        this.ensureWatcher();
+        this.ensureIntervalSync();
+        memoryLog.info("manager initialized", {
+            agentId: this.agentId,
+            memoryDir: this.memoryDir,
+            sessionTranscriptsDir: this.sessionTranscriptsDir,
+            includeMemorySource: this.includeMemorySource,
+            includeSessionsSource: this.includeSessionsSource,
+            indexPath: getMemoryIndexPath(),
+        });
+    }
+    async search(query, opts) {
+        const cleaned = query.trim();
+        if (!cleaned)
+            return [];
+        // Qwen3-Embedding instruct format: only on query side, documents stay raw.
+        // Format: "Instruct: {task}\nQuery:{text}" (no space after "Query:")
+        const DEFAULT_INSTRUCT = "Retrieve relevant code snippets or documentation passages that match the query";
+        const instruct = opts?.instruct?.trim() || DEFAULT_INSTRUCT;
+        const instructQuery = `Instruct: ${instruct}\nQuery:${cleaned}`;
+        if (this.settings.sync.onSearch && this.dirty) {
+            await this.sync({ reason: "search" }).catch((err) => {
+                console.warn(`[memory] sync failed (search): ${String(err)}`);
+                memoryLog.warn("sync failed (search)", { agentId: this.agentId, error: String(err) });
+            });
+        }
+        const maxResults = opts?.maxResults ?? this.settings.query.maxResults;
+        const minScore = opts?.minScore ?? this.settings.query.minScore;
+        const scope = opts?.scope ?? "self";
+        const targetAgentId = opts?.targetAgentId;
+        const agentFilter = scope === "all" ? undefined : scope === "agent" ? targetAgentId : this.agentId;
+        if (scope === "agent" && !targetAgentId) {
+            return [];
+        }
+        const defaultAgentId = scope === "all" ? undefined : agentFilter ?? this.agentId;
+        const resolveAgentId = (rowAgentId) => rowAgentId ?? defaultAgentId;
+        const querySnippet = cleaned.length > 200 ? `${cleaned.slice(0, 200)}...` : cleaned;
+        memoryLog.info("search start", {
+            agentId: this.agentId,
+            scope,
+            targetAgentId,
+            query: querySnippet,
+        });
+        const finish = (results, mode) => {
+            memoryLog.info("search complete", {
+                agentId: this.agentId,
+                scope,
+                targetAgentId,
+                mode,
+                resultCount: results.length,
+            });
+            return results;
+        };
+        const provider = await this.getProvider();
+        if (provider) {
+            try {
+                const queryEmbedding = await provider.embedQuery(instructQuery);
+                const vectorReady = await this.ensureVectorReady(queryEmbedding.length);
+                if (vectorReady) {
+                    const hybrid = this.settings.query.hybrid;
+                    const candidateLimit = Math.min(200, Math.max(1, Math.floor(maxResults * hybrid.candidateMultiplier)));
+                    const vectorSources = [];
+                    if (this.includeMemorySource)
+                        vectorSources.push("memory");
+                    if (this.includeSessionsSource)
+                        vectorSources.push("sessions");
+                    const vectorResults = (await Promise.all(vectorSources.map((source) => searchVector({
+                        db: this.db,
+                        queryVec: queryEmbedding,
+                        limit: hybrid.enabled ? candidateLimit : maxResults,
+                        snippetMaxChars: SNIPPET_MAX_CHARS,
+                        model: provider.model,
+                        agentId: agentFilter,
+                        source,
+                    })))).flat();
+                    if (!hybrid.enabled || !this.ftsAvailable) {
+                        return finish(vectorResults
+                            .map((row) => ({
+                            path: row.path,
+                            startLine: row.startLine,
+                            endLine: row.endLine,
+                            score: row.vectorScore,
+                            snippet: row.snippet,
+                            source: row.source,
+                            agentId: resolveAgentId(row.agentId),
+                        }))
+                            .filter((row) => row.score >= minScore)
+                            .slice(0, maxResults), "vector");
+                    }
+                    const keywordSources = [];
+                    if (this.includeMemorySource)
+                        keywordSources.push("memory");
+                    if (this.includeSessionsSource)
+                        keywordSources.push("sessions");
+                    const keywordResults = (await Promise.all(keywordSources.map((source) => Promise.resolve(searchFts({
+                        db: this.db,
+                        query: cleaned,
+                        limit: candidateLimit,
+                        minScore: 0,
+                        snippetMaxChars: SNIPPET_MAX_CHARS,
+                        model: provider.model,
+                        agentId: agentFilter,
+                        source,
+                    }))))).flat();
+                    const merged = mergeHybridResults({
+                        vector: vectorResults,
+                        keyword: keywordResults,
+                        vectorWeight: hybrid.vectorWeight,
+                        textWeight: hybrid.textWeight,
+                    });
+                    return finish(merged
+                        .filter((row) => row.score >= minScore)
+                        .sort((a, b) => b.score - a.score)
+                        .slice(0, maxResults)
+                        .map((row) => ({
+                        path: row.path,
+                        startLine: row.startLine,
+                        endLine: row.endLine,
+                        score: row.score,
+                        snippet: row.snippet,
+                        source: row.source,
+                        agentId: resolveAgentId(row.agentId),
+                    })), "hybrid");
+                }
+            }
+            catch (error) {
+                console.warn(`[memory] vector search failed: ${String(error)}`);
+                memoryLog.warn("vector search failed", {
+                    agentId: this.agentId,
+                    scope,
+                    targetAgentId,
+                    error: String(error),
+                });
+            }
+        }
+        if (!this.ftsAvailable)
+            return [];
+        const ftsSources = [];
+        if (this.includeMemorySource)
+            ftsSources.push("memory");
+        if (this.includeSessionsSource)
+            ftsSources.push("sessions");
+        const rows = (await Promise.all(ftsSources.map((source) => Promise.resolve(searchFts({
+            db: this.db,
+            query: cleaned,
+            limit: maxResults,
+            minScore,
+            snippetMaxChars: SNIPPET_MAX_CHARS,
+            model: provider?.model,
+            agentId: agentFilter,
+            source,
+        }))))).flat();
+        return finish(rows.map((row) => ({
+            path: row.path,
+            startLine: row.startLine,
+            endLine: row.endLine,
+            score: row.score,
+            snippet: row.snippet,
+            source: row.source,
+            agentId: resolveAgentId(row.agentId),
+        })), "fts");
+    }
+    async get(path, from, lines) {
+        const raw = await readFile(path, "utf-8");
+        if (from === undefined && lines === undefined) {
+            return { path, text: raw };
+        }
+        const startLine = Math.max(1, Math.floor(from ?? 1));
+        const maxLines = lines !== undefined ? Math.max(0, Math.floor(lines)) : undefined;
+        const allLines = raw.split("\n");
+        const startIndex = Math.min(allLines.length, Math.max(0, startLine - 1));
+        const endIndex = maxLines === undefined ? allLines.length : Math.min(allLines.length, startIndex + maxLines);
+        return { path, text: allLines.slice(startIndex, endIndex).join("\n") };
+    }
+    isDirty() {
+        return this.dirty;
+    }
+    async sync(params) {
+        const reason = params?.reason ?? "unknown";
+        const isAutoSyncReason = reason === "search" || reason === "watch" || reason === "interval";
+        if (this.syncInProgress) {
+            memoryLog.debug("sync skipped (already running)", {
+                agentId: this.agentId,
+                reason,
+            });
+            return;
+        }
+        this.syncInProgress = true;
+        let workDetected = false;
+        const notifyWorkDetected = () => {
+            if (workDetected) {
+                return;
+            }
+            workDetected = true;
+            if (isAutoSyncReason) {
+                this.onAutoSyncEvent?.({ reason, phase: "start" });
+            }
+            params?.onWorkDetected?.();
+        };
+        const start = Date.now();
+        memoryLog.info("sync start", {
+            agentId: this.agentId,
+            reason,
+            force: params?.force ?? false,
+        });
+        try {
+            const provider = await this.getProvider();
+            if (provider && !this.providerKey) {
+                this.providerKey = hashText(provider.model);
+            }
+            const embeddingsContext = provider
+                ? {
+                    provider,
+                    providerKey: this.providerKey ?? hashText(provider.model),
+                    cache: this.settings.cache,
+                    vector: {
+                        enabled: this.settings.store.vector.enabled,
+                        ensureReady: this.ensureVectorReady.bind(this),
+                    },
+                }
+                : undefined;
+            const files = await listMemoryFiles({
+                memoryDir: this.memoryDir,
+                sessionTranscriptsDir: this.sessionTranscriptsDir,
+                includeMemoryDir: this.includeMemorySource,
+                includeSessionTranscriptsDir: this.includeSessionsSource,
+                extraPaths: this.settings.extraPaths,
+                workspaceDir: this.workspaceDir,
+            });
+            const entries = (await Promise.all(files.map((file) => buildFileEntry(file)))).filter((entry) => entry !== null);
+            const activePaths = new Set(entries.map((entry) => entry.path));
+            let indexed = 0;
+            for (const entry of entries) {
+                const source = this.resolveSourceForPath(entry.path);
+                if (!source)
+                    continue;
+                const record = this.db
+                    .prepare(`SELECT hash FROM ${FILES_TABLE} WHERE path = ? AND source = ? AND agent_id = ?`)
+                    .get(entry.path, source, this.agentId);
+                if (!params?.force && record?.hash === entry.hash) {
+                    continue;
+                }
+                notifyWorkDetected();
+                await indexMemoryFile({
+                    db: this.db,
+                    agentId: this.agentId,
+                    entry,
+                    source,
+                    chunking: this.settings.chunking,
+                    ftsAvailable: this.ftsAvailable,
+                    embeddings: embeddingsContext,
+                });
+                indexed += 1;
+                this.db
+                    .prepare(`INSERT INTO ${FILES_TABLE} (path, agent_id, source, hash, mtime, size)
+						 VALUES (?, ?, ?, ?, ?, ?)
+						 ON CONFLICT(path, agent_id) DO UPDATE SET
+						   hash=excluded.hash,
+						   mtime=excluded.mtime,
+						   size=excluded.size`)
+                    .run(entry.path, this.agentId, source, entry.hash, Math.round(entry.mtimeMs), entry.size);
+            }
+            const trackedSources = [];
+            if (this.includeMemorySource)
+                trackedSources.push("memory");
+            if (this.includeSessionsSource)
+                trackedSources.push("sessions");
+            let staleDeleted = 0;
+            for (const source of trackedSources) {
+                const staleRows = this.db
+                    .prepare(`SELECT path FROM ${FILES_TABLE} WHERE source = ? AND agent_id = ?`)
+                    .all(source, this.agentId);
+                for (const stale of staleRows) {
+                    const staleSource = this.resolveSourceForPath(stale.path) ?? source;
+                    if (activePaths.has(stale.path) && staleSource === source)
+                        continue;
+                    notifyWorkDetected();
+                    this.db
+                        .prepare(`DELETE FROM ${FILES_TABLE} WHERE path = ? AND source = ? AND agent_id = ?`)
+                        .run(stale.path, source, this.agentId);
+                    if (this.settings.store.vector.enabled) {
+                        try {
+                            this.db
+                                .prepare(`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM ${CHUNKS_TABLE} WHERE path = ? AND source = ? AND agent_id = ?)`)
+                                .run(stale.path, source, this.agentId);
+                        }
+                        catch { }
+                    }
+                    if (this.ftsAvailable) {
+                        try {
+                            this.db
+                                .prepare(`DELETE FROM ${FTS_TABLE} WHERE id IN (SELECT id FROM ${CHUNKS_TABLE} WHERE path = ? AND source = ? AND agent_id = ?)`)
+                                .run(stale.path, source, this.agentId);
+                        }
+                        catch { }
+                    }
+                    this.db
+                        .prepare(`DELETE FROM ${CHUNKS_TABLE} WHERE path = ? AND source = ? AND agent_id = ?`)
+                        .run(stale.path, source, this.agentId);
+                    staleDeleted += 1;
+                }
+            }
+            this.dirty = false;
+            const durationMs = Date.now() - start;
+            memoryLog.info("sync complete", {
+                agentId: this.agentId,
+                reason,
+                indexed,
+                staleDeleted,
+                files: entries.length,
+                durationMs,
+            });
+            if (isAutoSyncReason && workDetected) {
+                this.onAutoSyncEvent?.({
+                    reason,
+                    phase: "complete",
+                    indexed,
+                    staleDeleted,
+                    files: entries.length,
+                    durationMs,
+                });
+            }
+        }
+        catch (error) {
+            if (isAutoSyncReason) {
+                this.onAutoSyncEvent?.({
+                    reason,
+                    phase: "failed",
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }
+            throw error;
+        }
+        finally {
+            this.syncInProgress = false;
+        }
+    }
+    async close() {
+        if (this.watchTimer) {
+            clearTimeout(this.watchTimer);
+            this.watchTimer = null;
+        }
+        if (this.intervalTimer) {
+            clearInterval(this.intervalTimer);
+            this.intervalTimer = null;
+        }
+        if (this.watcher) {
+            void this.watcher.close();
+            this.watcher = null;
+        }
+        this.provider = null;
+        this.db.close();
+    }
+    async getProvider() {
+        if (this.provider)
+            return this.provider;
+        if (!this.embedFn)
+            return null;
+        const embedFn = this.embedFn;
+        const model = this.embedModel ?? "local";
+        const provider = {
+            id: "local",
+            model,
+            embedQuery: async (text) => {
+                const [vec] = await embedFn([text]);
+                return vec;
+            },
+            embedBatch: embedFn,
+        };
+        this.provider = provider;
+        this.providerKey = hashText(model);
+        return provider;
+    }
+    async ensureVectorReady(dimensions) {
+        if (!this.settings.store.vector.enabled)
+            return false;
+        if (!dimensions || dimensions <= 0)
+            return false;
+        if (this.vectorAvailable === false)
+            return false;
+        if (this.vectorAvailable === null) {
+            const loaded = await loadSqliteVecExtension({
+                db: this.db,
+                extensionPath: this.settings.store.vector.extensionPath,
+            });
+            if (!loaded.ok) {
+                this.vectorAvailable = false;
+                console.warn(`[memory] sqlite-vec unavailable: ${loaded.error ?? "unknown"}`);
+                memoryLog.warn("sqlite-vec unavailable", {
+                    agentId: this.agentId,
+                    error: loaded.error ?? "unknown",
+                });
+                return false;
+            }
+            this.vectorAvailable = true;
+        }
+        this.ensureVectorTable(dimensions);
+        return true;
+    }
+    ensureVectorTable(dimensions) {
+        if (this.vectorDims === dimensions)
+            return;
+        if (this.vectorDims && this.vectorDims !== dimensions) {
+            try {
+                this.db.exec(`DROP TABLE IF EXISTS ${VECTOR_TABLE}`);
+            }
+            catch { }
+        }
+        this.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${VECTOR_TABLE} USING vec0(\n` +
+            `  id TEXT PRIMARY KEY,\n` +
+            `  embedding FLOAT[${dimensions}]\n` +
+            `)`);
+        this.vectorDims = dimensions;
+    }
+    ensureWatcher() {
+        if (!this.settings.sync.watch || this.watcher)
+            return;
+        const watchPaths = new Set();
+        if (this.includeMemorySource) {
+            watchPaths.add(this.memoryDir);
+            for (const extra of resolveExtraPaths(this.workspaceDir, this.settings.extraPaths)) {
+                watchPaths.add(extra);
+            }
+        }
+        if (this.includeSessionsSource) {
+            watchPaths.add(this.sessionTranscriptsDir);
+        }
+        if (watchPaths.size === 0)
+            return;
+        this.watcher = chokidar.watch(Array.from(watchPaths), {
+            ignoreInitial: true,
+            awaitWriteFinish: {
+                stabilityThreshold: this.settings.sync.watchDebounceMs,
+                pollInterval: 100,
+            },
+        });
+        const markDirty = () => {
+            this.dirty = true;
+            this.scheduleWatchSync();
+        };
+        this.watcher.on("add", markDirty);
+        this.watcher.on("change", markDirty);
+        this.watcher.on("unlink", markDirty);
+    }
+    scheduleWatchSync() {
+        if (!this.settings.sync.watch)
+            return;
+        if (this.watchTimer) {
+            clearTimeout(this.watchTimer);
+        }
+        this.watchTimer = setTimeout(() => {
+            this.watchTimer = null;
+            void this.sync({ reason: "watch" }).catch((err) => {
+                console.warn(`[memory] sync failed (watch): ${String(err)}`);
+                memoryLog.warn("sync failed (watch)", {
+                    agentId: this.agentId,
+                    error: String(err),
+                });
+            });
+        }, this.settings.sync.watchDebounceMs);
+    }
+    resolveSourceForPath(absPath) {
+        const relativeToSessions = relative(this.sessionTranscriptsDir, absPath);
+        const isSessionTranscript = relativeToSessions.length > 0 &&
+            !relativeToSessions.startsWith("..") &&
+            !relativeToSessions.startsWith(".");
+        if (isSessionTranscript) {
+            return this.includeSessionsSource ? "sessions" : null;
+        }
+        return this.includeMemorySource ? "memory" : null;
+    }
+    ensureIntervalSync() {
+        const minutes = this.settings.sync.intervalMinutes;
+        if (!minutes || minutes <= 0)
+            return;
+        if (this.intervalTimer)
+            return;
+        const intervalMs = Math.max(1, Math.floor(minutes * 60_000));
+        this.intervalTimer = setInterval(() => {
+            void this.sync({ reason: "interval" }).catch((err) => {
+                console.warn(`[memory] sync failed (interval): ${String(err)}`);
+                memoryLog.warn("sync failed (interval)", {
+                    agentId: this.agentId,
+                    error: String(err),
+                });
+            });
+        }, intervalMs);
+        this.intervalTimer.unref();
+        memoryLog.info("interval sync scheduled", {
+            agentId: this.agentId,
+            intervalMinutes: minutes,
+        });
+    }
+}

package/dist/src/memory/indexing/embeddings.js

@@ -0,0 +1,67 @@
+import { enforceEmbeddingMaxInputTokens } from "../embeddings/chunk-limits.js";
+import { runEmbeddingBatches } from "../embeddings/batch-runner.js";
+import { readEmbeddingCache, writeEmbeddingCache, pruneEmbeddingCache } from "../embeddings/cache.js";
+const DEFAULT_BATCH_SIZE = 2;
+const DEFAULT_BATCH_CONCURRENCY = 2;
+export async function embedChunks(params) {
+    const limited = enforceEmbeddingMaxInputTokens(params.provider, params.chunks);
+    const embeddings = Array.from({ length: limited.length });
+    const pendingTexts = [];
+    const pendingIndices = [];
+    for (let i = 0; i < limited.length; i += 1) {
+        const chunk = limited[i];
+        if (!chunk)
+            continue;
+        if (params.cache.enabled) {
+            const cached = readEmbeddingCache({
+                db: params.db,
+                provider: params.provider.id,
+                model: params.provider.model,
+                providerKey: params.providerKey,
+                hash: chunk.hash,
+            });
+            if (cached) {
+                embeddings[i] = cached;
+                continue;
+            }
+        }
+        pendingTexts.push(chunk.text);
+        pendingIndices.push(i);
+    }
+    if (pendingTexts.length > 0) {
+        const embedded = await runEmbeddingBatches({
+            items: pendingTexts,
+            maxBatchSize: DEFAULT_BATCH_SIZE,
+            concurrency: DEFAULT_BATCH_CONCURRENCY,
+            runBatch: async (batch) => params.provider.embedBatch(batch),
+        });
+        for (let i = 0; i < pendingIndices.length; i += 1) {
+            const index = pendingIndices[i];
+            const embedding = embedded[i] ?? [];
+            embeddings[index] = embedding;
+            if (params.cache.enabled) {
+                const chunk = limited[index];
+                if (chunk) {
+                    writeEmbeddingCache({
+                        db: params.db,
+                        provider: params.provider.id,
+                        model: params.provider.model,
+                        providerKey: params.providerKey,
+                        hash: chunk.hash,
+                        embedding,
+                    });
+                }
+            }
+        }
+        if (params.cache.enabled) {
+            pruneEmbeddingCache({
+                db: params.db,
+                provider: params.provider.id,
+                model: params.provider.model,
+                providerKey: params.providerKey,
+                maxEntries: params.cache.maxEntries,
+            });
+        }
+    }
+    return { chunks: limited, embeddings };
+}