mono-pilot 0.2.9 → 0.2.12

package/dist/src/memory/indexing/files.js

@@ -0,0 +1,180 @@
+import crypto from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+function isFileMissingError(error) {
+    return error?.code === "ENOENT";
+}
+export function hashText(value) {
+    return crypto.createHash("sha256").update(value).digest("hex");
+}
+function normalizePath(value) {
+    return value.replace(/\\/g, "/");
+}
+export function resolveExtraPaths(workspaceDir, extraPaths) {
+    if (!extraPaths?.length)
+        return [];
+    const resolved = extraPaths
+        .map((value) => value.trim())
+        .filter(Boolean)
+        .map((value) => path.isAbsolute(value)
+        ? path.resolve(value)
+        : workspaceDir
+            ? path.resolve(workspaceDir, value)
+            : path.resolve(value));
+    return Array.from(new Set(resolved));
+}
+async function walkDir(dir, files) {
+    const entries = await fs.readdir(dir, { withFileTypes: true });
+    for (const entry of entries) {
+        const full = path.join(dir, entry.name);
+        if (entry.isSymbolicLink())
+            continue;
+        if (entry.isDirectory()) {
+            if (entry.name === ".state")
+                continue;
+            await walkDir(full, files);
+            continue;
+        }
+        if (!entry.isFile())
+            continue;
+        if (!entry.name.endsWith(".md"))
+            continue;
+        files.push(full);
+    }
+}
+export async function listMemoryFiles(params) {
+    const result = [];
+    if (params.includeMemoryDir !== false && params.memoryDir) {
+        try {
+            const dirStat = await fs.lstat(params.memoryDir);
+            if (!dirStat.isSymbolicLink() && dirStat.isDirectory()) {
+                await walkDir(params.memoryDir, result);
+            }
+        }
+        catch { }
+    }
+    if (params.includeSessionTranscriptsDir !== false && params.sessionTranscriptsDir) {
+        try {
+            const transcriptsStat = await fs.lstat(params.sessionTranscriptsDir);
+            if (!transcriptsStat.isSymbolicLink() && transcriptsStat.isDirectory()) {
+                await walkDir(params.sessionTranscriptsDir, result);
+            }
+        }
+        catch { }
+    }
+    const normalizedExtraPaths = params.includeMemoryDir === false
+        ? []
+        : resolveExtraPaths(params.workspaceDir, params.extraPaths);
+    for (const inputPath of normalizedExtraPaths) {
+        try {
+            const stat = await fs.lstat(inputPath);
+            if (stat.isSymbolicLink())
+                continue;
+            if (stat.isDirectory()) {
+                await walkDir(inputPath, result);
+                continue;
+            }
+            if (stat.isFile() && inputPath.endsWith(".md")) {
+                result.push(inputPath);
+            }
+        }
+        catch { }
+    }
+    return Array.from(new Set(result));
+}
+export async function buildFileEntry(absPath) {
+    let stat;
+    try {
+        stat = await fs.stat(absPath);
+    }
+    catch (err) {
+        if (isFileMissingError(err))
+            return null;
+        throw err;
+    }
+    let content;
+    try {
+        content = await fs.readFile(absPath, "utf-8");
+    }
+    catch (err) {
+        if (isFileMissingError(err))
+            return null;
+        throw err;
+    }
+    const hash = hashText(content);
+    return {
+        path: normalizePath(path.resolve(absPath)),
+        absPath,
+        mtimeMs: stat.mtimeMs,
+        size: stat.size,
+        hash,
+    };
+}
+export function chunkMarkdown(content, chunking) {
+    const lines = content.split("\n");
+    if (lines.length === 0)
+        return [];
+    const maxChars = Math.max(32, chunking.tokens * 4);
+    const overlapChars = Math.max(0, chunking.overlap * 4);
+    const chunks = [];
+    let current = [];
+    let currentChars = 0;
+    const flush = () => {
+        if (current.length === 0)
+            return;
+        const firstEntry = current[0];
+        const lastEntry = current[current.length - 1];
+        if (!firstEntry || !lastEntry)
+            return;
+        const text = current.map((entry) => entry.line).join("\n");
+        const startLine = firstEntry.lineNo;
+        const endLine = lastEntry.lineNo;
+        chunks.push({ startLine, endLine, text, hash: hashText(text) });
+    };
+    const carryOverlap = () => {
+        if (overlapChars <= 0 || current.length === 0) {
+            current = [];
+            currentChars = 0;
+            return;
+        }
+        let acc = 0;
+        const kept = [];
+        for (let i = current.length - 1; i >= 0; i -= 1) {
+            const entry = current[i];
+            if (!entry)
+                continue;
+            acc += entry.line.length + 1;
+            kept.unshift(entry);
+            if (acc >= overlapChars)
+                break;
+        }
+        current = kept;
+        currentChars = kept.reduce((sum, entry) => sum + entry.line.length + 1, 0);
+    };
+    for (let i = 0; i < lines.length; i += 1) {
+        const line = lines[i] ?? "";
+        const lineNo = i + 1;
+        const lineSize = line.length + 1;
+        if (lineSize > maxChars) {
+            if (current.length > 0) {
+                flush();
+                carryOverlap();
+            }
+            chunks.push({
+                startLine: lineNo,
+                endLine: lineNo,
+                text: line,
+                hash: hashText(line),
+            });
+            continue;
+        }
+        if (currentChars + lineSize > maxChars && current.length > 0) {
+            flush();
+            carryOverlap();
+        }
+        current.push({ line, lineNo });
+        currentChars += lineSize;
+    }
+    flush();
+    return chunks;
+}

package/dist/src/memory/indexing/index-file.js

@@ -0,0 +1,105 @@
+import { readFile } from "node:fs/promises";
+import { chunkMarkdown, hashText } from "./files.js";
+import { CHUNKS_TABLE, FTS_TABLE, VECTOR_TABLE } from "../store/schema.js";
+import { embedChunks } from "./embeddings.js";
+const DEFAULT_MODEL_ID = "fts";
+export async function indexMemoryFile(params) {
+    const content = await readContent(params.entry);
+    const chunks = chunkMarkdown(content, params.chunking).filter((chunk) => chunk.text.trim().length > 0);
+    const now = Date.now();
+    const modelId = params.embeddings ? params.embeddings.provider.model : DEFAULT_MODEL_ID;
+    const vectorEnabled = params.embeddings?.vector.enabled ?? false;
+    if (vectorEnabled) {
+        try {
+            params.db
+                .prepare(`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM ${CHUNKS_TABLE} WHERE path = ? AND source = ? AND agent_id = ?)`)
+                .run(params.entry.path, params.source, params.agentId);
+        }
+        catch {
+            // Ignore vector cleanup errors.
+        }
+    }
+    if (params.ftsAvailable) {
+        try {
+            params.db
+                .prepare(`DELETE FROM ${FTS_TABLE} WHERE id IN (SELECT id FROM ${CHUNKS_TABLE} WHERE path = ? AND source = ? AND agent_id = ?)`)
+                .run(params.entry.path, params.source, params.agentId);
+        }
+        catch {
+            // Ignore FTS cleanup errors.
+        }
+    }
+    params.db
+        .prepare(`DELETE FROM ${CHUNKS_TABLE} WHERE path = ? AND source = ? AND agent_id = ?`)
+        .run(params.entry.path, params.source, params.agentId);
+    let indexedChunks = chunks;
+    let embeddings = indexedChunks.map(() => []);
+    if (params.embeddings) {
+        const embedded = await embedChunks({
+            db: params.db,
+            provider: params.embeddings.provider,
+            providerKey: params.embeddings.providerKey,
+            chunks: indexedChunks,
+            cache: params.embeddings.cache,
+        });
+        indexedChunks = embedded.chunks;
+        embeddings = embedded.embeddings;
+    }
+    const sampleEmbedding = embeddings.find((embedding) => embedding.length > 0);
+    const vectorReady = vectorEnabled && params.embeddings && sampleEmbedding
+        ? await params.embeddings.vector.ensureReady(sampleEmbedding.length)
+        : false;
+    for (let i = 0; i < indexedChunks.length; i += 1) {
+        const chunk = indexedChunks[i];
+        if (!chunk)
+            continue;
+        const embedding = embeddings[i] ?? [];
+        const id = buildChunkId({
+            agentId: params.agentId,
+            source: params.source,
+            path: params.entry.path,
+            chunk,
+            modelId,
+        });
+        params.db
+            .prepare(`INSERT INTO ${CHUNKS_TABLE} (id, path, agent_id, source, start_line, end_line, hash, model, text, embedding, updated_at)
+				 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+				 ON CONFLICT(id) DO UPDATE SET
+				   hash=excluded.hash,
+				   model=excluded.model,
+				   text=excluded.text,
+				   embedding=excluded.embedding,
+				   updated_at=excluded.updated_at`)
+            .run(id, params.entry.path, params.agentId, params.source, chunk.startLine, chunk.endLine, chunk.hash, modelId, chunk.text, JSON.stringify(embedding), now);
+        if (vectorReady && embedding.length > 0) {
+            try {
+                params.db
+                    .prepare(`INSERT INTO ${VECTOR_TABLE} (id, embedding) VALUES (?, ?)`)
+                    .run(id, vectorToBlob(embedding));
+            }
+            catch {
+                // Ignore vector insert errors.
+            }
+        }
+        if (params.ftsAvailable) {
+            try {
+                params.db
+                    .prepare(`INSERT INTO ${FTS_TABLE} (text, id, path, source, model, start_line, end_line)
+						 VALUES (?, ?, ?, ?, ?, ?, ?)`)
+                    .run(chunk.text, id, params.entry.path, params.source, modelId, chunk.startLine, chunk.endLine);
+            }
+            catch {
+                // Ignore FTS insert errors.
+            }
+        }
+    }
+}
+function buildChunkId(params) {
+    return hashText(`${params.source}:${params.agentId}:${params.path}:${params.chunk.startLine}:${params.chunk.endLine}:${params.chunk.hash}:${params.modelId}`);
+}
+async function readContent(entry) {
+    return await readFile(entry.absPath, "utf-8");
+}
+function vectorToBlob(embedding) {
+    return Buffer.from(new Float32Array(embedding).buffer);
+}

package/dist/src/memory/log.js

@@ -0,0 +1,38 @@
+import { appendFile, mkdir } from "node:fs/promises";
+import { getMemoryLogPath, getMemoryLogsDir } from "./paths.js";
+let writeQueue = Promise.resolve();
+function formatLine(level, message, data) {
+    const timestamp = new Date().toISOString();
+    const payload = data ? ` ${safeJson(data)}` : "";
+    return `${timestamp} [${level}] ${message}${payload}`;
+}
+function safeJson(data) {
+    try {
+        return JSON.stringify(data);
+    }
+    catch {
+        return "[unserializable]";
+    }
+}
+async function appendLogLine(line) {
+    await mkdir(getMemoryLogsDir(), { recursive: true });
+    await appendFile(getMemoryLogPath(), `${line}\n`, { encoding: "utf-8" });
+}
+function enqueue(level, message, data) {
+    const line = formatLine(level, message, data);
+    writeQueue = writeQueue.then(() => appendLogLine(line)).catch(() => { });
+}
+export const memoryLog = {
+    debug(message, data) {
+        enqueue("debug", message, data);
+    },
+    info(message, data) {
+        enqueue("info", message, data);
+    },
+    warn(message, data) {
+        enqueue("warn", message, data);
+    },
+    error(message, data) {
+        enqueue("error", message, data);
+    },
+};

package/dist/src/memory/paths.js

@@ -0,0 +1,15 @@
+import { homedir } from "node:os";
+import { join } from "node:path";
+export function getMemoryRootDir() {
+    return join(homedir(), ".mono-pilot", "memory");
+}
+export function getMemoryIndexPath() {
+    return join(getMemoryRootDir(), "index.sqlite");
+}
+export function getMemoryLogsDir() {
+    return join(homedir(), ".mono-pilot", "logs");
+}
+export function getMemoryLogPath(date = new Date()) {
+    const stamp = date.toISOString().slice(0, 10);
+    return join(getMemoryLogsDir(), `memory.${stamp}.log`);
+}

package/dist/src/memory/runtime/index.js

@@ -0,0 +1,299 @@
+import { Worker } from "node:worker_threads";
+import { fileURLToPath } from "node:url";
+import { dirname, join, resolve } from "node:path";
+import { readFile } from "node:fs/promises";
+import { loadResolvedMemorySearchConfig } from "../config/loader.js";
+import { publishSystemEvent } from "../../extensions/system-events.js";
+// When running via tsx (dev), import.meta.url points to src/ but worker threads
+// need compiled JS. Detect this and redirect to dist/.
+function resolveWorkerPath() {
+    const thisFile = fileURLToPath(import.meta.url);
+    const thisDir = dirname(thisFile);
+    const threadFile = join(thisDir, "thread.js");
+    if (thisDir.includes(`${join("dist", "src")}`)) {
+        return threadFile;
+    }
+    const projectRoot = resolve(thisDir, "../../..");
+    return join(projectRoot, "dist/src/memory/runtime/thread.js");
+}
+class WorkerMemoryProxy {
+    worker;
+    nextId = 1;
+    pending = new Map();
+    syncWorkDetectedHandlers = new Map();
+    dirty = true;
+    ready;
+    readyReject = null;
+    embeddingProvider = null;
+    constructor(params) {
+        const initData = {
+            agentId: params.agentId,
+            workspaceDir: params.workspaceDir,
+            settings: params.settings,
+            embedModel: params.embedModel,
+        };
+        this.worker = new Worker(resolveWorkerPath(), {
+            workerData: initData,
+            execArgv: ["--no-warnings"],
+        });
+        // Don't let the worker prevent process exit as a last resort.
+        this.worker.unref();
+        this.ready = new Promise((resolve, reject) => {
+            this.readyReject = reject;
+            const onMessage = (msg) => {
+                if (msg.type === "ready") {
+                    this.worker.removeListener("message", onMessage);
+                    this.readyReject = null;
+                    resolve();
+                }
+            };
+            this.worker.on("message", onMessage);
+        });
+        this.worker.on("message", (msg) => {
+            if (msg.type === "dirty") {
+                this.dirty = msg.value;
+                return;
+            }
+            if (msg.type === "autoSyncEvent") {
+                this.publishAutoSyncEvent(msg.event);
+                return;
+            }
+            if (msg.type === "syncWorkDetected") {
+                const handler = this.syncWorkDetectedHandlers.get(msg.requestId);
+                if (!handler) {
+                    return;
+                }
+                this.syncWorkDetectedHandlers.delete(msg.requestId);
+                handler();
+                return;
+            }
+            if (msg.type === "ready")
+                return;
+            // Embed request from worker — forward to main-thread provider
+            if (msg.type === "embed") {
+                void this.handleEmbedRequest(msg);
+                return;
+            }
+            // WorkerResponse
+            const pending = this.pending.get(msg.id);
+            if (!pending)
+                return;
+            this.pending.delete(msg.id);
+            this.syncWorkDetectedHandlers.delete(msg.id);
+            if (msg.type === "error") {
+                pending.reject(new Error(msg.message));
+            }
+            else {
+                pending.resolve(msg.data);
+            }
+        });
+        this.worker.on("error", (err) => {
+            this.rejectReady(err);
+            this.rejectAll(err);
+        });
+        this.worker.on("exit", (code) => {
+            const err = new Error(`Memory worker exited with code ${code}`);
+            this.rejectReady(err);
+            if (code !== 0) {
+                this.rejectAll(err);
+            }
+        });
+    }
+    async search(query, opts) {
+        await this.ready;
+        return this.send({ type: "search", query, opts });
+    }
+    // File read only — no DB, runs on main thread.
+    async get(path, from, lines) {
+        const raw = await readFile(path, "utf-8");
+        if (from === undefined && lines === undefined) {
+            return { path, text: raw };
+        }
+        const startLine = Math.max(1, Math.floor(from ?? 1));
+        const maxLines = lines !== undefined ? Math.max(0, Math.floor(lines)) : undefined;
+        const allLines = raw.split("\n");
+        const startIndex = Math.min(allLines.length, Math.max(0, startLine - 1));
+        const endIndex = maxLines === undefined ? allLines.length : Math.min(allLines.length, startIndex + maxLines);
+        return { path, text: allLines.slice(startIndex, endIndex).join("\n") };
+    }
+    async sync(opts) {
+        await this.ready;
+        const onWorkDetected = opts?.onWorkDetected;
+        let requestOpts;
+        if (opts) {
+            const { onWorkDetected: _ignored, ...rest } = opts;
+            requestOpts = rest;
+        }
+        const id = this.nextId++;
+        await new Promise((resolve, reject) => {
+            if (onWorkDetected) {
+                this.syncWorkDetectedHandlers.set(id, onWorkDetected);
+            }
+            this.pending.set(id, {
+                resolve: () => {
+                    this.syncWorkDetectedHandlers.delete(id);
+                    resolve();
+                },
+                reject: (err) => {
+                    this.syncWorkDetectedHandlers.delete(id);
+                    reject(err);
+                },
+            });
+            const req = {
+                id,
+                type: "sync",
+                opts: requestOpts,
+                notifyWorkDetected: Boolean(onWorkDetected),
+            };
+            this.worker.postMessage(req);
+        });
+    }
+    async syncDirty() {
+        await this.ready;
+        return this.send({ type: "syncDirty" });
+    }
+    isDirty() {
+        return this.dirty;
+    }
+    /** Set the embedding provider used to serve worker embed requests. */
+    setEmbeddingProvider(provider) {
+        this.embeddingProvider = provider;
+    }
+    async handleEmbedRequest(req) {
+        try {
+            if (!this.embeddingProvider) {
+                throw new Error("No embedding provider available");
+            }
+            const data = await this.embeddingProvider.embedBatch(req.texts);
+            this.worker.postMessage({ type: "embedResult", id: req.id, data });
+        }
+        catch (err) {
+            this.worker.postMessage({
+                type: "embedResult",
+                id: req.id,
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+    }
+    async close() {
+        const CLOSE_TIMEOUT_MS = 3000;
+        try {
+            await Promise.race([
+                this.send({ type: "close" }),
+                new Promise((_, reject) => {
+                    const timer = setTimeout(() => reject(new Error("close timeout")), CLOSE_TIMEOUT_MS);
+                    timer.unref();
+                }),
+            ]);
+        }
+        catch {
+            // Worker may already be gone.
+        }
+        // Force-terminate without waiting — don't block process exit.
+        this.worker.terminate().catch(() => { });
+        this.pending.clear();
+        this.syncWorkDetectedHandlers.clear();
+    }
+    send(partial) {
+        const id = this.nextId++;
+        const req = { ...partial, id };
+        return new Promise((resolve, reject) => {
+            this.pending.set(id, { resolve, reject });
+            this.worker.postMessage(req);
+        });
+    }
+    rejectReady(err) {
+        if (this.readyReject) {
+            this.readyReject(err);
+            this.readyReject = null;
+        }
+    }
+    publishAutoSyncEvent(event) {
+        const reasonLabel = event.reason === "search" ? "on-search" : event.reason;
+        if (event.phase === "start") {
+            publishSystemEvent({
+                source: "memory",
+                level: "info",
+                message: `Memory sync in progress (${reasonLabel}).`,
+                dedupeKey: `memory|auto_sync|${event.reason}|start`,
+                toast: false,
+            });
+            return;
+        }
+        if (event.phase === "complete") {
+            const indexed = event.indexed ?? 0;
+            const removed = event.staleDeleted ?? 0;
+            publishSystemEvent({
+                source: "memory",
+                level: "info",
+                message: `Memory sync complete (${reasonLabel}): indexed=${indexed}, removed=${removed}.`,
+                dedupeKey: `memory|auto_sync|${event.reason}|complete`,
+                toast: false,
+            });
+            return;
+        }
+        publishSystemEvent({
+            source: "memory",
+            level: "warning",
+            message: `Memory sync failed (${reasonLabel}): ${event.error ?? "unknown error"}`,
+            dedupeKey: `memory|auto_sync|${event.reason}|failed`,
+            toast: false,
+        });
+    }
+    rejectAll(err) {
+        this.syncWorkDetectedHandlers.clear();
+        for (const pending of this.pending.values()) {
+            pending.reject(err);
+        }
+        this.pending.clear();
+    }
+}
+// --- Singleton registry ---
+const managerCache = new Map();
+let defaultEmbeddingProvider = null;
+export async function getMemorySearchManager(params) {
+    const settings = await loadResolvedMemorySearchConfig();
+    if (!settings.enabled)
+        return null;
+    if (!settings.sources.includes("memory") && !settings.sources.includes("sessions"))
+        return null;
+    const key = params.agentId;
+    const cached = managerCache.get(key);
+    if (cached)
+        return cached;
+    const manager = new WorkerMemoryProxy({
+        agentId: params.agentId,
+        workspaceDir: params.workspaceDir,
+        settings,
+        embedModel: defaultEmbeddingProvider?.model,
+    });
+    if (defaultEmbeddingProvider) {
+        manager.setEmbeddingProvider(defaultEmbeddingProvider);
+    }
+    managerCache.set(key, manager);
+    return manager;
+}
+export async function closeMemorySearchManagers() {
+    const managers = Array.from(managerCache.values());
+    managerCache.clear();
+    for (const manager of managers) {
+        if (manager.close) {
+            await manager.close();
+        }
+    }
+}
+export function peekMemorySearchManager(params) {
+    const key = params.agentId;
+    return managerCache.get(key) ?? null;
+}
+/** Set the embedding provider on all active worker proxies. */
+export function setMemoryWorkersEmbeddingProvider(provider) {
+    defaultEmbeddingProvider = provider;
+    for (const proxy of managerCache.values()) {
+        proxy.setEmbeddingProvider(provider);
+    }
+}
+/** Get the current default embedding provider (if any). */
+export function getDefaultEmbeddingProvider() {
+    return defaultEmbeddingProvider;
+}