@kisev/memomatic 1.0.0-dev.100.g21366c4ad740

package/dist/inbox.js

@@ -0,0 +1,224 @@
+import { mkdir, open, readdir, rename, rm, unlink } from "node:fs/promises";
+import { join } from "node:path";
+import { randomBytes } from "node:crypto";
+import { check } from "./operations.js";
+import { writeAtomic } from "@kisev/safe-fs";
+import { appendDailyEntry, readTextIfExists, replaceEntryLine, writeCorpusFile } from "./corpus.js";
+import { entryLine, parseEntryLine } from "./entries.js";
+import { corpusFiles, reindex } from "./search.js";
+import { isForbidden } from "./rules.js";
+const SOURCE_PATTERN = /^[a-z][a-z0-9-]{0,63}$/;
+const LIST_PREFIX = /^- /;
+/** parseEntryLine keeps the leading "- "; strip it before re-serializing. */
+function entryText(parsed) {
+    return LIST_PREFIX.test(parsed.text) ? parsed.text.slice(2) : parsed.text;
+}
+const LOCK_STALE_MS = 30 * 60_000;
+export function inboxFileName(source, now = new Date()) {
+    const stamp = now.toISOString().replace(/[-:]/g, "").slice(0, 15);
+    return `${source}-${stamp}-${randomBytes(4).toString("hex")}.md`;
+}
+/**
+ * Append one inbox drop file. Producers write Markdown entry lines that the
+ * deterministic `process` pass later validates, gates, and moves into the
+ * corpus. Nothing here touches corpus files.
+ */
+export async function dropToInbox(paths, lines, source) {
+    if (!SOURCE_PATTERN.test(source))
+        throw new Error(`inbox source is invalid: ${source}`);
+    const content = `${lines.filter((line) => line.trim()).join("\n")}\n`;
+    if (!content.trim())
+        throw new Error("inbox drop is empty");
+    const file = join(paths.inboxDir, inboxFileName(source));
+    await writeAtomic(file, Buffer.from(content, "utf8"), 0o600);
+    return file;
+}
+async function scanCorpus(paths) {
+    const index = { texts: new Set(), keys: new Map() };
+    for (const file of await corpusFiles(paths)) {
+        const markdown = (await readTextIfExists(file)) ?? "";
+        const lines = markdown.split("\n");
+        for (let position = 0; position < lines.length; position += 1) {
+            const parsed = parseEntryLine(lines[position]);
+            if (!parsed)
+                continue;
+            index.texts.add(parsed.text);
+            if (parsed.annotations.key && parsed.annotations.status !== "superseded")
+                index.keys.set(parsed.annotations.key, { file, line: position + 1 });
+        }
+    }
+    return index;
+}
+async function moveRejected(paths, name) {
+    await mkdir(paths.rejectedDir, { mode: 0o700, recursive: true });
+    await rename(join(paths.inboxDir, name), join(paths.rejectedDir, name));
+}
+async function appendLine(context, file, line) {
+    const current = (await readTextIfExists(file)) ?? "";
+    await writeCorpusFile(context.paths, file, `${current.trimEnd()}\n${line}\n`);
+}
+function routeFor(annotations) {
+    if (annotations.target === "user" && annotations.origin === "user")
+        return "user";
+    if (annotations.target === "curated" && annotations.origin === "user")
+        return "curated";
+    return "episodic";
+}
+/**
+ * Deterministic inbox pass: validate entry lines, enforce never-save rules,
+ * deduplicate exact texts, supersede entries sharing a key, route user-origin
+ * targets, then rebuild the index with embeddings. No model turns.
+ */
+export async function processInbox(context, options = {}) {
+    const dryRun = options.dryRun === true;
+    const report = {
+        filesProcessed: 0,
+        filesRejected: 0,
+        entriesAppended: 0,
+        entriesSuperseded: 0,
+        entriesDuplicated: 0,
+        linesForbidden: 0,
+        linesInvalid: 0,
+        dryRun,
+    };
+    const names = (await readdir(context.paths.inboxDir).catch(() => []))
+        .filter((name) => name.endsWith(".md"))
+        .sort();
+    if (!names.length)
+        return report;
+    const corpus = await scanCorpus(context.paths);
+    for (const name of names) {
+        check(options);
+        options.observe?.({
+            phase: "inbox",
+            message: "Processing inbox file",
+            completed: report.filesProcessed + report.filesRejected,
+            total: names.length,
+        });
+        const markdown = (await readTextIfExists(join(context.paths.inboxDir, name))) ?? "";
+        const rawLines = markdown.split("\n");
+        const parsedLines = rawLines.filter((line) => line.trim()).map((line) => parseEntryLine(line));
+        report.linesInvalid += parsedLines.filter((parsed) => !parsed).length;
+        const allowed = parsedLines.filter((parsed) => {
+            if (!parsed)
+                return false;
+            if (isForbidden(parsed.text, context.rules)) {
+                report.linesForbidden += 1;
+                return false;
+            }
+            return true;
+        });
+        if (!allowed.length) {
+            if (!dryRun)
+                await moveRejected(context.paths, name);
+            report.filesRejected += 1;
+            continue;
+        }
+        let accepted = false;
+        for (const parsed of allowed) {
+            check(options);
+            if (corpus.texts.has(parsed.text)) {
+                report.entriesDuplicated += 1;
+                continue;
+            }
+            const annotations = {
+                ...parsed.annotations,
+                target: undefined,
+                observed: parsed.annotations.observed ?? new Date().toISOString().slice(0, 10),
+            };
+            const line = entryLine(entryText(parsed), annotations, { dropTarget: true });
+            const route = routeFor(parsed.annotations);
+            const existing = route === "episodic" && parsed.annotations.key
+                ? corpus.keys.get(parsed.annotations.key)
+                : undefined;
+            if (existing) {
+                if (!dryRun)
+                    await replaceEntryLine(context.paths, existing.file, existing.line, line);
+                report.entriesSuperseded += 1;
+            }
+            else if (route === "user" || route === "curated") {
+                if (!dryRun)
+                    await appendLine(context, route === "user" ? context.paths.userFile : context.paths.memoryFile, line);
+                report.entriesAppended += 1;
+            }
+            else {
+                if (!dryRun)
+                    await appendDailyEntry(context.paths, line);
+                report.entriesAppended += 1;
+            }
+            corpus.texts.add(parsed.text);
+            accepted = true;
+        }
+        if (accepted) {
+            if (!dryRun)
+                await unlink(join(context.paths.inboxDir, name)).catch(() => undefined);
+            report.filesProcessed += 1;
+        }
+        else {
+            if (!dryRun)
+                await moveRejected(context.paths, name);
+            report.filesRejected += 1;
+        }
+    }
+    if (!dryRun && !options.deferIndex)
+        await reindex(context.paths, context.settings, context.store, options);
+    return report;
+}
+/**
+ * Serialize whole runs (dream or process) with a stale-tolerant lock file so a
+ * nightly sweep and a manual CLI invocation never mutate the corpus together.
+ */
+export async function withRunLock(paths, work) {
+    const owner = `${process.pid} ${new Date().toISOString()} ${randomBytes(12).toString("hex")}\n`;
+    const acquire = async () => {
+        try {
+            const handle = await open(paths.runLockFile, "wx", 0o600);
+            try {
+                await handle.writeFile(owner);
+            }
+            finally {
+                await handle.close();
+            }
+        }
+        catch (error) {
+            if (error.code !== "EEXIST")
+                throw error;
+            let stale = false;
+            try {
+                const handle = await open(paths.runLockFile, "r");
+                try {
+                    const info = await handle.stat();
+                    const owner = Number.parseInt((await handle.readFile("utf8")).split(" ")[0], 10);
+                    if (Number.isSafeInteger(owner) && owner > 0) {
+                        try {
+                            process.kill(owner, 0);
+                        }
+                        catch (cause) {
+                            stale = cause.code === "ESRCH";
+                        }
+                    }
+                    else
+                        stale = Date.now() - info.mtimeMs > LOCK_STALE_MS;
+                }
+                finally {
+                    await handle.close();
+                }
+            }
+            catch {
+                return acquire();
+            }
+            if (!stale)
+                throw new Error("another memomatic run is active");
+            await rm(paths.runLockFile, { force: true });
+            return acquire();
+        }
+    };
+    await acquire();
+    try {
+        return await work();
+    }
+    finally {
+        if ((await readTextIfExists(paths.runLockFile)) === owner)
+            await rm(paths.runLockFile, { force: true }).catch(() => undefined);
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,7 @@
+export { writeCorpusFile } from "./corpus.js";
+export { entryLine, parseEntryLine } from "./entries.js";
+export { visibilityForSource } from "./visibility.js";
+export { dropToInbox, processInbox, withRunLock } from "./inbox.js";
+export { forgetEntry, getEntry, openMemomatic, searchMemory, searchMemoryResults, writeEntry, } from "./service.js";
+export type { MemomaticContext } from "./service.js";
+export { opencodeDatabasePath } from "./ingest.js";

package/dist/index.js

@@ -0,0 +1,6 @@
+export { writeCorpusFile } from "./corpus.js";
+export { entryLine, parseEntryLine } from "./entries.js";
+export { visibilityForSource } from "./visibility.js";
+export { dropToInbox, processInbox, withRunLock } from "./inbox.js";
+export { forgetEntry, getEntry, openMemomatic, searchMemory, searchMemoryResults, writeEntry, } from "./service.js";
+export { opencodeDatabasePath } from "./ingest.js";

package/dist/ingest.d.ts

@@ -0,0 +1 @@
+export declare function opencodeDatabasePath(): string;

package/dist/ingest.js

@@ -0,0 +1,15 @@
+import { homedir } from "node:os";
+import { isAbsolute, join } from "node:path";
+export function opencodeDatabasePath() {
+    const base = process.env.XDG_DATA_HOME ?? join(homedir(), ".local/share");
+    if (!isAbsolute(base))
+        throw new Error("XDG_DATA_HOME must be an absolute path");
+    if (process.env.OPENCODE_DB) {
+        if (process.env.OPENCODE_DB === ":memory:")
+            throw new Error("An in-memory OpenCode database cannot be ingested from another process");
+        return isAbsolute(process.env.OPENCODE_DB)
+            ? process.env.OPENCODE_DB
+            : join(base, "opencode", process.env.OPENCODE_DB);
+    }
+    return join(base, "opencode", "opencode.db");
+}

package/dist/ingestion.d.ts

@@ -0,0 +1,30 @@
+import type { MemoryStore } from "./store.js";
+export type Fragment = {
+    id: string;
+    sessionId: string;
+    messageIds: string[];
+    title: string;
+    directory: string;
+    text: string;
+    context: string;
+    checkpointKeys: string[];
+    seal?: [string, string];
+};
+export type IngestionPlan = {
+    fragments: Fragment[];
+    sessions: number;
+    messages: number;
+    cached: number;
+    skipped: number;
+    characters: number;
+    watermark: number;
+    cachedSessions: number;
+    unscannedSessions: number;
+    seals: Array<[string, string]>;
+};
+export declare function planIngestion(file: string, store: MemoryStore, options: {
+    before: number;
+    maxChars: number;
+    maxSessions?: number;
+    contextChars?: number;
+}): IngestionPlan;

package/dist/ingestion.js

@@ -0,0 +1,179 @@
+import { createHash } from "node:crypto";
+import { DatabaseSync } from "node:sqlite";
+const digest = (text) => createHash("sha256").update(text).digest("hex");
+export function planIngestion(file, store, options) {
+    if (!Number.isSafeInteger(options.maxChars) ||
+        options.maxChars < 2 ||
+        !Number.isFinite(options.before))
+        throw new Error("invalid ingestion limits");
+    const result = {
+        fragments: [],
+        sessions: 0,
+        messages: 0,
+        cached: 0,
+        skipped: 0,
+        characters: 0,
+        watermark: 0,
+        cachedSessions: 0,
+        unscannedSessions: 0,
+        seals: [],
+    };
+    const db = new DatabaseSync(file, { readOnly: true });
+    try {
+        db.exec("BEGIN");
+        const required = {
+            session_v2: ["id", "title", "directory", "time_created", "time_updated"],
+            session_message: ["id", "session_id", "type", "seq", "data", "time_created", "time_updated"],
+        };
+        for (const [table, fields] of Object.entries(required)) {
+            const columns = new Set(db
+                .prepare(`PRAGMA table_info(${table})`)
+                .all()
+                .map((row) => row.name));
+            if (!fields.every((field) => columns.has(field)))
+                throw new Error("OpenCode V2 session projections are unavailable; start V2 to migrate history before ingestion");
+        }
+        const sessions = db
+            .prepare("SELECT id,COALESCE(title,'') AS title,directory,time_created,time_updated AS revision FROM session_v2 WHERE time_updated < ? ORDER BY time_created,id")
+            .all(options.before);
+        for (const [position, session] of sessions.entries()) {
+            if (session.title.startsWith("[memomatic-internal]")) {
+                result.skipped++;
+                continue;
+            }
+            const sealKey = `ingest-session-v2:${digest(session.id)}`;
+            let seal;
+            {
+                const revisions = db
+                    .prepare("SELECT id,seq,time_updated FROM session_message WHERE session_id=? ORDER BY seq,id")
+                    .all(session.id);
+                if (revisions.some((message) => Number(message.time_updated) >= options.before)) {
+                    result.skipped++;
+                    continue;
+                }
+                const revision = digest(JSON.stringify([session.revision, session.title, session.directory, revisions]));
+                if (store.getMeta(sealKey) === revision) {
+                    result.cachedSessions++;
+                    continue;
+                }
+                seal = [sealKey, revision];
+            }
+            const rows = db
+                .prepare(`
+        SELECT id,type AS role,json_extract(data,'$.text') AS text,seq,0 AS part_index
+        FROM session_message WHERE session_id=? AND type='user'
+          AND CASE WHEN json_valid(data) THEN json_type(data,'$.text') END='text'
+        UNION ALL
+        SELECT m.id,m.type AS role,json_extract(p.value,'$.text') AS text,m.seq,CAST(p.key AS INTEGER) AS part_index
+        FROM session_message m JOIN json_each(CASE WHEN json_valid(m.data) THEN json_extract(m.data,'$.content') ELSE '[]' END) p
+        WHERE m.session_id=? AND m.type='assistant'
+          AND CASE WHEN json_valid(p.value) THEN json_extract(p.value,'$.type') END='text'
+          AND CASE WHEN json_valid(p.value) THEN json_type(p.value,'$.text') END='text'
+        ORDER BY seq,part_index
+      `)
+                .all(session.id, session.id);
+            const messages = new Map();
+            for (const row of rows) {
+                const item = messages.get(row.id) ?? { role: row.role, texts: [] };
+                item.texts.push(row.text);
+                messages.set(row.id, item);
+            }
+            const firstUser = [...messages.values()].find((message) => message.role === "user");
+            if (firstUser?.texts.join("\n").trimStart().startsWith("[memomatic-internal]")) {
+                result.skipped++;
+                continue;
+            }
+            let batch = [];
+            let ids = [];
+            let keys = [];
+            let context = "";
+            let previous = "";
+            let length = 0;
+            let changed = false;
+            const flush = () => {
+                if (!batch.length)
+                    return;
+                result.fragments.push({
+                    id: digest(keys.join("\n")),
+                    sessionId: session.id,
+                    messageIds: ids,
+                    title: session.title,
+                    directory: session.directory,
+                    text: batch.join("\n\n"),
+                    context,
+                    checkpointKeys: keys,
+                });
+                batch = [];
+                ids = [];
+                keys = [];
+                length = 0;
+            };
+            let lastMessage = "";
+            for (const [id, message] of messages) {
+                const body = message.texts.join("\n").trim();
+                if (!body) {
+                    result.skipped++;
+                    continue;
+                }
+                result.messages++;
+                const full = `${message.role}: ${body}`;
+                // Only adjacent exact duplicate messages with the same role are redundant.
+                if (full === lastMessage) {
+                    result.skipped++;
+                    continue;
+                }
+                lastMessage = full;
+                for (let offset = 0; offset < body.length;) {
+                    const start = offset;
+                    let end = Math.min(body.length, offset + options.maxChars);
+                    if (end < body.length &&
+                        /[\uD800-\uDBFF]/.test(body[end - 1]) &&
+                        /[\uDC00-\uDFFF]/.test(body[end]))
+                        end--;
+                    const text = `${message.role}: ${body.slice(start, end)}`;
+                    offset = end;
+                    const preceding = Array.from(previous)
+                        .slice(-(options.contextChars ?? 2000))
+                        .join("");
+                    const key = `ingest-v2:${digest(`${session.id}\n${id}\n${start}\n${preceding}\n${text}`)}`;
+                    if (store.getMeta(key)) {
+                        result.cached++;
+                        previous = text;
+                        continue;
+                    }
+                    if (batch.length && length + text.length > options.maxChars)
+                        flush();
+                    if (!batch.length)
+                        context = preceding;
+                    batch.push(text);
+                    ids.push(id);
+                    keys.push(key);
+                    length += text.length;
+                    result.characters += text.length;
+                    changed = true;
+                    previous = text;
+                }
+            }
+            flush();
+            if (seal) {
+                if (changed)
+                    result.fragments.at(-1).seal = seal;
+                else
+                    result.seals.push(seal);
+            }
+            if (changed) {
+                result.sessions++;
+                result.watermark = Math.max(result.watermark, session.time_created);
+            }
+            if (options.maxSessions && result.sessions >= options.maxSessions) {
+                result.unscannedSessions = sessions.length - position - 1;
+                break;
+            }
+        }
+        db.exec("COMMIT");
+        return result;
+    }
+    finally {
+        db.close();
+    }
+}

package/dist/mcp.d.ts

@@ -0,0 +1,15 @@
+type JsonRpcRequest = {
+    jsonrpc?: string;
+    id?: unknown;
+    method?: string;
+    params?: Record<string, unknown>;
+};
+export declare function mcpResponse(id: unknown, result: unknown): string;
+export declare function mcpError(id: unknown, code: number, message: string): string;
+export declare function handleMcpRequest(request: JsonRpcRequest): Promise<string>;
+export declare function runMcpServer(input?: NodeJS.ReadStream & {
+    fd: 0;
+}, output?: NodeJS.WriteStream & {
+    fd: 1;
+}): Promise<void>;
+export {};