open-multi-agent-kit 0.78.6 → 0.78.8

package/dist/memory/graph-delta-log.js

@@ -0,0 +1,650 @@
+import { createHash } from "node:crypto";
+import { closeSync, mkdirSync, openSync, readFileSync, statSync, truncateSync, unlinkSync, writeSync } from "node:fs";
+import { open as fsOpen, mkdir, readFile, rename, unlink, writeFile } from "node:fs/promises";
+import { hostname } from "node:os";
+import { dirname, join } from "node:path";
+// ── Path helpers ────────────────────────────────────────────────────────────
+function deltaDir(graphPath) {
+    return dirname(graphPath);
+}
+function snapshotPath(graphPath) {
+    return join(deltaDir(graphPath), "graph.snapshot.json");
+}
+function deltaLogPath(graphPath) {
+    return join(deltaDir(graphPath), "graph.delta.jsonl");
+}
+function manifestPath(graphPath) {
+    return join(deltaDir(graphPath), "graph.manifest.json");
+}
+// ── Durability mode detection ───────────────────────────────────────────────
+export function resolveDurabilityMode(env = process.env) {
+    const raw = (env.OMK_MEMORY_DURABILITY ?? "").trim().toLowerCase();
+    if (raw === "delta")
+        return "delta";
+    return "legacy";
+}
+// ── Write helpers ───────────────────────────────────────────────────────────
+async function writeFileAtomic(path, payload) {
+    await mkdir(dirname(path), { recursive: true });
+    const hash = createHash("sha256").update(payload).digest("hex").slice(0, 8);
+    const tempPath = `${path}.${process.pid}.${Date.now()}.${hash}.tmp`;
+    try {
+        await writeFile(tempPath, payload, "utf-8");
+        await rename(tempPath, path);
+    }
+    catch (err) {
+        await unlink(tempPath).catch(() => { });
+        throw err;
+    }
+}
+function errorCode(err) {
+    if (typeof err !== "object" || err === null || !("code" in err))
+        return undefined;
+    const code = err.code;
+    return typeof code === "string" ? code : undefined;
+}
+// ── CRC / hashing ───────────────────────────────────────────────────────────
+function computeRecordCrc(record) {
+    // Canonical: JSON.stringify(recordWithoutCrc) + "\n", then SHA256
+    const payload = `${JSON.stringify(record)}\n`;
+    return createHash("sha256").update(payload).digest("hex");
+}
+function verifyRecordCrc(record) {
+    // Rebuild record without crc, re-serialize, and compare against the stored crc.
+    const { crc, ...withoutCrc } = record;
+    const expected = computeRecordCrc(withoutCrc);
+    return expected === crc;
+}
+// ── Delta computation ───────────────────────────────────────────────────────
+/**
+ * Compute an order-faithful delta between pre-mutation and post-mutation state.
+ *
+ * The legacy full-rewrite produces array orders of the form
+ *   [survivors kept in pre-order (some updated in place)] ++ [appended at end]
+ * because `upsertNode`/`upsertEdge` update existing entries in place but push
+ * brand-new (or filter-then-re-added, e.g. `replaceGeneratedMindmap`) entries
+ * to the end. A naive `Map.set` replay keeps the OLD position for an existing
+ * key and therefore diverges when an entry is filtered out then re-added.
+ *
+ * To reproduce the EXACT raw array order on replay we:
+ *   1. Keep the longest front prefix of the final array whose entries exist in
+ *      pre-state with strictly increasing pre-positions (these stay in place).
+ *   2. Treat every entry from the first break point onward as "repositioned":
+ *      emit a `del` (so replay removes it from its old Map slot) then a `put`
+ *      in final order (so replay re-inserts it at the array end).
+ * Replay applies all `del` then all `put`, so del-before-put moves repositioned
+ * keys to the end exactly as the legacy filter+repush does.
+ */
+function computeOrderedDelta(pre, finalArr) {
+    // Pre-position index (Map iteration order == insertion order == pre array order).
+    const prePos = new Map();
+    let position = 0;
+    for (const id of pre.keys())
+        prePos.set(id, position++);
+    // Longest in-place prefix: exists in pre AND pre-position strictly increases.
+    let lastPrePos = -1;
+    let splitIdx = finalArr.length;
+    for (let i = 0; i < finalArr.length; i += 1) {
+        const pos = prePos.get(finalArr[i].id);
+        if (pos !== undefined && pos > lastPrePos) {
+            lastPrePos = pos;
+        }
+        else {
+            splitIdx = i;
+            break;
+        }
+    }
+    // Final index for suffix (repositioned) detection.
+    const finalIndex = new Map();
+    for (let i = 0; i < finalArr.length; i += 1)
+        finalIndex.set(finalArr[i].id, i);
+    // Deletes: pre entries dropped entirely, or kept but in the repositioned suffix.
+    const del = [];
+    for (const id of pre.keys()) {
+        const fi = finalIndex.get(id);
+        if (fi === undefined || fi >= splitIdx)
+            del.push(id);
+    }
+    // Puts (final order): suffix entries always re-put (append); prefix entries
+    // only when their value changed (in-place update keeps position).
+    const put = [];
+    for (let i = 0; i < finalArr.length; i += 1) {
+        const item = finalArr[i];
+        if (i >= splitIdx) {
+            put.push(item);
+        }
+        else if (pre.get(item.id) !== item) {
+            put.push(item);
+        }
+    }
+    return { del, put };
+}
+/**
+ * Compute an order-faithful delta between pre-mutation and post-mutation state.
+ * `put` entries are ordered by their index in the final array and repositioned
+ * entries are deleted-before-put, so Map-based replay reproduces the EXACT raw
+ * array order of the legacy full-rewrite (see `computeOrderedDelta`).
+ */
+export function computeDelta(preNodes, preEdges, state) {
+    return {
+        nodes: computeOrderedDelta(preNodes, state.nodes),
+        edges: computeOrderedDelta(preEdges, state.edges),
+    };
+}
+// ── Manifest I/O ────────────────────────────────────────────────────────────
+export async function readManifest(graphPath) {
+    const path = manifestPath(graphPath);
+    try {
+        const raw = await readFile(path, "utf-8");
+        const parsed = JSON.parse(raw);
+        if (parsed.formatVersion === 2 &&
+            typeof parsed.snapshot === "string" &&
+            typeof parsed.snapshotEpoch === "number" &&
+            typeof parsed.delta === "string") {
+            return {
+                formatVersion: 2,
+                snapshot: parsed.snapshot,
+                snapshotEpoch: parsed.snapshotEpoch,
+                delta: parsed.delta,
+                deltaOpCount: typeof parsed.deltaOpCount === "number" ? parsed.deltaOpCount : 0,
+            };
+        }
+        return null;
+    }
+    catch (err) {
+        if (errorCode(err) === "ENOENT")
+            return null;
+        throw err;
+    }
+}
+export async function writeManifest(graphPath, manifest) {
+    const payload = `${JSON.stringify(manifest, null, 2)}\n`;
+    await writeFileAtomic(manifestPath(graphPath), payload);
+}
+// ── Snapshot I/O ────────────────────────────────────────────────────────────
+async function writeSnapshot(graphPath, state) {
+    const payload = `${JSON.stringify(state, null, 2)}\n`;
+    await writeFileAtomic(snapshotPath(graphPath), payload);
+}
+async function readSnapshot(graphPath) {
+    const path = snapshotPath(graphPath);
+    try {
+        const raw = await readFile(path, "utf-8");
+        const parsed = JSON.parse(raw);
+        if (parsed.version === 1 && Array.isArray(parsed.nodes) && Array.isArray(parsed.edges)) {
+            return {
+                version: 1,
+                ontology: parsed.ontology ?? { version: "", classes: [], relationTypes: [], description: "" },
+                project: parsed.project ?? { key: "", name: "", root: "" },
+                updatedAt: typeof parsed.updatedAt === "string" ? parsed.updatedAt : new Date().toISOString(),
+                nodes: parsed.nodes,
+                edges: parsed.edges,
+            };
+        }
+        return null;
+    }
+    catch (err) {
+        if (errorCode(err) === "ENOENT")
+            return null;
+        throw err;
+    }
+}
+// ── Delta append ────────────────────────────────────────────────────────────
+let deltaFd = null;
+let deltaFdPath = null;
+async function getDeltaFd(path) {
+    if (deltaFd && deltaFdPath === path)
+        return deltaFd;
+    if (deltaFd) {
+        await deltaFd.close().catch(() => { });
+        deltaFd = null;
+        deltaFdPath = null;
+    }
+    deltaFd = await fsOpen(path, "a"); // O_APPEND
+    deltaFdPath = path;
+    return deltaFd;
+}
+function closeDeltaFd() {
+    if (deltaFd) {
+        deltaFd.close().catch(() => { });
+        deltaFd = null;
+        deltaFdPath = null;
+    }
+}
+/**
+ * Close the cached append fd and await the close. Used before a physical
+ * truncate so no stale O_APPEND handle survives the recovery rewrite.
+ */
+async function closeDeltaFdAsync() {
+    if (deltaFd) {
+        const fd = deltaFd;
+        deltaFd = null;
+        deltaFdPath = null;
+        await fd.close().catch(() => { });
+    }
+}
+/**
+ * Append a single delta record to the journal + fdatasync.
+ * Returns the seq number assigned to the record.
+ */
+export async function appendDelta(graphPath, epoch, seq, ts, meta, nodesDelta, edgesDelta) {
+    const recordWithoutCrc = {
+        v: 2,
+        epoch,
+        seq,
+        ts,
+        meta,
+        nodes: nodesDelta,
+        edges: edgesDelta,
+    };
+    const crc = computeRecordCrc(recordWithoutCrc);
+    const record = { ...recordWithoutCrc, crc };
+    const line = `${JSON.stringify(record)}\n`;
+    const logPath = deltaLogPath(graphPath);
+    await mkdir(dirname(logPath), { recursive: true });
+    const fd = await getDeltaFd(logPath);
+    await fd.write(line, null, "utf-8");
+    await fd.datasync();
+}
+// ── Replay ──────────────────────────────────────────────────────────────────
+/**
+ * Load snapshot from disk plus replay all valid delta records.
+ *
+ * Replay correctness lemma:
+ *   1. Base: nodes/edges loaded from snapshot into insertion-ordered Map.
+ *   2. For each record with epoch === currentEpoch:
+ *      a. Apply all `del` ids: Map.delete(key).
+ *      b. Apply all `put` entries in array order: Map.set(key, entry).
+ *         Map.set on existing key keeps position; on new/deleted key appends.
+ *   3. Materialize: `[...map.values()]` produces the exact same array order
+ *      as the full-rewrite saveState().
+ *
+ * Crash recovery: the last line of the delta log may be torn. If JSON.parse
+ * fails AND it is the last line, we treat it as a torn tail and discard it.
+ * A CRC mismatch on a non-final line is corruption → throws in strict mode.
+ */
+export function replayDeltas(baseState, records, epoch) {
+    const nodeMap = new Map();
+    for (const node of baseState.nodes)
+        nodeMap.set(node.id, node);
+    const edgeMap = new Map();
+    for (const edge of baseState.edges)
+        edgeMap.set(edge.id, edge);
+    for (const record of records) {
+        if (record.epoch !== epoch)
+            continue;
+        // Apply deletes
+        for (const id of record.nodes.del)
+            nodeMap.delete(id);
+        for (const id of record.edges.del)
+            edgeMap.delete(id);
+        // Apply puts (order is preserved by the Map)
+        for (const node of record.nodes.put)
+            nodeMap.set(node.id, node);
+        for (const edge of record.edges.put)
+            edgeMap.set(edge.id, edge);
+        // Update meta from the last record
+        baseState.updatedAt = record.meta.updatedAt;
+        baseState.project = record.meta.project;
+        baseState.ontology = {
+            version: record.meta.ontology,
+            classes: [],
+            relationTypes: [],
+            description: "",
+        };
+    }
+    return {
+        ...baseState,
+        nodes: [...nodeMap.values()],
+        edges: [...edgeMap.values()],
+    };
+}
+/**
+ * Byte-accurate parse of the delta log. Walks the raw bytes record-by-record
+ * (records are `\n`-framed) and tracks the byte offset of the last VALID record
+ * boundary so a recovering writer can truncate/append at exactly that offset.
+ *
+ * Framing policy:
+ *   - A trailing segment WITHOUT a terminating newline is a torn tail (a crash
+ *     mid-append): it is reported via `torn=true` and excluded from `validBytes`
+ *     so the caller can physically truncate it away before any new append.
+ *   - A newline-terminated line that fails JSON.parse or CRC is MID-FILE
+ *     corruption: strict mode throws; non-strict mode stops replay at the first
+ *     corruption (leaving `validBytes` at the last good boundary). Either way the
+ *     bytes are never silently interleaved with a resumed write.
+ */
+async function parseDeltaLog(graphPath, strict) {
+    const path = deltaLogPath(graphPath);
+    let buf;
+    try {
+        buf = await readFile(path);
+    }
+    catch (err) {
+        if (errorCode(err) === "ENOENT")
+            return { records: [], validBytes: 0, torn: false };
+        throw err;
+    }
+    const records = [];
+    const len = buf.length;
+    const NEWLINE = 0x0a;
+    let cursor = 0;
+    let validBytes = 0;
+    let torn = false;
+    while (cursor < len) {
+        const nl = buf.indexOf(NEWLINE, cursor);
+        if (nl === -1) {
+            // No terminating newline: trailing partial (torn) record from a crash.
+            torn = true;
+            break;
+        }
+        const recordEnd = nl + 1; // inclusive of the framing newline
+        const line = buf.toString("utf-8", cursor, nl);
+        if (line === "") {
+            // Blank line: tolerated framing artifact; advance the valid boundary.
+            validBytes = recordEnd;
+            cursor = recordEnd;
+            continue;
+        }
+        let record;
+        try {
+            record = JSON.parse(line);
+        }
+        catch {
+            // Newline-terminated but unparseable => mid-file corruption.
+            if (strict) {
+                throw new Error(`Delta log corruption: unparseable record at byte ${cursor}: ${line.slice(0, 80)}...`);
+            }
+            break; // non-strict: stop replay at first corruption (no interleave)
+        }
+        if (!verifyRecordCrc(record)) {
+            if (strict) {
+                throw new Error(`Delta log corruption: CRC mismatch at byte ${cursor} (seq=${record.seq})`);
+            }
+            break; // non-strict: stop replay at first corruption (no interleave)
+        }
+        records.push(record);
+        validBytes = recordEnd;
+        cursor = recordEnd;
+    }
+    return { records, validBytes, torn };
+}
+/**
+ * Full cold load in delta mode: manifest → snapshot → replay delta tail.
+ *
+ * Torn-tail recovery: if the log ends in a partial (non-newline-terminated)
+ * record, physically truncate the file to the last valid record boundary BEFORE
+ * returning, so the next O_APPEND write can never merge into the half-written
+ * line. Any cached append fd is closed first to drop the stale handle.
+ */
+export async function loadStateViaDelta(graphPath, strict, emptyState) {
+    const manifest = await readManifest(graphPath);
+    if (!manifest) {
+        // No manifest: try legacy migration path
+        return { state: emptyState, epoch: 1, lastSeq: 0, deltaOpCount: 0, validBytes: 0 };
+    }
+    let baseState;
+    try {
+        const snap = await readSnapshot(graphPath);
+        baseState = snap ?? emptyState;
+    }
+    catch {
+        baseState = emptyState;
+    }
+    const { records, validBytes, torn } = await parseDeltaLog(graphPath, strict);
+    if (torn) {
+        // Drop the torn trailing bytes at the last valid record boundary so a
+        // resumed append starts on a clean record frame (no mid-file interleave).
+        await closeDeltaFdAsync();
+        try {
+            truncateSync(deltaLogPath(graphPath), validBytes);
+        }
+        catch (err) {
+            if (errorCode(err) !== "ENOENT")
+                throw err;
+        }
+    }
+    const filtered = records.filter((r) => r.epoch === manifest.snapshotEpoch);
+    const state = replayDeltas(baseState, filtered, manifest.snapshotEpoch);
+    return {
+        state,
+        epoch: manifest.snapshotEpoch,
+        lastSeq: filtered.length > 0 ? filtered[filtered.length - 1].seq : 0,
+        deltaOpCount: filtered.length,
+        validBytes,
+    };
+}
+// ── Setup / migration ───────────────────────────────────────────────────────
+/**
+ * One-time setup: load legacy graph-state.json (or empty), write snapshot,
+ * write manifest, create empty delta log. Does NOT delete the legacy file.
+ * If manifest already exists, this is a no-op.
+ */
+export async function setupDeltaMode(graphPath, legacyState) {
+    const existing = await readManifest(graphPath);
+    if (existing)
+        return existing;
+    const manifest = {
+        formatVersion: 2,
+        snapshot: "graph.snapshot.json",
+        snapshotEpoch: 1,
+        delta: "graph.delta.jsonl",
+        deltaOpCount: 0,
+    };
+    await writeSnapshot(graphPath, legacyState);
+    await writeFile(deltaLogPath(graphPath), "", "utf-8").catch(async (err) => {
+        if (errorCode(err) === "ENOENT") {
+            await mkdir(dirname(deltaLogPath(graphPath)), { recursive: true });
+            await writeFile(deltaLogPath(graphPath), "", "utf-8");
+            return;
+        }
+        throw err;
+    });
+    await writeManifest(graphPath, manifest);
+    return manifest;
+}
+export function resolveCompactionThresholds(env = process.env) {
+    const maxBytes = Number.parseInt(env.OMK_MEMORY_COMPACT_BYTES ?? "", 10) || 8 * 1024 * 1024; // 8MB
+    const maxOps = Number.parseInt(env.OMK_MEMORY_COMPACT_OPS ?? "", 10) || 1000;
+    return { maxBytes, maxOps };
+}
+/**
+ * Check if compaction is needed and perform it if so.
+ * Procedure: write snapshot → write new manifest (commit pivot) → truncate delta.
+ * The manifest is the single commit point; crash before manifest pivot is safe.
+ */
+export async function compactIfNeeded(graphPath, state, currentEpoch, currentOpCount, thresholds) {
+    const logPath = deltaLogPath(graphPath);
+    let deltaBytes = 0;
+    try {
+        deltaBytes = statSync(logPath).size;
+    }
+    catch {
+        // File doesn't exist or can't be stat'd
+    }
+    const shouldCompact = deltaBytes > thresholds.maxBytes || currentOpCount > thresholds.maxOps;
+    if (!shouldCompact) {
+        return { compacted: false, newEpoch: currentEpoch, newOpCount: currentOpCount };
+    }
+    const newEpoch = currentEpoch + 1;
+    // 1. Write new snapshot (covers all state up to now)
+    await writeSnapshot(graphPath, state);
+    // 2. Write new manifest (commit pivot)
+    // Close the old delta fd so we can truncate
+    closeDeltaFd();
+    const manifest = {
+        formatVersion: 2,
+        snapshot: "graph.snapshot.json",
+        snapshotEpoch: newEpoch,
+        delta: "graph.delta.jsonl",
+        deltaOpCount: 0,
+    };
+    await writeManifest(graphPath, manifest);
+    // 3. Truncate delta log (records with old epoch are now folded into snapshot)
+    try {
+        await writeFile(logPath, "", "utf-8");
+    }
+    catch {
+        // If truncate fails, re-create
+        await unlink(logPath).catch(() => { });
+        await writeFile(logPath, "", "utf-8");
+    }
+    return { compacted: true, newEpoch, newOpCount: 0 };
+}
+export function statDeltaFiles(graphPath) {
+    try {
+        const snapStat = statSync(snapshotPath(graphPath));
+        let deltaSize = 0;
+        try {
+            deltaSize = statSync(deltaLogPath(graphPath)).size;
+        }
+        catch {
+            // delta file may not exist yet
+        }
+        return {
+            snapshotMtimeMs: snapStat.mtimeMs,
+            snapshotSize: snapStat.size,
+            snapshotCtimeMs: snapStat.ctimeMs,
+            snapshotIno: snapStat.ino,
+            deltaSize,
+        };
+    }
+    catch {
+        return null;
+    }
+}
+export function deltaStatsMatch(a, b) {
+    if (!a || !b)
+        return false;
+    return (a.snapshotMtimeMs === b.snapshotMtimeMs &&
+        a.snapshotSize === b.snapshotSize &&
+        a.snapshotCtimeMs === b.snapshotCtimeMs &&
+        a.snapshotIno === b.snapshotIno &&
+        a.deltaSize === b.deltaSize);
+}
+const LOCK_TTL_MS = 30000;
+const LOCK_MAX_WAIT_MS = 30000;
+const LOCK_BASE_BACKOFF_MS = 50;
+const LOCK_MAX_BACKOFF_MS = 1000;
+function deltaLockPath(graphPath) {
+    return `${graphPath}.delta.lock`;
+}
+function isLockHolderDead(holder) {
+    const started = Date.parse(holder.startedAt);
+    if (Number.isNaN(started))
+        return true;
+    if (Date.now() - started > LOCK_TTL_MS)
+        return true;
+    if (holder.hostname === hostname()) {
+        try {
+            process.kill(holder.pid, 0);
+            return false;
+        }
+        catch (err) {
+            const code = errorCode(err);
+            if (code === "ESRCH")
+                return true;
+            // EPERM or other errors mean the process exists but we cannot signal it;
+            // treat as alive to stay safe.
+            return false;
+        }
+    }
+    return false;
+}
+export async function acquireDeltaLock(graphPath, maxWaitMs = LOCK_MAX_WAIT_MS) {
+    const path = deltaLockPath(graphPath);
+    mkdirSync(dirname(path), { recursive: true });
+    const deadline = Date.now() + maxWaitMs;
+    while (true) {
+        try {
+            const record = {
+                pid: process.pid,
+                hostname: hostname(),
+                startedAt: new Date().toISOString(),
+            };
+            const fd = openSync(path, "wx");
+            try {
+                writeSync(fd, JSON.stringify(record));
+            }
+            finally {
+                closeSync(fd);
+            }
+            return;
+        }
+        catch (err) {
+            const code = errorCode(err);
+            if (code !== "EEXIST")
+                throw err;
+            let holder = null;
+            try {
+                const raw = readFileSync(path, "utf-8");
+                const parsed = JSON.parse(raw);
+                if (typeof parsed === "object" &&
+                    parsed !== null &&
+                    "pid" in parsed &&
+                    "hostname" in parsed &&
+                    "startedAt" in parsed) {
+                    holder = parsed;
+                }
+            }
+            catch {
+                // unreadable / unparseable → treat as stale below
+            }
+            if (holder && isLockHolderDead(holder)) {
+                try {
+                    unlinkSync(path);
+                }
+                catch (unlinkErr) {
+                    if (errorCode(unlinkErr) !== "ENOENT")
+                        throw unlinkErr;
+                }
+                continue;
+            }
+            if (Date.now() >= deadline) {
+                throw new Error(`Delta lock acquisition timed out after ${maxWaitMs}ms on ${path}` +
+                    (holder ? ` (held by pid=${holder.pid} host=${holder.hostname} since=${holder.startedAt})` : ""));
+            }
+            const backoff = Math.min(LOCK_MAX_BACKOFF_MS, LOCK_BASE_BACKOFF_MS * (1 + Math.random()));
+            await new Promise((resolve) => setTimeout(resolve, backoff));
+        }
+    }
+}
+export function releaseDeltaLock(graphPath) {
+    const path = deltaLockPath(graphPath);
+    let holder = null;
+    try {
+        const raw = readFileSync(path, "utf-8");
+        const parsed = JSON.parse(raw);
+        if (typeof parsed === "object" &&
+            parsed !== null &&
+            "pid" in parsed &&
+            "hostname" in parsed &&
+            "startedAt" in parsed) {
+            holder = parsed;
+        }
+    }
+    catch {
+        return;
+    }
+    if (holder && (holder.pid !== process.pid || holder.hostname !== hostname())) {
+        return;
+    }
+    try {
+        unlinkSync(path);
+    }
+    catch (err) {
+        if (errorCode(err) !== "ENOENT")
+            throw err;
+    }
+}
+export async function withDeltaLock(graphPath, fn, env = process.env, maxWaitMs = LOCK_MAX_WAIT_MS) {
+    if (resolveDurabilityMode(env) === "legacy") {
+        return fn();
+    }
+    await acquireDeltaLock(graphPath, maxWaitMs);
+    try {
+        return await fn();
+    }
+    finally {
+        releaseDeltaLock(graphPath);
+    }
+}