open-multi-agent-kit 0.78.8 → 0.79.1

package/dist/memory/graph-delta-log.js

@@ -1,650 +0,0 @@
-import { createHash } from "node:crypto";
-import { closeSync, mkdirSync, openSync, readFileSync, statSync, truncateSync, unlinkSync, writeSync } from "node:fs";
-import { open as fsOpen, mkdir, readFile, rename, unlink, writeFile } from "node:fs/promises";
-import { hostname } from "node:os";
-import { dirname, join } from "node:path";
-// ── Path helpers ────────────────────────────────────────────────────────────
-function deltaDir(graphPath) {
-    return dirname(graphPath);
-}
-function snapshotPath(graphPath) {
-    return join(deltaDir(graphPath), "graph.snapshot.json");
-}
-function deltaLogPath(graphPath) {
-    return join(deltaDir(graphPath), "graph.delta.jsonl");
-}
-function manifestPath(graphPath) {
-    return join(deltaDir(graphPath), "graph.manifest.json");
-}
-// ── Durability mode detection ───────────────────────────────────────────────
-export function resolveDurabilityMode(env = process.env) {
-    const raw = (env.OMK_MEMORY_DURABILITY ?? "").trim().toLowerCase();
-    if (raw === "delta")
-        return "delta";
-    return "legacy";
-}
-// ── Write helpers ───────────────────────────────────────────────────────────
-async function writeFileAtomic(path, payload) {
-    await mkdir(dirname(path), { recursive: true });
-    const hash = createHash("sha256").update(payload).digest("hex").slice(0, 8);
-    const tempPath = `${path}.${process.pid}.${Date.now()}.${hash}.tmp`;
-    try {
-        await writeFile(tempPath, payload, "utf-8");
-        await rename(tempPath, path);
-    }
-    catch (err) {
-        await unlink(tempPath).catch(() => { });
-        throw err;
-    }
-}
-function errorCode(err) {
-    if (typeof err !== "object" || err === null || !("code" in err))
-        return undefined;
-    const code = err.code;
-    return typeof code === "string" ? code : undefined;
-}
-// ── CRC / hashing ───────────────────────────────────────────────────────────
-function computeRecordCrc(record) {
-    // Canonical: JSON.stringify(recordWithoutCrc) + "\n", then SHA256
-    const payload = `${JSON.stringify(record)}\n`;
-    return createHash("sha256").update(payload).digest("hex");
-}
-function verifyRecordCrc(record) {
-    // Rebuild record without crc, re-serialize, and compare against the stored crc.
-    const { crc, ...withoutCrc } = record;
-    const expected = computeRecordCrc(withoutCrc);
-    return expected === crc;
-}
-// ── Delta computation ───────────────────────────────────────────────────────
-/**
- * Compute an order-faithful delta between pre-mutation and post-mutation state.
- *
- * The legacy full-rewrite produces array orders of the form
- *   [survivors kept in pre-order (some updated in place)] ++ [appended at end]
- * because `upsertNode`/`upsertEdge` update existing entries in place but push
- * brand-new (or filter-then-re-added, e.g. `replaceGeneratedMindmap`) entries
- * to the end. A naive `Map.set` replay keeps the OLD position for an existing
- * key and therefore diverges when an entry is filtered out then re-added.
- *
- * To reproduce the EXACT raw array order on replay we:
- *   1. Keep the longest front prefix of the final array whose entries exist in
- *      pre-state with strictly increasing pre-positions (these stay in place).
- *   2. Treat every entry from the first break point onward as "repositioned":
- *      emit a `del` (so replay removes it from its old Map slot) then a `put`
- *      in final order (so replay re-inserts it at the array end).
- * Replay applies all `del` then all `put`, so del-before-put moves repositioned
- * keys to the end exactly as the legacy filter+repush does.
- */
-function computeOrderedDelta(pre, finalArr) {
-    // Pre-position index (Map iteration order == insertion order == pre array order).
-    const prePos = new Map();
-    let position = 0;
-    for (const id of pre.keys())
-        prePos.set(id, position++);
-    // Longest in-place prefix: exists in pre AND pre-position strictly increases.
-    let lastPrePos = -1;
-    let splitIdx = finalArr.length;
-    for (let i = 0; i < finalArr.length; i += 1) {
-        const pos = prePos.get(finalArr[i].id);
-        if (pos !== undefined && pos > lastPrePos) {
-            lastPrePos = pos;
-        }
-        else {
-            splitIdx = i;
-            break;
-        }
-    }
-    // Final index for suffix (repositioned) detection.
-    const finalIndex = new Map();
-    for (let i = 0; i < finalArr.length; i += 1)
-        finalIndex.set(finalArr[i].id, i);
-    // Deletes: pre entries dropped entirely, or kept but in the repositioned suffix.
-    const del = [];
-    for (const id of pre.keys()) {
-        const fi = finalIndex.get(id);
-        if (fi === undefined || fi >= splitIdx)
-            del.push(id);
-    }
-    // Puts (final order): suffix entries always re-put (append); prefix entries
-    // only when their value changed (in-place update keeps position).
-    const put = [];
-    for (let i = 0; i < finalArr.length; i += 1) {
-        const item = finalArr[i];
-        if (i >= splitIdx) {
-            put.push(item);
-        }
-        else if (pre.get(item.id) !== item) {
-            put.push(item);
-        }
-    }
-    return { del, put };
-}
-/**
- * Compute an order-faithful delta between pre-mutation and post-mutation state.
- * `put` entries are ordered by their index in the final array and repositioned
- * entries are deleted-before-put, so Map-based replay reproduces the EXACT raw
- * array order of the legacy full-rewrite (see `computeOrderedDelta`).
- */
-export function computeDelta(preNodes, preEdges, state) {
-    return {
-        nodes: computeOrderedDelta(preNodes, state.nodes),
-        edges: computeOrderedDelta(preEdges, state.edges),
-    };
-}
-// ── Manifest I/O ────────────────────────────────────────────────────────────
-export async function readManifest(graphPath) {
-    const path = manifestPath(graphPath);
-    try {
-        const raw = await readFile(path, "utf-8");
-        const parsed = JSON.parse(raw);
-        if (parsed.formatVersion === 2 &&
-            typeof parsed.snapshot === "string" &&
-            typeof parsed.snapshotEpoch === "number" &&
-            typeof parsed.delta === "string") {
-            return {
-                formatVersion: 2,
-                snapshot: parsed.snapshot,
-                snapshotEpoch: parsed.snapshotEpoch,
-                delta: parsed.delta,
-                deltaOpCount: typeof parsed.deltaOpCount === "number" ? parsed.deltaOpCount : 0,
-            };
-        }
-        return null;
-    }
-    catch (err) {
-        if (errorCode(err) === "ENOENT")
-            return null;
-        throw err;
-    }
-}
-export async function writeManifest(graphPath, manifest) {
-    const payload = `${JSON.stringify(manifest, null, 2)}\n`;
-    await writeFileAtomic(manifestPath(graphPath), payload);
-}
-// ── Snapshot I/O ────────────────────────────────────────────────────────────
-async function writeSnapshot(graphPath, state) {
-    const payload = `${JSON.stringify(state, null, 2)}\n`;
-    await writeFileAtomic(snapshotPath(graphPath), payload);
-}
-async function readSnapshot(graphPath) {
-    const path = snapshotPath(graphPath);
-    try {
-        const raw = await readFile(path, "utf-8");
-        const parsed = JSON.parse(raw);
-        if (parsed.version === 1 && Array.isArray(parsed.nodes) && Array.isArray(parsed.edges)) {
-            return {
-                version: 1,
-                ontology: parsed.ontology ?? { version: "", classes: [], relationTypes: [], description: "" },
-                project: parsed.project ?? { key: "", name: "", root: "" },
-                updatedAt: typeof parsed.updatedAt === "string" ? parsed.updatedAt : new Date().toISOString(),
-                nodes: parsed.nodes,
-                edges: parsed.edges,
-            };
-        }
-        return null;
-    }
-    catch (err) {
-        if (errorCode(err) === "ENOENT")
-            return null;
-        throw err;
-    }
-}
-// ── Delta append ────────────────────────────────────────────────────────────
-let deltaFd = null;
-let deltaFdPath = null;
-async function getDeltaFd(path) {
-    if (deltaFd && deltaFdPath === path)
-        return deltaFd;
-    if (deltaFd) {
-        await deltaFd.close().catch(() => { });
-        deltaFd = null;
-        deltaFdPath = null;
-    }
-    deltaFd = await fsOpen(path, "a"); // O_APPEND
-    deltaFdPath = path;
-    return deltaFd;
-}
-function closeDeltaFd() {
-    if (deltaFd) {
-        deltaFd.close().catch(() => { });
-        deltaFd = null;
-        deltaFdPath = null;
-    }
-}
-/**
- * Close the cached append fd and await the close. Used before a physical
- * truncate so no stale O_APPEND handle survives the recovery rewrite.
- */
-async function closeDeltaFdAsync() {
-    if (deltaFd) {
-        const fd = deltaFd;
-        deltaFd = null;
-        deltaFdPath = null;
-        await fd.close().catch(() => { });
-    }
-}
-/**
- * Append a single delta record to the journal + fdatasync.
- * Returns the seq number assigned to the record.
- */
-export async function appendDelta(graphPath, epoch, seq, ts, meta, nodesDelta, edgesDelta) {
-    const recordWithoutCrc = {
-        v: 2,
-        epoch,
-        seq,
-        ts,
-        meta,
-        nodes: nodesDelta,
-        edges: edgesDelta,
-    };
-    const crc = computeRecordCrc(recordWithoutCrc);
-    const record = { ...recordWithoutCrc, crc };
-    const line = `${JSON.stringify(record)}\n`;
-    const logPath = deltaLogPath(graphPath);
-    await mkdir(dirname(logPath), { recursive: true });
-    const fd = await getDeltaFd(logPath);
-    await fd.write(line, null, "utf-8");
-    await fd.datasync();
-}
-// ── Replay ──────────────────────────────────────────────────────────────────
-/**
- * Load snapshot from disk plus replay all valid delta records.
- *
- * Replay correctness lemma:
- *   1. Base: nodes/edges loaded from snapshot into insertion-ordered Map.
- *   2. For each record with epoch === currentEpoch:
- *      a. Apply all `del` ids: Map.delete(key).
- *      b. Apply all `put` entries in array order: Map.set(key, entry).
- *         Map.set on existing key keeps position; on new/deleted key appends.
- *   3. Materialize: `[...map.values()]` produces the exact same array order
- *      as the full-rewrite saveState().
- *
- * Crash recovery: the last line of the delta log may be torn. If JSON.parse
- * fails AND it is the last line, we treat it as a torn tail and discard it.
- * A CRC mismatch on a non-final line is corruption → throws in strict mode.
- */
-export function replayDeltas(baseState, records, epoch) {
-    const nodeMap = new Map();
-    for (const node of baseState.nodes)
-        nodeMap.set(node.id, node);
-    const edgeMap = new Map();
-    for (const edge of baseState.edges)
-        edgeMap.set(edge.id, edge);
-    for (const record of records) {
-        if (record.epoch !== epoch)
-            continue;
-        // Apply deletes
-        for (const id of record.nodes.del)
-            nodeMap.delete(id);
-        for (const id of record.edges.del)
-            edgeMap.delete(id);
-        // Apply puts (order is preserved by the Map)
-        for (const node of record.nodes.put)
-            nodeMap.set(node.id, node);
-        for (const edge of record.edges.put)
-            edgeMap.set(edge.id, edge);
-        // Update meta from the last record
-        baseState.updatedAt = record.meta.updatedAt;
-        baseState.project = record.meta.project;
-        baseState.ontology = {
-            version: record.meta.ontology,
-            classes: [],
-            relationTypes: [],
-            description: "",
-        };
-    }
-    return {
-        ...baseState,
-        nodes: [...nodeMap.values()],
-        edges: [...edgeMap.values()],
-    };
-}
-/**
- * Byte-accurate parse of the delta log. Walks the raw bytes record-by-record
- * (records are `\n`-framed) and tracks the byte offset of the last VALID record
- * boundary so a recovering writer can truncate/append at exactly that offset.
- *
- * Framing policy:
- *   - A trailing segment WITHOUT a terminating newline is a torn tail (a crash
- *     mid-append): it is reported via `torn=true` and excluded from `validBytes`
- *     so the caller can physically truncate it away before any new append.
- *   - A newline-terminated line that fails JSON.parse or CRC is MID-FILE
- *     corruption: strict mode throws; non-strict mode stops replay at the first
- *     corruption (leaving `validBytes` at the last good boundary). Either way the
- *     bytes are never silently interleaved with a resumed write.
- */
-async function parseDeltaLog(graphPath, strict) {
-    const path = deltaLogPath(graphPath);
-    let buf;
-    try {
-        buf = await readFile(path);
-    }
-    catch (err) {
-        if (errorCode(err) === "ENOENT")
-            return { records: [], validBytes: 0, torn: false };
-        throw err;
-    }
-    const records = [];
-    const len = buf.length;
-    const NEWLINE = 0x0a;
-    let cursor = 0;
-    let validBytes = 0;
-    let torn = false;
-    while (cursor < len) {
-        const nl = buf.indexOf(NEWLINE, cursor);
-        if (nl === -1) {
-            // No terminating newline: trailing partial (torn) record from a crash.
-            torn = true;
-            break;
-        }
-        const recordEnd = nl + 1; // inclusive of the framing newline
-        const line = buf.toString("utf-8", cursor, nl);
-        if (line === "") {
-            // Blank line: tolerated framing artifact; advance the valid boundary.
-            validBytes = recordEnd;
-            cursor = recordEnd;
-            continue;
-        }
-        let record;
-        try {
-            record = JSON.parse(line);
-        }
-        catch {
-            // Newline-terminated but unparseable => mid-file corruption.
-            if (strict) {
-                throw new Error(`Delta log corruption: unparseable record at byte ${cursor}: ${line.slice(0, 80)}...`);
-            }
-            break; // non-strict: stop replay at first corruption (no interleave)
-        }
-        if (!verifyRecordCrc(record)) {
-            if (strict) {
-                throw new Error(`Delta log corruption: CRC mismatch at byte ${cursor} (seq=${record.seq})`);
-            }
-            break; // non-strict: stop replay at first corruption (no interleave)
-        }
-        records.push(record);
-        validBytes = recordEnd;
-        cursor = recordEnd;
-    }
-    return { records, validBytes, torn };
-}
-/**
- * Full cold load in delta mode: manifest → snapshot → replay delta tail.
- *
- * Torn-tail recovery: if the log ends in a partial (non-newline-terminated)
- * record, physically truncate the file to the last valid record boundary BEFORE
- * returning, so the next O_APPEND write can never merge into the half-written
- * line. Any cached append fd is closed first to drop the stale handle.
- */
-export async function loadStateViaDelta(graphPath, strict, emptyState) {
-    const manifest = await readManifest(graphPath);
-    if (!manifest) {
-        // No manifest: try legacy migration path
-        return { state: emptyState, epoch: 1, lastSeq: 0, deltaOpCount: 0, validBytes: 0 };
-    }
-    let baseState;
-    try {
-        const snap = await readSnapshot(graphPath);
-        baseState = snap ?? emptyState;
-    }
-    catch {
-        baseState = emptyState;
-    }
-    const { records, validBytes, torn } = await parseDeltaLog(graphPath, strict);
-    if (torn) {
-        // Drop the torn trailing bytes at the last valid record boundary so a
-        // resumed append starts on a clean record frame (no mid-file interleave).
-        await closeDeltaFdAsync();
-        try {
-            truncateSync(deltaLogPath(graphPath), validBytes);
-        }
-        catch (err) {
-            if (errorCode(err) !== "ENOENT")
-                throw err;
-        }
-    }
-    const filtered = records.filter((r) => r.epoch === manifest.snapshotEpoch);
-    const state = replayDeltas(baseState, filtered, manifest.snapshotEpoch);
-    return {
-        state,
-        epoch: manifest.snapshotEpoch,
-        lastSeq: filtered.length > 0 ? filtered[filtered.length - 1].seq : 0,
-        deltaOpCount: filtered.length,
-        validBytes,
-    };
-}
-// ── Setup / migration ───────────────────────────────────────────────────────
-/**
- * One-time setup: load legacy graph-state.json (or empty), write snapshot,
- * write manifest, create empty delta log. Does NOT delete the legacy file.
- * If manifest already exists, this is a no-op.
- */
-export async function setupDeltaMode(graphPath, legacyState) {
-    const existing = await readManifest(graphPath);
-    if (existing)
-        return existing;
-    const manifest = {
-        formatVersion: 2,
-        snapshot: "graph.snapshot.json",
-        snapshotEpoch: 1,
-        delta: "graph.delta.jsonl",
-        deltaOpCount: 0,
-    };
-    await writeSnapshot(graphPath, legacyState);
-    await writeFile(deltaLogPath(graphPath), "", "utf-8").catch(async (err) => {
-        if (errorCode(err) === "ENOENT") {
-            await mkdir(dirname(deltaLogPath(graphPath)), { recursive: true });
-            await writeFile(deltaLogPath(graphPath), "", "utf-8");
-            return;
-        }
-        throw err;
-    });
-    await writeManifest(graphPath, manifest);
-    return manifest;
-}
-export function resolveCompactionThresholds(env = process.env) {
-    const maxBytes = Number.parseInt(env.OMK_MEMORY_COMPACT_BYTES ?? "", 10) || 8 * 1024 * 1024; // 8MB
-    const maxOps = Number.parseInt(env.OMK_MEMORY_COMPACT_OPS ?? "", 10) || 1000;
-    return { maxBytes, maxOps };
-}
-/**
- * Check if compaction is needed and perform it if so.
- * Procedure: write snapshot → write new manifest (commit pivot) → truncate delta.
- * The manifest is the single commit point; crash before manifest pivot is safe.
- */
-export async function compactIfNeeded(graphPath, state, currentEpoch, currentOpCount, thresholds) {
-    const logPath = deltaLogPath(graphPath);
-    let deltaBytes = 0;
-    try {
-        deltaBytes = statSync(logPath).size;
-    }
-    catch {
-        // File doesn't exist or can't be stat'd
-    }
-    const shouldCompact = deltaBytes > thresholds.maxBytes || currentOpCount > thresholds.maxOps;
-    if (!shouldCompact) {
-        return { compacted: false, newEpoch: currentEpoch, newOpCount: currentOpCount };
-    }
-    const newEpoch = currentEpoch + 1;
-    // 1. Write new snapshot (covers all state up to now)
-    await writeSnapshot(graphPath, state);
-    // 2. Write new manifest (commit pivot)
-    // Close the old delta fd so we can truncate
-    closeDeltaFd();
-    const manifest = {
-        formatVersion: 2,
-        snapshot: "graph.snapshot.json",
-        snapshotEpoch: newEpoch,
-        delta: "graph.delta.jsonl",
-        deltaOpCount: 0,
-    };
-    await writeManifest(graphPath, manifest);
-    // 3. Truncate delta log (records with old epoch are now folded into snapshot)
-    try {
-        await writeFile(logPath, "", "utf-8");
-    }
-    catch {
-        // If truncate fails, re-create
-        await unlink(logPath).catch(() => { });
-        await writeFile(logPath, "", "utf-8");
-    }
-    return { compacted: true, newEpoch, newOpCount: 0 };
-}
-export function statDeltaFiles(graphPath) {
-    try {
-        const snapStat = statSync(snapshotPath(graphPath));
-        let deltaSize = 0;
-        try {
-            deltaSize = statSync(deltaLogPath(graphPath)).size;
-        }
-        catch {
-            // delta file may not exist yet
-        }
-        return {
-            snapshotMtimeMs: snapStat.mtimeMs,
-            snapshotSize: snapStat.size,
-            snapshotCtimeMs: snapStat.ctimeMs,
-            snapshotIno: snapStat.ino,
-            deltaSize,
-        };
-    }
-    catch {
-        return null;
-    }
-}
-export function deltaStatsMatch(a, b) {
-    if (!a || !b)
-        return false;
-    return (a.snapshotMtimeMs === b.snapshotMtimeMs &&
-        a.snapshotSize === b.snapshotSize &&
-        a.snapshotCtimeMs === b.snapshotCtimeMs &&
-        a.snapshotIno === b.snapshotIno &&
-        a.deltaSize === b.deltaSize);
-}
-const LOCK_TTL_MS = 30000;
-const LOCK_MAX_WAIT_MS = 30000;
-const LOCK_BASE_BACKOFF_MS = 50;
-const LOCK_MAX_BACKOFF_MS = 1000;
-function deltaLockPath(graphPath) {
-    return `${graphPath}.delta.lock`;
-}
-function isLockHolderDead(holder) {
-    const started = Date.parse(holder.startedAt);
-    if (Number.isNaN(started))
-        return true;
-    if (Date.now() - started > LOCK_TTL_MS)
-        return true;
-    if (holder.hostname === hostname()) {
-        try {
-            process.kill(holder.pid, 0);
-            return false;
-        }
-        catch (err) {
-            const code = errorCode(err);
-            if (code === "ESRCH")
-                return true;
-            // EPERM or other errors mean the process exists but we cannot signal it;
-            // treat as alive to stay safe.
-            return false;
-        }
-    }
-    return false;
-}
-export async function acquireDeltaLock(graphPath, maxWaitMs = LOCK_MAX_WAIT_MS) {
-    const path = deltaLockPath(graphPath);
-    mkdirSync(dirname(path), { recursive: true });
-    const deadline = Date.now() + maxWaitMs;
-    while (true) {
-        try {
-            const record = {
-                pid: process.pid,
-                hostname: hostname(),
-                startedAt: new Date().toISOString(),
-            };
-            const fd = openSync(path, "wx");
-            try {
-                writeSync(fd, JSON.stringify(record));
-            }
-            finally {
-                closeSync(fd);
-            }
-            return;
-        }
-        catch (err) {
-            const code = errorCode(err);
-            if (code !== "EEXIST")
-                throw err;
-            let holder = null;
-            try {
-                const raw = readFileSync(path, "utf-8");
-                const parsed = JSON.parse(raw);
-                if (typeof parsed === "object" &&
-                    parsed !== null &&
-                    "pid" in parsed &&
-                    "hostname" in parsed &&
-                    "startedAt" in parsed) {
-                    holder = parsed;
-                }
-            }
-            catch {
-                // unreadable / unparseable → treat as stale below
-            }
-            if (holder && isLockHolderDead(holder)) {
-                try {
-                    unlinkSync(path);
-                }
-                catch (unlinkErr) {
-                    if (errorCode(unlinkErr) !== "ENOENT")
-                        throw unlinkErr;
-                }
-                continue;
-            }
-            if (Date.now() >= deadline) {
-                throw new Error(`Delta lock acquisition timed out after ${maxWaitMs}ms on ${path}` +
-                    (holder ? ` (held by pid=${holder.pid} host=${holder.hostname} since=${holder.startedAt})` : ""));
-            }
-            const backoff = Math.min(LOCK_MAX_BACKOFF_MS, LOCK_BASE_BACKOFF_MS * (1 + Math.random()));
-            await new Promise((resolve) => setTimeout(resolve, backoff));
-        }
-    }
-}
-export function releaseDeltaLock(graphPath) {
-    const path = deltaLockPath(graphPath);
-    let holder = null;
-    try {
-        const raw = readFileSync(path, "utf-8");
-        const parsed = JSON.parse(raw);
-        if (typeof parsed === "object" &&
-            parsed !== null &&
-            "pid" in parsed &&
-            "hostname" in parsed &&
-            "startedAt" in parsed) {
-            holder = parsed;
-        }
-    }
-    catch {
-        return;
-    }
-    if (holder && (holder.pid !== process.pid || holder.hostname !== hostname())) {
-        return;
-    }
-    try {
-        unlinkSync(path);
-    }
-    catch (err) {
-        if (errorCode(err) !== "ENOENT")
-            throw err;
-    }
-}
-export async function withDeltaLock(graphPath, fn, env = process.env, maxWaitMs = LOCK_MAX_WAIT_MS) {
-    if (resolveDurabilityMode(env) === "legacy") {
-        return fn();
-    }
-    await acquireDeltaLock(graphPath, maxWaitMs);
-    try {
-        return await fn();
-    }
-    finally {
-        releaseDeltaLock(graphPath);
-    }
-}