chapterhouse 0.13.1 → 0.14.0

package/dist/memory/housekeeping.js

@@ -1,497 +0,0 @@
-import { performance } from "node:perf_hooks";
-import { config } from "../config.js";
-import { getDb } from "../store/db.js";
-import { childLogger } from "../util/logger.js";
-import { getActiveScope } from "./active-scope.js";
-import { releaseScopeWriteLocks, tryAcquireScopeWriteLocks } from "./scope-lock.js";
-import { listScopes } from "./scopes.js";
-import { tieringPass } from "./tiering.js";
-export { tieringPass };
-const log = childLogger("memory.housekeeping");
-const GLOBAL_PASS_SCOPE = Symbol("global-housekeeping-pass-scope");
-const inFlightScopesByPass = new Map();
-const PASS_ORDER = [
-    "dedup_observations",
-    "dedup_decisions",
-    "compact_supersede_chains",
-    "orphan_cleanup",
-    "decay",
-    "compact_inbox",
-    "tiering",
-];
-function passSummary(pass, examined = 0, modified = 0, errors = []) {
-    return { pass, examined, modified, errors };
-}
-function tokens(value) {
-    const words = value
-        .toLowerCase()
-        .split(/[^a-z0-9]+/u)
-        .map((word) => word.replace(/s$/, ""))
-        .filter((word) => word.length > 1);
-    return new Set(words);
-}
-function jaccard(left, right) {
-    const leftTokens = tokens(left);
-    const rightTokens = tokens(right);
-    if (leftTokens.size === 0 || rightTokens.size === 0) {
-        return 0;
-    }
-    let intersection = 0;
-    for (const token of leftTokens) {
-        if (rightTokens.has(token)) {
-            intersection++;
-        }
-    }
-    const union = leftTokens.size + rightTokens.size - intersection;
-    return union === 0 ? 0 : intersection / union;
-}
-function isSimilar(left, right) {
-    if (left.trim().toLowerCase() === right.trim().toLowerCase()) {
-        return true;
-    }
-    return jaccard(left, right) >= config.memoryHousekeepingSimilarityThreshold;
-}
-function compareObservationKeeper(left, right) {
-    if (right.confidence !== left.confidence) {
-        return right.confidence - left.confidence;
-    }
-    if (right.created_at !== left.created_at) {
-        return right.created_at.localeCompare(left.created_at);
-    }
-    return left.id - right.id;
-}
-function compareDecisionKeeper(left, right) {
-    if (right.decided_at !== left.decided_at) {
-        return right.decided_at.localeCompare(left.decided_at);
-    }
-    if (right.created_at !== left.created_at) {
-        return right.created_at.localeCompare(left.created_at);
-    }
-    return left.id - right.id;
-}
-function normalizePassName(pass) {
-    const normalized = pass.trim().toLowerCase().replace(/-/g, "_");
-    const aliases = {
-        dedup_observations: "dedup_observations",
-        dedupobservations: "dedup_observations",
-        dedupobservationspass: "dedup_observations",
-        observations: "dedup_observations",
-        dedup_decisions: "dedup_decisions",
-        dedupdecisions: "dedup_decisions",
-        dedupdecisionspass: "dedup_decisions",
-        decisions: "dedup_decisions",
-        compact_supersede_chains: "compact_supersede_chains",
-        compactsupersedechains: "compact_supersede_chains",
-        compactsupersedechainspass: "compact_supersede_chains",
-        supersede: "compact_supersede_chains",
-        supersedechains: "compact_supersede_chains",
-        orphan_cleanup: "orphan_cleanup",
-        orphancleanup: "orphan_cleanup",
-        orphancleanuppass: "orphan_cleanup",
-        orphans: "orphan_cleanup",
-        decay: "decay",
-        decaypass: "decay",
-        compact_inbox: "compact_inbox",
-        compactinbox: "compact_inbox",
-        compactinboxpass: "compact_inbox",
-        inbox: "compact_inbox",
-        tiering: "tiering",
-        tieringpass: "tiering",
-        tiers: "tiering",
-    };
-    const resolved = aliases[normalized];
-    if (!resolved) {
-        throw new Error(`Unknown housekeeping pass '${pass}'. Valid passes: ${PASS_ORDER.join(", ")}`);
-    }
-    return resolved;
-}
-export function dedupObservationsPass(scopeId) {
-    try {
-        const db = getDb();
-        const candidates = db.prepare(`
-      SELECT id, content, confidence, created_at
-      FROM mem_observations
-      WHERE scope_id = ?
-        AND superseded_by IS NULL
-        AND archived_at IS NULL
-      ORDER BY id ASC
-    `).all(scopeId);
-        let modified = 0;
-        const visited = new Set();
-        const update = db.prepare(`UPDATE mem_observations SET superseded_by = ? WHERE id = ? AND superseded_by IS NULL`);
-        const tx = db.transaction(() => {
-            for (const candidate of candidates) {
-                if (visited.has(candidate.id)) {
-                    continue;
-                }
-                const cluster = candidates.filter((entry) => !visited.has(entry.id) && isSimilar(candidate.content, entry.content));
-                for (const entry of cluster) {
-                    visited.add(entry.id);
-                }
-                if (cluster.length < 2) {
-                    continue;
-                }
-                const [keeper] = cluster.sort(compareObservationKeeper);
-                if (!keeper) {
-                    continue;
-                }
-                for (const entry of cluster) {
-                    if (entry.id === keeper.id) {
-                        continue;
-                    }
-                    modified += update.run(keeper.id, entry.id).changes;
-                }
-            }
-        });
-        tx();
-        return passSummary("dedupObservationsPass", candidates.length, modified);
-    }
-    catch (error) {
-        return passSummary("dedupObservationsPass", 0, 0, [error instanceof Error ? error.message : String(error)]);
-    }
-}
-export function dedupDecisionsPass(scopeId) {
-    try {
-        const db = getDb();
-        const candidates = db.prepare(`
-      SELECT id, entity_id, title, decided_at, created_at
-      FROM mem_decisions
-      WHERE scope_id = ?
-        AND superseded_by IS NULL
-        AND archived_at IS NULL
-      ORDER BY id ASC
-    `).all(scopeId);
-        let modified = 0;
-        const update = db.prepare(`UPDATE mem_decisions SET superseded_by = ? WHERE id = ? AND superseded_by IS NULL`);
-        const tx = db.transaction(() => {
-            const visited = new Set();
-            for (const candidate of candidates) {
-                if (visited.has(candidate.id)) {
-                    continue;
-                }
-                const cluster = candidates.filter((entry) => !visited.has(entry.id) && isSimilar(candidate.title, entry.title));
-                for (const entry of cluster) {
-                    visited.add(entry.id);
-                }
-                if (cluster.length < 2) {
-                    continue;
-                }
-                const [keeper] = cluster.sort(compareDecisionKeeper);
-                if (!keeper) {
-                    continue;
-                }
-                for (const entry of cluster) {
-                    if (entry.id === keeper.id) {
-                        continue;
-                    }
-                    modified += update.run(keeper.id, entry.id).changes;
-                }
-            }
-        });
-        tx();
-        return passSummary("dedupDecisionsPass", candidates.length, modified);
-    }
-    catch (error) {
-        return passSummary("dedupDecisionsPass", 0, 0, [error instanceof Error ? error.message : String(error)]);
-    }
-}
-function resolveSupersedeTerminal(startId, firstTargetId, supersededBy) {
-    const seen = new Set([startId]);
-    let current = firstTargetId;
-    while (true) {
-        if (seen.has(current)) {
-            return null;
-        }
-        seen.add(current);
-        const next = supersededBy.get(current);
-        if (next === undefined) {
-            return firstTargetId;
-        }
-        if (next === null) {
-            return current;
-        }
-        current = next;
-    }
-}
-export function compactSupersedeChainsPass(scopeId) {
-    try {
-        const db = getDb();
-        const observationRows = db.prepare(`
-      SELECT id, superseded_by
-      FROM mem_observations
-      WHERE scope_id = ?
-        AND superseded_by IS NOT NULL
-      ORDER BY id ASC
-    `).all(scopeId);
-        const decisionRows = db.prepare(`
-      SELECT id, superseded_by
-      FROM mem_decisions
-      WHERE scope_id = ?
-        AND superseded_by IS NOT NULL
-      ORDER BY id ASC
-    `).all(scopeId);
-        let modified = 0;
-        const tx = db.transaction(() => {
-            for (const { table, rows } of [
-                { table: "mem_observations", rows: observationRows },
-                { table: "mem_decisions", rows: decisionRows },
-            ]) {
-                const targetIds = [...new Set(rows.map((row) => row.superseded_by))];
-                const activeTargets = targetIds.length === 0
-                    ? undefined
-                    : db.prepare(`
-              SELECT id, superseded_by
-              FROM ${table}
-              WHERE scope_id = ?
-                AND id IN (${targetIds.map(() => "?").join(",")})
-            `);
-                const targets = targetIds.length === 0
-                    ? []
-                    : activeTargets.all(scopeId, ...targetIds);
-                const supersededBy = new Map();
-                for (const row of rows) {
-                    supersededBy.set(row.id, row.superseded_by);
-                }
-                for (const target of targets) {
-                    supersededBy.set(target.id, target.superseded_by);
-                }
-                const update = db.prepare(`UPDATE ${table} SET superseded_by = ? WHERE id = ? AND scope_id = ?`);
-                for (const row of rows) {
-                    const terminal = resolveSupersedeTerminal(row.id, row.superseded_by, supersededBy);
-                    if (terminal !== null && terminal !== row.superseded_by) {
-                        modified += update.run(terminal, row.id, scopeId).changes;
-                        supersededBy.set(row.id, terminal);
-                    }
-                }
-            }
-        });
-        tx();
-        return passSummary("compactSupersedeChainsPass", observationRows.length + decisionRows.length, modified);
-    }
-    catch (error) {
-        return passSummary("compactSupersedeChainsPass", 0, 0, [error instanceof Error ? error.message : String(error)]);
-    }
-}
-export function orphanCleanupPass(scopeId) {
-    try {
-        const db = getDb();
-        const orphanIds = db.prepare(`
-      SELECT o.id
-      FROM mem_observations o
-      LEFT JOIN mem_entities e ON e.id = o.entity_id
-      WHERE o.scope_id = ?
-        AND o.entity_id IS NOT NULL
-        AND e.id IS NULL
-      ORDER BY o.id ASC
-    `).all(scopeId);
-        const tx = db.transaction(() => db.prepare(`
-      UPDATE mem_observations
-      SET entity_id = NULL
-      WHERE scope_id = ?
-        AND entity_id IS NOT NULL
-        AND NOT EXISTS (SELECT 1 FROM mem_entities e WHERE e.id = mem_observations.entity_id)
-    `).run(scopeId).changes);
-        const modified = tx();
-        if (modified > 0) {
-            log.info({ scope_id: scopeId, count: modified }, "memory.housekeeping.orphan_cleanup");
-        }
-        return passSummary("orphanCleanupPass", orphanIds.length, modified);
-    }
-    catch (error) {
-        return passSummary("orphanCleanupPass", 0, 0, [error instanceof Error ? error.message : String(error)]);
-    }
-}
-export function decayPass(scopeId) {
-    try {
-        const db = getDb();
-        const candidateIds = db.prepare(`
-      SELECT id
-      FROM mem_observations
-      WHERE scope_id = ?
-        AND superseded_by IS NULL
-        AND archived_at IS NULL
-        AND confidence < 0.3
-        AND datetime(created_at) < datetime('now', ?)
-      ORDER BY id ASC
-    `).all(scopeId, `-${config.memoryDecayDays} days`);
-        const tx = db.transaction(() => db.prepare(`
-      UPDATE mem_observations
-      SET archived_at = CURRENT_TIMESTAMP
-      WHERE scope_id = ?
-        AND superseded_by IS NULL
-        AND archived_at IS NULL
-        AND confidence < 0.3
-        AND datetime(created_at) < datetime('now', ?)
-    `).run(scopeId, `-${config.memoryDecayDays} days`).changes);
-        const modified = tx();
-        return passSummary("decayPass", candidateIds.length, modified);
-    }
-    catch (error) {
-        return passSummary("decayPass", 0, 0, [error instanceof Error ? error.message : String(error)]);
-    }
-}
-export function compactInboxPass() {
-    try {
-        const db = getDb();
-        const candidateIds = db.prepare(`
-      SELECT id
-      FROM mem_inbox
-      WHERE status IN ('accepted', 'rejected')
-        AND resolved_at IS NOT NULL
-        AND datetime(resolved_at) < datetime('now', ?)
-      ORDER BY id ASC
-    `).all(`-${config.memoryInboxRetentionDays} days`);
-        const tx = db.transaction(() => db.prepare(`
-      DELETE FROM mem_inbox
-      WHERE status IN ('accepted', 'rejected')
-        AND resolved_at IS NOT NULL
-        AND datetime(resolved_at) < datetime('now', ?)
-    `).run(`-${config.memoryInboxRetentionDays} days`).changes);
-        const modified = tx();
-        return passSummary("compactInboxPass", candidateIds.length, modified);
-    }
-    catch (error) {
-        return passSummary("compactInboxPass", 0, 0, [error instanceof Error ? error.message : String(error)]);
-    }
-}
-function resolveScopeIds(input) {
-    if (input?.scopeIds && input.scopeIds.length > 0) {
-        return [...new Set(input.scopeIds)];
-    }
-    if (input?.allScopes) {
-        return listScopes().filter((scope) => scope.active).map((scope) => scope.id);
-    }
-    const activeScope = getActiveScope();
-    return activeScope ? [activeScope.id] : [];
-}
-function getInFlightScopes(pass) {
-    let scopes = inFlightScopesByPass.get(pass);
-    if (!scopes) {
-        scopes = new Set();
-        inFlightScopesByPass.set(pass, scopes);
-    }
-    return scopes;
-}
-function getReservedPassScopes(scopeIds, passes) {
-    const reserved = [];
-    for (const pass of passes) {
-        if (pass === "compact_inbox") {
-            reserved.push({ pass, scope: GLOBAL_PASS_SCOPE });
-            continue;
-        }
-        for (const scopeId of scopeIds) {
-            reserved.push({ pass, scope: scopeId });
-        }
-    }
-    return reserved;
-}
-function reservePassScopes(reserved) {
-    if (reserved.some(({ pass, scope }) => getInFlightScopes(pass).has(scope))) {
-        return false;
-    }
-    for (const { pass, scope } of reserved) {
-        getInFlightScopes(pass).add(scope);
-    }
-    return true;
-}
-function releasePassScopes(reserved) {
-    for (const { pass, scope } of reserved) {
-        const scopes = inFlightScopesByPass.get(pass);
-        scopes?.delete(scope);
-        if (scopes && scopes.size === 0) {
-            inFlightScopesByPass.delete(pass);
-        }
-    }
-}
-async function runPass(pass, scopeId) {
-    switch (pass) {
-        case "dedup_observations":
-            return await Promise.resolve(dedupObservationsPass(scopeId));
-        case "dedup_decisions":
-            return await Promise.resolve(dedupDecisionsPass(scopeId));
-        case "compact_supersede_chains":
-            return await Promise.resolve(compactSupersedeChainsPass(scopeId));
-        case "orphan_cleanup":
-            return await Promise.resolve(orphanCleanupPass(scopeId));
-        case "decay":
-            return await Promise.resolve(decayPass(scopeId));
-        case "compact_inbox":
-            return await Promise.resolve(compactInboxPass());
-        case "tiering":
-            return await Promise.resolve(tieringPass(scopeId));
-    }
-}
-async function runScopePasses(scopeId, passes) {
-    const summaries = [];
-    for (const pass of passes) {
-        summaries.push(await runPass(pass, scopeId));
-    }
-    return summaries;
-}
-export function isHousekeepingInFlight(scopeIds, passes) {
-    const normalizedPasses = passes?.map(normalizePassName) ?? PASS_ORDER;
-    if (!scopeIds || scopeIds.length === 0) {
-        return normalizedPasses.some((pass) => (inFlightScopesByPass.get(pass)?.size ?? 0) > 0);
-    }
-    const uniqueScopeIds = [...new Set(scopeIds)].sort((a, b) => a - b);
-    return normalizedPasses.some((pass) => {
-        const scopes = inFlightScopesByPass.get(pass);
-        if (!scopes) {
-            return false;
-        }
-        if (pass === "compact_inbox") {
-            return scopes.has(GLOBAL_PASS_SCOPE);
-        }
-        return uniqueScopeIds.some((scopeId) => scopes.has(scopeId));
-    });
-}
-export async function runHousekeeping(opts = {}) {
-    const started = performance.now();
-    const scopeIds = resolveScopeIds(opts).sort((a, b) => a - b);
-    const passes = opts.passes?.length ? opts.passes.map(normalizePassName) : PASS_ORDER;
-    const reservedPassScopes = getReservedPassScopes(scopeIds, passes);
-    if (!reservePassScopes(reservedPassScopes)) {
-        return {
-            scopeIds,
-            summaries: [passSummary("runHousekeeping", 0, 0, ["Housekeeping is already in flight for this scope/pass set."])],
-            totalExamined: 0,
-            totalModified: 0,
-            durationMs: 0,
-        };
-    }
-    const scopedPasses = passes.filter((pass) => pass !== "compact_inbox");
-    const lockedScopeIds = scopedPasses.length > 0 ? scopeIds : [];
-    if (lockedScopeIds.length > 0 && !tryAcquireScopeWriteLocks(lockedScopeIds)) {
-        releasePassScopes(reservedPassScopes);
-        return {
-            scopeIds,
-            summaries: [passSummary("runHousekeeping", 0, 0, ["Memory writes are already in flight for this scope."])],
-            totalExamined: 0,
-            totalModified: 0,
-            durationMs: 0,
-        };
-    }
-    try {
-        const hasCompactInbox = passes.includes("compact_inbox");
-        const summaries = (await Promise.all(scopeIds.map((scopeId) => runScopePasses(scopeId, scopedPasses)))).flat();
-        if (hasCompactInbox) {
-            summaries.push(await runPass("compact_inbox", undefined));
-        }
-        const totalExamined = summaries.reduce((sum, summary) => sum + summary.examined, 0);
-        const totalModified = summaries.reduce((sum, summary) => sum + summary.modified, 0);
-        const durationMs = Math.round(performance.now() - started);
-        log.info({
-            passes_run: summaries.map((summary) => summary.pass),
-            total_examined: totalExamined,
-            total_modified: totalModified,
-            duration_ms: durationMs,
-            scope_ids: scopeIds,
-        }, "memory.housekeeping.run");
-        return { scopeIds, summaries, totalExamined, totalModified, durationMs };
-    }
-    finally {
-        releaseScopeWriteLocks(lockedScopeIds);
-        releasePassScopes(reservedPassScopes);
-    }
-}
-//# sourceMappingURL=housekeeping.js.map