chapterhouse 0.5.1 → 0.5.2

package/dist/memory/housekeeping.js

@@ -8,7 +8,8 @@ import { tieringPass } from "./tiering.js";
 export { tieringPass };
 const log = childLogger("memory.housekeeping");
 const SIMILARITY_THRESHOLD = 0.8;
-const inFlightKeys = new Set();
+const GLOBAL_PASS_SCOPE = Symbol("global-housekeeping-pass-scope");
+const inFlightScopesByPass = new Map();
 const PASS_ORDER = [
     "dedup_observations",
     "dedup_decisions",
@@ -280,38 +281,91 @@ function resolveScopeIds(input) {
     const activeScope = getActiveScope();
     return activeScope ? [activeScope.id] : [];
 }
-function runPass(pass, scopeId) {
+function getInFlightScopes(pass) {
+    let scopes = inFlightScopesByPass.get(pass);
+    if (!scopes) {
+        scopes = new Set();
+        inFlightScopesByPass.set(pass, scopes);
+    }
+    return scopes;
+}
+function getReservedPassScopes(scopeIds, passes) {
+    const reserved = [];
+    for (const pass of passes) {
+        if (pass === "compact_inbox") {
+            reserved.push({ pass, scope: GLOBAL_PASS_SCOPE });
+            continue;
+        }
+        for (const scopeId of scopeIds) {
+            reserved.push({ pass, scope: scopeId });
+        }
+    }
+    return reserved;
+}
+function reservePassScopes(reserved) {
+    if (reserved.some(({ pass, scope }) => getInFlightScopes(pass).has(scope))) {
+        return false;
+    }
+    for (const { pass, scope } of reserved) {
+        getInFlightScopes(pass).add(scope);
+    }
+    return true;
+}
+function releasePassScopes(reserved) {
+    for (const { pass, scope } of reserved) {
+        const scopes = inFlightScopesByPass.get(pass);
+        scopes?.delete(scope);
+        if (scopes && scopes.size === 0) {
+            inFlightScopesByPass.delete(pass);
+        }
+    }
+}
+async function runPass(pass, scopeId) {
     switch (pass) {
         case "dedup_observations":
-            return dedupObservationsPass(scopeId);
+            return await Promise.resolve(dedupObservationsPass(scopeId));
         case "dedup_decisions":
-            return dedupDecisionsPass(scopeId);
+            return await Promise.resolve(dedupDecisionsPass(scopeId));
         case "orphan_cleanup":
-            return orphanCleanupPass(scopeId);
+            return await Promise.resolve(orphanCleanupPass(scopeId));
         case "decay":
-            return decayPass(scopeId);
+            return await Promise.resolve(decayPass(scopeId));
         case "compact_inbox":
-            return compactInboxPass();
+            return await Promise.resolve(compactInboxPass());
         case "tiering":
-            return tieringPass(scopeId);
+            return await Promise.resolve(tieringPass(scopeId));
     }
 }
-function inFlightKey(scopeIds, passes) {
-    return `${scopeIds.join(",") || "none"}:${passes.join(",")}`;
+async function runScopePasses(scopeId, passes) {
+    const summaries = [];
+    for (const pass of passes) {
+        summaries.push(await runPass(pass, scopeId));
+    }
+    return summaries;
 }
 export function isHousekeepingInFlight(scopeIds, passes) {
+    const normalizedPasses = passes?.map(normalizePassName) ?? PASS_ORDER;
     if (!scopeIds || scopeIds.length === 0) {
-        return inFlightKeys.size > 0;
+        return normalizedPasses.some((pass) => (inFlightScopesByPass.get(pass)?.size ?? 0) > 0);
     }
-    const normalizedPasses = passes?.map(normalizePassName) ?? PASS_ORDER;
-    return inFlightKeys.has(inFlightKey([...new Set(scopeIds)].sort((a, b) => a - b), normalizedPasses));
+    const uniqueScopeIds = [...new Set(scopeIds)].sort((a, b) => a - b);
+    return normalizedPasses.some((pass) => {
+        const scopes = inFlightScopesByPass.get(pass);
+        if (!scopes) {
+            return false;
+        }
+        if (pass === "compact_inbox") {
+            return scopes.has(GLOBAL_PASS_SCOPE);
+        }
+        return uniqueScopeIds.some((scopeId) => scopes.has(scopeId));
+    });
 }
-export function runHousekeeping(opts = {}) {
+export async function runHousekeeping(opts = {}) {
     const started = performance.now();
     const scopeIds = resolveScopeIds(opts).sort((a, b) => a - b);
     const passes = opts.passes?.length ? opts.passes.map(normalizePassName) : PASS_ORDER;
-    const key = inFlightKey(scopeIds, passes);
-    if (inFlightKeys.has(key)) {
+    const reservedPassScopes = getReservedPassScopes(scopeIds, passes);
+    if (!reservePassScopes(reservedPassScopes)) {
         return {
             scopeIds,
             summaries: [passSummary("runHousekeeping", 0, 0, ["Housekeeping is already in flight for this scope/pass set."])],
@@ -320,18 +374,12 @@ export function runHousekeeping(opts = {}) {
             durationMs: 0,
         };
     }
-    inFlightKeys.add(key);
-    const summaries = [];
     try {
         const scopedPasses = passes.filter((pass) => pass !== "compact_inbox");
         const hasCompactInbox = passes.includes("compact_inbox");
-        for (const scopeId of scopeIds) {
-            for (const pass of scopedPasses) {
-                summaries.push(runPass(pass, scopeId));
-            }
-        }
+        const summaries = (await Promise.all(scopeIds.map((scopeId) => runScopePasses(scopeId, scopedPasses)))).flat();
         if (hasCompactInbox) {
-            summaries.push(compactInboxPass());
+            summaries.push(await runPass("compact_inbox", undefined));
         }
         const totalExamined = summaries.reduce((sum, summary) => sum + summary.examined, 0);
         const totalModified = summaries.reduce((sum, summary) => sum + summary.modified, 0);
@@ -346,7 +394,7 @@ export function runHousekeeping(opts = {}) {
         return { scopeIds, summaries, totalExamined, totalModified, durationMs };
     }
     finally {
-        inFlightKeys.delete(key);
+        releasePassScopes(reservedPassScopes);
     }
 }
 //# sourceMappingURL=housekeeping.js.map

package/dist/memory/housekeeping.test.js

@@ -16,9 +16,51 @@ function resetSandbox() {
 async function loadModules() {
     const dbModule = await import(new URL("../store/db.js", import.meta.url).href);
     const memoryModule = await import(new URL("./index.js", import.meta.url).href);
-    const housekeepingModule = await import(new URL("./housekeeping.js", import.meta.url).href);
+    const housekeepingModule = await import(new URL(`./housekeeping.js?case=${Date.now()}-${Math.random()}`, import.meta.url).href);
     return { dbModule, memoryModule, housekeepingModule };
 }
+async function loadMockedHousekeepingModule(t, options = {}) {
+    t.mock.module("../config.js", {
+        namedExports: {
+            config: {
+                memoryDecayDays: 30,
+                memoryInboxRetentionDays: 7,
+            },
+        },
+    });
+    t.mock.module("../store/db.js", {
+        namedExports: {
+            getDb: () => {
+                throw new Error("getDb should not be called in this test");
+            },
+        },
+    });
+    t.mock.module("../util/logger.js", {
+        namedExports: {
+            childLogger: () => ({
+                info: () => { },
+                warn: () => { },
+                error: () => { },
+            }),
+        },
+    });
+    t.mock.module("./active-scope.js", {
+        namedExports: {
+            getActiveScope: () => (options.activeScopeId === undefined ? null : { id: options.activeScopeId }),
+        },
+    });
+    t.mock.module("./scopes.js", {
+        namedExports: {
+            listScopes: () => options.scopes ?? [],
+        },
+    });
+    t.mock.module("./tiering.js", {
+        namedExports: {
+            tieringPass: (scopeId) => options.tieringPass?.(scopeId) ?? { pass: "tieringPass", examined: scopeId, modified: 1, errors: [] },
+        },
+    });
+    return await import(new URL(`./housekeeping.js?case=${Date.now()}-${Math.random()}`, import.meta.url).href);
+}
 function getFunction(module, name) {
     const value = module[name];
     assert.equal(typeof value, "function", `expected ${name} to be exported`);
@@ -215,17 +257,67 @@ test("runHousekeeping defaults to the active scope and can target all active sco
     const teamOld = recordObservation({ scope_id: team.id, content: "Team old low", source: "test", confidence: 0.1 });
     db.prepare(`UPDATE mem_observations SET created_at = datetime('now', '-31 days') WHERE id IN (?, ?)`).run(chapterhouseOld.id, teamOld.id);
     setActiveScope("chapterhouse");
-    const activeOnly = housekeepingModule.runHousekeeping({ passes: ["decay"] });
+    const activeOnly = await housekeepingModule.runHousekeeping({ passes: ["decay"] });
     assert.deepEqual(activeOnly.scopeIds, [chapterhouse.id]);
     assert.equal(activeOnly.summaries.length, 1);
     assert.equal(activeOnly.summaries[0]?.modified, 1);
     assert.ok(db.prepare(`SELECT archived_at FROM mem_observations WHERE id = ?`).get(chapterhouseOld.id).archived_at);
     assert.equal(db.prepare(`SELECT archived_at FROM mem_observations WHERE id = ?`).get(teamOld.id).archived_at, null);
-    const allScopes = housekeepingModule.runHousekeeping({ allScopes: true, passes: ["decay"] });
+    const allScopes = await housekeepingModule.runHousekeeping({ allScopes: true, passes: ["decay"] });
     assert.ok(allScopes.scopeIds.includes(team.id));
     assert.equal(allScopes.summaries.some((summary) => summary.modified === 1), true);
     assert.ok(db.prepare(`SELECT archived_at FROM mem_observations WHERE id = ?`).get(teamOld.id).archived_at);
 });
+test("runHousekeeping starts all scoped passes before awaiting completion", async (t) => {
+    const releases = new Map();
+    const startedScopes = [];
+    const housekeepingModule = await loadMockedHousekeepingModule(t, {
+        scopes: [
+            { id: 11, active: true },
+            { id: 22, active: true },
+        ],
+        tieringPass: async (scopeId) => {
+            startedScopes.push(scopeId);
+            await new Promise((resolve) => {
+                releases.set(scopeId, resolve);
+            });
+            return { pass: `tieringPass:${scopeId}`, examined: 1, modified: 1, errors: [] };
+        },
+    });
+    const pending = housekeepingModule.runHousekeeping({ allScopes: true, passes: ["tiering"] });
+    assert.equal(typeof pending.then, "function");
+    await Promise.resolve();
+    assert.deepEqual(startedScopes.sort((left, right) => left - right), [11, 22]);
+    releases.get(11)?.();
+    releases.get(22)?.();
+    const result = await pending;
+    assert.deepEqual(result.scopeIds, [11, 22]);
+    assert.deepEqual(result.summaries.map((summary) => summary.pass), ["tieringPass:11", "tieringPass:22"]);
+});
+test("runHousekeeping rejects overlapping runs that share an in-flight scope", async (t) => {
+    const releases = new Map();
+    const housekeepingModule = await loadMockedHousekeepingModule(t, {
+        scopes: [
+            { id: 11, active: true },
+            { id: 22, active: true },
+        ],
+        tieringPass: async (scopeId) => {
+            await new Promise((resolve) => {
+                releases.set(scopeId, resolve);
+            });
+            return { pass: `tieringPass:${scopeId}`, examined: 1, modified: 1, errors: [] };
+        },
+    });
+    const firstRun = housekeepingModule.runHousekeeping({ scopeIds: [11, 22], passes: ["tiering"] });
+    await Promise.resolve();
+    assert.equal(housekeepingModule.isHousekeepingInFlight([22], ["tiering"]), true);
+    const secondRun = await housekeepingModule.runHousekeeping({ scopeIds: [22], passes: ["tiering"] });
+    assert.deepEqual(secondRun.scopeIds, [22]);
+    assert.match(secondRun.summaries[0]?.errors[0] ?? "", /already in flight/i);
+    releases.get(11)?.();
+    releases.get(22)?.();
+    await firstRun;
+});
 test("tieringPass promotes and demotes rows from lifecycle signals and is idempotent", async () => {
     const { dbModule, memoryModule, housekeepingModule } = await loadModules();
     const db = dbModule.getDb();

package/dist/memory/inbox.test.js

@@ -0,0 +1,178 @@
+import assert from "node:assert/strict";
+import { mkdirSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import test from "node:test";
+const repoRoot = process.cwd();
+const sandboxRoot = join(repoRoot, ".test-work", `memory-inbox-${process.pid}`);
+const chapterhouseHome = join(sandboxRoot, ".chapterhouse");
+process.env.CHAPTERHOUSE_HOME = sandboxRoot;
+function resetSandbox() {
+    mkdirSync(join(repoRoot, ".test-work"), { recursive: true });
+    rmSync(sandboxRoot, { recursive: true, force: true });
+    mkdirSync(chapterhouseHome, { recursive: true });
+}
+async function loadRealModules(cacheBust = `${Date.now()}-${Math.random()}`) {
+    const dbModule = await import(new URL("../store/db.js", import.meta.url).href);
+    const memoryModule = await import(new URL(`./index.js?case=${cacheBust}`, import.meta.url).href);
+    const inboxModule = await import(new URL(`./inbox.js?case=${cacheBust}`, import.meta.url).href);
+    return { dbModule, memoryModule, inboxModule };
+}
+async function loadMockedInboxModule(t, options) {
+    t.mock.module("../store/db.js", {
+        namedExports: {
+            getDb: () => ({ prepare: options.prepare }),
+        },
+    });
+    t.mock.module("./active-scope.js", {
+        namedExports: {
+            getActiveScope: () => options.activeScope ?? null,
+        },
+    });
+    t.mock.module("./scopes.js", {
+        namedExports: {
+            getScope: (slug) => options.explicitScopes?.[slug],
+        },
+    });
+    return await import(new URL(`./inbox.js?case=${Date.now()}-${Math.random()}`, import.meta.url).href);
+}
+function getFunction(module, name) {
+    const value = module[name];
+    assert.equal(typeof value, "function", `expected ${name} to be exported`);
+    return value;
+}
+function createTestScope(memoryModule, slug) {
+    const createScope = getFunction(memoryModule, "createScope");
+    return createScope({
+        slug,
+        title: slug.split("-").map((part) => part[0]?.toUpperCase() + part.slice(1)).join(" "),
+        description: `${slug} test scope`,
+        keywords: [slug],
+    });
+}
+test.beforeEach(async () => {
+    process.env.CHAPTERHOUSE_HOME = sandboxRoot;
+    const dbModule = await import(new URL("../store/db.js", import.meta.url).href);
+    dbModule.closeDb();
+    resetSandbox();
+});
+test.after(async () => {
+    const dbModule = await import(new URL("../store/db.js", import.meta.url).href);
+    dbModule.closeDb();
+    rmSync(sandboxRoot, { recursive: true, force: true });
+});
+test("resolveProposalScope returns explicit scopes and falls back to the active scope for missing or empty values", async () => {
+    const { dbModule, memoryModule, inboxModule } = await loadRealModules("resolve-fallback");
+    dbModule.getDb();
+    const docs = createTestScope(memoryModule, "docs");
+    const setActiveScope = getFunction(memoryModule, "setActiveScope");
+    const active = createTestScope(memoryModule, "team-memory");
+    setActiveScope(active.slug);
+    assert.deepEqual(inboxModule.resolveProposalScope(docs.slug), { scopeId: docs.id, scopeSlug: docs.slug });
+    assert.deepEqual(inboxModule.resolveProposalScope(undefined), { scopeId: active.id, scopeSlug: active.slug });
+    assert.deepEqual(inboxModule.resolveProposalScope(""), { scopeId: active.id, scopeSlug: active.slug });
+});
+test("resolveProposalScope rejects invalid explicit scopes and missing default scopes", async () => {
+    const { dbModule, inboxModule } = await loadRealModules("resolve-errors");
+    dbModule.getDb();
+    assert.throws(() => inboxModule.resolveProposalScope("does-not-exist"), /Unknown memory scope 'does-not-exist'\./);
+    assert.throws(() => inboxModule.resolveProposalScope(), /No active memory scope is set\. Use memory_set_scope or pass scope_slug explicitly\./);
+});
+test("queueMemoryProposal enqueues explicit-scope proposals with the expected envelope defaults", async () => {
+    const { dbModule, memoryModule, inboxModule } = await loadRealModules("explicit-queue");
+    const db = dbModule.getDb();
+    const docs = createTestScope(memoryModule, "docs-queue");
+    const queued = inboxModule.queueMemoryProposal({
+        kind: "observation",
+        scopeSlug: docs.slug,
+        payload: {
+            content: "Inbox tests should verify the stored proposal envelope.",
+            source: "oracle",
+        },
+        sourceAgent: "oracle",
+        sourceTaskId: "task-inbox-explicit",
+    });
+    assert.equal(typeof queued.id, "number");
+    assert.equal(queued.scopeId, docs.id);
+    assert.equal(queued.kind, "memory_proposal");
+    assert.equal(queued.status, "pending");
+    assert.equal(queued.sourceAgent, "oracle");
+    assert.equal(queued.sourceTaskId, "task-inbox-explicit");
+    assert.match(queued.createdAt, /\d{4}-\d{2}-\d{2}/);
+    const persisted = db.prepare(`
+    SELECT payload
+    FROM mem_inbox
+    WHERE id = ?
+  `).get(queued.id);
+    assert.ok(persisted, "queueMemoryProposal should persist a mem_inbox row");
+    assert.deepEqual(JSON.parse(persisted.payload), {
+        kind: "observation",
+        scope_slug: docs.slug,
+        confidence: 0.5,
+        payload: {
+            content: "Inbox tests should verify the stored proposal envelope.",
+            source: "oracle",
+        },
+    });
+    assert.deepEqual(inboxModule.getInboxItem(queued.id), queued);
+});
+test("queueMemoryProposal falls back to the active scope, does not deduplicate duplicates, and pending lists exclude resolved items", async () => {
+    const { dbModule, memoryModule, inboxModule } = await loadRealModules("active-scope-queue");
+    dbModule.getDb();
+    const setActiveScope = getFunction(memoryModule, "setActiveScope");
+    setActiveScope("chapterhouse");
+    const first = inboxModule.queueMemoryProposal({
+        kind: "observation",
+        scopeSlug: "",
+        payload: { content: "Duplicate proposal payloads should still queue separately." },
+        confidence: 0.8,
+        reason: "First proposal",
+        sourceAgent: "oracle",
+        sourceTaskId: "task-inbox-duplicates",
+    });
+    const second = inboxModule.queueMemoryProposal({
+        kind: "observation",
+        payload: { content: "Duplicate proposal payloads should still queue separately." },
+        confidence: 0.8,
+        reason: "Second proposal",
+        sourceAgent: "oracle",
+        sourceTaskId: "task-inbox-duplicates",
+    });
+    assert.notEqual(first.id, second.id);
+    assert.equal(first.scopeId, second.scopeId);
+    assert.deepEqual(inboxModule.listPendingMemoryProposalsForTask("task-inbox-duplicates").map((item) => item.id), [first.id, second.id]);
+    inboxModule.resolveInboxItem(first.id, "accepted", "Durable enough to keep");
+    const resolved = inboxModule.getInboxItem(first.id);
+    assert.equal(resolved?.status, "accepted");
+    assert.equal(resolved?.resolutionReason, "Durable enough to keep");
+    assert.match(resolved?.resolvedAt ?? "", /\d{4}-\d{2}-\d{2}/);
+    assert.deepEqual(inboxModule.listPendingMemoryProposalsForTask("task-inbox-duplicates").map((item) => item.id), [second.id]);
+});
+test("queueMemoryProposal surfaces insert failures such as a full queue", async (t) => {
+    const fullError = new Error("database or disk is full");
+    const seenStatements = [];
+    const inboxModule = await loadMockedInboxModule(t, {
+        activeScope: { id: 99, slug: "chapterhouse" },
+        prepare: (sql) => {
+            seenStatements.push(sql);
+            if (sql.includes("INSERT INTO mem_inbox")) {
+                return {
+                    run: () => {
+                        throw fullError;
+                    },
+                };
+            }
+            return {
+                run: () => ({ changes: 0 }),
+                get: () => undefined,
+                all: () => [],
+            };
+        },
+    });
+    assert.throws(() => inboxModule.queueMemoryProposal({
+        kind: "observation",
+        payload: { content: "This proposal should fail before it is persisted." },
+        sourceAgent: "oracle",
+    }), /database or disk is full/);
+    assert.equal(seenStatements.some((sql) => sql.includes("INSERT INTO mem_inbox")), true);
+});
+//# sourceMappingURL=inbox.test.js.map