chapterhouse 0.7.0 → 0.8.1

package/dist/wiki/consolidation.test.js

@@ -0,0 +1,143 @@
+import assert from "node:assert/strict";
+import { mkdirSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import test from "node:test";
+import { resetSingletons } from "../test/helpers/reset-singletons.js";
+const repoRoot = process.cwd();
+const sandboxRoot = join(repoRoot, ".test-work", `wiki-consolidation-${process.pid}`);
+const chapterhouseHome = join(sandboxRoot, ".chapterhouse");
+function resetSandbox() {
+    mkdirSync(join(repoRoot, ".test-work"), { recursive: true });
+    rmSync(sandboxRoot, { recursive: true, force: true });
+    mkdirSync(chapterhouseHome, { recursive: true });
+}
+async function loadModules() {
+    const nonce = `${Date.now()}-${Math.random()}`;
+    const dbModule = await import(new URL(`../store/db.js?c=${nonce}`, import.meta.url).href);
+    const wikiFs = await import(new URL(`./fs.js?c=${nonce}`, import.meta.url).href);
+    const indexManager = await import(new URL(`./index-manager.js?c=${nonce}`, import.meta.url).href);
+    const consolidation = await import(new URL(`./consolidation.js?c=${nonce}`, import.meta.url).href);
+    const memory = await import(new URL(`../memory/index.js?c=${nonce}`, import.meta.url).href);
+    return { dbModule, wikiFs, indexManager, consolidation, memory };
+}
+function makePage(title, summary, updated, summaryBody, timelineBlocks, pinned = false) {
+    return `---\ntitle: ${title}\nsummary: ${summary}\nupdated: ${updated}\ntags: []\npinned: ${pinned ? "true" : "false"}\n---\n\n# ${title}\n\n## Summary\n\n${summaryBody}\n\n## Timeline\n\n${timelineBlocks.trim()}\n`;
+}
+test.beforeEach(async () => {
+    process.env.CHAPTERHOUSE_HOME = sandboxRoot;
+    const dbModule = await import(new URL("../store/db.js", import.meta.url).href);
+    dbModule.closeDb();
+    resetSandbox();
+    resetSingletons();
+});
+test.after(async () => {
+    const dbModule = await import(new URL("../store/db.js", import.meta.url).href);
+    dbModule.closeDb();
+    resetSingletons();
+    rmSync(sandboxRoot, { recursive: true, force: true });
+});
+test("runConsolidation rewrites stale compiled truth and skips pinned pages", async () => {
+    const { dbModule, wikiFs, indexManager, consolidation } = await loadModules();
+    const db = dbModule.getDb();
+    wikiFs.ensureWikiStructure();
+    wikiFs.writePage("pages/topics/rust/index.md", makePage("Rust", "Systems language", "2026-05-01", "Old summary.", `### 2026-05-02T10:00:00.000Z\n\nStabilized async patterns.\n\n### 2026-05-04T10:00:00.000Z\n\nAdopted by backend team.`));
+    wikiFs.writePage("pages/topics/pinned/index.md", makePage("Pinned", "Pinned page", "2026-05-01", "Keep this summary.", `### 2026-05-10T10:00:00.000Z\n\nNew fact that should not trigger rewrite.`, true));
+    indexManager.rebuildWikiIndex();
+    const result = await consolidation.runConsolidationWithDeps(db, {
+        now: () => new Date("2026-05-14T22:30:03.086Z"),
+        synthesizeTruth: async ({ pagePath, pendingEntries }) => `Synthesized ${pagePath} from ${pendingEntries.length} entries.`,
+        commitWikiChanges: async () => false,
+    });
+    const rewritten = wikiFs.readPage("pages/topics/rust/index.md") ?? "";
+    const pinned = wikiFs.readPage("pages/topics/pinned/index.md") ?? "";
+    const row = db.prepare(`SELECT compiled_truth_hash FROM wiki_pages WHERE path = ?`).get("pages/topics/rust/index.md");
+    assert.match(rewritten, /Synthesized pages\/topics\/rust\/index.md from 2 entries\./);
+    assert.match(rewritten, /last_updated:/);
+    assert.ok(row.compiled_truth_hash, "compiled truth hash should be stored");
+    assert.equal(result.truthRewrites, 1);
+    assert.equal(result.llmCallsUsed, 1);
+    assert.match(pinned, /Keep this summary\./);
+    assert.doesNotMatch(pinned, /Synthesized/);
+});
+test("runConsolidation merges fragment pages into a canonical page and repoints links", async () => {
+    const { dbModule, wikiFs, indexManager, consolidation } = await loadModules();
+    const db = dbModule.getDb();
+    wikiFs.ensureWikiStructure();
+    wikiFs.writePage("pages/topics/rust/index.md", makePage("Rust", "Main Rust page", "2026-05-10", "Canonical summary.", `### 2026-05-10T10:00:00.000Z\n\nCanonical fact.`));
+    wikiFs.writePage("pages/topics/ruts/index.md", makePage("Ruts", "Fragment Rust page", "2026-05-11", "Fragment summary.", `### 2026-05-11T10:00:00.000Z\n\nFragment fact.`));
+    wikiFs.writePage("pages/topics/async/index.md", makePage("Async", "Async topic", "2026-05-10", "Async summary with [[Ruts]].", `### 2026-05-10T08:00:00.000Z\n\nReferences [[Ruts]].`));
+    indexManager.rebuildWikiIndex();
+    const result = await consolidation.runConsolidationWithDeps(db, {
+        now: () => new Date("2026-05-14T22:30:03.086Z"),
+        synthesizeTruth: async () => "unused",
+        commitWikiChanges: async () => false,
+    });
+    const canonical = wikiFs.readPage("pages/topics/rust/index.md") ?? "";
+    const fragmentExists = wikiFs.pageExists("pages/topics/ruts/index.md");
+    const repointed = db.prepare(`SELECT COUNT(*) as c FROM wiki_links WHERE to_page = ?`).get("pages/topics/rust/index.md");
+    const fragmentRow = db.prepare(`SELECT path FROM wiki_pages WHERE path = ?`).get("pages/topics/ruts/index.md");
+    assert.equal(result.fragmentsMerged, 1);
+    assert.match(canonical, /Canonical fact\./);
+    assert.match(canonical, /Fragment fact\./);
+    assert.equal(fragmentExists, false);
+    assert.equal(fragmentRow, undefined);
+    assert.ok(repointed.c >= 1, "links should be repointed to the canonical page");
+});
+test("runConsolidation removes orphaned wiki_links rows", async () => {
+    const { dbModule, wikiFs, indexManager, consolidation } = await loadModules();
+    const db = dbModule.getDb();
+    wikiFs.ensureWikiStructure();
+    wikiFs.writePage("pages/topics/alpha/index.md", makePage("Alpha", "Alpha page", "2026-05-10", "Alpha summary.", `### 2026-05-10T10:00:00.000Z\n\nAlpha fact.`));
+    indexManager.rebuildWikiIndex();
+    db.prepare(`INSERT INTO wiki_links (from_page, to_page, link_type, extracted_at) VALUES (?, ?, ?, ?)`)
+        .run("pages/topics/alpha/index.md", "pages/topics/missing/index.md", "references", "2026-05-12T00:00:00.000Z");
+    const result = await consolidation.runConsolidationWithDeps(db, {
+        now: () => new Date("2026-05-14T22:30:03.086Z"),
+        synthesizeTruth: async () => "unused",
+        commitWikiChanges: async () => false,
+    });
+    const orphanCount = db.prepare(`SELECT COUNT(*) as c FROM wiki_links WHERE to_page = ?`).get("pages/topics/missing/index.md");
+    assert.equal(result.linksRepaired, 1);
+    assert.equal(orphanCount.c, 0);
+});
+test("runConsolidation creates an action item for research sessions inactive for 7+ days", async () => {
+    const { dbModule, wikiFs, consolidation, memory } = await loadModules();
+    const db = dbModule.getDb();
+    wikiFs.ensureWikiStructure();
+    const globalScope = memory.getScope("global");
+    assert.ok(globalScope);
+    db.prepare(`
+    INSERT INTO wiki_sources (id, source_type, origin, title, ingested_at, raw_path, parsed_content, pages_updated, status, session_id, session_name)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `).run("source-1", "url", "https://example.test/1", "Research note", "2026-05-01T10:00:00.000Z", "sources/source-1.md", "content", "[]", "active", "session-123", "Compiler research");
+    const result = await consolidation.runConsolidationWithDeps(db, {
+        now: () => new Date("2026-05-14T22:30:03.086Z"),
+        synthesizeTruth: async () => "unused",
+        commitWikiChanges: async () => false,
+    });
+    const actionItems = memory.listActionItems({ scope_id: globalScope.id, includeArchived: true });
+    assert.equal(result.staleSessionsNotified, 1);
+    assert.equal(actionItems.some((item) => item.title.includes("Research session 'Compiler research' has been inactive for 7+ days")), true);
+});
+test("runConsolidation caps truth rewrites before exceeding the LLM budget", async () => {
+    const { dbModule, wikiFs, indexManager, consolidation } = await loadModules();
+    const db = dbModule.getDb();
+    wikiFs.ensureWikiStructure();
+    for (let i = 0; i < 25; i++) {
+        wikiFs.writePage(`pages/topics/topic-${i}/index.md`, makePage(`Topic ${i}`, `Topic ${i}`, "2026-05-01", `Old topic ${i} summary.`, `### 2026-05-13T10:00:00.000Z\n\nFresh fact ${i}.`));
+    }
+    indexManager.rebuildWikiIndex();
+    let llmCalls = 0;
+    const result = await consolidation.runConsolidationWithDeps(db, {
+        now: () => new Date("2026-05-14T22:30:03.086Z"),
+        synthesizeTruth: async () => {
+            llmCalls += 1;
+            return `Synthesized call ${llmCalls}`;
+        },
+        commitWikiChanges: async () => false,
+    });
+    assert.equal(result.truthRewrites, 18);
+    assert.equal(result.llmCallsUsed, 18);
+    assert.equal(llmCalls, 18);
+});
+//# sourceMappingURL=consolidation.test.js.map

package/dist/wiki/frontmatter.js

@@ -1,3 +1,4 @@
+import { normalizeWikiPath } from "./path-utils.js";
 const FRONTMATTER_RE = /^---\s*\n([\s\S]*?)\n---\s*\n?/;
 const SUMMARY_MARKDOWN_RE = /(\*\*|__|[_*`~]|^\s*#+\s|\[[^\]]+\]\([^)]+\)|!\[[^\]]*\]\([^)]+\)|^\s*>)/m;
 const FRONTMATTER_TEMPLATE = `---\ntitle: <title>\nsummary: <plain-text one-line summary, max 200 chars>\nupdated: YYYY-MM-DD\ntags: []\nrelated: []\n---`;
@@ -208,6 +209,53 @@ export function validateProjectRulesFrontmatter(content, options = {}) {
         warnings,
     };
 }
+/**
+ * Validate frontmatter and backfill required fields that are missing.
+ * Required fields: title, entity_type (null default), last_updated, version.
+ * Returns updated content and whether any changes were made.
+ */
+export function validateAndBackfillFrontmatter(path, content) {
+    const { parsed: fm } = parseWikiFrontmatter(content);
+    // Also handle missing-frontmatter case: inject a minimal block
+    if (!hasWikiFrontmatter(content)) {
+        const title = deriveTitleFromPath(path);
+        const now = new Date().toISOString();
+        const injected = `---\ntitle: ${title}\nlast_updated: ${now}\nversion: 1\n---\n\n${content}`;
+        return { content: injected, changed: true };
+    }
+    let changed = false;
+    const fmMatch = content.match(/^---\s*\n([\s\S]*?)\n---\s*\n?/);
+    if (!fmMatch)
+        return { content, changed: false };
+    const body = content.slice(fmMatch[0].length);
+    const fmLines = fmMatch[1].split("\n");
+    const fmKeys = new Set(fmLines.map((l) => l.split(":")[0].trim()).filter(Boolean));
+    if (!fmKeys.has("title") || !fm.title?.trim()) {
+        fmLines.push(`title: ${deriveTitleFromPath(path)}`);
+        changed = true;
+    }
+    if (!fmKeys.has("last_updated")) {
+        fmLines.push(`last_updated: ${new Date().toISOString()}`);
+        changed = true;
+    }
+    if (!fmKeys.has("version")) {
+        fmLines.push("version: 1");
+        changed = true;
+    }
+    // entity_type: only add if not already present (null default is fine — don't inject "null" string)
+    if (!changed)
+        return { content, changed: false };
+    const newContent = `---\n${fmLines.join("\n")}\n---\n${body}`;
+    return { content: newContent, changed: true };
+}
+function deriveTitleFromPath(path) {
+    const normalizedPath = normalizeWikiPath(path);
+    const segs = normalizedPath.split("/").filter(Boolean);
+    const file = segs[segs.length - 1] || normalizedPath;
+    const base = file.replace(/\.md$/, "");
+    const titleBase = base === "index" && segs.length >= 2 ? segs[segs.length - 2] : base;
+    return titleBase.split(/[-_]+/).map((w) => w.charAt(0).toUpperCase() + w.slice(1)).join(" ");
+}
 function formatFrontmatterMessage(reason) {
     return `Wiki page frontmatter violates the required shape: ${reason}. Use:\n${FRONTMATTER_TEMPLATE}`;
 }

package/dist/wiki/frontmatter.test.js

@@ -226,4 +226,46 @@ custom_rule: preserve-me
         },
     ]);
 });
+test("validateAndBackfillFrontmatter backfills missing required fields", async () => {
+    const { validateAndBackfillFrontmatter } = await loadFrontmatterModule();
+    const result = validateAndBackfillFrontmatter("pages/topics/rust/index.md", "---\ntitle: Rust\n---\n\n# Rust\n\nContent.\n");
+    assert.equal(result.changed, true);
+    assert.match(result.content, /last_updated:/);
+    assert.match(result.content, /version:/);
+});
+test("validateAndBackfillFrontmatter passes valid frontmatter unchanged", async () => {
+    const { validateAndBackfillFrontmatter } = await loadFrontmatterModule();
+    const content = "---\ntitle: Rust\nlast_updated: 2026-05-14T00:00:00.000Z\nversion: 1\n---\n\n# Rust\n";
+    const result = validateAndBackfillFrontmatter("pages/topics/rust/index.md", content);
+    assert.equal(result.changed, false);
+    assert.equal(result.content, content);
+});
+test("validateAndBackfillFrontmatter backfills title from filename when missing", async () => {
+    const { validateAndBackfillFrontmatter } = await loadFrontmatterModule();
+    const result = validateAndBackfillFrontmatter("pages/topics/rust-async/index.md", "---\nlast_updated: 2026-05-14T00:00:00.000Z\nversion: 1\n---\n\n# Content\n");
+    assert.equal(result.changed, true);
+    assert.match(result.content, /title: Rust Async/);
+});
+test("validateAndBackfillFrontmatter injects frontmatter block when completely missing", async () => {
+    const { validateAndBackfillFrontmatter } = await loadFrontmatterModule();
+    const result = validateAndBackfillFrontmatter("pages/topics/new-page/index.md", "# New Page\n\nContent here.\n");
+    assert.equal(result.changed, true);
+    assert.match(result.content, /^---\n/);
+    assert.match(result.content, /title: /);
+    assert.match(result.content, /last_updated: /);
+    assert.match(result.content, /version: 1/);
+    // ISO timestamp
+    const match = result.content.match(/last_updated: (.+)/);
+    assert.ok(match, "Should have last_updated");
+    assert.doesNotThrow(() => new Date(match[1].trim()).toISOString(), "last_updated should be valid ISO timestamp");
+});
+test("validateAndBackfillFrontmatter backfilled last_updated is valid ISO timestamp", async () => {
+    const { validateAndBackfillFrontmatter } = await loadFrontmatterModule();
+    const result = validateAndBackfillFrontmatter("pages/people/ada/index.md", "---\ntitle: Ada\n---\n\n# Ada\n");
+    assert.equal(result.changed, true);
+    const match = result.content.match(/last_updated: (.+)/);
+    assert.ok(match, "Should have last_updated after backfill");
+    const ts = match[1].trim();
+    assert.doesNotThrow(() => new Date(ts).toISOString(), `${ts} should be valid ISO timestamp`);
+});
 //# sourceMappingURL=frontmatter.test.js.map

package/dist/wiki/fs.js

@@ -6,9 +6,15 @@ import { join, dirname, relative, resolve, sep } from "path";
 import { WIKI_DIR, WIKI_PAGES_DIR, WIKI_SOURCES_DIR } from "../paths.js";
 import { normalizeWikiPath } from "./path-utils.js";
 import { topicPathError } from "./topic-structure.js";
-const INDEX_PATH = join(WIKI_DIR, "index.md");
-const PAGES_INDEX_PATH = join(WIKI_PAGES_DIR, "index.md");
-const LOG_PATH = join(WIKI_PAGES_DIR, "_meta", "log.md");
+function getIndexPath() {
+    return join(WIKI_DIR, "index.md");
+}
+function getPagesIndexPath() {
+    return join(WIKI_PAGES_DIR, "index.md");
+}
+function getLogPath() {
+    return join(WIKI_PAGES_DIR, "_meta", "log.md");
+}
 /**
  * Write a file atomically: write to a temp file in the same directory, fsync,
  * then rename over the destination. Prevents partial writes on crash and
@@ -89,14 +95,17 @@ export function ensureWikiStructure() {
     const isNew = !existsSync(WIKI_DIR);
     mkdirSync(WIKI_PAGES_DIR, { recursive: true });
     mkdirSync(WIKI_SOURCES_DIR, { recursive: true });
-    if (!existsSync(INDEX_PATH)) {
-        writeFileAtomic(INDEX_PATH, getInitialIndex());
+    const indexPath = getIndexPath();
+    const pagesIndexPath = getPagesIndexPath();
+    const logPath = getLogPath();
+    if (!existsSync(indexPath)) {
+        writeFileAtomic(indexPath, getInitialIndex());
     }
-    if (!existsSync(PAGES_INDEX_PATH)) {
-        writeFileAtomic(PAGES_INDEX_PATH, getInitialPagesIndex());
+    if (!existsSync(pagesIndexPath)) {
+        writeFileAtomic(pagesIndexPath, getInitialPagesIndex());
     }
-    if (!existsSync(LOG_PATH)) {
-        writeFileAtomic(LOG_PATH, INITIAL_LOG);
+    if (!existsSync(logPath)) {
+        writeFileAtomic(logPath, INITIAL_LOG);
     }
     return isNew;
 }
@@ -161,20 +170,20 @@ export function listSources() {
 /** Read index.md raw content. */
 export function readIndexFile() {
     ensureWikiStructure();
-    return readFileSync(INDEX_PATH, "utf-8");
+    return readFileSync(getIndexPath(), "utf-8");
 }
 /** Write index.md content atomically. */
 export function writeIndexFile(content) {
-    writeFileAtomic(INDEX_PATH, content);
+    writeFileAtomic(getIndexPath(), content);
 }
 /** Read log.md raw content. */
 export function readLogFile() {
     ensureWikiStructure();
-    return readFileSync(LOG_PATH, "utf-8");
+    return readFileSync(getLogPath(), "utf-8");
 }
 /** Write log.md content atomically. */
 export function writeLogFile(content) {
-    writeFileAtomic(LOG_PATH, content);
+    writeFileAtomic(getLogPath(), content);
 }
 /** Get the full wiki directory path (for external tools that need it). */
 export function getWikiDir() {