chapterhouse 0.8.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/copilot/agents.js +1 -1
- package/dist/copilot/system-message.js +1 -0
- package/dist/copilot/tools.js +11 -1
- package/dist/copilot/tools.wiki.test.js +27 -0
- package/dist/copilot/turn-event-log-env.test.js +11 -15
- package/dist/daemon.js +10 -0
- package/dist/memory/eot.js +30 -8
- package/dist/memory/eot.test.js +220 -6
- package/dist/memory/migration.test.js +10 -2
- package/dist/paths.js +31 -11
- package/dist/store/db.js +68 -0
- package/dist/store/db.test.js +47 -1
- package/dist/test/helpers/reset-singletons.js +8 -0
- package/dist/test/helpers/reset-singletons.test.js +37 -0
- package/dist/test/setup-env.js +8 -1
- package/dist/wiki/consolidation.test.js +3 -0
- package/dist/wiki/fs.js +22 -13
- package/dist/wiki/index-manager.js +82 -23
- package/dist/wiki/index-manager.test.js +129 -1
- package/dist/wiki/log-manager.js +8 -5
- package/dist/wiki/log-manager.test.js +4 -0
- package/package.json +1 -1
package/dist/store/db.js
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import Database from "better-sqlite3";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
3
|
import { ensureChapterhouseHome, getDbPath } from "../paths.js";
|
|
4
|
+
import { ensureWikiStructure, listPages, readPage } from "../wiki/fs.js";
|
|
5
|
+
import { parseWikiFrontmatter } from "../wiki/frontmatter.js";
|
|
6
|
+
// Reset in tests via src/test/helpers/reset-singletons.ts
|
|
4
7
|
let db;
|
|
5
8
|
let logInsertCount = 0;
|
|
6
9
|
let fts5Available = false;
|
|
@@ -49,6 +52,63 @@ function tableCreateSql(database, table) {
|
|
|
49
52
|
`).get(table);
|
|
50
53
|
return row?.sql ?? "";
|
|
51
54
|
}
|
|
55
|
+
const ACTION_LOG_PAGE_RE = /^pages\/_meta\/log(?:-\d{4})?\.md$/;
|
|
56
|
+
const LEGACY_INDEX_PAGE = "pages/index.md";
|
|
57
|
+
function isIgnoredWikiIndexPage(path) {
|
|
58
|
+
return path === LEGACY_INDEX_PAGE || ACTION_LOG_PAGE_RE.test(path);
|
|
59
|
+
}
|
|
60
|
+
function wikiBasenameTitle(path) {
|
|
61
|
+
const segs = path.split("/").filter(Boolean);
|
|
62
|
+
const file = segs[segs.length - 1] || path;
|
|
63
|
+
const base = file.replace(/\.md$/, "");
|
|
64
|
+
const titleBase = base === "index" && segs.length >= 2 ? segs[segs.length - 2] : base;
|
|
65
|
+
return titleBase.split(/[-_]+/).map((word) => word.charAt(0).toUpperCase() + word.slice(1)).join(" ");
|
|
66
|
+
}
|
|
67
|
+
function summarizeWikiBody(body) {
|
|
68
|
+
for (const raw of body.split("\n")) {
|
|
69
|
+
const line = raw.trim();
|
|
70
|
+
if (!line || line.startsWith("#") || line.startsWith("<!--")) {
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
const summary = line.replace(/^[-*]\s+/, "").replace(/_\(\d{4}-\d{2}-\d{2}\)_$/, "").trim();
|
|
74
|
+
if (summary) {
|
|
75
|
+
return summary.length > 160 ? `${summary.slice(0, 157)}…` : summary;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return "";
|
|
79
|
+
}
|
|
80
|
+
function seedWikiPagesFromDisk(database) {
|
|
81
|
+
ensureWikiStructure();
|
|
82
|
+
const pages = listPages().filter((page) => !isIgnoredWikiIndexPage(page));
|
|
83
|
+
if (pages.length === 0) {
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
const wikiPageCount = database.prepare(`SELECT COUNT(*) AS count FROM wiki_pages`).get().count;
|
|
87
|
+
if (wikiPageCount > 0) {
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
const upsert = database.prepare(`
|
|
91
|
+
INSERT INTO wiki_pages (path, title, entity_type, tags, summary, last_updated)
|
|
92
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
93
|
+
ON CONFLICT(path) DO UPDATE SET
|
|
94
|
+
title = excluded.title,
|
|
95
|
+
entity_type = excluded.entity_type,
|
|
96
|
+
tags = excluded.tags,
|
|
97
|
+
summary = excluded.summary,
|
|
98
|
+
last_updated = excluded.last_updated,
|
|
99
|
+
version = wiki_pages.version + 1
|
|
100
|
+
`);
|
|
101
|
+
for (const page of pages) {
|
|
102
|
+
const content = readPage(page);
|
|
103
|
+
if (!content) {
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
const { parsed: fm, body } = parseWikiFrontmatter(content);
|
|
107
|
+
const summary = fm.summary?.trim() || summarizeWikiBody(body) || fm.title || wikiBasenameTitle(page);
|
|
108
|
+
const entityType = fm.metadata?.["entity_type"] ?? null;
|
|
109
|
+
upsert.run(page, fm.title ?? wikiBasenameTitle(page), entityType, JSON.stringify(fm.tags ?? []), summary, fm.updated ?? new Date().toISOString());
|
|
110
|
+
}
|
|
111
|
+
}
|
|
52
112
|
function rebuildMemoryTierTables(database) {
|
|
53
113
|
const needsRebuild = ["mem_entities", "mem_observations", "mem_decisions"]
|
|
54
114
|
.some((table) => tableCreateSql(database, table).includes("'glacier'"));
|
|
@@ -1108,6 +1168,7 @@ export function getDb() {
|
|
|
1108
1168
|
VALUES(new.rowid, new.path, new.title, new.entity_type, new.tags, new.summary);
|
|
1109
1169
|
END
|
|
1110
1170
|
`);
|
|
1171
|
+
seedWikiPagesFromDisk(db);
|
|
1111
1172
|
// Backfill: check if FTS is in sync by comparing row counts
|
|
1112
1173
|
const memCount = db.prepare(`SELECT COUNT(*) as c FROM memories`).get().c;
|
|
1113
1174
|
const ftsCount = db.prepare(`SELECT COUNT(*) as c FROM memories_fts`).get().c;
|
|
@@ -1427,4 +1488,11 @@ export function closeDb() {
|
|
|
1427
1488
|
daemonRunRecorded = false;
|
|
1428
1489
|
}
|
|
1429
1490
|
}
|
|
1491
|
+
export function resetDbForTests() {
|
|
1492
|
+
closeDb();
|
|
1493
|
+
logInsertCount = 0;
|
|
1494
|
+
fts5Available = false;
|
|
1495
|
+
currentDaemonRunId = undefined;
|
|
1496
|
+
daemonRunRecorded = false;
|
|
1497
|
+
}
|
|
1430
1498
|
//# sourceMappingURL=db.js.map
|
package/dist/store/db.test.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
|
-
import { mkdirSync, rmSync } from "node:fs";
|
|
2
|
+
import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import test from "node:test";
|
|
5
5
|
import Database from "better-sqlite3";
|
|
6
|
+
import { resetSingletons } from "../test/helpers/reset-singletons.js";
|
|
6
7
|
const repoRoot = process.cwd();
|
|
7
8
|
const sandboxRoot = join(repoRoot, ".test-work", `store-db-${process.pid}`);
|
|
8
9
|
const chapterhouseHome = join(sandboxRoot, ".chapterhouse");
|
|
@@ -17,11 +18,56 @@ function resetSandbox() {
|
|
|
17
18
|
mkdirSync(chapterhouseHome, { recursive: true });
|
|
18
19
|
}
|
|
19
20
|
test.beforeEach(() => {
|
|
21
|
+
process.env.CHAPTERHOUSE_HOME = sandboxRoot;
|
|
20
22
|
resetSandbox();
|
|
23
|
+
resetSingletons();
|
|
21
24
|
});
|
|
22
25
|
test.after(() => {
|
|
26
|
+
resetSingletons();
|
|
23
27
|
rmSync(sandboxRoot, { recursive: true, force: true });
|
|
24
28
|
});
|
|
29
|
+
test("getDb startup reindex populates wiki_pages when pages exist on disk", async () => {
|
|
30
|
+
mkdirSync(join(chapterhouseHome, "wiki", "pages", "topics", "rust"), { recursive: true });
|
|
31
|
+
writeFileSync(join(chapterhouseHome, "wiki", "pages", "topics", "rust", "index.md"), "---\ntitle: Rust\nsummary: Systems language\nupdated: 2026-05-15\n---\n\n# Rust\n\nFearless concurrency.\n", "utf-8");
|
|
32
|
+
const dbModule = await loadDbModule();
|
|
33
|
+
try {
|
|
34
|
+
const db = dbModule.getDb();
|
|
35
|
+
const count = db.prepare(`SELECT COUNT(*) AS count FROM wiki_pages`).get().count;
|
|
36
|
+
assert.equal(count, 1);
|
|
37
|
+
}
|
|
38
|
+
finally {
|
|
39
|
+
dbModule.closeDb();
|
|
40
|
+
}
|
|
41
|
+
});
|
|
42
|
+
test("getDb startup reindex is idempotent across repeated startups", async () => {
|
|
43
|
+
mkdirSync(join(chapterhouseHome, "wiki", "pages", "topics", "rust"), { recursive: true });
|
|
44
|
+
writeFileSync(join(chapterhouseHome, "wiki", "pages", "topics", "rust", "index.md"), "---\ntitle: Rust\nsummary: Systems language\nupdated: 2026-05-15\n---\n\n# Rust\n\nFearless concurrency.\n", "utf-8");
|
|
45
|
+
const dbModule = await loadDbModule();
|
|
46
|
+
try {
|
|
47
|
+
dbModule.getDb();
|
|
48
|
+
dbModule.closeDb();
|
|
49
|
+
dbModule.getDb();
|
|
50
|
+
const reopened = new Database(dbPath, { readonly: true });
|
|
51
|
+
const count = reopened.prepare(`SELECT COUNT(*) AS count FROM wiki_pages WHERE path = 'pages/topics/rust/index.md'`).get().count;
|
|
52
|
+
reopened.close();
|
|
53
|
+
assert.equal(count, 1);
|
|
54
|
+
}
|
|
55
|
+
finally {
|
|
56
|
+
dbModule.closeDb();
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
test("getDb startup reindex creates the wiki log directory when it is missing", async () => {
|
|
60
|
+
mkdirSync(join(chapterhouseHome, "wiki", "pages", "topics", "rust"), { recursive: true });
|
|
61
|
+
writeFileSync(join(chapterhouseHome, "wiki", "pages", "topics", "rust", "index.md"), "---\ntitle: Rust\nsummary: Systems language\nupdated: 2026-05-15\n---\n\n# Rust\n\nFearless concurrency.\n", "utf-8");
|
|
62
|
+
const dbModule = await loadDbModule();
|
|
63
|
+
try {
|
|
64
|
+
dbModule.getDb();
|
|
65
|
+
assert.equal(existsSync(join(chapterhouseHome, "wiki", "pages", "_meta", "log.md")), true);
|
|
66
|
+
}
|
|
67
|
+
finally {
|
|
68
|
+
dbModule.closeDb();
|
|
69
|
+
}
|
|
70
|
+
});
|
|
25
71
|
test("getDb initializes schema, state helpers, and conversation formatting", async () => {
|
|
26
72
|
const dbModule = await loadDbModule();
|
|
27
73
|
try {
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { resetPathsForTests } from "../../paths.js";
|
|
2
|
+
import { resetDbForTests } from "../../store/db.js";
|
|
3
|
+
export function resetSingletons() {
|
|
4
|
+
resetDbForTests();
|
|
5
|
+
resetPathsForTests();
|
|
6
|
+
}
|
|
7
|
+
export const resetModuleCache = resetSingletons;
|
|
8
|
+
//# sourceMappingURL=reset-singletons.js.map
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { existsSync, mkdirSync, rmSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import test from "node:test";
|
|
5
|
+
const repoRoot = process.cwd();
|
|
6
|
+
const sandboxRoot = join(repoRoot, ".test-work", `reset-singletons-${process.pid}`);
|
|
7
|
+
function sandboxPath(name) {
|
|
8
|
+
return join(sandboxRoot, name);
|
|
9
|
+
}
|
|
10
|
+
test.before(() => {
|
|
11
|
+
mkdirSync(sandboxRoot, { recursive: true });
|
|
12
|
+
});
|
|
13
|
+
test.after(() => {
|
|
14
|
+
rmSync(sandboxRoot, { recursive: true, force: true });
|
|
15
|
+
});
|
|
16
|
+
test("resetSingletons refreshes cached db and path singletons after CHAPTERHOUSE_HOME changes", async () => {
|
|
17
|
+
const helper = await import("./reset-singletons.js");
|
|
18
|
+
const paths = await import("../../paths.js");
|
|
19
|
+
const dbModule = await import("../../store/db.js");
|
|
20
|
+
const firstHome = sandboxPath("first-home");
|
|
21
|
+
const secondHome = sandboxPath("second-home");
|
|
22
|
+
mkdirSync(firstHome, { recursive: true });
|
|
23
|
+
process.env.CHAPTERHOUSE_HOME = firstHome;
|
|
24
|
+
await helper.resetSingletons();
|
|
25
|
+
dbModule.getDb().prepare("SELECT 1").get();
|
|
26
|
+
assert.equal(paths.CHAPTERHOUSE_HOME, join(firstHome, ".chapterhouse"));
|
|
27
|
+
assert.equal(paths.DB_PATH, join(firstHome, ".chapterhouse", "chapterhouse.db"));
|
|
28
|
+
assert.equal(existsSync(join(firstHome, ".chapterhouse", "chapterhouse.db")), true);
|
|
29
|
+
mkdirSync(secondHome, { recursive: true });
|
|
30
|
+
process.env.CHAPTERHOUSE_HOME = secondHome;
|
|
31
|
+
await helper.resetSingletons();
|
|
32
|
+
dbModule.getDb().prepare("SELECT 1").get();
|
|
33
|
+
assert.equal(paths.CHAPTERHOUSE_HOME, join(secondHome, ".chapterhouse"));
|
|
34
|
+
assert.equal(paths.DB_PATH, join(secondHome, ".chapterhouse", "chapterhouse.db"));
|
|
35
|
+
assert.equal(existsSync(join(secondHome, ".chapterhouse", "chapterhouse.db")), true);
|
|
36
|
+
});
|
|
37
|
+
//# sourceMappingURL=reset-singletons.test.js.map
|
package/dist/test/setup-env.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import test from "node:test";
|
|
2
|
+
import { resetSingletons } from "./helpers/reset-singletons.js";
|
|
1
3
|
const RUNTIME_OVERRIDE_ENV_VARS = [
|
|
2
4
|
"CHAPTERHOUSE_MODE",
|
|
3
5
|
"CHAPTERHOUSE_SELF_EDIT",
|
|
@@ -26,5 +28,10 @@ for (const name of [...RUNTIME_OVERRIDE_ENV_VARS, ...AUTH_ENV_VARS]) {
|
|
|
26
28
|
delete process.env[name];
|
|
27
29
|
}
|
|
28
30
|
process.env.CHAPTERHOUSE_DISABLE_DOTENV = "1";
|
|
29
|
-
|
|
31
|
+
test.beforeEach(() => {
|
|
32
|
+
resetSingletons();
|
|
33
|
+
});
|
|
34
|
+
test.afterEach(() => {
|
|
35
|
+
resetSingletons();
|
|
36
|
+
});
|
|
30
37
|
//# sourceMappingURL=setup-env.js.map
|
|
@@ -2,6 +2,7 @@ import assert from "node:assert/strict";
|
|
|
2
2
|
import { mkdirSync, rmSync } from "node:fs";
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import test from "node:test";
|
|
5
|
+
import { resetSingletons } from "../test/helpers/reset-singletons.js";
|
|
5
6
|
const repoRoot = process.cwd();
|
|
6
7
|
const sandboxRoot = join(repoRoot, ".test-work", `wiki-consolidation-${process.pid}`);
|
|
7
8
|
const chapterhouseHome = join(sandboxRoot, ".chapterhouse");
|
|
@@ -27,10 +28,12 @@ test.beforeEach(async () => {
|
|
|
27
28
|
const dbModule = await import(new URL("../store/db.js", import.meta.url).href);
|
|
28
29
|
dbModule.closeDb();
|
|
29
30
|
resetSandbox();
|
|
31
|
+
resetSingletons();
|
|
30
32
|
});
|
|
31
33
|
test.after(async () => {
|
|
32
34
|
const dbModule = await import(new URL("../store/db.js", import.meta.url).href);
|
|
33
35
|
dbModule.closeDb();
|
|
36
|
+
resetSingletons();
|
|
34
37
|
rmSync(sandboxRoot, { recursive: true, force: true });
|
|
35
38
|
});
|
|
36
39
|
test("runConsolidation rewrites stale compiled truth and skips pinned pages", async () => {
|
package/dist/wiki/fs.js
CHANGED
|
@@ -6,9 +6,15 @@ import { join, dirname, relative, resolve, sep } from "path";
|
|
|
6
6
|
import { WIKI_DIR, WIKI_PAGES_DIR, WIKI_SOURCES_DIR } from "../paths.js";
|
|
7
7
|
import { normalizeWikiPath } from "./path-utils.js";
|
|
8
8
|
import { topicPathError } from "./topic-structure.js";
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
function getIndexPath() {
|
|
10
|
+
return join(WIKI_DIR, "index.md");
|
|
11
|
+
}
|
|
12
|
+
function getPagesIndexPath() {
|
|
13
|
+
return join(WIKI_PAGES_DIR, "index.md");
|
|
14
|
+
}
|
|
15
|
+
function getLogPath() {
|
|
16
|
+
return join(WIKI_PAGES_DIR, "_meta", "log.md");
|
|
17
|
+
}
|
|
12
18
|
/**
|
|
13
19
|
* Write a file atomically: write to a temp file in the same directory, fsync,
|
|
14
20
|
* then rename over the destination. Prevents partial writes on crash and
|
|
@@ -89,14 +95,17 @@ export function ensureWikiStructure() {
|
|
|
89
95
|
const isNew = !existsSync(WIKI_DIR);
|
|
90
96
|
mkdirSync(WIKI_PAGES_DIR, { recursive: true });
|
|
91
97
|
mkdirSync(WIKI_SOURCES_DIR, { recursive: true });
|
|
92
|
-
|
|
93
|
-
|
|
98
|
+
const indexPath = getIndexPath();
|
|
99
|
+
const pagesIndexPath = getPagesIndexPath();
|
|
100
|
+
const logPath = getLogPath();
|
|
101
|
+
if (!existsSync(indexPath)) {
|
|
102
|
+
writeFileAtomic(indexPath, getInitialIndex());
|
|
94
103
|
}
|
|
95
|
-
if (!existsSync(
|
|
96
|
-
writeFileAtomic(
|
|
104
|
+
if (!existsSync(pagesIndexPath)) {
|
|
105
|
+
writeFileAtomic(pagesIndexPath, getInitialPagesIndex());
|
|
97
106
|
}
|
|
98
|
-
if (!existsSync(
|
|
99
|
-
writeFileAtomic(
|
|
107
|
+
if (!existsSync(logPath)) {
|
|
108
|
+
writeFileAtomic(logPath, INITIAL_LOG);
|
|
100
109
|
}
|
|
101
110
|
return isNew;
|
|
102
111
|
}
|
|
@@ -161,20 +170,20 @@ export function listSources() {
|
|
|
161
170
|
/** Read index.md raw content. */
|
|
162
171
|
export function readIndexFile() {
|
|
163
172
|
ensureWikiStructure();
|
|
164
|
-
return readFileSync(
|
|
173
|
+
return readFileSync(getIndexPath(), "utf-8");
|
|
165
174
|
}
|
|
166
175
|
/** Write index.md content atomically. */
|
|
167
176
|
export function writeIndexFile(content) {
|
|
168
|
-
writeFileAtomic(
|
|
177
|
+
writeFileAtomic(getIndexPath(), content);
|
|
169
178
|
}
|
|
170
179
|
/** Read log.md raw content. */
|
|
171
180
|
export function readLogFile() {
|
|
172
181
|
ensureWikiStructure();
|
|
173
|
-
return readFileSync(
|
|
182
|
+
return readFileSync(getLogPath(), "utf-8");
|
|
174
183
|
}
|
|
175
184
|
/** Write log.md content atomically. */
|
|
176
185
|
export function writeLogFile(content) {
|
|
177
|
-
writeFileAtomic(
|
|
186
|
+
writeFileAtomic(getLogPath(), content);
|
|
178
187
|
}
|
|
179
188
|
/** Get the full wiki directory path (for external tools that need it). */
|
|
180
189
|
export function getWikiDir() {
|
|
@@ -2,15 +2,44 @@
|
|
|
2
2
|
// Wiki index manager — SQLite FTS5-backed wiki page catalog
|
|
3
3
|
// ---------------------------------------------------------------------------
|
|
4
4
|
import { getDb, isFts5Available } from "../store/db.js";
|
|
5
|
+
import { childLogger } from "../util/logger.js";
|
|
5
6
|
import { listPages, readPage } from "./fs.js";
|
|
6
7
|
import { parseWikiFrontmatter } from "./frontmatter.js";
|
|
7
8
|
import { normalizeWikiPath } from "./path-utils.js";
|
|
8
9
|
import { updateLinks } from "./links.js";
|
|
9
10
|
const ACTION_LOG_PAGE_RE = /^pages\/_meta\/log(?:-\d{4})?\.md$/;
|
|
10
11
|
const LEGACY_INDEX_PAGE = "pages/index.md";
|
|
12
|
+
const log = childLogger("wiki-index");
|
|
11
13
|
function isIgnoredIndexPage(path) {
|
|
12
14
|
return path === LEGACY_INDEX_PAGE || ACTION_LOG_PAGE_RE.test(path);
|
|
13
15
|
}
|
|
16
|
+
function getIndexablePages() {
|
|
17
|
+
return listPages().filter((p) => !isIgnoredIndexPage(p));
|
|
18
|
+
}
|
|
19
|
+
function getWikiPageCount() {
|
|
20
|
+
const db = getDb();
|
|
21
|
+
return db.prepare(`SELECT COUNT(*) as c FROM wiki_pages`).get().c;
|
|
22
|
+
}
|
|
23
|
+
function getErrorMessage(err) {
|
|
24
|
+
return err instanceof Error ? err.message : String(err);
|
|
25
|
+
}
|
|
26
|
+
function buildPageSummary(summary, body) {
|
|
27
|
+
let nextSummary = summary?.trim() ?? "";
|
|
28
|
+
if (!nextSummary) {
|
|
29
|
+
for (const raw of body.split("\n")) {
|
|
30
|
+
const line = raw.trim();
|
|
31
|
+
if (!line || line.startsWith("#") || line.startsWith("<!--"))
|
|
32
|
+
continue;
|
|
33
|
+
nextSummary = line.replace(/^[-*]\s+/, "").replace(/_\(\d{4}-\d{2}-\d{2}\)_$/, "").trim();
|
|
34
|
+
if (!nextSummary)
|
|
35
|
+
continue;
|
|
36
|
+
break;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
if (nextSummary.length > 160)
|
|
40
|
+
nextSummary = nextSummary.slice(0, 157) + "…";
|
|
41
|
+
return nextSummary;
|
|
42
|
+
}
|
|
14
43
|
function categoryOfPath(path) {
|
|
15
44
|
const rest = path.startsWith("pages/") ? path.slice("pages/".length) : path;
|
|
16
45
|
const segs = rest.split("/").filter(Boolean);
|
|
@@ -72,12 +101,11 @@ export function removeWikiPage(path) {
|
|
|
72
101
|
* Rebuild the wiki_pages SQLite index from the filesystem.
|
|
73
102
|
* Filesystem is the source of truth: insert missing pages, remove stale entries.
|
|
74
103
|
*/
|
|
75
|
-
|
|
76
|
-
const pages =
|
|
104
|
+
function runWikiReindex() {
|
|
105
|
+
const pages = getIndexablePages();
|
|
77
106
|
const db = getDb();
|
|
78
107
|
const onDisk = new Set(pages.map(normalizeWikiPath));
|
|
79
108
|
db.transaction(() => {
|
|
80
|
-
// Remove DB entries not on disk; also clean their wiki_links
|
|
81
109
|
const inDb = db.prepare(`SELECT path FROM wiki_pages`).all();
|
|
82
110
|
for (const { path } of inDb) {
|
|
83
111
|
if (!onDisk.has(path)) {
|
|
@@ -85,33 +113,35 @@ export function rebuildWikiIndex() {
|
|
|
85
113
|
db.prepare(`DELETE FROM wiki_links WHERE from_page = ? OR to_page = ?`).run(path, path);
|
|
86
114
|
}
|
|
87
115
|
}
|
|
88
|
-
|
|
89
|
-
|
|
116
|
+
})();
|
|
117
|
+
let indexedPageCount = 0;
|
|
118
|
+
let skippedPageCount = 0;
|
|
119
|
+
for (const p of pages) {
|
|
120
|
+
try {
|
|
90
121
|
const content = readPage(p);
|
|
91
122
|
if (!content)
|
|
92
123
|
continue;
|
|
93
124
|
const { parsed: fm, body } = parseWikiFrontmatter(content);
|
|
94
|
-
|
|
95
|
-
if (!summary) {
|
|
96
|
-
for (const raw of body.split("\n")) {
|
|
97
|
-
const line = raw.trim();
|
|
98
|
-
if (!line || line.startsWith("#") || line.startsWith("<!--"))
|
|
99
|
-
continue;
|
|
100
|
-
summary = line.replace(/^[-*]\s+/, "").replace(/_\(\d{4}-\d{2}-\d{2}\)_$/, "").trim();
|
|
101
|
-
if (!summary)
|
|
102
|
-
continue;
|
|
103
|
-
break;
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
if (summary.length > 160)
|
|
107
|
-
summary = summary.slice(0, 157) + "…";
|
|
125
|
+
const summary = buildPageSummary(fm.summary, body);
|
|
108
126
|
upsertWikiPage(p, fm, summary);
|
|
127
|
+
updateLinks(p);
|
|
128
|
+
indexedPageCount++;
|
|
129
|
+
}
|
|
130
|
+
catch (err) {
|
|
131
|
+
skippedPageCount++;
|
|
132
|
+
log.warn({ path: p, err: getErrorMessage(err) }, "Skipping wiki page during reindex");
|
|
133
|
+
continue;
|
|
109
134
|
}
|
|
110
|
-
})();
|
|
111
|
-
// Backfill wiki_links for all on-disk pages (outside transaction — each call is self-contained)
|
|
112
|
-
for (const p of pages) {
|
|
113
|
-
updateLinks(p);
|
|
114
135
|
}
|
|
136
|
+
log.info({ indexedPageCount, skippedPageCount }, `Reindexed ${indexedPageCount} pages, skipped ${skippedPageCount} pages (errors)`);
|
|
137
|
+
return {
|
|
138
|
+
diskPageCount: pages.length,
|
|
139
|
+
indexedPageCount,
|
|
140
|
+
skippedPageCount,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
export function rebuildWikiIndex() {
|
|
144
|
+
runWikiReindex();
|
|
115
145
|
}
|
|
116
146
|
/**
|
|
117
147
|
* Search wiki pages using FTS5 BM25 ranking.
|
|
@@ -120,6 +150,35 @@ export function rebuildWikiIndex() {
|
|
|
120
150
|
* Each direct result includes 1-hop neighbors from wiki_links in `related`.
|
|
121
151
|
* Neighbor pages not already in direct results are appended (deduplicated).
|
|
122
152
|
*/
|
|
153
|
+
export function ensureWikiIndexPopulated() {
|
|
154
|
+
const pages = getIndexablePages();
|
|
155
|
+
const indexedPageCount = getWikiPageCount();
|
|
156
|
+
if (indexedPageCount === 0 && pages.length > 0) {
|
|
157
|
+
log.warn({ diskPageCount: pages.length }, "wiki_pages empty while wiki files exist; rebuilding from disk");
|
|
158
|
+
const result = runWikiReindex();
|
|
159
|
+
return {
|
|
160
|
+
reindexed: true,
|
|
161
|
+
diskPageCount: result.diskPageCount,
|
|
162
|
+
indexedPageCount: result.indexedPageCount,
|
|
163
|
+
skippedPageCount: result.skippedPageCount,
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
return {
|
|
167
|
+
reindexed: false,
|
|
168
|
+
diskPageCount: pages.length,
|
|
169
|
+
indexedPageCount,
|
|
170
|
+
skippedPageCount: 0,
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
export function reindexWikiPages() {
|
|
174
|
+
const result = runWikiReindex();
|
|
175
|
+
return {
|
|
176
|
+
reindexed: true,
|
|
177
|
+
diskPageCount: result.diskPageCount,
|
|
178
|
+
indexedPageCount: result.indexedPageCount,
|
|
179
|
+
skippedPageCount: result.skippedPageCount,
|
|
180
|
+
};
|
|
181
|
+
}
|
|
123
182
|
export function wikiSearch(query, limit = 20) {
|
|
124
183
|
const db = getDb();
|
|
125
184
|
const trimmed = query.trim();
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
|
-
import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
|
|
2
|
+
import { chmodSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
|
|
3
|
+
import { resetSingletons } from "../test/helpers/reset-singletons.js";
|
|
3
4
|
import { join } from "node:path";
|
|
4
5
|
import test from "node:test";
|
|
5
6
|
// Sandbox: every test gets a fresh CHAPTERHOUSE_HOME
|
|
6
7
|
function makeSandbox() {
|
|
7
8
|
const dir = mkdtempSync(join(process.cwd(), ".test-work", "wiki-idx-"));
|
|
8
9
|
process.env.CHAPTERHOUSE_HOME = dir;
|
|
10
|
+
resetSingletons();
|
|
9
11
|
return dir;
|
|
10
12
|
}
|
|
11
13
|
async function loadModules(sandbox) {
|
|
@@ -14,6 +16,39 @@ async function loadModules(sandbox) {
|
|
|
14
16
|
const wikiFs = await import(new URL(`./fs.js?c=${nonce}`, import.meta.url).href);
|
|
15
17
|
return { indexManager, wikiFs };
|
|
16
18
|
}
|
|
19
|
+
function resetWikiState(indexManager, wikiFs) {
|
|
20
|
+
rmSync(wikiFs.getWikiDir(), { recursive: true, force: true });
|
|
21
|
+
for (const entry of indexManager.parseIndex()) {
|
|
22
|
+
indexManager.removeFromIndex(entry.path);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
async function loadModulesWithMocks(t, sandbox, options = {}) {
|
|
26
|
+
const warnings = [];
|
|
27
|
+
const infos = [];
|
|
28
|
+
t.mock.module("../util/logger.js", {
|
|
29
|
+
namedExports: {
|
|
30
|
+
childLogger: () => ({
|
|
31
|
+
info: (obj, msg) => infos.push({ obj, msg }),
|
|
32
|
+
warn: (obj, msg) => warnings.push({ obj, msg }),
|
|
33
|
+
error: () => { },
|
|
34
|
+
}),
|
|
35
|
+
},
|
|
36
|
+
});
|
|
37
|
+
if (options.malformedMarker) {
|
|
38
|
+
const actualFrontmatter = await import(new URL(`./frontmatter.js?actual=${Date.now()}-${Math.random()}`, import.meta.url).href);
|
|
39
|
+
t.mock.module("./frontmatter.js", {
|
|
40
|
+
namedExports: {
|
|
41
|
+
parseWikiFrontmatter: (content) => {
|
|
42
|
+
if (content.includes(options.malformedMarker)) {
|
|
43
|
+
throw new Error("Malformed wiki page");
|
|
44
|
+
}
|
|
45
|
+
return actualFrontmatter.parseWikiFrontmatter(content);
|
|
46
|
+
},
|
|
47
|
+
},
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
return { ...(await loadModules(sandbox)), warnings, infos };
|
|
51
|
+
}
|
|
17
52
|
test.before(() => {
|
|
18
53
|
mkdirSync(join(process.cwd(), ".test-work"), { recursive: true });
|
|
19
54
|
});
|
|
@@ -66,6 +101,25 @@ test("rebuildWikiIndex populates wiki_pages from filesystem", async () => {
|
|
|
66
101
|
rmSync(sandbox, { recursive: true, force: true });
|
|
67
102
|
}
|
|
68
103
|
});
|
|
104
|
+
test("ensureWikiIndexPopulated rebuilds from disk when wiki_pages starts empty", async () => {
|
|
105
|
+
const sandbox = makeSandbox();
|
|
106
|
+
try {
|
|
107
|
+
const { indexManager, wikiFs } = await loadModules(sandbox);
|
|
108
|
+
wikiFs.writePage("pages/topics/rust/index.md", "---\ntitle: Rust\nsummary: Systems programming\nupdated: 2026-05-10\n---\n\n# Rust\n");
|
|
109
|
+
for (const entry of indexManager.parseIndex()) {
|
|
110
|
+
indexManager.removeFromIndex(entry.path);
|
|
111
|
+
}
|
|
112
|
+
assert.equal(indexManager.parseIndex().length, 0, "Precondition: wiki_pages should start empty");
|
|
113
|
+
const result = indexManager.ensureWikiIndexPopulated();
|
|
114
|
+
assert.equal(result.reindexed, true);
|
|
115
|
+
assert.ok(result.diskPageCount >= 1);
|
|
116
|
+
assert.ok(result.indexedPageCount >= 1);
|
|
117
|
+
assert.ok(indexManager.parseIndex().some((entry) => entry.path === "pages/topics/rust/index.md"));
|
|
118
|
+
}
|
|
119
|
+
finally {
|
|
120
|
+
rmSync(sandbox, { recursive: true, force: true });
|
|
121
|
+
}
|
|
122
|
+
});
|
|
69
123
|
test("upsertWikiPage inserts and updates correctly", async () => {
|
|
70
124
|
const sandbox = makeSandbox();
|
|
71
125
|
try {
|
|
@@ -150,4 +204,78 @@ test("rebuildWikiIndex removes stale entries not on disk", async () => {
|
|
|
150
204
|
rmSync(sandbox, { recursive: true, force: true });
|
|
151
205
|
}
|
|
152
206
|
});
|
|
207
|
+
test("reindexWikiPages skips unreadable pages and continues indexing others", async (t) => {
|
|
208
|
+
const sandbox = makeSandbox();
|
|
209
|
+
try {
|
|
210
|
+
const { indexManager, wikiFs, warnings } = await loadModulesWithMocks(t, sandbox);
|
|
211
|
+
resetWikiState(indexManager, wikiFs);
|
|
212
|
+
const unreadablePath = join(wikiFs.getWikiDir(), "pages", "topics", "blocked", "index.md");
|
|
213
|
+
wikiFs.writePage("pages/topics/alpha/index.md", "---\ntitle: Alpha\nsummary: First topic\nupdated: 2026-05-01\n---\n\n# Alpha\n");
|
|
214
|
+
wikiFs.writePage("pages/topics/blocked/index.md", "---\ntitle: Blocked\nsummary: Unreadable topic\nupdated: 2026-05-02\n---\n\n# Blocked\n");
|
|
215
|
+
wikiFs.writePage("pages/topics/gamma/index.md", "---\ntitle: Gamma\nsummary: Third topic\nupdated: 2026-05-03\n---\n\n# Gamma\n");
|
|
216
|
+
chmodSync(unreadablePath, 0o000);
|
|
217
|
+
try {
|
|
218
|
+
const result = indexManager.reindexWikiPages();
|
|
219
|
+
const paths = indexManager.parseIndex().map((entry) => entry.path).sort();
|
|
220
|
+
assert.deepEqual(paths, [
|
|
221
|
+
"pages/topics/alpha/index.md",
|
|
222
|
+
"pages/topics/gamma/index.md",
|
|
223
|
+
]);
|
|
224
|
+
assert.equal(result.diskPageCount, 3);
|
|
225
|
+
assert.equal(result.indexedPageCount, 2);
|
|
226
|
+
assert.ok(warnings.some((entry) => entry.obj.path === "pages/topics/blocked/index.md"), "Expected a warning for the unreadable page");
|
|
227
|
+
}
|
|
228
|
+
finally {
|
|
229
|
+
chmodSync(unreadablePath, 0o644);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
finally {
|
|
233
|
+
rmSync(sandbox, { recursive: true, force: true });
|
|
234
|
+
}
|
|
235
|
+
});
|
|
236
|
+
test("reindexWikiPages skips malformed pages and continues indexing others", async (t) => {
|
|
237
|
+
const sandbox = makeSandbox();
|
|
238
|
+
try {
|
|
239
|
+
const { indexManager, wikiFs, warnings } = await loadModulesWithMocks(t, sandbox, { malformedMarker: "UNPARSEABLE" });
|
|
240
|
+
resetWikiState(indexManager, wikiFs);
|
|
241
|
+
wikiFs.writePage("pages/topics/alpha/index.md", "---\ntitle: Alpha\nsummary: First topic\nupdated: 2026-05-01\n---\n\n# Alpha\n");
|
|
242
|
+
wikiFs.writePage("pages/topics/bad/index.md", "---\ntitle: Bad\nsummary: Broken topic\nupdated: 2026-05-02\n---\n\nUNPARSEABLE\n");
|
|
243
|
+
wikiFs.writePage("pages/topics/gamma/index.md", "---\ntitle: Gamma\nsummary: Third topic\nupdated: 2026-05-03\n---\n\n# Gamma\n");
|
|
244
|
+
const result = indexManager.reindexWikiPages();
|
|
245
|
+
const paths = indexManager.parseIndex().map((entry) => entry.path).sort();
|
|
246
|
+
assert.deepEqual(paths, [
|
|
247
|
+
"pages/topics/alpha/index.md",
|
|
248
|
+
"pages/topics/gamma/index.md",
|
|
249
|
+
]);
|
|
250
|
+
assert.equal(result.diskPageCount, 3);
|
|
251
|
+
assert.equal(result.indexedPageCount, 2);
|
|
252
|
+
assert.ok(warnings.some((entry) => entry.obj.path === "pages/topics/bad/index.md"), "Expected a warning for the malformed page");
|
|
253
|
+
}
|
|
254
|
+
finally {
|
|
255
|
+
rmSync(sandbox, { recursive: true, force: true });
|
|
256
|
+
}
|
|
257
|
+
});
|
|
258
|
+
test("reindexWikiPages logs a summary with indexed and skipped counts", async (t) => {
|
|
259
|
+
const sandbox = makeSandbox();
|
|
260
|
+
try {
|
|
261
|
+
const { indexManager, wikiFs, infos } = await loadModulesWithMocks(t, sandbox);
|
|
262
|
+
resetWikiState(indexManager, wikiFs);
|
|
263
|
+
const unreadablePath = join(wikiFs.getWikiDir(), "pages", "topics", "blocked", "index.md");
|
|
264
|
+
wikiFs.writePage("pages/topics/alpha/index.md", "---\ntitle: Alpha\nsummary: First topic\nupdated: 2026-05-01\n---\n\n# Alpha\n");
|
|
265
|
+
wikiFs.writePage("pages/topics/blocked/index.md", "---\ntitle: Blocked\nsummary: Unreadable topic\nupdated: 2026-05-02\n---\n\n# Blocked\n");
|
|
266
|
+
wikiFs.writePage("pages/topics/gamma/index.md", "---\ntitle: Gamma\nsummary: Third topic\nupdated: 2026-05-03\n---\n\n# Gamma\n");
|
|
267
|
+
chmodSync(unreadablePath, 0o000);
|
|
268
|
+
try {
|
|
269
|
+
const result = indexManager.reindexWikiPages();
|
|
270
|
+
assert.equal(result.indexedPageCount, 2);
|
|
271
|
+
assert.ok(infos.some((entry) => /Reindexed 2 pages, skipped 1/.test(entry.msg)), "Expected a summary log with indexed and skipped counts");
|
|
272
|
+
}
|
|
273
|
+
finally {
|
|
274
|
+
chmodSync(unreadablePath, 0o644);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
finally {
|
|
278
|
+
rmSync(sandbox, { recursive: true, force: true });
|
|
279
|
+
}
|
|
280
|
+
});
|
|
153
281
|
//# sourceMappingURL=index-manager.test.js.map
|
package/dist/wiki/log-manager.js
CHANGED
|
@@ -6,7 +6,9 @@ import { join } from "path";
|
|
|
6
6
|
import { WIKI_PAGES_DIR } from "../paths.js";
|
|
7
7
|
import { ensureWikiStructure, writeFileAtomic } from "./fs.js";
|
|
8
8
|
export const ACTION_LOG_PATH = "pages/_meta/log.md";
|
|
9
|
-
|
|
9
|
+
function getLogPath() {
|
|
10
|
+
return join(WIKI_PAGES_DIR, "_meta", "log.md");
|
|
11
|
+
}
|
|
10
12
|
const MAX_LOG_ENTRIES = 500;
|
|
11
13
|
const LOG_ENTRY_RE = /^## \[/gm;
|
|
12
14
|
const INITIAL_LOG = `# Wiki Action Log
|
|
@@ -24,17 +26,18 @@ export function appendLog(type, subject, agent = resolveAgentName()) {
|
|
|
24
26
|
const now = new Date();
|
|
25
27
|
const ts = now.toISOString().slice(0, 16).replace("T", " ");
|
|
26
28
|
const entry = `## [${ts}] ${type} | ${subject} | ${agent}\n\n`;
|
|
27
|
-
appendFileSync(
|
|
29
|
+
appendFileSync(getLogPath(), entry, "utf-8");
|
|
28
30
|
}
|
|
29
31
|
function rotateLogIfNeeded() {
|
|
30
|
-
const
|
|
32
|
+
const logPath = getLogPath();
|
|
33
|
+
const currentLog = readFileSync(logPath, "utf-8");
|
|
31
34
|
const entryCount = (currentLog.match(LOG_ENTRY_RE) ?? []).length;
|
|
32
35
|
if (entryCount < MAX_LOG_ENTRIES) {
|
|
33
36
|
return;
|
|
34
37
|
}
|
|
35
38
|
const archivePath = join(WIKI_PAGES_DIR, "_meta", `log-${new Date().getFullYear()}.md`);
|
|
36
39
|
if (!existsSync(archivePath)) {
|
|
37
|
-
renameSync(
|
|
40
|
+
renameSync(logPath, archivePath);
|
|
38
41
|
}
|
|
39
42
|
else {
|
|
40
43
|
const archivedEntries = extractEntries(currentLog);
|
|
@@ -43,7 +46,7 @@ function rotateLogIfNeeded() {
|
|
|
43
46
|
appendFileSync(archivePath, `${separator}${archivedEntries.join("\n\n")}\n\n`, "utf-8");
|
|
44
47
|
}
|
|
45
48
|
}
|
|
46
|
-
writeFileAtomic(
|
|
49
|
+
writeFileAtomic(logPath, INITIAL_LOG);
|
|
47
50
|
}
|
|
48
51
|
function extractEntries(content) {
|
|
49
52
|
return content
|