chapterhouse 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/korg.agent.md +65 -0
- package/dist/api/korg.js +34 -0
- package/dist/api/korg.test.js +42 -0
- package/dist/api/server.js +238 -2
- package/dist/api/server.test.js +199 -0
- package/dist/config.js +28 -0
- package/dist/config.test.js +20 -0
- package/dist/copilot/agents.js +3 -4
- package/dist/copilot/agents.test.js +12 -1
- package/dist/copilot/orchestrator.js +12 -1
- package/dist/copilot/orchestrator.test.js +3 -7
- package/dist/copilot/system-message.js +11 -10
- package/dist/copilot/system-message.test.js +6 -1
- package/dist/copilot/tools.js +184 -376
- package/dist/copilot/tools.memory.test.js +32 -0
- package/dist/copilot/tools.wiki.test.js +53 -59
- package/dist/daemon.js +9 -0
- package/dist/memory/decisions.js +6 -5
- package/dist/memory/entities.js +20 -9
- package/dist/memory/hooks.js +151 -0
- package/dist/memory/hooks.test.js +325 -0
- package/dist/memory/hot-tier.js +37 -0
- package/dist/memory/hot-tier.test.js +30 -0
- package/dist/memory/housekeeping-scheduler.js +35 -0
- package/dist/memory/housekeeping-scheduler.test.js +50 -0
- package/dist/memory/inbox.js +10 -0
- package/dist/memory/index.js +3 -1
- package/dist/memory/migration.js +244 -0
- package/dist/memory/migration.test.js +100 -0
- package/dist/memory/reflect.js +273 -0
- package/dist/memory/reflect.test.js +254 -0
- package/dist/store/db.js +119 -4
- package/dist/store/db.test.js +19 -1
- package/dist/test/setup-env.js +1 -0
- package/dist/wiki/consolidation.js +641 -0
- package/dist/wiki/consolidation.test.js +140 -0
- package/dist/wiki/frontmatter.js +48 -0
- package/dist/wiki/frontmatter.test.js +42 -0
- package/dist/wiki/index-manager.js +246 -330
- package/dist/wiki/index-manager.test.js +138 -145
- package/dist/wiki/ingest.js +347 -0
- package/dist/wiki/ingest.test.js +111 -0
- package/dist/wiki/links.js +151 -0
- package/dist/wiki/links.test.js +176 -0
- package/dist/wiki/migrate-topics.test.js +16 -6
- package/dist/wiki/scheduler.js +118 -0
- package/dist/wiki/scheduler.test.js +64 -0
- package/dist/wiki/timeline.js +51 -0
- package/dist/wiki/timeline.test.js +65 -0
- package/dist/wiki/topic-structure.js +1 -1
- package/package.json +1 -1
- package/skills/pkb-ideas/SKILL.md +78 -0
- package/skills/pkb-ideas/_meta.json +4 -0
- package/skills/pkb-org/SKILL.md +82 -0
- package/skills/pkb-org/_meta.json +4 -0
- package/skills/pkb-people/SKILL.md +74 -0
- package/skills/pkb-people/_meta.json +4 -0
- package/skills/pkb-research/SKILL.md +83 -0
- package/skills/pkb-research/_meta.json +4 -0
- package/skills/pkb-source/SKILL.md +38 -0
- package/skills/pkb-source/_meta.json +4 -0
- package/skills/wiki-conventions/SKILL.md +5 -5
- package/web/dist/assets/{index-DuKYxMIR.css → index-5kz9aRU9.css} +1 -1
- package/web/dist/assets/{index-DytB69KC.js → index-BbX9RKf3.js} +91 -89
- package/web/dist/assets/index-BbX9RKf3.js.map +1 -0
- package/web/dist/index.html +2 -2
- package/dist/wiki/context.js +0 -138
- package/dist/wiki/fix.js +0 -335
- package/dist/wiki/fix.test.js +0 -350
- package/dist/wiki/lint.js +0 -451
- package/dist/wiki/lint.test.js +0 -329
- package/web/dist/assets/index-DytB69KC.js.map +0 -1
|
@@ -0,0 +1,641 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { execFileSync } from "node:child_process";
|
|
3
|
+
import { existsSync, mkdirSync, renameSync } from "node:fs";
|
|
4
|
+
import { dirname, join, basename } from "node:path";
|
|
5
|
+
import { config } from "../config.js";
|
|
6
|
+
import { recordActionItem } from "../memory/action-items.js";
|
|
7
|
+
import { getScope } from "../memory/scopes.js";
|
|
8
|
+
import { getChapterhouseHome, resolveWikiRelativePath } from "../paths.js";
|
|
9
|
+
import { childLogger } from "../util/logger.js";
|
|
10
|
+
import { deletePage, listPages, readPage, writePage } from "./fs.js";
|
|
11
|
+
import { parseWikiFrontmatter, validateAndBackfillFrontmatter } from "./frontmatter.js";
|
|
12
|
+
import { rebuildWikiIndex, upsertWikiPage, removeWikiPage } from "./index-manager.js";
|
|
13
|
+
import { normalizeWikiPath } from "./path-utils.js";
|
|
14
|
+
const log = childLogger("wiki.consolidation");
|
|
15
|
+
const TRUTH_REWRITE_BUDGET = 18;
|
|
16
|
+
const ONE_DAY_MS = 24 * 60 * 60 * 1000;
|
|
17
|
+
const STALE_SESSION_MS = 7 * ONE_DAY_MS;
|
|
18
|
+
const DEFAULT_MODEL = "claude-haiku-4.5";
|
|
19
|
+
export async function runConsolidation(db) {
|
|
20
|
+
return runConsolidationWithDeps(db, createDefaultDeps(db));
|
|
21
|
+
}
|
|
22
|
+
export async function runConsolidationWithDeps(db, partialDeps = {}) {
|
|
23
|
+
ensureConsolidationSchema(db);
|
|
24
|
+
const deps = { ...createDefaultDeps(db), ...partialDeps };
|
|
25
|
+
const runAt = deps.now();
|
|
26
|
+
const modifiedPaths = new Set();
|
|
27
|
+
const result = {
|
|
28
|
+
truthRewrites: 0,
|
|
29
|
+
fragmentsMerged: 0,
|
|
30
|
+
linksRepaired: 0,
|
|
31
|
+
pagesReindexed: 0,
|
|
32
|
+
sourcesArchived: 0,
|
|
33
|
+
staleSessionsNotified: 0,
|
|
34
|
+
llmCallsUsed: 0,
|
|
35
|
+
};
|
|
36
|
+
const rewriteCandidates = collectRewriteCandidates(db);
|
|
37
|
+
for (const candidate of rewriteCandidates) {
|
|
38
|
+
if (result.llmCallsUsed >= TRUTH_REWRITE_BUDGET) {
|
|
39
|
+
break;
|
|
40
|
+
}
|
|
41
|
+
const row = db.prepare(`SELECT title FROM wiki_pages WHERE path = ?`).get(candidate.path);
|
|
42
|
+
if (!row) {
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
const synthesized = await deps.synthesizeTruth({
|
|
46
|
+
pagePath: candidate.path,
|
|
47
|
+
pageTitle: row.title,
|
|
48
|
+
existingTruth: candidate.currentTruth,
|
|
49
|
+
pendingEntries: candidate.pendingEntries,
|
|
50
|
+
});
|
|
51
|
+
result.llmCallsUsed += 1;
|
|
52
|
+
applyTruthRewrite(db, candidate.path, candidate.content, synthesized, runAt.toISOString());
|
|
53
|
+
modifiedPaths.add(candidate.path);
|
|
54
|
+
result.truthRewrites += 1;
|
|
55
|
+
}
|
|
56
|
+
result.fragmentsMerged += mergeFragments(db, runAt.toISOString(), modifiedPaths);
|
|
57
|
+
result.linksRepaired += repairOrphanLinks(db);
|
|
58
|
+
result.sourcesArchived += archiveOrphanSources(db);
|
|
59
|
+
result.staleSessionsNotified += notifyStaleSessions(db, deps, runAt);
|
|
60
|
+
ensureConsolidationStateTable(db);
|
|
61
|
+
deps.rebuildIndex();
|
|
62
|
+
result.linksRepaired += repairOrphanLinks(db);
|
|
63
|
+
result.pagesReindexed = modifiedPaths.size;
|
|
64
|
+
db.prepare(`
|
|
65
|
+
INSERT INTO wiki_consolidation_state (id, last_run, pages_processed)
|
|
66
|
+
VALUES (1, ?, ?)
|
|
67
|
+
ON CONFLICT(id) DO UPDATE SET last_run = excluded.last_run, pages_processed = excluded.pages_processed
|
|
68
|
+
`).run(runAt.toISOString(), modifiedPaths.size);
|
|
69
|
+
await deps.commitWikiChanges(runAt);
|
|
70
|
+
return result;
|
|
71
|
+
}
|
|
72
|
+
function createDefaultDeps(db) {
|
|
73
|
+
return {
|
|
74
|
+
now: () => new Date(),
|
|
75
|
+
synthesizeTruth: synthesizeTruthWithCopilot,
|
|
76
|
+
rebuildIndex: rebuildWikiIndex,
|
|
77
|
+
commitWikiChanges: async (runAt) => commitWikiChanges(runAt),
|
|
78
|
+
createActionItem: ({ title, detail, source }) => createStaleSessionActionItem(db, { title, detail, source }),
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
function ensureConsolidationSchema(db) {
|
|
82
|
+
const wikiSourceColumns = db.prepare(`PRAGMA table_info(wiki_sources)`).all();
|
|
83
|
+
if (!wikiSourceColumns.some((column) => column.name === "status")) {
|
|
84
|
+
db.exec(`ALTER TABLE wiki_sources ADD COLUMN status TEXT NOT NULL DEFAULT 'active'`);
|
|
85
|
+
}
|
|
86
|
+
if (!wikiSourceColumns.some((column) => column.name === "session_id")) {
|
|
87
|
+
db.exec(`ALTER TABLE wiki_sources ADD COLUMN session_id TEXT`);
|
|
88
|
+
}
|
|
89
|
+
if (!wikiSourceColumns.some((column) => column.name === "session_name")) {
|
|
90
|
+
db.exec(`ALTER TABLE wiki_sources ADD COLUMN session_name TEXT`);
|
|
91
|
+
}
|
|
92
|
+
ensureConsolidationStateTable(db);
|
|
93
|
+
}
|
|
94
|
+
function ensureConsolidationStateTable(db) {
|
|
95
|
+
db.exec(`
|
|
96
|
+
CREATE TABLE IF NOT EXISTS wiki_consolidation_state (
|
|
97
|
+
id INTEGER PRIMARY KEY,
|
|
98
|
+
last_run TEXT,
|
|
99
|
+
pages_processed INTEGER NOT NULL DEFAULT 0
|
|
100
|
+
)
|
|
101
|
+
`);
|
|
102
|
+
}
|
|
103
|
+
function collectRewriteCandidates(db) {
|
|
104
|
+
const rows = db.prepare(`
|
|
105
|
+
SELECT p.path, p.title, p.summary, p.last_updated, p.pinned,
|
|
106
|
+
(SELECT COUNT(*) FROM wiki_links WHERE to_page = p.path) AS inbound_links
|
|
107
|
+
FROM wiki_pages p
|
|
108
|
+
`).all();
|
|
109
|
+
const candidates = [];
|
|
110
|
+
for (const row of rows) {
|
|
111
|
+
const content = readPage(row.path);
|
|
112
|
+
if (!content) {
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
const { parsed: frontmatter } = parseWikiFrontmatter(content);
|
|
116
|
+
const frontmatterPinned = frontmatter.metadata.pinned === true;
|
|
117
|
+
const pinned = row.pinned === 1 || frontmatterPinned;
|
|
118
|
+
db.prepare(`UPDATE wiki_pages SET pinned = ? WHERE path = ?`).run(pinned ? 1 : 0, row.path);
|
|
119
|
+
if (pinned) {
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
122
|
+
const timelineEntries = parseTimelineEntries(content);
|
|
123
|
+
if (timelineEntries.length === 0) {
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
const pendingEntries = timelineEntries.filter((entry) => !row.last_updated || entry.timestamp > row.last_updated);
|
|
127
|
+
if (pendingEntries.length === 0) {
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
candidates.push({
|
|
131
|
+
path: row.path,
|
|
132
|
+
inboundLinks: row.inbound_links,
|
|
133
|
+
pendingEntries,
|
|
134
|
+
mostRecentTimelineAt: pendingEntries[pendingEntries.length - 1].timestamp,
|
|
135
|
+
currentTruth: extractSummary(content),
|
|
136
|
+
content,
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
candidates.sort((left, right) => {
|
|
140
|
+
if (right.pendingEntries.length !== left.pendingEntries.length) {
|
|
141
|
+
return right.pendingEntries.length - left.pendingEntries.length;
|
|
142
|
+
}
|
|
143
|
+
if (right.mostRecentTimelineAt !== left.mostRecentTimelineAt) {
|
|
144
|
+
return right.mostRecentTimelineAt.localeCompare(left.mostRecentTimelineAt);
|
|
145
|
+
}
|
|
146
|
+
return right.inboundLinks - left.inboundLinks;
|
|
147
|
+
});
|
|
148
|
+
return candidates;
|
|
149
|
+
}
|
|
150
|
+
function applyTruthRewrite(db, pagePath, content, compiledTruth, lastUpdatedIso) {
|
|
151
|
+
const normalizedPath = normalizeWikiPath(pagePath);
|
|
152
|
+
const { parsed: frontmatter } = parseWikiFrontmatter(content);
|
|
153
|
+
const summary = summarizeCompiledTruth(compiledTruth) || frontmatter.summary || frontmatter.title || normalizedPath;
|
|
154
|
+
const rewrittenContent = replaceSummarySection(content, compiledTruth);
|
|
155
|
+
const withFrontmatter = updateFrontmatterFields(rewrittenContent, {
|
|
156
|
+
summary,
|
|
157
|
+
updated: lastUpdatedIso.slice(0, 10),
|
|
158
|
+
last_updated: lastUpdatedIso,
|
|
159
|
+
});
|
|
160
|
+
const { content: backfilled } = validateAndBackfillFrontmatter(normalizedPath, withFrontmatter);
|
|
161
|
+
writePage(normalizedPath, backfilled);
|
|
162
|
+
const parsedAfter = parseWikiFrontmatter(backfilled).parsed;
|
|
163
|
+
upsertWikiPage(normalizedPath, parsedAfter, summary);
|
|
164
|
+
db.prepare(`
|
|
165
|
+
UPDATE wiki_pages
|
|
166
|
+
SET compiled_truth_hash = ?, last_updated = ?, pinned = ?
|
|
167
|
+
WHERE path = ?
|
|
168
|
+
`).run(hashCompiledTruth(compiledTruth), lastUpdatedIso, frontmatter.metadata.pinned === true ? 1 : 0, normalizedPath);
|
|
169
|
+
}
|
|
170
|
+
function mergeFragments(db, updatedAt, modifiedPaths) {
|
|
171
|
+
const pages = db.prepare(`
|
|
172
|
+
SELECT p.path, p.last_updated,
|
|
173
|
+
(SELECT COUNT(*) FROM wiki_links WHERE to_page = p.path) AS inbound_links
|
|
174
|
+
FROM wiki_pages p
|
|
175
|
+
ORDER BY p.path ASC
|
|
176
|
+
`).all();
|
|
177
|
+
const active = new Map(pages.map((page) => [page.path, page]));
|
|
178
|
+
let merged = 0;
|
|
179
|
+
for (const page of pages) {
|
|
180
|
+
if (!active.has(page.path)) {
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
for (const candidate of pages) {
|
|
184
|
+
if (page.path === candidate.path || !active.has(candidate.path)) {
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
if (!arePotentialFragments(db, page.path, candidate.path)) {
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
const canonical = chooseCanonicalPage(active.get(page.path), active.get(candidate.path));
|
|
191
|
+
const fragment = canonical.path === page.path ? candidate : page;
|
|
192
|
+
mergePageIntoCanonical(db, canonical.path, fragment.path, updatedAt, modifiedPaths);
|
|
193
|
+
modifiedPaths.add(canonical.path);
|
|
194
|
+
active.delete(fragment.path);
|
|
195
|
+
merged += 1;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
return merged;
|
|
199
|
+
}
|
|
200
|
+
function arePotentialFragments(db, leftPath, rightPath) {
|
|
201
|
+
const leftInfo = pathInfo(leftPath);
|
|
202
|
+
const rightInfo = pathInfo(rightPath);
|
|
203
|
+
if (!leftInfo || !rightInfo || leftInfo.category !== rightInfo.category) {
|
|
204
|
+
return false;
|
|
205
|
+
}
|
|
206
|
+
if (leftInfo.slug.startsWith(rightInfo.slug) || rightInfo.slug.startsWith(leftInfo.slug)) {
|
|
207
|
+
return true;
|
|
208
|
+
}
|
|
209
|
+
if (levenshtein(leftInfo.slug, rightInfo.slug) <= 2) {
|
|
210
|
+
return true;
|
|
211
|
+
}
|
|
212
|
+
const sharedTarget = db.prepare(`
|
|
213
|
+
SELECT 1
|
|
214
|
+
FROM wiki_links l1
|
|
215
|
+
JOIN wiki_links l2 ON l1.to_page = l2.to_page
|
|
216
|
+
WHERE l1.from_page = ? AND l2.from_page = ?
|
|
217
|
+
LIMIT 1
|
|
218
|
+
`).get(leftPath, rightPath);
|
|
219
|
+
return Boolean(sharedTarget);
|
|
220
|
+
}
|
|
221
|
+
function chooseCanonicalPage(left, right) {
|
|
222
|
+
const leftSlug = pathInfo(left.path)?.slug ?? left.path;
|
|
223
|
+
const rightSlug = pathInfo(right.path)?.slug ?? right.path;
|
|
224
|
+
if (leftSlug.length !== rightSlug.length) {
|
|
225
|
+
return leftSlug.length < rightSlug.length ? left : right;
|
|
226
|
+
}
|
|
227
|
+
if (leftSlug !== rightSlug) {
|
|
228
|
+
return leftSlug.localeCompare(rightSlug) <= 0 ? left : right;
|
|
229
|
+
}
|
|
230
|
+
if (left.inbound_links !== right.inbound_links) {
|
|
231
|
+
return left.inbound_links > right.inbound_links ? left : right;
|
|
232
|
+
}
|
|
233
|
+
if ((left.last_updated ?? "") !== (right.last_updated ?? "")) {
|
|
234
|
+
return (left.last_updated ?? "") > (right.last_updated ?? "") ? left : right;
|
|
235
|
+
}
|
|
236
|
+
return left.path.localeCompare(right.path) <= 0 ? left : right;
|
|
237
|
+
}
|
|
238
|
+
function mergePageIntoCanonical(db, canonicalPath, fragmentPath, updatedAt, modifiedPaths) {
|
|
239
|
+
const canonicalContent = readPage(canonicalPath);
|
|
240
|
+
const fragmentContent = readPage(fragmentPath);
|
|
241
|
+
if (!canonicalContent || !fragmentContent) {
|
|
242
|
+
return;
|
|
243
|
+
}
|
|
244
|
+
const fragmentTimeline = extractTimelineBody(fragmentContent).trim();
|
|
245
|
+
let mergedContent = canonicalContent;
|
|
246
|
+
if (fragmentTimeline) {
|
|
247
|
+
mergedContent = appendTimelineRaw(canonicalContent, fragmentTimeline);
|
|
248
|
+
mergedContent = updateFrontmatterFields(mergedContent, {
|
|
249
|
+
updated: updatedAt.slice(0, 10),
|
|
250
|
+
last_updated: updatedAt,
|
|
251
|
+
});
|
|
252
|
+
const { content: backfilled } = validateAndBackfillFrontmatter(canonicalPath, mergedContent);
|
|
253
|
+
writePage(canonicalPath, backfilled);
|
|
254
|
+
const parsed = parseWikiFrontmatter(backfilled).parsed;
|
|
255
|
+
const summary = parsed.summary ?? (summarizeCompiledTruth(extractSummary(backfilled)) || parsed.title || canonicalPath);
|
|
256
|
+
upsertWikiPage(canonicalPath, parsed, summary);
|
|
257
|
+
db.prepare(`UPDATE wiki_pages SET last_updated = ? WHERE path = ?`).run(updatedAt, canonicalPath);
|
|
258
|
+
}
|
|
259
|
+
rewritePageReferences(db, fragmentPath, canonicalPath, updatedAt, modifiedPaths);
|
|
260
|
+
repointLinks(db, canonicalPath, fragmentPath);
|
|
261
|
+
deletePage(fragmentPath);
|
|
262
|
+
removeWikiPage(fragmentPath);
|
|
263
|
+
}
|
|
264
|
+
function rewritePageReferences(db, fragmentPath, canonicalPath, updatedAt, modifiedPaths) {
|
|
265
|
+
const fragmentContent = readPage(fragmentPath) ?? "";
|
|
266
|
+
const canonicalContent = readPage(canonicalPath) ?? "";
|
|
267
|
+
const fragmentTitle = parseWikiFrontmatter(fragmentContent).parsed.title ?? humanizeSlug(pathInfo(fragmentPath)?.slug ?? fragmentPath);
|
|
268
|
+
const canonicalTitle = parseWikiFrontmatter(canonicalContent).parsed.title ?? humanizeSlug(pathInfo(canonicalPath)?.slug ?? canonicalPath);
|
|
269
|
+
const fragmentSlug = pathInfo(fragmentPath)?.slug;
|
|
270
|
+
const canonicalSlug = pathInfo(canonicalPath)?.slug;
|
|
271
|
+
for (const pagePath of listPages()) {
|
|
272
|
+
if (pagePath === fragmentPath || pagePath === canonicalPath) {
|
|
273
|
+
continue;
|
|
274
|
+
}
|
|
275
|
+
const content = readPage(pagePath);
|
|
276
|
+
if (!content) {
|
|
277
|
+
continue;
|
|
278
|
+
}
|
|
279
|
+
let nextContent = content.replaceAll(`[[${fragmentTitle}]]`, `[[${canonicalTitle}]]`);
|
|
280
|
+
if (fragmentSlug && canonicalSlug) {
|
|
281
|
+
nextContent = nextContent.replaceAll(`[[${fragmentSlug}]]`, `[[${canonicalSlug}]]`);
|
|
282
|
+
}
|
|
283
|
+
if (nextContent === content) {
|
|
284
|
+
continue;
|
|
285
|
+
}
|
|
286
|
+
nextContent = updateFrontmatterFields(nextContent, {
|
|
287
|
+
updated: updatedAt.slice(0, 10),
|
|
288
|
+
last_updated: updatedAt,
|
|
289
|
+
});
|
|
290
|
+
const { content: backfilled } = validateAndBackfillFrontmatter(pagePath, nextContent);
|
|
291
|
+
writePage(pagePath, backfilled);
|
|
292
|
+
const parsed = parseWikiFrontmatter(backfilled).parsed;
|
|
293
|
+
upsertWikiPage(pagePath, parsed, parsed.summary ?? canonicalTitle);
|
|
294
|
+
db.prepare(`UPDATE wiki_pages SET last_updated = ? WHERE path = ?`).run(updatedAt, pagePath);
|
|
295
|
+
modifiedPaths.add(pagePath);
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
function repointLinks(db, canonicalPath, fragmentPath) {
|
|
299
|
+
const rows = db.prepare(`
|
|
300
|
+
SELECT from_page, to_page, link_type, extracted_at
|
|
301
|
+
FROM wiki_links
|
|
302
|
+
WHERE from_page = ? OR to_page = ?
|
|
303
|
+
`).all(fragmentPath, fragmentPath);
|
|
304
|
+
const tx = db.transaction(() => {
|
|
305
|
+
db.prepare(`DELETE FROM wiki_links WHERE from_page = ? OR to_page = ?`).run(fragmentPath, fragmentPath);
|
|
306
|
+
const insert = db.prepare(`
|
|
307
|
+
INSERT OR IGNORE INTO wiki_links (from_page, to_page, link_type, extracted_at)
|
|
308
|
+
VALUES (?, ?, ?, ?)
|
|
309
|
+
`);
|
|
310
|
+
for (const row of rows) {
|
|
311
|
+
const nextFrom = row.from_page === fragmentPath ? canonicalPath : row.from_page;
|
|
312
|
+
const nextTo = row.to_page === fragmentPath ? canonicalPath : row.to_page;
|
|
313
|
+
if (nextFrom === nextTo) {
|
|
314
|
+
continue;
|
|
315
|
+
}
|
|
316
|
+
insert.run(nextFrom, nextTo, row.link_type, row.extracted_at);
|
|
317
|
+
}
|
|
318
|
+
});
|
|
319
|
+
tx();
|
|
320
|
+
}
|
|
321
|
+
function repairOrphanLinks(db) {
|
|
322
|
+
const orphans = db.prepare(`
|
|
323
|
+
SELECT l.from_page, l.to_page, l.link_type, l.extracted_at
|
|
324
|
+
FROM wiki_links l
|
|
325
|
+
LEFT JOIN wiki_pages p ON p.path = l.to_page
|
|
326
|
+
WHERE p.path IS NULL
|
|
327
|
+
`).all();
|
|
328
|
+
if (orphans.length === 0) {
|
|
329
|
+
return 0;
|
|
330
|
+
}
|
|
331
|
+
const remove = db.prepare(`DELETE FROM wiki_links WHERE from_page = ? AND to_page = ? AND link_type = ?`);
|
|
332
|
+
const insert = db.prepare(`
|
|
333
|
+
INSERT OR IGNORE INTO wiki_links (from_page, to_page, link_type, extracted_at)
|
|
334
|
+
VALUES (?, ?, ?, ?)
|
|
335
|
+
`);
|
|
336
|
+
let repaired = 0;
|
|
337
|
+
for (const orphan of orphans) {
|
|
338
|
+
const resolvedTarget = resolvePageBySlug(db, orphan.to_page);
|
|
339
|
+
remove.run(orphan.from_page, orphan.to_page, orphan.link_type);
|
|
340
|
+
if (resolvedTarget && resolvedTarget !== orphan.from_page) {
|
|
341
|
+
insert.run(orphan.from_page, resolvedTarget, orphan.link_type, orphan.extracted_at);
|
|
342
|
+
repaired += 1;
|
|
343
|
+
log.info({ orphan, resolved_target: resolvedTarget }, "wiki.consolidation.repointed_orphan_link");
|
|
344
|
+
continue;
|
|
345
|
+
}
|
|
346
|
+
repaired += 1;
|
|
347
|
+
log.info({ orphan }, "wiki.consolidation.removed_orphan_link");
|
|
348
|
+
}
|
|
349
|
+
return repaired;
|
|
350
|
+
}
|
|
351
|
+
function resolvePageBySlug(db, missingPath) {
|
|
352
|
+
const slug = pathInfo(missingPath)?.slug ?? normalizeWikiPath(missingPath).split("/").filter(Boolean).at(-2);
|
|
353
|
+
if (!slug) {
|
|
354
|
+
return undefined;
|
|
355
|
+
}
|
|
356
|
+
const matches = db.prepare(`SELECT path FROM wiki_pages WHERE path LIKE ? ORDER BY path ASC`).all(`pages/%/${slug}/index.md`);
|
|
357
|
+
return matches.length === 1 ? matches[0].path : undefined;
|
|
358
|
+
}
|
|
359
|
+
function archiveOrphanSources(db) {
|
|
360
|
+
const rows = db.prepare(`
|
|
361
|
+
SELECT id, title, origin, raw_path, pages_updated, status
|
|
362
|
+
FROM wiki_sources
|
|
363
|
+
WHERE COALESCE(status, 'active') != 'archived'
|
|
364
|
+
`).all();
|
|
365
|
+
let archived = 0;
|
|
366
|
+
for (const row of rows) {
|
|
367
|
+
if (isSourceReferenced(row.id, row.title, row.origin, row.pages_updated)) {
|
|
368
|
+
continue;
|
|
369
|
+
}
|
|
370
|
+
if (row.raw_path) {
|
|
371
|
+
moveSourceToArchive(row.raw_path);
|
|
372
|
+
}
|
|
373
|
+
db.prepare(`UPDATE wiki_sources SET status = 'archived' WHERE id = ?`).run(row.id);
|
|
374
|
+
archived += 1;
|
|
375
|
+
}
|
|
376
|
+
return archived;
|
|
377
|
+
}
|
|
378
|
+
function isSourceReferenced(id, title, origin, pagesUpdatedRaw) {
|
|
379
|
+
const paths = parseJsonArray(pagesUpdatedRaw);
|
|
380
|
+
if (paths.length === 0) {
|
|
381
|
+
return false;
|
|
382
|
+
}
|
|
383
|
+
for (const pagePath of paths) {
|
|
384
|
+
const content = readPage(pagePath);
|
|
385
|
+
if (!content) {
|
|
386
|
+
continue;
|
|
387
|
+
}
|
|
388
|
+
if (content.includes(id)) {
|
|
389
|
+
return true;
|
|
390
|
+
}
|
|
391
|
+
if (title && content.includes(`Source ingested: ${title}`)) {
|
|
392
|
+
return true;
|
|
393
|
+
}
|
|
394
|
+
if (origin && content.includes(origin)) {
|
|
395
|
+
return true;
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
return false;
|
|
399
|
+
}
|
|
400
|
+
function moveSourceToArchive(rawPath) {
|
|
401
|
+
const sourcePath = resolveWikiRelativePath(rawPath);
|
|
402
|
+
if (!existsSync(sourcePath)) {
|
|
403
|
+
return;
|
|
404
|
+
}
|
|
405
|
+
const archivePath = join(dirname(sourcePath), "archive", basename(sourcePath));
|
|
406
|
+
mkdirSync(dirname(archivePath), { recursive: true });
|
|
407
|
+
renameSync(sourcePath, archivePath);
|
|
408
|
+
}
|
|
409
|
+
function notifyStaleSessions(db, deps, runAt) {
|
|
410
|
+
const cutoff = new Date(runAt.getTime() - STALE_SESSION_MS).toISOString();
|
|
411
|
+
const rows = db.prepare(`
|
|
412
|
+
SELECT session_id, COALESCE(session_name, session_id) AS session_name, MAX(ingested_at) AS last_ingested_at
|
|
413
|
+
FROM wiki_sources
|
|
414
|
+
WHERE session_id IS NOT NULL AND TRIM(session_id) != ''
|
|
415
|
+
GROUP BY session_id, COALESCE(session_name, session_id)
|
|
416
|
+
HAVING MAX(ingested_at) < ?
|
|
417
|
+
`).all(cutoff);
|
|
418
|
+
let created = 0;
|
|
419
|
+
for (const row of rows) {
|
|
420
|
+
const title = `Research session '${row.session_name}' has been inactive for 7+ days — consider closing or continuing`;
|
|
421
|
+
const detail = `Latest source for session '${row.session_name}' arrived at ${row.last_ingested_at}.`;
|
|
422
|
+
if (deps.createActionItem({ title, detail, source: "wiki:consolidation" })) {
|
|
423
|
+
created += 1;
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
return created;
|
|
427
|
+
}
|
|
428
|
+
function createStaleSessionActionItem(db, input) {
|
|
429
|
+
const globalScope = getScope("global");
|
|
430
|
+
if (!globalScope) {
|
|
431
|
+
return false;
|
|
432
|
+
}
|
|
433
|
+
const existing = recordDuplicateActionItemCheck(db, globalScope.id, input.title);
|
|
434
|
+
if (existing) {
|
|
435
|
+
return false;
|
|
436
|
+
}
|
|
437
|
+
recordActionItem({
|
|
438
|
+
scope_id: globalScope.id,
|
|
439
|
+
title: input.title,
|
|
440
|
+
detail: input.detail,
|
|
441
|
+
source: input.source,
|
|
442
|
+
tier: "warm",
|
|
443
|
+
});
|
|
444
|
+
return true;
|
|
445
|
+
}
|
|
446
|
+
function recordDuplicateActionItemCheck(db, scopeId, title) {
|
|
447
|
+
const row = db.prepare(`
|
|
448
|
+
SELECT 1
|
|
449
|
+
FROM mem_action_items
|
|
450
|
+
WHERE scope_id = ? AND title = ? AND status IN ('open', 'snoozed')
|
|
451
|
+
LIMIT 1
|
|
452
|
+
`).get(scopeId, title);
|
|
453
|
+
return Boolean(row);
|
|
454
|
+
}
|
|
455
|
+
async function synthesizeTruthWithCopilot(input) {
|
|
456
|
+
const token = config.copilotAuthToken || process.env.COPILOT_TOKEN || process.env.GITHUB_TOKEN;
|
|
457
|
+
if (!token) {
|
|
458
|
+
throw new Error(`Cannot consolidate '${input.pagePath}' without a Copilot auth token.`);
|
|
459
|
+
}
|
|
460
|
+
const prompt = [
|
|
461
|
+
`Rewrite the compiled truth for the wiki page '${input.pageTitle}' (${input.pagePath}).`,
|
|
462
|
+
"Keep it factual and concise.",
|
|
463
|
+
"Return Markdown only for the ## Summary section body, without the heading.",
|
|
464
|
+
"Existing compiled truth:",
|
|
465
|
+
input.existingTruth || "(none)",
|
|
466
|
+
"New timeline entries:",
|
|
467
|
+
input.pendingEntries.map((entry) => `- ${entry.timestamp}: ${entry.content}`).join("\n"),
|
|
468
|
+
].join("\n\n");
|
|
469
|
+
const { CopilotClient, approveAll } = await import("@github/copilot-sdk");
|
|
470
|
+
const client = new CopilotClient({
|
|
471
|
+
autoStart: true,
|
|
472
|
+
autoRestart: false,
|
|
473
|
+
gitHubToken: token,
|
|
474
|
+
});
|
|
475
|
+
await client.start();
|
|
476
|
+
try {
|
|
477
|
+
const session = await client.createSession({
|
|
478
|
+
model: DEFAULT_MODEL,
|
|
479
|
+
tools: [],
|
|
480
|
+
onPermissionRequest: approveAll,
|
|
481
|
+
});
|
|
482
|
+
try {
|
|
483
|
+
const response = await session.sendAndWait({ prompt }, 30_000);
|
|
484
|
+
return String(response).trim();
|
|
485
|
+
}
|
|
486
|
+
finally {
|
|
487
|
+
try {
|
|
488
|
+
session.destroy();
|
|
489
|
+
}
|
|
490
|
+
catch { /* best effort */ }
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
finally {
|
|
494
|
+
try {
|
|
495
|
+
await client.stop();
|
|
496
|
+
}
|
|
497
|
+
catch { /* best effort */ }
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
async function commitWikiChanges(runAt) {
|
|
501
|
+
const repoRoot = getChapterhouseHome();
|
|
502
|
+
try {
|
|
503
|
+
execFileSync("git", ["-C", repoRoot, "rev-parse", "--is-inside-work-tree"], { stdio: "ignore" });
|
|
504
|
+
}
|
|
505
|
+
catch {
|
|
506
|
+
log.debug({ repoRoot }, "wiki.consolidation.git_skip_not_repo");
|
|
507
|
+
return false;
|
|
508
|
+
}
|
|
509
|
+
const status = execFileSync("git", ["-C", repoRoot, "status", "--porcelain"], { encoding: "utf-8" }).trim();
|
|
510
|
+
if (!status) {
|
|
511
|
+
return false;
|
|
512
|
+
}
|
|
513
|
+
execFileSync("git", ["-C", repoRoot, "add", "-A"], { stdio: "ignore" });
|
|
514
|
+
execFileSync("git", ["-C", repoRoot, "commit", "-m", `pkb: nightly consolidation [${runAt.toISOString().slice(0, 10)}]`, "--no-verify"], {
|
|
515
|
+
stdio: "ignore",
|
|
516
|
+
});
|
|
517
|
+
return true;
|
|
518
|
+
}
|
|
519
|
+
function parseTimelineEntries(content) {
|
|
520
|
+
const body = extractSectionBody(content, "Timeline");
|
|
521
|
+
if (!body.trim()) {
|
|
522
|
+
return [];
|
|
523
|
+
}
|
|
524
|
+
const matches = [...body.matchAll(/^###\s+([^\n]+)\n\n([\s\S]*?)(?=\n###\s+|\n##\s+|$)/gm)];
|
|
525
|
+
return matches
|
|
526
|
+
.map((match) => ({
|
|
527
|
+
timestamp: match[1].trim(),
|
|
528
|
+
content: match[2].trim(),
|
|
529
|
+
raw: match[0].trim(),
|
|
530
|
+
}))
|
|
531
|
+
.filter((entry) => !Number.isNaN(Date.parse(entry.timestamp)));
|
|
532
|
+
}
|
|
533
|
+
function extractTimelineBody(content) {
|
|
534
|
+
return extractSectionBody(content, "Timeline");
|
|
535
|
+
}
|
|
536
|
+
function extractSummary(content) {
|
|
537
|
+
return extractSectionBody(content, "Summary").trim();
|
|
538
|
+
}
|
|
539
|
+
function replaceSummarySection(content, summaryBody) {
|
|
540
|
+
const replacement = `## Summary\n\n${summaryBody.trim()}\n`;
|
|
541
|
+
const range = findSectionRange(content, "Summary");
|
|
542
|
+
if (range) {
|
|
543
|
+
return `${content.slice(0, range.start)}${replacement}${content.slice(range.end)}`;
|
|
544
|
+
}
|
|
545
|
+
const timelineRange = findSectionRange(content, "Timeline");
|
|
546
|
+
if (timelineRange) {
|
|
547
|
+
return `${content.slice(0, timelineRange.start)}${replacement}\n${content.slice(timelineRange.start)}`;
|
|
548
|
+
}
|
|
549
|
+
return `${content.trimEnd()}\n\n${replacement}`;
|
|
550
|
+
}
|
|
551
|
+
function appendTimelineRaw(content, timelineBody) {
|
|
552
|
+
const existingTimeline = extractTimelineBody(content).trim();
|
|
553
|
+
const combined = [existingTimeline, timelineBody.trim()].filter(Boolean).join("\n\n");
|
|
554
|
+
const replacement = `## Timeline\n\n${combined}\n`;
|
|
555
|
+
const range = findSectionRange(content, "Timeline");
|
|
556
|
+
if (range) {
|
|
557
|
+
return `${content.slice(0, range.start)}${replacement}${content.slice(range.end)}`;
|
|
558
|
+
}
|
|
559
|
+
return `${content.trimEnd()}\n\n${replacement}`;
|
|
560
|
+
}
|
|
561
|
+
function extractSectionBody(content, heading) {
|
|
562
|
+
const range = findSectionRange(content, heading);
|
|
563
|
+
return range ? content.slice(range.bodyStart, range.end) : "";
|
|
564
|
+
}
|
|
565
|
+
function findSectionRange(content, heading) {
|
|
566
|
+
const token = `## ${heading}`;
|
|
567
|
+
const start = content.indexOf(token);
|
|
568
|
+
if (start === -1) {
|
|
569
|
+
return undefined;
|
|
570
|
+
}
|
|
571
|
+
const headingLineEnd = content.indexOf("\n", start);
|
|
572
|
+
const bodyStart = headingLineEnd === -1 ? content.length : headingLineEnd + 1;
|
|
573
|
+
const nextSection = content.indexOf("\n## ", bodyStart);
|
|
574
|
+
return {
|
|
575
|
+
start,
|
|
576
|
+
bodyStart,
|
|
577
|
+
end: nextSection === -1 ? content.length : nextSection,
|
|
578
|
+
};
|
|
579
|
+
}
|
|
580
|
+
function updateFrontmatterFields(content, fields) {
|
|
581
|
+
const match = content.match(/^---\s*\n([\s\S]*?)\n---\s*\n?/);
|
|
582
|
+
if (!match) {
|
|
583
|
+
return content;
|
|
584
|
+
}
|
|
585
|
+
const lines = match[1].split("\n");
|
|
586
|
+
for (const [key, value] of Object.entries(fields)) {
|
|
587
|
+
const serialized = typeof value === "boolean" ? `${key}: ${value ? "true" : "false"}` : `${key}: ${value}`;
|
|
588
|
+
const index = lines.findIndex((line) => line.startsWith(`${key}:`));
|
|
589
|
+
if (index >= 0) {
|
|
590
|
+
lines[index] = serialized;
|
|
591
|
+
}
|
|
592
|
+
else {
|
|
593
|
+
lines.push(serialized);
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
return `---\n${lines.join("\n")}\n---\n${content.slice(match[0].length)}`;
|
|
597
|
+
}
|
|
598
|
+
function summarizeCompiledTruth(compiledTruth) {
|
|
599
|
+
const firstMeaningfulLine = compiledTruth
|
|
600
|
+
.split("\n")
|
|
601
|
+
.map((line) => line.trim().replace(/^[-*]\s+/, ""))
|
|
602
|
+
.find((line) => Boolean(line));
|
|
603
|
+
return (firstMeaningfulLine ?? "").slice(0, 200);
|
|
604
|
+
}
|
|
605
|
+
function hashCompiledTruth(compiledTruth) {
|
|
606
|
+
return createHash("sha256").update(compiledTruth.trim()).digest("hex");
|
|
607
|
+
}
|
|
608
|
+
function parseJsonArray(raw) {
|
|
609
|
+
try {
|
|
610
|
+
const parsed = JSON.parse(raw || "[]");
|
|
611
|
+
return Array.isArray(parsed) ? parsed.filter((value) => typeof value === "string") : [];
|
|
612
|
+
}
|
|
613
|
+
catch {
|
|
614
|
+
return [];
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
function humanizeSlug(value) {
|
|
618
|
+
return value.split(/[-_]+/).map((segment) => segment.charAt(0).toUpperCase() + segment.slice(1)).join(" ");
|
|
619
|
+
}
|
|
620
|
+
function pathInfo(path) {
|
|
621
|
+
const match = normalizeWikiPath(path).match(/^pages\/([^/]+)\/([^/]+)\/index\.md$/);
|
|
622
|
+
if (!match) {
|
|
623
|
+
return undefined;
|
|
624
|
+
}
|
|
625
|
+
return { category: match[1], slug: match[2] };
|
|
626
|
+
}
|
|
627
|
+
function levenshtein(left, right) {
|
|
628
|
+
const dp = Array.from({ length: left.length + 1 }, () => Array(right.length + 1).fill(0));
|
|
629
|
+
for (let i = 0; i <= left.length; i++)
|
|
630
|
+
dp[i][0] = i;
|
|
631
|
+
for (let j = 0; j <= right.length; j++)
|
|
632
|
+
dp[0][j] = j;
|
|
633
|
+
for (let i = 1; i <= left.length; i++) {
|
|
634
|
+
for (let j = 1; j <= right.length; j++) {
|
|
635
|
+
const cost = left[i - 1] === right[j - 1] ? 0 : 1;
|
|
636
|
+
dp[i][j] = Math.min(dp[i - 1][j] + 1, dp[i][j - 1] + 1, dp[i - 1][j - 1] + cost);
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
return dp[left.length][right.length];
|
|
640
|
+
}
|
|
641
|
+
//# sourceMappingURL=consolidation.js.map
|