chapterhouse 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/korg.agent.md +65 -0
- package/dist/api/agent-edit-access.js +11 -0
- package/dist/api/agents.api.test.js +48 -0
- package/dist/api/korg.js +34 -0
- package/dist/api/korg.test.js +42 -0
- package/dist/api/server.js +420 -13
- package/dist/api/server.test.js +533 -3
- package/dist/config.js +28 -0
- package/dist/config.test.js +20 -0
- package/dist/copilot/agent-event-bus.js +1 -0
- package/dist/copilot/agents.js +117 -50
- package/dist/copilot/agents.mcp-servers.test.js +87 -0
- package/dist/copilot/agents.parse.test.js +69 -0
- package/dist/copilot/agents.test.js +137 -2
- package/dist/copilot/orchestrator.js +62 -13
- package/dist/copilot/orchestrator.test.js +130 -8
- package/dist/copilot/session-manager.js +34 -0
- package/dist/copilot/system-message.js +11 -10
- package/dist/copilot/system-message.test.js +6 -1
- package/dist/copilot/tools.js +184 -376
- package/dist/copilot/tools.memory.test.js +32 -0
- package/dist/copilot/tools.wiki.test.js +53 -59
- package/dist/daemon.js +9 -0
- package/dist/memory/decisions.js +6 -5
- package/dist/memory/entities.js +20 -9
- package/dist/memory/hooks.js +151 -0
- package/dist/memory/hooks.test.js +325 -0
- package/dist/memory/hot-tier.js +37 -0
- package/dist/memory/hot-tier.test.js +30 -0
- package/dist/memory/housekeeping-scheduler.js +35 -0
- package/dist/memory/housekeeping-scheduler.test.js +50 -0
- package/dist/memory/inbox.js +10 -0
- package/dist/memory/index.js +3 -1
- package/dist/memory/migration.js +244 -0
- package/dist/memory/migration.test.js +100 -0
- package/dist/memory/reflect.js +273 -0
- package/dist/memory/reflect.test.js +254 -0
- package/dist/store/db.js +119 -4
- package/dist/store/db.test.js +19 -1
- package/dist/test/setup-env.js +3 -1
- package/dist/test/setup-env.test.js +8 -1
- package/dist/wiki/consolidation.js +641 -0
- package/dist/wiki/consolidation.test.js +140 -0
- package/dist/wiki/frontmatter.js +48 -0
- package/dist/wiki/frontmatter.test.js +42 -0
- package/dist/wiki/index-manager.js +246 -330
- package/dist/wiki/index-manager.test.js +138 -145
- package/dist/wiki/ingest.js +347 -0
- package/dist/wiki/ingest.test.js +111 -0
- package/dist/wiki/links.js +151 -0
- package/dist/wiki/links.test.js +176 -0
- package/dist/wiki/migrate-topics.test.js +16 -6
- package/dist/wiki/scheduler.js +118 -0
- package/dist/wiki/scheduler.test.js +64 -0
- package/dist/wiki/timeline.js +51 -0
- package/dist/wiki/timeline.test.js +65 -0
- package/dist/wiki/topic-structure.js +1 -1
- package/package.json +3 -1
- package/skills/pkb-ideas/SKILL.md +78 -0
- package/skills/pkb-ideas/_meta.json +4 -0
- package/skills/pkb-org/SKILL.md +82 -0
- package/skills/pkb-org/_meta.json +4 -0
- package/skills/pkb-people/SKILL.md +74 -0
- package/skills/pkb-people/_meta.json +4 -0
- package/skills/pkb-research/SKILL.md +83 -0
- package/skills/pkb-research/_meta.json +4 -0
- package/skills/pkb-source/SKILL.md +38 -0
- package/skills/pkb-source/_meta.json +4 -0
- package/skills/wiki-conventions/SKILL.md +5 -5
- package/web/dist/assets/index-5kz9aRU9.css +10 -0
- package/web/dist/assets/{index-B5oDsQ5y.js → index-BbX9RKf3.js} +101 -99
- package/web/dist/assets/index-BbX9RKf3.js.map +1 -0
- package/web/dist/index.html +2 -2
- package/dist/wiki/context.js +0 -138
- package/dist/wiki/fix.js +0 -335
- package/dist/wiki/fix.test.js +0 -350
- package/dist/wiki/lint.js +0 -451
- package/dist/wiki/lint.test.js +0 -329
- package/web/dist/assets/index-B5oDsQ5y.js.map +0 -1
- package/web/dist/assets/index-DknKAtDS.css +0 -10
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
|
3
|
+
import { join, relative } from "node:path";
|
|
4
|
+
import { WIKI_PAGES_DIR } from "../paths.js";
|
|
5
|
+
import { parseWikiFrontmatter } from "../wiki/frontmatter.js";
|
|
6
|
+
const MIGRATION_NAME = "p6-wiki-seed";
|
|
7
|
+
const MIGRATION_SOURCE = "migration:p6";
|
|
8
|
+
export async function runP6Migration(db) {
|
|
9
|
+
db.exec(`
|
|
10
|
+
CREATE TABLE IF NOT EXISTS mem_migrations (
|
|
11
|
+
id INTEGER PRIMARY KEY,
|
|
12
|
+
name TEXT UNIQUE,
|
|
13
|
+
run_at TEXT
|
|
14
|
+
)
|
|
15
|
+
`);
|
|
16
|
+
const alreadyRan = db.prepare(`SELECT 1 FROM mem_migrations WHERE name = ?`).get(MIGRATION_NAME);
|
|
17
|
+
if (alreadyRan) {
|
|
18
|
+
return {
|
|
19
|
+
entitiesCreated: 0,
|
|
20
|
+
observationsCreated: 0,
|
|
21
|
+
decisionsCreated: 0,
|
|
22
|
+
skipped: -1,
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
const globalScopeId = getScopeId(db, "global");
|
|
26
|
+
if (!globalScopeId) {
|
|
27
|
+
throw new Error("Missing required memory scope 'global'.");
|
|
28
|
+
}
|
|
29
|
+
const chapterhouseScopeId = getScopeId(db, "chapterhouse") ?? globalScopeId;
|
|
30
|
+
const result = {
|
|
31
|
+
entitiesCreated: 0,
|
|
32
|
+
observationsCreated: 0,
|
|
33
|
+
decisionsCreated: 0,
|
|
34
|
+
skipped: 0,
|
|
35
|
+
};
|
|
36
|
+
if (existsSync(WIKI_PAGES_DIR)) {
|
|
37
|
+
for (const kindDir of ["projects", "people", "topics"]) {
|
|
38
|
+
for (const pagePath of findFiles(join(WIKI_PAGES_DIR, kindDir), "index.md")) {
|
|
39
|
+
const relativeSlug = normalizeRelativePageSlug(pagePath);
|
|
40
|
+
const content = readFileSync(pagePath, "utf-8");
|
|
41
|
+
const { parsed, body } = parseWikiFrontmatter(content);
|
|
42
|
+
const kind = toEntityKind(kindDir);
|
|
43
|
+
const title = parsed.title?.trim() || deriveTitleFromSlug(relativeSlug);
|
|
44
|
+
const summary = parsed.summary?.trim() || null;
|
|
45
|
+
const entityId = upsertEntityBySlug(db, {
|
|
46
|
+
scopeId: globalScopeId,
|
|
47
|
+
slug: relativeSlug,
|
|
48
|
+
kind,
|
|
49
|
+
title,
|
|
50
|
+
summary,
|
|
51
|
+
});
|
|
52
|
+
if (entityId.created) {
|
|
53
|
+
result.entitiesCreated += 1;
|
|
54
|
+
}
|
|
55
|
+
const compiledTruth = extractCompiledTruth(body, title);
|
|
56
|
+
if (!compiledTruth) {
|
|
57
|
+
result.skipped += 1;
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
const observationContent = compiledTruth.slice(0, 500);
|
|
61
|
+
if (insertObservationIfMissing(db, {
|
|
62
|
+
scopeId: globalScopeId,
|
|
63
|
+
entityId: entityId.id,
|
|
64
|
+
content: observationContent,
|
|
65
|
+
})) {
|
|
66
|
+
result.observationsCreated += 1;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
for (const decisionPath of findFiles(WIKI_PAGES_DIR, "decisions.md")) {
|
|
71
|
+
const entries = parseDecisionEntries(readFileSync(decisionPath, "utf-8"));
|
|
72
|
+
for (const entry of entries) {
|
|
73
|
+
if (!entry.rationale) {
|
|
74
|
+
result.skipped += 1;
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
if (insertDecisionIfMissing(db, {
|
|
78
|
+
scopeId: chapterhouseScopeId,
|
|
79
|
+
title: entry.title,
|
|
80
|
+
rationale: entry.rationale,
|
|
81
|
+
})) {
|
|
82
|
+
result.decisionsCreated += 1;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
db.prepare(`INSERT INTO mem_migrations (name, run_at) VALUES (?, ?)`)
|
|
88
|
+
.run(MIGRATION_NAME, new Date().toISOString());
|
|
89
|
+
return result;
|
|
90
|
+
}
|
|
91
|
+
function getScopeId(db, slug) {
|
|
92
|
+
const row = db.prepare(`SELECT id FROM mem_scopes WHERE slug = ?`).get(slug);
|
|
93
|
+
return row?.id;
|
|
94
|
+
}
|
|
95
|
+
function findFiles(root, targetName) {
|
|
96
|
+
if (!existsSync(root)) {
|
|
97
|
+
return [];
|
|
98
|
+
}
|
|
99
|
+
const matches = [];
|
|
100
|
+
for (const entry of readdirSync(root, { withFileTypes: true })) {
|
|
101
|
+
const fullPath = join(root, entry.name);
|
|
102
|
+
if (entry.isDirectory()) {
|
|
103
|
+
matches.push(...findFiles(fullPath, targetName));
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
if (entry.isFile() && entry.name === targetName) {
|
|
107
|
+
matches.push(fullPath);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
return matches.sort((left, right) => left.localeCompare(right));
|
|
111
|
+
}
|
|
112
|
+
function normalizeRelativePageSlug(fullPath) {
|
|
113
|
+
return relative(WIKI_PAGES_DIR, fullPath).replace(/\\/g, "/");
|
|
114
|
+
}
|
|
115
|
+
function toEntityKind(directory) {
|
|
116
|
+
switch (directory) {
|
|
117
|
+
case "projects": return "project";
|
|
118
|
+
case "people": return "person";
|
|
119
|
+
case "topics": return "topic";
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
function deriveTitleFromSlug(slug) {
|
|
123
|
+
const parts = slug.split("/").filter(Boolean);
|
|
124
|
+
const base = parts[parts.length - 2] || parts[parts.length - 1] || slug;
|
|
125
|
+
return base.split(/[-_]+/).map((part) => part.charAt(0).toUpperCase() + part.slice(1)).join(" ");
|
|
126
|
+
}
|
|
127
|
+
function extractCompiledTruth(body, fallback) {
|
|
128
|
+
let text = body.replace(/\r\n/g, "\n").trim();
|
|
129
|
+
text = text.replace(/^#\s+.+?(\n+|$)/, "").trim();
|
|
130
|
+
const summaryMatch = text.match(/^##\s+Summary\s*\n+([\s\S]*?)(?=\n##\s+|$)/m);
|
|
131
|
+
if (summaryMatch?.[1]?.trim()) {
|
|
132
|
+
text = summaryMatch[1].trim();
|
|
133
|
+
}
|
|
134
|
+
const beforeTimeline = text.split(/\n##\s+Timeline\b/m)[0]?.trim() ?? "";
|
|
135
|
+
return beforeTimeline || fallback;
|
|
136
|
+
}
|
|
137
|
+
function upsertEntityBySlug(db, input) {
|
|
138
|
+
const existingBySlug = db.prepare(`SELECT id FROM mem_entities WHERE scope_id = ? AND slug = ?`).get(input.scopeId, input.slug);
|
|
139
|
+
if (existingBySlug) {
|
|
140
|
+
db.prepare(`
|
|
141
|
+
UPDATE mem_entities
|
|
142
|
+
SET kind = ?, name = ?, summary = ?, updated_at = CURRENT_TIMESTAMP
|
|
143
|
+
WHERE id = ?
|
|
144
|
+
`).run(input.kind, input.title, input.summary, existingBySlug.id);
|
|
145
|
+
return { id: existingBySlug.id, created: false };
|
|
146
|
+
}
|
|
147
|
+
const existingByName = db.prepare(`
|
|
148
|
+
SELECT id
|
|
149
|
+
FROM mem_entities
|
|
150
|
+
WHERE scope_id = ? AND kind = ? AND name = ?
|
|
151
|
+
`).get(input.scopeId, input.kind, input.title);
|
|
152
|
+
if (existingByName) {
|
|
153
|
+
db.prepare(`
|
|
154
|
+
UPDATE mem_entities
|
|
155
|
+
SET slug = ?, summary = ?, updated_at = CURRENT_TIMESTAMP
|
|
156
|
+
WHERE id = ?
|
|
157
|
+
`).run(input.slug, input.summary, existingByName.id);
|
|
158
|
+
return { id: existingByName.id, created: false };
|
|
159
|
+
}
|
|
160
|
+
const inserted = db.prepare(`
|
|
161
|
+
INSERT INTO mem_entities (scope_id, slug, kind, name, summary, tier, confidence, created_at, updated_at)
|
|
162
|
+
VALUES (?, ?, ?, ?, ?, 'warm', 1.0, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
163
|
+
`).run(input.scopeId, input.slug, input.kind, input.title, input.summary);
|
|
164
|
+
return { id: Number(inserted.lastInsertRowid), created: true };
|
|
165
|
+
}
|
|
166
|
+
function insertObservationIfMissing(db, input) {
|
|
167
|
+
const hash = contentHash(input.content);
|
|
168
|
+
const existing = db.prepare(`
|
|
169
|
+
SELECT id
|
|
170
|
+
FROM mem_observations
|
|
171
|
+
WHERE scope_id = ? AND entity_id = ? AND source = ? AND content = ?
|
|
172
|
+
`).get(input.scopeId, input.entityId, MIGRATION_SOURCE, input.content);
|
|
173
|
+
if (existing) {
|
|
174
|
+
return false;
|
|
175
|
+
}
|
|
176
|
+
const duplicateByHash = db.prepare(`
|
|
177
|
+
SELECT id, content
|
|
178
|
+
FROM mem_observations
|
|
179
|
+
WHERE scope_id = ? AND entity_id = ? AND source = ?
|
|
180
|
+
`).all(input.scopeId, input.entityId, MIGRATION_SOURCE);
|
|
181
|
+
if (duplicateByHash.some((row) => contentHash(row.content) === hash)) {
|
|
182
|
+
return false;
|
|
183
|
+
}
|
|
184
|
+
db.prepare(`
|
|
185
|
+
INSERT INTO mem_observations (scope_id, entity_id, content, source, tier, confidence, created_at)
|
|
186
|
+
VALUES (?, ?, ?, ?, 'warm', 1.0, CURRENT_TIMESTAMP)
|
|
187
|
+
`).run(input.scopeId, input.entityId, input.content, MIGRATION_SOURCE);
|
|
188
|
+
return true;
|
|
189
|
+
}
|
|
190
|
+
function insertDecisionIfMissing(db, input) {
|
|
191
|
+
const hash = contentHash(`${input.title}\n${input.rationale}`);
|
|
192
|
+
const rows = db.prepare(`
|
|
193
|
+
SELECT id, title, rationale
|
|
194
|
+
FROM mem_decisions
|
|
195
|
+
WHERE scope_id = ?
|
|
196
|
+
`).all(input.scopeId);
|
|
197
|
+
if (rows.some((row) => contentHash(`${row.title}\n${row.rationale}`) === hash)) {
|
|
198
|
+
return false;
|
|
199
|
+
}
|
|
200
|
+
db.prepare(`
|
|
201
|
+
INSERT INTO mem_decisions (scope_id, title, rationale, decided_at, source, tier, created_at)
|
|
202
|
+
VALUES (?, ?, ?, ?, ?, 'warm', CURRENT_TIMESTAMP)
|
|
203
|
+
`).run(input.scopeId, input.title, input.rationale, new Date().toISOString().slice(0, 10), MIGRATION_SOURCE);
|
|
204
|
+
return true;
|
|
205
|
+
}
|
|
206
|
+
function parseDecisionEntries(content) {
|
|
207
|
+
const lines = content.replace(/\r\n/g, "\n").split("\n");
|
|
208
|
+
const entries = [];
|
|
209
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
210
|
+
const match = lines[index]?.match(/^###?\s+(.+)$/);
|
|
211
|
+
if (!match) {
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
214
|
+
const paragraph = [];
|
|
215
|
+
let cursor = index + 1;
|
|
216
|
+
while (cursor < lines.length && !lines[cursor]?.trim()) {
|
|
217
|
+
cursor += 1;
|
|
218
|
+
}
|
|
219
|
+
while (cursor < lines.length) {
|
|
220
|
+
const line = lines[cursor] ?? "";
|
|
221
|
+
if (/^###?\s+/.test(line)) {
|
|
222
|
+
break;
|
|
223
|
+
}
|
|
224
|
+
if (!line.trim()) {
|
|
225
|
+
if (paragraph.length > 0) {
|
|
226
|
+
break;
|
|
227
|
+
}
|
|
228
|
+
cursor += 1;
|
|
229
|
+
continue;
|
|
230
|
+
}
|
|
231
|
+
paragraph.push(line.trim());
|
|
232
|
+
cursor += 1;
|
|
233
|
+
}
|
|
234
|
+
entries.push({
|
|
235
|
+
title: match[1].trim(),
|
|
236
|
+
rationale: paragraph.join(" ").trim(),
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
return entries;
|
|
240
|
+
}
|
|
241
|
+
function contentHash(content) {
|
|
242
|
+
return createHash("sha256").update(content).digest("hex");
|
|
243
|
+
}
|
|
244
|
+
//# sourceMappingURL=migration.js.map
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import test from "node:test";
|
|
5
|
+
let sandboxRoot;
|
|
6
|
+
let chapterhouseHome;
|
|
7
|
+
let dbModule;
|
|
8
|
+
let migrationModule;
|
|
9
|
+
function resetSandbox() {
|
|
10
|
+
rmSync(chapterhouseHome, { recursive: true, force: true });
|
|
11
|
+
mkdirSync(chapterhouseHome, { recursive: true });
|
|
12
|
+
}
|
|
13
|
+
function writePage(relativePath, content) {
|
|
14
|
+
const fullPath = join(chapterhouseHome, "wiki", relativePath);
|
|
15
|
+
mkdirSync(join(fullPath, ".."), { recursive: true });
|
|
16
|
+
writeFileSync(fullPath, content, "utf-8");
|
|
17
|
+
}
|
|
18
|
+
test.before(async () => {
|
|
19
|
+
mkdirSync(join(process.cwd(), ".test-work"), { recursive: true });
|
|
20
|
+
sandboxRoot = mkdtempSync(join(process.cwd(), ".test-work", "memory-migration-"));
|
|
21
|
+
process.env.CHAPTERHOUSE_HOME = sandboxRoot;
|
|
22
|
+
chapterhouseHome = join(sandboxRoot, ".chapterhouse");
|
|
23
|
+
const nonce = `${Date.now()}-${Math.random()}`;
|
|
24
|
+
dbModule = await import(new URL(`../store/db.js?case=${nonce}`, import.meta.url).href);
|
|
25
|
+
migrationModule = await import(new URL(`./migration.js?case=${nonce}`, import.meta.url).href);
|
|
26
|
+
});
|
|
27
|
+
test.beforeEach(() => {
|
|
28
|
+
dbModule.closeDb();
|
|
29
|
+
resetSandbox();
|
|
30
|
+
});
|
|
31
|
+
test.after(() => {
|
|
32
|
+
try {
|
|
33
|
+
dbModule.closeDb();
|
|
34
|
+
}
|
|
35
|
+
catch { }
|
|
36
|
+
try {
|
|
37
|
+
rmSync(sandboxRoot, { recursive: true, force: true });
|
|
38
|
+
}
|
|
39
|
+
catch { }
|
|
40
|
+
});
|
|
41
|
+
test("runP6Migration seeds project, person, topic, and decision pages from the wiki", async () => {
|
|
42
|
+
const longTruth = `${"Compiled truth. ".repeat(60)}Detailed migration note.`;
|
|
43
|
+
writePage("pages/projects/project-x/index.md", `---\ntitle: Project X\nsummary: Flagship delivery program\ntags: [alpha, backend]\n---\n\n# Project X\n\n${longTruth}\n`);
|
|
44
|
+
writePage("pages/people/alice-example/index.md", `---\ntitle: Alice Example\nsummary: Principal engineer\ntags: [people]\n---\n\n# Alice Example\n\nAlice leads backend delivery.\n`);
|
|
45
|
+
writePage("pages/topics/observability/index.md", `---\ntitle: Observability\nsummary: Shared telemetry topic\ntags: [topic]\n---\n\n# Observability\n\nTracing, logs, and metrics.\n`);
|
|
46
|
+
writePage("pages/projects/chapterhouse/decisions.md", `# Decisions\n\n## Keep SQLite for memory\nSQLite keeps the daemon self-contained.\n\n### Append-only timelines\nTimeline history remains append-only for auditability.\n`);
|
|
47
|
+
const db = dbModule.getDb();
|
|
48
|
+
const result = await migrationModule.runP6Migration(db);
|
|
49
|
+
assert.deepEqual(result, {
|
|
50
|
+
entitiesCreated: 3,
|
|
51
|
+
observationsCreated: 3,
|
|
52
|
+
decisionsCreated: 2,
|
|
53
|
+
skipped: 0,
|
|
54
|
+
});
|
|
55
|
+
const entities = db.prepare(`SELECT slug, kind, name, summary FROM mem_entities ORDER BY slug`).all();
|
|
56
|
+
assert.deepEqual(entities.filter((row) => row.slug?.startsWith("projects/") || row.slug?.startsWith("people/") || row.slug?.startsWith("topics/")), [
|
|
57
|
+
{ slug: "people/alice-example/index.md", kind: "person", name: "Alice Example", summary: "Principal engineer" },
|
|
58
|
+
{ slug: "projects/project-x/index.md", kind: "project", name: "Project X", summary: "Flagship delivery program" },
|
|
59
|
+
{ slug: "topics/observability/index.md", kind: "topic", name: "Observability", summary: "Shared telemetry topic" },
|
|
60
|
+
]);
|
|
61
|
+
const observation = db.prepare(`SELECT content, source, tier FROM mem_observations WHERE source = 'migration:p6' ORDER BY id LIMIT 1`).get();
|
|
62
|
+
assert.equal(observation.source, "migration:p6");
|
|
63
|
+
assert.equal(observation.tier, "warm");
|
|
64
|
+
assert.equal(observation.content.length, 500);
|
|
65
|
+
assert.equal(observation.content, longTruth.slice(0, 500));
|
|
66
|
+
const chapterhouseScope = db.prepare(`SELECT id FROM mem_scopes WHERE slug = 'chapterhouse'`).get();
|
|
67
|
+
const decisions = db.prepare(`SELECT title, rationale, scope_id FROM mem_decisions WHERE title IN (?, ?) ORDER BY title`).all("Append-only timelines", "Keep SQLite for memory");
|
|
68
|
+
assert.deepEqual(decisions, [
|
|
69
|
+
{ title: "Append-only timelines", rationale: "Timeline history remains append-only for auditability.", scope_id: chapterhouseScope.id },
|
|
70
|
+
{ title: "Keep SQLite for memory", rationale: "SQLite keeps the daemon self-contained.", scope_id: chapterhouseScope.id },
|
|
71
|
+
]);
|
|
72
|
+
const migrationRow = db.prepare(`SELECT name FROM mem_migrations WHERE name = 'p6-wiki-seed'`).get();
|
|
73
|
+
assert.deepEqual(migrationRow, { name: "p6-wiki-seed" });
|
|
74
|
+
});
|
|
75
|
+
test("runP6Migration is idempotent across repeated runs", async () => {
|
|
76
|
+
writePage("pages/projects/project-x/index.md", `---\ntitle: Project X\nsummary: Flagship delivery program\ntags: []\n---\n\n# Project X\n\nCompiled truth lives here.\n`);
|
|
77
|
+
const db = dbModule.getDb();
|
|
78
|
+
const first = await migrationModule.runP6Migration(db);
|
|
79
|
+
const second = await migrationModule.runP6Migration(db);
|
|
80
|
+
assert.equal(first.entitiesCreated, 1);
|
|
81
|
+
assert.equal(first.observationsCreated, 1);
|
|
82
|
+
assert.equal(second.skipped, -1);
|
|
83
|
+
assert.equal(db.prepare(`SELECT COUNT(*) AS count FROM mem_entities WHERE slug = 'projects/project-x/index.md'`).get().count, 1);
|
|
84
|
+
assert.equal(db.prepare(`SELECT COUNT(*) AS count FROM mem_observations WHERE source = 'migration:p6'`).get().count, 1);
|
|
85
|
+
assert.equal(db.prepare(`SELECT COUNT(*) AS count FROM mem_migrations WHERE name = 'p6-wiki-seed'`).get().count, 1);
|
|
86
|
+
});
|
|
87
|
+
test("runP6Migration skips gracefully when the wiki pages directory is empty or absent", async () => {
|
|
88
|
+
const db = dbModule.getDb();
|
|
89
|
+
assert.equal(existsSync(join(chapterhouseHome, "wiki", "pages")), false);
|
|
90
|
+
const result = await migrationModule.runP6Migration(db);
|
|
91
|
+
assert.deepEqual(result, {
|
|
92
|
+
entitiesCreated: 0,
|
|
93
|
+
observationsCreated: 0,
|
|
94
|
+
decisionsCreated: 0,
|
|
95
|
+
skipped: 0,
|
|
96
|
+
});
|
|
97
|
+
assert.equal(db.prepare(`SELECT COUNT(*) AS count FROM mem_entities`).get().count >= 0, true);
|
|
98
|
+
assert.equal(db.prepare(`SELECT COUNT(*) AS count FROM mem_migrations WHERE name = 'p6-wiki-seed'`).get().count, 1);
|
|
99
|
+
});
|
|
100
|
+
//# sourceMappingURL=migration.test.js.map
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
import { config } from "../config.js";
|
|
2
|
+
import { getClient } from "../copilot/client.js";
|
|
3
|
+
import { runOneShotPrompt } from "../copilot/oneshot.js";
|
|
4
|
+
import { childLogger } from "../util/logger.js";
|
|
5
|
+
import { getScope, listScopes } from "./scopes.js";
|
|
6
|
+
const log = childLogger("memory.reflect");
|
|
7
|
+
const MIN_GROUP_SIZE = 3;
|
|
8
|
+
const SIMILARITY_THRESHOLD = 0.35;
|
|
9
|
+
const MAX_GROUP_OBSERVATIONS = 8;
|
|
10
|
+
const GLOBAL_SCOPE_SLUG = "global";
|
|
11
|
+
function normalizeText(value) {
|
|
12
|
+
return value
|
|
13
|
+
.toLowerCase()
|
|
14
|
+
.replace(/[^a-z0-9\s]/g, " ")
|
|
15
|
+
.replace(/\s+/g, " ")
|
|
16
|
+
.trim();
|
|
17
|
+
}
|
|
18
|
+
function tokenize(value) {
|
|
19
|
+
return new Set(normalizeText(value)
|
|
20
|
+
.split(" ")
|
|
21
|
+
.map((token) => token.replace(/s$/u, ""))
|
|
22
|
+
.filter((token) => token.length > 2));
|
|
23
|
+
}
|
|
24
|
+
function overlapCount(left, right) {
|
|
25
|
+
let overlap = 0;
|
|
26
|
+
for (const token of left) {
|
|
27
|
+
if (right.has(token)) {
|
|
28
|
+
overlap++;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
return overlap;
|
|
32
|
+
}
|
|
33
|
+
function jaccard(left, right) {
|
|
34
|
+
if (left.size === 0 || right.size === 0) {
|
|
35
|
+
return 0;
|
|
36
|
+
}
|
|
37
|
+
const overlap = overlapCount(left, right);
|
|
38
|
+
const union = left.size + right.size - overlap;
|
|
39
|
+
return union === 0 ? 0 : overlap / union;
|
|
40
|
+
}
|
|
41
|
+
function compareObservation(left, right) {
|
|
42
|
+
if (left.created_at !== right.created_at) {
|
|
43
|
+
return left.created_at.localeCompare(right.created_at);
|
|
44
|
+
}
|
|
45
|
+
return left.id - right.id;
|
|
46
|
+
}
|
|
47
|
+
function canonicalObservationIds(observations) {
|
|
48
|
+
return observations
|
|
49
|
+
.map((observation) => observation.id)
|
|
50
|
+
.sort((left, right) => left - right)
|
|
51
|
+
.join(",");
|
|
52
|
+
}
|
|
53
|
+
function buildEntityGroups(observations) {
|
|
54
|
+
const grouped = new Map();
|
|
55
|
+
for (const observation of observations) {
|
|
56
|
+
if (!observation.entity_id) {
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
const existing = grouped.get(observation.entity_id) ?? [];
|
|
60
|
+
existing.push(observation);
|
|
61
|
+
grouped.set(observation.entity_id, existing);
|
|
62
|
+
}
|
|
63
|
+
return [...grouped.entries()]
|
|
64
|
+
.map(([entityId, rows]) => ({
|
|
65
|
+
kind: "entity",
|
|
66
|
+
key: `entity:${entityId}`,
|
|
67
|
+
observations: rows.sort(compareObservation),
|
|
68
|
+
}))
|
|
69
|
+
.filter((group) => group.observations.length >= MIN_GROUP_SIZE);
|
|
70
|
+
}
|
|
71
|
+
function buildTopicGroups(observations) {
|
|
72
|
+
const tokensByObservation = new Map();
|
|
73
|
+
for (const observation of observations) {
|
|
74
|
+
tokensByObservation.set(observation.id, tokenize(observation.content));
|
|
75
|
+
}
|
|
76
|
+
const groups = [];
|
|
77
|
+
const seenKeys = new Set();
|
|
78
|
+
const ordered = [...observations].sort(compareObservation);
|
|
79
|
+
for (const seed of ordered) {
|
|
80
|
+
const seedTokens = tokensByObservation.get(seed.id) ?? new Set();
|
|
81
|
+
const cluster = ordered.filter((candidate) => {
|
|
82
|
+
const candidateTokens = tokensByObservation.get(candidate.id) ?? new Set();
|
|
83
|
+
return overlapCount(seedTokens, candidateTokens) >= 3
|
|
84
|
+
&& jaccard(seedTokens, candidateTokens) >= SIMILARITY_THRESHOLD;
|
|
85
|
+
});
|
|
86
|
+
if (cluster.length < MIN_GROUP_SIZE) {
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
const key = `topic:${canonicalObservationIds(cluster)}`;
|
|
90
|
+
if (seenKeys.has(key)) {
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
seenKeys.add(key);
|
|
94
|
+
groups.push({ kind: "topic", key, observations: cluster.sort(compareObservation) });
|
|
95
|
+
}
|
|
96
|
+
return groups;
|
|
97
|
+
}
|
|
98
|
+
function buildGroups(observations) {
|
|
99
|
+
const groups = [...buildEntityGroups(observations), ...buildTopicGroups(observations)];
|
|
100
|
+
const deduped = new Map();
|
|
101
|
+
for (const group of groups) {
|
|
102
|
+
const key = canonicalObservationIds(group.observations);
|
|
103
|
+
if (!deduped.has(key)) {
|
|
104
|
+
deduped.set(key, group);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return [...deduped.values()];
|
|
108
|
+
}
|
|
109
|
+
function buildFallbackPattern(group) {
|
|
110
|
+
const [first] = group.observations;
|
|
111
|
+
const entityTitle = first?.entity_name ? `${first.entity_name} pattern` : "Observed pattern";
|
|
112
|
+
return {
|
|
113
|
+
title: entityTitle,
|
|
114
|
+
summary: group.observations.slice(0, 3).map((observation) => observation.content).join(" "),
|
|
115
|
+
confidence: Math.max(0.5, Math.min(0.95, Number((group.observations.reduce((sum, observation) => sum + observation.confidence, 0) / group.observations.length).toFixed(2)))),
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
function parseSynthesizedPattern(raw, group) {
|
|
119
|
+
try {
|
|
120
|
+
const parsed = JSON.parse(raw);
|
|
121
|
+
if (typeof parsed.title === "string" && typeof parsed.summary === "string") {
|
|
122
|
+
return {
|
|
123
|
+
title: parsed.title.trim() || buildFallbackPattern(group).title,
|
|
124
|
+
summary: parsed.summary.trim() || buildFallbackPattern(group).summary,
|
|
125
|
+
confidence: typeof parsed.confidence === "number"
|
|
126
|
+
? Math.max(0, Math.min(1, parsed.confidence))
|
|
127
|
+
: buildFallbackPattern(group).confidence,
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
catch {
|
|
132
|
+
// Fall through to the heuristic fallback.
|
|
133
|
+
}
|
|
134
|
+
return buildFallbackPattern(group);
|
|
135
|
+
}
|
|
136
|
+
async function synthesizePattern(scopeSlug, group) {
|
|
137
|
+
const client = await getClient();
|
|
138
|
+
const system = [
|
|
139
|
+
"You synthesize durable memory patterns from repeated observations.",
|
|
140
|
+
"Return JSON only with keys: title, summary, confidence.",
|
|
141
|
+
"title should be short and specific.",
|
|
142
|
+
"summary should describe the stable pattern in 1-2 sentences.",
|
|
143
|
+
"confidence must be a number between 0 and 1.",
|
|
144
|
+
].join("\n");
|
|
145
|
+
const user = JSON.stringify({
|
|
146
|
+
scope: scopeSlug,
|
|
147
|
+
group_kind: group.kind,
|
|
148
|
+
observations: group.observations.slice(0, MAX_GROUP_OBSERVATIONS).map((observation) => ({
|
|
149
|
+
id: observation.id,
|
|
150
|
+
scope: observation.scope_slug,
|
|
151
|
+
entity: observation.entity_name,
|
|
152
|
+
content: observation.content,
|
|
153
|
+
source: observation.source,
|
|
154
|
+
confidence: observation.confidence,
|
|
155
|
+
created_at: observation.created_at,
|
|
156
|
+
})),
|
|
157
|
+
}, null, 2);
|
|
158
|
+
const response = await runOneShotPrompt({
|
|
159
|
+
client,
|
|
160
|
+
model: config.copilotModel,
|
|
161
|
+
system,
|
|
162
|
+
user,
|
|
163
|
+
expectJson: true,
|
|
164
|
+
});
|
|
165
|
+
return parseSynthesizedPattern(response.content, group);
|
|
166
|
+
}
|
|
167
|
+
function containsContradictionSignal(value) {
|
|
168
|
+
return /\b(?:no longer|changed to|now uses|now use|moved to|switched to|was .+ now .+)\b/i.test(value);
|
|
169
|
+
}
|
|
170
|
+
function countContradictions(groups) {
|
|
171
|
+
let contradictions = 0;
|
|
172
|
+
for (const group of groups) {
|
|
173
|
+
if (group.kind !== "entity") {
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
if (group.observations.some((observation) => containsContradictionSignal(observation.content))) {
|
|
177
|
+
contradictions++;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
return contradictions;
|
|
181
|
+
}
|
|
182
|
+
function loadObservations(scopeSlug, db) {
|
|
183
|
+
const scope = getScope(scopeSlug);
|
|
184
|
+
if (!scope) {
|
|
185
|
+
throw new Error(`Unknown memory scope '${scopeSlug}'.`);
|
|
186
|
+
}
|
|
187
|
+
const globalScope = scopeSlug === GLOBAL_SCOPE_SLUG ? null : getScope(GLOBAL_SCOPE_SLUG);
|
|
188
|
+
const scopeIds = globalScope ? [scope.id, globalScope.id] : [scope.id];
|
|
189
|
+
const placeholders = scopeIds.map(() => "?").join(", ");
|
|
190
|
+
return db.prepare(`
|
|
191
|
+
SELECT
|
|
192
|
+
o.id,
|
|
193
|
+
o.scope_id,
|
|
194
|
+
s.slug AS scope_slug,
|
|
195
|
+
o.entity_id,
|
|
196
|
+
e.name AS entity_name,
|
|
197
|
+
e.kind AS entity_kind,
|
|
198
|
+
o.content,
|
|
199
|
+
o.source,
|
|
200
|
+
o.tier,
|
|
201
|
+
o.confidence,
|
|
202
|
+
o.created_at
|
|
203
|
+
FROM mem_observations o
|
|
204
|
+
JOIN mem_scopes s ON s.id = o.scope_id
|
|
205
|
+
LEFT JOIN mem_entities e ON e.id = o.entity_id
|
|
206
|
+
WHERE o.scope_id IN (${placeholders})
|
|
207
|
+
AND o.tier IN ('hot', 'warm')
|
|
208
|
+
AND o.superseded_by IS NULL
|
|
209
|
+
AND o.archived_at IS NULL
|
|
210
|
+
ORDER BY o.created_at ASC, o.id ASC
|
|
211
|
+
`).all(...scopeIds);
|
|
212
|
+
}
|
|
213
|
+
function upsertPattern(db, scopeId, synthesized, observations) {
|
|
214
|
+
const sourceObservationIds = JSON.stringify(observations.map((observation) => observation.id));
|
|
215
|
+
const existing = db.prepare(`
|
|
216
|
+
SELECT id, title, source_observation_ids
|
|
217
|
+
FROM mem_patterns
|
|
218
|
+
WHERE scope_id = ? AND title = ?
|
|
219
|
+
ORDER BY id DESC
|
|
220
|
+
LIMIT 1
|
|
221
|
+
`).get(scopeId, synthesized.title);
|
|
222
|
+
if (existing) {
|
|
223
|
+
db.prepare(`
|
|
224
|
+
UPDATE mem_patterns
|
|
225
|
+
SET summary = ?,
|
|
226
|
+
source_observation_ids = ?,
|
|
227
|
+
confidence = ?,
|
|
228
|
+
tier = 'warm',
|
|
229
|
+
last_updated = CURRENT_TIMESTAMP
|
|
230
|
+
WHERE id = ?
|
|
231
|
+
`).run(synthesized.summary, sourceObservationIds, synthesized.confidence, existing.id);
|
|
232
|
+
return "updated";
|
|
233
|
+
}
|
|
234
|
+
db.prepare(`
|
|
235
|
+
INSERT INTO mem_patterns (
|
|
236
|
+
scope_id, title, summary, source_observation_ids, confidence, tier, created_at, last_updated
|
|
237
|
+
)
|
|
238
|
+
VALUES (?, ?, ?, ?, ?, 'warm', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
239
|
+
`).run(scopeId, synthesized.title, synthesized.summary, sourceObservationIds, synthesized.confidence);
|
|
240
|
+
return "created";
|
|
241
|
+
}
|
|
242
|
+
export async function reflectOnScope(scopeSlug, db) {
|
|
243
|
+
const scope = getScope(scopeSlug);
|
|
244
|
+
if (!scope) {
|
|
245
|
+
throw new Error(`Unknown memory scope '${scopeSlug}'.`);
|
|
246
|
+
}
|
|
247
|
+
const observations = loadObservations(scopeSlug, db);
|
|
248
|
+
const groups = buildGroups(observations);
|
|
249
|
+
const contradictionsFound = countContradictions(groups);
|
|
250
|
+
let patternsCreated = 0;
|
|
251
|
+
let patternsUpdated = 0;
|
|
252
|
+
for (const group of groups) {
|
|
253
|
+
const synthesized = await synthesizePattern(scopeSlug, group);
|
|
254
|
+
const outcome = upsertPattern(db, scope.id, synthesized, group.observations);
|
|
255
|
+
if (outcome === "created") {
|
|
256
|
+
patternsCreated++;
|
|
257
|
+
}
|
|
258
|
+
else {
|
|
259
|
+
patternsUpdated++;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
const result = { patternsCreated, patternsUpdated, contradictionsFound };
|
|
263
|
+
log.info({ scope: scopeSlug, ...result }, "memory.reflect.scope.complete");
|
|
264
|
+
return result;
|
|
265
|
+
}
|
|
266
|
+
export async function reflectAllScopes(db) {
|
|
267
|
+
const results = {};
|
|
268
|
+
for (const scope of listScopes().filter((entry) => entry.active)) {
|
|
269
|
+
results[scope.slug] = await reflectOnScope(scope.slug, db);
|
|
270
|
+
}
|
|
271
|
+
return results;
|
|
272
|
+
}
|
|
273
|
+
//# sourceMappingURL=reflect.js.map
|