obsidian-second-brain 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/config/projects.example.json +10 -0
- package/dist/claude-runner.js +34 -0
- package/dist/cli-claude-backend.js +2 -18
- package/dist/cli.js +72 -22
- package/dist/code-boundaries.js +154 -0
- package/dist/code-capture.js +122 -0
- package/dist/code-classify.js +150 -0
- package/dist/code-evidence.js +113 -0
- package/dist/code-frontmatter.js +147 -0
- package/dist/code-ingest.js +289 -0
- package/dist/code-map-backend.js +70 -0
- package/dist/code-map-config.js +0 -0
- package/dist/code-map-prompt.js +84 -0
- package/dist/code-scan.js +168 -0
- package/dist/config.js +5 -0
- package/dist/consolidation.js +1 -1
- package/dist/managed-section.js +40 -19
- package/dist/map-command.js +191 -0
- package/dist/report.js +34 -0
- package/dist/shell.js +55 -1
- package/dist/vault-paths.js +23 -0
- package/package.json +2 -2
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
import { readdir, readFile, rm } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { compareBytewise } from "./code-boundaries.js";
|
|
4
|
+
import { scanCodeEvidence } from "./code-classify.js";
|
|
5
|
+
import { renderCodePage, tryParseCodePage } from "./code-frontmatter.js";
|
|
6
|
+
import { ensureDirectory, writeFileAtomic } from "./filesystem.js";
|
|
7
|
+
import { appendLogEntry } from "./ingest-writer.js";
|
|
8
|
+
import { applyManagedBlock, CODE_MAPS_MARKERS } from "./managed-section.js";
|
|
9
|
+
import { loadManifest, saveManifest, withManifestEntries } from "./manifest.js";
|
|
10
|
+
export async function runCodeMapIngest(input) {
|
|
11
|
+
const scan = await scanCodeEvidence(input.rawCodeRoot);
|
|
12
|
+
const manifest = await loadManifest(input.manifestPath);
|
|
13
|
+
const captureFailures = input.captureFailures ?? [];
|
|
14
|
+
const freshQuarantined = scan.quarantined.filter((entry) => manifest.entries[quarantineKey(entry)] === undefined);
|
|
15
|
+
const byProject = groupByProject(scan.sources);
|
|
16
|
+
const failures = [...captureFailures];
|
|
17
|
+
const pagesRendered = [];
|
|
18
|
+
const pagesPruned = [];
|
|
19
|
+
const advancedEntries = {};
|
|
20
|
+
const now = input.now();
|
|
21
|
+
const date = now.toISOString().slice(0, 10);
|
|
22
|
+
const ingestedAt = now.toISOString();
|
|
23
|
+
// PLANNING — pure reads; decides per project whether any work exists.
|
|
24
|
+
const plans = new Map();
|
|
25
|
+
for (const [project, evidence] of byProject) {
|
|
26
|
+
try {
|
|
27
|
+
plans.set(project, await planProject(input, manifest, project, evidence));
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
failures.push({
|
|
31
|
+
project,
|
|
32
|
+
reason: error instanceof Error ? error.message : "Unknown error"
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
const hasRenderWork = [...plans.values()].some((plan) => plan.renderModules.length > 0 || plan.renderOverview || plan.prunePages.length > 0);
|
|
37
|
+
if (!hasRenderWork &&
|
|
38
|
+
failures.length === 0 &&
|
|
39
|
+
freshQuarantined.length === 0 &&
|
|
40
|
+
input.evidenceCaptured === 0) {
|
|
41
|
+
return {
|
|
42
|
+
failures: [],
|
|
43
|
+
pagesPruned: [],
|
|
44
|
+
pagesRendered: [],
|
|
45
|
+
projects: [...byProject.keys()],
|
|
46
|
+
quarantined: [],
|
|
47
|
+
skipped: true
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
// RENDER — per-project isolation: a throw records a failure and skips the
|
|
51
|
+
// manifest advance for THAT project only (design 10).
|
|
52
|
+
for (const [project, plan] of [...plans.entries()].sort((a, b) => compareBytewise(a[0], b[0]))) {
|
|
53
|
+
try {
|
|
54
|
+
const rendered = await renderProject(input, project, plan, date);
|
|
55
|
+
pagesRendered.push(...rendered.pagesRendered);
|
|
56
|
+
pagesPruned.push(...rendered.pagesPruned);
|
|
57
|
+
for (const source of [
|
|
58
|
+
...(plan.evidence.skeleton === undefined ? [] : [plan.evidence.skeleton]),
|
|
59
|
+
...plan.evidence.modules
|
|
60
|
+
]) {
|
|
61
|
+
advancedEntries[source.key] = {
|
|
62
|
+
contentHash: source.bodyHash,
|
|
63
|
+
ingestedAt
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
failures.push({
|
|
69
|
+
project,
|
|
70
|
+
reason: error instanceof Error ? error.message : "Unknown error"
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
// BOOKKEEPING — pages (index) -> report -> log -> manifest LAST.
|
|
75
|
+
await updateIndex(input.indexPath, byProject, failures);
|
|
76
|
+
let report;
|
|
77
|
+
if (input.reportWriter) {
|
|
78
|
+
report = await input.reportWriter.addCodeMap({
|
|
79
|
+
evidenceCaptured: input.evidenceCaptured,
|
|
80
|
+
failures,
|
|
81
|
+
pagesPruned,
|
|
82
|
+
pagesRendered,
|
|
83
|
+
quarantined: freshQuarantined
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
if (pagesRendered.length > 0 ||
|
|
87
|
+
pagesPruned.length > 0 ||
|
|
88
|
+
failures.length > 0 ||
|
|
89
|
+
freshQuarantined.length > 0 ||
|
|
90
|
+
report !== undefined) {
|
|
91
|
+
await appendLogEntry(input.logPath, buildLogEntry({ date, failures, pagesPruned, pagesRendered, quarantined: freshQuarantined, reportLink: report?.wikiLink }));
|
|
92
|
+
}
|
|
93
|
+
const quarantineEntries = Object.fromEntries(freshQuarantined.map((entry) => [
|
|
94
|
+
quarantineKey(entry),
|
|
95
|
+
{ contentHash: entry.contentHash, ingestedAt }
|
|
96
|
+
]));
|
|
97
|
+
await saveManifest(input.manifestPath, withManifestEntries(manifest, { ...advancedEntries, ...quarantineEntries }));
|
|
98
|
+
return {
|
|
99
|
+
failures,
|
|
100
|
+
pagesPruned,
|
|
101
|
+
pagesRendered,
|
|
102
|
+
projects: [...byProject.keys()],
|
|
103
|
+
quarantined: freshQuarantined,
|
|
104
|
+
...(report ? { reportPath: report.path } : {}),
|
|
105
|
+
skipped: false
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
function quarantineKey(entry) {
|
|
109
|
+
return `quarantine:code:${entry.contentHash}`;
|
|
110
|
+
}
|
|
111
|
+
function groupByProject(sources) {
|
|
112
|
+
const projects = new Map();
|
|
113
|
+
for (const source of sources) {
|
|
114
|
+
const entry = projects.get(source.project) ?? { modules: [] };
|
|
115
|
+
if (source.kind === "skeleton") {
|
|
116
|
+
entry.skeleton = source;
|
|
117
|
+
}
|
|
118
|
+
else {
|
|
119
|
+
entry.modules.push(source);
|
|
120
|
+
}
|
|
121
|
+
projects.set(source.project, entry);
|
|
122
|
+
}
|
|
123
|
+
for (const entry of projects.values()) {
|
|
124
|
+
entry.modules.sort((a, b) => compareBytewise(a.slug ?? "", b.slug ?? ""));
|
|
125
|
+
}
|
|
126
|
+
return new Map([...projects.entries()].sort((a, b) => compareBytewise(a[0], b[0])));
|
|
127
|
+
}
|
|
128
|
+
async function planProject(input, manifest, project, evidence) {
|
|
129
|
+
if (evidence.skeleton === undefined) {
|
|
130
|
+
throw new Error("Missing skeleton evidence — re-run `second-brain map`");
|
|
131
|
+
}
|
|
132
|
+
const projectDir = join(input.wikiCodeRoot, project);
|
|
133
|
+
const existingPages = await listMarkdownFiles(projectDir);
|
|
134
|
+
const parsedPages = new Map();
|
|
135
|
+
for (const page of existingPages) {
|
|
136
|
+
const content = await readFile(join(projectDir, page), "utf8");
|
|
137
|
+
parsedPages.set(page, tryParseCodePage(content) !== undefined);
|
|
138
|
+
}
|
|
139
|
+
// Ownership (design 4): a non-empty directory where NOTHING carries our
|
|
140
|
+
// generated frontmatter was not produced by this tool.
|
|
141
|
+
if (existingPages.length > 0 && ![...parsedPages.values()].some(Boolean)) {
|
|
142
|
+
throw new Error(`wiki/code/${project}/ exists but was not produced by this tool — move or remove it`);
|
|
143
|
+
}
|
|
144
|
+
const desired = new Set([
|
|
145
|
+
"overview.md",
|
|
146
|
+
...evidence.modules.map((source) => `${source.slug ?? ""}.md`)
|
|
147
|
+
]);
|
|
148
|
+
const changed = (source) => manifest.entries[source.key]?.contentHash !== source.bodyHash;
|
|
149
|
+
// Render work (design 6.3): changed digest, missing page, or unparseable
|
|
150
|
+
// page (self-healing).
|
|
151
|
+
const renderModules = evidence.modules.filter((source) => {
|
|
152
|
+
const pageName = `${source.slug ?? ""}.md`;
|
|
153
|
+
return (changed(source) ||
|
|
154
|
+
!parsedPages.has(pageName) ||
|
|
155
|
+
parsedPages.get(pageName) === false);
|
|
156
|
+
});
|
|
157
|
+
// Prune only OUR pages outside the desired set; foreign files in a mixed
|
|
158
|
+
// directory are left untouched (design 4).
|
|
159
|
+
const prunePages = existingPages.filter((page) => !desired.has(page) && parsedPages.get(page) === true);
|
|
160
|
+
const overviewMissing = !parsedPages.has("overview.md") || parsedPages.get("overview.md") === false;
|
|
161
|
+
const renderOverview = changed(evidence.skeleton) ||
|
|
162
|
+
overviewMissing ||
|
|
163
|
+
renderModules.length > 0 ||
|
|
164
|
+
prunePages.length > 0;
|
|
165
|
+
return { evidence, prunePages, renderModules, renderOverview };
|
|
166
|
+
}
|
|
167
|
+
async function renderProject(input, project, plan, date) {
|
|
168
|
+
const projectDir = join(input.wikiCodeRoot, project);
|
|
169
|
+
const pagesRendered = [];
|
|
170
|
+
const pagesPruned = [];
|
|
171
|
+
if (plan.renderModules.length > 0 || plan.renderOverview) {
|
|
172
|
+
await ensureDirectory(projectDir);
|
|
173
|
+
}
|
|
174
|
+
for (const source of plan.renderModules) {
|
|
175
|
+
const slug = source.slug ?? "";
|
|
176
|
+
const coverage = source.frontmatter.coverage;
|
|
177
|
+
const body = await input.backend.renderModulePage({
|
|
178
|
+
...(coverage === undefined ? {} : { coverage }),
|
|
179
|
+
digestBody: source.body,
|
|
180
|
+
module: slug,
|
|
181
|
+
project
|
|
182
|
+
});
|
|
183
|
+
const frontmatter = {
|
|
184
|
+
...(coverage === undefined ? {} : { coverage }),
|
|
185
|
+
generated: date,
|
|
186
|
+
module: slug,
|
|
187
|
+
project,
|
|
188
|
+
source_commit: source.frontmatter.commit,
|
|
189
|
+
type: "code-map"
|
|
190
|
+
};
|
|
191
|
+
await writeFileAtomic(join(projectDir, `${slug}.md`), renderCodePage(frontmatter, body));
|
|
192
|
+
pagesRendered.push(`${project}/${slug}`);
|
|
193
|
+
}
|
|
194
|
+
for (const page of plan.prunePages) {
|
|
195
|
+
await rm(join(projectDir, page), { force: true });
|
|
196
|
+
pagesPruned.push(`${project}/${page.replace(/\.md$/u, "")}`);
|
|
197
|
+
}
|
|
198
|
+
if (plan.renderOverview) {
|
|
199
|
+
const skeleton = plan.evidence.skeleton;
|
|
200
|
+
if (skeleton === undefined) {
|
|
201
|
+
throw new Error("Missing skeleton evidence — re-run `second-brain map`");
|
|
202
|
+
}
|
|
203
|
+
// Overview input = current module PAGE bodies (summaries of summaries),
|
|
204
|
+
// never the raw digests again (design 6.4).
|
|
205
|
+
const modulePages = [];
|
|
206
|
+
for (const source of plan.evidence.modules) {
|
|
207
|
+
const slug = source.slug ?? "";
|
|
208
|
+
const content = await readFile(join(projectDir, `${slug}.md`), "utf8");
|
|
209
|
+
const parsed = tryParseCodePage(content);
|
|
210
|
+
if (parsed === undefined) {
|
|
211
|
+
throw new Error(`Module page ${slug}.md unreadable after render`);
|
|
212
|
+
}
|
|
213
|
+
modulePages.push({ body: parsed.body, slug });
|
|
214
|
+
}
|
|
215
|
+
const body = await input.backend.renderOverview({
|
|
216
|
+
modulePages,
|
|
217
|
+
project,
|
|
218
|
+
skeletonBody: skeleton.body
|
|
219
|
+
});
|
|
220
|
+
const frontmatter = {
|
|
221
|
+
generated: date,
|
|
222
|
+
project,
|
|
223
|
+
source_commit: skeleton.frontmatter.commit,
|
|
224
|
+
type: "code-map"
|
|
225
|
+
};
|
|
226
|
+
await writeFileAtomic(join(projectDir, "overview.md"), renderCodePage(frontmatter, body));
|
|
227
|
+
pagesRendered.push(`${project}/overview`);
|
|
228
|
+
}
|
|
229
|
+
return { pagesPruned, pagesRendered };
|
|
230
|
+
}
|
|
231
|
+
async function updateIndex(indexPath, byProject, failures) {
|
|
232
|
+
const failedProjects = new Set(failures.map((failure) => failure.project));
|
|
233
|
+
const lines = ["## Code maps", ""];
|
|
234
|
+
for (const [project, evidence] of byProject) {
|
|
235
|
+
const suffix = failedProjects.has(project) ? " (last run failed)" : "";
|
|
236
|
+
lines.push(`- [[code/${project}/overview|${project}]] — ${evidence.modules.length} modules${suffix}`);
|
|
237
|
+
}
|
|
238
|
+
let existing;
|
|
239
|
+
try {
|
|
240
|
+
existing = await readFile(indexPath, "utf8");
|
|
241
|
+
}
|
|
242
|
+
catch (error) {
|
|
243
|
+
if (!isMissingFileError(error)) {
|
|
244
|
+
throw error;
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
const next = applyManagedBlock(existing, CODE_MAPS_MARKERS, lines.join("\n"));
|
|
248
|
+
if (next !== existing) {
|
|
249
|
+
await writeFileAtomic(indexPath, next);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
function buildLogEntry(input) {
|
|
253
|
+
const lines = [`## [${input.date}] code map | hourly pages`];
|
|
254
|
+
if (input.reportLink !== undefined) {
|
|
255
|
+
lines.push(`Run report: [[${input.reportLink}]]`);
|
|
256
|
+
}
|
|
257
|
+
if (input.pagesRendered.length > 0) {
|
|
258
|
+
lines.push(`Pages rendered: ${input.pagesRendered.map((page) => `[[code/${page}]]`).join(", ")}`);
|
|
259
|
+
}
|
|
260
|
+
if (input.pagesPruned.length > 0) {
|
|
261
|
+
lines.push(`Pages pruned: ${input.pagesPruned.map((page) => `\`code/${page}\``).join(", ")}`);
|
|
262
|
+
}
|
|
263
|
+
for (const failure of input.failures) {
|
|
264
|
+
lines.push(`Failed project (manifest not advanced): ${failure.project} — ${failure.reason}`);
|
|
265
|
+
}
|
|
266
|
+
for (const entry of input.quarantined) {
|
|
267
|
+
lines.push(`Quarantined evidence: ${entry.path} — ${entry.reason}`);
|
|
268
|
+
}
|
|
269
|
+
return lines.join("\n");
|
|
270
|
+
}
|
|
271
|
+
async function listMarkdownFiles(directory) {
|
|
272
|
+
try {
|
|
273
|
+
return (await readdir(directory))
|
|
274
|
+
.filter((entry) => entry.endsWith(".md"))
|
|
275
|
+
.sort(compareBytewise);
|
|
276
|
+
}
|
|
277
|
+
catch (error) {
|
|
278
|
+
if (isMissingFileError(error)) {
|
|
279
|
+
return [];
|
|
280
|
+
}
|
|
281
|
+
throw error;
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
function isMissingFileError(error) {
|
|
285
|
+
return (error instanceof Error &&
|
|
286
|
+
"code" in error &&
|
|
287
|
+
typeof error.code === "string" &&
|
|
288
|
+
error.code === "ENOENT");
|
|
289
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { join } from "node:path";
|
|
2
|
+
import { createClaudeRunner } from "./claude-runner.js";
|
|
3
|
+
import { buildModulePagePrompt, buildOverviewPrompt } from "./code-map-prompt.js";
|
|
4
|
+
import { ensureDirectory, writeFileAtomic } from "./filesystem.js";
|
|
5
|
+
export function createCliClaudeCodeMapBackend(options, dependencies = {}) {
|
|
6
|
+
const runner = dependencies.runner ??
|
|
7
|
+
createClaudeRunner({
|
|
8
|
+
...(options.claudeBin === undefined ? {} : { claudeBin: options.claudeBin }),
|
|
9
|
+
delivery: "stdin"
|
|
10
|
+
});
|
|
11
|
+
return {
|
|
12
|
+
name: "cli-claude",
|
|
13
|
+
renderModulePage: async (input) => {
|
|
14
|
+
const prompt = buildModulePagePrompt(input);
|
|
15
|
+
const raw = await runner({ model: options.models.pages, prompt });
|
|
16
|
+
return validateOrPreserve(raw, `${input.project}-${input.module}`, options.logsRoot);
|
|
17
|
+
},
|
|
18
|
+
renderOverview: async (input) => {
|
|
19
|
+
const prompt = buildOverviewPrompt(input);
|
|
20
|
+
const raw = await runner({ model: options.models.structure, prompt });
|
|
21
|
+
return validateOrPreserve(raw, `${input.project}-overview`, options.logsRoot);
|
|
22
|
+
}
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
// Failed output is debugging evidence, not garbage (design 10): preserve it
|
|
26
|
+
// under logsRoot, then fail so the manifest does not advance.
|
|
27
|
+
async function validateOrPreserve(raw, label, logsRoot) {
|
|
28
|
+
try {
|
|
29
|
+
return validatePageBody(raw);
|
|
30
|
+
}
|
|
31
|
+
catch (error) {
|
|
32
|
+
if (logsRoot === undefined) {
|
|
33
|
+
throw error;
|
|
34
|
+
}
|
|
35
|
+
const stamp = new Date().toISOString().replaceAll(":", "-");
|
|
36
|
+
const dumpPath = join(logsRoot, `code-map-${label}-${stamp}.txt`);
|
|
37
|
+
await ensureDirectory(logsRoot);
|
|
38
|
+
await writeFileAtomic(dumpPath, raw);
|
|
39
|
+
throw new Error(`${error instanceof Error ? error.message : "Invalid page body"} — raw output saved to ${dumpPath}`);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
// Light validation (design 4): hard structural failures only — content
|
|
43
|
+
// quality is the prompt's job, and a strict gate would just loop retries.
|
|
44
|
+
export function validatePageBody(raw) {
|
|
45
|
+
const body = stripFences(raw).trim();
|
|
46
|
+
if (body.length === 0) {
|
|
47
|
+
throw new Error("Backend returned an empty page body");
|
|
48
|
+
}
|
|
49
|
+
if (body.startsWith("---")) {
|
|
50
|
+
throw new Error("Backend returned frontmatter — page bodies must be bare markdown");
|
|
51
|
+
}
|
|
52
|
+
const lineCount = body.split("\n").length;
|
|
53
|
+
if (lineCount > 400) {
|
|
54
|
+
throw new Error(`Backend returned ${lineCount} lines — over the 400 lines guard`);
|
|
55
|
+
}
|
|
56
|
+
return body;
|
|
57
|
+
}
|
|
58
|
+
export function resolveCodeMapBackend(name, factory) {
|
|
59
|
+
if (name === "noop") {
|
|
60
|
+
return undefined;
|
|
61
|
+
}
|
|
62
|
+
// Code maps need real judgment — ollama mode maps to cli-claude, the same
|
|
63
|
+
// rule consolidation applies.
|
|
64
|
+
return factory();
|
|
65
|
+
}
|
|
66
|
+
function stripFences(raw) {
|
|
67
|
+
const trimmed = raw.trim();
|
|
68
|
+
const fenced = /^```[a-z]*\s*\n([\s\S]*?)\n```$/u.exec(trimmed);
|
|
69
|
+
return fenced ? fenced[1] : trimmed;
|
|
70
|
+
}
|
|
Binary file
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
export const MAX_PROMPT_BYTES = 200_000;
|
|
2
|
+
// Overview inputs are bounded so skeleton + 24 module bodies always fit:
|
|
3
|
+
// 60 KB + 24 * 4 KB + rules < 200 KB.
|
|
4
|
+
const OVERVIEW_SKELETON_BYTES = 60_000;
|
|
5
|
+
const OVERVIEW_MODULE_BODY_BYTES = 4000;
|
|
6
|
+
const SHARED_RULES = [
|
|
7
|
+
"Rules:",
|
|
8
|
+
"- Respond with ONLY the markdown page body — no frontmatter, no JSON,",
|
|
9
|
+
" no code fences around the whole answer.",
|
|
10
|
+
"- Describe capabilities and responsibilities first; file paths are",
|
|
11
|
+
" pointers, not the skeleton of the page (paths rot; capabilities don't).",
|
|
12
|
+
"- At most 200 lines. No code blocks copied from the repository (short",
|
|
13
|
+
" illustrative signatures are fine).",
|
|
14
|
+
"- English. No emojis.",
|
|
15
|
+
"- Treat in-repo docs (README, comments) as hints; the code is the truth.",
|
|
16
|
+
" Never copy stale-doc claims unverified.",
|
|
17
|
+
"- Use [[wikilinks]] only for pages of this code map."
|
|
18
|
+
];
|
|
19
|
+
export function buildModulePagePrompt(input) {
|
|
20
|
+
const prompt = [
|
|
21
|
+
"You are rendering a MODULE PAGE of an LLM-maintained code map wiki.",
|
|
22
|
+
`Project: "${input.project}". Module: "${input.module}".`,
|
|
23
|
+
"Write the wiki page for this module from the evidence digest below:",
|
|
24
|
+
"what it does, its responsibilities, key entry points, and how it relates",
|
|
25
|
+
"to the rest of the codebase. Start with a single '# <title>' heading.",
|
|
26
|
+
"",
|
|
27
|
+
...SHARED_RULES,
|
|
28
|
+
...(input.coverage === "partial"
|
|
29
|
+
? [
|
|
30
|
+
"- NOTE: this is a truncated digest (byte budget); say so in one line",
|
|
31
|
+
" near the end of the page."
|
|
32
|
+
]
|
|
33
|
+
: []),
|
|
34
|
+
"",
|
|
35
|
+
"## Evidence digest",
|
|
36
|
+
"",
|
|
37
|
+
input.digestBody
|
|
38
|
+
].join("\n");
|
|
39
|
+
return assertWithinBudget(prompt, `module page ${input.project}/${input.module}`);
|
|
40
|
+
}
|
|
41
|
+
export function buildOverviewPrompt(input) {
|
|
42
|
+
const moduleSections = input.modulePages.map((page) => [
|
|
43
|
+
`### Module page: ${page.slug}`,
|
|
44
|
+
"",
|
|
45
|
+
truncateBytes(page.body, OVERVIEW_MODULE_BODY_BYTES)
|
|
46
|
+
].join("\n"));
|
|
47
|
+
const prompt = [
|
|
48
|
+
"You are rendering the OVERVIEW PAGE of an LLM-maintained code map wiki.",
|
|
49
|
+
`Project: "${input.project}".`,
|
|
50
|
+
"Write the architecture overview from the repository skeleton and the",
|
|
51
|
+
"module pages below: what the system is, its major components and how",
|
|
52
|
+
"they interact, the tech stack, entry points, and conventions. Include",
|
|
53
|
+
"exactly ONE Mermaid diagram (graph TD) of the major components. Link",
|
|
54
|
+
"each module page once as [[<slug>]]. Start with a single '# <title>'",
|
|
55
|
+
"heading.",
|
|
56
|
+
"",
|
|
57
|
+
...SHARED_RULES,
|
|
58
|
+
"",
|
|
59
|
+
"## Repository skeleton",
|
|
60
|
+
"",
|
|
61
|
+
truncateBytes(input.skeletonBody, OVERVIEW_SKELETON_BYTES),
|
|
62
|
+
"",
|
|
63
|
+
"## Module pages",
|
|
64
|
+
"",
|
|
65
|
+
moduleSections.join("\n\n---\n\n")
|
|
66
|
+
].join("\n");
|
|
67
|
+
return assertWithinBudget(prompt, `overview ${input.project}`);
|
|
68
|
+
}
|
|
69
|
+
function truncateBytes(text, maxBytes) {
|
|
70
|
+
if (Buffer.byteLength(text, "utf8") <= maxBytes) {
|
|
71
|
+
return text;
|
|
72
|
+
}
|
|
73
|
+
// Slicing by chars can overshoot multi-byte runes slightly; Buffer slice
|
|
74
|
+
// then lossy decode keeps it deterministic and within budget.
|
|
75
|
+
const sliced = Buffer.from(text, "utf8").subarray(0, maxBytes).toString("utf8");
|
|
76
|
+
return `${sliced.replace(/�+$/u, "")}\n[truncated]`;
|
|
77
|
+
}
|
|
78
|
+
function assertWithinBudget(prompt, label) {
|
|
79
|
+
const bytes = Buffer.byteLength(prompt, "utf8");
|
|
80
|
+
if (bytes > MAX_PROMPT_BYTES) {
|
|
81
|
+
throw new Error(`Prompt for ${label} exceeds ${MAX_PROMPT_BYTES} bytes (${bytes})`);
|
|
82
|
+
}
|
|
83
|
+
return prompt;
|
|
84
|
+
}
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import { lstat, readFile } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { compareBytewise } from "./code-boundaries.js";
|
|
4
|
+
import { runCommand } from "./shell.js";
|
|
5
|
+
const defaultDependencies = {
|
|
6
|
+
fileSize: async (absolutePath) => {
|
|
7
|
+
try {
|
|
8
|
+
const stat = await lstat(absolutePath);
|
|
9
|
+
return stat.isFile() ? stat.size : undefined;
|
|
10
|
+
}
|
|
11
|
+
catch {
|
|
12
|
+
return undefined;
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
readHead: async (absolutePath, maxBytes) => {
|
|
16
|
+
try {
|
|
17
|
+
return (await readFile(absolutePath, "utf8")).slice(0, maxBytes);
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
return undefined;
|
|
21
|
+
}
|
|
22
|
+
},
|
|
23
|
+
run: runCommand
|
|
24
|
+
};
|
|
25
|
+
const MANIFEST_FILENAMES = new Set([
|
|
26
|
+
"CMakeLists.txt",
|
|
27
|
+
"Cargo.toml",
|
|
28
|
+
"Dockerfile",
|
|
29
|
+
"Gemfile",
|
|
30
|
+
"Makefile",
|
|
31
|
+
"build.gradle",
|
|
32
|
+
"build.gradle.kts",
|
|
33
|
+
"composer.json",
|
|
34
|
+
"docker-compose.yml",
|
|
35
|
+
"go.mod",
|
|
36
|
+
"mix.exs",
|
|
37
|
+
"package.json",
|
|
38
|
+
"pom.xml",
|
|
39
|
+
"pyproject.toml",
|
|
40
|
+
"setup.py"
|
|
41
|
+
]);
|
|
42
|
+
const DOC_FILENAMES = new Set(["AGENTS.md", "CLAUDE.md", "README", "README.md"]);
|
|
43
|
+
const MANIFEST_LIMIT = 12;
|
|
44
|
+
const MANIFEST_HEAD_BYTES = 4096;
|
|
45
|
+
const DOC_HEAD_BYTES = 8192;
|
|
46
|
+
const EXCLUDED_FILENAMES = new Set([
|
|
47
|
+
".DS_Store",
|
|
48
|
+
"Cargo.lock",
|
|
49
|
+
"Gemfile.lock",
|
|
50
|
+
"Pipfile.lock",
|
|
51
|
+
"bun.lock",
|
|
52
|
+
"bun.lockb",
|
|
53
|
+
"composer.lock",
|
|
54
|
+
"go.sum",
|
|
55
|
+
"package-lock.json",
|
|
56
|
+
"pnpm-lock.yaml",
|
|
57
|
+
"poetry.lock",
|
|
58
|
+
"uv.lock",
|
|
59
|
+
"yarn.lock"
|
|
60
|
+
]);
|
|
61
|
+
const EXCLUDED_EXTENSIONS = new Set([
|
|
62
|
+
"7z", "a", "avi", "avif", "bin", "bmp", "class", "dat", "db", "dll", "dylib",
|
|
63
|
+
"eot", "exe", "flac", "gif", "gz", "icns", "ico", "jar", "jpeg", "jpg",
|
|
64
|
+
"map", "mov", "mp3", "mp4", "o", "ogg", "otf", "pdf", "png", "rar", "so",
|
|
65
|
+
"sqlite", "sqlite3", "tar", "tgz", "tiff", "ttf", "war", "wasm", "wav",
|
|
66
|
+
"webm", "webp", "woff", "woff2", "zip"
|
|
67
|
+
]);
|
|
68
|
+
const EXCLUDED_DIR_SEGMENTS = new Set([
|
|
69
|
+
".cache", ".next", ".nuxt", ".turbo", ".venv", "__pycache__", "build",
|
|
70
|
+
"coverage", "dist", "node_modules", "out", "target", "vendor"
|
|
71
|
+
]);
|
|
72
|
+
export function isExcludedPath(path) {
|
|
73
|
+
const segments = path.split("/");
|
|
74
|
+
const filename = segments[segments.length - 1];
|
|
75
|
+
if (EXCLUDED_FILENAMES.has(filename)) {
|
|
76
|
+
return true;
|
|
77
|
+
}
|
|
78
|
+
if (filename.endsWith(".min.js") || filename.endsWith(".min.css")) {
|
|
79
|
+
return true;
|
|
80
|
+
}
|
|
81
|
+
const dotIndex = filename.lastIndexOf(".");
|
|
82
|
+
if (dotIndex > 0 && EXCLUDED_EXTENSIONS.has(filename.slice(dotIndex + 1).toLowerCase())) {
|
|
83
|
+
return true;
|
|
84
|
+
}
|
|
85
|
+
return segments.slice(0, -1).some((segment) => EXCLUDED_DIR_SEGMENTS.has(segment));
|
|
86
|
+
}
|
|
87
|
+
export async function scanRepository(input, dependencies = defaultDependencies) {
|
|
88
|
+
const { repoPath, scope } = input;
|
|
89
|
+
const scopeArgs = scope === undefined ? [] : ["--", scope];
|
|
90
|
+
const workTree = await dependencies.run("git", ["rev-parse", "--is-inside-work-tree"], repoPath);
|
|
91
|
+
if (workTree.stdout.trim() !== "true") {
|
|
92
|
+
throw new Error(`Not a git work tree: ${repoPath}`);
|
|
93
|
+
}
|
|
94
|
+
const commit = (await dependencies.run("git", ["rev-parse", "HEAD"], repoPath)).stdout.trim();
|
|
95
|
+
const lsFiles = await dependencies.run("git", ["ls-files", "-z", "--cached", "--others", "--exclude-standard", ...scopeArgs], repoPath);
|
|
96
|
+
const allPaths = lsFiles.stdout
|
|
97
|
+
.split("\u0000")
|
|
98
|
+
.filter((entry) => entry.length > 0)
|
|
99
|
+
.sort(compareBytewise);
|
|
100
|
+
const keptPaths = allPaths.filter((path) => !isExcludedPath(path));
|
|
101
|
+
const files = [];
|
|
102
|
+
for (const path of keptPaths) {
|
|
103
|
+
const size = await dependencies.fileSize(join(repoPath, path));
|
|
104
|
+
// ls-files lists cached entries deleted from the work tree; nothing to
|
|
105
|
+
// digest there.
|
|
106
|
+
if (size !== undefined) {
|
|
107
|
+
files.push({ path, size });
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
const lastCommitIso = (await dependencies.run("git", ["log", "-1", "--format=%cI"], repoPath)).stdout.trim();
|
|
111
|
+
const log90 = await dependencies.run("git", ["log", "--since=90 days ago", "--name-only", "--pretty=format:%cI", ...scopeArgs], repoPath);
|
|
112
|
+
const activity = parseActivity(log90.stdout);
|
|
113
|
+
const manifests = [];
|
|
114
|
+
for (const file of files) {
|
|
115
|
+
if (manifests.length >= MANIFEST_LIMIT) {
|
|
116
|
+
break;
|
|
117
|
+
}
|
|
118
|
+
if (MANIFEST_FILENAMES.has(file.path.split("/").at(-1) ?? "")) {
|
|
119
|
+
const text = await dependencies.readHead(join(repoPath, file.path), MANIFEST_HEAD_BYTES);
|
|
120
|
+
if (text !== undefined) {
|
|
121
|
+
manifests.push({ path: file.path, text });
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
const docs = [];
|
|
126
|
+
const scopePrefix = scope === undefined ? "" : `${scope}/`;
|
|
127
|
+
for (const file of files) {
|
|
128
|
+
const relative = file.path.startsWith(scopePrefix)
|
|
129
|
+
? file.path.slice(scopePrefix.length)
|
|
130
|
+
: file.path;
|
|
131
|
+
// Only scope-root docs — nested READMEs belong to module digests.
|
|
132
|
+
if (!relative.includes("/") && DOC_FILENAMES.has(relative)) {
|
|
133
|
+
const text = await dependencies.readHead(join(repoPath, file.path), DOC_HEAD_BYTES);
|
|
134
|
+
if (text !== undefined) {
|
|
135
|
+
docs.push({ path: file.path, text });
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
return {
|
|
140
|
+
activity,
|
|
141
|
+
commit,
|
|
142
|
+
docs,
|
|
143
|
+
excludedCount: allPaths.length - keptPaths.length,
|
|
144
|
+
files,
|
|
145
|
+
lastCommitDate: lastCommitIso.slice(0, 10),
|
|
146
|
+
manifests
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
const ISO_LINE = /^\d{4}-\d{2}-\d{2}T/u;
|
|
150
|
+
function parseActivity(stdout) {
|
|
151
|
+
const activity = new Map();
|
|
152
|
+
let currentDate = "";
|
|
153
|
+
for (const line of stdout.split("\n")) {
|
|
154
|
+
const trimmed = line.trim();
|
|
155
|
+
if (trimmed.length === 0) {
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
if (ISO_LINE.test(trimmed)) {
|
|
159
|
+
currentDate = trimmed.slice(0, 10);
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
// git log is newest-first: the first date seen for a path is its last touch.
|
|
163
|
+
if (currentDate !== "" && !activity.has(trimmed)) {
|
|
164
|
+
activity.set(trimmed, currentDate);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
return activity;
|
|
168
|
+
}
|
package/dist/config.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { readFile } from "node:fs/promises";
|
|
2
2
|
import { isAbsolute, normalize, resolve } from "node:path";
|
|
3
3
|
import { z } from "zod";
|
|
4
|
+
import { codeMapSectionSchema, resolveCodeMapConfig } from "./code-map-config.js";
|
|
4
5
|
const providerNameSchema = z.enum(["codex", "claude", "gemini"]);
|
|
5
6
|
const runKindSchema = z.enum(["hourly", "manual", "weekly"]);
|
|
6
7
|
const syncProviderSchema = z.object({
|
|
@@ -16,6 +17,7 @@ const synthesisConfigSchema = z.object({
|
|
|
16
17
|
ollamaModel: z.string().trim().min(1).optional()
|
|
17
18
|
});
|
|
18
19
|
const syncConfigSchema = z.object({
|
|
20
|
+
codeMap: codeMapSectionSchema.optional(),
|
|
19
21
|
logsRoot: z.string().trim().min(1).optional(),
|
|
20
22
|
providers: z.array(syncProviderSchema),
|
|
21
23
|
synthesis: synthesisConfigSchema.optional(),
|
|
@@ -65,6 +67,9 @@ export async function loadConfig(configPath) {
|
|
|
65
67
|
const vaultRoot = resolveConfigPath(parsedConfig.vaultRoot, configDirectory);
|
|
66
68
|
const logsRoot = resolveConfigPath(parsedConfig.logsRoot ?? "logs", configDirectory);
|
|
67
69
|
return {
|
|
70
|
+
...(parsedConfig.codeMap === undefined
|
|
71
|
+
? {}
|
|
72
|
+
: { codeMap: resolveCodeMapConfig(parsedConfig.codeMap, configDirectory) }),
|
|
68
73
|
logsRoot,
|
|
69
74
|
providers: parsedConfig.providers.map((provider) => ({
|
|
70
75
|
destinationPath: normalizeRelativePath(provider.destinationPath),
|
package/dist/consolidation.js
CHANGED
|
@@ -7,7 +7,7 @@ import { appendLogEntry } from "./ingest-writer.js";
|
|
|
7
7
|
import { buildSourceKey, loadManifest, saveManifest, withManifestEntries } from "./manifest.js";
|
|
8
8
|
const CONSOLIDATION_PROVIDER = "consolidation";
|
|
9
9
|
// Structural wiki files a project page must never collide with (design 7.2).
|
|
10
|
-
const RESERVED_PAGES = new Set(["episodes", "index", "log", "reports"]);
|
|
10
|
+
const RESERVED_PAGES = new Set(["code", "episodes", "index", "log", "reports"]);
|
|
11
11
|
export async function runConsolidation(input) {
|
|
12
12
|
const scan = await scanEpisodes(input.episodesRoot);
|
|
13
13
|
const manifest = await loadManifest(input.manifestPath);
|