obsidian-second-brain 0.1.1 → 0.2.0

package/README.md

@@ -106,6 +106,28 @@ If init warned `claude CLI preflight failed`, synthesis fell back to `noop` (mir
 
 Paths support `${ENV_VAR}` expansion; the file is validated with clear errors. Edit it and the next run picks it up — no reload step.
 
+## Code maps
+
+`second-brain map <repoPath>` captures a repository's structure as
+deterministic evidence under `raw/code/<project>/` ($0 — no LLM in capture)
+and renders wiki pages under `wiki/code/<project>/`: one architecture
+overview (with a Mermaid component diagram) plus one page per module.
+Registered repos are re-captured by the hourly job; only changed evidence
+re-renders (content-hash manifest), so idle repos cost nothing.
+
+```
+second-brain map <repoPath> [--project <name>] [--path <subdir>]
+                 [--max-modules N] [--force]
+```
+
+- `--path` scopes a monorepo package; register each package as its own project.
+- `--force` clears the project's code manifest entries and re-renders every page.
+- Module boundaries are deterministic (top-level directories, with descent
+  into a dominant directory); override per repo via `codeMap.repos[].modules`
+  in `config/projects.json` when directories do not reflect architecture.
+- Page bodies are generated artifacts — manual edits are overwritten. Evidence
+  under `raw/code/` is the LLM-readable source of truth for structure.
+
 ## Uninstall
 
 ```bash

package/config/projects.example.json

@@ -2,6 +2,16 @@
   "vaultRoot": "${HOME}/Documents/Obsidian Vault/raw/projects",
   "logsRoot": "../logs",
   "synthesis": { "backend": "noop" },
+  "codeMap": {
+    "repos": [
+      {
+        "path": "/Users/me/Sources/my-repo",
+        "project": "my-repo"
+      }
+    ],
+    "models": { "structure": "sonnet", "pages": "haiku" },
+    "maxModules": 24
+  },
   "providers": [
     {
       "name": "claude",

package/dist/claude-runner.js

@@ -0,0 +1,34 @@
+import { runCommand, runCommandWithInput } from "./shell.js";
+const defaultDependencies = {
+    run: runCommand,
+    runWithInput: runCommandWithInput
+};
+export function createClaudeRunner(options = {}, dependencies = defaultDependencies) {
+    const claudeBin = options.claudeBin ?? "claude";
+    const delivery = options.delivery ?? "argv";
+    return async (input) => {
+        const args = ["-p", "--model", input.model, "--output-format", "json"];
+        const result = delivery === "stdin"
+            ? await dependencies.runWithInput(claudeBin, args, process.cwd(), input.prompt)
+            : await dependencies.run(claudeBin, [...args, input.prompt], process.cwd());
+        return extractResultText(result.stdout);
+    };
+}
+export function extractResultText(stdout) {
+    let parsed;
+    try {
+        parsed = JSON.parse(stdout);
+    }
+    catch {
+        // The CLI can print a banner, login prompt, or usage-limit notice instead
+        // of the JSON envelope; a bare SyntaxError would be useless in failures[].
+        throw new Error(`claude CLI emitted non-JSON output: ${stdout.slice(0, 120).trim()}`);
+    }
+    if (typeof parsed === "object" &&
+        parsed !== null &&
+        "result" in parsed &&
+        typeof parsed.result === "string") {
+        return parsed.result;
+    }
+    throw new Error("Unexpected claude CLI output envelope: missing result text");
+}

package/dist/cli-claude-backend.js

@@ -1,7 +1,9 @@
 import { readFile } from "node:fs/promises";
+import { extractResultText } from "./claude-runner.js";
 import { parseEpisodePatch } from "./episode-patch.js";
 import { assertEpisodesMatchProject, buildEpisodePrompt } from "./episode-prompt.js";
 import { runCommand } from "./shell.js";
+export { extractResultText } from "./claude-runner.js";
 const defaultDependencies = {
     readSource: (path) => readFile(path, "utf8"),
     run: runCommand
@@ -20,21 +22,3 @@ export function createCliClaudeBackend(options = {}, dependencies = defaultDepen
         }
     };
 }
-export function extractResultText(stdout) {
-    let parsed;
-    try {
-        parsed = JSON.parse(stdout);
-    }
-    catch {
-        // The CLI can print a banner, login prompt, or usage-limit notice instead
-        // of the JSON envelope; a bare SyntaxError would be useless in failures[].
-        throw new Error(`claude CLI emitted non-JSON output: ${stdout.slice(0, 120).trim()}`);
-    }
-    if (typeof parsed === "object" &&
-        parsed !== null &&
-        "result" in parsed &&
-        typeof parsed.result === "string") {
-        return parsed.result;
-    }
-    throw new Error("Unexpected claude CLI output envelope: missing result text");
-}

package/dist/cli.js

@@ -1,8 +1,11 @@
 #!/usr/bin/env node
 import { realpathSync } from "node:fs";
 import { fileURLToPath } from "node:url";
-import { join, resolve, sep } from "node:path";
+import { join, resolve } from "node:path";
 import { createCliClaudeBackend } from "./cli-claude-backend.js";
+import { captureRepo, defaultCaptureDependencies } from "./code-capture.js";
+import { runCodeMapIngest } from "./code-ingest.js";
+import { createCliClaudeCodeMapBackend, resolveCodeMapBackend } from "./code-map-backend.js";
 import { loadConfig, parseCliArgs, selectProviders } from "./config.js";
 import { createCliClaudeConsolidationBackend, resolveConsolidationBackend } from "./consolidation-backend.js";
 import { runConsolidation } from "./consolidation.js";
@@ -13,6 +16,7 @@ import { createOllamaBackend } from "./ollama-backend.js";
 import { createReportWriter } from "./report.js";
 import { resolveBackend } from "./synthesis.js";
 import { syncProviderTree } from "./sync.js";
+import { resolveVaultPaths } from "./vault-paths.js";
 const defaultDependencies = {
     createLogger,
     loadConfig,
@@ -54,6 +58,30 @@ export async function runCli(argv, dependencies = defaultDependencies) {
                 vaultRoot: vaultPaths.vaultDir
             })
             : undefined;
+        // Code capture (design 6.1): deterministic, $0 — runs even when the
+        // synthesis backend is noop, so evidence accumulates for later.
+        let evidenceCaptured = 0;
+        const captureFailures = [];
+        const codeMap = config.codeMap;
+        if (vaultPaths && args.ingest && codeMap !== undefined) {
+            for (const repo of codeMap.repos) {
+                try {
+                    const captureSummary = await captureRepo({
+                        date: generatedAt.slice(0, 10),
+                        globalMaxModules: codeMap.maxModules,
+                        rawCodeRoot: vaultPaths.rawCodeRoot,
+                        repo
+                    }, defaultCaptureDependencies(repo.path));
+                    evidenceCaptured += captureSummary.evidenceWritten;
+                    logger.info(`[code-map] ${repo.project}: written=${captureSummary.evidenceWritten}, unchanged=${captureSummary.evidenceUnchanged}, modules=${captureSummary.moduleCount}, pruned=${captureSummary.digestsDeleted}`);
+                }
+                catch (error) {
+                    const reason = error instanceof Error ? error.message : "Unknown error";
+                    captureFailures.push({ project: repo.project, reason });
+                    logger.info(`[code-map] ${repo.project} capture error: ${reason}`);
+                }
+            }
+        }
         if (vaultPaths && args.ingest) {
             const backend = resolveBackend(config.synthesis.backend, {
                 "cli-claude": createCliClaudeBackend({
@@ -75,6 +103,36 @@ export async function runCli(argv, dependencies = defaultDependencies) {
                 ? "Ingest: skipped (no changes)"
                 : `Ingest: episodes=${ingestSummary.episodePaths.length}, transcripts_tracked=${ingestSummary.trackedTranscripts}, failures=${ingestSummary.failures.length}, quarantined=${ingestSummary.quarantined.length}`);
         }
+        let codeMapSummary;
+        if (vaultPaths && args.ingest && codeMap !== undefined) {
+            const codeBackend = resolveCodeMapBackend(config.synthesis.backend, () => createCliClaudeCodeMapBackend({
+                ...(config.synthesis.claudeBin === undefined
+                    ? {}
+                    : { claudeBin: config.synthesis.claudeBin }),
+                logsRoot: config.logsRoot,
+                models: codeMap.models
+            }));
+            if (codeBackend === undefined) {
+                logger.info("Code map: skipped (noop backend)");
+            }
+            else {
+                codeMapSummary = await runCodeMapIngest({
+                    backend: codeBackend,
+                    captureFailures,
+                    evidenceCaptured,
+                    indexPath: vaultPaths.indexPath,
+                    logPath: vaultPaths.wikiLogPath,
+                    manifestPath: vaultPaths.manifestPath,
+                    now: dependencies.now,
+                    rawCodeRoot: vaultPaths.rawCodeRoot,
+                    ...(reportWriter === undefined ? {} : { reportWriter }),
+                    wikiCodeRoot: vaultPaths.wikiCodeRoot
+                });
+                logger.info(codeMapSummary.skipped
+                    ? "Code map: skipped (no changes)"
+                    : `Code map: pages_rendered=${codeMapSummary.pagesRendered.length}, pages_pruned=${codeMapSummary.pagesPruned.length}, failures=${codeMapSummary.failures.length}`);
+            }
+        }
         if (vaultPaths && args.consolidate) {
             const backend = resolveConsolidationBackend(config.synthesis.backend, {
                 "cli-claude": createCliClaudeConsolidationBackend({
@@ -96,7 +154,9 @@ export async function runCli(argv, dependencies = defaultDependencies) {
                 : `Consolidation: pages_created=${consolidationSummary.pagesCreated.length}, pages_updated=${consolidationSummary.pagesUpdated.length}, episodes=${consolidationSummary.episodesConsolidated}, failures=${consolidationSummary.failures.length}`);
         }
         // Both stages write into the same report file, so either path works.
-        const reportPath = ingestSummary?.reportPath ?? consolidationSummary?.reportPath;
+        const reportPath = ingestSummary?.reportPath ??
+            codeMapSummary?.reportPath ??
+            consolidationSummary?.reportPath;
         if (reportPath !== undefined) {
             logger.info(`Run report: ${reportPath}`);
         }
@@ -104,6 +164,7 @@ export async function runCli(argv, dependencies = defaultDependencies) {
         const logPath = await logger.flush();
         dependencies.stdout(`Log written to ${logPath}\n`);
         return {
+            ...(codeMapSummary ? { codeMap: codeMapSummary } : {}),
             ...(consolidationSummary ? { consolidation: consolidationSummary } : {}),
             errorCount,
             ...(ingestSummary ? { ingest: ingestSummary } : {}),
@@ -117,26 +178,6 @@ export async function runCli(argv, dependencies = defaultDependencies) {
         await lock?.release();
     }
 }
-function resolveVaultPaths(vaultRoot) {
-    // Defense-in-depth (design 7.6): the prune scope must stay confined to
-    // raw/projects, so a hand-edited config cannot point ingest at a stray tree.
-    const normalized = resolve(vaultRoot);
-    if (!normalized.endsWith(join(`${sep}raw`, "projects"))) {
-        throw new Error(`vaultRoot must end in raw/projects to run ingest, got: ${vaultRoot}`);
-    }
-    const vault = resolve(normalized, "..", "..");
-    return {
-        artifactsRoot: join(vault, "raw", "artifacts"),
-        episodesRoot: join(vault, "wiki", "episodes"),
-        indexPath: join(vault, "wiki", "index.md"),
-        lockPath: join(vault, "wiki", ".ingest.lock"),
-        manifestPath: join(vault, "wiki", ".ingest-state.json"),
-        reportsRoot: join(vault, "wiki", "reports"),
-        vaultDir: vault,
-        wikiLogPath: join(vault, "wiki", "log.md"),
-        wikiRoot: join(vault, "wiki")
-    };
-}
 async function syncProvider(provider, vaultRoot, generatedAt, logger, dependencies) {
     const destinationRoot = join(vaultRoot, provider.destinationPath);
     try {
@@ -307,6 +348,15 @@ async function main() {
         await runUninstallCommand(argv.slice(1));
         return;
     }
+    if (command === "map") {
+        const { runMapCommand } = await import("./map-command.js");
+        await runMapCommand(argv.slice(1), {
+            defaultConfigPath: join(import.meta.dirname, "..", "config", "projects.json"),
+            log: (message) => stdout(`${message}\n`),
+            now: () => new Date()
+        });
+        return;
+    }
     const syncArgv = command === "sync" ? argv.slice(1) : argv;
     const result = await runCli(syncArgv);
     if (result.errorCount > 0) {

package/dist/code-boundaries.js

@@ -0,0 +1,154 @@
+import { slugify } from "./slug.js";
+const DESCEND_SHARE = 0.4;
+const MIN_MODULE_FILES = 3;
+const MISC_NAME = "misc";
+const RESERVED_MODULE_SLUG = "overview";
+// UTF-16 code-unit order — identical to bytewise order for the ASCII paths
+// git emits. Every tie in the algorithm resolves through this comparator
+// (design 5: same tree in, same boundaries out).
+export function compareBytewise(a, b) {
+    return a < b ? -1 : a > b ? 1 : 0;
+}
+export function proposeModules(input) {
+    const files = [...input.files].sort(compareBytewise);
+    if (files.length === 0) {
+        return [];
+    }
+    if (input.overrides !== undefined && input.overrides.length > 0) {
+        // The override replaces the heuristic entirely (design 5) — no small-
+        // candidate merging, the user's decomposition is taken as-is.
+        return finalize(applyOverrides(files, input.overrides), input.maxModules, false);
+    }
+    let candidates = splitByDirectory(files, "");
+    const total = files.length;
+    // Descend one level into every dominant directory, in bytewise order.
+    for (const candidate of [...candidates]) {
+        if (candidate.kind === "directory" &&
+            candidate.files.length / total > DESCEND_SHARE) {
+            candidates = [
+                ...candidates.filter((entry) => entry !== candidate),
+                ...splitByDirectory(candidate.files, `${candidate.name}/`)
+            ];
+        }
+    }
+    return finalize(candidates, input.maxModules, true);
+}
+function splitByDirectory(files, prefix) {
+    const groups = new Map();
+    const loose = [];
+    for (const file of files) {
+        const rest = file.slice(prefix.length);
+        const slashIndex = rest.indexOf("/");
+        if (slashIndex === -1) {
+            loose.push(file);
+            continue;
+        }
+        const name = prefix + rest.slice(0, slashIndex);
+        const group = groups.get(name);
+        if (group === undefined) {
+            groups.set(name, [file]);
+        }
+        else {
+            group.push(file);
+        }
+    }
+    const candidates = [...groups.entries()].map(([name, groupFiles]) => ({
+        files: groupFiles,
+        kind: "directory",
+        name
+    }));
+    if (loose.length > 0) {
+        candidates.push({
+            files: loose,
+            kind: "loose",
+            name: prefix === "" ? "root" : prefix.slice(0, -1)
+        });
+    }
+    return candidates.sort((a, b) => compareBytewise(a.name, b.name));
+}
+function applyOverrides(files, overrides) {
+    const candidates = overrides.map((override) => ({
+        files: [],
+        kind: "directory",
+        name: override.name
+    }));
+    const misc = [];
+    for (const file of files) {
+        const index = overrides.findIndex((override) => override.paths.some((path) => matchesPrefix(file, path)));
+        if (index === -1) {
+            misc.push(file);
+        }
+        else {
+            candidates[index].files.push(file);
+        }
+    }
+    if (misc.length > 0) {
+        candidates.push({ files: misc, kind: "loose", name: MISC_NAME });
+    }
+    return candidates.filter((candidate) => candidate.files.length > 0);
+}
+// Prefix entries are repo-relative files or directories, POSIX separators —
+// deliberately not globs (design 5).
+function matchesPrefix(file, prefix) {
+    const cleaned = prefix.replace(/\/+$/u, "");
+    return file === cleaned || file.startsWith(`${cleaned}/`);
+}
+function finalize(input, maxModules, mergeSmall) {
+    let candidates = input
+        .filter((candidate) => candidate.files.length > 0)
+        .map((candidate) => ({
+        ...candidate,
+        files: [...candidate.files].sort(compareBytewise)
+    }));
+    if (mergeSmall) {
+        const small = candidates
+            .filter((candidate) => candidate.files.length < MIN_MODULE_FILES && candidate.name !== MISC_NAME)
+            .sort(byCountThenName);
+        if (small.length > 0) {
+            const smallSet = new Set(small);
+            candidates = addToMisc(candidates.filter((candidate) => !smallSet.has(candidate)), small.flatMap((candidate) => candidate.files));
+        }
+    }
+    // Cap: merge ascending (file count, then name) into misc until within
+    // bounds. Merging into a fresh misc keeps the count constant on the first
+    // iteration, then every iteration strictly shrinks it.
+    while (candidates.length > maxModules) {
+        const nonMisc = candidates
+            .filter((candidate) => candidate.name !== MISC_NAME)
+            .sort(byCountThenName);
+        const smallest = nonMisc[0];
+        if (smallest === undefined) {
+            break;
+        }
+        candidates = addToMisc(candidates.filter((candidate) => candidate !== smallest), smallest.files);
+    }
+    const ordered = [...candidates].sort((a, b) => compareBytewise(a.name, b.name));
+    const usedSlugs = new Set([RESERVED_MODULE_SLUG]);
+    const modules = ordered.map((candidate) => {
+        const base = slugify(candidate.name.replaceAll("/", "-")) || MISC_NAME;
+        let slug = base;
+        let suffix = 2;
+        while (usedSlugs.has(slug)) {
+            slug = `${base}-${suffix}`;
+            suffix += 1;
+        }
+        usedSlugs.add(slug);
+        return { files: candidate.files, name: candidate.name, slug };
+    });
+    return modules.sort((a, b) => compareBytewise(a.slug, b.slug));
+}
+function addToMisc(candidates, files) {
+    const misc = candidates.find((candidate) => candidate.name === MISC_NAME);
+    if (misc === undefined) {
+        return [...candidates, { files: [...files].sort(compareBytewise), kind: "loose", name: MISC_NAME }];
+    }
+    return candidates.map((candidate) => candidate === misc
+        ? {
+            ...candidate,
+            files: [...candidate.files, ...files].sort(compareBytewise)
+        }
+        : candidate);
+}
+function byCountThenName(a, b) {
+    return a.files.length - b.files.length || compareBytewise(a.name, b.name);
+}

package/dist/code-capture.js

@@ -0,0 +1,122 @@
+import { readdir, readFile, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { proposeModules } from "./code-boundaries.js";
+import { parseCodeEvidence, renderCodeEvidence } from "./code-frontmatter.js";
+import { buildModuleDigestBody, buildSkeletonBody, DIGEST_BUDGET_BYTES } from "./code-evidence.js";
+import { scanRepository } from "./code-scan.js";
+import { ensureDirectory, writeFileAtomic } from "./filesystem.js";
+export function defaultCaptureDependencies(repoPath) {
+    return {
+        readText: async (repoRelativePath) => {
+            try {
+                return await readFile(join(repoPath, repoRelativePath), "utf8");
+            }
+            catch {
+                return undefined;
+            }
+        }
+    };
+}
+export async function captureRepo(input, dependencies) {
+    const { repo } = input;
+    // READ PHASE — every git command and file read completes before the first
+    // byte is written; a throw anywhere here leaves raw/code untouched
+    // (design 6.1, codex F2).
+    const scan = await (dependencies.scan === undefined
+        ? scanRepository({ repoPath: repo.path, ...(repo.scope === undefined ? {} : { scope: repo.scope }) })
+        : scanRepository({ repoPath: repo.path, ...(repo.scope === undefined ? {} : { scope: repo.scope }) }, dependencies.scan));
+    const scopePrefix = repo.scope === undefined ? "" : `${repo.scope}/`;
+    const scopedPaths = scan.files.map((file) => file.path.startsWith(scopePrefix) ? file.path.slice(scopePrefix.length) : file.path);
+    const scopedModules = proposeModules({
+        files: scopedPaths,
+        maxModules: repo.maxModules ?? input.globalMaxModules,
+        ...(repo.modules === undefined ? {} : { overrides: repo.modules })
+    });
+    // Boundary names/slugs are scope-relative; evidence paths stay repo-relative.
+    const modules = scopedModules.map((module) => ({
+        ...module,
+        files: module.files.map((file) => `${scopePrefix}${file}`)
+    }));
+    const projectDir = join(input.rawCodeRoot, repo.project);
+    const modulesDir = join(projectDir, "modules");
+    const sizeByPath = new Map(scan.files.map((file) => [file.path, file.size]));
+    const planned = [
+        {
+            body: buildSkeletonBody({
+                modules,
+                scan,
+                ...(repo.scope === undefined ? {} : { scope: repo.scope })
+            }),
+            path: join(projectDir, "skeleton.md")
+        }
+    ];
+    for (const module of modules) {
+        const digest = await buildModuleDigestBody({
+            activity: scan.activity,
+            budgetBytes: DIGEST_BUDGET_BYTES,
+            files: module.files.map((path) => ({ path, size: sizeByPath.get(path) ?? 0 })),
+            readText: dependencies.readText
+        });
+        planned.push({
+            body: digest.body,
+            ...(digest.coverage === undefined ? {} : { coverage: digest.coverage }),
+            module,
+            path: join(modulesDir, `${module.slug}.md`)
+        });
+    }
+    // WRITE PHASE — mirror with churn guard.
+    await ensureDirectory(modulesDir);
+    let evidenceWritten = 0;
+    let evidenceUnchanged = 0;
+    for (const evidence of planned) {
+        const frontmatter = {
+            captured: input.date,
+            commit: scan.commit,
+            ...(evidence.coverage === undefined ? {} : { coverage: evidence.coverage }),
+            ...(evidence.module === undefined
+                ? {}
+                : { module: evidence.module.slug, paths: [...evidence.module.files] }),
+            project: repo.project,
+            repo: repo.path
+        };
+        if (await isBodyUnchanged(evidence.path, evidence.body)) {
+            evidenceUnchanged += 1;
+            continue;
+        }
+        await writeFileAtomic(evidence.path, renderCodeEvidence(frontmatter, evidence.body));
+        evidenceWritten += 1;
+    }
+    // Prune digests for modules that no longer exist — mirror semantics,
+    // confined to this project's modules directory (design 4).
+    const desired = new Set(modules.map((module) => `${module.slug}.md`));
+    let digestsDeleted = 0;
+    for (const entry of (await readdir(modulesDir)).sort()) {
+        if (entry.endsWith(".md") && !desired.has(entry)) {
+            await rm(join(modulesDir, entry), { force: true });
+            digestsDeleted += 1;
+        }
+    }
+    return {
+        digestsDeleted,
+        evidenceUnchanged,
+        evidenceWritten,
+        moduleCount: modules.length,
+        project: repo.project
+    };
+}
+async function isBodyUnchanged(path, nextBody) {
+    let existing;
+    try {
+        existing = await readFile(path, "utf8");
+    }
+    catch {
+        return false;
+    }
+    try {
+        return parseCodeEvidence(existing).body === nextBody.trim();
+    }
+    catch {
+        // Unparseable evidence is replaced wholesale.
+        return false;
+    }
+}