obsidian-second-brain 0.1.0

package/dist/config.js

@@ -0,0 +1,208 @@
+import { readFile } from "node:fs/promises";
+import { isAbsolute, normalize, resolve } from "node:path";
+import { z } from "zod";
+const providerNameSchema = z.enum(["codex", "claude", "gemini"]);
+const runKindSchema = z.enum(["hourly", "manual", "weekly"]);
+const syncProviderSchema = z.object({
+    destinationPath: z.string().trim().min(1),
+    enabled: z.boolean(),
+    name: providerNameSchema,
+    sourceRoot: z.string().trim().min(1)
+});
+const synthesisBackendNameSchema = z.enum(["cli-claude", "noop", "ollama"]);
+const synthesisConfigSchema = z.object({
+    backend: synthesisBackendNameSchema,
+    claudeBin: z.string().trim().min(1).optional(),
+    ollamaModel: z.string().trim().min(1).optional()
+});
+const syncConfigSchema = z.object({
+    logsRoot: z.string().trim().min(1).optional(),
+    providers: z.array(syncProviderSchema),
+    synthesis: synthesisConfigSchema.optional(),
+    vaultRoot: z.string().trim().min(1)
+}).superRefine((config, context) => {
+    const seenNames = new Set();
+    for (const [index, provider] of config.providers.entries()) {
+        if (seenNames.has(provider.name)) {
+            context.addIssue({
+                code: z.ZodIssueCode.custom,
+                message: `Duplicate provider name: ${provider.name}`,
+                path: ["providers", index, "name"]
+            });
+            continue;
+        }
+        seenNames.add(provider.name);
+    }
+});
+const cliRequestSchema = z.object({
+    all: z.boolean(),
+    configPath: z.string().trim().min(1).optional(),
+    consolidate: z.boolean().optional(),
+    ingest: z.boolean().optional(),
+    providerName: providerNameSchema.optional(),
+    runKind: runKindSchema.optional()
+});
+const providerAliasSchema = z.union([
+    providerNameSchema,
+    z.literal("--claude"),
+    z.literal("--codex"),
+    z.literal("--gemini")
+]);
+const providerAliasToName = {
+    "--claude": "claude",
+    "--codex": "codex",
+    "--gemini": "gemini",
+    claude: "claude",
+    codex: "codex",
+    gemini: "gemini"
+};
+export async function loadConfig(configPath) {
+    const fileContent = await readFile(configPath, "utf8");
+    const rawConfig = JSON.parse(fileContent);
+    const parsedConfig = syncConfigSchema.parse(rawConfig);
+    const resolvedConfigPath = resolve(configPath);
+    const configDirectory = resolve(resolvedConfigPath, "..");
+    const vaultRoot = resolveConfigPath(parsedConfig.vaultRoot, configDirectory);
+    const logsRoot = resolveConfigPath(parsedConfig.logsRoot ?? "logs", configDirectory);
+    return {
+        logsRoot,
+        providers: parsedConfig.providers.map((provider) => ({
+            destinationPath: normalizeRelativePath(provider.destinationPath),
+            enabled: provider.enabled,
+            name: provider.name,
+            sourceRoot: resolveConfigPath(provider.sourceRoot, configDirectory)
+        })),
+        // Default stays noop until the cli-claude backend lands (phase 6).
+        synthesis: resolveSynthesisConfig(parsedConfig.synthesis),
+        vaultRoot
+    };
+}
+export function parseCliArgs(argv) {
+    let all = false;
+    let configPath;
+    let consolidate = false;
+    let ingest = false;
+    let providerName;
+    let runKind;
+    for (let index = 0; index < argv.length; index += 1) {
+        const currentArg = argv[index];
+        if (currentArg === "--all") {
+            all = true;
+            continue;
+        }
+        if (currentArg === "--ingest") {
+            ingest = true;
+            continue;
+        }
+        if (currentArg === "--consolidate") {
+            consolidate = true;
+            continue;
+        }
+        if (currentArg === "--config") {
+            const nextArg = argv[index + 1];
+            if (!nextArg) {
+                throw new Error("Missing value for --config");
+            }
+            configPath = nextArg;
+            index += 1;
+            continue;
+        }
+        if (currentArg === "--provider") {
+            const nextArg = argv[index + 1];
+            if (!nextArg) {
+                throw new Error("Missing value for --provider");
+            }
+            providerName = parseProviderName(nextArg);
+            index += 1;
+            continue;
+        }
+        if (currentArg === "--run") {
+            const nextArg = argv[index + 1];
+            if (!nextArg) {
+                throw new Error("Missing value for --run");
+            }
+            const parsedRunKind = runKindSchema.safeParse(nextArg);
+            if (!parsedRunKind.success) {
+                throw new Error(`Unknown run kind: ${nextArg}`);
+            }
+            runKind = parsedRunKind.data;
+            index += 1;
+            continue;
+        }
+        if (providerAliasSchema.safeParse(currentArg).success) {
+            providerName = parseProviderName(currentArg);
+            continue;
+        }
+        throw new Error(`Unknown argument: ${currentArg}`);
+    }
+    const request = {
+        ...(configPath ? { configPath } : {}),
+        ...(consolidate ? { consolidate } : {}),
+        ...(ingest ? { ingest } : {}),
+        ...(providerName ? { providerName } : {}),
+        ...(runKind ? { runKind } : {}),
+        all
+    };
+    return cliRequestSchema.parse(request);
+}
+export function selectProviders(configuredProviders, request) {
+    const selectedProviders = request.providerName
+        ? selectSingleProvider(configuredProviders, request.providerName)
+        : request.all
+            ? [...configuredProviders]
+            : configuredProviders.filter((provider) => provider.enabled);
+    if (selectedProviders.length === 0) {
+        throw new Error("No providers selected");
+    }
+    return selectedProviders;
+}
+function selectSingleProvider(configuredProviders, providerName) {
+    const selectedProvider = configuredProviders.find((provider) => provider.name === providerName);
+    if (!selectedProvider) {
+        throw new Error(`Unknown provider: ${providerName}`);
+    }
+    return [selectedProvider];
+}
+function parseProviderName(input) {
+    const parsedProvider = providerAliasSchema.safeParse(input);
+    if (!parsedProvider.success) {
+        throw new Error(`Unknown provider: ${input}`);
+    }
+    return providerAliasToName[parsedProvider.data];
+}
+function resolveConfigPath(inputPath, configDirectory) {
+    const expandedPath = expandEnvironmentVariables(inputPath);
+    return resolve(configDirectory, expandedPath);
+}
+function resolveSynthesisConfig(synthesis) {
+    if (synthesis === undefined) {
+        return { backend: "noop" };
+    }
+    // claudeBin supports ${ENV} expansion but is never resolved against the
+    // config directory: a bare command name ("claude") must stay a PATH lookup.
+    return {
+        ...synthesis,
+        ...(synthesis.claudeBin === undefined
+            ? {}
+            : { claudeBin: expandEnvironmentVariables(synthesis.claudeBin) })
+    };
+}
+function expandEnvironmentVariables(inputPath) {
+    return inputPath.replaceAll(/\$\{([A-Z0-9_]+)\}/gi, (_, variableName) => {
+        const variableValue = process.env[variableName];
+        if (!variableValue) {
+            throw new Error(`Missing environment variable: ${variableName}`);
+        }
+        return variableValue;
+    });
+}
+function normalizeRelativePath(inputPath) {
+    const expandedPath = expandEnvironmentVariables(inputPath);
+    const normalizedPath = normalize(expandedPath);
+    if (normalizedPath === "." ||
+        isAbsolute(normalizedPath) ||
+        normalizedPath.startsWith("..")) {
+        throw new Error(`Invalid destinationPath: ${inputPath}`);
+    }
+    return normalizedPath;
+}

package/dist/consolidation-backend.js

@@ -0,0 +1,86 @@
+import { z } from "zod";
+import { extractResultText } from "./cli-claude-backend.js";
+import { runCommand } from "./shell.js";
+const MAX_PAGE_CONTEXT_CHARS = 24_000;
+const MAX_EPISODE_CHARS = 8_000;
+const consolidationPatchSchema = z.object({
+    section: z.union([z.string().trim().min(1).max(20_000), z.null()])
+});
+export const noopConsolidationBackend = {
+    consolidate: async () => ({ section: null }),
+    name: "noop"
+};
+const defaultDependencies = {
+    run: runCommand
+};
+export function createCliClaudeConsolidationBackend(options = {}, dependencies = defaultDependencies) {
+    const claudeBin = options.claudeBin ?? "claude";
+    return {
+        consolidate: async (input) => {
+            const prompt = buildConsolidationPrompt(input);
+            // Weekly pass uses a frontier-class model on purpose (design 4.3):
+            // it edits trusted pages, so quality beats cost here.
+            const result = await dependencies.run(claudeBin, ["-p", "--model", "sonnet", "--output-format", "json", prompt], process.cwd());
+            const parsed = JSON.parse(stripFences(extractResultText(result.stdout)));
+            return consolidationPatchSchema.parse(parsed);
+        },
+        name: "cli-claude"
+    };
+}
+export function resolveConsolidationBackend(name, backends = {}) {
+    if (name === "noop") {
+        return noopConsolidationBackend;
+    }
+    // The weekly roll-up stays on the paid backend even in all-local mode
+    // (design section 6) — ollama maps to cli-claude here.
+    const effective = name === "ollama" ? "cli-claude" : name;
+    const backend = backends[effective];
+    if (backend !== undefined) {
+        return backend;
+    }
+    throw new Error(`Unknown consolidation backend: ${name} (available: noop, cli-claude, ollama -> cli-claude)`);
+}
+function buildConsolidationPrompt(input) {
+    const episodes = input.episodes
+        .map((episode) => {
+        const truncated = episode.content.length > MAX_EPISODE_CHARS
+            ? `${episode.content.slice(0, MAX_EPISODE_CHARS)}\n[truncated]`
+            : episode.content;
+        return [`### Episode (${episode.date}) ${episode.title}`, "", truncated].join("\n");
+    })
+        .join("\n\n---\n\n");
+    const existing = input.existingPage
+        ? input.existingPage.length > MAX_PAGE_CONTEXT_CHARS
+            ? `${input.existingPage.slice(0, MAX_PAGE_CONTEXT_CHARS)}\n[truncated]`
+            : input.existingPage
+        : "(no page exists yet)";
+    return [
+        "You are the weekly consolidation pass of an LLM-maintained wiki. Compress",
+        `the episodic notes below into ONE addition for the trusted entity page of`,
+        `the project "${input.project}".`,
+        "",
+        "Rules:",
+        '- Respond with ONLY strict JSON: {"section": "..."} — no prose, no fences.',
+        "- The section is markdown BODY text that will be APPENDED to the page",
+        "  under a dated heading; never include frontmatter or a top-level # title.",
+        "- Additive and factual: summarize what happened, decisions made and why,",
+        "  and open follow-ups. Cite episode dates.",
+        "- If the episodes contradict the existing page, state the contradiction",
+        "  explicitly and cite both sides — do not silently overwrite.",
+        "- Use [[wikilinks]] only for project-specific entities. No emojis.",
+        '- If the episodes contain nothing wiki-worthy, respond {"section": null}.',
+        "",
+        "## Existing page",
+        "",
+        existing,
+        "",
+        "## Episodes",
+        "",
+        episodes
+    ].join("\n");
+}
+function stripFences(raw) {
+    const trimmed = raw.trim();
+    const fenced = /^```(?:json)?\s*\n([\s\S]*?)\n```$/u.exec(trimmed);
+    return fenced ? fenced[1] : trimmed;
+}

package/dist/consolidation.js

@@ -0,0 +1,321 @@
+import { lstat, readFile, realpath } from "node:fs/promises";
+import { join, resolve, sep } from "node:path";
+import { hashContent } from "./changeset.js";
+import { parseEpisodeFile } from "./episode-file.js";
+import { collectFiles, writeFileAtomic } from "./filesystem.js";
+import { appendLogEntry } from "./ingest-writer.js";
+import { buildSourceKey, loadManifest, saveManifest, withManifestEntries } from "./manifest.js";
+const CONSOLIDATION_PROVIDER = "consolidation";
+// Structural wiki files a project page must never collide with (design 7.2).
+const RESERVED_PAGES = new Set(["episodes", "index", "log", "reports"]);
+export async function runConsolidation(input) {
+    const scan = await scanEpisodes(input.episodesRoot);
+    const manifest = await loadManifest(input.manifestPath);
+    const work = scan.sources.filter((source) => {
+        const entry = manifest.entries[source.key];
+        return entry === undefined || entry.contentHash !== source.contentHash;
+    });
+    const freshQuarantined = scan.quarantined.filter((artifact) => manifest.entries[`quarantine:episode:${artifact.contentHash}`] === undefined);
+    if (work.length === 0 && freshQuarantined.length === 0) {
+        return {
+            episodesConsolidated: 0,
+            failures: [],
+            pagesCreated: [],
+            pagesUpdated: [],
+            quarantined: [],
+            skipped: true,
+            skippedProjects: []
+        };
+    }
+    const now = input.now();
+    const date = now.toISOString().slice(0, 10);
+    const groups = new Map();
+    for (const source of work) {
+        const group = groups.get(source.project);
+        if (group === undefined) {
+            groups.set(source.project, [source]);
+        }
+        else {
+            group.push(source);
+        }
+    }
+    const failures = [];
+    const skippedProjects = [];
+    const pagesCreated = [];
+    const pagesUpdated = [];
+    const consolidatedEntries = {};
+    const ingestedAt = now.toISOString();
+    const wikiRoot = resolve(input.wikiRoot);
+    let episodesConsolidated = 0;
+    for (const [project, sources] of groups) {
+        if (RESERVED_PAGES.has(project)) {
+            failures.push({
+                project,
+                reason: "Project name collides with a reserved wiki file"
+            });
+            continue;
+        }
+        const pagePath = join(wikiRoot, `${project}.md`);
+        if (!pagePath.startsWith(`${wikiRoot}${sep}`)) {
+            failures.push({ project, reason: "Page path escapes the wiki root" });
+            continue;
+        }
+        let existingPage;
+        try {
+            existingPage = await readFile(pagePath, "utf8");
+        }
+        catch (error) {
+            if (!isMissingFileError(error)) {
+                failures.push({
+                    project,
+                    reason: error instanceof Error ? error.message : "Unknown error"
+                });
+                continue;
+            }
+        }
+        let section;
+        try {
+            const output = await input.backend.consolidate({
+                episodes: sources.map((source) => source.episode),
+                existingPage,
+                project
+            });
+            section = output.section;
+        }
+        catch (error) {
+            failures.push({
+                project,
+                reason: error instanceof Error ? error.message : "Unknown error"
+            });
+            continue;
+        }
+        if (section === null) {
+            skippedProjects.push(project);
+            continue;
+        }
+        // A write failure must only fail THIS project: earlier projects keep
+        // their manifest advance and the run still reaches log + manifest.
+        try {
+            // The backend call takes seconds; re-read the page right before
+            // writing so a concurrent manual edit is appended to, not clobbered.
+            let pageNow;
+            try {
+                pageNow = await readFile(pagePath, "utf8");
+            }
+            catch (error) {
+                if (!isMissingFileError(error)) {
+                    throw error;
+                }
+            }
+            // Additive edit only (vault AGENTS.md): a dated section is appended;
+            // the existing page body is never rewritten.
+            const nextPage = pageNow === undefined
+                ? buildNewPage(project, date, section)
+                : `${pageNow.replace(/\n+$/u, "")}\n\n## Consolidation ${date}\n\n${section.trim()}\n`;
+            await writeFileAtomic(pagePath, nextPage);
+            if (pageNow === undefined) {
+                pagesCreated.push(project);
+                await addToIndex(input.indexPath, project, date);
+            }
+            else {
+                pagesUpdated.push(project);
+            }
+        }
+        catch (error) {
+            failures.push({
+                project,
+                reason: error instanceof Error ? error.message : "Unknown error"
+            });
+            continue;
+        }
+        for (const source of sources) {
+            consolidatedEntries[source.key] = {
+                contentHash: source.contentHash,
+                ingestedAt
+            };
+            episodesConsolidated += 1;
+        }
+    }
+    let report;
+    // The report is a wiki page: it belongs to the pages step, before log and
+    // manifest, so a crashed run redoes it instead of losing it forever.
+    if (input.reportWriter) {
+        report = await input.reportWriter.addConsolidation({
+            episodesConsolidated,
+            failures,
+            pagesCreated,
+            pagesUpdated,
+            quarantined: freshQuarantined,
+            skippedProjects
+        });
+    }
+    if (pagesCreated.length > 0 ||
+        pagesUpdated.length > 0 ||
+        failures.length > 0 ||
+        freshQuarantined.length > 0 ||
+        report !== undefined) {
+        await appendLogEntry(input.logPath, buildLogEntry({
+            date,
+            failures,
+            freshQuarantined,
+            pagesCreated,
+            pagesUpdated,
+            reportLink: report?.wikiLink
+        }));
+    }
+    const quarantineEntries = Object.fromEntries(freshQuarantined.map((artifact) => [
+        `quarantine:episode:${artifact.contentHash}`,
+        { contentHash: artifact.contentHash, ingestedAt }
+    ]));
+    await saveManifest(input.manifestPath, withManifestEntries(manifest, { ...consolidatedEntries, ...quarantineEntries }));
+    return {
+        episodesConsolidated,
+        failures,
+        pagesCreated,
+        pagesUpdated,
+        quarantined: freshQuarantined,
+        ...(report ? { reportPath: report.path } : {}),
+        skipped: false,
+        skippedProjects
+    };
+}
+async function scanEpisodes(episodesRoot) {
+    let files;
+    try {
+        files = await collectFiles(episodesRoot);
+    }
+    catch (error) {
+        if (isMissingFileError(error)) {
+            return { quarantined: [], sources: [] };
+        }
+        throw error;
+    }
+    const quarantined = [];
+    const sources = [];
+    const resolvedRoot = await realpath(episodesRoot);
+    for (const file of files) {
+        if (!file.relativePath.endsWith(".md")) {
+            continue;
+        }
+        const stat = await lstat(file.absolutePath);
+        if (stat.isSymbolicLink()) {
+            quarantined.push({
+                contentHash: hashContent(`symlink:${file.absolutePath}`),
+                path: file.absolutePath,
+                reason: "Refused symlinked episode"
+            });
+            continue;
+        }
+        const resolved = await realpath(file.absolutePath);
+        if (!resolved.startsWith(`${resolvedRoot}${sep}`)) {
+            quarantined.push({
+                contentHash: hashContent(`escape:${file.absolutePath}`),
+                path: file.absolutePath,
+                reason: "Episode resolves outside the episodes root"
+            });
+            continue;
+        }
+        let content = "";
+        try {
+            content = await readFile(file.absolutePath, "utf8");
+            const parsed = parseEpisodeFile(content);
+            sources.push({
+                contentHash: hashContent(content),
+                episode: {
+                    content: parsed.body,
+                    date: parsed.frontmatter.date,
+                    title: parsed.frontmatter.title
+                },
+                key: buildSourceKey({
+                    provider: CONSOLIDATION_PROVIDER,
+                    sessionId: file.relativePath.replace(/\.md$/u, ""),
+                    type: "episode"
+                }),
+                project: parsed.frontmatter.project
+            });
+        }
+        catch (error) {
+            quarantined.push({
+                contentHash: hashContent(`${file.absolutePath}\n${content}`),
+                path: file.absolutePath,
+                reason: error instanceof Error ? error.message : "Unknown error"
+            });
+        }
+    }
+    return { quarantined, sources };
+}
+function buildNewPage(project, date, section) {
+    return [
+        "---",
+        `title: ${project}`,
+        "type: entity",
+        `tags: [${project}]`,
+        "sources: []",
+        `created: ${date}`,
+        `updated: ${date}`,
+        "---",
+        "",
+        `# ${project}`,
+        "",
+        `## Consolidation ${date}`,
+        "",
+        section.trim(),
+        ""
+    ].join("\n");
+}
+async function addToIndex(indexPath, project, date) {
+    let index = "";
+    try {
+        index = await readFile(indexPath, "utf8");
+    }
+    catch (error) {
+        if (!isMissingFileError(error)) {
+            throw error;
+        }
+    }
+    const entry = `- [[${project}]] — Project page created by weekly consolidation on ${date}.`;
+    const lines = index.length > 0 ? index.split("\n") : ["# Wiki Index", ""];
+    const headingIndex = lines.findIndex((line) => line.trim() === "## Entities");
+    if (headingIndex === -1) {
+        lines.push("", "## Entities", "", entry);
+    }
+    else {
+        let insertAt = lines.length;
+        for (let cursor = headingIndex + 1; cursor < lines.length; cursor += 1) {
+            if (lines[cursor].startsWith("## ")) {
+                insertAt = cursor;
+                break;
+            }
+        }
+        while (insertAt > headingIndex + 1 && lines[insertAt - 1].trim().length === 0) {
+            insertAt -= 1;
+        }
+        lines.splice(insertAt, 0, entry);
+    }
+    await writeFileAtomic(indexPath, `${lines.join("\n").replace(/\n+$/u, "")}\n`);
+}
+function buildLogEntry(input) {
+    const lines = [`## [${input.date}] consolidation | weekly roll-up`];
+    if (input.reportLink !== undefined) {
+        lines.push(`Run report: [[${input.reportLink}]]`);
+    }
+    if (input.pagesCreated.length > 0) {
+        lines.push(`Pages created: ${input.pagesCreated.map((page) => `[[${page}]]`).join(", ")}`);
+    }
+    if (input.pagesUpdated.length > 0) {
+        lines.push(`Pages updated: ${input.pagesUpdated.map((page) => `[[${page}]]`).join(", ")}`);
+    }
+    for (const failure of input.failures) {
+        lines.push(`Failed project (manifest not advanced): ${failure.project} — ${failure.reason}`);
+    }
+    for (const artifact of input.freshQuarantined) {
+        lines.push(`Quarantined episode: ${artifact.path} — ${artifact.reason}`);
+    }
+    return lines.join("\n");
+}
+function isMissingFileError(error) {
+    return (error instanceof Error &&
+        "code" in error &&
+        typeof error.code === "string" &&
+        error.code === "ENOENT");
+}

package/dist/episode-file.js

@@ -0,0 +1,61 @@
+import { z } from "zod";
+import { isProjectSegment } from "./slug.js";
+const episodeFrontmatterSchema = z.object({
+    date: z.string().regex(/^\d{4}-\d{2}-\d{2}$/u),
+    project: z
+        .string()
+        .refine(isProjectSegment, {
+        message: "project must be a kebab-case slug or the reserved _inbox"
+    }),
+    title: z.string().min(1),
+    type: z.literal("episode")
+});
+export function parseEpisodeFile(content) {
+    // Editors on other machines may save episodes with a BOM or CRLF endings;
+    // both would otherwise quarantine a perfectly valid file.
+    const lines = content.replace(/^\uFEFF/u, "").split(/\r?\n/u);
+    if (lines[0] !== "---") {
+        throw new Error("Episode is missing its opening frontmatter fence");
+    }
+    const closingIndex = lines.indexOf("---", 1);
+    if (closingIndex === -1) {
+        throw new Error("Episode is missing its closing frontmatter fence");
+    }
+    const fields = {};
+    for (const line of lines.slice(1, closingIndex)) {
+        if (line.trim().length === 0) {
+            continue;
+        }
+        const separatorIndex = line.indexOf(":");
+        if (separatorIndex === -1) {
+            throw new Error(`Malformed frontmatter line: ${line}`);
+        }
+        const key = line.slice(0, separatorIndex).trim();
+        // Last-wins merging would let a later line silently shadow an already
+        // validated value; duplicates are author mistakes worth surfacing.
+        if (Object.hasOwn(fields, key)) {
+            throw new Error(`Duplicate frontmatter key: ${key}`);
+        }
+        fields[key] = decodeScalar(line.slice(separatorIndex + 1).trim());
+    }
+    return {
+        body: lines.slice(closingIndex + 1).join("\n").replace(/^\n+/u, ""),
+        frontmatter: episodeFrontmatterSchema.parse(fields)
+    };
+}
+// The episode writer JSON-quotes backend-controlled scalars (titles); decode
+// them back, falling back to the raw value for plain scalars.
+function decodeScalar(value) {
+    if (value.startsWith('"') && value.endsWith('"')) {
+        try {
+            const decoded = JSON.parse(value);
+            if (typeof decoded === "string") {
+                return decoded;
+            }
+        }
+        catch {
+            return value;
+        }
+    }
+    return value;
+}