obsidian-second-brain 0.1.0

package/dist/episode-patch.js

@@ -0,0 +1,28 @@
+import { z } from "zod";
+import { isProjectSegment } from "./slug.js";
+const episodeSchema = z.object({
+    content: z.string().trim().min(1),
+    project: z
+        .string()
+        .refine(isProjectSegment, {
+        message: "project must be a kebab-case slug or the reserved _inbox"
+    }),
+    title: z
+        .string()
+        .trim()
+        .min(1)
+        .max(120)
+        .regex(/^[^\n\r]+$/u, "title must be a single line")
+});
+export const episodePatchSchema = z.object({
+    episodes: z.array(episodeSchema)
+});
+export function parseEpisodePatch(raw) {
+    const parsed = JSON.parse(stripJsonFences(raw));
+    return episodePatchSchema.parse(parsed);
+}
+function stripJsonFences(raw) {
+    const trimmed = raw.trim();
+    const fenced = /^```(?:json)?\s*\n([\s\S]*?)\n```$/u.exec(trimmed);
+    return fenced ? fenced[1] : trimmed;
+}

package/dist/episode-prompt.js

@@ -0,0 +1,43 @@
+const MAX_SOURCE_CHARS = 16_000;
+export async function buildEpisodePrompt(input, readSource) {
+    const sections = await Promise.all(input.sources.map(async (source) => {
+        const content = await readSource(source.path);
+        const truncated = content.length > MAX_SOURCE_CHARS
+            ? `${content.slice(0, MAX_SOURCE_CHARS)}\n[truncated]`
+            : content;
+        return [
+            `### Source (${source.type}, session ${source.sessionId})`,
+            "",
+            truncated
+        ].join("\n");
+    }));
+    return [
+        "You are the ingest step of an LLM-maintained wiki. Distill the curated",
+        `task artifacts below from the project "${input.project}" into episodic`,
+        "notes for the wiki's quarantined episodes folder.",
+        "",
+        "Rules:",
+        `- Respond with ONLY strict JSON: {"episodes": [{"project": "${input.project}",`,
+        '  "title": "...", "content": "..."}]} — no prose, no markdown fences.',
+        `- Every episode's project MUST be exactly "${input.project}".`,
+        "- One episode per coherent task; merge a plan/output pair for the same",
+        "  session into one episode.",
+        "- title: short, specific, factual (max 120 chars, single line).",
+        "- content: Obsidian-flavored markdown. Capture what was done, decisions",
+        "  made and why, and follow-ups left open. Cite session ids. Use",
+        "  [[wikilinks]] only for project-specific entities. No emojis.",
+        "- Do not invent facts that are not in the sources.",
+        "",
+        "## Sources",
+        "",
+        sections.join("\n\n---\n\n")
+    ].join("\n");
+}
+export function assertEpisodesMatchProject(episodes, project) {
+    const foreign = episodes.filter((episode) => episode.project !== project);
+    if (foreign.length > 0) {
+        throw new Error(`Backend returned episodes for the wrong project: ${foreign
+            .map((episode) => episode.project)
+            .join(", ")} (expected ${project})`);
+    }
+}

package/dist/filesystem.js

@@ -0,0 +1,86 @@
+import { randomUUID } from "node:crypto";
+import { lstat, mkdir, readdir, rename, rm, unlink, writeFile } from "node:fs/promises";
+import { join, relative } from "node:path";
+export async function ensureDirectory(directoryPath) {
+    await mkdir(directoryPath, { recursive: true });
+}
+export async function writeFileAtomic(filePath, content) {
+    const tempPath = `${filePath}.tmp-${process.pid}-${randomUUID()}`;
+    try {
+        await writeFile(tempPath, content);
+        await rename(tempPath, filePath);
+    }
+    catch (error) {
+        await rm(tempPath, { force: true });
+        throw error;
+    }
+}
+export async function ensureWritableFilePath(filePath) {
+    try {
+        await removePathEntry(filePath);
+    }
+    catch (error) {
+        if (!isMissingPathError(error)) {
+            throw error;
+        }
+    }
+}
+export async function removePathEntry(path) {
+    const stat = await lstat(path);
+    if (stat.isSymbolicLink()) {
+        await unlink(path);
+        return;
+    }
+    if (stat.isDirectory()) {
+        await rm(path, { force: true, recursive: true });
+        return;
+    }
+    await rm(path, { force: true });
+}
+export async function collectFiles(root) {
+    const files = [];
+    await walkDirectory(root, async (absolutePath) => {
+        files.push({
+            absolutePath,
+            relativePath: relative(root, absolutePath)
+        });
+    });
+    return files;
+}
+export async function cleanupEmptyDirectories(root) {
+    await removeEmptyDirectories(root, false);
+}
+async function walkDirectory(directoryPath, visitFile) {
+    const entries = await readdir(directoryPath, { withFileTypes: true });
+    for (const entry of entries) {
+        const absolutePath = join(directoryPath, entry.name);
+        if (entry.isDirectory()) {
+            await walkDirectory(absolutePath, visitFile);
+            continue;
+        }
+        if (entry.isFile() || entry.isSymbolicLink()) {
+            await visitFile(absolutePath);
+        }
+    }
+}
+async function removeEmptyDirectories(directoryPath, removeCurrentDirectory) {
+    const entries = await readdir(directoryPath, { withFileTypes: true });
+    for (const entry of entries) {
+        if (!entry.isDirectory()) {
+            continue;
+        }
+        await removeEmptyDirectories(join(directoryPath, entry.name), true);
+    }
+    const remainingEntries = await readdir(directoryPath);
+    const isEmpty = remainingEntries.length === 0;
+    if (isEmpty && removeCurrentDirectory) {
+        await rm(directoryPath, { recursive: true });
+    }
+    return isEmpty;
+}
+function isMissingPathError(error) {
+    return (error instanceof Error &&
+        "code" in error &&
+        typeof error.code === "string" &&
+        error.code === "ENOENT");
+}

package/dist/git.js

@@ -0,0 +1,61 @@
+import { runCommand, runCommandAllowFailure } from "./shell.js";
+export async function ensureGitRepository(projectRoot) {
+    const result = await runCommandAllowFailure("git", ["rev-parse", "--is-inside-work-tree"], projectRoot);
+    if (result.exitCode !== 0 || result.stdout.trim() !== "true") {
+        throw new Error(`Not a git repository: ${projectRoot}`);
+    }
+}
+export async function resolveDefaultBranch(projectRoot) {
+    const mainExists = await branchExists(projectRoot, "main");
+    if (mainExists) {
+        return "main";
+    }
+    const masterExists = await branchExists(projectRoot, "master");
+    if (masterExists) {
+        return "master";
+    }
+    throw new Error("Could not find local main or master branch");
+}
+export async function getCurrentBranch(projectRoot) {
+    const result = await runCommand("git", ["rev-parse", "--abbrev-ref", "HEAD"], projectRoot);
+    return result.stdout.trim();
+}
+export async function checkoutBranch(projectRoot, branchName) {
+    const currentBranch = await getCurrentBranch(projectRoot);
+    if (currentBranch === branchName) {
+        return;
+    }
+    await runCommand("git", ["checkout", branchName], projectRoot);
+}
+export async function pullFastForward(projectRoot) {
+    await runCommand("git", ["pull", "--ff-only"], projectRoot);
+}
+export async function getDirtyStatus(projectRoot) {
+    const result = await runCommand("git", ["status", "--porcelain=v1", "-z"], projectRoot);
+    const entries = result.stdout.split("\0").filter((entry) => entry.length > 0);
+    const dirtyEntries = [];
+    for (let index = 0; index < entries.length; index += 1) {
+        const entry = entries[index];
+        const code = entry.slice(0, 2);
+        const path = entry.slice(3);
+        let originalPath;
+        if (code.includes("R") || code.includes("C")) {
+            originalPath = entries[index + 1];
+            index += 1;
+        }
+        dirtyEntries.push({
+            code,
+            originalPath,
+            path
+        });
+    }
+    return dirtyEntries;
+}
+export async function listSyncFiles(projectRoot) {
+    const result = await runCommand("git", ["ls-files", "-z", "--cached", "--others", "--exclude-standard"], projectRoot);
+    return result.stdout.split("\0").filter((entry) => entry.length > 0);
+}
+async function branchExists(projectRoot, branchName) {
+    const result = await runCommandAllowFailure("git", ["show-ref", "--verify", "--quiet", `refs/heads/${branchName}`], projectRoot);
+    return result.exitCode === 0;
+}

package/dist/ingest-writer.js

@@ -0,0 +1,86 @@
+import { lstat, readFile } from "node:fs/promises";
+import { resolve, sep } from "node:path";
+import { ensureDirectory, writeFileAtomic } from "./filesystem.js";
+import { isProjectSegment, slugify } from "./slug.js";
+export async function writeEpisodes(input) {
+    if (input.episodes.length === 0) {
+        return [];
+    }
+    await ensureDirectory(input.episodesRoot);
+    const root = resolve(input.episodesRoot);
+    const written = [];
+    const claimed = new Set();
+    for (const episode of input.episodes) {
+        if (!isProjectSegment(episode.project)) {
+            throw new Error(`Episode project is not a valid segment: ${episode.project}`);
+        }
+        const titleSlug = slugify(episode.title);
+        if (titleSlug.length === 0) {
+            throw new Error(`Episode title does not slugify: ${episode.title}`);
+        }
+        const baseName = `${input.date}-${episode.project}-${titleSlug}`;
+        const episodePath = await claimEpisodePath(root, baseName, claimed);
+        if (!episodePath.startsWith(`${root}${sep}`)) {
+            throw new Error(`Episode path escapes the episodes root: ${episodePath}`);
+        }
+        // JSON-quoting is valid YAML and neutralizes newlines, colons, and quotes
+        // in backend-controlled values — frontmatter keys cannot be injected.
+        const content = [
+            "---",
+            `title: ${JSON.stringify(episode.title)}`,
+            `project: ${episode.project}`,
+            `date: ${input.date}`,
+            "type: episode",
+            "---",
+            "",
+            episode.content.trim(),
+            ""
+        ].join("\n");
+        await writeFileAtomic(episodePath, content);
+        written.push(episodePath);
+    }
+    return written;
+}
+export async function appendLogEntry(logPath, entry) {
+    let existing = "";
+    try {
+        existing = await readFile(logPath, "utf8");
+    }
+    catch (error) {
+        if (!isMissingFileError(error)) {
+            throw error;
+        }
+    }
+    const base = existing.length > 0 ? `${existing.replace(/\n+$/u, "")}\n\n` : "";
+    await writeFileAtomic(logPath, `${base}${entry.trim()}\n`);
+}
+async function claimEpisodePath(root, baseName, claimed) {
+    for (let suffix = 1; suffix <= 50; suffix += 1) {
+        const name = suffix === 1 ? `${baseName}.md` : `${baseName}-${suffix}.md`;
+        const candidate = resolve(root, name);
+        if (claimed.has(candidate) || (await pathExists(candidate))) {
+            continue;
+        }
+        claimed.add(candidate);
+        return candidate;
+    }
+    throw new Error(`Could not find a free episode filename for ${baseName}`);
+}
+async function pathExists(path) {
+    try {
+        await lstat(path);
+        return true;
+    }
+    catch (error) {
+        if (isMissingFileError(error)) {
+            return false;
+        }
+        return true;
+    }
+}
+function isMissingFileError(error) {
+    return (error instanceof Error &&
+        "code" in error &&
+        typeof error.code === "string" &&
+        error.code === "ENOENT");
+}

package/dist/ingest.js

@@ -0,0 +1,217 @@
+import { lstat, readFile, realpath } from "node:fs/promises";
+import { sep } from "node:path";
+import { parseArtifact } from "./artifact-frontmatter.js";
+import { hashContent } from "./changeset.js";
+import { classifySources, groupByProject, hasWork, renderedFileToIngestSource, toManifestEntries } from "./classify.js";
+import { collectFiles } from "./filesystem.js";
+import { appendLogEntry, writeEpisodes } from "./ingest-writer.js";
+import { loadManifest, saveManifest, withManifestEntries } from "./manifest.js";
+// Artifacts are provider-agnostic captures; they live in their own key
+// namespace so they can never collide with provider transcript entries.
+const ARTIFACT_PROVIDER = "artifact";
+// Per-call source cap keeps the synthesized prompt within argv limits; a
+// project with more work is processed in batches within the same run.
+const SYNTHESIS_BATCH_SIZE = 8;
+export async function runIngest(input) {
+    const { quarantined, sources: artifactSources } = await scanArtifacts(input.artifactsRoot);
+    const transcriptSources = input.renderedFiles.map(renderedFileToIngestSource);
+    const manifest = await loadManifest(input.manifestPath);
+    const classified = classifySources(manifest, [
+        ...artifactSources,
+        ...transcriptSources
+    ]);
+    // A broken artifact stays on disk forever; report it once per content
+    // version instead of spamming the log every hour.
+    const freshQuarantined = quarantined.filter((artifact) => manifest.entries[quarantineKey(artifact)] === undefined);
+    if (!hasWork(classified) && freshQuarantined.length === 0) {
+        return {
+            artifactsIngested: [],
+            episodePaths: [],
+            failures: [],
+            quarantined: [],
+            skipped: true,
+            trackedTranscripts: 0,
+            transcripts: { added: [], changed: [] }
+        };
+    }
+    const workSources = [...classified.added, ...classified.changed];
+    const artifactWork = workSources.filter((source) => source.type !== "transcript");
+    const transcriptWork = workSources.filter((source) => source.type === "transcript");
+    const failures = [];
+    const synthesized = [];
+    for (const [project, sources] of groupByProject(artifactWork)) {
+        for (const batch of chunk(sources, SYNTHESIS_BATCH_SIZE)) {
+            try {
+                const output = await input.backend.synthesize({ project, sources: batch });
+                synthesized.push({ episodes: output.episodes, sources: batch });
+            }
+            catch (error) {
+                failures.push({
+                    project,
+                    reason: error instanceof Error ? error.message : "Unknown error"
+                });
+            }
+        }
+    }
+    const now = input.now();
+    const date = now.toISOString().slice(0, 10);
+    const episodes = synthesized.flatMap((group) => group.episodes);
+    // Transactional order (design 4.2): pages -> log -> manifest last.
+    const episodePaths = await writeEpisodes({
+        date,
+        episodes,
+        episodesRoot: input.episodesRoot
+    });
+    const transcriptsAdded = classified.added.filter((source) => source.type === "transcript");
+    const transcriptsChanged = classified.changed.filter((source) => source.type === "transcript");
+    const artifactsIngested = synthesized.flatMap((group) => group.sources);
+    let report;
+    // The report is a wiki page: it belongs to the pages step, before log and
+    // manifest, so a crashed run redoes it instead of losing it forever.
+    if (input.reportWriter) {
+        report = await input.reportWriter.addIngest({
+            artifacts: artifactsIngested,
+            // writeEpisodes preserves input order, so paths zip with drafts.
+            episodes: episodePaths.map((path, index) => ({
+                path,
+                project: episodes[index].project
+            })),
+            failures,
+            quarantined: freshQuarantined,
+            transcriptsAdded,
+            transcriptsChanged
+        });
+    }
+    if (episodePaths.length > 0 ||
+        failures.length > 0 ||
+        freshQuarantined.length > 0 ||
+        report !== undefined) {
+        await appendLogEntry(input.logPath, buildLogEntry({
+            date,
+            episodePaths,
+            failures,
+            quarantined: freshQuarantined,
+            reportLink: report?.wikiLink
+        }));
+    }
+    const ingestedAt = now.toISOString();
+    const ingestedSources = [...artifactsIngested, ...transcriptWork];
+    const quarantineEntries = Object.fromEntries(freshQuarantined.map((artifact) => [
+        quarantineKey(artifact),
+        { contentHash: artifact.contentHash, ingestedAt }
+    ]));
+    const nextManifest = withManifestEntries(manifest, {
+        ...toManifestEntries(ingestedSources, ingestedAt),
+        ...quarantineEntries
+    });
+    await saveManifest(input.manifestPath, nextManifest);
+    return {
+        artifactsIngested,
+        episodePaths,
+        failures,
+        quarantined: freshQuarantined,
+        ...(report ? { reportPath: report.path } : {}),
+        skipped: false,
+        trackedTranscripts: transcriptWork.length,
+        transcripts: { added: transcriptsAdded, changed: transcriptsChanged }
+    };
+}
+function quarantineKey(artifact) {
+    return `quarantine:artifact:${artifact.contentHash}`;
+}
+function chunk(items, size) {
+    const batches = [];
+    for (let index = 0; index < items.length; index += size) {
+        batches.push(items.slice(index, index + size));
+    }
+    return batches;
+}
+async function scanArtifacts(artifactsRoot) {
+    let files;
+    try {
+        files = await collectFiles(artifactsRoot);
+    }
+    catch (error) {
+        if (isMissingFileError(error)) {
+            return { quarantined: [], sources: [] };
+        }
+        throw error;
+    }
+    const quarantined = [];
+    const sources = [];
+    const resolvedRoot = await realpath(artifactsRoot);
+    for (const file of files) {
+        if (!file.relativePath.endsWith(".md")) {
+            continue;
+        }
+        // Ingest-side re-validation (design 7.6): refuse symlinks and anything
+        // that resolves outside the artifacts root before reading a byte.
+        const stat = await lstat(file.absolutePath);
+        if (stat.isSymbolicLink()) {
+            quarantined.push({
+                contentHash: hashContent(`symlink:${file.absolutePath}`),
+                path: file.absolutePath,
+                reason: "Refused symlinked artifact"
+            });
+            continue;
+        }
+        const resolved = await realpath(file.absolutePath);
+        if (!resolved.startsWith(`${resolvedRoot}${sep}`)) {
+            quarantined.push({
+                contentHash: hashContent(`escape:${file.absolutePath}`),
+                path: file.absolutePath,
+                reason: "Artifact resolves outside the artifacts root"
+            });
+            continue;
+        }
+        let content = "";
+        try {
+            content = await readFile(file.absolutePath, "utf8");
+            const parsed = parseArtifact(content);
+            sources.push({
+                contentHash: hashContent(content),
+                path: file.absolutePath,
+                project: parsed.frontmatter.project,
+                provider: ARTIFACT_PROVIDER,
+                sessionId: parsed.frontmatter.session_id,
+                type: parsed.frontmatter.type
+            });
+        }
+        catch (error) {
+            quarantined.push({
+                contentHash: hashContent(`${file.absolutePath}\n${content}`),
+                path: file.absolutePath,
+                reason: error instanceof Error ? error.message : "Unknown error"
+            });
+        }
+    }
+    return { quarantined, sources };
+}
+function buildLogEntry(input) {
+    const lines = [`## [${input.date}] ingest | hourly episodes`];
+    if (input.reportLink !== undefined) {
+        lines.push(`Run report: [[${input.reportLink}]]`);
+    }
+    if (input.episodePaths.length > 0) {
+        const links = input.episodePaths
+            .map((path) => {
+            const name = path.split("/").at(-1)?.replace(/\.md$/u, "") ?? path;
+            return `[[episodes/${name}]]`;
+        })
+            .join(", ");
+        lines.push(`Episodes created: ${links}`);
+    }
+    for (const failure of input.failures) {
+        lines.push(`Failed project (manifest not advanced): ${failure.project} — ${failure.reason}`);
+    }
+    for (const artifact of input.quarantined) {
+        lines.push(`Quarantined artifact: ${artifact.path} — ${artifact.reason}`);
+    }
+    return lines.join("\n");
+}
+function isMissingFileError(error) {
+    return (error instanceof Error &&
+        "code" in error &&
+        typeof error.code === "string" &&
+        error.code === "ENOENT");
+}