chapterhouse 0.6.0 → 0.8.0

package/dist/wiki/index-manager.test.js

@@ -1,160 +1,153 @@
 import assert from "node:assert/strict";
-import { mkdirSync, rmSync } from "node:fs";
+import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
 import { join } from "node:path";
 import test from "node:test";
-const repoRoot = process.cwd();
-const sandboxRoot = join(repoRoot, ".test-work", `wiki-index-${process.pid}`);
-process.env.CHAPTERHOUSE_HOME = sandboxRoot;
-async function loadModules() {
+// Sandbox: every test gets a fresh CHAPTERHOUSE_HOME
+function makeSandbox() {
+    const dir = mkdtempSync(join(process.cwd(), ".test-work", "wiki-idx-"));
+    process.env.CHAPTERHOUSE_HOME = dir;
+    return dir;
+}
+async function loadModules(sandbox) {
     const nonce = `${Date.now()}-${Math.random()}`;
-    const indexManager = await import(new URL(`./index-manager.js?case=${nonce}`, import.meta.url).href);
-    const wikiFs = await import(new URL(`./fs.js?case=${nonce}`, import.meta.url).href);
+    const indexManager = await import(new URL(`./index-manager.js?c=${nonce}`, import.meta.url).href);
+    const wikiFs = await import(new URL(`./fs.js?c=${nonce}`, import.meta.url).href);
     return { indexManager, wikiFs };
 }
-function resetSandbox() {
-    mkdirSync(join(repoRoot, ".test-work"), { recursive: true });
-    rmSync(sandboxRoot, { recursive: true, force: true });
-}
-test.beforeEach(() => {
-    resetSandbox();
+test.before(() => {
+    mkdirSync(join(process.cwd(), ".test-work"), { recursive: true });
+});
+test("wikiSearch returns FTS5 results for matching query", async () => {
+    const sandbox = makeSandbox();
+    try {
+        const { indexManager, wikiFs } = await loadModules(sandbox);
+        wikiFs.writePage("pages/topics/rust/index.md", "---\ntitle: Rust Programming\nsummary: Systems programming with async support\ntags: [rust, async]\nupdated: 2026-05-01\n---\n\n# Rust\n\nSystems language.\n");
+        wikiFs.writePage("pages/topics/typescript/index.md", "---\ntitle: TypeScript\nsummary: Typed JavaScript for large projects\ntags: [ts, web]\nupdated: 2026-05-02\n---\n\n# TypeScript\n\nJS with types.\n");
+        indexManager.rebuildWikiIndex();
+        const results = indexManager.wikiSearch("rust async");
+        assert.ok(results.length > 0, "Should return results for 'rust async'");
+        assert.ok(results.some((r) => r.path === "pages/topics/rust/index.md"), "Should include rust page");
+    }
+    finally {
+        rmSync(sandbox, { recursive: true, force: true });
+    }
 });
-test.after(() => {
-    rmSync(sandboxRoot, { recursive: true, force: true });
+test("wikiSearch empty query returns most recently updated pages", async () => {
+    const sandbox = makeSandbox();
+    try {
+        const { indexManager, wikiFs } = await loadModules(sandbox);
+        wikiFs.writePage("pages/topics/alpha/index.md", "---\ntitle: Alpha\nsummary: First topic\nupdated: 2026-01-01\n---\n\n# Alpha\n");
+        wikiFs.writePage("pages/topics/beta/index.md", "---\ntitle: Beta\nsummary: Second topic\nupdated: 2026-05-14\n---\n\n# Beta\n");
+        indexManager.rebuildWikiIndex();
+        const results = indexManager.wikiSearch("", 10);
+        assert.ok(results.length >= 2, "Should return pages for empty query");
+        // Most recent first
+        const betaIdx = results.findIndex((r) => r.path === "pages/topics/beta/index.md");
+        const alphaIdx = results.findIndex((r) => r.path === "pages/topics/alpha/index.md");
+        assert.ok(betaIdx < alphaIdx, "More recently updated page should come first");
+    }
+    finally {
+        rmSync(sandbox, { recursive: true, force: true });
+    }
 });
-test("parseIndex reads sections, summaries, tags, and updated dates", async () => {
-    const { indexManager, wikiFs } = await loadModules();
-    wikiFs.writeIndexFile(`# Wiki Index\n\n## People\n\n- [Ada Lovelace](pages/people/ada.md) — Platform owner | tags: engineer, compiler | updated: 2026-05-01\n\n## Projects\n\n- [Roadmap](pages/projects/roadmap.md) - Shared priorities\n`);
-    assert.deepEqual(indexManager.parseIndex(), [
-        {
-            path: "pages/people/ada.md",
-            title: "Ada Lovelace",
-            summary: "Platform owner",
-            section: "People",
-            tags: ["engineer", "compiler"],
-            updated: "2026-05-01",
-        },
-        {
-            path: "pages/projects/roadmap.md",
-            title: "Roadmap",
-            summary: "Shared priorities",
-            section: "Projects",
-            tags: undefined,
-            updated: undefined,
-        },
-    ]);
+test("rebuildWikiIndex populates wiki_pages from filesystem", async () => {
+    const sandbox = makeSandbox();
+    try {
+        const { indexManager, wikiFs } = await loadModules(sandbox);
+        wikiFs.writePage("pages/projects/chapterhouse/index.md", "---\ntitle: Chapterhouse\nsummary: AI orchestrator\ntags: [ai, orchestration]\nupdated: 2026-05-10\n---\n\n# Chapterhouse\n");
+        wikiFs.writePage("pages/projects/chapterhouse/decisions.md", "---\ntitle: Decisions\nsummary: Architectural decisions\nupdated: 2026-05-09\n---\n\n# Decisions\n");
+        indexManager.rebuildWikiIndex();
+        const entries = indexManager.parseIndex();
+        const paths = entries.map((e) => e.path).sort();
+        assert.ok(paths.includes("pages/projects/chapterhouse/index.md"), "Should include index.md");
+        assert.ok(paths.includes("pages/projects/chapterhouse/decisions.md"), "Should include decisions.md");
+    }
+    finally {
+        rmSync(sandbox, { recursive: true, force: true });
+    }
 });
-test("buildIndexEntryForPage treats frontmatter summary as the canonical index summary", async () => {
-    const { indexManager, wikiFs } = await loadModules();
-    wikiFs.writePage("pages/shared/runbooks/deploy.md", `---\ntitle: Deploy Runbook\nsummary: Production deployment checklist\ntags: [ops, release]\nupdated: 2026-05-04\n---\n\n# Deploy\n\n${"Deploy carefully ".repeat(20)}\n`);
-    const entry = indexManager.buildIndexEntryForPage("pages/shared/runbooks/deploy.md");
-    assert.deepEqual(entry, {
-        path: "pages/shared/runbooks/deploy.md",
-        title: "Deploy Runbook",
-        summary: "Production deployment checklist",
-        section: "Knowledge",
-        tags: ["ops", "release"],
-        updated: "2026-05-04",
-    });
+test("upsertWikiPage inserts and updates correctly", async () => {
+    const sandbox = makeSandbox();
+    try {
+        const { indexManager } = await loadModules(sandbox);
+        indexManager.upsertWikiPage("pages/people/ada/index.md", { title: "Ada Lovelace", summary: "Mathematician", tags: ["math"], updated: "2026-05-01", metadata: {} }, "First programmer");
+        const results = indexManager.wikiSearch("Ada");
+        assert.ok(results.some((r) => r.title === "Ada Lovelace"), "Should find Ada");
+        // Update
+        indexManager.upsertWikiPage("pages/people/ada/index.md", { title: "Ada Lovelace", summary: "Mathematician and programmer", tags: ["math", "history"], updated: "2026-05-02", metadata: {} }, "First programmer and mathematician");
+        const updated = indexManager.wikiSearch("programmer");
+        assert.ok(updated.length > 0, "Should find updated page");
+    }
+    finally {
+        rmSync(sandbox, { recursive: true, force: true });
+    }
 });
-test("parseIndex self-heals an empty index from on-disk pages", async () => {
-    const { indexManager, wikiFs } = await loadModules();
-    const today = new Date().toISOString().slice(0, 10);
-    wikiFs.writePage("pages/team/vision.md", "# Vision\n\nShared direction for the team.\n");
-    wikiFs.writeIndexFile("# Wiki Index\n\n");
-    const entries = indexManager.parseIndex();
-    assert.deepEqual(entries, [
-        {
-            path: "pages/index.md",
-            title: "Wiki",
-            summary: "Index of all wiki pages.",
-            section: "Knowledge",
-            tags: undefined,
-            updated: today,
-        },
-        {
-            path: "pages/team/vision.md",
-            title: "Vision",
-            summary: "Shared direction for the team.",
-            section: "Knowledge",
-            tags: undefined,
-            updated: undefined,
-        },
-    ]);
-    assert.match(wikiFs.readIndexFile(), /\[Vision\]\(pages\/team\/vision\.md\)/);
+test("FTS search returns results under 50ms", async () => {
+    const sandbox = makeSandbox();
+    try {
+        const { indexManager, wikiFs } = await loadModules(sandbox);
+        // Populate with 20 pages
+        for (let i = 0; i < 20; i++) {
+            wikiFs.writePage(`pages/topics/topic-${i}/index.md`, `---\ntitle: Topic ${i}\nsummary: Description for topic ${i} covering various subjects\ntags: [topic${i}]\nupdated: 2026-05-01\n---\n\n# Topic ${i}\n\nContent.\n`);
+        }
+        indexManager.rebuildWikiIndex();
+        const start = Date.now();
+        const results = indexManager.wikiSearch("topic description");
+        const elapsed = Date.now() - start;
+        assert.ok(results.length > 0, "Should return results");
+        assert.ok(elapsed < 50, `FTS search should complete in <50ms, took ${elapsed}ms`);
+    }
+    finally {
+        rmSync(sandbox, { recursive: true, force: true });
+    }
 });
-test("the index renders entity categories as topic groups with nested facet pages", async () => {
-    const { indexManager, wikiFs } = await loadModules();
-    wikiFs.writePage("pages/projects/chapterhouse/index.md", "---\ntitle: Chapterhouse\nupdated: 2026-05-09\n---\n\n# Chapterhouse\n\nThe per-session orchestrator.\n");
-    wikiFs.writePage("pages/projects/chapterhouse/decisions.md", "---\ntitle: Chapterhouse Decisions\nupdated: 2026-05-09\n---\n\n# Decisions\n\nUse SSE for streaming.\n");
-    wikiFs.writePage("pages/preferences.md", "---\ntitle: Preferences\n---\n\n# Preferences\n\nDark mode.\n");
-    indexManager.rebuildIndexFromPages();
-    const index = wikiFs.readIndexFile();
-    assert.match(index, /## Projects/);
-    assert.match(index, /^- \[Chapterhouse\]\(pages\/projects\/chapterhouse\/index\.md\) — /m);
-    assert.match(index, /^ {2}- \[Chapterhouse Decisions\]\(pages\/projects\/chapterhouse\/decisions\.md\) — /m);
-    assert.match(index, /## Preferences/);
-    // Indented facet bullets must still round-trip through parseIndex.
-    const paths = indexManager.parseIndex().map((entry) => entry.path).sort();
-    assert.deepEqual(paths, [
-        "pages/preferences.md",
-        "pages/projects/chapterhouse/decisions.md",
-        "pages/projects/chapterhouse/index.md",
-    ]);
+test("removeFromIndex removes from wiki_pages", async () => {
+    const sandbox = makeSandbox();
+    try {
+        const { indexManager } = await loadModules(sandbox);
+        indexManager.upsertWikiPage("pages/people/test/index.md", { title: "Test Person", summary: "A test", tags: [], updated: "2026-05-01", metadata: {} }, "A test");
+        const before = indexManager.wikiSearch("Test Person");
+        assert.ok(before.length > 0, "Should exist before removal");
+        const removed = indexManager.removeFromIndex("pages/people/test/index.md");
+        assert.equal(removed, true);
+        const after = indexManager.wikiSearch("Test Person");
+        assert.equal(after.length, 0, "Should not exist after removal");
+    }
+    finally {
+        rmSync(sandbox, { recursive: true, force: true });
+    }
 });
-test("searchIndex ranks strong metadata matches and falls back to page bodies", async () => {
-    const { indexManager, wikiFs } = await loadModules();
-    wikiFs.writePage("pages/team/api.md", "# API\n\nObservability budget and telemetry plans.\n");
-    wikiFs.writePage("pages/team/ops.md", "# Ops\n\nDaily operational notes.\n");
-    indexManager.writeIndex([
-        {
-            path: "pages/team/api.md",
-            title: "API",
-            summary: "Status of the platform",
-            section: "Team",
-            tags: ["api"],
-            updated: new Date().toISOString().slice(0, 10),
-        },
-        {
-            path: "pages/team/ops.md",
-            title: "Operations",
-            summary: "Runbooks and incident work",
-            section: "Team",
-        },
-    ]);
-    const metadataHit = indexManager.searchIndex("api", 1);
-    const bodyFallback = indexManager.searchIndex("telemetry", 1);
-    assert.deepEqual(metadataHit.map((entry) => entry.path), ["pages/team/api.md"]);
-    assert.deepEqual(bodyFallback.map((entry) => entry.path), ["pages/team/api.md"]);
+test("searchIndex delegates to wikiSearch and returns IndexEntry shape", async () => {
+    const sandbox = makeSandbox();
+    try {
+        const { indexManager, wikiFs } = await loadModules(sandbox);
+        wikiFs.writePage("pages/team/api.md", "---\ntitle: API Docs\nsummary: API documentation\ntags: [api]\nupdated: 2026-05-01\n---\n\n# API\n");
+        indexManager.rebuildWikiIndex();
+        const results = indexManager.searchIndex("api");
+        assert.ok(results.length > 0);
+        assert.ok("section" in results[0], "Should have section field");
+        assert.ok("title" in results[0], "Should have title field");
+    }
+    finally {
+        rmSync(sandbox, { recursive: true, force: true });
+    }
 });
-test("addToIndex, removeFromIndex, and getIndexSummary keep the catalog in sync", async () => {
-    const { indexManager } = await loadModules();
-    const today = new Date().toISOString().slice(0, 10);
-    indexManager.addToIndex({
-        path: "pages/people/ada.md",
-        title: "Ada Lovelace",
-        summary: "Owns release quality",
-        section: "People",
-        tags: ["qa"],
-        updated: "2026-05-05",
-    });
-    indexManager.addToIndex({
-        path: "pages/projects/launch.md",
-        title: "Launch",
-        summary: "Tracks release milestones",
-        section: "Projects",
-    });
-    indexManager.addToIndex({
-        path: "pages/people/ada.md",
-        title: "Ada Lovelace",
-        summary: "Owns regression coverage",
-        section: "People",
-        tags: ["qa", "testing"],
-        updated: "2026-05-06",
-    });
-    assert.equal(indexManager.removeFromIndex("pages/projects/launch.md"), true);
-    assert.equal(indexManager.removeFromIndex("pages/projects/missing.md"), false);
-    const summary = indexManager.getIndexSummary();
-    assert.match(summary, /\*\*People\*\*: Ada Lovelace: Owns regression coverage \[qa, testing\] \(2026-05-06\)/);
-    assert.match(summary, new RegExp(`\\*\\*Index\\*\\*: Wiki: Index of all wiki pages\\. \\(${today}\\)`));
+test("rebuildWikiIndex removes stale entries not on disk", async () => {
+    const sandbox = makeSandbox();
+    try {
+        const { indexManager, wikiFs } = await loadModules(sandbox);
+        wikiFs.writePage("pages/topics/keep/index.md", "---\ntitle: Keep\nsummary: Keep this\nupdated: 2026-05-01\n---\n\n# Keep\n");
+        // Insert stale entry directly
+        indexManager.upsertWikiPage("pages/topics/stale/index.md", { title: "Stale", summary: "Should be removed", tags: [], updated: "2026-01-01", metadata: {} }, "Stale");
+        // Rebuild syncs disk → DB
+        indexManager.rebuildWikiIndex();
+        const entries = indexManager.parseIndex();
+        const paths = entries.map((e) => e.path);
+        assert.ok(paths.includes("pages/topics/keep/index.md"), "Should keep on-disk page");
+        assert.ok(!paths.includes("pages/topics/stale/index.md"), "Should remove stale entry");
+    }
+    finally {
+        rmSync(sandbox, { recursive: true, force: true });
+    }
 });
 //# sourceMappingURL=index-manager.test.js.map

package/dist/wiki/ingest.js

@@ -0,0 +1,347 @@
+// ---------------------------------------------------------------------------
+// PKB ingestion pipeline — parse, extract entities, write wiki pages
+// ---------------------------------------------------------------------------
+import { createHash } from "node:crypto";
+import { exec } from "node:child_process";
+import { mkdirSync, readFileSync, existsSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import { getDb } from "../store/db.js";
+import { ensureWikiStructure, writeRawSource, assertPagePath } from "./fs.js";
+import { appendTimeline } from "./timeline.js";
+import { validateAndBackfillFrontmatter } from "./frontmatter.js";
+import { writePage, readPage } from "./fs.js";
+import { childLogger } from "../util/logger.js";
+const log = childLogger("ingest");
+const execAsync = promisify(exec);
+// ---------------------------------------------------------------------------
+// Source ID
+// ---------------------------------------------------------------------------
+export function computeSourceId(sourceType, origin) {
+    return createHash("sha256").update(sourceType + origin).digest("hex");
+}
+// ---------------------------------------------------------------------------
+// Content parsers
+// ---------------------------------------------------------------------------
+export function assertSafeRemoteUrl(url) {
+    const parsedUrl = new URL(url);
+    if (!["http:", "https:"].includes(parsedUrl.protocol)) {
+        throw new Error(`Only http/https URLs supported, got: ${parsedUrl.protocol}`);
+    }
+    const host = parsedUrl.hostname.toLowerCase();
+    const octets = host.split(".").map((part) => Number(part));
+    const isIpv4 = octets.length === 4 && octets.every((part) => Number.isInteger(part) && part >= 0 && part <= 255);
+    const isPrivateIpv4 = isIpv4 && (octets[0] === 10
+        || (octets[0] === 172 && octets[1] >= 16 && octets[1] <= 31)
+        || (octets[0] === 192 && octets[1] === 168));
+    if (host === "localhost" || host === "127.0.0.1" || host === "::1"
+        || isPrivateIpv4
+        || host.startsWith("169.254.") || host === "metadata.google.internal") {
+        throw new Error("Cannot fetch internal/private URLs.");
+    }
+    return parsedUrl;
+}
+export function createEntityPageContent({ pageTitle, pageSummary, entityType, updatedAt, }) {
+    return `---\ntitle: ${pageTitle}\nsummary: ${pageSummary}\nupdated: ${updatedAt}\ntags: []\nmetadata:\n  entity_type: ${entityType}\n---\n\n# ${pageTitle}\n\n## Summary\n\n${pageSummary}\n\n## Timeline\n`;
+}
+async function parseUrl(url) {
+    const parsedUrl = assertSafeRemoteUrl(url);
+    const res = await fetch(url, { headers: { "User-Agent": "Chapterhouse/1.0 PKB-Ingest" } });
+    if (!res.ok)
+        throw new Error(`HTTP ${res.status} ${res.statusText} fetching ${url}`);
+    const html = await res.text();
+    // Try @mozilla/readability if available
+    let text;
+    let title = parsedUrl.hostname;
+    try {
+        const { Readability } = await import("@mozilla/readability");
+        const { JSDOM } = await import("jsdom");
+        const dom = new JSDOM(html, { url });
+        const reader = new Readability(dom.window.document);
+        const article = reader.parse();
+        if (article) {
+            text = article.textContent;
+            title = article.title || title;
+        }
+        else {
+            text = stripHtml(html);
+        }
+    }
+    catch {
+        text = stripHtml(html);
+        // Try to extract title from <title> tag
+        const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
+        if (titleMatch)
+            title = titleMatch[1].trim();
+    }
+    if (text.length > 50_000)
+        text = text.slice(0, 50_000);
+    return { text, title };
+}
+function stripHtml(html) {
+    return html
+        .replace(/<script[\s\S]*?<\/script>/gi, "")
+        .replace(/<style[\s\S]*?<\/style>/gi, "")
+        .replace(/<[^>]+>/g, " ")
+        .replace(/\s{2,}/g, " ")
+        .trim()
+        .slice(0, 10_000);
+}
+async function parsePdf(filePath) {
+    try {
+        const pdfParse = await import("pdf-parse");
+        const buf = readFileSync(filePath);
+        const data = await pdfParse.default(buf);
+        return { text: data.text.slice(0, 50_000), title: filePath.replace(/.*\//, "").replace(/\.pdf$/i, "") };
+    }
+    catch {
+        throw new Error("PDF ingestion requires pdf-parse: npm install pdf-parse");
+    }
+}
+async function parseRepo(repoUrl) {
+    const tmpDir = join(process.cwd(), ".test-work", `repo-${Date.now()}`);
+    mkdirSync(tmpDir, { recursive: true });
+    try {
+        await execAsync(`git clone --depth 1 ${JSON.stringify(repoUrl)} ${JSON.stringify(tmpDir)}`, { timeout: 60_000 });
+        const parts = [];
+        // README
+        for (const name of ["README.md", "README.rst", "README.txt", "README"]) {
+            const p = join(tmpDir, name);
+            if (existsSync(p)) {
+                parts.push(`## README\n\n${readFileSync(p, "utf-8").slice(0, 10_000)}`);
+                break;
+            }
+        }
+        // package.json / Cargo.toml / go.mod
+        for (const name of ["package.json", "Cargo.toml", "go.mod"]) {
+            const p = join(tmpDir, name);
+            if (existsSync(p)) {
+                parts.push(`## ${name}\n\n\`\`\`\n${readFileSync(p, "utf-8").slice(0, 2_000)}\n\`\`\``);
+            }
+        }
+        const { stdout } = await execAsync(`ls -la ${JSON.stringify(tmpDir)}`);
+        parts.push(`## Directory listing\n\n\`\`\`\n${stdout}\n\`\`\``);
+        const title = repoUrl.replace(/.*\//, "").replace(/\.git$/, "");
+        return { text: parts.join("\n\n"), title };
+    }
+    finally {
+        try {
+            rmSync(tmpDir, { recursive: true, force: true });
+        }
+        catch { /* best-effort */ }
+    }
+}
+// ---------------------------------------------------------------------------
+// Entity extraction via LLM
+// ---------------------------------------------------------------------------
+async function extractEntities(text, topic) {
+    // Skip entity extraction if no auth token is configured
+    const { config } = await import("../config.js");
+    const token = config.copilotAuthToken || process.env.COPILOT_TOKEN || process.env.GITHUB_TOKEN;
+    if (!token) {
+        log.debug("No Copilot auth token configured, skipping entity extraction");
+        return { entities: [], relationships: [] };
+    }
+    const topicHint = topic ? ` Focus especially on entities related to: ${topic}.` : "";
+    const systemPrompt = "Extract entities and relationships from this content. Return JSON only, no other text: " +
+        `{ "entities": [{"name": string, "type": string, "description": string}], ` +
+        `"relationships": [{"from": string, "to": string, "type": string}] }`;
+    const userMessage = `${systemPrompt}${topicHint}\n\n---\n\n${text.slice(0, 8_000)}`;
+    try {
+        const { CopilotClient, approveAll } = await import("@github/copilot-sdk");
+        // Use a one-shot client (autoRestart: false) so it doesn't keep the process alive
+        const client = new CopilotClient({
+            autoStart: true,
+            autoRestart: false,
+            gitHubToken: token,
+        });
+        await client.start();
+        try {
+            const session = await client.createSession({
+                model: "claude-haiku-4.5",
+                tools: [],
+                onPermissionRequest: approveAll,
+            });
+            try {
+                const result = await session.sendAndWait({ prompt: userMessage }, 30_000);
+                const rawText = typeof result === "string" ? result : JSON.stringify(result);
+                // Extract JSON from the response (may be wrapped in markdown code blocks)
+                const jsonMatch = rawText.match(/\{[\s\S]*\}/);
+                if (!jsonMatch)
+                    return { entities: [], relationships: [] };
+                const parsed = JSON.parse(jsonMatch[0]);
+                return {
+                    entities: Array.isArray(parsed.entities) ? parsed.entities : [],
+                    relationships: Array.isArray(parsed.relationships) ? parsed.relationships : [],
+                };
+            }
+            finally {
+                try {
+                    session.destroy();
+                }
+                catch { /* best-effort */ }
+            }
+        }
+        finally {
+            try {
+                await client.stop();
+            }
+            catch { /* best-effort */ }
+        }
+    }
+    catch (err) {
+        log.warn({ err: err instanceof Error ? err.message : err }, "Entity extraction LLM call failed, skipping");
+        return { entities: [], relationships: [] };
+    }
+}
+// ---------------------------------------------------------------------------
+// Slug helper
+// ---------------------------------------------------------------------------
+function slugify(name) {
+    return name
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-+|-+$/g, "")
+        .slice(0, 64) || "unknown";
+}
+// ---------------------------------------------------------------------------
+// Main ingest function
+// ---------------------------------------------------------------------------
+export async function ingestSource(source, type, topic, session) {
+    ensureWikiStructure();
+    const db = getDb();
+    const origin = type === "text" ? source.slice(0, 200) : source;
+    const sourceId = computeSourceId(type, origin);
+    // Idempotency check
+    const existing = db.prepare(`SELECT id, pages_updated FROM wiki_sources WHERE id = ?`).get(sourceId);
+    if (existing) {
+        log.info({ sourceId, type, origin }, "Source already ingested, skipping");
+        const pagesUpdated = JSON.parse(existing.pages_updated || "[]");
+        return {
+            source_id: sourceId,
+            pages_created: [],
+            pages_updated: pagesUpdated,
+            entities: [],
+            already_existed: true,
+        };
+    }
+    // Parse content
+    let parsedText;
+    let title;
+    switch (type) {
+        case "url": {
+            const r = await parseUrl(source);
+            parsedText = r.text;
+            title = r.title;
+            break;
+        }
+        case "pdf": {
+            const r = await parsePdf(source);
+            parsedText = r.text;
+            title = r.title;
+            break;
+        }
+        case "repo": {
+            const r = await parseRepo(source);
+            parsedText = r.text;
+            title = r.title;
+            break;
+        }
+        case "text":
+        default:
+            parsedText = source;
+            title = topic ?? `text-${sourceId.slice(0, 8)}`;
+            break;
+    }
+    // Save raw source archive
+    const rawFileName = `${sourceId.slice(0, 16)}.md`;
+    writeRawSource(rawFileName, parsedText);
+    // Persist to wiki_sources
+    const ingestedAt = new Date().toISOString();
+    db.prepare(`
+    INSERT INTO wiki_sources (id, source_type, origin, title, ingested_at, raw_path, parsed_content, pages_updated, session_id, session_name)
+    VALUES (?, ?, ?, ?, ?, ?, ?, '[]', ?, ?)
+  `).run(sourceId, type, origin, title, ingestedAt, `sources/${rawFileName}`, parsedText.slice(0, 100_000), session?.sessionId ?? null, session?.sessionName ?? null);
+    // Extract entities
+    const extraction = await extractEntities(parsedText, topic);
+    const pagesCreated = [];
+    const pagesUpdated = [];
+    const entitySummaries = [];
+    for (const entity of extraction.entities) {
+        if (!entity.name || !entity.type)
+            continue;
+        const entitySlug = slugify(entity.name);
+        const typeLower = entity.type.toLowerCase().replace(/[^a-z0-9-]/g, "-");
+        const pagePath = `pages/${typeLower}/${entitySlug}/index.md`;
+        // Validate path is safe
+        try {
+            assertPagePath(pagePath);
+        }
+        catch {
+            // Use topics fallback for unknown entity types
+            const fallbackPath = `pages/topics/${entitySlug}/index.md`;
+            try {
+                assertPagePath(fallbackPath);
+            }
+            catch {
+                continue;
+            }
+        }
+        const safePagePath = (() => {
+            try {
+                assertPagePath(pagePath);
+                return pagePath;
+            }
+            catch {
+                return `pages/topics/${entitySlug}/index.md`;
+            }
+        })();
+        const existed = readPage(safePagePath) !== undefined;
+        const timelineEntry = `Source ingested: ${title}\n\n${entity.description || entity.name}`;
+        if (!existed) {
+            const pageTitle = entity.name;
+            const pageSummary = (entity.description || entity.name).slice(0, 180).replace(/\n/g, " ");
+            const pageContent = createEntityPageContent({
+                pageTitle,
+                pageSummary,
+                entityType: typeLower,
+                updatedAt: ingestedAt.slice(0, 10),
+            });
+            const { content: backfilled } = validateAndBackfillFrontmatter(safePagePath, pageContent);
+            writePage(safePagePath, backfilled);
+            pagesCreated.push(safePagePath);
+        }
+        else {
+            pagesUpdated.push(safePagePath);
+        }
+        appendTimeline(safePagePath, timelineEntry);
+        entitySummaries.push({ name: entity.name, type: entity.type, path: safePagePath });
+    }
+    // Update wiki_sources with pages_updated
+    const allPages = [...new Set([...pagesCreated, ...pagesUpdated])];
+    db.prepare(`UPDATE wiki_sources SET pages_updated = ? WHERE id = ?`).run(JSON.stringify(allPages), sourceId);
+    return {
+        source_id: sourceId,
+        pages_created: pagesCreated,
+        pages_updated: pagesUpdated,
+        entities: entitySummaries,
+        already_existed: false,
+    };
+}
+// ---------------------------------------------------------------------------
+// Type auto-detection
+// ---------------------------------------------------------------------------
+export function detectSourceType(source) {
+    const trimmed = source.trim();
+    if (trimmed.startsWith("http://") || trimmed.startsWith("https://")) {
+        if (trimmed.endsWith(".pdf"))
+            return "pdf";
+        if (trimmed.includes("github.com") || trimmed.includes("gitlab.com") || trimmed.endsWith(".git"))
+            return "repo";
+        return "url";
+    }
+    if (trimmed.endsWith(".pdf") && !trimmed.includes(" "))
+        return "pdf";
+    return "text";
+}
+//# sourceMappingURL=ingest.js.map