chapterhouse 0.8.1 → 0.9.0

package/dist/api/server.js

@@ -16,8 +16,9 @@ import { createAuthMiddleware, getBootstrapAuthResponse } from "./auth.js";
 import { assertAgentEditAccess } from "./agent-edit-access.js";
 import { createConcurrentConnectionLimiter, createFixedWindowRateLimiter } from "./rate-limit.js";
 import { createTeamRouter } from "./team.js";
-import { writePage, deletePage, pageExists, listPages, ensureWikiStructure, assertPagePath, } from "../wiki/fs.js";
+import { readPage, writePage, deletePage, pageExists, listPages, ensureWikiStructure, assertPagePath, getWikiDir, } from "../wiki/fs.js";
 import { parseWikiFrontmatter } from "../wiki/frontmatter.js";
+import { normalizeWikiPath } from "../wiki/path-utils.js";
 import { loadRegistry, saveRegistry } from "../wiki/project-registry.js";
 import { getProjectRulesPath, listTopLevelSoftRules, loadProjectRules, loadProjectRuleSummary, renderInitialProjectRulesPage, saveProjectRulesHardFields, saveProjectRulesSoftRules, } from "../wiki/project-rules.js";
 import { readWikiPage, teamWikiSync } from "../wiki/team-sync.js";
@@ -29,7 +30,7 @@ import { getCurrentRunId, getDb, getSessionMessages, getTaskEvents } from "../st
 import { getTaskLogEvents, subscribeTaskLog } from "../copilot/task-event-log.js";
 import { subscribeSession, getSessionEventsFromDb, getSessionMaxSeqFromDb, oldestSessionSeq, } from "../copilot/turn-event-log.js";
 import { getStatus, onStatusChange } from "../status.js";
-import { formatSseData, formatSseEvent } from "./sse.js";
+import { formatSseData } from "./sse.js";
 import { assertAuthenticationConfigured, createHealthPayload, createPublicConfigPayload, getDisplayHost, resolveApiToken, shouldServeSpaPath, } from "./server-runtime.js";
 import { BadRequestError, ForbiddenError, InternalServerError, NotFoundError, apiNotFoundHandler, asBadRequest, createApiErrorHandler, parseRequest, } from "./errors.js";
 import { childLogger } from "../util/logger.js";
@@ -40,6 +41,7 @@ import { recordObservation } from "../memory/observations.js";
 import { recordDecision } from "../memory/decisions.js";
 import { upsertEntity } from "../memory/entities.js";
 import { getInboxItem, listPendingInboxItems, resolveInboxItem } from "../memory/inbox.js";
+import { ingestSource } from "../wiki/ingest.js";
 import { listKorgResearchSessions, routeKorgMessage } from "./korg.js";
 const log = childLogger("server");
 const modeContext = new ModeContext(config);
@@ -146,7 +148,7 @@ const prMergeHookSchema = z.object({
 });
 const korgRequestSchema = z.object({
     message: requiredString("Missing 'message' in request body"),
-    session_id: z.string().trim().min(1).optional(),
+    sessionKey: z.string().trim().min(1).optional(),
 }).strict();
 const projectHardRulesSchema = z.object({
     hardRules: z.object({
@@ -192,6 +194,149 @@ function createProjectDetailPayload(slug, cwd) {
         softRules: listTopLevelSoftRules(rules.soft),
     };
 }
+function coerceWikiPageUpdated(path, updated) {
+    const normalized = updated?.trim();
+    if (normalized) {
+        return normalized;
+    }
+    try {
+        return statSync(join(getWikiDir(), path)).mtime.toISOString();
+    }
+    catch {
+        return "unknown";
+    }
+}
+function wikiPathToBrowserSlug(path) {
+    return path.replace(/^pages\//, "").replace(/\.md$/, "");
+}
+function normalizeOptionalQueryParam(value) {
+    if (typeof value !== "string") {
+        return undefined;
+    }
+    const trimmed = value.trim();
+    return trimmed ? trimmed : undefined;
+}
+function matchesWikiBrowserFilters(page, filters) {
+    if (filters.type && page.type !== filters.type) {
+        return false;
+    }
+    if (!filters.q) {
+        return true;
+    }
+    const query = filters.q.toLowerCase();
+    return page.title.toLowerCase().includes(query) || page.summary.toLowerCase().includes(query);
+}
+function mapIndexEntryToBrowserPage(entry) {
+    return {
+        slug: wikiPathToBrowserSlug(entry.path),
+        title: entry.title,
+        summary: entry.summary,
+        type: entry.section,
+        last_updated: coerceWikiPageUpdated(entry.path, entry.updated),
+    };
+}
+function listFallbackWikiBrowserPages(filters) {
+    const entries = parseIndex();
+    const indexed = new Set(entries.map((entry) => entry.path));
+    const indexedResults = entries.map(mapIndexEntryToBrowserPage);
+    const orphanResults = listPages()
+        .filter((path) => !indexed.has(path))
+        .map((path) => ({
+        slug: wikiPathToBrowserSlug(path),
+        title: path,
+        summary: "",
+        type: "Unindexed",
+        last_updated: coerceWikiPageUpdated(path, undefined),
+    }));
+    return [...indexedResults, ...orphanResults].filter((page) => matchesWikiBrowserFilters(page, filters));
+}
+function listDbWikiBrowserPages(filters) {
+    const db = getDb();
+    const clauses = [];
+    const params = [];
+    if (filters.type) {
+        clauses.push("entity_type = ?");
+        params.push(filters.type);
+    }
+    if (filters.q) {
+        clauses.push("(title LIKE ? COLLATE NOCASE OR COALESCE(summary, '') LIKE ? COLLATE NOCASE)");
+        params.push(`%${filters.q}%`, `%${filters.q}%`);
+    }
+    const rows = db.prepare(`
+    SELECT path, title, summary, entity_type, last_updated, pinned
+    FROM wiki_pages
+    ${clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : ""}
+    ORDER BY COALESCE(last_updated, '') DESC, title ASC
+  `).all(...params);
+    return rows.map((row) => ({
+        slug: wikiPathToBrowserSlug(row.path),
+        title: row.title,
+        summary: row.summary ?? "",
+        type: row.entity_type ?? "topics",
+        last_updated: coerceWikiPageUpdated(row.path, row.last_updated ?? undefined),
+        ...(row.pinned ? { pinned: true } : {}),
+    }));
+}
+function parseWikiSourcePages(value) {
+    if (!value) {
+        return [];
+    }
+    try {
+        const parsed = JSON.parse(value);
+        return Array.isArray(parsed)
+            ? parsed.filter((entry) => typeof entry === "string").map((entry) => normalizeWikiPath(entry))
+            : [];
+    }
+    catch {
+        return [];
+    }
+}
+function listWikiSources(page) {
+    const rows = getDb().prepare(`
+    SELECT source_type, origin, raw_path, pages_updated, status, session_id, ingested_at
+    FROM wiki_sources
+    ORDER BY ingested_at DESC, id ASC
+  `).all();
+    return rows
+        .filter((row) => !page || parseWikiSourcePages(row.pages_updated).includes(page))
+        .map((row) => ({
+        source_url: row.origin,
+        path: row.raw_path ?? undefined,
+        kind: row.source_type,
+        status: row.status,
+        session_id: row.session_id,
+        ingested_at: row.ingested_at,
+    }));
+}
+function wikiPathToSlug(path) {
+    const segments = normalizeWikiPath(path).split("/").filter(Boolean);
+    const file = segments[segments.length - 1] ?? path;
+    const base = file.endsWith(".md") ? file.slice(0, -3) : file;
+    if (base === "index" && segments.length >= 2) {
+        return segments[segments.length - 2] ?? base;
+    }
+    return base;
+}
+function listWikiLinks(page) {
+    const rows = page
+        ? getDb().prepare(`
+      SELECT from_page, to_page, link_type
+      FROM wiki_links
+      WHERE from_page = ? OR to_page = ?
+      ORDER BY from_page ASC, to_page ASC, link_type ASC
+    `).all(page, page)
+        : getDb().prepare(`
+      SELECT from_page, to_page, link_type
+      FROM wiki_links
+      ORDER BY from_page ASC, to_page ASC, link_type ASC
+    `).all();
+    return rows.map((row) => ({
+        source_slug: wikiPathToSlug(row.from_page),
+        target_slug: wikiPathToSlug(row.to_page),
+        link_type: row.link_type,
+        ...(page ? { direction: row.from_page === page ? "outgoing" : "incoming" } : {}),
+    }));
+}
 // Load a configured API token when present; startup validation below enforces auth.
 let apiToken = null;
 try {
@@ -312,6 +457,13 @@ function readPathParam(req) {
     }
     return raw;
 }
+function readOptionalPageFilter(req) {
+    const raw = req.query.page;
+    if (typeof raw !== "string" || !raw.trim()) {
+        return undefined;
+    }
+    return normalizeWikiPath(raw.trim());
+}
 function assertValidPagePath(path) {
     try {
         assertPagePath(path);
@@ -758,11 +910,11 @@ app.get("/stream", (req, res) => {
     }
     sseClients.set(connectionId, res);
     const unsubscribeStatus = onStatusChange((status, message) => {
-        res.write(formatSseEvent("status", { status, message }));
+        res.write(formatSseData({ type: "status", status, message }));
     });
     const currentStatus = getStatus();
     if (currentStatus.status !== "idle") {
-        res.write(formatSseEvent("status", currentStatus));
+        res.write(formatSseData({ type: "status", ...currentStatus }));
     }
     const heartbeat = setInterval(() => {
         res.write(`:ping\n\n`);
@@ -1448,7 +1600,7 @@ app.get("/api/wiki/pages", async (req, res) => {
         summary: e.summary,
         section: e.section,
         tags: e.tags || [],
-        updated: e.updated || "",
+        updated: coerceWikiPageUpdated(e.path, e.updated),
         scope: getWikiPageScope(e.path),
     }));
     const orphanResults = listPages()
@@ -1459,12 +1611,60 @@ app.get("/api/wiki/pages", async (req, res) => {
         summary: "",
         section: "Unindexed",
         tags: [],
-        updated: "",
+        updated: coerceWikiPageUpdated(p, undefined),
         scope: getWikiPageScope(p),
     }));
     res.json([...indexedResults, ...orphanResults]);
 });
+app.get("/api/wiki/browser-pages", async (req, res) => {
+    ensureWikiStructure();
+    const filters = {
+        q: normalizeOptionalQueryParam(req.query.q),
+        type: normalizeOptionalQueryParam(req.query.type),
+    };
+    const db = getDb();
+    const { count } = db.prepare("SELECT COUNT(*) AS count FROM wiki_pages").get();
+    const pages = count > 0
+        ? listDbWikiBrowserPages(filters)
+        : listFallbackWikiBrowserPages(filters);
+    res.json({ pages });
+});
+app.get("/api/wiki/sources", async (req, res) => {
+    ensureWikiStructure();
+    res.json({ sources: listWikiSources(readOptionalPageFilter(req)) });
+});
+app.get("/api/wiki/links", async (req, res) => {
+    ensureWikiStructure();
+    res.json({ links: listWikiLinks(readOptionalPageFilter(req)) });
+});
 app.get("/api/wiki/page", async (req, res) => {
+    const slugParam = normalizeOptionalQueryParam(req.query.slug);
+    if (slugParam) {
+        const path = `pages/${slugParam}.md`;
+        assertValidPagePath(path);
+        const db = getDb();
+        const row = db.prepare("SELECT title, entity_type, last_updated, pinned FROM wiki_pages WHERE path = ?").get(path);
+        const authorizationHeader = typeof req.headers.authorization === "string"
+            ? req.headers.authorization
+            : undefined;
+        const rawContent = await readWikiPage(path, { authorizationHeader });
+        if (rawContent === undefined) {
+            throw new NotFoundError("Page not found");
+        }
+        const { parsed: frontmatter } = parseWikiFrontmatter(rawContent);
+        res.json({
+            page: {
+                slug: slugParam,
+                title: row?.title ?? slugParam,
+                type: row?.entity_type ?? "topics",
+                last_updated: row?.last_updated ?? coerceWikiPageUpdated(path, undefined),
+                compiled_truth: rawContent,
+                pinned: Boolean(row?.pinned),
+                frontmatter,
+            },
+        });
+        return;
+    }
     const path = assertValidPagePath(readPathParam(req));
     const authorizationHeader = typeof req.headers.authorization === "string"
         ? req.headers.authorization
@@ -1489,6 +1689,54 @@ app.put("/api/wiki/page", async (req, res) => {
     });
     res.json({ ok: true, created, path });
 });
+const wikiUpdateSchema = z.object({
+    slug: requiredString("Missing 'slug' in request body"),
+    compiled_truth: z.string().optional(),
+    frontmatter: z.record(z.string(), z.unknown()).optional(),
+});
+app.post("/api/wiki/update", authMiddleware, async (req, res) => {
+    const { slug, compiled_truth, frontmatter: newFrontmatter } = parseRequest(wikiUpdateSchema, req.body);
+    const path = assertValidPagePath(`pages/${slug}.md`);
+    let finalContent = compiled_truth;
+    if (newFrontmatter !== undefined) {
+        const existing = readPage(path);
+        const base = finalContent ?? existing ?? "";
+        const { parsed: existingFrontmatter, body } = parseWikiFrontmatter(base);
+        const merged = { ...existingFrontmatter, ...newFrontmatter };
+        const fmLines = Object.entries(merged).map(([k, v]) => `${k}: ${JSON.stringify(v)}`).join("\n");
+        finalContent = `---\n${fmLines}\n---\n${body}`;
+    }
+    if (finalContent !== undefined) {
+        await withWikiWrite(() => writePage(path, finalContent));
+    }
+    const db = getDb();
+    const row = db.prepare("SELECT title, entity_type, last_updated, pinned FROM wiki_pages WHERE path = ?").get(path);
+    const content = readPage(path) ?? finalContent ?? "";
+    const { parsed: frontmatter } = parseWikiFrontmatter(content);
+    res.json({
+        ok: true,
+        page: {
+            slug,
+            title: row?.title ?? slug,
+            type: row?.entity_type ?? "topics",
+            last_updated: row?.last_updated ?? coerceWikiPageUpdated(path, undefined),
+            compiled_truth: content,
+            pinned: Boolean(row?.pinned),
+            frontmatter,
+        },
+    });
+});
+const wikiPinSchema = z.object({
+    slug: requiredString("Missing 'slug' in request body"),
+    pinned: z.boolean({ error: "Missing 'pinned' in request body" }),
+});
+app.post("/api/wiki/page/pin", authMiddleware, async (req, res) => {
+    const { slug, pinned } = parseRequest(wikiPinSchema, req.body);
+    const path = assertValidPagePath(`pages/${slug}.md`);
+    const db = getDb();
+    db.prepare("UPDATE wiki_pages SET pinned = ? WHERE path = ?").run(pinned ? 1 : 0, path);
+    res.json({ ok: true, pinned: Boolean(pinned) });
+});
 app.delete("/api/wiki/page", async (req, res) => {
     const path = assertValidPagePath(readPathParam(req));
     const removed = await withWikiWrite(() => deletePage(path));
@@ -1502,6 +1750,72 @@ app.post("/api/wiki/korg", authMiddleware, async (req, res) => {
 app.get("/api/wiki/korg/sessions", authMiddleware, (_req, res) => {
     res.json({ sessions: listKorgResearchSessions(getDb()) });
 });
+const wikiIngestSchema = z.object({
+    source_url: z.string().optional(),
+    path: z.string().optional(),
+    topic: z.string().optional(),
+});
+app.post("/api/wiki/ingest", authMiddleware, async (req, res) => {
+    const { source_url, path: ingestPath, topic } = parseRequest(wikiIngestSchema, req.body ?? {});
+    const source = source_url ?? ingestPath;
+    if (!source) {
+        throw new BadRequestError("Missing 'source_url' or 'path' in request body");
+    }
+    const type = source_url ? "url" : "text";
+    await withWikiWrite(() => ingestSource(source, type, topic));
+    res.json({ ok: true });
+});
+const wikiSearchSchema = z.object({
+    q: z.string().optional(),
+    type: z.string().optional(),
+});
+app.get("/api/wiki/search", authMiddleware, async (req, res) => {
+    ensureWikiStructure();
+    const q = normalizeOptionalQueryParam(req.query.q);
+    const type = normalizeOptionalQueryParam(req.query.type);
+    const db = getDb();
+    let rows;
+    try {
+        const clauses = ["wiki_pages_fts MATCH ?"];
+        const params = [q ? `${q}*` : "*"];
+        if (type) {
+            clauses.push("entity_type = ?");
+            params.push(type);
+        }
+        rows = db.prepare(`
+      SELECT path, title, entity_type, last_updated
+      FROM wiki_pages_fts
+      WHERE ${clauses.join(" AND ")}
+      ORDER BY rank
+      LIMIT 50
+    `).all(...params);
+    }
+    catch {
+        const clauses = [];
+        const params = [];
+        if (q) {
+            clauses.push("title LIKE ? COLLATE NOCASE");
+            params.push(`%${q}%`);
+        }
+        if (type) {
+            clauses.push("entity_type = ?");
+            params.push(type);
+        }
+        rows = db.prepare(`
+      SELECT path, title, entity_type, last_updated
+      FROM wiki_pages
+      ${clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : ""}
+      LIMIT 50
+    `).all(...params);
+    }
+    const pages = rows.map((row) => ({
+        slug: wikiPathToBrowserSlug(row.path),
+        title: row.title,
+        type: row.entity_type ?? "topics",
+        last_updated: coerceWikiPageUpdated(row.path, row.last_updated ?? undefined),
+    }));
+    res.json({ pages });
+});
 // ---------------------------------------------------------------------------
 // Skills
 // ---------------------------------------------------------------------------