@luuthuong/devboard 0.0.0-stage → 1.0.1

package/server-dist/server/harness/kbStore.js

@@ -0,0 +1,258 @@
+// Storage behind one knowledge base's index. With `node:sqlite` (Node ≥ 22.5)
+// each KB gets its own file, `<data dir>/kb-index/<kb id>.sqlite`: an FTS5
+// table for text search (BM25, title and summary weighted up) and the parsed
+// entries, so a restart re-reads only files whose mtime or size changed.
+// Without it (Node 20) the same interface is served from memory. The markdown
+// files in the harness repo stay the source of truth; a store can be deleted.
+import fs from "node:fs";
+import path from "node:path";
+import { tokenize } from "../transcripts.js";
+import { log } from "../util.js";
+/** Max query terms; more adds cost without changing the top results. */
+const MAX_TERMS = 32;
+/** FTS5 query from free text: each word as a phrase, 3+ letters as a prefix, any may match. */
+export function ftsQuery(text) {
+    const terms = [...new Set(tokenize(text))].slice(0, MAX_TERMS);
+    // tokenize() keeps only letters and digits, so nothing needs escaping inside the quotes.
+    return terms.map((t) => (t.length >= 3 ? `"${t}"*` : `"${t}"`)).join(" OR ");
+}
+let sqlite;
+/** `node:sqlite` when this Node has it, else null. Its one-time "experimental" warning is not shown. */
+export function loadSqlite() {
+    if (sqlite !== undefined)
+        return sqlite;
+    const emit = process.emitWarning;
+    process.emitWarning = ((warning, ...rest) => {
+        if (String(warning).includes("SQLite is an experimental feature"))
+            return;
+        return emit.call(process, warning, ...rest);
+    });
+    try {
+        sqlite = process.getBuiltinModule?.("node:sqlite") ?? null;
+    }
+    catch {
+        sqlite = null;
+    }
+    finally {
+        process.emitWarning = emit;
+    }
+    return sqlite;
+}
+/** Bump when the schema changes: the store is then rebuilt from the markdown files. */
+const SCHEMA_VERSION = "1";
+const SCHEMA = `
+  CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
+  CREATE TABLE IF NOT EXISTS files (
+    id INTEGER PRIMARY KEY,
+    rel TEXT NOT NULL UNIQUE,
+    mtime REAL NOT NULL,
+    size INTEGER NOT NULL,
+    entry_id TEXT NOT NULL,
+    entry TEXT NOT NULL
+  );
+  CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(
+    title, summary, scope, body,
+    tokenize = 'unicode61 remove_diacritics 2',
+    prefix = '3'
+  );
+`;
+/** Column weights for bm25(): title, summary, scope, body. */
+const BM25 = "bm25(fts, 3.0, 2.0, 1.0, 1.0)";
+class SqliteKbStore {
+    kind = "sqlite";
+    db;
+    constructor(lib, file) {
+        this.db = new lib.DatabaseSync(file);
+        // A throwing constructor leaves no object to close, and an open handle blocks deleting the file on Windows.
+        try {
+            this.db.exec("PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL;");
+            this.db.exec(SCHEMA);
+            const version = this.db.prepare("SELECT value FROM meta WHERE key = 'schema'").get();
+            if (version?.value !== SCHEMA_VERSION) {
+                this.db.exec("DROP TABLE IF EXISTS files; DROP TABLE IF EXISTS fts;");
+                this.db.exec(SCHEMA);
+                this.db.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES ('schema', ?)").run(SCHEMA_VERSION);
+            }
+        }
+        catch (err) {
+            this.db.close();
+            throw err;
+        }
+    }
+    load() {
+        const rows = this.db.prepare("SELECT rel, mtime, size, entry FROM files").all();
+        return rows.map((r) => ({ rel: r.rel, mtimeMs: r.mtime, size: r.size, entry: JSON.parse(r.entry) }));
+    }
+    apply(upserts, removed) {
+        if (upserts.length === 0 && removed.length === 0)
+            return;
+        const idOf = this.db.prepare("SELECT id FROM files WHERE rel = ?");
+        const delFts = this.db.prepare("DELETE FROM fts WHERE rowid = ?");
+        const delFile = this.db.prepare("DELETE FROM files WHERE id = ?");
+        const insFile = this.db.prepare("INSERT INTO files (rel, mtime, size, entry_id, entry) VALUES (?, ?, ?, ?, ?) RETURNING id");
+        const insFts = this.db.prepare("INSERT INTO fts (rowid, title, summary, scope, body) VALUES (?, ?, ?, ?, ?)");
+        const drop = (rel) => {
+            const row = idOf.get(rel);
+            if (!row)
+                return;
+            delFts.run(row.id);
+            delFile.run(row.id);
+        };
+        this.db.exec("BEGIN");
+        try {
+            for (const rel of removed)
+                drop(rel);
+            for (const f of upserts) {
+                drop(f.rel);
+                const e = f.entry;
+                const { id } = insFile.get(f.rel, f.mtimeMs, f.size, e.id, JSON.stringify(e));
+                insFts.run(id, e.title, e.summary, e.scope.join(" "), e.body);
+            }
+            this.db.exec("COMMIT");
+        }
+        catch (err) {
+            this.db.exec("ROLLBACK");
+            throw err;
+        }
+    }
+    searchText(text, limit) {
+        const q = ftsQuery(text);
+        if (!q)
+            return [];
+        const rows = this.db
+            .prepare(`SELECT f.entry_id AS id FROM fts JOIN files f ON f.id = fts.rowid WHERE fts MATCH ? ORDER BY ${BM25} LIMIT ?`)
+            .all(q, limit);
+        return rows.map((r) => r.id);
+    }
+    close() {
+        try {
+            this.db.close();
+        }
+        catch {
+            // already closed
+        }
+    }
+}
+// ── Memory (Node without node:sqlite) ─────────────────────────────────────────
+const K1 = 1.2;
+const B = 0.4;
+/** BM25 over in-memory postings; prefix terms via a sorted vocabulary. */
+class MemoryKbStore {
+    kind = "memory";
+    docs = new Map();
+    postings = null;
+    vocab = [];
+    avgLen = 0;
+    load() {
+        return [...this.docs.values()].map((d) => d.file);
+    }
+    apply(upserts, removed) {
+        for (const rel of removed)
+            this.docs.delete(rel);
+        for (const f of upserts) {
+            // Title and summary count more than the body, like the SQLite column weights.
+            const e = f.entry;
+            const tokens = tokenize(`${e.title} ${e.title} ${e.title} ${e.summary} ${e.summary} ${e.scope.join(" ")} ${e.body}`);
+            const tf = new Map();
+            for (const t of tokens)
+                tf.set(t, (tf.get(t) ?? 0) + 1);
+            this.docs.set(f.rel, { file: f, tf, len: tokens.length });
+        }
+        if (upserts.length || removed.length) {
+            this.postings = null;
+            this.build(); // now, so the first query (often a SessionStart hook) does not pay for it
+        }
+    }
+    build() {
+        if (this.postings)
+            return this.postings;
+        const postings = new Map();
+        let total = 0;
+        for (const d of this.docs.values()) {
+            total += d.len;
+            for (const [term, n] of d.tf) {
+                let p = postings.get(term);
+                if (!p)
+                    postings.set(term, (p = new Map()));
+                p.set(d, n);
+            }
+        }
+        this.vocab = [...postings.keys()].sort();
+        this.avgLen = this.docs.size ? total / this.docs.size : 0;
+        return (this.postings = postings);
+    }
+    searchText(text, limit) {
+        const terms = [...new Set(tokenize(text))].slice(0, MAX_TERMS);
+        const postings = this.build();
+        const n = this.docs.size;
+        if (terms.length === 0 || n === 0)
+            return [];
+        const scores = new Map();
+        for (const term of terms) {
+            const tfByDoc = new Map();
+            for (const t of matchingTerms(this.vocab, term)) {
+                for (const [d, c] of postings.get(t))
+                    tfByDoc.set(d, (tfByDoc.get(d) ?? 0) + c);
+            }
+            if (tfByDoc.size === 0)
+                continue;
+            const idf = Math.log((n - tfByDoc.size + 0.5) / (tfByDoc.size + 0.5) + 1);
+            for (const [d, tf] of tfByDoc) {
+                const norm = (tf * (K1 + 1)) / (tf + K1 * (1 - B + (B * d.len) / this.avgLen));
+                scores.set(d, (scores.get(d) ?? 0) + idf * norm);
+            }
+        }
+        return [...scores.entries()]
+            .sort((a, b) => b[1] - a[1])
+            .slice(0, limit)
+            .map(([d]) => d.file.entry.id);
+    }
+    close() {
+        this.docs.clear();
+        this.postings = null;
+    }
+}
+/** Vocabulary terms a query term matches: itself, or every term it prefixes (3+ characters). */
+function matchingTerms(vocab, term) {
+    let lo = 0;
+    let hi = vocab.length;
+    while (lo < hi) {
+        const mid = (lo + hi) >> 1;
+        if (vocab[mid] < term)
+            lo = mid + 1;
+        else
+            hi = mid;
+    }
+    if (term.length < 3)
+        return vocab[lo] === term ? [term] : [];
+    const out = [];
+    for (let i = lo; i < vocab.length && vocab[i].startsWith(term); i++)
+        out.push(vocab[i]);
+    return out;
+}
+// ── Opening ──────────────────────────────────────────────────────────────────
+/**
+ * The store for one KB: SQLite under `dir` when possible, else memory. A file
+ * SQLite cannot open (corrupt, wrong format) is deleted and rebuilt.
+ */
+export function openStore(dir, kbId) {
+    const lib = dir ? loadSqlite() : null;
+    if (!dir || !lib)
+        return new MemoryKbStore();
+    fs.mkdirSync(dir, { recursive: true });
+    const file = path.join(dir, `${kbId.replace(/[^\w.-]/g, "_")}.sqlite`);
+    try {
+        return new SqliteKbStore(lib, file);
+    }
+    catch (e) {
+        log.warn("harness", `index ${file} unusable (${e instanceof Error ? e.message : e}); rebuilding`);
+        for (const f of [file, `${file}-wal`, `${file}-shm`])
+            fs.rmSync(f, { force: true });
+        try {
+            return new SqliteKbStore(lib, file);
+        }
+        catch {
+            return new MemoryKbStore();
+        }
+    }
+}

package/server-dist/server/harness/librarian.js

@@ -0,0 +1,147 @@
+// The librarian turns finished work into proposed knowledge entries. It reads
+// the tail of a session transcript (or, when a project is onboarded, the
+// project's own docs), scrubs it, asks a small model through `claude -p`, and
+// returns proposals for a person to review. It never writes to a KB itself.
+import fs from "node:fs";
+import path from "node:path";
+import { KNOWLEDGE_TYPES } from "../../src/types/index.js";
+import { log, readLines, run } from "../util.js";
+import { isKnowledgeType } from "./entry.js";
+import { scrub } from "./scrub.js";
+const MAX_PROPOSALS = 5;
+export const LIBRARIAN_PROMPT_HEAD = "You are the librarian of a team knowledge base";
+export function buildPrompt(input) {
+    const what = input.mode === "session"
+        ? `Below is the end of a Claude Code session${input.cardTitle ? ` for the task "${input.cardTitle}"` : ""} in the project "${input.project}".`
+        : `Below are the README, CLAUDE.md, docs and recent commit history of the project "${input.project}", which was just added.`;
+    return [
+        `${LIBRARIAN_PROMPT_HEAD} for software projects.`,
+        what,
+        "",
+        `Extract at most ${MAX_PROPOSALS} pieces of durable project knowledge that would help a future agent working in this repository.`,
+        "Keep only what is true beyond this one task: decisions and their reasons, conventions, gotchas (things that broke and why), what a component is for, project terms, and how to run or debug things.",
+        "Skip task status, anything obvious from reading the code, personal preferences, secrets, and anything you are unsure about.",
+        "",
+        `Types: ${KNOWLEDGE_TYPES.join(", ")}.`,
+        "scope: repo-relative files or folders (folders end with /) the knowledge applies to; [] when project-wide.",
+        "summary: one sentence under 200 characters. body: a few short paragraphs in markdown.",
+        "",
+        "Already in the knowledge base (do not repeat):",
+        ...(input.existingTitles.length ? input.existingTitles.slice(0, 200).map((t) => `- ${t}`) : ["- (nothing yet)"]),
+        "",
+        'Answer with JSON only, no prose: {"proposals":[{"type":"gotcha","title":"…","summary":"…","body":"…","scope":["…"],"reason":"why this is worth keeping"}]}',
+        'Answer {"proposals":[]} when nothing qualifies.',
+        "",
+        "<material>",
+        input.material,
+        "</material>",
+    ].join("\n");
+}
+/** Pull the first JSON object out of a model reply and keep only valid drafts. */
+export function parseDrafts(reply) {
+    const start = reply.indexOf("{");
+    const end = reply.lastIndexOf("}");
+    if (start < 0 || end <= start)
+        return [];
+    let parsed;
+    try {
+        parsed = JSON.parse(reply.slice(start, end + 1));
+    }
+    catch {
+        return [];
+    }
+    const list = Array.isArray(parsed?.proposals) ? parsed.proposals : [];
+    const out = [];
+    for (const p of list) {
+        if (!p || !isKnowledgeType(p.type))
+            continue;
+        const title = typeof p.title === "string" ? p.title.trim() : "";
+        const body = typeof p.body === "string" ? p.body.trim() : "";
+        if (!title || !body)
+            continue;
+        out.push({
+            type: p.type,
+            title,
+            summary: typeof p.summary === "string" ? p.summary.trim() : title,
+            body,
+            scope: Array.isArray(p.scope) ? p.scope.filter((s) => typeof s === "string") : [],
+            reason: typeof p.reason === "string" ? p.reason : undefined,
+        });
+    }
+    return out.slice(0, MAX_PROPOSALS);
+}
+/** Run the prompt through `claude -p`. Resolves [] (and logs) on any failure. */
+export async function runLibrarian(input, model, cwd) {
+    const prompt = buildPrompt({ ...input, material: scrub(input.material) });
+    // No persisted session: a transcript would show up as a board card and be distilled again.
+    const r = await run("claude", ["-p", "--no-session-persistence", "--model", model, "--output-format", "text"], {
+        input: prompt,
+        cwd,
+        timeoutMs: 300_000,
+    });
+    if (!r.ok) {
+        log.warn("librarian", `claude -p failed: ${(r.stderr || r.stdout).trim().slice(0, 300)}`);
+        return [];
+    }
+    return parseDrafts(r.stdout);
+}
+/** The text of user and assistant turns, last `maxChars` characters. */
+export async function transcriptTail(jsonlPath, maxChars) {
+    const parts = [];
+    for await (const line of readLines(jsonlPath)) {
+        let obj;
+        try {
+            obj = JSON.parse(line);
+        }
+        catch {
+            continue;
+        }
+        const role = obj?.message?.role ?? obj?.type;
+        if (role !== "user" && role !== "assistant")
+            continue;
+        const content = obj?.message?.content;
+        const text = typeof content === "string"
+            ? content
+            : Array.isArray(content)
+                ? content
+                    .filter((b) => b?.type === "text" && typeof b.text === "string")
+                    .map((b) => b.text)
+                    .join("\n")
+                : "";
+        if (text.trim())
+            parts.push(`${role}: ${text.trim()}`);
+    }
+    const all = parts.join("\n\n");
+    return all.length > maxChars ? all.slice(all.length - maxChars) : all;
+}
+/** README, CLAUDE.md, docs/*.md and the last 50 commit subjects, capped at `maxChars`. */
+export async function projectMaterial(root, maxChars) {
+    const files = ["README.md", "CLAUDE.md", "CONTRIBUTING.md", "ARCHITECTURE.md"];
+    try {
+        const docs = await fs.promises.readdir(path.join(root, "docs"));
+        files.push(...docs.filter((f) => f.endsWith(".md")).map((f) => `docs/${f}`));
+    }
+    catch {
+        // no docs folder
+    }
+    const parts = [];
+    let budget = maxChars;
+    for (const rel of files) {
+        if (budget <= 0)
+            break;
+        let text;
+        try {
+            text = await fs.promises.readFile(path.join(root, rel), "utf8");
+        }
+        catch {
+            continue;
+        }
+        const chunk = `## ${rel}\n${text.slice(0, Math.min(budget, 12_000))}`;
+        parts.push(chunk);
+        budget -= chunk.length;
+    }
+    const logOut = await run("git", ["log", "-50", "--format=%s"], { cwd: root, timeoutMs: 20_000 });
+    if (logOut.ok && budget > 0)
+        parts.push(`## Recent commits\n${logOut.stdout.slice(0, budget)}`);
+    return parts.join("\n\n");
+}

package/server-dist/server/harness/pack.js

@@ -0,0 +1,182 @@
+// The agent pack: skills, subagents and the SessionStart hook from the
+// harness repo, installed at user level (~/.claude) so every project on the
+// machine can use them. On the server they are symlinked (a `git pull`
+// updates them); member machines get copies through `devboard harness install`.
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { IS_WINDOWS, log } from "../util.js";
+const MAX_PACK_FILE = 256 * 1024;
+/**
+ * Hook command for SessionStart: posts the hook payload to the server's
+ * `kb_session_context` and prints the result. `{"result":X}` is unwrapped
+ * with sed so the script needs nothing beyond bash and curl. Any failure
+ * prints nothing, so a server that is down never blocks a session.
+ */
+export function hookScriptContent(serverUrl, token) {
+    const q = (s) => `'${s.replaceAll("'", "'\\''")}'`;
+    return [
+        "#!/usr/bin/env bash",
+        "# Managed by DevBoard (knowledge harness). Edits here will be overwritten.",
+        "set -u",
+        `URL=\${DEVBOARD_URL:-${q(serverUrl)}}`,
+        `TOKEN=\${DEVBOARD_TOKEN:-${q(token ?? "")}}`,
+        "PAYLOAD=$(cat)",
+        'AUTH=(); [ -n "$TOKEN" ] && AUTH=(-H "Authorization: Bearer $TOKEN")',
+        'RESP=$(curl -fsS --max-time 5 "${AUTH[@]}" -H "Content-Type: application/json" \\',
+        '  --data-binary "$PAYLOAD" "$URL/api/invoke/kb_session_context" 2>/dev/null) || exit 0',
+        "printf '%s' \"$RESP\" | sed -e 's/^{\"result\"://' -e 's/}$//'",
+        "",
+    ].join("\n");
+}
+export const HOOK_SCRIPT_NAME = "kb-hook.sh";
+/** Add `command` to `event` in a Claude settings object, idempotently. */
+export function mergeHookCommand(data, event, command) {
+    data ??= {};
+    const hooks = (data.hooks ??= {});
+    const groups = (hooks[event] ??= []);
+    if (groups.length === 0)
+        groups.push({ hooks: [] });
+    const inner = (groups[0].hooks ??= []);
+    if (!inner.some((h) => h?.command === command))
+        inner.push({ type: "command", command });
+    return data;
+}
+export function claudeDir() {
+    return path.join(os.homedir(), ".claude");
+}
+async function writeSettingsHook(command) {
+    const settingsPath = path.join(claudeDir(), "settings.json");
+    await fs.promises.mkdir(path.dirname(settingsPath), { recursive: true });
+    let data = {};
+    try {
+        data = JSON.parse(await fs.promises.readFile(settingsPath, "utf8"));
+    }
+    catch (e) {
+        if (e?.code !== "ENOENT")
+            await fs.promises.rename(settingsPath, `${settingsPath}.bak`).catch(() => { });
+    }
+    await fs.promises.writeFile(settingsPath, JSON.stringify(mergeHookCommand(data, "SessionStart", command), null, 2));
+}
+/** Install the hook script that points at this server. */
+export async function installHook(dataDir, serverUrl, token) {
+    const script = path.join(dataDir, HOOK_SCRIPT_NAME);
+    await fs.promises.mkdir(dataDir, { recursive: true });
+    await fs.promises.writeFile(script, hookScriptContent(serverUrl, token), { mode: 0o700 });
+    await writeSettingsHook(IS_WINDOWS ? script.replaceAll("\\", "/") : script);
+    return script;
+}
+async function sameContent(a, b) {
+    try {
+        return (await fs.promises.readFile(a)).equals(await fs.promises.readFile(b));
+    }
+    catch {
+        return false;
+    }
+}
+/** Symlink each pack skill and subagent into ~/.claude, never replacing a user's own. */
+export async function linkPack(repoPath) {
+    const linked = [];
+    const skipped = [];
+    const link = async (src, dest, type) => {
+        try {
+            const cur = await fs.promises.lstat(dest).catch(() => null);
+            if (cur) {
+                // A copy made by the EPERM fallback below: identical content is ours.
+                if (type === "file" && cur.isFile() && (await sameContent(src, dest)))
+                    return;
+                const target = cur.isSymbolicLink() ? await fs.promises.readlink(dest) : null;
+                if (target && path.resolve(path.dirname(dest), target) === src)
+                    return;
+                // A dangling link (e.g. made from an unexpanded "~" path) is ours to replace.
+                const dangling = !!target && !fs.existsSync(dest);
+                if (!dangling && (!target || !target.startsWith(repoPath))) {
+                    skipped.push(dest);
+                    return;
+                }
+                await fs.promises.rm(dest, { force: true });
+            }
+            await fs.promises.mkdir(path.dirname(dest), { recursive: true });
+            try {
+                await fs.promises.symlink(src, dest, IS_WINDOWS ? (type === "dir" ? "junction" : "file") : undefined);
+            }
+            catch (e) {
+                // Windows file symlinks need admin or Developer Mode; copy instead.
+                if (e?.code !== "EPERM" || type !== "file")
+                    throw e;
+                await fs.promises.copyFile(src, dest);
+            }
+            linked.push(dest);
+        }
+        catch (e) {
+            log.warn("harness", `link ${dest} failed: ${e instanceof Error ? e.message : e}`);
+            skipped.push(dest);
+        }
+    };
+    for (const name of await listDirs(path.join(repoPath, "skills"))) {
+        const src = path.join(repoPath, "skills", name);
+        if (fs.existsSync(path.join(src, "SKILL.md")))
+            await link(src, path.join(claudeDir(), "skills", name), "dir");
+    }
+    for (const file of await listFiles(path.join(repoPath, "agents"), ".md")) {
+        await link(path.join(repoPath, "agents", file), path.join(claudeDir(), "agents", file), "file");
+    }
+    return { linked, skipped };
+}
+/** Everything a member machine needs, as file contents. */
+export async function buildBundle(repoPath, version) {
+    const skills = [];
+    for (const name of await listDirs(path.join(repoPath, "skills"))) {
+        const root = path.join(repoPath, "skills", name);
+        if (!fs.existsSync(path.join(root, "SKILL.md")))
+            continue;
+        skills.push({ name, files: await readTree(root) });
+    }
+    const agents = [];
+    for (const file of await listFiles(path.join(repoPath, "agents"), ".md")) {
+        agents.push({ path: file, content: await fs.promises.readFile(path.join(repoPath, "agents", file), "utf8") });
+    }
+    return { version, skills, agents };
+}
+async function readTree(root) {
+    const out = [];
+    const walk = async (dir) => {
+        for (const it of await fs.promises.readdir(dir, { withFileTypes: true })) {
+            if (it.name.startsWith("."))
+                continue;
+            const abs = path.join(dir, it.name);
+            if (it.isDirectory())
+                await walk(abs);
+            else if (it.isFile() && (await fs.promises.stat(abs)).size <= MAX_PACK_FILE) {
+                out.push({
+                    path: path.relative(root, abs).replaceAll("\\", "/"),
+                    content: await fs.promises.readFile(abs, "utf8"),
+                });
+            }
+        }
+    };
+    await walk(root);
+    return out;
+}
+async function listDirs(dir) {
+    try {
+        return (await fs.promises.readdir(dir, { withFileTypes: true }))
+            .filter((d) => d.isDirectory() && !d.name.startsWith("."))
+            .map((d) => d.name)
+            .sort();
+    }
+    catch {
+        return [];
+    }
+}
+async function listFiles(dir, ext) {
+    try {
+        return (await fs.promises.readdir(dir, { withFileTypes: true }))
+            .filter((d) => d.isFile() && d.name.endsWith(ext) && d.name.toLowerCase() !== "readme.md")
+            .map((d) => d.name)
+            .sort();
+    }
+    catch {
+        return [];
+    }
+}