@luuthuong/devboard 0.0.0-stage → 1.0.1

package/server-dist/server/harness/proposals.js

@@ -0,0 +1,113 @@
+// Proposed knowledge entries waiting for a person: kb-proposals.json in the
+// data dir. Nothing the librarian writes reaches the knowledge base until a
+// proposal is approved.
+import fs from "node:fs";
+import path from "node:path";
+import { Mutex, devboardDataDir, ksuid, writeFileAtomic } from "../util.js";
+import { isKnowledgeType, normalizeScope } from "./entry.js";
+import { scrub } from "./scrub.js";
+const MAX_TITLE = 120;
+const MAX_SUMMARY = 240;
+const MAX_BODY = 4000;
+const DECIDED_RETENTION_MS = 182 * 24 * 3_600_000;
+/** Validate and clean a proposal; throws with a readable message when unusable. */
+export function cleanProposal(p) {
+    if (!p.kbId)
+        throw new Error("kbId is required");
+    if (!isKnowledgeType(p.type))
+        throw new Error(`unknown type: ${String(p.type)}`);
+    const title = scrub(String(p.title ?? "").trim()).slice(0, MAX_TITLE);
+    const body = scrub(String(p.body ?? "").trim()).slice(0, MAX_BODY);
+    if (!title)
+        throw new Error("title is required");
+    if (!body)
+        throw new Error("body is required");
+    const summary = scrub(String(p.summary ?? "").trim()).slice(0, MAX_SUMMARY) || title;
+    const scope = [...new Set((p.scope ?? []).map(normalizeScope).filter((s) => s && !s.startsWith("..")))];
+    return {
+        kbId: p.kbId,
+        type: p.type,
+        title,
+        summary,
+        body,
+        scope,
+        source: p.source,
+        reason: p.reason ? scrub(p.reason).slice(0, 500) : undefined,
+        origin: p.origin,
+        // Entry ids are whatever the files say; only reject what cannot be an id at all.
+        supersedes: p.supersedes?.filter((id) => typeof id === "string" && /^[\w.-]{1,100}$/.test(id)).slice(0, 10),
+    };
+}
+export class ProposalStore {
+    filePath;
+    lock = new Mutex();
+    constructor(base) {
+        this.filePath = path.join(base ?? devboardDataDir(), "kb-proposals.json");
+    }
+    /** Newest first. `cardId` keeps proposals made from that card's sessions. */
+    async list(status, from) {
+        const sessions = new Set(from?.sessionIds ?? []);
+        return (await this.read())
+            .filter((p) => !status || p.status === status)
+            .filter((p) => !from ||
+            (!!from.cardId && p.source?.cardId === from.cardId) ||
+            (!!p.source?.sessionId && sessions.has(p.source.sessionId)))
+            .sort((a, b) => b.createdAt.localeCompare(a.createdAt));
+    }
+    async get(id) {
+        return (await this.read()).find((p) => p.id === id);
+    }
+    /** Adds proposals, skipping any whose title already waits in the same KB. */
+    add(inputs) {
+        return this.lock.run(async () => {
+            const all = await this.read();
+            const pendingTitles = new Set(all.filter((p) => p.status === "pending").map((p) => `${p.kbId}\u0000${p.title.toLowerCase()}`));
+            const added = [];
+            for (const input of inputs) {
+                const clean = cleanProposal(input);
+                const key = `${clean.kbId}\u0000${clean.title.toLowerCase()}`;
+                if (pendingTitles.has(key))
+                    continue;
+                pendingTitles.add(key);
+                const p = {
+                    ...clean,
+                    id: ksuid("prop"),
+                    status: "pending",
+                    createdAt: new Date().toISOString(),
+                };
+                all.push(p);
+                added.push(p);
+            }
+            if (added.length)
+                await this.write(all);
+            return added;
+        });
+    }
+    update(id, fn) {
+        return this.lock.run(async () => {
+            const all = await this.read();
+            const p = all.find((x) => x.id === id);
+            if (!p)
+                throw new Error(`proposal ${id} not found`);
+            fn(p);
+            await this.write(all);
+            return p;
+        });
+    }
+    async read() {
+        try {
+            const parsed = JSON.parse(await fs.promises.readFile(this.filePath, "utf8"));
+            return Array.isArray(parsed?.proposals) ? parsed.proposals : [];
+        }
+        catch {
+            return [];
+        }
+    }
+    async write(all) {
+        // Decided proposals are history; their entries live in the harness repo. Keep half a year.
+        const cutoff = new Date(Date.now() - DECIDED_RETENTION_MS).toISOString();
+        const kept = all.filter((p) => p.status === "pending" || (p.decidedAt ?? p.createdAt) >= cutoff);
+        await fs.promises.mkdir(path.dirname(this.filePath), { recursive: true });
+        await writeFileAtomic(this.filePath, JSON.stringify({ proposals: kept }, null, 2));
+    }
+}

package/server-dist/server/harness/repo.js

@@ -0,0 +1,150 @@
+// Git plumbing for the private harness repo. Reads use the server's clone;
+// every change goes through its own temporary worktree on a new branch and a
+// pull request, so the clone the index reads from is never mid-edit. A repo
+// without an `origin` (local trial) gets a plain commit instead.
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { Mutex, findOnPath, run } from "../util.js";
+const GIT_TIMEOUT = 120_000;
+const writeLock = new Mutex();
+async function git(cwd, args, timeoutMs = GIT_TIMEOUT) {
+    const r = await run("git", args, { cwd, timeoutMs });
+    if (!r.ok)
+        throw new Error(`git ${args[0]} failed: ${(r.stderr || r.stdout).trim()}`);
+    return r.stdout.trim();
+}
+/** `~` and `~/x` mean the user's home; node's `path` does not expand them. */
+export function expandHome(p) {
+    return p === "~" || /^~[\\/]/.test(p) ? path.join(os.homedir(), p.slice(1)) : p;
+}
+/** True only when `dir` is itself a repo root, not a folder inside some other repo. */
+export async function isGitRepo(dir) {
+    const r = await run("git", ["rev-parse", "--show-toplevel"], { cwd: dir, timeoutMs: 10_000 });
+    if (!r.ok)
+        return false;
+    const norm = (x) => path.resolve(x).toLowerCase();
+    return norm(r.stdout.trim()) === norm(dir);
+}
+export async function hasOrigin(repo) {
+    const r = await run("git", ["remote", "get-url", "origin"], { cwd: repo, timeoutMs: 10_000 });
+    return r.ok && !!r.stdout.trim();
+}
+/** Clone `gitUrl` into `repoPath` unless a repo is already there. */
+export async function ensureClone(repoPath, gitUrl) {
+    if (await isGitRepo(repoPath))
+        return;
+    if (!gitUrl)
+        throw new Error(`${repoPath} is not a git repository and no gitUrl is set`);
+    await fs.promises.mkdir(path.dirname(repoPath), { recursive: true });
+    const r = await run("git", ["clone", gitUrl, repoPath], { timeoutMs: 600_000 });
+    if (!r.ok)
+        throw new Error(`git clone failed: ${r.stderr.trim()}`);
+}
+/** Fast-forward the clone. Resolves true when HEAD moved. */
+export async function pull(repoPath) {
+    if (!(await hasOrigin(repoPath)))
+        return false;
+    const before = await git(repoPath, ["rev-parse", "HEAD"]).catch(() => "");
+    await git(repoPath, ["pull", "--ff-only", "--quiet"]);
+    const after = await git(repoPath, ["rev-parse", "HEAD"]).catch(() => "");
+    return before !== after;
+}
+async function defaultBranch(repo) {
+    const r = await run("git", ["symbolic-ref", "--short", "refs/remotes/origin/HEAD"], { cwd: repo, timeoutMs: 10_000 });
+    if (r.ok && r.stdout.trim())
+        return r.stdout.trim().replace(/^origin\//, "");
+    return (await git(repo, ["rev-parse", "--abbrev-ref", "HEAD"])) || "main";
+}
+/** `-c user.*` only when the repo has no identity, so commits never fail on a fresh server. */
+async function identityArgs(repo) {
+    const email = await run("git", ["config", "user.email"], { cwd: repo, timeoutMs: 10_000 });
+    return email.ok && email.stdout.trim() ? [] : ["-c", "user.name=DevBoard", "-c", "user.email=devboard@localhost"];
+}
+/**
+ * Apply `edit` to a fresh checkout of the default branch and propose it.
+ * `edit` gets the checkout's root and writes files there.
+ */
+export function proposeChange(repoPath, req, edit) {
+    return writeLock.run(async () => {
+        const ident = await identityArgs(repoPath);
+        if (!(await hasOrigin(repoPath))) {
+            await edit(repoPath);
+            await git(repoPath, ["add", "-A"]);
+            await git(repoPath, [...ident, "commit", "-q", "-m", req.message]);
+            return { commit: await git(repoPath, ["rev-parse", "HEAD"]), mode: "local" };
+        }
+        await git(repoPath, ["fetch", "--quiet", "origin"]);
+        const base = await defaultBranch(repoPath);
+        const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "devboard-harness-"));
+        await fs.promises.rm(dir, { recursive: true, force: true });
+        await git(repoPath, ["worktree", "add", "-q", "-b", req.branch, dir, `origin/${base}`]);
+        try {
+            await edit(dir);
+            await git(dir, ["add", "-A"]);
+            await git(dir, [...ident, "commit", "-q", "-m", req.message]);
+            const commit = await git(dir, ["rev-parse", "HEAD"]);
+            await git(dir, ["push", "-q", "-u", "origin", req.branch]);
+            if (!findOnPath("gh"))
+                return { commit, branch: req.branch, mode: "branch" };
+            const pr = await run("gh", ["pr", "create", "--base", base, "--head", req.branch, "--title", req.prTitle, "--body", req.prBody], { cwd: dir, timeoutMs: GIT_TIMEOUT });
+            const url = pr.ok
+                ? pr.stdout
+                    .trim()
+                    .split(/\s+/)
+                    .find((s) => s.startsWith("http"))
+                : undefined;
+            return { commit, branch: req.branch, url, mode: url ? "pr" : "branch" };
+        }
+        finally {
+            await run("git", ["worktree", "remove", "--force", dir], { cwd: repoPath, timeoutMs: GIT_TIMEOUT });
+            await run("git", ["branch", "-D", req.branch], { cwd: repoPath, timeoutMs: GIT_TIMEOUT });
+        }
+    });
+}
+/**
+ * Take back an entry file (`file`, relative to the repo root) that `proposeChange` added.
+ * Not merged yet: close its pull request and delete its branch. Already merged: propose a
+ * change that deletes the file.
+ */
+export async function retractChange(repoPath, file, prev, req) {
+    const posix = file.split(path.sep).join("/");
+    const remove = async (dir) => {
+        await fs.promises.rm(path.join(dir, file), { force: true });
+    };
+    if (!(await hasOrigin(repoPath))) {
+        if (!fs.existsSync(path.join(repoPath, file)))
+            return { closed: true };
+        return { revert: await proposeChange(repoPath, req, remove) };
+    }
+    await git(repoPath, ["fetch", "--quiet", "origin"]);
+    const base = await defaultBranch(repoPath);
+    const merged = (await run("git", ["cat-file", "-e", `origin/${base}:${posix}`], { cwd: repoPath, timeoutMs: 10_000 }))
+        .ok;
+    if (merged)
+        return { revert: await proposeChange(repoPath, req, remove) };
+    if (prev.prUrl?.startsWith("http") && findOnPath("gh")) {
+        await run("gh", ["pr", "close", prev.prUrl, "--delete-branch"], { cwd: repoPath, timeoutMs: GIT_TIMEOUT });
+    }
+    // `gh pr close --delete-branch` may be unavailable or already done; either way the branch must go.
+    await run("git", ["push", "-q", "origin", "--delete", prev.branch], { cwd: repoPath, timeoutMs: GIT_TIMEOUT });
+    return { closed: true };
+}
+/** Every file changed in `repo` since `sinceIso`, with the time of its latest change (ms). One `git log`. */
+export async function fileChangeTimes(repo, sinceIso) {
+    const r = await run("git", ["log", `--since=${sinceIso}`, "--name-only", "--format=%x01%ct"], {
+        cwd: repo,
+        timeoutMs: 60_000,
+    });
+    const out = new Map();
+    if (!r.ok)
+        return out;
+    let at = 0;
+    for (const line of r.stdout.split(/\r?\n/)) {
+        if (line.startsWith("\x01"))
+            at = Number(line.slice(1)) * 1000;
+        else if (line && !out.has(line))
+            out.set(line, at); // newest first: the first time seen is the latest
+    }
+    return out;
+}

package/server-dist/server/harness/scrub.js

@@ -0,0 +1,30 @@
+// Redaction applied to anything the librarian reads before it reaches a model,
+// and to every proposal before it is written to the harness repo.
+import os from "node:os";
+const PATTERNS = [
+    [/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, "<private-key>"],
+    [/\bsk-ant-[A-Za-z0-9_-]{20,}/g, "<anthropic-key>"],
+    [/\bsk-[A-Za-z0-9]{20,}/g, "<api-key>"],
+    [/\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{30,}/g, "<github-token>"],
+    [/\bgithub_pat_[A-Za-z0-9_]{30,}/g, "<github-token>"],
+    [/\bxox[abposr]-[A-Za-z0-9-]{10,}/g, "<slack-token>"],
+    [/\bAKIA[0-9A-Z]{16}\b/g, "<aws-key>"],
+    [/\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}/g, "<jwt>"],
+    [/\b(https?:\/\/)[^\s/:@]+:[^\s/@]+@/g, "$1<credentials>@"],
+    [
+        /\b((?:api[_-]?key|secret|token|password|passwd|client[_-]?secret|access[_-]?key)["']?\s*[:=]\s*["']?)[^\s"',;]{6,}/gi,
+        "$1<redacted>",
+    ],
+];
+export function scrub(text) {
+    let out = text;
+    for (const [re, repl] of PATTERNS)
+        out = out.replace(re, repl);
+    const home = os.homedir();
+    if (home && home.length > 1)
+        out = out.split(home).join("~");
+    const host = os.hostname();
+    if (host && host.length > 2)
+        out = out.split(host).join("<host>");
+    return out;
+}