@foldrun/core 2.4.0 → 2.5.0

package/dist/src/online-checks.js

@@ -0,0 +1,432 @@
+// The checks that need the network (FOL-58), kept apart from `scanFiles`,
+// which stays offline and free:
+//
+//   supply-chain/known-vulnerability — a runtime package pinned to an exact
+//     version that OSV.dev lists advisories for. Only exact pins are asked:
+//     an unpinned package is a different program tomorrow (and
+//     supply-chain/unpinned-package says so). Transitive dependencies are
+//     not resolved — a known limit, documented.
+//   content/mcp-tool-text — what an MCP server reached by URL tells the model
+//     about its tools, through the content rules as untrusted text. A server
+//     started as a command is never run to check it, and no credential is
+//     ever sent: a server that needs one is reported as not checked.
+//
+// Both answer findings in the scanner's shape and say, per source, what they
+// could not check — an unreachable or malformed answer is never a pass.
+// Shaped by a design debate and review with Codex (18 Oct 2026).
+import matter from "./matter.js";
+import { filterIgnored, findingFingerprint, untrustedTextFindings } from "./scan.js";
+import { platform } from "./platform.js";
+import { parseToolDef } from "./store.js";
+export const OSV_BATCH_URL = "https://api.osv.dev/v1/querybatch";
+const TIMEOUT_MS = 10_000;
+const MAX_PINS = 500;
+/** One server: pages of tools/list, tools read, characters of text, bytes read. */
+const MCP_LIMITS = { pages: 10, tools: 300, text: 200_000, bytes: 2_000_000 };
+/** The MCP protocol version this client asks for (2025-06-18 spec). */
+const MCP_VERSION = "2025-06-18";
+const front = (content) => {
+    try {
+        return matter(content).data;
+    }
+    catch {
+        return null;
+    }
+};
+const lineOf = (content, needle) => {
+    const i = content.split(/\r?\n/).findIndex((l) => l.includes(needle));
+    return i >= 0 ? i + 1 : undefined;
+};
+/** Files whose frontmatter can carry a runtime: or mcpServers: block. */
+const CONFIG_FILE = /^(?:library\/)?(?:AGENTS\.md|project\.md|agents\/[^/]+\/agent\.md|tools\/[^/]+(?:\/tool)?\.md|\.(?:claude|agents)\/agents\/[^/]+\.md)$/;
+// ── known vulnerabilities ──────────────────────────────────────────────────
+/** An exact pin: pip `name==1.2.3` (extras allowed), npm `name@1.2.3` or `@scope/name@1.2.3`. */
+export function parsePin(spec) {
+    const s = spec.trim();
+    const pip = /^([A-Za-z0-9][A-Za-z0-9._-]*)(?:\[[^\]]*\])?\s*===?\s*([0-9][A-Za-z0-9.+!-]*)$/.exec(s);
+    if (pip)
+        return { ecosystem: "PyPI", name: pip[1], version: pip[2] };
+    // npm's exact spellings: 1.2.3, =1.2.3, v1.2.3, =v1.2.3 (Codex review).
+    const npm = /^((?:@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*)@=?v?(\d+\.\d+\.\d+(?:[-+][A-Za-z0-9.-]+)?)$/.exec(s);
+    if (npm)
+        return { ecosystem: "npm", name: npm[1], version: npm[2] };
+    return null;
+}
+/** Every exact pin in the workspace's runtime: blocks (agents, AGENTS.md, tools). */
+export function exactPins(files) {
+    const out = [];
+    for (const f of files) {
+        if (!CONFIG_FILE.test(f.path))
+            continue;
+        const runtime = front(f.content)?.runtime;
+        if (!runtime || typeof runtime !== "object")
+            continue;
+        const r = runtime;
+        // The keys the runtime installs from (runtime.ts parseRuntime): packages,
+        // else its pip alias; and npm.
+        for (const list of [r.packages ?? r.pip, r.npm]) {
+            if (!Array.isArray(list))
+                continue;
+            for (const raw of list) {
+                const p = typeof raw === "string" ? parsePin(raw) : null;
+                if (p)
+                    out.push({ ...p, spec: raw, file: f.path, line: lineOf(f.content, raw) });
+            }
+        }
+    }
+    return out;
+}
+const pinKey = (p) => `${p.ecosystem}:${p.name}@${p.version}`;
+/** Ask OSV about every exact pin, once each, in one batch; findings fan out to every file that declares it. */
+export async function vulnerabilityFindings(files, fetchImpl = fetch, deadline = Infinity) {
+    const pins = exactPins(files);
+    const unique = [...new Map(pins.map((p) => [pinKey(p), p])).values()];
+    if (!unique.length)
+        return { findings: [], problems: [], sources: [{ source: "osv", checked: true }] };
+    const asked = unique.slice(0, MAX_PINS);
+    const incomplete = unique.length > MAX_PINS ? [`${unique.length} pinned packages — the first ${MAX_PINS} were checked`] : [];
+    const fail = (problem) => ({ findings: [], problems: [problem], sources: [{ source: "osv", checked: false, problem }] });
+    // The caller's deadline holds here too (Codex review).
+    if (Date.now() >= deadline) {
+        const problem = "the check ran out of time before OSV — packages not checked";
+        return { findings: [], problems: [problem], sources: [{ source: "osv", checked: false, skipped: true, problem }] };
+    }
+    let results;
+    try {
+        const res = await fetchImpl(OSV_BATCH_URL, {
+            method: "POST",
+            headers: { "content-type": "application/json" },
+            body: JSON.stringify({ queries: asked.map((p) => ({ package: { name: p.name, ecosystem: p.ecosystem }, version: p.version })) }),
+            signal: AbortSignal.timeout(Math.max(1, Math.min(TIMEOUT_MS, deadline - Date.now()))),
+        });
+        if (!res.ok)
+            return fail(`OSV answered HTTP ${res.status} — packages not checked`);
+        const body = (await res.json());
+        if (!Array.isArray(body.results) || body.results.length !== asked.length)
+            return fail("OSV's answer did not match the question — packages not checked");
+        results = body.results;
+    }
+    catch (err) {
+        return fail(`OSV could not be reached (${err instanceof Error ? err.message : String(err)}) — packages not checked`);
+    }
+    const findings = [];
+    for (let i = 0; i < asked.length; i++) {
+        const r = results[i];
+        if (!r || typeof r !== "object")
+            return fail("OSV's answer was malformed — packages not checked");
+        const vulns = r.vulns === undefined ? [] : r.vulns;
+        if (!Array.isArray(vulns))
+            return fail("OSV's answer was malformed — packages not checked");
+        const raw = vulns.map((v) => (v && typeof v === "object" ? v.id : null));
+        // An advisory with no id is not "no advisory" — unknown, never clean (Codex review).
+        if (raw.some((x) => typeof x !== "string" || !x))
+            return fail("OSV's answer held an advisory with no id — packages not checked");
+        const ids = [...new Set(raw)].sort();
+        const paged = typeof r.next_page_token === "string" && r.next_page_token.length > 0;
+        const p = asked[i];
+        if (paged && !ids.length) {
+            incomplete.push(`${p.spec}: OSV paged its answer with no advisories on the first page — unknown, not clean`);
+            continue;
+        }
+        if (!ids.length)
+            continue;
+        if (paged)
+            incomplete.push(`${p.spec}: OSV has more advisories than one page — the rest not read`);
+        const count = paged ? `${ids.length} advisories returned (results incomplete)` : `${ids.length} known ${ids.length === 1 ? "advisory" : "advisories"}`;
+        const rule = "supply-chain/known-vulnerability";
+        for (const decl of pins.filter((x) => pinKey(x) === pinKey(p))) {
+            const message = `runtime package ${decl.spec} has ${count}: ${ids.slice(0, 5).join(", ")}${ids.length > 5 ? `, +${ids.length - 5} more` : ""} — upgrade it (https://osv.dev/vulnerability/${ids[0]})`;
+            // The occurrence is the pin in this file; the advisories are its
+            // evidence, so a new advisory on the same pin is not a new finding.
+            findings.push({ rule, level: "warn", file: decl.file, ...(decl.line ? { line: decl.line } : {}), message, fingerprint: findingFingerprint(rule, decl.file, pinKey(decl)), source: "osv" });
+        }
+    }
+    return {
+        findings: filterIgnored(findings, files),
+        problems: incomplete,
+        // A cap or a paged-unknown answer is not a complete check (Codex review).
+        sources: [incomplete.length ? { source: "osv", checked: false, partial: true, problem: incomplete.join("; ") } : { source: "osv", checked: true }],
+    };
+}
+/** MCP servers declared by URL — in an agent's mcpServers: or a tool with transport: mcp. */
+export function remoteMcpServers(files) {
+    const out = [];
+    const add = (name, spec, file, content) => {
+        if (!spec || typeof spec !== "object")
+            return;
+        const s = spec;
+        if (typeof s.url !== "string" || !/^https?:\/\//i.test(s.url))
+            return;
+        let credentialUrl = /\$\{/.test(s.url);
+        try {
+            const u = new URL(s.url);
+            if (u.username || u.password || u.search)
+                credentialUrl = true;
+        }
+        catch {
+            credentialUrl = true;
+        }
+        const hasHeaders = Boolean(s.headers && typeof s.headers === "object" && Object.keys(s.headers).length);
+        out.push({ name, url: s.url, needsCredential: credentialUrl || hasHeaders, file, line: lineOf(content, s.url) });
+    };
+    for (const f of files) {
+        if (!CONFIG_FILE.test(f.path))
+            continue;
+        const data = front(f.content);
+        if (!data)
+            continue;
+        if (data.mcpServers && typeof data.mcpServers === "object") {
+            for (const [name, spec] of Object.entries(data.mcpServers))
+                add(name, spec, f.path, f.content);
+        }
+        // A tool file is an MCP server exactly when the runtime reads it as one:
+        // transport: mcp, the legacy type: mcp, or a url:/command: with neither.
+        const tool = /^(?:library\/)?tools\/([^/]+?)(?:\/tool)?\.md$/.exec(f.path);
+        if (tool) {
+            const def = parseToolDef(data, tool[1], f.content);
+            if (def?.kind === "mcp" && def.spec.url)
+                add(def.name ?? tool[1], { url: def.spec.url, headers: def.spec.headers }, f.path, f.content);
+        }
+    }
+    return out;
+}
+/** Parse one candidate message: the answer to `id`, an error, or not it. */
+function matchAnswer(raw, id) {
+    let msg;
+    try {
+        msg = JSON.parse(raw);
+    }
+    catch {
+        return null;
+    }
+    if (!msg || typeof msg !== "object" || msg.id !== id)
+        return null;
+    const m = msg;
+    if (m.error)
+        return { problem: `answered a JSON-RPC error (${String(m.error.message ?? "unknown").slice(0, 120)})` };
+    if (!m.result || typeof m.result !== "object")
+        return { problem: "answered without a result" };
+    return { msg: m };
+}
+/**
+ * The JSON-RPC message answering `id`: one JSON body, or an SSE stream read
+ * incrementally — events joined per the SSE standard, the reader cancelled
+ * the moment the answer arrives (a server may keep the stream open). Never
+ * more than the byte budget.
+ */
+async function rpcAnswer(res, id, budget) {
+    if (!res.body)
+        return { problem: "gave no answer to the request" };
+    const sse = (res.headers.get("content-type") ?? "").includes("text/event-stream");
+    const reader = res.body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = "";
+    let data = [];
+    const done = async (r) => {
+        await reader.cancel().catch(() => undefined);
+        return r;
+    };
+    for (;;) {
+        const { done: end, value } = await reader.read();
+        if (value) {
+            budget.bytes -= value.byteLength;
+            if (budget.bytes < 0)
+                return done({ problem: `answered more than ${MCP_LIMITS.bytes} bytes` });
+            buffer += decoder.decode(value, { stream: true });
+        }
+        if (sse) {
+            // Complete lines only; an event ends at a blank line. A trailing CR
+            // may be the first half of a CRLF the next chunk completes, so it
+            // waits for more (Codex review).
+            let nl;
+            while ((nl = buffer.search(end ? /\r\n|\r|\n/ : /\r\n|\r(?!$)|\n/)) >= 0) {
+                const line = buffer.slice(0, nl);
+                buffer = buffer.slice(nl + (buffer.startsWith("\r\n", nl) ? 2 : 1));
+                if (line === "") {
+                    if (data.length) {
+                        const hit = matchAnswer(data.join("\n"), id);
+                        data = [];
+                        if (hit)
+                            return done(hit);
+                    }
+                }
+                else if (line.startsWith("data:"))
+                    data.push(line.slice(5).replace(/^ /, ""));
+            }
+        }
+        if (end)
+            break;
+    }
+    if (sse) {
+        if (buffer.startsWith("data:"))
+            data.push(buffer.slice(5).replace(/^ /, ""));
+        if (data.length)
+            return matchAnswer(data.join("\n"), id) ?? { problem: "gave no answer to the request" };
+        return { problem: "gave no answer to the request" };
+    }
+    return matchAnswer(buffer, id) ?? { problem: "gave no answer to the request" };
+}
+/** Everything a tool shows the model: name, title, description, annotation title, schema descriptions. */
+function toolText(tool) {
+    const parts = [tool.name, tool.title, tool.description, tool.annotations?.title].filter((x) => typeof x === "string");
+    // Every string the schema shows the model — property names, defaults,
+    // enum values, examples — not only descriptions (Codex review).
+    const walk = (v) => {
+        if (typeof v === "string") {
+            parts.push(v);
+            return;
+        }
+        if (!v || typeof v !== "object")
+            return;
+        for (const [k, x] of Object.entries(v)) {
+            parts.push(k);
+            walk(x);
+        }
+    };
+    walk(tool.inputSchema);
+    walk(tool.outputSchema);
+    return parts.join("\n");
+}
+/** initialize → notifications/initialized → tools/list (every page, within limits), anonymously. */
+export async function mcpTools(url, fetchImpl, deadline = Infinity) {
+    const budget = { bytes: MCP_LIMITS.bytes };
+    const tools = [];
+    let session = null;
+    let version = MCP_VERSION;
+    const post = (body) => fetchImpl(url, {
+        method: "POST",
+        // Protocol headers only: nothing configured is forwarded, so no
+        // credential can leave on a probe (Codex review).
+        headers: {
+            "content-type": "application/json",
+            accept: "application/json, text/event-stream",
+            ...(session ? { "mcp-session-id": session } : {}),
+            ...(body.method !== "initialize" ? { "mcp-protocol-version": version } : {}),
+        },
+        body: JSON.stringify(body),
+        // The request's own limit, or what is left of the caller's — whichever is sooner.
+        signal: AbortSignal.timeout(Math.max(1, Math.min(TIMEOUT_MS, deadline - Date.now()))),
+        redirect: "manual",
+    });
+    try {
+        const init = await post({ jsonrpc: "2.0", id: 1, method: "initialize", params: { protocolVersion: MCP_VERSION, capabilities: {}, clientInfo: { name: "foldrun-check", version: "1" } } });
+        if (!init.ok)
+            return { problem: `answered HTTP ${init.status} to initialize` };
+        session = init.headers.get("mcp-session-id");
+        const initAnswer = await rpcAnswer(init, 1, budget);
+        if ("problem" in initAnswer)
+            return { problem: `initialize: ${initAnswer.problem}` };
+        const negotiated = initAnswer.msg.result.protocolVersion;
+        if (typeof negotiated !== "string" || !negotiated)
+            return { problem: "initialize: no protocolVersion in the answer" };
+        version = negotiated;
+        const ack = await post({ jsonrpc: "2.0", method: "notifications/initialized" });
+        await ack.body?.cancel().catch(() => undefined);
+        if (!ack.ok)
+            return { problem: `answered HTTP ${ack.status} to notifications/initialized` };
+        let text = 0;
+        let malformed = 0;
+        let cursor;
+        for (let page = 0; page < MCP_LIMITS.pages; page++) {
+            const id = 2 + page;
+            // A failure after the first page keeps what was already read: a
+            // server cannot hide text it sent by failing the next page (Codex review).
+            const stop = (why) => (page === 0 ? { problem: why } : { tools, incomplete: `page ${page + 1} of tools/list: ${why}` });
+            const list = await post({ jsonrpc: "2.0", id, method: "tools/list", ...(cursor ? { params: { cursor } } : {}) });
+            if (!list.ok)
+                return stop(`answered HTTP ${list.status} to tools/list`);
+            const answer = await rpcAnswer(list, id, budget);
+            if ("problem" in answer)
+                return stop(`tools/list: ${answer.problem}`);
+            const result = answer.msg.result;
+            if (!Array.isArray(result.tools))
+                return stop("tools/list: no tools array in the answer");
+            for (const t of result.tools) {
+                // A tool with no name is not a tool the model can call — but it is
+                // not a clean answer either (Codex review).
+                if (!t || typeof t !== "object" || typeof t.name !== "string" || !t.name) {
+                    malformed++;
+                    if (t && typeof t === "object")
+                        tools.push({ name: "(no name)", text: toolText(t) });
+                    continue;
+                }
+                const body = toolText(t);
+                tools.push({ name: t.name.slice(0, 120), text: body });
+                text += body.length;
+                if (tools.length >= MCP_LIMITS.tools || text >= MCP_LIMITS.text)
+                    return { tools, incomplete: `more than ${tools.length} tools or ${MCP_LIMITS.text} characters — the rest not read` };
+            }
+            cursor = typeof result.nextCursor === "string" && result.nextCursor ? result.nextCursor : undefined;
+            if (!cursor)
+                return malformed ? { tools, incomplete: `${malformed} malformed ${malformed === 1 ? "entry" : "entries"} in tools/list` } : { tools };
+        }
+        return { tools, incomplete: `more than ${MCP_LIMITS.pages} pages of tools — the rest not read` };
+    }
+    catch (err) {
+        const why = `could not be reached (${err instanceof Error ? err.message : String(err)})`;
+        // Thrown mid-way (a dropped connection on page 2): what was read stays (Codex review).
+        return tools.length ? { tools, incomplete: why } : { problem: why };
+    }
+}
+/** Every URL-reached MCP server's tool text, through the content rules as untrusted text. */
+export async function mcpDescriptionFindings(files, opts = {}) {
+    const deadline = opts.deadline ?? Infinity;
+    // `order`: source ids to try first (the rescan's least recently checked),
+    // so a slow server early in the list cannot starve the rest for ever.
+    const rank = (s) => {
+        const i = opts.order?.indexOf(`mcp:${s.file}#${s.name}`) ?? -1;
+        return i < 0 ? -1 : i;
+    };
+    const fetchImpl = opts.fetch ?? ((url, init) => platform.fetchUntrusted(url, init));
+    const findings = [];
+    const problems = [];
+    const sources = [];
+    for (const server of remoteMcpServers(files).map((s, i) => ({ s, i })).sort((a, b) => rank(a.s) - rank(b.s) || a.i - b.i).map(({ s }) => s)) {
+        const source = `mcp:${server.file}#${server.name}`;
+        if (server.needsCredential) {
+            const problem = `mcpServers.${server.name}: needs a credential (headers, or a URL carrying one) — no credential is sent on a check, so its tools were not checked`;
+            problems.push(problem);
+            sources.push({ source, checked: false, problem });
+            continue;
+        }
+        if (Date.now() >= deadline) {
+            const problem = `mcpServers.${server.name}: the check ran out of time before it — its tools were not checked`;
+            problems.push(problem);
+            sources.push({ source, checked: false, skipped: true, problem });
+            continue;
+        }
+        const got = await mcpTools(server.url, fetchImpl, deadline);
+        if ("problem" in got) {
+            const problem = `mcpServers.${server.name} (${server.url}) ${got.problem} — its tools were not checked`;
+            problems.push(problem);
+            sources.push({ source, checked: false, problem });
+            continue;
+        }
+        if (got.incomplete)
+            problems.push(`mcpServers.${server.name}: ${got.incomplete}`);
+        sources.push(got.incomplete ? { source, checked: false, partial: true, problem: got.incomplete } : { source, checked: true });
+        for (const tool of got.tools) {
+            for (const f of untrustedTextFindings(tool.text)) {
+                const rule = "content/mcp-tool-text";
+                const message = `mcpServers.${server.name} tool "${tool.name}": ${f.message} [${f.rule}]`;
+                findings.push({
+                    rule,
+                    level: "warn",
+                    file: server.file,
+                    ...(server.line ? { line: server.line } : {}),
+                    message,
+                    fingerprint: findingFingerprint(rule, server.file, `${server.name} ${tool.name} ${f.rule}`),
+                    source,
+                });
+            }
+        }
+    }
+    // The workspace's own foldrun-ignore can accept a finding — the server's
+    // text never can.
+    return { findings: filterIgnored(findings, files), problems, sources };
+}
+/** Both, as one answer. */
+export async function onlineFindings(files, opts = {}) {
+    const [v, m] = await Promise.all([vulnerabilityFindings(files, opts.osvFetch ?? fetch, opts.deadline), mcpDescriptionFindings(files, { fetch: opts.fetch, deadline: opts.deadline, order: opts.order })]);
+    return { findings: [...v.findings, ...m.findings], problems: [...v.problems, ...m.problems], sources: [...v.sources, ...m.sources] };
+}