@armadra/agent 0.9.3 → 0.11.0

package/dist/tools/web/html-to-markdown.js

@@ -0,0 +1,522 @@
+/**
+ * HTML → Markdown（`web_fetch` 用，零依赖）。宽松的分词 + 轻量树，不做完整 HTML5 解析：
+ *
+ * - 丢弃 `script` / `style` / `noscript` / `svg` / `template` / `iframe` / `nav` / `footer` / `aside` /
+ *   `form` / `button` 及其内容，`<!-- -->` 注释与 `<head>`（标题单独取 `<title>`）；
+ * - 正文优先 `<main>`，其次 `<article>`、`role="main"`，再次 `<body>`；
+ * - 保留标题、段落、列表（嵌套缩进）、链接（`[文字](绝对 URL)`，`javascript:` 只留文字）、行内代码与
+ *   `<pre>` 代码块（`language-xxx` 类名作语言）、引用、简单表格、`<br>` / `<hr>`；图片只留 alt；
+ * - 实体：常用命名实体 + 十进制 / 十六进制数字实体；非 `<pre>` 文本折叠空白。
+ *
+ * 输出只依赖输入，便于黄金测试。
+ */
+const VOID = new Set([
+    "area",
+    "base",
+    "br",
+    "col",
+    "embed",
+    "hr",
+    "img",
+    "input",
+    "link",
+    "meta",
+    "param",
+    "source",
+    "track",
+    "wbr",
+]);
+const RAW_TEXT = new Set(["script", "style", "textarea", "title", "xmp", "noscript"]);
+const DROP = new Set([
+    "script",
+    "style",
+    "noscript",
+    "svg",
+    "math",
+    "template",
+    "iframe",
+    "nav",
+    "footer",
+    "aside",
+    "form",
+    "button",
+    "select",
+    "textarea",
+    "head",
+    "title",
+    "canvas",
+    "object",
+    "dialog",
+]);
+const BLOCK = new Set([
+    "address",
+    "article",
+    "blockquote",
+    "body",
+    "center",
+    "dd",
+    "details",
+    "div",
+    "dl",
+    "dt",
+    "figcaption",
+    "figure",
+    "h1",
+    "h2",
+    "h3",
+    "h4",
+    "h5",
+    "h6",
+    "header",
+    "hr",
+    "html",
+    "li",
+    "main",
+    "ol",
+    "p",
+    "pre",
+    "section",
+    "summary",
+    "table",
+    "tbody",
+    "td",
+    "tfoot",
+    "th",
+    "thead",
+    "tr",
+    "ul",
+]);
+/** 打开这些标签时，先关掉还开着的同类（`<li>`、`<p>` 等常省略结束标签）。 */
+const AUTO_CLOSE = {
+    li: ["li"],
+    dt: ["dt", "dd"],
+    dd: ["dt", "dd"],
+    tr: ["tr"],
+    td: ["td", "th"],
+    th: ["td", "th"],
+    option: ["option"],
+};
+const CLOSES_P = new Set([
+    "p",
+    ...[...BLOCK].filter((t) => !["li", "td", "th", "dd", "dt"].includes(t)),
+]);
+/** 自动关闭不越过这些容器（嵌套列表、表格）。 */
+const SCOPE = new Set(["ul", "ol", "dl", "table", "html", "body"]);
+const ENTITIES = {
+    amp: "&",
+    lt: "<",
+    gt: ">",
+    quot: '"',
+    apos: "'",
+    nbsp: " ",
+    copy: "©",
+    reg: "®",
+    trade: "™",
+    hellip: "…",
+    mdash: "—",
+    ndash: "–",
+    lsquo: "‘",
+    rsquo: "’",
+    ldquo: "“",
+    rdquo: "”",
+    laquo: "«",
+    raquo: "»",
+    middot: "·",
+    bull: "•",
+    times: "×",
+    divide: "÷",
+    deg: "°",
+    euro: "€",
+    pound: "£",
+    yen: "¥",
+    cent: "¢",
+    sect: "§",
+    para: "¶",
+    larr: "←",
+    rarr: "→",
+    uarr: "↑",
+    darr: "↓",
+    harr: "↔",
+    rArr: "⇒",
+    lArr: "⇐",
+    le: "≤",
+    ge: "≥",
+    ne: "≠",
+    plusmn: "±",
+    minus: "−",
+    shy: "",
+    zwj: "",
+    zwnj: "",
+    thinsp: " ",
+    ensp: " ",
+    emsp: " ",
+    check: "✓",
+};
+/** 解码字符实体。 */
+export function decodeEntities(text) {
+    return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z][a-z0-9]*);?/gi, (whole, body) => {
+        if (body[0] === "#") {
+            const code = body[1] === "x" || body[1] === "X" ? parseInt(body.slice(2), 16) : Number(body.slice(1));
+            if (!Number.isFinite(code) ||
+                code <= 0 ||
+                code > 0x10ffff ||
+                (code >= 0xd800 && code <= 0xdfff))
+                return "\uFFFD";
+            return String.fromCodePoint(code);
+        }
+        const hit = ENTITIES[body] ?? ENTITIES[body.toLowerCase()];
+        return hit ?? whole;
+    });
+}
+const ATTR_RE = /([^\s"'<>/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;
+function parseAttrs(text) {
+    const attrs = {};
+    for (const m of text.matchAll(ATTR_RE)) {
+        const name = m[1].toLowerCase();
+        if (!(name in attrs))
+            attrs[name] = decodeEntities(m[2] ?? m[3] ?? m[4] ?? "");
+    }
+    return attrs;
+}
+/** 宽松解析成树。 */
+export function parseHtml(html) {
+    const root = { tag: "#root", attrs: {}, children: [] };
+    let cur = root;
+    const open = (tag) => {
+        for (let n = cur; n !== undefined; n = n.parent)
+            if (n.tag === tag)
+                return n;
+        return undefined;
+    };
+    const closeTo = (node) => {
+        cur = node.parent ?? root;
+    };
+    /** 从当前节点往上找还开着的 `tags` 之一并关掉；先碰到 `stop` 里的标签就作罢。 */
+    const closeOpen = (tags, stop) => {
+        for (let n = cur; n !== undefined && n !== root; n = n.parent) {
+            if (tags.includes(n.tag))
+                return closeTo(n);
+            if (stop.has(n.tag))
+                return;
+        }
+    };
+    let i = 0;
+    while (i < html.length) {
+        const lt = html.indexOf("<", i);
+        if (lt === -1) {
+            cur.children.push(html.slice(i));
+            break;
+        }
+        if (lt > i)
+            cur.children.push(html.slice(i, lt));
+        if (html.startsWith("<!--", lt)) {
+            const end = html.indexOf("-->", lt + 4);
+            i = end === -1 ? html.length : end + 3;
+            continue;
+        }
+        if (html[lt + 1] === "!" || html[lt + 1] === "?") {
+            const end = html.indexOf(">", lt);
+            i = end === -1 ? html.length : end + 1;
+            continue;
+        }
+        const m = /^<(\/?)([a-zA-Z][a-zA-Z0-9:-]*)((?:[^>"']|"[^"]*"|'[^']*')*)>/.exec(html.slice(lt, lt + 8192));
+        if (m === null) {
+            cur.children.push("<");
+            i = lt + 1;
+            continue;
+        }
+        i = lt + m[0].length;
+        const tag = m[2].toLowerCase();
+        if (m[1] === "/") {
+            const node = open(tag);
+            if (node !== undefined)
+                closeTo(node);
+            continue;
+        }
+        if (CLOSES_P.has(tag))
+            closeOpen(["p"], BLOCK);
+        const siblings = AUTO_CLOSE[tag];
+        if (siblings !== undefined)
+            closeOpen(siblings, SCOPE);
+        const node = { tag, attrs: parseAttrs(m[3]), children: [], parent: cur };
+        cur.children.push(node);
+        if (RAW_TEXT.has(tag)) {
+            const end = html.toLowerCase().indexOf(`</${tag}`, i);
+            const body = html.slice(i, end === -1 ? html.length : end);
+            node.children.push(body);
+            const close = end === -1 ? -1 : html.indexOf(">", end);
+            i = close === -1 ? html.length : close + 1;
+            continue;
+        }
+        if (!VOID.has(tag) && !/\/\s*$/.test(m[3]))
+            cur = node;
+    }
+    return root;
+}
+function find(node, test) {
+    for (const child of node.children) {
+        if (typeof child === "string")
+            continue;
+        if (test(child))
+            return child;
+        if (DROP.has(child.tag))
+            continue;
+        const hit = find(child, test);
+        if (hit !== undefined)
+            return hit;
+    }
+    return undefined;
+}
+function rawText(node) {
+    return node.children.map((c) => (typeof c === "string" ? c : rawText(c))).join("");
+}
+function resolveHref(href, base) {
+    const value = href.trim();
+    if (value === "" || /^(javascript|data|vbscript):/i.test(value))
+        return undefined;
+    if (value.startsWith("#"))
+        return undefined;
+    try {
+        return base === undefined ? new URL(value).href : new URL(value, base).href;
+    }
+    catch {
+        return undefined;
+    }
+}
+function collapse(text) {
+    return decodeEntities(text).replace(/[\s\u00a0]+/g, " ");
+}
+function inline(node, ctx) {
+    let out = "";
+    for (const child of node.children) {
+        if (typeof child === "string")
+            out += collapse(child);
+        else
+            out += renderInline(child, ctx);
+    }
+    return out;
+}
+function wrap(mark, text) {
+    const trimmed = text.trim();
+    if (trimmed === "")
+        return text;
+    const lead = text.startsWith(" ") ? " " : "";
+    const trail = text.endsWith(" ") ? " " : "";
+    return `${lead}${mark}${trimmed}${mark}${trail}`;
+}
+function renderInline(node, ctx) {
+    if (DROP.has(node.tag))
+        return "";
+    switch (node.tag) {
+        case "br":
+            return "\n";
+        case "img": {
+            const alt = (node.attrs["alt"] ?? "").trim();
+            return alt === "" ? "" : collapse(alt);
+        }
+        case "a": {
+            const text = inline(node, ctx);
+            const href = resolveHref(node.attrs["href"] ?? "", ctx.base);
+            if (href === undefined || text.trim() === "")
+                return text;
+            return `[${text.trim().replace(/[[\]]/g, "\\$&")}](${href.replace(/[()\s]/g, encodeURIComponent)})`;
+        }
+        case "code":
+        case "kbd":
+        case "samp":
+        case "tt": {
+            const text = decodeEntities(rawText(node)).replace(/\s+/g, " ");
+            if (text.trim() === "")
+                return text;
+            const fence = text.includes("`") ? "``" : "`";
+            return `${fence}${text}${fence}`;
+        }
+        case "strong":
+        case "b":
+            return wrap("**", inline(node, ctx));
+        case "em":
+        case "i":
+            return wrap("*", inline(node, ctx));
+        case "del":
+        case "s":
+            return wrap("~~", inline(node, ctx));
+        default:
+            return BLOCK.has(node.tag) ? `\n\n${blocks(node, ctx)}\n\n` : inline(node, ctx);
+    }
+}
+/** 子节点渲染成块序列（块之间空一行）。 */
+function blocks(node, ctx) {
+    const parts = [];
+    let run = "";
+    const flush = () => {
+        const text = run
+            .split("\n")
+            .map((l) => l.replace(/ {2,}/g, " ").trim())
+            .join("\n")
+            .trim();
+        if (text !== "")
+            parts.push(text);
+        run = "";
+    };
+    for (const child of node.children) {
+        if (typeof child === "string") {
+            run += collapse(child);
+            continue;
+        }
+        if (DROP.has(child.tag))
+            continue;
+        if (BLOCK.has(child.tag)) {
+            flush();
+            const text = renderBlock(child, ctx);
+            if (text.trim() !== "")
+                parts.push(text);
+        }
+        else
+            run += renderInline(child, ctx);
+    }
+    flush();
+    return parts.join("\n\n");
+}
+function prefixLines(text, first, rest) {
+    return text
+        .split("\n")
+        .map((line, i) => line === "" ? (i === 0 ? first.trimEnd() : "") : `${i === 0 ? first : rest}${line}`)
+        .join("\n");
+}
+function renderList(node, ctx) {
+    const ordered = node.tag === "ol";
+    let n = Number(node.attrs["start"] ?? "1");
+    if (!Number.isFinite(n))
+        n = 1;
+    const items = [];
+    const inner = { ...ctx, listDepth: ctx.listDepth + 1 };
+    for (const child of node.children) {
+        if (typeof child === "string" || DROP.has(child.tag))
+            continue;
+        const body = child.tag === "li" ? blocks(child, inner) : renderBlock(child, inner);
+        if (body.trim() === "")
+            continue;
+        const marker = ordered ? `${n++}. ` : "- ";
+        items.push(prefixLines(body.replace(/\n{2,}/g, "\n"), marker, " ".repeat(marker.length)));
+    }
+    return items.join("\n");
+}
+function cellText(node, ctx) {
+    return blocks(node, ctx)
+        .replace(/\s*\n+\s*/g, " ")
+        .replace(/\|/g, "\\|")
+        .trim();
+}
+function renderTable(node, ctx) {
+    const rows = [];
+    const walk = (n) => {
+        for (const child of n.children) {
+            if (typeof child === "string" || DROP.has(child.tag))
+                continue;
+            if (child.tag === "tr") {
+                const cells = child.children.filter((c) => typeof c !== "string" && (c.tag === "td" || c.tag === "th"));
+                if (cells.length > 0)
+                    rows.push({
+                        cells: cells.map((c) => cellText(c, ctx)),
+                        header: n.tag === "thead" || cells.every((c) => c.tag === "th"),
+                    });
+            }
+            else if (child.tag !== "table")
+                walk(child);
+        }
+    };
+    walk(node);
+    if (rows.length === 0)
+        return "";
+    const width = Math.max(...rows.map((r) => r.cells.length));
+    const line = (cells) => `| ${[...cells, ...new Array(width - cells.length).fill("")].join(" | ")} |`;
+    const headerIndex = rows[0]?.header === true ? 0 : -1;
+    const head = headerIndex === 0
+        ? rows.shift().cells
+        : new Array(width).fill("");
+    return [line(head), `|${" --- |".repeat(width)}`, ...rows.map((r) => line(r.cells))].join("\n");
+}
+function renderBlock(node, ctx) {
+    switch (node.tag) {
+        case "h1":
+        case "h2":
+        case "h3":
+        case "h4":
+        case "h5":
+        case "h6": {
+            const text = inline(node, ctx).replace(/\s+/g, " ").trim();
+            return text === "" ? "" : `${"#".repeat(Number(node.tag[1]))} ${text}`;
+        }
+        case "hr":
+            return "---";
+        case "pre": {
+            const code = find(node, (n) => n.tag === "code");
+            const cls = `${code?.attrs["class"] ?? ""} ${node.attrs["class"] ?? ""}`;
+            const lang = /(?:^|\s)(?:language|lang)-([\w+#-]+)/.exec(cls)?.[1] ?? "";
+            const text = decodeEntities(rawText(node)).replace(/^\n/, "").replace(/\s+$/, "");
+            const fence = text.includes("```") ? "~~~~" : "```";
+            return `${fence}${lang}\n${text}\n${fence}`;
+        }
+        case "blockquote":
+            return prefixLines(blocks(node, ctx), "> ", "> ");
+        case "ul":
+        case "ol":
+            return renderList(node, ctx);
+        case "table":
+            return renderTable(node, ctx);
+        default:
+            return blocks(node, ctx);
+    }
+}
+/** 页面标题：`<title>`，没有时取第一个 `<h1>`。 */
+function pageTitle(root) {
+    const title = find(root, (n) => n.tag === "title") ?? findIn(root, "title");
+    const text = title !== undefined ? collapse(rawText(title)).trim() : "";
+    if (text !== "")
+        return text;
+    const h1 = find(root, (n) => n.tag === "h1");
+    const heading = h1 !== undefined ? collapse(rawText(h1)).trim() : "";
+    return heading === "" ? undefined : heading;
+}
+/** 在丢弃的子树（`<head>`）里也找。 */
+function findIn(node, tag) {
+    for (const child of node.children) {
+        if (typeof child === "string")
+            continue;
+        if (child.tag === tag)
+            return child;
+        const hit = findIn(child, tag);
+        if (hit !== undefined)
+            return hit;
+    }
+    return undefined;
+}
+/** HTML 全文 → `{ title, markdown }`。`baseUrl` 用来把相对链接变成绝对 URL。 */
+export function htmlToMarkdown(html, baseUrl) {
+    const root = parseHtml(html);
+    let base;
+    try {
+        const declared = findIn(root, "base")?.attrs["href"];
+        base = baseUrl === undefined ? undefined : new URL(declared ?? "", baseUrl);
+    }
+    catch {
+        base = baseUrl === undefined ? undefined : new URL(baseUrl);
+    }
+    const main = find(root, (n) => n.tag === "main") ??
+        find(root, (n) => n.tag === "article") ??
+        find(root, (n) => n.attrs["role"] === "main") ??
+        find(root, (n) => n.tag === "body") ??
+        root;
+    const markdown = blocks(main, { base, listDepth: 0 })
+        .replace(/[ \t]+\n/g, "\n")
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+    const title = pageTitle(root);
+    return title === undefined ? { markdown } : { title, markdown };
+}
+/** `<meta charset>` / `http-equiv` 里声明的编码（只看前 2 KB）。 */
+export function sniffMetaCharset(head) {
+    const text = head.slice(0, 2048);
+    const direct = /<meta[^>]+charset\s*=\s*["']?\s*([\w.:-]+)/i.exec(text);
+    return direct?.[1]?.toLowerCase();
+}

package/dist/tools/web/url-guard.d.ts

@@ -0,0 +1,59 @@
+/**
+ * `web_fetch` 的地址守卫（docs/guides/permissions.md「网络类」）：URL 校验、私网 / 元数据地址判定、
+ * DNS 钉住与重定向策略。纯 `node:*`，模型侧固定英文。
+ *
+ * - URL：只收 `http:` / `https:`；拒 userinfo；全文 ≤ 2 048 字符、查询串 ≤ 1 024 字符；主机名必须含 `.`
+ *   （`localhost`、`*.localhost` 与无点主机一律拒——本机服务请走 bash 审批）；IP 字面量当场判定。
+ * - 地址：解析出的**全部**地址都不得落在环回、未指定、链路本地（含云元数据 `169.254.169.254`）、
+ *   RFC 1918、运营商 NAT `100.64/10`、基准测试 `198.18/15`、保留与广播、组播、IPv6 ULA `fc00::/7`、
+ *   站点本地 `fec0::/10`、链路本地 `fe80::/10`、组播 `ff00::/8`、未指定与已弃用的兼容段 `::/96`；IPv4 映射 / 兼容、NAT64 `64:ff9b::/96`、
+ *   6to4 `2002::/16` 里嵌的 IPv4 按 IPv4 再判一次。有一个落在里面就拒（防「多地址里夹一个内网」）。
+ * - DNS 钉住：连接用 `pinnedLookup()` 只返回已检查过的地址，防检查与连接之间被重新解析（DNS 重绑定）。
+ * - 重定向：同主机（主机名与端口相同、不从 https 降到 http）由调用方跟随并逐跳重检；跨主机不跟。
+ */
+import { type LookupAddress } from "node:dns";
+export declare const MAX_URL_LENGTH = 2048;
+export declare const MAX_QUERY_LENGTH = 1024;
+export declare const MAX_SAME_HOST_REDIRECTS = 5;
+/** IPv6 文本 → 8 个 16 位组；不合法返回 undefined。支持 `::` 与末尾点分 IPv4。 */
+export declare function ipv6Groups(address: string): number[] | undefined;
+/** IPv6 里嵌的 IPv4（映射 / 兼容 / NAT64 / 6to4）；没有返回 undefined。 */
+export declare function embeddedIPv4(address: string): string | undefined;
+/**
+ * 地址是否落在禁止段；落在里面返回英文原因（进工具结果），否则 undefined。
+ * 不是 IP 文本的一律按禁止（调用方只该传解析结果）。
+ */
+export declare function blockedAddressReason(address: string): string | undefined;
+export type UrlCheck = {
+    ok: true;
+    url: URL;
+    hostname: string;
+} | {
+    ok: false;
+    reason: string;
+};
+/** 主机名：去方括号与末尾点，小写。 */
+export declare function bareHostname(url: URL): string;
+/** 静态校验（不查 DNS）：协议、userinfo、长度、主机名形状、IP 字面量。 */
+export declare function checkFetchUrl(raw: string): UrlCheck;
+export type LookupAll = (hostname: string) => Promise<LookupAddress[]>;
+/** 缺省解析：系统解析器，返回全部地址。 */
+export declare const systemLookup: LookupAll;
+export type ResolveCheck = {
+    ok: true;
+    addresses: LookupAddress[];
+} | {
+    ok: false;
+    reason: string;
+};
+/** 解析主机名并检查全部地址；IP 字面量直接返回。 */
+export declare function resolvePublicAddresses(hostname: string, lookup?: LookupAll): Promise<ResolveCheck>;
+type LookupCallback = (err: NodeJS.ErrnoException | null, address: string | LookupAddress[], family?: number) => void;
+/** `http.request({ lookup })` 用：只返回已检查过的地址（兼容 `all: true` 与单地址两种调用）。 */
+export declare function pinnedLookup(addresses: readonly LookupAddress[]): (_hostname: string, options: unknown, callback?: LookupCallback) => void;
+/**
+ * 重定向能否自动跟：同主机名，且同协议同端口或 http → https 升级。其余（跨主机、换端口、
+ * https 降到 http）不跟：交给模型再调一次 web_fetch，下一次调用重新过权限。
+ */
+export declare function redirectPolicy(from: URL, to: URL): "follow" | "cross-host";
+export {};