minimal-agent 0.8.2 → 0.10.0

package/skills/web-crawler/scripts/crawl.mjs

@@ -0,0 +1,538 @@
+#!/usr/bin/env node
+/**
+ * ============================================================
+ *  skills/web-crawler/scripts/crawl.mjs —— 通用爬虫引擎（node + bun 通用，零 npm 依赖）
+ * ------------------------------------------------------------
+ *  集合主流爬虫该有的能力，做成 minimal-agent 的 bun 脚本 skill：
+ *    - 静态优先 + JS 渲染升级（疑似 JS-app 或 --render on 时 spawn agent-browser 取渲染后 DOM）
+ *    - 内容净抽：HTML→markdown（turndown 动态探测，回落内置转换）+ title + OpenGraph + JSON-LD + 主图
+ *    - 整站爬取：BFS 前沿 + URL 规范化去重 + depth/max-pages 双闸 + 同域/白名单 + robots.txt + sitemap 播种 + 礼貌限速 + retry
+ *    - 输出：每页 out/page_NNN.md（front-matter）+ out/crawl.json 清单
+ *
+ *  ★ 零 npm 依赖（只用 node 内置 fetch/fs/URL）——因为 skills/ 原样发布、绿色版目录无 node_modules，
+ *    依赖 npm 包的 .mjs 在绿色版跑不起来。turndown 用动态 import 探测：有就用（质量更好）、无则回落内置转换。
+ *  ★ agent-browser 经 PATH 调用（minimal-agent 启动时已把内置 agent-browser 目录前置进 PATH）。
+ *
+ *  纯函数（extract* / canonicalizeUrl / parseRobots / fallbackHtmlToMarkdown / looksJsRendered / shouldFollow）
+ *  全部 export 供单测；crawl() 接受 deps 注入（fetchImpl/renderImpl）不连真网测试。
+ *
+ *  用法：
+ *    node crawl.mjs <url> [--mode single|site] [--out ./build/crawl] [--depth 2] [--max-pages 20]
+ *         [--same-origin | --allow-domains a.com,b.com] [--render auto|on|off]
+ *         [--concurrency 3] [--delay 500] [--timeout 20000] [--json]
+ * ============================================================
+ */
+
+import { mkdir, writeFile } from 'node:fs/promises';
+import { spawn } from 'node:child_process';
+import { join } from 'node:path';
+
+const UA =
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36';
+
+// ───────────────────────── 纯函数：抽取 ─────────────────────────
+
+/** <title> 文本（去标签、解实体、trim）。 */
+export function extractTitle(html) {
+  const m = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(html || '');
+  return m ? decodeEntities(m[1]).replace(/\s+/g, ' ').trim() : '';
+}
+
+/** OpenGraph：{ 'og:title': '...', 'og:image': '...' }（property/content 任意顺序）。 */
+export function extractOpenGraph(html) {
+  const out = {};
+  const re = /<meta\b[^>]*>/gi;
+  for (const tag of (html || '').match(re) || []) {
+    const prop = /(?:property|name)\s*=\s*["']\s*(og:[^"']+?)\s*["']/i.exec(tag);
+    const content = /content\s*=\s*["']([\s\S]*?)["']/i.exec(tag);
+    if (prop && content) out[prop[1].toLowerCase()] = decodeEntities(content[1]).trim();
+  }
+  return out;
+}
+
+/** JSON-LD：解析所有 <script type="application/ld+json"> 块（坏块跳过）。 */
+export function extractJsonLd(html) {
+  const out = [];
+  const re = /<script\b[^>]*type\s*=\s*["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
+  let m;
+  while ((m = re.exec(html || '')) !== null) {
+    try {
+      out.push(JSON.parse(m[1].trim()));
+    } catch {
+      /* 坏 JSON-LD 跳过 */
+    }
+  }
+  return out;
+}
+
+/** 抽 <a href> → 绝对 http(s) URL 数组（相对地址按 base 解析、去重）。 */
+export function extractLinks(html, baseUrl) {
+  const out = new Set();
+  const re = /<a\b[^>]*\bhref\s*=\s*["']([^"']+)["']/gi;
+  let m;
+  while ((m = re.exec(html || '')) !== null) {
+    const href = m[1].trim();
+    if (!href || href.startsWith('#') || /^(javascript|mailto|tel):/i.test(href)) continue;
+    try {
+      const abs = new URL(href, baseUrl).href;
+      if (/^https?:/i.test(abs)) out.add(abs);
+    } catch {
+      /* 非法 URL 跳过 */
+    }
+  }
+  return [...out];
+}
+
+/** 主图：优先 og:image，否则第一张较大 <img src>。 */
+export function extractMainImage(html, og, baseUrl) {
+  if (og && og['og:image']) {
+    try {
+      return new URL(og['og:image'], baseUrl).href;
+    } catch {
+      /* fall through */
+    }
+  }
+  const m = /<img\b[^>]*\bsrc\s*=\s*["']([^"']+)["']/i.exec(html || '');
+  if (m) {
+    try {
+      return new URL(m[1], baseUrl).href;
+    } catch {
+      /* ignore */
+    }
+  }
+  return '';
+}
+
+// ───────────────────────── 纯函数：HTML→Markdown ─────────────────────────
+
+const ENTITIES = {
+  '&amp;': '&', '&lt;': '<', '&gt;': '>', '&quot;': '"', '&#39;': "'",
+  '&apos;': "'", '&nbsp;': ' ', '&mdash;': '—', '&ndash;': '–', '&hellip;': '…',
+};
+export function decodeEntities(s) {
+  return String(s || '')
+    .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(Number(n)))
+    .replace(/&#x([0-9a-f]+);/gi, (_, n) => String.fromCodePoint(parseInt(n, 16)))
+    .replace(/&[a-z]+;/gi, (e) => ENTITIES[e.toLowerCase()] ?? e);
+}
+
+/**
+ * 零依赖的内置 HTML→Markdown 回落（turndown 不可用时用）。粗但够爬虫消费：
+ * 剥 script/style/head/nav/footer，标题→#、链接→[t](u)、列表→-、p/br→换行、其余去标签。
+ */
+export function fallbackHtmlToMarkdown(html) {
+  let s = String(html || '');
+  s = s.replace(/<!--[\s\S]*?-->/g, '');
+  s = s.replace(/<(script|style|head|noscript|svg|template)\b[\s\S]*?<\/\1>/gi, '');
+  s = s.replace(/<(nav|footer|header|aside)\b[\s\S]*?<\/\1>/gi, ' ');
+  s = s.replace(/<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/gi, (_, n, t) => `\n\n${'#'.repeat(Number(n))} ${strip(t)}\n\n`);
+  s = s.replace(/<a\b[^>]*\bhref\s*=\s*["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, (_, href, t) => {
+    const text = strip(t);
+    return text ? `[${text}](${href})` : '';
+  });
+  s = s.replace(/<li\b[^>]*>([\s\S]*?)<\/li>/gi, (_, t) => `\n- ${strip(t)}`);
+  s = s.replace(/<(p|div|section|article|tr|h[1-6]|ul|ol|table|br|hr)\b[^>]*>/gi, '\n');
+  s = s.replace(/<\/(p|div|section|article|tr|li|ul|ol|table)>/gi, '\n');
+  s = s.replace(/<[^>]+>/g, ''); // 剩余标签全去
+  s = decodeEntities(s);
+  s = s.replace(/[ \t]+\n/g, '\n').replace(/\n{3,}/g, '\n\n').replace(/[ \t]{2,}/g, ' ');
+  return s.trim();
+}
+
+function strip(htmlFrag) {
+  return decodeEntities(String(htmlFrag || '').replace(/<[^>]+>/g, '')).replace(/\s+/g, ' ').trim();
+}
+
+/** turndown 有就用（质量更好），无则回落内置。异步（动态 import）。 */
+export async function htmlToMarkdown(html) {
+  try {
+    const mod = await import('turndown');
+    const Turndown = mod.default || mod;
+    const td = new Turndown({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+    // 先剥 script/style（turndown 不剥）
+    const cleaned = String(html || '').replace(/<(script|style|noscript)\b[\s\S]*?<\/\1>/gi, '');
+    return td.turndown(cleaned).trim();
+  } catch {
+    return fallbackHtmlToMarkdown(html);
+  }
+}
+
+// ───────────────────────── 纯函数：URL / 域名 / robots ─────────────────────────
+
+const TRACKING_PARAMS = /^(utm_|fbclid$|gclid$|gbraid$|wbraid$|msclkid$|ref$|ref_src$|spm$|from$|yclid$|igshid$)/i;
+
+/** URL 规范化：去 hash、剥 tracking 参数、host 小写、去默认端口、去末尾斜杠（非根）。用于去重。 */
+export function canonicalizeUrl(url) {
+  try {
+    const u = new URL(url);
+    u.hash = '';
+    u.hostname = u.hostname.toLowerCase();
+    if (u.pathname !== '/') u.pathname = u.pathname.replace(/\/+$/, '') || '/'; // 去末尾斜杠（非根）
+    const keep = [];
+    for (const [k, v] of u.searchParams) if (!TRACKING_PARAMS.test(k)) keep.push([k, v]);
+    keep.sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
+    u.search = '';
+    for (const [k, v] of keep) u.searchParams.append(k, v);
+    return u.href;
+  } catch {
+    return url;
+  }
+}
+
+/** 该 URL 是否在爬取域范围内。 */
+export function shouldFollow(url, { origin, allowDomains, sameOrigin }) {
+  let host;
+  try {
+    const u = new URL(url);
+    if (!/^https?:$/i.test(u.protocol)) return false;
+    host = u.hostname.toLowerCase();
+    if (sameOrigin && origin) return u.origin === origin;
+  } catch {
+    return false;
+  }
+  if (allowDomains && allowDomains.length) {
+    return allowDomains.some((d) => host === d || host.endsWith(`.${d}`));
+  }
+  return true;
+}
+
+/** 解析 robots.txt：取 User-agent:* （和 *=fallback）组的 Disallow + Crawl-delay。 */
+export function parseRobots(txt) {
+  const lines = String(txt || '').split(/\r?\n/);
+  const groups = [];
+  let cur = null;
+  for (let raw of lines) {
+    raw = raw.replace(/#.*$/, '').trim();
+    if (!raw) continue;
+    const m = /^([a-z-]+)\s*:\s*(.*)$/i.exec(raw);
+    if (!m) continue;
+    const field = m[1].toLowerCase();
+    const val = m[2].trim();
+    if (field === 'user-agent') {
+      if (!cur || cur.hasRules) {
+        cur = { agents: [], disallow: [], crawlDelay: 0, hasRules: false };
+        groups.push(cur);
+      }
+      cur.agents.push(val.toLowerCase());
+    } else if (cur && field === 'disallow') {
+      if (val) cur.disallow.push(val);
+      cur.hasRules = true;
+    } else if (cur && field === 'crawl-delay') {
+      const n = Number(val);
+      if (Number.isFinite(n)) cur.crawlDelay = n;
+      cur.hasRules = true;
+    }
+  }
+  const star = groups.find((g) => g.agents.includes('*'));
+  return star
+    ? { disallow: star.disallow, crawlDelay: star.crawlDelay }
+    : { disallow: [], crawlDelay: 0 };
+}
+
+/** robots 是否允许该路径。 */
+export function isAllowedByRobots(url, robots) {
+  if (!robots || !robots.disallow || !robots.disallow.length) return true;
+  let path;
+  try {
+    path = new URL(url).pathname;
+  } catch {
+    return true;
+  }
+  return !robots.disallow.some((d) => d && path.startsWith(d));
+}
+
+/** 静态 HTML 疑似 JS-app（正文太空、靠 JS 渲染）→ 建议升级浏览器渲染。 */
+export function looksJsRendered(html) {
+  const h = String(html || '');
+  const text = strip(h.replace(/<(script|style|noscript)\b[\s\S]*?<\/\1>/gi, ''));
+  const hasAppShell = /__NEXT_DATA__|id=["']root["']|id=["']app["']|window\.__NUXT__/i.test(h);
+  return text.length < 200 && (hasAppShell || /<script/i.test(h));
+}
+
+// ───────────────────────── 引擎 ─────────────────────────
+
+/** 默认静态抓取：node 内置 fetch + 超时 + 浏览器 UA。 */
+async function defaultFetch(url, { timeout = 20000, headers = {} } = {}) {
+  const ctrl = new AbortController();
+  const t = setTimeout(() => ctrl.abort(), timeout);
+  try {
+    const res = await fetch(url, {
+      redirect: 'follow',
+      headers: { 'User-Agent': UA, Accept: 'text/html,*/*', ...headers },
+      signal: ctrl.signal,
+    });
+    const ct = res.headers.get('content-type') || '';
+    const body = /text|html|json|xml|javascript/i.test(ct) ? await res.text() : '';
+    return { ok: res.ok, status: res.status, contentType: ct, body, url: res.url || url };
+  } catch (e) {
+    return { ok: false, status: 0, contentType: '', body: '', url, error: e?.message || String(e) };
+  } finally {
+    clearTimeout(t);
+  }
+}
+
+/** 默认 JS 渲染：spawn agent-browser 取渲染后 outerHTML（eval 输出双重编码，JSON.parse 一次解开）。 */
+function defaultRender(url, { timeout = 30000 } = {}) {
+  return new Promise((resolve) => {
+    const session = 'crawl';
+    const run = (args) =>
+      new Promise((res) => {
+        let out = '';
+        let err = '';
+        let child;
+        try {
+          child = spawn('agent-browser', ['--session', session, ...args], {
+            stdio: ['ignore', 'pipe', 'pipe'],
+            windowsHide: true,
+          });
+        } catch (e) {
+          res({ code: -1, out: '', err: e?.message || String(e) });
+          return;
+        }
+        const kill = setTimeout(() => {
+          try {
+            child.kill();
+          } catch {}
+        }, timeout);
+        child.stdout.on('data', (c) => (out += c));
+        child.stderr.on('data', (c) => (err += c));
+        child.on('error', (e) => {
+          clearTimeout(kill);
+          res({ code: -1, out, err: err || e?.message });
+        });
+        child.on('close', (code) => {
+          clearTimeout(kill);
+          res({ code: code ?? -1, out, err });
+        });
+      });
+
+    (async () => {
+      const open = await run(['open', url]);
+      if (open.code !== 0) {
+        resolve({ ok: false, html: '', error: open.err || 'agent-browser open 失败' });
+        return;
+      }
+      const ev = await run(['eval', 'document.documentElement.outerHTML']);
+      let html = ev.out.trim();
+      // eval 输出双重编码：结果字符串被 agent-browser 再 JSON 编码一次 → JSON.parse 解开。
+      try {
+        const once = JSON.parse(html);
+        if (typeof once === 'string') html = once;
+      } catch {
+        /* 非双编码则原样用 */
+      }
+      resolve({ ok: ev.code === 0 && html.length > 0, html, error: ev.err });
+    })();
+  });
+}
+
+/** 抓 + 抽一个页面（含按需 JS 渲染升级）。返回页面记录（含 markdown）。 */
+async function fetchPage(url, opts, deps) {
+  const fetchImpl = deps.fetchImpl || defaultFetch;
+  const renderImpl = deps.renderImpl || defaultRender;
+
+  let r = await fetchImpl(url, { timeout: opts.timeout });
+  let rendered = false;
+  let html = r.body || '';
+
+  const wantRender =
+    opts.render === 'on' || (opts.render !== 'off' && (!r.ok || looksJsRendered(html)));
+  if (wantRender) {
+    const rr = await renderImpl(url, { timeout: opts.timeout + 10000 });
+    if (rr && rr.ok && rr.html) {
+      html = rr.html;
+      rendered = true;
+    }
+  }
+
+  const og = extractOpenGraph(html);
+  const md = await htmlToMarkdown(html);
+  return {
+    url: r.url || url,
+    ok: r.ok || rendered,
+    status: r.status,
+    title: extractOpenGraph(html)['og:title'] || extractTitle(html),
+    rendered,
+    markdown: md,
+    og,
+    jsonld: extractJsonLd(html),
+    image: extractMainImage(html, og, r.url || url),
+    links: extractLinks(html, r.url || url),
+    error: r.error,
+  };
+}
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+/**
+ * 主引擎。opts: { url, mode, out, depth, maxPages, sameOrigin, allowDomains[], render, concurrency, delay, timeout, respectRobots }
+ * deps（DI 测试）: { fetchImpl, renderImpl, writeFileImpl, mkdirImpl }
+ * 返回 manifest 对象（同时落盘 out/page_NNN.md + out/crawl.json，除非 deps 提供假实现）。
+ */
+export async function crawl(opts, deps = {}) {
+  const o = {
+    mode: 'single',
+    out: './build/crawl',
+    depth: 2,
+    maxPages: 20,
+    sameOrigin: false,
+    allowDomains: [],
+    render: 'auto',
+    concurrency: 3,
+    delay: 500,
+    timeout: 20000,
+    respectRobots: true,
+    ...opts,
+  };
+  const writeFileImpl = deps.writeFileImpl || writeFile;
+  const mkdirImpl = deps.mkdirImpl || mkdir;
+  const fetchImpl = deps.fetchImpl || defaultFetch;
+
+  const origin = (() => {
+    try {
+      return new URL(o.url).origin;
+    } catch {
+      return '';
+    }
+  })();
+
+  // robots（site 模式且 respectRobots）
+  let robots = { disallow: [], crawlDelay: 0 };
+  if (o.mode === 'site' && o.respectRobots && origin) {
+    const rb = await fetchImpl(`${origin}/robots.txt`, { timeout: o.timeout });
+    if (rb.ok && rb.body) robots = parseRobots(rb.body);
+  }
+  const perDomainDelay = Math.max(o.delay, (robots.crawlDelay || 0) * 1000);
+
+  const pages = [];
+  const seen = new Set();
+  const queue = [{ url: o.url, depth: 0 }];
+  seen.add(canonicalizeUrl(o.url));
+
+  await mkdirImpl(o.out, { recursive: true });
+
+  while (queue.length) {
+    const done = pages.filter((p) => p.file).length;
+    if (done >= o.maxPages) break;
+    // 批大小按剩余配额收，避免一个并发批一次超过 maxPages。
+    const room = o.mode === 'single' ? 1 : Math.max(1, Math.min(o.concurrency, o.maxPages - done));
+    const batch = queue.splice(0, room);
+    const results = await Promise.all(
+      batch.map(async (item) => {
+        if (o.mode === 'site' && !isAllowedByRobots(item.url, robots)) {
+          return { skipped: 'robots', url: item.url };
+        }
+        const page = await fetchPage(item.url, o, deps);
+        return { page, depth: item.depth };
+      }),
+    );
+
+    for (const res of results) {
+      if (res.skipped) {
+        pages.push({ url: res.url, skipped: res.skipped });
+        continue;
+      }
+      const { page, depth } = res;
+      const n = pages.filter((p) => p.file).length + 1;
+      const file = join(o.out, `page_${String(n).padStart(3, '0')}.md`);
+      const front =
+        `---\nurl: ${page.url}\ntitle: ${JSON.stringify(page.title)}\nrendered: ${page.rendered}\n` +
+        (page.image ? `image: ${page.image}\n` : '') +
+        `---\n\n`;
+      await writeFileImpl(file, front + (page.markdown || ''), 'utf8');
+      pages.push({
+        url: page.url,
+        title: page.title,
+        file,
+        rendered: page.rendered,
+        status: page.status,
+        links: page.links.length,
+        jsonld: page.jsonld.length,
+        og: page.og,
+        error: page.error,
+      });
+
+      // site：把新链接入队
+      if (o.mode === 'site' && depth < o.depth) {
+        for (const link of page.links) {
+          const canon = canonicalizeUrl(link);
+          if (seen.has(canon)) continue;
+          if (!shouldFollow(link, { origin, allowDomains: o.allowDomains, sameOrigin: o.sameOrigin })) continue;
+          seen.add(canon);
+          queue.push({ url: link, depth: depth + 1 });
+          if (seen.size > o.maxPages * 20) break; // 前沿失控保护
+        }
+      }
+    }
+    if (o.mode === 'site' && queue.length && perDomainDelay) await sleep(perDomainDelay);
+  }
+
+  const manifest = {
+    seed: o.url,
+    mode: o.mode,
+    generated_pages: pages.filter((p) => p.file).length,
+    crawl_delay_ms: perDomainDelay,
+    pages,
+  };
+  await writeFileImpl(join(o.out, 'crawl.json'), JSON.stringify(manifest, null, 2), 'utf8');
+  return manifest;
+}
+
+// ───────────────────────── CLI ─────────────────────────
+
+function parseArgs(argv) {
+  const o = { allowDomains: [] };
+  const pos = [];
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === '--same-origin') o.sameOrigin = true;
+    else if (a === '--json') o.json = true;
+    else if (a === '--mode') o.mode = argv[++i];
+    else if (a === '--out') o.out = argv[++i];
+    else if (a === '--depth') o.depth = Number(argv[++i]);
+    else if (a === '--max-pages') o.maxPages = Number(argv[++i]);
+    else if (a === '--allow-domains') o.allowDomains = String(argv[++i] || '').split(',').map((s) => s.trim()).filter(Boolean);
+    else if (a === '--render') o.render = argv[++i];
+    else if (a === '--concurrency') o.concurrency = Number(argv[++i]);
+    else if (a === '--delay') o.delay = Number(argv[++i]);
+    else if (a === '--timeout') o.timeout = Number(argv[++i]);
+    else if (a === '--no-robots') o.respectRobots = false;
+    else if (!a.startsWith('--')) pos.push(a);
+  }
+  o.url = pos[0];
+  return o;
+}
+
+async function main() {
+  const o = parseArgs(process.argv.slice(2));
+  if (!o.url) {
+    process.stderr.write(
+      'CRAWL_ERROR: 缺 URL。用法 node crawl.mjs <url> [--mode single|site] [--out ./build/crawl] [--depth N] [--max-pages N] [--same-origin|--allow-domains a,b] [--render auto|on|off] [--json]\n',
+    );
+    process.exit(2);
+  }
+  try {
+    const manifest = await crawl(o);
+    if (o.json) process.stdout.write(`${JSON.stringify(manifest)}\n`);
+    process.stderr.write(
+      `CRAWL_DONE: ${manifest.generated_pages} pages → ${join(o.out || './build/crawl', 'crawl.json')}\n`,
+    );
+    process.exit(0);
+  } catch (e) {
+    process.stderr.write(`CRAWL_ERROR: ${e?.message || String(e)}\n`);
+    process.exit(1);
+  }
+}
+
+// main 守卫：node 与 bun 通用（直接跑本文件才执行 main，被 import 时不跑）。
+const isMain = (() => {
+  try {
+    if (typeof import.meta !== 'undefined' && import.meta.main) return true; // bun
+  } catch {}
+  try {
+    return process.argv[1] && import.meta.url === new URL(`file://${process.argv[1].replace(/\\/g, '/')}`).href;
+  } catch {
+    return false;
+  }
+})();
+if (isMain) main();