079project 1.0.0 → 3.0.0

package/crawler/index.cjs

@@ -0,0 +1,515 @@
+// ...existing code...
+const fs = require('fs');
+const path = require('path');
+const urlLib = require('url');
+const http = require('http');
+const https = require('https');
+const { CrawlerStorage } = require('./storage.cjs');
+// 新增：定向检索工具（若不存在也不影响本改造）
+let planQueries, extractSerpLinks, relevanceScore;
+try {
+  ({ planQueries, extractSerpLinks, relevanceScore } = require('./agent.cjs'));
+} catch { /* 忽略 */ }
+
+// 可选代理 Agent（若未安装对应包，将自动降级直连）
+let HttpsProxyAgent = null;
+let SocksProxyAgent = null;
+try { HttpsProxyAgent = require('https-proxy-agent'); } catch {}
+try { SocksProxyAgent = require('socks-proxy-agent'); } catch {}
+
+// 新增：尝试动态渲染（Puppeteer / Puppeteer-Core）
+let Puppeteer = null;
+try { Puppeteer = require('puppeteer'); } catch {
+  try { Puppeteer = require('puppeteer-core'); } catch {}
+}
+
+const DEFAULT_UAS = [
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36',
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0',
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 13_5_2) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15',
+  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36',
+];
+
+function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
+function randomIPv4() { return Array(4).fill(0).map(() => Math.floor(Math.random() * 254) + 1).join('.'); }
+
+class RobotsCache {
+  constructor() {
+    this.cache = new Map(); // host -> { fetchedAt, rules: [{allow|disallow, path}] }
+    this.ttl = 12 * 60 * 60 * 1000; // 12h
+  }
+  async fetch(host, fetchFn) {
+    const hit = this.cache.get(host);
+    const now = Date.now();
+    if (hit && now - hit.fetchedAt < this.ttl) return hit.rules;
+    const url = `https://${host}/robots.txt`;
+    try {
+      const { status, body } = await fetchFn(url, { timeout: 8000 });
+      let rules = [];
+      if (status === 200 && body) {
+        rules = this.parse(body);
+      }
+      this.cache.set(host, { fetchedAt: now, rules });
+      return rules;
+    } catch {
+      this.cache.set(host, { fetchedAt: now, rules: [] });
+      return [];
+    }
+  }
+  parse(content) {
+    const lines = String(content).split(/\r?\n/);
+    let uaStar = false;
+    const rules = [];
+    for (const ln of lines) {
+      const s = ln.trim();
+      if (!s || s.startsWith('#')) continue;
+      const [k, v] = s.split(':').map(x => (x || '').trim());
+      if (/^user-agent$/i.test(k)) {
+        uaStar = v === '*' ? true : false;
+      } else if (uaStar && /^disallow$/i.test(k)) {
+        if (v) rules.push({ type: 'disallow', path: v });
+      } else if (uaStar && /^allow$/i.test(k)) {
+        if (v) rules.push({ type: 'allow', path: v });
+      }
+    }
+    return rules;
+  }
+  allowed(host, pathName, rules) {
+    if (!rules || rules.length === 0) return true;
+    let allowLen = -1, disallowLen = -1;
+    for (const r of rules) {
+      if (!r.path) continue;
+      if (pathName.startsWith(r.path)) {
+        if (r.type === 'allow') allowLen = Math.max(allowLen, r.path.length);
+        else if (r.type === 'disallow') disallowLen = Math.max(disallowLen, r.path.length);
+      }
+    }
+    if (disallowLen > allowLen) return false;
+    return true;
+  }
+}
+
+class CrawlerManager {
+  constructor(options = {}) {
+    this.opts = Object.assign({
+      concurrency: 4,
+      perHostDelayMs: 1500,
+      requestTimeoutMs: 12000,
+      userAgents: DEFAULT_UAS,
+      proxiesFile: path.join(__dirname, 'proxies.txt'),
+      seedsFile: path.join(__dirname, '..', 'robots', 'seeds.txt'),
+      maxDocPerTick: 8,
+      allowLang: ['en', 'zh'],
+      minTextLen: 200,
+      // 新增：动态渲染与递归抓取
+      renderDynamic: 'auto',          // 'never' | 'auto' | 'always'
+      renderTimeoutMs: 15000,
+      renderConcurrency: 1,           // 同时渲染页面数
+      dynamicHosts: [],               // 总是渲染的 host 列表
+      dynamicPathRegex: null,         // 匹配则渲染
+      maxDepth: 2,                    // crawlRecursive 默认深度
+      sameDomainOnly: true,           // 限制域内递归
+      maxPagesPerTask: 100            // crawlRecursive 默认最大页数
+    }, options);
+    this.running = false;
+    this.storage = new CrawlerStorage();
+    this.robots = new RobotsCache();
+    this.perHostTime = new Map();
+    this.queue = new Set();
+    this.uaIdx = 0;
+    this.proxies = this._loadProxies();
+    // 渲染器状态
+    this._puppeteer = Puppeteer || null;
+    this._browser = null;
+    this._renderInFlight = 0;
+  }
+
+  // ---------- 定向检索（如果 agent.cjs 可用） ----------
+  async directedSearch(prompt, {
+    vertical = 'general',
+    perQuery = 8,
+    maxEnqueue = 30,
+    crawl = true,
+    maxCrawl = 12
+  } = {}) {
+    if (!planQueries || !extractSerpLinks || !relevanceScore) {
+      return { prompt, vertical, queries: [], candidates: [], enqueued: 0, crawled: 0, saved: 0, note: 'agent.cjs 不可用' };
+    }
+    const queries = planQueries(prompt, vertical);
+    const serpHost = 'https://html.duckduckgo.com/html/';
+    const candidates = [];
+
+    for (const q of queries) {
+      const url = `${serpHost}?q=${encodeURIComponent(q)}`;
+      const { status, body } = await this.fetchUrl(url, { timeout: 12000 });
+      if (status !== 200 || !body) continue;
+
+      const links = extractSerpLinks(body, serpHost).slice(0, perQuery).map(x => ({ ...x, query: q }));
+      for (const it of links) {
+        const score = relevanceScore({ url: it.url, title: it.title }, prompt, vertical);
+        candidates.push({ ...it, score });
+      }
+      await sleep(300 + Math.floor(Math.random() * 400));
+    }
+
+    candidates.sort((a, b) => b.score - a.score);
+    const picked = candidates.slice(0, maxEnqueue);
+    const added = this.storage.enqueueLinks(picked.map(p => p.url), maxEnqueue);
+
+    let crawled = 0, savedDocs = 0;
+    if (crawl) {
+      for (const p of picked.slice(0, maxCrawl)) {
+        const r = await this._crawlOne(p.url, { parent: null, depth: 0 });
+        if (r && r.ok) { crawled++; savedDocs += r.saved ? 1 : 0; }
+      }
+    }
+
+    return {
+      prompt, vertical, queries,
+      candidates: candidates.slice(0, 50).map(({ url, title, score }) => ({ url, title, score: Number(score.toFixed(3)) })),
+      enqueued: added, crawled, saved: savedDocs
+    };
+  }
+
+  // ---------- 代理/UA ----------
+  _loadProxies() {
+    try {
+      if (!fs.existsSync(this.opts.proxiesFile)) return [];
+      const lines = fs.readFileSync(this.opts.proxiesFile, 'utf-8').split(/\r?\n/).map(s => s.trim()).filter(Boolean);
+      return lines;
+    } catch { return []; }
+  }
+  _pickUA() {
+    const ua = this.opts.userAgents[this.uaIdx % this.opts.userAgents.length];
+    this.uaIdx++;
+    return ua;
+  }
+  _pickProxy() {
+    if (!this.proxies.length) return null;
+    const i = Math.floor(Math.random() * this.proxies.length);
+    return this.proxies[i];
+  }
+  _agentForUrl(targetUrl, proxyStr) {
+    if (!proxyStr) return null;
+    try {
+      if (/^socks/i.test(proxyStr) && SocksProxyAgent) return new SocksProxyAgent.SocksProxyAgent(proxyStr);
+      if (/^http/i.test(proxyStr) && HttpsProxyAgent) return new HttpsProxyAgent.HttpsProxyAgent(proxyStr);
+    } catch {}
+    return null;
+  }
+
+  // ---------- 静态抓取 ----------
+  async fetchUrl(targetUrl, { timeout = 8000 } = {}) {
+    return new Promise((resolve) => {
+      try {
+        const u = new urlLib.URL(targetUrl);
+        const proxy = this._pickProxy();
+        const agent = this._agentForUrl(targetUrl, proxy);
+        const client = u.protocol === 'https:' ? https : http;
+        const headers = {
+          'User-Agent': this._pickUA(),
+          'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+          'Accept-Language': Math.random() < 0.5 ? 'en-US,en;q=0.8' : 'zh-CN,zh;q=0.8,en;q=0.6',
+          'Connection': 'keep-alive',
+          'X-Forwarded-For': randomIPv4(),
+          'X-Real-IP': randomIPv4()
+        };
+        const opts = {
+          protocol: u.protocol,
+          hostname: u.hostname,
+          port: u.port || (u.protocol === 'https:' ? 443 : 80),
+          path: u.pathname + (u.search || ''),
+          method: 'GET',
+          headers,
+          timeout,
+          agent: agent || undefined,
+        };
+        const req = client.request(opts, (res) => {
+          const { statusCode } = res;
+          const chunks = [];
+          const limit = 3 * 1024 * 1024;
+          res.on('data', d => {
+            chunks.push(d);
+            if (chunks.reduce((a, b) => a + b.length, 0) > limit) req.destroy();
+          });
+          res.on('end', () => {
+            const buf = Buffer.concat(chunks);
+            resolve({ status: statusCode, headers: res.headers, body: buf.toString('utf-8'), finalUrl: targetUrl });
+          });
+        });
+        req.on('timeout', () => { req.destroy(); resolve({ status: 0, error: 'timeout' }); });
+        req.on('error', () => resolve({ status: 0, error: 'network' }));
+        req.end();
+      } catch {
+        resolve({ status: 0, error: 'badurl' });
+      }
+    });
+  }
+
+  // ---------- 动态渲染 ----------
+  async _ensureRenderer() {
+    if (!this._puppeteer) return false;
+    if (this._browser) return true;
+    try {
+      this._browser = await this._puppeteer.launch({
+        headless: 'new',
+        args: [
+          '--no-sandbox',
+          '--disable-setuid-sandbox',
+          '--disable-dev-shm-usage'
+        ]
+      });
+      return true;
+    } catch (e) {
+      console.warn('[CRAWLER][RENDER] 启动失败:', e.message);
+      this._browser = null;
+      return false;
+    }
+  }
+
+  async _renderHtml(url, { timeout = 15000 } = {}) {
+    if (!await this._ensureRenderer()) return { ok: false };
+    // 控制并发
+    while (this._renderInFlight >= Math.max(1, this.opts.renderConcurrency)) {
+      await sleep(100);
+    }
+    this._renderInFlight++;
+    try {
+      const page = await this._browser.newPage();
+      await page.setUserAgent(this._pickUA());
+      await page.setExtraHTTPHeaders({
+        'Accept-Language': Math.random() < 0.5 ? 'en-US,en;q=0.8' : 'zh-CN,zh;q=0.8,en;q=0.6',
+        'X-Forwarded-For': randomIPv4(),
+        'X-Real-IP': randomIPv4()
+      });
+      await page.setRequestInterception(true);
+      page.on('request', (req) => {
+        const rtype = req.resourceType();
+        // 阻断大资源
+        if (['image', 'media', 'font'].includes(rtype)) return req.abort();
+        req.continue();
+      });
+      await page.goto(url, { waitUntil: 'networkidle2', timeout });
+      // 滚动以触发懒加载
+      await page.evaluate(async () => {
+        await new Promise(resolve => {
+          let t = 0;
+          const id = setInterval(() => {
+            window.scrollBy(0, 800);
+            if ((t += 200) >= 2000) { clearInterval(id); resolve(); }
+          }, 200);
+        });
+      });
+      const content = await page.content();
+      // 提取链接（渲染后）
+      const links = await page.$$eval('a[href]', as => as.map(a => a.href).filter(Boolean));
+      const finalUrl = page.url();
+      await page.close();
+      return { ok: true, html: content, links, finalUrl };
+    } catch (e) {
+      return { ok: false, error: e.message };
+    } finally {
+      this._renderInFlight--;
+    }
+  }
+
+  // ---------- 解析 ----------
+  extractContent(html, baseUrl) {
+    let text = String(html || '');
+    text = text.replace(/<script[\s\S]*?<\/script>/gi, ' ')
+               .replace(/<style[\s\S]*?<\/style>/gi, ' ')
+               .replace(/<!--[\s\S]*?-->/g, ' ');
+    const mTitle = text.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+    const title = mTitle ? mTitle[1].replace(/\s+/g, ' ').trim() : '';
+    let body = text.replace(/<\/(p|div|h\d|br|li)>/gi, '\n')
+                   .replace(/<[^>]+>/g, ' ')
+                   .replace(/&nbsp;/g, ' ')
+                   .replace(/&amp;/g, '&')
+                   .replace(/\s+\n/g, '\n')
+                   .replace(/\n{3,}/g, '\n\n')
+                   .replace(/[ \t]{2,}/g, ' ')
+                   .trim();
+    const links = [];
+    const aTagRe = /<a\s+[^>]*href=["']([^"']+)["'][^>]*>/gi;
+    let m;
+    let base;
+    try { base = new urlLib.URL(baseUrl); } catch { base = null; }
+    while ((m = aTagRe.exec(html)) !== null) {
+      let href = (m[1] || '').trim();
+      if (!href) continue;
+      try {
+        const abs = base ? new urlLib.URL(href, base).toString() : href;
+        links.push(abs);
+      } catch {}
+    }
+    const nonAscii = (body.match(/[^\x00-\x7F]/g) || []).length;
+    const ratio = nonAscii / Math.max(1, body.length);
+    const lang = ratio > 0.3 ? 'zh' : 'en';
+    return { title, text: body, links, lang };
+  }
+
+  _hostDelayOk(host) {
+    const last = this.perHostTime.get(host) || 0;
+    const now = Date.now();
+    if (now - last >= this.opts.perHostDelayMs) {
+      this.perHostTime.set(host, now);
+      return true;
+    }
+    return false;
+  }
+
+  // ---------- 单页抓取（含动态渲染回退） ----------
+  async _crawlOne(url, ctx = { parent: null, depth: 0 }) {
+    try {
+      const u = new urlLib.URL(url);
+      const rules = await this.robots.fetch(u.host, (u2, o) => this.fetchUrl(u2, o));
+      const ok = this.robots.allowed(u.host, u.pathname, rules);
+      if (!ok) return { ok: false, reason: 'robots' };
+
+      while (!this._hostDelayOk(u.host)) await sleep(200);
+
+      // 静态抓
+      let { status, body, finalUrl } = await this.fetchUrl(url, { timeout: this.opts.requestTimeoutMs });
+      if (status !== 200) body = '';
+
+      // 解析静态
+      let parsed = this.extractContent(body || '', finalUrl || url);
+      let useRender = false;
+
+      // 动态渲染触发条件
+      const alwaysRenderHost = this.opts.dynamicHosts.some(h => u.hostname.endsWith(h));
+      const matchPath = this.opts.dynamicPathRegex && this.opts.dynamicPathRegex.test(u.pathname);
+      const heuristicHeavy = (body.match(/<script/gi) || []).length >= 10 || (parsed.text || '').length < this.opts.minTextLen;
+
+      if (this.opts.renderDynamic === 'always' || alwaysRenderHost || matchPath ||
+         (this.opts.renderDynamic === 'auto' && heuristicHeavy)) {
+        useRender = true;
+      }
+
+      let renderedLinks = [];
+      if (useRender) {
+        const r = await this._renderHtml(url, { timeout: this.opts.renderTimeoutMs });
+        if (r.ok && r.html) {
+          parsed = this.extractContent(r.html, r.finalUrl || url);
+          renderedLinks = Array.isArray(r.links) ? r.links : [];
+          finalUrl = r.finalUrl || finalUrl || url;
+        }
+      }
+
+      if (!this.opts.allowLang.includes(parsed.lang) || !parsed.text || parsed.text.length < this.opts.minTextLen) {
+        this.storage.state.stats.deduped++;
+        return { ok: false, reason: 'short-or-lang', lang: parsed.lang };
+      }
+
+      const saved = this.storage.saveDocument({
+        url: finalUrl || url,
+        title: parsed.title,
+        text: parsed.text,
+        lang: parsed.lang,
+        depth: ctx.depth || 0,
+        parent: ctx.parent || ''
+      });
+
+      // 扩展链接（合并渲染后的链接）
+      const links = Array.from(new Set([...(parsed.links || []), ...renderedLinks])).slice(0, 800);
+      const added = this.storage.enqueueLinks(links, 200);
+
+      this.storage.markVisited(url);
+      this.storage.state.stats.fetched++;
+      return { ok: true, saved, added, lang: parsed.lang, links };
+    } catch (e) {
+      this.storage.state.stats.errors++;
+      return { ok: false, reason: e.message };
+    }
+  }
+
+  // ---------- 递归抓取（多层链接） ----------
+  async crawlRecursive(seedUrl, {
+    maxDepth = this.opts.maxDepth,
+    sameDomainOnly = this.opts.sameDomainOnly,
+    maxPages = this.opts.maxPagesPerTask
+  } = {}) {
+    let seed;
+    try { seed = new urlLib.URL(seedUrl); } catch { return { ok: false, error: 'bad seed url' }; }
+
+    const visitedLocal = new Set();
+    const q = [{ url: seedUrl, depth: 0, parent: '' }];
+    let crawled = 0, savedDocs = 0;
+
+    while (q.length > 0) {
+      if (crawled >= maxPages) break;
+      const { url, depth, parent } = q.shift();
+      if (visitedLocal.has(url)) continue;
+      visitedLocal.add(url);
+
+      // 域限制
+      if (sameDomainOnly) {
+        try {
+          const u = new urlLib.URL(url);
+          if (u.hostname !== seed.hostname) continue;
+        } catch { continue; }
+      }
+
+      const res = await this._crawlOne(url, { parent, depth });
+      if (res && res.ok) {
+        crawled++;
+        if (res.saved) savedDocs++;
+        if (depth + 1 <= maxDepth && Array.isArray(res.links)) {
+          for (const l of res.links) {
+            if (!l || visitedLocal.has(l)) continue;
+            q.push({ url: l, depth: depth + 1, parent: url });
+          }
+        }
+      }
+
+      // 轻微退避，避免单站被打爆
+      await sleep(40 + Math.floor(Math.random() * 120));
+    }
+
+    return { ok: true, seed: seedUrl, crawled, saved: savedDocs, maxDepth, sameDomainOnly, maxPages };
+  }
+
+  async _workerLoop(slotId) {
+    while (this.running) {
+      const url = this.storage.nextUrl();
+      if (!url) { await sleep(1000); continue; }
+      if (this.queue.has(url)) continue;
+      this.queue.add(url);
+      await this._crawlOne(url, { parent: null, depth: 0 });
+      this.queue.delete(url);
+      if (!this.running) break;
+      await sleep(50 + Math.floor(Math.random() * 150));
+    }
+  }
+
+  start() {
+    if (this.running) return;
+    if (fs.existsSync(this.opts.seedsFile)) {
+      const seeds = fs.readFileSync(this.opts.seedsFile, 'utf-8').split(/\r?\n/).map(s => s.trim()).filter(x => /^https?:\/\//i.test(x));
+      if (seeds.length) this.storage.addSeed(seeds);
+    }
+    this.running = true;
+    this.workers = Array.from({ length: this.opts.concurrency }, (_, i) => this._workerLoop(i));
+    console.log(`[CRAWLER] started, concurrency=${this.opts.concurrency}, proxies=${this.proxies.length}, render=${this.opts.renderDynamic}${this._puppeteer ? '' : ' (no-puppeteer)'}`);
+  }
+
+  stop() {
+    this.running = false;
+    console.log('[CRAWLER] stopping...');
+    // 关闭渲染器
+    setTimeout(async () => {
+      try { if (this._browser) await this._browser.close(); } catch {}
+      this._browser = null;
+    }, 0);
+  }
+
+  addSeeds(urls = []) { return this.storage.addSeed(urls); }
+  stats() {
+    const s = this.storage.stats();
+    return Object.assign(s, { running: this.running, inflight: this.queue.size, renderInFlight: this._renderInFlight });
+  }
+  loadRecentDocs(maxFiles = 20) { return this.storage.loadRecentDocs(maxFiles); }
+}
+
+module.exports = { CrawlerManager };
+// ...existing code...

package/crawler/storage.cjs

@@ -0,0 +1,163 @@
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const { planQueries, extractSerpLinks, relevanceScore } = require('./agent.cjs');
+// ...existing code...
+
+function randomIPv4() {
+  return Array(4).fill(0).map(() => Math.floor(Math.random() * 254) + 1).join('.');
+}
+class CrawlerStorage {
+  constructor(baseDir = path.join(__dirname, '..', 'crawler_data')) {
+    this.baseDir = baseDir;
+    this.stateFile = path.join(this.baseDir, 'state.json');
+    this.docsDir = path.join(this.baseDir, 'docs');
+    this.ensureDirs();
+    this.state = {
+      frontier: [],            // 待抓取队列（url）
+      visited: {},             // urlHash -> timestamp
+      enqueued: {},            // urlHash -> 1
+      stats: { fetched: 0, saved: 0, deduped: 0, errors: 0, lastSave: 0 }
+    };
+    this._loadState();
+  }
+
+  ensureDirs() {
+    fs.mkdirSync(this.baseDir, { recursive: true });
+    fs.mkdirSync(this.docsDir, { recursive: true });
+  }
+
+  _hash(str) {
+    return crypto.createHash('sha1').update(String(str)).digest('hex');
+  }
+
+  _loadState() {
+    try {
+      if (fs.existsSync(this.stateFile)) {
+        const obj = JSON.parse(fs.readFileSync(this.stateFile, 'utf-8'));
+        if (obj && typeof obj === 'object') this.state = obj;
+      }
+    } catch (e) {
+      console.warn('[CRAWLER][STATE] load failed:', e.message);
+    }
+  }
+
+  _saveState() {
+    try {
+      fs.writeFileSync(this.stateFile, JSON.stringify(this.state, null, 2), 'utf-8');
+      this.state.stats.lastSave = Date.now();
+    } catch (e) {
+      console.warn('[CRAWLER][STATE] save failed:', e.message);
+    }
+  }
+
+  addSeed(urls = []) {
+    let n = 0;
+    for (const u of urls) {
+      const url = String(u || '').trim();
+      if (!url || !/^https?:\/\//i.test(url)) continue;
+      const h = this._hash(url);
+      if (this.state.enqueued[h] || this.state.visited[h]) continue;
+      this.state.frontier.push(url);
+      this.state.enqueued[h] = 1;
+      n++;
+    }
+    if (n > 0) this._saveState();
+    return n;
+  }
+
+  nextUrl() {
+    while (this.state.frontier.length > 0) {
+      const url = this.state.frontier.shift();
+      if (!url) continue;
+      const h = this._hash(url);
+      if (this.state.visited[h]) continue;
+      return url;
+    }
+    return null;
+  }
+
+  markVisited(url) {
+    const h = this._hash(url);
+    this.state.visited[h] = Date.now();
+    this._saveState();
+  }
+
+  // 新链接入队（去重）
+  enqueueLinks(links = [], limitPerBatch = 200) {
+    let n = 0;
+    for (const l of links) {
+      if (!l || !/^https?:\/\//i.test(l)) continue;
+      const h = this._hash(l);
+      if (this.state.enqueued[h] || this.state.visited[h]) continue;
+      this.state.frontier.push(l);
+      this.state.enqueued[h] = 1;
+      n++;
+      if (n >= limitPerBatch) break;
+    }
+    if (n > 0) this._saveState();
+    return n;
+  }
+
+  // 保存清洗后的正文为 txt，返回保存路径（增强：记录 parent/depth）
+  saveDocument(doc) {
+    try {
+      const day = new Date();
+      const dir = path.join(this.docsDir,
+        `${day.getFullYear()}-${String(day.getMonth() + 1).padStart(2, '0')}-${String(day.getDate()).padStart(2, '0')}`
+      );
+      fs.mkdirSync(dir, { recursive: true });
+      const name = `${Date.now()}_${Math.floor(Math.random() * 1e6)}.txt`;
+      const file = path.join(dir, name);
+      const meta = [
+        `URL: ${doc.url || ''}`,
+        `Title: ${doc.title || ''}`,
+        `FetchedAt: ${new Date().toISOString()}`,
+        `Lang: ${doc.lang || ''}`,
+        `Depth: ${doc.depth ?? 0}`,
+        `Parent: ${doc.parent || ''}`
+      ].join('\n');
+      const body = (doc.text || '').trim();
+      if (!body) return null;
+      fs.writeFileSync(file, meta + '\n\n' + body, 'utf-8');
+      this.state.stats.saved++;
+      this._saveState();
+      return file;
+    } catch (e) {
+      this.state.stats.errors++;
+      this._saveState();
+      return null;
+    }
+  }
+
+  stats() {
+    return Object.assign({}, this.state.stats, {
+      frontier: this.state.frontier.length,
+      visited: Object.keys(this.state.visited).length
+    });
+  }
+
+  // 消费已保存文档，按批次读入返回文本数组
+  // 返回：[{ path, text }]
+  loadRecentDocs(maxFiles = 20) {
+    const d = this.docsDir;
+    if (!fs.existsSync(d)) return [];
+    const days = fs.readdirSync(d).filter(f => /^\d{4}-\d{2}-\d{2}$/.test(f)).sort().reverse();
+    const out = [];
+    for (const day of days) {
+      const dir = path.join(d, day);
+      const files = fs.readdirSync(dir).filter(f => f.endsWith('.txt')).sort().reverse();
+      for (const f of files) {
+        try {
+          const p = path.join(dir, f);
+          const text = fs.readFileSync(p, 'utf-8');
+          out.push({ path: p, text });
+          if (out.length >= maxFiles) return out;
+        } catch {}
+      }
+    }
+    return out;
+  }
+}
+
+module.exports = { CrawlerStorage };

package/groupmanager.cjs

@@ -51,7 +51,8 @@ const REQUIRED_DIRS = [
   'robots',
   'tests',
   'public',
-  'snapshots'
+  'snapshots',
+  'crawler'
 ];
 
 // 检查与准备目标 groups 文件夹