079project 2.0.0 → 3.0.0

package/crawler/agent.cjs

@@ -0,0 +1,97 @@
+const urlLib = require('url');
+
+// 基础 tokenizer（英文/中文混合，清停用词可在调用层做）
+function tokenize(s) {
+  return String(s || '')
+    .toLowerCase()
+    .replace(/[^a-z0-9\u4e00-\u9fa5\s]/g, ' ')
+    .split(/\s+/)
+    .filter(Boolean);
+}
+
+// 简单查询规划：原始提示 + 派生关键词 + 站点/纵向限定
+function planQueries(prompt, vertical = 'general') {
+  const base = String(prompt || '').trim();
+  const tk = tokenize(base);
+  const heads = tk.slice(0, 6);
+  const queries = new Set();
+
+  if (heads.length) queries.add(heads.join(' '));
+  if (tk.length > 6) queries.add(tk.slice(0, 10).join(' '));
+  if (tk.length >= 2) queries.add(`${tk[0]} ${tk[1]} analysis`);
+  if (tk.length >= 2) queries.add(`${tk[0]} vs ${tk[1]} review`);
+  queries.add(`${heads.join(' ')} site:wikipedia.org`);
+  queries.add(`${heads.join(' ')} filetype:pdf`);
+
+  // 简单纵向词汇
+  const vMap = {
+    science: ['site:nature.com', 'site:sciencedirect.com', 'site:arxiv.org'],
+    news: ['site:reuters.com', 'site:bbc.com', 'site:apnews.com'],
+    tech: ['site:stackoverflow.com', 'site:github.com', 'site:arstechnica.com'],
+    finance: ['site:bloomberg.com', 'site:ft.com', 'site:wsj.com'],
+    general: []
+  };
+  (vMap[vertical] || []).forEach(s => queries.add(`${heads.join(' ')} ${s}`));
+
+  return Array.from(queries).filter(q => q.length > 0).slice(0, 10);
+}
+
+// 从 SERP HTML 提取候选链接（DuckDuckGo/简单通用版）
+function extractSerpLinks(html, base = 'https://duckduckgo.com') {
+  const aTagRe = /<a\s+[^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi;
+  const results = [];
+  let m;
+  while ((m = aTagRe.exec(html)) !== null) {
+    const href = (m[1] || '').trim();
+    const text = String(m[2] || '').replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
+    if (!href) continue;
+    try {
+      const u = new urlLib.URL(href, base);
+      if (!/^https?:$/i.test(u.protocol)) continue;
+      // 过滤自身域/跳转器
+      if (/duckduckgo\.com|bing\.com|google\./i.test(u.hostname)) continue;
+      results.push({ url: u.toString(), title: text });
+    } catch {}
+  }
+  return results;
+}
+
+// 相关性与方向性评分
+function relevanceScore({ url, title = '', snippet = '' }, prompt, vertical = 'general') {
+  const pt = tokenize(prompt);
+  const tt = tokenize(title + ' ' + snippet);
+  if (!pt.length || !tt.length) return 0;
+
+  const pSet = new Set(pt);
+  const tSet = new Set(tt);
+  let inter = 0;
+  for (const w of pSet) if (tSet.has(w)) inter++;
+
+  // Jaccard + 竖向/域名加权
+  const j = inter / Math.max(1, new Set([...pSet, ...tSet]).size);
+
+  let vBoost = 0;
+  const host = safeHost(url);
+  const verticalDomains = {
+    science: ['nature.com', 'sciencedirect.com', 'arxiv.org', 'acm.org'],
+    news: ['reuters.com', 'bbc.com', 'apnews.com', 'nytimes.com', 'theguardian.com'],
+    tech: ['stackoverflow.com', 'github.com', 'arstechnica.com', 'wired.com'],
+    finance: ['bloomberg.com', 'ft.com', 'wsj.com', 'reuters.com'],
+    general: []
+  };
+  if ((verticalDomains[vertical] || []).some(d => host.endsWith(d))) vBoost += 0.08;
+  if (/\.pdf$/i.test(url)) vBoost += 0.05;
+  if (/wikipedia\.org$/i.test(host)) vBoost += 0.03;
+
+  return j + vBoost;
+}
+
+function safeHost(u) {
+  try { return new urlLib.URL(u).hostname || ''; } catch { return ''; }
+}
+
+module.exports = {
+  planQueries,
+  extractSerpLinks,
+  relevanceScore
+};

package/crawler/index.cjs

@@ -0,0 +1,515 @@
+// ...existing code...
+const fs = require('fs');
+const path = require('path');
+const urlLib = require('url');
+const http = require('http');
+const https = require('https');
+const { CrawlerStorage } = require('./storage.cjs');
+// 新增：定向检索工具（若不存在也不影响本改造）
+let planQueries, extractSerpLinks, relevanceScore;
+try {
+  ({ planQueries, extractSerpLinks, relevanceScore } = require('./agent.cjs'));
+} catch { /* 忽略 */ }
+
+// 可选代理 Agent（若未安装对应包，将自动降级直连）
+let HttpsProxyAgent = null;
+let SocksProxyAgent = null;
+try { HttpsProxyAgent = require('https-proxy-agent'); } catch {}
+try { SocksProxyAgent = require('socks-proxy-agent'); } catch {}
+
+// 新增：尝试动态渲染（Puppeteer / Puppeteer-Core）
+let Puppeteer = null;
+try { Puppeteer = require('puppeteer'); } catch {
+  try { Puppeteer = require('puppeteer-core'); } catch {}
+}
+
+const DEFAULT_UAS = [
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36',
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0',
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 13_5_2) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15',
+  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36',
+];
+
+function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
+function randomIPv4() { return Array(4).fill(0).map(() => Math.floor(Math.random() * 254) + 1).join('.'); }
+
+class RobotsCache {
+  constructor() {
+    this.cache = new Map(); // host -> { fetchedAt, rules: [{allow|disallow, path}] }
+    this.ttl = 12 * 60 * 60 * 1000; // 12h
+  }
+  async fetch(host, fetchFn) {
+    const hit = this.cache.get(host);
+    const now = Date.now();
+    if (hit && now - hit.fetchedAt < this.ttl) return hit.rules;
+    const url = `https://${host}/robots.txt`;
+    try {
+      const { status, body } = await fetchFn(url, { timeout: 8000 });
+      let rules = [];
+      if (status === 200 && body) {
+        rules = this.parse(body);
+      }
+      this.cache.set(host, { fetchedAt: now, rules });
+      return rules;
+    } catch {
+      this.cache.set(host, { fetchedAt: now, rules: [] });
+      return [];
+    }
+  }
+  parse(content) {
+    const lines = String(content).split(/\r?\n/);
+    let uaStar = false;
+    const rules = [];
+    for (const ln of lines) {
+      const s = ln.trim();
+      if (!s || s.startsWith('#')) continue;
+      const [k, v] = s.split(':').map(x => (x || '').trim());
+      if (/^user-agent$/i.test(k)) {
+        uaStar = v === '*' ? true : false;
+      } else if (uaStar && /^disallow$/i.test(k)) {
+        if (v) rules.push({ type: 'disallow', path: v });
+      } else if (uaStar && /^allow$/i.test(k)) {
+        if (v) rules.push({ type: 'allow', path: v });
+      }
+    }
+    return rules;
+  }
+  allowed(host, pathName, rules) {
+    if (!rules || rules.length === 0) return true;
+    let allowLen = -1, disallowLen = -1;
+    for (const r of rules) {
+      if (!r.path) continue;
+      if (pathName.startsWith(r.path)) {
+        if (r.type === 'allow') allowLen = Math.max(allowLen, r.path.length);
+        else if (r.type === 'disallow') disallowLen = Math.max(disallowLen, r.path.length);
+      }
+    }
+    if (disallowLen > allowLen) return false;
+    return true;
+  }
+}
+
+class CrawlerManager {
+  constructor(options = {}) {
+    this.opts = Object.assign({
+      concurrency: 4,
+      perHostDelayMs: 1500,
+      requestTimeoutMs: 12000,
+      userAgents: DEFAULT_UAS,
+      proxiesFile: path.join(__dirname, 'proxies.txt'),
+      seedsFile: path.join(__dirname, '..', 'robots', 'seeds.txt'),
+      maxDocPerTick: 8,
+      allowLang: ['en', 'zh'],
+      minTextLen: 200,
+      // 新增：动态渲染与递归抓取
+      renderDynamic: 'auto',          // 'never' | 'auto' | 'always'
+      renderTimeoutMs: 15000,
+      renderConcurrency: 1,           // 同时渲染页面数
+      dynamicHosts: [],               // 总是渲染的 host 列表
+      dynamicPathRegex: null,         // 匹配则渲染
+      maxDepth: 2,                    // crawlRecursive 默认深度
+      sameDomainOnly: true,           // 限制域内递归
+      maxPagesPerTask: 100            // crawlRecursive 默认最大页数
+    }, options);
+    this.running = false;
+    this.storage = new CrawlerStorage();
+    this.robots = new RobotsCache();
+    this.perHostTime = new Map();
+    this.queue = new Set();
+    this.uaIdx = 0;
+    this.proxies = this._loadProxies();
+    // 渲染器状态
+    this._puppeteer = Puppeteer || null;
+    this._browser = null;
+    this._renderInFlight = 0;
+  }
+
+  // ---------- 定向检索（如果 agent.cjs 可用） ----------
+  async directedSearch(prompt, {
+    vertical = 'general',
+    perQuery = 8,
+    maxEnqueue = 30,
+    crawl = true,
+    maxCrawl = 12
+  } = {}) {
+    if (!planQueries || !extractSerpLinks || !relevanceScore) {
+      return { prompt, vertical, queries: [], candidates: [], enqueued: 0, crawled: 0, saved: 0, note: 'agent.cjs 不可用' };
+    }
+    const queries = planQueries(prompt, vertical);
+    const serpHost = 'https://html.duckduckgo.com/html/';
+    const candidates = [];
+
+    for (const q of queries) {
+      const url = `${serpHost}?q=${encodeURIComponent(q)}`;
+      const { status, body } = await this.fetchUrl(url, { timeout: 12000 });
+      if (status !== 200 || !body) continue;
+
+      const links = extractSerpLinks(body, serpHost).slice(0, perQuery).map(x => ({ ...x, query: q }));
+      for (const it of links) {
+        const score = relevanceScore({ url: it.url, title: it.title }, prompt, vertical);
+        candidates.push({ ...it, score });
+      }
+      await sleep(300 + Math.floor(Math.random() * 400));
+    }
+
+    candidates.sort((a, b) => b.score - a.score);
+    const picked = candidates.slice(0, maxEnqueue);
+    const added = this.storage.enqueueLinks(picked.map(p => p.url), maxEnqueue);
+
+    let crawled = 0, savedDocs = 0;
+    if (crawl) {
+      for (const p of picked.slice(0, maxCrawl)) {
+        const r = await this._crawlOne(p.url, { parent: null, depth: 0 });
+        if (r && r.ok) { crawled++; savedDocs += r.saved ? 1 : 0; }
+      }
+    }
+
+    return {
+      prompt, vertical, queries,
+      candidates: candidates.slice(0, 50).map(({ url, title, score }) => ({ url, title, score: Number(score.toFixed(3)) })),
+      enqueued: added, crawled, saved: savedDocs
+    };
+  }
+
+  // ---------- 代理/UA ----------
+  _loadProxies() {
+    try {
+      if (!fs.existsSync(this.opts.proxiesFile)) return [];
+      const lines = fs.readFileSync(this.opts.proxiesFile, 'utf-8').split(/\r?\n/).map(s => s.trim()).filter(Boolean);
+      return lines;
+    } catch { return []; }
+  }
+  _pickUA() {
+    const ua = this.opts.userAgents[this.uaIdx % this.opts.userAgents.length];
+    this.uaIdx++;
+    return ua;
+  }
+  _pickProxy() {
+    if (!this.proxies.length) return null;
+    const i = Math.floor(Math.random() * this.proxies.length);
+    return this.proxies[i];
+  }
+  _agentForUrl(targetUrl, proxyStr) {
+    if (!proxyStr) return null;
+    try {
+      if (/^socks/i.test(proxyStr) && SocksProxyAgent) return new SocksProxyAgent.SocksProxyAgent(proxyStr);
+      if (/^http/i.test(proxyStr) && HttpsProxyAgent) return new HttpsProxyAgent.HttpsProxyAgent(proxyStr);
+    } catch {}
+    return null;
+  }
+
+  // ---------- 静态抓取 ----------
+  async fetchUrl(targetUrl, { timeout = 8000 } = {}) {
+    return new Promise((resolve) => {
+      try {
+        const u = new urlLib.URL(targetUrl);
+        const proxy = this._pickProxy();
+        const agent = this._agentForUrl(targetUrl, proxy);
+        const client = u.protocol === 'https:' ? https : http;
+        const headers = {
+          'User-Agent': this._pickUA(),
+          'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+          'Accept-Language': Math.random() < 0.5 ? 'en-US,en;q=0.8' : 'zh-CN,zh;q=0.8,en;q=0.6',
+          'Connection': 'keep-alive',
+          'X-Forwarded-For': randomIPv4(),
+          'X-Real-IP': randomIPv4()
+        };
+        const opts = {
+          protocol: u.protocol,
+          hostname: u.hostname,
+          port: u.port || (u.protocol === 'https:' ? 443 : 80),
+          path: u.pathname + (u.search || ''),
+          method: 'GET',
+          headers,
+          timeout,
+          agent: agent || undefined,
+        };
+        const req = client.request(opts, (res) => {
+          const { statusCode } = res;
+          const chunks = [];
+          const limit = 3 * 1024 * 1024;
+          res.on('data', d => {
+            chunks.push(d);
+            if (chunks.reduce((a, b) => a + b.length, 0) > limit) req.destroy();
+          });
+          res.on('end', () => {
+            const buf = Buffer.concat(chunks);
+            resolve({ status: statusCode, headers: res.headers, body: buf.toString('utf-8'), finalUrl: targetUrl });
+          });
+        });
+        req.on('timeout', () => { req.destroy(); resolve({ status: 0, error: 'timeout' }); });
+        req.on('error', () => resolve({ status: 0, error: 'network' }));
+        req.end();
+      } catch {
+        resolve({ status: 0, error: 'badurl' });
+      }
+    });
+  }
+
+  // ---------- 动态渲染 ----------
+  async _ensureRenderer() {
+    if (!this._puppeteer) return false;
+    if (this._browser) return true;
+    try {
+      this._browser = await this._puppeteer.launch({
+        headless: 'new',
+        args: [
+          '--no-sandbox',
+          '--disable-setuid-sandbox',
+          '--disable-dev-shm-usage'
+        ]
+      });
+      return true;
+    } catch (e) {
+      console.warn('[CRAWLER][RENDER] 启动失败:', e.message);
+      this._browser = null;
+      return false;
+    }
+  }
+
+  async _renderHtml(url, { timeout = 15000 } = {}) {
+    if (!await this._ensureRenderer()) return { ok: false };
+    // 控制并发
+    while (this._renderInFlight >= Math.max(1, this.opts.renderConcurrency)) {
+      await sleep(100);
+    }
+    this._renderInFlight++;
+    try {
+      const page = await this._browser.newPage();
+      await page.setUserAgent(this._pickUA());
+      await page.setExtraHTTPHeaders({
+        'Accept-Language': Math.random() < 0.5 ? 'en-US,en;q=0.8' : 'zh-CN,zh;q=0.8,en;q=0.6',
+        'X-Forwarded-For': randomIPv4(),
+        'X-Real-IP': randomIPv4()
+      });
+      await page.setRequestInterception(true);
+      page.on('request', (req) => {
+        const rtype = req.resourceType();
+        // 阻断大资源
+        if (['image', 'media', 'font'].includes(rtype)) return req.abort();
+        req.continue();
+      });
+      await page.goto(url, { waitUntil: 'networkidle2', timeout });
+      // 滚动以触发懒加载
+      await page.evaluate(async () => {
+        await new Promise(resolve => {
+          let t = 0;
+          const id = setInterval(() => {
+            window.scrollBy(0, 800);
+            if ((t += 200) >= 2000) { clearInterval(id); resolve(); }
+          }, 200);
+        });
+      });
+      const content = await page.content();
+      // 提取链接（渲染后）
+      const links = await page.$$eval('a[href]', as => as.map(a => a.href).filter(Boolean));
+      const finalUrl = page.url();
+      await page.close();
+      return { ok: true, html: content, links, finalUrl };
+    } catch (e) {
+      return { ok: false, error: e.message };
+    } finally {
+      this._renderInFlight--;
+    }
+  }
+
+  // ---------- 解析 ----------
+  extractContent(html, baseUrl) {
+    let text = String(html || '');
+    text = text.replace(/<script[\s\S]*?<\/script>/gi, ' ')
+               .replace(/<style[\s\S]*?<\/style>/gi, ' ')
+               .replace(/<!--[\s\S]*?-->/g, ' ');
+    const mTitle = text.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+    const title = mTitle ? mTitle[1].replace(/\s+/g, ' ').trim() : '';
+    let body = text.replace(/<\/(p|div|h\d|br|li)>/gi, '\n')
+                   .replace(/<[^>]+>/g, ' ')
+                   .replace(/&nbsp;/g, ' ')
+                   .replace(/&amp;/g, '&')
+                   .replace(/\s+\n/g, '\n')
+                   .replace(/\n{3,}/g, '\n\n')
+                   .replace(/[ \t]{2,}/g, ' ')
+                   .trim();
+    const links = [];
+    const aTagRe = /<a\s+[^>]*href=["']([^"']+)["'][^>]*>/gi;
+    let m;
+    let base;
+    try { base = new urlLib.URL(baseUrl); } catch { base = null; }
+    while ((m = aTagRe.exec(html)) !== null) {
+      let href = (m[1] || '').trim();
+      if (!href) continue;
+      try {
+        const abs = base ? new urlLib.URL(href, base).toString() : href;
+        links.push(abs);
+      } catch {}
+    }
+    const nonAscii = (body.match(/[^\x00-\x7F]/g) || []).length;
+    const ratio = nonAscii / Math.max(1, body.length);
+    const lang = ratio > 0.3 ? 'zh' : 'en';
+    return { title, text: body, links, lang };
+  }
+
+  _hostDelayOk(host) {
+    const last = this.perHostTime.get(host) || 0;
+    const now = Date.now();
+    if (now - last >= this.opts.perHostDelayMs) {
+      this.perHostTime.set(host, now);
+      return true;
+    }
+    return false;
+  }
+
+  // ---------- 单页抓取（含动态渲染回退） ----------
+  async _crawlOne(url, ctx = { parent: null, depth: 0 }) {
+    try {
+      const u = new urlLib.URL(url);
+      const rules = await this.robots.fetch(u.host, (u2, o) => this.fetchUrl(u2, o));
+      const ok = this.robots.allowed(u.host, u.pathname, rules);
+      if (!ok) return { ok: false, reason: 'robots' };
+
+      while (!this._hostDelayOk(u.host)) await sleep(200);
+
+      // 静态抓
+      let { status, body, finalUrl } = await this.fetchUrl(url, { timeout: this.opts.requestTimeoutMs });
+      if (status !== 200) body = '';
+
+      // 解析静态
+      let parsed = this.extractContent(body || '', finalUrl || url);
+      let useRender = false;
+
+      // 动态渲染触发条件
+      const alwaysRenderHost = this.opts.dynamicHosts.some(h => u.hostname.endsWith(h));
+      const matchPath = this.opts.dynamicPathRegex && this.opts.dynamicPathRegex.test(u.pathname);
+      const heuristicHeavy = (body.match(/<script/gi) || []).length >= 10 || (parsed.text || '').length < this.opts.minTextLen;
+
+      if (this.opts.renderDynamic === 'always' || alwaysRenderHost || matchPath ||
+         (this.opts.renderDynamic === 'auto' && heuristicHeavy)) {
+        useRender = true;
+      }
+
+      let renderedLinks = [];
+      if (useRender) {
+        const r = await this._renderHtml(url, { timeout: this.opts.renderTimeoutMs });
+        if (r.ok && r.html) {
+          parsed = this.extractContent(r.html, r.finalUrl || url);
+          renderedLinks = Array.isArray(r.links) ? r.links : [];
+          finalUrl = r.finalUrl || finalUrl || url;
+        }
+      }
+
+      if (!this.opts.allowLang.includes(parsed.lang) || !parsed.text || parsed.text.length < this.opts.minTextLen) {
+        this.storage.state.stats.deduped++;
+        return { ok: false, reason: 'short-or-lang', lang: parsed.lang };
+      }
+
+      const saved = this.storage.saveDocument({
+        url: finalUrl || url,
+        title: parsed.title,
+        text: parsed.text,
+        lang: parsed.lang,
+        depth: ctx.depth || 0,
+        parent: ctx.parent || ''
+      });
+
+      // 扩展链接（合并渲染后的链接）
+      const links = Array.from(new Set([...(parsed.links || []), ...renderedLinks])).slice(0, 800);
+      const added = this.storage.enqueueLinks(links, 200);
+
+      this.storage.markVisited(url);
+      this.storage.state.stats.fetched++;
+      return { ok: true, saved, added, lang: parsed.lang, links };
+    } catch (e) {
+      this.storage.state.stats.errors++;
+      return { ok: false, reason: e.message };
+    }
+  }
+
+  // ---------- 递归抓取（多层链接） ----------
+  async crawlRecursive(seedUrl, {
+    maxDepth = this.opts.maxDepth,
+    sameDomainOnly = this.opts.sameDomainOnly,
+    maxPages = this.opts.maxPagesPerTask
+  } = {}) {
+    let seed;
+    try { seed = new urlLib.URL(seedUrl); } catch { return { ok: false, error: 'bad seed url' }; }
+
+    const visitedLocal = new Set();
+    const q = [{ url: seedUrl, depth: 0, parent: '' }];
+    let crawled = 0, savedDocs = 0;
+
+    while (q.length > 0) {
+      if (crawled >= maxPages) break;
+      const { url, depth, parent } = q.shift();
+      if (visitedLocal.has(url)) continue;
+      visitedLocal.add(url);
+
+      // 域限制
+      if (sameDomainOnly) {
+        try {
+          const u = new urlLib.URL(url);
+          if (u.hostname !== seed.hostname) continue;
+        } catch { continue; }
+      }
+
+      const res = await this._crawlOne(url, { parent, depth });
+      if (res && res.ok) {
+        crawled++;
+        if (res.saved) savedDocs++;
+        if (depth + 1 <= maxDepth && Array.isArray(res.links)) {
+          for (const l of res.links) {
+            if (!l || visitedLocal.has(l)) continue;
+            q.push({ url: l, depth: depth + 1, parent: url });
+          }
+        }
+      }
+
+      // 轻微退避，避免单站被打爆
+      await sleep(40 + Math.floor(Math.random() * 120));
+    }
+
+    return { ok: true, seed: seedUrl, crawled, saved: savedDocs, maxDepth, sameDomainOnly, maxPages };
+  }
+
+  async _workerLoop(slotId) {
+    while (this.running) {
+      const url = this.storage.nextUrl();
+      if (!url) { await sleep(1000); continue; }
+      if (this.queue.has(url)) continue;
+      this.queue.add(url);
+      await this._crawlOne(url, { parent: null, depth: 0 });
+      this.queue.delete(url);
+      if (!this.running) break;
+      await sleep(50 + Math.floor(Math.random() * 150));
+    }
+  }
+
+  start() {
+    if (this.running) return;
+    if (fs.existsSync(this.opts.seedsFile)) {
+      const seeds = fs.readFileSync(this.opts.seedsFile, 'utf-8').split(/\r?\n/).map(s => s.trim()).filter(x => /^https?:\/\//i.test(x));
+      if (seeds.length) this.storage.addSeed(seeds);
+    }
+    this.running = true;
+    this.workers = Array.from({ length: this.opts.concurrency }, (_, i) => this._workerLoop(i));
+    console.log(`[CRAWLER] started, concurrency=${this.opts.concurrency}, proxies=${this.proxies.length}, render=${this.opts.renderDynamic}${this._puppeteer ? '' : ' (no-puppeteer)'}`);
+  }
+
+  stop() {
+    this.running = false;
+    console.log('[CRAWLER] stopping...');
+    // 关闭渲染器
+    setTimeout(async () => {
+      try { if (this._browser) await this._browser.close(); } catch {}
+      this._browser = null;
+    }, 0);
+  }
+
+  addSeeds(urls = []) { return this.storage.addSeed(urls); }
+  stats() {
+    const s = this.storage.stats();
+    return Object.assign(s, { running: this.running, inflight: this.queue.size, renderInFlight: this._renderInFlight });
+  }
+  loadRecentDocs(maxFiles = 20) { return this.storage.loadRecentDocs(maxFiles); }
+}
+
+module.exports = { CrawlerManager };
+// ...existing code...