079project 2.0.0 → 3.0.0

package/crawler/storage.cjs

@@ -0,0 +1,163 @@
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const { planQueries, extractSerpLinks, relevanceScore } = require('./agent.cjs');
+// ...existing code...
+
+function randomIPv4() {
+  return Array(4).fill(0).map(() => Math.floor(Math.random() * 254) + 1).join('.');
+}
+class CrawlerStorage {
+  constructor(baseDir = path.join(__dirname, '..', 'crawler_data')) {
+    this.baseDir = baseDir;
+    this.stateFile = path.join(this.baseDir, 'state.json');
+    this.docsDir = path.join(this.baseDir, 'docs');
+    this.ensureDirs();
+    this.state = {
+      frontier: [],            // 待抓取队列（url）
+      visited: {},             // urlHash -> timestamp
+      enqueued: {},            // urlHash -> 1
+      stats: { fetched: 0, saved: 0, deduped: 0, errors: 0, lastSave: 0 }
+    };
+    this._loadState();
+  }
+
+  ensureDirs() {
+    fs.mkdirSync(this.baseDir, { recursive: true });
+    fs.mkdirSync(this.docsDir, { recursive: true });
+  }
+
+  _hash(str) {
+    return crypto.createHash('sha1').update(String(str)).digest('hex');
+  }
+
+  _loadState() {
+    try {
+      if (fs.existsSync(this.stateFile)) {
+        const obj = JSON.parse(fs.readFileSync(this.stateFile, 'utf-8'));
+        if (obj && typeof obj === 'object') this.state = obj;
+      }
+    } catch (e) {
+      console.warn('[CRAWLER][STATE] load failed:', e.message);
+    }
+  }
+
+  _saveState() {
+    try {
+      fs.writeFileSync(this.stateFile, JSON.stringify(this.state, null, 2), 'utf-8');
+      this.state.stats.lastSave = Date.now();
+    } catch (e) {
+      console.warn('[CRAWLER][STATE] save failed:', e.message);
+    }
+  }
+
+  addSeed(urls = []) {
+    let n = 0;
+    for (const u of urls) {
+      const url = String(u || '').trim();
+      if (!url || !/^https?:\/\//i.test(url)) continue;
+      const h = this._hash(url);
+      if (this.state.enqueued[h] || this.state.visited[h]) continue;
+      this.state.frontier.push(url);
+      this.state.enqueued[h] = 1;
+      n++;
+    }
+    if (n > 0) this._saveState();
+    return n;
+  }
+
+  nextUrl() {
+    while (this.state.frontier.length > 0) {
+      const url = this.state.frontier.shift();
+      if (!url) continue;
+      const h = this._hash(url);
+      if (this.state.visited[h]) continue;
+      return url;
+    }
+    return null;
+  }
+
+  markVisited(url) {
+    const h = this._hash(url);
+    this.state.visited[h] = Date.now();
+    this._saveState();
+  }
+
+  // 新链接入队（去重）
+  enqueueLinks(links = [], limitPerBatch = 200) {
+    let n = 0;
+    for (const l of links) {
+      if (!l || !/^https?:\/\//i.test(l)) continue;
+      const h = this._hash(l);
+      if (this.state.enqueued[h] || this.state.visited[h]) continue;
+      this.state.frontier.push(l);
+      this.state.enqueued[h] = 1;
+      n++;
+      if (n >= limitPerBatch) break;
+    }
+    if (n > 0) this._saveState();
+    return n;
+  }
+
+  // 保存清洗后的正文为 txt，返回保存路径（增强：记录 parent/depth）
+  saveDocument(doc) {
+    try {
+      const day = new Date();
+      const dir = path.join(this.docsDir,
+        `${day.getFullYear()}-${String(day.getMonth() + 1).padStart(2, '0')}-${String(day.getDate()).padStart(2, '0')}`
+      );
+      fs.mkdirSync(dir, { recursive: true });
+      const name = `${Date.now()}_${Math.floor(Math.random() * 1e6)}.txt`;
+      const file = path.join(dir, name);
+      const meta = [
+        `URL: ${doc.url || ''}`,
+        `Title: ${doc.title || ''}`,
+        `FetchedAt: ${new Date().toISOString()}`,
+        `Lang: ${doc.lang || ''}`,
+        `Depth: ${doc.depth ?? 0}`,
+        `Parent: ${doc.parent || ''}`
+      ].join('\n');
+      const body = (doc.text || '').trim();
+      if (!body) return null;
+      fs.writeFileSync(file, meta + '\n\n' + body, 'utf-8');
+      this.state.stats.saved++;
+      this._saveState();
+      return file;
+    } catch (e) {
+      this.state.stats.errors++;
+      this._saveState();
+      return null;
+    }
+  }
+
+  stats() {
+    return Object.assign({}, this.state.stats, {
+      frontier: this.state.frontier.length,
+      visited: Object.keys(this.state.visited).length
+    });
+  }
+
+  // 消费已保存文档，按批次读入返回文本数组
+  // 返回：[{ path, text }]
+  loadRecentDocs(maxFiles = 20) {
+    const d = this.docsDir;
+    if (!fs.existsSync(d)) return [];
+    const days = fs.readdirSync(d).filter(f => /^\d{4}-\d{2}-\d{2}$/.test(f)).sort().reverse();
+    const out = [];
+    for (const day of days) {
+      const dir = path.join(d, day);
+      const files = fs.readdirSync(dir).filter(f => f.endsWith('.txt')).sort().reverse();
+      for (const f of files) {
+        try {
+          const p = path.join(dir, f);
+          const text = fs.readFileSync(p, 'utf-8');
+          out.push({ path: p, text });
+          if (out.length >= maxFiles) return out;
+        } catch {}
+      }
+    }
+    return out;
+  }
+}
+
+module.exports = { CrawlerStorage };

package/groupmanager.cjs

@@ -51,7 +51,8 @@ const REQUIRED_DIRS = [
   'robots',
   'tests',
   'public',
-  'snapshots'
+  'snapshots',
+  'crawler'
 ];
 
 // 检查与准备目标 groups 文件夹