079project 2.0.0 → 4.0.0

package/crawler/storage.cjs

@@ -0,0 +1,163 @@
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const { planQueries, extractSerpLinks, relevanceScore } = require('./agent.cjs');
+// ...existing code...
+
+function randomIPv4() {
+  return Array(4).fill(0).map(() => Math.floor(Math.random() * 254) + 1).join('.');
+}
+class CrawlerStorage {
+  constructor(baseDir = path.join(__dirname, '..', 'crawler_data')) {
+    this.baseDir = baseDir;
+    this.stateFile = path.join(this.baseDir, 'state.json');
+    this.docsDir = path.join(this.baseDir, 'docs');
+    this.ensureDirs();
+    this.state = {
+      frontier: [],            // 待抓取队列（url）
+      visited: {},             // urlHash -> timestamp
+      enqueued: {},            // urlHash -> 1
+      stats: { fetched: 0, saved: 0, deduped: 0, errors: 0, lastSave: 0 }
+    };
+    this._loadState();
+  }
+
+  ensureDirs() {
+    fs.mkdirSync(this.baseDir, { recursive: true });
+    fs.mkdirSync(this.docsDir, { recursive: true });
+  }
+
+  _hash(str) {
+    return crypto.createHash('sha1').update(String(str)).digest('hex');
+  }
+
+  _loadState() {
+    try {
+      if (fs.existsSync(this.stateFile)) {
+        const obj = JSON.parse(fs.readFileSync(this.stateFile, 'utf-8'));
+        if (obj && typeof obj === 'object') this.state = obj;
+      }
+    } catch (e) {
+      console.warn('[CRAWLER][STATE] load failed:', e.message);
+    }
+  }
+
+  _saveState() {
+    try {
+      fs.writeFileSync(this.stateFile, JSON.stringify(this.state, null, 2), 'utf-8');
+      this.state.stats.lastSave = Date.now();
+    } catch (e) {
+      console.warn('[CRAWLER][STATE] save failed:', e.message);
+    }
+  }
+
+  addSeed(urls = []) {
+    let n = 0;
+    for (const u of urls) {
+      const url = String(u || '').trim();
+      if (!url || !/^https?:\/\//i.test(url)) continue;
+      const h = this._hash(url);
+      if (this.state.enqueued[h] || this.state.visited[h]) continue;
+      this.state.frontier.push(url);
+      this.state.enqueued[h] = 1;
+      n++;
+    }
+    if (n > 0) this._saveState();
+    return n;
+  }
+
+  nextUrl() {
+    while (this.state.frontier.length > 0) {
+      const url = this.state.frontier.shift();
+      if (!url) continue;
+      const h = this._hash(url);
+      if (this.state.visited[h]) continue;
+      return url;
+    }
+    return null;
+  }
+
+  markVisited(url) {
+    const h = this._hash(url);
+    this.state.visited[h] = Date.now();
+    this._saveState();
+  }
+
+  // 新链接入队（去重）
+  enqueueLinks(links = [], limitPerBatch = 200) {
+    let n = 0;
+    for (const l of links) {
+      if (!l || !/^https?:\/\//i.test(l)) continue;
+      const h = this._hash(l);
+      if (this.state.enqueued[h] || this.state.visited[h]) continue;
+      this.state.frontier.push(l);
+      this.state.enqueued[h] = 1;
+      n++;
+      if (n >= limitPerBatch) break;
+    }
+    if (n > 0) this._saveState();
+    return n;
+  }
+
+  // 保存清洗后的正文为 txt，返回保存路径（增强：记录 parent/depth）
+  saveDocument(doc) {
+    try {
+      const day = new Date();
+      const dir = path.join(this.docsDir,
+        `${day.getFullYear()}-${String(day.getMonth() + 1).padStart(2, '0')}-${String(day.getDate()).padStart(2, '0')}`
+      );
+      fs.mkdirSync(dir, { recursive: true });
+      const name = `${Date.now()}_${Math.floor(Math.random() * 1e6)}.txt`;
+      const file = path.join(dir, name);
+      const meta = [
+        `URL: ${doc.url || ''}`,
+        `Title: ${doc.title || ''}`,
+        `FetchedAt: ${new Date().toISOString()}`,
+        `Lang: ${doc.lang || ''}`,
+        `Depth: ${doc.depth ?? 0}`,
+        `Parent: ${doc.parent || ''}`
+      ].join('\n');
+      const body = (doc.text || '').trim();
+      if (!body) return null;
+      fs.writeFileSync(file, meta + '\n\n' + body, 'utf-8');
+      this.state.stats.saved++;
+      this._saveState();
+      return file;
+    } catch (e) {
+      this.state.stats.errors++;
+      this._saveState();
+      return null;
+    }
+  }
+
+  stats() {
+    return Object.assign({}, this.state.stats, {
+      frontier: this.state.frontier.length,
+      visited: Object.keys(this.state.visited).length
+    });
+  }
+
+  // 消费已保存文档，按批次读入返回文本数组
+  // 返回：[{ path, text }]
+  loadRecentDocs(maxFiles = 20) {
+    const d = this.docsDir;
+    if (!fs.existsSync(d)) return [];
+    const days = fs.readdirSync(d).filter(f => /^\d{4}-\d{2}-\d{2}$/.test(f)).sort().reverse();
+    const out = [];
+    for (const day of days) {
+      const dir = path.join(d, day);
+      const files = fs.readdirSync(dir).filter(f => f.endsWith('.txt')).sort().reverse();
+      for (const f of files) {
+        try {
+          const p = path.join(dir, f);
+          const text = fs.readFileSync(p, 'utf-8');
+          out.push({ path: p, text });
+          if (out.length >= maxFiles) return out;
+        } catch {}
+      }
+    }
+    return out;
+  }
+}
+
+module.exports = { CrawlerStorage };

package/forwarder.js

@@ -285,6 +285,43 @@ app.get('/', (req, res) => {
                                 <button type="submit" class="btn btn-primary">应用参数</button>
                                 <button type="button" id="resetParams" class="btn btn-secondary">重置默认值</button>
                             </div>
+                             <hr class="my-4"/>
+                            <h6 class="mb-3">激活与传递函数</h6>
+                            <div class="row g-3">
+                                <div class="col-md-6">
+                                    <label for="activationType" class="form-label">Activation Function</label>
+                                    <select id="activationType" class="form-select">
+                                        <option value="identity">identity</option>
+                                        <option value="relu" selected>relu</option>
+                                        <option value="leaky_relu">leaky_relu</option>
+                                        <option value="tanh">tanh</option>
+                                        <option value="sigmoid">sigmoid</option>
+                                        <option value="elu">elu</option>
+                                        <option value="softplus">softplus</option>
+                                        <option value="gelu">gelu</option>
+                                        <option value="custom">custom</option>
+                                    </select>
+                                    <small class="text-muted">Node 侧将对节点聚合与扩散值应用该激活</small>
+                                    <textarea id="activationCustom" class="form-control mt-2" rows="3" style="display:none;" placeholder="Custom activation JS. Example: return x > 0 ? x : 0;"></textarea>
+                                </div>
+                                <div class="col-md-6">
+                                    <label for="transferType" class="form-label">Transfer Function</label>
+                                    <select id="transferType" class="form-select">
+                                        <option value="linear" selected>linear</option>
+                                        <option value="exp">exp</option>
+                                        <option value="inverse">inverse</option>
+                                        <option value="capped">capped</option>
+                                        <option value="custom">custom</option>
+                                    </select>
+                                    <small class="text-muted">传递函数决定沿边传播的信号衰减方式</small>
+                                    <textarea id="transferCustom" class="form-control mt-2" rows="3" style="display:none;" placeholder="Custom transfer JS. Args: value, weight, decayK, ctx. Example: return value * Math.exp(-(decayK*weight));"></textarea>
+                                </div>
+                            </div>
+
+                            <div class="d-flex justify-content-between mt-4">
+                                <button type="submit" class="btn btn-primary">应用参数</button>
+                                <button type="button" id="resetParams" class="btn btn-secondary">重置默认值</button>
+                            </div>
                             
                         </form>
                     </div>
@@ -534,6 +571,7 @@ app.get('/', (req, res) => {
             document.getElementById('confirmDeleteSnapshot').addEventListener('click', function() {
                 deleteSnapshot();
             });
+                        bindActivationEditors(); // 新增：绑定选择器与自定义编辑显隐
         });
         
         // 发送消息
@@ -595,6 +633,8 @@ app.get('/', (req, res) => {
         }
         
         // 加载模型参数
+
+        // 加载模型参数（保持接口不变）
         function loadModelParams() {
             fetch('/api/model/params')
                 .then(response => response.json())
@@ -672,6 +712,13 @@ function updateParamSliders(params) {
         document.getElementById('waitingTimeRange').value = params.waitingTime;
         document.getElementById('waitingTimeValue').textContent = params.waitingTime;
     }
+                    if (params.activationType) document.getElementById('activationType').value = params.activationType;
+            if (params.transferType) document.getElementById('transferType').value = params.transferType;
+            if (typeof params.activationCustom === 'string') document.getElementById('activationCustom').value = params.activationCustom;
+            if (typeof params.transferCustom === 'string') document.getElementById('transferCustom').value = params.transferCustom;
+            
+            // 触发一次显隐同步
+            if (typeof bindActivationEditors === 'function') bindActivationEditors();
 } // 添加这个花括号
 
 function bindSliderEvents() {
@@ -705,43 +752,47 @@ function bindSliderEvents() {
     });
 }
         // 更新模型参数
-function updateModelParams() {
-    const params = {
-        decayFactor: parseFloat(document.getElementById('decayFactorRange').value),
-        maxMemeWords: parseInt(document.getElementById('maxMemeWordsRange').value),
-        minOverlapThreshold: parseInt(document.getElementById('minOverlapThresholdRange').value),
-        maliciousThreshold: parseFloat(document.getElementById('maliciousThresholdRange').value),
-        learningIterations: parseInt(document.getElementById('learningIterationsRange').value),
-        iteration: parseInt(document.getElementById('iterationRange').value),
-        threshold: parseInt(document.getElementById('thresholdRange').value),
-        decay: parseFloat(document.getElementById('decayRange').value),
-        decayK: parseFloat(document.getElementById('decayKRange').value),
-        maxLen: parseInt(document.getElementById('maxLenRange').value),
-        edgeWeight: parseFloat(document.getElementById('edgeWeightRange').value),
-        communicateCount: parseInt(document.getElementById('communicateCountRange').value),
-        waitingTime: parseInt(document.getElementById('waitingTime').value) // 添加这个参数
-    };
-    
-    fetch('/api/model/params', {
-        method: 'POST',
-        headers: {
-            'Content-Type': 'application/json',
-        },
-        body: JSON.stringify(params),
-    })
-    .then(response => response.json())
-    .then(data => {
-        if (data.success) {
-            alert('参数更新成功!');
-        } else {
-            alert('参数更新失败: ' + (data.error || '未知错误'));
+        // 更新模型参数：增加激活/传递函数字段
+        function updateModelParams() {
+            const params = {
+                decayFactor: parseFloat(document.getElementById('decayFactorRange').value),
+                maxMemeWords: parseInt(document.getElementById('maxMemeWordsRange').value),
+                minOverlapThreshold: parseInt(document.getElementById('minOverlapThresholdRange').value),
+                maliciousThreshold: parseFloat(document.getElementById('maliciousThresholdRange').value),
+                learningIterations: parseInt(document.getElementById('learningIterationsRange').value),
+                iteration: parseInt(document.getElementById('iterationRange').value),
+                threshold: parseInt(document.getElementById('thresholdRange').value),
+                decay: parseFloat(document.getElementById('decayRange').value),
+                decayK: parseFloat(document.getElementById('decayKRange').value),
+                maxLen: parseInt(document.getElementById('maxLenRange').value),
+                edgeWeight: parseFloat(document.getElementById('edgeWeightRange').value),
+                communicateCount: parseInt(document.getElementById('communicateCountRange').value),
+                waitingTime: parseInt(document.getElementById('waitingTime').value),
+                // 新增
+                activationType: document.getElementById('activationType').value,
+                transferType: document.getElementById('transferType').value,
+                activationCustom: document.getElementById('activationCustom').value,
+                transferCustom: document.getElementById('transferCustom').value
+            };
+
+            fetch('/api/model/params', {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify(params),
+            })
+            .then(response => response.json())
+            .then(data => {
+                if (data.success) {
+                    alert('参数更新成功!');
+                } else {
+                    alert('参数更新失败: ' + (data.error || '未知错误'));
+                }
+            })
+            .catch(error => {
+                console.error('Error updating parameters:', error);
+                alert('更新参数时发生错误: ' + error.message);
+            });
         }
-    })
-    .catch(error => {
-        console.error('Error updating parameters:', error);
-        alert('更新参数时发生错误: ' + error.message);
-    });
-}
         
         // 重置模型参数
         function resetModelParams() {
@@ -776,6 +827,19 @@ function updateModelParams() {
                 .catch(error => console.error('Error loading system status:', error));
         }
         
+                function bindActivationEditors() {
+            const actSel = document.getElementById('activationType');
+            const actTxt = document.getElementById('activationCustom');
+            const trSel  = document.getElementById('transferType');
+            const trTxt  = document.getElementById('transferCustom');
+            const sync = () => {
+                actTxt.style.display = (actSel.value === 'custom') ? '' : 'none';
+                trTxt.style.display  = (trSel.value  === 'custom') ? '' : 'none';
+            };
+            actSel.addEventListener('change', sync);
+            trSel.addEventListener('change', sync);
+            sync();
+        }
 function updateParamSliders(params) {
     // 定义一个更新滑块的辅助函数
     function updateSlider(paramName, sliderId, valueId) {
@@ -1080,7 +1144,12 @@ const modelDefaults = {
     minOverlapThreshold: 2,
     maliciousThreshold: 0.7,
     learningIterations: 3,
-    communicateCount: 1 // 新增
+    communicateCount: 1,
+    // 新增：激活与传递函数选择
+    activationType: 'relu',
+    transferType: 'linear',
+    activationCustom: '',
+    transferCustom: ''
 };
 
 // 当前应用的模型参数

package/groupmanager.cjs

@@ -51,7 +51,8 @@ const REQUIRED_DIRS = [
   'robots',
   'tests',
   'public',
-  'snapshots'
+  'snapshots',
+  'crawler'
 ];
 
 // 检查与准备目标 groups 文件夹