079project 1.0.0

package/GroupStarter.cjs

@@ -0,0 +1,647 @@
+// ...existing code...
+const fs = require('fs');
+const path = require('path');
+const { spawn } = require('child_process');
+const express = require('express');
+const os = require('os');
+const { exec } = require('child_process');
+const axios = require('axios');
+
+// 轻量参数解析（不引入依赖）
+function parseArgs(argv) {
+  const out = {};
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a.startsWith('--')) {
+      const key = a.slice(2);
+      const val = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : true;
+      out[key] = val;
+    }
+  }
+  return out;
+}
+const args = parseArgs(process.argv.slice(2));
+
+// 配置（可被 CLI 参数或环境变量覆盖）
+let GROUPS_DIR = path.resolve(args['groups-dir'] || process.env.GROUPS_DIR || path.join(__dirname, 'groups'));
+let SERVE_COUNT = Number(args['serve-count'] || process.env.SERVE_COUNT || 4);
+let PORT_BASE = Number(args['port-base'] || process.env.PORT_BASE || 12001);
+let MASTER_PORT = Number(args['master-port'] || process.env.MASTER_PORT || 9100);
+let GROUP_START_DELAY = Number(args['group-start-delay'] || process.env.GROUP_START_DELAY || 3000);
+let ROBOTS_DIR_OVERRIDE = args['robots-dir'] ? path.resolve(args['robots-dir']) : '';
+let TESTS_DIR_OVERRIDE = args['tests-dir'] ? path.resolve(args['tests-dir']) : '';
+let PUBLIC_DIR_OVERRIDE = args['public-dir'] ? path.resolve(args['public-dir']) : '';
+let SNAPSHOTS_DIR_OVERRIDE = args['snapshots-dir'] ? path.resolve(args['snapshots-dir']) : '';
+const FORWARDER_OFFSET = SERVE_COUNT; // forwarder端口偏移
+
+console.log('[CFG] GROUPS_DIR=', GROUPS_DIR);
+console.log('[CFG] SERVE_COUNT=', SERVE_COUNT, ' PORT_BASE=', PORT_BASE, ' MASTER_PORT=', MASTER_PORT);
+if (ROBOTS_DIR_OVERRIDE) console.log('[CFG] ROBOTS_DIR_OVERRIDE=', ROBOTS_DIR_OVERRIDE);
+if (TESTS_DIR_OVERRIDE) console.log('[CFG] TESTS_DIR_OVERRIDE=', TESTS_DIR_OVERRIDE);
+if (PUBLIC_DIR_OVERRIDE) console.log('[CFG] PUBLIC_DIR_OVERRIDE=', PUBLIC_DIR_OVERRIDE);
+if (SNAPSHOTS_DIR_OVERRIDE) console.log('[CFG] SNAPSHOTS_DIR_OVERRIDE=', SNAPSHOTS_DIR_OVERRIDE);
+
+// 读取所有 group_x 文件夹
+const groupFolders = fs.existsSync(GROUPS_DIR)
+  ? fs.readdirSync(GROUPS_DIR).filter((f) => /^group_\d+$/.test(f)).map((f) => path.join(GROUPS_DIR, f))
+  : [];
+// ...existing code...
+// 记录所有 group 的 forwarder 端口与子进程
+const groupForwarderPorts = [];
+const groupProcesses = [];
+
+// 运行时注册表，便于管理与重启
+const groupRegistry = []; // { id, dir, servePorts: [{pid,port}], study: {pid,port}, forwarder: {pid,port} }
+
+// 资源监控与自适应节流
+const METRICS_WINDOW = 300; // 保留最近300个点
+const metrics = {
+  ts: [],
+  cpu: [], // 0..1
+  mem: [], // 使用率 0..1
+  disk: [], // [{ drive, free, total }]
+  gpu: [], // 简化 [{ util, memUsed, memTotal }]
+};
+
+let lastCpuSample = os.cpus();
+let emergencyBuffer = null; // 应急释放缓冲，避免立刻 OOM
+let swapActive = false; // 是否处于“内存应急”状态
+
+// 计算CPU占用（基于两次 /os.cpus 快照）
+function sampleCpu() {
+  const now = os.cpus();
+  const util = now.map((core, idx) => {
+    const p = lastCpuSample[idx];
+    const idle = core.times.idle - p.times.idle;
+    const total = Object.keys(core.times).reduce((s, k) => s + (core.times[k] - p.times[k]), 0);
+    const busy = Math.max(0, total - idle);
+    return total > 0 ? busy / total : 0;
+  });
+  lastCpuSample = now;
+  return util.reduce((a, b) => a + b, 0) / Math.max(1, util.length);
+}
+
+function sampleMem() {
+  const total = os.totalmem();
+  const free = os.freemem();
+  const usedRatio = (total - free) / total;
+  return { usedRatio, total, free };
+}
+
+function pushMetric(key, value) {
+  const ts = Date.now();
+  metrics.ts.push(ts);
+  if (metrics.ts.length > METRICS_WINDOW) metrics.ts.shift();
+  const pushArr = (arr, v) => {
+    arr.push(v);
+    if (arr.length > METRICS_WINDOW) arr.shift();
+  };
+  if (key === 'cpu') pushArr(metrics.cpu, value);
+  if (key === 'mem') pushArr(metrics.mem, value);
+}
+
+// 尝试采样磁盘（Windows: wmic）
+async function sampleDiskOnce() {
+  return new Promise((resolve) => {
+    exec(
+      'wmic logicaldisk get size,freespace,caption',
+      { windowsHide: true },
+      (err, stdout) => {
+        if (err || !stdout) return resolve([]);
+        const lines = stdout.split(/\r?\n/).filter(Boolean).slice(1);
+        const out = [];
+        for (const line of lines) {
+          const parts = line.trim().split(/\s+/);
+          if (parts.length >= 3) {
+            const [caption, free, size] = [
+              parts[0],
+              Number(parts[1] || 0),
+              Number(parts[2] || 0),
+            ];
+            out.push({ drive: caption, free, total: size });
+          }
+        }
+        resolve(out);
+      }
+    );
+  });
+}
+
+// 尝试采样GPU（若有NVIDIA）
+async function sampleGpuOnce() {
+  return new Promise((resolve) => {
+    exec(
+      'nvidia-smi --query-gpu=utilization.gpu,memory.used,memory.total --format=csv,noheader,nounits',
+      { windowsHide: true },
+      (err, stdout) => {
+        if (err || !stdout) return resolve([]);
+        const lines = stdout.split(/\r?\n/).filter(Boolean);
+        const out = lines.map((l) => {
+          const [util, memUsed, memTotal] = l
+            .split(',')
+            .map((s) => Number(s.trim()));
+          return {
+            util: util / 100,
+            memUsed: memUsed * 1024 * 1024,
+            memTotal: memTotal * 1024 * 1024,
+          };
+        });
+        resolve(out);
+      }
+    );
+  });
+}
+
+function tryAllocEmergencyBuffer() {
+  if (!emergencyBuffer) {
+    try {
+      // 预留一块可释放内存（64MB，不至于影响正常运行）
+      emergencyBuffer = Buffer.allocUnsafe(64 * 1024 * 1024);
+    } catch (_) {
+      emergencyBuffer = null;
+    }
+  }
+}
+
+function releaseEmergencyBuffer() {
+  if (emergencyBuffer) {
+    emergencyBuffer = null;
+    global.gc && global.gc(); // 如 --expose-gc 则主动GC
+  }
+}
+
+function startResourceMonitor() {
+  tryAllocEmergencyBuffer();
+  setInterval(async () => {
+    const cpu = sampleCpu();
+    pushMetric('cpu', cpu);
+    const { usedRatio } = sampleMem();
+    pushMetric('mem', usedRatio);
+    // 每30秒采样一次磁盘与GPU
+    if (metrics.ts.length % 30 === 0) {
+      try {
+        metrics.disk = await sampleDiskOnce();
+      } catch (_) {}
+      try {
+        metrics.gpu = await sampleGpuOnce();
+      } catch (_) {}
+    }
+
+    // 自适应：高内存时进入应急模式，释放预留；恢复后重新预留
+    if (usedRatio >= 0.92 && !swapActive) {
+      swapActive = true;
+      releaseEmergencyBuffer();
+    } else if (swapActive && usedRatio <= 0.8) {
+      swapActive = false;
+      tryAllocEmergencyBuffer();
+    }
+  }, 1000);
+}
+
+async function startAllGroups() {
+  for (let i = 0; i < groupFolders.length; i++) {
+    const groupDir = groupFolders[i];
+    const reg = { id: i, dir: groupDir, servePorts: [], study: null, forwarder: null };
+    const servePorts = [];
+
+    for (let j = 0; j < SERVE_COUNT; j++) {
+      const port = PORT_BASE + i * (SERVE_COUNT + 2) + j;
+      servePorts.push(port);
+      const serveScript = path.join(groupDir, 'main_Serve.cjs');
+      if (fs.existsSync(serveScript)) {
+        let attempts = 0;
+        while (attempts < 3) {
+          try {
+            const proc = spawn('node', [serveScript, port.toString(),'--expose-gc'], {
+              cwd: groupDir,
+              stdio: 'inherit',
+            });
+            groupProcesses.push(proc);
+            reg.servePorts.push({ pid: null, port });
+            proc.on('spawn', () => {
+              const idx = reg.servePorts.findIndex((p) => p.port === port);
+              if (idx >= 0) reg.servePorts[idx].pid = proc.pid;
+            });
+            break;
+          } catch (error) {
+            attempts++;
+            console.warn(
+              `[WARN] 启动服务 ${serveScript} 失败 (尝试 ${attempts}/3): ${error.message}`
+            );
+            if (attempts === 3)
+              console.error(`[ERROR] 无法启动服务 ${serveScript}`);
+            await new Promise((resolve) => setTimeout(resolve, 3000));
+          }
+        }
+      }
+    }
+
+    // study端口
+    const studyPort = PORT_BASE + i * (SERVE_COUNT + 2) + SERVE_COUNT;
+    const studyScript = path.join(groupDir, 'main_Study.cjs');
+    if (fs.existsSync(studyScript)) {
+      const proc = spawn(
+        'node',
+        [studyScript, studyPort.toString(), '--max-old-space-size=12288', '--expose-gc'],
+        { cwd: groupDir, stdio: 'inherit' }
+      );
+      groupProcesses.push(proc);
+      reg.study = { pid: null, port: studyPort };
+      proc.on('spawn', () => {
+        reg.study.pid = proc.pid;
+      });
+    }
+
+    // forwarder端口
+    const forwarderPort = PORT_BASE + i * (SERVE_COUNT + 2) + FORWARDER_OFFSET + 1;
+    groupForwarderPorts.push(forwarderPort);
+    const forwarderScript = path.join(groupDir, 'forwarder.js');
+    if (fs.existsSync(forwarderScript)) {
+      const proc = spawn(
+        'node',
+        [
+          forwarderScript,
+          forwarderPort.toString(),
+          ...servePorts.map((p) => p.toString()),
+          studyPort.toString(),
+          '--expose-gc'
+        ],
+        { cwd: groupDir, stdio: 'inherit' }
+      );
+      groupProcesses.push(proc);
+      reg.forwarder = { pid: null, port: forwarderPort };
+      proc.on('spawn', () => {
+        reg.forwarder.pid = proc.pid;
+      });
+    }
+
+    // 写入组配置文件
+    const config = { servePorts, studyPort, forwarderPort };
+    fs.writeFileSync(
+      path.join(groupDir, 'group_ports.json'),
+      JSON.stringify(config, null, 2)
+    );
+    console.log(
+      `[START] group_${i} 端口分配: serve=${servePorts.join(',')}, study=${studyPort}, forwarder=${forwarderPort}`
+    );
+    groupRegistry.push(reg);
+    await new Promise((r) => setTimeout(r, GROUP_START_DELAY));
+  }
+  console.log(`[INFO] 共启动 ${groupFolders.length} 个 group`);
+}
+
+function startMasterForwarder() {
+  const app = express();
+  app.use(express.json({ limit: '800mb' }));
+  // 新增：托管前端静态资源
+  app.use(express.static(path.join(__dirname, 'public')));
+
+  // 启动资源监控
+  startResourceMonitor();
+
+  // 新增：语料流接口（从各组 robots/*.txt 逐行推送至前端）
+  app.get('/api/corpus', async (req, res) => {
+    try {
+      const max = Math.min(parseInt(req.query.max || '2000', 10), 200000);
+      const gStart = Math.max(parseInt(req.query.groupStart || '0', 10), 0);
+      const gEnd = Math.min(
+        parseInt(req.query.groupEnd || String(groupFolders.length - 1), 10),
+        groupFolders.length - 1
+      );
+      const format = (req.query.format || 'ndjson').toLowerCase(); // ndjson | text
+
+      if (format === 'ndjson') {
+        res.setHeader('Content-Type', 'application/x-ndjson; charset=utf-8');
+      } else {
+        res.setHeader('Content-Type', 'text/plain; charset=utf-8');
+      }
+
+      const { createInterface } = require('readline');
+      let count = 0;
+
+      for (let i = gStart; i <= gEnd && count < max; i++) {
+        const robotsDir = ROBOTS_DIR_OVERRIDE || path.join(groupFolders[i], 'robots');
+        if (!fs.existsSync(robotsDir)) continue;
+        const files = fs
+          .readdirSync(robotsDir)
+          .filter((f) => f.toLowerCase().endsWith('.txt'));
+
+        for (const f of files) {
+          if (count >= max) break;
+          const full = path.join(robotsDir, f);
+          const stream = fs.createReadStream(full, { encoding: 'utf-8' });
+          const rl = createInterface({ input: stream, crlfDelay: Infinity });
+
+          for await (const line of rl) {
+            const t = (line || '').trim();
+            if (!t) continue;
+            if (format === 'ndjson') {
+              res.write(JSON.stringify({ g: i, t }) + '\n');
+            } else {
+              res.write(t + '\n');
+            }
+            count++;
+            if (count >= max) {
+              rl.close();
+              break;
+            }
+          }
+        }
+      }
+      res.end();
+    } catch (e) {
+      res.status(500).json({ error: e.message });
+    }
+  });
+
+  // 指标接口：返回最近窗口的时序数据
+  app.get('/api/metrics', (req, res) => {
+    res.json({
+      ts: metrics.ts,
+      cpu: metrics.cpu,
+      mem: metrics.mem,
+      disk: metrics.disk,
+      gpu: metrics.gpu,
+      swapActive,
+    });
+  });
+
+  // 管理：重启指定组或类型
+  app.post('/api/admin/restart', async (req, res) => {
+    try {
+      const { groupId, type } = req.body; // type: 'serve'|'study'|'forwarder'|'all'
+      if (
+        typeof groupId !== 'number' ||
+        groupId < 0 ||
+        groupId >= groupRegistry.length
+      ) {
+        return res.status(400).json({ error: 'invalid groupId' });
+      }
+      const reg = groupRegistry[groupId];
+      const spawnServe = () => {
+        const serveScript = path.join(reg.dir, 'main_Serve.cjs');
+        for (const sp of reg.servePorts) {
+          const p = spawn('node', [serveScript, sp.port.toString(), '--expose-gc','--max-old-space-size=16384'], {
+            cwd: reg.dir,
+            stdio: 'inherit',
+          });
+          p.on('spawn', () => {
+            sp.pid = p.pid;
+          });
+        }
+      };
+      const spawnStudy = () => {
+        const studyScript = path.join(reg.dir, 'main_Study.cjs');
+        const p = spawn(
+          'node',
+          [studyScript, reg.study.port.toString(), '--max-old-space-size=16384', '--expose-gc'],
+          { cwd: reg.dir, stdio: 'inherit' }
+        );
+        p.on('spawn', () => {
+          reg.study.pid = p.pid;
+        });
+      };
+      const spawnForwarder = () => {
+        const forwarderScript = path.join(reg.dir, 'forwarder.js');
+        const p = spawn(
+          'node',
+          [
+            forwarderScript,
+            reg.forwarder.port.toString(),
+            ...reg.servePorts.map((s) => s.port.toString()),
+            reg.study.port.toString(),
+            '--expose-gc',
+            '--max-old-space-size=16384'
+          ],
+          { cwd: reg.dir, stdio: 'inherit' }
+        );
+        p.on('spawn', () => {
+          reg.forwarder.pid = p.pid;
+        });
+      };
+
+      const killIf = (pid) => {
+        try {
+          process.kill(pid);
+        } catch (_) {}
+      };
+      if (type === 'serve' || type === 'all') {
+        reg.servePorts.forEach((sp) => sp.pid && killIf(sp.pid));
+        spawnServe();
+      }
+      if (type === 'study' || type === 'all') {
+        reg.study?.pid && killIf(reg.study.pid);
+        spawnStudy();
+      }
+      if (type === 'forwarder' || type === 'all') {
+        reg.forwarder?.pid && killIf(reg.forwarder.pid);
+        spawnForwarder();
+      }
+      res.json({ ok: true });
+    } catch (e) {
+      res.status(500).json({ error: e.message });
+    }
+  });
+
+  // 管理：批量更新某一类文件到所有组（如 robots 资源或脚本）
+  app.post('/api/admin/batch-update', async (req, res) => {
+    try {
+      const { relativePath, content, sourcePath } = req.body; // 二选一：直接内容或从源文件复制
+      if (!relativePath)
+        return res.status(400).json({ error: 'relativePath required' });
+      const updated = [];
+      for (const reg of groupRegistry) {
+        const dst = path.join(reg.dir, relativePath);
+        fs.mkdirSync(path.dirname(dst), { recursive: true });
+        if (typeof content === 'string') {
+          fs.writeFileSync(dst, content, 'utf-8');
+        } else if (typeof sourcePath === 'string' && fs.existsSync(sourcePath)) {
+          fs.copyFileSync(sourcePath, dst);
+        } else {
+          continue;
+        }
+        updated.push(dst);
+      }
+      res.json({ ok: true, updatedCount: updated.length });
+    } catch (e) {
+      res.status(500).json({ error: e.message });
+    }
+  });
+
+  // 聚合器：自适应采样与节流
+  app.post('/api/chat', async (req, res) => {
+    const { message } = req.body || {};
+    if (!message || typeof message !== 'string') {
+      return res.status(400).json({ error: '消息不能为空' });
+    }
+
+    // 轻微节流：高负载时延迟处理，避免CPU尖峰
+    const curCpu = metrics.cpu[metrics.cpu.length - 1] || 0;
+    const curMem = metrics.mem[metrics.mem.length - 1] || 0;
+    const throttleMs = curCpu > 0.9 ? 150 : curCpu > 0.8 ? 60 : 0;
+    if (throttleMs) await new Promise((r) => setTimeout(r, throttleMs));
+
+    // 采样规模自适应
+    const ALL = groupForwarderPorts.slice();
+    let baseSize = Math.min(
+      Math.max(8, Math.floor(Math.sqrt(ALL.length || 100))),
+      64
+    );
+    let factor = 1.0;
+    if (curCpu > 0.85) factor *= 0.6;
+    if (curMem > 0.9 || swapActive) factor *= 0.6;
+    const SAMPLE_SIZE = Math.max(2, Math.floor(baseSize * factor));
+
+    const samplePorts = (allPorts, m) => {
+      const arr = [...allPorts];
+      const out = [];
+      for (let i = 0; i < Math.min(m, arr.length); i++) {
+        const idx = Math.floor(Math.random() * arr.length);
+        out.push(arr.splice(idx, 1)[0]);
+      }
+      return out;
+    };
+
+    const SELECTED = samplePorts(ALL, SAMPLE_SIZE);
+
+    // 历史统计持久化
+    const STATS_PATH = path.join(__dirname, 'group_stats.json');
+    const loadStats = () => {
+      try {
+        return JSON.parse(fs.readFileSync(STATS_PATH, 'utf-8'));
+      } catch {
+        return {};
+      }
+    };
+    const saveStats = (s) => {
+      try {
+        fs.writeFileSync(STATS_PATH, JSON.stringify(s, null, 2));
+      } catch {}
+    };
+    const stats = loadStats();
+
+    async function queryPorts(ports) {
+      const promises = ports.map((port) => {
+        const t0 = Date.now();
+        return axios
+          .post(`http://localhost:${port}/api/chat`, { message }, { timeout: 20000 })
+          .then((r) => ({
+            port,
+            ok: true,
+            text: r.data && r.data.response ? r.data.response : '',
+            latency: Date.now() - t0,
+          }))
+          .catch(() => ({ port, ok: false, text: '', latency: Date.now() - t0 }));
+      });
+      return Promise.all(promises);
+    }
+
+    // 中间层“周期拆分”：CPU>80%时将一次查询分两批执行，批间让出事件循环
+    let settled = [];
+    if (curCpu > 0.8 && SELECTED.length > 2) {
+      const mid = Math.ceil(SELECTED.length / 2);
+      const first = SELECTED.slice(0, mid);
+      const second = SELECTED.slice(mid);
+      const r1 = await queryPorts(first);
+      settled.push(...r1);
+      await new Promise((r) => setImmediate(r));
+      const r2 = await queryPorts(second);
+      settled.push(...r2);
+    } else {
+      settled = await queryPorts(SELECTED);
+    }
+
+    // 更新统计（简单EWMA/计数）
+    settled.forEach((r) => {
+      if (!stats[r.port])
+        stats[r.port] = { score: 1, requests: 0, success: 0, avgLatency: r.latency || 0 };
+      const s = stats[r.port];
+      s.requests = (s.requests || 0) + 1;
+      if (r.ok && r.text) s.success = (s.success || 0) + 1;
+      s.avgLatency = ((s.avgLatency * (s.requests - 1)) + r.latency) / s.requests;
+      s.score = Math.max(
+        0.1,
+        Math.min(10, ((s.success || 0) / s.requests) * 2 + (s.score || 1) * 0.8)
+      );
+    });
+    saveStats(stats);
+
+    const scoreResp = (text, port) => {
+      const s = stats[port] || { score: 1, avgLatency: 1000, requests: 0, success: 0 };
+      const lenScore = Math.min(1, (text.length || 0) / 200);
+      const latencyScore = 1 / (1 + (s.avgLatency || 1000) / 500);
+      return 0.6 * (s.score || 1) + 0.3 * lenScore + 0.1 * latencyScore;
+    };
+
+    // 只保留有效响应并打分
+    const scored = settled
+      .filter((r) => r.ok && r.text && r.text.length > 0)
+      .map((r) => ({ ...r, score: scoreResp(r.text, r.port) }))
+      .sort((a, b) => b.score - a.score);
+
+    if (scored.length === 0) {
+      return res.status(502).json({ error: '所有组无有效响应' });
+    }
+
+    // 去重并返回 top K
+    const seen = new Set();
+    const unique = [];
+    for (const item of scored) {
+      const key = item.text.trim().slice(0, 200);
+      if (!seen.has(key)) {
+        seen.add(key);
+        unique.push(item);
+      }
+      if (unique.length >= 10) break;
+    }
+
+    res.json({
+      top: unique[0].text,
+      responses: unique.map((u) => ({ port: u.port, response: u.text, score: u.score })),
+      sampled: SELECTED.length,
+      totalCandidates: ALL.length,
+      cpu: curCpu,
+      mem: curMem,
+      throttledMs: throttleMs,
+    });
+  });
+
+  // 健康检查
+  app.get('/health', (req, res) => {
+    res.json({ status: 'ok', groups: groupForwarderPorts.length });
+  });
+  app.get('/', (req, res) => {
+    res.sendFile(path.join(__dirname, 'public', 'index.html'));
+  });
+  // 监控页面
+  app.get('/monitor', (req, res) => {
+    res.sendFile(path.join(__dirname, 'public', 'monitor.html'));
+  });
+
+  // 启动服务
+  app.listen(MASTER_PORT, () => {
+    console.log(`[MASTER] GroupStarter master forwarder已启动，端口 ${MASTER_PORT}`);
+    console.log(`[MASTER] 静态前端: http://localhost:${MASTER_PORT}/`);
+    console.log(
+      `[MASTER] 语料流:  GET /api/corpus?max=1000&groupStart=0&groupEnd=10&format=ndjson`
+    );
+    console.log(`[MASTER] 对外API: POST /api/chat（前端不会调用，可留作他用）`);
+  });
+}
+
+// 启动所有组和总控 forwarder
+(async () => {
+  await startAllGroups();
+  startMasterForwarder();
+})();
+
+// 进程退出处理
+process.on('SIGINT', () => {
+  console.log('收到退出信号，正在关闭所有工作组进程...');
+  for (const proc of groupProcesses) {
+    try {
+      proc.kill();
+    } catch (e) {}
+  }
+  process.exit(0);
+});