079project 3.0.0 → 5.0.0

package/GroupStarter.cjs

@@ -99,6 +99,196 @@ const metrics = {
   gpu: [], // 简化 [{ util, memUsed, memTotal }]
 };
 
+// 记录所有 group 的“模型分片”状态
+// shardId 就是 groupId（0,1,2,...），后续可以映射到 notes.txt 里的 ModelShard 概念
+
+// --- 轻量文本清洗与 hash-embedding，用 group 的语料生成一个语义中心 ---
+function basicClean(s) {
+  if (!s) return '';
+  let t = String(s)
+    .replace(/https?:\/\/[\w\-._~:/?#[\]@!$&'()*+,;=%]+/gi, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+  if (t.length > 512) t = t.slice(0, 512);
+  return t;
+}
+
+function hashStrSimple(str, seed = 1315423911) {
+  let h = seed >>> 0;
+  for (let i = 0; i < str.length; i++) {
+    h ^= ((h << 5) + str.charCodeAt(i) + (h >>> 2)) >>> 0;
+  }
+  return h >>> 0;
+}
+
+function textToMiniEmbedding(text, dim = 64) {
+  const vec = new Float32Array(dim);
+  const toks = basicClean(text).toLowerCase().split(/[^a-z0-9_\-\u4e00-\u9fa5]+/).filter(Boolean);
+  if (!toks.length) return Array.from(vec);
+  for (const t of toks) {
+    const h = hashStrSimple(t);
+    const idx = h % dim;
+    vec[idx] += 1;
+  }
+  let n2 = 0;
+  for (let i = 0; i < dim; i++) n2 += vec[i] * vec[i];
+  n2 = Math.sqrt(n2) || 1;
+  for (let i = 0; i < dim; i++) vec[i] /= n2;
+  return Array.from(vec);
+}
+
+function averageEmbedding(vectors, dim = 64) {
+  if (!vectors || !vectors.length) return Array.from(new Float32Array(dim));
+  const acc = new Float32Array(dim);
+  for (const v of vectors) {
+    const arr = Array.isArray(v) ? v : [];
+    for (let i = 0; i < dim && i < arr.length; i++) acc[i] += arr[i];
+  }
+  for (let i = 0; i < dim; i++) acc[i] /= vectors.length;
+  return Array.from(acc);
+}
+
+// 每组用 robots/*.txt 采样若干行，构成其“语料摘要+embedding”
+function sampleGroupCorpusEmbedding(groupDir, { maxFiles = 16, maxLines = 200, dim = 64 } = {}) {
+  const robotsDir = path.join(groupDir, 'robots');
+  if (!fs.existsSync(robotsDir)) return { dim, centroid: Array.from(new Float32Array(dim)), lines: [] };
+
+  const files = fs.readdirSync(robotsDir).filter(f => f.toLowerCase().endsWith('.txt')).slice(0, maxFiles);
+  const lines = [];
+  for (const f of files) {
+    if (lines.length >= maxLines) break;
+    const full = path.join(robotsDir, f);
+    try {
+      const content = fs.readFileSync(full, 'utf-8');
+      const arr = content.split(/\r?\n/).map(s => s.trim()).filter(Boolean);
+      for (const ln of arr) {
+        if (lines.length >= maxLines) break;
+        if (ln.length < 4) continue;
+        lines.push(ln);
+      }
+    } catch (_) {}
+  }
+  const vecs = lines.map(ln => textToMiniEmbedding(ln, dim));
+  const centroid = averageEmbedding(vecs, dim);
+  return { dim, centroid, lines };
+}
+// 记录每个 group 的“模型分片”状态
+// shardId 就是 groupId（0,1,2,...），后续可以映射到 notes.txt 里的 ModelShard 概念
+const shardMeta = new Map(); // shardId -> { id, groupId, type, tags, state, createdAt, readyAt, warmScore, memHistory, centroid }
+
+// 初始化时创建空 meta，后续由监控填充
+function initShardMetaForGroup(groupId, type = 'text') {
+  if (!shardMeta.has(groupId)) {
+    shardMeta.set(groupId, {
+      id: `group_${groupId}`,
+      groupId,
+      type,                     // 'text' | 'image' | 'audio' 等
+      tags: [],                 // 领域标签（后续可以从 robots/ 目录名或配置推断）
+      state: 'booting',         // booting -> training -> sealed -> warm/cold
+      createdAt: Date.now(),
+      readyAt: null,
+      warmScore: 0,             // 使用/访问次数等加权得到
+      memHistory: [],           // 最近若干次采样的内存用量（MB）
+      centroid: null            // 语义中心向量，下一轮从 Study/Serve 导入
+    });
+  }
+  return shardMeta.get(groupId);
+}
+function startResourceMonitor() {
+  tryAllocEmergencyBuffer();
+  setInterval(async () => {
+    const cpu = sampleCpu();
+    pushMetric('cpu', cpu);
+    const { usedRatio } = sampleMem();
+    pushMetric('mem', usedRatio);
+
+    // === 新增：按 group 聚合 RSS（MB），用于判断训练/封存时机 ===
+    const perGroupMem = new Map(); // groupId -> { rss: number, procs: number }
+    for (const reg of groupRegistry) {
+      let totalRss = 0;
+      let count = 0;
+      const allPids = [
+        ...(reg.servePorts || []).map(s => s.pid).filter(Boolean),
+        reg.study?.pid,
+        reg.forwarder?.pid
+      ].filter(Boolean);
+      for (const pid of allPids) {
+        try {
+          const stat = process.platform === 'win32'
+            ? null
+            : fs.readFileSync(`/proc/${pid}/statm`, 'utf-8'); // Linux 下可用；Windows 下暂忽略
+          if (stat) {
+            const parts = stat.trim().split(/\s+/);
+            const rssPages = Number(parts[1] || 0);
+            const rssBytes = rssPages * 4096;
+            totalRss += rssBytes;
+            count++;
+          }
+        } catch (_) {
+          // 读取失败忽略
+        }
+      }
+      if (count > 0) {
+        perGroupMem.set(reg.id, { rss: totalRss / (1024 * 1024), procs: count }); // MB
+      }
+    }
+
+    // 更新 shardMeta.memHistory 并尝试检测“内存急剧下降”事件
+    for (const [gid, info] of perGroupMem.entries()) {
+      const meta = initShardMetaForGroup(gid, 'text');
+      meta.memHistory.push(info.rss);
+      if (meta.memHistory.length > 32) meta.memHistory.shift();
+
+      // 当 state 仍在 booting/training 时，检测“急剧下降”
+      if (meta.state === 'booting' || meta.state === 'training') {
+        // 简单判断：过去 N=10 个点的最大值和当前值的差，如果超过阈值则认为完成封存
+        const N = 10;
+        if (meta.memHistory.length >= N) {
+          const recent = meta.memHistory.slice(-N);
+          const maxRecent = Math.max(...recent);
+          const nowMem = recent[recent.length - 1];
+          // 阈值：下降超过 30% 且绝对下降 > 200MB，视为一次“完成训练/快照刷盘”
+          if (maxRecent > 0 && (maxRecent - nowMem) / maxRecent >= 0.3 && (maxRecent - nowMem) >= 200) {
+            if (!meta.readyAt) {
+              meta.readyAt = Date.now();
+              meta.state = 'sealed'; // 训练完成并封存（可进入冷池）
+              console.log(`[SHARD] group_${gid} 训练完成并封存：内存峰值约 ${maxRecent.toFixed(1)}MB -> ${nowMem.toFixed(1)}MB`);
+            }
+          } else {
+            // 尚未明显下降，认为仍在训练/构建
+            if (meta.state === 'booting') meta.state = 'training';
+          }
+        }
+      }
+
+      // 对已封存但仍在线的 shard，根据使用次数/最近访问情况更新 warmScore，后面给API用
+      // 这里只是为后续预留；当前版本 warmScore 仅简单随时间衰减
+      if (meta.state === 'sealed' || meta.state === 'warm') {
+        const decay = 0.99;
+        meta.warmScore = meta.warmScore * decay;
+      }
+    }
+
+    // 每30秒采样一次磁盘与GPU
+    if (metrics.ts.length % 30 === 0) {
+      try {
+        metrics.disk = await sampleDiskOnce();
+      } catch (_) {}
+      try {
+        metrics.gpu = await sampleGpuOnce();
+      } catch (_) {}
+    }
+
+    // 自适应：高内存时进入应急模式，释放预留；恢复后重新预留
+    if (usedRatio >= 0.92 && !swapActive) {
+      swapActive = true;
+      releaseEmergencyBuffer();
+    } else if (swapActive && usedRatio <= 0.8) {
+      swapActive = false;
+      tryAllocEmergencyBuffer();
+    }
+  }, 1000);
+}
 let lastCpuSample = os.cpus();
 let emergencyBuffer = null; // 应急释放缓冲，避免立刻 OOM
 let swapActive = false; // 是否处于“内存应急”状态
@@ -206,40 +396,81 @@ function releaseEmergencyBuffer() {
   }
 }
 
-function startResourceMonitor() {
-  tryAllocEmergencyBuffer();
-  setInterval(async () => {
-    const cpu = sampleCpu();
-    pushMetric('cpu', cpu);
-    const { usedRatio } = sampleMem();
-    pushMetric('mem', usedRatio);
-    // 每30秒采样一次磁盘与GPU
-    if (metrics.ts.length % 30 === 0) {
-      try {
-        metrics.disk = await sampleDiskOnce();
-      } catch (_) {}
-      try {
-        metrics.gpu = await sampleGpuOnce();
-      } catch (_) {}
+
+
+async function waitGroupSealed(groupId, { timeoutMs = 60 * 60 * 1000, pollMs = 5000 } = {}) {
+  const start = Date.now();
+  for (;;) {
+    const meta = shardMeta.get(groupId);
+    if (meta && meta.state === 'sealed') {
+      console.log(`[TRAIN] group_${groupId} 已检测到 sealed 状态`);
+      return true;
+    }
+    if (Date.now() - start > timeoutMs) {
+      console.warn(`[TRAIN] group_${groupId} 等待 sealed 超时 (${timeoutMs}ms)`); 
+      return false;
     }
+    await new Promise(r => setTimeout(r, pollMs));
+  }
+}
+// 封存后杀死该组所有子进程
+function killGroupProcesses(reg) {
+  const all = [];
+  for (const sp of (reg.servePorts || [])) if (sp.pid) all.push(sp.pid);
+  if (reg.study?.pid) all.push(reg.study.pid);
+  if (reg.forwarder?.pid) all.push(reg.forwarder.pid);
+  for (const pid of all) {
+    try { process.kill(pid); } catch (_) {}
+  }
+  console.log(`[TRAIN] group_${reg.id} 所有子进程已终止（进入冷态）`);
+}
 
-    // 自适应：高内存时进入应急模式，释放预留；恢复后重新预留
-    if (usedRatio >= 0.92 && !swapActive) {
-      swapActive = true;
-      releaseEmergencyBuffer();
-    } else if (swapActive && usedRatio <= 0.8) {
-      swapActive = false;
-      tryAllocEmergencyBuffer();
+// 调用该 group forwarder 的 /api/snapshot/create ，再 /api/snapshot/list 做一次硬盘快照
+async function snapshotGroup(groupId, forwarderPort) {
+  try {
+    const apiBase = `http://localhost:${forwarderPort}`;
+    // 这里假设 serve/main_* 那边已经暴露 snapshot API（你 main_Serve.cjs 里有）
+    // 对于 group forwarder，我们只要简单调用 /api/snapshots/create 即可，如果你以后加的话。
+    // 当前版本没有 forwarder 级 snapshot，这里调用 serve 主实例的 snapshot。
+    console.log(`[TRAIN] group_${groupId} 尝试触发快照（通过任一 serve 实例）`);
+
+    // 读 group_ports.json，拿一个 serve port
+    const reg = groupRegistry[groupId];
+    const firstServe = reg?.servePorts?.[0]?.port;
+    if (!firstServe) {
+      console.warn(`[TRAIN] group_${groupId} 无 serve 端口，跳过快照`);
+      return false;
     }
-  }, 1000);
+    const serveApiBase = `http://localhost:${firstServe}`;
+    await axios.post(`${serveApiBase}/api/snapshot/create`, { name: 'auto-train-complete' }, { timeout: 600_000 }).catch(() => {});
+    console.log(`[TRAIN] group_${groupId} 快照请求已发送到 serve@${firstServe}`);
+    return true;
+  } catch (e) {
+    console.warn(`[TRAIN] group_${groupId} 快照失败:`, e.message);
+    return false;
+  }
 }
 
+// 封存后杀死该组所有子进程
+function killGroupProcesses(reg) {
+  const all = [];
+  for (const sp of (reg.servePorts || [])) if (sp.pid) all.push(sp.pid);
+  if (reg.study?.pid) all.push(reg.study.pid);
+  if (reg.forwarder?.pid) all.push(reg.forwarder.pid);
+  for (const pid of all) {
+    try { process.kill(pid); } catch (_) {}
+  }
+  console.log(`[TRAIN] group_${reg.id} 所有子进程已终止（进入冷态）`);
+}
 
 async function startAllGroups() {
   startLogger();
   for (let i = 0; i < groupFolders.length; i++) {
     const groupDir = groupFolders[i];
     const reg = { id: i, dir: groupDir, servePorts: [], study: null, forwarder: null };
+    // 初始化 shard 元信息（类型按 GROUPS_DIR 推断）
+    const type = GROUPS_DIR.includes('image') ? 'image' : GROUPS_DIR.includes('audio') ? 'audio' : 'text';
+    initShardMetaForGroup(i, type);
 
     // 预计算本组端口
     const groupServePorts = Array.from({ length: SERVE_COUNT }, (_, j) => PORT_BASE + i * (SERVE_COUNT + 2) + j);
@@ -268,7 +499,6 @@ async function startAllGroups() {
               cwd: groupDir,
               stdio: 'inherit',
             });
-            // 注册到日志子进程
             portIndex.set(port, { groupId: i, role: 'serve' });
             loggerSend({ type: 'register', processes: [{ groupId: i, role: 'serve', port, pid: null }] });
             groupProcesses.push(proc);
@@ -289,7 +519,7 @@ async function startAllGroups() {
       }
     }
 
-    // study 进程（保持不变）
+    // Study
     if (fs.existsSync(path.join(groupDir, 'main_Study.cjs'))) {
       const proc = spawn(
         'node',
@@ -305,8 +535,8 @@ async function startAllGroups() {
         loggerSend({ type: 'update', port: studyPort, pid: proc.pid, role: 'study', groupId: i });
       });
     }
-
-    // forwarder 进程（保持不变）
+    
+    // Forwarder
     groupForwarderPorts.push(forwarderPort);
     const forwarderScript = path.join(groupDir, 'forwarder.js');
     if (fs.existsSync(forwarderScript)) {
@@ -331,14 +561,132 @@ async function startAllGroups() {
       });
     }
 
-    // 写入组配置文件
     const config = { servePorts, studyPort, forwarderPort };
     fs.writeFileSync(path.join(groupDir, 'group_ports.json'), JSON.stringify(config, null, 2));
     console.log(`[START] group_${i} 端口分配: serve=${servePorts.join(',')}, study=${studyPort}, forwarder=${forwarderPort}`);
     groupRegistry.push(reg);
+
+    // === 关键：串行训练 + 封存 + 冷杀 ===
+    console.log(`[TRAIN] 等待 group_${i} 完成首次训练/封存...`);
+    await waitGroupSealed(i, { timeoutMs: 60 * 60 * 1000, pollMs: 5000 });
+
+    // 生成语义 embedding + 标签，并写入 shard_meta.json
+    const meta = shardMeta.get(i) || initShardMetaForGroup(i, type);
+    const { dim, centroid, lines } = sampleGroupCorpusEmbedding(groupDir, { maxFiles: 16, maxLines: 200, dim: 64 });
+    meta.type = type;
+    meta.tags = meta.tags && meta.tags.length ? meta.tags : [type];
+    meta.embedding = centroid;
+    meta.sampleLines = lines.slice(0, 16);
+    try {
+      fs.writeFileSync(
+        path.join(groupDir, 'shard_meta.json'),
+        JSON.stringify({
+          id: meta.id,
+          groupId: meta.groupId,
+          type: meta.type,
+          tags: meta.tags,
+          dim,
+          centroid,
+          createdAt: meta.createdAt,
+          readyAt: meta.readyAt
+        }, null, 2),
+        'utf-8'
+      );
+      console.log(`[SHARD] 已写入 ${groupDir}/shard_meta.json`);
+    } catch (e) {
+      console.warn(`[SHARD] 写入 shard_meta.json 失败:`, e.message);
+    }
+
+    // 触发一次快照（使训练结果落盘）
+    await snapshotGroup(i, forwarderPort);
+
+    // 冷杀该组所有进程，避免同时训练引发 OOM
+    killGroupProcesses(reg);
+
+    // 小间隔再启动下一组，避免端口释放延迟
     await new Promise((r) => setTimeout(r, GROUP_START_DELAY));
   }
-  console.log(`[INFO] 共启动 ${groupFolders.length} 个 group`);
+  console.log(`[INFO] 所有 group 已串行训练并封存，接下来将统一冷启动`);
+
+  // === 统一冷启动：把所有 group 再启动一次，此时不再等待 sealed，直接作为推理实例运行 ===
+  for (let i = 0; i < groupFolders.length; i++) {
+    const groupDir = groupFolders[i];
+    const reg = groupRegistry[i];
+    const portsConf = JSON.parse(fs.readFileSync(path.join(groupDir, 'group_ports.json'), 'utf-8'));
+    const servePorts = portsConf.servePorts || [];
+    const studyPort = portsConf.studyPort;
+    const forwarderPort = portsConf.forwarderPort;
+
+    // Serve
+    reg.servePorts = [];
+    for (const port of servePorts) {
+      const serveScript = path.join(groupDir, 'main_Serve.cjs');
+      if (!fs.existsSync(serveScript)) continue;
+      const peers = servePorts.filter(p => p !== port).join(',');
+      const proc = spawn('node', [
+        serveScript,
+        String(port),
+        '--expose-gc',
+        '--group-id', String(i),
+        '--forwarder-port', String(forwarderPort),
+        '--study-port', String(studyPort),
+        '--peers', peers
+      ], { cwd: groupDir, stdio: 'inherit' });
+      portIndex.set(port, { groupId: i, role: 'serve' });
+      loggerSend({ type: 'register', processes: [{ groupId: i, role: 'serve', port, pid: null }] });
+      groupProcesses.push(proc);
+      reg.servePorts.push({ pid: null, port });
+      proc.on('spawn', () => {
+        const idx = reg.servePorts.findIndex((p) => p.port === port);
+        if (idx >= 0) reg.servePorts[idx].pid = proc.pid;
+        loggerSend({ type: 'update', port, pid: proc.pid, role: 'serve', groupId: i });
+      });
+    }
+
+    // Study
+    if (fs.existsSync(path.join(groupDir, 'main_Study.cjs'))) {
+      const proc = spawn(
+        'node',
+        [path.join(groupDir, 'main_Study.cjs'), String(studyPort), '--max-old-space-size=16384', '--expose-gc'],
+        { cwd: groupDir, stdio: 'inherit' }
+      );
+      portIndex.set(studyPort, { groupId: i, role: 'study' });
+      loggerSend({ type: 'register', processes: [{ groupId: i, role: 'study', port: studyPort, pid: null }] });
+      groupProcesses.push(proc);
+      reg.study = { pid: null, port: studyPort };
+      proc.on('spawn', () => {
+        reg.study.pid = proc.pid;
+        loggerSend({ type: 'update', port: studyPort, pid: proc.pid, role: 'study', groupId: i });
+      });
+    }
+        // Forwarder
+    if (fs.existsSync(path.join(groupDir, 'forwarder.js'))) {
+      const proc = spawn(
+        'node',
+        [
+          path.join(groupDir, 'forwarder.js'),
+          String(forwarderPort),
+          ...servePorts.map(p => String(p)),
+          String(studyPort),
+          '--expose-gc'
+        ],
+        { cwd: groupDir, stdio: 'inherit' }
+      );
+      portIndex.set(forwarderPort, { groupId: i, role: 'forwarder' });
+      loggerSend({ type: 'register', processes: [{ groupId: i, role: 'forwarder', port: forwarderPort, pid: null }] });
+      groupProcesses.push(proc);
+      reg.forwarder = { pid: null, port: forwarderPort };
+      proc.on('spawn', () => {
+        reg.forwarder.pid = proc.pid;
+        loggerSend({ type: 'update', port: forwarderPort, pid: proc.pid, role: 'forwarder', groupId: i });
+      });
+    }
+
+    console.log(`[COLD-START] group_${i} 已从封存状态冷启动: serve=${servePorts.join(',')}, study=${studyPort}, forwarder=${forwarderPort}`);
+    await new Promise((r) => setTimeout(r, GROUP_START_DELAY));
+  }
+
+  console.log(`[INFO] 共启动 ${groupFolders.length} 个 group（冷启动完成）`);
 }
 
 function startMasterForwarder() {
@@ -689,7 +1037,24 @@ function startMasterForwarder() {
   app.get('/monitor', (req, res) => {
     res.sendFile(path.join(__dirname, 'public', 'monitor.html'));
   });
-
+app.get('/api/shards', (req, res) => {
+  const out = [];
+  for (const [gid, meta] of shardMeta.entries()) {
+    out.push({
+      id: meta.id,
+      groupId: gid,
+      type: meta.type,
+      tags: meta.tags,
+      state: meta.state,
+      createdAt: meta.createdAt,
+      readyAt: meta.readyAt,
+      warmScore: meta.warmScore,
+      memHistory: meta.memHistory.slice(-8),
+      embedding: meta.embedding || null
+    });
+  }
+  res.json({ ok: true, shards: out });
+});
   // 启动服务
   app.listen(MASTER_PORT, () => {
     console.log(`[MASTER] GroupStarter master forwarder已启动，端口 ${MASTER_PORT}`);
@@ -699,6 +1064,7 @@ function startMasterForwarder() {
     );
     console.log(`[MASTER] 对外API: POST /api/chat（前端不会调用，可留作他用）`);
   });
+  
 }
 // 僵尸端口检测 + 自动重启
 async function probePort(port) {