079project 1.0.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/GroupStarter.cjs +211 -35
- package/README.md +3 -3
- package/crawler/agent.cjs +97 -0
- package/crawler/index.cjs +515 -0
- package/crawler/storage.cjs +163 -0
- package/groupmanager.cjs +2 -1
- package/loggerworker.cjs +202 -0
- package/main_Serve.cjs +1132 -115
- package/main_Study.cjs +1749 -365
- package/package.json +2 -1
- package/robots/seeds.txt +2 -0
- package/schedule.cjs +745 -0
- package/wikitext/wikitext-103-all.txt +0 -0
- package/wikitext/.gitattributes +0 -27
- package/wikitext/README.md +0 -344
- package/wikitext/describtion.txt +0 -1
package/main_Serve.cjs
CHANGED
|
@@ -18,6 +18,7 @@ const csvParse = require('csv-parse/sync')
|
|
|
18
18
|
const os = require('os');
|
|
19
19
|
const MAX_WORKERS = Math.max(1, os.cpus().length - 1); // 留一个核心给主线程
|
|
20
20
|
const natural = require('natural');
|
|
21
|
+
const { AdversaryScheduler } = require('./schedule.cjs');
|
|
21
22
|
const STOP_WORDS = natural.stopwords; // 英文停用词
|
|
22
23
|
const pool = workerpool.pool(path.join(__dirname, 'memeMergeWorker.cjs'), {
|
|
23
24
|
minWorkers: 1,
|
|
@@ -31,11 +32,28 @@ protobuf.load(runtimeProtoPath, (err, root) => {
|
|
|
31
32
|
if (err) throw err;
|
|
32
33
|
RuntimeMessage = root.lookupType('Runtime');
|
|
33
34
|
});
|
|
34
|
-
|
|
35
|
+
function parseArgs(argv) {
|
|
36
|
+
const out = {};
|
|
37
|
+
for (let i = 0; i < argv.length; i++) {
|
|
38
|
+
const a = argv[i];
|
|
39
|
+
if (a && a.startsWith('--')) {
|
|
40
|
+
const k = a.slice(2);
|
|
41
|
+
const v = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : true;
|
|
42
|
+
out[k] = v;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return out;
|
|
46
|
+
}
|
|
47
|
+
const __args = parseArgs(process.argv.slice(2));
|
|
35
48
|
global.config = {
|
|
36
49
|
masterPortOfSql: 3125,
|
|
37
50
|
masterPortOfMain: process.argv[2],
|
|
38
|
-
emitExitport: process.argv[3] || 8641
|
|
51
|
+
emitExitport: process.argv[3] || 8641,
|
|
52
|
+
groupId: Number(__args['group-id'] || -1),
|
|
53
|
+
forwarderPort: Number(__args['forwarder-port'] || 0),
|
|
54
|
+
studyPort: Number(__args['study-port'] || 0),
|
|
55
|
+
peerServePorts: String(__args['peers'] || '').split(',').map(s => Number(s)).filter(n => Number.isFinite(n) && n > 0),
|
|
56
|
+
isStudy: !!__args['study']
|
|
39
57
|
};
|
|
40
58
|
const modelDefaults = {
|
|
41
59
|
decayFactor: 0.5,
|
|
@@ -248,8 +266,8 @@ class SnapshotManager {
|
|
|
248
266
|
}
|
|
249
267
|
}
|
|
250
268
|
|
|
269
|
+
|
|
251
270
|
async createSnapshot(name = 'auto') {
|
|
252
|
-
// 防止并发创建
|
|
253
271
|
if (this.isCreatingSnapshot) {
|
|
254
272
|
console.log('[SNAPSHOT] 另一个快照正在创建中,跳过');
|
|
255
273
|
return null;
|
|
@@ -265,31 +283,39 @@ class SnapshotManager {
|
|
|
265
283
|
|
|
266
284
|
console.log(`[SNAPSHOT] 开始创建快照: ${snapshotId}`);
|
|
267
285
|
|
|
268
|
-
//
|
|
286
|
+
// 优先使用分区图的全量导出(避免仅导出窗口)
|
|
287
|
+
let memesAll = [];
|
|
288
|
+
if (this.runtime.graph && typeof this.runtime.graph.exportAllPoints === 'function') {
|
|
289
|
+
try {
|
|
290
|
+
memesAll = await this.runtime.graph.exportAllPoints();
|
|
291
|
+
} catch (e) {
|
|
292
|
+
console.warn('[SNAPSHOT] 分区图导出失败,回退窗口:', e.message);
|
|
293
|
+
memesAll = this.runtime.graph.getAllPoints();
|
|
294
|
+
}
|
|
295
|
+
} else {
|
|
296
|
+
memesAll = this.runtime.graph.getAllPoints();
|
|
297
|
+
}
|
|
298
|
+
|
|
269
299
|
const snapshotData = {
|
|
270
300
|
id: snapshotId,
|
|
271
301
|
timestamp,
|
|
272
302
|
name,
|
|
273
303
|
createDate: new Date().toISOString(),
|
|
274
|
-
memes:
|
|
304
|
+
memes: memesAll,
|
|
275
305
|
wordGraph: Array.from(this.runtime.wordGraph.points.values()),
|
|
276
|
-
kvm: this.runtime.kvm.
|
|
306
|
+
kvm: Array.from(this.runtime.kvm.memory.entries()),
|
|
277
307
|
vocab: this.runtime.vocabManager.vocab,
|
|
278
|
-
wordAccessLog: Array.from(this.runtime.wordAccessLog
|
|
308
|
+
wordAccessLog: Array.from(this.runtime.wordAccessLog.entries()).map(([w, per]) =>
|
|
309
|
+
[w, per instanceof Map ? Array.from(per.entries()) : (Array.isArray(per) ? [['legacy', per.length]] : [])]
|
|
310
|
+
),
|
|
311
|
+
sessions: this.runtime.session.export()
|
|
279
312
|
};
|
|
280
313
|
|
|
281
|
-
// 写入临时文件,然后原子重命名以确保数据完整性
|
|
282
314
|
const tempPath = `${filePath}.temp`;
|
|
283
315
|
await fs.promises.writeFile(tempPath, JSON.stringify(snapshotData), 'utf-8');
|
|
284
316
|
await fs.promises.rename(tempPath, filePath);
|
|
285
317
|
|
|
286
|
-
|
|
287
|
-
const snapshotInfo = {
|
|
288
|
-
id: snapshotId,
|
|
289
|
-
timestamp,
|
|
290
|
-
name,
|
|
291
|
-
path: filePath
|
|
292
|
-
};
|
|
318
|
+
const snapshotInfo = { id: snapshotId, timestamp, name, path: filePath };
|
|
293
319
|
this.snapshotList.unshift(snapshotInfo);
|
|
294
320
|
|
|
295
321
|
console.timeEnd('snapshotCreation');
|
|
@@ -303,11 +329,11 @@ class SnapshotManager {
|
|
|
303
329
|
}
|
|
304
330
|
}
|
|
305
331
|
|
|
332
|
+
|
|
306
333
|
async restoreSnapshot(snapshotId) {
|
|
307
334
|
console.log(`[SNAPSHOT] 开始从快照恢复: ${snapshotId}`);
|
|
308
335
|
console.time('snapshotRestore');
|
|
309
336
|
|
|
310
|
-
// 查找快照
|
|
311
337
|
const snapshot = this.snapshotList.find(s => s.id === snapshotId);
|
|
312
338
|
if (!snapshot) {
|
|
313
339
|
console.error(`[SNAPSHOT] 快照不存在: ${snapshotId}`);
|
|
@@ -315,37 +341,27 @@ class SnapshotManager {
|
|
|
315
341
|
}
|
|
316
342
|
|
|
317
343
|
try {
|
|
318
|
-
// 读取快照文件
|
|
319
|
-
console.log(`[SNAPSHOT] 从文件读取数据: ${snapshot.path}`);
|
|
320
344
|
const dataStr = await fs.promises.readFile(snapshot.path, 'utf-8');
|
|
321
345
|
const data = JSON.parse(dataStr);
|
|
322
346
|
|
|
323
|
-
// 在恢复前创建自动备份
|
|
324
347
|
await this.createSnapshot(`auto_before_restore_${snapshotId}`);
|
|
325
348
|
|
|
326
|
-
//
|
|
327
|
-
console.log('[SNAPSHOT] 清空当前运行时...');
|
|
328
|
-
this.runtime.graph = new GraphDB();
|
|
349
|
+
// 清空当前运行时(词图/KVM 内存)
|
|
329
350
|
this.runtime.wordGraph = new GraphDB();
|
|
330
351
|
this.runtime.kvm = new KVM();
|
|
331
352
|
this.runtime.wordAccessLog = new Map();
|
|
332
353
|
|
|
333
|
-
//
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
this.runtime.graph.addPoint(point.pointID, point.connect);
|
|
341
|
-
}
|
|
342
|
-
// 让事件循环有机会处理其他事件
|
|
343
|
-
await new Promise(resolve => setImmediate(resolve));
|
|
354
|
+
// 恢复模因图:走分区导入(覆盖分区存储)
|
|
355
|
+
if (data.memes && this.runtime.graph && typeof this.runtime.graph.importAllPoints === 'function') {
|
|
356
|
+
await this.runtime.graph.importAllPoints(data.memes);
|
|
357
|
+
} else if (data.memes) {
|
|
358
|
+
// 窗口回退(不推荐)
|
|
359
|
+
for (const point of data.memes) {
|
|
360
|
+
await this.runtime.graph.addPoint(point.pointID, point.connect);
|
|
344
361
|
}
|
|
345
362
|
}
|
|
346
363
|
|
|
347
364
|
// 恢复词图
|
|
348
|
-
console.log('[SNAPSHOT] 恢复词语网络...');
|
|
349
365
|
if (data.wordGraph) {
|
|
350
366
|
const BATCH_SIZE = 1000;
|
|
351
367
|
for (let i = 0; i < data.wordGraph.length; i += BATCH_SIZE) {
|
|
@@ -358,29 +374,39 @@ class SnapshotManager {
|
|
|
358
374
|
}
|
|
359
375
|
|
|
360
376
|
// 恢复KVM
|
|
361
|
-
console.log('[SNAPSHOT] 恢复键值存储...');
|
|
362
377
|
if (data.kvm) {
|
|
363
378
|
const BATCH_SIZE = 1000;
|
|
364
379
|
for (let i = 0; i < data.kvm.length; i += BATCH_SIZE) {
|
|
365
380
|
const batch = data.kvm.slice(i, i + BATCH_SIZE);
|
|
366
|
-
for (const [k, v] of batch)
|
|
367
|
-
this.runtime.kvm.set(k, v);
|
|
368
|
-
}
|
|
381
|
+
for (const [k, v] of batch) this.runtime.kvm.set(k, v);
|
|
369
382
|
await new Promise(resolve => setImmediate(resolve));
|
|
370
383
|
}
|
|
371
384
|
}
|
|
372
385
|
|
|
373
386
|
// 恢复词表
|
|
374
|
-
console.log('[SNAPSHOT] 恢复词表...');
|
|
375
387
|
if (data.vocab) {
|
|
376
388
|
this.runtime.vocabManager.vocab = data.vocab;
|
|
377
389
|
this.runtime.vocabManager.updateMappings();
|
|
378
390
|
}
|
|
379
391
|
|
|
380
392
|
// 恢复词访问日志
|
|
381
|
-
console.log('[SNAPSHOT] 恢复词访问日志...');
|
|
382
393
|
if (data.wordAccessLog) {
|
|
383
|
-
|
|
394
|
+
const restored = new Map();
|
|
395
|
+
for (const [word, per] of data.wordAccessLog) {
|
|
396
|
+
if (Array.isArray(per) && per.length > 0 && Array.isArray(per[0])) {
|
|
397
|
+
restored.set(word, new Map(per));
|
|
398
|
+
} else if (Array.isArray(per)) {
|
|
399
|
+
restored.set(word, new Map([['legacy', per.length]]));
|
|
400
|
+
} else {
|
|
401
|
+
restored.set(word, new Map());
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
this.runtime.wordAccessLog = restored;
|
|
405
|
+
}
|
|
406
|
+
if (data.sessions) {
|
|
407
|
+
this.runtime.session.import(data.sessions);
|
|
408
|
+
} else {
|
|
409
|
+
this.runtime.session.startNewSession({ reason: 'snapshot-legacy' });
|
|
384
410
|
}
|
|
385
411
|
|
|
386
412
|
console.timeEnd('snapshotRestore');
|
|
@@ -698,6 +724,727 @@ class GraphDB {
|
|
|
698
724
|
}
|
|
699
725
|
}
|
|
700
726
|
}
|
|
727
|
+
// ...existing code...
|
|
728
|
+
const crypto = require('crypto');
|
|
729
|
+
// ...existing code...
|
|
730
|
+
|
|
731
|
+
// ========================== 分区图存储适配层与滑动窗口 ==========================
|
|
732
|
+
|
|
733
|
+
// 简易日志辅助
|
|
734
|
+
function logPart(...args) { console.log('[PART]', ...args); }
|
|
735
|
+
function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
|
|
736
|
+
|
|
737
|
+
// 存储适配层(FS/LMDB/Level 多后端,按需加载)
|
|
738
|
+
class GraphStorageAdapter {
|
|
739
|
+
constructor({ baseDir, backend = 'fs' } = {}) {
|
|
740
|
+
this.baseDir = baseDir || path.join(__dirname, 'graph_parts');
|
|
741
|
+
this.backend = backend;
|
|
742
|
+
this.ready = false;
|
|
743
|
+
|
|
744
|
+
// 尝试创建目录
|
|
745
|
+
fs.mkdirSync(this.baseDir, { recursive: true });
|
|
746
|
+
|
|
747
|
+
// 可选依赖
|
|
748
|
+
this.lmdb = null;
|
|
749
|
+
this.level = null;
|
|
750
|
+
|
|
751
|
+
if (backend === 'lmdb') {
|
|
752
|
+
try {
|
|
753
|
+
this.lmdb = require('lmdb');
|
|
754
|
+
} catch (e) {
|
|
755
|
+
console.warn('[PART][ADAPTER] LMDB 不可用,降级为 FS:', e.message);
|
|
756
|
+
this.backend = 'fs';
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
if (backend === 'level') {
|
|
760
|
+
try {
|
|
761
|
+
this.level = require('level');
|
|
762
|
+
} catch (e) {
|
|
763
|
+
console.warn('[PART][ADAPTER] level 不可用,降级为 FS:', e.message);
|
|
764
|
+
this.backend = 'fs';
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
// 初始化后端
|
|
769
|
+
this._initBackend();
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
_initBackend() {
|
|
773
|
+
if (this.backend === 'fs') {
|
|
774
|
+
// FS: 每个分区一个 .jsonl(节点),边界事件一个独立 .jsonl
|
|
775
|
+
this.ready = true;
|
|
776
|
+
return;
|
|
777
|
+
}
|
|
778
|
+
if (this.backend === 'lmdb' && this.lmdb) {
|
|
779
|
+
try {
|
|
780
|
+
const storeDir = path.join(this.baseDir, 'lmdb');
|
|
781
|
+
fs.mkdirSync(storeDir, { recursive: true });
|
|
782
|
+
this.env = this.lmdb.open({
|
|
783
|
+
path: storeDir,
|
|
784
|
+
mapSize: 1024n * 1024n * 1024n * 64n,
|
|
785
|
+
compression: true,
|
|
786
|
+
});
|
|
787
|
+
this.ready = true;
|
|
788
|
+
} catch (e) {
|
|
789
|
+
console.warn('[PART][ADAPTER] LMDB 初始化失败,降级 FS:', e.message);
|
|
790
|
+
this.backend = 'fs';
|
|
791
|
+
this.ready = true;
|
|
792
|
+
}
|
|
793
|
+
return;
|
|
794
|
+
}
|
|
795
|
+
if (this.backend === 'level' && this.level) {
|
|
796
|
+
try {
|
|
797
|
+
const dbDir = path.join(this.baseDir, 'leveldb');
|
|
798
|
+
fs.mkdirSync(dbDir, { recursive: true });
|
|
799
|
+
this.db = new this.level.Level(dbDir, { valueEncoding: 'json' });
|
|
800
|
+
this.ready = true;
|
|
801
|
+
} catch (e) {
|
|
802
|
+
console.warn('[PART][ADAPTER] level 初始化失败,降级 FS:', e.message);
|
|
803
|
+
this.backend = 'fs';
|
|
804
|
+
this.ready = true;
|
|
805
|
+
}
|
|
806
|
+
return;
|
|
807
|
+
}
|
|
808
|
+
this.ready = true;
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
// 分区文件名(FS)
|
|
812
|
+
_partFile(pid) { return path.join(this.baseDir, `p_${pid}.jsonl`); }
|
|
813
|
+
_eventFile(pid) { return path.join(this.baseDir, `p_${pid}.events.jsonl`); }
|
|
814
|
+
|
|
815
|
+
// 读取分区(返回 { points: Map<string,{pointID,connect:[]}> })
|
|
816
|
+
async loadPartition(pid) {
|
|
817
|
+
if (this.backend === 'fs') {
|
|
818
|
+
const file = this._partFile(pid);
|
|
819
|
+
const out = new Map();
|
|
820
|
+
if (!fs.existsSync(file)) return { points: out };
|
|
821
|
+
const rs = fs.createReadStream(file, { encoding: 'utf-8' });
|
|
822
|
+
let buf = '';
|
|
823
|
+
for await (const chunk of rs) {
|
|
824
|
+
buf += chunk;
|
|
825
|
+
let idx;
|
|
826
|
+
while ((idx = buf.indexOf('\n')) >= 0) {
|
|
827
|
+
const line = buf.slice(0, idx);
|
|
828
|
+
buf = buf.slice(idx + 1);
|
|
829
|
+
if (!line.trim()) continue;
|
|
830
|
+
try {
|
|
831
|
+
const obj = JSON.parse(line);
|
|
832
|
+
if (obj && obj.pointID) {
|
|
833
|
+
out.set(obj.pointID, { pointID: obj.pointID, connect: obj.connect || [] });
|
|
834
|
+
}
|
|
835
|
+
} catch { /* ignore */ }
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
return { points: out };
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
if (this.backend === 'lmdb' && this.env) {
|
|
842
|
+
const points = new Map();
|
|
843
|
+
const txn = this.env.beginTxn({ readOnly: true });
|
|
844
|
+
try {
|
|
845
|
+
const cursor = new this.lmdb.Cursors.Cursor(txn, this.env.openDB({ name: `p_${pid}`, create: true }));
|
|
846
|
+
for (let found = cursor.goToFirst(); found; found = cursor.goToNext()) {
|
|
847
|
+
const key = cursor.getCurrentString();
|
|
848
|
+
const val = cursor.getCurrentBinary();
|
|
849
|
+
try {
|
|
850
|
+
const obj = JSON.parse(Buffer.from(val).toString('utf-8'));
|
|
851
|
+
if (obj && obj.pointID) points.set(obj.pointID, obj);
|
|
852
|
+
} catch { }
|
|
853
|
+
}
|
|
854
|
+
cursor.close();
|
|
855
|
+
} catch { }
|
|
856
|
+
txn.abort();
|
|
857
|
+
return { points };
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
if (this.backend === 'level' && this.db) {
|
|
861
|
+
const points = new Map();
|
|
862
|
+
try {
|
|
863
|
+
for await (const { key, value } of this.db.iterator({ gte: `p:${pid}:`, lt: `p:${pid};` })) {
|
|
864
|
+
const obj = value;
|
|
865
|
+
if (obj && obj.pointID) points.set(obj.pointID, obj);
|
|
866
|
+
}
|
|
867
|
+
} catch { }
|
|
868
|
+
return { points };
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
return { points: new Map() };
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
// 保存分区(全量覆盖写)
|
|
875
|
+
async savePartition(pid, pointsMap) {
|
|
876
|
+
if (!(pointsMap instanceof Map)) return;
|
|
877
|
+
if (this.backend === 'fs') {
|
|
878
|
+
const file = this._partFile(pid);
|
|
879
|
+
const tmp = `${file}.tmp`;
|
|
880
|
+
const ws = fs.createWriteStream(tmp, { encoding: 'utf-8' });
|
|
881
|
+
for (const [, p] of pointsMap.entries()) {
|
|
882
|
+
ws.write(JSON.stringify({ pointID: p.pointID, connect: p.connect || [] }) + '\n');
|
|
883
|
+
}
|
|
884
|
+
await new Promise((res, rej) => ws.end(res));
|
|
885
|
+
await fs.promises.rename(tmp, file);
|
|
886
|
+
return;
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
if (this.backend === 'lmdb' && this.env) {
|
|
890
|
+
const dbi = this.env.openDB({ name: `p_${pid}`, create: true });
|
|
891
|
+
const txn = this.env.beginTxn();
|
|
892
|
+
try {
|
|
893
|
+
// 先清空:简化实现
|
|
894
|
+
const cur = new this.lmdb.Cursors.Cursor(txn, dbi);
|
|
895
|
+
for (let found = cur.goToFirst(); found; found = cur.goToNext()) {
|
|
896
|
+
const k = cur.getCurrentString();
|
|
897
|
+
txn.del(dbi, k);
|
|
898
|
+
}
|
|
899
|
+
cur.close();
|
|
900
|
+
for (const [, p] of pointsMap.entries()) {
|
|
901
|
+
txn.put(dbi, p.pointID, JSON.stringify(p));
|
|
902
|
+
}
|
|
903
|
+
txn.commit();
|
|
904
|
+
} catch (e) {
|
|
905
|
+
try { txn.abort(); } catch { }
|
|
906
|
+
console.warn('[PART][ADAPTER][LMDB] savePartition err:', e.message);
|
|
907
|
+
}
|
|
908
|
+
return;
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
if (this.backend === 'level' && this.db) {
|
|
912
|
+
const ops = [];
|
|
913
|
+
// 简化:清理旧 key 不容易,直接覆盖同 key
|
|
914
|
+
for (const [, p] of pointsMap.entries()) {
|
|
915
|
+
ops.push({ type: 'put', key: `p:${pid}:${p.pointID}`, value: p });
|
|
916
|
+
}
|
|
917
|
+
await this.db.batch(ops);
|
|
918
|
+
return;
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
// 追加边界事件(跨分区边)
|
|
923
|
+
async appendEdgeEvent(pid, event) {
|
|
924
|
+
if (!event || !event.type) return;
|
|
925
|
+
if (this.backend === 'fs') {
|
|
926
|
+
const file = this._eventFile(pid);
|
|
927
|
+
fs.appendFileSync(file, JSON.stringify(event) + '\n', 'utf-8');
|
|
928
|
+
return;
|
|
929
|
+
}
|
|
930
|
+
if (this.backend === 'lmdb' && this.env) {
|
|
931
|
+
const dbi = this.env.openDB({ name: `e_${pid}`, create: true });
|
|
932
|
+
const txn = this.env.beginTxn();
|
|
933
|
+
try {
|
|
934
|
+
const key = `e:${Date.now()}_${Math.random().toString(36).slice(2, 10)}`;
|
|
935
|
+
txn.put(dbi, key, JSON.stringify(event));
|
|
936
|
+
txn.commit();
|
|
937
|
+
} catch (e) {
|
|
938
|
+
try { txn.abort(); } catch { }
|
|
939
|
+
}
|
|
940
|
+
return;
|
|
941
|
+
}
|
|
942
|
+
if (this.backend === 'level' && this.db) {
|
|
943
|
+
const key = `e:${pid}:${Date.now()}_${Math.random().toString(36).slice(2, 10)}`;
|
|
944
|
+
await this.db.put(key, event);
|
|
945
|
+
return;
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
// 读取并消费边界事件(与该分区相关的)
|
|
950
|
+
async consumeEdgeEvents(pid, filterFn = null, limit = 2000) {
|
|
951
|
+
const events = [];
|
|
952
|
+
if (this.backend === 'fs') {
|
|
953
|
+
const file = this._eventFile(pid);
|
|
954
|
+
if (!fs.existsSync(file)) return events;
|
|
955
|
+
|
|
956
|
+
const tmp = `${file}.tmp`;
|
|
957
|
+
// 将不消费的事件写入 tmp,再覆盖原文件;已消费事件返回
|
|
958
|
+
const lines = fs.readFileSync(file, 'utf-8').split(/\r?\n/).filter(Boolean);
|
|
959
|
+
const remain = [];
|
|
960
|
+
for (const line of lines) {
|
|
961
|
+
try {
|
|
962
|
+
const e = JSON.parse(line);
|
|
963
|
+
const ok = filterFn ? filterFn(e) : true;
|
|
964
|
+
if (ok && events.length < limit) {
|
|
965
|
+
events.push(e);
|
|
966
|
+
} else {
|
|
967
|
+
remain.push(line);
|
|
968
|
+
}
|
|
969
|
+
} catch {
|
|
970
|
+
remain.push(line);
|
|
971
|
+
}
|
|
972
|
+
}
|
|
973
|
+
fs.writeFileSync(tmp, remain.join('\n') + (remain.length ? '\n' : ''), 'utf-8');
|
|
974
|
+
await fs.promises.rename(tmp, file);
|
|
975
|
+
return events;
|
|
976
|
+
}
|
|
977
|
+
|
|
978
|
+
if (this.backend === 'lmdb' && this.env) {
|
|
979
|
+
const dbi = this.env.openDB({ name: `e_${pid}`, create: true });
|
|
980
|
+
const txn = this.env.beginTxn();
|
|
981
|
+
const toDel = [];
|
|
982
|
+
try {
|
|
983
|
+
const cur = new this.lmdb.Cursors.Cursor(txn, dbi);
|
|
984
|
+
for (let found = cur.goToFirst(); found; found = cur.goToNext()) {
|
|
985
|
+
const k = cur.getCurrentString();
|
|
986
|
+
const v = cur.getCurrentBinary();
|
|
987
|
+
const e = JSON.parse(Buffer.from(v).toString('utf-8'));
|
|
988
|
+
const ok = filterFn ? filterFn(e) : true;
|
|
989
|
+
if (ok && events.length < limit) {
|
|
990
|
+
events.push(e);
|
|
991
|
+
toDel.push(k);
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
cur.close();
|
|
995
|
+
for (const k of toDel) txn.del(dbi, k);
|
|
996
|
+
txn.commit();
|
|
997
|
+
} catch (e) {
|
|
998
|
+
try { txn.abort(); } catch { }
|
|
999
|
+
}
|
|
1000
|
+
return events;
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
if (this.backend === 'level' && this.db) {
|
|
1004
|
+
// 简化:扫描全库 keys 读取该 pid 的事件
|
|
1005
|
+
try {
|
|
1006
|
+
const toDel = [];
|
|
1007
|
+
for await (const { key, value } of this.db.iterator({ gte: `e:${pid}:`, lt: `e:${pid};` })) {
|
|
1008
|
+
const ok = filterFn ? filterFn(value) : true;
|
|
1009
|
+
if (ok && events.length < limit) {
|
|
1010
|
+
events.push(value);
|
|
1011
|
+
toDel.push(key);
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
1014
|
+
// 删除已消费
|
|
1015
|
+
const ops = toDel.map(k => ({ type: 'del', key: k }));
|
|
1016
|
+
if (ops.length) await this.db.batch(ops);
|
|
1017
|
+
} catch { }
|
|
1018
|
+
return events;
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
return events;
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
// 枚举所有分区 ID(FS 模式)
|
|
1025
|
+
async listPartitionIds() {
|
|
1026
|
+
if (this.backend === 'fs') {
|
|
1027
|
+
const files = fs.readdirSync(this.baseDir).filter(f => /^p_\d+\.jsonl$/.test(f));
|
|
1028
|
+
const ids = files.map(f => Number(f.match(/^p_(\d+)\.jsonl$/)[1])).sort((a, b) => a - b);
|
|
1029
|
+
return ids;
|
|
1030
|
+
}
|
|
1031
|
+
// LMDB/level 不易列举,约定 0..N-1 尝试加载
|
|
1032
|
+
return [];
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
// 分区器(哈希 -> 分区ID)
|
|
1037
|
+
class GraphPartitioner {
|
|
1038
|
+
constructor({ partitions = 64 } = {}) {
|
|
1039
|
+
this.partitions = Math.max(4, partitions);
|
|
1040
|
+
}
|
|
1041
|
+
idOf(pointID) {
|
|
1042
|
+
if (!pointID) return 0;
|
|
1043
|
+
const h = crypto.createHash('sha1').update(String(pointID)).digest();
|
|
1044
|
+
// 使用前 4 字节构造 uint32
|
|
1045
|
+
const u32 = h.readUInt32BE(0);
|
|
1046
|
+
return u32 % this.partitions;
|
|
1047
|
+
}
|
|
1048
|
+
neighborsOf(pid, radius = 1) {
|
|
1049
|
+
const out = new Set([pid]);
|
|
1050
|
+
for (let r = 1; r <= radius; r++) {
|
|
1051
|
+
out.add((pid - r + this.partitions) % this.partitions);
|
|
1052
|
+
out.add((pid + r) % this.partitions);
|
|
1053
|
+
}
|
|
1054
|
+
return Array.from(out).sort((a, b) => a - b);
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
// 分区图 + 滑动窗口 + 边界事件消费
|
|
1059
|
+
class PartitionedGraphDB {
|
|
1060
|
+
constructor({
|
|
1061
|
+
partitions = 64,
|
|
1062
|
+
maxLoadedPartitions = 8,
|
|
1063
|
+
windowRadius = 1,
|
|
1064
|
+
baseDir = path.join(__dirname, 'graph_parts'),
|
|
1065
|
+
backend = 'fs'
|
|
1066
|
+
} = {}) {
|
|
1067
|
+
this.partitioner = new GraphPartitioner({ partitions });
|
|
1068
|
+
this.adapter = new GraphStorageAdapter({ baseDir, backend });
|
|
1069
|
+
this.maxLoadedPartitions = Math.max(2, maxLoadedPartitions);
|
|
1070
|
+
this.windowRadius = Math.max(0, windowRadius);
|
|
1071
|
+
|
|
1072
|
+
// 已加载分区:pid -> { points: Map, dirty, lastAccess }
|
|
1073
|
+
this.loaded = new Map();
|
|
1074
|
+
// 兼容旧代码:合并视图(仅包含已加载分区的点)
|
|
1075
|
+
this.points = new Map();
|
|
1076
|
+
// LRU
|
|
1077
|
+
this.accessTick = 0;
|
|
1078
|
+
this.centerPid = null;
|
|
1079
|
+
|
|
1080
|
+
// 并发保护
|
|
1081
|
+
this.loading = new Set();
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
// ---------- 内部:加载/保存/淘汰 ----------
|
|
1085
|
+
async ensureLoaded(pid) {
|
|
1086
|
+
if (this.loaded.has(pid)) {
|
|
1087
|
+
this._touch(pid);
|
|
1088
|
+
return this.loaded.get(pid);
|
|
1089
|
+
}
|
|
1090
|
+
if (this.loading.has(pid)) {
|
|
1091
|
+
// 等待已有加载完成
|
|
1092
|
+
while (this.loading.has(pid)) { await sleep(10); }
|
|
1093
|
+
return this.loaded.get(pid);
|
|
1094
|
+
}
|
|
1095
|
+
this.loading.add(pid);
|
|
1096
|
+
try {
|
|
1097
|
+
const part = await this.adapter.loadPartition(pid);
|
|
1098
|
+
const bundle = {
|
|
1099
|
+
points: part.points || new Map(),
|
|
1100
|
+
dirty: false,
|
|
1101
|
+
lastAccess: ++this.accessTick
|
|
1102
|
+
};
|
|
1103
|
+
this.loaded.set(pid, bundle);
|
|
1104
|
+
// 合并到全局视图
|
|
1105
|
+
for (const [id, p] of bundle.points.entries()) this.points.set(id, p);
|
|
1106
|
+
|
|
1107
|
+
// 消费边界事件:把指向本分区的事件落库
|
|
1108
|
+
const events = await this.adapter.consumeEdgeEvents(pid, (e) =>
|
|
1109
|
+
e && e.type === 'cross-edge' && (e.toPid === pid || e.fromPid === pid), 5000);
|
|
1110
|
+
if (events.length) {
|
|
1111
|
+
for (const e of events) this._applyEdgeEvent(bundle, e);
|
|
1112
|
+
bundle.dirty = true;
|
|
1113
|
+
}
|
|
1114
|
+
|
|
1115
|
+
// 控制内存:若超容量,执行淘汰
|
|
1116
|
+
await this._evictIfNeeded();
|
|
1117
|
+
return bundle;
|
|
1118
|
+
} finally {
|
|
1119
|
+
this.loading.delete(pid);
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
async savePartitionIfDirty(pid) {
|
|
1124
|
+
const entry = this.loaded.get(pid);
|
|
1125
|
+
if (!entry) return;
|
|
1126
|
+
if (!entry.dirty) return;
|
|
1127
|
+
await this.adapter.savePartition(pid, entry.points);
|
|
1128
|
+
entry.dirty = false;
|
|
1129
|
+
}
|
|
1130
|
+
|
|
1131
|
+
async _evictIfNeeded() {
|
|
1132
|
+
if (this.loaded.size <= this.maxLoadedPartitions) return;
|
|
1133
|
+
// 淘汰最近最少访问的分区(除中心窗口)
|
|
1134
|
+
const avoid = new Set(this.partitioner.neighborsOf(this.centerPid ?? 0, this.windowRadius));
|
|
1135
|
+
// 构建按 lastAccess 升序
|
|
1136
|
+
const list = Array.from(this.loaded.entries())
|
|
1137
|
+
.filter(([pid]) => !avoid.has(pid))
|
|
1138
|
+
.sort((a, b) => a[1].lastAccess - b[1].lastAccess);
|
|
1139
|
+
while (this.loaded.size > this.maxLoadedPartitions && list.length) {
|
|
1140
|
+
const [pid, entry] = list.shift();
|
|
1141
|
+
await this.savePartitionIfDirty(pid);
|
|
1142
|
+
// 从全局视图移除
|
|
1143
|
+
for (const [id] of entry.points.entries()) this.points.delete(id);
|
|
1144
|
+
this.loaded.delete(pid);
|
|
1145
|
+
logPart('evicted partition', pid);
|
|
1146
|
+
}
|
|
1147
|
+
}
|
|
1148
|
+
|
|
1149
|
+
_touch(pid) {
|
|
1150
|
+
const entry = this.loaded.get(pid);
|
|
1151
|
+
if (entry) entry.lastAccess = ++this.accessTick;
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
_applyEdgeEvent(targetBundle, e) {
|
|
1155
|
+
// 事件格式:{ type:'cross-edge', from:'id', to:'id', weight, direction, fromPid, toPid }
|
|
1156
|
+
if (!e || e.type !== 'cross-edge') return;
|
|
1157
|
+
const ensurePoint = (m, id) => {
|
|
1158
|
+
if (!m.has(id)) m.set(id, { pointID: id, connect: [] });
|
|
1159
|
+
return m.get(id);
|
|
1160
|
+
};
|
|
1161
|
+
const mp = targetBundle.points;
|
|
1162
|
+
const pFrom = ensurePoint(mp, e.from);
|
|
1163
|
+
const pTo = ensurePoint(mp, e.to);
|
|
1164
|
+
// 在 from 中落边(若 from 属于本分区)
|
|
1165
|
+
if (e.toPid === e.fromPid) {
|
|
1166
|
+
// 同分区事件(理论上不会在事件日志里)
|
|
1167
|
+
if (!pFrom.connect.some(([w, id, d]) => id === e.to && d === e.direction)) {
|
|
1168
|
+
pFrom.connect.push([e.weight, e.to, e.direction]);
|
|
1169
|
+
}
|
|
1170
|
+
} else {
|
|
1171
|
+
// 当前 bundle 即为 toPid 或 fromPid 的载体
|
|
1172
|
+
if (e.toPid === this.partitioner.idOf(pTo.pointID)) {
|
|
1173
|
+
// 对于目标分区,至少要保证可被 selectPath 遍历;保留边终点即可(可选:反向提示边)
|
|
1174
|
+
// 不在 pTo 里写边(避免双写),仅保证 from 的边会在 from 分区生效
|
|
1175
|
+
}
|
|
1176
|
+
if (e.fromPid === this.partitioner.idOf(pFrom.pointID)) {
|
|
1177
|
+
if (!pFrom.connect.some(([w, id, d]) => id === e.to && d === e.direction)) {
|
|
1178
|
+
pFrom.connect.push([e.weight, e.to, e.direction]);
|
|
1179
|
+
}
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
// ---------- 滑动窗口 ----------
|
|
1185
|
+
async focusOnPoint(pointID) {
|
|
1186
|
+
const pid = this.partitioner.idOf(pointID);
|
|
1187
|
+
this.centerPid = pid;
|
|
1188
|
+
const toLoad = this.partitioner.neighborsOf(pid, this.windowRadius);
|
|
1189
|
+
for (const id of toLoad) await this.ensureLoaded(id);
|
|
1190
|
+
await this._evictIfNeeded();
|
|
1191
|
+
}
|
|
1192
|
+
|
|
1193
|
+
// ---------- 兼容 API:点/边 操作 ----------
|
|
1194
|
+
addPoint(pointID, connect = []) {
|
|
1195
|
+
const pid = this.partitioner.idOf(pointID);
|
|
1196
|
+
const ensure = (bundle) => {
|
|
1197
|
+
if (!bundle.points.has(pointID)) bundle.points.set(pointID, { pointID, connect: [] });
|
|
1198
|
+
this.points.set(pointID, bundle.points.get(pointID));
|
|
1199
|
+
return bundle.points.get(pointID);
|
|
1200
|
+
};
|
|
1201
|
+
return this.ensureLoaded(pid).then(bundle => {
|
|
1202
|
+
const p = ensure(bundle);
|
|
1203
|
+
// 添加本地边;跨分区写事件
|
|
1204
|
+
for (const [w, nid, dir] of connect) this._addEdgeInternal(pid, p, w, nid, dir, bundle);
|
|
1205
|
+
bundle.dirty = true;
|
|
1206
|
+
});
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
_addEdgeInternal(fromPid, fromPoint, weight, toID, direction, bundleOfFrom) {
|
|
1210
|
+
const toPid = this.partitioner.idOf(toID);
|
|
1211
|
+
const w = (typeof weight === 'number' && isFinite(weight)) ? weight : 1;
|
|
1212
|
+
const d = (direction === 0 || direction === 1 || direction === 2) ? direction : 0;
|
|
1213
|
+
|
|
1214
|
+
if (toPid === fromPid) {
|
|
1215
|
+
// 同分区直接写
|
|
1216
|
+
if (!fromPoint.connect.some(([ww, id, dd]) => id === toID && dd === d)) {
|
|
1217
|
+
fromPoint.connect.push([w, toID, d]);
|
|
1218
|
+
bundleOfFrom.dirty = true;
|
|
1219
|
+
}
|
|
1220
|
+
} else {
|
|
1221
|
+
// 跨分区 -> 记录边界事件至 fromPid(或 toPid 都可,这里记录到 fromPid,toPid 加载时也会消费相关事件)
|
|
1222
|
+
this.adapter.appendEdgeEvent(fromPid, {
|
|
1223
|
+
type: 'cross-edge',
|
|
1224
|
+
from: fromPoint.pointID,
|
|
1225
|
+
to: toID,
|
|
1226
|
+
weight: w,
|
|
1227
|
+
direction: d,
|
|
1228
|
+
fromPid,
|
|
1229
|
+
toPid
|
|
1230
|
+
});
|
|
1231
|
+
// 同时对“已加载且包含 toPid 的 bundle”进行即时应用(若存在)
|
|
1232
|
+
const toBundle = this.loaded.get(toPid);
|
|
1233
|
+
if (toBundle) {
|
|
1234
|
+
// 在 from 分区已经写入 from->to 事件;对于 to 分区无需写边(避免双写),可选择记录提示(此处略)
|
|
1235
|
+
}
|
|
1236
|
+
}
|
|
1237
|
+
}
|
|
1238
|
+
|
|
1239
|
+
addBidirectionalEdge(id1, id2, weight = 1) {
|
|
1240
|
+
return this.addEdge(id1, id2, weight, 0);
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
async addEdge(fromID, toID, weight = 1, direction = 0) {
|
|
1244
|
+
const fromPid = this.partitioner.idOf(fromID);
|
|
1245
|
+
const fromBundle = await this.ensureLoaded(fromPid);
|
|
1246
|
+
if (!fromBundle.points.has(fromID)) {
|
|
1247
|
+
fromBundle.points.set(fromID, { pointID: fromID, connect: [] });
|
|
1248
|
+
this.points.set(fromID, fromBundle.points.get(fromID));
|
|
1249
|
+
}
|
|
1250
|
+
const fromPoint = fromBundle.points.get(fromID);
|
|
1251
|
+
this._addEdgeInternal(fromPid, fromPoint, weight, toID, direction, fromBundle);
|
|
1252
|
+
|
|
1253
|
+
if (direction === 0) {
|
|
1254
|
+
// 双向边:反向写入
|
|
1255
|
+
const toPid = this.partitioner.idOf(toID);
|
|
1256
|
+
const toBundle = await this.ensureLoaded(toPid);
|
|
1257
|
+
if (!toBundle.points.has(toID)) {
|
|
1258
|
+
toBundle.points.set(toID, { pointID: toID, connect: [] });
|
|
1259
|
+
this.points.set(toID, toBundle.points.get(toID));
|
|
1260
|
+
}
|
|
1261
|
+
const toPoint = toBundle.points.get(toID);
|
|
1262
|
+
this._addEdgeInternal(toPid, toPoint, weight, fromID, 0, toBundle);
|
|
1263
|
+
}
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1266
|
+
async updateEdge(fromID, toID, newWeight, direction = 0) {
|
|
1267
|
+
const fromPid = this.partitioner.idOf(fromID);
|
|
1268
|
+
const b = await this.ensureLoaded(fromPid);
|
|
1269
|
+
const p = b.points.get(fromID);
|
|
1270
|
+
if (!p) return;
|
|
1271
|
+
const idx = p.connect.findIndex(([w, id, d]) => id === toID && d === direction);
|
|
1272
|
+
if (idx >= 0) {
|
|
1273
|
+
p.connect[idx][0] = newWeight;
|
|
1274
|
+
b.dirty = true;
|
|
1275
|
+
} else {
|
|
1276
|
+
// 不存在则添加
|
|
1277
|
+
this._addEdgeInternal(fromPid, p, newWeight, toID, direction, b);
|
|
1278
|
+
}
|
|
1279
|
+
}
|
|
1280
|
+
|
|
1281
|
+
existEdge(fromID, toID) {
|
|
1282
|
+
const fromPid = this.partitioner.idOf(fromID);
|
|
1283
|
+
const entry = this.loaded.get(fromPid);
|
|
1284
|
+
if (!entry) return { exist: false, weight: undefined, type: undefined };
|
|
1285
|
+
const p = entry.points.get(fromID);
|
|
1286
|
+
if (!p) return { exist: false, weight: undefined, type: undefined };
|
|
1287
|
+
const found = p.connect.find(([w, id]) => id === toID);
|
|
1288
|
+
return { exist: !!found, weight: found ? found[0] : undefined, type: found ? found[2] : undefined };
|
|
1289
|
+
}
|
|
1290
|
+
|
|
1291
|
+
existPoint(pointID) {
|
|
1292
|
+
// 仅检查已加载窗口
|
|
1293
|
+
const p = this.points.get(pointID);
|
|
1294
|
+
return { exist: !!p, connect: p ? p.connect : [] };
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1297
|
+
deleteEdge(a, b) {
|
|
1298
|
+
const pid = this.partitioner.idOf(a);
|
|
1299
|
+
const entry = this.loaded.get(pid);
|
|
1300
|
+
if (!entry) return;
|
|
1301
|
+
const p = entry.points.get(a);
|
|
1302
|
+
if (!p) return;
|
|
1303
|
+
const before = p.connect.length;
|
|
1304
|
+
p.connect = p.connect.filter(([_, id]) => id !== b);
|
|
1305
|
+
entry.dirty = entry.dirty || (p.connect.length !== before);
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1308
|
+
deletePoint(pointID) {
|
|
1309
|
+
const pid = this.partitioner.idOf(pointID);
|
|
1310
|
+
const entry = this.loaded.get(pid);
|
|
1311
|
+
if (!entry) return;
|
|
1312
|
+
if (entry.points.has(pointID)) {
|
|
1313
|
+
entry.points.delete(pointID);
|
|
1314
|
+
this.points.delete(pointID);
|
|
1315
|
+
entry.dirty = true;
|
|
1316
|
+
}
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
// 仅遍历窗口内点(兼容旧 getAllPoints 调用)
|
|
1320
|
+
getAllPoints() {
|
|
1321
|
+
return Array.from(this.points.values());
|
|
1322
|
+
}
|
|
1323
|
+
|
|
1324
|
+
// 导出全量点(跨所有分区),用于快照/发布
|
|
1325
|
+
async exportAllPoints() {
|
|
1326
|
+
const out = [];
|
|
1327
|
+
// 尝试枚举 FS 分区;其他后端可按 0..N-1 遍历或仅导出已加载窗口
|
|
1328
|
+
const ids = await this.adapter.listPartitionIds();
|
|
1329
|
+
if (ids.length === 0) {
|
|
1330
|
+
// 回退:导出窗口
|
|
1331
|
+
return this.getAllPoints();
|
|
1332
|
+
}
|
|
1333
|
+
for (const pid of ids) {
|
|
1334
|
+
const part = await this.adapter.loadPartition(pid);
|
|
1335
|
+
for (const [, p] of part.points.entries()) out.push({ pointID: p.pointID, connect: p.connect || [] });
|
|
1336
|
+
}
|
|
1337
|
+
return out;
|
|
1338
|
+
}
|
|
1339
|
+
|
|
1340
|
+
// 批量导入(将 legacy 点集落到分区)
|
|
1341
|
+
async importAllPoints(pointsArr) {
|
|
1342
|
+
if (!Array.isArray(pointsArr)) return;
|
|
1343
|
+
// 分桶
|
|
1344
|
+
const buckets = new Map();
|
|
1345
|
+
for (const p of pointsArr) {
|
|
1346
|
+
const pid = this.partitioner.idOf(p.pointID);
|
|
1347
|
+
if (!buckets.has(pid)) buckets.set(pid, new Map());
|
|
1348
|
+
const bm = buckets.get(pid);
|
|
1349
|
+
bm.set(p.pointID, { pointID: p.pointID, connect: Array.isArray(p.connect) ? p.connect.slice() : [] });
|
|
1350
|
+
}
|
|
1351
|
+
// 写入并更新窗口视图(懒加载)
|
|
1352
|
+
for (const [pid, map] of buckets.entries()) {
|
|
1353
|
+
await this.adapter.savePartition(pid, map);
|
|
1354
|
+
// 若已加载该分区,刷新内存镜像
|
|
1355
|
+
if (this.loaded.has(pid)) {
|
|
1356
|
+
const entry = this.loaded.get(pid);
|
|
1357
|
+
// 从全局视图移除旧
|
|
1358
|
+
for (const [id] of entry.points.entries()) this.points.delete(id);
|
|
1359
|
+
entry.points = map;
|
|
1360
|
+
entry.dirty = false;
|
|
1361
|
+
entry.lastAccess = ++this.accessTick;
|
|
1362
|
+
for (const [id, p] of map.entries()) this.points.set(id, p);
|
|
1363
|
+
}
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
|
|
1367
|
+
// 聚合邻居(窗口内),供传播使用
|
|
1368
|
+
getNeighbors(pointID, maxNeighbors = 50) {
|
|
1369
|
+
const p = this.points.get(pointID);
|
|
1370
|
+
if (!p) return [];
|
|
1371
|
+
return p.connect.slice(0, maxNeighbors);
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1374
|
+
// A* 简化:仅在窗口内搜索;跳出窗口时,尝试预取邻接分区后再继续
|
|
1375
|
+
async selectPath(fromID, toID) {
|
|
1376
|
+
if (fromID === toID) return [fromID];
|
|
1377
|
+
// 优先保证焦点加载
|
|
1378
|
+
await this.focusOnPoint(fromID);
|
|
1379
|
+
|
|
1380
|
+
const reconstruct = (came, cur) => {
|
|
1381
|
+
const path = [];
|
|
1382
|
+
let t = cur;
|
|
1383
|
+
while (came.has(t)) { path.push(t); t = came.get(t); }
|
|
1384
|
+
path.push(fromID);
|
|
1385
|
+
return path.reverse();
|
|
1386
|
+
};
|
|
1387
|
+
|
|
1388
|
+
const open = new Set([fromID]);
|
|
1389
|
+
const came = new Map();
|
|
1390
|
+
const g = new Map([[fromID, 0]]);
|
|
1391
|
+
const f = new Map([[fromID, 1]]);
|
|
1392
|
+
const closed = new Set();
|
|
1393
|
+
|
|
1394
|
+
const heuristic = () => 1;
|
|
1395
|
+
let iter = 0;
|
|
1396
|
+
const MAX_ITERS = 5000;
|
|
1397
|
+
|
|
1398
|
+
while (open.size && iter++ < MAX_ITERS) {
|
|
1399
|
+
// 取 f 最小
|
|
1400
|
+
let cur = null; let minF = Infinity;
|
|
1401
|
+
for (const id of open) {
|
|
1402
|
+
const val = f.get(id) ?? Infinity;
|
|
1403
|
+
if (val < minF) { minF = val; cur = id; }
|
|
1404
|
+
}
|
|
1405
|
+
if (cur == null) break;
|
|
1406
|
+
if (cur === toID) return reconstruct(came, cur);
|
|
1407
|
+
|
|
1408
|
+
open.delete(cur);
|
|
1409
|
+
closed.add(cur);
|
|
1410
|
+
|
|
1411
|
+
// 若遇到未知点,尝试加载其分区(滑动窗口)
|
|
1412
|
+
if (!this.points.has(cur)) {
|
|
1413
|
+
await this.focusOnPoint(cur);
|
|
1414
|
+
}
|
|
1415
|
+
|
|
1416
|
+
const neighbors = this.getNeighbors(cur, 50);
|
|
1417
|
+
// 如果邻居为空,尝试边界事件预取(根据邻居 ID 的分区预取)
|
|
1418
|
+
if (neighbors.length === 0) {
|
|
1419
|
+
const pid = this.partitioner.idOf(cur);
|
|
1420
|
+
const ring = this.partitioner.neighborsOf(pid, 1);
|
|
1421
|
+
for (const rid of ring) await this.ensureLoaded(rid);
|
|
1422
|
+
}
|
|
1423
|
+
|
|
1424
|
+
for (const [w, nb] of neighbors) {
|
|
1425
|
+
if (closed.has(nb)) continue;
|
|
1426
|
+
const tentative = (g.get(cur) || Infinity) + w;
|
|
1427
|
+
if (!open.has(nb)) open.add(nb);
|
|
1428
|
+
else if (tentative >= (g.get(nb) || Infinity)) continue;
|
|
1429
|
+
|
|
1430
|
+
came.set(nb, cur);
|
|
1431
|
+
g.set(nb, tentative);
|
|
1432
|
+
f.set(nb, tentative + heuristic());
|
|
1433
|
+
}
|
|
1434
|
+
}
|
|
1435
|
+
return null;
|
|
1436
|
+
}
|
|
1437
|
+
|
|
1438
|
+
// 刷盘所有已加载分区
|
|
1439
|
+
async flushAll() {
|
|
1440
|
+
for (const [pid] of this.loaded.entries()) await this.savePartitionIfDirty(pid);
|
|
1441
|
+
}
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1444
|
+
// ========================== 替换 Runtime.graph 为分区图 ==========================
|
|
1445
|
+
// 说明:wordGraph 仍使用内存 GraphDB;模因图使用 PartitionedGraphDB
|
|
1446
|
+
|
|
1447
|
+
// ...existing code...
|
|
701
1448
|
|
|
702
1449
|
class KVM {
|
|
703
1450
|
// this KVM is the key-value memory
|
|
@@ -753,7 +1500,7 @@ class KVM {
|
|
|
753
1500
|
}
|
|
754
1501
|
delete(key) {
|
|
755
1502
|
if (this.useLMDB) {
|
|
756
|
-
try { this.db.remove(key); } catch (_) {}
|
|
1503
|
+
try { this.db.remove(key); } catch (_) { }
|
|
757
1504
|
return;
|
|
758
1505
|
}
|
|
759
1506
|
this.memory.delete(key);
|
|
@@ -766,7 +1513,7 @@ class KVM {
|
|
|
766
1513
|
for (const k of this.db.getKeys({ snapshot: true })) {
|
|
767
1514
|
let v = this.db.get(k);
|
|
768
1515
|
if (typeof v === 'string') {
|
|
769
|
-
try { v = JSON.parse(v); } catch (_) {}
|
|
1516
|
+
try { v = JSON.parse(v); } catch (_) { }
|
|
770
1517
|
}
|
|
771
1518
|
out.push([k, v]);
|
|
772
1519
|
}
|
|
@@ -860,7 +1607,15 @@ class Runtime {
|
|
|
860
1607
|
// 运行时负责AI核心的调度、模因转换、信号传递与主流程控制
|
|
861
1608
|
constructor(config = {}) {
|
|
862
1609
|
this.config = config;
|
|
863
|
-
|
|
1610
|
+
// 使用分区图作为模因图;词图仍用内存图
|
|
1611
|
+
this.graph = new PartitionedGraphDB({
|
|
1612
|
+
partitions: this.config.partitions || 64,
|
|
1613
|
+
maxLoadedPartitions: this.config.maxLoadedPartitions || 8,
|
|
1614
|
+
windowRadius: this.config.windowRadius || 1,
|
|
1615
|
+
baseDir: path.join(__dirname, 'graph_parts'),
|
|
1616
|
+
backend: this.config.graphBackend || 'fs' // 可选 'fs' | 'lmdb' | 'level'
|
|
1617
|
+
});
|
|
1618
|
+
|
|
864
1619
|
this.wordGraph = new GraphDB();
|
|
865
1620
|
this.kvm = new KVM();
|
|
866
1621
|
this.changer = new Changer();
|
|
@@ -1042,19 +1797,15 @@ class Runtime {
|
|
|
1042
1797
|
visitCount++;
|
|
1043
1798
|
activatedOrder.push(id);
|
|
1044
1799
|
|
|
1045
|
-
// 仅在是“词”时记录访问,避免把模因ID写入词访问日志
|
|
1046
1800
|
if (this.wordGraph.points.has(id)) {
|
|
1047
1801
|
this.logWordAccess(id);
|
|
1048
1802
|
}
|
|
1049
1803
|
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
if (!visited.has(neighborID)) {
|
|
1056
|
-
next.push({ id: neighborID, value: value - decayK * weight });
|
|
1057
|
-
}
|
|
1804
|
+
// 改为通过 graph.getNeighbors 访问(窗口内)
|
|
1805
|
+
const neighbors = this.graph.getNeighbors(id, 50);
|
|
1806
|
+
for (const [weight, neighborID] of neighbors) {
|
|
1807
|
+
if (!visited.has(neighborID)) {
|
|
1808
|
+
next.push({ id: neighborID, value: value - decayK * weight });
|
|
1058
1809
|
}
|
|
1059
1810
|
}
|
|
1060
1811
|
}
|
|
@@ -1171,7 +1922,7 @@ class Runtime {
|
|
|
1171
1922
|
processInput(wordsArr, { addNewWords = true } = {}) {
|
|
1172
1923
|
wordsArr = this.filterStopWords(wordsArr);
|
|
1173
1924
|
if (wordsArr.length === 0) { console.log('[FILTER] 输入全为停用词,已全部过滤'); return; }
|
|
1174
|
-
|
|
1925
|
+
// console.log('Processing input:', wordsArr);
|
|
1175
1926
|
// 批量处理新词添加
|
|
1176
1927
|
if (addNewWords) {
|
|
1177
1928
|
// 一次性检查哪些词不在词表中
|
|
@@ -1257,7 +2008,7 @@ class Runtime {
|
|
|
1257
2008
|
const overlap = wordsArr.filter(w => memeWords.includes(w)).length;
|
|
1258
2009
|
if (overlap >= this.MIN_OVERLAP && memeWords.length + wordsArr.length <= this.MAX_MEME_WORDS) {
|
|
1259
2010
|
this.kvm.set(minMemeID, Array.from(new Set([...memeWords, ...wordsArr])));
|
|
1260
|
-
|
|
2011
|
+
// console.log(`Merged to existing meme: ${minMemeID}`);
|
|
1261
2012
|
} else {
|
|
1262
2013
|
// 创建新模因,使用有向连接
|
|
1263
2014
|
const newID = 'meme_' + Date.now();
|
|
@@ -1267,9 +2018,9 @@ class Runtime {
|
|
|
1267
2018
|
// 单向连接到最近的模因 (方向:2表示指向对方)
|
|
1268
2019
|
if (minMemeID) {
|
|
1269
2020
|
this.graph.addDirectionalEdge(newID, minMemeID, minDistance, 2);
|
|
1270
|
-
|
|
2021
|
+
// console.log(`[LINK] 新模因 ${newID} 单向连接到最近模因 ${minMemeID}`);
|
|
1271
2022
|
}
|
|
1272
|
-
|
|
2023
|
+
// console.log(`Created new meme: ${newID}`);
|
|
1273
2024
|
}
|
|
1274
2025
|
} else {
|
|
1275
2026
|
// 创建新模因
|
|
@@ -1280,9 +2031,9 @@ class Runtime {
|
|
|
1280
2031
|
// 如果有较近的模因,仍然创建单向连接
|
|
1281
2032
|
if (minMemeID) {
|
|
1282
2033
|
this.graph.addDirectionalEdge(newID, minMemeID, Math.min(minDistance, 5), 2);
|
|
1283
|
-
|
|
2034
|
+
// console.log(`[LINK] 新模因 ${newID} 单向连接到最近模因 ${minMemeID}`);
|
|
1284
2035
|
}
|
|
1285
|
-
|
|
2036
|
+
// console.log(`Created new meme: ${newID}`);
|
|
1286
2037
|
}
|
|
1287
2038
|
}
|
|
1288
2039
|
// 新增批量添加边的辅助方法
|
|
@@ -1599,7 +2350,7 @@ class Runtime {
|
|
|
1599
2350
|
this.kvm.delete(memeB.pointID);
|
|
1600
2351
|
memesToDelete.add(memeB.pointID);
|
|
1601
2352
|
|
|
1602
|
-
|
|
2353
|
+
// console.log(`Merged memes: ${memeA.pointID} <- ${memeB.pointID}`);
|
|
1603
2354
|
// 合并后立即尝试分裂
|
|
1604
2355
|
this.splitMemeIfNeeded(memeA.pointID);
|
|
1605
2356
|
} else {
|
|
@@ -1616,7 +2367,7 @@ class Runtime {
|
|
|
1616
2367
|
// 如果没有双向边,则添加双向边
|
|
1617
2368
|
if (!(existAtoB.exist && existAtoB.type === 0) && !(existBtoA.exist && existBtoA.type === 0)) {
|
|
1618
2369
|
this.graph.addBidirectionalEdge(memeA.pointID, memeB.pointID, avgDist);
|
|
1619
|
-
|
|
2370
|
+
// console.log(`[LINK] 添加双向边: ${memeA.pointID} <-> ${memeB.pointID} (avgDist=${avgDist})`);
|
|
1620
2371
|
}
|
|
1621
2372
|
}
|
|
1622
2373
|
}
|
|
@@ -1648,14 +2399,14 @@ class Runtime {
|
|
|
1648
2399
|
const newID = newIDs[i];
|
|
1649
2400
|
this.graph.addPoint(newID, []);
|
|
1650
2401
|
this.kvm.set(newID, chunk);
|
|
1651
|
-
|
|
2402
|
+
// console.log(`[SPLIT-FORCE] 新建模因: ${newID} 词数: ${chunk.length}`);
|
|
1652
2403
|
}
|
|
1653
2404
|
}
|
|
1654
2405
|
|
|
1655
2406
|
// 删除原模因
|
|
1656
2407
|
this.graph.points.delete(memeID);
|
|
1657
2408
|
this.kvm.delete(memeID);
|
|
1658
|
-
|
|
2409
|
+
// console.log(`[SPLIT-FORCE] 删除原模因: ${memeID}`);
|
|
1659
2410
|
return;
|
|
1660
2411
|
}
|
|
1661
2412
|
|
|
@@ -1704,12 +2455,12 @@ class Runtime {
|
|
|
1704
2455
|
const newID = 'meme_' + Date.now() + '_' + Math.floor(Math.random() * 10000);
|
|
1705
2456
|
this.graph.addPoint(newID, []);
|
|
1706
2457
|
this.kvm.set(newID, comp);
|
|
1707
|
-
|
|
2458
|
+
// console.log(`[SPLIT] 新建模因: ${newID} 词数: ${comp.length}`);
|
|
1708
2459
|
}
|
|
1709
2460
|
// 删除原节点
|
|
1710
2461
|
this.graph.points.delete(memeID);
|
|
1711
2462
|
this.kvm.delete(memeID);
|
|
1712
|
-
|
|
2463
|
+
// console.log(`[SPLIT] 删除原模因: ${memeID}`);
|
|
1713
2464
|
}
|
|
1714
2465
|
}
|
|
1715
2466
|
}
|
|
@@ -2136,7 +2887,7 @@ class controller {
|
|
|
2136
2887
|
const words = text.toLowerCase().split(' ').filter(w => w.length > 0);
|
|
2137
2888
|
this.runtime.processInput(words, { addNewWords: false });
|
|
2138
2889
|
// 用模因网络参与推理
|
|
2139
|
-
|
|
2890
|
+
// console.log('[DEBUG] 当前所有模因节点:', this.runtime.kvm.memory);
|
|
2140
2891
|
return await this.runtime.generateResponseWithMemes(words);
|
|
2141
2892
|
}
|
|
2142
2893
|
|
|
@@ -2200,7 +2951,7 @@ function saveAll(runtime) {
|
|
|
2200
2951
|
const data = {
|
|
2201
2952
|
memes: runtime.graph.getAllPoints(),
|
|
2202
2953
|
wordGraph: Array.from(runtime.wordGraph.points.values()),
|
|
2203
|
-
|
|
2954
|
+
kvm: runtime.kvm.exportEntries(),
|
|
2204
2955
|
vocab: runtime.vocabManager.vocab,
|
|
2205
2956
|
wordAccessLog: Array.from(runtime.wordAccessLog ? runtime.wordAccessLog.entries() : [])
|
|
2206
2957
|
};
|
|
@@ -2280,6 +3031,135 @@ async function boot() {
|
|
|
2280
3031
|
return model;
|
|
2281
3032
|
}
|
|
2282
3033
|
|
|
3034
|
+
|
|
3035
|
+
// 激活副本:从快照加载 ctrlB/ctrlC 并加入轮询
|
|
3036
|
+
let activeControllers = []; // 轮询数组
|
|
3037
|
+
let rrIdx = 0;
|
|
3038
|
+
// 新增:双实例热切换状态
|
|
3039
|
+
let servingCtrl = null; // 当前对外服务的控制器
|
|
3040
|
+
let standbyCtrl = null; // 后台承接 Redis 模型的控制器
|
|
3041
|
+
let updatingModel = false;
|
|
3042
|
+
let pendingModelObj = null;
|
|
3043
|
+
|
|
3044
|
+
function setServingController(ctrl) {
|
|
3045
|
+
servingCtrl = ctrl;
|
|
3046
|
+
activeControllers = [servingCtrl]; // 仅对外暴露一份,避免切换瞬间不一致
|
|
3047
|
+
global.ctrlA = servingCtrl;
|
|
3048
|
+
global.ctrl = servingCtrl;
|
|
3049
|
+
}
|
|
3050
|
+
|
|
3051
|
+
|
|
3052
|
+
// 替换 getNextController:始终返回当前服务控制器
|
|
3053
|
+
function getNextController() {
|
|
3054
|
+
if (servingCtrl) return servingCtrl;
|
|
3055
|
+
if (activeControllers.length) return activeControllers[rrIdx % activeControllers.length];
|
|
3056
|
+
return global.ctrlA; // 兜底
|
|
3057
|
+
}
|
|
3058
|
+
async function ensureStandbyReady() {
|
|
3059
|
+
if (!standbyCtrl) {
|
|
3060
|
+
standbyCtrl = new controller(); // 构造函数会初始化 Runtime
|
|
3061
|
+
// 初始保持与serving词表一致,减少首次同步成本(可选)
|
|
3062
|
+
if (servingCtrl?.runtime?.vocabManager?.vocab?.length) {
|
|
3063
|
+
standbyCtrl.runtime.vocabManager.vocab = [...servingCtrl.runtime.vocabManager.vocab];
|
|
3064
|
+
standbyCtrl.runtime.vocabManager.updateMappings();
|
|
3065
|
+
}
|
|
3066
|
+
}
|
|
3067
|
+
return standbyCtrl;
|
|
3068
|
+
}
|
|
3069
|
+
|
|
3070
|
+
// 将 protobuf 反序列化对象应用到 standby 并热切换
|
|
3071
|
+
async function handleRedisModelSwap(modelObj) {
|
|
3072
|
+
if (updatingModel) {
|
|
3073
|
+
pendingModelObj = modelObj; // 只保留最近一条
|
|
3074
|
+
return;
|
|
3075
|
+
}
|
|
3076
|
+
updatingModel = true;
|
|
3077
|
+
try {
|
|
3078
|
+
const standby = await ensureStandbyReady();
|
|
3079
|
+
// 在 standby 上同步模型
|
|
3080
|
+
await plainObjToRuntime(standby.runtime, modelObj);
|
|
3081
|
+
applyModelParams(standby.runtime);
|
|
3082
|
+
|
|
3083
|
+
// 原子切换:standby 上位,serving 变为 standby
|
|
3084
|
+
const oldServing = servingCtrl;
|
|
3085
|
+
setServingController(standbyCtrl);
|
|
3086
|
+
standbyCtrl = oldServing;
|
|
3087
|
+
|
|
3088
|
+
console.log('[MODEL SYNC] 新模型已在后台准备完毕并无缝切换为在线实例');
|
|
3089
|
+
} catch (e) {
|
|
3090
|
+
console.error('[MODEL SYNC] 后台模型切换失败:', e.message);
|
|
3091
|
+
} finally {
|
|
3092
|
+
updatingModel = false;
|
|
3093
|
+
if (pendingModelObj) {
|
|
3094
|
+
const next = pendingModelObj;
|
|
3095
|
+
pendingModelObj = null;
|
|
3096
|
+
// 递归处理最新一条待处理消息
|
|
3097
|
+
setImmediate(() => handleRedisModelSwap(next));
|
|
3098
|
+
}
|
|
3099
|
+
}
|
|
3100
|
+
}
|
|
3101
|
+
async function activateReplicasIfNeeded() {
|
|
3102
|
+
if (global.config.isStudy) return; // study 进程不启用副本扩容
|
|
3103
|
+
if (global.ctrlB && global.ctrlC) return; // 已激活
|
|
3104
|
+
try {
|
|
3105
|
+
const loadReplicaByName = async (name) => {
|
|
3106
|
+
const ctrl = new controller();
|
|
3107
|
+
// 为副本单独挂载快照管理器
|
|
3108
|
+
ctrl.snapshotManager = new SnapshotManager(ctrl.runtime);
|
|
3109
|
+
// 在 A 的快照目录中寻找对应 id
|
|
3110
|
+
const smA = global.ctrlA.snapshotManager;
|
|
3111
|
+
smA.loadSnapshotList();
|
|
3112
|
+
const snap = smA.snapshotList.find(s => s.name === name || s.id.endsWith(`_${name}`));
|
|
3113
|
+
if (!snap) {
|
|
3114
|
+
console.warn(`[REPLICA] 未找到快照: ${name},尝试现生成...`);
|
|
3115
|
+
await smA.createSnapshot(name);
|
|
3116
|
+
}
|
|
3117
|
+
const snap2 = smA.snapshotList.find(s => s.name === name || s.id.endsWith(`_${name}`));
|
|
3118
|
+
if (snap2) {
|
|
3119
|
+
// 直接读取该文件内容并恢复到 ctrl.runtime
|
|
3120
|
+
const dataStr = fs.readFileSync(snap2.path, 'utf-8');
|
|
3121
|
+
const data = JSON.parse(dataStr);
|
|
3122
|
+
// 简单恢复(与 SnapshotManager.restoreSnapshot 类似)
|
|
3123
|
+
ctrl.runtime.graph = new GraphDB();
|
|
3124
|
+
ctrl.runtime.wordGraph = new GraphDB();
|
|
3125
|
+
ctrl.runtime.kvm = new KVM();
|
|
3126
|
+
ctrl.runtime.wordAccessLog = new Map(data.wordAccessLog || []);
|
|
3127
|
+
if (data.memes) for (const p of data.memes) ctrl.runtime.graph.addPoint(p.pointID, p.connect);
|
|
3128
|
+
if (data.wordGraph) for (const p of data.wordGraph) ctrl.runtime.wordGraph.addPoint(p.pointID, p.connect);
|
|
3129
|
+
if (data.kvm) for (const [k, v] of data.kvm) ctrl.runtime.kvm.set(k, v);
|
|
3130
|
+
if (data.vocab) { ctrl.runtime.vocabManager.vocab = data.vocab; ctrl.runtime.vocabManager.updateMappings(); }
|
|
3131
|
+
return ctrl;
|
|
3132
|
+
}
|
|
3133
|
+
return null;
|
|
3134
|
+
};
|
|
3135
|
+
|
|
3136
|
+
if (!global.ctrlB) global.ctrlB = await loadReplicaByName('replica_B');
|
|
3137
|
+
if (!global.ctrlC) global.ctrlC = await loadReplicaByName('replica_C');
|
|
3138
|
+
activeControllers = [global.ctrlA, ...(global.ctrlB ? [global.ctrlB] : []), ...(global.ctrlC ? [global.ctrlC] : [])];
|
|
3139
|
+
console.log(`[REPLICA] 已激活副本: ${activeControllers.length} 个控制器`);
|
|
3140
|
+
} catch (e) {
|
|
3141
|
+
console.warn('[REPLICA] 激活失败:', e.message);
|
|
3142
|
+
}
|
|
3143
|
+
}
|
|
3144
|
+
// 对端探测/反触发:同组任一对端端口死亡则激活副本
|
|
3145
|
+
function installPeerFailoverMonitor() {
|
|
3146
|
+
const peers = global.config.peerServePorts || [];
|
|
3147
|
+
if (!Array.isArray(peers) || peers.length === 0) return;
|
|
3148
|
+
const axiosInst = axios.create({ timeout: 1500 });
|
|
3149
|
+
setInterval(async () => {
|
|
3150
|
+
try {
|
|
3151
|
+
const checks = await Promise.all(peers.map(p =>
|
|
3152
|
+
axiosInst.get(`http://127.0.0.1:${p}/health`).then(() => true).catch(() => false)
|
|
3153
|
+
));
|
|
3154
|
+
const dead = checks.some(ok => !ok);
|
|
3155
|
+
if (dead) {
|
|
3156
|
+
console.warn('[ANTI-TRIGGER] 检测到对端 serve 进程离线,启动副本扩容...');
|
|
3157
|
+
await activateReplicasIfNeeded();
|
|
3158
|
+
}
|
|
3159
|
+
} catch (_) { /* 忽略一次错误 */ }
|
|
3160
|
+
}, 3000);
|
|
3161
|
+
}
|
|
3162
|
+
|
|
2283
3163
|
// 在main函数末尾添加
|
|
2284
3164
|
function setupExitHandler() {
|
|
2285
3165
|
let isExiting = false;
|
|
@@ -2324,6 +3204,27 @@ function setupExitHandler() {
|
|
|
2324
3204
|
exitHandler();
|
|
2325
3205
|
});
|
|
2326
3206
|
}
|
|
3207
|
+
|
|
3208
|
+
// 预缓存副本快照(不在内存常驻)
|
|
3209
|
+
async function preCacheReplicas() {
|
|
3210
|
+
try {
|
|
3211
|
+
if (!global.ctrlA?.snapshotManager) return;
|
|
3212
|
+
const sm = global.ctrlA.snapshotManager;
|
|
3213
|
+
sm.loadSnapshotList();
|
|
3214
|
+
const hasB = sm.getSnapshotList().some(s => s.name.includes('replica_B'));
|
|
3215
|
+
const hasC = sm.getSnapshotList().some(s => s.name.includes('replica_C'));
|
|
3216
|
+
if (!hasB) {
|
|
3217
|
+
console.log('[REPLICA] 预生成 ctrlB 快照(replica_B)...');
|
|
3218
|
+
await sm.createSnapshot('replica_B');
|
|
3219
|
+
}
|
|
3220
|
+
if (!hasC) {
|
|
3221
|
+
console.log('[REPLICA] 预生成 ctrlC 快照(replica_C)...');
|
|
3222
|
+
await sm.createSnapshot('replica_C');
|
|
3223
|
+
}
|
|
3224
|
+
} catch (e) {
|
|
3225
|
+
console.warn('[REPLICA] 预缓存失败:', e.message);
|
|
3226
|
+
}
|
|
3227
|
+
}
|
|
2327
3228
|
// 添加定期垃圾回收帮助函数
|
|
2328
3229
|
function optimizeMemory() {
|
|
2329
3230
|
const memUsage = process.memoryUsage();
|
|
@@ -2337,16 +3238,24 @@ function optimizeMemory() {
|
|
|
2337
3238
|
}
|
|
2338
3239
|
}
|
|
2339
3240
|
async function main() {
|
|
3241
|
+
let __adv = null;
|
|
3242
|
+
if (String(process.env.SERVE_ADV_AUTOSTART || '').toLowerCase() === 'true') {
|
|
3243
|
+
__adv = new AdversaryScheduler(global.ctrlA.runtime, {
|
|
3244
|
+
providerSpec: process.env.SERVE_ADV_MODEL || process.env.ADV_MODEL || 'ollama:llama3:70b',
|
|
3245
|
+
judgeMode: process.env.SERVE_ADV_JUDGE || process.env.ADV_JUDGE || 'heuristic',
|
|
3246
|
+
intervalMs: Number(process.env.SERVE_ADV_INTERVAL || 120_000),
|
|
3247
|
+
batchSize: Number(process.env.SERVE_ADV_BATCH || 2),
|
|
3248
|
+
adjustParams: false // 只记录,不修改服务实例的参数
|
|
3249
|
+
});
|
|
3250
|
+
__adv.start();
|
|
3251
|
+
}
|
|
2340
3252
|
|
|
2341
3253
|
let spiderPrivate = new Spider();
|
|
2342
3254
|
// 创建三个全局控制器副本
|
|
2343
3255
|
const ctrlA = new controller();
|
|
2344
|
-
const ctrlB = new controller();
|
|
2345
|
-
const ctrlC = new controller();
|
|
2346
3256
|
global.ctrlA = ctrlA;
|
|
2347
|
-
global.ctrlB = ctrlB;
|
|
2348
|
-
global.ctrlC = ctrlC;
|
|
2349
3257
|
global.ctrl = ctrlA;
|
|
3258
|
+
setServingController(ctrlA);
|
|
2350
3259
|
// 启动快照自动保存(每30分钟)
|
|
2351
3260
|
if (ctrlA.snapshotManager) {
|
|
2352
3261
|
ctrlA.snapshotManager.setupAutoSnapshot(30 * 60 * 1000);
|
|
@@ -2354,10 +3263,9 @@ async function main() {
|
|
|
2354
3263
|
}
|
|
2355
3264
|
setupExitHandler();
|
|
2356
3265
|
console.log('Starting AI system...');
|
|
2357
|
-
//
|
|
3266
|
+
// 恢复 ctrlA
|
|
2358
3267
|
loadAll(ctrlA.runtime);
|
|
2359
|
-
|
|
2360
|
-
loadAll(ctrlC.runtime);
|
|
3268
|
+
await preCacheReplicas();
|
|
2361
3269
|
const redisClient = redis.createClient();
|
|
2362
3270
|
redisClient.connect();
|
|
2363
3271
|
redisClient.on("error", function (err) {
|
|
@@ -2388,19 +3296,13 @@ async function main() {
|
|
|
2388
3296
|
redisClient.on("message", function (channel, message) {
|
|
2389
3297
|
if (channel === `AI-model-${__dirname}` && RuntimeMessage) {
|
|
2390
3298
|
try {
|
|
2391
|
-
// 反序列化为对象
|
|
2392
3299
|
const modelObj = RuntimeMessage.decode(Buffer.from(message));
|
|
2393
|
-
// 可选:校验
|
|
2394
3300
|
const errMsg = RuntimeMessage.verify(modelObj);
|
|
2395
3301
|
if (errMsg) throw Error(errMsg);
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
// 你需要实现一个 plainObjToRuntime 方法
|
|
2399
|
-
plainObjToRuntime(ctrlA.runtime, modelObj);
|
|
2400
|
-
|
|
2401
|
-
console.log("[MODEL SYNC] 已同步最新模型到 ctrlA.runtime");
|
|
3302
|
+
// 后台准备并切换
|
|
3303
|
+
handleRedisModelSwap(modelObj);
|
|
2402
3304
|
} catch (e) {
|
|
2403
|
-
console.error("[MODEL SYNC]
|
|
3305
|
+
console.error("[MODEL SYNC] 反序列化或切换失败:", e.message);
|
|
2404
3306
|
}
|
|
2405
3307
|
}
|
|
2406
3308
|
});
|
|
@@ -2451,30 +3353,66 @@ async function main() {
|
|
|
2451
3353
|
await ctrlA.handleInput('I like apple');
|
|
2452
3354
|
await ctrlA.handleInput('I love orange');
|
|
2453
3355
|
await ctrlA.handleInput('you are good');
|
|
2454
|
-
|
|
2455
|
-
// 启动主循环(只用A副本)
|
|
3356
|
+
// 启动主循环(仅 A)
|
|
2456
3357
|
ctrlA.startMainLoop();
|
|
2457
3358
|
|
|
3359
|
+
// 安装对端监控触发器
|
|
3360
|
+
installPeerFailoverMonitor();
|
|
3361
|
+
app.get('/api/graph/partitions/status', async (req, res) => {
|
|
3362
|
+
try {
|
|
3363
|
+
const g = global.ctrlA?.runtime?.graph;
|
|
3364
|
+
if (!g || !(g instanceof PartitionedGraphDB)) {
|
|
3365
|
+
return res.json({ ok: true, mode: 'in-memory', loaded: 0 });
|
|
3366
|
+
}
|
|
3367
|
+
const loaded = Array.from(g.loaded.keys());
|
|
3368
|
+
res.json({
|
|
3369
|
+
ok: true,
|
|
3370
|
+
mode: 'partitioned',
|
|
3371
|
+
partitions: g.partitioner.partitions,
|
|
3372
|
+
loaded,
|
|
3373
|
+
maxLoaded: g.maxLoadedPartitions,
|
|
3374
|
+
windowRadius: g.windowRadius,
|
|
3375
|
+
centerPid: g.centerPid
|
|
3376
|
+
});
|
|
3377
|
+
} catch (e) {
|
|
3378
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3379
|
+
}
|
|
3380
|
+
});
|
|
3381
|
+
|
|
3382
|
+
app.post('/api/graph/partitions/flush', async (req, res) => {
|
|
3383
|
+
try {
|
|
3384
|
+
const g = global.ctrlA?.runtime?.graph;
|
|
3385
|
+
if (g && g.flushAll) await g.flushAll();
|
|
3386
|
+
res.json({ ok: true });
|
|
3387
|
+
} catch (e) {
|
|
3388
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3389
|
+
}
|
|
3390
|
+
});
|
|
3391
|
+
|
|
3392
|
+
app.post('/api/graph/prefetch', async (req, res) => {
|
|
3393
|
+
try {
|
|
3394
|
+
const { node } = req.body || {};
|
|
3395
|
+
const g = global.ctrlA?.runtime?.graph;
|
|
3396
|
+
if (!node || !(g instanceof PartitionedGraphDB)) return res.status(400).json({ ok: false, error: 'node 必填/或非分区图' });
|
|
3397
|
+
await g.focusOnPoint(String(node));
|
|
3398
|
+
res.json({ ok: true });
|
|
3399
|
+
} catch (e) {
|
|
3400
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3401
|
+
}
|
|
3402
|
+
});
|
|
2458
3403
|
app.post('/api/chat', async (req, res) => {
|
|
2459
3404
|
try {
|
|
2460
|
-
const { message } = req.body;
|
|
2461
|
-
//
|
|
3405
|
+
const { message } = req.body || {};
|
|
3406
|
+
const ctrl = getNextController(); // 始终路由到当前服务实例
|
|
2462
3407
|
const words = typeof message === 'string'
|
|
2463
3408
|
? message.toLowerCase().split(/\s+/).filter(w => w.length > 0)
|
|
2464
3409
|
: [];
|
|
2465
|
-
const normWords =
|
|
3410
|
+
const normWords = (new Spider()).lemmatizeWords(words);
|
|
2466
3411
|
const normMessage = normWords.join(' ');
|
|
2467
|
-
|
|
2468
|
-
/// console.log(`[DEBUG] 收到消息: ${message}`);
|
|
2469
|
-
/// console.log(`[DEBUG] 归一化后: ${normMessage}`);
|
|
2470
|
-
|
|
2471
|
-
const response = await ctrlA.handleInput(normMessage);
|
|
2472
|
-
|
|
3412
|
+
const response = await ctrl.handleInput(normMessage);
|
|
2473
3413
|
if (!response || response.trim() === '') {
|
|
2474
|
-
console.warn('[WARN] AI
|
|
3414
|
+
console.warn('[WARN] AI响应为空');
|
|
2475
3415
|
}
|
|
2476
|
-
|
|
2477
|
-
// console.log(`[DEBUG] 响应: ${response}`);
|
|
2478
3416
|
res.json({ response });
|
|
2479
3417
|
} catch (error) {
|
|
2480
3418
|
console.error('[ERROR] 处理请求失败:', error);
|
|
@@ -2580,8 +3518,7 @@ async function main() {
|
|
|
2580
3518
|
// 参数校验(可根据需要扩展)
|
|
2581
3519
|
Object.assign(currentModelParams, updates);
|
|
2582
3520
|
applyModelParams(global.ctrlA?.runtime);
|
|
2583
|
-
|
|
2584
|
-
applyModelParams(global.ctrlC?.runtime);
|
|
3521
|
+
|
|
2585
3522
|
res.json({ success: true, params: currentModelParams });
|
|
2586
3523
|
} catch (err) {
|
|
2587
3524
|
res.status(500).json({ error: err.message });
|
|
@@ -2592,15 +3529,81 @@ async function main() {
|
|
|
2592
3529
|
try {
|
|
2593
3530
|
Object.assign(currentModelParams, modelDefaults);
|
|
2594
3531
|
applyModelParams(global.ctrlA?.runtime);
|
|
2595
|
-
|
|
2596
|
-
applyModelParams(global.ctrlC?.runtime);
|
|
3532
|
+
|
|
2597
3533
|
res.json({ success: true, params: currentModelParams });
|
|
2598
3534
|
} catch (err) {
|
|
2599
3535
|
res.status(500).json({ error: err.message });
|
|
2600
3536
|
}
|
|
2601
3537
|
});
|
|
2602
|
-
|
|
3538
|
+
app.get('/api/adversary/status', (req, res) => {
|
|
3539
|
+
try {
|
|
3540
|
+
res.json({ ok: true, status: __adv ? __adv.getStatus() : { running: false } });
|
|
3541
|
+
} catch (e) {
|
|
3542
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3543
|
+
}
|
|
3544
|
+
});
|
|
2603
3545
|
|
|
3546
|
+
app.post('/api/adversary/start', (req, res) => {
|
|
3547
|
+
try {
|
|
3548
|
+
if (!__adv) __adv = new AdversaryScheduler(global.ctrlA.runtime, { adjustParams: false });
|
|
3549
|
+
__adv.start();
|
|
3550
|
+
res.json({ ok: true, status: __adv.getStatus() });
|
|
3551
|
+
} catch (e) {
|
|
3552
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3553
|
+
}
|
|
3554
|
+
});
|
|
3555
|
+
|
|
3556
|
+
app.post('/api/adversary/stop', (req, res) => {
|
|
3557
|
+
try { __adv?.stop?.(); res.json({ ok: true }); }
|
|
3558
|
+
catch (e) { res.status(500).json({ ok: false, error: e.message }); }
|
|
3559
|
+
});
|
|
3560
|
+
// 新增:serve 侧参数调优 API(默认不启用自动调参,仅手动设置)
|
|
3561
|
+
app.get('/api/tune/get', (req, res) => {
|
|
3562
|
+
try {
|
|
3563
|
+
const rt = global.ctrlA?.runtime;
|
|
3564
|
+
if (!rt) return res.status(500).json({ ok: false, error: 'runtime missing' });
|
|
3565
|
+
res.json({
|
|
3566
|
+
ok: true,
|
|
3567
|
+
params: {
|
|
3568
|
+
decayK: rt.config?.decayK ?? 1,
|
|
3569
|
+
maxLen: rt.config?.maxLen ?? 16,
|
|
3570
|
+
spiderMix: rt.config?.spiderMix ?? { onlineWeight: 0.5, offlineWeight: 0.5 },
|
|
3571
|
+
crawler: {
|
|
3572
|
+
perQuery: global.__crawler?.__tune_perQuery ?? 8,
|
|
3573
|
+
maxCrawl: global.__crawler?.__tune_maxCrawl ?? 12
|
|
3574
|
+
}
|
|
3575
|
+
}
|
|
3576
|
+
});
|
|
3577
|
+
} catch (e) {
|
|
3578
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3579
|
+
}
|
|
3580
|
+
});
|
|
3581
|
+
|
|
3582
|
+
app.post('/api/tune/set', (req, res) => {
|
|
3583
|
+
try {
|
|
3584
|
+
const rt = global.ctrlA?.runtime;
|
|
3585
|
+
if (!rt) return res.status(500).json({ ok: false, error: 'runtime missing' });
|
|
3586
|
+
const snap = applyServeTunableParams(rt, req.body || {});
|
|
3587
|
+
res.json({ ok: true, snapshot: snap });
|
|
3588
|
+
} catch (e) {
|
|
3589
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3590
|
+
}
|
|
3591
|
+
});
|
|
3592
|
+
|
|
3593
|
+
// 可选:服务侧对抗调度保持 adjustParams: false,但允许设置 promptMode/targets
|
|
3594
|
+
// ...existing code...
|
|
3595
|
+
app.post('/api/adversary/start', (req, res) => {
|
|
3596
|
+
try {
|
|
3597
|
+
if (!__adv) __adv = new AdversaryScheduler(global.ctrlA.runtime, { adjustParams: false });
|
|
3598
|
+
const { promptMode, targetWeights } = req.body || {};
|
|
3599
|
+
if (promptMode) __adv.setPromptMode(promptMode);
|
|
3600
|
+
if (targetWeights) __adv.setTargets(targetWeights);
|
|
3601
|
+
__adv.start();
|
|
3602
|
+
res.json({ ok: true, status: __adv.getStatus() });
|
|
3603
|
+
} catch (e) {
|
|
3604
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3605
|
+
}
|
|
3606
|
+
});
|
|
2604
3607
|
|
|
2605
3608
|
// 设置退出保存
|
|
2606
3609
|
setupExitHandler();
|
|
@@ -2718,20 +3721,34 @@ async function plainObjToRuntime(runtime, obj) {
|
|
|
2718
3721
|
|
|
2719
3722
|
console.log('[MODEL SYNC] 模型同步完成');
|
|
2720
3723
|
}
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
if (
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
3724
|
+
function applyServeTunableParams(runtime, partial = {}) {
|
|
3725
|
+
if (!runtime) return null;
|
|
3726
|
+
runtime.config = runtime.config || {};
|
|
3727
|
+
if (partial.spiderMix) {
|
|
3728
|
+
const ow = Math.max(0, Math.min(1, Number(partial.spiderMix.onlineWeight ?? runtime.config.spiderMix?.onlineWeight ?? 0.5)));
|
|
3729
|
+
runtime.config.spiderMix = { onlineWeight: ow, offlineWeight: Math.max(0, Math.min(1, 1 - ow)) };
|
|
3730
|
+
}
|
|
3731
|
+
if (typeof partial.decayK === 'number') runtime.config.decayK = Math.max(0.1, Math.min(2.0, partial.decayK));
|
|
3732
|
+
if (typeof partial.maxLen === 'number') runtime.config.maxLen = Math.max(8, Math.min(64, Math.round(partial.maxLen)));
|
|
3733
|
+
if (typeof partial.edgeWeight === 'number' && runtime.graph) {
|
|
3734
|
+
for (const p of runtime.graph.getAllPoints()) for (const e of p.connect) e[0] = Math.max(0.1, Math.min(5, partial.edgeWeight));
|
|
3735
|
+
}
|
|
3736
|
+
if (global.__crawler) {
|
|
3737
|
+
if (typeof partial.perQuery === 'number') global.__crawler.__tune_perQuery = Math.max(2, Math.min(16, Math.round(partial.perQuery)));
|
|
3738
|
+
if (typeof partial.maxCrawl === 'number') global.__crawler.__tune_maxCrawl = Math.max(2, Math.min(24, Math.round(partial.maxCrawl)));
|
|
3739
|
+
}
|
|
3740
|
+
return {
|
|
3741
|
+
decayK: runtime.config.decayK,
|
|
3742
|
+
maxLen: runtime.config.maxLen,
|
|
3743
|
+
spiderMix: runtime.config.spiderMix || { onlineWeight: 0.5, offlineWeight: 0.5 },
|
|
3744
|
+
crawler: {
|
|
3745
|
+
perQuery: global.__crawler?.__tune_perQuery ?? 8,
|
|
3746
|
+
maxCrawl: global.__crawler?.__tune_maxCrawl ?? 12
|
|
2732
3747
|
}
|
|
2733
|
-
|
|
2734
|
-
|
|
3748
|
+
};
|
|
3749
|
+
}
|
|
3750
|
+
if (require.main === module) {
|
|
3751
|
+
|
|
2735
3752
|
main().catch(console.error)
|
|
2736
3753
|
}
|
|
2737
3754
|
|