079project 2.0.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/crawler/agent.cjs +97 -0
- package/crawler/index.cjs +515 -0
- package/crawler/storage.cjs +163 -0
- package/groupmanager.cjs +2 -1
- package/main_Serve.cjs +1136 -210
- package/main_Study.cjs +1584 -349
- package/package.json +2 -1
- package/robots/seeds.txt +2 -0
- package/schedule.cjs +745 -0
- package/todo-list.txt +0 -86
package/main_Serve.cjs
CHANGED
|
@@ -18,6 +18,7 @@ const csvParse = require('csv-parse/sync')
|
|
|
18
18
|
const os = require('os');
|
|
19
19
|
const MAX_WORKERS = Math.max(1, os.cpus().length - 1); // 留一个核心给主线程
|
|
20
20
|
const natural = require('natural');
|
|
21
|
+
const { AdversaryScheduler } = require('./schedule.cjs');
|
|
21
22
|
const STOP_WORDS = natural.stopwords; // 英文停用词
|
|
22
23
|
const pool = workerpool.pool(path.join(__dirname, 'memeMergeWorker.cjs'), {
|
|
23
24
|
minWorkers: 1,
|
|
@@ -32,27 +33,27 @@ protobuf.load(runtimeProtoPath, (err, root) => {
|
|
|
32
33
|
RuntimeMessage = root.lookupType('Runtime');
|
|
33
34
|
});
|
|
34
35
|
function parseArgs(argv) {
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
36
|
+
const out = {};
|
|
37
|
+
for (let i = 0; i < argv.length; i++) {
|
|
38
|
+
const a = argv[i];
|
|
39
|
+
if (a && a.startsWith('--')) {
|
|
40
|
+
const k = a.slice(2);
|
|
41
|
+
const v = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : true;
|
|
42
|
+
out[k] = v;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return out;
|
|
45
46
|
}
|
|
46
47
|
const __args = parseArgs(process.argv.slice(2));
|
|
47
48
|
global.config = {
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
49
|
+
masterPortOfSql: 3125,
|
|
50
|
+
masterPortOfMain: process.argv[2],
|
|
51
|
+
emitExitport: process.argv[3] || 8641,
|
|
52
|
+
groupId: Number(__args['group-id'] || -1),
|
|
53
|
+
forwarderPort: Number(__args['forwarder-port'] || 0),
|
|
54
|
+
studyPort: Number(__args['study-port'] || 0),
|
|
55
|
+
peerServePorts: String(__args['peers'] || '').split(',').map(s => Number(s)).filter(n => Number.isFinite(n) && n > 0),
|
|
56
|
+
isStudy: !!__args['study']
|
|
56
57
|
};
|
|
57
58
|
const modelDefaults = {
|
|
58
59
|
decayFactor: 0.5,
|
|
@@ -265,8 +266,8 @@ class SnapshotManager {
|
|
|
265
266
|
}
|
|
266
267
|
}
|
|
267
268
|
|
|
269
|
+
|
|
268
270
|
async createSnapshot(name = 'auto') {
|
|
269
|
-
// 防止并发创建
|
|
270
271
|
if (this.isCreatingSnapshot) {
|
|
271
272
|
console.log('[SNAPSHOT] 另一个快照正在创建中,跳过');
|
|
272
273
|
return null;
|
|
@@ -282,31 +283,39 @@ class SnapshotManager {
|
|
|
282
283
|
|
|
283
284
|
console.log(`[SNAPSHOT] 开始创建快照: ${snapshotId}`);
|
|
284
285
|
|
|
285
|
-
//
|
|
286
|
+
// 优先使用分区图的全量导出(避免仅导出窗口)
|
|
287
|
+
let memesAll = [];
|
|
288
|
+
if (this.runtime.graph && typeof this.runtime.graph.exportAllPoints === 'function') {
|
|
289
|
+
try {
|
|
290
|
+
memesAll = await this.runtime.graph.exportAllPoints();
|
|
291
|
+
} catch (e) {
|
|
292
|
+
console.warn('[SNAPSHOT] 分区图导出失败,回退窗口:', e.message);
|
|
293
|
+
memesAll = this.runtime.graph.getAllPoints();
|
|
294
|
+
}
|
|
295
|
+
} else {
|
|
296
|
+
memesAll = this.runtime.graph.getAllPoints();
|
|
297
|
+
}
|
|
298
|
+
|
|
286
299
|
const snapshotData = {
|
|
287
300
|
id: snapshotId,
|
|
288
301
|
timestamp,
|
|
289
302
|
name,
|
|
290
303
|
createDate: new Date().toISOString(),
|
|
291
|
-
memes:
|
|
304
|
+
memes: memesAll,
|
|
292
305
|
wordGraph: Array.from(this.runtime.wordGraph.points.values()),
|
|
293
|
-
kvm: this.runtime.kvm.
|
|
306
|
+
kvm: Array.from(this.runtime.kvm.memory.entries()),
|
|
294
307
|
vocab: this.runtime.vocabManager.vocab,
|
|
295
|
-
wordAccessLog: Array.from(this.runtime.wordAccessLog
|
|
308
|
+
wordAccessLog: Array.from(this.runtime.wordAccessLog.entries()).map(([w, per]) =>
|
|
309
|
+
[w, per instanceof Map ? Array.from(per.entries()) : (Array.isArray(per) ? [['legacy', per.length]] : [])]
|
|
310
|
+
),
|
|
311
|
+
sessions: this.runtime.session.export()
|
|
296
312
|
};
|
|
297
313
|
|
|
298
|
-
// 写入临时文件,然后原子重命名以确保数据完整性
|
|
299
314
|
const tempPath = `${filePath}.temp`;
|
|
300
315
|
await fs.promises.writeFile(tempPath, JSON.stringify(snapshotData), 'utf-8');
|
|
301
316
|
await fs.promises.rename(tempPath, filePath);
|
|
302
317
|
|
|
303
|
-
|
|
304
|
-
const snapshotInfo = {
|
|
305
|
-
id: snapshotId,
|
|
306
|
-
timestamp,
|
|
307
|
-
name,
|
|
308
|
-
path: filePath
|
|
309
|
-
};
|
|
318
|
+
const snapshotInfo = { id: snapshotId, timestamp, name, path: filePath };
|
|
310
319
|
this.snapshotList.unshift(snapshotInfo);
|
|
311
320
|
|
|
312
321
|
console.timeEnd('snapshotCreation');
|
|
@@ -320,11 +329,11 @@ class SnapshotManager {
|
|
|
320
329
|
}
|
|
321
330
|
}
|
|
322
331
|
|
|
332
|
+
|
|
323
333
|
async restoreSnapshot(snapshotId) {
|
|
324
334
|
console.log(`[SNAPSHOT] 开始从快照恢复: ${snapshotId}`);
|
|
325
335
|
console.time('snapshotRestore');
|
|
326
336
|
|
|
327
|
-
// 查找快照
|
|
328
337
|
const snapshot = this.snapshotList.find(s => s.id === snapshotId);
|
|
329
338
|
if (!snapshot) {
|
|
330
339
|
console.error(`[SNAPSHOT] 快照不存在: ${snapshotId}`);
|
|
@@ -332,37 +341,27 @@ class SnapshotManager {
|
|
|
332
341
|
}
|
|
333
342
|
|
|
334
343
|
try {
|
|
335
|
-
// 读取快照文件
|
|
336
|
-
console.log(`[SNAPSHOT] 从文件读取数据: ${snapshot.path}`);
|
|
337
344
|
const dataStr = await fs.promises.readFile(snapshot.path, 'utf-8');
|
|
338
345
|
const data = JSON.parse(dataStr);
|
|
339
346
|
|
|
340
|
-
// 在恢复前创建自动备份
|
|
341
347
|
await this.createSnapshot(`auto_before_restore_${snapshotId}`);
|
|
342
348
|
|
|
343
|
-
//
|
|
344
|
-
console.log('[SNAPSHOT] 清空当前运行时...');
|
|
345
|
-
this.runtime.graph = new GraphDB();
|
|
349
|
+
// 清空当前运行时(词图/KVM 内存)
|
|
346
350
|
this.runtime.wordGraph = new GraphDB();
|
|
347
351
|
this.runtime.kvm = new KVM();
|
|
348
352
|
this.runtime.wordAccessLog = new Map();
|
|
349
353
|
|
|
350
|
-
//
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
this.runtime.graph.addPoint(point.pointID, point.connect);
|
|
358
|
-
}
|
|
359
|
-
// 让事件循环有机会处理其他事件
|
|
360
|
-
await new Promise(resolve => setImmediate(resolve));
|
|
354
|
+
// 恢复模因图:走分区导入(覆盖分区存储)
|
|
355
|
+
if (data.memes && this.runtime.graph && typeof this.runtime.graph.importAllPoints === 'function') {
|
|
356
|
+
await this.runtime.graph.importAllPoints(data.memes);
|
|
357
|
+
} else if (data.memes) {
|
|
358
|
+
// 窗口回退(不推荐)
|
|
359
|
+
for (const point of data.memes) {
|
|
360
|
+
await this.runtime.graph.addPoint(point.pointID, point.connect);
|
|
361
361
|
}
|
|
362
362
|
}
|
|
363
363
|
|
|
364
364
|
// 恢复词图
|
|
365
|
-
console.log('[SNAPSHOT] 恢复词语网络...');
|
|
366
365
|
if (data.wordGraph) {
|
|
367
366
|
const BATCH_SIZE = 1000;
|
|
368
367
|
for (let i = 0; i < data.wordGraph.length; i += BATCH_SIZE) {
|
|
@@ -375,29 +374,39 @@ class SnapshotManager {
|
|
|
375
374
|
}
|
|
376
375
|
|
|
377
376
|
// 恢复KVM
|
|
378
|
-
console.log('[SNAPSHOT] 恢复键值存储...');
|
|
379
377
|
if (data.kvm) {
|
|
380
378
|
const BATCH_SIZE = 1000;
|
|
381
379
|
for (let i = 0; i < data.kvm.length; i += BATCH_SIZE) {
|
|
382
380
|
const batch = data.kvm.slice(i, i + BATCH_SIZE);
|
|
383
|
-
for (const [k, v] of batch)
|
|
384
|
-
this.runtime.kvm.set(k, v);
|
|
385
|
-
}
|
|
381
|
+
for (const [k, v] of batch) this.runtime.kvm.set(k, v);
|
|
386
382
|
await new Promise(resolve => setImmediate(resolve));
|
|
387
383
|
}
|
|
388
384
|
}
|
|
389
385
|
|
|
390
386
|
// 恢复词表
|
|
391
|
-
console.log('[SNAPSHOT] 恢复词表...');
|
|
392
387
|
if (data.vocab) {
|
|
393
388
|
this.runtime.vocabManager.vocab = data.vocab;
|
|
394
389
|
this.runtime.vocabManager.updateMappings();
|
|
395
390
|
}
|
|
396
391
|
|
|
397
392
|
// 恢复词访问日志
|
|
398
|
-
console.log('[SNAPSHOT] 恢复词访问日志...');
|
|
399
393
|
if (data.wordAccessLog) {
|
|
400
|
-
|
|
394
|
+
const restored = new Map();
|
|
395
|
+
for (const [word, per] of data.wordAccessLog) {
|
|
396
|
+
if (Array.isArray(per) && per.length > 0 && Array.isArray(per[0])) {
|
|
397
|
+
restored.set(word, new Map(per));
|
|
398
|
+
} else if (Array.isArray(per)) {
|
|
399
|
+
restored.set(word, new Map([['legacy', per.length]]));
|
|
400
|
+
} else {
|
|
401
|
+
restored.set(word, new Map());
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
this.runtime.wordAccessLog = restored;
|
|
405
|
+
}
|
|
406
|
+
if (data.sessions) {
|
|
407
|
+
this.runtime.session.import(data.sessions);
|
|
408
|
+
} else {
|
|
409
|
+
this.runtime.session.startNewSession({ reason: 'snapshot-legacy' });
|
|
401
410
|
}
|
|
402
411
|
|
|
403
412
|
console.timeEnd('snapshotRestore');
|
|
@@ -715,6 +724,727 @@ class GraphDB {
|
|
|
715
724
|
}
|
|
716
725
|
}
|
|
717
726
|
}
|
|
727
|
+
// ...existing code...
|
|
728
|
+
const crypto = require('crypto');
|
|
729
|
+
// ...existing code...
|
|
730
|
+
|
|
731
|
+
// ========================== 分区图存储适配层与滑动窗口 ==========================
|
|
732
|
+
|
|
733
|
+
// 简易日志辅助
|
|
734
|
+
function logPart(...args) { console.log('[PART]', ...args); }
|
|
735
|
+
function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
|
|
736
|
+
|
|
737
|
+
// 存储适配层(FS/LMDB/Level 多后端,按需加载)
|
|
738
|
+
class GraphStorageAdapter {
|
|
739
|
+
constructor({ baseDir, backend = 'fs' } = {}) {
|
|
740
|
+
this.baseDir = baseDir || path.join(__dirname, 'graph_parts');
|
|
741
|
+
this.backend = backend;
|
|
742
|
+
this.ready = false;
|
|
743
|
+
|
|
744
|
+
// 尝试创建目录
|
|
745
|
+
fs.mkdirSync(this.baseDir, { recursive: true });
|
|
746
|
+
|
|
747
|
+
// 可选依赖
|
|
748
|
+
this.lmdb = null;
|
|
749
|
+
this.level = null;
|
|
750
|
+
|
|
751
|
+
if (backend === 'lmdb') {
|
|
752
|
+
try {
|
|
753
|
+
this.lmdb = require('lmdb');
|
|
754
|
+
} catch (e) {
|
|
755
|
+
console.warn('[PART][ADAPTER] LMDB 不可用,降级为 FS:', e.message);
|
|
756
|
+
this.backend = 'fs';
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
if (backend === 'level') {
|
|
760
|
+
try {
|
|
761
|
+
this.level = require('level');
|
|
762
|
+
} catch (e) {
|
|
763
|
+
console.warn('[PART][ADAPTER] level 不可用,降级为 FS:', e.message);
|
|
764
|
+
this.backend = 'fs';
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
// 初始化后端
|
|
769
|
+
this._initBackend();
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
_initBackend() {
|
|
773
|
+
if (this.backend === 'fs') {
|
|
774
|
+
// FS: 每个分区一个 .jsonl(节点),边界事件一个独立 .jsonl
|
|
775
|
+
this.ready = true;
|
|
776
|
+
return;
|
|
777
|
+
}
|
|
778
|
+
if (this.backend === 'lmdb' && this.lmdb) {
|
|
779
|
+
try {
|
|
780
|
+
const storeDir = path.join(this.baseDir, 'lmdb');
|
|
781
|
+
fs.mkdirSync(storeDir, { recursive: true });
|
|
782
|
+
this.env = this.lmdb.open({
|
|
783
|
+
path: storeDir,
|
|
784
|
+
mapSize: 1024n * 1024n * 1024n * 64n,
|
|
785
|
+
compression: true,
|
|
786
|
+
});
|
|
787
|
+
this.ready = true;
|
|
788
|
+
} catch (e) {
|
|
789
|
+
console.warn('[PART][ADAPTER] LMDB 初始化失败,降级 FS:', e.message);
|
|
790
|
+
this.backend = 'fs';
|
|
791
|
+
this.ready = true;
|
|
792
|
+
}
|
|
793
|
+
return;
|
|
794
|
+
}
|
|
795
|
+
if (this.backend === 'level' && this.level) {
|
|
796
|
+
try {
|
|
797
|
+
const dbDir = path.join(this.baseDir, 'leveldb');
|
|
798
|
+
fs.mkdirSync(dbDir, { recursive: true });
|
|
799
|
+
this.db = new this.level.Level(dbDir, { valueEncoding: 'json' });
|
|
800
|
+
this.ready = true;
|
|
801
|
+
} catch (e) {
|
|
802
|
+
console.warn('[PART][ADAPTER] level 初始化失败,降级 FS:', e.message);
|
|
803
|
+
this.backend = 'fs';
|
|
804
|
+
this.ready = true;
|
|
805
|
+
}
|
|
806
|
+
return;
|
|
807
|
+
}
|
|
808
|
+
this.ready = true;
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
// 分区文件名(FS)
|
|
812
|
+
_partFile(pid) { return path.join(this.baseDir, `p_${pid}.jsonl`); }
|
|
813
|
+
_eventFile(pid) { return path.join(this.baseDir, `p_${pid}.events.jsonl`); }
|
|
814
|
+
|
|
815
|
+
// 读取分区(返回 { points: Map<string,{pointID,connect:[]}> })
|
|
816
|
+
async loadPartition(pid) {
|
|
817
|
+
if (this.backend === 'fs') {
|
|
818
|
+
const file = this._partFile(pid);
|
|
819
|
+
const out = new Map();
|
|
820
|
+
if (!fs.existsSync(file)) return { points: out };
|
|
821
|
+
const rs = fs.createReadStream(file, { encoding: 'utf-8' });
|
|
822
|
+
let buf = '';
|
|
823
|
+
for await (const chunk of rs) {
|
|
824
|
+
buf += chunk;
|
|
825
|
+
let idx;
|
|
826
|
+
while ((idx = buf.indexOf('\n')) >= 0) {
|
|
827
|
+
const line = buf.slice(0, idx);
|
|
828
|
+
buf = buf.slice(idx + 1);
|
|
829
|
+
if (!line.trim()) continue;
|
|
830
|
+
try {
|
|
831
|
+
const obj = JSON.parse(line);
|
|
832
|
+
if (obj && obj.pointID) {
|
|
833
|
+
out.set(obj.pointID, { pointID: obj.pointID, connect: obj.connect || [] });
|
|
834
|
+
}
|
|
835
|
+
} catch { /* ignore */ }
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
return { points: out };
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
if (this.backend === 'lmdb' && this.env) {
|
|
842
|
+
const points = new Map();
|
|
843
|
+
const txn = this.env.beginTxn({ readOnly: true });
|
|
844
|
+
try {
|
|
845
|
+
const cursor = new this.lmdb.Cursors.Cursor(txn, this.env.openDB({ name: `p_${pid}`, create: true }));
|
|
846
|
+
for (let found = cursor.goToFirst(); found; found = cursor.goToNext()) {
|
|
847
|
+
const key = cursor.getCurrentString();
|
|
848
|
+
const val = cursor.getCurrentBinary();
|
|
849
|
+
try {
|
|
850
|
+
const obj = JSON.parse(Buffer.from(val).toString('utf-8'));
|
|
851
|
+
if (obj && obj.pointID) points.set(obj.pointID, obj);
|
|
852
|
+
} catch { }
|
|
853
|
+
}
|
|
854
|
+
cursor.close();
|
|
855
|
+
} catch { }
|
|
856
|
+
txn.abort();
|
|
857
|
+
return { points };
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
if (this.backend === 'level' && this.db) {
|
|
861
|
+
const points = new Map();
|
|
862
|
+
try {
|
|
863
|
+
for await (const { key, value } of this.db.iterator({ gte: `p:${pid}:`, lt: `p:${pid};` })) {
|
|
864
|
+
const obj = value;
|
|
865
|
+
if (obj && obj.pointID) points.set(obj.pointID, obj);
|
|
866
|
+
}
|
|
867
|
+
} catch { }
|
|
868
|
+
return { points };
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
return { points: new Map() };
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
// 保存分区(全量覆盖写)
|
|
875
|
+
async savePartition(pid, pointsMap) {
|
|
876
|
+
if (!(pointsMap instanceof Map)) return;
|
|
877
|
+
if (this.backend === 'fs') {
|
|
878
|
+
const file = this._partFile(pid);
|
|
879
|
+
const tmp = `${file}.tmp`;
|
|
880
|
+
const ws = fs.createWriteStream(tmp, { encoding: 'utf-8' });
|
|
881
|
+
for (const [, p] of pointsMap.entries()) {
|
|
882
|
+
ws.write(JSON.stringify({ pointID: p.pointID, connect: p.connect || [] }) + '\n');
|
|
883
|
+
}
|
|
884
|
+
await new Promise((res, rej) => ws.end(res));
|
|
885
|
+
await fs.promises.rename(tmp, file);
|
|
886
|
+
return;
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
if (this.backend === 'lmdb' && this.env) {
|
|
890
|
+
const dbi = this.env.openDB({ name: `p_${pid}`, create: true });
|
|
891
|
+
const txn = this.env.beginTxn();
|
|
892
|
+
try {
|
|
893
|
+
// 先清空:简化实现
|
|
894
|
+
const cur = new this.lmdb.Cursors.Cursor(txn, dbi);
|
|
895
|
+
for (let found = cur.goToFirst(); found; found = cur.goToNext()) {
|
|
896
|
+
const k = cur.getCurrentString();
|
|
897
|
+
txn.del(dbi, k);
|
|
898
|
+
}
|
|
899
|
+
cur.close();
|
|
900
|
+
for (const [, p] of pointsMap.entries()) {
|
|
901
|
+
txn.put(dbi, p.pointID, JSON.stringify(p));
|
|
902
|
+
}
|
|
903
|
+
txn.commit();
|
|
904
|
+
} catch (e) {
|
|
905
|
+
try { txn.abort(); } catch { }
|
|
906
|
+
console.warn('[PART][ADAPTER][LMDB] savePartition err:', e.message);
|
|
907
|
+
}
|
|
908
|
+
return;
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
if (this.backend === 'level' && this.db) {
|
|
912
|
+
const ops = [];
|
|
913
|
+
// 简化:清理旧 key 不容易,直接覆盖同 key
|
|
914
|
+
for (const [, p] of pointsMap.entries()) {
|
|
915
|
+
ops.push({ type: 'put', key: `p:${pid}:${p.pointID}`, value: p });
|
|
916
|
+
}
|
|
917
|
+
await this.db.batch(ops);
|
|
918
|
+
return;
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
// 追加边界事件(跨分区边)
|
|
923
|
+
async appendEdgeEvent(pid, event) {
|
|
924
|
+
if (!event || !event.type) return;
|
|
925
|
+
if (this.backend === 'fs') {
|
|
926
|
+
const file = this._eventFile(pid);
|
|
927
|
+
fs.appendFileSync(file, JSON.stringify(event) + '\n', 'utf-8');
|
|
928
|
+
return;
|
|
929
|
+
}
|
|
930
|
+
if (this.backend === 'lmdb' && this.env) {
|
|
931
|
+
const dbi = this.env.openDB({ name: `e_${pid}`, create: true });
|
|
932
|
+
const txn = this.env.beginTxn();
|
|
933
|
+
try {
|
|
934
|
+
const key = `e:${Date.now()}_${Math.random().toString(36).slice(2, 10)}`;
|
|
935
|
+
txn.put(dbi, key, JSON.stringify(event));
|
|
936
|
+
txn.commit();
|
|
937
|
+
} catch (e) {
|
|
938
|
+
try { txn.abort(); } catch { }
|
|
939
|
+
}
|
|
940
|
+
return;
|
|
941
|
+
}
|
|
942
|
+
if (this.backend === 'level' && this.db) {
|
|
943
|
+
const key = `e:${pid}:${Date.now()}_${Math.random().toString(36).slice(2, 10)}`;
|
|
944
|
+
await this.db.put(key, event);
|
|
945
|
+
return;
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
// 读取并消费边界事件(与该分区相关的)
|
|
950
|
+
async consumeEdgeEvents(pid, filterFn = null, limit = 2000) {
|
|
951
|
+
const events = [];
|
|
952
|
+
if (this.backend === 'fs') {
|
|
953
|
+
const file = this._eventFile(pid);
|
|
954
|
+
if (!fs.existsSync(file)) return events;
|
|
955
|
+
|
|
956
|
+
const tmp = `${file}.tmp`;
|
|
957
|
+
// 将不消费的事件写入 tmp,再覆盖原文件;已消费事件返回
|
|
958
|
+
const lines = fs.readFileSync(file, 'utf-8').split(/\r?\n/).filter(Boolean);
|
|
959
|
+
const remain = [];
|
|
960
|
+
for (const line of lines) {
|
|
961
|
+
try {
|
|
962
|
+
const e = JSON.parse(line);
|
|
963
|
+
const ok = filterFn ? filterFn(e) : true;
|
|
964
|
+
if (ok && events.length < limit) {
|
|
965
|
+
events.push(e);
|
|
966
|
+
} else {
|
|
967
|
+
remain.push(line);
|
|
968
|
+
}
|
|
969
|
+
} catch {
|
|
970
|
+
remain.push(line);
|
|
971
|
+
}
|
|
972
|
+
}
|
|
973
|
+
fs.writeFileSync(tmp, remain.join('\n') + (remain.length ? '\n' : ''), 'utf-8');
|
|
974
|
+
await fs.promises.rename(tmp, file);
|
|
975
|
+
return events;
|
|
976
|
+
}
|
|
977
|
+
|
|
978
|
+
if (this.backend === 'lmdb' && this.env) {
|
|
979
|
+
const dbi = this.env.openDB({ name: `e_${pid}`, create: true });
|
|
980
|
+
const txn = this.env.beginTxn();
|
|
981
|
+
const toDel = [];
|
|
982
|
+
try {
|
|
983
|
+
const cur = new this.lmdb.Cursors.Cursor(txn, dbi);
|
|
984
|
+
for (let found = cur.goToFirst(); found; found = cur.goToNext()) {
|
|
985
|
+
const k = cur.getCurrentString();
|
|
986
|
+
const v = cur.getCurrentBinary();
|
|
987
|
+
const e = JSON.parse(Buffer.from(v).toString('utf-8'));
|
|
988
|
+
const ok = filterFn ? filterFn(e) : true;
|
|
989
|
+
if (ok && events.length < limit) {
|
|
990
|
+
events.push(e);
|
|
991
|
+
toDel.push(k);
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
cur.close();
|
|
995
|
+
for (const k of toDel) txn.del(dbi, k);
|
|
996
|
+
txn.commit();
|
|
997
|
+
} catch (e) {
|
|
998
|
+
try { txn.abort(); } catch { }
|
|
999
|
+
}
|
|
1000
|
+
return events;
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
if (this.backend === 'level' && this.db) {
|
|
1004
|
+
// 简化:扫描全库 keys 读取该 pid 的事件
|
|
1005
|
+
try {
|
|
1006
|
+
const toDel = [];
|
|
1007
|
+
for await (const { key, value } of this.db.iterator({ gte: `e:${pid}:`, lt: `e:${pid};` })) {
|
|
1008
|
+
const ok = filterFn ? filterFn(value) : true;
|
|
1009
|
+
if (ok && events.length < limit) {
|
|
1010
|
+
events.push(value);
|
|
1011
|
+
toDel.push(key);
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
1014
|
+
// 删除已消费
|
|
1015
|
+
const ops = toDel.map(k => ({ type: 'del', key: k }));
|
|
1016
|
+
if (ops.length) await this.db.batch(ops);
|
|
1017
|
+
} catch { }
|
|
1018
|
+
return events;
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
return events;
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
// 枚举所有分区 ID(FS 模式)
|
|
1025
|
+
async listPartitionIds() {
|
|
1026
|
+
if (this.backend === 'fs') {
|
|
1027
|
+
const files = fs.readdirSync(this.baseDir).filter(f => /^p_\d+\.jsonl$/.test(f));
|
|
1028
|
+
const ids = files.map(f => Number(f.match(/^p_(\d+)\.jsonl$/)[1])).sort((a, b) => a - b);
|
|
1029
|
+
return ids;
|
|
1030
|
+
}
|
|
1031
|
+
// LMDB/level 不易列举,约定 0..N-1 尝试加载
|
|
1032
|
+
return [];
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
// 分区器(哈希 -> 分区ID)
|
|
1037
|
+
class GraphPartitioner {
|
|
1038
|
+
constructor({ partitions = 64 } = {}) {
|
|
1039
|
+
this.partitions = Math.max(4, partitions);
|
|
1040
|
+
}
|
|
1041
|
+
idOf(pointID) {
|
|
1042
|
+
if (!pointID) return 0;
|
|
1043
|
+
const h = crypto.createHash('sha1').update(String(pointID)).digest();
|
|
1044
|
+
// 使用前 4 字节构造 uint32
|
|
1045
|
+
const u32 = h.readUInt32BE(0);
|
|
1046
|
+
return u32 % this.partitions;
|
|
1047
|
+
}
|
|
1048
|
+
neighborsOf(pid, radius = 1) {
|
|
1049
|
+
const out = new Set([pid]);
|
|
1050
|
+
for (let r = 1; r <= radius; r++) {
|
|
1051
|
+
out.add((pid - r + this.partitions) % this.partitions);
|
|
1052
|
+
out.add((pid + r) % this.partitions);
|
|
1053
|
+
}
|
|
1054
|
+
return Array.from(out).sort((a, b) => a - b);
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
// 分区图 + 滑动窗口 + 边界事件消费
|
|
1059
|
+
class PartitionedGraphDB {
|
|
1060
|
+
constructor({
|
|
1061
|
+
partitions = 64,
|
|
1062
|
+
maxLoadedPartitions = 8,
|
|
1063
|
+
windowRadius = 1,
|
|
1064
|
+
baseDir = path.join(__dirname, 'graph_parts'),
|
|
1065
|
+
backend = 'fs'
|
|
1066
|
+
} = {}) {
|
|
1067
|
+
this.partitioner = new GraphPartitioner({ partitions });
|
|
1068
|
+
this.adapter = new GraphStorageAdapter({ baseDir, backend });
|
|
1069
|
+
this.maxLoadedPartitions = Math.max(2, maxLoadedPartitions);
|
|
1070
|
+
this.windowRadius = Math.max(0, windowRadius);
|
|
1071
|
+
|
|
1072
|
+
// 已加载分区:pid -> { points: Map, dirty, lastAccess }
|
|
1073
|
+
this.loaded = new Map();
|
|
1074
|
+
// 兼容旧代码:合并视图(仅包含已加载分区的点)
|
|
1075
|
+
this.points = new Map();
|
|
1076
|
+
// LRU
|
|
1077
|
+
this.accessTick = 0;
|
|
1078
|
+
this.centerPid = null;
|
|
1079
|
+
|
|
1080
|
+
// 并发保护
|
|
1081
|
+
this.loading = new Set();
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
// ---------- 内部:加载/保存/淘汰 ----------
|
|
1085
|
+
async ensureLoaded(pid) {
|
|
1086
|
+
if (this.loaded.has(pid)) {
|
|
1087
|
+
this._touch(pid);
|
|
1088
|
+
return this.loaded.get(pid);
|
|
1089
|
+
}
|
|
1090
|
+
if (this.loading.has(pid)) {
|
|
1091
|
+
// 等待已有加载完成
|
|
1092
|
+
while (this.loading.has(pid)) { await sleep(10); }
|
|
1093
|
+
return this.loaded.get(pid);
|
|
1094
|
+
}
|
|
1095
|
+
this.loading.add(pid);
|
|
1096
|
+
try {
|
|
1097
|
+
const part = await this.adapter.loadPartition(pid);
|
|
1098
|
+
const bundle = {
|
|
1099
|
+
points: part.points || new Map(),
|
|
1100
|
+
dirty: false,
|
|
1101
|
+
lastAccess: ++this.accessTick
|
|
1102
|
+
};
|
|
1103
|
+
this.loaded.set(pid, bundle);
|
|
1104
|
+
// 合并到全局视图
|
|
1105
|
+
for (const [id, p] of bundle.points.entries()) this.points.set(id, p);
|
|
1106
|
+
|
|
1107
|
+
// 消费边界事件:把指向本分区的事件落库
|
|
1108
|
+
const events = await this.adapter.consumeEdgeEvents(pid, (e) =>
|
|
1109
|
+
e && e.type === 'cross-edge' && (e.toPid === pid || e.fromPid === pid), 5000);
|
|
1110
|
+
if (events.length) {
|
|
1111
|
+
for (const e of events) this._applyEdgeEvent(bundle, e);
|
|
1112
|
+
bundle.dirty = true;
|
|
1113
|
+
}
|
|
1114
|
+
|
|
1115
|
+
// 控制内存:若超容量,执行淘汰
|
|
1116
|
+
await this._evictIfNeeded();
|
|
1117
|
+
return bundle;
|
|
1118
|
+
} finally {
|
|
1119
|
+
this.loading.delete(pid);
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
async savePartitionIfDirty(pid) {
|
|
1124
|
+
const entry = this.loaded.get(pid);
|
|
1125
|
+
if (!entry) return;
|
|
1126
|
+
if (!entry.dirty) return;
|
|
1127
|
+
await this.adapter.savePartition(pid, entry.points);
|
|
1128
|
+
entry.dirty = false;
|
|
1129
|
+
}
|
|
1130
|
+
|
|
1131
|
+
async _evictIfNeeded() {
|
|
1132
|
+
if (this.loaded.size <= this.maxLoadedPartitions) return;
|
|
1133
|
+
// 淘汰最近最少访问的分区(除中心窗口)
|
|
1134
|
+
const avoid = new Set(this.partitioner.neighborsOf(this.centerPid ?? 0, this.windowRadius));
|
|
1135
|
+
// 构建按 lastAccess 升序
|
|
1136
|
+
const list = Array.from(this.loaded.entries())
|
|
1137
|
+
.filter(([pid]) => !avoid.has(pid))
|
|
1138
|
+
.sort((a, b) => a[1].lastAccess - b[1].lastAccess);
|
|
1139
|
+
while (this.loaded.size > this.maxLoadedPartitions && list.length) {
|
|
1140
|
+
const [pid, entry] = list.shift();
|
|
1141
|
+
await this.savePartitionIfDirty(pid);
|
|
1142
|
+
// 从全局视图移除
|
|
1143
|
+
for (const [id] of entry.points.entries()) this.points.delete(id);
|
|
1144
|
+
this.loaded.delete(pid);
|
|
1145
|
+
logPart('evicted partition', pid);
|
|
1146
|
+
}
|
|
1147
|
+
}
|
|
1148
|
+
|
|
1149
|
+
_touch(pid) {
|
|
1150
|
+
const entry = this.loaded.get(pid);
|
|
1151
|
+
if (entry) entry.lastAccess = ++this.accessTick;
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
_applyEdgeEvent(targetBundle, e) {
|
|
1155
|
+
// 事件格式:{ type:'cross-edge', from:'id', to:'id', weight, direction, fromPid, toPid }
|
|
1156
|
+
if (!e || e.type !== 'cross-edge') return;
|
|
1157
|
+
const ensurePoint = (m, id) => {
|
|
1158
|
+
if (!m.has(id)) m.set(id, { pointID: id, connect: [] });
|
|
1159
|
+
return m.get(id);
|
|
1160
|
+
};
|
|
1161
|
+
const mp = targetBundle.points;
|
|
1162
|
+
const pFrom = ensurePoint(mp, e.from);
|
|
1163
|
+
const pTo = ensurePoint(mp, e.to);
|
|
1164
|
+
// 在 from 中落边(若 from 属于本分区)
|
|
1165
|
+
if (e.toPid === e.fromPid) {
|
|
1166
|
+
// 同分区事件(理论上不会在事件日志里)
|
|
1167
|
+
if (!pFrom.connect.some(([w, id, d]) => id === e.to && d === e.direction)) {
|
|
1168
|
+
pFrom.connect.push([e.weight, e.to, e.direction]);
|
|
1169
|
+
}
|
|
1170
|
+
} else {
|
|
1171
|
+
// 当前 bundle 即为 toPid 或 fromPid 的载体
|
|
1172
|
+
if (e.toPid === this.partitioner.idOf(pTo.pointID)) {
|
|
1173
|
+
// 对于目标分区,至少要保证可被 selectPath 遍历;保留边终点即可(可选:反向提示边)
|
|
1174
|
+
// 不在 pTo 里写边(避免双写),仅保证 from 的边会在 from 分区生效
|
|
1175
|
+
}
|
|
1176
|
+
if (e.fromPid === this.partitioner.idOf(pFrom.pointID)) {
|
|
1177
|
+
if (!pFrom.connect.some(([w, id, d]) => id === e.to && d === e.direction)) {
|
|
1178
|
+
pFrom.connect.push([e.weight, e.to, e.direction]);
|
|
1179
|
+
}
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
// ---------- 滑动窗口 ----------
|
|
1185
|
+
async focusOnPoint(pointID) {
|
|
1186
|
+
const pid = this.partitioner.idOf(pointID);
|
|
1187
|
+
this.centerPid = pid;
|
|
1188
|
+
const toLoad = this.partitioner.neighborsOf(pid, this.windowRadius);
|
|
1189
|
+
for (const id of toLoad) await this.ensureLoaded(id);
|
|
1190
|
+
await this._evictIfNeeded();
|
|
1191
|
+
}
|
|
1192
|
+
|
|
1193
|
+
// ---------- 兼容 API:点/边 操作 ----------
|
|
1194
|
+
addPoint(pointID, connect = []) {
|
|
1195
|
+
const pid = this.partitioner.idOf(pointID);
|
|
1196
|
+
const ensure = (bundle) => {
|
|
1197
|
+
if (!bundle.points.has(pointID)) bundle.points.set(pointID, { pointID, connect: [] });
|
|
1198
|
+
this.points.set(pointID, bundle.points.get(pointID));
|
|
1199
|
+
return bundle.points.get(pointID);
|
|
1200
|
+
};
|
|
1201
|
+
return this.ensureLoaded(pid).then(bundle => {
|
|
1202
|
+
const p = ensure(bundle);
|
|
1203
|
+
// 添加本地边;跨分区写事件
|
|
1204
|
+
for (const [w, nid, dir] of connect) this._addEdgeInternal(pid, p, w, nid, dir, bundle);
|
|
1205
|
+
bundle.dirty = true;
|
|
1206
|
+
});
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
_addEdgeInternal(fromPid, fromPoint, weight, toID, direction, bundleOfFrom) {
|
|
1210
|
+
const toPid = this.partitioner.idOf(toID);
|
|
1211
|
+
const w = (typeof weight === 'number' && isFinite(weight)) ? weight : 1;
|
|
1212
|
+
const d = (direction === 0 || direction === 1 || direction === 2) ? direction : 0;
|
|
1213
|
+
|
|
1214
|
+
if (toPid === fromPid) {
|
|
1215
|
+
// 同分区直接写
|
|
1216
|
+
if (!fromPoint.connect.some(([ww, id, dd]) => id === toID && dd === d)) {
|
|
1217
|
+
fromPoint.connect.push([w, toID, d]);
|
|
1218
|
+
bundleOfFrom.dirty = true;
|
|
1219
|
+
}
|
|
1220
|
+
} else {
|
|
1221
|
+
// 跨分区 -> 记录边界事件至 fromPid(或 toPid 都可,这里记录到 fromPid,toPid 加载时也会消费相关事件)
|
|
1222
|
+
this.adapter.appendEdgeEvent(fromPid, {
|
|
1223
|
+
type: 'cross-edge',
|
|
1224
|
+
from: fromPoint.pointID,
|
|
1225
|
+
to: toID,
|
|
1226
|
+
weight: w,
|
|
1227
|
+
direction: d,
|
|
1228
|
+
fromPid,
|
|
1229
|
+
toPid
|
|
1230
|
+
});
|
|
1231
|
+
// 同时对“已加载且包含 toPid 的 bundle”进行即时应用(若存在)
|
|
1232
|
+
const toBundle = this.loaded.get(toPid);
|
|
1233
|
+
if (toBundle) {
|
|
1234
|
+
// 在 from 分区已经写入 from->to 事件;对于 to 分区无需写边(避免双写),可选择记录提示(此处略)
|
|
1235
|
+
}
|
|
1236
|
+
}
|
|
1237
|
+
}
|
|
1238
|
+
|
|
1239
|
+
addBidirectionalEdge(id1, id2, weight = 1) {
|
|
1240
|
+
return this.addEdge(id1, id2, weight, 0);
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
async addEdge(fromID, toID, weight = 1, direction = 0) {
|
|
1244
|
+
const fromPid = this.partitioner.idOf(fromID);
|
|
1245
|
+
const fromBundle = await this.ensureLoaded(fromPid);
|
|
1246
|
+
if (!fromBundle.points.has(fromID)) {
|
|
1247
|
+
fromBundle.points.set(fromID, { pointID: fromID, connect: [] });
|
|
1248
|
+
this.points.set(fromID, fromBundle.points.get(fromID));
|
|
1249
|
+
}
|
|
1250
|
+
const fromPoint = fromBundle.points.get(fromID);
|
|
1251
|
+
this._addEdgeInternal(fromPid, fromPoint, weight, toID, direction, fromBundle);
|
|
1252
|
+
|
|
1253
|
+
if (direction === 0) {
|
|
1254
|
+
// 双向边:反向写入
|
|
1255
|
+
const toPid = this.partitioner.idOf(toID);
|
|
1256
|
+
const toBundle = await this.ensureLoaded(toPid);
|
|
1257
|
+
if (!toBundle.points.has(toID)) {
|
|
1258
|
+
toBundle.points.set(toID, { pointID: toID, connect: [] });
|
|
1259
|
+
this.points.set(toID, toBundle.points.get(toID));
|
|
1260
|
+
}
|
|
1261
|
+
const toPoint = toBundle.points.get(toID);
|
|
1262
|
+
this._addEdgeInternal(toPid, toPoint, weight, fromID, 0, toBundle);
|
|
1263
|
+
}
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1266
|
+
async updateEdge(fromID, toID, newWeight, direction = 0) {
|
|
1267
|
+
const fromPid = this.partitioner.idOf(fromID);
|
|
1268
|
+
const b = await this.ensureLoaded(fromPid);
|
|
1269
|
+
const p = b.points.get(fromID);
|
|
1270
|
+
if (!p) return;
|
|
1271
|
+
const idx = p.connect.findIndex(([w, id, d]) => id === toID && d === direction);
|
|
1272
|
+
if (idx >= 0) {
|
|
1273
|
+
p.connect[idx][0] = newWeight;
|
|
1274
|
+
b.dirty = true;
|
|
1275
|
+
} else {
|
|
1276
|
+
// 不存在则添加
|
|
1277
|
+
this._addEdgeInternal(fromPid, p, newWeight, toID, direction, b);
|
|
1278
|
+
}
|
|
1279
|
+
}
|
|
1280
|
+
|
|
1281
|
+
existEdge(fromID, toID) {
|
|
1282
|
+
const fromPid = this.partitioner.idOf(fromID);
|
|
1283
|
+
const entry = this.loaded.get(fromPid);
|
|
1284
|
+
if (!entry) return { exist: false, weight: undefined, type: undefined };
|
|
1285
|
+
const p = entry.points.get(fromID);
|
|
1286
|
+
if (!p) return { exist: false, weight: undefined, type: undefined };
|
|
1287
|
+
const found = p.connect.find(([w, id]) => id === toID);
|
|
1288
|
+
return { exist: !!found, weight: found ? found[0] : undefined, type: found ? found[2] : undefined };
|
|
1289
|
+
}
|
|
1290
|
+
|
|
1291
|
+
existPoint(pointID) {
|
|
1292
|
+
// 仅检查已加载窗口
|
|
1293
|
+
const p = this.points.get(pointID);
|
|
1294
|
+
return { exist: !!p, connect: p ? p.connect : [] };
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1297
|
+
deleteEdge(a, b) {
|
|
1298
|
+
const pid = this.partitioner.idOf(a);
|
|
1299
|
+
const entry = this.loaded.get(pid);
|
|
1300
|
+
if (!entry) return;
|
|
1301
|
+
const p = entry.points.get(a);
|
|
1302
|
+
if (!p) return;
|
|
1303
|
+
const before = p.connect.length;
|
|
1304
|
+
p.connect = p.connect.filter(([_, id]) => id !== b);
|
|
1305
|
+
entry.dirty = entry.dirty || (p.connect.length !== before);
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1308
|
+
deletePoint(pointID) {
|
|
1309
|
+
const pid = this.partitioner.idOf(pointID);
|
|
1310
|
+
const entry = this.loaded.get(pid);
|
|
1311
|
+
if (!entry) return;
|
|
1312
|
+
if (entry.points.has(pointID)) {
|
|
1313
|
+
entry.points.delete(pointID);
|
|
1314
|
+
this.points.delete(pointID);
|
|
1315
|
+
entry.dirty = true;
|
|
1316
|
+
}
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
// 仅遍历窗口内点(兼容旧 getAllPoints 调用)
|
|
1320
|
+
getAllPoints() {
|
|
1321
|
+
return Array.from(this.points.values());
|
|
1322
|
+
}
|
|
1323
|
+
|
|
1324
|
+
// 导出全量点(跨所有分区),用于快照/发布
|
|
1325
|
+
async exportAllPoints() {
|
|
1326
|
+
const out = [];
|
|
1327
|
+
// 尝试枚举 FS 分区;其他后端可按 0..N-1 遍历或仅导出已加载窗口
|
|
1328
|
+
const ids = await this.adapter.listPartitionIds();
|
|
1329
|
+
if (ids.length === 0) {
|
|
1330
|
+
// 回退:导出窗口
|
|
1331
|
+
return this.getAllPoints();
|
|
1332
|
+
}
|
|
1333
|
+
for (const pid of ids) {
|
|
1334
|
+
const part = await this.adapter.loadPartition(pid);
|
|
1335
|
+
for (const [, p] of part.points.entries()) out.push({ pointID: p.pointID, connect: p.connect || [] });
|
|
1336
|
+
}
|
|
1337
|
+
return out;
|
|
1338
|
+
}
|
|
1339
|
+
|
|
1340
|
+
// 批量导入(将 legacy 点集落到分区)
|
|
1341
|
+
async importAllPoints(pointsArr) {
|
|
1342
|
+
if (!Array.isArray(pointsArr)) return;
|
|
1343
|
+
// 分桶
|
|
1344
|
+
const buckets = new Map();
|
|
1345
|
+
for (const p of pointsArr) {
|
|
1346
|
+
const pid = this.partitioner.idOf(p.pointID);
|
|
1347
|
+
if (!buckets.has(pid)) buckets.set(pid, new Map());
|
|
1348
|
+
const bm = buckets.get(pid);
|
|
1349
|
+
bm.set(p.pointID, { pointID: p.pointID, connect: Array.isArray(p.connect) ? p.connect.slice() : [] });
|
|
1350
|
+
}
|
|
1351
|
+
// 写入并更新窗口视图(懒加载)
|
|
1352
|
+
for (const [pid, map] of buckets.entries()) {
|
|
1353
|
+
await this.adapter.savePartition(pid, map);
|
|
1354
|
+
// 若已加载该分区,刷新内存镜像
|
|
1355
|
+
if (this.loaded.has(pid)) {
|
|
1356
|
+
const entry = this.loaded.get(pid);
|
|
1357
|
+
// 从全局视图移除旧
|
|
1358
|
+
for (const [id] of entry.points.entries()) this.points.delete(id);
|
|
1359
|
+
entry.points = map;
|
|
1360
|
+
entry.dirty = false;
|
|
1361
|
+
entry.lastAccess = ++this.accessTick;
|
|
1362
|
+
for (const [id, p] of map.entries()) this.points.set(id, p);
|
|
1363
|
+
}
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
|
|
1367
|
+
// 聚合邻居(窗口内),供传播使用
|
|
1368
|
+
getNeighbors(pointID, maxNeighbors = 50) {
|
|
1369
|
+
const p = this.points.get(pointID);
|
|
1370
|
+
if (!p) return [];
|
|
1371
|
+
return p.connect.slice(0, maxNeighbors);
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1374
|
+
// A* 简化:仅在窗口内搜索;跳出窗口时,尝试预取邻接分区后再继续
|
|
1375
|
+
async selectPath(fromID, toID) {
|
|
1376
|
+
if (fromID === toID) return [fromID];
|
|
1377
|
+
// 优先保证焦点加载
|
|
1378
|
+
await this.focusOnPoint(fromID);
|
|
1379
|
+
|
|
1380
|
+
const reconstruct = (came, cur) => {
|
|
1381
|
+
const path = [];
|
|
1382
|
+
let t = cur;
|
|
1383
|
+
while (came.has(t)) { path.push(t); t = came.get(t); }
|
|
1384
|
+
path.push(fromID);
|
|
1385
|
+
return path.reverse();
|
|
1386
|
+
};
|
|
1387
|
+
|
|
1388
|
+
const open = new Set([fromID]);
|
|
1389
|
+
const came = new Map();
|
|
1390
|
+
const g = new Map([[fromID, 0]]);
|
|
1391
|
+
const f = new Map([[fromID, 1]]);
|
|
1392
|
+
const closed = new Set();
|
|
1393
|
+
|
|
1394
|
+
const heuristic = () => 1;
|
|
1395
|
+
let iter = 0;
|
|
1396
|
+
const MAX_ITERS = 5000;
|
|
1397
|
+
|
|
1398
|
+
while (open.size && iter++ < MAX_ITERS) {
|
|
1399
|
+
// 取 f 最小
|
|
1400
|
+
let cur = null; let minF = Infinity;
|
|
1401
|
+
for (const id of open) {
|
|
1402
|
+
const val = f.get(id) ?? Infinity;
|
|
1403
|
+
if (val < minF) { minF = val; cur = id; }
|
|
1404
|
+
}
|
|
1405
|
+
if (cur == null) break;
|
|
1406
|
+
if (cur === toID) return reconstruct(came, cur);
|
|
1407
|
+
|
|
1408
|
+
open.delete(cur);
|
|
1409
|
+
closed.add(cur);
|
|
1410
|
+
|
|
1411
|
+
// 若遇到未知点,尝试加载其分区(滑动窗口)
|
|
1412
|
+
if (!this.points.has(cur)) {
|
|
1413
|
+
await this.focusOnPoint(cur);
|
|
1414
|
+
}
|
|
1415
|
+
|
|
1416
|
+
const neighbors = this.getNeighbors(cur, 50);
|
|
1417
|
+
// 如果邻居为空,尝试边界事件预取(根据邻居 ID 的分区预取)
|
|
1418
|
+
if (neighbors.length === 0) {
|
|
1419
|
+
const pid = this.partitioner.idOf(cur);
|
|
1420
|
+
const ring = this.partitioner.neighborsOf(pid, 1);
|
|
1421
|
+
for (const rid of ring) await this.ensureLoaded(rid);
|
|
1422
|
+
}
|
|
1423
|
+
|
|
1424
|
+
for (const [w, nb] of neighbors) {
|
|
1425
|
+
if (closed.has(nb)) continue;
|
|
1426
|
+
const tentative = (g.get(cur) || Infinity) + w;
|
|
1427
|
+
if (!open.has(nb)) open.add(nb);
|
|
1428
|
+
else if (tentative >= (g.get(nb) || Infinity)) continue;
|
|
1429
|
+
|
|
1430
|
+
came.set(nb, cur);
|
|
1431
|
+
g.set(nb, tentative);
|
|
1432
|
+
f.set(nb, tentative + heuristic());
|
|
1433
|
+
}
|
|
1434
|
+
}
|
|
1435
|
+
return null;
|
|
1436
|
+
}
|
|
1437
|
+
|
|
1438
|
+
// 刷盘所有已加载分区
|
|
1439
|
+
async flushAll() {
|
|
1440
|
+
for (const [pid] of this.loaded.entries()) await this.savePartitionIfDirty(pid);
|
|
1441
|
+
}
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1444
|
+
// ========================== 替换 Runtime.graph 为分区图 ==========================
|
|
1445
|
+
// 说明:wordGraph 仍使用内存 GraphDB;模因图使用 PartitionedGraphDB
|
|
1446
|
+
|
|
1447
|
+
// ...existing code...
|
|
718
1448
|
|
|
719
1449
|
class KVM {
|
|
720
1450
|
// this KVM is the key-value memory
|
|
@@ -770,7 +1500,7 @@ class KVM {
|
|
|
770
1500
|
}
|
|
771
1501
|
delete(key) {
|
|
772
1502
|
if (this.useLMDB) {
|
|
773
|
-
try { this.db.remove(key); } catch (_) {}
|
|
1503
|
+
try { this.db.remove(key); } catch (_) { }
|
|
774
1504
|
return;
|
|
775
1505
|
}
|
|
776
1506
|
this.memory.delete(key);
|
|
@@ -783,7 +1513,7 @@ class KVM {
|
|
|
783
1513
|
for (const k of this.db.getKeys({ snapshot: true })) {
|
|
784
1514
|
let v = this.db.get(k);
|
|
785
1515
|
if (typeof v === 'string') {
|
|
786
|
-
try { v = JSON.parse(v); } catch (_) {}
|
|
1516
|
+
try { v = JSON.parse(v); } catch (_) { }
|
|
787
1517
|
}
|
|
788
1518
|
out.push([k, v]);
|
|
789
1519
|
}
|
|
@@ -877,7 +1607,15 @@ class Runtime {
|
|
|
877
1607
|
// 运行时负责AI核心的调度、模因转换、信号传递与主流程控制
|
|
878
1608
|
constructor(config = {}) {
|
|
879
1609
|
this.config = config;
|
|
880
|
-
|
|
1610
|
+
// 使用分区图作为模因图;词图仍用内存图
|
|
1611
|
+
this.graph = new PartitionedGraphDB({
|
|
1612
|
+
partitions: this.config.partitions || 64,
|
|
1613
|
+
maxLoadedPartitions: this.config.maxLoadedPartitions || 8,
|
|
1614
|
+
windowRadius: this.config.windowRadius || 1,
|
|
1615
|
+
baseDir: path.join(__dirname, 'graph_parts'),
|
|
1616
|
+
backend: this.config.graphBackend || 'fs' // 可选 'fs' | 'lmdb' | 'level'
|
|
1617
|
+
});
|
|
1618
|
+
|
|
881
1619
|
this.wordGraph = new GraphDB();
|
|
882
1620
|
this.kvm = new KVM();
|
|
883
1621
|
this.changer = new Changer();
|
|
@@ -1059,19 +1797,15 @@ class Runtime {
|
|
|
1059
1797
|
visitCount++;
|
|
1060
1798
|
activatedOrder.push(id);
|
|
1061
1799
|
|
|
1062
|
-
// 仅在是“词”时记录访问,避免把模因ID写入词访问日志
|
|
1063
1800
|
if (this.wordGraph.points.has(id)) {
|
|
1064
1801
|
this.logWordAccess(id);
|
|
1065
1802
|
}
|
|
1066
1803
|
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
if (!visited.has(neighborID)) {
|
|
1073
|
-
next.push({ id: neighborID, value: value - decayK * weight });
|
|
1074
|
-
}
|
|
1804
|
+
// 改为通过 graph.getNeighbors 访问(窗口内)
|
|
1805
|
+
const neighbors = this.graph.getNeighbors(id, 50);
|
|
1806
|
+
for (const [weight, neighborID] of neighbors) {
|
|
1807
|
+
if (!visited.has(neighborID)) {
|
|
1808
|
+
next.push({ id: neighborID, value: value - decayK * weight });
|
|
1075
1809
|
}
|
|
1076
1810
|
}
|
|
1077
1811
|
}
|
|
@@ -1188,7 +1922,7 @@ class Runtime {
|
|
|
1188
1922
|
processInput(wordsArr, { addNewWords = true } = {}) {
|
|
1189
1923
|
wordsArr = this.filterStopWords(wordsArr);
|
|
1190
1924
|
if (wordsArr.length === 0) { console.log('[FILTER] 输入全为停用词,已全部过滤'); return; }
|
|
1191
|
-
|
|
1925
|
+
// console.log('Processing input:', wordsArr);
|
|
1192
1926
|
// 批量处理新词添加
|
|
1193
1927
|
if (addNewWords) {
|
|
1194
1928
|
// 一次性检查哪些词不在词表中
|
|
@@ -1274,7 +2008,7 @@ class Runtime {
|
|
|
1274
2008
|
const overlap = wordsArr.filter(w => memeWords.includes(w)).length;
|
|
1275
2009
|
if (overlap >= this.MIN_OVERLAP && memeWords.length + wordsArr.length <= this.MAX_MEME_WORDS) {
|
|
1276
2010
|
this.kvm.set(minMemeID, Array.from(new Set([...memeWords, ...wordsArr])));
|
|
1277
|
-
|
|
2011
|
+
// console.log(`Merged to existing meme: ${minMemeID}`);
|
|
1278
2012
|
} else {
|
|
1279
2013
|
// 创建新模因,使用有向连接
|
|
1280
2014
|
const newID = 'meme_' + Date.now();
|
|
@@ -1284,9 +2018,9 @@ class Runtime {
|
|
|
1284
2018
|
// 单向连接到最近的模因 (方向:2表示指向对方)
|
|
1285
2019
|
if (minMemeID) {
|
|
1286
2020
|
this.graph.addDirectionalEdge(newID, minMemeID, minDistance, 2);
|
|
1287
|
-
|
|
2021
|
+
// console.log(`[LINK] 新模因 ${newID} 单向连接到最近模因 ${minMemeID}`);
|
|
1288
2022
|
}
|
|
1289
|
-
|
|
2023
|
+
// console.log(`Created new meme: ${newID}`);
|
|
1290
2024
|
}
|
|
1291
2025
|
} else {
|
|
1292
2026
|
// 创建新模因
|
|
@@ -1297,9 +2031,9 @@ class Runtime {
|
|
|
1297
2031
|
// 如果有较近的模因,仍然创建单向连接
|
|
1298
2032
|
if (minMemeID) {
|
|
1299
2033
|
this.graph.addDirectionalEdge(newID, minMemeID, Math.min(minDistance, 5), 2);
|
|
1300
|
-
|
|
2034
|
+
// console.log(`[LINK] 新模因 ${newID} 单向连接到最近模因 ${minMemeID}`);
|
|
1301
2035
|
}
|
|
1302
|
-
|
|
2036
|
+
// console.log(`Created new meme: ${newID}`);
|
|
1303
2037
|
}
|
|
1304
2038
|
}
|
|
1305
2039
|
// 新增批量添加边的辅助方法
|
|
@@ -1616,7 +2350,7 @@ class Runtime {
|
|
|
1616
2350
|
this.kvm.delete(memeB.pointID);
|
|
1617
2351
|
memesToDelete.add(memeB.pointID);
|
|
1618
2352
|
|
|
1619
|
-
|
|
2353
|
+
// console.log(`Merged memes: ${memeA.pointID} <- ${memeB.pointID}`);
|
|
1620
2354
|
// 合并后立即尝试分裂
|
|
1621
2355
|
this.splitMemeIfNeeded(memeA.pointID);
|
|
1622
2356
|
} else {
|
|
@@ -1633,7 +2367,7 @@ class Runtime {
|
|
|
1633
2367
|
// 如果没有双向边,则添加双向边
|
|
1634
2368
|
if (!(existAtoB.exist && existAtoB.type === 0) && !(existBtoA.exist && existBtoA.type === 0)) {
|
|
1635
2369
|
this.graph.addBidirectionalEdge(memeA.pointID, memeB.pointID, avgDist);
|
|
1636
|
-
|
|
2370
|
+
// console.log(`[LINK] 添加双向边: ${memeA.pointID} <-> ${memeB.pointID} (avgDist=${avgDist})`);
|
|
1637
2371
|
}
|
|
1638
2372
|
}
|
|
1639
2373
|
}
|
|
@@ -1665,14 +2399,14 @@ class Runtime {
|
|
|
1665
2399
|
const newID = newIDs[i];
|
|
1666
2400
|
this.graph.addPoint(newID, []);
|
|
1667
2401
|
this.kvm.set(newID, chunk);
|
|
1668
|
-
|
|
2402
|
+
// console.log(`[SPLIT-FORCE] 新建模因: ${newID} 词数: ${chunk.length}`);
|
|
1669
2403
|
}
|
|
1670
2404
|
}
|
|
1671
2405
|
|
|
1672
2406
|
// 删除原模因
|
|
1673
2407
|
this.graph.points.delete(memeID);
|
|
1674
2408
|
this.kvm.delete(memeID);
|
|
1675
|
-
|
|
2409
|
+
// console.log(`[SPLIT-FORCE] 删除原模因: ${memeID}`);
|
|
1676
2410
|
return;
|
|
1677
2411
|
}
|
|
1678
2412
|
|
|
@@ -1721,12 +2455,12 @@ class Runtime {
|
|
|
1721
2455
|
const newID = 'meme_' + Date.now() + '_' + Math.floor(Math.random() * 10000);
|
|
1722
2456
|
this.graph.addPoint(newID, []);
|
|
1723
2457
|
this.kvm.set(newID, comp);
|
|
1724
|
-
|
|
2458
|
+
// console.log(`[SPLIT] 新建模因: ${newID} 词数: ${comp.length}`);
|
|
1725
2459
|
}
|
|
1726
2460
|
// 删除原节点
|
|
1727
2461
|
this.graph.points.delete(memeID);
|
|
1728
2462
|
this.kvm.delete(memeID);
|
|
1729
|
-
|
|
2463
|
+
// console.log(`[SPLIT] 删除原模因: ${memeID}`);
|
|
1730
2464
|
}
|
|
1731
2465
|
}
|
|
1732
2466
|
}
|
|
@@ -2153,7 +2887,7 @@ class controller {
|
|
|
2153
2887
|
const words = text.toLowerCase().split(' ').filter(w => w.length > 0);
|
|
2154
2888
|
this.runtime.processInput(words, { addNewWords: false });
|
|
2155
2889
|
// 用模因网络参与推理
|
|
2156
|
-
|
|
2890
|
+
// console.log('[DEBUG] 当前所有模因节点:', this.runtime.kvm.memory);
|
|
2157
2891
|
return await this.runtime.generateResponseWithMemes(words);
|
|
2158
2892
|
}
|
|
2159
2893
|
|
|
@@ -2217,7 +2951,7 @@ function saveAll(runtime) {
|
|
|
2217
2951
|
const data = {
|
|
2218
2952
|
memes: runtime.graph.getAllPoints(),
|
|
2219
2953
|
wordGraph: Array.from(runtime.wordGraph.points.values()),
|
|
2220
|
-
|
|
2954
|
+
kvm: runtime.kvm.exportEntries(),
|
|
2221
2955
|
vocab: runtime.vocabManager.vocab,
|
|
2222
2956
|
wordAccessLog: Array.from(runtime.wordAccessLog ? runtime.wordAccessLog.entries() : [])
|
|
2223
2957
|
};
|
|
@@ -2301,72 +3035,129 @@ async function boot() {
|
|
|
2301
3035
|
// 激活副本:从快照加载 ctrlB/ctrlC 并加入轮询
|
|
2302
3036
|
let activeControllers = []; // 轮询数组
|
|
2303
3037
|
let rrIdx = 0;
|
|
3038
|
+
// 新增:双实例热切换状态
|
|
3039
|
+
let servingCtrl = null; // 当前对外服务的控制器
|
|
3040
|
+
let standbyCtrl = null; // 后台承接 Redis 模型的控制器
|
|
3041
|
+
let updatingModel = false;
|
|
3042
|
+
let pendingModelObj = null;
|
|
3043
|
+
|
|
3044
|
+
function setServingController(ctrl) {
|
|
3045
|
+
servingCtrl = ctrl;
|
|
3046
|
+
activeControllers = [servingCtrl]; // 仅对外暴露一份,避免切换瞬间不一致
|
|
3047
|
+
global.ctrlA = servingCtrl;
|
|
3048
|
+
global.ctrl = servingCtrl;
|
|
3049
|
+
}
|
|
3050
|
+
|
|
3051
|
+
|
|
3052
|
+
// 替换 getNextController:始终返回当前服务控制器
|
|
2304
3053
|
function getNextController() {
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
3054
|
+
if (servingCtrl) return servingCtrl;
|
|
3055
|
+
if (activeControllers.length) return activeControllers[rrIdx % activeControllers.length];
|
|
3056
|
+
return global.ctrlA; // 兜底
|
|
3057
|
+
}
|
|
3058
|
+
async function ensureStandbyReady() {
|
|
3059
|
+
if (!standbyCtrl) {
|
|
3060
|
+
standbyCtrl = new controller(); // 构造函数会初始化 Runtime
|
|
3061
|
+
// 初始保持与serving词表一致,减少首次同步成本(可选)
|
|
3062
|
+
if (servingCtrl?.runtime?.vocabManager?.vocab?.length) {
|
|
3063
|
+
standbyCtrl.runtime.vocabManager.vocab = [...servingCtrl.runtime.vocabManager.vocab];
|
|
3064
|
+
standbyCtrl.runtime.vocabManager.updateMappings();
|
|
3065
|
+
}
|
|
3066
|
+
}
|
|
3067
|
+
return standbyCtrl;
|
|
3068
|
+
}
|
|
3069
|
+
|
|
3070
|
+
// 将 protobuf 反序列化对象应用到 standby 并热切换
|
|
3071
|
+
async function handleRedisModelSwap(modelObj) {
|
|
3072
|
+
if (updatingModel) {
|
|
3073
|
+
pendingModelObj = modelObj; // 只保留最近一条
|
|
3074
|
+
return;
|
|
3075
|
+
}
|
|
3076
|
+
updatingModel = true;
|
|
3077
|
+
try {
|
|
3078
|
+
const standby = await ensureStandbyReady();
|
|
3079
|
+
// 在 standby 上同步模型
|
|
3080
|
+
await plainObjToRuntime(standby.runtime, modelObj);
|
|
3081
|
+
applyModelParams(standby.runtime);
|
|
3082
|
+
|
|
3083
|
+
// 原子切换:standby 上位,serving 变为 standby
|
|
3084
|
+
const oldServing = servingCtrl;
|
|
3085
|
+
setServingController(standbyCtrl);
|
|
3086
|
+
standbyCtrl = oldServing;
|
|
3087
|
+
|
|
3088
|
+
console.log('[MODEL SYNC] 新模型已在后台准备完毕并无缝切换为在线实例');
|
|
3089
|
+
} catch (e) {
|
|
3090
|
+
console.error('[MODEL SYNC] 后台模型切换失败:', e.message);
|
|
3091
|
+
} finally {
|
|
3092
|
+
updatingModel = false;
|
|
3093
|
+
if (pendingModelObj) {
|
|
3094
|
+
const next = pendingModelObj;
|
|
3095
|
+
pendingModelObj = null;
|
|
3096
|
+
// 递归处理最新一条待处理消息
|
|
3097
|
+
setImmediate(() => handleRedisModelSwap(next));
|
|
3098
|
+
}
|
|
3099
|
+
}
|
|
2309
3100
|
}
|
|
2310
3101
|
async function activateReplicasIfNeeded() {
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
3102
|
+
if (global.config.isStudy) return; // study 进程不启用副本扩容
|
|
3103
|
+
if (global.ctrlB && global.ctrlC) return; // 已激活
|
|
3104
|
+
try {
|
|
3105
|
+
const loadReplicaByName = async (name) => {
|
|
3106
|
+
const ctrl = new controller();
|
|
3107
|
+
// 为副本单独挂载快照管理器
|
|
3108
|
+
ctrl.snapshotManager = new SnapshotManager(ctrl.runtime);
|
|
3109
|
+
// 在 A 的快照目录中寻找对应 id
|
|
3110
|
+
const smA = global.ctrlA.snapshotManager;
|
|
3111
|
+
smA.loadSnapshotList();
|
|
3112
|
+
const snap = smA.snapshotList.find(s => s.name === name || s.id.endsWith(`_${name}`));
|
|
3113
|
+
if (!snap) {
|
|
3114
|
+
console.warn(`[REPLICA] 未找到快照: ${name},尝试现生成...`);
|
|
3115
|
+
await smA.createSnapshot(name);
|
|
3116
|
+
}
|
|
3117
|
+
const snap2 = smA.snapshotList.find(s => s.name === name || s.id.endsWith(`_${name}`));
|
|
3118
|
+
if (snap2) {
|
|
3119
|
+
// 直接读取该文件内容并恢复到 ctrl.runtime
|
|
3120
|
+
const dataStr = fs.readFileSync(snap2.path, 'utf-8');
|
|
3121
|
+
const data = JSON.parse(dataStr);
|
|
3122
|
+
// 简单恢复(与 SnapshotManager.restoreSnapshot 类似)
|
|
3123
|
+
ctrl.runtime.graph = new GraphDB();
|
|
3124
|
+
ctrl.runtime.wordGraph = new GraphDB();
|
|
3125
|
+
ctrl.runtime.kvm = new KVM();
|
|
3126
|
+
ctrl.runtime.wordAccessLog = new Map(data.wordAccessLog || []);
|
|
3127
|
+
if (data.memes) for (const p of data.memes) ctrl.runtime.graph.addPoint(p.pointID, p.connect);
|
|
3128
|
+
if (data.wordGraph) for (const p of data.wordGraph) ctrl.runtime.wordGraph.addPoint(p.pointID, p.connect);
|
|
3129
|
+
if (data.kvm) for (const [k, v] of data.kvm) ctrl.runtime.kvm.set(k, v);
|
|
3130
|
+
if (data.vocab) { ctrl.runtime.vocabManager.vocab = data.vocab; ctrl.runtime.vocabManager.updateMappings(); }
|
|
3131
|
+
return ctrl;
|
|
3132
|
+
}
|
|
3133
|
+
return null;
|
|
3134
|
+
};
|
|
2344
3135
|
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
3136
|
+
if (!global.ctrlB) global.ctrlB = await loadReplicaByName('replica_B');
|
|
3137
|
+
if (!global.ctrlC) global.ctrlC = await loadReplicaByName('replica_C');
|
|
3138
|
+
activeControllers = [global.ctrlA, ...(global.ctrlB ? [global.ctrlB] : []), ...(global.ctrlC ? [global.ctrlC] : [])];
|
|
3139
|
+
console.log(`[REPLICA] 已激活副本: ${activeControllers.length} 个控制器`);
|
|
3140
|
+
} catch (e) {
|
|
3141
|
+
console.warn('[REPLICA] 激活失败:', e.message);
|
|
3142
|
+
}
|
|
2352
3143
|
}
|
|
2353
3144
|
// 对端探测/反触发:同组任一对端端口死亡则激活副本
|
|
2354
3145
|
function installPeerFailoverMonitor() {
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
3146
|
+
const peers = global.config.peerServePorts || [];
|
|
3147
|
+
if (!Array.isArray(peers) || peers.length === 0) return;
|
|
3148
|
+
const axiosInst = axios.create({ timeout: 1500 });
|
|
3149
|
+
setInterval(async () => {
|
|
3150
|
+
try {
|
|
3151
|
+
const checks = await Promise.all(peers.map(p =>
|
|
3152
|
+
axiosInst.get(`http://127.0.0.1:${p}/health`).then(() => true).catch(() => false)
|
|
3153
|
+
));
|
|
3154
|
+
const dead = checks.some(ok => !ok);
|
|
3155
|
+
if (dead) {
|
|
3156
|
+
console.warn('[ANTI-TRIGGER] 检测到对端 serve 进程离线,启动副本扩容...');
|
|
3157
|
+
await activateReplicasIfNeeded();
|
|
3158
|
+
}
|
|
3159
|
+
} catch (_) { /* 忽略一次错误 */ }
|
|
3160
|
+
}, 3000);
|
|
2370
3161
|
}
|
|
2371
3162
|
|
|
2372
3163
|
// 在main函数末尾添加
|
|
@@ -2416,23 +3207,23 @@ function setupExitHandler() {
|
|
|
2416
3207
|
|
|
2417
3208
|
// 预缓存副本快照(不在内存常驻)
|
|
2418
3209
|
async function preCacheReplicas() {
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
3210
|
+
try {
|
|
3211
|
+
if (!global.ctrlA?.snapshotManager) return;
|
|
3212
|
+
const sm = global.ctrlA.snapshotManager;
|
|
3213
|
+
sm.loadSnapshotList();
|
|
3214
|
+
const hasB = sm.getSnapshotList().some(s => s.name.includes('replica_B'));
|
|
3215
|
+
const hasC = sm.getSnapshotList().some(s => s.name.includes('replica_C'));
|
|
3216
|
+
if (!hasB) {
|
|
3217
|
+
console.log('[REPLICA] 预生成 ctrlB 快照(replica_B)...');
|
|
3218
|
+
await sm.createSnapshot('replica_B');
|
|
3219
|
+
}
|
|
3220
|
+
if (!hasC) {
|
|
3221
|
+
console.log('[REPLICA] 预生成 ctrlC 快照(replica_C)...');
|
|
3222
|
+
await sm.createSnapshot('replica_C');
|
|
3223
|
+
}
|
|
3224
|
+
} catch (e) {
|
|
3225
|
+
console.warn('[REPLICA] 预缓存失败:', e.message);
|
|
3226
|
+
}
|
|
2436
3227
|
}
|
|
2437
3228
|
// 添加定期垃圾回收帮助函数
|
|
2438
3229
|
function optimizeMemory() {
|
|
@@ -2447,21 +3238,33 @@ function optimizeMemory() {
|
|
|
2447
3238
|
}
|
|
2448
3239
|
}
|
|
2449
3240
|
async function main() {
|
|
3241
|
+
let __adv = null;
|
|
3242
|
+
if (String(process.env.SERVE_ADV_AUTOSTART || '').toLowerCase() === 'true') {
|
|
3243
|
+
__adv = new AdversaryScheduler(global.ctrlA.runtime, {
|
|
3244
|
+
providerSpec: process.env.SERVE_ADV_MODEL || process.env.ADV_MODEL || 'ollama:llama3:70b',
|
|
3245
|
+
judgeMode: process.env.SERVE_ADV_JUDGE || process.env.ADV_JUDGE || 'heuristic',
|
|
3246
|
+
intervalMs: Number(process.env.SERVE_ADV_INTERVAL || 120_000),
|
|
3247
|
+
batchSize: Number(process.env.SERVE_ADV_BATCH || 2),
|
|
3248
|
+
adjustParams: false // 只记录,不修改服务实例的参数
|
|
3249
|
+
});
|
|
3250
|
+
__adv.start();
|
|
3251
|
+
}
|
|
2450
3252
|
|
|
2451
3253
|
let spiderPrivate = new Spider();
|
|
2452
3254
|
// 创建三个全局控制器副本
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
ctrlA.snapshotManager
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
3255
|
+
const ctrlA = new controller();
|
|
3256
|
+
global.ctrlA = ctrlA;
|
|
3257
|
+
global.ctrl = ctrlA;
|
|
3258
|
+
setServingController(ctrlA);
|
|
3259
|
+
// 启动快照自动保存(每30分钟)
|
|
3260
|
+
if (ctrlA.snapshotManager) {
|
|
3261
|
+
ctrlA.snapshotManager.setupAutoSnapshot(30 * 60 * 1000);
|
|
3262
|
+
console.log('[MAIN] 已设置自动快照,每30分钟检查一次');
|
|
3263
|
+
}
|
|
3264
|
+
setupExitHandler();
|
|
3265
|
+
console.log('Starting AI system...');
|
|
3266
|
+
// 恢复 ctrlA
|
|
3267
|
+
loadAll(ctrlA.runtime);
|
|
2465
3268
|
await preCacheReplicas();
|
|
2466
3269
|
const redisClient = redis.createClient();
|
|
2467
3270
|
redisClient.connect();
|
|
@@ -2493,19 +3296,13 @@ async function main() {
|
|
|
2493
3296
|
redisClient.on("message", function (channel, message) {
|
|
2494
3297
|
if (channel === `AI-model-${__dirname}` && RuntimeMessage) {
|
|
2495
3298
|
try {
|
|
2496
|
-
// 反序列化为对象
|
|
2497
3299
|
const modelObj = RuntimeMessage.decode(Buffer.from(message));
|
|
2498
|
-
// 可选:校验
|
|
2499
3300
|
const errMsg = RuntimeMessage.verify(modelObj);
|
|
2500
3301
|
if (errMsg) throw Error(errMsg);
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
// 你需要实现一个 plainObjToRuntime 方法
|
|
2504
|
-
plainObjToRuntime(ctrlA.runtime, modelObj);
|
|
2505
|
-
|
|
2506
|
-
console.log("[MODEL SYNC] 已同步最新模型到 ctrlA.runtime");
|
|
3302
|
+
// 后台准备并切换
|
|
3303
|
+
handleRedisModelSwap(modelObj);
|
|
2507
3304
|
} catch (e) {
|
|
2508
|
-
console.error("[MODEL SYNC]
|
|
3305
|
+
console.error("[MODEL SYNC] 反序列化或切换失败:", e.message);
|
|
2509
3306
|
}
|
|
2510
3307
|
}
|
|
2511
3308
|
});
|
|
@@ -2556,33 +3353,72 @@ async function main() {
|
|
|
2556
3353
|
await ctrlA.handleInput('I like apple');
|
|
2557
3354
|
await ctrlA.handleInput('I love orange');
|
|
2558
3355
|
await ctrlA.handleInput('you are good');
|
|
2559
|
-
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
3356
|
+
// 启动主循环(仅 A)
|
|
3357
|
+
ctrlA.startMainLoop();
|
|
3358
|
+
|
|
3359
|
+
// 安装对端监控触发器
|
|
3360
|
+
installPeerFailoverMonitor();
|
|
3361
|
+
app.get('/api/graph/partitions/status', async (req, res) => {
|
|
3362
|
+
try {
|
|
3363
|
+
const g = global.ctrlA?.runtime?.graph;
|
|
3364
|
+
if (!g || !(g instanceof PartitionedGraphDB)) {
|
|
3365
|
+
return res.json({ ok: true, mode: 'in-memory', loaded: 0 });
|
|
3366
|
+
}
|
|
3367
|
+
const loaded = Array.from(g.loaded.keys());
|
|
3368
|
+
res.json({
|
|
3369
|
+
ok: true,
|
|
3370
|
+
mode: 'partitioned',
|
|
3371
|
+
partitions: g.partitioner.partitions,
|
|
3372
|
+
loaded,
|
|
3373
|
+
maxLoaded: g.maxLoadedPartitions,
|
|
3374
|
+
windowRadius: g.windowRadius,
|
|
3375
|
+
centerPid: g.centerPid
|
|
3376
|
+
});
|
|
3377
|
+
} catch (e) {
|
|
3378
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3379
|
+
}
|
|
3380
|
+
});
|
|
3381
|
+
|
|
3382
|
+
app.post('/api/graph/partitions/flush', async (req, res) => {
|
|
3383
|
+
try {
|
|
3384
|
+
const g = global.ctrlA?.runtime?.graph;
|
|
3385
|
+
if (g && g.flushAll) await g.flushAll();
|
|
3386
|
+
res.json({ ok: true });
|
|
3387
|
+
} catch (e) {
|
|
3388
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3389
|
+
}
|
|
3390
|
+
});
|
|
3391
|
+
|
|
3392
|
+
app.post('/api/graph/prefetch', async (req, res) => {
|
|
3393
|
+
try {
|
|
3394
|
+
const { node } = req.body || {};
|
|
3395
|
+
const g = global.ctrlA?.runtime?.graph;
|
|
3396
|
+
if (!node || !(g instanceof PartitionedGraphDB)) return res.status(400).json({ ok: false, error: 'node 必填/或非分区图' });
|
|
3397
|
+
await g.focusOnPoint(String(node));
|
|
3398
|
+
res.json({ ok: true });
|
|
3399
|
+
} catch (e) {
|
|
3400
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3401
|
+
}
|
|
2585
3402
|
});
|
|
3403
|
+
app.post('/api/chat', async (req, res) => {
|
|
3404
|
+
try {
|
|
3405
|
+
const { message } = req.body || {};
|
|
3406
|
+
const ctrl = getNextController(); // 始终路由到当前服务实例
|
|
3407
|
+
const words = typeof message === 'string'
|
|
3408
|
+
? message.toLowerCase().split(/\s+/).filter(w => w.length > 0)
|
|
3409
|
+
: [];
|
|
3410
|
+
const normWords = (new Spider()).lemmatizeWords(words);
|
|
3411
|
+
const normMessage = normWords.join(' ');
|
|
3412
|
+
const response = await ctrl.handleInput(normMessage);
|
|
3413
|
+
if (!response || response.trim() === '') {
|
|
3414
|
+
console.warn('[WARN] AI响应为空');
|
|
3415
|
+
}
|
|
3416
|
+
res.json({ response });
|
|
3417
|
+
} catch (error) {
|
|
3418
|
+
console.error('[ERROR] 处理请求失败:', error);
|
|
3419
|
+
res.status(500).json({ error: error.message });
|
|
3420
|
+
}
|
|
3421
|
+
});
|
|
2586
3422
|
// 添加健康检查路由
|
|
2587
3423
|
app.get('/health', (req, res) => {
|
|
2588
3424
|
res.status(200).send('OK');
|
|
@@ -2699,8 +3535,75 @@ app.post('/api/chat', async (req, res) => {
|
|
|
2699
3535
|
res.status(500).json({ error: err.message });
|
|
2700
3536
|
}
|
|
2701
3537
|
});
|
|
2702
|
-
|
|
3538
|
+
app.get('/api/adversary/status', (req, res) => {
|
|
3539
|
+
try {
|
|
3540
|
+
res.json({ ok: true, status: __adv ? __adv.getStatus() : { running: false } });
|
|
3541
|
+
} catch (e) {
|
|
3542
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3543
|
+
}
|
|
3544
|
+
});
|
|
3545
|
+
|
|
3546
|
+
app.post('/api/adversary/start', (req, res) => {
|
|
3547
|
+
try {
|
|
3548
|
+
if (!__adv) __adv = new AdversaryScheduler(global.ctrlA.runtime, { adjustParams: false });
|
|
3549
|
+
__adv.start();
|
|
3550
|
+
res.json({ ok: true, status: __adv.getStatus() });
|
|
3551
|
+
} catch (e) {
|
|
3552
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3553
|
+
}
|
|
3554
|
+
});
|
|
3555
|
+
|
|
3556
|
+
app.post('/api/adversary/stop', (req, res) => {
|
|
3557
|
+
try { __adv?.stop?.(); res.json({ ok: true }); }
|
|
3558
|
+
catch (e) { res.status(500).json({ ok: false, error: e.message }); }
|
|
3559
|
+
});
|
|
3560
|
+
// 新增:serve 侧参数调优 API(默认不启用自动调参,仅手动设置)
|
|
3561
|
+
app.get('/api/tune/get', (req, res) => {
|
|
3562
|
+
try {
|
|
3563
|
+
const rt = global.ctrlA?.runtime;
|
|
3564
|
+
if (!rt) return res.status(500).json({ ok: false, error: 'runtime missing' });
|
|
3565
|
+
res.json({
|
|
3566
|
+
ok: true,
|
|
3567
|
+
params: {
|
|
3568
|
+
decayK: rt.config?.decayK ?? 1,
|
|
3569
|
+
maxLen: rt.config?.maxLen ?? 16,
|
|
3570
|
+
spiderMix: rt.config?.spiderMix ?? { onlineWeight: 0.5, offlineWeight: 0.5 },
|
|
3571
|
+
crawler: {
|
|
3572
|
+
perQuery: global.__crawler?.__tune_perQuery ?? 8,
|
|
3573
|
+
maxCrawl: global.__crawler?.__tune_maxCrawl ?? 12
|
|
3574
|
+
}
|
|
3575
|
+
}
|
|
3576
|
+
});
|
|
3577
|
+
} catch (e) {
|
|
3578
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3579
|
+
}
|
|
3580
|
+
});
|
|
2703
3581
|
|
|
3582
|
+
app.post('/api/tune/set', (req, res) => {
|
|
3583
|
+
try {
|
|
3584
|
+
const rt = global.ctrlA?.runtime;
|
|
3585
|
+
if (!rt) return res.status(500).json({ ok: false, error: 'runtime missing' });
|
|
3586
|
+
const snap = applyServeTunableParams(rt, req.body || {});
|
|
3587
|
+
res.json({ ok: true, snapshot: snap });
|
|
3588
|
+
} catch (e) {
|
|
3589
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3590
|
+
}
|
|
3591
|
+
});
|
|
3592
|
+
|
|
3593
|
+
// 可选:服务侧对抗调度保持 adjustParams: false,但允许设置 promptMode/targets
|
|
3594
|
+
// ...existing code...
|
|
3595
|
+
app.post('/api/adversary/start', (req, res) => {
|
|
3596
|
+
try {
|
|
3597
|
+
if (!__adv) __adv = new AdversaryScheduler(global.ctrlA.runtime, { adjustParams: false });
|
|
3598
|
+
const { promptMode, targetWeights } = req.body || {};
|
|
3599
|
+
if (promptMode) __adv.setPromptMode(promptMode);
|
|
3600
|
+
if (targetWeights) __adv.setTargets(targetWeights);
|
|
3601
|
+
__adv.start();
|
|
3602
|
+
res.json({ ok: true, status: __adv.getStatus() });
|
|
3603
|
+
} catch (e) {
|
|
3604
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3605
|
+
}
|
|
3606
|
+
});
|
|
2704
3607
|
|
|
2705
3608
|
// 设置退出保存
|
|
2706
3609
|
setupExitHandler();
|
|
@@ -2818,9 +3721,32 @@ async function plainObjToRuntime(runtime, obj) {
|
|
|
2818
3721
|
|
|
2819
3722
|
console.log('[MODEL SYNC] 模型同步完成');
|
|
2820
3723
|
}
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
|
|
3724
|
+
function applyServeTunableParams(runtime, partial = {}) {
|
|
3725
|
+
if (!runtime) return null;
|
|
3726
|
+
runtime.config = runtime.config || {};
|
|
3727
|
+
if (partial.spiderMix) {
|
|
3728
|
+
const ow = Math.max(0, Math.min(1, Number(partial.spiderMix.onlineWeight ?? runtime.config.spiderMix?.onlineWeight ?? 0.5)));
|
|
3729
|
+
runtime.config.spiderMix = { onlineWeight: ow, offlineWeight: Math.max(0, Math.min(1, 1 - ow)) };
|
|
3730
|
+
}
|
|
3731
|
+
if (typeof partial.decayK === 'number') runtime.config.decayK = Math.max(0.1, Math.min(2.0, partial.decayK));
|
|
3732
|
+
if (typeof partial.maxLen === 'number') runtime.config.maxLen = Math.max(8, Math.min(64, Math.round(partial.maxLen)));
|
|
3733
|
+
if (typeof partial.edgeWeight === 'number' && runtime.graph) {
|
|
3734
|
+
for (const p of runtime.graph.getAllPoints()) for (const e of p.connect) e[0] = Math.max(0.1, Math.min(5, partial.edgeWeight));
|
|
3735
|
+
}
|
|
3736
|
+
if (global.__crawler) {
|
|
3737
|
+
if (typeof partial.perQuery === 'number') global.__crawler.__tune_perQuery = Math.max(2, Math.min(16, Math.round(partial.perQuery)));
|
|
3738
|
+
if (typeof partial.maxCrawl === 'number') global.__crawler.__tune_maxCrawl = Math.max(2, Math.min(24, Math.round(partial.maxCrawl)));
|
|
3739
|
+
}
|
|
3740
|
+
return {
|
|
3741
|
+
decayK: runtime.config.decayK,
|
|
3742
|
+
maxLen: runtime.config.maxLen,
|
|
3743
|
+
spiderMix: runtime.config.spiderMix || { onlineWeight: 0.5, offlineWeight: 0.5 },
|
|
3744
|
+
crawler: {
|
|
3745
|
+
perQuery: global.__crawler?.__tune_perQuery ?? 8,
|
|
3746
|
+
maxCrawl: global.__crawler?.__tune_maxCrawl ?? 12
|
|
3747
|
+
}
|
|
3748
|
+
};
|
|
3749
|
+
}
|
|
2824
3750
|
if (require.main === module) {
|
|
2825
3751
|
|
|
2826
3752
|
main().catch(console.error)
|