079project 2.0.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/crawler/agent.cjs +97 -0
- package/crawler/index.cjs +515 -0
- package/crawler/storage.cjs +163 -0
- package/forwarder.js +106 -37
- package/groupmanager.cjs +2 -1
- package/main_Serve.cjs +1281 -270
- package/main_Study.cjs +1731 -375
- package/package.json +6 -1
- package/robots/seeds.txt +2 -0
- package/schedule.cjs +745 -0
- package/todo-list.txt +0 -86
package/main_Serve.cjs
CHANGED
|
@@ -18,6 +18,7 @@ const csvParse = require('csv-parse/sync')
|
|
|
18
18
|
const os = require('os');
|
|
19
19
|
const MAX_WORKERS = Math.max(1, os.cpus().length - 1); // 留一个核心给主线程
|
|
20
20
|
const natural = require('natural');
|
|
21
|
+
const { AdversaryScheduler } = require('./schedule.cjs');
|
|
21
22
|
const STOP_WORDS = natural.stopwords; // 英文停用词
|
|
22
23
|
const pool = workerpool.pool(path.join(__dirname, 'memeMergeWorker.cjs'), {
|
|
23
24
|
minWorkers: 1,
|
|
@@ -32,28 +33,93 @@ protobuf.load(runtimeProtoPath, (err, root) => {
|
|
|
32
33
|
RuntimeMessage = root.lookupType('Runtime');
|
|
33
34
|
});
|
|
34
35
|
function parseArgs(argv) {
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
36
|
+
const out = {};
|
|
37
|
+
for (let i = 0; i < argv.length; i++) {
|
|
38
|
+
const a = argv[i];
|
|
39
|
+
if (a && a.startsWith('--')) {
|
|
40
|
+
const k = a.slice(2);
|
|
41
|
+
const v = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : true;
|
|
42
|
+
out[k] = v;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return out;
|
|
45
46
|
}
|
|
46
47
|
const __args = parseArgs(process.argv.slice(2));
|
|
47
48
|
global.config = {
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
49
|
+
masterPortOfSql: 3125,
|
|
50
|
+
masterPortOfMain: process.argv[2],
|
|
51
|
+
emitExitport: process.argv[3] || 8641,
|
|
52
|
+
groupId: Number(__args['group-id'] || -1),
|
|
53
|
+
forwarderPort: Number(__args['forwarder-port'] || 0),
|
|
54
|
+
studyPort: Number(__args['study-port'] || 0),
|
|
55
|
+
peerServePorts: String(__args['peers'] || '').split(',').map(s => Number(s)).filter(n => Number.isFinite(n) && n > 0),
|
|
56
|
+
isStudy: !!__args['study']
|
|
57
|
+
};
|
|
58
|
+
// ...existing code...
|
|
59
|
+
const vm = require('vm'); // 新增:沙箱编译
|
|
60
|
+
// ...existing code...
|
|
61
|
+
|
|
62
|
+
// ==== 内置激活/传递函数注册表 + 安全编译工具 ====
|
|
63
|
+
const BuiltinActivations = {
|
|
64
|
+
identity: (x) => x,
|
|
65
|
+
relu: (x) => (x > 0 ? x : 0),
|
|
66
|
+
leaky_relu: (x) => (x > 0 ? x : 0.01 * x),
|
|
67
|
+
tanh: (x) => Math.tanh(x),
|
|
68
|
+
sigmoid: (x) => 1 / (1 + Math.exp(-x)),
|
|
69
|
+
elu: (x) => (x >= 0 ? x : (Math.exp(x) - 1)),
|
|
70
|
+
softplus: (x) => Math.log(1 + Math.exp(x)),
|
|
71
|
+
// 近似 GELU
|
|
72
|
+
gelu: (x) => 0.5 * x * (1 + Math.tanh(Math.sqrt(2 / Math.PI) * (x + 0.044715 * Math.pow(x, 3))))
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
const BuiltinTransfers = {
|
|
76
|
+
// 线性衰减:next = value - decayK*weight*(dirMult)
|
|
77
|
+
linear: (value, weight, decayK, ctx) => {
|
|
78
|
+
const dm = ctx?.direction === 0 ? (ctx?.bidirectionalMultiplier ?? 1.2) : (ctx?.directionalMultiplier ?? 0.7);
|
|
79
|
+
return value - (decayK * weight * dm);
|
|
80
|
+
},
|
|
81
|
+
// 指数衰减:next = value * exp(-decayK*weight*(dirMult))
|
|
82
|
+
exp: (value, weight, decayK, ctx) => {
|
|
83
|
+
const dm = ctx?.direction === 0 ? (ctx?.bidirectionalMultiplier ?? 1.2) : (ctx?.directionalMultiplier ?? 0.7);
|
|
84
|
+
return value * Math.exp(-(decayK * weight * dm));
|
|
85
|
+
},
|
|
86
|
+
// 反比例:next = value / (1 + decayK*weight*(dirMult))
|
|
87
|
+
inverse: (value, weight, decayK, ctx) => {
|
|
88
|
+
const dm = ctx?.direction === 0 ? (ctx?.bidirectionalMultiplier ?? 1.2) : (ctx?.directionalMultiplier ?? 0.7);
|
|
89
|
+
return value / (1 + (decayK * weight * dm));
|
|
90
|
+
},
|
|
91
|
+
// 截断线性:线性后下限截断为0,上限截断为value
|
|
92
|
+
capped: (value, weight, decayK, ctx) => {
|
|
93
|
+
const dm = ctx?.direction === 0 ? (ctx?.bidirectionalMultiplier ?? 1.2) : (ctx?.directionalMultiplier ?? 0.7);
|
|
94
|
+
const raw = value - (decayK * weight * dm);
|
|
95
|
+
return Math.max(0, Math.min(value, raw));
|
|
96
|
+
}
|
|
56
97
|
};
|
|
98
|
+
|
|
99
|
+
function compileCustomFunctionSafely(source, argNames, fallback) {
|
|
100
|
+
try {
|
|
101
|
+
const ctx = vm.createContext({ Math });
|
|
102
|
+
// 如果用户提供的是“表达式”,包一层 return
|
|
103
|
+
const body = source.includes('return') || source.includes('=>') || source.includes('function')
|
|
104
|
+
? source
|
|
105
|
+
: `return (${source});`;
|
|
106
|
+
|
|
107
|
+
// 统一包成 function 体
|
|
108
|
+
const wrapper = `(function(${argNames.join(',')}) { "use strict"; ${body} })`;
|
|
109
|
+
const script = new vm.Script(wrapper, { timeout: 50 });
|
|
110
|
+
const fn = script.runInContext(ctx, { timeout: 50 });
|
|
111
|
+
if (typeof fn !== 'function') return fallback;
|
|
112
|
+
// 再包一层,避免传入异常导致抛出
|
|
113
|
+
return (...args) => {
|
|
114
|
+
try { return fn(...args); } catch (_e) { return fallback(...args); }
|
|
115
|
+
};
|
|
116
|
+
} catch (_e) {
|
|
117
|
+
return fallback;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
// ...existing code...
|
|
121
|
+
|
|
122
|
+
// 顶部 modelDefaults 增加参数(与本文件后半段的重复默认值保持一致)
|
|
57
123
|
const modelDefaults = {
|
|
58
124
|
decayFactor: 0.5,
|
|
59
125
|
maxMemeWords: 100,
|
|
@@ -65,7 +131,12 @@ const modelDefaults = {
|
|
|
65
131
|
decay: 1,
|
|
66
132
|
decayK: 1,
|
|
67
133
|
maxLen: 16,
|
|
68
|
-
edgeWeight: 1
|
|
134
|
+
edgeWeight: 1,
|
|
135
|
+
// 新增:激活/传递函数选择与自定义
|
|
136
|
+
activationType: 'relu', // identity|relu|leaky_relu|tanh|sigmoid|elu|softplus|gelu|custom
|
|
137
|
+
transferType: 'linear', // linear|exp|inverse|capped|custom
|
|
138
|
+
activationCustom: '', // 自定义激活函数源码/表达式:f(x) 或 return ...
|
|
139
|
+
transferCustom: '' // 自定义传递函数源码/表达式:f(value, weight, decayK, ctx) 或 return ...
|
|
69
140
|
};
|
|
70
141
|
const currentModelParams = { ...modelDefaults };
|
|
71
142
|
// 反触发机制
|
|
@@ -265,8 +336,8 @@ class SnapshotManager {
|
|
|
265
336
|
}
|
|
266
337
|
}
|
|
267
338
|
|
|
339
|
+
|
|
268
340
|
async createSnapshot(name = 'auto') {
|
|
269
|
-
// 防止并发创建
|
|
270
341
|
if (this.isCreatingSnapshot) {
|
|
271
342
|
console.log('[SNAPSHOT] 另一个快照正在创建中,跳过');
|
|
272
343
|
return null;
|
|
@@ -282,31 +353,39 @@ class SnapshotManager {
|
|
|
282
353
|
|
|
283
354
|
console.log(`[SNAPSHOT] 开始创建快照: ${snapshotId}`);
|
|
284
355
|
|
|
285
|
-
//
|
|
356
|
+
// 优先使用分区图的全量导出(避免仅导出窗口)
|
|
357
|
+
let memesAll = [];
|
|
358
|
+
if (this.runtime.graph && typeof this.runtime.graph.exportAllPoints === 'function') {
|
|
359
|
+
try {
|
|
360
|
+
memesAll = await this.runtime.graph.exportAllPoints();
|
|
361
|
+
} catch (e) {
|
|
362
|
+
console.warn('[SNAPSHOT] 分区图导出失败,回退窗口:', e.message);
|
|
363
|
+
memesAll = this.runtime.graph.getAllPoints();
|
|
364
|
+
}
|
|
365
|
+
} else {
|
|
366
|
+
memesAll = this.runtime.graph.getAllPoints();
|
|
367
|
+
}
|
|
368
|
+
|
|
286
369
|
const snapshotData = {
|
|
287
370
|
id: snapshotId,
|
|
288
371
|
timestamp,
|
|
289
372
|
name,
|
|
290
373
|
createDate: new Date().toISOString(),
|
|
291
|
-
memes:
|
|
374
|
+
memes: memesAll,
|
|
292
375
|
wordGraph: Array.from(this.runtime.wordGraph.points.values()),
|
|
293
|
-
kvm: this.runtime.kvm.
|
|
376
|
+
kvm: Array.from(this.runtime.kvm.memory.entries()),
|
|
294
377
|
vocab: this.runtime.vocabManager.vocab,
|
|
295
|
-
wordAccessLog: Array.from(this.runtime.wordAccessLog
|
|
378
|
+
wordAccessLog: Array.from(this.runtime.wordAccessLog.entries()).map(([w, per]) =>
|
|
379
|
+
[w, per instanceof Map ? Array.from(per.entries()) : (Array.isArray(per) ? [['legacy', per.length]] : [])]
|
|
380
|
+
),
|
|
381
|
+
sessions: this.runtime.session.export()
|
|
296
382
|
};
|
|
297
383
|
|
|
298
|
-
// 写入临时文件,然后原子重命名以确保数据完整性
|
|
299
384
|
const tempPath = `${filePath}.temp`;
|
|
300
385
|
await fs.promises.writeFile(tempPath, JSON.stringify(snapshotData), 'utf-8');
|
|
301
386
|
await fs.promises.rename(tempPath, filePath);
|
|
302
387
|
|
|
303
|
-
|
|
304
|
-
const snapshotInfo = {
|
|
305
|
-
id: snapshotId,
|
|
306
|
-
timestamp,
|
|
307
|
-
name,
|
|
308
|
-
path: filePath
|
|
309
|
-
};
|
|
388
|
+
const snapshotInfo = { id: snapshotId, timestamp, name, path: filePath };
|
|
310
389
|
this.snapshotList.unshift(snapshotInfo);
|
|
311
390
|
|
|
312
391
|
console.timeEnd('snapshotCreation');
|
|
@@ -320,11 +399,11 @@ class SnapshotManager {
|
|
|
320
399
|
}
|
|
321
400
|
}
|
|
322
401
|
|
|
402
|
+
|
|
323
403
|
async restoreSnapshot(snapshotId) {
|
|
324
404
|
console.log(`[SNAPSHOT] 开始从快照恢复: ${snapshotId}`);
|
|
325
405
|
console.time('snapshotRestore');
|
|
326
406
|
|
|
327
|
-
// 查找快照
|
|
328
407
|
const snapshot = this.snapshotList.find(s => s.id === snapshotId);
|
|
329
408
|
if (!snapshot) {
|
|
330
409
|
console.error(`[SNAPSHOT] 快照不存在: ${snapshotId}`);
|
|
@@ -332,37 +411,27 @@ class SnapshotManager {
|
|
|
332
411
|
}
|
|
333
412
|
|
|
334
413
|
try {
|
|
335
|
-
// 读取快照文件
|
|
336
|
-
console.log(`[SNAPSHOT] 从文件读取数据: ${snapshot.path}`);
|
|
337
414
|
const dataStr = await fs.promises.readFile(snapshot.path, 'utf-8');
|
|
338
415
|
const data = JSON.parse(dataStr);
|
|
339
416
|
|
|
340
|
-
// 在恢复前创建自动备份
|
|
341
417
|
await this.createSnapshot(`auto_before_restore_${snapshotId}`);
|
|
342
418
|
|
|
343
|
-
//
|
|
344
|
-
console.log('[SNAPSHOT] 清空当前运行时...');
|
|
345
|
-
this.runtime.graph = new GraphDB();
|
|
419
|
+
// 清空当前运行时(词图/KVM 内存)
|
|
346
420
|
this.runtime.wordGraph = new GraphDB();
|
|
347
421
|
this.runtime.kvm = new KVM();
|
|
348
422
|
this.runtime.wordAccessLog = new Map();
|
|
349
423
|
|
|
350
|
-
//
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
this.runtime.graph.addPoint(point.pointID, point.connect);
|
|
358
|
-
}
|
|
359
|
-
// 让事件循环有机会处理其他事件
|
|
360
|
-
await new Promise(resolve => setImmediate(resolve));
|
|
424
|
+
// 恢复模因图:走分区导入(覆盖分区存储)
|
|
425
|
+
if (data.memes && this.runtime.graph && typeof this.runtime.graph.importAllPoints === 'function') {
|
|
426
|
+
await this.runtime.graph.importAllPoints(data.memes);
|
|
427
|
+
} else if (data.memes) {
|
|
428
|
+
// 窗口回退(不推荐)
|
|
429
|
+
for (const point of data.memes) {
|
|
430
|
+
await this.runtime.graph.addPoint(point.pointID, point.connect);
|
|
361
431
|
}
|
|
362
432
|
}
|
|
363
433
|
|
|
364
434
|
// 恢复词图
|
|
365
|
-
console.log('[SNAPSHOT] 恢复词语网络...');
|
|
366
435
|
if (data.wordGraph) {
|
|
367
436
|
const BATCH_SIZE = 1000;
|
|
368
437
|
for (let i = 0; i < data.wordGraph.length; i += BATCH_SIZE) {
|
|
@@ -375,29 +444,39 @@ class SnapshotManager {
|
|
|
375
444
|
}
|
|
376
445
|
|
|
377
446
|
// 恢复KVM
|
|
378
|
-
console.log('[SNAPSHOT] 恢复键值存储...');
|
|
379
447
|
if (data.kvm) {
|
|
380
448
|
const BATCH_SIZE = 1000;
|
|
381
449
|
for (let i = 0; i < data.kvm.length; i += BATCH_SIZE) {
|
|
382
450
|
const batch = data.kvm.slice(i, i + BATCH_SIZE);
|
|
383
|
-
for (const [k, v] of batch)
|
|
384
|
-
this.runtime.kvm.set(k, v);
|
|
385
|
-
}
|
|
451
|
+
for (const [k, v] of batch) this.runtime.kvm.set(k, v);
|
|
386
452
|
await new Promise(resolve => setImmediate(resolve));
|
|
387
453
|
}
|
|
388
454
|
}
|
|
389
455
|
|
|
390
456
|
// 恢复词表
|
|
391
|
-
console.log('[SNAPSHOT] 恢复词表...');
|
|
392
457
|
if (data.vocab) {
|
|
393
458
|
this.runtime.vocabManager.vocab = data.vocab;
|
|
394
459
|
this.runtime.vocabManager.updateMappings();
|
|
395
460
|
}
|
|
396
461
|
|
|
397
462
|
// 恢复词访问日志
|
|
398
|
-
console.log('[SNAPSHOT] 恢复词访问日志...');
|
|
399
463
|
if (data.wordAccessLog) {
|
|
400
|
-
|
|
464
|
+
const restored = new Map();
|
|
465
|
+
for (const [word, per] of data.wordAccessLog) {
|
|
466
|
+
if (Array.isArray(per) && per.length > 0 && Array.isArray(per[0])) {
|
|
467
|
+
restored.set(word, new Map(per));
|
|
468
|
+
} else if (Array.isArray(per)) {
|
|
469
|
+
restored.set(word, new Map([['legacy', per.length]]));
|
|
470
|
+
} else {
|
|
471
|
+
restored.set(word, new Map());
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
this.runtime.wordAccessLog = restored;
|
|
475
|
+
}
|
|
476
|
+
if (data.sessions) {
|
|
477
|
+
this.runtime.session.import(data.sessions);
|
|
478
|
+
} else {
|
|
479
|
+
this.runtime.session.startNewSession({ reason: 'snapshot-legacy' });
|
|
401
480
|
}
|
|
402
481
|
|
|
403
482
|
console.timeEnd('snapshotRestore');
|
|
@@ -715,6 +794,727 @@ class GraphDB {
|
|
|
715
794
|
}
|
|
716
795
|
}
|
|
717
796
|
}
|
|
797
|
+
// ...existing code...
|
|
798
|
+
const crypto = require('crypto');
|
|
799
|
+
// ...existing code...
|
|
800
|
+
|
|
801
|
+
// ========================== 分区图存储适配层与滑动窗口 ==========================
|
|
802
|
+
|
|
803
|
+
// 简易日志辅助
|
|
804
|
+
function logPart(...args) { console.log('[PART]', ...args); }
|
|
805
|
+
function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
|
|
806
|
+
|
|
807
|
+
// 存储适配层(FS/LMDB/Level 多后端,按需加载)
|
|
808
|
+
class GraphStorageAdapter {
|
|
809
|
+
constructor({ baseDir, backend = 'fs' } = {}) {
|
|
810
|
+
this.baseDir = baseDir || path.join(__dirname, 'graph_parts');
|
|
811
|
+
this.backend = backend;
|
|
812
|
+
this.ready = false;
|
|
813
|
+
|
|
814
|
+
// 尝试创建目录
|
|
815
|
+
fs.mkdirSync(this.baseDir, { recursive: true });
|
|
816
|
+
|
|
817
|
+
// 可选依赖
|
|
818
|
+
this.lmdb = null;
|
|
819
|
+
this.level = null;
|
|
820
|
+
|
|
821
|
+
if (backend === 'lmdb') {
|
|
822
|
+
try {
|
|
823
|
+
this.lmdb = require('lmdb');
|
|
824
|
+
} catch (e) {
|
|
825
|
+
console.warn('[PART][ADAPTER] LMDB 不可用,降级为 FS:', e.message);
|
|
826
|
+
this.backend = 'fs';
|
|
827
|
+
}
|
|
828
|
+
}
|
|
829
|
+
if (backend === 'level') {
|
|
830
|
+
try {
|
|
831
|
+
this.level = require('level');
|
|
832
|
+
} catch (e) {
|
|
833
|
+
console.warn('[PART][ADAPTER] level 不可用,降级为 FS:', e.message);
|
|
834
|
+
this.backend = 'fs';
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
|
|
838
|
+
// 初始化后端
|
|
839
|
+
this._initBackend();
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
_initBackend() {
|
|
843
|
+
if (this.backend === 'fs') {
|
|
844
|
+
// FS: 每个分区一个 .jsonl(节点),边界事件一个独立 .jsonl
|
|
845
|
+
this.ready = true;
|
|
846
|
+
return;
|
|
847
|
+
}
|
|
848
|
+
if (this.backend === 'lmdb' && this.lmdb) {
|
|
849
|
+
try {
|
|
850
|
+
const storeDir = path.join(this.baseDir, 'lmdb');
|
|
851
|
+
fs.mkdirSync(storeDir, { recursive: true });
|
|
852
|
+
this.env = this.lmdb.open({
|
|
853
|
+
path: storeDir,
|
|
854
|
+
mapSize: 1024n * 1024n * 1024n * 64n,
|
|
855
|
+
compression: true,
|
|
856
|
+
});
|
|
857
|
+
this.ready = true;
|
|
858
|
+
} catch (e) {
|
|
859
|
+
console.warn('[PART][ADAPTER] LMDB 初始化失败,降级 FS:', e.message);
|
|
860
|
+
this.backend = 'fs';
|
|
861
|
+
this.ready = true;
|
|
862
|
+
}
|
|
863
|
+
return;
|
|
864
|
+
}
|
|
865
|
+
if (this.backend === 'level' && this.level) {
|
|
866
|
+
try {
|
|
867
|
+
const dbDir = path.join(this.baseDir, 'leveldb');
|
|
868
|
+
fs.mkdirSync(dbDir, { recursive: true });
|
|
869
|
+
this.db = new this.level.Level(dbDir, { valueEncoding: 'json' });
|
|
870
|
+
this.ready = true;
|
|
871
|
+
} catch (e) {
|
|
872
|
+
console.warn('[PART][ADAPTER] level 初始化失败,降级 FS:', e.message);
|
|
873
|
+
this.backend = 'fs';
|
|
874
|
+
this.ready = true;
|
|
875
|
+
}
|
|
876
|
+
return;
|
|
877
|
+
}
|
|
878
|
+
this.ready = true;
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
// 分区文件名(FS)
|
|
882
|
+
_partFile(pid) { return path.join(this.baseDir, `p_${pid}.jsonl`); }
|
|
883
|
+
_eventFile(pid) { return path.join(this.baseDir, `p_${pid}.events.jsonl`); }
|
|
884
|
+
|
|
885
|
+
// 读取分区(返回 { points: Map<string,{pointID,connect:[]}> })
|
|
886
|
+
async loadPartition(pid) {
|
|
887
|
+
if (this.backend === 'fs') {
|
|
888
|
+
const file = this._partFile(pid);
|
|
889
|
+
const out = new Map();
|
|
890
|
+
if (!fs.existsSync(file)) return { points: out };
|
|
891
|
+
const rs = fs.createReadStream(file, { encoding: 'utf-8' });
|
|
892
|
+
let buf = '';
|
|
893
|
+
for await (const chunk of rs) {
|
|
894
|
+
buf += chunk;
|
|
895
|
+
let idx;
|
|
896
|
+
while ((idx = buf.indexOf('\n')) >= 0) {
|
|
897
|
+
const line = buf.slice(0, idx);
|
|
898
|
+
buf = buf.slice(idx + 1);
|
|
899
|
+
if (!line.trim()) continue;
|
|
900
|
+
try {
|
|
901
|
+
const obj = JSON.parse(line);
|
|
902
|
+
if (obj && obj.pointID) {
|
|
903
|
+
out.set(obj.pointID, { pointID: obj.pointID, connect: obj.connect || [] });
|
|
904
|
+
}
|
|
905
|
+
} catch { /* ignore */ }
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
return { points: out };
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
if (this.backend === 'lmdb' && this.env) {
|
|
912
|
+
const points = new Map();
|
|
913
|
+
const txn = this.env.beginTxn({ readOnly: true });
|
|
914
|
+
try {
|
|
915
|
+
const cursor = new this.lmdb.Cursors.Cursor(txn, this.env.openDB({ name: `p_${pid}`, create: true }));
|
|
916
|
+
for (let found = cursor.goToFirst(); found; found = cursor.goToNext()) {
|
|
917
|
+
const key = cursor.getCurrentString();
|
|
918
|
+
const val = cursor.getCurrentBinary();
|
|
919
|
+
try {
|
|
920
|
+
const obj = JSON.parse(Buffer.from(val).toString('utf-8'));
|
|
921
|
+
if (obj && obj.pointID) points.set(obj.pointID, obj);
|
|
922
|
+
} catch { }
|
|
923
|
+
}
|
|
924
|
+
cursor.close();
|
|
925
|
+
} catch { }
|
|
926
|
+
txn.abort();
|
|
927
|
+
return { points };
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
if (this.backend === 'level' && this.db) {
|
|
931
|
+
const points = new Map();
|
|
932
|
+
try {
|
|
933
|
+
for await (const { key, value } of this.db.iterator({ gte: `p:${pid}:`, lt: `p:${pid};` })) {
|
|
934
|
+
const obj = value;
|
|
935
|
+
if (obj && obj.pointID) points.set(obj.pointID, obj);
|
|
936
|
+
}
|
|
937
|
+
} catch { }
|
|
938
|
+
return { points };
|
|
939
|
+
}
|
|
940
|
+
|
|
941
|
+
return { points: new Map() };
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
// 保存分区(全量覆盖写)
|
|
945
|
+
async savePartition(pid, pointsMap) {
|
|
946
|
+
if (!(pointsMap instanceof Map)) return;
|
|
947
|
+
if (this.backend === 'fs') {
|
|
948
|
+
const file = this._partFile(pid);
|
|
949
|
+
const tmp = `${file}.tmp`;
|
|
950
|
+
const ws = fs.createWriteStream(tmp, { encoding: 'utf-8' });
|
|
951
|
+
for (const [, p] of pointsMap.entries()) {
|
|
952
|
+
ws.write(JSON.stringify({ pointID: p.pointID, connect: p.connect || [] }) + '\n');
|
|
953
|
+
}
|
|
954
|
+
await new Promise((res, rej) => ws.end(res));
|
|
955
|
+
await fs.promises.rename(tmp, file);
|
|
956
|
+
return;
|
|
957
|
+
}
|
|
958
|
+
|
|
959
|
+
if (this.backend === 'lmdb' && this.env) {
|
|
960
|
+
const dbi = this.env.openDB({ name: `p_${pid}`, create: true });
|
|
961
|
+
const txn = this.env.beginTxn();
|
|
962
|
+
try {
|
|
963
|
+
// 先清空:简化实现
|
|
964
|
+
const cur = new this.lmdb.Cursors.Cursor(txn, dbi);
|
|
965
|
+
for (let found = cur.goToFirst(); found; found = cur.goToNext()) {
|
|
966
|
+
const k = cur.getCurrentString();
|
|
967
|
+
txn.del(dbi, k);
|
|
968
|
+
}
|
|
969
|
+
cur.close();
|
|
970
|
+
for (const [, p] of pointsMap.entries()) {
|
|
971
|
+
txn.put(dbi, p.pointID, JSON.stringify(p));
|
|
972
|
+
}
|
|
973
|
+
txn.commit();
|
|
974
|
+
} catch (e) {
|
|
975
|
+
try { txn.abort(); } catch { }
|
|
976
|
+
console.warn('[PART][ADAPTER][LMDB] savePartition err:', e.message);
|
|
977
|
+
}
|
|
978
|
+
return;
|
|
979
|
+
}
|
|
980
|
+
|
|
981
|
+
if (this.backend === 'level' && this.db) {
|
|
982
|
+
const ops = [];
|
|
983
|
+
// 简化:清理旧 key 不容易,直接覆盖同 key
|
|
984
|
+
for (const [, p] of pointsMap.entries()) {
|
|
985
|
+
ops.push({ type: 'put', key: `p:${pid}:${p.pointID}`, value: p });
|
|
986
|
+
}
|
|
987
|
+
await this.db.batch(ops);
|
|
988
|
+
return;
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
|
|
992
|
+
// 追加边界事件(跨分区边)
|
|
993
|
+
async appendEdgeEvent(pid, event) {
|
|
994
|
+
if (!event || !event.type) return;
|
|
995
|
+
if (this.backend === 'fs') {
|
|
996
|
+
const file = this._eventFile(pid);
|
|
997
|
+
fs.appendFileSync(file, JSON.stringify(event) + '\n', 'utf-8');
|
|
998
|
+
return;
|
|
999
|
+
}
|
|
1000
|
+
if (this.backend === 'lmdb' && this.env) {
|
|
1001
|
+
const dbi = this.env.openDB({ name: `e_${pid}`, create: true });
|
|
1002
|
+
const txn = this.env.beginTxn();
|
|
1003
|
+
try {
|
|
1004
|
+
const key = `e:${Date.now()}_${Math.random().toString(36).slice(2, 10)}`;
|
|
1005
|
+
txn.put(dbi, key, JSON.stringify(event));
|
|
1006
|
+
txn.commit();
|
|
1007
|
+
} catch (e) {
|
|
1008
|
+
try { txn.abort(); } catch { }
|
|
1009
|
+
}
|
|
1010
|
+
return;
|
|
1011
|
+
}
|
|
1012
|
+
if (this.backend === 'level' && this.db) {
|
|
1013
|
+
const key = `e:${pid}:${Date.now()}_${Math.random().toString(36).slice(2, 10)}`;
|
|
1014
|
+
await this.db.put(key, event);
|
|
1015
|
+
return;
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
// 读取并消费边界事件(与该分区相关的)
|
|
1020
|
+
async consumeEdgeEvents(pid, filterFn = null, limit = 2000) {
|
|
1021
|
+
const events = [];
|
|
1022
|
+
if (this.backend === 'fs') {
|
|
1023
|
+
const file = this._eventFile(pid);
|
|
1024
|
+
if (!fs.existsSync(file)) return events;
|
|
1025
|
+
|
|
1026
|
+
const tmp = `${file}.tmp`;
|
|
1027
|
+
// 将不消费的事件写入 tmp,再覆盖原文件;已消费事件返回
|
|
1028
|
+
const lines = fs.readFileSync(file, 'utf-8').split(/\r?\n/).filter(Boolean);
|
|
1029
|
+
const remain = [];
|
|
1030
|
+
for (const line of lines) {
|
|
1031
|
+
try {
|
|
1032
|
+
const e = JSON.parse(line);
|
|
1033
|
+
const ok = filterFn ? filterFn(e) : true;
|
|
1034
|
+
if (ok && events.length < limit) {
|
|
1035
|
+
events.push(e);
|
|
1036
|
+
} else {
|
|
1037
|
+
remain.push(line);
|
|
1038
|
+
}
|
|
1039
|
+
} catch {
|
|
1040
|
+
remain.push(line);
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
fs.writeFileSync(tmp, remain.join('\n') + (remain.length ? '\n' : ''), 'utf-8');
|
|
1044
|
+
await fs.promises.rename(tmp, file);
|
|
1045
|
+
return events;
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
if (this.backend === 'lmdb' && this.env) {
|
|
1049
|
+
const dbi = this.env.openDB({ name: `e_${pid}`, create: true });
|
|
1050
|
+
const txn = this.env.beginTxn();
|
|
1051
|
+
const toDel = [];
|
|
1052
|
+
try {
|
|
1053
|
+
const cur = new this.lmdb.Cursors.Cursor(txn, dbi);
|
|
1054
|
+
for (let found = cur.goToFirst(); found; found = cur.goToNext()) {
|
|
1055
|
+
const k = cur.getCurrentString();
|
|
1056
|
+
const v = cur.getCurrentBinary();
|
|
1057
|
+
const e = JSON.parse(Buffer.from(v).toString('utf-8'));
|
|
1058
|
+
const ok = filterFn ? filterFn(e) : true;
|
|
1059
|
+
if (ok && events.length < limit) {
|
|
1060
|
+
events.push(e);
|
|
1061
|
+
toDel.push(k);
|
|
1062
|
+
}
|
|
1063
|
+
}
|
|
1064
|
+
cur.close();
|
|
1065
|
+
for (const k of toDel) txn.del(dbi, k);
|
|
1066
|
+
txn.commit();
|
|
1067
|
+
} catch (e) {
|
|
1068
|
+
try { txn.abort(); } catch { }
|
|
1069
|
+
}
|
|
1070
|
+
return events;
|
|
1071
|
+
}
|
|
1072
|
+
|
|
1073
|
+
if (this.backend === 'level' && this.db) {
|
|
1074
|
+
// 简化:扫描全库 keys 读取该 pid 的事件
|
|
1075
|
+
try {
|
|
1076
|
+
const toDel = [];
|
|
1077
|
+
for await (const { key, value } of this.db.iterator({ gte: `e:${pid}:`, lt: `e:${pid};` })) {
|
|
1078
|
+
const ok = filterFn ? filterFn(value) : true;
|
|
1079
|
+
if (ok && events.length < limit) {
|
|
1080
|
+
events.push(value);
|
|
1081
|
+
toDel.push(key);
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
// 删除已消费
|
|
1085
|
+
const ops = toDel.map(k => ({ type: 'del', key: k }));
|
|
1086
|
+
if (ops.length) await this.db.batch(ops);
|
|
1087
|
+
} catch { }
|
|
1088
|
+
return events;
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1091
|
+
return events;
|
|
1092
|
+
}
|
|
1093
|
+
|
|
1094
|
+
// 枚举所有分区 ID(FS 模式)
|
|
1095
|
+
async listPartitionIds() {
|
|
1096
|
+
if (this.backend === 'fs') {
|
|
1097
|
+
const files = fs.readdirSync(this.baseDir).filter(f => /^p_\d+\.jsonl$/.test(f));
|
|
1098
|
+
const ids = files.map(f => Number(f.match(/^p_(\d+)\.jsonl$/)[1])).sort((a, b) => a - b);
|
|
1099
|
+
return ids;
|
|
1100
|
+
}
|
|
1101
|
+
// LMDB/level 不易列举,约定 0..N-1 尝试加载
|
|
1102
|
+
return [];
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
// 分区器(哈希 -> 分区ID)
|
|
1107
|
+
class GraphPartitioner {
|
|
1108
|
+
constructor({ partitions = 64 } = {}) {
|
|
1109
|
+
this.partitions = Math.max(4, partitions);
|
|
1110
|
+
}
|
|
1111
|
+
idOf(pointID) {
|
|
1112
|
+
if (!pointID) return 0;
|
|
1113
|
+
const h = crypto.createHash('sha1').update(String(pointID)).digest();
|
|
1114
|
+
// 使用前 4 字节构造 uint32
|
|
1115
|
+
const u32 = h.readUInt32BE(0);
|
|
1116
|
+
return u32 % this.partitions;
|
|
1117
|
+
}
|
|
1118
|
+
neighborsOf(pid, radius = 1) {
|
|
1119
|
+
const out = new Set([pid]);
|
|
1120
|
+
for (let r = 1; r <= radius; r++) {
|
|
1121
|
+
out.add((pid - r + this.partitions) % this.partitions);
|
|
1122
|
+
out.add((pid + r) % this.partitions);
|
|
1123
|
+
}
|
|
1124
|
+
return Array.from(out).sort((a, b) => a - b);
|
|
1125
|
+
}
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
// 分区图 + 滑动窗口 + 边界事件消费
|
|
1129
|
+
class PartitionedGraphDB {
|
|
1130
|
+
constructor({
|
|
1131
|
+
partitions = 64,
|
|
1132
|
+
maxLoadedPartitions = 8,
|
|
1133
|
+
windowRadius = 1,
|
|
1134
|
+
baseDir = path.join(__dirname, 'graph_parts'),
|
|
1135
|
+
backend = 'fs'
|
|
1136
|
+
} = {}) {
|
|
1137
|
+
this.partitioner = new GraphPartitioner({ partitions });
|
|
1138
|
+
this.adapter = new GraphStorageAdapter({ baseDir, backend });
|
|
1139
|
+
this.maxLoadedPartitions = Math.max(2, maxLoadedPartitions);
|
|
1140
|
+
this.windowRadius = Math.max(0, windowRadius);
|
|
1141
|
+
|
|
1142
|
+
// 已加载分区:pid -> { points: Map, dirty, lastAccess }
|
|
1143
|
+
this.loaded = new Map();
|
|
1144
|
+
// 兼容旧代码:合并视图(仅包含已加载分区的点)
|
|
1145
|
+
this.points = new Map();
|
|
1146
|
+
// LRU
|
|
1147
|
+
this.accessTick = 0;
|
|
1148
|
+
this.centerPid = null;
|
|
1149
|
+
|
|
1150
|
+
// 并发保护
|
|
1151
|
+
this.loading = new Set();
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
// ---------- 内部:加载/保存/淘汰 ----------
|
|
1155
|
+
async ensureLoaded(pid) {
|
|
1156
|
+
if (this.loaded.has(pid)) {
|
|
1157
|
+
this._touch(pid);
|
|
1158
|
+
return this.loaded.get(pid);
|
|
1159
|
+
}
|
|
1160
|
+
if (this.loading.has(pid)) {
|
|
1161
|
+
// 等待已有加载完成
|
|
1162
|
+
while (this.loading.has(pid)) { await sleep(10); }
|
|
1163
|
+
return this.loaded.get(pid);
|
|
1164
|
+
}
|
|
1165
|
+
this.loading.add(pid);
|
|
1166
|
+
try {
|
|
1167
|
+
const part = await this.adapter.loadPartition(pid);
|
|
1168
|
+
const bundle = {
|
|
1169
|
+
points: part.points || new Map(),
|
|
1170
|
+
dirty: false,
|
|
1171
|
+
lastAccess: ++this.accessTick
|
|
1172
|
+
};
|
|
1173
|
+
this.loaded.set(pid, bundle);
|
|
1174
|
+
// 合并到全局视图
|
|
1175
|
+
for (const [id, p] of bundle.points.entries()) this.points.set(id, p);
|
|
1176
|
+
|
|
1177
|
+
// 消费边界事件:把指向本分区的事件落库
|
|
1178
|
+
const events = await this.adapter.consumeEdgeEvents(pid, (e) =>
|
|
1179
|
+
e && e.type === 'cross-edge' && (e.toPid === pid || e.fromPid === pid), 5000);
|
|
1180
|
+
if (events.length) {
|
|
1181
|
+
for (const e of events) this._applyEdgeEvent(bundle, e);
|
|
1182
|
+
bundle.dirty = true;
|
|
1183
|
+
}
|
|
1184
|
+
|
|
1185
|
+
// 控制内存:若超容量,执行淘汰
|
|
1186
|
+
await this._evictIfNeeded();
|
|
1187
|
+
return bundle;
|
|
1188
|
+
} finally {
|
|
1189
|
+
this.loading.delete(pid);
|
|
1190
|
+
}
|
|
1191
|
+
}
|
|
1192
|
+
|
|
1193
|
+
async savePartitionIfDirty(pid) {
|
|
1194
|
+
const entry = this.loaded.get(pid);
|
|
1195
|
+
if (!entry) return;
|
|
1196
|
+
if (!entry.dirty) return;
|
|
1197
|
+
await this.adapter.savePartition(pid, entry.points);
|
|
1198
|
+
entry.dirty = false;
|
|
1199
|
+
}
|
|
1200
|
+
|
|
1201
|
+
async _evictIfNeeded() {
|
|
1202
|
+
if (this.loaded.size <= this.maxLoadedPartitions) return;
|
|
1203
|
+
// 淘汰最近最少访问的分区(除中心窗口)
|
|
1204
|
+
const avoid = new Set(this.partitioner.neighborsOf(this.centerPid ?? 0, this.windowRadius));
|
|
1205
|
+
// 构建按 lastAccess 升序
|
|
1206
|
+
const list = Array.from(this.loaded.entries())
|
|
1207
|
+
.filter(([pid]) => !avoid.has(pid))
|
|
1208
|
+
.sort((a, b) => a[1].lastAccess - b[1].lastAccess);
|
|
1209
|
+
while (this.loaded.size > this.maxLoadedPartitions && list.length) {
|
|
1210
|
+
const [pid, entry] = list.shift();
|
|
1211
|
+
await this.savePartitionIfDirty(pid);
|
|
1212
|
+
// 从全局视图移除
|
|
1213
|
+
for (const [id] of entry.points.entries()) this.points.delete(id);
|
|
1214
|
+
this.loaded.delete(pid);
|
|
1215
|
+
logPart('evicted partition', pid);
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
_touch(pid) {
|
|
1220
|
+
const entry = this.loaded.get(pid);
|
|
1221
|
+
if (entry) entry.lastAccess = ++this.accessTick;
|
|
1222
|
+
}
|
|
1223
|
+
|
|
1224
|
+
_applyEdgeEvent(targetBundle, e) {
|
|
1225
|
+
// 事件格式:{ type:'cross-edge', from:'id', to:'id', weight, direction, fromPid, toPid }
|
|
1226
|
+
if (!e || e.type !== 'cross-edge') return;
|
|
1227
|
+
const ensurePoint = (m, id) => {
|
|
1228
|
+
if (!m.has(id)) m.set(id, { pointID: id, connect: [] });
|
|
1229
|
+
return m.get(id);
|
|
1230
|
+
};
|
|
1231
|
+
const mp = targetBundle.points;
|
|
1232
|
+
const pFrom = ensurePoint(mp, e.from);
|
|
1233
|
+
const pTo = ensurePoint(mp, e.to);
|
|
1234
|
+
// 在 from 中落边(若 from 属于本分区)
|
|
1235
|
+
if (e.toPid === e.fromPid) {
|
|
1236
|
+
// 同分区事件(理论上不会在事件日志里)
|
|
1237
|
+
if (!pFrom.connect.some(([w, id, d]) => id === e.to && d === e.direction)) {
|
|
1238
|
+
pFrom.connect.push([e.weight, e.to, e.direction]);
|
|
1239
|
+
}
|
|
1240
|
+
} else {
|
|
1241
|
+
// 当前 bundle 即为 toPid 或 fromPid 的载体
|
|
1242
|
+
if (e.toPid === this.partitioner.idOf(pTo.pointID)) {
|
|
1243
|
+
// 对于目标分区,至少要保证可被 selectPath 遍历;保留边终点即可(可选:反向提示边)
|
|
1244
|
+
// 不在 pTo 里写边(避免双写),仅保证 from 的边会在 from 分区生效
|
|
1245
|
+
}
|
|
1246
|
+
if (e.fromPid === this.partitioner.idOf(pFrom.pointID)) {
|
|
1247
|
+
if (!pFrom.connect.some(([w, id, d]) => id === e.to && d === e.direction)) {
|
|
1248
|
+
pFrom.connect.push([e.weight, e.to, e.direction]);
|
|
1249
|
+
}
|
|
1250
|
+
}
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
1253
|
+
|
|
1254
|
+
// ---------- 滑动窗口 ----------
|
|
1255
|
+
async focusOnPoint(pointID) {
|
|
1256
|
+
const pid = this.partitioner.idOf(pointID);
|
|
1257
|
+
this.centerPid = pid;
|
|
1258
|
+
const toLoad = this.partitioner.neighborsOf(pid, this.windowRadius);
|
|
1259
|
+
for (const id of toLoad) await this.ensureLoaded(id);
|
|
1260
|
+
await this._evictIfNeeded();
|
|
1261
|
+
}
|
|
1262
|
+
|
|
1263
|
+
// ---------- 兼容 API:点/边 操作 ----------
|
|
1264
|
+
addPoint(pointID, connect = []) {
|
|
1265
|
+
const pid = this.partitioner.idOf(pointID);
|
|
1266
|
+
const ensure = (bundle) => {
|
|
1267
|
+
if (!bundle.points.has(pointID)) bundle.points.set(pointID, { pointID, connect: [] });
|
|
1268
|
+
this.points.set(pointID, bundle.points.get(pointID));
|
|
1269
|
+
return bundle.points.get(pointID);
|
|
1270
|
+
};
|
|
1271
|
+
return this.ensureLoaded(pid).then(bundle => {
|
|
1272
|
+
const p = ensure(bundle);
|
|
1273
|
+
// 添加本地边;跨分区写事件
|
|
1274
|
+
for (const [w, nid, dir] of connect) this._addEdgeInternal(pid, p, w, nid, dir, bundle);
|
|
1275
|
+
bundle.dirty = true;
|
|
1276
|
+
});
|
|
1277
|
+
}
|
|
1278
|
+
|
|
1279
|
+
_addEdgeInternal(fromPid, fromPoint, weight, toID, direction, bundleOfFrom) {
|
|
1280
|
+
const toPid = this.partitioner.idOf(toID);
|
|
1281
|
+
const w = (typeof weight === 'number' && isFinite(weight)) ? weight : 1;
|
|
1282
|
+
const d = (direction === 0 || direction === 1 || direction === 2) ? direction : 0;
|
|
1283
|
+
|
|
1284
|
+
if (toPid === fromPid) {
|
|
1285
|
+
// 同分区直接写
|
|
1286
|
+
if (!fromPoint.connect.some(([ww, id, dd]) => id === toID && dd === d)) {
|
|
1287
|
+
fromPoint.connect.push([w, toID, d]);
|
|
1288
|
+
bundleOfFrom.dirty = true;
|
|
1289
|
+
}
|
|
1290
|
+
} else {
|
|
1291
|
+
// 跨分区 -> 记录边界事件至 fromPid(或 toPid 都可,这里记录到 fromPid,toPid 加载时也会消费相关事件)
|
|
1292
|
+
this.adapter.appendEdgeEvent(fromPid, {
|
|
1293
|
+
type: 'cross-edge',
|
|
1294
|
+
from: fromPoint.pointID,
|
|
1295
|
+
to: toID,
|
|
1296
|
+
weight: w,
|
|
1297
|
+
direction: d,
|
|
1298
|
+
fromPid,
|
|
1299
|
+
toPid
|
|
1300
|
+
});
|
|
1301
|
+
// 同时对“已加载且包含 toPid 的 bundle”进行即时应用(若存在)
|
|
1302
|
+
const toBundle = this.loaded.get(toPid);
|
|
1303
|
+
if (toBundle) {
|
|
1304
|
+
// 在 from 分区已经写入 from->to 事件;对于 to 分区无需写边(避免双写),可选择记录提示(此处略)
|
|
1305
|
+
}
|
|
1306
|
+
}
|
|
1307
|
+
}
|
|
1308
|
+
|
|
1309
|
+
addBidirectionalEdge(id1, id2, weight = 1) {
|
|
1310
|
+
return this.addEdge(id1, id2, weight, 0);
|
|
1311
|
+
}
|
|
1312
|
+
|
|
1313
|
+
async addEdge(fromID, toID, weight = 1, direction = 0) {
|
|
1314
|
+
const fromPid = this.partitioner.idOf(fromID);
|
|
1315
|
+
const fromBundle = await this.ensureLoaded(fromPid);
|
|
1316
|
+
if (!fromBundle.points.has(fromID)) {
|
|
1317
|
+
fromBundle.points.set(fromID, { pointID: fromID, connect: [] });
|
|
1318
|
+
this.points.set(fromID, fromBundle.points.get(fromID));
|
|
1319
|
+
}
|
|
1320
|
+
const fromPoint = fromBundle.points.get(fromID);
|
|
1321
|
+
this._addEdgeInternal(fromPid, fromPoint, weight, toID, direction, fromBundle);
|
|
1322
|
+
|
|
1323
|
+
if (direction === 0) {
|
|
1324
|
+
// 双向边:反向写入
|
|
1325
|
+
const toPid = this.partitioner.idOf(toID);
|
|
1326
|
+
const toBundle = await this.ensureLoaded(toPid);
|
|
1327
|
+
if (!toBundle.points.has(toID)) {
|
|
1328
|
+
toBundle.points.set(toID, { pointID: toID, connect: [] });
|
|
1329
|
+
this.points.set(toID, toBundle.points.get(toID));
|
|
1330
|
+
}
|
|
1331
|
+
const toPoint = toBundle.points.get(toID);
|
|
1332
|
+
this._addEdgeInternal(toPid, toPoint, weight, fromID, 0, toBundle);
|
|
1333
|
+
}
|
|
1334
|
+
}
|
|
1335
|
+
|
|
1336
|
+
async updateEdge(fromID, toID, newWeight, direction = 0) {
|
|
1337
|
+
const fromPid = this.partitioner.idOf(fromID);
|
|
1338
|
+
const b = await this.ensureLoaded(fromPid);
|
|
1339
|
+
const p = b.points.get(fromID);
|
|
1340
|
+
if (!p) return;
|
|
1341
|
+
const idx = p.connect.findIndex(([w, id, d]) => id === toID && d === direction);
|
|
1342
|
+
if (idx >= 0) {
|
|
1343
|
+
p.connect[idx][0] = newWeight;
|
|
1344
|
+
b.dirty = true;
|
|
1345
|
+
} else {
|
|
1346
|
+
// 不存在则添加
|
|
1347
|
+
this._addEdgeInternal(fromPid, p, newWeight, toID, direction, b);
|
|
1348
|
+
}
|
|
1349
|
+
}
|
|
1350
|
+
|
|
1351
|
+
existEdge(fromID, toID) {
|
|
1352
|
+
const fromPid = this.partitioner.idOf(fromID);
|
|
1353
|
+
const entry = this.loaded.get(fromPid);
|
|
1354
|
+
if (!entry) return { exist: false, weight: undefined, type: undefined };
|
|
1355
|
+
const p = entry.points.get(fromID);
|
|
1356
|
+
if (!p) return { exist: false, weight: undefined, type: undefined };
|
|
1357
|
+
const found = p.connect.find(([w, id]) => id === toID);
|
|
1358
|
+
return { exist: !!found, weight: found ? found[0] : undefined, type: found ? found[2] : undefined };
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
existPoint(pointID) {
|
|
1362
|
+
// 仅检查已加载窗口
|
|
1363
|
+
const p = this.points.get(pointID);
|
|
1364
|
+
return { exist: !!p, connect: p ? p.connect : [] };
|
|
1365
|
+
}
|
|
1366
|
+
|
|
1367
|
+
deleteEdge(a, b) {
|
|
1368
|
+
const pid = this.partitioner.idOf(a);
|
|
1369
|
+
const entry = this.loaded.get(pid);
|
|
1370
|
+
if (!entry) return;
|
|
1371
|
+
const p = entry.points.get(a);
|
|
1372
|
+
if (!p) return;
|
|
1373
|
+
const before = p.connect.length;
|
|
1374
|
+
p.connect = p.connect.filter(([_, id]) => id !== b);
|
|
1375
|
+
entry.dirty = entry.dirty || (p.connect.length !== before);
|
|
1376
|
+
}
|
|
1377
|
+
|
|
1378
|
+
deletePoint(pointID) {
|
|
1379
|
+
const pid = this.partitioner.idOf(pointID);
|
|
1380
|
+
const entry = this.loaded.get(pid);
|
|
1381
|
+
if (!entry) return;
|
|
1382
|
+
if (entry.points.has(pointID)) {
|
|
1383
|
+
entry.points.delete(pointID);
|
|
1384
|
+
this.points.delete(pointID);
|
|
1385
|
+
entry.dirty = true;
|
|
1386
|
+
}
|
|
1387
|
+
}
|
|
1388
|
+
|
|
1389
|
+
// 仅遍历窗口内点(兼容旧 getAllPoints 调用)
|
|
1390
|
+
getAllPoints() {
|
|
1391
|
+
return Array.from(this.points.values());
|
|
1392
|
+
}
|
|
1393
|
+
|
|
1394
|
+
// 导出全量点(跨所有分区),用于快照/发布
|
|
1395
|
+
async exportAllPoints() {
|
|
1396
|
+
const out = [];
|
|
1397
|
+
// 尝试枚举 FS 分区;其他后端可按 0..N-1 遍历或仅导出已加载窗口
|
|
1398
|
+
const ids = await this.adapter.listPartitionIds();
|
|
1399
|
+
if (ids.length === 0) {
|
|
1400
|
+
// 回退:导出窗口
|
|
1401
|
+
return this.getAllPoints();
|
|
1402
|
+
}
|
|
1403
|
+
for (const pid of ids) {
|
|
1404
|
+
const part = await this.adapter.loadPartition(pid);
|
|
1405
|
+
for (const [, p] of part.points.entries()) out.push({ pointID: p.pointID, connect: p.connect || [] });
|
|
1406
|
+
}
|
|
1407
|
+
return out;
|
|
1408
|
+
}
|
|
1409
|
+
|
|
1410
|
+
// 批量导入(将 legacy 点集落到分区)
|
|
1411
|
+
async importAllPoints(pointsArr) {
|
|
1412
|
+
if (!Array.isArray(pointsArr)) return;
|
|
1413
|
+
// 分桶
|
|
1414
|
+
const buckets = new Map();
|
|
1415
|
+
for (const p of pointsArr) {
|
|
1416
|
+
const pid = this.partitioner.idOf(p.pointID);
|
|
1417
|
+
if (!buckets.has(pid)) buckets.set(pid, new Map());
|
|
1418
|
+
const bm = buckets.get(pid);
|
|
1419
|
+
bm.set(p.pointID, { pointID: p.pointID, connect: Array.isArray(p.connect) ? p.connect.slice() : [] });
|
|
1420
|
+
}
|
|
1421
|
+
// 写入并更新窗口视图(懒加载)
|
|
1422
|
+
for (const [pid, map] of buckets.entries()) {
|
|
1423
|
+
await this.adapter.savePartition(pid, map);
|
|
1424
|
+
// 若已加载该分区,刷新内存镜像
|
|
1425
|
+
if (this.loaded.has(pid)) {
|
|
1426
|
+
const entry = this.loaded.get(pid);
|
|
1427
|
+
// 从全局视图移除旧
|
|
1428
|
+
for (const [id] of entry.points.entries()) this.points.delete(id);
|
|
1429
|
+
entry.points = map;
|
|
1430
|
+
entry.dirty = false;
|
|
1431
|
+
entry.lastAccess = ++this.accessTick;
|
|
1432
|
+
for (const [id, p] of map.entries()) this.points.set(id, p);
|
|
1433
|
+
}
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
|
|
1437
|
+
// 聚合邻居(窗口内),供传播使用
|
|
1438
|
+
getNeighbors(pointID, maxNeighbors = 50) {
|
|
1439
|
+
const p = this.points.get(pointID);
|
|
1440
|
+
if (!p) return [];
|
|
1441
|
+
return p.connect.slice(0, maxNeighbors);
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1444
|
+
// A* 简化:仅在窗口内搜索;跳出窗口时,尝试预取邻接分区后再继续
|
|
1445
|
+
async selectPath(fromID, toID) {
|
|
1446
|
+
if (fromID === toID) return [fromID];
|
|
1447
|
+
// 优先保证焦点加载
|
|
1448
|
+
await this.focusOnPoint(fromID);
|
|
1449
|
+
|
|
1450
|
+
const reconstruct = (came, cur) => {
|
|
1451
|
+
const path = [];
|
|
1452
|
+
let t = cur;
|
|
1453
|
+
while (came.has(t)) { path.push(t); t = came.get(t); }
|
|
1454
|
+
path.push(fromID);
|
|
1455
|
+
return path.reverse();
|
|
1456
|
+
};
|
|
1457
|
+
|
|
1458
|
+
const open = new Set([fromID]);
|
|
1459
|
+
const came = new Map();
|
|
1460
|
+
const g = new Map([[fromID, 0]]);
|
|
1461
|
+
const f = new Map([[fromID, 1]]);
|
|
1462
|
+
const closed = new Set();
|
|
1463
|
+
|
|
1464
|
+
const heuristic = () => 1;
|
|
1465
|
+
let iter = 0;
|
|
1466
|
+
const MAX_ITERS = 5000;
|
|
1467
|
+
|
|
1468
|
+
while (open.size && iter++ < MAX_ITERS) {
|
|
1469
|
+
// 取 f 最小
|
|
1470
|
+
let cur = null; let minF = Infinity;
|
|
1471
|
+
for (const id of open) {
|
|
1472
|
+
const val = f.get(id) ?? Infinity;
|
|
1473
|
+
if (val < minF) { minF = val; cur = id; }
|
|
1474
|
+
}
|
|
1475
|
+
if (cur == null) break;
|
|
1476
|
+
if (cur === toID) return reconstruct(came, cur);
|
|
1477
|
+
|
|
1478
|
+
open.delete(cur);
|
|
1479
|
+
closed.add(cur);
|
|
1480
|
+
|
|
1481
|
+
// 若遇到未知点,尝试加载其分区(滑动窗口)
|
|
1482
|
+
if (!this.points.has(cur)) {
|
|
1483
|
+
await this.focusOnPoint(cur);
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
const neighbors = this.getNeighbors(cur, 50);
|
|
1487
|
+
// 如果邻居为空,尝试边界事件预取(根据邻居 ID 的分区预取)
|
|
1488
|
+
if (neighbors.length === 0) {
|
|
1489
|
+
const pid = this.partitioner.idOf(cur);
|
|
1490
|
+
const ring = this.partitioner.neighborsOf(pid, 1);
|
|
1491
|
+
for (const rid of ring) await this.ensureLoaded(rid);
|
|
1492
|
+
}
|
|
1493
|
+
|
|
1494
|
+
for (const [w, nb] of neighbors) {
|
|
1495
|
+
if (closed.has(nb)) continue;
|
|
1496
|
+
const tentative = (g.get(cur) || Infinity) + w;
|
|
1497
|
+
if (!open.has(nb)) open.add(nb);
|
|
1498
|
+
else if (tentative >= (g.get(nb) || Infinity)) continue;
|
|
1499
|
+
|
|
1500
|
+
came.set(nb, cur);
|
|
1501
|
+
g.set(nb, tentative);
|
|
1502
|
+
f.set(nb, tentative + heuristic());
|
|
1503
|
+
}
|
|
1504
|
+
}
|
|
1505
|
+
return null;
|
|
1506
|
+
}
|
|
1507
|
+
|
|
1508
|
+
// 刷盘所有已加载分区
|
|
1509
|
+
async flushAll() {
|
|
1510
|
+
for (const [pid] of this.loaded.entries()) await this.savePartitionIfDirty(pid);
|
|
1511
|
+
}
|
|
1512
|
+
}
|
|
1513
|
+
|
|
1514
|
+
// ========================== 替换 Runtime.graph 为分区图 ==========================
|
|
1515
|
+
// 说明:wordGraph 仍使用内存 GraphDB;模因图使用 PartitionedGraphDB
|
|
1516
|
+
|
|
1517
|
+
// ...existing code...
|
|
718
1518
|
|
|
719
1519
|
class KVM {
|
|
720
1520
|
// this KVM is the key-value memory
|
|
@@ -770,7 +1570,7 @@ class KVM {
|
|
|
770
1570
|
}
|
|
771
1571
|
delete(key) {
|
|
772
1572
|
if (this.useLMDB) {
|
|
773
|
-
try { this.db.remove(key); } catch (_) {}
|
|
1573
|
+
try { this.db.remove(key); } catch (_) { }
|
|
774
1574
|
return;
|
|
775
1575
|
}
|
|
776
1576
|
this.memory.delete(key);
|
|
@@ -783,7 +1583,7 @@ class KVM {
|
|
|
783
1583
|
for (const k of this.db.getKeys({ snapshot: true })) {
|
|
784
1584
|
let v = this.db.get(k);
|
|
785
1585
|
if (typeof v === 'string') {
|
|
786
|
-
try { v = JSON.parse(v); } catch (_) {}
|
|
1586
|
+
try { v = JSON.parse(v); } catch (_) { }
|
|
787
1587
|
}
|
|
788
1588
|
out.push([k, v]);
|
|
789
1589
|
}
|
|
@@ -877,7 +1677,17 @@ class Runtime {
|
|
|
877
1677
|
// 运行时负责AI核心的调度、模因转换、信号传递与主流程控制
|
|
878
1678
|
constructor(config = {}) {
|
|
879
1679
|
this.config = config;
|
|
880
|
-
|
|
1680
|
+
// 使用分区图作为模因图;词图仍用内存图
|
|
1681
|
+
this.graph = new PartitionedGraphDB({
|
|
1682
|
+
partitions: this.config.partitions || 64,
|
|
1683
|
+
maxLoadedPartitions: this.config.maxLoadedPartitions || 8,
|
|
1684
|
+
windowRadius: this.config.windowRadius || 1,
|
|
1685
|
+
baseDir: path.join(__dirname, 'graph_parts'),
|
|
1686
|
+
backend: this.config.graphBackend || 'fs' // 可选 'fs' | 'lmdb' | 'level'
|
|
1687
|
+
});
|
|
1688
|
+
this._act = BuiltinActivations.relu;
|
|
1689
|
+
this._transfer = BuiltinTransfers.linear;
|
|
1690
|
+
this._activationMeta = { activationType: 'relu', transferType: 'linear' };
|
|
881
1691
|
this.wordGraph = new GraphDB();
|
|
882
1692
|
this.kvm = new KVM();
|
|
883
1693
|
this.changer = new Changer();
|
|
@@ -898,6 +1708,37 @@ class Runtime {
|
|
|
898
1708
|
batchSizeMultiplier: 1
|
|
899
1709
|
};
|
|
900
1710
|
this.memeBarrier = new memeBarrier(this);
|
|
1711
|
+
}
|
|
1712
|
+
// 获取/设置激活-传递函数配置
|
|
1713
|
+
getActivationConfig() {
|
|
1714
|
+
return {
|
|
1715
|
+
activationType: this._activationMeta.activationType,
|
|
1716
|
+
transferType: this._activationMeta.transferType,
|
|
1717
|
+
activationCustom: this.config?.activationCustom || '',
|
|
1718
|
+
transferCustom: this.config?.transferCustom || ''
|
|
1719
|
+
};
|
|
1720
|
+
}
|
|
1721
|
+
|
|
1722
|
+
setActivationConfig({ activationType, transferType, activationCustom, transferCustom } = {}) {
|
|
1723
|
+
const aType = String(activationType || this._activationMeta.activationType || 'relu');
|
|
1724
|
+
const tType = String(transferType || this._activationMeta.transferType || 'linear');
|
|
1725
|
+
|
|
1726
|
+
let act = BuiltinActivations[aType] || BuiltinActivations.relu;
|
|
1727
|
+
let tr = BuiltinTransfers[tType] || BuiltinTransfers.linear;
|
|
1728
|
+
|
|
1729
|
+
if (aType === 'custom' && activationCustom) {
|
|
1730
|
+
act = compileCustomFunctionSafely(activationCustom, ['x'], BuiltinActivations.relu);
|
|
1731
|
+
}
|
|
1732
|
+
if (tType === 'custom' && transferCustom) {
|
|
1733
|
+
tr = compileCustomFunctionSafely(transferCustom, ['value', 'weight', 'decayK', 'ctx'], BuiltinTransfers.linear);
|
|
1734
|
+
}
|
|
1735
|
+
|
|
1736
|
+
this._act = (typeof act === 'function') ? act : BuiltinActivations.relu;
|
|
1737
|
+
this._transfer = (typeof tr === 'function') ? tr : BuiltinTransfers.linear;
|
|
1738
|
+
this._activationMeta = { activationType: aType, transferType: tType };
|
|
1739
|
+
this.config = this.config || {};
|
|
1740
|
+
this.config.activationCustom = activationCustom || this.config.activationCustom || '';
|
|
1741
|
+
this.config.transferCustom = transferCustom || this.config.transferCustom || '';
|
|
901
1742
|
}
|
|
902
1743
|
// 新增资源监控方法
|
|
903
1744
|
monitorSystemLoad() {
|
|
@@ -1059,19 +1900,15 @@ class Runtime {
|
|
|
1059
1900
|
visitCount++;
|
|
1060
1901
|
activatedOrder.push(id);
|
|
1061
1902
|
|
|
1062
|
-
// 仅在是“词”时记录访问,避免把模因ID写入词访问日志
|
|
1063
1903
|
if (this.wordGraph.points.has(id)) {
|
|
1064
1904
|
this.logWordAccess(id);
|
|
1065
1905
|
}
|
|
1066
1906
|
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
if (!visited.has(neighborID)) {
|
|
1073
|
-
next.push({ id: neighborID, value: value - decayK * weight });
|
|
1074
|
-
}
|
|
1907
|
+
// 改为通过 graph.getNeighbors 访问(窗口内)
|
|
1908
|
+
const neighbors = this.graph.getNeighbors(id, 50);
|
|
1909
|
+
for (const [weight, neighborID] of neighbors) {
|
|
1910
|
+
if (!visited.has(neighborID)) {
|
|
1911
|
+
next.push({ id: neighborID, value: value - decayK * weight });
|
|
1075
1912
|
}
|
|
1076
1913
|
}
|
|
1077
1914
|
}
|
|
@@ -1188,7 +2025,7 @@ class Runtime {
|
|
|
1188
2025
|
processInput(wordsArr, { addNewWords = true } = {}) {
|
|
1189
2026
|
wordsArr = this.filterStopWords(wordsArr);
|
|
1190
2027
|
if (wordsArr.length === 0) { console.log('[FILTER] 输入全为停用词,已全部过滤'); return; }
|
|
1191
|
-
|
|
2028
|
+
// console.log('Processing input:', wordsArr);
|
|
1192
2029
|
// 批量处理新词添加
|
|
1193
2030
|
if (addNewWords) {
|
|
1194
2031
|
// 一次性检查哪些词不在词表中
|
|
@@ -1274,7 +2111,7 @@ class Runtime {
|
|
|
1274
2111
|
const overlap = wordsArr.filter(w => memeWords.includes(w)).length;
|
|
1275
2112
|
if (overlap >= this.MIN_OVERLAP && memeWords.length + wordsArr.length <= this.MAX_MEME_WORDS) {
|
|
1276
2113
|
this.kvm.set(minMemeID, Array.from(new Set([...memeWords, ...wordsArr])));
|
|
1277
|
-
|
|
2114
|
+
// console.log(`Merged to existing meme: ${minMemeID}`);
|
|
1278
2115
|
} else {
|
|
1279
2116
|
// 创建新模因,使用有向连接
|
|
1280
2117
|
const newID = 'meme_' + Date.now();
|
|
@@ -1284,9 +2121,9 @@ class Runtime {
|
|
|
1284
2121
|
// 单向连接到最近的模因 (方向:2表示指向对方)
|
|
1285
2122
|
if (minMemeID) {
|
|
1286
2123
|
this.graph.addDirectionalEdge(newID, minMemeID, minDistance, 2);
|
|
1287
|
-
|
|
2124
|
+
// console.log(`[LINK] 新模因 ${newID} 单向连接到最近模因 ${minMemeID}`);
|
|
1288
2125
|
}
|
|
1289
|
-
|
|
2126
|
+
// console.log(`Created new meme: ${newID}`);
|
|
1290
2127
|
}
|
|
1291
2128
|
} else {
|
|
1292
2129
|
// 创建新模因
|
|
@@ -1297,9 +2134,9 @@ class Runtime {
|
|
|
1297
2134
|
// 如果有较近的模因,仍然创建单向连接
|
|
1298
2135
|
if (minMemeID) {
|
|
1299
2136
|
this.graph.addDirectionalEdge(newID, minMemeID, Math.min(minDistance, 5), 2);
|
|
1300
|
-
|
|
2137
|
+
// console.log(`[LINK] 新模因 ${newID} 单向连接到最近模因 ${minMemeID}`);
|
|
1301
2138
|
}
|
|
1302
|
-
|
|
2139
|
+
// console.log(`Created new meme: ${newID}`);
|
|
1303
2140
|
}
|
|
1304
2141
|
}
|
|
1305
2142
|
// 新增批量添加边的辅助方法
|
|
@@ -1341,95 +2178,65 @@ class Runtime {
|
|
|
1341
2178
|
* options.bidirectionalMultiplier: 双向连接的衰减倍率
|
|
1342
2179
|
* @returns {Object|Map} 激活结果
|
|
1343
2180
|
*/
|
|
2181
|
+
// 用于多源扩散:将“传递函数+激活函数”应用在每一步
|
|
1344
2182
|
propagateSignalMultiSource(startIDs, strengths, decayK, maxStep, options = {}) {
|
|
1345
2183
|
decayK = decayK !== undefined ? decayK : (this.config.decayK !== undefined ? this.config.decayK : 1);
|
|
1346
2184
|
maxStep = maxStep !== undefined ? maxStep : (this.config.maxStep !== undefined ? this.config.maxStep : 10);
|
|
1347
2185
|
const maxActiveNodes = options.maxActiveNodes || 5000;
|
|
1348
2186
|
const minSignal = options.minSignal || 0.01;
|
|
1349
2187
|
const trackPath = options.trackPath || false;
|
|
1350
|
-
|
|
1351
|
-
const
|
|
1352
|
-
|
|
2188
|
+
const directionalMultiplier = options.directionalMultiplier || 0.7;
|
|
2189
|
+
const bidirectionalMultiplier = options.bidirectionalMultiplier || 1.2;
|
|
2190
|
+
|
|
2191
|
+
const actFn = this._act || BuiltinActivations.relu;
|
|
2192
|
+
const transferFn = this._transfer || BuiltinTransfers.linear;
|
|
1353
2193
|
|
|
1354
|
-
// 节点信号累加表
|
|
1355
2194
|
const signalMap = new Map();
|
|
1356
|
-
// 路径追踪表(可选)
|
|
1357
2195
|
const activationPaths = trackPath ? new Map() : null;
|
|
1358
|
-
// 记录节点激活类型
|
|
1359
2196
|
const activationTypes = trackPath ? new Map() : null;
|
|
1360
2197
|
|
|
1361
|
-
// 初始化活跃队列,每个元素{id, value, from, connectionType}
|
|
1362
2198
|
let active = startIDs.map((id, i) => ({
|
|
1363
|
-
id,
|
|
1364
|
-
value: strengths[i],
|
|
1365
|
-
from: null,
|
|
1366
|
-
connectionType: -1 // 起点无连接类型
|
|
2199
|
+
id, value: strengths[i], from: null, connectionType: -1
|
|
1367
2200
|
}));
|
|
1368
|
-
|
|
1369
2201
|
let step = 0;
|
|
1370
2202
|
|
|
1371
2203
|
while (active.length > 0 && step < maxStep) {
|
|
1372
|
-
// 限制活跃节点数,优先保留信号强的
|
|
1373
2204
|
if (active.length > maxActiveNodes) {
|
|
1374
2205
|
active.sort((a, b) => b.value - a.value);
|
|
1375
2206
|
active = active.slice(0, maxActiveNodes);
|
|
1376
|
-
console.log(`[LIMIT] 多源扩散活跃节点数已限制为 ${maxActiveNodes}`);
|
|
1377
2207
|
}
|
|
1378
2208
|
|
|
1379
2209
|
const next = [];
|
|
1380
2210
|
for (const { id, value, from, connectionType } of active) {
|
|
1381
2211
|
if (value < minSignal) continue;
|
|
1382
2212
|
|
|
1383
|
-
//
|
|
1384
|
-
|
|
2213
|
+
// 节点处应用激活函数(融合累计)
|
|
2214
|
+
const prev = signalMap.get(id) || 0;
|
|
2215
|
+
const merged = actFn(prev + value);
|
|
2216
|
+
signalMap.set(id, merged);
|
|
1385
2217
|
|
|
1386
|
-
// 记录激活类型
|
|
1387
2218
|
if (trackPath && connectionType !== -1) {
|
|
1388
|
-
if (!activationTypes.has(id))
|
|
1389
|
-
activationTypes.set(id, new Set());
|
|
1390
|
-
}
|
|
2219
|
+
if (!activationTypes.has(id)) activationTypes.set(id, new Set());
|
|
1391
2220
|
activationTypes.get(id).add(connectionType);
|
|
1392
2221
|
}
|
|
1393
|
-
|
|
1394
|
-
// 路径追踪
|
|
1395
2222
|
if (trackPath && from) {
|
|
1396
|
-
if (!activationPaths.has(id))
|
|
1397
|
-
activationPaths.set(id, []);
|
|
1398
|
-
}
|
|
2223
|
+
if (!activationPaths.has(id)) activationPaths.set(id, []);
|
|
1399
2224
|
activationPaths.get(id).push({ from, connectionType, value });
|
|
1400
2225
|
}
|
|
1401
2226
|
|
|
1402
|
-
// 传播到邻居(考虑连接方向)
|
|
1403
2227
|
const point = this.graph.points.get(id);
|
|
1404
2228
|
if (!point) continue;
|
|
1405
2229
|
|
|
1406
|
-
// 限制每个节点最多处理的邻居数量
|
|
1407
2230
|
const MAX_NEIGHBORS = 30;
|
|
1408
2231
|
const neighbors = point.connect.slice(0, MAX_NEIGHBORS);
|
|
1409
2232
|
|
|
1410
2233
|
for (const [weight, neighborID, direction = 0] of neighbors) {
|
|
1411
|
-
|
|
1412
|
-
|
|
2234
|
+
const ctx = { direction, directionalMultiplier, bidirectionalMultiplier };
|
|
2235
|
+
const rawNext = transferFn(value, weight, decayK, ctx);
|
|
2236
|
+
const nextValue = actFn(rawNext);
|
|
1413
2237
|
|
|
1414
|
-
if (direction === 0) {
|
|
1415
|
-
// 双向连接 - 衰减较大(语义关联较弱)
|
|
1416
|
-
effectiveDecay *= bidirectionalMultiplier;
|
|
1417
|
-
} else {
|
|
1418
|
-
// 单向连接 - 衰减较小(语义流向强)
|
|
1419
|
-
effectiveDecay *= directionalMultiplier;
|
|
1420
|
-
}
|
|
1421
|
-
|
|
1422
|
-
// 计算传播后的信号强度
|
|
1423
|
-
const nextValue = value - effectiveDecay * weight;
|
|
1424
|
-
|
|
1425
|
-
// 仅当信号足够强时才继续传播
|
|
1426
2238
|
if (nextValue >= minSignal) {
|
|
1427
|
-
next.push({
|
|
1428
|
-
id: neighborID,
|
|
1429
|
-
value: nextValue,
|
|
1430
|
-
from: id,
|
|
1431
|
-
connectionType: direction
|
|
1432
|
-
});
|
|
2239
|
+
next.push({ id: neighborID, value: nextValue, from: id, connectionType: direction });
|
|
1433
2240
|
}
|
|
1434
2241
|
}
|
|
1435
2242
|
}
|
|
@@ -1437,15 +2244,9 @@ class Runtime {
|
|
|
1437
2244
|
step++;
|
|
1438
2245
|
}
|
|
1439
2246
|
|
|
1440
|
-
// 返回结果,根据跟踪路径选项决定返回格式
|
|
1441
2247
|
if (trackPath) {
|
|
1442
|
-
return {
|
|
1443
|
-
signalMap,
|
|
1444
|
-
activationPaths,
|
|
1445
|
-
activationTypes
|
|
1446
|
-
};
|
|
2248
|
+
return { signalMap, activationPaths, activationTypes };
|
|
1447
2249
|
}
|
|
1448
|
-
|
|
1449
2250
|
return signalMap;
|
|
1450
2251
|
}
|
|
1451
2252
|
|
|
@@ -1616,7 +2417,7 @@ class Runtime {
|
|
|
1616
2417
|
this.kvm.delete(memeB.pointID);
|
|
1617
2418
|
memesToDelete.add(memeB.pointID);
|
|
1618
2419
|
|
|
1619
|
-
|
|
2420
|
+
// console.log(`Merged memes: ${memeA.pointID} <- ${memeB.pointID}`);
|
|
1620
2421
|
// 合并后立即尝试分裂
|
|
1621
2422
|
this.splitMemeIfNeeded(memeA.pointID);
|
|
1622
2423
|
} else {
|
|
@@ -1633,7 +2434,7 @@ class Runtime {
|
|
|
1633
2434
|
// 如果没有双向边,则添加双向边
|
|
1634
2435
|
if (!(existAtoB.exist && existAtoB.type === 0) && !(existBtoA.exist && existBtoA.type === 0)) {
|
|
1635
2436
|
this.graph.addBidirectionalEdge(memeA.pointID, memeB.pointID, avgDist);
|
|
1636
|
-
|
|
2437
|
+
// console.log(`[LINK] 添加双向边: ${memeA.pointID} <-> ${memeB.pointID} (avgDist=${avgDist})`);
|
|
1637
2438
|
}
|
|
1638
2439
|
}
|
|
1639
2440
|
}
|
|
@@ -1665,14 +2466,14 @@ class Runtime {
|
|
|
1665
2466
|
const newID = newIDs[i];
|
|
1666
2467
|
this.graph.addPoint(newID, []);
|
|
1667
2468
|
this.kvm.set(newID, chunk);
|
|
1668
|
-
|
|
2469
|
+
// console.log(`[SPLIT-FORCE] 新建模因: ${newID} 词数: ${chunk.length}`);
|
|
1669
2470
|
}
|
|
1670
2471
|
}
|
|
1671
2472
|
|
|
1672
2473
|
// 删除原模因
|
|
1673
2474
|
this.graph.points.delete(memeID);
|
|
1674
2475
|
this.kvm.delete(memeID);
|
|
1675
|
-
|
|
2476
|
+
// console.log(`[SPLIT-FORCE] 删除原模因: ${memeID}`);
|
|
1676
2477
|
return;
|
|
1677
2478
|
}
|
|
1678
2479
|
|
|
@@ -1721,12 +2522,12 @@ class Runtime {
|
|
|
1721
2522
|
const newID = 'meme_' + Date.now() + '_' + Math.floor(Math.random() * 10000);
|
|
1722
2523
|
this.graph.addPoint(newID, []);
|
|
1723
2524
|
this.kvm.set(newID, comp);
|
|
1724
|
-
|
|
2525
|
+
// console.log(`[SPLIT] 新建模因: ${newID} 词数: ${comp.length}`);
|
|
1725
2526
|
}
|
|
1726
2527
|
// 删除原节点
|
|
1727
2528
|
this.graph.points.delete(memeID);
|
|
1728
2529
|
this.kvm.delete(memeID);
|
|
1729
|
-
|
|
2530
|
+
// console.log(`[SPLIT] 删除原模因: ${memeID}`);
|
|
1730
2531
|
}
|
|
1731
2532
|
}
|
|
1732
2533
|
}
|
|
@@ -2153,7 +2954,7 @@ class controller {
|
|
|
2153
2954
|
const words = text.toLowerCase().split(' ').filter(w => w.length > 0);
|
|
2154
2955
|
this.runtime.processInput(words, { addNewWords: false });
|
|
2155
2956
|
// 用模因网络参与推理
|
|
2156
|
-
|
|
2957
|
+
// console.log('[DEBUG] 当前所有模因节点:', this.runtime.kvm.memory);
|
|
2157
2958
|
return await this.runtime.generateResponseWithMemes(words);
|
|
2158
2959
|
}
|
|
2159
2960
|
|
|
@@ -2217,7 +3018,7 @@ function saveAll(runtime) {
|
|
|
2217
3018
|
const data = {
|
|
2218
3019
|
memes: runtime.graph.getAllPoints(),
|
|
2219
3020
|
wordGraph: Array.from(runtime.wordGraph.points.values()),
|
|
2220
|
-
|
|
3021
|
+
kvm: runtime.kvm.exportEntries(),
|
|
2221
3022
|
vocab: runtime.vocabManager.vocab,
|
|
2222
3023
|
wordAccessLog: Array.from(runtime.wordAccessLog ? runtime.wordAccessLog.entries() : [])
|
|
2223
3024
|
};
|
|
@@ -2301,72 +3102,129 @@ async function boot() {
|
|
|
2301
3102
|
// 激活副本:从快照加载 ctrlB/ctrlC 并加入轮询
|
|
2302
3103
|
let activeControllers = []; // 轮询数组
|
|
2303
3104
|
let rrIdx = 0;
|
|
3105
|
+
// 新增:双实例热切换状态
|
|
3106
|
+
let servingCtrl = null; // 当前对外服务的控制器
|
|
3107
|
+
let standbyCtrl = null; // 后台承接 Redis 模型的控制器
|
|
3108
|
+
let updatingModel = false;
|
|
3109
|
+
let pendingModelObj = null;
|
|
3110
|
+
|
|
3111
|
+
function setServingController(ctrl) {
|
|
3112
|
+
servingCtrl = ctrl;
|
|
3113
|
+
activeControllers = [servingCtrl]; // 仅对外暴露一份,避免切换瞬间不一致
|
|
3114
|
+
global.ctrlA = servingCtrl;
|
|
3115
|
+
global.ctrl = servingCtrl;
|
|
3116
|
+
}
|
|
3117
|
+
|
|
3118
|
+
|
|
3119
|
+
// 替换 getNextController:始终返回当前服务控制器
|
|
2304
3120
|
function getNextController() {
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
3121
|
+
if (servingCtrl) return servingCtrl;
|
|
3122
|
+
if (activeControllers.length) return activeControllers[rrIdx % activeControllers.length];
|
|
3123
|
+
return global.ctrlA; // 兜底
|
|
3124
|
+
}
|
|
3125
|
+
async function ensureStandbyReady() {
|
|
3126
|
+
if (!standbyCtrl) {
|
|
3127
|
+
standbyCtrl = new controller(); // 构造函数会初始化 Runtime
|
|
3128
|
+
// 初始保持与serving词表一致,减少首次同步成本(可选)
|
|
3129
|
+
if (servingCtrl?.runtime?.vocabManager?.vocab?.length) {
|
|
3130
|
+
standbyCtrl.runtime.vocabManager.vocab = [...servingCtrl.runtime.vocabManager.vocab];
|
|
3131
|
+
standbyCtrl.runtime.vocabManager.updateMappings();
|
|
3132
|
+
}
|
|
3133
|
+
}
|
|
3134
|
+
return standbyCtrl;
|
|
3135
|
+
}
|
|
3136
|
+
|
|
3137
|
+
// 将 protobuf 反序列化对象应用到 standby 并热切换
|
|
3138
|
+
async function handleRedisModelSwap(modelObj) {
|
|
3139
|
+
if (updatingModel) {
|
|
3140
|
+
pendingModelObj = modelObj; // 只保留最近一条
|
|
3141
|
+
return;
|
|
3142
|
+
}
|
|
3143
|
+
updatingModel = true;
|
|
3144
|
+
try {
|
|
3145
|
+
const standby = await ensureStandbyReady();
|
|
3146
|
+
// 在 standby 上同步模型
|
|
3147
|
+
await plainObjToRuntime(standby.runtime, modelObj);
|
|
3148
|
+
applyModelParams(standby.runtime);
|
|
3149
|
+
|
|
3150
|
+
// 原子切换:standby 上位,serving 变为 standby
|
|
3151
|
+
const oldServing = servingCtrl;
|
|
3152
|
+
setServingController(standbyCtrl);
|
|
3153
|
+
standbyCtrl = oldServing;
|
|
3154
|
+
|
|
3155
|
+
console.log('[MODEL SYNC] 新模型已在后台准备完毕并无缝切换为在线实例');
|
|
3156
|
+
} catch (e) {
|
|
3157
|
+
console.error('[MODEL SYNC] 后台模型切换失败:', e.message);
|
|
3158
|
+
} finally {
|
|
3159
|
+
updatingModel = false;
|
|
3160
|
+
if (pendingModelObj) {
|
|
3161
|
+
const next = pendingModelObj;
|
|
3162
|
+
pendingModelObj = null;
|
|
3163
|
+
// 递归处理最新一条待处理消息
|
|
3164
|
+
setImmediate(() => handleRedisModelSwap(next));
|
|
3165
|
+
}
|
|
3166
|
+
}
|
|
2309
3167
|
}
|
|
2310
3168
|
async function activateReplicasIfNeeded() {
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
3169
|
+
if (global.config.isStudy) return; // study 进程不启用副本扩容
|
|
3170
|
+
if (global.ctrlB && global.ctrlC) return; // 已激活
|
|
3171
|
+
try {
|
|
3172
|
+
const loadReplicaByName = async (name) => {
|
|
3173
|
+
const ctrl = new controller();
|
|
3174
|
+
// 为副本单独挂载快照管理器
|
|
3175
|
+
ctrl.snapshotManager = new SnapshotManager(ctrl.runtime);
|
|
3176
|
+
// 在 A 的快照目录中寻找对应 id
|
|
3177
|
+
const smA = global.ctrlA.snapshotManager;
|
|
3178
|
+
smA.loadSnapshotList();
|
|
3179
|
+
const snap = smA.snapshotList.find(s => s.name === name || s.id.endsWith(`_${name}`));
|
|
3180
|
+
if (!snap) {
|
|
3181
|
+
console.warn(`[REPLICA] 未找到快照: ${name},尝试现生成...`);
|
|
3182
|
+
await smA.createSnapshot(name);
|
|
3183
|
+
}
|
|
3184
|
+
const snap2 = smA.snapshotList.find(s => s.name === name || s.id.endsWith(`_${name}`));
|
|
3185
|
+
if (snap2) {
|
|
3186
|
+
// 直接读取该文件内容并恢复到 ctrl.runtime
|
|
3187
|
+
const dataStr = fs.readFileSync(snap2.path, 'utf-8');
|
|
3188
|
+
const data = JSON.parse(dataStr);
|
|
3189
|
+
// 简单恢复(与 SnapshotManager.restoreSnapshot 类似)
|
|
3190
|
+
ctrl.runtime.graph = new GraphDB();
|
|
3191
|
+
ctrl.runtime.wordGraph = new GraphDB();
|
|
3192
|
+
ctrl.runtime.kvm = new KVM();
|
|
3193
|
+
ctrl.runtime.wordAccessLog = new Map(data.wordAccessLog || []);
|
|
3194
|
+
if (data.memes) for (const p of data.memes) ctrl.runtime.graph.addPoint(p.pointID, p.connect);
|
|
3195
|
+
if (data.wordGraph) for (const p of data.wordGraph) ctrl.runtime.wordGraph.addPoint(p.pointID, p.connect);
|
|
3196
|
+
if (data.kvm) for (const [k, v] of data.kvm) ctrl.runtime.kvm.set(k, v);
|
|
3197
|
+
if (data.vocab) { ctrl.runtime.vocabManager.vocab = data.vocab; ctrl.runtime.vocabManager.updateMappings(); }
|
|
3198
|
+
return ctrl;
|
|
3199
|
+
}
|
|
3200
|
+
return null;
|
|
3201
|
+
};
|
|
2344
3202
|
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
3203
|
+
if (!global.ctrlB) global.ctrlB = await loadReplicaByName('replica_B');
|
|
3204
|
+
if (!global.ctrlC) global.ctrlC = await loadReplicaByName('replica_C');
|
|
3205
|
+
activeControllers = [global.ctrlA, ...(global.ctrlB ? [global.ctrlB] : []), ...(global.ctrlC ? [global.ctrlC] : [])];
|
|
3206
|
+
console.log(`[REPLICA] 已激活副本: ${activeControllers.length} 个控制器`);
|
|
3207
|
+
} catch (e) {
|
|
3208
|
+
console.warn('[REPLICA] 激活失败:', e.message);
|
|
3209
|
+
}
|
|
2352
3210
|
}
|
|
2353
3211
|
// 对端探测/反触发:同组任一对端端口死亡则激活副本
|
|
2354
3212
|
function installPeerFailoverMonitor() {
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
3213
|
+
const peers = global.config.peerServePorts || [];
|
|
3214
|
+
if (!Array.isArray(peers) || peers.length === 0) return;
|
|
3215
|
+
const axiosInst = axios.create({ timeout: 1500 });
|
|
3216
|
+
setInterval(async () => {
|
|
3217
|
+
try {
|
|
3218
|
+
const checks = await Promise.all(peers.map(p =>
|
|
3219
|
+
axiosInst.get(`http://127.0.0.1:${p}/health`).then(() => true).catch(() => false)
|
|
3220
|
+
));
|
|
3221
|
+
const dead = checks.some(ok => !ok);
|
|
3222
|
+
if (dead) {
|
|
3223
|
+
console.warn('[ANTI-TRIGGER] 检测到对端 serve 进程离线,启动副本扩容...');
|
|
3224
|
+
await activateReplicasIfNeeded();
|
|
3225
|
+
}
|
|
3226
|
+
} catch (_) { /* 忽略一次错误 */ }
|
|
3227
|
+
}, 3000);
|
|
2370
3228
|
}
|
|
2371
3229
|
|
|
2372
3230
|
// 在main函数末尾添加
|
|
@@ -2416,23 +3274,23 @@ function setupExitHandler() {
|
|
|
2416
3274
|
|
|
2417
3275
|
// 预缓存副本快照(不在内存常驻)
|
|
2418
3276
|
async function preCacheReplicas() {
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
3277
|
+
try {
|
|
3278
|
+
if (!global.ctrlA?.snapshotManager) return;
|
|
3279
|
+
const sm = global.ctrlA.snapshotManager;
|
|
3280
|
+
sm.loadSnapshotList();
|
|
3281
|
+
const hasB = sm.getSnapshotList().some(s => s.name.includes('replica_B'));
|
|
3282
|
+
const hasC = sm.getSnapshotList().some(s => s.name.includes('replica_C'));
|
|
3283
|
+
if (!hasB) {
|
|
3284
|
+
console.log('[REPLICA] 预生成 ctrlB 快照(replica_B)...');
|
|
3285
|
+
await sm.createSnapshot('replica_B');
|
|
3286
|
+
}
|
|
3287
|
+
if (!hasC) {
|
|
3288
|
+
console.log('[REPLICA] 预生成 ctrlC 快照(replica_C)...');
|
|
3289
|
+
await sm.createSnapshot('replica_C');
|
|
3290
|
+
}
|
|
3291
|
+
} catch (e) {
|
|
3292
|
+
console.warn('[REPLICA] 预缓存失败:', e.message);
|
|
3293
|
+
}
|
|
2436
3294
|
}
|
|
2437
3295
|
// 添加定期垃圾回收帮助函数
|
|
2438
3296
|
function optimizeMemory() {
|
|
@@ -2447,21 +3305,33 @@ function optimizeMemory() {
|
|
|
2447
3305
|
}
|
|
2448
3306
|
}
|
|
2449
3307
|
async function main() {
|
|
3308
|
+
let __adv = null;
|
|
3309
|
+
if (String(process.env.SERVE_ADV_AUTOSTART || '').toLowerCase() === 'true') {
|
|
3310
|
+
__adv = new AdversaryScheduler(global.ctrlA.runtime, {
|
|
3311
|
+
providerSpec: process.env.SERVE_ADV_MODEL || process.env.ADV_MODEL || 'ollama:llama3:70b',
|
|
3312
|
+
judgeMode: process.env.SERVE_ADV_JUDGE || process.env.ADV_JUDGE || 'heuristic',
|
|
3313
|
+
intervalMs: Number(process.env.SERVE_ADV_INTERVAL || 120_000),
|
|
3314
|
+
batchSize: Number(process.env.SERVE_ADV_BATCH || 2),
|
|
3315
|
+
adjustParams: false // 只记录,不修改服务实例的参数
|
|
3316
|
+
});
|
|
3317
|
+
__adv.start();
|
|
3318
|
+
}
|
|
2450
3319
|
|
|
2451
3320
|
let spiderPrivate = new Spider();
|
|
2452
3321
|
// 创建三个全局控制器副本
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
ctrlA.snapshotManager
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
3322
|
+
const ctrlA = new controller();
|
|
3323
|
+
global.ctrlA = ctrlA;
|
|
3324
|
+
global.ctrl = ctrlA;
|
|
3325
|
+
setServingController(ctrlA);
|
|
3326
|
+
// 启动快照自动保存(每30分钟)
|
|
3327
|
+
if (ctrlA.snapshotManager) {
|
|
3328
|
+
ctrlA.snapshotManager.setupAutoSnapshot(30 * 60 * 1000);
|
|
3329
|
+
console.log('[MAIN] 已设置自动快照,每30分钟检查一次');
|
|
3330
|
+
}
|
|
3331
|
+
setupExitHandler();
|
|
3332
|
+
console.log('Starting AI system...');
|
|
3333
|
+
// 恢复 ctrlA
|
|
3334
|
+
loadAll(ctrlA.runtime);
|
|
2465
3335
|
await preCacheReplicas();
|
|
2466
3336
|
const redisClient = redis.createClient();
|
|
2467
3337
|
redisClient.connect();
|
|
@@ -2493,19 +3363,13 @@ async function main() {
|
|
|
2493
3363
|
redisClient.on("message", function (channel, message) {
|
|
2494
3364
|
if (channel === `AI-model-${__dirname}` && RuntimeMessage) {
|
|
2495
3365
|
try {
|
|
2496
|
-
// 反序列化为对象
|
|
2497
3366
|
const modelObj = RuntimeMessage.decode(Buffer.from(message));
|
|
2498
|
-
// 可选:校验
|
|
2499
3367
|
const errMsg = RuntimeMessage.verify(modelObj);
|
|
2500
3368
|
if (errMsg) throw Error(errMsg);
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
// 你需要实现一个 plainObjToRuntime 方法
|
|
2504
|
-
plainObjToRuntime(ctrlA.runtime, modelObj);
|
|
2505
|
-
|
|
2506
|
-
console.log("[MODEL SYNC] 已同步最新模型到 ctrlA.runtime");
|
|
3369
|
+
// 后台准备并切换
|
|
3370
|
+
handleRedisModelSwap(modelObj);
|
|
2507
3371
|
} catch (e) {
|
|
2508
|
-
console.error("[MODEL SYNC]
|
|
3372
|
+
console.error("[MODEL SYNC] 反序列化或切换失败:", e.message);
|
|
2509
3373
|
}
|
|
2510
3374
|
}
|
|
2511
3375
|
});
|
|
@@ -2556,33 +3420,72 @@ async function main() {
|
|
|
2556
3420
|
await ctrlA.handleInput('I like apple');
|
|
2557
3421
|
await ctrlA.handleInput('I love orange');
|
|
2558
3422
|
await ctrlA.handleInput('you are good');
|
|
2559
|
-
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
3423
|
+
// 启动主循环(仅 A)
|
|
3424
|
+
ctrlA.startMainLoop();
|
|
3425
|
+
|
|
3426
|
+
// 安装对端监控触发器
|
|
3427
|
+
installPeerFailoverMonitor();
|
|
3428
|
+
app.get('/api/graph/partitions/status', async (req, res) => {
|
|
3429
|
+
try {
|
|
3430
|
+
const g = global.ctrlA?.runtime?.graph;
|
|
3431
|
+
if (!g || !(g instanceof PartitionedGraphDB)) {
|
|
3432
|
+
return res.json({ ok: true, mode: 'in-memory', loaded: 0 });
|
|
3433
|
+
}
|
|
3434
|
+
const loaded = Array.from(g.loaded.keys());
|
|
3435
|
+
res.json({
|
|
3436
|
+
ok: true,
|
|
3437
|
+
mode: 'partitioned',
|
|
3438
|
+
partitions: g.partitioner.partitions,
|
|
3439
|
+
loaded,
|
|
3440
|
+
maxLoaded: g.maxLoadedPartitions,
|
|
3441
|
+
windowRadius: g.windowRadius,
|
|
3442
|
+
centerPid: g.centerPid
|
|
3443
|
+
});
|
|
3444
|
+
} catch (e) {
|
|
3445
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3446
|
+
}
|
|
3447
|
+
});
|
|
3448
|
+
|
|
3449
|
+
app.post('/api/graph/partitions/flush', async (req, res) => {
|
|
3450
|
+
try {
|
|
3451
|
+
const g = global.ctrlA?.runtime?.graph;
|
|
3452
|
+
if (g && g.flushAll) await g.flushAll();
|
|
3453
|
+
res.json({ ok: true });
|
|
3454
|
+
} catch (e) {
|
|
3455
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3456
|
+
}
|
|
3457
|
+
});
|
|
3458
|
+
|
|
3459
|
+
app.post('/api/graph/prefetch', async (req, res) => {
|
|
3460
|
+
try {
|
|
3461
|
+
const { node } = req.body || {};
|
|
3462
|
+
const g = global.ctrlA?.runtime?.graph;
|
|
3463
|
+
if (!node || !(g instanceof PartitionedGraphDB)) return res.status(400).json({ ok: false, error: 'node 必填/或非分区图' });
|
|
3464
|
+
await g.focusOnPoint(String(node));
|
|
3465
|
+
res.json({ ok: true });
|
|
3466
|
+
} catch (e) {
|
|
3467
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3468
|
+
}
|
|
2585
3469
|
});
|
|
3470
|
+
app.post('/api/chat', async (req, res) => {
|
|
3471
|
+
try {
|
|
3472
|
+
const { message } = req.body || {};
|
|
3473
|
+
const ctrl = getNextController(); // 始终路由到当前服务实例
|
|
3474
|
+
const words = typeof message === 'string'
|
|
3475
|
+
? message.toLowerCase().split(/\s+/).filter(w => w.length > 0)
|
|
3476
|
+
: [];
|
|
3477
|
+
const normWords = (new Spider()).lemmatizeWords(words);
|
|
3478
|
+
const normMessage = normWords.join(' ');
|
|
3479
|
+
const response = await ctrl.handleInput(normMessage);
|
|
3480
|
+
if (!response || response.trim() === '') {
|
|
3481
|
+
console.warn('[WARN] AI响应为空');
|
|
3482
|
+
}
|
|
3483
|
+
res.json({ response });
|
|
3484
|
+
} catch (error) {
|
|
3485
|
+
console.error('[ERROR] 处理请求失败:', error);
|
|
3486
|
+
res.status(500).json({ error: error.message });
|
|
3487
|
+
}
|
|
3488
|
+
});
|
|
2586
3489
|
// 添加健康检查路由
|
|
2587
3490
|
app.get('/health', (req, res) => {
|
|
2588
3491
|
res.status(200).send('OK');
|
|
@@ -2668,7 +3571,11 @@ app.post('/api/chat', async (req, res) => {
|
|
|
2668
3571
|
decay: 1, // 新增
|
|
2669
3572
|
decayK: 1, // 新增
|
|
2670
3573
|
maxLen: 16, // 新增
|
|
2671
|
-
edgeWeight: 1
|
|
3574
|
+
edgeWeight: 1, // 新增
|
|
3575
|
+
activationType: 'relu',
|
|
3576
|
+
transferType: 'linear',
|
|
3577
|
+
activationCustom: '',
|
|
3578
|
+
transferCustom: ''
|
|
2672
3579
|
};
|
|
2673
3580
|
const currentModelParams = { ...modelDefaults };
|
|
2674
3581
|
|
|
@@ -2699,18 +3606,85 @@ app.post('/api/chat', async (req, res) => {
|
|
|
2699
3606
|
res.status(500).json({ error: err.message });
|
|
2700
3607
|
}
|
|
2701
3608
|
});
|
|
2702
|
-
|
|
3609
|
+
app.get('/api/adversary/status', (req, res) => {
|
|
3610
|
+
try {
|
|
3611
|
+
res.json({ ok: true, status: __adv ? __adv.getStatus() : { running: false } });
|
|
3612
|
+
} catch (e) {
|
|
3613
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3614
|
+
}
|
|
3615
|
+
});
|
|
2703
3616
|
|
|
3617
|
+
app.post('/api/adversary/start', (req, res) => {
|
|
3618
|
+
try {
|
|
3619
|
+
if (!__adv) __adv = new AdversaryScheduler(global.ctrlA.runtime, { adjustParams: false });
|
|
3620
|
+
__adv.start();
|
|
3621
|
+
res.json({ ok: true, status: __adv.getStatus() });
|
|
3622
|
+
} catch (e) {
|
|
3623
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3624
|
+
}
|
|
3625
|
+
});
|
|
3626
|
+
|
|
3627
|
+
app.post('/api/adversary/stop', (req, res) => {
|
|
3628
|
+
try { __adv?.stop?.(); res.json({ ok: true }); }
|
|
3629
|
+
catch (e) { res.status(500).json({ ok: false, error: e.message }); }
|
|
3630
|
+
});
|
|
3631
|
+
// 新增:serve 侧参数调优 API(默认不启用自动调参,仅手动设置)
|
|
3632
|
+
app.get('/api/tune/get', (req, res) => {
|
|
3633
|
+
try {
|
|
3634
|
+
const rt = global.ctrlA?.runtime;
|
|
3635
|
+
if (!rt) return res.status(500).json({ ok: false, error: 'runtime missing' });
|
|
3636
|
+
res.json({
|
|
3637
|
+
ok: true,
|
|
3638
|
+
params: {
|
|
3639
|
+
decayK: rt.config?.decayK ?? 1,
|
|
3640
|
+
maxLen: rt.config?.maxLen ?? 16,
|
|
3641
|
+
spiderMix: rt.config?.spiderMix ?? { onlineWeight: 0.5, offlineWeight: 0.5 },
|
|
3642
|
+
crawler: {
|
|
3643
|
+
perQuery: global.__crawler?.__tune_perQuery ?? 8,
|
|
3644
|
+
maxCrawl: global.__crawler?.__tune_maxCrawl ?? 12
|
|
3645
|
+
}
|
|
3646
|
+
}
|
|
3647
|
+
});
|
|
3648
|
+
} catch (e) {
|
|
3649
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3650
|
+
}
|
|
3651
|
+
});
|
|
3652
|
+
|
|
3653
|
+
app.post('/api/tune/set', (req, res) => {
|
|
3654
|
+
try {
|
|
3655
|
+
const rt = global.ctrlA?.runtime;
|
|
3656
|
+
if (!rt) return res.status(500).json({ ok: false, error: 'runtime missing' });
|
|
3657
|
+
const snap = applyServeTunableParams(rt, req.body || {});
|
|
3658
|
+
res.json({ ok: true, snapshot: snap });
|
|
3659
|
+
} catch (e) {
|
|
3660
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3661
|
+
}
|
|
3662
|
+
});
|
|
3663
|
+
|
|
3664
|
+
// 可选:服务侧对抗调度保持 adjustParams: false,但允许设置 promptMode/targets
|
|
3665
|
+
// ...existing code...
|
|
3666
|
+
app.post('/api/adversary/start', (req, res) => {
|
|
3667
|
+
try {
|
|
3668
|
+
if (!__adv) __adv = new AdversaryScheduler(global.ctrlA.runtime, { adjustParams: false });
|
|
3669
|
+
const { promptMode, targetWeights } = req.body || {};
|
|
3670
|
+
if (promptMode) __adv.setPromptMode(promptMode);
|
|
3671
|
+
if (targetWeights) __adv.setTargets(targetWeights);
|
|
3672
|
+
__adv.start();
|
|
3673
|
+
res.json({ ok: true, status: __adv.getStatus() });
|
|
3674
|
+
} catch (e) {
|
|
3675
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3676
|
+
}
|
|
3677
|
+
});
|
|
2704
3678
|
|
|
2705
3679
|
// 设置退出保存
|
|
2706
3680
|
setupExitHandler();
|
|
2707
3681
|
|
|
2708
3682
|
console.log('已设置交错自主学习定时任务,每200s执行一次');
|
|
2709
3683
|
}
|
|
2710
|
-
// plainObjToRuntime: 将protobuf对象同步到runtime实例
|
|
2711
|
-
// 扩展 applyModelParams
|
|
2712
3684
|
function applyModelParams(runtime) {
|
|
2713
3685
|
if (!runtime) return;
|
|
3686
|
+
|
|
3687
|
+
// 同步通用参数
|
|
2714
3688
|
runtime.MAX_MEME_WORDS = currentModelParams.maxMemeWords;
|
|
2715
3689
|
runtime.MIN_OVERLAP = currentModelParams.minOverlapThreshold;
|
|
2716
3690
|
runtime.config = runtime.config || {};
|
|
@@ -2721,11 +3695,12 @@ function applyModelParams(runtime) {
|
|
|
2721
3695
|
runtime.config.iteration = currentModelParams.iteration;
|
|
2722
3696
|
runtime.config.threshold = currentModelParams.threshold;
|
|
2723
3697
|
runtime.config.decay = currentModelParams.decay;
|
|
3698
|
+
|
|
2724
3699
|
// memeBarrier
|
|
2725
3700
|
if (runtime.memeBarrier) {
|
|
2726
3701
|
runtime.memeBarrier.maliciousThreshold = currentModelParams.maliciousThreshold;
|
|
2727
3702
|
}
|
|
2728
|
-
//
|
|
3703
|
+
// 全局边权
|
|
2729
3704
|
if (runtime.graph && currentModelParams.edgeWeight !== undefined) {
|
|
2730
3705
|
for (const point of runtime.graph.getAllPoints()) {
|
|
2731
3706
|
for (const conn of point.connect) {
|
|
@@ -2733,7 +3708,20 @@ function applyModelParams(runtime) {
|
|
|
2733
3708
|
}
|
|
2734
3709
|
}
|
|
2735
3710
|
}
|
|
2736
|
-
|
|
3711
|
+
|
|
3712
|
+
// 新增:激活/传递函数配置
|
|
3713
|
+
runtime.setActivationConfig({
|
|
3714
|
+
activationType: currentModelParams.activationType,
|
|
3715
|
+
transferType: currentModelParams.transferType,
|
|
3716
|
+
activationCustom: currentModelParams.activationCustom,
|
|
3717
|
+
transferCustom: currentModelParams.transferCustom
|
|
3718
|
+
});
|
|
3719
|
+
|
|
3720
|
+
console.log('[PARAMS] 已更新运行时参数:', {
|
|
3721
|
+
...currentModelParams,
|
|
3722
|
+
activationType: runtime.getActivationConfig().activationType,
|
|
3723
|
+
transferType: runtime.getActivationConfig().transferType
|
|
3724
|
+
});
|
|
2737
3725
|
}
|
|
2738
3726
|
// plainObjToRuntime: 将protobuf对象同步到runtime实例
|
|
2739
3727
|
async function plainObjToRuntime(runtime, obj) {
|
|
@@ -2818,9 +3806,32 @@ async function plainObjToRuntime(runtime, obj) {
|
|
|
2818
3806
|
|
|
2819
3807
|
console.log('[MODEL SYNC] 模型同步完成');
|
|
2820
3808
|
}
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
|
|
3809
|
+
function applyServeTunableParams(runtime, partial = {}) {
|
|
3810
|
+
if (!runtime) return null;
|
|
3811
|
+
runtime.config = runtime.config || {};
|
|
3812
|
+
if (partial.spiderMix) {
|
|
3813
|
+
const ow = Math.max(0, Math.min(1, Number(partial.spiderMix.onlineWeight ?? runtime.config.spiderMix?.onlineWeight ?? 0.5)));
|
|
3814
|
+
runtime.config.spiderMix = { onlineWeight: ow, offlineWeight: Math.max(0, Math.min(1, 1 - ow)) };
|
|
3815
|
+
}
|
|
3816
|
+
if (typeof partial.decayK === 'number') runtime.config.decayK = Math.max(0.1, Math.min(2.0, partial.decayK));
|
|
3817
|
+
if (typeof partial.maxLen === 'number') runtime.config.maxLen = Math.max(8, Math.min(64, Math.round(partial.maxLen)));
|
|
3818
|
+
if (typeof partial.edgeWeight === 'number' && runtime.graph) {
|
|
3819
|
+
for (const p of runtime.graph.getAllPoints()) for (const e of p.connect) e[0] = Math.max(0.1, Math.min(5, partial.edgeWeight));
|
|
3820
|
+
}
|
|
3821
|
+
if (global.__crawler) {
|
|
3822
|
+
if (typeof partial.perQuery === 'number') global.__crawler.__tune_perQuery = Math.max(2, Math.min(16, Math.round(partial.perQuery)));
|
|
3823
|
+
if (typeof partial.maxCrawl === 'number') global.__crawler.__tune_maxCrawl = Math.max(2, Math.min(24, Math.round(partial.maxCrawl)));
|
|
3824
|
+
}
|
|
3825
|
+
return {
|
|
3826
|
+
decayK: runtime.config.decayK,
|
|
3827
|
+
maxLen: runtime.config.maxLen,
|
|
3828
|
+
spiderMix: runtime.config.spiderMix || { onlineWeight: 0.5, offlineWeight: 0.5 },
|
|
3829
|
+
crawler: {
|
|
3830
|
+
perQuery: global.__crawler?.__tune_perQuery ?? 8,
|
|
3831
|
+
maxCrawl: global.__crawler?.__tune_maxCrawl ?? 12
|
|
3832
|
+
}
|
|
3833
|
+
};
|
|
3834
|
+
}
|
|
2824
3835
|
if (require.main === module) {
|
|
2825
3836
|
|
|
2826
3837
|
main().catch(console.error)
|