079project 2.0.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/crawler/agent.cjs +97 -0
- package/crawler/index.cjs +515 -0
- package/crawler/storage.cjs +163 -0
- package/forwarder.js +106 -37
- package/groupmanager.cjs +2 -1
- package/main_Serve.cjs +1281 -270
- package/main_Study.cjs +1731 -375
- package/package.json +6 -1
- package/robots/seeds.txt +2 -0
- package/schedule.cjs +745 -0
- package/todo-list.txt +0 -86
package/main_Study.cjs
CHANGED
|
@@ -29,7 +29,8 @@ const pool = workerpool.pool(path.join(__dirname, 'memeMergeWorker.cjs'), {
|
|
|
29
29
|
});
|
|
30
30
|
const natural = require('natural');
|
|
31
31
|
const STOP_WORDS = natural.stopwords; // 英文停用词
|
|
32
|
-
|
|
32
|
+
const { CrawlerManager } = require('./crawler/index.cjs');
|
|
33
|
+
const { AdversaryScheduler } = require('./schedule.cjs');
|
|
33
34
|
console.log(`[WORKERS] 工作池已创建,最大工作进程数: ${MAX_WORKERS}`);
|
|
34
35
|
protobuf.load(runtimeProtoPath, (err, root) => {
|
|
35
36
|
if (err) throw err;
|
|
@@ -40,6 +41,71 @@ global.config = {
|
|
|
40
41
|
masterPortOfMain: process.argv[2],
|
|
41
42
|
emitExitport: process.argv[3] || 8641
|
|
42
43
|
};
|
|
44
|
+
// ...existing code...
|
|
45
|
+
const vm = require('vm'); // 新增:沙箱编译
|
|
46
|
+
// ...existing code...
|
|
47
|
+
|
|
48
|
+
// ==== 内置激活/传递函数注册表 + 安全编译工具 ====
|
|
49
|
+
const BuiltinActivations = {
|
|
50
|
+
identity: (x) => x,
|
|
51
|
+
relu: (x) => (x > 0 ? x : 0),
|
|
52
|
+
leaky_relu: (x) => (x > 0 ? x : 0.01 * x),
|
|
53
|
+
tanh: (x) => Math.tanh(x),
|
|
54
|
+
sigmoid: (x) => 1 / (1 + Math.exp(-x)),
|
|
55
|
+
elu: (x) => (x >= 0 ? x : (Math.exp(x) - 1)),
|
|
56
|
+
softplus: (x) => Math.log(1 + Math.exp(x)),
|
|
57
|
+
// 近似 GELU
|
|
58
|
+
gelu: (x) => 0.5 * x * (1 + Math.tanh(Math.sqrt(2 / Math.PI) * (x + 0.044715 * Math.pow(x, 3))))
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
const BuiltinTransfers = {
|
|
62
|
+
// 线性衰减:next = value - decayK*weight*(dirMult)
|
|
63
|
+
linear: (value, weight, decayK, ctx) => {
|
|
64
|
+
const dm = ctx?.direction === 0 ? (ctx?.bidirectionalMultiplier ?? 1.2) : (ctx?.directionalMultiplier ?? 0.7);
|
|
65
|
+
return value - (decayK * weight * dm);
|
|
66
|
+
},
|
|
67
|
+
// 指数衰减:next = value * exp(-decayK*weight*(dirMult))
|
|
68
|
+
exp: (value, weight, decayK, ctx) => {
|
|
69
|
+
const dm = ctx?.direction === 0 ? (ctx?.bidirectionalMultiplier ?? 1.2) : (ctx?.directionalMultiplier ?? 0.7);
|
|
70
|
+
return value * Math.exp(-(decayK * weight * dm));
|
|
71
|
+
},
|
|
72
|
+
// 反比例:next = value / (1 + decayK*weight*(dirMult))
|
|
73
|
+
inverse: (value, weight, decayK, ctx) => {
|
|
74
|
+
const dm = ctx?.direction === 0 ? (ctx?.bidirectionalMultiplier ?? 1.2) : (ctx?.directionalMultiplier ?? 0.7);
|
|
75
|
+
return value / (1 + (decayK * weight * dm));
|
|
76
|
+
},
|
|
77
|
+
// 截断线性:线性后下限截断为0,上限截断为value
|
|
78
|
+
capped: (value, weight, decayK, ctx) => {
|
|
79
|
+
const dm = ctx?.direction === 0 ? (ctx?.bidirectionalMultiplier ?? 1.2) : (ctx?.directionalMultiplier ?? 0.7);
|
|
80
|
+
const raw = value - (decayK * weight * dm);
|
|
81
|
+
return Math.max(0, Math.min(value, raw));
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
function compileCustomFunctionSafely(source, argNames, fallback) {
|
|
86
|
+
try {
|
|
87
|
+
const ctx = vm.createContext({ Math });
|
|
88
|
+
// 如果用户提供的是“表达式”,包一层 return
|
|
89
|
+
const body = source.includes('return') || source.includes('=>') || source.includes('function')
|
|
90
|
+
? source
|
|
91
|
+
: `return (${source});`;
|
|
92
|
+
|
|
93
|
+
// 统一包成 function 体
|
|
94
|
+
const wrapper = `(function(${argNames.join(',')}) { "use strict"; ${body} })`;
|
|
95
|
+
const script = new vm.Script(wrapper, { timeout: 50 });
|
|
96
|
+
const fn = script.runInContext(ctx, { timeout: 50 });
|
|
97
|
+
if (typeof fn !== 'function') return fallback;
|
|
98
|
+
// 再包一层,避免传入异常导致抛出
|
|
99
|
+
return (...args) => {
|
|
100
|
+
try { return fn(...args); } catch (_e) { return fallback(...args); }
|
|
101
|
+
};
|
|
102
|
+
} catch (_e) {
|
|
103
|
+
return fallback;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
// ...existing code...
|
|
107
|
+
|
|
108
|
+
// 顶部 modelDefaults 增加参数(与本文件后半段的重复默认值保持一致)
|
|
43
109
|
const modelDefaults = {
|
|
44
110
|
decayFactor: 0.5,
|
|
45
111
|
maxMemeWords: 100,
|
|
@@ -51,9 +117,415 @@ const modelDefaults = {
|
|
|
51
117
|
decay: 1,
|
|
52
118
|
decayK: 1,
|
|
53
119
|
maxLen: 16,
|
|
54
|
-
edgeWeight: 1
|
|
120
|
+
edgeWeight: 1,
|
|
121
|
+
// 新增:激活/传递函数选择与自定义
|
|
122
|
+
activationType: 'relu', // identity|relu|leaky_relu|tanh|sigmoid|elu|softplus|gelu|custom
|
|
123
|
+
transferType: 'linear', // linear|exp|inverse|capped|custom
|
|
124
|
+
activationCustom: '', // 自定义激活函数源码/表达式:f(x) 或 return ...
|
|
125
|
+
transferCustom: '' // 自定义传递函数源码/表达式:f(value, weight, decayK, ctx) 或 return ...
|
|
55
126
|
};
|
|
56
127
|
const currentModelParams = { ...modelDefaults };
|
|
128
|
+
// ...existing code...
|
|
129
|
+
|
|
130
|
+
// 统一发布到 Redis(沿用现有 RuntimeMessage)
|
|
131
|
+
async function publishRuntimeToRedis(runtime) {
|
|
132
|
+
try {
|
|
133
|
+
if (!RuntimeMessage) return;
|
|
134
|
+
if (!redisClient || !redisClient.isOpen) {
|
|
135
|
+
console.warn('[REDIS] 客户端未连接,跳过发布');
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
const plainObj = runtimeToPlain(runtime);
|
|
139
|
+
const errMsg = RuntimeMessage.verify(plainObj);
|
|
140
|
+
if (errMsg) throw Error(errMsg);
|
|
141
|
+
const message = RuntimeMessage.create(plainObj);
|
|
142
|
+
const buffer = RuntimeMessage.encode(message).finish();
|
|
143
|
+
await redisClient.publish(`AI-model-${__dirname}`, buffer);
|
|
144
|
+
console.log('[REDIS] 已发布运行时状态');
|
|
145
|
+
} catch (e) {
|
|
146
|
+
console.warn('[REDIS] 发布失败:', e.message);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
// ...existing code...
|
|
150
|
+
const crypto = require('crypto');
|
|
151
|
+
// ...existing code...
|
|
152
|
+
|
|
153
|
+
// 差量复制辅助(递归分区哈希)
|
|
154
|
+
const DELTA = {
|
|
155
|
+
BUCKET_SIZE: 256,
|
|
156
|
+
hash(buf) {
|
|
157
|
+
return crypto.createHash('sha1').update(buf).digest('hex');
|
|
158
|
+
},
|
|
159
|
+
hashPoint(point) {
|
|
160
|
+
if (!point) return '0';
|
|
161
|
+
const arr = (point.connect || []).slice().sort((a, b) => {
|
|
162
|
+
if (a[1] !== b[1]) return String(a[1]).localeCompare(String(b[1]));
|
|
163
|
+
if ((a[2] || 0) !== (b[2] || 0)) return (a[2] || 0) - (b[2] || 0);
|
|
164
|
+
return (a[0] || 0) - (b[0] || 0);
|
|
165
|
+
});
|
|
166
|
+
const buf = Buffer.from(JSON.stringify([point.pointID, arr]));
|
|
167
|
+
return DELTA.hash(buf);
|
|
168
|
+
},
|
|
169
|
+
buildGraphIndex(graph) {
|
|
170
|
+
const ids = Array.from(graph.points.keys()).sort();
|
|
171
|
+
const nodeHash = new Map();
|
|
172
|
+
const buckets = [];
|
|
173
|
+
for (let i = 0; i < ids.length; i += DELTA.BUCKET_SIZE) {
|
|
174
|
+
const chunk = ids.slice(i, i + DELTA.BUCKET_SIZE);
|
|
175
|
+
let acc = '';
|
|
176
|
+
for (const id of chunk) {
|
|
177
|
+
const h = DELTA.hashPoint(graph.points.get(id));
|
|
178
|
+
nodeHash.set(id, h);
|
|
179
|
+
acc += h;
|
|
180
|
+
}
|
|
181
|
+
buckets.push({ start: chunk[0], end: chunk[chunk.length - 1], ids: chunk, hash: DELTA.hash(Buffer.from(acc)) });
|
|
182
|
+
}
|
|
183
|
+
const summary = DELTA.hash(Buffer.from(buckets.map(b => b.hash).join('')));
|
|
184
|
+
return { nodeHash, buckets, summary };
|
|
185
|
+
},
|
|
186
|
+
equalArray(a, b) {
|
|
187
|
+
if (a === b) return true;
|
|
188
|
+
if (!Array.isArray(a) || !Array.isArray(b)) return false;
|
|
189
|
+
if (a.length !== b.length) return false;
|
|
190
|
+
for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
|
|
191
|
+
return true;
|
|
192
|
+
}
|
|
193
|
+
};
|
|
194
|
+
|
|
195
|
+
// 基于递归分区哈希的差量克隆
|
|
196
|
+
async function deltaCloneRuntime(prevClone, srcRuntime) {
|
|
197
|
+
if (!prevClone) {
|
|
198
|
+
const clone = new Runtime({ ...srcRuntime.config });
|
|
199
|
+
clone.registerClone();
|
|
200
|
+
clone.spider = srcRuntime.spider;
|
|
201
|
+
|
|
202
|
+
clone.vocabManager.vocab = [...srcRuntime.vocabManager.vocab];
|
|
203
|
+
clone.vocabManager.updateMappings();
|
|
204
|
+
|
|
205
|
+
for (const [id, p] of srcRuntime.wordGraph.points.entries()) {
|
|
206
|
+
clone.wordGraph.addPoint(id, Array.isArray(p.connect) ? p.connect.map(e => [...e]) : []);
|
|
207
|
+
}
|
|
208
|
+
for (const [id, p] of srcRuntime.graph.points.entries()) {
|
|
209
|
+
clone.graph.addPoint(id, Array.isArray(p.connect) ? p.connect.map(e => [...e]) : []);
|
|
210
|
+
}
|
|
211
|
+
for (const [k, v] of srcRuntime.kvm.memory.entries()) {
|
|
212
|
+
clone.kvm.set(k, Array.isArray(v) ? [...v] : (v == null ? [] : [String(v)]));
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
clone.__deltaIndexes = {
|
|
216
|
+
graph: DELTA.buildGraphIndex(clone.graph),
|
|
217
|
+
wordGraph: DELTA.buildGraphIndex(clone.wordGraph),
|
|
218
|
+
vocabHash: DELTA.hash(Buffer.from(JSON.stringify(clone.vocabManager.vocab)))
|
|
219
|
+
};
|
|
220
|
+
return clone;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
const clone = prevClone;
|
|
224
|
+
clone.spider = srcRuntime.spider;
|
|
225
|
+
|
|
226
|
+
const srcVocabHash = DELTA.hash(Buffer.from(JSON.stringify(srcRuntime.vocabManager.vocab)));
|
|
227
|
+
if (!clone.__deltaIndexes || clone.__deltaIndexes.vocabHash !== srcVocabHash) {
|
|
228
|
+
clone.vocabManager.vocab = [...srcRuntime.vocabManager.vocab];
|
|
229
|
+
clone.vocabManager.updateMappings();
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
const syncGraph = (cloneGraph, srcGraph, name) => {
|
|
233
|
+
const prevIdx = (clone.__deltaIndexes && clone.__deltaIndexes[name]) || { nodeHash: new Map(), buckets: [] };
|
|
234
|
+
const srcIdx = DELTA.buildGraphIndex(srcGraph);
|
|
235
|
+
|
|
236
|
+
for (const id of Array.from(cloneGraph.points.keys())) {
|
|
237
|
+
if (!srcGraph.points.has(id)) cloneGraph.points.delete(id);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
for (const b of srcIdx.buckets) {
|
|
241
|
+
const prevBucket = prevIdx.buckets.find(x => x.start === b.start && x.end === b.end && x.ids.length === b.ids.length);
|
|
242
|
+
if (prevBucket && prevBucket.hash === b.hash) continue;
|
|
243
|
+
for (const id of b.ids) {
|
|
244
|
+
const srcPoint = srcGraph.points.get(id);
|
|
245
|
+
const srcH = srcIdx.nodeHash.get(id);
|
|
246
|
+
const prevH = prevIdx.nodeHash.get(id);
|
|
247
|
+
if (srcH === prevH && cloneGraph.points.has(id)) continue;
|
|
248
|
+
cloneGraph.points.set(id, {
|
|
249
|
+
pointID: id,
|
|
250
|
+
connect: Array.isArray(srcPoint.connect) ? srcPoint.connect.map(e => [...e]) : []
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
clone.__deltaIndexes = clone.__deltaIndexes || {};
|
|
256
|
+
clone.__deltaIndexes[name] = DELTA.buildGraphIndex(cloneGraph);
|
|
257
|
+
};
|
|
258
|
+
|
|
259
|
+
syncGraph(clone.wordGraph, srcRuntime.wordGraph, 'wordGraph');
|
|
260
|
+
syncGraph(clone.graph, srcRuntime.graph, 'graph');
|
|
261
|
+
|
|
262
|
+
for (const k of Array.from(clone.kvm.memory.keys())) {
|
|
263
|
+
if (!srcRuntime.kvm.memory.has(k)) clone.kvm.memory.delete(k);
|
|
264
|
+
}
|
|
265
|
+
for (const [k, v] of srcRuntime.kvm.memory.entries()) {
|
|
266
|
+
const nv = Array.isArray(v) ? v : (v == null ? [] : [String(v)]);
|
|
267
|
+
const ov = clone.kvm.memory.get(k) || [];
|
|
268
|
+
if (!DELTA.equalArray(ov, nv)) {
|
|
269
|
+
clone.kvm.set(k, [...nv]);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
clone.__deltaIndexes.vocabHash = srcVocabHash;
|
|
274
|
+
return clone;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
// 按需检索器:在用户输入(processInput)时触发网络检索与增量学习
|
|
278
|
+
class OnlineResearcher {
|
|
279
|
+
constructor(runtime, options = {}) {
|
|
280
|
+
this.runtime = runtime;
|
|
281
|
+
this.opts = Object.assign({
|
|
282
|
+
cooldownMs: 25 * 1000, // 触发冷却窗口
|
|
283
|
+
maxCrawl: 8, // 单次抓取最大页面
|
|
284
|
+
perQuery: 6,
|
|
285
|
+
maxEnqueue: 20,
|
|
286
|
+
minPromptLen: 3, // 过滤过短提示
|
|
287
|
+
ingestMinLen: 6, // ingest 的最小分词长度
|
|
288
|
+
recentCapacity: 64, // 近提示去重窗口
|
|
289
|
+
}, options);
|
|
290
|
+
this.running = false;
|
|
291
|
+
this.lastRunAt = 0;
|
|
292
|
+
this.pendingPrompt = null;
|
|
293
|
+
this.timer = null;
|
|
294
|
+
this.recent = new Set();
|
|
295
|
+
this.recentQueue = [];
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
_normalizePromptFromWords(words) {
|
|
299
|
+
const ws = (Array.isArray(words) ? words : [])
|
|
300
|
+
.map(w => String(w || '').toLowerCase().trim())
|
|
301
|
+
.filter(Boolean);
|
|
302
|
+
// 去停用词后再判断长度
|
|
303
|
+
const filtered = this.runtime.filterStopWords ? this.runtime.filterStopWords(ws) : ws;
|
|
304
|
+
if (filtered.length < this.opts.minPromptLen) return '';
|
|
305
|
+
// 取前若干关键词,避免提示过长
|
|
306
|
+
return Array.from(new Set(filtered)).slice(0, 16).join(' ');
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
_seen(key) {
|
|
310
|
+
if (this.recent.has(key)) return true;
|
|
311
|
+
this.recent.add(key);
|
|
312
|
+
this.recentQueue.push(key);
|
|
313
|
+
if (this.recentQueue.length > this.opts.recentCapacity) {
|
|
314
|
+
const old = this.recentQueue.shift();
|
|
315
|
+
this.recent.delete(old);
|
|
316
|
+
}
|
|
317
|
+
return false;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
scheduleFromWords(words) {
|
|
321
|
+
if (!global.__crawler) return; // 爬虫未初始化时跳过
|
|
322
|
+
const prompt = this._normalizePromptFromWords(words);
|
|
323
|
+
if (!prompt) return;
|
|
324
|
+
|
|
325
|
+
// 近似去重:同一时间窗内同类提示不重复抓取
|
|
326
|
+
const key = prompt.slice(0, 64);
|
|
327
|
+
if (this._seen(key)) return;
|
|
328
|
+
|
|
329
|
+
const now = Date.now();
|
|
330
|
+
const remain = this.opts.cooldownMs - (now - this.lastRunAt);
|
|
331
|
+
|
|
332
|
+
// 若正在运行或处于冷却,合并为待执行
|
|
333
|
+
if (this.running || remain > 0) {
|
|
334
|
+
this.pendingPrompt = prompt;
|
|
335
|
+
if (!this.timer) {
|
|
336
|
+
this.timer = setTimeout(() => {
|
|
337
|
+
this.timer = null;
|
|
338
|
+
const p = this.pendingPrompt;
|
|
339
|
+
this.pendingPrompt = null;
|
|
340
|
+
if (p) this._doSearchAndIngest(p);
|
|
341
|
+
}, Math.max(100, remain));
|
|
342
|
+
}
|
|
343
|
+
return;
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
this._doSearchAndIngest(prompt);
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
async _doSearchAndIngest(prompt) {
|
|
350
|
+
this.running = true;
|
|
351
|
+
this.lastRunAt = Date.now();
|
|
352
|
+
try {
|
|
353
|
+
const report = await global.__crawler.directedSearch(prompt, {
|
|
354
|
+
vertical: 'general',
|
|
355
|
+
perQuery: this.opts.perQuery,
|
|
356
|
+
maxEnqueue: this.opts.maxEnqueue,
|
|
357
|
+
crawl: true,
|
|
358
|
+
maxCrawl: this.opts.maxCrawl
|
|
359
|
+
});
|
|
360
|
+
// 抓取完成后立即增量学习(只读最近若干文档)
|
|
361
|
+
const docs = global.__crawler.loadRecentDocs(24);
|
|
362
|
+
let fed = 0;
|
|
363
|
+
for (const d of docs) {
|
|
364
|
+
fed += await this.runtime.ingestTextDocument(d.text, {
|
|
365
|
+
addNewWords: true,
|
|
366
|
+
minLen: this.opts.ingestMinLen
|
|
367
|
+
});
|
|
368
|
+
}
|
|
369
|
+
if (fed > 0) {
|
|
370
|
+
this.runtime.updateAttentionLinks();
|
|
371
|
+
console.log(`[RESEARCH] "${prompt}" 抓取并增量学习完成: fed=${fed}, crawled=${report?.crawled ?? 0}`);
|
|
372
|
+
} else {
|
|
373
|
+
console.log(`[RESEARCH] "${prompt}" 未产生有效增量`);
|
|
374
|
+
}
|
|
375
|
+
} catch (e) {
|
|
376
|
+
console.warn('[RESEARCH] 抓取/增量学习失败:', e.message);
|
|
377
|
+
} finally {
|
|
378
|
+
this.running = false;
|
|
379
|
+
// 合并突发多次提示:优先处理最近一次
|
|
380
|
+
if (this.pendingPrompt) {
|
|
381
|
+
const p = this.pendingPrompt;
|
|
382
|
+
this.pendingPrompt = null;
|
|
383
|
+
this._doSearchAndIngest(p);
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
// ...existing code...
|
|
389
|
+
// 轮换管理器:在克隆上训练,评估通过后原子切换,避免占用在线副本
|
|
390
|
+
class RotationManager {
|
|
391
|
+
constructor(ctrlA, ctrlB, ctrlC, options = {}) {
|
|
392
|
+
this.ctrlA = ctrlA;
|
|
393
|
+
this.ctrlB = ctrlB;
|
|
394
|
+
this.ctrlC = ctrlC;
|
|
395
|
+
this.isRotating = false;
|
|
396
|
+
this.timer = null;
|
|
397
|
+
this.opts = Object.assign({
|
|
398
|
+
cycleMs: 30 * 60 * 1000, // 每轮步进间隔:默认30分钟
|
|
399
|
+
cooldownMs: 60 * 1000, // 单步完成后的冷却时间
|
|
400
|
+
learnIters: 3, // 克隆训练迭代
|
|
401
|
+
minImprove: 0.005 // 最小收益阈值(0.5%)
|
|
402
|
+
}, options);
|
|
403
|
+
this.sequence = [
|
|
404
|
+
{ from: 'A', to: 'B' },
|
|
405
|
+
{ from: 'B', to: 'C' },
|
|
406
|
+
{ from: 'C', to: 'A' }
|
|
407
|
+
];
|
|
408
|
+
this.idx = 0;
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
start() {
|
|
412
|
+
if (this.timer) return;
|
|
413
|
+
const runStep = async () => {
|
|
414
|
+
if (this.isRotating || isShuttingDown) return;
|
|
415
|
+
this.isRotating = true;
|
|
416
|
+
try {
|
|
417
|
+
const step = this.sequence[this.idx % this.sequence.length];
|
|
418
|
+
await this._runOne(step.from, step.to);
|
|
419
|
+
this.idx++;
|
|
420
|
+
} catch (e) {
|
|
421
|
+
console.warn('[ROTATE] 步骤失败:', e.message);
|
|
422
|
+
} finally {
|
|
423
|
+
this.isRotating = false;
|
|
424
|
+
}
|
|
425
|
+
};
|
|
426
|
+
// 立即跑一次,再按周期跑
|
|
427
|
+
runStep();
|
|
428
|
+
this.timer = registerInterval(runStep, this.opts.cycleMs);
|
|
429
|
+
console.log(`[ROTATE] 轮换学习已启动:每${Math.round(this.opts.cycleMs / 60000)}分钟步进一次`);
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
stop() {
|
|
433
|
+
if (this.timer) {
|
|
434
|
+
clearInterval(this.timer);
|
|
435
|
+
}
|
|
436
|
+
this.timer = null;
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
_getCtrl(name) {
|
|
440
|
+
return name === 'A' ? this.ctrlA : (name === 'B' ? this.ctrlB : this.ctrlC);
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
async _runOne(fromName, toName) {
|
|
444
|
+
const fromCtrl = this._getCtrl(fromName);
|
|
445
|
+
const toCtrl = this._getCtrl(toName);
|
|
446
|
+
if (!fromCtrl || !toCtrl) return;
|
|
447
|
+
|
|
448
|
+
if (fromCtrl.isLearning || toCtrl.isLearning) {
|
|
449
|
+
console.log(`[ROTATE] ${fromName} 或 ${toName} 正在学习,跳过本步`);
|
|
450
|
+
return;
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
console.log(`[ROTATE] ${fromName} -> ${toName} 开始克隆与训练(离线)`);
|
|
454
|
+
|
|
455
|
+
// 1) 识别模式 + 克隆
|
|
456
|
+
const association = new AssociationLayer(fromCtrl.runtime);
|
|
457
|
+
association.patterns = association.identifyPatterns();
|
|
458
|
+
const systemClone = await association.cloneSystem();
|
|
459
|
+
systemClone.spider = fromCtrl.runtime.spider;
|
|
460
|
+
association.applyPatternsToClone(systemClone);
|
|
461
|
+
|
|
462
|
+
// 2) 预热克隆(确保索引、边权等就绪)
|
|
463
|
+
const sampleWords = Array.from(new Set(systemClone.vocabManager.vocab.slice(4, 64))).filter(w => w.length > 1).slice(0, 16);
|
|
464
|
+
if (sampleWords.length) {
|
|
465
|
+
try {
|
|
466
|
+
await systemClone.processInput(sampleWords, { addNewWords: false });
|
|
467
|
+
} catch (_) { }
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
// 3) 在克隆上训练而非在线副本
|
|
471
|
+
try {
|
|
472
|
+
await systemClone.startSelfLearning(this.opts.learnIters);
|
|
473
|
+
} catch (e) {
|
|
474
|
+
console.warn('[ROTATE] 克隆训练失败:', e.message);
|
|
475
|
+
return;
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
// 4) 评估:对比 from 与 clone
|
|
479
|
+
const { origScore, cloneScore } = await this._evaluateTwo(fromCtrl.runtime, systemClone);
|
|
480
|
+
const improve = origScore > 0 ? (cloneScore - origScore) / origScore : (cloneScore > 0 ? 1 : 0);
|
|
481
|
+
|
|
482
|
+
console.log(`[ROTATE] 评估:orig=${origScore.toFixed(4)} clone=${cloneScore.toFixed(4)} improve=${(improve * 100).toFixed(2)}%`);
|
|
483
|
+
|
|
484
|
+
if (improve >= this.opts.minImprove) {
|
|
485
|
+
// 5) 提交:原子切换 toCtrl.runtime = clone
|
|
486
|
+
toCtrl.updateRuntime(systemClone);
|
|
487
|
+
console.log(`[ROTATE] 已将 ${fromName} 的成果切换到 ${toName}`);
|
|
488
|
+
|
|
489
|
+
// 如切到 A,发布到 Redis
|
|
490
|
+
if (toName === 'A') {
|
|
491
|
+
await publishRuntimeToRedis(systemClone);
|
|
492
|
+
}
|
|
493
|
+
} else {
|
|
494
|
+
console.log('[ROTATE] 改进不足,放弃本次提交');
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
// 6) 冷却
|
|
498
|
+
await new Promise(r => setTimeout(r, this.opts.cooldownMs));
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
async _evaluateTwo(baseRuntime, cloneRuntime) {
|
|
502
|
+
try {
|
|
503
|
+
// 复用 RL 的评估数据来源与方法
|
|
504
|
+
const rl = new ReinforcementLearner(baseRuntime);
|
|
505
|
+
const articles = rl.loadTestArticles();
|
|
506
|
+
if (!articles.length) {
|
|
507
|
+
// 无测试集:退化为用词表覆盖率近似
|
|
508
|
+
const s1 = baseRuntime.vocabManager.getsize();
|
|
509
|
+
const s2 = cloneRuntime.vocabManager.getsize();
|
|
510
|
+
return { origScore: s1, cloneScore: s2 };
|
|
511
|
+
}
|
|
512
|
+
let origCoverages = 0;
|
|
513
|
+
let cloneCoverages = 0;
|
|
514
|
+
for (const article of articles) {
|
|
515
|
+
origCoverages += rl.evaluateSystem(baseRuntime, article);
|
|
516
|
+
cloneCoverages += rl.evaluateSystem(cloneRuntime, article);
|
|
517
|
+
}
|
|
518
|
+
// 使用“总和-方差的平方根”的稳健度量(与 RL.learn 相同思想)
|
|
519
|
+
const avgOrig = origCoverages;
|
|
520
|
+
const avgClone = cloneCoverages;
|
|
521
|
+
return { origScore: avgOrig, cloneScore: avgClone };
|
|
522
|
+
} catch (e) {
|
|
523
|
+
console.warn('[ROTATE] 评估失败,回退到零分:', e.message);
|
|
524
|
+
return { origScore: 0, cloneScore: 0 };
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
// ...existing code...
|
|
57
529
|
async function batchAddPoints(graph, pointsArr, batchSize = 500) {
|
|
58
530
|
if (!graph || !pointsArr || !Array.isArray(pointsArr)) {
|
|
59
531
|
console.error('[BATCH] 无效的参数:', {
|
|
@@ -300,7 +772,6 @@ class SnapshotManager {
|
|
|
300
772
|
}
|
|
301
773
|
|
|
302
774
|
async createSnapshot(name = 'auto') {
|
|
303
|
-
// 防止并发创建
|
|
304
775
|
if (this.isCreatingSnapshot) {
|
|
305
776
|
console.log('[SNAPSHOT] 另一个快照正在创建中,跳过');
|
|
306
777
|
return null;
|
|
@@ -316,36 +787,39 @@ class SnapshotManager {
|
|
|
316
787
|
|
|
317
788
|
console.log(`[SNAPSHOT] 开始创建快照: ${snapshotId}`);
|
|
318
789
|
|
|
319
|
-
//
|
|
790
|
+
// 优先使用分区图的全量导出(避免仅导出窗口)
|
|
791
|
+
let memesAll = [];
|
|
792
|
+
if (this.runtime.graph && typeof this.runtime.graph.exportAllPoints === 'function') {
|
|
793
|
+
try {
|
|
794
|
+
memesAll = await this.runtime.graph.exportAllPoints();
|
|
795
|
+
} catch (e) {
|
|
796
|
+
console.warn('[SNAPSHOT] 分区图导出失败,回退窗口:', e.message);
|
|
797
|
+
memesAll = this.runtime.graph.getAllPoints();
|
|
798
|
+
}
|
|
799
|
+
} else {
|
|
800
|
+
memesAll = this.runtime.graph.getAllPoints();
|
|
801
|
+
}
|
|
802
|
+
|
|
320
803
|
const snapshotData = {
|
|
321
804
|
id: snapshotId,
|
|
322
805
|
timestamp,
|
|
323
806
|
name,
|
|
324
807
|
createDate: new Date().toISOString(),
|
|
325
|
-
memes:
|
|
808
|
+
memes: memesAll,
|
|
326
809
|
wordGraph: Array.from(this.runtime.wordGraph.points.values()),
|
|
327
810
|
kvm: Array.from(this.runtime.kvm.memory.entries()),
|
|
328
811
|
vocab: this.runtime.vocabManager.vocab,
|
|
329
|
-
// 修正:序列化为 [word, [[sessionId,count], ...]]
|
|
330
812
|
wordAccessLog: Array.from(this.runtime.wordAccessLog.entries()).map(([w, per]) =>
|
|
331
813
|
[w, per instanceof Map ? Array.from(per.entries()) : (Array.isArray(per) ? [['legacy', per.length]] : [])]
|
|
332
814
|
),
|
|
333
|
-
// 新增:保存会话状态
|
|
334
815
|
sessions: this.runtime.session.export()
|
|
335
816
|
};
|
|
336
817
|
|
|
337
|
-
// 写入临时文件,然后原子重命名以确保数据完整性
|
|
338
818
|
const tempPath = `${filePath}.temp`;
|
|
339
819
|
await fs.promises.writeFile(tempPath, JSON.stringify(snapshotData), 'utf-8');
|
|
340
820
|
await fs.promises.rename(tempPath, filePath);
|
|
341
821
|
|
|
342
|
-
|
|
343
|
-
const snapshotInfo = {
|
|
344
|
-
id: snapshotId,
|
|
345
|
-
timestamp,
|
|
346
|
-
name,
|
|
347
|
-
path: filePath
|
|
348
|
-
};
|
|
822
|
+
const snapshotInfo = { id: snapshotId, timestamp, name, path: filePath };
|
|
349
823
|
this.snapshotList.unshift(snapshotInfo);
|
|
350
824
|
|
|
351
825
|
console.timeEnd('snapshotCreation');
|
|
@@ -359,11 +833,11 @@ class SnapshotManager {
|
|
|
359
833
|
}
|
|
360
834
|
}
|
|
361
835
|
|
|
836
|
+
|
|
362
837
|
async restoreSnapshot(snapshotId) {
|
|
363
838
|
console.log(`[SNAPSHOT] 开始从快照恢复: ${snapshotId}`);
|
|
364
839
|
console.time('snapshotRestore');
|
|
365
840
|
|
|
366
|
-
// 查找快照
|
|
367
841
|
const snapshot = this.snapshotList.find(s => s.id === snapshotId);
|
|
368
842
|
if (!snapshot) {
|
|
369
843
|
console.error(`[SNAPSHOT] 快照不存在: ${snapshotId}`);
|
|
@@ -371,37 +845,27 @@ class SnapshotManager {
|
|
|
371
845
|
}
|
|
372
846
|
|
|
373
847
|
try {
|
|
374
|
-
// 读取快照文件
|
|
375
|
-
console.log(`[SNAPSHOT] 从文件读取数据: ${snapshot.path}`);
|
|
376
848
|
const dataStr = await fs.promises.readFile(snapshot.path, 'utf-8');
|
|
377
849
|
const data = JSON.parse(dataStr);
|
|
378
850
|
|
|
379
|
-
// 在恢复前创建自动备份
|
|
380
851
|
await this.createSnapshot(`auto_before_restore_${snapshotId}`);
|
|
381
852
|
|
|
382
|
-
//
|
|
383
|
-
console.log('[SNAPSHOT] 清空当前运行时...');
|
|
384
|
-
this.runtime.graph = new GraphDB();
|
|
853
|
+
// 清空当前运行时(词图/KVM 内存)
|
|
385
854
|
this.runtime.wordGraph = new GraphDB();
|
|
386
855
|
this.runtime.kvm = new KVM();
|
|
387
856
|
this.runtime.wordAccessLog = new Map();
|
|
388
857
|
|
|
389
|
-
//
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
this.runtime.graph.addPoint(point.pointID, point.connect);
|
|
397
|
-
}
|
|
398
|
-
// 让事件循环有机会处理其他事件
|
|
399
|
-
await new Promise(resolve => setImmediate(resolve));
|
|
858
|
+
// 恢复模因图:走分区导入(覆盖分区存储)
|
|
859
|
+
if (data.memes && this.runtime.graph && typeof this.runtime.graph.importAllPoints === 'function') {
|
|
860
|
+
await this.runtime.graph.importAllPoints(data.memes);
|
|
861
|
+
} else if (data.memes) {
|
|
862
|
+
// 窗口回退(不推荐)
|
|
863
|
+
for (const point of data.memes) {
|
|
864
|
+
await this.runtime.graph.addPoint(point.pointID, point.connect);
|
|
400
865
|
}
|
|
401
866
|
}
|
|
402
867
|
|
|
403
868
|
// 恢复词图
|
|
404
|
-
console.log('[SNAPSHOT] 恢复词语网络...');
|
|
405
869
|
if (data.wordGraph) {
|
|
406
870
|
const BATCH_SIZE = 1000;
|
|
407
871
|
for (let i = 0; i < data.wordGraph.length; i += BATCH_SIZE) {
|
|
@@ -414,27 +878,22 @@ class SnapshotManager {
|
|
|
414
878
|
}
|
|
415
879
|
|
|
416
880
|
// 恢复KVM
|
|
417
|
-
console.log('[SNAPSHOT] 恢复键值存储...');
|
|
418
881
|
if (data.kvm) {
|
|
419
882
|
const BATCH_SIZE = 1000;
|
|
420
883
|
for (let i = 0; i < data.kvm.length; i += BATCH_SIZE) {
|
|
421
884
|
const batch = data.kvm.slice(i, i + BATCH_SIZE);
|
|
422
|
-
for (const [k, v] of batch)
|
|
423
|
-
this.runtime.kvm.set(k, v);
|
|
424
|
-
}
|
|
885
|
+
for (const [k, v] of batch) this.runtime.kvm.set(k, v);
|
|
425
886
|
await new Promise(resolve => setImmediate(resolve));
|
|
426
887
|
}
|
|
427
888
|
}
|
|
428
889
|
|
|
429
890
|
// 恢复词表
|
|
430
|
-
console.log('[SNAPSHOT] 恢复词表...');
|
|
431
891
|
if (data.vocab) {
|
|
432
892
|
this.runtime.vocabManager.vocab = data.vocab;
|
|
433
893
|
this.runtime.vocabManager.updateMappings();
|
|
434
894
|
}
|
|
435
895
|
|
|
436
896
|
// 恢复词访问日志
|
|
437
|
-
console.log('[SNAPSHOT] 恢复词访问日志...');
|
|
438
897
|
if (data.wordAccessLog) {
|
|
439
898
|
const restored = new Map();
|
|
440
899
|
for (const [word, per] of data.wordAccessLog) {
|
|
@@ -448,13 +907,12 @@ class SnapshotManager {
|
|
|
448
907
|
}
|
|
449
908
|
this.runtime.wordAccessLog = restored;
|
|
450
909
|
}
|
|
451
|
-
// 恢复会话信息
|
|
452
910
|
if (data.sessions) {
|
|
453
911
|
this.runtime.session.import(data.sessions);
|
|
454
912
|
} else {
|
|
455
|
-
// 无会话信息时,创建一个遗留会话
|
|
456
913
|
this.runtime.session.startNewSession({ reason: 'snapshot-legacy' });
|
|
457
914
|
}
|
|
915
|
+
|
|
458
916
|
console.timeEnd('snapshotRestore');
|
|
459
917
|
console.log(`[SNAPSHOT] 成功从快照恢复: ${snapshotId}`);
|
|
460
918
|
return true;
|
|
@@ -754,65 +1212,775 @@ class GraphDB {
|
|
|
754
1212
|
return path.reverse();
|
|
755
1213
|
}
|
|
756
1214
|
|
|
757
|
-
for (const [weight, neighborID, direction] of this.points.get(current).connect) {
|
|
758
|
-
// 只考虑允许的方向
|
|
759
|
-
if (direction === 1 && neighborID !== current) continue; // 只指向自己
|
|
760
|
-
if (direction === 2 && current !== fromID) continue; // 只指向对方
|
|
761
|
-
if (closedSet.has(neighborID)) continue;
|
|
1215
|
+
for (const [weight, neighborID, direction] of this.points.get(current).connect) {
|
|
1216
|
+
// 只考虑允许的方向
|
|
1217
|
+
if (direction === 1 && neighborID !== current) continue; // 只指向自己
|
|
1218
|
+
if (direction === 2 && current !== fromID) continue; // 只指向对方
|
|
1219
|
+
if (closedSet.has(neighborID)) continue;
|
|
1220
|
+
|
|
1221
|
+
const tentativeGScore = (gScore.get(current) || Infinity) + weight;
|
|
1222
|
+
if (!openSet.has(neighborID)) {
|
|
1223
|
+
openSet.add(neighborID);
|
|
1224
|
+
} else if (tentativeGScore >= (gScore.get(neighborID) || Infinity)) {
|
|
1225
|
+
continue;
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
cameFrom.set(neighborID, current);
|
|
1229
|
+
gScore.set(neighborID, tentativeGScore);
|
|
1230
|
+
fScore.set(neighborID, tentativeGScore + this.heuristic(neighborID, toID));
|
|
1231
|
+
}
|
|
1232
|
+
}
|
|
1233
|
+
return null;
|
|
1234
|
+
}
|
|
1235
|
+
|
|
1236
|
+
// 获取所有点
|
|
1237
|
+
getAllPoints() {
|
|
1238
|
+
return Array.from(this.points.values());
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1241
|
+
// 启发式函数:简单的常数启发式
|
|
1242
|
+
heuristic(pointID, toID) {
|
|
1243
|
+
return 1; // 简化的启发式函数
|
|
1244
|
+
}
|
|
1245
|
+
existEdge(pointID, neighborID) {
|
|
1246
|
+
const point = this.points.get(pointID);
|
|
1247
|
+
if (!point) {
|
|
1248
|
+
return { exist: false, weight: undefined, type: undefined };
|
|
1249
|
+
}
|
|
1250
|
+
const connectArr = point.connect || [];
|
|
1251
|
+
const found = connectArr.find(([_, id]) => id === neighborID);
|
|
1252
|
+
return {
|
|
1253
|
+
exist: connectArr.some(([_, id]) => id === neighborID),
|
|
1254
|
+
weight: found ? found[0] : undefined,
|
|
1255
|
+
type: found ? found[2] : undefined
|
|
1256
|
+
};
|
|
1257
|
+
}
|
|
1258
|
+
existPoint(pointID) {
|
|
1259
|
+
return { exist: this.points.has(pointID), connect: this.points.get(pointID)?.connect || [] };
|
|
1260
|
+
}
|
|
1261
|
+
deleteEdge(pointID, neighborID) {
|
|
1262
|
+
if (this.existEdge(pointID, neighborID).exist) {
|
|
1263
|
+
this.points.get(pointID).connect = this.points.get(pointID).connect.filter(([_, id]) => id !== neighborID);
|
|
1264
|
+
this.points.get(neighborID).connect = this.points.get(neighborID).connect.filter(([_, id]) => id !== pointID);
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
deletePoint(pointID) {
|
|
1268
|
+
if (this.existPoint(pointID).exist) {
|
|
1269
|
+
this.points.delete(pointID);
|
|
1270
|
+
}
|
|
1271
|
+
}
|
|
1272
|
+
}
|
|
1273
|
+
// 简易日志辅助
|
|
1274
|
+
function logPart(...args) { console.log('[PART]', ...args); }
|
|
1275
|
+
function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
|
|
1276
|
+
|
|
1277
|
+
// 存储适配层(FS/LMDB/Level 多后端,按需加载)
|
|
1278
|
+
class GraphStorageAdapter {
|
|
1279
|
+
constructor({ baseDir, backend = 'fs' } = {}) {
|
|
1280
|
+
this.baseDir = baseDir || path.join(__dirname, 'graph_parts');
|
|
1281
|
+
this.backend = backend;
|
|
1282
|
+
this.ready = false;
|
|
1283
|
+
|
|
1284
|
+
// 尝试创建目录
|
|
1285
|
+
fs.mkdirSync(this.baseDir, { recursive: true });
|
|
1286
|
+
|
|
1287
|
+
// 可选依赖
|
|
1288
|
+
this.lmdb = null;
|
|
1289
|
+
this.level = null;
|
|
1290
|
+
|
|
1291
|
+
if (backend === 'lmdb') {
|
|
1292
|
+
try {
|
|
1293
|
+
this.lmdb = require('lmdb');
|
|
1294
|
+
} catch (e) {
|
|
1295
|
+
console.warn('[PART][ADAPTER] LMDB 不可用,降级为 FS:', e.message);
|
|
1296
|
+
this.backend = 'fs';
|
|
1297
|
+
}
|
|
1298
|
+
}
|
|
1299
|
+
if (backend === 'level') {
|
|
1300
|
+
try {
|
|
1301
|
+
this.level = require('level');
|
|
1302
|
+
} catch (e) {
|
|
1303
|
+
console.warn('[PART][ADAPTER] level 不可用,降级为 FS:', e.message);
|
|
1304
|
+
this.backend = 'fs';
|
|
1305
|
+
}
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1308
|
+
// 初始化后端
|
|
1309
|
+
this._initBackend();
|
|
1310
|
+
}
|
|
1311
|
+
|
|
1312
|
+
_initBackend() {
|
|
1313
|
+
if (this.backend === 'fs') {
|
|
1314
|
+
// FS: 每个分区一个 .jsonl(节点),边界事件一个独立 .jsonl
|
|
1315
|
+
this.ready = true;
|
|
1316
|
+
return;
|
|
1317
|
+
}
|
|
1318
|
+
if (this.backend === 'lmdb' && this.lmdb) {
|
|
1319
|
+
try {
|
|
1320
|
+
const storeDir = path.join(this.baseDir, 'lmdb');
|
|
1321
|
+
fs.mkdirSync(storeDir, { recursive: true });
|
|
1322
|
+
this.env = this.lmdb.open({
|
|
1323
|
+
path: storeDir,
|
|
1324
|
+
mapSize: 1024n * 1024n * 1024n * 64n,
|
|
1325
|
+
compression: true,
|
|
1326
|
+
});
|
|
1327
|
+
this.ready = true;
|
|
1328
|
+
} catch (e) {
|
|
1329
|
+
console.warn('[PART][ADAPTER] LMDB 初始化失败,降级 FS:', e.message);
|
|
1330
|
+
this.backend = 'fs';
|
|
1331
|
+
this.ready = true;
|
|
1332
|
+
}
|
|
1333
|
+
return;
|
|
1334
|
+
}
|
|
1335
|
+
if (this.backend === 'level' && this.level) {
|
|
1336
|
+
try {
|
|
1337
|
+
const dbDir = path.join(this.baseDir, 'leveldb');
|
|
1338
|
+
fs.mkdirSync(dbDir, { recursive: true });
|
|
1339
|
+
this.db = new this.level.Level(dbDir, { valueEncoding: 'json' });
|
|
1340
|
+
this.ready = true;
|
|
1341
|
+
} catch (e) {
|
|
1342
|
+
console.warn('[PART][ADAPTER] level 初始化失败,降级 FS:', e.message);
|
|
1343
|
+
this.backend = 'fs';
|
|
1344
|
+
this.ready = true;
|
|
1345
|
+
}
|
|
1346
|
+
return;
|
|
1347
|
+
}
|
|
1348
|
+
this.ready = true;
|
|
1349
|
+
}
|
|
1350
|
+
|
|
1351
|
+
// 分区文件名(FS)
|
|
1352
|
+
_partFile(pid) { return path.join(this.baseDir, `p_${pid}.jsonl`); }
|
|
1353
|
+
_eventFile(pid) { return path.join(this.baseDir, `p_${pid}.events.jsonl`); }
|
|
1354
|
+
|
|
1355
|
+
// 读取分区(返回 { points: Map<string,{pointID,connect:[]}> })
|
|
1356
|
+
async loadPartition(pid) {
|
|
1357
|
+
if (this.backend === 'fs') {
|
|
1358
|
+
const file = this._partFile(pid);
|
|
1359
|
+
const out = new Map();
|
|
1360
|
+
if (!fs.existsSync(file)) return { points: out };
|
|
1361
|
+
const rs = fs.createReadStream(file, { encoding: 'utf-8' });
|
|
1362
|
+
let buf = '';
|
|
1363
|
+
for await (const chunk of rs) {
|
|
1364
|
+
buf += chunk;
|
|
1365
|
+
let idx;
|
|
1366
|
+
while ((idx = buf.indexOf('\n')) >= 0) {
|
|
1367
|
+
const line = buf.slice(0, idx);
|
|
1368
|
+
buf = buf.slice(idx + 1);
|
|
1369
|
+
if (!line.trim()) continue;
|
|
1370
|
+
try {
|
|
1371
|
+
const obj = JSON.parse(line);
|
|
1372
|
+
if (obj && obj.pointID) {
|
|
1373
|
+
out.set(obj.pointID, { pointID: obj.pointID, connect: obj.connect || [] });
|
|
1374
|
+
}
|
|
1375
|
+
} catch { /* ignore */ }
|
|
1376
|
+
}
|
|
1377
|
+
}
|
|
1378
|
+
return { points: out };
|
|
1379
|
+
}
|
|
1380
|
+
|
|
1381
|
+
if (this.backend === 'lmdb' && this.env) {
|
|
1382
|
+
const points = new Map();
|
|
1383
|
+
const txn = this.env.beginTxn({ readOnly: true });
|
|
1384
|
+
try {
|
|
1385
|
+
const cursor = new this.lmdb.Cursors.Cursor(txn, this.env.openDB({ name: `p_${pid}`, create: true }));
|
|
1386
|
+
for (let found = cursor.goToFirst(); found; found = cursor.goToNext()) {
|
|
1387
|
+
const key = cursor.getCurrentString();
|
|
1388
|
+
const val = cursor.getCurrentBinary();
|
|
1389
|
+
try {
|
|
1390
|
+
const obj = JSON.parse(Buffer.from(val).toString('utf-8'));
|
|
1391
|
+
if (obj && obj.pointID) points.set(obj.pointID, obj);
|
|
1392
|
+
} catch { }
|
|
1393
|
+
}
|
|
1394
|
+
cursor.close();
|
|
1395
|
+
} catch { }
|
|
1396
|
+
txn.abort();
|
|
1397
|
+
return { points };
|
|
1398
|
+
}
|
|
1399
|
+
|
|
1400
|
+
if (this.backend === 'level' && this.db) {
|
|
1401
|
+
const points = new Map();
|
|
1402
|
+
try {
|
|
1403
|
+
for await (const { key, value } of this.db.iterator({ gte: `p:${pid}:`, lt: `p:${pid};` })) {
|
|
1404
|
+
const obj = value;
|
|
1405
|
+
if (obj && obj.pointID) points.set(obj.pointID, obj);
|
|
1406
|
+
}
|
|
1407
|
+
} catch { }
|
|
1408
|
+
return { points };
|
|
1409
|
+
}
|
|
1410
|
+
|
|
1411
|
+
return { points: new Map() };
|
|
1412
|
+
}
|
|
1413
|
+
|
|
1414
|
+
|
|
1415
|
+
|
|
1416
|
+
// 保存分区(全量覆盖写)
|
|
1417
|
+
async savePartition(pid, pointsMap) {
|
|
1418
|
+
if (!(pointsMap instanceof Map)) return;
|
|
1419
|
+
if (this.backend === 'fs') {
|
|
1420
|
+
const file = this._partFile(pid);
|
|
1421
|
+
const tmp = `${file}.tmp`;
|
|
1422
|
+
const ws = fs.createWriteStream(tmp, { encoding: 'utf-8' });
|
|
1423
|
+
for (const [, p] of pointsMap.entries()) {
|
|
1424
|
+
ws.write(JSON.stringify({ pointID: p.pointID, connect: p.connect || [] }) + '\n');
|
|
1425
|
+
}
|
|
1426
|
+
await new Promise((res, rej) => ws.end(res));
|
|
1427
|
+
await fs.promises.rename(tmp, file);
|
|
1428
|
+
return;
|
|
1429
|
+
}
|
|
1430
|
+
|
|
1431
|
+
if (this.backend === 'lmdb' && this.env) {
|
|
1432
|
+
const dbi = this.env.openDB({ name: `p_${pid}`, create: true });
|
|
1433
|
+
const txn = this.env.beginTxn();
|
|
1434
|
+
try {
|
|
1435
|
+
// 先清空:简化实现
|
|
1436
|
+
const cur = new this.lmdb.Cursors.Cursor(txn, dbi);
|
|
1437
|
+
for (let found = cur.goToFirst(); found; found = cur.goToNext()) {
|
|
1438
|
+
const k = cur.getCurrentString();
|
|
1439
|
+
txn.del(dbi, k);
|
|
1440
|
+
}
|
|
1441
|
+
cur.close();
|
|
1442
|
+
for (const [, p] of pointsMap.entries()) {
|
|
1443
|
+
txn.put(dbi, p.pointID, JSON.stringify(p));
|
|
1444
|
+
}
|
|
1445
|
+
txn.commit();
|
|
1446
|
+
} catch (e) {
|
|
1447
|
+
try { txn.abort(); } catch { }
|
|
1448
|
+
console.warn('[PART][ADAPTER][LMDB] savePartition err:', e.message);
|
|
1449
|
+
}
|
|
1450
|
+
return;
|
|
1451
|
+
}
|
|
1452
|
+
|
|
1453
|
+
if (this.backend === 'level' && this.db) {
|
|
1454
|
+
const ops = [];
|
|
1455
|
+
// 简化:清理旧 key 不容易,直接覆盖同 key
|
|
1456
|
+
for (const [, p] of pointsMap.entries()) {
|
|
1457
|
+
ops.push({ type: 'put', key: `p:${pid}:${p.pointID}`, value: p });
|
|
1458
|
+
}
|
|
1459
|
+
await this.db.batch(ops);
|
|
1460
|
+
return;
|
|
1461
|
+
}
|
|
1462
|
+
}
|
|
1463
|
+
|
|
1464
|
+
// 追加边界事件(跨分区边)
|
|
1465
|
+
async appendEdgeEvent(pid, event) {
|
|
1466
|
+
if (!event || !event.type) return;
|
|
1467
|
+
if (this.backend === 'fs') {
|
|
1468
|
+
const file = this._eventFile(pid);
|
|
1469
|
+
fs.appendFileSync(file, JSON.stringify(event) + '\n', 'utf-8');
|
|
1470
|
+
return;
|
|
1471
|
+
}
|
|
1472
|
+
if (this.backend === 'lmdb' && this.env) {
|
|
1473
|
+
const dbi = this.env.openDB({ name: `e_${pid}`, create: true });
|
|
1474
|
+
const txn = this.env.beginTxn();
|
|
1475
|
+
try {
|
|
1476
|
+
const key = `e:${Date.now()}_${Math.random().toString(36).slice(2, 10)}`;
|
|
1477
|
+
txn.put(dbi, key, JSON.stringify(event));
|
|
1478
|
+
txn.commit();
|
|
1479
|
+
} catch (e) {
|
|
1480
|
+
try { txn.abort(); } catch { }
|
|
1481
|
+
}
|
|
1482
|
+
return;
|
|
1483
|
+
}
|
|
1484
|
+
if (this.backend === 'level' && this.db) {
|
|
1485
|
+
const key = `e:${pid}:${Date.now()}_${Math.random().toString(36).slice(2, 10)}`;
|
|
1486
|
+
await this.db.put(key, event);
|
|
1487
|
+
return;
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
|
|
1491
|
+
// 读取并消费边界事件(与该分区相关的)
|
|
1492
|
+
async consumeEdgeEvents(pid, filterFn = null, limit = 2000) {
|
|
1493
|
+
const events = [];
|
|
1494
|
+
if (this.backend === 'fs') {
|
|
1495
|
+
const file = this._eventFile(pid);
|
|
1496
|
+
if (!fs.existsSync(file)) return events;
|
|
1497
|
+
|
|
1498
|
+
const tmp = `${file}.tmp`;
|
|
1499
|
+
// 将不消费的事件写入 tmp,再覆盖原文件;已消费事件返回
|
|
1500
|
+
const lines = fs.readFileSync(file, 'utf-8').split(/\r?\n/).filter(Boolean);
|
|
1501
|
+
const remain = [];
|
|
1502
|
+
for (const line of lines) {
|
|
1503
|
+
try {
|
|
1504
|
+
const e = JSON.parse(line);
|
|
1505
|
+
const ok = filterFn ? filterFn(e) : true;
|
|
1506
|
+
if (ok && events.length < limit) {
|
|
1507
|
+
events.push(e);
|
|
1508
|
+
} else {
|
|
1509
|
+
remain.push(line);
|
|
1510
|
+
}
|
|
1511
|
+
} catch {
|
|
1512
|
+
remain.push(line);
|
|
1513
|
+
}
|
|
1514
|
+
}
|
|
1515
|
+
fs.writeFileSync(tmp, remain.join('\n') + (remain.length ? '\n' : ''), 'utf-8');
|
|
1516
|
+
await fs.promises.rename(tmp, file);
|
|
1517
|
+
return events;
|
|
1518
|
+
}
|
|
1519
|
+
|
|
1520
|
+
if (this.backend === 'lmdb' && this.env) {
|
|
1521
|
+
const dbi = this.env.openDB({ name: `e_${pid}`, create: true });
|
|
1522
|
+
const txn = this.env.beginTxn();
|
|
1523
|
+
const toDel = [];
|
|
1524
|
+
try {
|
|
1525
|
+
const cur = new this.lmdb.Cursors.Cursor(txn, dbi);
|
|
1526
|
+
for (let found = cur.goToFirst(); found; found = cur.goToNext()) {
|
|
1527
|
+
const k = cur.getCurrentString();
|
|
1528
|
+
const v = cur.getCurrentBinary();
|
|
1529
|
+
const e = JSON.parse(Buffer.from(v).toString('utf-8'));
|
|
1530
|
+
const ok = filterFn ? filterFn(e) : true;
|
|
1531
|
+
if (ok && events.length < limit) {
|
|
1532
|
+
events.push(e);
|
|
1533
|
+
toDel.push(k);
|
|
1534
|
+
}
|
|
1535
|
+
}
|
|
1536
|
+
cur.close();
|
|
1537
|
+
for (const k of toDel) txn.del(dbi, k);
|
|
1538
|
+
txn.commit();
|
|
1539
|
+
} catch (e) {
|
|
1540
|
+
try { txn.abort(); } catch { }
|
|
1541
|
+
}
|
|
1542
|
+
return events;
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1545
|
+
if (this.backend === 'level' && this.db) {
|
|
1546
|
+
// 简化:扫描全库 keys 读取该 pid 的事件
|
|
1547
|
+
try {
|
|
1548
|
+
const toDel = [];
|
|
1549
|
+
for await (const { key, value } of this.db.iterator({ gte: `e:${pid}:`, lt: `e:${pid};` })) {
|
|
1550
|
+
const ok = filterFn ? filterFn(value) : true;
|
|
1551
|
+
if (ok && events.length < limit) {
|
|
1552
|
+
events.push(value);
|
|
1553
|
+
toDel.push(key);
|
|
1554
|
+
}
|
|
1555
|
+
}
|
|
1556
|
+
// 删除已消费
|
|
1557
|
+
const ops = toDel.map(k => ({ type: 'del', key: k }));
|
|
1558
|
+
if (ops.length) await this.db.batch(ops);
|
|
1559
|
+
} catch { }
|
|
1560
|
+
return events;
|
|
1561
|
+
}
|
|
1562
|
+
|
|
1563
|
+
return events;
|
|
1564
|
+
}
|
|
1565
|
+
|
|
1566
|
+
// 枚举所有分区 ID(FS 模式)
|
|
1567
|
+
async listPartitionIds() {
|
|
1568
|
+
if (this.backend === 'fs') {
|
|
1569
|
+
const files = fs.readdirSync(this.baseDir).filter(f => /^p_\d+\.jsonl$/.test(f));
|
|
1570
|
+
const ids = files.map(f => Number(f.match(/^p_(\d+)\.jsonl$/)[1])).sort((a, b) => a - b);
|
|
1571
|
+
return ids;
|
|
1572
|
+
}
|
|
1573
|
+
// LMDB/level 不易列举,约定 0..N-1 尝试加载
|
|
1574
|
+
return [];
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
1577
|
+
|
|
1578
|
+
// 分区器(哈希 -> 分区ID)
|
|
1579
|
+
class GraphPartitioner {
|
|
1580
|
+
constructor({ partitions = 64 } = {}) {
|
|
1581
|
+
this.partitions = Math.max(4, partitions);
|
|
1582
|
+
}
|
|
1583
|
+
idOf(pointID) {
|
|
1584
|
+
if (!pointID) return 0;
|
|
1585
|
+
const h = crypto.createHash('sha1').update(String(pointID)).digest();
|
|
1586
|
+
// 使用前 4 字节构造 uint32
|
|
1587
|
+
const u32 = h.readUInt32BE(0);
|
|
1588
|
+
return u32 % this.partitions;
|
|
1589
|
+
}
|
|
1590
|
+
neighborsOf(pid, radius = 1) {
|
|
1591
|
+
const out = new Set([pid]);
|
|
1592
|
+
for (let r = 1; r <= radius; r++) {
|
|
1593
|
+
out.add((pid - r + this.partitions) % this.partitions);
|
|
1594
|
+
out.add((pid + r) % this.partitions);
|
|
1595
|
+
}
|
|
1596
|
+
return Array.from(out).sort((a, b) => a - b);
|
|
1597
|
+
}
|
|
1598
|
+
}
|
|
1599
|
+
// 分区图 + 滑动窗口 + 边界事件消费
|
|
1600
|
+
class PartitionedGraphDB {
|
|
1601
|
+
constructor({
|
|
1602
|
+
partitions = 64,
|
|
1603
|
+
maxLoadedPartitions = 8,
|
|
1604
|
+
windowRadius = 1,
|
|
1605
|
+
baseDir = path.join(__dirname, 'graph_parts'),
|
|
1606
|
+
backend = 'fs'
|
|
1607
|
+
} = {}) {
|
|
1608
|
+
this.partitioner = new GraphPartitioner({ partitions });
|
|
1609
|
+
this.adapter = new GraphStorageAdapter({ baseDir, backend });
|
|
1610
|
+
this.maxLoadedPartitions = Math.max(2, maxLoadedPartitions);
|
|
1611
|
+
this.windowRadius = Math.max(0, windowRadius);
|
|
1612
|
+
|
|
1613
|
+
// 已加载分区:pid -> { points: Map, dirty, lastAccess }
|
|
1614
|
+
this.loaded = new Map();
|
|
1615
|
+
// 兼容旧代码:合并视图(仅包含已加载分区的点)
|
|
1616
|
+
this.points = new Map();
|
|
1617
|
+
// LRU
|
|
1618
|
+
this.accessTick = 0;
|
|
1619
|
+
this.centerPid = null;
|
|
1620
|
+
|
|
1621
|
+
// 并发保护
|
|
1622
|
+
this.loading = new Set();
|
|
1623
|
+
}
|
|
1624
|
+
|
|
1625
|
+
// ---------- 内部:加载/保存/淘汰 ----------
|
|
1626
|
+
async ensureLoaded(pid) {
|
|
1627
|
+
if (this.loaded.has(pid)) {
|
|
1628
|
+
this._touch(pid);
|
|
1629
|
+
return this.loaded.get(pid);
|
|
1630
|
+
}
|
|
1631
|
+
if (this.loading.has(pid)) {
|
|
1632
|
+
// 等待已有加载完成
|
|
1633
|
+
while (this.loading.has(pid)) { await sleep(10); }
|
|
1634
|
+
return this.loaded.get(pid);
|
|
1635
|
+
}
|
|
1636
|
+
this.loading.add(pid);
|
|
1637
|
+
try {
|
|
1638
|
+
const part = await this.adapter.loadPartition(pid);
|
|
1639
|
+
const bundle = {
|
|
1640
|
+
points: part.points || new Map(),
|
|
1641
|
+
dirty: false,
|
|
1642
|
+
lastAccess: ++this.accessTick
|
|
1643
|
+
};
|
|
1644
|
+
this.loaded.set(pid, bundle);
|
|
1645
|
+
// 合并到全局视图
|
|
1646
|
+
for (const [id, p] of bundle.points.entries()) this.points.set(id, p);
|
|
1647
|
+
|
|
1648
|
+
// 消费边界事件:把指向本分区的事件落库
|
|
1649
|
+
const events = await this.adapter.consumeEdgeEvents(pid, (e) =>
|
|
1650
|
+
e && e.type === 'cross-edge' && (e.toPid === pid || e.fromPid === pid), 5000);
|
|
1651
|
+
if (events.length) {
|
|
1652
|
+
for (const e of events) this._applyEdgeEvent(bundle, e);
|
|
1653
|
+
bundle.dirty = true;
|
|
1654
|
+
}
|
|
1655
|
+
|
|
1656
|
+
// 控制内存:若超容量,执行淘汰
|
|
1657
|
+
await this._evictIfNeeded();
|
|
1658
|
+
return bundle;
|
|
1659
|
+
} finally {
|
|
1660
|
+
this.loading.delete(pid);
|
|
1661
|
+
}
|
|
1662
|
+
}
|
|
1663
|
+
|
|
1664
|
+
async savePartitionIfDirty(pid) {
|
|
1665
|
+
const entry = this.loaded.get(pid);
|
|
1666
|
+
if (!entry) return;
|
|
1667
|
+
if (!entry.dirty) return;
|
|
1668
|
+
await this.adapter.savePartition(pid, entry.points);
|
|
1669
|
+
entry.dirty = false;
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
async _evictIfNeeded() {
|
|
1673
|
+
if (this.loaded.size <= this.maxLoadedPartitions) return;
|
|
1674
|
+
// 淘汰最近最少访问的分区(除中心窗口)
|
|
1675
|
+
const avoid = new Set(this.partitioner.neighborsOf(this.centerPid ?? 0, this.windowRadius));
|
|
1676
|
+
// 构建按 lastAccess 升序
|
|
1677
|
+
const list = Array.from(this.loaded.entries())
|
|
1678
|
+
.filter(([pid]) => !avoid.has(pid))
|
|
1679
|
+
.sort((a, b) => a[1].lastAccess - b[1].lastAccess);
|
|
1680
|
+
while (this.loaded.size > this.maxLoadedPartitions && list.length) {
|
|
1681
|
+
const [pid, entry] = list.shift();
|
|
1682
|
+
await this.savePartitionIfDirty(pid);
|
|
1683
|
+
// 从全局视图移除
|
|
1684
|
+
for (const [id] of entry.points.entries()) this.points.delete(id);
|
|
1685
|
+
this.loaded.delete(pid);
|
|
1686
|
+
logPart('evicted partition', pid);
|
|
1687
|
+
}
|
|
1688
|
+
}
|
|
1689
|
+
|
|
1690
|
+
_touch(pid) {
|
|
1691
|
+
const entry = this.loaded.get(pid);
|
|
1692
|
+
if (entry) entry.lastAccess = ++this.accessTick;
|
|
1693
|
+
}
|
|
1694
|
+
|
|
1695
|
+
_applyEdgeEvent(targetBundle, e) {
|
|
1696
|
+
// 事件格式:{ type:'cross-edge', from:'id', to:'id', weight, direction, fromPid, toPid }
|
|
1697
|
+
if (!e || e.type !== 'cross-edge') return;
|
|
1698
|
+
const ensurePoint = (m, id) => {
|
|
1699
|
+
if (!m.has(id)) m.set(id, { pointID: id, connect: [] });
|
|
1700
|
+
return m.get(id);
|
|
1701
|
+
};
|
|
1702
|
+
const mp = targetBundle.points;
|
|
1703
|
+
const pFrom = ensurePoint(mp, e.from);
|
|
1704
|
+
const pTo = ensurePoint(mp, e.to);
|
|
1705
|
+
// 在 from 中落边(若 from 属于本分区)
|
|
1706
|
+
if (e.toPid === e.fromPid) {
|
|
1707
|
+
// 同分区事件(理论上不会在事件日志里)
|
|
1708
|
+
if (!pFrom.connect.some(([w, id, d]) => id === e.to && d === e.direction)) {
|
|
1709
|
+
pFrom.connect.push([e.weight, e.to, e.direction]);
|
|
1710
|
+
}
|
|
1711
|
+
} else {
|
|
1712
|
+
// 当前 bundle 即为 toPid 或 fromPid 的载体
|
|
1713
|
+
if (e.toPid === this.partitioner.idOf(pTo.pointID)) {
|
|
1714
|
+
// 对于目标分区,至少要保证可被 selectPath 遍历;保留边终点即可(可选:反向提示边)
|
|
1715
|
+
// 不在 pTo 里写边(避免双写),仅保证 from 的边会在 from 分区生效
|
|
1716
|
+
}
|
|
1717
|
+
if (e.fromPid === this.partitioner.idOf(pFrom.pointID)) {
|
|
1718
|
+
if (!pFrom.connect.some(([w, id, d]) => id === e.to && d === e.direction)) {
|
|
1719
|
+
pFrom.connect.push([e.weight, e.to, e.direction]);
|
|
1720
|
+
}
|
|
1721
|
+
}
|
|
1722
|
+
}
|
|
1723
|
+
}
|
|
1724
|
+
|
|
1725
|
+
// ---------- 滑动窗口 ----------
|
|
1726
|
+
async focusOnPoint(pointID) {
|
|
1727
|
+
const pid = this.partitioner.idOf(pointID);
|
|
1728
|
+
this.centerPid = pid;
|
|
1729
|
+
const toLoad = this.partitioner.neighborsOf(pid, this.windowRadius);
|
|
1730
|
+
for (const id of toLoad) await this.ensureLoaded(id);
|
|
1731
|
+
await this._evictIfNeeded();
|
|
1732
|
+
}
|
|
1733
|
+
|
|
1734
|
+
// ---------- 兼容 API:点/边 操作 ----------
|
|
1735
|
+
addPoint(pointID, connect = []) {
|
|
1736
|
+
const pid = this.partitioner.idOf(pointID);
|
|
1737
|
+
const ensure = (bundle) => {
|
|
1738
|
+
if (!bundle.points.has(pointID)) bundle.points.set(pointID, { pointID, connect: [] });
|
|
1739
|
+
this.points.set(pointID, bundle.points.get(pointID));
|
|
1740
|
+
return bundle.points.get(pointID);
|
|
1741
|
+
};
|
|
1742
|
+
return this.ensureLoaded(pid).then(bundle => {
|
|
1743
|
+
const p = ensure(bundle);
|
|
1744
|
+
// 添加本地边;跨分区写事件
|
|
1745
|
+
for (const [w, nid, dir] of connect) this._addEdgeInternal(pid, p, w, nid, dir, bundle);
|
|
1746
|
+
bundle.dirty = true;
|
|
1747
|
+
});
|
|
1748
|
+
}
|
|
1749
|
+
|
|
1750
|
+
_addEdgeInternal(fromPid, fromPoint, weight, toID, direction, bundleOfFrom) {
|
|
1751
|
+
const toPid = this.partitioner.idOf(toID);
|
|
1752
|
+
const w = (typeof weight === 'number' && isFinite(weight)) ? weight : 1;
|
|
1753
|
+
const d = (direction === 0 || direction === 1 || direction === 2) ? direction : 0;
|
|
1754
|
+
|
|
1755
|
+
if (toPid === fromPid) {
|
|
1756
|
+
// 同分区直接写
|
|
1757
|
+
if (!fromPoint.connect.some(([ww, id, dd]) => id === toID && dd === d)) {
|
|
1758
|
+
fromPoint.connect.push([w, toID, d]);
|
|
1759
|
+
bundleOfFrom.dirty = true;
|
|
1760
|
+
}
|
|
1761
|
+
} else {
|
|
1762
|
+
// 跨分区 -> 记录边界事件至 fromPid(或 toPid 都可,这里记录到 fromPid,toPid 加载时也会消费相关事件)
|
|
1763
|
+
this.adapter.appendEdgeEvent(fromPid, {
|
|
1764
|
+
type: 'cross-edge',
|
|
1765
|
+
from: fromPoint.pointID,
|
|
1766
|
+
to: toID,
|
|
1767
|
+
weight: w,
|
|
1768
|
+
direction: d,
|
|
1769
|
+
fromPid,
|
|
1770
|
+
toPid
|
|
1771
|
+
});
|
|
1772
|
+
// 同时对“已加载且包含 toPid 的 bundle”进行即时应用(若存在)
|
|
1773
|
+
const toBundle = this.loaded.get(toPid);
|
|
1774
|
+
if (toBundle) {
|
|
1775
|
+
// 在 from 分区已经写入 from->to 事件;对于 to 分区无需写边(避免双写),可选择记录提示(此处略)
|
|
1776
|
+
}
|
|
1777
|
+
}
|
|
1778
|
+
}
|
|
1779
|
+
|
|
1780
|
+
addBidirectionalEdge(id1, id2, weight = 1) {
|
|
1781
|
+
return this.addEdge(id1, id2, weight, 0);
|
|
1782
|
+
}
|
|
1783
|
+
|
|
1784
|
+
async addEdge(fromID, toID, weight = 1, direction = 0) {
|
|
1785
|
+
const fromPid = this.partitioner.idOf(fromID);
|
|
1786
|
+
const fromBundle = await this.ensureLoaded(fromPid);
|
|
1787
|
+
if (!fromBundle.points.has(fromID)) {
|
|
1788
|
+
fromBundle.points.set(fromID, { pointID: fromID, connect: [] });
|
|
1789
|
+
this.points.set(fromID, fromBundle.points.get(fromID));
|
|
1790
|
+
}
|
|
1791
|
+
const fromPoint = fromBundle.points.get(fromID);
|
|
1792
|
+
this._addEdgeInternal(fromPid, fromPoint, weight, toID, direction, fromBundle);
|
|
1793
|
+
|
|
1794
|
+
if (direction === 0) {
|
|
1795
|
+
// 双向边:反向写入
|
|
1796
|
+
const toPid = this.partitioner.idOf(toID);
|
|
1797
|
+
const toBundle = await this.ensureLoaded(toPid);
|
|
1798
|
+
if (!toBundle.points.has(toID)) {
|
|
1799
|
+
toBundle.points.set(toID, { pointID: toID, connect: [] });
|
|
1800
|
+
this.points.set(toID, toBundle.points.get(toID));
|
|
1801
|
+
}
|
|
1802
|
+
const toPoint = toBundle.points.get(toID);
|
|
1803
|
+
this._addEdgeInternal(toPid, toPoint, weight, fromID, 0, toBundle);
|
|
1804
|
+
}
|
|
1805
|
+
}
|
|
1806
|
+
|
|
1807
|
+
async updateEdge(fromID, toID, newWeight, direction = 0) {
|
|
1808
|
+
const fromPid = this.partitioner.idOf(fromID);
|
|
1809
|
+
const b = await this.ensureLoaded(fromPid);
|
|
1810
|
+
const p = b.points.get(fromID);
|
|
1811
|
+
if (!p) return;
|
|
1812
|
+
const idx = p.connect.findIndex(([w, id, d]) => id === toID && d === direction);
|
|
1813
|
+
if (idx >= 0) {
|
|
1814
|
+
p.connect[idx][0] = newWeight;
|
|
1815
|
+
b.dirty = true;
|
|
1816
|
+
} else {
|
|
1817
|
+
// 不存在则添加
|
|
1818
|
+
this._addEdgeInternal(fromPid, p, newWeight, toID, direction, b);
|
|
1819
|
+
}
|
|
1820
|
+
}
|
|
762
1821
|
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
1822
|
+
existEdge(fromID, toID) {
|
|
1823
|
+
const fromPid = this.partitioner.idOf(fromID);
|
|
1824
|
+
const entry = this.loaded.get(fromPid);
|
|
1825
|
+
if (!entry) return { exist: false, weight: undefined, type: undefined };
|
|
1826
|
+
const p = entry.points.get(fromID);
|
|
1827
|
+
if (!p) return { exist: false, weight: undefined, type: undefined };
|
|
1828
|
+
const found = p.connect.find(([w, id]) => id === toID);
|
|
1829
|
+
return { exist: !!found, weight: found ? found[0] : undefined, type: found ? found[2] : undefined };
|
|
1830
|
+
}
|
|
769
1831
|
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
1832
|
+
existPoint(pointID) {
|
|
1833
|
+
// 仅检查已加载窗口
|
|
1834
|
+
const p = this.points.get(pointID);
|
|
1835
|
+
return { exist: !!p, connect: p ? p.connect : [] };
|
|
1836
|
+
}
|
|
1837
|
+
|
|
1838
|
+
deleteEdge(a, b) {
|
|
1839
|
+
const pid = this.partitioner.idOf(a);
|
|
1840
|
+
const entry = this.loaded.get(pid);
|
|
1841
|
+
if (!entry) return;
|
|
1842
|
+
const p = entry.points.get(a);
|
|
1843
|
+
if (!p) return;
|
|
1844
|
+
const before = p.connect.length;
|
|
1845
|
+
p.connect = p.connect.filter(([_, id]) => id !== b);
|
|
1846
|
+
entry.dirty = entry.dirty || (p.connect.length !== before);
|
|
1847
|
+
}
|
|
1848
|
+
|
|
1849
|
+
deletePoint(pointID) {
|
|
1850
|
+
const pid = this.partitioner.idOf(pointID);
|
|
1851
|
+
const entry = this.loaded.get(pid);
|
|
1852
|
+
if (!entry) return;
|
|
1853
|
+
if (entry.points.has(pointID)) {
|
|
1854
|
+
entry.points.delete(pointID);
|
|
1855
|
+
this.points.delete(pointID);
|
|
1856
|
+
entry.dirty = true;
|
|
774
1857
|
}
|
|
775
|
-
return null;
|
|
776
1858
|
}
|
|
777
1859
|
|
|
778
|
-
//
|
|
1860
|
+
// 仅遍历窗口内点(兼容旧 getAllPoints 调用)
|
|
779
1861
|
getAllPoints() {
|
|
780
1862
|
return Array.from(this.points.values());
|
|
781
1863
|
}
|
|
782
1864
|
|
|
783
|
-
//
|
|
784
|
-
|
|
785
|
-
|
|
1865
|
+
// 导出全量点(跨所有分区),用于快照/发布
|
|
1866
|
+
async exportAllPoints() {
|
|
1867
|
+
const out = [];
|
|
1868
|
+
// 尝试枚举 FS 分区;其他后端可按 0..N-1 遍历或仅导出已加载窗口
|
|
1869
|
+
const ids = await this.adapter.listPartitionIds();
|
|
1870
|
+
if (ids.length === 0) {
|
|
1871
|
+
// 回退:导出窗口
|
|
1872
|
+
return this.getAllPoints();
|
|
1873
|
+
}
|
|
1874
|
+
for (const pid of ids) {
|
|
1875
|
+
const part = await this.adapter.loadPartition(pid);
|
|
1876
|
+
for (const [, p] of part.points.entries()) out.push({ pointID: p.pointID, connect: p.connect || [] });
|
|
1877
|
+
}
|
|
1878
|
+
return out;
|
|
786
1879
|
}
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
1880
|
+
|
|
1881
|
+
// 批量导入(将 legacy 点集落到分区)
|
|
1882
|
+
async importAllPoints(pointsArr) {
|
|
1883
|
+
if (!Array.isArray(pointsArr)) return;
|
|
1884
|
+
// 分桶
|
|
1885
|
+
const buckets = new Map();
|
|
1886
|
+
for (const p of pointsArr) {
|
|
1887
|
+
const pid = this.partitioner.idOf(p.pointID);
|
|
1888
|
+
if (!buckets.has(pid)) buckets.set(pid, new Map());
|
|
1889
|
+
const bm = buckets.get(pid);
|
|
1890
|
+
bm.set(p.pointID, { pointID: p.pointID, connect: Array.isArray(p.connect) ? p.connect.slice() : [] });
|
|
1891
|
+
}
|
|
1892
|
+
// 写入并更新窗口视图(懒加载)
|
|
1893
|
+
for (const [pid, map] of buckets.entries()) {
|
|
1894
|
+
await this.adapter.savePartition(pid, map);
|
|
1895
|
+
// 若已加载该分区,刷新内存镜像
|
|
1896
|
+
if (this.loaded.has(pid)) {
|
|
1897
|
+
const entry = this.loaded.get(pid);
|
|
1898
|
+
// 从全局视图移除旧
|
|
1899
|
+
for (const [id] of entry.points.entries()) this.points.delete(id);
|
|
1900
|
+
entry.points = map;
|
|
1901
|
+
entry.dirty = false;
|
|
1902
|
+
entry.lastAccess = ++this.accessTick;
|
|
1903
|
+
for (const [id, p] of map.entries()) this.points.set(id, p);
|
|
1904
|
+
}
|
|
791
1905
|
}
|
|
792
|
-
const connectArr = point.connect || [];
|
|
793
|
-
const found = connectArr.find(([_, id]) => id === neighborID);
|
|
794
|
-
return {
|
|
795
|
-
exist: connectArr.some(([_, id]) => id === neighborID),
|
|
796
|
-
weight: found ? found[0] : undefined,
|
|
797
|
-
type: found ? found[2] : undefined
|
|
798
|
-
};
|
|
799
1906
|
}
|
|
800
|
-
|
|
801
|
-
|
|
1907
|
+
|
|
1908
|
+
// 聚合邻居(窗口内),供传播使用
|
|
1909
|
+
getNeighbors(pointID, maxNeighbors = 50) {
|
|
1910
|
+
const p = this.points.get(pointID);
|
|
1911
|
+
if (!p) return [];
|
|
1912
|
+
return p.connect.slice(0, maxNeighbors);
|
|
802
1913
|
}
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
1914
|
+
|
|
1915
|
+
// A* 简化:仅在窗口内搜索;跳出窗口时,尝试预取邻接分区后再继续
|
|
1916
|
+
async selectPath(fromID, toID) {
|
|
1917
|
+
if (fromID === toID) return [fromID];
|
|
1918
|
+
// 优先保证焦点加载
|
|
1919
|
+
await this.focusOnPoint(fromID);
|
|
1920
|
+
|
|
1921
|
+
const reconstruct = (came, cur) => {
|
|
1922
|
+
const path = [];
|
|
1923
|
+
let t = cur;
|
|
1924
|
+
while (came.has(t)) { path.push(t); t = came.get(t); }
|
|
1925
|
+
path.push(fromID);
|
|
1926
|
+
return path.reverse();
|
|
1927
|
+
};
|
|
1928
|
+
|
|
1929
|
+
const open = new Set([fromID]);
|
|
1930
|
+
const came = new Map();
|
|
1931
|
+
const g = new Map([[fromID, 0]]);
|
|
1932
|
+
const f = new Map([[fromID, 1]]);
|
|
1933
|
+
const closed = new Set();
|
|
1934
|
+
|
|
1935
|
+
const heuristic = () => 1;
|
|
1936
|
+
let iter = 0;
|
|
1937
|
+
const MAX_ITERS = 5000;
|
|
1938
|
+
|
|
1939
|
+
while (open.size && iter++ < MAX_ITERS) {
|
|
1940
|
+
// 取 f 最小
|
|
1941
|
+
let cur = null; let minF = Infinity;
|
|
1942
|
+
for (const id of open) {
|
|
1943
|
+
const val = f.get(id) ?? Infinity;
|
|
1944
|
+
if (val < minF) { minF = val; cur = id; }
|
|
1945
|
+
}
|
|
1946
|
+
if (cur == null) break;
|
|
1947
|
+
if (cur === toID) return reconstruct(came, cur);
|
|
1948
|
+
|
|
1949
|
+
open.delete(cur);
|
|
1950
|
+
closed.add(cur);
|
|
1951
|
+
|
|
1952
|
+
// 若遇到未知点,尝试加载其分区(滑动窗口)
|
|
1953
|
+
if (!this.points.has(cur)) {
|
|
1954
|
+
await this.focusOnPoint(cur);
|
|
1955
|
+
}
|
|
1956
|
+
|
|
1957
|
+
const neighbors = this.getNeighbors(cur, 50);
|
|
1958
|
+
// 如果邻居为空,尝试边界事件预取(根据邻居 ID 的分区预取)
|
|
1959
|
+
if (neighbors.length === 0) {
|
|
1960
|
+
const pid = this.partitioner.idOf(cur);
|
|
1961
|
+
const ring = this.partitioner.neighborsOf(pid, 1);
|
|
1962
|
+
for (const rid of ring) await this.ensureLoaded(rid);
|
|
1963
|
+
}
|
|
1964
|
+
|
|
1965
|
+
for (const [w, nb] of neighbors) {
|
|
1966
|
+
if (closed.has(nb)) continue;
|
|
1967
|
+
const tentative = (g.get(cur) || Infinity) + w;
|
|
1968
|
+
if (!open.has(nb)) open.add(nb);
|
|
1969
|
+
else if (tentative >= (g.get(nb) || Infinity)) continue;
|
|
1970
|
+
|
|
1971
|
+
came.set(nb, cur);
|
|
1972
|
+
g.set(nb, tentative);
|
|
1973
|
+
f.set(nb, tentative + heuristic());
|
|
1974
|
+
}
|
|
807
1975
|
}
|
|
1976
|
+
return null;
|
|
808
1977
|
}
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
1978
|
+
|
|
1979
|
+
// 刷盘所有已加载分区
|
|
1980
|
+
async flushAll() {
|
|
1981
|
+
for (const [pid] of this.loaded.entries()) await this.savePartitionIfDirty(pid);
|
|
813
1982
|
}
|
|
814
1983
|
}
|
|
815
|
-
|
|
816
1984
|
class KVM {
|
|
817
1985
|
constructor() {
|
|
818
1986
|
this.memory = new Map();
|
|
@@ -910,14 +2078,22 @@ class Runtime {
|
|
|
910
2078
|
// 运行时负责AI核心的调度、模因转换、信号传递与主流程控制
|
|
911
2079
|
constructor(config = {}) {
|
|
912
2080
|
this.config = config;
|
|
913
|
-
|
|
2081
|
+
// 使用分区图作为模因图;词图仍用内存图
|
|
2082
|
+
this.graph = new PartitionedGraphDB({
|
|
2083
|
+
partitions: this.config.partitions || 64,
|
|
2084
|
+
maxLoadedPartitions: this.config.maxLoadedPartitions || 8,
|
|
2085
|
+
windowRadius: this.config.windowRadius || 1,
|
|
2086
|
+
baseDir: path.join(__dirname, 'graph_parts'),
|
|
2087
|
+
backend: this.config.graphBackend || 'lmdb' // 可选 'fs' | 'lmdb' | 'level'
|
|
2088
|
+
});
|
|
914
2089
|
this.wordGraph = new GraphDB();
|
|
915
2090
|
this.kvm = new KVM();
|
|
916
2091
|
|
|
917
2092
|
this.transformer = null;
|
|
918
2093
|
this.vocabManager = global.vocabmanager;
|
|
919
2094
|
this.spider = new Spider();
|
|
920
|
-
|
|
2095
|
+
// 新:按需检索器(基于全局爬虫)
|
|
2096
|
+
this.researcher = new OnlineResearcher(this);
|
|
921
2097
|
// 新:以“会话”为尺度的访问日志与会话管理
|
|
922
2098
|
this.session = new SessionManager({
|
|
923
2099
|
idleMs: this.config.sessionIdleMs || 10 * 60 * 1000,
|
|
@@ -925,7 +2101,7 @@ class Runtime {
|
|
|
925
2101
|
});
|
|
926
2102
|
// Map<word, Map<sessionId, count>>
|
|
927
2103
|
this.wordAccessLog = new Map();
|
|
928
|
-
|
|
2104
|
+
this.config.spiderMix = this.config.spiderMix || { onlineWeight: 0.5, offlineWeight: 0.5 };
|
|
929
2105
|
this.initWordGraph();
|
|
930
2106
|
this.forgetTimer = setInterval(() => this.forgetWords(), 350 * 1000);
|
|
931
2107
|
this.MAX_MEME_WORDS = 100;
|
|
@@ -939,12 +2115,95 @@ class Runtime {
|
|
|
939
2115
|
batchSizeMultiplier: 1
|
|
940
2116
|
};
|
|
941
2117
|
this.memeBarrier = new memeBarrier(this);
|
|
2118
|
+
this._act = BuiltinActivations.relu;
|
|
2119
|
+
this._transfer = BuiltinTransfers.linear;
|
|
2120
|
+
this._activationMeta = { activationType: 'relu', transferType: 'linear' };
|
|
2121
|
+
}
|
|
2122
|
+
// 获取/设置激活-传递函数配置
|
|
2123
|
+
getActivationConfig() {
|
|
2124
|
+
return {
|
|
2125
|
+
activationType: this._activationMeta.activationType,
|
|
2126
|
+
transferType: this._activationMeta.transferType,
|
|
2127
|
+
activationCustom: this.config?.activationCustom || '',
|
|
2128
|
+
transferCustom: this.config?.transferCustom || ''
|
|
2129
|
+
};
|
|
2130
|
+
}
|
|
2131
|
+
|
|
2132
|
+
setActivationConfig({ activationType, transferType, activationCustom, transferCustom } = {}) {
|
|
2133
|
+
const aType = String(activationType || this._activationMeta.activationType || 'relu');
|
|
2134
|
+
const tType = String(transferType || this._activationMeta.transferType || 'linear');
|
|
2135
|
+
|
|
2136
|
+
let act = BuiltinActivations[aType] || BuiltinActivations.relu;
|
|
2137
|
+
let tr = BuiltinTransfers[tType] || BuiltinTransfers.linear;
|
|
2138
|
+
|
|
2139
|
+
if (aType === 'custom' && activationCustom) {
|
|
2140
|
+
act = compileCustomFunctionSafely(activationCustom, ['x'], BuiltinActivations.relu);
|
|
2141
|
+
}
|
|
2142
|
+
if (tType === 'custom' && transferCustom) {
|
|
2143
|
+
tr = compileCustomFunctionSafely(transferCustom, ['value', 'weight', 'decayK', 'ctx'], BuiltinTransfers.linear);
|
|
2144
|
+
}
|
|
2145
|
+
|
|
2146
|
+
this._act = (typeof act === 'function') ? act : BuiltinActivations.relu;
|
|
2147
|
+
this._transfer = (typeof tr === 'function') ? tr : BuiltinTransfers.linear;
|
|
2148
|
+
this._activationMeta = { activationType: aType, transferType: tType };
|
|
2149
|
+
this.config = this.config || {};
|
|
2150
|
+
this.config.activationCustom = activationCustom || this.config.activationCustom || '';
|
|
2151
|
+
this.config.transferCustom = transferCustom || this.config.transferCustom || '';
|
|
2152
|
+
}
|
|
2153
|
+
// 新增:应用可调参数(含 spiderMix / decayK / maxLen 等)
|
|
2154
|
+
applyTunableParams(partial = {}) {
|
|
2155
|
+
this.config = this.config || {};
|
|
2156
|
+
if (partial.spiderMix) {
|
|
2157
|
+
const ow = Math.max(0, Math.min(1, Number(partial.spiderMix.onlineWeight ?? this.config.spiderMix.onlineWeight ?? 0.5)));
|
|
2158
|
+
this.config.spiderMix = { onlineWeight: ow, offlineWeight: Math.max(0, Math.min(1, 1 - ow)) };
|
|
2159
|
+
}
|
|
2160
|
+
if (typeof partial.decayK === 'number') this.config.decayK = Math.max(0.1, Math.min(2.0, partial.decayK));
|
|
2161
|
+
if (typeof partial.maxLen === 'number') this.config.maxLen = Math.max(8, Math.min(64, Math.round(partial.maxLen)));
|
|
2162
|
+
if (typeof partial.edgeWeight === 'number') {
|
|
2163
|
+
for (const p of this.graph.getAllPoints()) for (const e of p.connect) e[0] = Math.max(0.1, Math.min(5, partial.edgeWeight));
|
|
2164
|
+
}
|
|
2165
|
+
// 可选:调节 crawler 抓取强度(若存在)
|
|
2166
|
+
if (global.__crawler) {
|
|
2167
|
+
if (typeof partial.perQuery === 'number') global.__crawler.__tune_perQuery = Math.max(2, Math.min(16, Math.round(partial.perQuery)));
|
|
2168
|
+
if (typeof partial.maxCrawl === 'number') global.__crawler.__tune_maxCrawl = Math.max(2, Math.min(24, Math.round(partial.maxCrawl)));
|
|
2169
|
+
}
|
|
2170
|
+
return {
|
|
2171
|
+
decayK: this.config.decayK,
|
|
2172
|
+
maxLen: this.config.maxLen,
|
|
2173
|
+
spiderMix: this.config.spiderMix,
|
|
2174
|
+
crawler: {
|
|
2175
|
+
perQuery: global.__crawler?.__tune_perQuery ?? 8,
|
|
2176
|
+
maxCrawl: global.__crawler?.__tune_maxCrawl ?? 12
|
|
2177
|
+
}
|
|
2178
|
+
};
|
|
942
2179
|
}
|
|
943
2180
|
// 添加到Runtime类内部
|
|
944
2181
|
filterStopWords(words) {
|
|
945
2182
|
return words.filter(word => !STOP_WORDS.includes(word.toLowerCase()));
|
|
946
2183
|
}
|
|
947
|
-
|
|
2184
|
+
async ingestTextDocument(raw, { addNewWords = true, minLen = 8 } = {}) {
|
|
2185
|
+
if (!raw) return 0;
|
|
2186
|
+
// 去除 meta,正文在空行后
|
|
2187
|
+
const parts = String(raw).split(/\r?\n\r?\n/);
|
|
2188
|
+
const body = parts.length > 1 ? parts.slice(1).join('\n') : parts[0];
|
|
2189
|
+
const sentences = body.split(/\r?\n+/).map(s => s.trim()).filter(Boolean);
|
|
2190
|
+
let fed = 0;
|
|
2191
|
+
for (const line of sentences) {
|
|
2192
|
+
// 分词 -> 归一化 -> 停用词过滤 -> processInput
|
|
2193
|
+
const words = line
|
|
2194
|
+
.toLowerCase()
|
|
2195
|
+
.replace(/[^a-z\s\u4e00-\u9fa5]/g, ' ')
|
|
2196
|
+
.split(/\s+/)
|
|
2197
|
+
.filter(w => w.length >= 2);
|
|
2198
|
+
if (!words.length) continue;
|
|
2199
|
+
const normalized = this.spider ? this.spider.lemmatizeWords(words) : words;
|
|
2200
|
+
const filtered = this.filterStopWords ? this.filterStopWords(normalized) : normalized;
|
|
2201
|
+
if (filtered.length < minLen) continue;
|
|
2202
|
+
this.processInput(filtered, { addNewWords });
|
|
2203
|
+
fed++;
|
|
2204
|
+
}
|
|
2205
|
+
return fed;
|
|
2206
|
+
}
|
|
948
2207
|
// 新增资源监控方法
|
|
949
2208
|
monitorSystemLoad() {
|
|
950
2209
|
const now = Date.now();
|
|
@@ -979,12 +2238,12 @@ class Runtime {
|
|
|
979
2238
|
return this.systemLoad.batchSizeMultiplier;
|
|
980
2239
|
}
|
|
981
2240
|
// 清理定时器
|
|
2241
|
+
// 清理定时器/刷盘
|
|
982
2242
|
cleanup() {
|
|
983
|
-
if (this.forgetTimer)
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
this.memeBarrier.stop();
|
|
2243
|
+
if (this.forgetTimer) clearInterval(this.forgetTimer);
|
|
2244
|
+
if (this.memeBarrier) this.memeBarrier.stop();
|
|
2245
|
+
if (this.graph && this.graph.flushAll) {
|
|
2246
|
+
this.graph.flushAll().catch(() => { });
|
|
988
2247
|
}
|
|
989
2248
|
}
|
|
990
2249
|
// Runtime类中添加监控函数
|
|
@@ -993,7 +2252,7 @@ class Runtime {
|
|
|
993
2252
|
for (const meme of memes) {
|
|
994
2253
|
const words = this.kvm.get(meme.pointID) || [];
|
|
995
2254
|
if (words.length > this.MAX_MEME_WORDS * 0.8) { // 如果接近最大限制
|
|
996
|
-
|
|
2255
|
+
// console.log(`[MONITOR] 检测到大模因: ${meme.pointID}, 词数: ${words.length}`);
|
|
997
2256
|
this.splitMemeIfNeeded(meme.pointID); // 尝试分裂
|
|
998
2257
|
}
|
|
999
2258
|
}
|
|
@@ -1062,14 +2321,14 @@ class Runtime {
|
|
|
1062
2321
|
// 不自动销毁
|
|
1063
2322
|
}
|
|
1064
2323
|
|
|
1065
|
-
dispose() {
|
|
2324
|
+
dispose() {
|
|
1066
2325
|
this.graph.points.clear();
|
|
1067
2326
|
this.wordGraph.points.clear();
|
|
1068
2327
|
this.kvm.memory.clear();
|
|
1069
2328
|
if (this.wordAccessLog) this.wordAccessLog.clear();
|
|
1070
2329
|
if (this.forgetTimer) clearInterval(this.forgetTimer);
|
|
1071
2330
|
}
|
|
1072
|
-
|
|
2331
|
+
// 将遗忘策略改为“最近N个会话窗口”
|
|
1073
2332
|
forgetWords() {
|
|
1074
2333
|
// 保护:收集所有被KVM引用的词
|
|
1075
2334
|
const protectedWords = new Set();
|
|
@@ -1203,19 +2462,15 @@ dispose() {
|
|
|
1203
2462
|
visitCount++;
|
|
1204
2463
|
activatedOrder.push(id);
|
|
1205
2464
|
|
|
1206
|
-
// 仅在是“词”时记录访问,避免把模因ID写入词访问日志
|
|
1207
2465
|
if (this.wordGraph.points.has(id)) {
|
|
1208
2466
|
this.logWordAccess(id);
|
|
1209
2467
|
}
|
|
1210
2468
|
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
if (!visited.has(neighborID)) {
|
|
1217
|
-
next.push({ id: neighborID, value: value - decayK * weight });
|
|
1218
|
-
}
|
|
2469
|
+
// 改为通过 graph.getNeighbors 访问(窗口内)
|
|
2470
|
+
const neighbors = this.graph.getNeighbors(id, 50);
|
|
2471
|
+
for (const [weight, neighborID] of neighbors) {
|
|
2472
|
+
if (!visited.has(neighborID)) {
|
|
2473
|
+
next.push({ id: neighborID, value: value - decayK * weight });
|
|
1219
2474
|
}
|
|
1220
2475
|
}
|
|
1221
2476
|
}
|
|
@@ -1270,7 +2525,11 @@ dispose() {
|
|
|
1270
2525
|
processInput(wordsArr, { addNewWords = true } = {}) {
|
|
1271
2526
|
wordsArr = this.filterStopWords(wordsArr);
|
|
1272
2527
|
if (wordsArr.length === 0) { console.log('[FILTER] 输入全为停用词,已全部过滤'); return; }
|
|
1273
|
-
|
|
2528
|
+
// console.log('Processing input:', wordsArr);
|
|
2529
|
+
// 异步触发在线检索(不阻塞)
|
|
2530
|
+
if (triggerResearch && this.researcher) {
|
|
2531
|
+
try { this.researcher.scheduleFromWords(wordsArr); } catch (_) { }
|
|
2532
|
+
}
|
|
1274
2533
|
// 批量处理新词添加
|
|
1275
2534
|
if (addNewWords) {
|
|
1276
2535
|
// 一次性检查哪些词不在词表中
|
|
@@ -1356,7 +2615,7 @@ dispose() {
|
|
|
1356
2615
|
const overlap = wordsArr.filter(w => memeWords.includes(w)).length;
|
|
1357
2616
|
if (overlap >= this.MIN_OVERLAP && memeWords.length + wordsArr.length <= this.MAX_MEME_WORDS) {
|
|
1358
2617
|
this.kvm.set(minMemeID, Array.from(new Set([...memeWords, ...wordsArr])));
|
|
1359
|
-
|
|
2618
|
+
/// console.log(`Merged to existing meme: ${minMemeID}`);
|
|
1360
2619
|
} else {
|
|
1361
2620
|
// 创建新模因,使用有向连接
|
|
1362
2621
|
const newID = 'meme_' + Date.now();
|
|
@@ -1366,9 +2625,9 @@ dispose() {
|
|
|
1366
2625
|
// 单向连接到最近的模因 (方向:2表示指向对方)
|
|
1367
2626
|
if (minMemeID) {
|
|
1368
2627
|
this.graph.addDirectionalEdge(newID, minMemeID, minDistance, 2);
|
|
1369
|
-
|
|
2628
|
+
// console.log(`[LINK] 新模因 ${newID} 单向连接到最近模因 ${minMemeID}`);
|
|
1370
2629
|
}
|
|
1371
|
-
|
|
2630
|
+
// console.log(`Created new meme: ${newID}`);
|
|
1372
2631
|
}
|
|
1373
2632
|
} else {
|
|
1374
2633
|
// 创建新模因
|
|
@@ -1379,9 +2638,9 @@ dispose() {
|
|
|
1379
2638
|
// 如果有较近的模因,仍然创建单向连接
|
|
1380
2639
|
if (minMemeID) {
|
|
1381
2640
|
this.graph.addDirectionalEdge(newID, minMemeID, Math.min(minDistance, 5), 2);
|
|
1382
|
-
|
|
2641
|
+
// console.log(`[LINK] 新模因 ${newID} 单向连接到最近模因 ${minMemeID}`);
|
|
1383
2642
|
}
|
|
1384
|
-
|
|
2643
|
+
// console.log(`Created new meme: ${newID}`);
|
|
1385
2644
|
}
|
|
1386
2645
|
}
|
|
1387
2646
|
// 新增批量添加边的辅助方法
|
|
@@ -1430,59 +2689,58 @@ dispose() {
|
|
|
1430
2689
|
const maxActiveNodes = options.maxActiveNodes || 5000;
|
|
1431
2690
|
const minSignal = options.minSignal || 0.01;
|
|
1432
2691
|
const trackPath = options.trackPath || false;
|
|
2692
|
+
const directionalMultiplier = options.directionalMultiplier || 0.7;
|
|
2693
|
+
const bidirectionalMultiplier = options.bidirectionalMultiplier || 1.2;
|
|
2694
|
+
|
|
2695
|
+
const actFn = this._act || BuiltinActivations.relu;
|
|
2696
|
+
const transferFn = this._transfer || BuiltinTransfers.linear;
|
|
1433
2697
|
|
|
1434
|
-
// 节点信号累加表
|
|
1435
2698
|
const signalMap = new Map();
|
|
1436
|
-
// 路径追踪表(可选)
|
|
1437
2699
|
const activationPaths = trackPath ? new Map() : null;
|
|
2700
|
+
const activationTypes = trackPath ? new Map() : null;
|
|
1438
2701
|
|
|
1439
|
-
// 初始化活跃队列,每个元素{id, value, from}
|
|
1440
2702
|
let active = startIDs.map((id, i) => ({
|
|
1441
|
-
id,
|
|
1442
|
-
value: strengths[i],
|
|
1443
|
-
from: null // 起点无前驱
|
|
2703
|
+
id, value: strengths[i], from: null, connectionType: -1
|
|
1444
2704
|
}));
|
|
1445
|
-
|
|
1446
2705
|
let step = 0;
|
|
1447
2706
|
|
|
1448
2707
|
while (active.length > 0 && step < maxStep) {
|
|
1449
|
-
// 1. 限制活跃节点数,优先保留信号强的
|
|
1450
2708
|
if (active.length > maxActiveNodes) {
|
|
1451
2709
|
active.sort((a, b) => b.value - a.value);
|
|
1452
2710
|
active = active.slice(0, maxActiveNodes);
|
|
1453
|
-
console.log(`[LIMIT] 多源扩散活跃节点数已限制为 ${maxActiveNodes}`);
|
|
1454
2711
|
}
|
|
1455
2712
|
|
|
1456
2713
|
const next = [];
|
|
1457
|
-
for (const { id, value, from } of active) {
|
|
2714
|
+
for (const { id, value, from, connectionType } of active) {
|
|
1458
2715
|
if (value < minSignal) continue;
|
|
1459
2716
|
|
|
1460
|
-
//
|
|
1461
|
-
|
|
2717
|
+
// 节点处应用激活函数(融合累计)
|
|
2718
|
+
const prev = signalMap.get(id) || 0;
|
|
2719
|
+
const merged = actFn(prev + value);
|
|
2720
|
+
signalMap.set(id, merged);
|
|
1462
2721
|
|
|
1463
|
-
|
|
1464
|
-
|
|
2722
|
+
if (trackPath && connectionType !== -1) {
|
|
2723
|
+
if (!activationTypes.has(id)) activationTypes.set(id, new Set());
|
|
2724
|
+
activationTypes.get(id).add(connectionType);
|
|
2725
|
+
}
|
|
2726
|
+
if (trackPath && from) {
|
|
1465
2727
|
if (!activationPaths.has(id)) activationPaths.set(id, []);
|
|
1466
|
-
|
|
2728
|
+
activationPaths.get(id).push({ from, connectionType, value });
|
|
1467
2729
|
}
|
|
1468
2730
|
|
|
1469
|
-
// 传播到邻居(不防环,允许信号多次叠加)
|
|
1470
2731
|
const point = this.graph.points.get(id);
|
|
1471
|
-
if (point)
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
from: id
|
|
1484
|
-
});
|
|
1485
|
-
}
|
|
2732
|
+
if (!point) continue;
|
|
2733
|
+
|
|
2734
|
+
const MAX_NEIGHBORS = 30;
|
|
2735
|
+
const neighbors = point.connect.slice(0, MAX_NEIGHBORS);
|
|
2736
|
+
|
|
2737
|
+
for (const [weight, neighborID, direction = 0] of neighbors) {
|
|
2738
|
+
const ctx = { direction, directionalMultiplier, bidirectionalMultiplier };
|
|
2739
|
+
const rawNext = transferFn(value, weight, decayK, ctx);
|
|
2740
|
+
const nextValue = actFn(rawNext);
|
|
2741
|
+
|
|
2742
|
+
if (nextValue >= minSignal) {
|
|
2743
|
+
next.push({ id: neighborID, value: nextValue, from: id, connectionType: direction });
|
|
1486
2744
|
}
|
|
1487
2745
|
}
|
|
1488
2746
|
}
|
|
@@ -1491,7 +2749,7 @@ dispose() {
|
|
|
1491
2749
|
}
|
|
1492
2750
|
|
|
1493
2751
|
if (trackPath) {
|
|
1494
|
-
return { signalMap, activationPaths };
|
|
2752
|
+
return { signalMap, activationPaths, activationTypes };
|
|
1495
2753
|
}
|
|
1496
2754
|
return signalMap;
|
|
1497
2755
|
}
|
|
@@ -1665,7 +2923,7 @@ dispose() {
|
|
|
1665
2923
|
this.kvm.memory.delete(memeB.pointID);
|
|
1666
2924
|
memesToDelete.add(memeB.pointID);
|
|
1667
2925
|
|
|
1668
|
-
|
|
2926
|
+
// console.log(`Merged memes: ${memeA.pointID} <- ${memeB.pointID}`);
|
|
1669
2927
|
// 合并后立即尝试分裂
|
|
1670
2928
|
this.splitMemeIfNeeded(memeA.pointID);
|
|
1671
2929
|
} else {
|
|
@@ -1682,7 +2940,7 @@ dispose() {
|
|
|
1682
2940
|
// 如果没有双向边,则添加双向边
|
|
1683
2941
|
if (!(existAtoB.exist && existAtoB.type === 0) && !(existBtoA.exist && existBtoA.type === 0)) {
|
|
1684
2942
|
this.graph.addBidirectionalEdge(memeA.pointID, memeB.pointID, avgDist);
|
|
1685
|
-
|
|
2943
|
+
// console.log(`[LINK] 添加双向边: ${memeA.pointID} <-> ${memeB.pointID} (avgDist=${avgDist})`);
|
|
1686
2944
|
}
|
|
1687
2945
|
}
|
|
1688
2946
|
}
|
|
@@ -1713,14 +2971,14 @@ dispose() {
|
|
|
1713
2971
|
const newID = newIDs[i];
|
|
1714
2972
|
this.graph.addPoint(newID, []);
|
|
1715
2973
|
this.kvm.set(newID, chunk);
|
|
1716
|
-
|
|
2974
|
+
// console.log(`[SPLIT-FORCE] 新建模因: ${newID} 词数: ${chunk.length}`);
|
|
1717
2975
|
}
|
|
1718
2976
|
}
|
|
1719
2977
|
|
|
1720
2978
|
// 删除原模因
|
|
1721
2979
|
this.graph.points.delete(memeID);
|
|
1722
2980
|
this.kvm.memory.delete(memeID);
|
|
1723
|
-
|
|
2981
|
+
// console.log(`[SPLIT-FORCE] 删除原模因: ${memeID}`);
|
|
1724
2982
|
return;
|
|
1725
2983
|
}
|
|
1726
2984
|
|
|
@@ -1768,12 +3026,12 @@ dispose() {
|
|
|
1768
3026
|
const newID = 'meme_' + Date.now() + '_' + Math.floor(Math.random() * 10000);
|
|
1769
3027
|
this.graph.addPoint(newID, []);
|
|
1770
3028
|
this.kvm.set(newID, comp);
|
|
1771
|
-
|
|
3029
|
+
// console.log(`[SPLIT] 新建模因: ${newID} 词数: ${comp.length}`);
|
|
1772
3030
|
}
|
|
1773
3031
|
// 删除原节点
|
|
1774
3032
|
this.graph.points.delete(memeID);
|
|
1775
3033
|
this.kvm.memory.delete(memeID);
|
|
1776
|
-
|
|
3034
|
+
// console.log(`[SPLIT] 删除原模因: ${memeID}`);
|
|
1777
3035
|
}
|
|
1778
3036
|
}
|
|
1779
3037
|
}
|
|
@@ -1793,6 +3051,7 @@ class AssociationLayer {
|
|
|
1793
3051
|
constructor(runtime) {
|
|
1794
3052
|
this.runtime = runtime;
|
|
1795
3053
|
this.patterns = []; // 存储发现的模因边关系模式
|
|
3054
|
+
this._prevClone = null; // 差量克隆基线
|
|
1796
3055
|
}
|
|
1797
3056
|
|
|
1798
3057
|
// 识别模因层的边关系模式
|
|
@@ -1849,83 +3108,33 @@ class AssociationLayer {
|
|
|
1849
3108
|
// ...前面的代码...
|
|
1850
3109
|
|
|
1851
3110
|
// 创建系统副本
|
|
3111
|
+
// 创建系统副本(差量克隆)
|
|
1852
3112
|
async cloneSystem() {
|
|
1853
|
-
console.log('[CLONE]
|
|
1854
|
-
const clone = new Runtime();
|
|
1855
|
-
clone.registerClone();
|
|
1856
|
-
|
|
3113
|
+
console.log('[CLONE] 差量克隆系统(递归分区哈希)');
|
|
1857
3114
|
try {
|
|
1858
|
-
|
|
1859
|
-
clone.
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
3115
|
+
const clone = await deltaCloneRuntime(this._prevClone, this.runtime);
|
|
3116
|
+
clone.registerClone();
|
|
3117
|
+
this._prevClone = clone;
|
|
3118
|
+
return clone;
|
|
3119
|
+
} catch (err) {
|
|
3120
|
+
console.warn('[CLONE] 差量克隆失败,回退全量:', err.message);
|
|
3121
|
+
const fallback = new Runtime();
|
|
3122
|
+
fallback.registerClone();
|
|
3123
|
+
fallback.spider = this.runtime.spider;
|
|
3124
|
+
fallback.vocabManager.vocab = [...this.runtime.vocabManager.vocab];
|
|
3125
|
+
fallback.vocabManager.updateMappings();
|
|
1867
3126
|
for (const [key, value] of this.runtime.wordGraph.points.entries()) {
|
|
1868
|
-
|
|
1869
|
-
value.connect.map(conn => [...conn]) : []);
|
|
3127
|
+
fallback.wordGraph.addPoint(key, Array.isArray(value.connect) ? value.connect.map(conn => [...conn]) : []);
|
|
1870
3128
|
}
|
|
1871
|
-
|
|
1872
|
-
// 4. 复制模因网络
|
|
1873
|
-
console.log('[CLONE] 开始复制模因网络...');
|
|
1874
3129
|
for (const [key, value] of this.runtime.graph.points.entries()) {
|
|
1875
|
-
|
|
1876
|
-
value.connect.map(conn => [...conn]) : []);
|
|
3130
|
+
fallback.graph.addPoint(key, Array.isArray(value.connect) ? value.connect.map(conn => [...conn]) : []);
|
|
1877
3131
|
}
|
|
1878
|
-
|
|
1879
|
-
// 5. 复制KVM - 确保一致性和类型
|
|
1880
|
-
console.log('[CLONE] 开始复制KVM...');
|
|
1881
|
-
let nonEmptyCount = 0;
|
|
1882
|
-
let totalWordCount = 0;
|
|
1883
|
-
|
|
1884
3132
|
for (const [key, value] of this.runtime.kvm.memory.entries()) {
|
|
1885
|
-
|
|
1886
|
-
if (Array.isArray(value)) {
|
|
1887
|
-
// 确保数组中每个元素都是字符串且归一化
|
|
1888
|
-
const normalizedWords = value.map(word =>
|
|
1889
|
-
typeof word === 'string' ? word.toLowerCase().trim() : String(word)
|
|
1890
|
-
);
|
|
1891
|
-
|
|
1892
|
-
// 应用词形归一化
|
|
1893
|
-
const lemmatizedWords = clone.spider.lemmatizeWords(normalizedWords);
|
|
1894
|
-
clone.kvm.set(key, lemmatizedWords);
|
|
1895
|
-
|
|
1896
|
-
if (lemmatizedWords.length > 0) {
|
|
1897
|
-
nonEmptyCount++;
|
|
1898
|
-
totalWordCount += lemmatizedWords.length;
|
|
1899
|
-
}
|
|
1900
|
-
} else if (value != null) {
|
|
1901
|
-
// 非数组值转换为单元素数组
|
|
1902
|
-
const singleWord = String(value).toLowerCase().trim();
|
|
1903
|
-
const lemmatizedWord = clone.spider.lemmatize(singleWord);
|
|
1904
|
-
clone.kvm.set(key, [lemmatizedWord]);
|
|
1905
|
-
|
|
1906
|
-
nonEmptyCount++;
|
|
1907
|
-
totalWordCount++;
|
|
1908
|
-
} else {
|
|
1909
|
-
// null或undefined情况,设为空数组
|
|
1910
|
-
clone.kvm.set(key, []);
|
|
1911
|
-
}
|
|
3133
|
+
fallback.kvm.set(key, Array.isArray(value) ? [...value] : (value == null ? [] : [String(value)]));
|
|
1912
3134
|
}
|
|
1913
|
-
|
|
1914
|
-
console.log(`[CLONE] KVM复制完成: ${nonEmptyCount}个非空模因,${totalWordCount}个词语`);
|
|
1915
|
-
|
|
1916
|
-
// 6. 复制其他配置和参数
|
|
1917
|
-
console.log('[CLONE] 开始复制词表和其他属性...');
|
|
1918
|
-
clone.MAX_MEME_WORDS = this.runtime.MAX_MEME_WORDS;
|
|
1919
|
-
clone.MIN_OVERLAP = this.runtime.MIN_OVERLAP;
|
|
1920
|
-
clone.config = { ...this.runtime.config };
|
|
1921
|
-
|
|
1922
|
-
console.log('[CLONE] 系统副本创建完成');
|
|
1923
|
-
return clone;
|
|
1924
|
-
} catch (error) {
|
|
1925
|
-
console.error('[CLONE ERROR]', error);
|
|
1926
|
-
return clone;
|
|
3135
|
+
return fallback;
|
|
1927
3136
|
}
|
|
1928
|
-
}
|
|
3137
|
+
}
|
|
1929
3138
|
|
|
1930
3139
|
applyPatternsToClone(systemClone) {
|
|
1931
3140
|
console.log('[CLONE] 应用关系模式到副本');
|
|
@@ -1982,7 +3191,7 @@ class AssociationLayer {
|
|
|
1982
3191
|
meme.connect[connIdx][0] = newWeight;
|
|
1983
3192
|
meme.connect[connIdx][2] = direction;
|
|
1984
3193
|
|
|
1985
|
-
|
|
3194
|
+
// console.log(`[CLONE] 修改边权重: ${meme.pointID}->${meme.connect[connIdx][1]}, ${oldWeight}->${newWeight.toFixed(2)}, 保留方向: ${direction}`);
|
|
1986
3195
|
modified++;
|
|
1987
3196
|
}
|
|
1988
3197
|
}
|
|
@@ -2697,7 +3906,7 @@ class controller {
|
|
|
2697
3906
|
const sid = this.runtime.session.ensureActive();
|
|
2698
3907
|
this.runtime.session.incMessage(sid);
|
|
2699
3908
|
const words = text.toLowerCase().split(' ').filter(w => w.length > 0);
|
|
2700
|
-
this.runtime.processInput(words, { addNewWords: false });
|
|
3909
|
+
this.runtime.processInput(words, { addNewWords: false, triggerResearch: true });
|
|
2701
3910
|
return await this.runtime.generateResponseWithMemes(words);
|
|
2702
3911
|
}
|
|
2703
3912
|
// 启动自主学习
|
|
@@ -2799,6 +4008,27 @@ setInterval(() => {
|
|
|
2799
4008
|
saveQueued = false;
|
|
2800
4009
|
}
|
|
2801
4010
|
}, 10000); // 每10秒最多写盘一次
|
|
4011
|
+
setInterval(async () => {
|
|
4012
|
+
try {
|
|
4013
|
+
if (!global.__crawler || !global.ctrlA) return;
|
|
4014
|
+
const mix = global.ctrlA.runtime.config.spiderMix || { onlineWeight: 0.5, offlineWeight: 0.5 };
|
|
4015
|
+
// 以 1 - onlineWeight 的概率跳过本轮(控制强度)
|
|
4016
|
+
if (Math.random() > (mix.onlineWeight || 0.5)) return;
|
|
4017
|
+
|
|
4018
|
+
const docs = global.__crawler.loadRecentDocs(12);
|
|
4019
|
+
if (!docs.length) return;
|
|
4020
|
+
let fed = 0;
|
|
4021
|
+
for (const d of docs) {
|
|
4022
|
+
fed += await global.ctrlA.runtime.ingestTextDocument(d.text, { addNewWords: true, minLen: 6 });
|
|
4023
|
+
}
|
|
4024
|
+
if (fed > 0) {
|
|
4025
|
+
console.log(`[INGEST] 在线文档本轮投喂 ${fed} 段 (mix=${mix.onlineWeight.toFixed(2)})`);
|
|
4026
|
+
global.ctrlA.runtime.updateAttentionLinks();
|
|
4027
|
+
}
|
|
4028
|
+
} catch (e) {
|
|
4029
|
+
console.warn('[INGEST] 失败:', e.message);
|
|
4030
|
+
}
|
|
4031
|
+
}, 20_000);
|
|
2802
4032
|
// 从硬盘恢复
|
|
2803
4033
|
// 从硬盘恢复
|
|
2804
4034
|
function loadAll(runtime) {
|
|
@@ -2845,138 +4075,16 @@ function loadAll(runtime) {
|
|
|
2845
4075
|
|
|
2846
4076
|
|
|
2847
4077
|
function scheduleCrossLearning() {
|
|
2848
|
-
const
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
|
|
2853
|
-
|
|
2854
|
-
|
|
2855
|
-
global.ctrlA.startSelfLearning(3).then(() => {
|
|
2856
|
-
setTimeout(async () => {
|
|
2857
|
-
console.log('[CROSS] A将成果传递给B');
|
|
2858
|
-
const associationLayer = new AssociationLayer(global.ctrlA.runtime);
|
|
2859
|
-
|
|
2860
|
-
// 归一化一致性测试
|
|
2861
|
-
const testWord = "testing";
|
|
2862
|
-
const normalizedWord = global.ctrlA.runtime.spider.lemmatize(testWord);
|
|
2863
|
-
console.log(`[CROSS-CHECK] 词归一化测试: "${testWord}" -> "${normalizedWord}"`);
|
|
2864
|
-
//const associationLayer = new AssociationLayer(global.ctrlA.runtime);
|
|
2865
|
-
// 这里要加上
|
|
2866
|
-
associationLayer.patterns = associationLayer.identifyPatterns();
|
|
2867
|
-
// 克隆
|
|
2868
|
-
const systemClone = await associationLayer.cloneSystem();
|
|
2869
|
-
systemClone.spider = global.ctrlA.runtime.spider;
|
|
2870
|
-
associationLayer.applyPatternsToClone(systemClone);
|
|
2871
|
-
|
|
2872
|
-
verifySystemConsistency(global.ctrlA.runtime, systemClone);
|
|
2873
|
-
|
|
2874
|
-
// 预热克隆系统
|
|
2875
|
-
console.log('[CROSS] 预热克隆系统...');
|
|
2876
|
-
const sampleWords = Array.from(
|
|
2877
|
-
new Set([...systemClone.vocabManager.vocab].slice(4, 20))
|
|
2878
|
-
).filter(w => w.length > 1);
|
|
2879
|
-
await systemClone.processInput(sampleWords, { addNewWords: false });
|
|
2880
|
-
|
|
2881
|
-
await systemClone.startSelfLearning(3);
|
|
2882
|
-
global.ctrlB.updateRuntime(systemClone);
|
|
2883
|
-
|
|
2884
|
-
verifySystemConsistency(global.ctrlA.runtime, global.ctrlB.runtime);
|
|
2885
|
-
|
|
2886
|
-
console.log('[CROSS] 已将A的系统更新到B');
|
|
2887
|
-
testCloneMatching(global.ctrlB.runtime);
|
|
2888
|
-
}, learnTime);
|
|
2889
|
-
});
|
|
2890
|
-
}, cycle);
|
|
2891
|
-
|
|
2892
|
-
// B -> C
|
|
2893
|
-
registerInterval(() => {
|
|
2894
|
-
if (global.ctrlA.isLearning || global.ctrlB.isLearning || global.ctrlC.isLearning || isShuttingDown) return;
|
|
2895
|
-
global.ctrlB.startSelfLearning(3).then(() => {
|
|
2896
|
-
setTimeout(async () => {
|
|
2897
|
-
console.log('[CROSS] B将成果传递给C');
|
|
2898
|
-
const associationLayer = new AssociationLayer(global.ctrlB.runtime);
|
|
2899
|
-
|
|
2900
|
-
const testWord = "testing";
|
|
2901
|
-
const normalizedWord = global.ctrlB.runtime.spider.lemmatize(testWord);
|
|
2902
|
-
console.log(`[CROSS-CHECK] 词归一化测试: "${testWord}" -> "${normalizedWord}"`);
|
|
2903
|
-
// 这里要加上
|
|
2904
|
-
associationLayer.patterns = associationLayer.identifyPatterns();
|
|
2905
|
-
const systemClone = await associationLayer.cloneSystem();
|
|
2906
|
-
systemClone.spider = global.ctrlB.runtime.spider;
|
|
2907
|
-
associationLayer.applyPatternsToClone(systemClone);
|
|
2908
|
-
|
|
2909
|
-
verifySystemConsistency(global.ctrlB.runtime, systemClone);
|
|
2910
|
-
|
|
2911
|
-
console.log('[CROSS] 预热克隆系统...');
|
|
2912
|
-
const sampleWords = Array.from(
|
|
2913
|
-
new Set([...systemClone.vocabManager.vocab].slice(4, 20))
|
|
2914
|
-
).filter(w => w.length > 1);
|
|
2915
|
-
await systemClone.processInput(sampleWords, { addNewWords: false });
|
|
2916
|
-
|
|
2917
|
-
await systemClone.startSelfLearning(3);
|
|
2918
|
-
global.ctrlC.updateRuntime(systemClone);
|
|
2919
|
-
|
|
2920
|
-
verifySystemConsistency(global.ctrlB.runtime, global.ctrlC.runtime);
|
|
2921
|
-
|
|
2922
|
-
console.log('[CROSS] 已将B的系统更新到C');
|
|
2923
|
-
testCloneMatching(global.ctrlC.runtime);
|
|
2924
|
-
}, learnTime);
|
|
2925
|
-
});
|
|
2926
|
-
}, cycle);
|
|
2927
|
-
|
|
2928
|
-
// C -> A
|
|
2929
|
-
registerInterval(() => {
|
|
2930
|
-
if (global.ctrlA.isLearning || global.ctrlB.isLearning || global.ctrlC.isLearning || isShuttingDown) return;
|
|
2931
|
-
global.ctrlC.startSelfLearning(3).then(() => {
|
|
2932
|
-
setTimeout(async () => {
|
|
2933
|
-
console.log('[CROSS] C将成果传递给A');
|
|
2934
|
-
const associationLayer = new AssociationLayer(global.ctrlC.runtime);
|
|
2935
|
-
|
|
2936
|
-
const testWord = "testing";
|
|
2937
|
-
const normalizedWord = global.ctrlC.runtime.spider.lemmatize(testWord);
|
|
2938
|
-
console.log(`[CROSS-CHECK] 词归一化测试: "${testWord}" -> "${normalizedWord}"`);
|
|
2939
|
-
// 这里要加上
|
|
2940
|
-
associationLayer.patterns = associationLayer.identifyPatterns();
|
|
2941
|
-
const systemClone = await associationLayer.cloneSystem();
|
|
2942
|
-
systemClone.spider = global.ctrlC.runtime.spider;
|
|
2943
|
-
associationLayer.applyPatternsToClone(systemClone);
|
|
2944
|
-
|
|
2945
|
-
verifySystemConsistency(global.ctrlC.runtime, systemClone);
|
|
2946
|
-
|
|
2947
|
-
console.log('[CROSS] 预热克隆系统...');
|
|
2948
|
-
const sampleWords = Array.from(
|
|
2949
|
-
new Set([...systemClone.vocabManager.vocab].slice(4, 20))
|
|
2950
|
-
).filter(w => w.length > 1);
|
|
2951
|
-
await systemClone.processInput(sampleWords, { addNewWords: false });
|
|
2952
|
-
|
|
2953
|
-
await systemClone.startSelfLearning(3);
|
|
2954
|
-
global.ctrlA.updateRuntime(systemClone);
|
|
2955
|
-
|
|
2956
|
-
verifySystemConsistency(global.ctrlC.runtime, global.ctrlA.runtime);
|
|
2957
|
-
|
|
2958
|
-
console.log('[CROSS] 已将C的系统更新到A');
|
|
2959
|
-
testCloneMatching(global.ctrlA.runtime);
|
|
2960
|
-
}, learnTime);
|
|
2961
|
-
});
|
|
2962
|
-
console.log('Publishing runtime state to Redis...');
|
|
2963
|
-
if (!RuntimeMessage) return; // protobuf未加载完成
|
|
2964
|
-
if (!redisClient || !redisClient.isOpen) {
|
|
2965
|
-
console.warn('[REDIS] 客户端未连接,跳过发布');
|
|
2966
|
-
return;
|
|
2967
|
-
}
|
|
2968
|
-
if (!RuntimeMessage) return;
|
|
2969
|
-
if (global.ctrlA.runtime.isLearning || global.ctrlA.runtime.isMainLoopRunning) return;
|
|
2970
|
-
const plainObj = runtimeToPlain(global.ctrlA.runtime);
|
|
2971
|
-
const errMsg = RuntimeMessage.verify(plainObj);
|
|
2972
|
-
if (errMsg) throw Error(errMsg);
|
|
2973
|
-
const message = RuntimeMessage.create(plainObj);
|
|
2974
|
-
const buffer = RuntimeMessage.encode(message).finish();
|
|
2975
|
-
redisClient.publish(`AI-model-${__dirname}`, buffer);
|
|
2976
|
-
console.log('已发布运行时状态到Redis');
|
|
2977
|
-
}, cycle);
|
|
4078
|
+
const rot = new RotationManager(global.ctrlA, global.ctrlB, global.ctrlC, {
|
|
4079
|
+
cycleMs: 15 * 60 * 1000, // 可根据需要调整
|
|
4080
|
+
cooldownMs: 60 * 1000,
|
|
4081
|
+
learnIters: 3,
|
|
4082
|
+
minImprove: 0.005
|
|
4083
|
+
});
|
|
4084
|
+
rot.start();
|
|
2978
4085
|
}
|
|
2979
4086
|
|
|
4087
|
+
|
|
2980
4088
|
// 新增:测试克隆系统的词汇匹配能力
|
|
2981
4089
|
function testCloneMatching(runtime) {
|
|
2982
4090
|
// 从词表中随机选取10个词
|
|
@@ -3115,7 +4223,7 @@ function optimizeMemory() {
|
|
|
3115
4223
|
}
|
|
3116
4224
|
}
|
|
3117
4225
|
async function main() {
|
|
3118
|
-
|
|
4226
|
+
console.log('Starting AI system...');
|
|
3119
4227
|
redisClient = redis.createClient();
|
|
3120
4228
|
// 创建三个全局控制器副本
|
|
3121
4229
|
const ctrlA = new controller();
|
|
@@ -3130,11 +4238,38 @@ async function main() {
|
|
|
3130
4238
|
loadAll(ctrlA.runtime);
|
|
3131
4239
|
loadAll(ctrlB.runtime);
|
|
3132
4240
|
loadAll(ctrlC.runtime);
|
|
3133
|
-
|
|
4241
|
+
const crawler = new CrawlerManager({
|
|
4242
|
+
concurrency: 5,
|
|
4243
|
+
perHostDelayMs: 2000,
|
|
4244
|
+
requestTimeoutMs: 12000,
|
|
4245
|
+
allowLang: ['en', 'zh'],
|
|
4246
|
+
seedsFile: path.join(__dirname, 'robots', 'seeds.txt'),
|
|
4247
|
+
proxiesFile: path.join(__dirname, 'crawler', 'proxies.txt')
|
|
4248
|
+
});
|
|
4249
|
+
global.__crawler = crawler;
|
|
4250
|
+
crawler.start();
|
|
4251
|
+
setInterval(async () => {
|
|
4252
|
+
try {
|
|
4253
|
+
const docs = crawler.loadRecentDocs(12);
|
|
4254
|
+
if (!docs.length) return;
|
|
4255
|
+
let fed = 0;
|
|
4256
|
+
for (const d of docs) {
|
|
4257
|
+
fed += await ctrlA.runtime.ingestTextDocument(d.text, { addNewWords: true, minLen: 6 });
|
|
4258
|
+
}
|
|
4259
|
+
if (fed > 0) {
|
|
4260
|
+
console.log(`[INGEST] 在线文档本轮投喂 ${fed} 段`);
|
|
4261
|
+
// 适当刷新注意力连接
|
|
4262
|
+
ctrlA.runtime.updateAttentionLinks();
|
|
4263
|
+
}
|
|
4264
|
+
} catch (e) {
|
|
4265
|
+
console.warn('[INGEST] 失败:', e.message);
|
|
4266
|
+
}
|
|
4267
|
+
}, 20_000); // 每20秒消费一批
|
|
3134
4268
|
// 用A副本初始化语料和模因
|
|
3135
4269
|
console.time('articleProcessing');
|
|
3136
4270
|
const articles = ctrlA.runtime.buildVocabFromSpider();
|
|
3137
4271
|
console.log(`Spider: 加载文章数: ${articles.length}`);
|
|
4272
|
+
// 周期性消费在线文档 -> 投喂到 ctrlA.runtime
|
|
3138
4273
|
|
|
3139
4274
|
// 修复:在首次使用前定义 lemmaCsvPath
|
|
3140
4275
|
const BATCH_SIZE = 20;
|
|
@@ -3187,30 +4322,177 @@ async function main() {
|
|
|
3187
4322
|
//每12分钟尝试启动memebarrier
|
|
3188
4323
|
ctrlA.runtime.memeBarrier.start();
|
|
3189
4324
|
}, 1000 * 60 * 12);
|
|
3190
|
-
// API路由 - 只做学习,不返回结果
|
|
3191
|
-
app.post('/api/chat', async (req, res) => {
|
|
3192
|
-
try {
|
|
3193
|
-
const { message, sessionId } = req.body || {};
|
|
3194
|
-
// 支持从Header透传会话
|
|
3195
|
-
const headerSid = req.headers['x-session-id'];
|
|
3196
|
-
const sid = sessionId || headerSid || global.ctrlA.runtime.session.ensureActive();
|
|
3197
4325
|
|
|
3198
|
-
// 使用/续接会话并计数
|
|
3199
|
-
global.ctrlA.runtime.session.useSession(sid);
|
|
3200
|
-
global.ctrlA.runtime.session.incMessage(sid);
|
|
3201
4326
|
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
|
|
4327
|
+
// 可选:启动(默认不开启,避免未配置API Key)
|
|
4328
|
+
if (String(process.env.ADV_AUTOSTART || '').toLowerCase() === 'true') {
|
|
4329
|
+
adv.start();
|
|
4330
|
+
}
|
|
4331
|
+
// 新增:serve 侧参数调优 API(默认不启用自动调参,仅手动设置)
|
|
4332
|
+
app.get('/api/tune/get', (req, res) => {
|
|
4333
|
+
try {
|
|
4334
|
+
const rt = global.ctrlA?.runtime;
|
|
4335
|
+
if (!rt) return res.status(500).json({ ok: false, error: 'runtime missing' });
|
|
4336
|
+
res.json({
|
|
4337
|
+
ok: true,
|
|
4338
|
+
params: {
|
|
4339
|
+
decayK: rt.config?.decayK ?? 1,
|
|
4340
|
+
maxLen: rt.config?.maxLen ?? 16,
|
|
4341
|
+
spiderMix: rt.config?.spiderMix ?? { onlineWeight: 0.5, offlineWeight: 0.5 },
|
|
4342
|
+
crawler: {
|
|
4343
|
+
perQuery: global.__crawler?.__tune_perQuery ?? 8,
|
|
4344
|
+
maxCrawl: global.__crawler?.__tune_maxCrawl ?? 12
|
|
4345
|
+
}
|
|
4346
|
+
}
|
|
4347
|
+
});
|
|
4348
|
+
} catch (e) {
|
|
4349
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
4350
|
+
}
|
|
4351
|
+
});
|
|
4352
|
+
|
|
4353
|
+
app.post('/api/tune/set', (req, res) => {
|
|
4354
|
+
try {
|
|
4355
|
+
const rt = global.ctrlA?.runtime;
|
|
4356
|
+
if (!rt) return res.status(500).json({ ok: false, error: 'runtime missing' });
|
|
4357
|
+
const snap = applyServeTunableParams(rt, req.body || {});
|
|
4358
|
+
res.json({ ok: true, snapshot: snap });
|
|
4359
|
+
} catch (e) {
|
|
4360
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
4361
|
+
}
|
|
4362
|
+
});
|
|
4363
|
+
// API
|
|
4364
|
+
// 初始化对抗学习调度器时传递 promptMode/targetWeights 可选参数
|
|
4365
|
+
// ...existing code...
|
|
4366
|
+
const adv = new AdversaryScheduler(ctrlA.runtime, {
|
|
4367
|
+
providerSpec: process.env.ADV_MODEL || 'ollama:llama3.1:405b',
|
|
4368
|
+
judgeMode: process.env.ADV_JUDGE || 'llm',
|
|
4369
|
+
intervalMs: Number(process.env.ADV_INTERVAL || 60_000),
|
|
4370
|
+
batchSize: Number(process.env.ADV_BATCH || 3),
|
|
4371
|
+
promptMode: process.env.ADV_PROMPT_MODE || 'mixed',
|
|
4372
|
+
targetWeights: {
|
|
4373
|
+
decayK: Number(process.env.TUNE_W_DECAYK || 1.0),
|
|
4374
|
+
maxLen: Number(process.env.TUNE_W_MAXLEN || 0.7),
|
|
4375
|
+
onlineWeight: Number(process.env.TUNE_W_ONLINE || 0.8),
|
|
4376
|
+
edgeWeight: Number(process.env.TUNE_W_EDGE || 0.4),
|
|
4377
|
+
perQuery: Number(process.env.TUNE_W_PERQ || 0.5),
|
|
4378
|
+
maxCrawl: Number(process.env.TUNE_W_MAXC || 0.5),
|
|
4379
|
+
}
|
|
4380
|
+
});
|
|
4381
|
+
// ...existing code...
|
|
4382
|
+
global.__adversary = adv;
|
|
3205
4383
|
|
|
3206
|
-
|
|
3207
|
-
|
|
3208
|
-
|
|
3209
|
-
|
|
3210
|
-
|
|
4384
|
+
// 对抗学习控制 API 增补 promptMode / targets
|
|
4385
|
+
app.post('/api/adversary/start', (req, res) => {
|
|
4386
|
+
try {
|
|
4387
|
+
const { provider, judgeMode, intervalMs, batchSize, promptMode, targetWeights } = req.body || {};
|
|
4388
|
+
if (provider) {
|
|
4389
|
+
const neo = new AdversaryScheduler(global.ctrlA.runtime, {
|
|
4390
|
+
providerSpec: provider,
|
|
4391
|
+
judgeMode: judgeMode || adv.opts.judgeMode,
|
|
4392
|
+
intervalMs: Number(intervalMs || adv.opts.intervalMs),
|
|
4393
|
+
batchSize: Number(batchSize || adv.opts.batchSize),
|
|
4394
|
+
promptMode: promptMode || adv.opts.promptMode,
|
|
4395
|
+
targetWeights: targetWeights || adv.opts.targetWeights
|
|
4396
|
+
});
|
|
4397
|
+
global.__adversary?.stop?.();
|
|
4398
|
+
global.__adversary = neo;
|
|
4399
|
+
global.__adversary.start();
|
|
4400
|
+
} else {
|
|
4401
|
+
if (promptMode) adv.setPromptMode(promptMode);
|
|
4402
|
+
if (targetWeights) adv.setTargets(targetWeights);
|
|
4403
|
+
adv.start();
|
|
4404
|
+
}
|
|
4405
|
+
res.json({ ok: true, status: global.__adversary.getStatus() });
|
|
4406
|
+
} catch (e) {
|
|
4407
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
3211
4408
|
}
|
|
3212
4409
|
});
|
|
3213
4410
|
|
|
4411
|
+
app.post('/api/adversary/stop', (req, res) => {
|
|
4412
|
+
try { global.__adversary?.stop?.(); res.json({ ok: true }); }
|
|
4413
|
+
catch (e) { res.status(500).json({ ok: false, error: e.message }); }
|
|
4414
|
+
});
|
|
4415
|
+
|
|
4416
|
+
app.get('/api/adversary/status', (req, res) => {
|
|
4417
|
+
try { res.json({ ok: true, status: global.__adversary?.getStatus?.() || { running: false } }); }
|
|
4418
|
+
catch (e) { res.status(500).json({ ok: false, error: e.message }); }
|
|
4419
|
+
});
|
|
4420
|
+
|
|
4421
|
+
// 触发一次性对抗评估(可指定 prompts 数组)
|
|
4422
|
+
app.post('/api/adversary/once', async (req, res) => {
|
|
4423
|
+
try {
|
|
4424
|
+
const prompts = Array.isArray(req.body?.prompts) ? req.body.prompts.slice(0, 5) : null;
|
|
4425
|
+
const report = await global.__adversary.evaluateOnce(prompts);
|
|
4426
|
+
res.json({ ok: true, report });
|
|
4427
|
+
} catch (e) {
|
|
4428
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
4429
|
+
}
|
|
4430
|
+
});
|
|
4431
|
+
app.get('/api/graph/partitions/status', async (req, res) => {
|
|
4432
|
+
try {
|
|
4433
|
+
const g = global.ctrlA?.runtime?.graph;
|
|
4434
|
+
if (!g || !(g instanceof PartitionedGraphDB)) {
|
|
4435
|
+
return res.json({ ok: true, mode: 'in-memory', loaded: 0 });
|
|
4436
|
+
}
|
|
4437
|
+
const loaded = Array.from(g.loaded.keys());
|
|
4438
|
+
res.json({
|
|
4439
|
+
ok: true,
|
|
4440
|
+
mode: 'partitioned',
|
|
4441
|
+
partitions: g.partitioner.partitions,
|
|
4442
|
+
loaded,
|
|
4443
|
+
maxLoaded: g.maxLoadedPartitions,
|
|
4444
|
+
windowRadius: g.windowRadius,
|
|
4445
|
+
centerPid: g.centerPid
|
|
4446
|
+
});
|
|
4447
|
+
} catch (e) {
|
|
4448
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
4449
|
+
}
|
|
4450
|
+
});
|
|
4451
|
+
|
|
4452
|
+
app.post('/api/graph/partitions/flush', async (req, res) => {
|
|
4453
|
+
try {
|
|
4454
|
+
const g = global.ctrlA?.runtime?.graph;
|
|
4455
|
+
if (g && g.flushAll) await g.flushAll();
|
|
4456
|
+
res.json({ ok: true });
|
|
4457
|
+
} catch (e) {
|
|
4458
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
4459
|
+
}
|
|
4460
|
+
});
|
|
4461
|
+
|
|
4462
|
+
app.post('/api/graph/prefetch', async (req, res) => {
|
|
4463
|
+
try {
|
|
4464
|
+
const { node } = req.body || {};
|
|
4465
|
+
const g = global.ctrlA?.runtime?.graph;
|
|
4466
|
+
if (!node || !(g instanceof PartitionedGraphDB)) return res.status(400).json({ ok: false, error: 'node 必填/或非分区图' });
|
|
4467
|
+
await g.focusOnPoint(String(node));
|
|
4468
|
+
res.json({ ok: true });
|
|
4469
|
+
} catch (e) {
|
|
4470
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
4471
|
+
}
|
|
4472
|
+
});
|
|
4473
|
+
// API路由 - 只做学习,不返回结果
|
|
4474
|
+
app.post('/api/chat', async (req, res) => {
|
|
4475
|
+
try {
|
|
4476
|
+
const { message, sessionId } = req.body || {};
|
|
4477
|
+
const headerSid = req.headers['x-session-id'];
|
|
4478
|
+
const sid = sessionId || headerSid || global.ctrlA.runtime.session.ensureActive();
|
|
4479
|
+
|
|
4480
|
+
global.ctrlA.runtime.session.useSession(sid);
|
|
4481
|
+
global.ctrlA.runtime.session.incMessage(sid);
|
|
4482
|
+
|
|
4483
|
+
const words = String(message || '').toLowerCase().split(/\s+/).filter(w => w.length > 0);
|
|
4484
|
+
// 开启按需检索
|
|
4485
|
+
global.ctrlA.runtime.processInput(words, { triggerResearch: true });
|
|
4486
|
+
global.ctrlA.runtime.updateAttentionLinks();
|
|
4487
|
+
|
|
4488
|
+
res.set('X-Session-Id', sid);
|
|
4489
|
+
res.status(204).end();
|
|
4490
|
+
} catch (error) {
|
|
4491
|
+
res.status(500).json({ error: error.message });
|
|
4492
|
+
console.error('Error in /api/chat:', error);
|
|
4493
|
+
}
|
|
4494
|
+
});
|
|
4495
|
+
|
|
3214
4496
|
app.get('/api/status', (req, res) => {
|
|
3215
4497
|
res.json({ status: 'running', timestamp: new Date().toISOString() });
|
|
3216
4498
|
});
|
|
@@ -3265,7 +4547,20 @@ app.post('/api/chat', async (req, res) => {
|
|
|
3265
4547
|
res.status(500).json({ success: false, error: error.message });
|
|
3266
4548
|
}
|
|
3267
4549
|
});
|
|
3268
|
-
|
|
4550
|
+
app.post('/api/crawler/start', (req, res) => {
|
|
4551
|
+
try { global.__crawler?.start(); res.json({ ok: true }); } catch (e) { res.status(500).json({ ok: false, error: e.message }); }
|
|
4552
|
+
});
|
|
4553
|
+
app.post('/api/crawler/stop', (req, res) => {
|
|
4554
|
+
try { global.__crawler?.stop(); res.json({ ok: true }); } catch (e) { res.status(500).json({ ok: false, error: e.message }); }
|
|
4555
|
+
});
|
|
4556
|
+
app.post('/api/crawler/seed', (req, res) => {
|
|
4557
|
+
const { urls } = req.body || {};
|
|
4558
|
+
const n = global.__crawler?.addSeeds(Array.isArray(urls) ? urls : []) || 0;
|
|
4559
|
+
res.json({ ok: true, added: n });
|
|
4560
|
+
});
|
|
4561
|
+
app.get('/api/crawler/status', (req, res) => {
|
|
4562
|
+
res.json({ ok: true, stats: global.__crawler?.stats() || {} });
|
|
4563
|
+
});
|
|
3269
4564
|
|
|
3270
4565
|
// 模型默认参数
|
|
3271
4566
|
const modelDefaults = {
|
|
@@ -3279,7 +4574,11 @@ app.post('/api/chat', async (req, res) => {
|
|
|
3279
4574
|
decay: 1, // 新增
|
|
3280
4575
|
decayK: 1, // 新增
|
|
3281
4576
|
maxLen: 16, // 新增
|
|
3282
|
-
edgeWeight: 1
|
|
4577
|
+
edgeWeight: 1, // 新增
|
|
4578
|
+
activationType: 'relu',
|
|
4579
|
+
transferType: 'linear',
|
|
4580
|
+
activationCustom: '',
|
|
4581
|
+
transferCustom: ''
|
|
3283
4582
|
};
|
|
3284
4583
|
const currentModelParams = { ...modelDefaults };
|
|
3285
4584
|
|
|
@@ -3318,6 +4617,49 @@ app.post('/api/chat', async (req, res) => {
|
|
|
3318
4617
|
console.log(`\nAI system running on port ${global.config.masterPortOfMain}`);
|
|
3319
4618
|
console.log(`API available at http://localhost:${global.config.masterPortOfMain}/api/`);
|
|
3320
4619
|
});
|
|
4620
|
+
app.post('/api/search-crawl', async (req, res) => {
|
|
4621
|
+
try {
|
|
4622
|
+
const { prompt, vertical, perQuery, maxEnqueue, crawl, maxCrawl, ingest = true, minLen = 6 } = req.body || {};
|
|
4623
|
+
if (!prompt || String(prompt).trim().length < 2) {
|
|
4624
|
+
return res.status(400).json({ ok: false, error: 'prompt 不能为空' });
|
|
4625
|
+
}
|
|
4626
|
+
if (!global.__crawler) {
|
|
4627
|
+
return res.status(500).json({ ok: false, error: 'crawler 未初始化' });
|
|
4628
|
+
}
|
|
4629
|
+
const report = await global.__crawler.directedSearch(String(prompt), {
|
|
4630
|
+
vertical: vertical || 'general',
|
|
4631
|
+
perQuery: perQuery || 8,
|
|
4632
|
+
maxEnqueue: maxEnqueue || 30,
|
|
4633
|
+
crawl: crawl !== false,
|
|
4634
|
+
maxCrawl: maxCrawl || 12
|
|
4635
|
+
});
|
|
4636
|
+
|
|
4637
|
+
let fed = 0;
|
|
4638
|
+
if (ingest !== false) {
|
|
4639
|
+
const docs = global.__crawler.loadRecentDocs(24);
|
|
4640
|
+
for (const d of docs) {
|
|
4641
|
+
fed += await global.ctrlA.runtime.ingestTextDocument(d.text, { addNewWords: true, minLen });
|
|
4642
|
+
}
|
|
4643
|
+
if (fed > 0) global.ctrlA.runtime.updateAttentionLinks();
|
|
4644
|
+
}
|
|
4645
|
+
|
|
4646
|
+
res.json({ ok: true, report, ingestedSegments: fed });
|
|
4647
|
+
} catch (e) {
|
|
4648
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
4649
|
+
}
|
|
4650
|
+
});
|
|
4651
|
+
|
|
4652
|
+
// 仅添加待抓取URL(不立即抓)
|
|
4653
|
+
app.post('/api/crawler/query', async (req, res) => {
|
|
4654
|
+
try {
|
|
4655
|
+
const { prompt, vertical } = req.body || {};
|
|
4656
|
+
if (!prompt) return res.status(400).json({ ok: false, error: 'prompt 不能为空' });
|
|
4657
|
+
const report = await global.__crawler.directedSearch(String(prompt), { vertical: vertical || 'general', crawl: false });
|
|
4658
|
+
res.json({ ok: true, report });
|
|
4659
|
+
} catch (e) {
|
|
4660
|
+
res.status(500).json({ ok: false, error: e.message });
|
|
4661
|
+
}
|
|
4662
|
+
});
|
|
3321
4663
|
// 启动Redis客户端
|
|
3322
4664
|
redisClient.on('error', (err) => {
|
|
3323
4665
|
console.error('Redis error:', err);
|
|
@@ -3336,10 +4678,10 @@ app.post('/api/chat', async (req, res) => {
|
|
|
3336
4678
|
console.log('已设置交错自主学习定时任务,每200s执行一次');
|
|
3337
4679
|
}
|
|
3338
4680
|
|
|
3339
|
-
// 将参数应用到运行时
|
|
3340
|
-
// 扩展 applyModelParams
|
|
3341
4681
|
function applyModelParams(runtime) {
|
|
3342
4682
|
if (!runtime) return;
|
|
4683
|
+
|
|
4684
|
+
// 同步通用参数
|
|
3343
4685
|
runtime.MAX_MEME_WORDS = currentModelParams.maxMemeWords;
|
|
3344
4686
|
runtime.MIN_OVERLAP = currentModelParams.minOverlapThreshold;
|
|
3345
4687
|
runtime.config = runtime.config || {};
|
|
@@ -3350,11 +4692,12 @@ function applyModelParams(runtime) {
|
|
|
3350
4692
|
runtime.config.iteration = currentModelParams.iteration;
|
|
3351
4693
|
runtime.config.threshold = currentModelParams.threshold;
|
|
3352
4694
|
runtime.config.decay = currentModelParams.decay;
|
|
4695
|
+
|
|
3353
4696
|
// memeBarrier
|
|
3354
4697
|
if (runtime.memeBarrier) {
|
|
3355
4698
|
runtime.memeBarrier.maliciousThreshold = currentModelParams.maliciousThreshold;
|
|
3356
4699
|
}
|
|
3357
|
-
//
|
|
4700
|
+
// 全局边权
|
|
3358
4701
|
if (runtime.graph && currentModelParams.edgeWeight !== undefined) {
|
|
3359
4702
|
for (const point of runtime.graph.getAllPoints()) {
|
|
3360
4703
|
for (const conn of point.connect) {
|
|
@@ -3362,7 +4705,20 @@ function applyModelParams(runtime) {
|
|
|
3362
4705
|
}
|
|
3363
4706
|
}
|
|
3364
4707
|
}
|
|
3365
|
-
|
|
4708
|
+
|
|
4709
|
+
// 新增:激活/传递函数配置
|
|
4710
|
+
runtime.setActivationConfig({
|
|
4711
|
+
activationType: currentModelParams.activationType,
|
|
4712
|
+
transferType: currentModelParams.transferType,
|
|
4713
|
+
activationCustom: currentModelParams.activationCustom,
|
|
4714
|
+
transferCustom: currentModelParams.transferCustom
|
|
4715
|
+
});
|
|
4716
|
+
|
|
4717
|
+
console.log('[PARAMS] 已更新运行时参数:', {
|
|
4718
|
+
...currentModelParams,
|
|
4719
|
+
activationType: runtime.getActivationConfig().activationType,
|
|
4720
|
+
transferType: runtime.getActivationConfig().transferType
|
|
4721
|
+
});
|
|
3366
4722
|
}
|
|
3367
4723
|
// 如果直接运行此文件,启动主函数
|
|
3368
4724
|
if (require.main === module) {
|