@kira905/dsh-memory-toolkit 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +92 -0
- package/README.en.md +225 -2
- package/README.md +180 -2
- package/lib/deploy-guard.mjs +463 -275
- package/lib/guard-http.mjs +59 -62
- package/lib/pitfall-form-map.mjs +94 -84
- package/lib/pitfall-score.mjs +262 -185
- package/lib/session-file-name.mjs +116 -100
- package/lib/zstd-frames.mjs +88 -67
- package/memory-db-tools/README.md +29 -1
- package/memory-db-tools/audit-analyze.mjs +1 -1
- package/memory-db-tools/audit-export.mjs +60 -16
- package/memory-db-tools/audit-struct-probe.mjs +29 -8
- package/memory-db-tools/lib/known-devices.mjs +137 -0
- package/memory-engine/README.md +58 -4
- package/memory-engine/bench/bench.mjs +1 -1
- package/memory-engine/cli/band-sentinel.mjs +164 -0
- package/memory-engine/cli/distill-rollback.mjs +112 -44
- package/memory-engine/cli/distill-sessions.mjs +237 -0
- package/memory-engine/cli/distill.mjs +196 -95
- package/memory-engine/cli/doc-index-rebuild.mjs +142 -108
- package/memory-engine/cli/doc-index.mjs +47 -38
- package/memory-engine/cli/domain-tags.mjs +246 -162
- package/memory-engine/cli/layer-migrate.mjs +176 -0
- package/memory-engine/cli/layer-move.mjs +189 -0
- package/memory-engine/cli/pitfall-hook.mjs +136 -70
- package/memory-engine/cli/recall.mjs +243 -110
- package/memory-engine/cli/selfcheck.mjs +93 -44
- package/memory-engine/cli/serve-start.mjs +221 -140
- package/memory-engine/cli/sweep.mjs +83 -44
- package/memory-engine/cli/sync.mjs +93 -57
- package/memory-engine/config/audit-devices.json +6 -0
- package/memory-engine/config/bands.json +7 -0
- package/memory-engine/config/layers.json +7 -0
- package/memory-engine/dict/LICENSE-/350/257/215/345/205/270.md +53 -0
- package/memory-engine/dict/lexicon-config.json +11 -0
- package/memory-engine/dict//350/257/215/345/205/270.txt +109750 -0
- package/memory-engine/dict//350/257/215/351/242/221.txt +20000 -0
- package/memory-engine/lib/band-sentinel.mjs +417 -0
- package/memory-engine/lib/cli-strict.mjs +128 -0
- package/memory-engine/lib/cli-util.mjs +147 -125
- package/memory-engine/lib/config.mjs +83 -96
- package/memory-engine/lib/dirty-ledger.mjs +127 -94
- package/memory-engine/lib/distill-rollback.mjs +173 -133
- package/memory-engine/lib/distill.mjs +878 -585
- package/memory-engine/lib/doc-index-store.mjs +208 -235
- package/memory-engine/lib/doc-index.mjs +351 -350
- package/memory-engine/lib/doc-scan.mjs +178 -156
- package/memory-engine/lib/doc-summarize.mjs +220 -205
- package/memory-engine/lib/domain-tags.mjs +358 -277
- package/memory-engine/lib/embedder.mjs +175 -194
- package/memory-engine/lib/hashes.mjs +22 -7
- package/memory-engine/lib/layer-migrate.mjs +330 -0
- package/memory-engine/lib/layer-move.mjs +307 -0
- package/memory-engine/lib/layers.mjs +191 -0
- package/memory-engine/lib/lexicon-recall.mjs +473 -0
- package/memory-engine/lib/memory-source.mjs +78 -80
- package/memory-engine/lib/pitfall-source.mjs +115 -114
- package/memory-engine/lib/policy-source.mjs +279 -261
- package/memory-engine/lib/recall-log.mjs +285 -0
- package/memory-engine/lib/recall.mjs +565 -240
- package/memory-engine/lib/selfcheck.mjs +142 -106
- package/memory-engine/lib/session-distill.mjs +603 -0
- package/memory-engine/lib/session-source.mjs +421 -0
- package/memory-engine/lib/sweep.mjs +107 -130
- package/memory-engine/lib/sync.mjs +110 -140
- package/memory-engine/lib/tick-hook.mjs +123 -148
- package/memory-engine/lib/vectors-store.mjs +213 -188
- package/memory-engine/test/12-layering.test.mjs +539 -0
- package/memory-engine/test/13-session-distill.test.mjs +616 -0
- package/memory-engine/tools/bridge-smoke.mjs +94 -55
- package/memory-engine/tools/fetch-model.mjs +167 -87
- package/memory-engine/tools/smoke-recall.mjs +94 -38
- package/memory-engine/tools/verify-recall-chain.mjs +413 -0
- package/package.json +1 -1
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
//
|
|
1
|
+
// 命令行公共件:参数解析、入口守卫、日志出口、运行时装配。
|
|
2
2
|
//
|
|
3
|
-
//
|
|
4
|
-
// `process.argv[1]`
|
|
3
|
+
// 入口守卫这条是铁律:工作区里可能挂着目录链接,`import.meta.url` 会被解析成物理路径,
|
|
4
|
+
// 而 `process.argv[1]` 仍是调用方给的那条逻辑路径 ⇒ 两边都 realpath 之后再比;
|
|
5
|
+
// 否则 main 永远不触发,脚本一声不响地以 0 退出。
|
|
6
|
+
|
|
5
7
|
import fs from 'node:fs';
|
|
6
8
|
import path from 'node:path';
|
|
7
9
|
import { fileURLToPath } from 'node:url';
|
|
@@ -12,186 +14,206 @@ import { openPitfallSource } from './pitfall-source.mjs';
|
|
|
12
14
|
import { openPolicySource, scanPolicySources } from './policy-source.mjs';
|
|
13
15
|
import { createStubEmbedder, createHttpEmbedder, createLocalEmbedder } from './embedder.mjs';
|
|
14
16
|
|
|
15
|
-
/**
|
|
17
|
+
/** 已实现的向量来源(取值即 `vectors.source` 里的字符串)。 */
|
|
16
18
|
export const KNOWN_SOURCES = ['memory', 'pitfall', 'policy'];
|
|
17
19
|
|
|
18
|
-
|
|
20
|
+
// `--source` 的默认值:没给、给了空值、给了裸开关,一律当"没给" ⇒ 仍是 memory。
|
|
19
21
|
export function resolveSourceName(flags = {}, dflt = 'memory') {
|
|
20
|
-
const
|
|
21
|
-
|
|
22
|
-
return String(
|
|
22
|
+
const given = flags.source;
|
|
23
|
+
const missing = given === undefined || given === true || given === '';
|
|
24
|
+
return missing ? dflt : String(given);
|
|
23
25
|
}
|
|
24
26
|
|
|
25
27
|
/**
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
+
* 按名字挑数据源 —— 运行时选源的唯一分发点。
|
|
29
|
+
* 下游就算全参数化了,命令行也只从这里才拿得到坑源与规则源。
|
|
28
30
|
*/
|
|
29
31
|
export function openDataSource(name, config) {
|
|
30
|
-
const
|
|
31
|
-
if (
|
|
32
|
-
if (
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
personaDir: config.policyPersonaDir,
|
|
39
|
-
skillsRoot: config.policySkillsRoot,
|
|
40
|
-
docsPaths: config.policyDocsPaths,
|
|
41
|
-
}),
|
|
42
|
-
});
|
|
32
|
+
const which = String(name);
|
|
33
|
+
if (which === 'memory') return openMemorySource(config.memoryDb);
|
|
34
|
+
if (which === 'pitfall') return openPitfallSource(config.pitfallsFile);
|
|
35
|
+
if (which === 'policy') {
|
|
36
|
+
// 规则源没有库:语料本身就是文件,每次现扫一份段清单喂进去
|
|
37
|
+
// (增量靠 refId + textHash 幂等,不需要另建状态文件)。
|
|
38
|
+
const segments = scanPolicySources({ personaDir: config.policyPersonaDir, skillsRoot: config.policySkillsRoot, docsPaths: config.policyDocsPaths });
|
|
39
|
+
return openPolicySource({ segments });
|
|
43
40
|
}
|
|
44
|
-
throw new Error(`未知来源: ${
|
|
41
|
+
throw new Error(`未知来源: ${which}(现支持 memory / pitfall / policy)`);
|
|
45
42
|
}
|
|
46
43
|
|
|
44
|
+
// 两边都取物理路径再比;取不到(文件不在 / 权限不够)就当"不是入口",不抛错。
|
|
47
45
|
export function isMainEntry(importMetaUrl, argv1 = process.argv[1]) {
|
|
48
46
|
if (!argv1) return false;
|
|
49
47
|
try {
|
|
50
|
-
const
|
|
51
|
-
|
|
52
|
-
return a === b;
|
|
48
|
+
const self = fs.realpathSync(fileURLToPath(importMetaUrl));
|
|
49
|
+
return self === fs.realpathSync(argv1);
|
|
53
50
|
} catch {
|
|
54
51
|
return false;
|
|
55
52
|
}
|
|
56
53
|
}
|
|
57
54
|
|
|
55
|
+
// 长开关的正则:`--name` 或 `--name=value` 两种形态一把抓(值里允许再出现等号)。
|
|
56
|
+
const LONG_OPTION = /^--([^=]*)(?:=([\s\S]*))?$/;
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* 极简参数解析:`--k=v`、`--k v`、`--k`(布尔)三种写法都认,其余进 rest。
|
|
60
|
+
* 不认识的开关不做校验 —— 交给各命令自己决定要不要报错。
|
|
61
|
+
*/
|
|
58
62
|
export function parseArgs(argv = process.argv.slice(2)) {
|
|
59
63
|
const flags = {};
|
|
60
64
|
const rest = [];
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
rest.push(a);
|
|
65
|
+
let at = 0;
|
|
66
|
+
while (at < argv.length) {
|
|
67
|
+
const token = argv[at];
|
|
68
|
+
const hit = LONG_OPTION.exec(token);
|
|
69
|
+
if (!hit) {
|
|
70
|
+
rest.push(token);
|
|
71
|
+
at += 1;
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
const [, name, inlineValue] = hit;
|
|
75
|
+
if (inlineValue !== undefined) {
|
|
76
|
+
flags[name] = inlineValue;
|
|
77
|
+
at += 1;
|
|
78
|
+
continue;
|
|
76
79
|
}
|
|
80
|
+
const next = argv[at + 1];
|
|
81
|
+
// 下一个词本身还是开关 ⇒ 当前这个只能当布尔开关
|
|
82
|
+
if (next !== undefined && !next.startsWith('--')) {
|
|
83
|
+
flags[name] = next;
|
|
84
|
+
at += 2;
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
flags[name] = true;
|
|
88
|
+
at += 1;
|
|
77
89
|
}
|
|
78
90
|
return { flags, rest };
|
|
79
91
|
}
|
|
80
92
|
|
|
81
|
-
/**
|
|
93
|
+
/** `--json` 时人话日志一律走 stderr —— 机器只解析 stdout 上那一份 JSON。 */
|
|
82
94
|
export function makeLogger({ json = false, quiet = false, prefix = '' } = {}) {
|
|
83
|
-
const
|
|
84
|
-
if (quiet)
|
|
85
|
-
|
|
86
|
-
|
|
95
|
+
const emit = (line) => {
|
|
96
|
+
if (!quiet) {
|
|
97
|
+
const sink = json ? process.stderr : process.stdout;
|
|
98
|
+
sink.write(`${line}\n`);
|
|
99
|
+
}
|
|
87
100
|
};
|
|
88
|
-
return (msg) =>
|
|
101
|
+
return (msg) => emit(prefix ? `${prefix} ${msg}` : String(msg));
|
|
89
102
|
}
|
|
90
103
|
|
|
104
|
+
// 数值兜底:没给 / null / 裸开关都回默认值;转不成有限数字也回默认值。
|
|
91
105
|
export function num(v, dflt) {
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
106
|
+
const absent = v === undefined || v === null || v === true;
|
|
107
|
+
if (absent) return dflt;
|
|
108
|
+
const parsed = Number(v);
|
|
109
|
+
return Number.isFinite(parsed) ? parsed : dflt;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// 纯字符串型开关 → 配置字段(给了就带过去;空串与裸开关算没给)。
|
|
113
|
+
const STRING_FLAGS = [
|
|
114
|
+
['memoryDb', 'memory-db'],
|
|
115
|
+
['vectorsDb', 'vectors-db'],
|
|
116
|
+
['vectorsDir', 'vectors-dir'],
|
|
117
|
+
['stateFile', 'state-file'],
|
|
118
|
+
['pitfallsFile', 'pitfalls-file'],
|
|
119
|
+
['pitfallVectorDirtyFile', 'pitfall-dirty-file'],
|
|
120
|
+
['policyPersonaDir', 'policy-persona-dir'],
|
|
121
|
+
['policySkillsRoot', 'policy-skills-root'],
|
|
122
|
+
['modelDir', 'model-dir'],
|
|
123
|
+
['modelId', 'model'],
|
|
124
|
+
['hostId', 'host'],
|
|
125
|
+
['dshHome', 'dsh-home'],
|
|
126
|
+
];
|
|
127
|
+
|
|
128
|
+
// 数值型开关 → 配置字段(裸开关按 Number(true) = 1 算,与历史口径一致)。
|
|
129
|
+
const NUMBER_FLAGS = [
|
|
130
|
+
['dim', 'dim'],
|
|
131
|
+
['batchSize', 'batch'],
|
|
132
|
+
['bridgePort', 'bridge-port'],
|
|
133
|
+
];
|
|
134
|
+
|
|
135
|
+
const wasGiven = (raw) => raw !== undefined && raw !== '' && raw !== false;
|
|
136
|
+
|
|
137
|
+
// 命令行 → 覆盖项。只带真给了的那几项,剩下的留给环境变量与默认值。
|
|
138
|
+
function configOverrides(flags) {
|
|
139
|
+
const overrides = {};
|
|
140
|
+
for (const [field, flag] of STRING_FLAGS) {
|
|
141
|
+
if (wasGiven(flags[flag])) overrides[field] = flags[flag];
|
|
142
|
+
}
|
|
143
|
+
for (const [field, flag] of NUMBER_FLAGS) {
|
|
144
|
+
if (wasGiven(flags[flag])) overrides[field] = Number(flags[flag]);
|
|
145
|
+
}
|
|
146
|
+
const docs = flags['policy-docs'];
|
|
147
|
+
if (docs) {
|
|
148
|
+
overrides.policyDocsPaths = String(docs).split(',').map((piece) => piece.trim()).filter(Boolean);
|
|
149
|
+
}
|
|
150
|
+
return overrides;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// 三种编码器按开关挑:stub(骨架)> http(回环桥)> local(默认)。
|
|
154
|
+
function buildEmbedder(flags, config) {
|
|
155
|
+
if (flags['stub-embedder']) {
|
|
156
|
+
return createStubEmbedder({ dim: config.dim, modelId: `${config.modelId}-stub` });
|
|
157
|
+
}
|
|
158
|
+
if (flags['http-embedder']) {
|
|
159
|
+
const bridge = { host: config.bridgeHost, port: config.bridgePort, modelId: config.modelId, dim: config.dim };
|
|
160
|
+
return createHttpEmbedder({ ...bridge, timeoutMs: num(flags['bridge-timeout'], 250) });
|
|
161
|
+
}
|
|
162
|
+
return createLocalEmbedder({ modelDir: config.modelDir, modelId: config.modelId, dim: config.dim, quantized: Boolean(flags.quantized), dtype: flags.dtype || null });
|
|
95
163
|
}
|
|
96
164
|
|
|
97
165
|
/**
|
|
98
|
-
*
|
|
99
|
-
*
|
|
166
|
+
* 装配一次运行时:配置 + 向量库 + 数据源 + 编码器。
|
|
167
|
+
* 所有路径都能注入;命令行没给时落默认(≈ 用户主目录下的引擎根)。
|
|
100
168
|
*/
|
|
101
169
|
export async function openRuntime(flags = {}, { withSource = true, withEmbedder = true } = {}) {
|
|
102
|
-
const config = resolveConfig(
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
policySkillsRoot: flags['policy-skills-root'] || undefined,
|
|
113
|
-
policyDocsPaths: flags['policy-docs']
|
|
114
|
-
? String(flags['policy-docs']).split(',').map((s) => s.trim()).filter(Boolean)
|
|
115
|
-
: undefined,
|
|
116
|
-
modelDir: flags['model-dir'] || undefined,
|
|
117
|
-
modelId: flags.model || undefined,
|
|
118
|
-
dim: flags.dim ? Number(flags.dim) : undefined,
|
|
119
|
-
hostId: flags.host || undefined,
|
|
120
|
-
batchSize: flags.batch ? Number(flags.batch) : undefined,
|
|
121
|
-
bridgePort: flags['bridge-port'] ? Number(flags['bridge-port']) : undefined,
|
|
122
|
-
dshHome: flags['dsh-home'] || undefined,
|
|
123
|
-
});
|
|
124
|
-
|
|
125
|
-
const store = openVectorsStore(config.vectorsDb, { create: true });
|
|
126
|
-
store.hostId = config.hostId;
|
|
127
|
-
|
|
128
|
-
// 数据源按名字缓存(多来源召回要同时开 memory + pitfall;同一来源只开一次)
|
|
129
|
-
const cache = new Map();
|
|
130
|
-
const openSource = (name) => {
|
|
131
|
-
const n = String(name);
|
|
132
|
-
if (!cache.has(n)) cache.set(n, openDataSource(n, config));
|
|
133
|
-
return cache.get(n);
|
|
170
|
+
const config = resolveConfig(configOverrides(flags));
|
|
171
|
+
const vault = openVectorsStore(config.vectorsDb, { create: true });
|
|
172
|
+
vault.hostId = config.hostId;
|
|
173
|
+
|
|
174
|
+
// 数据源按名字缓存:多来源召回会同时开 memory 与 pitfall,同一个名字只开一次、收尾一次关掉。
|
|
175
|
+
const opened = new Map();
|
|
176
|
+
const sourceFor = (name) => {
|
|
177
|
+
const key = String(name);
|
|
178
|
+
if (!opened.has(key)) opened.set(key, openDataSource(key, config));
|
|
179
|
+
return opened.get(key);
|
|
134
180
|
};
|
|
135
|
-
const
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
let embedder = null;
|
|
140
|
-
if (!withEmbedder) {
|
|
141
|
-
embedder = null;
|
|
142
|
-
} else if (flags['stub-embedder']) {
|
|
143
|
-
embedder = createStubEmbedder({ dim: config.dim, modelId: `${config.modelId}-stub` });
|
|
144
|
-
} else if (flags['http-embedder']) {
|
|
145
|
-
embedder = createHttpEmbedder({
|
|
146
|
-
host: config.bridgeHost,
|
|
147
|
-
port: config.bridgePort,
|
|
148
|
-
timeoutMs: num(flags['bridge-timeout'], 250),
|
|
149
|
-
modelId: config.modelId,
|
|
150
|
-
dim: config.dim,
|
|
151
|
-
});
|
|
152
|
-
} else {
|
|
153
|
-
embedder = await createLocalEmbedder({
|
|
154
|
-
modelDir: config.modelDir,
|
|
155
|
-
modelId: config.modelId,
|
|
156
|
-
dim: config.dim,
|
|
157
|
-
quantized: Boolean(flags.quantized),
|
|
158
|
-
dtype: flags.dtype || null,
|
|
159
|
-
});
|
|
160
|
-
}
|
|
181
|
+
const chosen = resolveSourceName(flags);
|
|
182
|
+
const source = withSource ? sourceFor(chosen) : null;
|
|
183
|
+
const embedder = withEmbedder ? await buildEmbedder(flags, config) : null;
|
|
161
184
|
|
|
162
185
|
return {
|
|
163
|
-
config,
|
|
164
|
-
store,
|
|
165
|
-
source,
|
|
166
|
-
sourceName,
|
|
167
|
-
openSource,
|
|
168
|
-
embedder,
|
|
186
|
+
config, store: vault, source, sourceName: chosen, openSource: sourceFor, embedder,
|
|
169
187
|
close() {
|
|
170
|
-
for (const
|
|
188
|
+
for (const handle of opened.values()) {
|
|
171
189
|
try {
|
|
172
|
-
if (
|
|
190
|
+
if (handle) handle.close();
|
|
173
191
|
} catch {
|
|
174
|
-
|
|
192
|
+
// 关不掉不算错
|
|
175
193
|
}
|
|
176
194
|
}
|
|
177
|
-
|
|
195
|
+
opened.clear();
|
|
178
196
|
try {
|
|
179
|
-
|
|
197
|
+
vault.db.close();
|
|
180
198
|
} catch {
|
|
181
|
-
|
|
199
|
+
// 同上
|
|
182
200
|
}
|
|
183
201
|
},
|
|
184
202
|
};
|
|
185
203
|
}
|
|
186
204
|
|
|
205
|
+
// 日志里尽量给相对路径(短、好读);算不出来就原样回。
|
|
187
206
|
export function relForLog(p) {
|
|
188
207
|
try {
|
|
189
|
-
|
|
208
|
+
const rel = path.relative(process.cwd(), p);
|
|
209
|
+
return rel || p;
|
|
190
210
|
} catch {
|
|
191
211
|
return p;
|
|
192
212
|
}
|
|
193
213
|
}
|
|
194
214
|
|
|
215
|
+
// 毫秒读数统一保留一位小数,后缀写死。
|
|
195
216
|
export function fmtMs(v) {
|
|
196
|
-
|
|
217
|
+
const fixed = Number(v).toFixed(1);
|
|
218
|
+
return `${fixed}ms`;
|
|
197
219
|
}
|
|
@@ -1,135 +1,122 @@
|
|
|
1
|
-
//
|
|
1
|
+
// 配置解析层:所有落盘位置、模型参数、同步与召回阈值都从这一处成形,别处不再自己拼路径。
|
|
2
2
|
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
|
|
3
|
+
// 取值优先级是固定的三段:显式入参 > 环境变量 > 内置默认。
|
|
4
|
+
// 只有三段都没给,才退回用户主目录下的默认根;因此同一份代码换台机器只需换环境变量。
|
|
5
|
+
// 这里不写死任何机器名/主机名;测试必须自带临时目录,不许落默认位置。
|
|
6
|
+
|
|
7
7
|
import fs from 'node:fs';
|
|
8
8
|
import os from 'node:os';
|
|
9
9
|
import path from 'node:path';
|
|
10
10
|
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
// 老库靠 `migrateMetaNamespace()` 幂等迁移 + 兼容读,回滚靠「冗余写」保持旧键同值)。
|
|
14
|
-
// ⚠️ 文档索引库(doc-index.db)的 DDL 与本库无关,**不要**跟着改 —— 它有独立常量(见下)。
|
|
11
|
+
// 向量库的结构版本:只有键位/表结构真的改了才动它。
|
|
12
|
+
// 文档索引库是另一只库,版本号各算各的,别跟着这里一起抬。
|
|
15
13
|
export const SCHEMA_VERSION = '2';
|
|
16
|
-
/** 文档索引库(doc-index.db)的 schema 版本(本轮未动其 DDL ⇒ 保持 '1') */
|
|
17
14
|
export const DOC_INDEX_SCHEMA_VERSION = '1';
|
|
18
15
|
export const DEFAULT_MODEL_ID = 'bge-small-zh-v1.5';
|
|
19
16
|
export const DEFAULT_DIM = 512;
|
|
20
17
|
|
|
21
|
-
|
|
18
|
+
// 权重目录的三个候选,按优先级排:环境变量 > 用户目录 > 本地缓存目录。
|
|
22
19
|
export function defaultModelDirCandidates() {
|
|
20
|
+
const home = os.homedir();
|
|
23
21
|
return [
|
|
24
22
|
process.env.MEMORY_ENGINE_MODEL_DIR,
|
|
25
|
-
path.join(
|
|
26
|
-
path.join(
|
|
23
|
+
path.join(home, '.dsh', 'memory-engine', 'models'),
|
|
24
|
+
path.join(home, '.cache', 'dsh-memory-engine', 'models'),
|
|
27
25
|
].filter(Boolean);
|
|
28
26
|
}
|
|
29
27
|
|
|
28
|
+
// 取第一个真实存在的候选;一个都不在就返回末位那个(调用方负责去创建)。
|
|
30
29
|
export function defaultModelDir() {
|
|
31
|
-
const
|
|
32
|
-
for (const
|
|
30
|
+
const dirs = defaultModelDirCandidates();
|
|
31
|
+
for (const dir of dirs) {
|
|
33
32
|
try {
|
|
34
|
-
if (fs.existsSync(
|
|
33
|
+
if (fs.existsSync(dir)) return dir;
|
|
35
34
|
} catch {
|
|
36
|
-
|
|
35
|
+
// 探测失败当作"不存在",继续看下一个
|
|
37
36
|
}
|
|
38
37
|
}
|
|
39
|
-
return
|
|
38
|
+
return dirs[dirs.length - 1];
|
|
40
39
|
}
|
|
41
40
|
|
|
41
|
+
// 字符串字段:入参 > 环境变量 > 兜底。用 || 是故意的 —— 空串同样算"没给"。
|
|
42
|
+
const pick = (value, envValue, fallback) => value || envValue || fallback;
|
|
43
|
+
|
|
44
|
+
// 路径字段:兜底值由 dshHome 拼出来,换根目录时整棵树跟着搬。
|
|
45
|
+
const pickPath = (dshHome, value, envValue, segments) =>
|
|
46
|
+
pick(value, envValue, path.join(dshHome, ...segments));
|
|
47
|
+
|
|
48
|
+
// 数值字段:只做一次 Number,非数字怎么处理交给上层(保持既有口径,不在这里纠偏)。
|
|
49
|
+
const pickNumber = (value, envValue, fallback) => Number(pick(value, envValue, fallback));
|
|
50
|
+
|
|
42
51
|
/**
|
|
43
|
-
*
|
|
44
|
-
*
|
|
52
|
+
* 把三层来源折成一份配置。
|
|
53
|
+
* 关键位置(记忆库 / 向量库 / 状态文件 / 模型目录)测试可以全部覆盖。
|
|
45
54
|
*/
|
|
46
55
|
export function resolveConfig(overrides = {}) {
|
|
47
56
|
const env = overrides.env || process.env;
|
|
48
|
-
const dshHome =
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
overrides.vectorsDir ||
|
|
55
|
-
env.MEMORY_ENGINE_VECTORS_DIR ||
|
|
56
|
-
path.join(dshHome, 'memory-vectors');
|
|
57
|
+
const dshHome = pick(overrides.dshHome, env.DSH_HOME, path.join(os.homedir(), '.dsh'));
|
|
58
|
+
const vectorsDir = pick(
|
|
59
|
+
overrides.vectorsDir,
|
|
60
|
+
env.MEMORY_ENGINE_VECTORS_DIR,
|
|
61
|
+
path.join(dshHome, 'memory-vectors'),
|
|
62
|
+
);
|
|
57
63
|
|
|
58
64
|
return {
|
|
59
65
|
dshHome,
|
|
60
|
-
|
|
61
|
-
memoryDb:
|
|
62
|
-
overrides.memoryDb ||
|
|
63
|
-
env.MEMORY_ENGINE_MEMORY_DB ||
|
|
64
|
-
path.join(dshHome, 'memory', 'memory.db'),
|
|
66
|
+
memoryDb: pickPath(dshHome, overrides.memoryDb, env.MEMORY_ENGINE_MEMORY_DB, ['memory', 'memory.db']),
|
|
65
67
|
vectorsDir,
|
|
66
|
-
vectorsDb:
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
policyDocsPaths: overrides.policyDocsPaths || null, // 默认空(后置项,不默认扫全量 docs)
|
|
100
|
-
|
|
101
|
-
// 文档索引层(方案 §3.2:独立库出主库;§4 载体清单)
|
|
102
|
-
docIndexDb:
|
|
103
|
-
overrides.docIndexDb ||
|
|
104
|
-
env.DOC_INDEX_DB ||
|
|
105
|
-
path.join(vectorsDir, 'doc-index.db'),
|
|
106
|
-
docIndexState:
|
|
107
|
-
overrides.docIndexState ||
|
|
108
|
-
env.DOC_INDEX_STATE ||
|
|
109
|
-
path.join(dshHome, 'butler-doc-index-state.json'),
|
|
110
|
-
// 扫描面与 path 口径锚(不写死机器名;方案 §4 宿主迁移契约)
|
|
111
|
-
docsRoot: overrides.docsRoot || env.DOC_INDEX_DOCS_ROOT || path.join(process.cwd(), 'docs'),
|
|
112
|
-
dshRoot: overrides.dshRoot || env.DOC_INDEX_DSH_ROOT || process.cwd(),
|
|
68
|
+
vectorsDb: pick(overrides.vectorsDb, env.MEMORY_ENGINE_VECTORS_DB, path.join(vectorsDir, 'vectors.db')),
|
|
69
|
+
stateFile: pickPath(dshHome, overrides.stateFile, env.MEMORY_ENGINE_STATE, [
|
|
70
|
+
'butler-memory-vectors-state.json',
|
|
71
|
+
]),
|
|
72
|
+
|
|
73
|
+
// 坑库这条来源的两个文件:一个只读输入,一个置/清标记用(都不由本层写)。
|
|
74
|
+
pitfallsFile: pickPath(dshHome, overrides.pitfallsFile, env.MEMORY_ENGINE_PITFALLS_FILE, [
|
|
75
|
+
'butler-pitfalls.json',
|
|
76
|
+
]),
|
|
77
|
+
pitfallVectorDirtyFile: pickPath(
|
|
78
|
+
dshHome,
|
|
79
|
+
overrides.pitfallVectorDirtyFile,
|
|
80
|
+
env.MEMORY_ENGINE_PITFALL_DIRTY,
|
|
81
|
+
['butler-pitfall-vector-dirty.json'],
|
|
82
|
+
),
|
|
83
|
+
|
|
84
|
+
// 规则/技能语料面:persona 与技能目录各一处;文档面默认关(要扫时由调用方显式给)。
|
|
85
|
+
// 这三处只读,且只在产生它的那台机器上有意义,不跟着任何同步走。
|
|
86
|
+
policyPersonaDir: pickPath(dshHome, overrides.policyPersonaDir, env.MEMORY_ENGINE_POLICY_PERSONA_DIR, [
|
|
87
|
+
'.agent-presets',
|
|
88
|
+
]),
|
|
89
|
+
policySkillsRoot: pickPath(dshHome, overrides.policySkillsRoot, env.MEMORY_ENGINE_POLICY_SKILLS_ROOT, [
|
|
90
|
+
'skills',
|
|
91
|
+
]),
|
|
92
|
+
policyDocsPaths: overrides.policyDocsPaths || null,
|
|
93
|
+
|
|
94
|
+
// 文档索引是独立库:库文件和运行状态都与向量库分开摆。
|
|
95
|
+
docIndexDb: pick(overrides.docIndexDb, env.DOC_INDEX_DB, path.join(vectorsDir, 'doc-index.db')),
|
|
96
|
+
docIndexState: pickPath(dshHome, overrides.docIndexState, env.DOC_INDEX_STATE, [
|
|
97
|
+
'butler-doc-index-state.json',
|
|
98
|
+
]),
|
|
99
|
+
docsRoot: pick(overrides.docsRoot, env.DOC_INDEX_DOCS_ROOT, path.join(process.cwd(), 'docs')),
|
|
100
|
+
dshRoot: pick(overrides.dshRoot, env.DOC_INDEX_DSH_ROOT, process.cwd()),
|
|
113
101
|
|
|
114
102
|
modelDir: overrides.modelDir || defaultModelDir(),
|
|
115
|
-
modelId: overrides.modelId
|
|
116
|
-
dim:
|
|
103
|
+
modelId: pick(overrides.modelId, env.MEMORY_ENGINE_MODEL_ID, DEFAULT_MODEL_ID),
|
|
104
|
+
dim: pickNumber(overrides.dim, env.MEMORY_ENGINE_DIM, DEFAULT_DIM),
|
|
117
105
|
|
|
118
|
-
//
|
|
119
|
-
batchSize:
|
|
120
|
-
|
|
121
|
-
sweepSegments: Number(overrides.sweepSegments || env.MEMORY_ENGINE_SWEEP_SEGMENTS || 20),
|
|
106
|
+
// 同步:一批多少条;兜底轮转把全表切几段。
|
|
107
|
+
batchSize: pickNumber(overrides.batchSize, env.MEMORY_ENGINE_BATCH, 50),
|
|
108
|
+
sweepSegments: pickNumber(overrides.sweepSegments, env.MEMORY_ENGINE_SWEEP_SEGMENTS, 20),
|
|
122
109
|
|
|
123
|
-
//
|
|
124
|
-
vectorTopK:
|
|
125
|
-
ftsTopK:
|
|
126
|
-
rrfK:
|
|
127
|
-
resultTopK:
|
|
110
|
+
// 召回:向量候选数 / 关键词候选数 / 融合常数 / 最终条数。
|
|
111
|
+
vectorTopK: pickNumber(overrides.vectorTopK, env.MEMORY_ENGINE_VEC_TOPK, 20),
|
|
112
|
+
ftsTopK: pickNumber(overrides.ftsTopK, env.MEMORY_ENGINE_FTS_TOPK, 20),
|
|
113
|
+
rrfK: pickNumber(overrides.rrfK, env.MEMORY_ENGINE_RRF_K, 60),
|
|
114
|
+
resultTopK: pickNumber(overrides.resultTopK, env.MEMORY_ENGINE_RESULT_TOPK, 8),
|
|
128
115
|
|
|
129
|
-
//
|
|
130
|
-
bridgeHost: overrides.bridgeHost
|
|
131
|
-
bridgePort:
|
|
116
|
+
// 编码桥只走回环地址,端口默认 8201。
|
|
117
|
+
bridgeHost: pick(overrides.bridgeHost, env.MEMORY_ENGINE_BRIDGE_HOST, '127.0.0.1'),
|
|
118
|
+
bridgePort: pickNumber(overrides.bridgePort, env.MEMORY_ENGINE_BRIDGE_PORT, 8201),
|
|
132
119
|
|
|
133
|
-
hostId: overrides.hostId
|
|
120
|
+
hostId: pick(overrides.hostId, env.MEMORY_ENGINE_HOST_ID, os.hostname()),
|
|
134
121
|
};
|
|
135
122
|
}
|