@kira905/dsh-memory-toolkit 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/LICENSE +661 -0
- package/README.en.md +110 -0
- package/README.md +102 -2
- package/dsh-memory-local/FORK-BASE.json +24 -0
- package/dsh-memory-local/README.md +113 -0
- package/dsh-memory-local/build.mjs +101 -0
- package/dsh-memory-local/cordis.patch.yml +20 -0
- package/dsh-memory-local/deploy.mjs +306 -0
- package/dsh-memory-local/entity-status.mjs +75 -0
- package/dsh-memory-local/lib/bridge.mjs +108 -0
- package/dsh-memory-local/lib/host-adapter.js +1171 -0
- package/dsh-memory-local/lib/index.d.ts +128 -0
- package/dsh-memory-local/lib/index.js +944 -0
- package/dsh-memory-local/lib/injector.mjs +400 -0
- package/dsh-memory-local/lib/memory-read.mjs +116 -0
- package/dsh-memory-local/lib/module-gate.mjs +68 -0
- package/dsh-memory-local/lib/pointer.mjs +152 -0
- package/dsh-memory-local/lib/prod-gate.mjs +121 -0
- package/dsh-memory-local/lib/recall-log.mjs +118 -0
- package/dsh-memory-local/lib/recall.mjs +192 -0
- package/dsh-memory-local/lib/schema-migrate.mjs +365 -0
- package/dsh-memory-local/lib/user-message.mjs +32 -0
- package/dsh-memory-local/lib/vectors-read.mjs +110 -0
- package/dsh-memory-local/package.json +51 -0
- package/dsh-memory-local/restore.mjs +259 -0
- package/dsh-memory-local/rollback-injector-fix.mjs +166 -0
- package/dsh-memory-local/src/bridge.mjs +108 -0
- package/dsh-memory-local/src/host-adapter.js +1171 -0
- package/dsh-memory-local/src/index.d.ts +128 -0
- package/dsh-memory-local/src/index.js +944 -0
- package/dsh-memory-local/src/injector.mjs +400 -0
- package/dsh-memory-local/src/memory-read.mjs +116 -0
- package/dsh-memory-local/src/module-gate.mjs +68 -0
- package/dsh-memory-local/src/pointer.mjs +152 -0
- package/dsh-memory-local/src/recall-log.mjs +118 -0
- package/dsh-memory-local/src/recall.mjs +192 -0
- package/dsh-memory-local/src/schema-migrate.mjs +365 -0
- package/dsh-memory-local/src/user-message.mjs +32 -0
- package/dsh-memory-local/src/vectors-read.mjs +110 -0
- package/dsh-memory-local/test/fixtures-s2b.mjs +216 -0
- package/dsh-memory-local/test/probe-entity.mjs +170 -0
- package/dsh-memory-local/test/regression.mjs +270 -0
- package/dsh-memory-local/test/s2b.test.mjs +568 -0
- package/dsh-memory-local/test/schema-migrate.test.mjs +334 -0
- package/dsh-memory-local/test/test-adapter-converge.mjs +127 -0
- package/dsh-memory-local/test/visible-injections.test.mjs +239 -0
- package/dsh-memory-local/verify-after-restart.mjs +181 -0
- package/lib/deploy-guard.mjs +391 -0
- package/lib/guard-http.mjs +93 -0
- package/lib/pitfall-form-map.mjs +130 -0
- package/lib/pitfall-score.mjs +185 -0
- package/lib/session-file-name.mjs +160 -0
- package/lib/zstd-frames.mjs +78 -0
- package/memory-db-tools/README.md +68 -0
- package/memory-db-tools/_test/test-memory-scope.mjs +302 -0
- package/memory-db-tools/audit-analyze.mjs +352 -0
- package/memory-db-tools/audit-dump-ids.mjs +245 -0
- package/memory-db-tools/audit-export.mjs +270 -0
- package/memory-db-tools/audit-hit-probe.mjs +227 -0
- package/memory-db-tools/audit-ledger-probe.mjs +236 -0
- package/memory-db-tools/audit-persona-overlap.mjs +277 -0
- package/memory-db-tools/audit-struct-probe.mjs +213 -0
- package/memory-db-tools/audit-verify-unchanged.mjs +276 -0
- package/memory-db-tools/check-memory-db.mjs +8 -0
- package/memory-db-tools/dump-memory-db.mjs +12 -0
- package/memory-db-tools/lib/memory-scope.mjs +446 -0
- package/memory-db-tools/list-memory-summary.mjs +477 -0
- package/memory-db-tools/memory-identity-check.mjs +101 -0
- package/memory-db-tools/memory-schema-migrate.mjs +290 -0
- package/memory-db-tools/memory-scope-fixture.mjs +197 -0
- package/memory-db-tools/read-memories.mjs +26 -0
- package/memory-db-tools/read-memory-db.mjs +20 -0
- package/memory-db-tools/refine-memories.mjs +247 -0
- package/memory-db-tools/show-mem-times.mjs +22 -0
- package/memory-engine/README.md +70 -0
- package/memory-engine/bench/bench.mjs +221 -0
- package/memory-engine/cli/distill-rollback.mjs +86 -0
- package/memory-engine/cli/distill.mjs +156 -0
- package/memory-engine/cli/doc-index-rebuild.mjs +155 -0
- package/memory-engine/cli/doc-index.mjs +48 -0
- package/memory-engine/cli/domain-tags.mjs +251 -0
- package/memory-engine/cli/pitfall-hook.mjs +91 -0
- package/memory-engine/cli/recall.mjs +132 -0
- package/memory-engine/cli/selfcheck.mjs +57 -0
- package/memory-engine/cli/serve-start.mjs +247 -0
- package/memory-engine/cli/sweep.mjs +58 -0
- package/memory-engine/cli/sync.mjs +69 -0
- package/memory-engine/lib/cli-util.mjs +197 -0
- package/memory-engine/lib/config.mjs +135 -0
- package/memory-engine/lib/dirty-ledger.mjs +174 -0
- package/memory-engine/lib/distill-rollback.mjs +178 -0
- package/memory-engine/lib/distill.mjs +898 -0
- package/memory-engine/lib/doc-index-store.mjs +325 -0
- package/memory-engine/lib/doc-index.mjs +529 -0
- package/memory-engine/lib/doc-scan.mjs +271 -0
- package/memory-engine/lib/doc-summarize.mjs +305 -0
- package/memory-engine/lib/domain-tags.mjs +429 -0
- package/memory-engine/lib/embedder.mjs +242 -0
- package/memory-engine/lib/hashes.mjs +12 -0
- package/memory-engine/lib/memory-source.mjs +127 -0
- package/memory-engine/lib/pitfall-source.mjs +170 -0
- package/memory-engine/lib/policy-source.mjs +371 -0
- package/memory-engine/lib/recall.mjs +331 -0
- package/memory-engine/lib/selfcheck.mjs +131 -0
- package/memory-engine/lib/sweep.mjs +159 -0
- package/memory-engine/lib/sync.mjs +180 -0
- package/memory-engine/lib/tick-hook.mjs +218 -0
- package/memory-engine/lib/vectors-store.mjs +292 -0
- package/memory-engine/package-lock.json +1086 -0
- package/memory-engine/package.json +22 -0
- package/memory-engine/serve.mjs +171 -0
- package/memory-engine/test/01-store.test.mjs +127 -0
- package/memory-engine/test/02-sync.test.mjs +196 -0
- package/memory-engine/test/03-sweep.test.mjs +145 -0
- package/memory-engine/test/04-recall.test.mjs +214 -0
- package/memory-engine/test/05-selfcheck.test.mjs +175 -0
- package/memory-engine/test/06-doc-index.test.mjs +928 -0
- package/memory-engine/test/07-domain-tags.test.mjs +298 -0
- package/memory-engine/test/08-distill.test.mjs +686 -0
- package/memory-engine/test/09-pitfall-source.test.mjs +365 -0
- package/memory-engine/test/10-tick-hook.test.mjs +252 -0
- package/memory-engine/test/11-policy-source.test.mjs +224 -0
- package/memory-engine/test/_tickhook-worker.mjs +56 -0
- package/memory-engine/test/fixtures.mjs +205 -0
- package/memory-engine/test/harness.mjs +73 -0
- package/memory-engine/test/run.mjs +35 -0
- package/memory-engine/tools/bridge-smoke.mjs +80 -0
- package/memory-engine/tools/fetch-model.mjs +128 -0
- package/memory-engine/tools/smoke-recall.mjs +76 -0
- package/package.json +53 -4
- package/persona-memory-audit.mjs +217 -0
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
// 记忆引擎 v2 · 离线层 —— 坑库向量同步钩子 CLI(A 点 · 标记/状态/消费三用)
|
|
2
|
+
//
|
|
3
|
+
// 用法:
|
|
4
|
+
// node cli/pitfall-hook.mjs --status --json # 只读:标记现值 + 两个来源的向量条数
|
|
5
|
+
// node cli/pitfall-hook.mjs --run --json # **预演**(默认 dry-run:只算不写、不清标记)
|
|
6
|
+
// node cli/pitfall-hook.mjs --run --apply --json # 真消费(写向量 + 清标记)
|
|
7
|
+
// node cli/pitfall-hook.mjs --mark --reason "pitfall job 变动" # 置标记(pitfall job 侧一行调用)
|
|
8
|
+
// 路径可注入:--marker-file / --pitfalls-file / --vectors-db / --dsh-home
|
|
9
|
+
//
|
|
10
|
+
// ⚠️ 生态规矩:动数据的动作一律**默认 dry-run**,`--apply` 才真写。
|
|
11
|
+
import { isMainEntry, parseArgs, makeLogger, openRuntime, num } from '../lib/cli-util.mjs';
|
|
12
|
+
import { readMarker, markDirty, runPitfallHook } from '../lib/tick-hook.mjs';
|
|
13
|
+
import { countVectors } from '../lib/vectors-store.mjs';
|
|
14
|
+
|
|
15
|
+
async function main() {
|
|
16
|
+
const { flags } = parseArgs();
|
|
17
|
+
const json = Boolean(flags.json);
|
|
18
|
+
const log = makeLogger({ json, quiet: Boolean(flags.quiet) });
|
|
19
|
+
const mode = flags.run ? 'run' : flags.mark ? 'mark' : 'status';
|
|
20
|
+
const apply = Boolean(flags.apply);
|
|
21
|
+
let rt = null;
|
|
22
|
+
try {
|
|
23
|
+
// 钩子只关心坑源 ⇒ 不预开默认来源(默认来源 = memory,会被守卫拒掉)
|
|
24
|
+
rt = await openRuntime(flags, { withSource: false, withEmbedder: mode === 'run' });
|
|
25
|
+
const markerPath = rt.config.pitfallVectorDirtyFile;
|
|
26
|
+
const out = {
|
|
27
|
+
ok: true,
|
|
28
|
+
mode,
|
|
29
|
+
apply,
|
|
30
|
+
markerPath,
|
|
31
|
+
pitfallsFile: rt.config.pitfallsFile,
|
|
32
|
+
vectorsDb: rt.config.vectorsDb,
|
|
33
|
+
counts: { memory: countVectors(rt.store.db, 'memory'), pitfall: countVectors(rt.store.db, 'pitfall') },
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
if (mode === 'status') {
|
|
37
|
+
const r = readMarker(markerPath);
|
|
38
|
+
out.marker = { present: r.present, value: r.marker, error: r.error };
|
|
39
|
+
} else if (mode === 'mark') {
|
|
40
|
+
out.mark = markDirty(markerPath, {
|
|
41
|
+
reason: flags.reason === undefined || flags.reason === true ? null : String(flags.reason),
|
|
42
|
+
by: 'cli/pitfall-hook.mjs',
|
|
43
|
+
});
|
|
44
|
+
} else {
|
|
45
|
+
out.hook = await runPitfallHook({
|
|
46
|
+
markerPath,
|
|
47
|
+
store: rt.store,
|
|
48
|
+
// ⚠️ 必须是**坑源**:默认来源(memory)会被 runPitfallHook 的守卫拒掉
|
|
49
|
+
source: rt.openSource('pitfall'),
|
|
50
|
+
embedder: rt.embedder,
|
|
51
|
+
modelId: rt.config.modelId,
|
|
52
|
+
dim: rt.config.dim,
|
|
53
|
+
batchSize: num(flags.batch, rt.config.batchSize),
|
|
54
|
+
full: Boolean(flags.full),
|
|
55
|
+
dryRun: !apply,
|
|
56
|
+
log,
|
|
57
|
+
});
|
|
58
|
+
out.countsAfter = {
|
|
59
|
+
memory: countVectors(rt.store.db, 'memory'),
|
|
60
|
+
pitfall: countVectors(rt.store.db, 'pitfall'),
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
if (json) process.stdout.write(`${JSON.stringify(out, null, 2)}\n`);
|
|
65
|
+
else {
|
|
66
|
+
log(`=== 坑库向量钩子(${mode}${apply ? ' · --apply' : ' · 预演'}) ===`);
|
|
67
|
+
log(`标记文件 : ${markerPath}`);
|
|
68
|
+
log(`坑库 : ${rt.config.pitfallsFile}`);
|
|
69
|
+
log(`向量库 : ${rt.config.vectorsDb}`);
|
|
70
|
+
log(`向量条数 : memory=${out.counts.memory} / pitfall=${out.counts.pitfall}`);
|
|
71
|
+
if (mode === 'status') log(`标记 : ${out.marker.present ? JSON.stringify(out.marker.value) : `无${out.marker.error ? `(读失败:${out.marker.error})` : ''}`}`);
|
|
72
|
+
if (mode === 'mark') log(`已置标记 : updatedAt=${out.mark.updatedAt}`);
|
|
73
|
+
if (mode === 'run') {
|
|
74
|
+
log(`钩子结果 : ${out.hook.ok ? '✅' : '❌'} ${out.hook.skipped ? `(${out.hook.skipped})` : ''}`);
|
|
75
|
+
if (out.hook.stats) log(`同步 : 扫 ${out.hook.stats.scanned} / 编码 ${out.hook.stats.encoded}(跳过 ${out.hook.stats.skippedSameHash})向量 ${out.hook.stats.vectorsBefore} → ${out.hook.stats.vectorsAfter}`);
|
|
76
|
+
log(`清标记 : ${JSON.stringify(out.hook.cleared)}`);
|
|
77
|
+
log(`向量条数 : memory=${out.countsAfter.memory} / pitfall=${out.countsAfter.pitfall}`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
process.exitCode = out.hook && out.hook.ok === false ? 1 : 0;
|
|
81
|
+
} catch (e) {
|
|
82
|
+
const msg = String(e && e.stack ? e.stack : e);
|
|
83
|
+
if (json) process.stdout.write(`${JSON.stringify({ ok: false, error: msg }, null, 2)}\n`);
|
|
84
|
+
else log(`❌ ${msg}`);
|
|
85
|
+
process.exitCode = 2;
|
|
86
|
+
} finally {
|
|
87
|
+
if (rt) rt.close();
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
if (isMainEntry(import.meta.url)) await main();
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
// 记忆引擎 v2 · 离线层 —— 召回 CLI(向量 + FTS 补位 + RRF 融合 + 排除 pinned)
|
|
2
|
+
//
|
|
3
|
+
// 用法:
|
|
4
|
+
// node cli/recall.mjs --query "退下吧" [--top-k 8]
|
|
5
|
+
// node cli/recall.mjs --query "task-board 看板" --json
|
|
6
|
+
// node cli/recall.mjs --query "..." --repeat 100 # 计时(V2 用 bench 更完整)
|
|
7
|
+
// node cli/recall.mjs --query "..." --source pitfall # 只召回坑源(默认 memory)
|
|
8
|
+
// node cli/recall.mjs --query "..." --sources memory,pitfall # 多来源召回(RRF 跨源融合)
|
|
9
|
+
import { isMainEntry, parseArgs, makeLogger, openRuntime, num } from '../lib/cli-util.mjs';
|
|
10
|
+
import {
|
|
11
|
+
recall,
|
|
12
|
+
recallMulti,
|
|
13
|
+
buildVectorIndex,
|
|
14
|
+
shouldTrigger,
|
|
15
|
+
extractFtsTokens,
|
|
16
|
+
buildFtsMatch,
|
|
17
|
+
} from '../lib/recall.mjs';
|
|
18
|
+
|
|
19
|
+
function parseSourceList(flags) {
|
|
20
|
+
const raw = flags.sources;
|
|
21
|
+
if (raw === undefined || raw === true || raw === '') return null;
|
|
22
|
+
const list = String(raw)
|
|
23
|
+
.split(',')
|
|
24
|
+
.map((s) => s.trim())
|
|
25
|
+
.filter(Boolean);
|
|
26
|
+
return list.length ? list : null;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
async function main() {
|
|
30
|
+
const { flags } = parseArgs();
|
|
31
|
+
const json = Boolean(flags.json);
|
|
32
|
+
const log = makeLogger({ json, quiet: Boolean(flags.quiet) });
|
|
33
|
+
const query = flags.query === undefined || flags.query === true ? '' : String(flags.query);
|
|
34
|
+
const multiSources = parseSourceList(flags);
|
|
35
|
+
let rt = null;
|
|
36
|
+
try {
|
|
37
|
+
rt = await openRuntime(flags);
|
|
38
|
+
const repeat = num(flags.repeat, 1);
|
|
39
|
+
let last = null;
|
|
40
|
+
const times = [];
|
|
41
|
+
let indexSize = 0;
|
|
42
|
+
|
|
43
|
+
for (let i = 0; i < repeat; i += 1) {
|
|
44
|
+
if (multiSources) {
|
|
45
|
+
const entries = multiSources.map((name) => ({
|
|
46
|
+
name,
|
|
47
|
+
source: rt.openSource(name),
|
|
48
|
+
index: buildVectorIndex(rt.store.db, { source: name }),
|
|
49
|
+
}));
|
|
50
|
+
indexSize = entries.reduce((a, e) => a + (e.index.count || 0), 0);
|
|
51
|
+
last = await recallMulti({
|
|
52
|
+
query,
|
|
53
|
+
store: rt.store,
|
|
54
|
+
entries,
|
|
55
|
+
embedder: rt.embedder,
|
|
56
|
+
perSourceTopK: num(flags['per-source-top-k'], 5),
|
|
57
|
+
rrfK: num(flags['rrf-k'], rt.config.rrfK),
|
|
58
|
+
resultTopK: num(flags['top-k'], rt.config.resultTopK),
|
|
59
|
+
log,
|
|
60
|
+
});
|
|
61
|
+
} else {
|
|
62
|
+
const index = buildVectorIndex(rt.store.db, { source: rt.sourceName });
|
|
63
|
+
indexSize = index.count;
|
|
64
|
+
last = await recall({
|
|
65
|
+
query,
|
|
66
|
+
store: rt.store,
|
|
67
|
+
source: rt.source,
|
|
68
|
+
embedder: rt.embedder,
|
|
69
|
+
index,
|
|
70
|
+
sourceName: rt.sourceName,
|
|
71
|
+
vectorTopK: num(flags['vector-top-k'], rt.config.vectorTopK),
|
|
72
|
+
ftsTopK: num(flags['fts-top-k'], rt.config.ftsTopK),
|
|
73
|
+
rrfK: num(flags['rrf-k'], rt.config.rrfK),
|
|
74
|
+
resultTopK: num(flags['top-k'], rt.config.resultTopK),
|
|
75
|
+
log,
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
times.push(last.timings.totalMs);
|
|
79
|
+
}
|
|
80
|
+
times.sort((a, b) => a - b);
|
|
81
|
+
const pct = (p) => times[Math.min(times.length - 1, Math.floor((p / 100) * times.length))];
|
|
82
|
+
|
|
83
|
+
const out = {
|
|
84
|
+
ok: true,
|
|
85
|
+
source: multiSources ? multiSources.join(',') : rt.sourceName,
|
|
86
|
+
vectorsDb: rt.config.vectorsDb,
|
|
87
|
+
indexSize,
|
|
88
|
+
gate: shouldTrigger(query),
|
|
89
|
+
ftsTokens: extractFtsTokens(query),
|
|
90
|
+
ftsMatchExpr: buildFtsMatch(extractFtsTokens(query)),
|
|
91
|
+
...last,
|
|
92
|
+
repeat: { n: times.length, p50: pct(50), p95: pct(95), min: times[0], max: times[times.length - 1] },
|
|
93
|
+
};
|
|
94
|
+
if (json) process.stdout.write(JSON.stringify(out, null, 2) + '\n');
|
|
95
|
+
else {
|
|
96
|
+
log(`=== 召回 ===`);
|
|
97
|
+
log(`来源 : ${out.source}`);
|
|
98
|
+
log(`查询 : ${JSON.stringify(query)}`);
|
|
99
|
+
log(`触发 : ${last.triggered ? `是(${last.reason})` : `否(${last.reason})`}`);
|
|
100
|
+
if (last.triggered) {
|
|
101
|
+
if (multiSources) {
|
|
102
|
+
for (const [name, rec] of Object.entries(last.perSource || {})) {
|
|
103
|
+
log(
|
|
104
|
+
` · ${name}: 索引 ${rec.indexSize} 条 / 排除 pinned ${rec.pinnedExcluded} / 向量命中 ${rec.vectorHits} / FTS 命中 ${rec.ftsHits}${rec.error ? ` / ⚠️ ${rec.error}` : ''}`,
|
|
105
|
+
);
|
|
106
|
+
}
|
|
107
|
+
} else {
|
|
108
|
+
log(`索引规模 : ${last.stats.indexSize} 条向量 / 排除 pinned ${last.stats.pinnedExcluded} 条`);
|
|
109
|
+
log(`FTS 补位 : tokens=${JSON.stringify(last.stats.ftsTokens)} → 命中 ${last.stats.ftsMatched}`);
|
|
110
|
+
}
|
|
111
|
+
log(`耗时 : 编码 ${last.timings.encodeMs.toFixed(1)}ms / 向量 ${last.timings.vectorMs.toFixed(1)}ms / 融合 ${last.timings.fuseMs.toFixed(1)}ms / 合计 ${last.timings.totalMs.toFixed(1)}ms`);
|
|
112
|
+
log(`命中 top-${last.hits.length}:`);
|
|
113
|
+
for (const h of last.hits) {
|
|
114
|
+
log(
|
|
115
|
+
` #${h.refId}${multiSources ? ` [${h.source}]` : ''} score=${h.score.toFixed(5)} 来源=${h.sources.join('+')} vecRank=${h.vectorRank ?? '-'} ftsRank=${h.ftsRank ?? '-'}`,
|
|
116
|
+
);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
if (repeat > 1) log(`重复 ${times.length} 次:P50 ${pct(50).toFixed(1)}ms / P95 ${pct(95).toFixed(1)}ms`);
|
|
120
|
+
}
|
|
121
|
+
process.exitCode = 0;
|
|
122
|
+
} catch (e) {
|
|
123
|
+
const msg = String(e && e.stack ? e.stack : e);
|
|
124
|
+
if (json) process.stdout.write(JSON.stringify({ ok: false, error: msg }, null, 2) + '\n');
|
|
125
|
+
else log(`❌ ${msg}`);
|
|
126
|
+
process.exitCode = 2;
|
|
127
|
+
} finally {
|
|
128
|
+
if (rt) rt.close();
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
if (isMainEntry(import.meta.url)) await main();
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
// 记忆引擎 v2 · 离线层 —— 启动自检 CLI
|
|
2
|
+
//
|
|
3
|
+
// 用法:node cli/selfcheck.mjs [--json] [--stub-embedder] [--memory-db X] [--vectors-db Y] [--model-dir Z]
|
|
4
|
+
// 退出码:0 = 全过(warn 不算失败)· 1 = 有 fail · 2 = 运行异常
|
|
5
|
+
import { isMainEntry, parseArgs, makeLogger, openRuntime } from '../lib/cli-util.mjs';
|
|
6
|
+
import { runSelfcheck } from '../lib/selfcheck.mjs';
|
|
7
|
+
|
|
8
|
+
const ICON = { ok: '✅', warn: '⚠️ ', fail: '❌', info: 'ℹ️ ' };
|
|
9
|
+
|
|
10
|
+
async function main() {
|
|
11
|
+
const { flags } = parseArgs();
|
|
12
|
+
const json = Boolean(flags.json);
|
|
13
|
+
const log = makeLogger({ json, quiet: Boolean(flags.quiet) });
|
|
14
|
+
let rt = null;
|
|
15
|
+
try {
|
|
16
|
+
rt = await openRuntime(flags);
|
|
17
|
+
const result = await runSelfcheck({
|
|
18
|
+
store: rt.store,
|
|
19
|
+
source: rt.source,
|
|
20
|
+
embedder: rt.embedder,
|
|
21
|
+
config: rt.config,
|
|
22
|
+
allowHostMismatch: !flags['strict-host'],
|
|
23
|
+
});
|
|
24
|
+
const out = {
|
|
25
|
+
ok: result.ok,
|
|
26
|
+
host: rt.config.hostId,
|
|
27
|
+
memoryDb: rt.config.memoryDb,
|
|
28
|
+
vectorsDb: rt.config.vectorsDb,
|
|
29
|
+
modelId: rt.config.modelId,
|
|
30
|
+
modelDir: rt.config.modelDir,
|
|
31
|
+
embedder: rt.embedder.describe(),
|
|
32
|
+
...result,
|
|
33
|
+
};
|
|
34
|
+
if (json) process.stdout.write(JSON.stringify(out, null, 2) + '\n');
|
|
35
|
+
else {
|
|
36
|
+
log('=== 启动自检 ===');
|
|
37
|
+
log(`主机 : ${rt.config.hostId}`);
|
|
38
|
+
log(`记忆库 : ${rt.config.memoryDb}`);
|
|
39
|
+
log(`向量库 : ${rt.config.vectorsDb}`);
|
|
40
|
+
log(`编码器 : ${JSON.stringify(rt.embedder.describe())}`);
|
|
41
|
+
log('');
|
|
42
|
+
for (const c of result.checks) log(`${ICON[c.status] || '·'} ${c.name}:${c.detail}`);
|
|
43
|
+
log('');
|
|
44
|
+
log(result.ok ? `✅ 自检通过(warn ${result.warned})` : `❌ 自检失败(fail ${result.failed})`);
|
|
45
|
+
}
|
|
46
|
+
process.exitCode = result.ok ? 0 : 1;
|
|
47
|
+
} catch (e) {
|
|
48
|
+
const msg = String(e && e.stack ? e.stack : e);
|
|
49
|
+
if (json) process.stdout.write(JSON.stringify({ ok: false, error: msg }, null, 2) + '\n');
|
|
50
|
+
else log(`❌ ${msg}`);
|
|
51
|
+
process.exitCode = 2;
|
|
52
|
+
} finally {
|
|
53
|
+
if (rt) rt.close();
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
if (isMainEntry(import.meta.url)) await main();
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
// 记忆引擎 v2 · S2c —— 编码桥**幂等启动器**(方案 §3.1「谁起」+ §3.5 第 5 条)
|
|
2
|
+
//
|
|
3
|
+
// 契约(照方案原文,本文件是它的落地件):
|
|
4
|
+
// · 幂等:先 `GET /health` 探活;**已在跑 → 直接退出 0 并打印端口**(不重复拉起、不抢端口);
|
|
5
|
+
// · 不在跑 → 拉起 `serve.mjs`(detached),轮询 `/health` 直到 `ready:true`(默认 30s 上限);
|
|
6
|
+
// · 端口顺延:`serve.mjs` 自己 +1 顺延(最多 10 次);启动器**按子进程 pid 在
|
|
7
|
+
// [port, port+10] 里定位真实端口**,顺势落 `bridge-port.json` ⇒ 顺延后调用方仍查得到;
|
|
8
|
+
// · 崩溃 / 冷启:桥不在 / 模型仍在加载 / 返回异常 ⇒ **调用方 fail-open**(S2b 已做),
|
|
9
|
+
// 启动器**只负责把桥拉起来**,不做无限重试(每轮 tick 幂等重启,靠 job 节奏兜底);
|
|
10
|
+
// · 仅绑回环:`--bind` 非回环一律拒绝(与 serve.mjs 同款铁律,双层防呆)。
|
|
11
|
+
//
|
|
12
|
+
// 退出码:0 = 已在跑(跳过)或拉起成功且 ready / 1 = 拉起失败(超时 / spawn 失败 / 非回环)
|
|
13
|
+
// 3 = `--no-start` 且桥不在(只探活模式,供调用方判让路)/ 2 = 用法错
|
|
14
|
+
//
|
|
15
|
+
// 用法:
|
|
16
|
+
// node cli/serve-start.mjs # 幂等确保 127.0.0.1:8201 在跑
|
|
17
|
+
// node cli/serve-start.mjs --json # 机器可读
|
|
18
|
+
// node cli/serve-start.mjs --no-start # 只探活,不拉起
|
|
19
|
+
// node cli/serve-start.mjs --port-file <path> # 覆盖端口台账落点(默认 <DSH_HOME>/memory-vectors/bridge-port.json)
|
|
20
|
+
// node cli/serve-start.mjs --port 18201 --wait-ms 20000 --log <file>
|
|
21
|
+
import fs from 'node:fs';
|
|
22
|
+
import path from 'node:path';
|
|
23
|
+
import { spawn } from 'node:child_process';
|
|
24
|
+
import { fileURLToPath } from 'node:url';
|
|
25
|
+
import { resolveConfig } from '../lib/config.mjs';
|
|
26
|
+
import { parseArgs, num, isMainEntry } from '../lib/cli-util.mjs';
|
|
27
|
+
|
|
28
|
+
export const SERVE_START_VERSION = '0.1.0';
|
|
29
|
+
|
|
30
|
+
/** 与 serve.mjs 的 MAX_PORT_TRIES 保持一致(顺延搜索范围 = 同款 +1 逻辑的镜像)。 */
|
|
31
|
+
export const MAX_PORT_TRIES = 10;
|
|
32
|
+
const PROBE_TIMEOUT_MS = 1200;
|
|
33
|
+
const PROBE_TIMEOUT_FAST_MS = 400;
|
|
34
|
+
const POLL_INTERVAL_MS = 500;
|
|
35
|
+
const DEFAULT_WAIT_MS = 30_000;
|
|
36
|
+
const LOG_MAX_BYTES = 2 * 1024 * 1024;
|
|
37
|
+
const PORT_FILE_SCHEMA = 1;
|
|
38
|
+
|
|
39
|
+
export const EXIT = { OK: 0, FAIL: 1, USAGE: 2, NOT_RUNNING: 3 };
|
|
40
|
+
|
|
41
|
+
const LOOPBACK_RE = /^(127\.0\.0\.1|localhost|::1)$/;
|
|
42
|
+
|
|
43
|
+
/** `/health` 响应是否为**本编码桥**(而不是恰好占着端口的路人服务,如 3080 的 web)。 */
|
|
44
|
+
export function isBridgeHealth(body) {
|
|
45
|
+
return Boolean(body) && typeof body === 'object'
|
|
46
|
+
&& typeof body.ready === 'boolean' && typeof body.modelLoading === 'boolean'
|
|
47
|
+
&& (typeof body.model === 'string' || typeof body.dim === 'number');
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** 单端口探活:命中返回 { ok:true, port, body };否则 { ok:false, port, error }。 */
|
|
51
|
+
export async function probeHealth(port, { host = '127.0.0.1', timeoutMs = PROBE_TIMEOUT_MS } = {}) {
|
|
52
|
+
const ac = new AbortController();
|
|
53
|
+
const t = setTimeout(() => ac.abort(), timeoutMs);
|
|
54
|
+
try {
|
|
55
|
+
const res = await fetch(`http://${host}:${port}/health`, { signal: ac.signal });
|
|
56
|
+
if (!res.ok) return { ok: false, port, error: `HTTP ${res.status}` };
|
|
57
|
+
const body = await res.json();
|
|
58
|
+
if (!isBridgeHealth(body)) return { ok: false, port, error: 'not-a-bridge' };
|
|
59
|
+
return { ok: true, port, body };
|
|
60
|
+
} catch (e) {
|
|
61
|
+
const name = e && e.name;
|
|
62
|
+
return { ok: false, port, error: name === 'AbortError' || name === 'TimeoutError' ? 'timeout' : String(e?.message ?? e) };
|
|
63
|
+
} finally {
|
|
64
|
+
clearTimeout(t);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** 端口台账(bridge-port.json)读取:读不到/坏 JSON 一律 null(台账是派生物,不阻断)。 */
|
|
69
|
+
export function readPortFile(file) {
|
|
70
|
+
try {
|
|
71
|
+
const txt = fs.readFileSync(file, 'utf8');
|
|
72
|
+
const obj = JSON.parse(txt);
|
|
73
|
+
return obj && typeof obj === 'object' ? obj : null;
|
|
74
|
+
} catch {
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** 原子写端口台账(同目录 tmp → rename;失败不致命)。 */
|
|
80
|
+
export function writePortFile(file, obj) {
|
|
81
|
+
try {
|
|
82
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
83
|
+
const tmp = `${file}.tmp-${process.pid}`;
|
|
84
|
+
fs.writeFileSync(tmp, JSON.stringify(obj, null, 2) + '\n', 'utf8');
|
|
85
|
+
fs.renameSync(tmp, file);
|
|
86
|
+
return true;
|
|
87
|
+
} catch {
|
|
88
|
+
return false;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** 候选端口(去重、保序):显式 `--port` / 台账记录 / 顺延窗。 */
|
|
93
|
+
export function candidatePorts(basePort, recorded = null) {
|
|
94
|
+
const out = [];
|
|
95
|
+
const push = (p) => {
|
|
96
|
+
const n = Number(p);
|
|
97
|
+
if (!Number.isInteger(n) || n <= 0 || n > 65535) return;
|
|
98
|
+
if (!out.includes(n)) out.push(n);
|
|
99
|
+
};
|
|
100
|
+
push(basePort);
|
|
101
|
+
push(recorded);
|
|
102
|
+
for (let i = 1; i <= MAX_PORT_TRIES; i += 1) push(Number(basePort) + i);
|
|
103
|
+
return out;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** 在候选端口里找**已在跑的桥**(顺延后仍能查到)。 */
|
|
107
|
+
export async function findRunningBridge({ host = '127.0.0.1', ports = [], timeoutMs = PROBE_TIMEOUT_MS } = {}) {
|
|
108
|
+
for (const port of ports) {
|
|
109
|
+
// eslint-disable-next-line no-await-in-loop
|
|
110
|
+
const r = await probeHealth(port, { host, timeoutMs });
|
|
111
|
+
if (r.ok) return r;
|
|
112
|
+
}
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function truncateLog(file) {
|
|
117
|
+
try {
|
|
118
|
+
const st = fs.existsSync(file) ? fs.statSync(file) : null;
|
|
119
|
+
if (st && st.size > LOG_MAX_BYTES) fs.writeFileSync(file, '', 'utf8');
|
|
120
|
+
} catch { /* 忽略 */ }
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* 幂等确保桥在跑(job / 人工都走这一个入口)。
|
|
125
|
+
* @param {{flags?:object, log?:(msg:string)=>void}} opts
|
|
126
|
+
* @returns {Promise<{ok:boolean, started:boolean, port:number|null, pid:number|null, ready:boolean,
|
|
127
|
+
* skippedReason?:string, error?:string, portFile:string, bypass?:string}>}
|
|
128
|
+
*/
|
|
129
|
+
export async function ensureBridge({ flags = {}, log = (m) => process.stdout.write(m + '\n') } = {}) {
|
|
130
|
+
const config = resolveConfig({
|
|
131
|
+
modelDir: flags['model-dir'] || undefined,
|
|
132
|
+
modelId: flags.model || undefined,
|
|
133
|
+
dim: flags.dim ? Number(flags.dim) : undefined,
|
|
134
|
+
bridgeHost: flags.host || undefined,
|
|
135
|
+
bridgePort: flags.port ? Number(flags.port) : undefined,
|
|
136
|
+
});
|
|
137
|
+
const host = flags.bind || config.bridgeHost;
|
|
138
|
+
if (!LOOPBACK_RE.test(String(host))) {
|
|
139
|
+
return { ok: false, started: false, port: null, pid: null, ready: false, portFile: '', error: `只允许回环绑定(方案 §3.1),收到 ${host}` };
|
|
140
|
+
}
|
|
141
|
+
const basePort = Number(flags.port || config.bridgePort);
|
|
142
|
+
const portFile = flags['port-file']
|
|
143
|
+
? path.resolve(String(flags['port-file']))
|
|
144
|
+
: path.join(config.vectorsDir, 'bridge-port.json');
|
|
145
|
+
const logFile = flags.log ? path.resolve(String(flags.log)) : path.join(path.dirname(portFile), 'bridge.log');
|
|
146
|
+
const recorded = readPortFile(portFile);
|
|
147
|
+
const ports = candidatePorts(basePort, recorded?.port ?? null);
|
|
148
|
+
|
|
149
|
+
// ① 探活:已在跑 → 幂等跳过(唯一权威判据 = /health,不是 pid 文件)
|
|
150
|
+
const found = await findRunningBridge({ host, ports });
|
|
151
|
+
if (found) {
|
|
152
|
+
const b = found.body;
|
|
153
|
+
writePortFile(portFile, {
|
|
154
|
+
schemaVersion: PORT_FILE_SCHEMA, host, port: found.port, pid: b.pid ?? null,
|
|
155
|
+
modelId: b.model ?? config.modelId, modelDir: b.modelDir ?? config.modelDir,
|
|
156
|
+
ready: b.ready === true, modelLoading: b.modelLoading === true,
|
|
157
|
+
startedAt: b.uptimeMs ? new Date(Date.now() - Number(b.uptimeMs)).toISOString() : null,
|
|
158
|
+
checkedAt: new Date().toISOString(), via: 'serve-start(probe)',
|
|
159
|
+
});
|
|
160
|
+
log(`[serve-start] 已在跑,跳过拉起:http://${host}:${found.port} pid=${b.pid ?? '?'} ready=${b.ready} model=${b.model ?? '?'}(台账 ${portFile})`);
|
|
161
|
+
return { ok: true, started: false, port: found.port, pid: b.pid ?? null, ready: b.ready === true, portFile, health: b };
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
if (flags['no-start']) {
|
|
165
|
+
log(`[serve-start] 桥不在跑(探了 ${ports.length} 个端口),--no-start ⇒ 不拉起`);
|
|
166
|
+
return { ok: false, started: false, port: null, pid: null, ready: false, portFile, skippedReason: 'not-running' };
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
// ② 拉起(detached:本进程退出后桥继续活着;日志走文件,不污染调用方 stdout)
|
|
170
|
+
const serveScript = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', 'serve.mjs');
|
|
171
|
+
fs.mkdirSync(path.dirname(logFile), { recursive: true });
|
|
172
|
+
truncateLog(logFile);
|
|
173
|
+
const fd = fs.openSync(logFile, 'a');
|
|
174
|
+
let child;
|
|
175
|
+
try {
|
|
176
|
+
child = spawn(process.execPath, [
|
|
177
|
+
serveScript,
|
|
178
|
+
'--model-dir', config.modelDir,
|
|
179
|
+
'--port', String(basePort),
|
|
180
|
+
'--bind', host,
|
|
181
|
+
], { detached: true, stdio: ['ignore', fd, fd], windowsHide: true });
|
|
182
|
+
} catch (e) {
|
|
183
|
+
try { fs.closeSync(fd); } catch { /* 忽略 */ }
|
|
184
|
+
return { ok: false, started: false, port: null, pid: null, ready: false, portFile, error: `spawn 失败:${e?.message ?? e}` };
|
|
185
|
+
}
|
|
186
|
+
child.unref();
|
|
187
|
+
try { fs.closeSync(fd); } catch { /* 忽略 */ }
|
|
188
|
+
|
|
189
|
+
// ③ 等 ready:按 pid 在顺延窗里定位真实端口
|
|
190
|
+
const waitMs = num(flags['wait-ms'], DEFAULT_WAIT_MS);
|
|
191
|
+
const deadline = Date.now() + waitMs;
|
|
192
|
+
let lastSeen = null;
|
|
193
|
+
while (Date.now() < deadline) {
|
|
194
|
+
// eslint-disable-next-line no-await-in-loop
|
|
195
|
+
await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
|
|
196
|
+
for (const port of ports) {
|
|
197
|
+
// eslint-disable-next-line no-await-in-loop
|
|
198
|
+
const r = await probeHealth(port, { host, timeoutMs: PROBE_TIMEOUT_FAST_MS });
|
|
199
|
+
if (!r.ok) continue;
|
|
200
|
+
if (r.body.pid !== child.pid) continue; // 别人占着这个端口 → 不是我们拉的那只
|
|
201
|
+
lastSeen = r;
|
|
202
|
+
if (r.body.ready === true) {
|
|
203
|
+
writePortFile(portFile, {
|
|
204
|
+
schemaVersion: PORT_FILE_SCHEMA, host, port, pid: child.pid,
|
|
205
|
+
modelId: r.body.model ?? config.modelId, modelDir: r.body.modelDir ?? config.modelDir,
|
|
206
|
+
ready: true, modelLoading: false,
|
|
207
|
+
startedAt: new Date().toISOString(), readyAt: new Date().toISOString(), checkedAt: new Date().toISOString(),
|
|
208
|
+
via: 'serve-start(spawn)', logFile,
|
|
209
|
+
});
|
|
210
|
+
log(`[serve-start] 已拉起并就绪:http://${host}:${port} pid=${child.pid} model=${r.body.model ?? '?'}`
|
|
211
|
+
+ `${port !== basePort ? `(端口 ${basePort} 被占,顺延到 ${port})` : ''} 日志 ${logFile}`);
|
|
212
|
+
return { ok: true, started: true, port, pid: child.pid, ready: true, portFile, health: r.body };
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
writePortFile(portFile, {
|
|
218
|
+
schemaVersion: PORT_FILE_SCHEMA, host, port: lastSeen?.port ?? basePort, pid: child.pid,
|
|
219
|
+
modelId: lastSeen?.body?.model ?? config.modelId, modelDir: config.modelDir,
|
|
220
|
+
ready: false, modelLoading: lastSeen?.body?.modelLoading ?? null,
|
|
221
|
+
startedAt: new Date().toISOString(), checkedAt: new Date().toISOString(),
|
|
222
|
+
via: 'serve-start(spawn)', logFile, error: 'not-ready-in-wait-window',
|
|
223
|
+
});
|
|
224
|
+
return {
|
|
225
|
+
ok: false, started: true, port: lastSeen?.port ?? null, pid: child.pid, ready: false, portFile,
|
|
226
|
+
error: `拉起后 ${waitMs}ms 内未就绪${lastSeen ? `(pid 已探到,modelLoading=${lastSeen.body.modelLoading})` : '(未探到 pid)'};日志 ${logFile}`,
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
async function main() {
|
|
231
|
+
const { flags } = parseArgs();
|
|
232
|
+
const json = Boolean(flags.json);
|
|
233
|
+
const log = (m) => { if (!json) process.stdout.write(m + '\n'); };
|
|
234
|
+
if (flags.help) {
|
|
235
|
+
process.stdout.write('用法:node cli/serve-start.mjs [--port N] [--bind 127.0.0.1] [--model-dir P] '
|
|
236
|
+
+ '[--port-file P] [--log P] [--wait-ms N] [--no-start] [--json]\n');
|
|
237
|
+
return EXIT.OK;
|
|
238
|
+
}
|
|
239
|
+
const r = await ensureBridge({ flags, log });
|
|
240
|
+
if (json) process.stdout.write(JSON.stringify(r, null, 2) + '\n');
|
|
241
|
+
if (r.ok) return EXIT.OK;
|
|
242
|
+
// 失败原因必须说人话(否则 `--bind 0.0.0.0` 这类用法错只会得到一个静默的 exit 1)
|
|
243
|
+
if (r.error) log(`[serve-start] 失败:${r.error}`);
|
|
244
|
+
return r.skippedReason === 'not-running' ? EXIT.NOT_RUNNING : EXIT.FAIL;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
if (isMainEntry(import.meta.url)) process.exitCode = await main();
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
// 记忆引擎 v2 · 离线层 —— 兜底 hash 段轮转 CLI(方案 §3.1 兜底)
|
|
2
|
+
//
|
|
3
|
+
// 用法:
|
|
4
|
+
// node cli/sweep.mjs # 跑 1 段(默认 20 段轮转;每周跑一次即可)
|
|
5
|
+
// node cli/sweep.mjs --rounds 5
|
|
6
|
+
// node cli/sweep.mjs --all # 一轮完整覆盖(= --rounds <segments>)
|
|
7
|
+
// node cli/sweep.mjs --dry-run --json
|
|
8
|
+
// node cli/sweep.mjs --source pitfall # 坑源兜底轮转(默认 memory;坑源按 lastSeen 排序切片)
|
|
9
|
+
import { isMainEntry, parseArgs, makeLogger, openRuntime, num } from '../lib/cli-util.mjs';
|
|
10
|
+
import { runHashSweep } from '../lib/sweep.mjs';
|
|
11
|
+
|
|
12
|
+
async function main() {
|
|
13
|
+
const { flags } = parseArgs();
|
|
14
|
+
const json = Boolean(flags.json);
|
|
15
|
+
const log = makeLogger({ json, quiet: Boolean(flags.quiet) });
|
|
16
|
+
let rt = null;
|
|
17
|
+
try {
|
|
18
|
+
rt = await openRuntime(flags);
|
|
19
|
+
const segments = num(flags.segments, rt.config.sweepSegments);
|
|
20
|
+
const rounds = flags.all ? segments : num(flags.rounds, 1);
|
|
21
|
+
const stats = await runHashSweep({
|
|
22
|
+
source: rt.source,
|
|
23
|
+
store: rt.store,
|
|
24
|
+
embedder: rt.embedder,
|
|
25
|
+
modelId: rt.config.modelId,
|
|
26
|
+
dim: rt.config.dim,
|
|
27
|
+
sourceName: rt.sourceName,
|
|
28
|
+
segments,
|
|
29
|
+
rounds,
|
|
30
|
+
dryRun: Boolean(flags['dry-run']),
|
|
31
|
+
log,
|
|
32
|
+
});
|
|
33
|
+
const out = { ok: !stats.error, source: rt.sourceName, vectorsDb: rt.config.vectorsDb, ...stats };
|
|
34
|
+
if (json) process.stdout.write(JSON.stringify(out, null, 2) + '\n');
|
|
35
|
+
else {
|
|
36
|
+
log('');
|
|
37
|
+
log('=== 兜底 hash 段轮转结果 ===');
|
|
38
|
+
log(`来源 : ${rt.sourceName}`);
|
|
39
|
+
log(`id 段 : ${stats.range.lo} - ${stats.range.hi}(${stats.range.n} 条)`);
|
|
40
|
+
log(`轮转游标 : ${stats.cursorStart} → ${stats.cursorEnd} / ${stats.segments}`);
|
|
41
|
+
log(`扫描/重编码 : ${stats.scanned} / ${stats.fixed}(缺失 ${stats.missing} / 哈希不符 ${stats.mismatched})`);
|
|
42
|
+
log(`一轮完整覆盖 : ${stats.completedFullSweep ? '是(已更新 lastFullHashSweepAt)' : '否'}`);
|
|
43
|
+
log(`耗时 : ${(stats.elapsedMs / 1000).toFixed(2)}s`);
|
|
44
|
+
if (stats.error) log(`❌ 失败 : ${stats.error}`);
|
|
45
|
+
else log('✅ 完成');
|
|
46
|
+
}
|
|
47
|
+
process.exitCode = stats.error ? 1 : 0;
|
|
48
|
+
} catch (e) {
|
|
49
|
+
const msg = String(e && e.stack ? e.stack : e);
|
|
50
|
+
if (json) process.stdout.write(JSON.stringify({ ok: false, error: msg }, null, 2) + '\n');
|
|
51
|
+
else log(`❌ ${msg}`);
|
|
52
|
+
process.exitCode = 2;
|
|
53
|
+
} finally {
|
|
54
|
+
if (rt) rt.close();
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
if (isMainEntry(import.meta.url)) await main();
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
// 记忆引擎 v2 · 离线层 —— 增量同步 CLI
|
|
2
|
+
//
|
|
3
|
+
// 用法:
|
|
4
|
+
// node cli/sync.mjs # 增量(只读 memory.db → 写 vectors.db)
|
|
5
|
+
// node cli/sync.mjs --full # 全量重建(换模型/维度也自动触发)
|
|
6
|
+
// node cli/sync.mjs --dry-run # 只算不写(计划与差异)
|
|
7
|
+
// node cli/sync.mjs --json # 机器可读(日志走 stderr)
|
|
8
|
+
// node cli/sync.mjs --stub-embedder # 骨架模式(无模型依赖)
|
|
9
|
+
// node cli/sync.mjs --source pitfall # 同步**坑库**来源(默认 memory;两来源共库、meta 分命名空间)
|
|
10
|
+
// 路径可注入:--memory-db / --vectors-db / --pitfalls-file / --model-dir / --model / --dim / --batch
|
|
11
|
+
import { isMainEntry, parseArgs, makeLogger, openRuntime, num } from '../lib/cli-util.mjs';
|
|
12
|
+
import { runIncrementalSync } from '../lib/sync.mjs';
|
|
13
|
+
|
|
14
|
+
async function main() {
|
|
15
|
+
const { flags } = parseArgs();
|
|
16
|
+
const json = Boolean(flags.json);
|
|
17
|
+
const log = makeLogger({ json, quiet: Boolean(flags.quiet) });
|
|
18
|
+
let rt = null;
|
|
19
|
+
try {
|
|
20
|
+
rt = await openRuntime(flags);
|
|
21
|
+
const stats = await runIncrementalSync({
|
|
22
|
+
source: rt.source,
|
|
23
|
+
store: rt.store,
|
|
24
|
+
embedder: rt.embedder,
|
|
25
|
+
modelId: rt.config.modelId,
|
|
26
|
+
dim: rt.config.dim,
|
|
27
|
+
sourceName: rt.sourceName,
|
|
28
|
+
batchSize: num(flags.batch, rt.config.batchSize),
|
|
29
|
+
full: Boolean(flags.full),
|
|
30
|
+
dryRun: Boolean(flags['dry-run']),
|
|
31
|
+
log,
|
|
32
|
+
});
|
|
33
|
+
const out = {
|
|
34
|
+
ok: !stats.error,
|
|
35
|
+
source: rt.sourceName,
|
|
36
|
+
// 兼容字段:默认来源(memory)时语义与旧版一致
|
|
37
|
+
memoryDb: rt.config.memoryDb,
|
|
38
|
+
dataSource: rt.source.dbPath,
|
|
39
|
+
vectorsDb: rt.config.vectorsDb,
|
|
40
|
+
modelId: rt.config.modelId,
|
|
41
|
+
...stats,
|
|
42
|
+
};
|
|
43
|
+
if (json) process.stdout.write(JSON.stringify(out, null, 2) + '\n');
|
|
44
|
+
else {
|
|
45
|
+
log('');
|
|
46
|
+
log('=== 增量同步结果 ===');
|
|
47
|
+
log(`模式 : ${stats.mode}${stats.reasons.length ? `(${stats.reasons.join(' / ')})` : ''}`);
|
|
48
|
+
log(`来源 : ${rt.sourceName}(${rt.source.dbPath})`);
|
|
49
|
+
log(`向量库 : ${rt.config.vectorsDb}`);
|
|
50
|
+
log(`模型 : ${rt.config.modelId}(${rt.embedder.dim} 维)`);
|
|
51
|
+
log(`扫描/编码 : ${stats.scanned} / ${stats.encoded}(同哈希跳过 ${stats.skippedSameHash})`);
|
|
52
|
+
log(`向量条数 : ${stats.vectorsBefore} → ${stats.vectorsAfter}`);
|
|
53
|
+
log(`水位 : ${stats.watermarkStart} → ${stats.watermarkEnd}`);
|
|
54
|
+
log(`耗时 : ${(stats.elapsedMs / 1000).toFixed(2)}s`);
|
|
55
|
+
if (stats.error) log(`❌ 失败 : ${stats.error}(进度已保留,可直接续跑)`);
|
|
56
|
+
else log('✅ 完成');
|
|
57
|
+
}
|
|
58
|
+
process.exitCode = stats.error ? 1 : 0;
|
|
59
|
+
} catch (e) {
|
|
60
|
+
const msg = String(e && e.stack ? e.stack : e);
|
|
61
|
+
if (json) process.stdout.write(JSON.stringify({ ok: false, error: msg }, null, 2) + '\n');
|
|
62
|
+
else log(`❌ ${msg}`);
|
|
63
|
+
process.exitCode = 2;
|
|
64
|
+
} finally {
|
|
65
|
+
if (rt) rt.close();
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
if (isMainEntry(import.meta.url)) await main();
|