@kira905/dsh-memory-toolkit 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +92 -0
- package/README.en.md +225 -2
- package/README.md +180 -2
- package/lib/deploy-guard.mjs +463 -275
- package/lib/guard-http.mjs +59 -62
- package/lib/pitfall-form-map.mjs +94 -84
- package/lib/pitfall-score.mjs +262 -185
- package/lib/session-file-name.mjs +116 -100
- package/lib/zstd-frames.mjs +88 -67
- package/memory-db-tools/README.md +29 -1
- package/memory-db-tools/audit-analyze.mjs +1 -1
- package/memory-db-tools/audit-export.mjs +60 -16
- package/memory-db-tools/audit-struct-probe.mjs +29 -8
- package/memory-db-tools/lib/known-devices.mjs +137 -0
- package/memory-engine/README.md +58 -4
- package/memory-engine/bench/bench.mjs +1 -1
- package/memory-engine/cli/band-sentinel.mjs +164 -0
- package/memory-engine/cli/distill-rollback.mjs +112 -44
- package/memory-engine/cli/distill-sessions.mjs +237 -0
- package/memory-engine/cli/distill.mjs +196 -95
- package/memory-engine/cli/doc-index-rebuild.mjs +142 -108
- package/memory-engine/cli/doc-index.mjs +47 -38
- package/memory-engine/cli/domain-tags.mjs +246 -162
- package/memory-engine/cli/layer-migrate.mjs +176 -0
- package/memory-engine/cli/layer-move.mjs +189 -0
- package/memory-engine/cli/pitfall-hook.mjs +136 -70
- package/memory-engine/cli/recall.mjs +243 -110
- package/memory-engine/cli/selfcheck.mjs +93 -44
- package/memory-engine/cli/serve-start.mjs +221 -140
- package/memory-engine/cli/sweep.mjs +83 -44
- package/memory-engine/cli/sync.mjs +93 -57
- package/memory-engine/config/audit-devices.json +6 -0
- package/memory-engine/config/bands.json +7 -0
- package/memory-engine/config/layers.json +7 -0
- package/memory-engine/dict/LICENSE-/350/257/215/345/205/270.md +53 -0
- package/memory-engine/dict/lexicon-config.json +11 -0
- package/memory-engine/dict//350/257/215/345/205/270.txt +109750 -0
- package/memory-engine/dict//350/257/215/351/242/221.txt +20000 -0
- package/memory-engine/lib/band-sentinel.mjs +417 -0
- package/memory-engine/lib/cli-strict.mjs +128 -0
- package/memory-engine/lib/cli-util.mjs +147 -125
- package/memory-engine/lib/config.mjs +83 -96
- package/memory-engine/lib/dirty-ledger.mjs +127 -94
- package/memory-engine/lib/distill-rollback.mjs +173 -133
- package/memory-engine/lib/distill.mjs +878 -585
- package/memory-engine/lib/doc-index-store.mjs +208 -235
- package/memory-engine/lib/doc-index.mjs +351 -350
- package/memory-engine/lib/doc-scan.mjs +178 -156
- package/memory-engine/lib/doc-summarize.mjs +220 -205
- package/memory-engine/lib/domain-tags.mjs +358 -277
- package/memory-engine/lib/embedder.mjs +175 -194
- package/memory-engine/lib/hashes.mjs +22 -7
- package/memory-engine/lib/layer-migrate.mjs +330 -0
- package/memory-engine/lib/layer-move.mjs +307 -0
- package/memory-engine/lib/layers.mjs +191 -0
- package/memory-engine/lib/lexicon-recall.mjs +473 -0
- package/memory-engine/lib/memory-source.mjs +78 -80
- package/memory-engine/lib/pitfall-source.mjs +115 -114
- package/memory-engine/lib/policy-source.mjs +279 -261
- package/memory-engine/lib/recall-log.mjs +285 -0
- package/memory-engine/lib/recall.mjs +565 -240
- package/memory-engine/lib/selfcheck.mjs +142 -106
- package/memory-engine/lib/session-distill.mjs +603 -0
- package/memory-engine/lib/session-source.mjs +421 -0
- package/memory-engine/lib/sweep.mjs +107 -130
- package/memory-engine/lib/sync.mjs +110 -140
- package/memory-engine/lib/tick-hook.mjs +123 -148
- package/memory-engine/lib/vectors-store.mjs +213 -188
- package/memory-engine/test/12-layering.test.mjs +539 -0
- package/memory-engine/test/13-session-distill.test.mjs +616 -0
- package/memory-engine/tools/bridge-smoke.mjs +94 -55
- package/memory-engine/tools/fetch-model.mjs +167 -87
- package/memory-engine/tools/smoke-recall.mjs +94 -38
- package/memory-engine/tools/verify-recall-chain.mjs +413 -0
- package/package.json +1 -1
|
@@ -1,159 +1,136 @@
|
|
|
1
|
-
//
|
|
1
|
+
// 兜底轮转:把全表按 id 段切开,一段一段核对「正文哈希」与「向量库里存的那份」是否还一致。
|
|
2
2
|
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
3
|
+
// 为什么非它不可:外部脚本改正文时未必顺手更新 updated_at(真实写口里就有一批这样的),
|
|
4
|
+
// 这类勘误永远落在增量窗口之外。所以兜底单列一条路,不与增量主路径共用候选集,
|
|
5
|
+
// 靠一圈一圈轮转最终覆盖整张表。
|
|
6
|
+
|
|
6
7
|
import { textHash } from './hashes.mjs';
|
|
7
|
-
import
|
|
8
|
-
// 【S3c 挂钩(2026-09-21)】轮转发现的"正文变了"差异**必须落 `memories_dirty` 台账**:
|
|
9
|
-
// 本函数发现差异后会**立刻重编码并更新 `vectors.text_hash`** ⇒ 差异信号当场消失。
|
|
10
|
-
// 若此刻只用于重编码而不留痕,被外部改过 text 的条(A7 的 41 处写口)就再也没人知道要提炼它们。
|
|
11
|
-
// 台账是**只加不改**的旁路;`recordDirty` 幂等(同一条重复扫到只 hits+1,不复活已消费项)。
|
|
8
|
+
import * as vectors from './vectors-store.mjs';
|
|
12
9
|
import { recordDirty } from './dirty-ledger.mjs';
|
|
13
10
|
|
|
11
|
+
// 台账的语义是「哪条记忆的正文被外部改过」,因此只对记忆源记账;
|
|
12
|
+
// 其它来源(例如坑源的十六位十六进制 id)一律不往台账塞,免得污染提炼侧输入。
|
|
13
|
+
const LEDGER_SOURCE = 'memory';
|
|
14
|
+
|
|
15
|
+
const noteOf = (stage, e) => ({ stage, message: e && e.message ? e.message : String(e) });
|
|
16
|
+
|
|
14
17
|
/**
|
|
15
18
|
* @param {object} p
|
|
16
|
-
* @param {object} p.source
|
|
17
|
-
* @param {object} p.store
|
|
18
|
-
* @param {object} p.embedder
|
|
19
|
-
* @param {
|
|
20
|
-
* @param {number} p.
|
|
21
|
-
* @param {string} [p.sourceName='memory']
|
|
22
|
-
*
|
|
23
|
-
*
|
|
19
|
+
* @param {object} p.source 数据源(idRange / segment)
|
|
20
|
+
* @param {object} p.store 向量库({ db })
|
|
21
|
+
* @param {object} p.embedder 编码器
|
|
22
|
+
* @param {string} p.modelId
|
|
23
|
+
* @param {number} p.dim
|
|
24
|
+
* @param {string} [p.sourceName='memory'] 来源名(决定写哪一列、游标落哪个命名空间)
|
|
25
|
+
* @param {number} [p.segments=20] 全表切几段
|
|
26
|
+
* @param {number} [p.rounds=1] 本次走几段(传 segments 即一整圈)
|
|
24
27
|
*/
|
|
25
|
-
export async function runHashSweep({
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
segments = 20,
|
|
33
|
-
rounds = 1,
|
|
34
|
-
dryRun = false,
|
|
35
|
-
log = () => {},
|
|
36
|
-
now = () => Date.now(),
|
|
37
|
-
}) {
|
|
38
|
-
const db = store.db;
|
|
39
|
-
const src = String(sourceName || 'memory');
|
|
40
|
-
// ⚠️ `memories_dirty` 台账的语义是「哪条**记忆**的正文被外部改过、待提炼」——
|
|
41
|
-
// 坑源跑兜底时**不得**把 16 位 hex 的坑 id 记进去(会污染提炼侧输入)。故只对 memory 源落台账。
|
|
42
|
-
const useDirtyLedger = src === 'memory';
|
|
43
|
-
const startedAt = now();
|
|
44
|
-
const range = source.idRange();
|
|
28
|
+
export async function runHashSweep({ source, store, embedder, modelId, dim, sourceName = 'memory', segments = 20, rounds = 1, dryRun = false, log = () => {}, now = () => Date.now() }) {
|
|
29
|
+
const conn = store.db;
|
|
30
|
+
const origin = String(sourceName || 'memory');
|
|
31
|
+
const ledgerApplies = origin === LEDGER_SOURCE;
|
|
32
|
+
const openedAt = now();
|
|
33
|
+
const bounds = source.idRange();
|
|
34
|
+
const slotCount = Math.max(1, segments);
|
|
45
35
|
const stats = {
|
|
46
|
-
source:
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
range,
|
|
51
|
-
cursorStart: Number(getMetaFor(db, META_KEYS.hashSweepCursor, src) || 0) % Math.max(1, segments),
|
|
52
|
-
cursorEnd: 0,
|
|
53
|
-
scanned: 0,
|
|
54
|
-
mismatched: 0,
|
|
55
|
-
missing: 0,
|
|
56
|
-
fixed: 0,
|
|
57
|
-
dirtyRecorded: 0,
|
|
58
|
-
segmentReports: [],
|
|
59
|
-
completedFullSweep: false,
|
|
60
|
-
elapsedMs: 0,
|
|
61
|
-
failures: [],
|
|
36
|
+
source: origin, mode: 'hash-sweep', segments, rounds, range: bounds,
|
|
37
|
+
cursorStart: Number(vectors.getMetaFor(conn, vectors.META_KEYS.hashSweepCursor, origin) || 0) % slotCount,
|
|
38
|
+
cursorEnd: 0, scanned: 0, mismatched: 0, missing: 0, fixed: 0, dirtyRecorded: 0,
|
|
39
|
+
segmentReports: [], completedFullSweep: false, elapsedMs: 0, failures: [],
|
|
62
40
|
};
|
|
63
|
-
|
|
64
|
-
stats.cursorEnd = stats.cursorStart;
|
|
41
|
+
const closeBook = () => {
|
|
65
42
|
stats.finishedAt = now();
|
|
66
|
-
stats.elapsedMs = stats.finishedAt -
|
|
67
|
-
|
|
43
|
+
stats.elapsedMs = stats.finishedAt - openedAt;
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
// 空表:游标原地不动,直接收工(一个 meta 都不写)
|
|
47
|
+
if (bounds.lo == null) {
|
|
48
|
+
stats.cursorEnd = stats.cursorStart;
|
|
49
|
+
closeBook();
|
|
50
|
+
log(`[sweep] ${origin} 源为空,跳过`);
|
|
68
51
|
return stats;
|
|
69
52
|
}
|
|
70
53
|
|
|
71
|
-
const width = Math.max(1, Math.ceil((
|
|
72
|
-
let
|
|
54
|
+
const width = Math.max(1, Math.ceil((bounds.hi - bounds.lo + 1) / slotCount));
|
|
55
|
+
let slot = stats.cursorStart;
|
|
73
56
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
mismatched += 1;
|
|
93
|
-
need.push({ row: r, hash: h });
|
|
94
|
-
dirtyItems.push({ refId: r.id, kind: 'mismatch', oldHash: prev, newHash: h });
|
|
95
|
-
}
|
|
57
|
+
// 单段作业:先对哈希,再落台账,最后重编码 —— 顺序不能换。
|
|
58
|
+
// 重编码会把旧的 text_hash 覆盖掉,差异就再也说不清了。
|
|
59
|
+
const runSegment = async (lo, hi, index) => {
|
|
60
|
+
const rows = source.segment(lo, hi);
|
|
61
|
+
const known = vectors.getHashes(conn, origin, rows.map((row) => row.id));
|
|
62
|
+
const redo = [];
|
|
63
|
+
const drifted = [];
|
|
64
|
+
let absent = 0;
|
|
65
|
+
let stale = 0;
|
|
66
|
+
for (const row of rows) {
|
|
67
|
+
const digest = textHash(row.text);
|
|
68
|
+
const stored = known.get(String(row.id));
|
|
69
|
+
if (stored === digest) continue;
|
|
70
|
+
if (stored === undefined) {
|
|
71
|
+
absent += 1;
|
|
72
|
+
} else {
|
|
73
|
+
stale += 1;
|
|
74
|
+
drifted.push({ refId: row.id, kind: 'mismatch', oldHash: stored, newHash: digest });
|
|
96
75
|
}
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
76
|
+
redo.push({ row, digest });
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
if (drifted.length && ledgerApplies && !dryRun) {
|
|
80
|
+
try {
|
|
81
|
+
recordDirty(conn, drifted, { now: now() });
|
|
82
|
+
stats.dirtyRecorded += drifted.length;
|
|
83
|
+
} catch (e) {
|
|
84
|
+
// 台账只是提炼侧的补充输入源 ⇒ 它写失败绝不许拖垮主路径
|
|
85
|
+
stats.failures.push(noteOf('dirty-ledger', e));
|
|
106
86
|
}
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
);
|
|
122
|
-
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
if (redo.length) {
|
|
90
|
+
const encoded = await embedder.encodeBatch(redo.map((item) => item.row.text));
|
|
91
|
+
if (!dryRun) {
|
|
92
|
+
vectors.upsertVectors(conn, redo.map((item, at) => ({
|
|
93
|
+
source: origin,
|
|
94
|
+
refId: item.row.id,
|
|
95
|
+
model: modelId,
|
|
96
|
+
dim: encoded[at].length,
|
|
97
|
+
vec: encoded[at],
|
|
98
|
+
textHash: item.digest,
|
|
99
|
+
updatedAt: item.row.updated_at,
|
|
100
|
+
})));
|
|
123
101
|
}
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
);
|
|
140
|
-
|
|
141
|
-
if (cursor === 0) stats.completedFullSweep = true;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
stats.scanned += rows.length;
|
|
105
|
+
stats.missing += absent;
|
|
106
|
+
stats.mismatched += stale;
|
|
107
|
+
stats.fixed += redo.length;
|
|
108
|
+
stats.segmentReports.push({ segment: index, lo, hi, rows: rows.length, missing: absent, mismatched: stale, fixed: redo.length });
|
|
109
|
+
log(`[sweep] 段 ${index}(id ${lo}-${hi}):${rows.length} 条,缺失 ${absent},哈希不符 ${stale} ⇒ 重编码 ${redo.length}`);
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
try {
|
|
113
|
+
for (let step = 0; step < rounds; step += 1) {
|
|
114
|
+
const from = bounds.lo + slot * width;
|
|
115
|
+
const until = Math.min(bounds.hi, from + width - 1);
|
|
116
|
+
await runSegment(from, until, slot);
|
|
117
|
+
slot = (slot + 1) % slotCount;
|
|
118
|
+
if (slot === 0) stats.completedFullSweep = true;
|
|
142
119
|
}
|
|
143
120
|
} catch (e) {
|
|
144
|
-
stats.failures.push(
|
|
121
|
+
stats.failures.push(noteOf('sweep', e));
|
|
145
122
|
stats.error = stats.failures[0].message;
|
|
146
123
|
}
|
|
147
124
|
|
|
148
|
-
stats.cursorEnd =
|
|
125
|
+
stats.cursorEnd = slot;
|
|
149
126
|
if (!dryRun) {
|
|
150
|
-
setMetaFor(
|
|
127
|
+
vectors.setMetaFor(conn, vectors.META_KEYS.hashSweepCursor, origin, String(slot));
|
|
151
128
|
if (stats.completedFullSweep) {
|
|
152
129
|
stats.lastFullHashSweepAt = now();
|
|
153
|
-
|
|
130
|
+
const fullAt = vectors.META_KEYS.lastFullHashSweepAt;
|
|
131
|
+
vectors.setMetaFor(conn, fullAt, origin, String(stats.lastFullHashSweepAt));
|
|
154
132
|
}
|
|
155
133
|
}
|
|
156
|
-
|
|
157
|
-
stats.elapsedMs = stats.finishedAt - startedAt;
|
|
134
|
+
closeBook();
|
|
158
135
|
return stats;
|
|
159
136
|
}
|
|
@@ -1,180 +1,150 @@
|
|
|
1
|
-
//
|
|
1
|
+
// 增量同步:按 updated_at 水位,把新增或改过的条目编码进独立向量库。
|
|
2
2
|
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
3
|
+
// 这条路只读源库、只写向量库;一批一批往前推,某一批炸了就停在那儿 ——
|
|
4
|
+
// 之前跑完的批次进度已经落盘,下一轮从断点接着编即可。
|
|
5
|
+
// 换模型 / 换维度 / 显式全量都会转成"先清掉该来源、再整份重建"。
|
|
6
|
+
// 那种「改正文却不改时间戳」的盲区不归这里管,另有兜底轮转那条路。
|
|
7
|
+
|
|
6
8
|
import { textHash } from './hashes.mjs';
|
|
7
|
-
import
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
9
|
+
import * as vectors from './vectors-store.mjs';
|
|
10
|
+
|
|
11
|
+
const errText = (e) => (e && e.message ? e.message : String(e));
|
|
12
|
+
|
|
13
|
+
// 重建判据:模型变了、维度变了、或者调用方明确要全量。
|
|
14
|
+
function decideMode({ knownModel, knownDim, modelId, dim, full }) {
|
|
15
|
+
const causes = [];
|
|
16
|
+
if (knownModel && knownModel !== modelId) causes.push(`模型变更 ${knownModel} → ${modelId}`);
|
|
17
|
+
if (knownDim && Number(knownDim) !== Number(dim)) causes.push(`维度变更 ${knownDim} → ${dim}`);
|
|
18
|
+
if (full) causes.push('显式 --full');
|
|
19
|
+
return { causes, rebuild: causes.length > 0 };
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
// 一批里真正要编码的那些:哈希与库里存的一致就跳过(幂等全靠这一步)。
|
|
23
|
+
function splitBatch(rows, stored) {
|
|
24
|
+
const pending = [];
|
|
25
|
+
let skipped = 0;
|
|
26
|
+
for (const row of rows) {
|
|
27
|
+
const digest = textHash(row.text);
|
|
28
|
+
if (stored.get(String(row.id)) === digest) {
|
|
29
|
+
skipped += 1;
|
|
30
|
+
continue;
|
|
31
|
+
}
|
|
32
|
+
pending.push({ row, digest });
|
|
33
|
+
}
|
|
34
|
+
return { pending, skipped };
|
|
35
|
+
}
|
|
18
36
|
|
|
19
37
|
/**
|
|
20
38
|
* @param {object} p
|
|
21
|
-
* @param {object} p.source
|
|
22
|
-
* @param {object} p.store
|
|
23
|
-
* @param {object} p.embedder 编码器(
|
|
39
|
+
* @param {object} p.source 记忆源 / 坑源(只读)
|
|
40
|
+
* @param {object} p.store 向量库({ db })
|
|
41
|
+
* @param {object} p.embedder 编码器(encodeBatch)
|
|
24
42
|
* @param {string} p.modelId
|
|
25
43
|
* @param {number} p.dim
|
|
26
|
-
* @param {string} [p.sourceName='memory']
|
|
44
|
+
* @param {string} [p.sourceName='memory'] 来源名(同时决定 vectors.source 与 meta 命名空间)
|
|
27
45
|
*/
|
|
28
|
-
export async function runIncrementalSync({
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
dryRun = false,
|
|
38
|
-
log = () => {},
|
|
39
|
-
now = () => Date.now(),
|
|
40
|
-
untilEmpty = true,
|
|
41
|
-
}) {
|
|
42
|
-
const db = store.db;
|
|
43
|
-
const src = String(sourceName || 'memory');
|
|
44
|
-
const startedAt = now();
|
|
45
|
-
// ⚠️ 水位/模型/维度键一律**按来源**读写(共库不串味:方案 §A.6 首要风险 / §1.3-③.1)
|
|
46
|
-
const metaModel = getMetaFor(db, META_KEYS.modelId, src);
|
|
47
|
-
const metaDim = getMetaFor(db, META_KEYS.dim, src);
|
|
48
|
-
const metaHost = getMeta(db, META_KEYS.hostId);
|
|
49
|
-
|
|
50
|
-
const reasons = [];
|
|
51
|
-
if (metaModel && metaModel !== modelId) reasons.push(`模型变更 ${metaModel} → ${modelId}`);
|
|
52
|
-
if (metaDim && Number(metaDim) !== Number(dim)) reasons.push(`维度变更 ${metaDim} → ${dim}`);
|
|
53
|
-
if (full) reasons.push('显式 --full');
|
|
54
|
-
|
|
55
|
-
let rebuild = reasons.length > 0;
|
|
56
|
-
let watermark = 0;
|
|
57
|
-
let cleared = 0;
|
|
46
|
+
export async function runIncrementalSync({ source, store, embedder, modelId, dim, sourceName = 'memory', batchSize = 50, full = false, dryRun = false, log = () => {}, now = () => Date.now(), untilEmpty = true }) {
|
|
47
|
+
const conn = store.db;
|
|
48
|
+
const origin = String(sourceName || 'memory');
|
|
49
|
+
const openedAt = now();
|
|
50
|
+
// 水位/模型/维度一律按来源读写:两个来源共用一只库,键位串味会表现成"某来源静默漏编码"。
|
|
51
|
+
const knownModel = vectors.getMetaFor(conn, vectors.META_KEYS.modelId, origin);
|
|
52
|
+
const knownDim = vectors.getMetaFor(conn, vectors.META_KEYS.dim, origin);
|
|
53
|
+
const hostKnown = vectors.getMeta(conn, vectors.META_KEYS.hostId);
|
|
54
|
+
const { causes, rebuild } = decideMode({ knownModel, knownDim, modelId, dim, full });
|
|
58
55
|
|
|
56
|
+
let level = 0;
|
|
57
|
+
let purged = 0;
|
|
59
58
|
if (rebuild) {
|
|
60
59
|
if (!dryRun) {
|
|
61
|
-
|
|
62
|
-
setMetaFor(
|
|
63
|
-
setMetaFor(
|
|
64
|
-
setMetaFor(
|
|
60
|
+
purged = vectors.deleteSource(conn, origin);
|
|
61
|
+
vectors.setMetaFor(conn, vectors.META_KEYS.modelId, origin, modelId);
|
|
62
|
+
vectors.setMetaFor(conn, vectors.META_KEYS.dim, origin, String(dim));
|
|
63
|
+
vectors.setMetaFor(conn, vectors.META_KEYS.lastIncrementalSyncAt, origin, '0');
|
|
65
64
|
}
|
|
66
|
-
log(`[sync] 全量重建:${
|
|
65
|
+
log(`[sync] 全量重建:${causes.join(' / ')}(清空 ${purged} 条向量)`);
|
|
67
66
|
} else {
|
|
68
|
-
|
|
67
|
+
level = Number(vectors.getMetaFor(conn, vectors.META_KEYS.lastIncrementalSyncAt, origin) || 0);
|
|
69
68
|
}
|
|
70
69
|
|
|
71
70
|
const stats = {
|
|
72
|
-
source:
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
encoded: 0,
|
|
79
|
-
skippedSameHash: 0,
|
|
80
|
-
upserted: 0,
|
|
81
|
-
batches: 0,
|
|
82
|
-
failures: [],
|
|
83
|
-
cleared,
|
|
84
|
-
vectorsBefore: countVectors(db, src),
|
|
85
|
-
vectorsAfter: 0,
|
|
86
|
-
elapsedMs: 0,
|
|
87
|
-
startedAt,
|
|
88
|
-
finishedAt: 0,
|
|
71
|
+
source: origin, mode: rebuild ? 'rebuild' : 'incremental', reasons: causes,
|
|
72
|
+
watermarkStart: level, watermarkEnd: level,
|
|
73
|
+
scanned: 0, encoded: 0, skippedSameHash: 0, upserted: 0, batches: 0,
|
|
74
|
+
failures: [], cleared: purged,
|
|
75
|
+
vectorsBefore: vectors.countVectors(conn, origin), vectorsAfter: 0,
|
|
76
|
+
elapsedMs: 0, startedAt: openedAt, finishedAt: 0,
|
|
89
77
|
};
|
|
90
78
|
|
|
91
79
|
let afterId = 0;
|
|
92
|
-
let
|
|
80
|
+
let high = level;
|
|
81
|
+
|
|
82
|
+
// 每批成功后把四类键一起前移(主机的键只在库里没有时补一次)。
|
|
83
|
+
const advanceLevel = (value) => {
|
|
84
|
+
vectors.setMetaFor(conn, vectors.META_KEYS.modelId, origin, modelId);
|
|
85
|
+
vectors.setMetaFor(conn, vectors.META_KEYS.dim, origin, String(dim));
|
|
86
|
+
if (!hostKnown) vectors.setMeta(conn, vectors.META_KEYS.hostId, String(store.hostId || ''));
|
|
87
|
+
vectors.setMetaFor(conn, vectors.META_KEYS.lastIncrementalSyncAt, origin, String(value));
|
|
88
|
+
};
|
|
93
89
|
|
|
94
90
|
try {
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
if (!page.length) break;
|
|
91
|
+
let page = source.incrementalPage(level, afterId, batchSize);
|
|
92
|
+
while (page.length) {
|
|
98
93
|
stats.batches += 1;
|
|
99
|
-
|
|
94
|
+
const tailId = page[page.length - 1].id;
|
|
100
95
|
stats.scanned += page.length;
|
|
101
|
-
for (const
|
|
96
|
+
for (const row of page) if (row.updated_at > high) high = row.updated_at;
|
|
102
97
|
|
|
103
|
-
const
|
|
104
|
-
|
|
105
|
-
for (const r of page) {
|
|
106
|
-
const h = textHash(r.text);
|
|
107
|
-
if (hashes.get(String(r.id)) === h) {
|
|
108
|
-
stats.skippedSameHash += 1;
|
|
109
|
-
continue;
|
|
110
|
-
}
|
|
111
|
-
need.push({ row: r, hash: h });
|
|
112
|
-
}
|
|
98
|
+
const { pending, skipped } = splitBatch(page, vectors.getHashes(conn, origin, page.map((row) => row.id)));
|
|
99
|
+
stats.skippedSameHash += skipped;
|
|
113
100
|
|
|
114
|
-
if (
|
|
115
|
-
const
|
|
116
|
-
if (
|
|
117
|
-
throw new Error(`编码器返回数量不符:期望 ${
|
|
101
|
+
if (pending.length) {
|
|
102
|
+
const encoded = await embedder.encodeBatch(pending.map((item) => item.row.text));
|
|
103
|
+
if (encoded.length !== pending.length) {
|
|
104
|
+
throw new Error(`编码器返回数量不符:期望 ${pending.length},实际 ${encoded.length}`);
|
|
118
105
|
}
|
|
119
106
|
if (!dryRun) {
|
|
120
|
-
upsertVectors(
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
updatedAt: n.row.updated_at,
|
|
130
|
-
})),
|
|
131
|
-
);
|
|
107
|
+
vectors.upsertVectors(conn, pending.map((item, at) => ({
|
|
108
|
+
source: origin,
|
|
109
|
+
refId: item.row.id,
|
|
110
|
+
model: modelId,
|
|
111
|
+
dim: encoded[at].length,
|
|
112
|
+
vec: encoded[at],
|
|
113
|
+
textHash: item.digest,
|
|
114
|
+
updatedAt: item.row.updated_at,
|
|
115
|
+
})));
|
|
132
116
|
}
|
|
133
|
-
stats.encoded +=
|
|
134
|
-
stats.upserted +=
|
|
117
|
+
stats.encoded += pending.length;
|
|
118
|
+
stats.upserted += pending.length;
|
|
135
119
|
}
|
|
136
120
|
|
|
137
|
-
//
|
|
138
|
-
if (!dryRun)
|
|
139
|
-
|
|
140
|
-
setMetaFor(db, META_KEYS.dim, src, String(dim));
|
|
141
|
-
if (!metaHost) setMeta(db, META_KEYS.hostId, String(store.hostId || ''));
|
|
142
|
-
setMetaFor(db, META_KEYS.lastIncrementalSyncAt, src, String(maxSeenUpdatedAt));
|
|
143
|
-
}
|
|
144
|
-
log(
|
|
145
|
-
`[sync] 批 ${stats.batches}:扫 ${page.length} 条(新编码 ${need.length} / 同哈希跳过 ${
|
|
146
|
-
page.length - need.length
|
|
147
|
-
})水位=${maxSeenUpdatedAt}`,
|
|
148
|
-
);
|
|
121
|
+
// 进度前移取「本批扫到的最大 updated_at」:边界条下轮会再扫一遍,靠同哈希幂等兜住。
|
|
122
|
+
if (!dryRun) advanceLevel(high);
|
|
123
|
+
log(`[sync] 批 ${stats.batches}:扫 ${page.length} 条(新编码 ${pending.length} / 同哈希跳过 ${page.length - pending.length})水位=${high}`);
|
|
149
124
|
if (!untilEmpty) break;
|
|
125
|
+
afterId = tailId;
|
|
126
|
+
page = source.incrementalPage(level, afterId, batchSize);
|
|
150
127
|
}
|
|
151
|
-
stats.watermarkEnd =
|
|
128
|
+
stats.watermarkEnd = high;
|
|
152
129
|
} catch (e) {
|
|
153
|
-
stats.failures.push({ stage: 'sync', message:
|
|
154
|
-
|
|
130
|
+
stats.failures.push({ stage: 'sync', message: errText(e), atBatch: stats.batches });
|
|
131
|
+
// 已成功批次的进度早就落盘了,这里只是把读数对齐
|
|
132
|
+
stats.watermarkEnd = high;
|
|
155
133
|
stats.error = stats.failures[0].message;
|
|
156
134
|
}
|
|
157
135
|
|
|
158
|
-
stats.vectorsAfter = countVectors(
|
|
159
|
-
|
|
160
|
-
stats.
|
|
136
|
+
stats.vectorsAfter = vectors.countVectors(conn, origin);
|
|
137
|
+
const finishedAt = now();
|
|
138
|
+
stats.finishedAt = finishedAt;
|
|
139
|
+
stats.elapsedMs = finishedAt - stats.startedAt;
|
|
161
140
|
if (!dryRun) {
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
scanned: stats.scanned,
|
|
170
|
-
encoded: stats.encoded,
|
|
171
|
-
skippedSameHash: stats.skippedSameHash,
|
|
172
|
-
watermarkEnd: stats.watermarkEnd,
|
|
173
|
-
elapsedMs: stats.elapsedMs,
|
|
174
|
-
failures: stats.failures.length,
|
|
175
|
-
at: new Date(stats.finishedAt).toISOString(),
|
|
176
|
-
}),
|
|
177
|
-
);
|
|
141
|
+
const brief = {
|
|
142
|
+
source: origin, mode: stats.mode, scanned: stats.scanned,
|
|
143
|
+
encoded: stats.encoded, skippedSameHash: stats.skippedSameHash,
|
|
144
|
+
watermarkEnd: stats.watermarkEnd, elapsedMs: stats.elapsedMs,
|
|
145
|
+
failures: stats.failures.length, at: new Date(finishedAt).toISOString(),
|
|
146
|
+
};
|
|
147
|
+
vectors.setMetaFor(conn, vectors.META_KEYS.lastSyncStats, origin, JSON.stringify(brief));
|
|
178
148
|
}
|
|
179
149
|
return stats;
|
|
180
150
|
}
|