@modusensus/dsh-mneme 0.3.5 → 0.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -6
- package/lib/api.js +30 -8
- package/lib/index.js +27 -17
- package/lib/local-embedder.js +7 -1
- package/lib/service.js +154 -22
- package/lib/store.js +142 -26
- package/package.json +1 -1
- package/src/api.js +30 -8
- package/src/index.js +27 -17
- package/src/local-embedder.js +7 -1
- package/src/service.js +154 -22
- package/src/store.js +142 -26
- package/test/fnew-03.test.js +11 -4
- package/test/mirror-dirty.test.js +12 -8
- package/test/mirror-generation.test.js +463 -0
- package/test/recall-layer.test.js +1 -0
- package/test/semantic.test.js +1 -0
- package/test/service-search.test.js +1 -0
package/lib/store.js
CHANGED
|
@@ -169,12 +169,20 @@ CREATE INDEX IF NOT EXISTS idx_relations_type ON entity_relations(relation_type)
|
|
|
169
169
|
-- syncMirror 失败不再只靠瞬时 console.warn —— dirty=1 提示镜像脏了需重渲染,
|
|
170
170
|
-- last_error/last_attempt 记录失败原因与最近尝试,success_at 记录最近成功。
|
|
171
171
|
-- 上层可据此在启动时重试、提供人工 reconcile 入口与健康状态查询。
|
|
172
|
+
-- v0.3.6: 新增 generation/applied_generation/type_status —— desired-applied
|
|
173
|
+
-- 建模镜像债务:generation 是期望同步轮次,applied_generation 是已成功应用
|
|
174
|
+
-- 轮次(成功清 dirty 必须 CAS/fence 到具体轮次,旧 worker 不能清新故障),
|
|
175
|
+
-- type_status 逐 type 记录部分成功状态。旧库经 PRAGMA table_info 检查后
|
|
176
|
+
-- ALTER 补列,幂等且不丢数据。
|
|
172
177
|
CREATE TABLE IF NOT EXISTS mirror_state (
|
|
173
178
|
id TEXT PRIMARY KEY, -- 单一状态行(用 'main')
|
|
174
179
|
dirty INTEGER NOT NULL DEFAULT 0, -- 1=镜像脏了需重渲染
|
|
175
180
|
last_error TEXT, -- 最近失败原因
|
|
176
181
|
last_attempt TEXT, -- 最近尝试时间(ISO)
|
|
177
|
-
success_at TEXT
|
|
182
|
+
success_at TEXT, -- 最近成功时间(ISO)
|
|
183
|
+
generation INTEGER NOT NULL DEFAULT 0, -- 期望的同步轮次(desired)
|
|
184
|
+
applied_generation INTEGER NOT NULL DEFAULT 0, -- 已成功应用的轮次
|
|
185
|
+
type_status TEXT -- JSON: 逐 type 状态 {type: {dirty, applied_gen, last_error}}
|
|
178
186
|
);
|
|
179
187
|
`;
|
|
180
188
|
|
|
@@ -337,14 +345,33 @@ function toRelation(row) {
|
|
|
337
345
|
|
|
338
346
|
function toMirrorState(row) {
|
|
339
347
|
if (!row) {
|
|
340
|
-
return {
|
|
348
|
+
return {
|
|
349
|
+
dirty: false,
|
|
350
|
+
last_error: null,
|
|
351
|
+
last_attempt: null,
|
|
352
|
+
success_at: null,
|
|
353
|
+
generation: 0,
|
|
354
|
+
applied_generation: 0,
|
|
355
|
+
type_status: {}
|
|
356
|
+
};
|
|
357
|
+
}
|
|
358
|
+
let typeStatus = {};
|
|
359
|
+
if (row.type_status) {
|
|
360
|
+
try {
|
|
361
|
+
typeStatus = JSON.parse(row.type_status) || {};
|
|
362
|
+
} catch {
|
|
363
|
+
typeStatus = {};
|
|
364
|
+
}
|
|
341
365
|
}
|
|
342
366
|
return {
|
|
343
367
|
id: row.id,
|
|
344
368
|
dirty: row.dirty === 1,
|
|
345
369
|
last_error: row.last_error,
|
|
346
370
|
last_attempt: row.last_attempt,
|
|
347
|
-
success_at: row.success_at
|
|
371
|
+
success_at: row.success_at,
|
|
372
|
+
generation: Number(row.generation) || 0,
|
|
373
|
+
applied_generation: Number(row.applied_generation) || 0,
|
|
374
|
+
type_status: typeStatus
|
|
348
375
|
};
|
|
349
376
|
}
|
|
350
377
|
|
|
@@ -377,6 +404,19 @@ export function createStore(path) {
|
|
|
377
404
|
db.exec("ALTER TABLE dream_runs ADD COLUMN policy_epoch INTEGER NOT NULL DEFAULT 0");
|
|
378
405
|
}
|
|
379
406
|
|
|
407
|
+
// Legacy mirror_state without v0.3.6 generation columns → add each missing
|
|
408
|
+
// column idempotently (old DBs open cleanly, no data loss).
|
|
409
|
+
const mirrorCols = db.prepare("PRAGMA table_info(mirror_state)").all().map((c) => c.name);
|
|
410
|
+
if (!mirrorCols.includes("generation")) {
|
|
411
|
+
db.exec("ALTER TABLE mirror_state ADD COLUMN generation INTEGER NOT NULL DEFAULT 0");
|
|
412
|
+
}
|
|
413
|
+
if (!mirrorCols.includes("applied_generation")) {
|
|
414
|
+
db.exec("ALTER TABLE mirror_state ADD COLUMN applied_generation INTEGER NOT NULL DEFAULT 0");
|
|
415
|
+
}
|
|
416
|
+
if (!mirrorCols.includes("type_status")) {
|
|
417
|
+
db.exec("ALTER TABLE mirror_state ADD COLUMN type_status TEXT");
|
|
418
|
+
}
|
|
419
|
+
|
|
380
420
|
// Per-instance monotonic timestamp guard: consecutive writes within the same
|
|
381
421
|
// millisecond must still produce strictly increasing timestamps (test asserts
|
|
382
422
|
// updated_at != created_at). State lives in the store closure, not module scope.
|
|
@@ -1113,14 +1153,25 @@ export function createStore(path) {
|
|
|
1113
1153
|
|
|
1114
1154
|
/**
|
|
1115
1155
|
* Upsert the single mirror_state row (id='main'). patch accepts
|
|
1116
|
-
* {dirty?, last_error?, last_attempt?, success_at
|
|
1117
|
-
*
|
|
1118
|
-
*
|
|
1156
|
+
* {dirty?, last_error?, last_attempt?, success_at?, generation?,
|
|
1157
|
+
* applied_generation?, type_status?} — only the keys present on the object
|
|
1158
|
+
* are written, everything else is left untouched (partial upsert). type_status
|
|
1159
|
+
* is stored as JSON text (objects are serialized on write), generation /
|
|
1160
|
+
* applied_generation are coerced to non-negative integers. Returns the freshly
|
|
1161
|
+
* read state row (default shape when absent).
|
|
1119
1162
|
*/
|
|
1120
1163
|
function setMirrorState(patch) {
|
|
1121
|
-
const ALLOWED = new Set([
|
|
1122
|
-
|
|
1123
|
-
|
|
1164
|
+
const ALLOWED = new Set([
|
|
1165
|
+
"dirty",
|
|
1166
|
+
"last_error",
|
|
1167
|
+
"last_attempt",
|
|
1168
|
+
"success_at",
|
|
1169
|
+
"generation",
|
|
1170
|
+
"applied_generation",
|
|
1171
|
+
"type_status"
|
|
1172
|
+
]);
|
|
1173
|
+
const keys = Object.keys(patch).filter(
|
|
1174
|
+
(key) => ALLOWED.has(key) && Object.prototype.hasOwnProperty.call(patch, key)
|
|
1124
1175
|
);
|
|
1125
1176
|
if (keys.length === 0) {
|
|
1126
1177
|
db.prepare(
|
|
@@ -1130,41 +1181,78 @@ export function createStore(path) {
|
|
|
1130
1181
|
}
|
|
1131
1182
|
// 列同时出现在 INSERT 与 ON CONFLICT 里(excluded.*),保证首次插入也写入
|
|
1132
1183
|
// patch 值,而不只是默认值;未传入的列保持不变(partial upsert)。
|
|
1133
|
-
const cols =
|
|
1134
|
-
const
|
|
1135
|
-
const updates =
|
|
1136
|
-
const
|
|
1137
|
-
|
|
1138
|
-
|
|
1184
|
+
const cols = [];
|
|
1185
|
+
const values = [];
|
|
1186
|
+
const updates = [];
|
|
1187
|
+
for (const key of keys) {
|
|
1188
|
+
let value = patch[key];
|
|
1189
|
+
if (key === "dirty") {
|
|
1190
|
+
value = value ? 1 : 0;
|
|
1191
|
+
} else if (key === "generation" || key === "applied_generation") {
|
|
1192
|
+
value = Math.trunc(Number(value)) || 0;
|
|
1193
|
+
} else if (key === "type_status" && value != null && typeof value !== "string") {
|
|
1194
|
+
value = JSON.stringify(value);
|
|
1195
|
+
}
|
|
1196
|
+
cols.push(key);
|
|
1197
|
+
values.push(value);
|
|
1198
|
+
updates.push(`${key} = excluded.${key}`);
|
|
1199
|
+
}
|
|
1200
|
+
const placeholders = cols.map(() => "?").join(", ");
|
|
1139
1201
|
db.prepare(
|
|
1140
|
-
`INSERT INTO mirror_state (id, ${cols}) VALUES ('main', ${placeholders})
|
|
1141
|
-
ON CONFLICT(id) DO UPDATE SET ${updates}`
|
|
1202
|
+
`INSERT INTO mirror_state (id, ${cols.join(", ")}) VALUES ('main', ${placeholders})
|
|
1203
|
+
ON CONFLICT(id) DO UPDATE SET ${updates.join(", ")}`
|
|
1142
1204
|
).run(...values);
|
|
1143
1205
|
return getMirrorState();
|
|
1144
1206
|
}
|
|
1145
1207
|
|
|
1146
|
-
/** Current mirror state; default {dirty:0, last_error:null, last_attempt:null, success_at:null} when absent. */
|
|
1208
|
+
/** Current mirror state; default {dirty:0, last_error:null, last_attempt:null, success_at:null, generation:0, applied_generation:0, type_status:{}} when absent. */
|
|
1147
1209
|
function getMirrorState() {
|
|
1148
1210
|
const row = db.prepare("SELECT * FROM mirror_state WHERE id = 'main'").get();
|
|
1149
1211
|
return toMirrorState(row);
|
|
1150
1212
|
}
|
|
1151
1213
|
|
|
1152
|
-
/**
|
|
1214
|
+
/**
|
|
1215
|
+
* Mark the mirror dirty after a failed sync (dirty=1 + last_error +
|
|
1216
|
+
* last_attempt). v0.3.6: also bumps the desired generation so the debt is
|
|
1217
|
+
* bound to a specific sync round; applied_generation is left untouched
|
|
1218
|
+
* (the round was NOT applied). A stale worker that started earlier cannot
|
|
1219
|
+
* clear this newer debt — only a clean fenced to a generation at least as
|
|
1220
|
+
* recent as this one may.
|
|
1221
|
+
*/
|
|
1153
1222
|
function markMirrorDirty(error, now) {
|
|
1223
|
+
const current = getMirrorState();
|
|
1154
1224
|
return setMirrorState({
|
|
1155
1225
|
dirty: 1,
|
|
1156
1226
|
last_error: error,
|
|
1157
|
-
last_attempt: now ?? nowIso()
|
|
1227
|
+
last_attempt: now ?? nowIso(),
|
|
1228
|
+
generation: (current.generation || 0) + 1
|
|
1158
1229
|
});
|
|
1159
1230
|
}
|
|
1160
1231
|
|
|
1161
|
-
/**
|
|
1232
|
+
/**
|
|
1233
|
+
* Fenced clean (CAS): mark the mirror clean for a specific generation.
|
|
1234
|
+
* First records that generation `gen` has been applied
|
|
1235
|
+
* (applied_generation = MAX(applied_generation, gen)), then clears dirty only
|
|
1236
|
+
* when the current desired generation has not advanced past gen — a stale
|
|
1237
|
+
* worker cleaning an older round must not wipe a newer failure's debt.
|
|
1238
|
+
* Returns the resulting state (dirty stays set when the fence holds).
|
|
1239
|
+
*/
|
|
1240
|
+
function markMirrorCleanForGeneration(gen, now) {
|
|
1241
|
+
const current = getMirrorState();
|
|
1242
|
+
const applied = Math.max(current.applied_generation || 0, gen);
|
|
1243
|
+
const patch = { applied_generation: applied };
|
|
1244
|
+
if (applied >= gen && (current.generation || 0) <= gen) {
|
|
1245
|
+
patch.dirty = 0;
|
|
1246
|
+
patch.last_error = null;
|
|
1247
|
+
patch.success_at = now ?? nowIso();
|
|
1248
|
+
}
|
|
1249
|
+
return setMirrorState(patch);
|
|
1250
|
+
}
|
|
1251
|
+
|
|
1252
|
+
/** Convenience: mark the mirror clean for the current desired generation (backward-compatible with pre-v0.3.6 callers). */
|
|
1162
1253
|
function markMirrorClean(now) {
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
last_error: null,
|
|
1166
|
-
success_at: now ?? nowIso()
|
|
1167
|
-
});
|
|
1254
|
+
const current = getMirrorState();
|
|
1255
|
+
return markMirrorCleanForGeneration(current.generation || 0, now);
|
|
1168
1256
|
}
|
|
1169
1257
|
|
|
1170
1258
|
/** Convenience: clear only the dirty flag + last_error, leaving success_at untouched (manual reconcile / retry path). */
|
|
@@ -1172,6 +1260,30 @@ export function createStore(path) {
|
|
|
1172
1260
|
return setMirrorState({ dirty: 0, last_error: null });
|
|
1173
1261
|
}
|
|
1174
1262
|
|
|
1263
|
+
/**
|
|
1264
|
+
* Record per-type mirror status (partial success bookkeeping). `status` is a
|
|
1265
|
+
* partial patch {dirty?, applied_gen?, last_error?} merged into the existing
|
|
1266
|
+
* entry for `type` (other types untouched). Returns the updated full state.
|
|
1267
|
+
*/
|
|
1268
|
+
function setTypeStatus(type, status) {
|
|
1269
|
+
const current = getMirrorState();
|
|
1270
|
+
const statuses = current.type_status || {};
|
|
1271
|
+
statuses[type] = { ...(statuses[type] || {}), ...status };
|
|
1272
|
+
return setMirrorState({ type_status: JSON.stringify(statuses) });
|
|
1273
|
+
}
|
|
1274
|
+
|
|
1275
|
+
/** Per-type mirror status map {type: {dirty, applied_gen, last_error}}, {} when unset. */
|
|
1276
|
+
function getTypeStatus() {
|
|
1277
|
+
const current = getMirrorState();
|
|
1278
|
+
return current.type_status || {};
|
|
1279
|
+
}
|
|
1280
|
+
|
|
1281
|
+
/** Bump the desired generation (new sync round), returning the new state. */
|
|
1282
|
+
function incrementGeneration() {
|
|
1283
|
+
const current = getMirrorState();
|
|
1284
|
+
return setMirrorState({ generation: (current.generation || 0) + 1 });
|
|
1285
|
+
}
|
|
1286
|
+
|
|
1175
1287
|
return {
|
|
1176
1288
|
db,
|
|
1177
1289
|
count,
|
|
@@ -1224,7 +1336,11 @@ export function createStore(path) {
|
|
|
1224
1336
|
getMirrorState,
|
|
1225
1337
|
markMirrorDirty,
|
|
1226
1338
|
markMirrorClean,
|
|
1339
|
+
markMirrorCleanForGeneration,
|
|
1227
1340
|
clearMirrorDirty,
|
|
1341
|
+
setTypeStatus,
|
|
1342
|
+
getTypeStatus,
|
|
1343
|
+
incrementGeneration,
|
|
1228
1344
|
close() {
|
|
1229
1345
|
db.close();
|
|
1230
1346
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@modusensus/dsh-mneme",
|
|
3
3
|
"description": "Cross-session memory plugin for DeepSeek Harness with autoDream consolidation: SQLite store, Markdown mirrors, 6 model tools, automatic injection, session summarization, user profile/rules, custom slash commands, vector (semantic) search, and a Web GUI panel",
|
|
4
|
-
"version": "0.3.
|
|
4
|
+
"version": "0.3.7",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"main": "lib/index.js",
|
package/src/api.js
CHANGED
|
@@ -268,21 +268,43 @@ export function createApi(ctx, service, settings, commands, embedder, semantic =
|
|
|
268
268
|
}
|
|
269
269
|
});
|
|
270
270
|
|
|
271
|
-
// --- health: mirror sync state (F-NEW-03) ---
|
|
271
|
+
// --- health: mirror sync state (F-NEW-03 / v0.3.6) ---
|
|
272
|
+
// Auth-gated; only returns a sanitized error code (never raw last_error which
|
|
273
|
+
// may leak paths/token-like strings/internal hosts). On state read failure it
|
|
274
|
+
// reports unknown/degraded (fail-closed) instead of a false dirty=false.
|
|
272
275
|
register({
|
|
273
276
|
kind: "exact",
|
|
274
277
|
path: "/api/dsh-mneme/health",
|
|
275
278
|
handler(req, res) {
|
|
279
|
+
if (!requireAuth(req, res, apiToken)) return;
|
|
280
|
+
let state = null;
|
|
276
281
|
try {
|
|
277
|
-
|
|
278
|
-
sendJson(res, 200, {
|
|
279
|
-
mirror: state
|
|
280
|
-
? { dirty: state.dirty === true, last_error: state.last_error ?? null, last_attempt: state.last_attempt ?? null, success_at: state.success_at ?? null }
|
|
281
|
-
: null
|
|
282
|
-
});
|
|
282
|
+
state = service.getMirrorHealth?.() ?? null;
|
|
283
283
|
} catch {
|
|
284
|
-
|
|
284
|
+
// read failure is itself a health signal: do not report a false clean
|
|
285
|
+
sendJson(res, 200, { mirror: { dirty: null, status: "unknown", last_error: null, last_attempt: null, success_at: null } });
|
|
286
|
+
return;
|
|
285
287
|
}
|
|
288
|
+
if (!state) {
|
|
289
|
+
sendJson(res, 200, { mirror: { dirty: null, status: "unknown", last_error: null, last_attempt: null, success_at: null } });
|
|
290
|
+
return;
|
|
291
|
+
}
|
|
292
|
+
// Sanitized: boolean dirty + coarse status only; error string is mapped to
|
|
293
|
+
// a bounded code, never echoed verbatim.
|
|
294
|
+
let code = null;
|
|
295
|
+
if (state.last_error) {
|
|
296
|
+
const e = String(state.last_error);
|
|
297
|
+
code = /enospc|no space/i.test(e) ? "no-space" : /permission|eacces/i.test(e) ? "permission" : "sync-failed";
|
|
298
|
+
}
|
|
299
|
+
sendJson(res, 200, {
|
|
300
|
+
mirror: {
|
|
301
|
+
dirty: state.dirty === true,
|
|
302
|
+
status: state.dirty === true ? "degraded" : (code ? "degraded" : "ok"),
|
|
303
|
+
last_error: code,
|
|
304
|
+
last_attempt: state.last_attempt ?? null,
|
|
305
|
+
success_at: state.success_at ?? null
|
|
306
|
+
}
|
|
307
|
+
});
|
|
286
308
|
}
|
|
287
309
|
});
|
|
288
310
|
|
package/src/index.js
CHANGED
|
@@ -80,11 +80,28 @@ export const apply = (ctx, config) => {
|
|
|
80
80
|
const vectorIndex = createVectorIndex({ store, logger: ctx.logger });
|
|
81
81
|
service.setVectorIndex(vectorIndex);
|
|
82
82
|
|
|
83
|
+
// Human edits in mirror files win on every sync; merge them back first.
|
|
84
|
+
// TYPE_FILE maps each memory type to its mirror filename. Read every type's
|
|
85
|
+
// edits up front: mergeHumanEdits re-renders ALL mirror files on success, so
|
|
86
|
+
// a per-type read-then-merge loop would overwrite edits in files not yet read
|
|
87
|
+
// (e.g. preferences.md merging would clobber unsynced projects.md edits).
|
|
88
|
+
const humanEdits = new Map();
|
|
89
|
+
for (const type of Object.keys(TYPE_FILE)) {
|
|
90
|
+
humanEdits.set(type, mirror.readHumanEdits(type));
|
|
91
|
+
}
|
|
92
|
+
const applyHumanEdits = () => {
|
|
93
|
+
for (const [type, edits] of humanEdits) {
|
|
94
|
+
if (edits.length) service.mergeHumanEdits(type, edits);
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
|
|
83
98
|
let embedder = null;
|
|
84
99
|
let reranker = null;
|
|
85
100
|
if (cfg.embedProvider === "openai") {
|
|
86
101
|
embedder = createEmbedder({ store, settings, logger: ctx.logger });
|
|
87
102
|
service.setEmbedder(embedder);
|
|
103
|
+
// legacy OpenAI embedder is immediately usable
|
|
104
|
+
applyHumanEdits();
|
|
88
105
|
} else {
|
|
89
106
|
try {
|
|
90
107
|
embedder = createEmbedderByProvider(cfg.embedProvider, {
|
|
@@ -97,12 +114,18 @@ export const apply = (ctx, config) => {
|
|
|
97
114
|
logger: ctx.logger
|
|
98
115
|
});
|
|
99
116
|
service.setEmbedder(embedder);
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
117
|
+
// issue #6: wait for extractor init before applying human edits, so
|
|
118
|
+
// scheduled embeddings see a ready embedder.
|
|
119
|
+
embedder.init()
|
|
120
|
+
.then(() => applyHumanEdits())
|
|
121
|
+
.catch((error) => {
|
|
122
|
+
ctx.logger?.warn?.(`[dsh-mneme] embedder init failed, search degrades to keyword: ${String(error)}`);
|
|
123
|
+
service.setEmbedder(null);
|
|
124
|
+
applyHumanEdits();
|
|
125
|
+
});
|
|
104
126
|
} catch (error) {
|
|
105
127
|
ctx.logger?.warn?.(`[dsh-mneme] embedder unavailable, search degrades to keyword: ${String(error)}`);
|
|
128
|
+
applyHumanEdits();
|
|
106
129
|
}
|
|
107
130
|
}
|
|
108
131
|
|
|
@@ -139,19 +162,6 @@ export const apply = (ctx, config) => {
|
|
|
139
162
|
commands.sync();
|
|
140
163
|
}
|
|
141
164
|
|
|
142
|
-
// Human edits in mirror files win on every sync; merge them back first.
|
|
143
|
-
// TYPE_FILE maps each memory type to its mirror filename. Read every type's
|
|
144
|
-
// edits up front: mergeHumanEdits re-renders ALL mirror files on success, so
|
|
145
|
-
// a per-type read-then-merge loop would overwrite edits in files not yet read
|
|
146
|
-
// (e.g. preferences.md merging would clobber unsynced projects.md edits).
|
|
147
|
-
const humanEdits = new Map();
|
|
148
|
-
for (const type of Object.keys(TYPE_FILE)) {
|
|
149
|
-
humanEdits.set(type, mirror.readHumanEdits(type));
|
|
150
|
-
}
|
|
151
|
-
for (const [type, edits] of humanEdits) {
|
|
152
|
-
if (edits.length) service.mergeHumanEdits(type, edits);
|
|
153
|
-
}
|
|
154
|
-
|
|
155
165
|
// Dream scheduler: automatic consolidation + summary runs, triggered by
|
|
156
166
|
// store growth. Writes through the service fire the dream hook, which asks
|
|
157
167
|
// the scheduler to (re)schedule a run once absolute and since-last-run
|
package/src/local-embedder.js
CHANGED
|
@@ -58,16 +58,21 @@ export class LocalEmbedder {
|
|
|
58
58
|
// Test hook: replace the pipeline factory without touching modules.
|
|
59
59
|
this.engineFactory = opts.engineFactory || defaultPipelineLoader;
|
|
60
60
|
this.extractor = null;
|
|
61
|
+
// issue #6: readiness flag for the service's scheduleEmbed gate. False until
|
|
62
|
+
// init() succeeds, so "ready" in embedder is observable even pre-init.
|
|
63
|
+
this.ready = false;
|
|
61
64
|
}
|
|
62
65
|
|
|
63
|
-
/** Load the model; throws when it cannot be loaded. */
|
|
66
|
+
/** Load the model; throws when it cannot be loaded. Idempotent. */
|
|
64
67
|
async init() {
|
|
68
|
+
if (this.extractor) return this; // already initialized: no-op
|
|
65
69
|
const options = {
|
|
66
70
|
dtype: this.useDtype,
|
|
67
71
|
device: this.device
|
|
68
72
|
};
|
|
69
73
|
if (this.cacheDir) options.cache_dir = this.cacheDir;
|
|
70
74
|
this.extractor = await this.engineFactory("feature-extraction", this.model, options);
|
|
75
|
+
this.ready = true; // service reads this to flush queued re-embeds
|
|
71
76
|
this.logger?.info?.(
|
|
72
77
|
`[dsh-mneme] local embedder ready: ${this.model} (dim=${this._dimension}, device=${this.device})`
|
|
73
78
|
);
|
|
@@ -107,6 +112,7 @@ export class LocalEmbedder {
|
|
|
107
112
|
// best-effort: some engines free resources on GC
|
|
108
113
|
}
|
|
109
114
|
this.extractor = null;
|
|
115
|
+
this.ready = false;
|
|
110
116
|
}
|
|
111
117
|
}
|
|
112
118
|
|
package/src/service.js
CHANGED
|
@@ -37,11 +37,69 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
37
37
|
// replays them exactly once against the committed state.
|
|
38
38
|
let txDepth = 0;
|
|
39
39
|
|
|
40
|
+
// issue #6 (part 2): startup race defense. A local embedder (LocalEmbedder /
|
|
41
|
+
// Ollama) exposes an async init(), so between `setEmbedder` and init()
|
|
42
|
+
// resolving there is a window where embedSingle would throw "not initialized"
|
|
43
|
+
// and the re-embed would be silently dropped. When the embedder carries a
|
|
44
|
+
// `ready` flag we queue writes in embedPending until init sets ready=true,
|
|
45
|
+
// then flush them through the embedder's real interface. Embedders without a
|
|
46
|
+
// `ready` flag (legacy OpenAI, instantly usable) keep their old behavior.
|
|
47
|
+
let embedPending = [];
|
|
48
|
+
let embedReadyTimer = null;
|
|
49
|
+
const EMBED_PENDING_MAX = 100; // bound the queue; drop oldest beyond this
|
|
50
|
+
const EMBED_READY_POLL_MS = 100;
|
|
51
|
+
const EMBED_READY_POLL_LIMIT = 30; // ~3s ceiling; never poll forever
|
|
52
|
+
|
|
53
|
+
/** Flush the queued re-embeds once the embedder is ready. Fail-safe. */
|
|
54
|
+
function flushEmbedPending() {
|
|
55
|
+
if (!embedder || embedPending.length === 0) return;
|
|
56
|
+
const batch = embedPending.splice(0, embedPending.length);
|
|
57
|
+
for (const memory of batch) {
|
|
58
|
+
try {
|
|
59
|
+
if (!memory?.id) continue;
|
|
60
|
+
if (typeof embedder.schedule === "function") {
|
|
61
|
+
embedder.schedule(memory);
|
|
62
|
+
} else if (typeof embedder.embedSingle === "function") {
|
|
63
|
+
const text = [memory.title, memory.content].filter(Boolean).join("\n");
|
|
64
|
+
if (!text) continue;
|
|
65
|
+
embedder
|
|
66
|
+
.embedSingle(text)
|
|
67
|
+
.then((vec) => {
|
|
68
|
+
if (Array.isArray(vec) && vec.length) {
|
|
69
|
+
store.setEmbedding(memory.id, vec);
|
|
70
|
+
}
|
|
71
|
+
})
|
|
72
|
+
.catch((err) => {
|
|
73
|
+
logger?.warn?.("flushEmbedPending embedSingle failed:", err);
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
} catch (err) {
|
|
77
|
+
logger?.warn?.("flushEmbedPending failed:", err);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function stopEmbedReadyPolling() {
|
|
83
|
+
if (embedReadyTimer) {
|
|
84
|
+
clearInterval(embedReadyTimer);
|
|
85
|
+
embedReadyTimer = null;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
40
89
|
function scheduleEmbed(memory) {
|
|
41
90
|
try {
|
|
42
91
|
if (txDepth > 0) return; // deferred to the transaction's commit
|
|
43
92
|
if (!embedder || !memory?.id) return;
|
|
44
93
|
|
|
94
|
+
// Readiness gate: embedder exposes `ready` (async init) and is not ready
|
|
95
|
+
// yet — queue instead of firing embedSingle into a half-built extractor.
|
|
96
|
+
const hasReady = "ready" in embedder;
|
|
97
|
+
if (hasReady && embedder.ready !== true) {
|
|
98
|
+
if (embedPending.length >= EMBED_PENDING_MAX) embedPending.shift();
|
|
99
|
+
embedPending.push(memory);
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
|
|
45
103
|
if (typeof embedder.schedule === "function") {
|
|
46
104
|
embedder.schedule(memory);
|
|
47
105
|
return;
|
|
@@ -110,7 +168,7 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
110
168
|
|
|
111
169
|
/**
|
|
112
170
|
* Search for memories attached to a named entity (v0.3.0 Phase 3).
|
|
113
|
-
*
|
|
171
|
+
* 合并优先级:entity_attrs.memory_id 精确关联 = 1.0 > 关键词提及 = 0.7;
|
|
114
172
|
* attr 命中不覆盖,keyword 只补充召回,最后按 _score 降序取 topK。
|
|
115
173
|
* @param {string} entityName
|
|
116
174
|
* @param {object} [options]
|
|
@@ -511,38 +569,82 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
511
569
|
* human-editable file (a human "edit" could otherwise resurrect them).
|
|
512
570
|
*/
|
|
513
571
|
// syncMirror: 同步 mirror,并在失败/成功时持久记录 dirty 状态;保证自身不抛出。
|
|
514
|
-
//
|
|
515
|
-
//
|
|
516
|
-
//
|
|
572
|
+
// v0.3.6(audit peer 4 阻断):
|
|
573
|
+
// - 开始时 incrementGeneration 绑定本次期望轮次 gen;成功用
|
|
574
|
+
// markMirrorCleanForGeneration(gen, now) CAS/fence 清 dirty——旧 worker
|
|
575
|
+
// (gen 已过期)不会误清另一 worker 未恢复的故障债务;
|
|
576
|
+
// - 失败写 markMirrorDirty(递增 desired 绑定新债务),下次 recover 恢复;
|
|
577
|
+
// - 逐 type 用 setTypeStatus 记录部分成功/失败(type_status JSON);
|
|
578
|
+
// - 所有 store 状态写入各自 try/catch,失败只 warn,绝不向外抛(F-NEW-03)。
|
|
517
579
|
function syncMirror() {
|
|
518
580
|
if (txDepth > 0 || !mirror) return; // deferred to the transaction's commit
|
|
519
581
|
const now = new Date().toISOString();
|
|
582
|
+
let gen;
|
|
520
583
|
try {
|
|
521
|
-
|
|
522
|
-
|
|
584
|
+
// 绑定本次期望轮次,必须在任何渲染之前,避免制造幽灵债务
|
|
585
|
+
const state = store.incrementGeneration();
|
|
586
|
+
gen = state.generation;
|
|
587
|
+
} catch (stateError) {
|
|
588
|
+
logger?.warn?.("syncMirror: incrementGeneration failed:", stateError);
|
|
589
|
+
return;
|
|
590
|
+
}
|
|
591
|
+
// coveredTypes 提到 try 外初始化:即使 store.list 先抛错,catch 分支也有
|
|
592
|
+
// 合法的空 Set 可迭代,保证 syncMirror 自身绝不抛(fail-safe)。
|
|
593
|
+
const coveredTypes = new Set();
|
|
594
|
+
try {
|
|
595
|
+
// 预先获取本次要覆盖的 type 集合(只调一次 store.list)
|
|
596
|
+
const list = store.list({ limit: 500, includeForgotten: false });
|
|
597
|
+
for (const memory of list) {
|
|
598
|
+
if (memory?.type && TYPE_FILE[memory.type]) {
|
|
599
|
+
coveredTypes.add(memory.type);
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
// 全量渲染
|
|
604
|
+
mirror.sync(reconcileHumanEdits(list));
|
|
605
|
+
|
|
606
|
+
// 成功:CAS/fence 绑定到本地 gen,旧 worker(gen 已过期)会被拦截
|
|
523
607
|
try {
|
|
524
|
-
store.
|
|
608
|
+
store.markMirrorCleanForGeneration(gen, now);
|
|
525
609
|
} catch (stateError) {
|
|
526
|
-
|
|
527
|
-
|
|
610
|
+
logger?.warn?.("syncMirror: markMirrorCleanForGeneration failed:", stateError);
|
|
611
|
+
}
|
|
612
|
+
// 逐 type 标记为 clean
|
|
613
|
+
for (const type of coveredTypes) {
|
|
614
|
+
try {
|
|
615
|
+
store.setTypeStatus(type, { dirty: false, applied_gen: gen, last_error: null });
|
|
616
|
+
} catch (stateError) {
|
|
617
|
+
logger?.warn?.(`syncMirror: setTypeStatus(${type}) clean failed:`, stateError);
|
|
618
|
+
}
|
|
528
619
|
}
|
|
529
620
|
} catch (error) {
|
|
530
|
-
// 同步失败:写 dirty 状态
|
|
531
621
|
const errMsg = error?.message ?? String(error);
|
|
532
622
|
logger?.warn?.("syncMirror failed:", error);
|
|
533
623
|
try {
|
|
624
|
+
// 债务绑定到新的一轮(desired generation +1)
|
|
534
625
|
store.markMirrorDirty(errMsg, now);
|
|
535
626
|
} catch (stateError) {
|
|
536
|
-
// markMirrorDirty 失败同样不能向外抛出
|
|
537
627
|
logger?.warn?.("syncMirror: markMirrorDirty failed:", stateError);
|
|
538
628
|
}
|
|
629
|
+
// 逐 type 标记为 dirty(applied_gen 不动)
|
|
630
|
+
for (const type of coveredTypes) {
|
|
631
|
+
try {
|
|
632
|
+
store.setTypeStatus(type, { dirty: true, last_error: errMsg });
|
|
633
|
+
} catch (stateError) {
|
|
634
|
+
logger?.warn?.(`syncMirror: setTypeStatus(${type}) dirty failed:`, stateError);
|
|
635
|
+
}
|
|
636
|
+
}
|
|
539
637
|
}
|
|
540
638
|
}
|
|
541
639
|
|
|
542
640
|
// recoverMirror: 启动/手动 reconcile 时根据持久 dirty 状态决定是否恢复同步
|
|
543
|
-
// (F-NEW-03
|
|
544
|
-
//
|
|
545
|
-
//
|
|
641
|
+
// (F-NEW-03 + v0.3.6)。触发条件不只是 dirty——还检查
|
|
642
|
+
// generation > applied_generation(有未应用的债务),这样 COMMIT→dirty 崩溃
|
|
643
|
+
// 窗口(DB 提交后、markMirrorDirty/clean 前进程退出 → dirty=false 但
|
|
644
|
+
// generation 不一致)也能被捕获。有界重试(最多 3 次)重跑 syncMirror 收敛;
|
|
645
|
+
// 某次成功后 dirty=false 且无更新债务(generation <= applied_generation)
|
|
646
|
+
// 立即停止。返回 { recovered, error } 供 index.js 启动 / api.js health 判断。
|
|
647
|
+
// 一切 fail-safe,绝不向外抛。
|
|
546
648
|
function recoverMirror() {
|
|
547
649
|
const MAX_ATTEMPTS = 3;
|
|
548
650
|
let lastError = null;
|
|
@@ -550,24 +652,29 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
550
652
|
|
|
551
653
|
try {
|
|
552
654
|
const state = store.getMirrorState();
|
|
553
|
-
|
|
655
|
+
// 崩溃窗口检测:dirty 或 generation > applied_generation(COMMIT→dirty 窗口)
|
|
656
|
+
if (!state?.dirty && !(state.generation > state.applied_generation)) {
|
|
554
657
|
// 本来就干净:无需恢复,视为成功
|
|
555
658
|
return { recovered: true, error: null };
|
|
556
659
|
}
|
|
557
660
|
|
|
558
|
-
// dirty
|
|
661
|
+
// 有 dirty 或有未应用债务:最多尝试 3 次 sync
|
|
559
662
|
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|
|
560
663
|
try {
|
|
561
664
|
syncMirror(); // syncMirror 内部已 catch,不会向外抛
|
|
562
665
|
const currentState = store.getMirrorState();
|
|
563
|
-
|
|
564
|
-
|
|
666
|
+
// 成功条件:dirty 为 false 且没有更新一轮的债务
|
|
667
|
+
// (generation <= applied_generation,恢复后由 syncMirror 里
|
|
668
|
+
// markMirrorCleanForGeneration 自动把 applied 跟上)
|
|
669
|
+
if (!currentState?.dirty && currentState.generation <= currentState.applied_generation) {
|
|
565
670
|
recovered = true;
|
|
566
671
|
lastError = null;
|
|
567
672
|
break;
|
|
568
673
|
}
|
|
569
|
-
// 仍 dirty
|
|
570
|
-
|
|
674
|
+
// 仍 dirty 或仍有更新债务:记录最后一次错误供重试耗尽后上报。
|
|
675
|
+
// 注意:若别的 worker 又失败产生新债务(dirty 仍 true),这是"新债务"
|
|
676
|
+
// 不是本次失败,继续重试直到耗尽次数。
|
|
677
|
+
lastError = currentState.last_error || `Sync attempt ${attempt + 1} left mirror dirty or has pending debt`;
|
|
571
678
|
} catch (syncError) {
|
|
572
679
|
// syncMirror 理论不抛,fail-safe 兜底
|
|
573
680
|
const errMsg = syncError?.message ?? String(syncError);
|
|
@@ -579,7 +686,7 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
579
686
|
if (!recovered) {
|
|
580
687
|
logger?.warn?.("dsh-mneme mirror: recover failed after", MAX_ATTEMPTS, "attempts");
|
|
581
688
|
} else {
|
|
582
|
-
logger?.warn?.("dsh-mneme mirror: recovered from dirty state");
|
|
689
|
+
logger?.warn?.("dsh-mneme mirror: recovered from dirty/pending state");
|
|
583
690
|
}
|
|
584
691
|
} catch (error) {
|
|
585
692
|
// fail-safe:任何意外异常不向外抛
|
|
@@ -632,7 +739,32 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
632
739
|
toApiList,
|
|
633
740
|
transaction,
|
|
634
741
|
setDreamHook(fn) { dreamHook = fn; },
|
|
635
|
-
setEmbedder(emb) {
|
|
742
|
+
setEmbedder(emb) {
|
|
743
|
+
embedder = emb;
|
|
744
|
+
if (!emb) {
|
|
745
|
+
// embedder removed (init failed in index.js): stop polling and drop
|
|
746
|
+
// queued re-embeds — search just degrades to keyword.
|
|
747
|
+
stopEmbedReadyPolling();
|
|
748
|
+
embedPending = [];
|
|
749
|
+
return;
|
|
750
|
+
}
|
|
751
|
+
if (emb.ready === true) {
|
|
752
|
+
flushEmbedPending();
|
|
753
|
+
return;
|
|
754
|
+
}
|
|
755
|
+
// Async-initializing embedder: poll `ready` until it flips, then flush.
|
|
756
|
+
if ("ready" in emb && embedReadyTimer === null) {
|
|
757
|
+
let attempts = 0;
|
|
758
|
+
embedReadyTimer = setInterval(() => {
|
|
759
|
+
attempts++;
|
|
760
|
+
if (emb.ready === true || attempts >= EMBED_READY_POLL_LIMIT) {
|
|
761
|
+
stopEmbedReadyPolling();
|
|
762
|
+
if (emb.ready === true) flushEmbedPending();
|
|
763
|
+
else embedPending = []; // init never landed: drop the queue
|
|
764
|
+
}
|
|
765
|
+
}, EMBED_READY_POLL_MS);
|
|
766
|
+
}
|
|
767
|
+
},
|
|
636
768
|
setEntityExtractor(fn) { entityExtractor = fn; },
|
|
637
769
|
setVectorIndex(vi) { vectorIndex = vi; },
|
|
638
770
|
setReranker(rn) { reranker = rn; },
|