@modusensus/dsh-mneme 0.3.6 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -3
- package/lib/api.js +8 -0
- package/lib/index.js +27 -17
- package/lib/local-embedder.js +7 -1
- package/lib/service.js +132 -29
- package/lib/store.js +114 -34
- package/package.json +1 -1
- package/src/api.js +8 -0
- package/src/index.js +27 -17
- package/src/local-embedder.js +7 -1
- package/src/service.js +132 -29
- package/src/store.js +114 -34
- package/test/mirror-generation.test.js +24 -21
- package/test/peer-blockers.test.js +148 -0
package/src/api.js
CHANGED
|
@@ -289,6 +289,14 @@ export function createApi(ctx, service, settings, commands, embedder, semantic =
|
|
|
289
289
|
sendJson(res, 200, { mirror: { dirty: null, status: "unknown", last_error: null, last_attempt: null, success_at: null } });
|
|
290
290
|
return;
|
|
291
291
|
}
|
|
292
|
+
// Real read failure surfaces as dirty === null (peer blocker 5): report
|
|
293
|
+
// unknown explicitly instead of collapsing into a false "ok"/"degraded".
|
|
294
|
+
if (state.dirty === null) {
|
|
295
|
+
sendJson(res, 200, {
|
|
296
|
+
mirror: { dirty: null, status: "unknown", last_error: null, last_attempt: null, success_at: null }
|
|
297
|
+
});
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
292
300
|
// Sanitized: boolean dirty + coarse status only; error string is mapped to
|
|
293
301
|
// a bounded code, never echoed verbatim.
|
|
294
302
|
let code = null;
|
package/src/index.js
CHANGED
|
@@ -80,11 +80,28 @@ export const apply = (ctx, config) => {
|
|
|
80
80
|
const vectorIndex = createVectorIndex({ store, logger: ctx.logger });
|
|
81
81
|
service.setVectorIndex(vectorIndex);
|
|
82
82
|
|
|
83
|
+
// Human edits in mirror files win on every sync; merge them back first.
|
|
84
|
+
// TYPE_FILE maps each memory type to its mirror filename. Read every type's
|
|
85
|
+
// edits up front: mergeHumanEdits re-renders ALL mirror files on success, so
|
|
86
|
+
// a per-type read-then-merge loop would overwrite edits in files not yet read
|
|
87
|
+
// (e.g. preferences.md merging would clobber unsynced projects.md edits).
|
|
88
|
+
const humanEdits = new Map();
|
|
89
|
+
for (const type of Object.keys(TYPE_FILE)) {
|
|
90
|
+
humanEdits.set(type, mirror.readHumanEdits(type));
|
|
91
|
+
}
|
|
92
|
+
const applyHumanEdits = () => {
|
|
93
|
+
for (const [type, edits] of humanEdits) {
|
|
94
|
+
if (edits.length) service.mergeHumanEdits(type, edits);
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
|
|
83
98
|
let embedder = null;
|
|
84
99
|
let reranker = null;
|
|
85
100
|
if (cfg.embedProvider === "openai") {
|
|
86
101
|
embedder = createEmbedder({ store, settings, logger: ctx.logger });
|
|
87
102
|
service.setEmbedder(embedder);
|
|
103
|
+
// legacy OpenAI embedder is immediately usable
|
|
104
|
+
applyHumanEdits();
|
|
88
105
|
} else {
|
|
89
106
|
try {
|
|
90
107
|
embedder = createEmbedderByProvider(cfg.embedProvider, {
|
|
@@ -97,12 +114,18 @@ export const apply = (ctx, config) => {
|
|
|
97
114
|
logger: ctx.logger
|
|
98
115
|
});
|
|
99
116
|
service.setEmbedder(embedder);
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
117
|
+
// issue #6: wait for extractor init before applying human edits, so
|
|
118
|
+
// scheduled embeddings see a ready embedder.
|
|
119
|
+
embedder.init()
|
|
120
|
+
.then(() => applyHumanEdits())
|
|
121
|
+
.catch((error) => {
|
|
122
|
+
ctx.logger?.warn?.(`[dsh-mneme] embedder init failed, search degrades to keyword: ${String(error)}`);
|
|
123
|
+
service.setEmbedder(null);
|
|
124
|
+
applyHumanEdits();
|
|
125
|
+
});
|
|
104
126
|
} catch (error) {
|
|
105
127
|
ctx.logger?.warn?.(`[dsh-mneme] embedder unavailable, search degrades to keyword: ${String(error)}`);
|
|
128
|
+
applyHumanEdits();
|
|
106
129
|
}
|
|
107
130
|
}
|
|
108
131
|
|
|
@@ -139,19 +162,6 @@ export const apply = (ctx, config) => {
|
|
|
139
162
|
commands.sync();
|
|
140
163
|
}
|
|
141
164
|
|
|
142
|
-
// Human edits in mirror files win on every sync; merge them back first.
|
|
143
|
-
// TYPE_FILE maps each memory type to its mirror filename. Read every type's
|
|
144
|
-
// edits up front: mergeHumanEdits re-renders ALL mirror files on success, so
|
|
145
|
-
// a per-type read-then-merge loop would overwrite edits in files not yet read
|
|
146
|
-
// (e.g. preferences.md merging would clobber unsynced projects.md edits).
|
|
147
|
-
const humanEdits = new Map();
|
|
148
|
-
for (const type of Object.keys(TYPE_FILE)) {
|
|
149
|
-
humanEdits.set(type, mirror.readHumanEdits(type));
|
|
150
|
-
}
|
|
151
|
-
for (const [type, edits] of humanEdits) {
|
|
152
|
-
if (edits.length) service.mergeHumanEdits(type, edits);
|
|
153
|
-
}
|
|
154
|
-
|
|
155
165
|
// Dream scheduler: automatic consolidation + summary runs, triggered by
|
|
156
166
|
// store growth. Writes through the service fire the dream hook, which asks
|
|
157
167
|
// the scheduler to (re)schedule a run once absolute and since-last-run
|
package/src/local-embedder.js
CHANGED
|
@@ -58,16 +58,21 @@ export class LocalEmbedder {
|
|
|
58
58
|
// Test hook: replace the pipeline factory without touching modules.
|
|
59
59
|
this.engineFactory = opts.engineFactory || defaultPipelineLoader;
|
|
60
60
|
this.extractor = null;
|
|
61
|
+
// issue #6: readiness flag for the service's scheduleEmbed gate. False until
|
|
62
|
+
// init() succeeds, so "ready" in embedder is observable even pre-init.
|
|
63
|
+
this.ready = false;
|
|
61
64
|
}
|
|
62
65
|
|
|
63
|
-
/** Load the model; throws when it cannot be loaded. */
|
|
66
|
+
/** Load the model; throws when it cannot be loaded. Idempotent. */
|
|
64
67
|
async init() {
|
|
68
|
+
if (this.extractor) return this; // already initialized: no-op
|
|
65
69
|
const options = {
|
|
66
70
|
dtype: this.useDtype,
|
|
67
71
|
device: this.device
|
|
68
72
|
};
|
|
69
73
|
if (this.cacheDir) options.cache_dir = this.cacheDir;
|
|
70
74
|
this.extractor = await this.engineFactory("feature-extraction", this.model, options);
|
|
75
|
+
this.ready = true; // service reads this to flush queued re-embeds
|
|
71
76
|
this.logger?.info?.(
|
|
72
77
|
`[dsh-mneme] local embedder ready: ${this.model} (dim=${this._dimension}, device=${this.device})`
|
|
73
78
|
);
|
|
@@ -107,6 +112,7 @@ export class LocalEmbedder {
|
|
|
107
112
|
// best-effort: some engines free resources on GC
|
|
108
113
|
}
|
|
109
114
|
this.extractor = null;
|
|
115
|
+
this.ready = false;
|
|
110
116
|
}
|
|
111
117
|
}
|
|
112
118
|
|
package/src/service.js
CHANGED
|
@@ -37,11 +37,69 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
37
37
|
// replays them exactly once against the committed state.
|
|
38
38
|
let txDepth = 0;
|
|
39
39
|
|
|
40
|
+
// issue #6 (part 2): startup race defense. A local embedder (LocalEmbedder /
|
|
41
|
+
// Ollama) exposes an async init(), so between `setEmbedder` and init()
|
|
42
|
+
// resolving there is a window where embedSingle would throw "not initialized"
|
|
43
|
+
// and the re-embed would be silently dropped. When the embedder carries a
|
|
44
|
+
// `ready` flag we queue writes in embedPending until init sets ready=true,
|
|
45
|
+
// then flush them through the embedder's real interface. Embedders without a
|
|
46
|
+
// `ready` flag (legacy OpenAI, instantly usable) keep their old behavior.
|
|
47
|
+
let embedPending = [];
|
|
48
|
+
let embedReadyTimer = null;
|
|
49
|
+
const EMBED_PENDING_MAX = 100; // bound the queue; drop oldest beyond this
|
|
50
|
+
const EMBED_READY_POLL_MS = 100;
|
|
51
|
+
const EMBED_READY_POLL_LIMIT = 30; // ~3s ceiling; never poll forever
|
|
52
|
+
|
|
53
|
+
/** Flush the queued re-embeds once the embedder is ready. Fail-safe. */
|
|
54
|
+
function flushEmbedPending() {
|
|
55
|
+
if (!embedder || embedPending.length === 0) return;
|
|
56
|
+
const batch = embedPending.splice(0, embedPending.length);
|
|
57
|
+
for (const memory of batch) {
|
|
58
|
+
try {
|
|
59
|
+
if (!memory?.id) continue;
|
|
60
|
+
if (typeof embedder.schedule === "function") {
|
|
61
|
+
embedder.schedule(memory);
|
|
62
|
+
} else if (typeof embedder.embedSingle === "function") {
|
|
63
|
+
const text = [memory.title, memory.content].filter(Boolean).join("\n");
|
|
64
|
+
if (!text) continue;
|
|
65
|
+
embedder
|
|
66
|
+
.embedSingle(text)
|
|
67
|
+
.then((vec) => {
|
|
68
|
+
if (Array.isArray(vec) && vec.length) {
|
|
69
|
+
store.setEmbedding(memory.id, vec);
|
|
70
|
+
}
|
|
71
|
+
})
|
|
72
|
+
.catch((err) => {
|
|
73
|
+
logger?.warn?.("flushEmbedPending embedSingle failed:", err);
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
} catch (err) {
|
|
77
|
+
logger?.warn?.("flushEmbedPending failed:", err);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function stopEmbedReadyPolling() {
|
|
83
|
+
if (embedReadyTimer) {
|
|
84
|
+
clearInterval(embedReadyTimer);
|
|
85
|
+
embedReadyTimer = null;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
40
89
|
function scheduleEmbed(memory) {
|
|
41
90
|
try {
|
|
42
91
|
if (txDepth > 0) return; // deferred to the transaction's commit
|
|
43
92
|
if (!embedder || !memory?.id) return;
|
|
44
93
|
|
|
94
|
+
// Readiness gate: embedder exposes `ready` (async init) and is not ready
|
|
95
|
+
// yet — queue instead of firing embedSingle into a half-built extractor.
|
|
96
|
+
const hasReady = "ready" in embedder;
|
|
97
|
+
if (hasReady && embedder.ready !== true) {
|
|
98
|
+
if (embedPending.length >= EMBED_PENDING_MAX) embedPending.shift();
|
|
99
|
+
embedPending.push(memory);
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
|
|
45
103
|
if (typeof embedder.schedule === "function") {
|
|
46
104
|
embedder.schedule(memory);
|
|
47
105
|
return;
|
|
@@ -326,10 +384,12 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
326
384
|
throw error;
|
|
327
385
|
} finally {
|
|
328
386
|
txDepth--;
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
387
|
+
// Sync failures are surfaced, not swallowed (peer blocker 2): the mirror
|
|
388
|
+
// debt was already recorded by markMirrorDirty inside syncMirror, so a
|
|
389
|
+
// restart recovers — but the operator must see it now, not after restart.
|
|
390
|
+
const syncResult = syncMirror();
|
|
391
|
+
if (!syncResult?.success && !syncResult?.deferred) {
|
|
392
|
+
logger?.warn?.("mirror sync failed after transaction:", syncResult?.error);
|
|
333
393
|
}
|
|
334
394
|
notifyWrite();
|
|
335
395
|
}
|
|
@@ -350,7 +410,7 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
350
410
|
tags: memory.tags ?? existing.tags,
|
|
351
411
|
title: memory.title ?? existing.title
|
|
352
412
|
});
|
|
353
|
-
|
|
413
|
+
afterSync("write");
|
|
354
414
|
notifyWrite();
|
|
355
415
|
scheduleEmbed(merged);
|
|
356
416
|
return { action: "merged", memory: merged };
|
|
@@ -363,7 +423,7 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
363
423
|
importance: memory.importance ?? 3,
|
|
364
424
|
source: memory.source ?? "manual"
|
|
365
425
|
});
|
|
366
|
-
|
|
426
|
+
afterSync("write");
|
|
367
427
|
notifyWrite();
|
|
368
428
|
scheduleEmbed(created);
|
|
369
429
|
scheduleEntityExtraction(created);
|
|
@@ -423,7 +483,7 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
423
483
|
}
|
|
424
484
|
}
|
|
425
485
|
if (applied) {
|
|
426
|
-
|
|
486
|
+
afterSync("write");
|
|
427
487
|
notifyWrite();
|
|
428
488
|
}
|
|
429
489
|
return applied;
|
|
@@ -519,16 +579,17 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
519
579
|
// - 逐 type 用 setTypeStatus 记录部分成功/失败(type_status JSON);
|
|
520
580
|
// - 所有 store 状态写入各自 try/catch,失败只 warn,绝不向外抛(F-NEW-03)。
|
|
521
581
|
function syncMirror() {
|
|
522
|
-
if (txDepth > 0 || !mirror) return; // deferred to the transaction's commit
|
|
582
|
+
if (txDepth > 0 || !mirror) return { success: true, deferred: true }; // deferred to the transaction's commit
|
|
523
583
|
const now = new Date().toISOString();
|
|
524
584
|
let gen;
|
|
525
585
|
try {
|
|
526
|
-
//
|
|
527
|
-
|
|
528
|
-
|
|
586
|
+
// desired generation 已在业务写事务中原子递增(peer blocker 1);这里
|
|
587
|
+
// 直接读当前值作为本次同步的目标轮次,不再自行 incrementGeneration。
|
|
588
|
+
const state = store.getMirrorState();
|
|
589
|
+
gen = state?.generation ?? 0;
|
|
529
590
|
} catch (stateError) {
|
|
530
|
-
logger?.warn?.("syncMirror:
|
|
531
|
-
return;
|
|
591
|
+
logger?.warn?.("syncMirror: getMirrorState failed:", stateError);
|
|
592
|
+
return { success: false, error: stateError?.message ?? String(stateError) };
|
|
532
593
|
}
|
|
533
594
|
// coveredTypes 提到 try 外初始化:即使 store.list 先抛错,catch 分支也有
|
|
534
595
|
// 合法的空 Set 可迭代,保证 syncMirror 自身绝不抛(fail-safe)。
|
|
@@ -545,37 +606,53 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
545
606
|
// 全量渲染
|
|
546
607
|
mirror.sync(reconcileHumanEdits(list));
|
|
547
608
|
|
|
548
|
-
// 成功:CAS/fence 绑定到本地 gen,旧 worker(gen
|
|
609
|
+
// 成功:CAS/fence 绑定到本地 gen,旧 worker(gen 已过期)会被拦截。
|
|
610
|
+
// 此步失败说明核心 clean 状态没写成功,向上层报失败(不再静默)。
|
|
549
611
|
try {
|
|
550
612
|
store.markMirrorCleanForGeneration(gen, now);
|
|
551
613
|
} catch (stateError) {
|
|
552
614
|
logger?.warn?.("syncMirror: markMirrorCleanForGeneration failed:", stateError);
|
|
615
|
+
return { success: false, error: stateError?.message ?? String(stateError) };
|
|
553
616
|
}
|
|
554
|
-
// 逐 type 标记为
|
|
617
|
+
// 逐 type 标记为 committed(peer blocker 4: per-type receipt)
|
|
555
618
|
for (const type of coveredTypes) {
|
|
556
619
|
try {
|
|
557
|
-
store.setTypeStatus(type, {
|
|
620
|
+
store.setTypeStatus(type, { status: "committed", applied_gen: gen, last_error: null });
|
|
558
621
|
} catch (stateError) {
|
|
559
|
-
logger?.warn?.(`syncMirror: setTypeStatus(${type})
|
|
622
|
+
logger?.warn?.(`syncMirror: setTypeStatus(${type}) committed failed:`, stateError);
|
|
560
623
|
}
|
|
561
624
|
}
|
|
625
|
+
return { success: true };
|
|
562
626
|
} catch (error) {
|
|
563
627
|
const errMsg = error?.message ?? String(error);
|
|
564
628
|
logger?.warn?.("syncMirror failed:", error);
|
|
565
629
|
try {
|
|
566
|
-
// 债务绑定到新的一轮(desired generation
|
|
630
|
+
// 债务绑定到新的一轮(desired generation 原子递增;即便 dirty 写失败,
|
|
631
|
+
// generation 已推进,recoverMirror 仍能捕获,不产生 false-clean)。
|
|
567
632
|
store.markMirrorDirty(errMsg, now);
|
|
568
633
|
} catch (stateError) {
|
|
569
634
|
logger?.warn?.("syncMirror: markMirrorDirty failed:", stateError);
|
|
570
635
|
}
|
|
571
|
-
// 逐 type 标记为
|
|
636
|
+
// 逐 type 标记为 failed(applied_gen 不动)
|
|
572
637
|
for (const type of coveredTypes) {
|
|
573
638
|
try {
|
|
574
|
-
store.setTypeStatus(type, {
|
|
639
|
+
store.setTypeStatus(type, { status: "failed", last_error: errMsg });
|
|
575
640
|
} catch (stateError) {
|
|
576
|
-
logger?.warn?.(`syncMirror: setTypeStatus(${type})
|
|
641
|
+
logger?.warn?.(`syncMirror: setTypeStatus(${type}) failed:`, stateError);
|
|
577
642
|
}
|
|
578
643
|
}
|
|
644
|
+
return { success: false, error: errMsg };
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
// afterSync: run syncMirror and surface a failure to the operator instead of
|
|
649
|
+
// swallowing it (peer blocker 2). The mirror debt has already been persisted
|
|
650
|
+
// by markMirrorDirty inside syncMirror, so a restart recovers — but the
|
|
651
|
+
// calling write path must not report clean while the mirror is known-stale.
|
|
652
|
+
function afterSync(label) {
|
|
653
|
+
const r = syncMirror();
|
|
654
|
+
if (!r?.success && !r?.deferred) {
|
|
655
|
+
logger?.warn?.(`${label}: mirror sync failed (will recover on restart):`, r?.error);
|
|
579
656
|
}
|
|
580
657
|
}
|
|
581
658
|
|
|
@@ -660,10 +737,11 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
660
737
|
success_at: state.success_at ?? null
|
|
661
738
|
};
|
|
662
739
|
} catch (error) {
|
|
663
|
-
// fail-safe
|
|
740
|
+
// fail-safe:状态读取失败也不向外抛,但必须显式表达"未知"而非伪装成
|
|
741
|
+
// 干净(peer blocker 5:真实读取失败要显式 unknown,不得归一为 dirty:false)。
|
|
664
742
|
logger?.warn?.("getMirrorHealth failed:", error);
|
|
665
743
|
return {
|
|
666
|
-
dirty:
|
|
744
|
+
dirty: null,
|
|
667
745
|
last_error: error?.message ?? String(error),
|
|
668
746
|
last_attempt: null,
|
|
669
747
|
success_at: null
|
|
@@ -681,7 +759,32 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
681
759
|
toApiList,
|
|
682
760
|
transaction,
|
|
683
761
|
setDreamHook(fn) { dreamHook = fn; },
|
|
684
|
-
setEmbedder(emb) {
|
|
762
|
+
setEmbedder(emb) {
|
|
763
|
+
embedder = emb;
|
|
764
|
+
if (!emb) {
|
|
765
|
+
// embedder removed (init failed in index.js): stop polling and drop
|
|
766
|
+
// queued re-embeds — search just degrades to keyword.
|
|
767
|
+
stopEmbedReadyPolling();
|
|
768
|
+
embedPending = [];
|
|
769
|
+
return;
|
|
770
|
+
}
|
|
771
|
+
if (emb.ready === true) {
|
|
772
|
+
flushEmbedPending();
|
|
773
|
+
return;
|
|
774
|
+
}
|
|
775
|
+
// Async-initializing embedder: poll `ready` until it flips, then flush.
|
|
776
|
+
if ("ready" in emb && embedReadyTimer === null) {
|
|
777
|
+
let attempts = 0;
|
|
778
|
+
embedReadyTimer = setInterval(() => {
|
|
779
|
+
attempts++;
|
|
780
|
+
if (emb.ready === true || attempts >= EMBED_READY_POLL_LIMIT) {
|
|
781
|
+
stopEmbedReadyPolling();
|
|
782
|
+
if (emb.ready === true) flushEmbedPending();
|
|
783
|
+
else embedPending = []; // init never landed: drop the queue
|
|
784
|
+
}
|
|
785
|
+
}, EMBED_READY_POLL_MS);
|
|
786
|
+
}
|
|
787
|
+
},
|
|
685
788
|
setEntityExtractor(fn) { entityExtractor = fn; },
|
|
686
789
|
setVectorIndex(vi) { vectorIndex = vi; },
|
|
687
790
|
setReranker(rn) { reranker = rn; },
|
|
@@ -697,7 +800,7 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
697
800
|
getById: (id) => store.getById(id),
|
|
698
801
|
remove: (id) => {
|
|
699
802
|
store.remove(id);
|
|
700
|
-
|
|
803
|
+
afterSync("write");
|
|
701
804
|
notifyWrite();
|
|
702
805
|
},
|
|
703
806
|
update: (id, p, ctx = {}) => {
|
|
@@ -723,7 +826,7 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
723
826
|
memory_id: id
|
|
724
827
|
});
|
|
725
828
|
}
|
|
726
|
-
|
|
829
|
+
afterSync("write");
|
|
727
830
|
notifyWrite();
|
|
728
831
|
scheduleEmbed(updated);
|
|
729
832
|
return updated;
|
|
@@ -752,19 +855,19 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
752
855
|
memory_id: id
|
|
753
856
|
});
|
|
754
857
|
}
|
|
755
|
-
|
|
858
|
+
afterSync("write");
|
|
756
859
|
notifyWrite();
|
|
757
860
|
scheduleEmbed(updated);
|
|
758
861
|
return updated;
|
|
759
862
|
},
|
|
760
863
|
setForget: (id, f) => {
|
|
761
864
|
const updated = store.setForget(id, f);
|
|
762
|
-
|
|
865
|
+
afterSync("write");
|
|
763
866
|
return updated;
|
|
764
867
|
},
|
|
765
868
|
setArchived: (id, f) => {
|
|
766
869
|
const updated = store.setArchived(id, f);
|
|
767
|
-
|
|
870
|
+
afterSync("write");
|
|
768
871
|
return updated;
|
|
769
872
|
},
|
|
770
873
|
// autoDream audit trail: passthroughs deliberately bypass write hooks —
|
package/src/store.js
CHANGED
|
@@ -180,14 +180,19 @@ CREATE TABLE IF NOT EXISTS mirror_state (
|
|
|
180
180
|
last_error TEXT, -- 最近失败原因
|
|
181
181
|
last_attempt TEXT, -- 最近尝试时间(ISO)
|
|
182
182
|
success_at TEXT, -- 最近成功时间(ISO)
|
|
183
|
-
generation INTEGER NOT NULL DEFAULT 0, -- 期望的同步轮次(desired)
|
|
184
|
-
applied_generation INTEGER NOT NULL DEFAULT 0, -- 已成功应用的轮次
|
|
183
|
+
generation INTEGER NOT NULL DEFAULT 0 CHECK (generation >= 0 AND generation <= 9007199254740991), -- 期望的同步轮次(desired)
|
|
184
|
+
applied_generation INTEGER NOT NULL DEFAULT 0 CHECK (applied_generation >= 0 AND applied_generation <= 9007199254740991), -- 已成功应用的轮次
|
|
185
185
|
type_status TEXT -- JSON: 逐 type 状态 {type: {dirty, applied_gen, last_error}}
|
|
186
186
|
);
|
|
187
187
|
`;
|
|
188
188
|
|
|
189
189
|
const TYPES = new Set(["preference", "project", "decision", "history", "summary"]);
|
|
190
190
|
|
|
191
|
+
// Per-type mirror sync receipts (peer blocker 4): a type is either committed
|
|
192
|
+
// (file written + fence applied), failed (last sync round errored for it), or
|
|
193
|
+
// pending (still owed a write).
|
|
194
|
+
const VALID_TYPE_STATUS = new Set(["committed", "failed", "pending"]);
|
|
195
|
+
|
|
191
196
|
// Pure helpers: no shared module state.
|
|
192
197
|
|
|
193
198
|
function sanitizePage(limit, offset, defaultLimit) {
|
|
@@ -387,6 +392,10 @@ function parseJsonArray(raw) {
|
|
|
387
392
|
export function createStore(path) {
|
|
388
393
|
const db = new DatabaseSync(path);
|
|
389
394
|
db.exec("PRAGMA journal_mode = WAL;");
|
|
395
|
+
// Concurrent writers (peer probe: 8 independent processes) must wait for the
|
|
396
|
+
// write lock instead of failing immediately with SQLITE_BUSY — otherwise the
|
|
397
|
+
// atomic generation increment loses whole writes, not just increments.
|
|
398
|
+
db.exec("PRAGMA busy_timeout = 5000;");
|
|
390
399
|
db.exec(SCHEMA);
|
|
391
400
|
|
|
392
401
|
// Schema migrations for legacy databases (idempotent).
|
|
@@ -467,10 +476,17 @@ export function createStore(path) {
|
|
|
467
476
|
const embedding = Array.isArray(memory.embedding) && memory.embedding.length
|
|
468
477
|
? JSON.stringify(memory.embedding)
|
|
469
478
|
: null;
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
479
|
+
runAtomically(() => {
|
|
480
|
+
db.prepare(
|
|
481
|
+
`INSERT INTO memories (id, type, title, content, tags, importance, forgotten, source, embedding, created_at, updated_at)
|
|
482
|
+
VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?)`
|
|
483
|
+
).run(id, type, memory.title, memory.content, tags, importance, memory.source ?? null, embedding, now, now);
|
|
484
|
+
// desired generation bumped in the same transaction as the write: once
|
|
485
|
+
// this commits, generation > applied_generation, so a crash right after
|
|
486
|
+
// (before syncMirror) is caught by recoverMirror on restart (peer
|
|
487
|
+
// blocker 1). ROLLBACK on error rolls this back with the write.
|
|
488
|
+
incrementGeneration();
|
|
489
|
+
});
|
|
474
490
|
return getById(id);
|
|
475
491
|
}
|
|
476
492
|
|
|
@@ -486,24 +502,34 @@ export function createStore(path) {
|
|
|
486
502
|
const embedding = patch.embedding !== undefined
|
|
487
503
|
? (Array.isArray(patch.embedding) && patch.embedding.length ? JSON.stringify(patch.embedding) : null)
|
|
488
504
|
: existing.embedding ?? null;
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
505
|
+
runAtomically(() => {
|
|
506
|
+
db.prepare(
|
|
507
|
+
`UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, embedding=?, updated_at=? WHERE id=?`
|
|
508
|
+
).run(
|
|
509
|
+
type,
|
|
510
|
+
patch.title ?? existing.title,
|
|
511
|
+
patch.content ?? existing.content,
|
|
512
|
+
JSON.stringify(patch.tags ?? existing.tags),
|
|
513
|
+
Number.isInteger(patch.importance) ? patch.importance : existing.importance,
|
|
514
|
+
patch.source !== undefined ? patch.source : (existing.source ?? null),
|
|
515
|
+
embedding,
|
|
516
|
+
now,
|
|
517
|
+
id
|
|
518
|
+
);
|
|
519
|
+
// Desired generation bumped in the same transaction as the update (peer
|
|
520
|
+
// blocker 1: crash between write and sync must still be recoverable).
|
|
521
|
+
incrementGeneration();
|
|
522
|
+
});
|
|
502
523
|
return getById(id);
|
|
503
524
|
}
|
|
504
525
|
|
|
505
526
|
function remove(id) {
|
|
506
|
-
|
|
527
|
+
runAtomically(() => {
|
|
528
|
+
db.prepare("DELETE FROM memories WHERE id = ?").run(id);
|
|
529
|
+
// Mirror sync must reflect the deletion; bump desired generation so a
|
|
530
|
+
// crash between the delete and syncMirror leaves a recoverable debt.
|
|
531
|
+
incrementGeneration();
|
|
532
|
+
});
|
|
507
533
|
}
|
|
508
534
|
|
|
509
535
|
/**
|
|
@@ -542,18 +568,26 @@ export function createStore(path) {
|
|
|
542
568
|
expectedUpdatedAt
|
|
543
569
|
);
|
|
544
570
|
if (result.changes === 0) return undefined; // CAS miss: a concurrent write won
|
|
571
|
+
// Only bump desired generation on a successful CAS — a miss writes nothing.
|
|
572
|
+
runAtomically(() => { incrementGeneration(); });
|
|
545
573
|
return getById(id);
|
|
546
574
|
}
|
|
547
575
|
|
|
548
576
|
function setForget(id, forgotten) {
|
|
549
|
-
|
|
550
|
-
.
|
|
577
|
+
runAtomically(() => {
|
|
578
|
+
db.prepare("UPDATE memories SET forgotten = ?, updated_at = ? WHERE id = ?")
|
|
579
|
+
.run(forgotten === true || forgotten === 1 ? 1 : 0, nowIso(), id);
|
|
580
|
+
incrementGeneration();
|
|
581
|
+
});
|
|
551
582
|
return getById(id);
|
|
552
583
|
}
|
|
553
584
|
|
|
554
585
|
function setArchived(id, archived) {
|
|
555
|
-
|
|
556
|
-
.
|
|
586
|
+
runAtomically(() => {
|
|
587
|
+
db.prepare("UPDATE memories SET archived = ?, updated_at = ? WHERE id = ?")
|
|
588
|
+
.run(archived ? 1 : 0, nowIso(), id);
|
|
589
|
+
incrementGeneration();
|
|
590
|
+
});
|
|
557
591
|
return getById(id);
|
|
558
592
|
}
|
|
559
593
|
|
|
@@ -1189,7 +1223,10 @@ export function createStore(path) {
|
|
|
1189
1223
|
if (key === "dirty") {
|
|
1190
1224
|
value = value ? 1 : 0;
|
|
1191
1225
|
} else if (key === "generation" || key === "applied_generation") {
|
|
1192
|
-
value = Math.trunc(Number(value))
|
|
1226
|
+
value = Math.trunc(Number(value));
|
|
1227
|
+
if (!Number.isFinite(value) || value < 0 || value > Number.MAX_SAFE_INTEGER) {
|
|
1228
|
+
throw new RangeError(`mirror_state.${key} out of range: ${value}`);
|
|
1229
|
+
}
|
|
1193
1230
|
} else if (key === "type_status" && value != null && typeof value !== "string") {
|
|
1194
1231
|
value = JSON.stringify(value);
|
|
1195
1232
|
}
|
|
@@ -1220,12 +1257,15 @@ export function createStore(path) {
|
|
|
1220
1257
|
* recent as this one may.
|
|
1221
1258
|
*/
|
|
1222
1259
|
function markMirrorDirty(error, now) {
|
|
1223
|
-
|
|
1260
|
+
// Bump the desired generation atomically first — the new debt must be bound
|
|
1261
|
+
// to a fresh round so a stale worker cannot fence-clean it. Even if this
|
|
1262
|
+
// write fails (peer blocker 2), generation still advanced, so recoverMirror
|
|
1263
|
+
// sees generation > applied_generation and retries rather than false-clean.
|
|
1264
|
+
incrementGeneration();
|
|
1224
1265
|
return setMirrorState({
|
|
1225
1266
|
dirty: 1,
|
|
1226
1267
|
last_error: error,
|
|
1227
|
-
last_attempt: now ?? nowIso()
|
|
1228
|
-
generation: (current.generation || 0) + 1
|
|
1268
|
+
last_attempt: now ?? nowIso()
|
|
1229
1269
|
});
|
|
1230
1270
|
}
|
|
1231
1271
|
|
|
@@ -1262,13 +1302,23 @@ export function createStore(path) {
|
|
|
1262
1302
|
|
|
1263
1303
|
/**
|
|
1264
1304
|
* Record per-type mirror status (partial success bookkeeping). `status` is a
|
|
1265
|
-
*
|
|
1266
|
-
* entry for `type` (other types untouched).
|
|
1305
|
+
* patch {status: 'committed'|'failed'|'pending', applied_gen?, last_error?}
|
|
1306
|
+
* replacing the entry for `type` (other types untouched). Standardizing on an
|
|
1307
|
+
* explicit status gives per-type committed/failed/pending receipts — a type
|
|
1308
|
+
* whose file was written while a sibling failed is recorded as such, not
|
|
1309
|
+
* collapsed into a bulk "dirty" (peer blocker 4). Returns the updated state.
|
|
1267
1310
|
*/
|
|
1268
1311
|
function setTypeStatus(type, status) {
|
|
1312
|
+
if (!VALID_TYPE_STATUS.has(status?.status)) {
|
|
1313
|
+
throw new TypeError(`setTypeStatus: status must be one of committed|failed|pending, got ${status?.status}`);
|
|
1314
|
+
}
|
|
1269
1315
|
const current = getMirrorState();
|
|
1270
1316
|
const statuses = current.type_status || {};
|
|
1271
|
-
statuses[type] = {
|
|
1317
|
+
statuses[type] = {
|
|
1318
|
+
status: status.status,
|
|
1319
|
+
...(status.applied_gen !== undefined ? { applied_gen: status.applied_gen } : {}),
|
|
1320
|
+
...(status.last_error !== undefined ? { last_error: status.last_error } : {})
|
|
1321
|
+
};
|
|
1272
1322
|
return setMirrorState({ type_status: JSON.stringify(statuses) });
|
|
1273
1323
|
}
|
|
1274
1324
|
|
|
@@ -1278,10 +1328,40 @@ export function createStore(path) {
|
|
|
1278
1328
|
return current.type_status || {};
|
|
1279
1329
|
}
|
|
1280
1330
|
|
|
1281
|
-
/**
|
|
1331
|
+
/** Run fn atomically: when the connection is already inside a transaction
|
|
1332
|
+
* (service.transaction's BEGIN), just run it — the outer COMMIT covers us.
|
|
1333
|
+
* Otherwise wrap in BEGIN/COMMIT so a memory write and its desired-generation
|
|
1334
|
+
* bump commit together: a crash between them can never leave a mutated store
|
|
1335
|
+
* with generation == applied (audit peer blocker 1, "crash window"). */
|
|
1336
|
+
function runAtomically(fn) {
|
|
1337
|
+
if (db.isTransaction) return fn();
|
|
1338
|
+
db.exec("BEGIN");
|
|
1339
|
+
try {
|
|
1340
|
+
const result = fn();
|
|
1341
|
+
db.exec("COMMIT");
|
|
1342
|
+
return result;
|
|
1343
|
+
} catch (error) {
|
|
1344
|
+
try { db.exec("ROLLBACK"); } catch { /* connection may be closed */ }
|
|
1345
|
+
throw error;
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1349
|
+
/** Bump the desired generation atomically (SQLite single-statement increment,
|
|
1350
|
+
* no SELECT-then-UPSERT race: peer blocker 3 lost 10 of 91 concurrent
|
|
1351
|
+
* increments under an 8-process probe). Returns the new mirror state.
|
|
1352
|
+
* Guards the upper bound: generation must stay within MAX_SAFE_INTEGER so
|
|
1353
|
+
* reads never hit ERR_OUT_OF_RANGE (peer blocker 6). */
|
|
1282
1354
|
function incrementGeneration() {
|
|
1283
|
-
|
|
1284
|
-
|
|
1355
|
+
return runAtomically(() => {
|
|
1356
|
+
// Ensure the singleton row exists before incrementing (UPDATE alone would
|
|
1357
|
+
// match nothing on a fresh DB).
|
|
1358
|
+
db.prepare("INSERT OR IGNORE INTO mirror_state (id) VALUES ('main')").run();
|
|
1359
|
+
const row = db.prepare(
|
|
1360
|
+
"UPDATE mirror_state SET generation = generation + 1 WHERE id = 'main' AND generation < ? RETURNING generation"
|
|
1361
|
+
).get(Number.MAX_SAFE_INTEGER);
|
|
1362
|
+
if (!row) throw new RangeError("mirror_state.generation exceeded MAX_SAFE_INTEGER");
|
|
1363
|
+
return getMirrorState();
|
|
1364
|
+
});
|
|
1285
1365
|
}
|
|
1286
1366
|
|
|
1287
1367
|
return {
|