@modusensus/dsh-mneme 0.4.4 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/lib/config.js +18 -0
- package/lib/dream/decisions.js +43 -2
- package/lib/inject.js +6 -1
- package/lib/service.js +141 -1
- package/lib/store.js +159 -5
- package/package.json +1 -1
- package/src/config.js +18 -0
- package/src/dream/decisions.js +43 -2
- package/src/inject.js +6 -1
- package/src/service.js +141 -1
- package/src/store.js +159 -5
- package/test/epistemic.test.js +298 -0
- package/test/recall-evals.test.js +235 -0
package/README.md
CHANGED
|
@@ -163,6 +163,7 @@ v0.3.0 起新增**记忆基因**层:从记忆里抽取**命名实体**、**带
|
|
|
163
163
|
| **v0.4.2** | ✅ 完成 | autoSummarize 自定义模型 | `summarizeProvider`/`summarizeModel` 配置项支持,可独立指定轻量模型(如 qwen3.6-plus)用于会话摘要,节省主模型 token;473 测试全绿 |
|
|
164
164
|
| **v0.4.3** | ✅ 完成 | autoDream 大记忆量修复 | issue#9 B+A:`dreamMaxTokens` 上限 32768→131072 + `dreamReasoningEffort`/`sleepReasoningEffort` 思考开关(none 默认,主对话不受影响);478 测试全绿 |
|
|
165
165
|
| **v0.4.4** | ✅ 完成 | autoDream 决策覆盖修复 | issue#9 方案C:滑动窗口 `dreamMaxSnapshotSize`(默认200,updated_at 倒序截断) + 隐式 keep `dreamImplicitKeep`(默认true) + 覆盖率下限 `dreamMinExplicitCoverage`(默认50%) + 固定决策 schema;487 测试全绿 |
|
|
166
|
+
| **v0.4.5** | ✅ 完成 | epistemic trust + recall eval | 记忆可信度分级 `trustEpistemicWeighting`(observation>inferred>subjective:检索排序优先高可信、注入标注 `[verified]`、dream merge/conflict 偏向高可信;opt-in 默认关)+ 检索评估 `evaluateRetrieval` 落库 `recall_evals`(`evalPersistTestResults` opt-in 默认关,生产检索始终走 `recall_runs` 无条件隔离);518 测试全绿 |
|
|
166
167
|
| **v0.5.0+** | 🚀 远期 | 自进化记忆 | 兴趣漂移跟踪 + 跨 workspace 记忆共享(等 DSH 支持) |
|
|
167
168
|
|
|
168
169
|
> 新能力一律做成**可开关的功能**(配置启用/关闭),默认保守开启、不破坏现有行为。`failure_memories` 表与 autoDream 决策引擎已为后续反思性成长铺好路。
|
|
@@ -264,6 +265,8 @@ dsh web
|
|
|
264
265
|
| `entityExtractionMaxEntities` | `10` | 每次抽取实体数上限 |
|
|
265
266
|
| `entityExtractionMaxAttrs` | `20` | 每实体属性数上限 |
|
|
266
267
|
| `entitySearchEnabled` | `true` | `entity:` / `attr:` 前缀搜索开关 |
|
|
268
|
+
| `trustEpistemicWeighting` | `false` | 记忆可信度加权(v0.4.5,opt-in 默认关):记忆按来源分级 `observation`> `inferred` > `subjective`,开启后检索排序优先高可信记忆、注入对 observation 标注 `[verified]`、dream merge/conflict 偏向高可信一方;关闭时 `epistemic_status` 仅随保存落库、不参与行为 |
|
|
269
|
+
| `evalPersistTestResults` | `false` | 检索评估落库(v0.4.5,opt-in 默认关):开启后 `evaluateRetrieval` 把 precision/recall/mrr 快照写入 `recall_evals`;默认关时仅返回调用方不落库。生产 `searchMemories` 审计始终走 `recall_runs`,无条件不触碰 `recall_evals` |
|
|
267
270
|
|
|
268
271
|
> 🔐 **API 安全**:DSH 无内置鉴权且默认仅监听 `127.0.0.1`。插件 API 默认开放(便于 Web 面板即装即用)。如需防护(如局域网暴露),在配置中设置 `apiToken`:写操作(画像/规则/命令)与密钥端点(`vector-config`、`vector-reindex`)需携带 `Authorization: Bearer <token>`(前端设置面板可填入同一 token),只读的 `list` / `search` / `semantic` 保持开放。`/api/dsh-mneme/vector-config` 返回的 `apiKey` 已掩码(`sk-***…`),存储仍保留明文供调用;前端回传空或掩码值表示"不改 key"。
|
|
269
272
|
|
package/lib/config.js
CHANGED
|
@@ -167,4 +167,22 @@ export const Config = z.object({
|
|
|
167
167
|
z.const("high"),
|
|
168
168
|
z.const("none")
|
|
169
169
|
]).default("none"),
|
|
170
|
+
|
|
171
|
+
// --- epistemic trust: memory source credibility (v0.4.5) -----------------
|
|
172
|
+
// Distinguish memories by source: observation (measured / witnessed),
|
|
173
|
+
// subjective (opinion / guess) and inferred (derived from other evidence).
|
|
174
|
+
// Opt-in by default: when false (default) retrieval ranking, injection
|
|
175
|
+
// marking and dream merge/conflict keepSource are untouched and
|
|
176
|
+
// epistemic_status stays inert data (still written + inferred on save, just
|
|
177
|
+
// never used to influence behavior).
|
|
178
|
+
trustEpistemicWeighting: z.boolean().default(false),
|
|
179
|
+
|
|
180
|
+
// --- recall evaluation: test-result storage (v0.4.6, 方案 B) --------------
|
|
181
|
+
// Separate retrieval evaluation snapshots from the production recall audit.
|
|
182
|
+
// When false (default) evaluateRetrieval still computes precision/recall/mrr
|
|
183
|
+
// and returns them to the caller, but writes nothing to recall_evals — the
|
|
184
|
+
// eval table only grows when the operator opts in. Production searchMemories
|
|
185
|
+
// audits to recall_runs and NEVER touches recall_evals, regardless of this
|
|
186
|
+
// flag (production isolation is unconditional).
|
|
187
|
+
evalPersistTestResults: z.boolean().default(false),
|
|
170
188
|
});
|
package/lib/dream/decisions.js
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
const ACTIONS = new Set(["keep", "merge", "archive", "conflict", "update", "create"]);
|
|
2
2
|
|
|
3
|
+
// Epistemic trust (v0.4.5): when config.trustEpistemicWeighting is on, merge
|
|
4
|
+
// keepSource and conflict winners prefer the higher-trust memory. Higher value
|
|
5
|
+
// = preferred. observation (measured) > inferred (derived) > subjective (guess).
|
|
6
|
+
const EPISTEMIC_PRIORITY = { observation: 3, inferred: 2, subjective: 1 };
|
|
7
|
+
|
|
3
8
|
/**
|
|
4
9
|
* Validate a dream decision list against a snapshot of eligible memories.
|
|
5
10
|
* @param decisions - LLM-produced decision list.
|
|
@@ -252,12 +257,34 @@ function applyOne(d, service, snapshot, config = {}) {
|
|
|
252
257
|
switch (d.action) {
|
|
253
258
|
case "archive": return applyArchive(d, service, snapshot);
|
|
254
259
|
case "merge": return applyMerge(d, service, snapshot, config);
|
|
255
|
-
case "conflict": return applyConflict(d, service, snapshot);
|
|
260
|
+
case "conflict": return applyConflict(d, service, snapshot, config);
|
|
256
261
|
case "create": return applyCreate(d, service, config);
|
|
257
262
|
default: return applyUpdate(d, service, snapshot, config);
|
|
258
263
|
}
|
|
259
264
|
}
|
|
260
265
|
|
|
266
|
+
/**
|
|
267
|
+
* Highest-epistemic-priority UNARCHIVED id among `ids` (ties break toward
|
|
268
|
+
* `preferred`). Archived memories are never eligible keepers — promoting one
|
|
269
|
+
* would demote the real keepSource to a source and then hit the archived-keeper
|
|
270
|
+
* guard in applyMerge, silently skipping the whole merge. When `preferred`
|
|
271
|
+
* itself is archived (or missing), fall back to any unarchived candidate.
|
|
272
|
+
*/
|
|
273
|
+
function pickBestKeeper(ids, preferred, service) {
|
|
274
|
+
let best = null;
|
|
275
|
+
let bestP = -1;
|
|
276
|
+
for (const id of ids) {
|
|
277
|
+
const mem = service.getById(id);
|
|
278
|
+
if (!mem || mem.archived) continue; // archived/missing: ineligible keeper
|
|
279
|
+
const p = EPISTEMIC_PRIORITY[mem.epistemic_status] ?? 0;
|
|
280
|
+
if (p > bestP || (p === bestP && id === preferred)) {
|
|
281
|
+
bestP = p;
|
|
282
|
+
best = id;
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
return best ?? preferred;
|
|
286
|
+
}
|
|
287
|
+
|
|
261
288
|
/**
|
|
262
289
|
* Mint a fresh memory (pattern discovery). No existing target, so no CAS guard.
|
|
263
290
|
* Evidence ids ride in the content so a pattern stays traceable to its source
|
|
@@ -297,6 +324,13 @@ function applyArchive(d, service, snapshot) {
|
|
|
297
324
|
}
|
|
298
325
|
|
|
299
326
|
function applyMerge(d, service, snapshot, config = {}) {
|
|
327
|
+
// Epistemic trust (v0.4.5): when enabled, prefer an observation keeper over a
|
|
328
|
+
// subjective/inferred one. Mutating the decision keeps the receipt + committed
|
|
329
|
+
// record aligned with the actual keeper.
|
|
330
|
+
if (config.trustEpistemicWeighting === true) {
|
|
331
|
+
const best = pickBestKeeper(d.ids, d.keepSource, service);
|
|
332
|
+
if (best && best !== d.keepSource) d.keepSource = best;
|
|
333
|
+
}
|
|
300
334
|
const sources = d.ids.filter((id) => id !== d.keepSource);
|
|
301
335
|
// Idempotent replay: if every other source is already archived, this merge
|
|
302
336
|
// already landed — skip so a replayed/concurrent decision never double-counts
|
|
@@ -334,7 +368,14 @@ function applyMerge(d, service, snapshot, config = {}) {
|
|
|
334
368
|
};
|
|
335
369
|
}
|
|
336
370
|
|
|
337
|
-
function applyConflict(d, service, snapshot) {
|
|
371
|
+
function applyConflict(d, service, snapshot, config = {}) {
|
|
372
|
+
// Epistemic trust (v0.4.5): when enabled, the observation side of a conflict
|
|
373
|
+
// is preferred as winner over a subjective/inferred one.
|
|
374
|
+
if (config.trustEpistemicWeighting === true) {
|
|
375
|
+
const pw = EPISTEMIC_PRIORITY[service.getById(d.winner)?.epistemic_status] ?? 0;
|
|
376
|
+
const pl = EPISTEMIC_PRIORITY[service.getById(d.loser)?.epistemic_status] ?? 0;
|
|
377
|
+
if (pl > pw) [d.winner, d.loser] = [d.loser, d.winner];
|
|
378
|
+
}
|
|
338
379
|
const winner = service.getById(d.winner);
|
|
339
380
|
const loser = service.getById(d.loser);
|
|
340
381
|
if (!winner || !loser) return "skipped";
|
package/lib/inject.js
CHANGED
|
@@ -6,7 +6,12 @@ export function createInjector(ctx, service, settings, config) {
|
|
|
6
6
|
if (!candidates.length) return "";
|
|
7
7
|
const lines = ["[记忆库] 来自 dsh-mneme 的跨会话记忆(用户偏好与高优先级项目/决策):"];
|
|
8
8
|
for (const m of candidates) {
|
|
9
|
-
|
|
9
|
+
// Epistemic trust (v0.4.5): when enabled, measured observations are
|
|
10
|
+
// flagged so the agent can weigh them above guesses/opinions.
|
|
11
|
+
const verified = config.trustEpistemicWeighting === true && m.epistemic_status === "observation"
|
|
12
|
+
? "[verified] "
|
|
13
|
+
: "";
|
|
14
|
+
lines.push(`- [${m.type}] ${verified}${m.title}(重要性 ${m.importance}):${m.content}`);
|
|
10
15
|
}
|
|
11
16
|
return lines.join("\n");
|
|
12
17
|
}
|
package/lib/service.js
CHANGED
|
@@ -3,6 +3,39 @@ import { TYPE_FILE } from "./mirror.js";
|
|
|
3
3
|
|
|
4
4
|
const INJECT_TYPES = new Set(["preference", "project", "decision", "summary"]);
|
|
5
5
|
|
|
6
|
+
// Epistemic trust weights (v0.4.5): when config.trustEpistemicWeighting is on,
|
|
7
|
+
// each recall candidate's existing score is multiplied by the weight of its
|
|
8
|
+
// epistemic_status before ranking — measured facts outrank guesses. Missing /
|
|
9
|
+
// unknown statuses are unscaled (×1). Off by default, so nothing changes.
|
|
10
|
+
const EPISTEMIC_WEIGHTS = { observation: 1.0, inferred: 0.85, subjective: 0.7 };
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Standard retrieval-quality metrics over the ordered candidate ids actually
|
|
14
|
+
* returned vs the ids the evaluator marked relevant (方案 B). Pure + total, so
|
|
15
|
+
* callers (and tests) get deterministic numbers without touching a store:
|
|
16
|
+
* precision = |relevant ∩ retrieved| / |retrieved|
|
|
17
|
+
* recall = |relevant ∩ retrieved| / |expected|
|
|
18
|
+
* mrr = 1 / rank of the first relevant doc (0 when none retrieved)
|
|
19
|
+
* hit_count is the raw intersection size. Values are rounded to 4 decimals so
|
|
20
|
+
* repeated divisions (e.g. 1/3) never surface binary-float noise.
|
|
21
|
+
*/
|
|
22
|
+
export function computeRetrievalMetrics(actualIds, expectedIds) {
|
|
23
|
+
const expected = new Set(Array.isArray(expectedIds) ? expectedIds : []);
|
|
24
|
+
const actual = Array.isArray(actualIds) ? actualIds : [];
|
|
25
|
+
const relevant = actual.filter((id) => expected.has(id)).length;
|
|
26
|
+
const round4 = (x) => Math.round(x * 10000) / 10000;
|
|
27
|
+
let mrr = 0;
|
|
28
|
+
for (let i = 0; i < actual.length; i++) {
|
|
29
|
+
if (expected.has(actual[i])) { mrr = 1 / (i + 1); break; }
|
|
30
|
+
}
|
|
31
|
+
return {
|
|
32
|
+
precision: round4(actual.length ? relevant / actual.length : 0),
|
|
33
|
+
recall: round4(expected.size ? relevant / expected.size : 0),
|
|
34
|
+
mrr: round4(mrr),
|
|
35
|
+
hit_count: relevant
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
|
|
6
39
|
export function createService({ store, mirror, config, onWrite, logger }) {
|
|
7
40
|
// Optional dream scheduler hook, installed via setDreamHook after creation
|
|
8
41
|
// (the scheduler holds a reference back to the service, so it cannot be
|
|
@@ -357,9 +390,21 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
357
390
|
}
|
|
358
391
|
|
|
359
392
|
merged = merged.slice(0, lim);
|
|
360
|
-
|
|
393
|
+
let result = useRerank && reranker && merged.length
|
|
361
394
|
? await rerankCandidates(q, merged, lim)
|
|
362
395
|
: merged;
|
|
396
|
+
// Epistemic trust (v0.4.5): opt-in re-weighting of the final candidate
|
|
397
|
+
// scores by source credibility. When off (default) `result` is returned
|
|
398
|
+
// untouched — exactly the legacy behavior.
|
|
399
|
+
if (config.trustEpistemicWeighting === true) {
|
|
400
|
+
result = result
|
|
401
|
+
.map((m) => ({
|
|
402
|
+
...m,
|
|
403
|
+
score: (m.score ?? 0) * (EPISTEMIC_WEIGHTS[m.epistemic_status] ?? 1)
|
|
404
|
+
}))
|
|
405
|
+
.sort((a, b) => (b.score ?? 0) - (a.score ?? 0))
|
|
406
|
+
.slice(0, lim);
|
|
407
|
+
}
|
|
363
408
|
|
|
364
409
|
// Recall layer receipt: with recordRecall on, hand the actual merged
|
|
365
410
|
// candidate list (id/title/content/score/source) to the injected recorder
|
|
@@ -388,6 +433,93 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
388
433
|
return result;
|
|
389
434
|
}
|
|
390
435
|
|
|
436
|
+
/**
|
|
437
|
+
* Retrieval evaluation (方案 B): run one search for `query`, compare the ids
|
|
438
|
+
* it actually returned against `expectedIds`, and return the computed
|
|
439
|
+
* metrics. When persistence is on (config.evalPersistTestResults, or an
|
|
440
|
+
* explicit `persist` override per call) the snapshot is written to the
|
|
441
|
+
* recall_evals table — a SEPARATE store from the recall_runs production audit,
|
|
442
|
+
* so test/eval data never inflates the production trail.
|
|
443
|
+
*
|
|
444
|
+
* options:
|
|
445
|
+
* mode/topK/threshold/useRerank — passed through to searchMemories
|
|
446
|
+
* evalType — label for the snapshot (default 'manual')
|
|
447
|
+
* recordRecall — also write a recall_runs audit row for the
|
|
448
|
+
* same scene and link it via recall_run_id
|
|
449
|
+
* (default false: eval stays unlinked)
|
|
450
|
+
* recallRunId — explicit link to an existing recall_runs id
|
|
451
|
+
* persist — override the config gate for this call
|
|
452
|
+
*
|
|
453
|
+
* Returns { metrics, actualIds, expectedIds, recallRunId, persisted }.
|
|
454
|
+
* Never throws on persistence failures: a broken eval write must not break
|
|
455
|
+
* the retrieval quality measurement.
|
|
456
|
+
*/
|
|
457
|
+
async function evaluateRetrieval(query, expectedIds, options = {}) {
|
|
458
|
+
const q = String(query ?? "").trim();
|
|
459
|
+
const expected = Array.isArray(expectedIds) ? expectedIds : [];
|
|
460
|
+
const {
|
|
461
|
+
mode = "auto",
|
|
462
|
+
topK = 20,
|
|
463
|
+
threshold,
|
|
464
|
+
useRerank = true,
|
|
465
|
+
evalType = "manual",
|
|
466
|
+
recordRecall = false,
|
|
467
|
+
recallRunId = null,
|
|
468
|
+
persist = config.evalPersistTestResults === true
|
|
469
|
+
} = options;
|
|
470
|
+
if (!q) {
|
|
471
|
+
const empty = computeRetrievalMetrics([], expected);
|
|
472
|
+
return { metrics: empty, actualIds: [], expectedIds: expected, recallRunId: null, persisted: false };
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
const rows = await searchMemories(q, { mode, topK, threshold, useRerank, recordRecall: false });
|
|
476
|
+
const actualIds = rows.map((m) => m.id);
|
|
477
|
+
const metrics = computeRetrievalMetrics(actualIds, expected);
|
|
478
|
+
|
|
479
|
+
// Optional recall_runs audit for the same scene; the eval row then links to
|
|
480
|
+
// it. Kept separate from the production recorder (which fires only on
|
|
481
|
+
// recordRecall=true inside searchMemories) — eval never double-records.
|
|
482
|
+
// An explicit recallRunId wins; recordRecall only mints a NEW audit run when
|
|
483
|
+
// the caller did not already link one (never clobber an existing link).
|
|
484
|
+
let runId = recallRunId ?? null;
|
|
485
|
+
if (recordRecall && runId === null) {
|
|
486
|
+
try {
|
|
487
|
+
const run = store.saveRecallRun({
|
|
488
|
+
query: q,
|
|
489
|
+
mode,
|
|
490
|
+
topK,
|
|
491
|
+
threshold: threshold ?? null,
|
|
492
|
+
candidates: rows.map((m) => ({
|
|
493
|
+
id: m.id,
|
|
494
|
+
title: m.title,
|
|
495
|
+
content: m.content,
|
|
496
|
+
score: m.score ?? null,
|
|
497
|
+
source: m.source ?? "keyword"
|
|
498
|
+
})),
|
|
499
|
+
created_at: new Date().toISOString()
|
|
500
|
+
});
|
|
501
|
+
runId = run.id;
|
|
502
|
+
} catch { /* non-fatal: the eval itself still succeeds */ }
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
let persisted = false;
|
|
506
|
+
if (persist) {
|
|
507
|
+
try {
|
|
508
|
+
store.saveRecallEval({
|
|
509
|
+
recall_run_id: runId,
|
|
510
|
+
query: q,
|
|
511
|
+
expected_ids: expected,
|
|
512
|
+
actual_ids: actualIds,
|
|
513
|
+
metrics,
|
|
514
|
+
eval_type: evalType,
|
|
515
|
+
created_at: new Date().toISOString()
|
|
516
|
+
});
|
|
517
|
+
persisted = true;
|
|
518
|
+
} catch { /* non-fatal: measurement survives a failed eval write */ }
|
|
519
|
+
}
|
|
520
|
+
return { metrics, actualIds, expectedIds: expected, recallRunId: runId, persisted };
|
|
521
|
+
}
|
|
522
|
+
|
|
391
523
|
/**
|
|
392
524
|
* Fire-and-forget write notification; errors are swallowed to keep write
|
|
393
525
|
* paths clean. The store mutation has already committed, so a throwing
|
|
@@ -867,6 +999,8 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
867
999
|
setReranker(rn) { reranker = rn; },
|
|
868
1000
|
setRecallRecorder(fn) { recallRecorder = fn; },
|
|
869
1001
|
searchMemories,
|
|
1002
|
+
evaluateRetrieval,
|
|
1003
|
+
computeRetrievalMetrics,
|
|
870
1004
|
// passthroughs used by tools and api layers; mutations keep the mirror in sync
|
|
871
1005
|
search: (q, o) => store.search(q, o),
|
|
872
1006
|
searchVector: (v, o) => store.searchVector(v, o),
|
|
@@ -999,6 +1133,12 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
999
1133
|
listConflictPending: (opts) => store.listConflictPending(opts),
|
|
1000
1134
|
resolveConflictPending: (id, o) => store.resolveConflictPending(id, o),
|
|
1001
1135
|
countConflictPending: () => store.countConflictPending(),
|
|
1136
|
+
// Recall evaluation trail (方案 B): audit-bookkeeping semantics like the
|
|
1137
|
+
// dream/recall passthroughs above — a recall_evals write is a snapshot, not
|
|
1138
|
+
// a memory mutation, so it never triggers write hooks.
|
|
1139
|
+
saveRecallEval: (r) => store.saveRecallEval(r),
|
|
1140
|
+
getRecallEval: (id) => store.getRecallEval(id),
|
|
1141
|
+
listRecallEvals: (opts) => store.listRecallEvals(opts),
|
|
1002
1142
|
// Entity gene (v0.3.0) passthroughs for the autoDream apply path
|
|
1003
1143
|
// (applyDecisions): records supersedes relations after an update and
|
|
1004
1144
|
// migrates entity_attrs on merge. Bookkeeping writes like the audit
|
package/lib/store.js
CHANGED
|
@@ -13,6 +13,7 @@ CREATE TABLE IF NOT EXISTS memories (
|
|
|
13
13
|
archived INTEGER NOT NULL DEFAULT 0,
|
|
14
14
|
source TEXT,
|
|
15
15
|
embedding TEXT,
|
|
16
|
+
epistemic_status TEXT NOT NULL DEFAULT 'subjective',
|
|
16
17
|
last_accessed_at TEXT,
|
|
17
18
|
_full_content TEXT,
|
|
18
19
|
created_at TEXT NOT NULL,
|
|
@@ -61,6 +62,29 @@ CREATE TABLE IF NOT EXISTS recall_runs (
|
|
|
61
62
|
CREATE INDEX IF NOT EXISTS idx_recall_runs_created ON recall_runs(created_at);
|
|
62
63
|
CREATE INDEX IF NOT EXISTS idx_recall_runs_query ON recall_runs(query);
|
|
63
64
|
|
|
65
|
+
-- recall_evals: retrieval evaluation/test snapshots, kept SEPARATE from the
|
|
66
|
+
-- recall_runs production audit so test runs never inflate the production trail.
|
|
67
|
+
-- One row per evaluateRetrieval call that opted into persistence
|
|
68
|
+
-- (config.evalPersistTestResults): the query, the expected ids the operator
|
|
69
|
+
-- marked relevant, the actual ids retrieval returned, and the computed
|
|
70
|
+
-- metrics (precision/recall/mrr). recall_run_id optionally links to the
|
|
71
|
+
-- recall_runs audit row that captured the same retrieval scene (null when the
|
|
72
|
+
-- eval did not also record a run). Bookkeeping like the other audit tables: it
|
|
73
|
+
-- never triggers write hooks.
|
|
74
|
+
CREATE TABLE IF NOT EXISTS recall_evals (
|
|
75
|
+
id TEXT PRIMARY KEY,
|
|
76
|
+
recall_run_id TEXT, -- FK → recall_runs.id (optional linkage)
|
|
77
|
+
query TEXT NOT NULL,
|
|
78
|
+
expected_ids TEXT NOT NULL, -- JSON: relevant ids expected by the evaluator
|
|
79
|
+
actual_ids TEXT NOT NULL, -- JSON: ids actually retrieved
|
|
80
|
+
metrics TEXT NOT NULL, -- JSON: { precision, recall, mrr, hit_count }
|
|
81
|
+
eval_type TEXT NOT NULL DEFAULT 'manual',
|
|
82
|
+
created_at TEXT NOT NULL,
|
|
83
|
+
FOREIGN KEY (recall_run_id) REFERENCES recall_runs(id)
|
|
84
|
+
);
|
|
85
|
+
CREATE INDEX IF NOT EXISTS idx_recall_evals_created ON recall_evals(created_at);
|
|
86
|
+
CREATE INDEX IF NOT EXISTS idx_recall_evals_run ON recall_evals(recall_run_id);
|
|
87
|
+
|
|
64
88
|
-- failure_memories: records user corrections / reflection failures. Captures
|
|
65
89
|
-- what a memory was (actual) vs what the user changed it to (expected)
|
|
66
90
|
-- so later reflection passes can mine recurring correction patterns.
|
|
@@ -191,6 +215,48 @@ CREATE TABLE IF NOT EXISTS mirror_state (
|
|
|
191
215
|
|
|
192
216
|
const TYPES = new Set(["preference", "project", "decision", "history", "summary", "pattern"]);
|
|
193
217
|
|
|
218
|
+
// Epistemic status: what kind of evidence a memory rests on. Defaults to
|
|
219
|
+
// 'subjective' so legacy rows (and rows without any signal) stay compatible.
|
|
220
|
+
const EPISTEMIC_STATUSES = new Set(["observation", "subjective", "inferred"]);
|
|
221
|
+
// Rule-based inference markers, checked in priority order (observation >
|
|
222
|
+
// inferred > subjective). The default fallback is 'subjective'.
|
|
223
|
+
const OBSERVATION_RE = /实测|观察到|观测|测得|测量|结果表明|数据显示|实验|统计|结果/;
|
|
224
|
+
const INFERRED_RE = /推断|推测出|推导|推论|由此可|据此|综上|意味着|所以|因此/;
|
|
225
|
+
const SUBJECTIVE_RE = /我推测|我猜|我觉得|我感觉|可能|大概|也许|认为|猜想|似乎|猜测|感觉/;
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* Heuristically infer a memory's epistemic status from its content (and the
|
|
229
|
+
* AI-generated types). summary/pattern entries are always 'inferred' (derived
|
|
230
|
+
* from other memories); otherwise content markers decide. Pure rule-based, so
|
|
231
|
+
* it never throws and always returns a value in EPISTEMIC_STATUSES.
|
|
232
|
+
*/
|
|
233
|
+
function inferEpistemicStatus(memory) {
|
|
234
|
+
if (memory.type === "summary" || memory.type === "pattern") return "inferred";
|
|
235
|
+
const text = `${memory.title ?? ""} ${memory.content ?? ""}`;
|
|
236
|
+
if (OBSERVATION_RE.test(text)) return "observation";
|
|
237
|
+
if (INFERRED_RE.test(text)) return "inferred";
|
|
238
|
+
if (SUBJECTIVE_RE.test(text)) return "subjective";
|
|
239
|
+
return "subjective";
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/** Resolve a requested epistemic_status: explicit valid value wins, otherwise
|
|
243
|
+
* re-infer from (possibly updated) content. Never returns an invalid value. */
|
|
244
|
+
function resolveEpistemicStatus(memory, patch) {
|
|
245
|
+
if (patch?.epistemic_status !== undefined) {
|
|
246
|
+
return EPISTEMIC_STATUSES.has(patch.epistemic_status) ? patch.epistemic_status : "subjective";
|
|
247
|
+
}
|
|
248
|
+
// Re-infer whenever any signal that feeds the heuristic changed: content
|
|
249
|
+
// (marker words), title (marker words), or type (summary/pattern are always
|
|
250
|
+
// inferred). Otherwise keep the stored status.
|
|
251
|
+
const changed = ["content", "title", "type"].some(
|
|
252
|
+
(k) => patch?.[k] !== undefined && patch[k] !== memory?.[k]
|
|
253
|
+
);
|
|
254
|
+
if (changed) {
|
|
255
|
+
return inferEpistemicStatus({ ...memory, ...patch });
|
|
256
|
+
}
|
|
257
|
+
return memory?.epistemic_status ?? "subjective";
|
|
258
|
+
}
|
|
259
|
+
|
|
194
260
|
// Per-type mirror sync receipts (peer blocker 4): a type is either committed
|
|
195
261
|
// (file written + fence applied), failed (last sync round errored for it), or
|
|
196
262
|
// pending (still owed a write).
|
|
@@ -229,6 +295,7 @@ function toRow(row) {
|
|
|
229
295
|
forgotten: row.forgotten === 1,
|
|
230
296
|
archived: row.archived === 1,
|
|
231
297
|
source: row.source ?? undefined,
|
|
298
|
+
epistemic_status: row.epistemic_status ?? "subjective",
|
|
232
299
|
created_at: row.created_at,
|
|
233
300
|
updated_at: row.updated_at,
|
|
234
301
|
last_accessed_at: row.last_accessed_at ?? undefined,
|
|
@@ -305,6 +372,24 @@ function toRecallRun(row) {
|
|
|
305
372
|
};
|
|
306
373
|
}
|
|
307
374
|
|
|
375
|
+
function toRecallEval(row) {
|
|
376
|
+
if (!row) return undefined;
|
|
377
|
+
let metrics;
|
|
378
|
+
if (row.metrics != null) {
|
|
379
|
+
try { metrics = JSON.parse(row.metrics); } catch { metrics = undefined; }
|
|
380
|
+
}
|
|
381
|
+
return {
|
|
382
|
+
id: row.id,
|
|
383
|
+
recall_run_id: row.recall_run_id ?? undefined,
|
|
384
|
+
query: row.query,
|
|
385
|
+
expected_ids: parseJsonArray(row.expected_ids),
|
|
386
|
+
actual_ids: parseJsonArray(row.actual_ids),
|
|
387
|
+
metrics,
|
|
388
|
+
eval_type: row.eval_type,
|
|
389
|
+
created_at: row.created_at
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
|
|
308
393
|
function toEntity(row) {
|
|
309
394
|
if (!row) return undefined;
|
|
310
395
|
return {
|
|
@@ -421,6 +506,9 @@ export function createStore(path) {
|
|
|
421
506
|
if (!columns.includes("_full_content")) {
|
|
422
507
|
db.exec("ALTER TABLE memories ADD COLUMN _full_content TEXT");
|
|
423
508
|
}
|
|
509
|
+
if (!columns.includes("epistemic_status")) {
|
|
510
|
+
db.exec("ALTER TABLE memories ADD COLUMN epistemic_status TEXT NOT NULL DEFAULT 'subjective'");
|
|
511
|
+
}
|
|
424
512
|
|
|
425
513
|
// Legacy dream_runs without policy_epoch → backfill with the default epoch.
|
|
426
514
|
const dreamCols = db.prepare("PRAGMA table_info(dream_runs)").all().map((c) => c.name);
|
|
@@ -512,11 +600,16 @@ export function createStore(path) {
|
|
|
512
600
|
const embedding = Array.isArray(memory.embedding) && memory.embedding.length
|
|
513
601
|
? JSON.stringify(memory.embedding)
|
|
514
602
|
: null;
|
|
603
|
+
// Explicit valid status wins; otherwise infer from content/type. Falls back
|
|
604
|
+
// to 'subjective' (the column default) so legacy callers never break.
|
|
605
|
+
const epistemicStatus = EPISTEMIC_STATUSES.has(memory.epistemic_status)
|
|
606
|
+
? memory.epistemic_status
|
|
607
|
+
: inferEpistemicStatus(memory);
|
|
515
608
|
runAtomically(() => {
|
|
516
609
|
db.prepare(
|
|
517
|
-
`INSERT INTO memories (id, type, title, content, tags, importance, forgotten, source, embedding, created_at, updated_at)
|
|
518
|
-
VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?)`
|
|
519
|
-
).run(id, type, memory.title, memory.content, tags, importance, memory.source ?? null, embedding, now, now);
|
|
610
|
+
`INSERT INTO memories (id, type, title, content, tags, importance, forgotten, source, embedding, epistemic_status, created_at, updated_at)
|
|
611
|
+
VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?)`
|
|
612
|
+
).run(id, type, memory.title, memory.content, tags, importance, memory.source ?? null, embedding, epistemicStatus, now, now);
|
|
520
613
|
// desired generation bumped in the same transaction as the write: once
|
|
521
614
|
// this commits, generation > applied_generation, so a crash right after
|
|
522
615
|
// (before syncMirror) is caught by recoverMirror on restart (peer
|
|
@@ -538,9 +631,10 @@ export function createStore(path) {
|
|
|
538
631
|
const embedding = patch.embedding !== undefined
|
|
539
632
|
? (Array.isArray(patch.embedding) && patch.embedding.length ? JSON.stringify(patch.embedding) : null)
|
|
540
633
|
: existing.embedding ?? null;
|
|
634
|
+
const epistemicStatus = resolveEpistemicStatus(existing, patch);
|
|
541
635
|
runAtomically(() => {
|
|
542
636
|
db.prepare(
|
|
543
|
-
`UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, embedding=?, updated_at=? WHERE id=?`
|
|
637
|
+
`UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, embedding=?, epistemic_status=?, updated_at=? WHERE id=?`
|
|
544
638
|
).run(
|
|
545
639
|
type,
|
|
546
640
|
patch.title ?? existing.title,
|
|
@@ -549,6 +643,7 @@ export function createStore(path) {
|
|
|
549
643
|
Number.isInteger(patch.importance) ? patch.importance : existing.importance,
|
|
550
644
|
patch.source !== undefined ? patch.source : (existing.source ?? null),
|
|
551
645
|
embedding,
|
|
646
|
+
epistemicStatus,
|
|
552
647
|
now,
|
|
553
648
|
id
|
|
554
649
|
);
|
|
@@ -588,6 +683,7 @@ export function createStore(path) {
|
|
|
588
683
|
const embedding = patch.embedding !== undefined
|
|
589
684
|
? (Array.isArray(patch.embedding) && patch.embedding.length ? JSON.stringify(patch.embedding) : null)
|
|
590
685
|
: existing.embedding ?? null;
|
|
686
|
+
const epistemicStatus = resolveEpistemicStatus(existing, patch);
|
|
591
687
|
// The CAS UPDATE and the desired-generation bump must commit together (audit
|
|
592
688
|
// peer A): if the UPDATE autocommits first and the process dies before the
|
|
593
689
|
// increment, the store is mutated while generation == applied_generation and
|
|
@@ -597,7 +693,7 @@ export function createStore(path) {
|
|
|
597
693
|
let applied = false;
|
|
598
694
|
runAtomically(() => {
|
|
599
695
|
const result = db.prepare(
|
|
600
|
-
`UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, embedding=?, updated_at=?
|
|
696
|
+
`UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, embedding=?, epistemic_status=?, updated_at=?
|
|
601
697
|
WHERE id=? AND updated_at=?`
|
|
602
698
|
).run(
|
|
603
699
|
type,
|
|
@@ -607,6 +703,7 @@ export function createStore(path) {
|
|
|
607
703
|
Number.isInteger(patch.importance) ? patch.importance : existing.importance,
|
|
608
704
|
patch.source !== undefined ? patch.source : (existing.source ?? null),
|
|
609
705
|
embedding,
|
|
706
|
+
epistemicStatus,
|
|
610
707
|
now,
|
|
611
708
|
id,
|
|
612
709
|
expectedUpdatedAt
|
|
@@ -1001,6 +1098,60 @@ export function createStore(path) {
|
|
|
1001
1098
|
return rows.map(toRecallRun);
|
|
1002
1099
|
}
|
|
1003
1100
|
|
|
1101
|
+
// --- recall evaluation trail (方案 B: separate from the production audit) -
|
|
1102
|
+
|
|
1103
|
+
/**
|
|
1104
|
+
* Persist one retrieval-evaluation snapshot into recall_evals — the test/eval
|
|
1105
|
+
* sibling of recall_runs, deliberately stored apart so eval snapshots never
|
|
1106
|
+
* inflate the production recall audit. Like the other audit tables this is
|
|
1107
|
+
* bookkeeping: it never triggers write hooks. Writes are idempotent on id
|
|
1108
|
+
* (replay overwrites, never duplicates), matching saveRecallRun. recall_run_id
|
|
1109
|
+
* optionally links the eval to the recall_runs row that captured the same
|
|
1110
|
+
* retrieval scene (FK-referenced, null when no run was recorded).
|
|
1111
|
+
*/
|
|
1112
|
+
function saveRecallEval(evalRow) {
|
|
1113
|
+
const id = evalRow.id ?? randomUUID();
|
|
1114
|
+
db.prepare(
|
|
1115
|
+
`INSERT INTO recall_evals (id, recall_run_id, query, expected_ids, actual_ids, metrics, eval_type, created_at)
|
|
1116
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1117
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
1118
|
+
recall_run_id=excluded.recall_run_id, query=excluded.query,
|
|
1119
|
+
expected_ids=excluded.expected_ids, actual_ids=excluded.actual_ids,
|
|
1120
|
+
metrics=excluded.metrics, eval_type=excluded.eval_type,
|
|
1121
|
+
created_at=excluded.created_at`
|
|
1122
|
+
).run(
|
|
1123
|
+
id,
|
|
1124
|
+
evalRow.recall_run_id ?? null,
|
|
1125
|
+
evalRow.query,
|
|
1126
|
+
JSON.stringify(evalRow.expected_ids ?? []),
|
|
1127
|
+
JSON.stringify(evalRow.actual_ids ?? []),
|
|
1128
|
+
JSON.stringify(evalRow.metrics ?? {}),
|
|
1129
|
+
evalRow.eval_type ?? "manual",
|
|
1130
|
+
evalRow.created_at ?? nowIso()
|
|
1131
|
+
);
|
|
1132
|
+
return getRecallEval(id);
|
|
1133
|
+
}
|
|
1134
|
+
|
|
1135
|
+
function getRecallEval(id) {
|
|
1136
|
+
const row = db.prepare("SELECT * FROM recall_evals WHERE id = ?").get(id);
|
|
1137
|
+
return toRecallEval(row);
|
|
1138
|
+
}
|
|
1139
|
+
|
|
1140
|
+
function listRecallEvals({ limit = 50, offset = 0, query } = {}) {
|
|
1141
|
+
const { limit: lim, offset: off } = sanitizePage(limit, offset, 50);
|
|
1142
|
+
const clauses = [];
|
|
1143
|
+
const params = [];
|
|
1144
|
+
if (query) {
|
|
1145
|
+
clauses.push("query LIKE ? ESCAPE '\\'");
|
|
1146
|
+
params.push(`%${escapeLike(String(query))}%`);
|
|
1147
|
+
}
|
|
1148
|
+
const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
|
|
1149
|
+
const rows = db.prepare(
|
|
1150
|
+
`SELECT * FROM recall_evals ${where} ORDER BY created_at DESC, id LIMIT ? OFFSET ?`
|
|
1151
|
+
).all(...params, lim, off);
|
|
1152
|
+
return rows.map(toRecallEval);
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1004
1155
|
// --- failure memories ----------------------------------------------------
|
|
1005
1156
|
|
|
1006
1157
|
/**
|
|
@@ -1524,6 +1675,9 @@ export function createStore(path) {
|
|
|
1524
1675
|
saveRecallRun,
|
|
1525
1676
|
getRecallRun,
|
|
1526
1677
|
listRecallRuns,
|
|
1678
|
+
saveRecallEval,
|
|
1679
|
+
getRecallEval,
|
|
1680
|
+
listRecallEvals,
|
|
1527
1681
|
saveFailure,
|
|
1528
1682
|
listFailures,
|
|
1529
1683
|
getFailureStats,
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@modusensus/dsh-mneme",
|
|
3
3
|
"description": "Cross-session memory plugin for DeepSeek Harness with autoDream consolidation: SQLite store, Markdown mirrors, 7 model tools, automatic injection, session summarization, user profile/rules, custom slash commands, vector (semantic) search, and a Web GUI panel",
|
|
4
|
-
"version": "0.4.
|
|
4
|
+
"version": "0.4.5",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"main": "lib/index.js",
|
package/src/config.js
CHANGED
|
@@ -167,4 +167,22 @@ export const Config = z.object({
|
|
|
167
167
|
z.const("high"),
|
|
168
168
|
z.const("none")
|
|
169
169
|
]).default("none"),
|
|
170
|
+
|
|
171
|
+
// --- epistemic trust: memory source credibility (v0.4.5) -----------------
|
|
172
|
+
// Distinguish memories by source: observation (measured / witnessed),
|
|
173
|
+
// subjective (opinion / guess) and inferred (derived from other evidence).
|
|
174
|
+
// Opt-in by default: when false (default) retrieval ranking, injection
|
|
175
|
+
// marking and dream merge/conflict keepSource are untouched and
|
|
176
|
+
// epistemic_status stays inert data (still written + inferred on save, just
|
|
177
|
+
// never used to influence behavior).
|
|
178
|
+
trustEpistemicWeighting: z.boolean().default(false),
|
|
179
|
+
|
|
180
|
+
// --- recall evaluation: test-result storage (v0.4.6, 方案 B) --------------
|
|
181
|
+
// Separate retrieval evaluation snapshots from the production recall audit.
|
|
182
|
+
// When false (default) evaluateRetrieval still computes precision/recall/mrr
|
|
183
|
+
// and returns them to the caller, but writes nothing to recall_evals — the
|
|
184
|
+
// eval table only grows when the operator opts in. Production searchMemories
|
|
185
|
+
// audits to recall_runs and NEVER touches recall_evals, regardless of this
|
|
186
|
+
// flag (production isolation is unconditional).
|
|
187
|
+
evalPersistTestResults: z.boolean().default(false),
|
|
170
188
|
});
|