@modusensus/dsh-mneme 0.5.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +434 -419
- package/lib/client.js +1302 -1302
- package/lib/config.js +15 -6
- package/lib/dream/sleep.js +4 -4
- package/lib/dream.js +96 -4
- package/lib/hot-memory.js +53 -53
- package/lib/index.js +20 -0
- package/lib/reranker.js +218 -218
- package/lib/service.js +1517 -1489
- package/lib/store.js +65 -6
- package/lib/tools.js +16 -5
- package/package.json +1 -1
- package/src/config.js +15 -6
- package/src/dream/sleep.js +4 -4
- package/src/dream.js +96 -4
- package/src/hot-memory.js +53 -53
- package/src/index.js +20 -0
- package/src/reranker.js +218 -218
- package/src/service.js +1517 -1489
- package/src/store.js +65 -6
- package/src/tools.js +16 -5
- package/test/hot-memory.test.js +174 -174
- package/test/normalize-decisions.test.js +120 -0
- package/test/reasoning-effort.test.js +27 -0
- package/test/reranker.test.js +240 -240
- package/test/service-search.test.js +199 -199
- package/test/service.test.js +106 -0
- package/test/store.test.js +76 -0
- package/test/tools.test.js +20 -0
package/src/store.js
CHANGED
|
@@ -11,6 +11,7 @@ CREATE TABLE IF NOT EXISTS memories (
|
|
|
11
11
|
importance INTEGER NOT NULL DEFAULT 3,
|
|
12
12
|
forgotten INTEGER NOT NULL DEFAULT 0,
|
|
13
13
|
archived INTEGER NOT NULL DEFAULT 0,
|
|
14
|
+
session_disposed_at TEXT,
|
|
14
15
|
source TEXT,
|
|
15
16
|
session_id TEXT,
|
|
16
17
|
content_history TEXT,
|
|
@@ -321,6 +322,7 @@ function toRow(row) {
|
|
|
321
322
|
importance: row.importance,
|
|
322
323
|
forgotten: row.forgotten === 1,
|
|
323
324
|
archived: row.archived === 1,
|
|
325
|
+
session_disposed_at: row.session_disposed_at ?? undefined,
|
|
324
326
|
source: row.source ?? undefined,
|
|
325
327
|
session_id: row.session_id ?? undefined,
|
|
326
328
|
content_history: parseJsonArray(row.content_history),
|
|
@@ -567,6 +569,7 @@ export function createStore(path) {
|
|
|
567
569
|
};
|
|
568
570
|
|
|
569
571
|
addColumn("memories", "archived", "ALTER TABLE memories ADD COLUMN archived INTEGER NOT NULL DEFAULT 0");
|
|
572
|
+
addColumn("memories", "session_disposed_at", "ALTER TABLE memories ADD COLUMN session_disposed_at TEXT");
|
|
570
573
|
addColumn("memories", "embedding", "ALTER TABLE memories ADD COLUMN embedding TEXT");
|
|
571
574
|
addColumn("memories", "last_accessed_at", "ALTER TABLE memories ADD COLUMN last_accessed_at TEXT");
|
|
572
575
|
addColumn("memories", "_full_content", "ALTER TABLE memories ADD COLUMN _full_content TEXT");
|
|
@@ -575,6 +578,12 @@ export function createStore(path) {
|
|
|
575
578
|
addColumn("memories", "quality_score", "ALTER TABLE memories ADD COLUMN quality_score REAL");
|
|
576
579
|
addColumn("memories", "session_id", "ALTER TABLE memories ADD COLUMN session_id TEXT");
|
|
577
580
|
|
|
581
|
+
// Composite index for session-lifecycle queries (dispose/restore/listBySession).
|
|
582
|
+
// Created post-migration, NOT in SCHEMA: on legacy DBs both columns arrive via
|
|
583
|
+
// ADD COLUMN above, so the index would fail at db.exec(SCHEMA) time. CREATE
|
|
584
|
+
// INDEX IF NOT EXISTS is atomic, so the two-process race is safe here.
|
|
585
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_memories_session ON memories(session_id, session_disposed_at)");
|
|
586
|
+
|
|
578
587
|
// Legacy dream_runs without policy_epoch → backfill with the default epoch.
|
|
579
588
|
addColumn("dream_runs", "policy_epoch", "ALTER TABLE dream_runs ADD COLUMN policy_epoch INTEGER NOT NULL DEFAULT 0");
|
|
580
589
|
addColumn("dream_runs", "run_type", "ALTER TABLE dream_runs ADD COLUMN run_type TEXT NOT NULL DEFAULT 'auto'");
|
|
@@ -618,7 +627,7 @@ export function createStore(path) {
|
|
|
618
627
|
return ts;
|
|
619
628
|
}
|
|
620
629
|
|
|
621
|
-
function count(type, { includeForgotten = false, includeArchived = false } = {}) {
|
|
630
|
+
function count(type, { includeForgotten = false, includeArchived = false, includeDisposed = false } = {}) {
|
|
622
631
|
const clauses = [];
|
|
623
632
|
const params = [];
|
|
624
633
|
if (type !== undefined) {
|
|
@@ -631,6 +640,9 @@ export function createStore(path) {
|
|
|
631
640
|
if (!includeArchived) {
|
|
632
641
|
clauses.push("archived = 0");
|
|
633
642
|
}
|
|
643
|
+
if (!includeDisposed) {
|
|
644
|
+
clauses.push("session_disposed_at IS NULL");
|
|
645
|
+
}
|
|
634
646
|
const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
|
|
635
647
|
return db.prepare(`SELECT count(*) AS c FROM memories ${where}`).get(...params).c;
|
|
636
648
|
}
|
|
@@ -820,6 +832,45 @@ export function createStore(path) {
|
|
|
820
832
|
return getById(id);
|
|
821
833
|
}
|
|
822
834
|
|
|
835
|
+
// --- session lifecycle (v0.6.0) ------------------------------------------
|
|
836
|
+
// Session dispose is orthogonal to `archived`: memory_archive is the user/AI
|
|
837
|
+
// choosing to keep an entry long-term-but-quiet, while session_disposed_at
|
|
838
|
+
// marks entries hidden because the session they were born in was deleted
|
|
839
|
+
// (a reversible "undo" — restoreBySession clears it). They never clobber each
|
|
840
|
+
// other: restoreBySession must not resurrect user-archived memories.
|
|
841
|
+
// Mirrors list/search: disposed rows are hidden by default. A consumer that
|
|
842
|
+
// needs to see the full picture (e.g. a restore flow that tells the user
|
|
843
|
+
// "these N entries were hidden") opts in via includeDisposed.
|
|
844
|
+
function listBySession(sessionId, { includeDisposed = false } = {}) {
|
|
845
|
+
const disposedFilter = includeDisposed ? "" : "AND session_disposed_at IS NULL";
|
|
846
|
+
const rows = db.prepare(
|
|
847
|
+
`SELECT * FROM memories WHERE session_id = ? ${disposedFilter} ORDER BY updated_at DESC`
|
|
848
|
+
).all(sessionId);
|
|
849
|
+
return rows.map(toRow);
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
// Idempotent by state guard, not timestamp compare (nowIso() differs every
|
|
853
|
+
// call, so a fresh-timestamp re-dispose would spuriously count): dispose only
|
|
854
|
+
// touches rows that are NOT yet disposed; restore only touches rows that ARE.
|
|
855
|
+
// updated_at is deliberately left alone — this is a lifecycle flag, not
|
|
856
|
+
// content — so a true flip is the sole trigger for a mirror generation.
|
|
857
|
+
function setDisposedBySession(sessionId, disposed) {
|
|
858
|
+
const at = disposed ? nowIso() : null;
|
|
859
|
+
let affected = 0;
|
|
860
|
+
runAtomically(() => {
|
|
861
|
+
const result = disposed
|
|
862
|
+
? db.prepare(
|
|
863
|
+
"UPDATE memories SET session_disposed_at = ? WHERE session_id = ? AND session_disposed_at IS NULL"
|
|
864
|
+
).run(at, sessionId)
|
|
865
|
+
: db.prepare(
|
|
866
|
+
"UPDATE memories SET session_disposed_at = NULL WHERE session_id = ? AND session_disposed_at IS NOT NULL"
|
|
867
|
+
).run(sessionId);
|
|
868
|
+
affected = result.changes;
|
|
869
|
+
if (affected > 0) incrementGeneration();
|
|
870
|
+
});
|
|
871
|
+
return affected;
|
|
872
|
+
}
|
|
873
|
+
|
|
823
874
|
// --- sleep-mode storage support (v0.4.0) ---------------------------------
|
|
824
875
|
// touchLastAccess stamps the read time on recall/inject paths. It deliberately
|
|
825
876
|
// does NOT bump the mirror generation: reads must not mark the mirror dirty.
|
|
@@ -876,6 +927,7 @@ export function createStore(path) {
|
|
|
876
927
|
const rows = db.prepare(
|
|
877
928
|
`SELECT * FROM memories
|
|
878
929
|
WHERE forgotten = 0 AND archived = 0
|
|
930
|
+
AND session_disposed_at IS NULL
|
|
879
931
|
AND (last_accessed_at IS NULL OR last_accessed_at < ?)
|
|
880
932
|
ORDER BY COALESCE(last_accessed_at, created_at) ASC, id
|
|
881
933
|
LIMIT ?`
|
|
@@ -883,7 +935,7 @@ export function createStore(path) {
|
|
|
883
935
|
return rows.map(toRow);
|
|
884
936
|
}
|
|
885
937
|
|
|
886
|
-
function list({ type, limit = 50, offset = 0, includeForgotten = false, includeArchived = false } = {}) {
|
|
938
|
+
function list({ type, limit = 50, offset = 0, includeForgotten = false, includeArchived = false, includeDisposed = false } = {}) {
|
|
887
939
|
const clauses = [];
|
|
888
940
|
const params = [];
|
|
889
941
|
if (type) {
|
|
@@ -896,6 +948,9 @@ export function createStore(path) {
|
|
|
896
948
|
if (!includeArchived) {
|
|
897
949
|
clauses.push("archived = 0");
|
|
898
950
|
}
|
|
951
|
+
if (!includeDisposed) {
|
|
952
|
+
clauses.push("session_disposed_at IS NULL");
|
|
953
|
+
}
|
|
899
954
|
const { limit: lim, offset: off } = sanitizePage(limit, offset, 50);
|
|
900
955
|
const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
|
|
901
956
|
const rows = db.prepare(
|
|
@@ -953,7 +1008,7 @@ export function createStore(path) {
|
|
|
953
1008
|
).all(limit);
|
|
954
1009
|
}
|
|
955
1010
|
|
|
956
|
-
function search(query, { limit = 20, includeArchived = false } = {}) {
|
|
1011
|
+
function search(query, { limit = 20, includeArchived = false, includeDisposed = false } = {}) {
|
|
957
1012
|
const q = String(query).trim();
|
|
958
1013
|
if (!q) return [];
|
|
959
1014
|
// Plain LIKE substring scan over title/content/tags (wildcards escaped so
|
|
@@ -962,9 +1017,10 @@ export function createStore(path) {
|
|
|
962
1017
|
const like = `%${escapeLike(q)}%`;
|
|
963
1018
|
const { limit: lim } = sanitizePage(limit, 0, 20);
|
|
964
1019
|
const archivedFilter = includeArchived ? "" : "archived = 0 AND ";
|
|
1020
|
+
const disposedFilter = includeDisposed ? "" : "session_disposed_at IS NULL AND ";
|
|
965
1021
|
const rows = db.prepare(
|
|
966
1022
|
`SELECT * FROM memories
|
|
967
|
-
WHERE ${archivedFilter}forgotten = 0 AND (title LIKE ? ESCAPE '\\' OR content LIKE ? ESCAPE '\\' OR tags LIKE ? ESCAPE '\\')
|
|
1023
|
+
WHERE ${archivedFilter}${disposedFilter}forgotten = 0 AND (title LIKE ? ESCAPE '\\' OR content LIKE ? ESCAPE '\\' OR tags LIKE ? ESCAPE '\\')
|
|
968
1024
|
ORDER BY
|
|
969
1025
|
CASE WHEN title LIKE ? ESCAPE '\\' THEN 0 ELSE 1 END,
|
|
970
1026
|
importance DESC,
|
|
@@ -995,12 +1051,13 @@ export function createStore(path) {
|
|
|
995
1051
|
* Brute-force cosine similarity over embedded rows. Returns rows decorated
|
|
996
1052
|
* with a `score` (0..1). Only rows with a stored embedding participate.
|
|
997
1053
|
*/
|
|
998
|
-
function searchVector(vector, { limit = 20, includeArchived = false, threshold = 0 } = {}) {
|
|
1054
|
+
function searchVector(vector, { limit = 20, includeArchived = false, includeDisposed = false, threshold = 0 } = {}) {
|
|
999
1055
|
if (!Array.isArray(vector) || !vector.length) return [];
|
|
1000
1056
|
const archivedFilter = includeArchived ? "" : "archived = 0 AND ";
|
|
1057
|
+
const disposedFilter = includeDisposed ? "" : "session_disposed_at IS NULL AND ";
|
|
1001
1058
|
const rows = db.prepare(
|
|
1002
1059
|
`SELECT * FROM memories
|
|
1003
|
-
WHERE ${archivedFilter}forgotten = 0 AND embedding IS NOT NULL AND embedding != ''`
|
|
1060
|
+
WHERE ${archivedFilter}${disposedFilter}forgotten = 0 AND embedding IS NOT NULL AND embedding != ''`
|
|
1004
1061
|
).all();
|
|
1005
1062
|
const scored = [];
|
|
1006
1063
|
for (const row of rows) {
|
|
@@ -1871,6 +1928,8 @@ export function createStore(path) {
|
|
|
1871
1928
|
remove,
|
|
1872
1929
|
setForget,
|
|
1873
1930
|
setArchived,
|
|
1931
|
+
listBySession,
|
|
1932
|
+
setDisposedBySession,
|
|
1874
1933
|
touchLastAccess,
|
|
1875
1934
|
demoteToSummary,
|
|
1876
1935
|
restoreContent,
|
package/src/tools.js
CHANGED
|
@@ -180,9 +180,10 @@ export function createTools(ctx, service, config, embedder) {
|
|
|
180
180
|
|
|
181
181
|
defineTool({
|
|
182
182
|
name: "memory_delete",
|
|
183
|
-
description: "Permanently delete a memory entry.",
|
|
183
|
+
description: "Permanently delete a memory entry. Pass id for exact delete, or query to delete the single best-matching entry by text — lets the agent honor 'delete the memory about X' without a prior list/search round trip.",
|
|
184
184
|
parameters: {
|
|
185
|
-
id: { type: "string",
|
|
185
|
+
id: { type: "string", description: "Exact memory id to delete (from memory_list/memory_search output)" },
|
|
186
|
+
query: { type: "string", description: "Delete the best-matching entry for this text (searches title/content/tags; uses hybrid recall when an embedder is configured)" }
|
|
186
187
|
},
|
|
187
188
|
output: {
|
|
188
189
|
schema: {
|
|
@@ -193,9 +194,19 @@ export function createTools(ctx, service, config, embedder) {
|
|
|
193
194
|
render: (_args, value) => TEXT_OUTPUT(value.deleted ? "Memory deleted." : "Memory not found.")
|
|
194
195
|
},
|
|
195
196
|
async execute(args) {
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
197
|
+
if (args.id) {
|
|
198
|
+
const existed = service.getById(args.id) !== undefined;
|
|
199
|
+
if (existed) service.remove(args.id);
|
|
200
|
+
return { deleted: existed };
|
|
201
|
+
}
|
|
202
|
+
if (args.query) {
|
|
203
|
+
const [best] = await service.searchMemories(args.query, { mode: "auto", topK: 1, useRerank: true });
|
|
204
|
+
if (best) {
|
|
205
|
+
service.remove(best.id);
|
|
206
|
+
return { deleted: true };
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
return { deleted: false };
|
|
199
210
|
}
|
|
200
211
|
}),
|
|
201
212
|
|
package/test/hot-memory.test.js
CHANGED
|
@@ -1,174 +1,174 @@
|
|
|
1
|
-
import test from "node:test";
|
|
2
|
-
import assert from "node:assert/strict";
|
|
3
|
-
import { createHotMemory, estimateTokens } from "../src/hot-memory.js";
|
|
4
|
-
import { createStore } from "../src/store.js";
|
|
5
|
-
import { createService } from "../src/service.js";
|
|
6
|
-
import { createVectorIndex } from "../src/vector-index.js";
|
|
7
|
-
|
|
8
|
-
// --- hot memory buffer ---
|
|
9
|
-
|
|
10
|
-
test("hot memory keeps the latest rounds within maxRounds", () => {
|
|
11
|
-
const hot = createHotMemory({ maxRounds: 2, maxTokens: 10000 });
|
|
12
|
-
hot.add({ query: "第一轮", response: "答一" });
|
|
13
|
-
hot.add({ query: "第二轮", response: "答二" });
|
|
14
|
-
hot.add({ query: "第三轮", response: "答三" });
|
|
15
|
-
assert.equal(hot.rounds().length, 2);
|
|
16
|
-
assert.ok(hot.getContext().includes("第三轮"));
|
|
17
|
-
assert.ok(!hot.getContext().includes("第一轮"));
|
|
18
|
-
});
|
|
19
|
-
|
|
20
|
-
test("hot memory enforces the token budget", () => {
|
|
21
|
-
const hot = createHotMemory({ maxRounds: 10, maxTokens: 30 });
|
|
22
|
-
hot.add({ query: "很长的第一轮问题".repeat(10), response: "很长的回答".repeat(10) });
|
|
23
|
-
hot.add({ query: "第二轮", response: "答二" });
|
|
24
|
-
// The first round alone blows the budget; the newest round survives and
|
|
25
|
-
// the buffer never empties completely.
|
|
26
|
-
const rounds = hot.rounds();
|
|
27
|
-
assert.ok(rounds.length >= 1);
|
|
28
|
-
assert.equal(rounds[rounds.length - 1].query, "第二轮");
|
|
29
|
-
});
|
|
30
|
-
|
|
31
|
-
test("hot memory getContext uses the Q/A round format", () => {
|
|
32
|
-
const hot = createHotMemory({ maxRounds: 5, maxTokens: 10000 });
|
|
33
|
-
hot.add({ query: "Q1", response: "A1" });
|
|
34
|
-
hot.add({ query: "Q2", response: "A2" });
|
|
35
|
-
assert.equal(hot.getContext(), "Q: Q1\nA: A1\n\nQ: Q2\nA: A2");
|
|
36
|
-
});
|
|
37
|
-
|
|
38
|
-
test("hot memory ignores empty rounds and clears", () => {
|
|
39
|
-
const hot = createHotMemory({ maxRounds: 5, maxTokens: 10000 });
|
|
40
|
-
hot.add({ query: "", response: "x" });
|
|
41
|
-
hot.add(null);
|
|
42
|
-
assert.equal(hot.rounds().length, 0);
|
|
43
|
-
hot.add({ query: "q" });
|
|
44
|
-
hot.clear();
|
|
45
|
-
assert.equal(hot.getContext(), "");
|
|
46
|
-
});
|
|
47
|
-
|
|
48
|
-
test("estimateTokens counts CJK heavier than ASCII", () => {
|
|
49
|
-
assert.ok(estimateTokens("中文内容") > estimateTokens("abcd"));
|
|
50
|
-
});
|
|
51
|
-
|
|
52
|
-
// --- entry defense: non-positive / non-integer bounds fall back to defaults ---
|
|
53
|
-
// Bug: createHotMemory({ maxRounds: -1 }) made the eviction while-loop
|
|
54
|
-
// `while (buffer.length > maxRounds)` unbounded — after the buffer emptied,
|
|
55
|
-
// `0 > -1` stayed true and buffer.shift() on an empty array is a no-op, so
|
|
56
|
-
// every add() spun forever. Non-integer values (1.5, NaN, null) were also
|
|
57
|
-
// silently wrong. The fix clamps them to the 5/2000 defaults at the door.
|
|
58
|
-
|
|
59
|
-
test("hot memory falls back to maxRounds=5 for non-positive/invalid values", () => {
|
|
60
|
-
for (const bad of [0, -1, 1.5, NaN, null]) {
|
|
61
|
-
const hot = createHotMemory({ maxRounds: bad, maxTokens: 10000 });
|
|
62
|
-
for (let i = 0; i < 8; i++) hot.add({ query: `第${i}轮`, response: "x" });
|
|
63
|
-
assert.equal(hot.rounds().length, 5, `maxRounds=${bad} must fall back to 5, no infinite loop`);
|
|
64
|
-
assert.ok(hot.getContext().includes("第7轮"), `maxRounds=${bad}: newest round survives`);
|
|
65
|
-
assert.ok(!hot.getContext().includes("第0轮"), `maxRounds=${bad}: oldest round evicted`);
|
|
66
|
-
}
|
|
67
|
-
});
|
|
68
|
-
|
|
69
|
-
test("hot memory falls back to maxTokens=2000 for non-positive/infinite values", () => {
|
|
70
|
-
for (const bad of [0, -1, Infinity]) {
|
|
71
|
-
const hot = createHotMemory({ maxRounds: 50, maxTokens: bad });
|
|
72
|
-
// 50 rounds at ~74 tokens each blow a 2000-token budget; the fallback must
|
|
73
|
-
// evict into (1, 50). A broken budget of 0/-1 would squeeze to 1 round and
|
|
74
|
-
// Infinity would keep all 50 — both are the pre-fix behavior.
|
|
75
|
-
for (let i = 0; i < 50; i++) hot.add({ query: `第${i}轮`, response: "长回答".repeat(40) });
|
|
76
|
-
const n = hot.rounds().length;
|
|
77
|
-
assert.ok(n > 1 && n < 50, `maxTokens=${bad} falls back to 2000 (kept ${n} rounds)`);
|
|
78
|
-
}
|
|
79
|
-
});
|
|
80
|
-
|
|
81
|
-
// --- service-level: BM25 fusion + semantic dedup + selective injection ---
|
|
82
|
-
|
|
83
|
-
function toyVec(text) {
|
|
84
|
-
const v = new Array(64).fill(0);
|
|
85
|
-
for (const t of String(text).toLowerCase().split(/[^\p{L}\p{N}]+/u).filter(Boolean)) {
|
|
86
|
-
let h = 0;
|
|
87
|
-
for (const ch of t) h = (h * 31 + ch.codePointAt(0)) >>> 0;
|
|
88
|
-
v[h % 64] += 1;
|
|
89
|
-
}
|
|
90
|
-
const norm = Math.sqrt(v.reduce((s, x) => s + x * x, 0)) || 1;
|
|
91
|
-
return v.map((x) => x / norm);
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
function setup(overrides = {}) {
|
|
95
|
-
const store = createStore(":memory:");
|
|
96
|
-
const config = {
|
|
97
|
-
entitySearchEnabled: false,
|
|
98
|
-
bm25SearchEnabled: true,
|
|
99
|
-
adaptiveThresholdEnabled: false,
|
|
100
|
-
searchSemanticDedup: true,
|
|
101
|
-
selectiveInjectEnabled: true,
|
|
102
|
-
...overrides
|
|
103
|
-
};
|
|
104
|
-
const service = createService({ store, mirror: null, config, logger: null });
|
|
105
|
-
service.setVectorIndex(createVectorIndex({ store, logger: null }));
|
|
106
|
-
service.setEmbedder({ embedSingle: async (t) => toyVec(t) });
|
|
107
|
-
return { store, service };
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
test("searchMemories fuses BM25: scattered-term queries recall both rows", async () => {
|
|
111
|
-
const { store, service } = setup();
|
|
112
|
-
const a = store.save({ type: "project", title: "异步并发模式", content: "async runtime 选用 tokio", importance: 3 });
|
|
113
|
-
const b = store.save({ type: "decision", title: "语言迁移", content: "编译模块迁移到 Rust", importance: 3 });
|
|
114
|
-
store.save({ type: "project", title: "无关", content: "夜间 ETL 脚本", importance: 3 });
|
|
115
|
-
|
|
116
|
-
// "rust 异步" is not a substring of either row — LIKE misses both; BM25
|
|
117
|
-
// must surface both rows in the merged result.
|
|
118
|
-
const results = await service.searchMemories("rust 异步", { mode: "auto", topK: 5 });
|
|
119
|
-
const ids = results.map((r) => r.id);
|
|
120
|
-
assert.ok(ids.includes(a.id), "async row must be recalled via BM25");
|
|
121
|
-
assert.ok(ids.includes(b.id), "rust row must be recalled via BM25");
|
|
122
|
-
assert.equal(results.find((r) => r.id === a.id)?.source, "bm25");
|
|
123
|
-
});
|
|
124
|
-
|
|
125
|
-
test("bm25SearchEnabled=false restores the two-path behavior", async () => {
|
|
126
|
-
const { store, service } = setup({ bm25SearchEnabled: false });
|
|
127
|
-
const a = store.save({ type: "project", title: "异步并发模式", content: "async runtime 选用 tokio", importance: 3 });
|
|
128
|
-
const results = await service.searchMemories("rust 异步", { mode: "auto", topK: 5 });
|
|
129
|
-
assert.ok(!results.some((r) => r.id === a.id), "no BM25 → scattered-term miss is back");
|
|
130
|
-
});
|
|
131
|
-
|
|
132
|
-
test("search-time semantic dedup drops near-identical embeddings", async () => {
|
|
133
|
-
const { store, service } = setup();
|
|
134
|
-
const a = store.save({ type: "project", title: "偏好 A", content: "用户喜欢深色主题编辑器", importance: 3 });
|
|
135
|
-
const b = store.save({ type: "project", title: "偏好 A 备份", content: "用户喜欢深色主题编辑器(备份)", importance: 3 });
|
|
136
|
-
store.setEmbedding(a.id, toyVec("用户喜欢深色主题编辑器"));
|
|
137
|
-
store.setEmbedding(b.id, toyVec("用户喜欢深色主题编辑器"));
|
|
138
|
-
|
|
139
|
-
// auto (not keyword): keyword mode is the documented text-only path and is
|
|
140
|
-
// exempt from dedup by design; auto exercises the dedup the way production
|
|
141
|
-
// searches run.
|
|
142
|
-
const results = await service.searchMemories("深色主题", { mode: "auto", topK: 5 });
|
|
143
|
-
const ids = results.map((r) => r.id);
|
|
144
|
-
assert.ok(ids.includes(a.id) !== ids.includes(b.id), "one of the near-duplicate pair is dropped");
|
|
145
|
-
});
|
|
146
|
-
|
|
147
|
-
test("searchSemanticDedup=false keeps duplicate embeddings", async () => {
|
|
148
|
-
const { store, service } = setup({ searchSemanticDedup: false });
|
|
149
|
-
const a = store.save({ type: "project", title: "偏好 A", content: "用户喜欢深色主题编辑器", importance: 3 });
|
|
150
|
-
const b = store.save({ type: "project", title: "偏好 A 备份", content: "用户喜欢深色主题编辑器(备份)", importance: 3 });
|
|
151
|
-
store.setEmbedding(a.id, toyVec("用户喜欢深色主题编辑器"));
|
|
152
|
-
store.setEmbedding(b.id, toyVec("用户喜欢深色主题编辑器"));
|
|
153
|
-
const results = await service.searchMemories("深色主题", { mode: "keyword", topK: 5 });
|
|
154
|
-
assert.equal(results.length, 2);
|
|
155
|
-
});
|
|
156
|
-
|
|
157
|
-
test("selective injection re-orders candidates by query similarity", () => {
|
|
158
|
-
const { store, service } = setup();
|
|
159
|
-
const thesis = store.save({ type: "project", title: "湿地论文", content: "毕业论文研究城市湿地公园", importance: 5 });
|
|
160
|
-
const plugin = store.save({ type: "project", title: "插件项目", content: "dsh-mneme 记忆插件开发", importance: 5 });
|
|
161
|
-
store.setEmbedding(thesis.id, toyVec("毕业论文研究城市湿地公园"));
|
|
162
|
-
store.setEmbedding(plugin.id, toyVec("dsh-mneme 记忆插件开发"));
|
|
163
|
-
|
|
164
|
-
// Rule-based order would put both at equal importance; the query vector is
|
|
165
|
-
// about the thesis, so topic ranking must put the thesis memory first.
|
|
166
|
-
const picked = service.injectCandidates({
|
|
167
|
-
query: "论文写作",
|
|
168
|
-
queryVector: toyVec("毕业论文研究城市湿地公园"),
|
|
169
|
-
maxItems: 2,
|
|
170
|
-
threshold: 3
|
|
171
|
-
});
|
|
172
|
-
assert.equal(picked[0].id, thesis.id);
|
|
173
|
-
assert.ok(picked.some((m) => m.id === plugin.id));
|
|
174
|
-
});
|
|
1
|
+
import test from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { createHotMemory, estimateTokens } from "../src/hot-memory.js";
|
|
4
|
+
import { createStore } from "../src/store.js";
|
|
5
|
+
import { createService } from "../src/service.js";
|
|
6
|
+
import { createVectorIndex } from "../src/vector-index.js";
|
|
7
|
+
|
|
8
|
+
// --- hot memory buffer ---
|
|
9
|
+
|
|
10
|
+
test("hot memory keeps the latest rounds within maxRounds", () => {
|
|
11
|
+
const hot = createHotMemory({ maxRounds: 2, maxTokens: 10000 });
|
|
12
|
+
hot.add({ query: "第一轮", response: "答一" });
|
|
13
|
+
hot.add({ query: "第二轮", response: "答二" });
|
|
14
|
+
hot.add({ query: "第三轮", response: "答三" });
|
|
15
|
+
assert.equal(hot.rounds().length, 2);
|
|
16
|
+
assert.ok(hot.getContext().includes("第三轮"));
|
|
17
|
+
assert.ok(!hot.getContext().includes("第一轮"));
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
test("hot memory enforces the token budget", () => {
|
|
21
|
+
const hot = createHotMemory({ maxRounds: 10, maxTokens: 30 });
|
|
22
|
+
hot.add({ query: "很长的第一轮问题".repeat(10), response: "很长的回答".repeat(10) });
|
|
23
|
+
hot.add({ query: "第二轮", response: "答二" });
|
|
24
|
+
// The first round alone blows the budget; the newest round survives and
|
|
25
|
+
// the buffer never empties completely.
|
|
26
|
+
const rounds = hot.rounds();
|
|
27
|
+
assert.ok(rounds.length >= 1);
|
|
28
|
+
assert.equal(rounds[rounds.length - 1].query, "第二轮");
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test("hot memory getContext uses the Q/A round format", () => {
|
|
32
|
+
const hot = createHotMemory({ maxRounds: 5, maxTokens: 10000 });
|
|
33
|
+
hot.add({ query: "Q1", response: "A1" });
|
|
34
|
+
hot.add({ query: "Q2", response: "A2" });
|
|
35
|
+
assert.equal(hot.getContext(), "Q: Q1\nA: A1\n\nQ: Q2\nA: A2");
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
test("hot memory ignores empty rounds and clears", () => {
|
|
39
|
+
const hot = createHotMemory({ maxRounds: 5, maxTokens: 10000 });
|
|
40
|
+
hot.add({ query: "", response: "x" });
|
|
41
|
+
hot.add(null);
|
|
42
|
+
assert.equal(hot.rounds().length, 0);
|
|
43
|
+
hot.add({ query: "q" });
|
|
44
|
+
hot.clear();
|
|
45
|
+
assert.equal(hot.getContext(), "");
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test("estimateTokens counts CJK heavier than ASCII", () => {
|
|
49
|
+
assert.ok(estimateTokens("中文内容") > estimateTokens("abcd"));
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
// --- entry defense: non-positive / non-integer bounds fall back to defaults ---
|
|
53
|
+
// Bug: createHotMemory({ maxRounds: -1 }) made the eviction while-loop
|
|
54
|
+
// `while (buffer.length > maxRounds)` unbounded — after the buffer emptied,
|
|
55
|
+
// `0 > -1` stayed true and buffer.shift() on an empty array is a no-op, so
|
|
56
|
+
// every add() spun forever. Non-integer values (1.5, NaN, null) were also
|
|
57
|
+
// silently wrong. The fix clamps them to the 5/2000 defaults at the door.
|
|
58
|
+
|
|
59
|
+
test("hot memory falls back to maxRounds=5 for non-positive/invalid values", () => {
|
|
60
|
+
for (const bad of [0, -1, 1.5, NaN, null]) {
|
|
61
|
+
const hot = createHotMemory({ maxRounds: bad, maxTokens: 10000 });
|
|
62
|
+
for (let i = 0; i < 8; i++) hot.add({ query: `第${i}轮`, response: "x" });
|
|
63
|
+
assert.equal(hot.rounds().length, 5, `maxRounds=${bad} must fall back to 5, no infinite loop`);
|
|
64
|
+
assert.ok(hot.getContext().includes("第7轮"), `maxRounds=${bad}: newest round survives`);
|
|
65
|
+
assert.ok(!hot.getContext().includes("第0轮"), `maxRounds=${bad}: oldest round evicted`);
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
test("hot memory falls back to maxTokens=2000 for non-positive/infinite values", () => {
|
|
70
|
+
for (const bad of [0, -1, Infinity]) {
|
|
71
|
+
const hot = createHotMemory({ maxRounds: 50, maxTokens: bad });
|
|
72
|
+
// 50 rounds at ~74 tokens each blow a 2000-token budget; the fallback must
|
|
73
|
+
// evict into (1, 50). A broken budget of 0/-1 would squeeze to 1 round and
|
|
74
|
+
// Infinity would keep all 50 — both are the pre-fix behavior.
|
|
75
|
+
for (let i = 0; i < 50; i++) hot.add({ query: `第${i}轮`, response: "长回答".repeat(40) });
|
|
76
|
+
const n = hot.rounds().length;
|
|
77
|
+
assert.ok(n > 1 && n < 50, `maxTokens=${bad} falls back to 2000 (kept ${n} rounds)`);
|
|
78
|
+
}
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
// --- service-level: BM25 fusion + semantic dedup + selective injection ---
|
|
82
|
+
|
|
83
|
+
function toyVec(text) {
|
|
84
|
+
const v = new Array(64).fill(0);
|
|
85
|
+
for (const t of String(text).toLowerCase().split(/[^\p{L}\p{N}]+/u).filter(Boolean)) {
|
|
86
|
+
let h = 0;
|
|
87
|
+
for (const ch of t) h = (h * 31 + ch.codePointAt(0)) >>> 0;
|
|
88
|
+
v[h % 64] += 1;
|
|
89
|
+
}
|
|
90
|
+
const norm = Math.sqrt(v.reduce((s, x) => s + x * x, 0)) || 1;
|
|
91
|
+
return v.map((x) => x / norm);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function setup(overrides = {}) {
|
|
95
|
+
const store = createStore(":memory:");
|
|
96
|
+
const config = {
|
|
97
|
+
entitySearchEnabled: false,
|
|
98
|
+
bm25SearchEnabled: true,
|
|
99
|
+
adaptiveThresholdEnabled: false,
|
|
100
|
+
searchSemanticDedup: true,
|
|
101
|
+
selectiveInjectEnabled: true,
|
|
102
|
+
...overrides
|
|
103
|
+
};
|
|
104
|
+
const service = createService({ store, mirror: null, config, logger: null });
|
|
105
|
+
service.setVectorIndex(createVectorIndex({ store, logger: null }));
|
|
106
|
+
service.setEmbedder({ embedSingle: async (t) => toyVec(t) });
|
|
107
|
+
return { store, service };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
test("searchMemories fuses BM25: scattered-term queries recall both rows", async () => {
|
|
111
|
+
const { store, service } = setup();
|
|
112
|
+
const a = store.save({ type: "project", title: "异步并发模式", content: "async runtime 选用 tokio", importance: 3 });
|
|
113
|
+
const b = store.save({ type: "decision", title: "语言迁移", content: "编译模块迁移到 Rust", importance: 3 });
|
|
114
|
+
store.save({ type: "project", title: "无关", content: "夜间 ETL 脚本", importance: 3 });
|
|
115
|
+
|
|
116
|
+
// "rust 异步" is not a substring of either row — LIKE misses both; BM25
|
|
117
|
+
// must surface both rows in the merged result.
|
|
118
|
+
const results = await service.searchMemories("rust 异步", { mode: "auto", topK: 5 });
|
|
119
|
+
const ids = results.map((r) => r.id);
|
|
120
|
+
assert.ok(ids.includes(a.id), "async row must be recalled via BM25");
|
|
121
|
+
assert.ok(ids.includes(b.id), "rust row must be recalled via BM25");
|
|
122
|
+
assert.equal(results.find((r) => r.id === a.id)?.source, "bm25");
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
test("bm25SearchEnabled=false restores the two-path behavior", async () => {
|
|
126
|
+
const { store, service } = setup({ bm25SearchEnabled: false });
|
|
127
|
+
const a = store.save({ type: "project", title: "异步并发模式", content: "async runtime 选用 tokio", importance: 3 });
|
|
128
|
+
const results = await service.searchMemories("rust 异步", { mode: "auto", topK: 5 });
|
|
129
|
+
assert.ok(!results.some((r) => r.id === a.id), "no BM25 → scattered-term miss is back");
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
test("search-time semantic dedup drops near-identical embeddings", async () => {
|
|
133
|
+
const { store, service } = setup();
|
|
134
|
+
const a = store.save({ type: "project", title: "偏好 A", content: "用户喜欢深色主题编辑器", importance: 3 });
|
|
135
|
+
const b = store.save({ type: "project", title: "偏好 A 备份", content: "用户喜欢深色主题编辑器(备份)", importance: 3 });
|
|
136
|
+
store.setEmbedding(a.id, toyVec("用户喜欢深色主题编辑器"));
|
|
137
|
+
store.setEmbedding(b.id, toyVec("用户喜欢深色主题编辑器"));
|
|
138
|
+
|
|
139
|
+
// auto (not keyword): keyword mode is the documented text-only path and is
|
|
140
|
+
// exempt from dedup by design; auto exercises the dedup the way production
|
|
141
|
+
// searches run.
|
|
142
|
+
const results = await service.searchMemories("深色主题", { mode: "auto", topK: 5 });
|
|
143
|
+
const ids = results.map((r) => r.id);
|
|
144
|
+
assert.ok(ids.includes(a.id) !== ids.includes(b.id), "one of the near-duplicate pair is dropped");
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
test("searchSemanticDedup=false keeps duplicate embeddings", async () => {
|
|
148
|
+
const { store, service } = setup({ searchSemanticDedup: false });
|
|
149
|
+
const a = store.save({ type: "project", title: "偏好 A", content: "用户喜欢深色主题编辑器", importance: 3 });
|
|
150
|
+
const b = store.save({ type: "project", title: "偏好 A 备份", content: "用户喜欢深色主题编辑器(备份)", importance: 3 });
|
|
151
|
+
store.setEmbedding(a.id, toyVec("用户喜欢深色主题编辑器"));
|
|
152
|
+
store.setEmbedding(b.id, toyVec("用户喜欢深色主题编辑器"));
|
|
153
|
+
const results = await service.searchMemories("深色主题", { mode: "keyword", topK: 5 });
|
|
154
|
+
assert.equal(results.length, 2);
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
test("selective injection re-orders candidates by query similarity", () => {
|
|
158
|
+
const { store, service } = setup();
|
|
159
|
+
const thesis = store.save({ type: "project", title: "湿地论文", content: "毕业论文研究城市湿地公园", importance: 5 });
|
|
160
|
+
const plugin = store.save({ type: "project", title: "插件项目", content: "dsh-mneme 记忆插件开发", importance: 5 });
|
|
161
|
+
store.setEmbedding(thesis.id, toyVec("毕业论文研究城市湿地公园"));
|
|
162
|
+
store.setEmbedding(plugin.id, toyVec("dsh-mneme 记忆插件开发"));
|
|
163
|
+
|
|
164
|
+
// Rule-based order would put both at equal importance; the query vector is
|
|
165
|
+
// about the thesis, so topic ranking must put the thesis memory first.
|
|
166
|
+
const picked = service.injectCandidates({
|
|
167
|
+
query: "论文写作",
|
|
168
|
+
queryVector: toyVec("毕业论文研究城市湿地公园"),
|
|
169
|
+
maxItems: 2,
|
|
170
|
+
threshold: 3
|
|
171
|
+
});
|
|
172
|
+
assert.equal(picked[0].id, thesis.id);
|
|
173
|
+
assert.ok(picked.some((m) => m.id === plugin.id));
|
|
174
|
+
});
|