@modusensus/dsh-mneme 0.5.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/lib/api.js +516 -516
- package/lib/client.js +1302 -1302
- package/lib/commands.js +64 -64
- package/lib/config.js +252 -252
- package/lib/dream.js +817 -817
- package/lib/embedding.js +154 -154
- package/lib/hot-memory.js +53 -46
- package/lib/index.js +341 -341
- package/lib/inject.js +208 -208
- package/lib/local-embedder.js +282 -282
- package/lib/reranker.js +4 -1
- package/lib/service.js +3 -1
- package/package.json +1 -1
- package/scripts/sync-lib.js +7 -2
- package/src/hot-memory.js +7 -0
- package/src/reranker.js +4 -1
- package/src/service.js +3 -1
- package/test/hot-memory.test.js +29 -0
- package/test/reranker.test.js +43 -0
- package/test/service-search.test.js +25 -0
package/test/hot-memory.test.js
CHANGED
|
@@ -49,6 +49,35 @@ test("estimateTokens counts CJK heavier than ASCII", () => {
|
|
|
49
49
|
assert.ok(estimateTokens("中文内容") > estimateTokens("abcd"));
|
|
50
50
|
});
|
|
51
51
|
|
|
52
|
+
// --- entry defense: non-positive / non-integer bounds fall back to defaults ---
|
|
53
|
+
// Bug: createHotMemory({ maxRounds: -1 }) made the eviction while-loop
|
|
54
|
+
// `while (buffer.length > maxRounds)` unbounded — after the buffer emptied,
|
|
55
|
+
// `0 > -1` stayed true and buffer.shift() on an empty array is a no-op, so
|
|
56
|
+
// every add() spun forever. Non-integer values (1.5, NaN, null) were also
|
|
57
|
+
// silently wrong. The fix clamps them to the 5/2000 defaults at the door.
|
|
58
|
+
|
|
59
|
+
test("hot memory falls back to maxRounds=5 for non-positive/invalid values", () => {
|
|
60
|
+
for (const bad of [0, -1, 1.5, NaN, null]) {
|
|
61
|
+
const hot = createHotMemory({ maxRounds: bad, maxTokens: 10000 });
|
|
62
|
+
for (let i = 0; i < 8; i++) hot.add({ query: `第${i}轮`, response: "x" });
|
|
63
|
+
assert.equal(hot.rounds().length, 5, `maxRounds=${bad} must fall back to 5, no infinite loop`);
|
|
64
|
+
assert.ok(hot.getContext().includes("第7轮"), `maxRounds=${bad}: newest round survives`);
|
|
65
|
+
assert.ok(!hot.getContext().includes("第0轮"), `maxRounds=${bad}: oldest round evicted`);
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
test("hot memory falls back to maxTokens=2000 for non-positive/infinite values", () => {
|
|
70
|
+
for (const bad of [0, -1, Infinity]) {
|
|
71
|
+
const hot = createHotMemory({ maxRounds: 50, maxTokens: bad });
|
|
72
|
+
// 50 rounds at ~74 tokens each blow a 2000-token budget; the fallback must
|
|
73
|
+
// evict into (1, 50). A broken budget of 0/-1 would squeeze to 1 round and
|
|
74
|
+
// Infinity would keep all 50 — both are the pre-fix behavior.
|
|
75
|
+
for (let i = 0; i < 50; i++) hot.add({ query: `第${i}轮`, response: "长回答".repeat(40) });
|
|
76
|
+
const n = hot.rounds().length;
|
|
77
|
+
assert.ok(n > 1 && n < 50, `maxTokens=${bad} falls back to 2000 (kept ${n} rounds)`);
|
|
78
|
+
}
|
|
79
|
+
});
|
|
80
|
+
|
|
52
81
|
// --- service-level: BM25 fusion + semantic dedup + selective injection ---
|
|
53
82
|
|
|
54
83
|
function toyVec(text) {
|
package/test/reranker.test.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import test from "node:test";
|
|
2
2
|
import assert from "node:assert/strict";
|
|
3
|
+
import { spawnSync } from "node:child_process";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
3
5
|
import { LocalReranker } from "../src/reranker.js";
|
|
4
6
|
|
|
5
7
|
/** Injected scorer: records (query, passage) calls, returns a fixed score. */
|
|
@@ -195,3 +197,44 @@ test("dispose releases the loaded pipeline", async () => {
|
|
|
195
197
|
assert.equal(r.pipeline, null);
|
|
196
198
|
assert.equal(extractor.disposed, true);
|
|
197
199
|
});
|
|
200
|
+
|
|
201
|
+
test("default pipeline loader mirrors cache_dir onto env.cacheDir (issue #13)", () => {
|
|
202
|
+
// The fix lives in the module's *default* loader — the dynamic-import of
|
|
203
|
+
// @huggingface/transformers that the in-process tests bypass by injecting
|
|
204
|
+
// engineFactory. So it is exercised in a child node process under the
|
|
205
|
+
// --experimental-test-module-mocks flag: the transformers module is mocked
|
|
206
|
+
// with an empty env, LocalReranker uses the real default loader, and we
|
|
207
|
+
// assert env.cacheDir picks up the constructor's cache_dir. A regression
|
|
208
|
+
// (loader no longer mirroring) fails the child and surfaces here as a
|
|
209
|
+
// non-zero exit.
|
|
210
|
+
const srcUrl = new URL("../src/reranker.js", import.meta.url).href;
|
|
211
|
+
const cacheDir = "/tmp/dsh-mneme-cache-mirror-test";
|
|
212
|
+
const script = `
|
|
213
|
+
import { test } from "node:test";
|
|
214
|
+
const cacheDir = ${JSON.stringify(cacheDir)};
|
|
215
|
+
test("cache_dir is mirrored onto env.cacheDir", async (t) => {
|
|
216
|
+
t.mock.module("@huggingface/transformers", {
|
|
217
|
+
namedExports: {
|
|
218
|
+
env: {},
|
|
219
|
+
pipeline: async () => ({ dispose: () => {} })
|
|
220
|
+
}
|
|
221
|
+
});
|
|
222
|
+
const { LocalReranker } = await import(${JSON.stringify(srcUrl)});
|
|
223
|
+
const r = new LocalReranker({ cacheDir, device: "cpu" });
|
|
224
|
+
await r.init();
|
|
225
|
+
const { env } = await import("@huggingface/transformers");
|
|
226
|
+
if (env.cacheDir !== cacheDir) {
|
|
227
|
+
throw new Error("env.cacheDir not mirrored from cache_dir: " + env.cacheDir);
|
|
228
|
+
}
|
|
229
|
+
console.log("CACHE_MIRROR_OK");
|
|
230
|
+
});
|
|
231
|
+
`;
|
|
232
|
+
const res = spawnSync(process.execPath, [
|
|
233
|
+
"--experimental-test-module-mocks",
|
|
234
|
+
"--input-type=module",
|
|
235
|
+
"-e",
|
|
236
|
+
script
|
|
237
|
+
], { encoding: "utf8", cwd: fileURLToPath(new URL("..", import.meta.url)) });
|
|
238
|
+
assert.equal(res.status, 0, `cache-dir mirror child failed:\n${res.stdout}\n${res.stderr}`);
|
|
239
|
+
assert.match(res.stdout, /CACHE_MIRROR_OK/);
|
|
240
|
+
});
|
|
@@ -85,6 +85,31 @@ test("hybrid blend honors configured vector/keyword weights", async () => {
|
|
|
85
85
|
assert.ok(Math.abs(t.score - 0.9) < 1e-9, `tuned blend ${t.score}`);
|
|
86
86
|
});
|
|
87
87
|
|
|
88
|
+
test("hybrid fusion clamps blended scores into [0,1]", async () => {
|
|
89
|
+
// Weights summing above 1 (1.0 + 1.0) push a same-memory blend over 1.0:
|
|
90
|
+
// title hit (importance 5) = 1 * (0.5 + 0.5) = 1.0, vector cosine = 1.0 →
|
|
91
|
+
// raw blend 1.0*1 + 1.0*1 = 2.0. The fused score must be clamped so
|
|
92
|
+
// consumers never see a score outside [0,1].
|
|
93
|
+
const store = createStore(":memory:");
|
|
94
|
+
const service = createService({ store, mirror: null, config: {
|
|
95
|
+
hybridSearchVectorWeight: 1,
|
|
96
|
+
hybridSearchKeywordWeight: 1,
|
|
97
|
+
adaptiveThresholdEnabled: false,
|
|
98
|
+
searchSemanticDedup: false
|
|
99
|
+
} });
|
|
100
|
+
const vi = createVectorIndex({ store });
|
|
101
|
+
service.setEmbedder(embedder);
|
|
102
|
+
service.setVectorIndex(vi);
|
|
103
|
+
const m = service.saveWithDedupe({ type: "preference", title: "量子计算", content: "量子计算入门", importance: 5 });
|
|
104
|
+
vi.saveEmbedding(m.memory.id, [1, 0, 0]);
|
|
105
|
+
|
|
106
|
+
const rows = await service.searchMemories("量子计算", { mode: "hybrid", topK: 10 });
|
|
107
|
+
const hit = rows.find((r) => r.id === m.memory.id);
|
|
108
|
+
assert.ok(hit, "blended row is recalled");
|
|
109
|
+
assert.equal(hit.score, 1, `over-weight blend clamped to 1 (got ${hit.score})`);
|
|
110
|
+
assert.ok(hit.score >= 0 && hit.score <= 1);
|
|
111
|
+
});
|
|
112
|
+
|
|
88
113
|
test("auto = keyword leads, vector fills the remaining slots", async () => {
|
|
89
114
|
const { service, vectorIndex } = setup();
|
|
90
115
|
const kw = service.saveWithDedupe({ type: "preference", title: "量子计算", content: "量子计算入门" });
|