@modusensus/dsh-mneme 0.7.28 → 0.7.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -6
- package/lib/api.js +17 -1
- package/lib/client.js +142 -11
- package/lib/config.js +27 -0
- package/lib/embedding.js +3 -0
- package/lib/entities/extractor.js +12 -5
- package/lib/index.js +104 -37
- package/lib/local-embedder.js +4 -0
- package/lib/service.js +171 -58
- package/lib/settings.js +13 -1
- package/lib/store.js +4 -1
- package/package.json +1 -1
- package/scripts/benchmark-recall.js +71 -5
- package/scripts/e2e-dsh.js +6 -4
- package/src/api.js +17 -1
- package/src/config.js +27 -0
- package/src/embedding.js +3 -0
- package/src/entities/extractor.js +12 -5
- package/src/index.js +104 -37
- package/src/local-embedder.js +4 -0
- package/src/service.js +171 -58
- package/src/settings.js +13 -1
- package/src/store.js +4 -1
- package/test/api.test.js +65 -5
- package/test/benchmark.test.js +60 -1
- package/test/client.test.js +39 -0
- package/test/entities.test.js +25 -0
- package/test/local-embedder.test.js +2 -0
- package/test/reasoning-effort.test.js +77 -0
- package/test/store.test.js +9 -0
- package/README.en.md +0 -488
package/src/api.js
CHANGED
|
@@ -161,6 +161,16 @@ export function createApi(ctx, service, settings, commands, embedder, semantic =
|
|
|
161
161
|
}
|
|
162
162
|
});
|
|
163
163
|
|
|
164
|
+
// 反馈入口预填用的插件版本(面板「帮助与反馈」卡片拉取)。读取失败(打包
|
|
165
|
+
// 环境)返回 "unknown",链接照常可用,纯展示信息不拦截。
|
|
166
|
+
register({
|
|
167
|
+
kind: "exact",
|
|
168
|
+
path: "/api/dsh-mneme/info",
|
|
169
|
+
handler(req, res) {
|
|
170
|
+
sendJson(res, 200, { version: PACKAGE_VERSION });
|
|
171
|
+
}
|
|
172
|
+
});
|
|
173
|
+
|
|
164
174
|
register({
|
|
165
175
|
kind: "exact",
|
|
166
176
|
path: "/api/dsh-mneme/list",
|
|
@@ -595,7 +605,13 @@ export function createApi(ctx, service, settings, commands, embedder, semantic =
|
|
|
595
605
|
try {
|
|
596
606
|
const stats = semantic?.vectorIndex?.getStats?.() ?? null;
|
|
597
607
|
sendJson(res, 200, {
|
|
598
|
-
|
|
608
|
+
// #118: expose readiness so the status card can tell "initializing /
|
|
609
|
+
// unreachable" from "disabled". Legacy OpenAI embedder has no `ready`
|
|
610
|
+
// prop and is immediately usable, so treat that as ready.
|
|
611
|
+
ready: embedder ? ("ready" in embedder ? embedder.ready === true : true) : null,
|
|
612
|
+
// Explicit display name first: the legacy OpenAI-compatible embedder
|
|
613
|
+
// is an object literal, so constructor.name would be "Object".
|
|
614
|
+
embedProvider: embedder ? (embedder.name ?? embedder.constructor?.name ?? "unknown") : null,
|
|
599
615
|
modelHash: embedder?.modelHash ?? null,
|
|
600
616
|
dimension: embedder?.dimension ?? null,
|
|
601
617
|
reranker: semantic?.reranker ? "ready" : null,
|
package/src/config.js
CHANGED
|
@@ -178,6 +178,19 @@ export const Config = z.object({
|
|
|
178
178
|
searchSemanticDedup: z.boolean().default(false),
|
|
179
179
|
searchSemanticDedupThreshold: z.number().min(0.5).max(1).default(0.95),
|
|
180
180
|
|
|
181
|
+
// Recall fusion recipe (plan #1): how the keyword/vector/BM25 ranked lists
|
|
182
|
+
// are combined into the final ranking. `blend` (default) is the legacy
|
|
183
|
+
// behavior — weighted sum for vector/hybrid, union backfill for auto —
|
|
184
|
+
// unchanged. `rrf` (Reciprocal Rank Fusion) and `minmax` (min-max normalized
|
|
185
|
+
// weighted sum) are rank/scale-aware recipes that fix the unit mismatch the
|
|
186
|
+
// issue describes (raw cosine vs keyword score vs normalized IDF are added
|
|
187
|
+
// directly). Off by default so existing behavior holds exactly.
|
|
188
|
+
recallFusion: z.union([z.const("blend"), z.const("rrf"), z.const("minmax")]).default("blend"),
|
|
189
|
+
// Attach a `signals` object { keyword, vector, bm25, final } to each search
|
|
190
|
+
// result for transparency/debugging (plan #2). Default off; when on it only
|
|
191
|
+
// decorates the returned rows, never changes the ranking.
|
|
192
|
+
signalTransparency: z.boolean().default(false),
|
|
193
|
+
|
|
181
194
|
// --- semantic: rerank layer (v0.2) --------------------------------------
|
|
182
195
|
// Opt-in by default (item ⑥): the local cross-encoder pulls in onnxruntime
|
|
183
196
|
// (transformers.js) at init, so a bare install must not load it. Only an
|
|
@@ -207,9 +220,23 @@ export const Config = z.object({
|
|
|
207
220
|
// The storage layer (entities/entity_attrs/entity_relations tables + CRUD)
|
|
208
221
|
// is always available regardless of this flag.
|
|
209
222
|
entityExtractionEnabled: z.boolean().default(false),
|
|
223
|
+
// Optional provider override for entity extraction; empty = use the caller's
|
|
224
|
+
// default provider/model. Combined with entityExtractionModel — provider
|
|
225
|
+
// without model (or vice versa) falls through to the caller default.
|
|
226
|
+
entityExtractionProvider: z.string().default(""),
|
|
210
227
|
// Optional model override for entity extraction; empty = use the caller's
|
|
211
228
|
// default provider/model.
|
|
212
229
|
entityExtractionModel: z.string().default(""),
|
|
230
|
+
// Reasoning effort for entity extraction (issue #109), mirrors
|
|
231
|
+
// dreamReasoningEffort: 'none' (default) omits the field / provider default;
|
|
232
|
+
// low/medium/high passed through. A provider that rejects the effort retries
|
|
233
|
+
// once without it, so opting in is safe to experiment with.
|
|
234
|
+
entityExtractionReasoning: z.union([
|
|
235
|
+
z.const("low"),
|
|
236
|
+
z.const("medium"),
|
|
237
|
+
z.const("high"),
|
|
238
|
+
z.const("none")
|
|
239
|
+
]).default("none"),
|
|
213
240
|
// Cap on entities per extraction pass and attributes per entity.
|
|
214
241
|
entityExtractionMaxEntities: z.natural().min(1).max(20).default(10),
|
|
215
242
|
entityExtractionMaxAttrs: z.natural().min(1).max(50).default(20),
|
package/src/embedding.js
CHANGED
|
@@ -97,6 +97,9 @@ export function createEmbedder({ store, settings, logger, vectorIndex }) {
|
|
|
97
97
|
}
|
|
98
98
|
|
|
99
99
|
return {
|
|
100
|
+
// Display name for /semantic: a literal's constructor.name is "Object",
|
|
101
|
+
// which the status card would render verbatim.
|
|
102
|
+
name: "OpenAI",
|
|
100
103
|
/** Fire-and-forget re-embed of a memory after any write. */
|
|
101
104
|
schedule(memory) {
|
|
102
105
|
if (!memory?.id) return;
|
|
@@ -146,16 +146,23 @@ export async function extractEntities(memory, { store, config, callLLM, logger }
|
|
|
146
146
|
return { ok: false, error: "Invalid memory: missing content" };
|
|
147
147
|
}
|
|
148
148
|
|
|
149
|
-
const model = config.entityExtractionModel || null;
|
|
150
149
|
const systemPrompt = buildSystemPrompt(config);
|
|
151
150
|
const userText = buildUserMessage(memory.content);
|
|
152
|
-
|
|
151
|
+
|
|
153
152
|
const messages = [
|
|
154
153
|
{ role: "system", content: [{ type: "text", text: systemPrompt }] },
|
|
155
154
|
{ role: "user", content: [{ type: "text", text: userText }] }
|
|
156
155
|
];
|
|
157
|
-
|
|
158
|
-
|
|
156
|
+
|
|
157
|
+
// Issue #109: optional provider/model override + reasoning effort are
|
|
158
|
+
// passed through to callLLM's options; the index.js adapter maps them onto
|
|
159
|
+
// the route (or falls back to the caller's default model). Empty provider/
|
|
160
|
+
// model both mean "use the caller default".
|
|
161
|
+
const options = {};
|
|
162
|
+
if (config.entityExtractionProvider) options.provider = config.entityExtractionProvider;
|
|
163
|
+
if (config.entityExtractionModel) options.model = config.entityExtractionModel;
|
|
164
|
+
const reasoning = config.entityExtractionReasoning;
|
|
165
|
+
if (reasoning && reasoning !== "none") options.reasoningEffort = reasoning;
|
|
159
166
|
const llmResponse = await callLLM(messages, options);
|
|
160
167
|
|
|
161
168
|
if (!llmResponse) {
|
|
@@ -219,7 +226,7 @@ export async function extractEntities(memory, { store, config, callLLM, logger }
|
|
|
219
226
|
to_entity: toId,
|
|
220
227
|
relation_type: rel.type,
|
|
221
228
|
memory_id: memory.id,
|
|
222
|
-
metadata: { model: model || "default" }
|
|
229
|
+
metadata: { model: options.model || "default" }
|
|
223
230
|
});
|
|
224
231
|
savedRelations.push(saved);
|
|
225
232
|
} catch (err) {
|
package/src/index.js
CHANGED
|
@@ -35,6 +35,57 @@ export { Config };
|
|
|
35
35
|
// value, so a `function apply` disposer would never run on unload. An arrow
|
|
36
36
|
// has no prototype, is called normally, and its returned disposer is collected
|
|
37
37
|
// and run by the fiber on unload.
|
|
38
|
+
// Entity-extraction LLM adapter (issue #108/#109): maps the extractor's
|
|
39
|
+
// options (provider/model override + reasoningEffort) onto a real dsh-llm
|
|
40
|
+
// stream route and retries once without the effort when the first attempt is
|
|
41
|
+
// rejected. Extracted from apply() so the effort-fallback branch is
|
|
42
|
+
// unit-testable; the extractor only ever sees a callLLM(messages, options)
|
|
43
|
+
// => Promise<string>. The route always carries a real provider/model (dsh-llm
|
|
44
|
+
// GenerateOptions requires both) — never a bare stream.
|
|
45
|
+
export function createEntityStreamAdapter({ llm, agentDefaultModel, logger }) {
|
|
46
|
+
return async function streamEntityText(messages, options = {}) {
|
|
47
|
+
let route = {};
|
|
48
|
+
if (options.provider) route.provider = options.provider;
|
|
49
|
+
if (options.model) route.model = options.model;
|
|
50
|
+
if (!route.provider || !route.model) {
|
|
51
|
+
try {
|
|
52
|
+
const sel = agentDefaultModel?.currentSelection?.();
|
|
53
|
+
if (sel?.provider && sel?.model) {
|
|
54
|
+
route.provider ??= sel.provider;
|
|
55
|
+
route.model ??= sel.model;
|
|
56
|
+
}
|
|
57
|
+
} catch { /* fall through to whatever route we already have */ }
|
|
58
|
+
}
|
|
59
|
+
const effort = options.reasoningEffort;
|
|
60
|
+
const tryStream = (withEffort) => {
|
|
61
|
+
let text = "";
|
|
62
|
+
return (async () => {
|
|
63
|
+
for await (const chunk of llm.stream({
|
|
64
|
+
...route,
|
|
65
|
+
maxTokens: 4096,
|
|
66
|
+
...(withEffort && effort ? { reasoningEffort: effort } : {}),
|
|
67
|
+
messages
|
|
68
|
+
})) {
|
|
69
|
+
if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
|
|
70
|
+
if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) return undefined;
|
|
71
|
+
}
|
|
72
|
+
return text;
|
|
73
|
+
})().catch((err) => {
|
|
74
|
+
logger?.warn?.(`[dsh-mneme] entity extraction llm stream failed: ${String(err)}`);
|
|
75
|
+
return undefined;
|
|
76
|
+
});
|
|
77
|
+
};
|
|
78
|
+
let text = await tryStream(true);
|
|
79
|
+
if (text === undefined && effort) {
|
|
80
|
+
// Mirror dream's effort fallback: a provider rejecting the reasoning
|
|
81
|
+
// effort must not sink the whole extraction — retry once without it.
|
|
82
|
+
logger?.warn?.(`[dsh-mneme] entity extraction: reasoningEffort "${effort}" rejected, retrying without it`);
|
|
83
|
+
text = await tryStream(false);
|
|
84
|
+
}
|
|
85
|
+
return text;
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
|
|
38
89
|
export const apply = (ctx, config) => {
|
|
39
90
|
const rawCfg = Config(config);
|
|
40
91
|
|
|
@@ -148,6 +199,8 @@ export const apply = (ctx, config) => {
|
|
|
148
199
|
|
|
149
200
|
let embedder = null;
|
|
150
201
|
let reranker = null;
|
|
202
|
+
// #118: pending embedder-init retry timer, cleared on unload.
|
|
203
|
+
let embedRetryTimer = null;
|
|
151
204
|
if (lightMode) {
|
|
152
205
|
// Light mode: the whole vector pipeline stays off — no embedder (nothing
|
|
153
206
|
// pulls in ONNX/transformers), no reranker, no boot backfill (the preset
|
|
@@ -175,13 +228,32 @@ export const apply = (ctx, config) => {
|
|
|
175
228
|
service.setEmbedder(embedder);
|
|
176
229
|
// issue #6: wait for extractor init before applying human edits, so
|
|
177
230
|
// scheduled embeddings see a ready embedder.
|
|
178
|
-
embedder.init()
|
|
179
|
-
.then(() => applyHumanEdits())
|
|
180
|
-
.catch((
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
231
|
+
const bootEmbedder = () => embedder.init()
|
|
232
|
+
.then(() => { applyHumanEdits(); return true; })
|
|
233
|
+
.catch(() => false);
|
|
234
|
+
// #118: the old one-shot probe permanently degraded search to keyword
|
|
235
|
+
// when Ollama was briefly unreachable at boot (recoverable only by
|
|
236
|
+
// restart). Retry briefly (5 attempts total: 1 initial + 4 × 15s);
|
|
237
|
+
// search degrades to keyword meanwhile because per-query embed failures
|
|
238
|
+
// are swallowed.
|
|
239
|
+
bootEmbedder().then((ok) => {
|
|
240
|
+
if (ok) return;
|
|
241
|
+
let tries = 4;
|
|
242
|
+
const retry = () => {
|
|
243
|
+
if (tries-- <= 0) {
|
|
244
|
+
ctx.logger?.warn?.("[dsh-mneme] embedder init retries exhausted, search degrades to keyword");
|
|
245
|
+
service.setEmbedder(null);
|
|
246
|
+
applyHumanEdits();
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
embedRetryTimer = setTimeout(async () => {
|
|
250
|
+
if (await bootEmbedder()) return;
|
|
251
|
+
retry();
|
|
252
|
+
}, 15_000);
|
|
253
|
+
};
|
|
254
|
+
ctx.logger?.warn?.("[dsh-mneme] embedder init failed, retrying");
|
|
255
|
+
retry();
|
|
256
|
+
});
|
|
185
257
|
} catch (error) {
|
|
186
258
|
ctx.logger?.warn?.(`[dsh-mneme] embedder unavailable, search degrades to keyword: ${String(error)}`);
|
|
187
259
|
applyHumanEdits();
|
|
@@ -320,40 +392,35 @@ export const apply = (ctx, config) => {
|
|
|
320
392
|
// expects, reusing the same ctx.llm.stream consumption pattern as dream.js.
|
|
321
393
|
// Explicit opt-in only (entityExtractionEnabled defaults to false); any LLM
|
|
322
394
|
// failure degrades inside the extractor to { ok:false }, never a write error.
|
|
323
|
-
if (cfg.entityExtractionEnabled
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
return text;
|
|
345
|
-
};
|
|
346
|
-
service.setEntityExtractor((memory) =>
|
|
347
|
-
extractEntities(memory, { store, config: cfg, callLLM: streamEntityText, logger: ctx.logger })
|
|
348
|
-
.catch((err) => {
|
|
349
|
-
ctx.logger?.warn?.(`[dsh-mneme] entity extraction failed: ${String(err)}`);
|
|
350
|
-
return { ok: false, error: String(err) };
|
|
351
|
-
})
|
|
352
|
-
);
|
|
395
|
+
if (cfg.entityExtractionEnabled) {
|
|
396
|
+
if (!ctx.llm) {
|
|
397
|
+
// Issue #108: an enabled-but-unwired extractor failed silently before —
|
|
398
|
+
// zero entities, zero llm_audit_logs, no log line anywhere. Make the
|
|
399
|
+
// missing dependency visible so a user can tell "extractor not installed"
|
|
400
|
+
// from "extraction failed".
|
|
401
|
+
ctx.logger?.warn?.("[dsh-mneme] entityExtractionEnabled=true but ctx.llm unavailable — entity extractor NOT installed");
|
|
402
|
+
} else {
|
|
403
|
+
const streamEntityText = createEntityStreamAdapter({
|
|
404
|
+
llm: ctx.llm,
|
|
405
|
+
agentDefaultModel: ctx.agentDefaultModel,
|
|
406
|
+
logger: ctx.logger
|
|
407
|
+
});
|
|
408
|
+
service.setEntityExtractor((memory) =>
|
|
409
|
+
extractEntities(memory, { store, config: cfg, callLLM: streamEntityText, logger: ctx.logger })
|
|
410
|
+
.catch((err) => {
|
|
411
|
+
ctx.logger?.warn?.(`[dsh-mneme] entity extraction failed: ${String(err)}`);
|
|
412
|
+
return { ok: false, error: String(err) };
|
|
413
|
+
})
|
|
414
|
+
);
|
|
415
|
+
}
|
|
353
416
|
}
|
|
354
417
|
|
|
355
418
|
const disposers = [];
|
|
356
419
|
|
|
420
|
+
// #118: never let a pending embedder init retry fire after unload and touch
|
|
421
|
+
// a torn-down context.
|
|
422
|
+
disposers.push(() => { if (embedRetryTimer !== null) clearTimeout(embedRetryTimer); });
|
|
423
|
+
|
|
357
424
|
ctx.inject(["systemPrompt"], (promptCtx) => {
|
|
358
425
|
if (cfg.autoInject) disposers.push(createInjector(promptCtx, service, settings, cfg));
|
|
359
426
|
});
|
package/src/local-embedder.js
CHANGED
|
@@ -139,6 +139,9 @@ export class OllamaEmbedder {
|
|
|
139
139
|
this.model = String(opts.model ?? "nomic-embed-text").trim();
|
|
140
140
|
this.logger = opts.logger ?? null;
|
|
141
141
|
this._dimension = null;
|
|
142
|
+
// #118: async-init embedders expose `ready` so the service queues re-embeds
|
|
143
|
+
// until init lands and the status card can show "initializing".
|
|
144
|
+
this.ready = false;
|
|
142
145
|
}
|
|
143
146
|
|
|
144
147
|
async _post(body) {
|
|
@@ -157,6 +160,7 @@ export class OllamaEmbedder {
|
|
|
157
160
|
const body = await res.json();
|
|
158
161
|
if (!Array.isArray(body?.embedding)) throw new Error(`Ollama ${this.model} returned no embedding`);
|
|
159
162
|
this._dimension = body.embedding.length;
|
|
163
|
+
this.ready = true;
|
|
160
164
|
this.logger?.info?.(
|
|
161
165
|
`[dsh-mneme] ollama embedder ready: ${this.model} (dim=${this._dimension})`
|
|
162
166
|
);
|
package/src/service.js
CHANGED
|
@@ -416,6 +416,171 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
416
416
|
}
|
|
417
417
|
}
|
|
418
418
|
|
|
419
|
+
/**
|
|
420
|
+
* Recall fusion (plan #1). Turns the three ranked signal lists (keyword,
|
|
421
|
+
* vector, BM25) into a single merged list. Three recipes, selected by
|
|
422
|
+
* config.recallFusion:
|
|
423
|
+
* - blend (default): legacy behavior — weighted sum for vector/hybrid, union
|
|
424
|
+
* backfill for auto. Byte-identical to pre-fusion code, so enabling the
|
|
425
|
+
* config never regresses anybody.
|
|
426
|
+
* - rrf: Reciprocal Rank Fusion — Σ 1/(k + rank + 1) over each list a row
|
|
427
|
+
* appears in. Rank-based, so the unit mismatch (raw cosine vs keyword
|
|
428
|
+
* score vs normalized IDF) is irrelevant.
|
|
429
|
+
* - minmax: min-max normalize each source list's scores to [0,1] then take
|
|
430
|
+
* the weighted sum — a scale-aware version of `blend`.
|
|
431
|
+
* Returns { merged, signals }, where signals is Map<id, {keyword, vector,
|
|
432
|
+
* bm25}> so searchMemories can decorate rows when signalTransparency is on.
|
|
433
|
+
*/
|
|
434
|
+
function fuseRecall({ keyword, vector, bm25, lim, mode, wv, wk, wb }) {
|
|
435
|
+
const recipe = config?.recallFusion ?? "blend";
|
|
436
|
+
|
|
437
|
+
// Per-source scores are recorded for every recipe so signalTransparency
|
|
438
|
+
// works regardless of how the ranking was produced.
|
|
439
|
+
const signals = new Map();
|
|
440
|
+
const addSig = (id, field, sc) => {
|
|
441
|
+
const cur = signals.get(id) ?? {};
|
|
442
|
+
cur[field] = sc;
|
|
443
|
+
signals.set(id, cur);
|
|
444
|
+
};
|
|
445
|
+
for (const m of keyword) addSig(m.id, "keyword", m.score ?? 0);
|
|
446
|
+
for (const m of vector) addSig(m.id, "vector", m.score ?? 0);
|
|
447
|
+
for (const m of bm25) addSig(m.id, "bm25", m.score ?? 0);
|
|
448
|
+
|
|
449
|
+
const vectorIds = new Set(vector.map((m) => m.id));
|
|
450
|
+
const keywordIds = new Set(keyword.map((m) => m.id));
|
|
451
|
+
|
|
452
|
+
// Mode contract (aligns rrf/minmax with blend): keyword-only searches must
|
|
453
|
+
// stay keyword-only regardless of recipe, so enabling an opt-in recipe can
|
|
454
|
+
// never pull vector/BM25 rows into a mode="keyword" request. This mirrors
|
|
455
|
+
// the blend branch's `mode === "keyword"` short-circuit (byte-for-byte).
|
|
456
|
+
if (mode === "keyword") {
|
|
457
|
+
return { merged: keyword.slice(0, lim), signals };
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
let merged;
|
|
461
|
+
if (recipe === "rrf") {
|
|
462
|
+
// Rank-based: only the position of a row inside each surviving source
|
|
463
|
+
// list matters, so no cross-signal scale calibration is needed.
|
|
464
|
+
const k = 60; // standard RRF constant (plan #1 documents k=60)
|
|
465
|
+
const rows = new Map();
|
|
466
|
+
const addList = (list) => list.forEach((m, idx) => {
|
|
467
|
+
const s = 1 / (k + idx + 1);
|
|
468
|
+
const cur = rows.get(m.id);
|
|
469
|
+
if (cur) cur.score += s;
|
|
470
|
+
else rows.set(m.id, { ...m, score: s });
|
|
471
|
+
});
|
|
472
|
+
addList(keyword);
|
|
473
|
+
addList(vector);
|
|
474
|
+
addList(bm25);
|
|
475
|
+
merged = [...rows.values()].sort((a, b) => (b.score ?? 0) - (a.score ?? 0)).slice(0, lim);
|
|
476
|
+
} else if (recipe === "minmax") {
|
|
477
|
+
// Scale-aware weighted sum: each source list is min-max normalized to
|
|
478
|
+
// [0,1] before blending, so raw cosine and keyword score live on the
|
|
479
|
+
// same footing.
|
|
480
|
+
const norm = (list) => {
|
|
481
|
+
if (!list.length) return new Map();
|
|
482
|
+
let min = Infinity, max = -Infinity;
|
|
483
|
+
for (const m of list) { const s = m.score ?? 0; if (s < min) min = s; if (s > max) max = s; }
|
|
484
|
+
const range = max - min;
|
|
485
|
+
const out = new Map();
|
|
486
|
+
for (const m of list) out.set(m.id, range > 0 ? ((m.score ?? 0) - min) / range : 0.5);
|
|
487
|
+
return out;
|
|
488
|
+
};
|
|
489
|
+
const kw = norm(keyword), ve = norm(vector), bm = norm(bm25);
|
|
490
|
+
const rows = new Map();
|
|
491
|
+
const seed = (m) => { if (!rows.has(m.id)) rows.set(m.id, { ...m, score: 0 }); };
|
|
492
|
+
for (const m of keyword) seed(m);
|
|
493
|
+
for (const m of vector) seed(m);
|
|
494
|
+
for (const m of bm25) seed(m);
|
|
495
|
+
for (const [id, row] of rows) {
|
|
496
|
+
const k = kw.get(id) ?? 0;
|
|
497
|
+
const v = ve.get(id) ?? 0;
|
|
498
|
+
const b = bm.get(id) ?? 0;
|
|
499
|
+
row.score = v * wv + k * wk + b * wb;
|
|
500
|
+
}
|
|
501
|
+
merged = [...rows.values()].sort((a, b) => (b.score ?? 0) - (a.score ?? 0)).slice(0, lim);
|
|
502
|
+
} else {
|
|
503
|
+
// blend — the pre-existing per-mode behavior, extracted verbatim.
|
|
504
|
+
if (mode === "keyword") {
|
|
505
|
+
merged = keyword;
|
|
506
|
+
} else if (mode === "vector" || mode === "hybrid") {
|
|
507
|
+
const byId = new Map();
|
|
508
|
+
for (const m of vector) {
|
|
509
|
+
const rec = byId.get(m.id);
|
|
510
|
+
byId.set(m.id, rec ? { ...rec, score: Math.max(rec.score ?? 0, m.score ?? 0) } : m);
|
|
511
|
+
}
|
|
512
|
+
for (const m of keyword) {
|
|
513
|
+
const rec = byId.get(m.id);
|
|
514
|
+
if (rec) {
|
|
515
|
+
byId.set(m.id, { ...rec, score: (rec.score ?? 0) * wv + (m.score ?? 0) * wk });
|
|
516
|
+
} else {
|
|
517
|
+
byId.set(m.id, m);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
for (const m of bm25) {
|
|
521
|
+
const rec = byId.get(m.id);
|
|
522
|
+
if (rec) {
|
|
523
|
+
if (keywordIds.has(m.id)) continue;
|
|
524
|
+
byId.set(m.id, { ...rec, score: (rec.score ?? 0) + wb * (m.score ?? 0) });
|
|
525
|
+
} else {
|
|
526
|
+
byId.set(m.id, { ...m, score: wb * (m.score ?? 0) });
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
const ranked = [...byId.values()].sort((a, b) => (b.score ?? 0) - (a.score ?? 0));
|
|
530
|
+
merged = ranked.slice(0, lim);
|
|
531
|
+
if (merged.length < lim && !merged.length) {
|
|
532
|
+
merged = keyword.slice(0, lim);
|
|
533
|
+
}
|
|
534
|
+
} else {
|
|
535
|
+
// auto: keyword leads, vector + BM25 fill remaining slots.
|
|
536
|
+
merged = keyword.slice(0, lim);
|
|
537
|
+
const seen = new Set(merged.map((m) => m.id));
|
|
538
|
+
for (const m of vector) {
|
|
539
|
+
if (merged.length >= lim) break;
|
|
540
|
+
if (!seen.has(m.id)) { seen.add(m.id); merged.push(m); }
|
|
541
|
+
}
|
|
542
|
+
for (const m of bm25) {
|
|
543
|
+
if (merged.length >= lim) break;
|
|
544
|
+
if (!seen.has(m.id)) { seen.add(m.id); merged.push(m); }
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
// Mode contract, auto (aligns rrf/minmax with blend): keyword leads, the
|
|
550
|
+
// recipe fills the remaining slots. blend.auto already front-loads keyword;
|
|
551
|
+
// rrf/minmax rank across sources, so re-apply the same "keyword first"
|
|
552
|
+
// ordering here to preserve the pre-fusion auto contract — the keyword
|
|
553
|
+
// hit list keeps its power, and only slots it couldn't fill go to the
|
|
554
|
+
// recipe's ranking.
|
|
555
|
+
if (recipe !== "blend" && mode === "auto" && keyword.length) {
|
|
556
|
+
const head = keyword.slice(0, lim);
|
|
557
|
+
const seen = new Set(head.map((m) => m.id));
|
|
558
|
+
const tail = merged.filter((m) => !seen.has(m.id)).slice(0, Math.max(0, lim - head.length));
|
|
559
|
+
merged = head.concat(tail);
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
return { merged, signals };
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
/**
|
|
566
|
+
* Search memories for a query. Merges up to three recall sources (keyword,
|
|
567
|
+
* vector, BM25) according to config.recallFusion (blend/rrf/minmax — see
|
|
568
|
+
* fuseRecall), then optionally decorates rows with per-source signals
|
|
569
|
+
* (config.signalTransparency), applies semantic dedup (non-keyword modes),
|
|
570
|
+
* reranking, and epistemic trust re-weighting, and finally hands the merged
|
|
571
|
+
* list to the recall-layer recorder.
|
|
572
|
+
*
|
|
573
|
+
* options:
|
|
574
|
+
* mode — 'auto' (default) | 'keyword' | 'vector' | 'hybrid'
|
|
575
|
+
* topK — max rows (default 20)
|
|
576
|
+
* threshold — explicit vector score floor (overrides adaptive)
|
|
577
|
+
* useRerank — apply the reranker if available (default true)
|
|
578
|
+
* recordRecall — write a recall_runs audit row (default from config)
|
|
579
|
+
*
|
|
580
|
+
* Returns an array of memory rows { id, title, content, score, source, ... },
|
|
581
|
+
* with `signals` added when config.signalTransparency is on. Never throws:
|
|
582
|
+
* a vector/rerank failure degrades to keyword results.
|
|
583
|
+
*/
|
|
419
584
|
async function searchMemories(query, options = {}) {
|
|
420
585
|
const { mode = "auto", topK = 20, threshold, useRerank = true, recordRecall = options.recordRecall ?? (config?.recallRecordDefault ?? true) } = options;
|
|
421
586
|
const q = String(query ?? "").trim();
|
|
@@ -477,69 +642,17 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
477
642
|
// Loose blend weight: BM25 confirms and backfills, never dominates the
|
|
478
643
|
// semantic signal. Same-memory overlap boosts, unseen ids backfill.
|
|
479
644
|
const wb = 0.3;
|
|
480
|
-
// Path bookkeeping for the boost rule below: which ids each semantic
|
|
481
|
-
// recall path surfaced.
|
|
482
|
-
const vectorIds = new Set(vector.map((m) => m.id));
|
|
483
|
-
const keywordIds = new Set(keyword.map((m) => m.id));
|
|
484
645
|
|
|
485
646
|
// Hybrid blending weights from config when provided.
|
|
486
647
|
const wv = config?.hybridSearchVectorWeight ?? DEFAULT_HYBRID_WEIGHTS.vector;
|
|
487
648
|
const wk = config?.hybridSearchKeywordWeight ?? DEFAULT_HYBRID_WEIGHTS.keyword;
|
|
488
649
|
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
// vector order leads (it is the semantic signal), lexical paths
|
|
496
|
-
// backfill.
|
|
497
|
-
const byId = new Map();
|
|
498
|
-
for (const m of vector) {
|
|
499
|
-
const rec = byId.get(m.id);
|
|
500
|
-
byId.set(m.id, rec ? { ...rec, score: Math.max(rec.score ?? 0, m.score ?? 0) } : m);
|
|
501
|
-
}
|
|
502
|
-
for (const m of keyword) {
|
|
503
|
-
const rec = byId.get(m.id);
|
|
504
|
-
if (rec) {
|
|
505
|
-
// Same memory from both sides: blend the scores.
|
|
506
|
-
byId.set(m.id, { ...rec, score: (rec.score ?? 0) * wv + (m.score ?? 0) * wk });
|
|
507
|
-
} else {
|
|
508
|
-
byId.set(m.id, m);
|
|
509
|
-
}
|
|
510
|
-
}
|
|
511
|
-
for (const m of bm25) {
|
|
512
|
-
const rec = byId.get(m.id);
|
|
513
|
-
if (rec) {
|
|
514
|
-
// Boost rule: a row the LIKE keyword path already hit carries the
|
|
515
|
-
// query as a substring, so BM25 tokens are trivially present —
|
|
516
|
-
// boosting it double-counts lexical evidence. Only vector-recalled
|
|
517
|
-
// rows (lexical hit is genuinely new information) get the boost.
|
|
518
|
-
if (keywordIds.has(m.id)) continue;
|
|
519
|
-
byId.set(m.id, { ...rec, score: (rec.score ?? 0) + wb * (m.score ?? 0) });
|
|
520
|
-
} else {
|
|
521
|
-
byId.set(m.id, { ...m, score: wb * (m.score ?? 0) });
|
|
522
|
-
}
|
|
523
|
-
}
|
|
524
|
-
const ranked = [...byId.values()].sort((a, b) => (b.score ?? 0) - (a.score ?? 0));
|
|
525
|
-
merged = ranked.slice(0, lim);
|
|
526
|
-
if (merged.length < lim && !merged.length) {
|
|
527
|
-
// Vector unavailable entirely: fall back to plain keyword.
|
|
528
|
-
merged = keyword.slice(0, lim);
|
|
529
|
-
}
|
|
530
|
-
} else {
|
|
531
|
-
// auto: keyword leads, vector + BM25 fill remaining slots (legacy
|
|
532
|
-
// behavior, extended with the third path)
|
|
533
|
-
merged = keyword.slice(0, lim);
|
|
534
|
-
const seen = new Set(merged.map((m) => m.id));
|
|
535
|
-
for (const m of vector) {
|
|
536
|
-
if (merged.length >= lim) break;
|
|
537
|
-
if (!seen.has(m.id)) { seen.add(m.id); merged.push(m); }
|
|
538
|
-
}
|
|
539
|
-
for (const m of bm25) {
|
|
540
|
-
if (merged.length >= lim) break;
|
|
541
|
-
if (!seen.has(m.id)) { seen.add(m.id); merged.push(m); }
|
|
542
|
-
}
|
|
650
|
+
const { merged: fusedMerged, signals } = fuseRecall({ keyword, vector, bm25, lim, mode, wv, wk, wb });
|
|
651
|
+
let merged = fusedMerged;
|
|
652
|
+
// Signal transparency (#2): decorate each returned row with its per-source
|
|
653
|
+
// scores and the final fused score. Purely additive — never changes rank.
|
|
654
|
+
if (config?.signalTransparency === true) {
|
|
655
|
+
merged = merged.map((m) => ({ ...m, signals: { ...(signals.get(m.id) ?? {}), final: m.score ?? 0 } }));
|
|
543
656
|
}
|
|
544
657
|
|
|
545
658
|
// Search-time semantic dedup (v0.5.0 2.3): near-duplicate rows are
|
package/src/settings.js
CHANGED
|
@@ -58,6 +58,9 @@ const FEATURE_FLAG_BOOLEANS = [
|
|
|
58
58
|
"bm25SearchEnabled",
|
|
59
59
|
"conflictFreezeEnabled",
|
|
60
60
|
"trustEpistemicWeighting",
|
|
61
|
+
// Plan #2: attach per-source {keyword, vector, bm25, final} signals to each
|
|
62
|
+
// search result for transparency/debugging. Default off, purely decorative.
|
|
63
|
+
"signalTransparency",
|
|
61
64
|
// Issue #89:宽容校验回归(默认开)+ 跨类型合并显式放宽(默认关)。
|
|
62
65
|
"dreamSkipInvalid",
|
|
63
66
|
"allowCrossTypeMerge",
|
|
@@ -86,6 +89,10 @@ const FEATURE_FLAG_STRINGS = [
|
|
|
86
89
|
// /llm-providers 端点一起提供,留空 = 用巩固模型或当前模型。
|
|
87
90
|
"sleepProvider",
|
|
88
91
|
"sleepModel",
|
|
92
|
+
// 实体抽取侧专用路由(issue #109):provider/model 显式指定,
|
|
93
|
+
// 留空 = 用当前默认模型。
|
|
94
|
+
"entityExtractionProvider",
|
|
95
|
+
"entityExtractionModel",
|
|
89
96
|
"localEmbedModel",
|
|
90
97
|
"ollamaModel"
|
|
91
98
|
];
|
|
@@ -94,7 +101,12 @@ const FEATURE_FLAG_STRINGS = [
|
|
|
94
101
|
const FEATURE_FLAG_URLS = ["ollamaBaseUrl"];
|
|
95
102
|
// 枚举开关(与 config.js 的 z.union(z.const(...)) 对齐):仅允许列出的值。
|
|
96
103
|
const FEATURE_FLAG_ENUMS = {
|
|
97
|
-
embedProvider: ["openai", "local", "ollama"]
|
|
104
|
+
embedProvider: ["openai", "local", "ollama"],
|
|
105
|
+
// Plan #1: recall fusion recipe. blend = legacy (default); rrf / minmax are
|
|
106
|
+
// rank/scale-aware alternatives selected by the panel.
|
|
107
|
+
recallFusion: ["blend", "rrf", "minmax"],
|
|
108
|
+
// 实体抽取思考强度(issue #109):与 dreamReasoningEffort 枚举对齐。
|
|
109
|
+
entityExtractionReasoning: ["low", "medium", "high", "none"]
|
|
98
110
|
};
|
|
99
111
|
const FEATURE_FLAG_STRING_MAX = 200;
|
|
100
112
|
|
package/src/store.js
CHANGED
|
@@ -1023,8 +1023,11 @@ export function createStore(path) {
|
|
|
1023
1023
|
}
|
|
1024
1024
|
|
|
1025
1025
|
function embeddedCount() {
|
|
1026
|
+
// Active rows only — getStats pairs this with count() as the status card's
|
|
1027
|
+
// "indexed N / M" denominator, and count() defaults exclude forgotten and
|
|
1028
|
+
// archived memories. Unfiltered, archived rows inflate N past M.
|
|
1026
1029
|
return db.prepare(
|
|
1027
|
-
"SELECT count(*) AS c FROM memories WHERE embedding IS NOT NULL AND embedding != ''"
|
|
1030
|
+
"SELECT count(*) AS c FROM memories WHERE embedding IS NOT NULL AND embedding != '' AND forgotten = 0 AND archived = 0"
|
|
1028
1031
|
).get().c;
|
|
1029
1032
|
}
|
|
1030
1033
|
|