@modusensus/dsh-mneme 0.4.5 → 0.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -0
- package/lib/api.js +40 -2
- package/lib/config.js +35 -0
- package/lib/dream.js +86 -6
- package/lib/embedding.js +59 -2
- package/lib/index.js +68 -1
- package/lib/inject.js +79 -4
- package/lib/quality-filter.js +123 -0
- package/lib/service.js +214 -13
- package/lib/store.js +212 -5
- package/lib/summarize.js +65 -7
- package/lib/vector-index.js +12 -2
- package/package.json +1 -1
- package/src/api.js +40 -2
- package/src/config.js +35 -0
- package/src/dream.js +86 -6
- package/src/embedding.js +59 -2
- package/src/index.js +68 -1
- package/src/inject.js +79 -4
- package/src/quality-filter.js +123 -0
- package/src/service.js +214 -13
- package/src/store.js +212 -5
- package/src/summarize.js +65 -7
- package/src/vector-index.js +12 -2
- package/test/api.test.js +84 -0
- package/test/dream.test.js +52 -0
- package/test/inject.test.js +21 -0
- package/test/llm-audit.test.js +279 -0
- package/test/mirror-edit-digest.test.js +3 -1
- package/test/quality-filter.test.js +118 -0
- package/test/service.test.js +133 -2
- package/test/vector-index.test.js +22 -6
package/src/dream.js
CHANGED
|
@@ -199,11 +199,13 @@ function buildRecordReceipts({ runId, committed, snapshot, policyEpoch }) {
|
|
|
199
199
|
* accumulation covers both the real protocol ({type:"text-delta", index, text})
|
|
200
200
|
* and looser test doubles ({type:"text-delta", text}); a terminal error/abort
|
|
201
201
|
* surfaces as undefined. The caller decides how to treat an empty result.
|
|
202
|
+
* `onUsage` (optional, Bug8) receives any usage chunk for token accounting.
|
|
202
203
|
*/
|
|
203
|
-
async function streamText(ctx, options) {
|
|
204
|
+
async function streamText(ctx, options, onUsage) {
|
|
204
205
|
let text = "";
|
|
205
206
|
for await (const chunk of ctx.llm.stream(options)) {
|
|
206
207
|
if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
|
|
208
|
+
if (chunk.type === "usage" && typeof onUsage === "function") onUsage(chunk);
|
|
207
209
|
if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) {
|
|
208
210
|
return undefined;
|
|
209
211
|
}
|
|
@@ -211,6 +213,66 @@ async function streamText(ctx, options) {
|
|
|
211
213
|
return text;
|
|
212
214
|
}
|
|
213
215
|
|
|
216
|
+
/**
|
|
217
|
+
* Bug8: wrap a background LLM call so its token/time/status are recorded in the
|
|
218
|
+
* llm_audit_logs table. Best-effort bookkeeping: a failure to WRITE the audit
|
|
219
|
+
* row is swallowed (never blocks the LLM call), while a failure of the call
|
|
220
|
+
* itself is captured as status='error' and re-thrown so the caller keeps its
|
|
221
|
+
* existing error path. `spec` carries the static metadata (trigger_source,
|
|
222
|
+
* operation_type, model_id, related_memory_ids); `body(reportUsage)` performs
|
|
223
|
+
* the actual stream consumption and is handed a usage reporter for the chunks.
|
|
224
|
+
*/
|
|
225
|
+
async function runAuditedLlm(ctx, service, config, spec, body) {
|
|
226
|
+
const audit = config?.llmAudit;
|
|
227
|
+
if (audit?.enabled === false || typeof service?.saveLlmAudit !== "function") return body(() => {});
|
|
228
|
+
const startedAt = Date.now();
|
|
229
|
+
const timestamp = new Date(startedAt).toISOString();
|
|
230
|
+
let inputTokens = 0;
|
|
231
|
+
let outputTokens = 0;
|
|
232
|
+
let status = "success";
|
|
233
|
+
let errorMessage = null;
|
|
234
|
+
let result;
|
|
235
|
+
try {
|
|
236
|
+
result = await body((usage) => {
|
|
237
|
+
if (!usage) return;
|
|
238
|
+
const i = usage.input_tokens ?? usage.inputTokens ?? usage.prompt_tokens ?? usage.promptTokens;
|
|
239
|
+
const o = usage.output_tokens ?? usage.outputTokens ?? usage.completion_tokens ?? usage.completionTokens;
|
|
240
|
+
if (Number.isFinite(i)) inputTokens = i;
|
|
241
|
+
if (Number.isFinite(o)) outputTokens = o;
|
|
242
|
+
});
|
|
243
|
+
if (result === undefined) {
|
|
244
|
+
// stream aborted/errored: the caller treats undefined as a failed run;
|
|
245
|
+
// record it as error here so the audit shows the truth.
|
|
246
|
+
status = "error";
|
|
247
|
+
errorMessage = errorMessage ?? "llm stream aborted or errored";
|
|
248
|
+
}
|
|
249
|
+
return result;
|
|
250
|
+
} catch (error) {
|
|
251
|
+
status = "error";
|
|
252
|
+
errorMessage = String(error?.message ?? error);
|
|
253
|
+
throw error;
|
|
254
|
+
} finally {
|
|
255
|
+
try {
|
|
256
|
+
service.saveLlmAudit({
|
|
257
|
+
timestamp,
|
|
258
|
+
trigger_source: spec.triggerSource,
|
|
259
|
+
operation_type: spec.operationType,
|
|
260
|
+
model_id: spec.modelId,
|
|
261
|
+
input_tokens: inputTokens,
|
|
262
|
+
output_tokens: outputTokens,
|
|
263
|
+
total_tokens: inputTokens + outputTokens,
|
|
264
|
+
cost_usd: 0,
|
|
265
|
+
duration_ms: Date.now() - startedAt,
|
|
266
|
+
status,
|
|
267
|
+
error_message: errorMessage,
|
|
268
|
+
related_memory_ids: spec.relatedMemoryIds ?? []
|
|
269
|
+
});
|
|
270
|
+
} catch (auditError) {
|
|
271
|
+
ctx.logger?.warn?.(`dsh-mneme: llm audit write failed: ${String(auditError)}`);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
214
276
|
/**
|
|
215
277
|
* Resolve the LLM route: agent default model (deployment) first, plugin config
|
|
216
278
|
* (dreamProvider/dreamModel) as fallback. Falls through to undefined when no
|
|
@@ -487,7 +549,15 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
487
549
|
: CONSOLIDATION_PROMPT;
|
|
488
550
|
let decisionText;
|
|
489
551
|
try {
|
|
490
|
-
|
|
552
|
+
// Bug8: the consolidation call is audited (tokens/time/status). A throw
|
|
553
|
+
// re-propagates to the catch below; an aborted stream returns undefined
|
|
554
|
+
// and is treated as a failed run after the check below.
|
|
555
|
+
decisionText = await runAuditedLlm(ctx, service, config, {
|
|
556
|
+
triggerSource: "autoDream",
|
|
557
|
+
operationType: "dream_consolidate",
|
|
558
|
+
modelId: `${route.provider}:${route.model}`,
|
|
559
|
+
relatedMemoryIds: [...snapshot.keys()]
|
|
560
|
+
}, (reportUsage) => streamText(ctx, {
|
|
491
561
|
provider: route.provider,
|
|
492
562
|
model: route.model,
|
|
493
563
|
purpose: "compaction",
|
|
@@ -499,7 +569,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
499
569
|
{ role: "system", content: [{ type: "text", text: consolidationPrompt }] },
|
|
500
570
|
{ role: "user", content: [{ type: "text", text: listText }] }
|
|
501
571
|
]
|
|
502
|
-
});
|
|
572
|
+
}, reportUsage));
|
|
503
573
|
} catch (error) {
|
|
504
574
|
logger?.warn?.(`dsh-mneme dream: consolidation llm call failed: ${String(error)}`);
|
|
505
575
|
return finish({ ok: false, error: "llm failed", summary: false });
|
|
@@ -637,7 +707,13 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
637
707
|
// a failed run; summary:false marks a run that produced no summary.
|
|
638
708
|
let summaryText;
|
|
639
709
|
try {
|
|
640
|
-
|
|
710
|
+
// Bug8: the summary call is audited too (operation dream_summarize).
|
|
711
|
+
summaryText = await runAuditedLlm(ctx, service, config, {
|
|
712
|
+
triggerSource: "autoDream",
|
|
713
|
+
operationType: "dream_summarize",
|
|
714
|
+
modelId: `${route.provider}:${route.model}`,
|
|
715
|
+
relatedMemoryIds: []
|
|
716
|
+
}, (reportUsage) => streamText(ctx, {
|
|
641
717
|
provider: route.provider,
|
|
642
718
|
model: route.model,
|
|
643
719
|
purpose: "compaction",
|
|
@@ -649,14 +725,18 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
649
725
|
{ role: "system", content: [{ type: "text", text: SUMMARY_PROMPT }] },
|
|
650
726
|
{ role: "user", content: [{ type: "text", text: service.all().filter((m) => !m.archived && m.type !== "summary").map((m) => `- ${m.title}: ${m.content}`).join("\n") }] }
|
|
651
727
|
]
|
|
652
|
-
});
|
|
728
|
+
}, reportUsage));
|
|
653
729
|
} catch (error) {
|
|
654
730
|
logger?.warn?.(`dsh-mneme dream: summary llm call failed: ${String(error)}`);
|
|
655
731
|
return finish({ ok: false, error: "llm failed", applied, decisions: auditDecisions, outcome, frozen: frozenCount, summary: false });
|
|
656
732
|
}
|
|
657
733
|
let summaryStored = false;
|
|
658
734
|
if (summaryText !== undefined && summaryText.trim()) {
|
|
659
|
-
|
|
735
|
+
// Bug5 carve-out: the library overview is regenerated every run, so it
|
|
736
|
+
// must REPLACE the previous overview (not append — that would grow the
|
|
737
|
+
// summary unboundedly). `_overwrite` still archives the old overview into
|
|
738
|
+
// content_history before replacing it.
|
|
739
|
+
service.saveWithDedupe({ type: "summary", title: "记忆库总览", content: summaryText.trim(), importance: 5, source: "dream", _overwrite: true });
|
|
660
740
|
summaryStored = true;
|
|
661
741
|
// Re-embed the fresh summary so the index stays in sync with the store.
|
|
662
742
|
if (semantic?.embedder && semantic?.vectorIndex) {
|
package/src/embedding.js
CHANGED
|
@@ -4,6 +4,20 @@
|
|
|
4
4
|
// proxy) and any provider exposing the standard embeddings API.
|
|
5
5
|
const DEFAULT_TIMEOUT_MS = 15000;
|
|
6
6
|
|
|
7
|
+
/** djb2 — stable, fast fingerprint for a provider/model string. Mirrors the
|
|
8
|
+
* hash used by the local embedders so all backends share one fingerprint
|
|
9
|
+
* format (model#hex) for vector_meta consistency checks. */
|
|
10
|
+
function hashString(s) {
|
|
11
|
+
let h = 5381;
|
|
12
|
+
for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) >>> 0;
|
|
13
|
+
return h.toString(16);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** Full provider+model fingerprint used for index-consistency checks. */
|
|
17
|
+
function modelHashOf(model) {
|
|
18
|
+
return `${model}#${hashString(model)}`;
|
|
19
|
+
}
|
|
20
|
+
|
|
7
21
|
/** Normalize a configured baseUrl into the full embeddings endpoint URL. */
|
|
8
22
|
function embeddingsUrl(baseUrl) {
|
|
9
23
|
const base = String(baseUrl ?? "").trim().replace(/\/+$/, "");
|
|
@@ -50,8 +64,23 @@ export async function embedText({ baseUrl, apiKey, model }, text) {
|
|
|
50
64
|
* Embedder bound to the current settings + store: on each write it re-embeds
|
|
51
65
|
* the row's title+content and stores the vector. Failures are swallowed so a
|
|
52
66
|
* flaky embedding endpoint never breaks memory writes.
|
|
67
|
+
*
|
|
68
|
+
* `vectorIndex` (optional) is the vector_meta fingerprint holder: after any
|
|
69
|
+
* successful embed the model that produced the vectors is recorded, so the
|
|
70
|
+
* index can detect drift and the auto-reindex backfill knows what to rebuild.
|
|
53
71
|
*/
|
|
54
|
-
export function createEmbedder({ store, settings, logger }) {
|
|
72
|
+
export function createEmbedder({ store, settings, logger, vectorIndex }) {
|
|
73
|
+
// Dimension of the most recent successful embed, exposed for fingerprinting.
|
|
74
|
+
let _dimension = 0;
|
|
75
|
+
|
|
76
|
+
/** Record the producing model fingerprint in vector_meta (best-effort). */
|
|
77
|
+
function markModel(cfg, dimension) {
|
|
78
|
+
if (!vectorIndex || typeof vectorIndex.markModel !== "function") return;
|
|
79
|
+
try {
|
|
80
|
+
vectorIndex.markModel(modelHashOf(cfg.model), dimension);
|
|
81
|
+
} catch { /* metadata write is best-effort */ }
|
|
82
|
+
}
|
|
83
|
+
|
|
55
84
|
async function embedFor(id, title, content) {
|
|
56
85
|
const cfg = settings.getVectorConfig();
|
|
57
86
|
if (!cfg?.enabled || !cfg.baseUrl || !cfg.apiKey || !cfg.model) return;
|
|
@@ -59,6 +88,10 @@ export function createEmbedder({ store, settings, logger }) {
|
|
|
59
88
|
const vector = await embedText(cfg, text);
|
|
60
89
|
if (vector) {
|
|
61
90
|
store.setEmbedding(id, vector);
|
|
91
|
+
_dimension = vector.length;
|
|
92
|
+
// Bug3: record which model produced the current vectors so the index can
|
|
93
|
+
// detect drift and skip a redundant backfill when nothing changed.
|
|
94
|
+
markModel(cfg, vector.length);
|
|
62
95
|
logger?.info?.(`[dsh-mneme] embedded memory ${id} (dim=${vector.length})`);
|
|
63
96
|
}
|
|
64
97
|
}
|
|
@@ -74,7 +107,29 @@ export function createEmbedder({ store, settings, logger }) {
|
|
|
74
107
|
async embed(query) {
|
|
75
108
|
const cfg = settings.getVectorConfig();
|
|
76
109
|
if (!cfg?.enabled || !cfg.baseUrl || !cfg.apiKey || !cfg.model) return null;
|
|
77
|
-
|
|
110
|
+
const vector = await embedText(cfg, query);
|
|
111
|
+
if (vector) _dimension = vector.length;
|
|
112
|
+
return vector;
|
|
113
|
+
},
|
|
114
|
+
|
|
115
|
+
// Bug1: single-text adapter. Local/ollama embedders expose embedSingle
|
|
116
|
+
// natively; the legacy OpenAI-compatible client only has embed. This
|
|
117
|
+
// adapter unifies the interface so vector-index rebuildIndex (which guards
|
|
118
|
+
// on `typeof embedder.embedSingle === "function"`) accepts this embedder.
|
|
119
|
+
async embedSingle(text) {
|
|
120
|
+
if (typeof this.embed === "function") return this.embed(text);
|
|
121
|
+
return null;
|
|
122
|
+
},
|
|
123
|
+
|
|
124
|
+
/** Model fingerprint (model#hex), or undefined when not configured. */
|
|
125
|
+
get modelHash() {
|
|
126
|
+
const cfg = settings.getVectorConfig();
|
|
127
|
+
return cfg?.enabled && cfg.model ? modelHashOf(cfg.model) : undefined;
|
|
128
|
+
},
|
|
129
|
+
|
|
130
|
+
/** Dimension of the last successful embed (0 when never embedded). */
|
|
131
|
+
get dimension() {
|
|
132
|
+
return _dimension || undefined;
|
|
78
133
|
},
|
|
79
134
|
|
|
80
135
|
/** Batch re-index rows still missing an embedding. */
|
|
@@ -88,9 +143,11 @@ export function createEmbedder({ store, settings, logger }) {
|
|
|
88
143
|
const vector = await embedText(cfg, text);
|
|
89
144
|
if (vector) {
|
|
90
145
|
store.setEmbedding(row.id, vector);
|
|
146
|
+
_dimension = vector.length;
|
|
91
147
|
indexed++;
|
|
92
148
|
}
|
|
93
149
|
}
|
|
150
|
+
if (indexed > 0) markModel(cfg, _dimension || undefined);
|
|
94
151
|
return { indexed, skipped: rows.length - indexed };
|
|
95
152
|
}
|
|
96
153
|
};
|
package/src/index.js
CHANGED
|
@@ -43,6 +43,15 @@ export const apply = (ctx, config) => {
|
|
|
43
43
|
try {
|
|
44
44
|
store.deleteOldFailures(new Date(Date.now() - 90 * 86400000).toISOString());
|
|
45
45
|
} catch { /* non-fatal */ }
|
|
46
|
+
// Bug8: enforce llm_audit_logs retention on boot (config.llmAudit.retentionDays,
|
|
47
|
+
// default 90). Best-effort like the failure prune — the audit trail is
|
|
48
|
+
// bookkeeping and a failed purge must never block plugin boot.
|
|
49
|
+
try {
|
|
50
|
+
if (cfg.llmAudit?.enabled !== false) {
|
|
51
|
+
const retentionMs = Number.isInteger(cfg.llmAudit?.retentionDays) ? cfg.llmAudit.retentionDays : 90;
|
|
52
|
+
store.deleteOldLlmAudits(new Date(Date.now() - retentionMs * 86400000).toISOString());
|
|
53
|
+
}
|
|
54
|
+
} catch { /* non-fatal */ }
|
|
46
55
|
const mirror = createMirror(memoryDir);
|
|
47
56
|
const service = createService({ store, mirror, config: cfg, logger: ctx.logger });
|
|
48
57
|
|
|
@@ -99,7 +108,9 @@ export const apply = (ctx, config) => {
|
|
|
99
108
|
let embedder = null;
|
|
100
109
|
let reranker = null;
|
|
101
110
|
if (cfg.embedProvider === "openai") {
|
|
102
|
-
|
|
111
|
+
// vectorIndex is passed so the legacy OpenAI embedder records the producing
|
|
112
|
+
// model fingerprint after each successful embed (Bug3).
|
|
113
|
+
embedder = createEmbedder({ store, settings, logger: ctx.logger, vectorIndex });
|
|
103
114
|
service.setEmbedder(embedder);
|
|
104
115
|
// legacy OpenAI embedder is immediately usable
|
|
105
116
|
applyHumanEdits();
|
|
@@ -155,6 +166,62 @@ export const apply = (ctx, config) => {
|
|
|
155
166
|
}
|
|
156
167
|
}
|
|
157
168
|
|
|
169
|
+
// Bug2: lazy auto-backfill of missing embeddings on boot. When the vector API
|
|
170
|
+
// is configured and rows still lack an embedding (e.g. written before vector
|
|
171
|
+
// search was enabled) AND the vector_meta fingerprint is absent or stale, the
|
|
172
|
+
// index is rebuilt in the background after a short delay. Gated on
|
|
173
|
+
// cfg.autoReindexOnBoot; rate-limited in small batches so a large backlog
|
|
174
|
+
// never floods the provider. Failures degrade silently — search stays keyword.
|
|
175
|
+
function scheduleAutoReindex() {
|
|
176
|
+
if (cfg.autoReindexOnBoot === false) return;
|
|
177
|
+
const attempt = (tries) => {
|
|
178
|
+
try {
|
|
179
|
+
if (!embedder || typeof embedder.embedSingle !== "function") return;
|
|
180
|
+
if ("ready" in embedder && embedder.ready !== true) {
|
|
181
|
+
// Local/ollama embedders init asynchronously; give them a moment
|
|
182
|
+
// before giving up on this boot (next boot retries).
|
|
183
|
+
if (tries > 0) setTimeout(() => attempt(tries - 1), 2000);
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
if (!store.needsEmbedding(1).length) return; // nothing to backfill
|
|
187
|
+
// Model fingerprint gate: vectors already produced by the same model
|
|
188
|
+
// mean there is no drift and no rebuild needed.
|
|
189
|
+
const current = embedder.modelHash;
|
|
190
|
+
if (current && vectorIndex.modelHash?.() === current) return;
|
|
191
|
+
const BATCH = 10;
|
|
192
|
+
const MAX_TOTAL = 500; // bound boot-time work
|
|
193
|
+
(async () => {
|
|
194
|
+
let indexed = 0;
|
|
195
|
+
for (let done = 0; done < MAX_TOTAL;) {
|
|
196
|
+
const rows = store.needsEmbedding(BATCH);
|
|
197
|
+
if (!rows.length) break;
|
|
198
|
+
for (const row of rows) {
|
|
199
|
+
try {
|
|
200
|
+
const text = [row.title, row.content].filter(Boolean).join("\n");
|
|
201
|
+
const vector = await embedder.embedSingle(text);
|
|
202
|
+
if (vector?.length) {
|
|
203
|
+
store.setEmbedding(row.id, vector);
|
|
204
|
+
indexed++;
|
|
205
|
+
}
|
|
206
|
+
} catch { /* skip the bad row */ }
|
|
207
|
+
}
|
|
208
|
+
done += rows.length;
|
|
209
|
+
// Rate limit: space out batches so the provider is not hammered.
|
|
210
|
+
if (store.needsEmbedding(1).length) await new Promise((r) => setTimeout(r, 200));
|
|
211
|
+
}
|
|
212
|
+
if (indexed > 0 && current) vectorIndex.markModel?.(current, embedder.dimension);
|
|
213
|
+
ctx.logger?.info?.(`[dsh-mneme] auto-reindex backfilled ${indexed} embeddings on boot`);
|
|
214
|
+
})().catch((error) => {
|
|
215
|
+
ctx.logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
|
|
216
|
+
});
|
|
217
|
+
} catch (error) {
|
|
218
|
+
ctx.logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
|
|
219
|
+
}
|
|
220
|
+
};
|
|
221
|
+
setTimeout(() => attempt(5), 5000);
|
|
222
|
+
}
|
|
223
|
+
scheduleAutoReindex();
|
|
224
|
+
|
|
158
225
|
// Custom commands: register persisted commands into the DSH command registry
|
|
159
226
|
// on boot; add/remove re-register live through the API.
|
|
160
227
|
let commands = null;
|
package/src/inject.js
CHANGED
|
@@ -1,21 +1,88 @@
|
|
|
1
|
+
// Best-effort extraction of the current user's latest message text from the
|
|
2
|
+
// live session, for semantic-first injection (Bug4). The system-prompt
|
|
3
|
+
// interpolator renders synchronously, so this walks the already-materialized
|
|
4
|
+
// session event log (same event shape summarize.js consumes) and returns the
|
|
5
|
+
// most recent human message. Any failure degrades to "" — the injector then
|
|
6
|
+
// falls back to the legacy rule-based pick, never breaking the render.
|
|
7
|
+
function lastUserQuery(ctx) {
|
|
8
|
+
try {
|
|
9
|
+
const events = ctx?.agent?.session?.events;
|
|
10
|
+
if (!Array.isArray(events) || events.length === 0) return "";
|
|
11
|
+
for (let i = events.length - 1; i >= 0; i--) {
|
|
12
|
+
const event = events[i];
|
|
13
|
+
if (event?.type !== "user/message") continue;
|
|
14
|
+
const kind = event.data?.source?.kind;
|
|
15
|
+
if (kind !== undefined && kind !== "user") continue;
|
|
16
|
+
const parts = event.data?.content;
|
|
17
|
+
if (!Array.isArray(parts) || parts.length === 0) continue;
|
|
18
|
+
return parts
|
|
19
|
+
.map((p) => (typeof p === "string" ? p : p?.text ?? ""))
|
|
20
|
+
.filter(Boolean)
|
|
21
|
+
.join("\n")
|
|
22
|
+
.slice(0, 500);
|
|
23
|
+
}
|
|
24
|
+
} catch { /* session internals unavailable: degrade to no query */ }
|
|
25
|
+
return "";
|
|
26
|
+
}
|
|
27
|
+
|
|
1
28
|
export function createInjector(ctx, service, settings, config) {
|
|
2
29
|
const maxItems = config.maxInjectedItems ?? 5;
|
|
3
30
|
const threshold = config.importanceThreshold ?? 3;
|
|
4
31
|
|
|
32
|
+
// Bug6: bound the injected memory block. Each entry's content is truncated to
|
|
33
|
+
// MAX_CONTENT chars (trailing `…`); the whole block gets a MAX_BLOCK budget
|
|
34
|
+
// and an entry that would exceed it collapses to its title only, so a long
|
|
35
|
+
// memory can never push the injected context past a few thousand chars.
|
|
36
|
+
const MAX_CONTENT = 300;
|
|
37
|
+
const MAX_BLOCK = 1500;
|
|
38
|
+
|
|
5
39
|
function render(candidates) {
|
|
6
40
|
if (!candidates.length) return "";
|
|
7
|
-
const
|
|
41
|
+
const header = "[记忆库] 来自 dsh-mneme 的跨会话记忆(用户偏好与高优先级项目/决策):";
|
|
42
|
+
const lines = [header];
|
|
43
|
+
let budget = MAX_BLOCK - header.length;
|
|
8
44
|
for (const m of candidates) {
|
|
9
45
|
// Epistemic trust (v0.4.5): when enabled, measured observations are
|
|
10
46
|
// flagged so the agent can weigh them above guesses/opinions.
|
|
11
47
|
const verified = config.trustEpistemicWeighting === true && m.epistemic_status === "observation"
|
|
12
48
|
? "[verified] "
|
|
13
49
|
: "";
|
|
14
|
-
|
|
50
|
+
const title = `${m.title}(重要性 ${m.importance})`;
|
|
51
|
+
let content = String(m.content ?? "");
|
|
52
|
+
if (content.length > MAX_CONTENT) content = `${content.slice(0, MAX_CONTENT)}…`;
|
|
53
|
+
const full = `- [${m.type}] ${verified}${title}:${content}`;
|
|
54
|
+
if (budget - full.length >= 0) {
|
|
55
|
+
lines.push(full);
|
|
56
|
+
budget -= full.length;
|
|
57
|
+
} else {
|
|
58
|
+
lines.push(`- [${m.type}] ${verified}${title}`);
|
|
59
|
+
}
|
|
15
60
|
}
|
|
16
61
|
return lines.join("\n");
|
|
17
62
|
}
|
|
18
63
|
|
|
64
|
+
// Bug4: the system-prompt render is synchronous, so the semantic query vector
|
|
65
|
+
// must be prefetched asynchronously and cached for the next assembly. The
|
|
66
|
+
// first render after a new user message may still fall back to the rule-based
|
|
67
|
+
// pick; later assemblies in the same session reuse the cached vector. Bounded
|
|
68
|
+
// cache (cap 8, drop oldest) so a long session never grows it unbounded.
|
|
69
|
+
const QUERY_VECTOR_CACHE_MAX = 8;
|
|
70
|
+
const queryVectorCache = new Map();
|
|
71
|
+
let lastPrefetched = "";
|
|
72
|
+
|
|
73
|
+
function prefetchQueryVector(query) {
|
|
74
|
+
if (!query || query === lastPrefetched || queryVectorCache.has(query)) return;
|
|
75
|
+
lastPrefetched = query;
|
|
76
|
+
service.embedQuery(query).then((vec) => {
|
|
77
|
+
if (Array.isArray(vec) && vec.length) {
|
|
78
|
+
queryVectorCache.set(query, vec);
|
|
79
|
+
if (queryVectorCache.size > QUERY_VECTOR_CACHE_MAX) {
|
|
80
|
+
queryVectorCache.delete(queryVectorCache.keys().next().value);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}).catch(() => { /* prefetch is best-effort */ });
|
|
84
|
+
}
|
|
85
|
+
|
|
19
86
|
// User profile + rules: injected ahead of the memory block because they are
|
|
20
87
|
// always-relevant instructions the agent should follow every turn.
|
|
21
88
|
function renderUserSettings() {
|
|
@@ -32,8 +99,15 @@ export function createInjector(ctx, service, settings, config) {
|
|
|
32
99
|
ctx.systemPrompt.context({
|
|
33
100
|
name: "memory",
|
|
34
101
|
order: 90,
|
|
35
|
-
text: () => {
|
|
36
|
-
|
|
102
|
+
text: (ctx) => {
|
|
103
|
+
// Bug4: pass the latest user query so injection prefers semantically
|
|
104
|
+
// relevant memories; lastUserQuery is best-effort (empty → legacy).
|
|
105
|
+
// The query vector is prefetched asynchronously (cached) because the
|
|
106
|
+
// render itself must stay synchronous.
|
|
107
|
+
const query = lastUserQuery(ctx);
|
|
108
|
+
if (query) prefetchQueryVector(query);
|
|
109
|
+
const queryVector = queryVectorCache.get(query);
|
|
110
|
+
const candidates = service.injectCandidates({ query, queryVector, maxItems, threshold });
|
|
37
111
|
return render(candidates);
|
|
38
112
|
}
|
|
39
113
|
}),
|
|
@@ -45,6 +119,7 @@ export function createInjector(ctx, service, settings, config) {
|
|
|
45
119
|
];
|
|
46
120
|
|
|
47
121
|
return () => {
|
|
122
|
+
queryVectorCache.clear();
|
|
48
123
|
for (const dispose of disposers) {
|
|
49
124
|
if (typeof dispose === "function") dispose();
|
|
50
125
|
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
// Rule-based memory quality filter (Bug7). Pure + total: no shared state, no
|
|
2
|
+
// async, no external calls, so it can be unit-tested in isolation and wired
|
|
3
|
+
// into the writer without any I/O or store access.
|
|
4
|
+
//
|
|
5
|
+
// evaluateMemoryQuality scores a memory 0-100 and tags low-value signals. The
|
|
6
|
+
// writer then decides (config.memoryQualityFilter):
|
|
7
|
+
// score >= degradeThreshold (60) → stored normally
|
|
8
|
+
// archiveThreshold (30) <= score < 60 → quality_score persisted; the
|
|
9
|
+
// injection sort re-ranks by importance * quality_score/100 (degraded)
|
|
10
|
+
// score < archiveThreshold (30) → archived + tagged low_quality (still
|
|
11
|
+
// recallable via explicit search, just never auto-injected)
|
|
12
|
+
//
|
|
13
|
+
// Signals and their deductions from the base 100:
|
|
14
|
+
// meta meta-memory vocabulary (the memory talks about the
|
|
15
|
+
// memory system itself, not the user's world) −45
|
|
16
|
+
// self_referential title/content mentions its own type label −15
|
|
17
|
+
// short_content content shorter than minContentLength −80
|
|
18
|
+
// repetitive dedup ratio (unique chars / total) < 0.3 −50
|
|
19
|
+
// duplicate bigram similarity to a recent memory > 0.85 −80
|
|
20
|
+
//
|
|
21
|
+
// The meta signal alone lands a well-formed memory in the degraded band
|
|
22
|
+
// (30..60) — it is still stored and searchable, just demoted in injection.
|
|
23
|
+
// Reaching the archive band (< 30) needs a degenerate body (short, repetitive
|
|
24
|
+
// or near-duplicated) or stacked signals.
|
|
25
|
+
|
|
26
|
+
export const META_MEMORY_RE =
|
|
27
|
+
/记忆|mneme|recall|inject|上下文|token|prompt|系统指令|作为AI|作为助手|我需要记住|总结一下刚才/;
|
|
28
|
+
|
|
29
|
+
// Own-type labels, used for self-reference detection (the English type value
|
|
30
|
+
// the AI writers emit plus the Chinese equivalent a human would type).
|
|
31
|
+
const TYPE_LABELS = {
|
|
32
|
+
preference: ["preference", "偏好"],
|
|
33
|
+
project: ["project", "项目"],
|
|
34
|
+
decision: ["decision", "决策", "决定"],
|
|
35
|
+
history: ["history", "历史", "事件"],
|
|
36
|
+
summary: ["summary", "总结", "摘要", "总览"],
|
|
37
|
+
pattern: ["pattern", "模式", "规律"]
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
/** Normalized bigram-overlap similarity in [0,1]; 0 for tiny/empty inputs. */
|
|
41
|
+
export function textSimilarity(a, b) {
|
|
42
|
+
const bigrams = (s) => {
|
|
43
|
+
const set = new Set();
|
|
44
|
+
const t = String(s).replace(/\s+/g, "");
|
|
45
|
+
for (let i = 0; i < t.length - 1; i++) set.add(t.slice(i, i + 2));
|
|
46
|
+
return set;
|
|
47
|
+
};
|
|
48
|
+
const A = bigrams(a);
|
|
49
|
+
const B = bigrams(b);
|
|
50
|
+
if (!A.size || !B.size) return 0;
|
|
51
|
+
let inter = 0;
|
|
52
|
+
for (const g of A) if (B.has(g)) inter++;
|
|
53
|
+
return inter / Math.min(A.size, B.size);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Fraction of characters that are unique (dedup ratio in [0,1]). */
|
|
57
|
+
export function dedupRatio(text) {
|
|
58
|
+
const t = String(text);
|
|
59
|
+
if (!t.length) return 0;
|
|
60
|
+
return new Set(t).size / t.length;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Score a memory's quality. `recentContents` (optional) is the list of recent
|
|
65
|
+
* memory contents used for near-duplicate detection; when omitted the duplicate
|
|
66
|
+
* signal is skipped. Never throws: every input is coerced defensively.
|
|
67
|
+
* @param {object} memory { type, title, content }
|
|
68
|
+
* @param {object} [options]
|
|
69
|
+
* @param {number} [options.minContentLength=10]
|
|
70
|
+
* @param {string[]} [options.recentContents] up to ~20 recent contents
|
|
71
|
+
* @returns {{score: number, tags: string[], reason: string}}
|
|
72
|
+
*/
|
|
73
|
+
export function evaluateMemoryQuality(memory, options = {}) {
|
|
74
|
+
const minContentLength = options.minContentLength ?? 10;
|
|
75
|
+
const recentContents = Array.isArray(options.recentContents) ? options.recentContents : [];
|
|
76
|
+
const title = String(memory?.title ?? "");
|
|
77
|
+
const content = String(memory?.content ?? "");
|
|
78
|
+
const text = `${title}\n${content}`;
|
|
79
|
+
const trimmed = content.trim();
|
|
80
|
+
const tags = [];
|
|
81
|
+
const reasons = [];
|
|
82
|
+
let score = 100;
|
|
83
|
+
|
|
84
|
+
if (META_MEMORY_RE.test(text)) {
|
|
85
|
+
score -= 45;
|
|
86
|
+
tags.push("meta");
|
|
87
|
+
reasons.push("meta-memory vocabulary");
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const labels = TYPE_LABELS[memory?.type];
|
|
91
|
+
if (labels && labels.some((l) => text.includes(l))) {
|
|
92
|
+
score -= 15;
|
|
93
|
+
tags.push("self_referential");
|
|
94
|
+
reasons.push("mentions its own type");
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
if (minContentLength > 0 && trimmed.length < minContentLength) {
|
|
98
|
+
score -= 80;
|
|
99
|
+
tags.push("short_content");
|
|
100
|
+
reasons.push(`content shorter than ${minContentLength} chars`);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
if (trimmed.length > 0 && dedupRatio(trimmed) < 0.3) {
|
|
104
|
+
score -= 50;
|
|
105
|
+
tags.push("repetitive");
|
|
106
|
+
reasons.push("repetitive content");
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
if (recentContents.length > 0 && trimmed.length > 0) {
|
|
110
|
+
for (const other of recentContents) {
|
|
111
|
+
if (textSimilarity(trimmed, other) > 0.85) {
|
|
112
|
+
score -= 80;
|
|
113
|
+
tags.push("duplicate");
|
|
114
|
+
reasons.push("near-duplicate of a recent memory");
|
|
115
|
+
break;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
score = Math.max(0, Math.min(100, Math.round(score)));
|
|
121
|
+
if (score < 30) tags.push("low_quality");
|
|
122
|
+
return { score, tags: [...new Set(tags)], reason: reasons.length ? reasons.join("; ") : "ok" };
|
|
123
|
+
}
|