@modusensus/dsh-mneme 0.4.4 → 0.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -0
- package/lib/api.js +40 -2
- package/lib/config.js +53 -0
- package/lib/dream/decisions.js +43 -2
- package/lib/dream.js +86 -6
- package/lib/embedding.js +59 -2
- package/lib/index.js +68 -1
- package/lib/inject.js +84 -4
- package/lib/quality-filter.js +123 -0
- package/lib/service.js +355 -14
- package/lib/store.js +366 -5
- package/lib/summarize.js +65 -7
- package/lib/vector-index.js +12 -2
- package/package.json +1 -1
- package/src/api.js +40 -2
- package/src/config.js +53 -0
- package/src/dream/decisions.js +43 -2
- package/src/dream.js +86 -6
- package/src/embedding.js +59 -2
- package/src/index.js +68 -1
- package/src/inject.js +84 -4
- package/src/quality-filter.js +123 -0
- package/src/service.js +355 -14
- package/src/store.js +366 -5
- package/src/summarize.js +65 -7
- package/src/vector-index.js +12 -2
- package/test/api.test.js +84 -0
- package/test/dream.test.js +52 -0
- package/test/epistemic.test.js +298 -0
- package/test/inject.test.js +21 -0
- package/test/llm-audit.test.js +279 -0
- package/test/mirror-edit-digest.test.js +3 -1
- package/test/quality-filter.test.js +118 -0
- package/test/recall-evals.test.js +235 -0
- package/test/service.test.js +133 -2
- package/test/vector-index.test.js +22 -6
package/src/config.js
CHANGED
|
@@ -82,6 +82,15 @@ export const Config = z.object({
|
|
|
82
82
|
vectorSearchThreshold: z.number().min(0).max(1).default(0.65),
|
|
83
83
|
hybridSearchVectorWeight: z.number().min(0).max(1).default(0.6),
|
|
84
84
|
hybridSearchKeywordWeight: z.number().min(0).max(1).default(0.4),
|
|
85
|
+
// Lazy auto-backfill of missing embeddings on boot (Bug2): when the vector
|
|
86
|
+
// API is configured and rows still lack an embedding, the index is rebuilt
|
|
87
|
+
// in the background after a short delay, rate-limited in batches. On by
|
|
88
|
+
// default; set false to keep the backfill manual only.
|
|
89
|
+
autoReindexOnBoot: z.boolean().default(true),
|
|
90
|
+
// Semantic-first injection (Bug4): when enabled, injectCandidates with a
|
|
91
|
+
// non-empty query recalls via the vector index first and falls back to the
|
|
92
|
+
// rule-based pick to fill/dedupe. Empty query / no vector → legacy behavior.
|
|
93
|
+
hybridInject: z.boolean().default(true),
|
|
85
94
|
|
|
86
95
|
// --- semantic: rerank layer (v0.2) --------------------------------------
|
|
87
96
|
// Opt-in by default (item ⑥): the local cross-encoder pulls in onnxruntime
|
|
@@ -167,4 +176,48 @@ export const Config = z.object({
|
|
|
167
176
|
z.const("high"),
|
|
168
177
|
z.const("none")
|
|
169
178
|
]).default("none"),
|
|
179
|
+
|
|
180
|
+
// --- epistemic trust: memory source credibility (v0.4.5) -----------------
|
|
181
|
+
// Distinguish memories by source: observation (measured / witnessed),
|
|
182
|
+
// subjective (opinion / guess) and inferred (derived from other evidence).
|
|
183
|
+
// Opt-in by default: when false (default) retrieval ranking, injection
|
|
184
|
+
// marking and dream merge/conflict keepSource are untouched and
|
|
185
|
+
// epistemic_status stays inert data (still written + inferred on save, just
|
|
186
|
+
// never used to influence behavior).
|
|
187
|
+
trustEpistemicWeighting: z.boolean().default(false),
|
|
188
|
+
|
|
189
|
+
// --- memory quality filter (Bug7) ------------------------------------------
|
|
190
|
+
// Heuristic gate on what deserves the injection/recall surface. When enabled,
|
|
191
|
+
// saveWithDedupe scores each new memory after dedupe and before write:
|
|
192
|
+
// score >= degradeThreshold (60) → stored normally
|
|
193
|
+
// archiveThreshold (30) <= score < 60 → quality_score persisted and the
|
|
194
|
+
// injection sort re-ranks by importance * quality_score/100 (degraded)
|
|
195
|
+
// score < 30 → archived + tagged low_quality (still explicitly searchable)
|
|
196
|
+
// Meta-memory markers, near-duplicates and repetitive filler lose points.
|
|
197
|
+
memoryQualityFilter: z.object({
|
|
198
|
+
enabled: z.boolean().default(true),
|
|
199
|
+
archiveThreshold: z.natural().min(1).max(100).default(30),
|
|
200
|
+
degradeThreshold: z.natural().min(1).max(100).default(60),
|
|
201
|
+
minContentLength: z.natural().min(1).max(1000).default(10)
|
|
202
|
+
}).default({}),
|
|
203
|
+
|
|
204
|
+
// --- LLM audit trail (Bug8) ------------------------------------------------
|
|
205
|
+
// Records every background LLM call (autoDream consolidation + summary,
|
|
206
|
+
// autoSummarize compression) into llm_audit_logs: tokens, duration, status
|
|
207
|
+
// and which trigger produced it. Failures are recorded as status=error and
|
|
208
|
+
// never block the feature. retentionDays bounds the table: older rows are
|
|
209
|
+
// purged on boot.
|
|
210
|
+
llmAudit: z.object({
|
|
211
|
+
enabled: z.boolean().default(true),
|
|
212
|
+
retentionDays: z.natural().min(1).max(3650).default(90)
|
|
213
|
+
}).default({}),
|
|
214
|
+
|
|
215
|
+
// --- recall evaluation: test-result storage (v0.4.6, 方案 B) --------------
|
|
216
|
+
// Separate retrieval evaluation snapshots from the production recall audit.
|
|
217
|
+
// When false (default) evaluateRetrieval still computes precision/recall/mrr
|
|
218
|
+
// and returns them to the caller, but writes nothing to recall_evals — the
|
|
219
|
+
// eval table only grows when the operator opts in. Production searchMemories
|
|
220
|
+
// audits to recall_runs and NEVER touches recall_evals, regardless of this
|
|
221
|
+
// flag (production isolation is unconditional).
|
|
222
|
+
evalPersistTestResults: z.boolean().default(false),
|
|
170
223
|
});
|
package/src/dream/decisions.js
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
const ACTIONS = new Set(["keep", "merge", "archive", "conflict", "update", "create"]);
|
|
2
2
|
|
|
3
|
+
// Epistemic trust (v0.4.5): when config.trustEpistemicWeighting is on, merge
|
|
4
|
+
// keepSource and conflict winners prefer the higher-trust memory. Higher value
|
|
5
|
+
// = preferred. observation (measured) > inferred (derived) > subjective (guess).
|
|
6
|
+
const EPISTEMIC_PRIORITY = { observation: 3, inferred: 2, subjective: 1 };
|
|
7
|
+
|
|
3
8
|
/**
|
|
4
9
|
* Validate a dream decision list against a snapshot of eligible memories.
|
|
5
10
|
* @param decisions - LLM-produced decision list.
|
|
@@ -252,12 +257,34 @@ function applyOne(d, service, snapshot, config = {}) {
|
|
|
252
257
|
switch (d.action) {
|
|
253
258
|
case "archive": return applyArchive(d, service, snapshot);
|
|
254
259
|
case "merge": return applyMerge(d, service, snapshot, config);
|
|
255
|
-
case "conflict": return applyConflict(d, service, snapshot);
|
|
260
|
+
case "conflict": return applyConflict(d, service, snapshot, config);
|
|
256
261
|
case "create": return applyCreate(d, service, config);
|
|
257
262
|
default: return applyUpdate(d, service, snapshot, config);
|
|
258
263
|
}
|
|
259
264
|
}
|
|
260
265
|
|
|
266
|
+
/**
|
|
267
|
+
* Highest-epistemic-priority UNARCHIVED id among `ids` (ties break toward
|
|
268
|
+
* `preferred`). Archived memories are never eligible keepers — promoting one
|
|
269
|
+
* would demote the real keepSource to a source and then hit the archived-keeper
|
|
270
|
+
* guard in applyMerge, silently skipping the whole merge. When `preferred`
|
|
271
|
+
* itself is archived (or missing), fall back to any unarchived candidate.
|
|
272
|
+
*/
|
|
273
|
+
function pickBestKeeper(ids, preferred, service) {
|
|
274
|
+
let best = null;
|
|
275
|
+
let bestP = -1;
|
|
276
|
+
for (const id of ids) {
|
|
277
|
+
const mem = service.getById(id);
|
|
278
|
+
if (!mem || mem.archived) continue; // archived/missing: ineligible keeper
|
|
279
|
+
const p = EPISTEMIC_PRIORITY[mem.epistemic_status] ?? 0;
|
|
280
|
+
if (p > bestP || (p === bestP && id === preferred)) {
|
|
281
|
+
bestP = p;
|
|
282
|
+
best = id;
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
return best ?? preferred;
|
|
286
|
+
}
|
|
287
|
+
|
|
261
288
|
/**
|
|
262
289
|
* Mint a fresh memory (pattern discovery). No existing target, so no CAS guard.
|
|
263
290
|
* Evidence ids ride in the content so a pattern stays traceable to its source
|
|
@@ -297,6 +324,13 @@ function applyArchive(d, service, snapshot) {
|
|
|
297
324
|
}
|
|
298
325
|
|
|
299
326
|
function applyMerge(d, service, snapshot, config = {}) {
|
|
327
|
+
// Epistemic trust (v0.4.5): when enabled, prefer an observation keeper over a
|
|
328
|
+
// subjective/inferred one. Mutating the decision keeps the receipt + committed
|
|
329
|
+
// record aligned with the actual keeper.
|
|
330
|
+
if (config.trustEpistemicWeighting === true) {
|
|
331
|
+
const best = pickBestKeeper(d.ids, d.keepSource, service);
|
|
332
|
+
if (best && best !== d.keepSource) d.keepSource = best;
|
|
333
|
+
}
|
|
300
334
|
const sources = d.ids.filter((id) => id !== d.keepSource);
|
|
301
335
|
// Idempotent replay: if every other source is already archived, this merge
|
|
302
336
|
// already landed — skip so a replayed/concurrent decision never double-counts
|
|
@@ -334,7 +368,14 @@ function applyMerge(d, service, snapshot, config = {}) {
|
|
|
334
368
|
};
|
|
335
369
|
}
|
|
336
370
|
|
|
337
|
-
function applyConflict(d, service, snapshot) {
|
|
371
|
+
function applyConflict(d, service, snapshot, config = {}) {
|
|
372
|
+
// Epistemic trust (v0.4.5): when enabled, the observation side of a conflict
|
|
373
|
+
// is preferred as winner over a subjective/inferred one.
|
|
374
|
+
if (config.trustEpistemicWeighting === true) {
|
|
375
|
+
const pw = EPISTEMIC_PRIORITY[service.getById(d.winner)?.epistemic_status] ?? 0;
|
|
376
|
+
const pl = EPISTEMIC_PRIORITY[service.getById(d.loser)?.epistemic_status] ?? 0;
|
|
377
|
+
if (pl > pw) [d.winner, d.loser] = [d.loser, d.winner];
|
|
378
|
+
}
|
|
338
379
|
const winner = service.getById(d.winner);
|
|
339
380
|
const loser = service.getById(d.loser);
|
|
340
381
|
if (!winner || !loser) return "skipped";
|
package/src/dream.js
CHANGED
|
@@ -199,11 +199,13 @@ function buildRecordReceipts({ runId, committed, snapshot, policyEpoch }) {
|
|
|
199
199
|
* accumulation covers both the real protocol ({type:"text-delta", index, text})
|
|
200
200
|
* and looser test doubles ({type:"text-delta", text}); a terminal error/abort
|
|
201
201
|
* surfaces as undefined. The caller decides how to treat an empty result.
|
|
202
|
+
* `onUsage` (optional, Bug8) receives any usage chunk for token accounting.
|
|
202
203
|
*/
|
|
203
|
-
async function streamText(ctx, options) {
|
|
204
|
+
async function streamText(ctx, options, onUsage) {
|
|
204
205
|
let text = "";
|
|
205
206
|
for await (const chunk of ctx.llm.stream(options)) {
|
|
206
207
|
if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
|
|
208
|
+
if (chunk.type === "usage" && typeof onUsage === "function") onUsage(chunk);
|
|
207
209
|
if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) {
|
|
208
210
|
return undefined;
|
|
209
211
|
}
|
|
@@ -211,6 +213,66 @@ async function streamText(ctx, options) {
|
|
|
211
213
|
return text;
|
|
212
214
|
}
|
|
213
215
|
|
|
216
|
+
/**
|
|
217
|
+
* Bug8: wrap a background LLM call so its token/time/status are recorded in the
|
|
218
|
+
* llm_audit_logs table. Best-effort bookkeeping: a failure to WRITE the audit
|
|
219
|
+
* row is swallowed (never blocks the LLM call), while a failure of the call
|
|
220
|
+
* itself is captured as status='error' and re-thrown so the caller keeps its
|
|
221
|
+
* existing error path. `spec` carries the static metadata (trigger_source,
|
|
222
|
+
* operation_type, model_id, related_memory_ids); `body(reportUsage)` performs
|
|
223
|
+
* the actual stream consumption and is handed a usage reporter for the chunks.
|
|
224
|
+
*/
|
|
225
|
+
async function runAuditedLlm(ctx, service, config, spec, body) {
|
|
226
|
+
const audit = config?.llmAudit;
|
|
227
|
+
if (audit?.enabled === false || typeof service?.saveLlmAudit !== "function") return body(() => {});
|
|
228
|
+
const startedAt = Date.now();
|
|
229
|
+
const timestamp = new Date(startedAt).toISOString();
|
|
230
|
+
let inputTokens = 0;
|
|
231
|
+
let outputTokens = 0;
|
|
232
|
+
let status = "success";
|
|
233
|
+
let errorMessage = null;
|
|
234
|
+
let result;
|
|
235
|
+
try {
|
|
236
|
+
result = await body((usage) => {
|
|
237
|
+
if (!usage) return;
|
|
238
|
+
const i = usage.input_tokens ?? usage.inputTokens ?? usage.prompt_tokens ?? usage.promptTokens;
|
|
239
|
+
const o = usage.output_tokens ?? usage.outputTokens ?? usage.completion_tokens ?? usage.completionTokens;
|
|
240
|
+
if (Number.isFinite(i)) inputTokens = i;
|
|
241
|
+
if (Number.isFinite(o)) outputTokens = o;
|
|
242
|
+
});
|
|
243
|
+
if (result === undefined) {
|
|
244
|
+
// stream aborted/errored: the caller treats undefined as a failed run;
|
|
245
|
+
// record it as error here so the audit shows the truth.
|
|
246
|
+
status = "error";
|
|
247
|
+
errorMessage = errorMessage ?? "llm stream aborted or errored";
|
|
248
|
+
}
|
|
249
|
+
return result;
|
|
250
|
+
} catch (error) {
|
|
251
|
+
status = "error";
|
|
252
|
+
errorMessage = String(error?.message ?? error);
|
|
253
|
+
throw error;
|
|
254
|
+
} finally {
|
|
255
|
+
try {
|
|
256
|
+
service.saveLlmAudit({
|
|
257
|
+
timestamp,
|
|
258
|
+
trigger_source: spec.triggerSource,
|
|
259
|
+
operation_type: spec.operationType,
|
|
260
|
+
model_id: spec.modelId,
|
|
261
|
+
input_tokens: inputTokens,
|
|
262
|
+
output_tokens: outputTokens,
|
|
263
|
+
total_tokens: inputTokens + outputTokens,
|
|
264
|
+
cost_usd: 0,
|
|
265
|
+
duration_ms: Date.now() - startedAt,
|
|
266
|
+
status,
|
|
267
|
+
error_message: errorMessage,
|
|
268
|
+
related_memory_ids: spec.relatedMemoryIds ?? []
|
|
269
|
+
});
|
|
270
|
+
} catch (auditError) {
|
|
271
|
+
ctx.logger?.warn?.(`dsh-mneme: llm audit write failed: ${String(auditError)}`);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
214
276
|
/**
|
|
215
277
|
* Resolve the LLM route: agent default model (deployment) first, plugin config
|
|
216
278
|
* (dreamProvider/dreamModel) as fallback. Falls through to undefined when no
|
|
@@ -487,7 +549,15 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
487
549
|
: CONSOLIDATION_PROMPT;
|
|
488
550
|
let decisionText;
|
|
489
551
|
try {
|
|
490
|
-
|
|
552
|
+
// Bug8: the consolidation call is audited (tokens/time/status). A throw
|
|
553
|
+
// re-propagates to the catch below; an aborted stream returns undefined
|
|
554
|
+
// and is treated as a failed run after the check below.
|
|
555
|
+
decisionText = await runAuditedLlm(ctx, service, config, {
|
|
556
|
+
triggerSource: "autoDream",
|
|
557
|
+
operationType: "dream_consolidate",
|
|
558
|
+
modelId: `${route.provider}:${route.model}`,
|
|
559
|
+
relatedMemoryIds: [...snapshot.keys()]
|
|
560
|
+
}, (reportUsage) => streamText(ctx, {
|
|
491
561
|
provider: route.provider,
|
|
492
562
|
model: route.model,
|
|
493
563
|
purpose: "compaction",
|
|
@@ -499,7 +569,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
499
569
|
{ role: "system", content: [{ type: "text", text: consolidationPrompt }] },
|
|
500
570
|
{ role: "user", content: [{ type: "text", text: listText }] }
|
|
501
571
|
]
|
|
502
|
-
});
|
|
572
|
+
}, reportUsage));
|
|
503
573
|
} catch (error) {
|
|
504
574
|
logger?.warn?.(`dsh-mneme dream: consolidation llm call failed: ${String(error)}`);
|
|
505
575
|
return finish({ ok: false, error: "llm failed", summary: false });
|
|
@@ -637,7 +707,13 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
637
707
|
// a failed run; summary:false marks a run that produced no summary.
|
|
638
708
|
let summaryText;
|
|
639
709
|
try {
|
|
640
|
-
|
|
710
|
+
// Bug8: the summary call is audited too (operation dream_summarize).
|
|
711
|
+
summaryText = await runAuditedLlm(ctx, service, config, {
|
|
712
|
+
triggerSource: "autoDream",
|
|
713
|
+
operationType: "dream_summarize",
|
|
714
|
+
modelId: `${route.provider}:${route.model}`,
|
|
715
|
+
relatedMemoryIds: []
|
|
716
|
+
}, (reportUsage) => streamText(ctx, {
|
|
641
717
|
provider: route.provider,
|
|
642
718
|
model: route.model,
|
|
643
719
|
purpose: "compaction",
|
|
@@ -649,14 +725,18 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
649
725
|
{ role: "system", content: [{ type: "text", text: SUMMARY_PROMPT }] },
|
|
650
726
|
{ role: "user", content: [{ type: "text", text: service.all().filter((m) => !m.archived && m.type !== "summary").map((m) => `- ${m.title}: ${m.content}`).join("\n") }] }
|
|
651
727
|
]
|
|
652
|
-
});
|
|
728
|
+
}, reportUsage));
|
|
653
729
|
} catch (error) {
|
|
654
730
|
logger?.warn?.(`dsh-mneme dream: summary llm call failed: ${String(error)}`);
|
|
655
731
|
return finish({ ok: false, error: "llm failed", applied, decisions: auditDecisions, outcome, frozen: frozenCount, summary: false });
|
|
656
732
|
}
|
|
657
733
|
let summaryStored = false;
|
|
658
734
|
if (summaryText !== undefined && summaryText.trim()) {
|
|
659
|
-
|
|
735
|
+
// Bug5 carve-out: the library overview is regenerated every run, so it
|
|
736
|
+
// must REPLACE the previous overview (not append — that would grow the
|
|
737
|
+
// summary unboundedly). `_overwrite` still archives the old overview into
|
|
738
|
+
// content_history before replacing it.
|
|
739
|
+
service.saveWithDedupe({ type: "summary", title: "记忆库总览", content: summaryText.trim(), importance: 5, source: "dream", _overwrite: true });
|
|
660
740
|
summaryStored = true;
|
|
661
741
|
// Re-embed the fresh summary so the index stays in sync with the store.
|
|
662
742
|
if (semantic?.embedder && semantic?.vectorIndex) {
|
package/src/embedding.js
CHANGED
|
@@ -4,6 +4,20 @@
|
|
|
4
4
|
// proxy) and any provider exposing the standard embeddings API.
|
|
5
5
|
const DEFAULT_TIMEOUT_MS = 15000;
|
|
6
6
|
|
|
7
|
+
/** djb2 — stable, fast fingerprint for a provider/model string. Mirrors the
|
|
8
|
+
* hash used by the local embedders so all backends share one fingerprint
|
|
9
|
+
* format (model#hex) for vector_meta consistency checks. */
|
|
10
|
+
function hashString(s) {
|
|
11
|
+
let h = 5381;
|
|
12
|
+
for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) >>> 0;
|
|
13
|
+
return h.toString(16);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** Full provider+model fingerprint used for index-consistency checks. */
|
|
17
|
+
function modelHashOf(model) {
|
|
18
|
+
return `${model}#${hashString(model)}`;
|
|
19
|
+
}
|
|
20
|
+
|
|
7
21
|
/** Normalize a configured baseUrl into the full embeddings endpoint URL. */
|
|
8
22
|
function embeddingsUrl(baseUrl) {
|
|
9
23
|
const base = String(baseUrl ?? "").trim().replace(/\/+$/, "");
|
|
@@ -50,8 +64,23 @@ export async function embedText({ baseUrl, apiKey, model }, text) {
|
|
|
50
64
|
* Embedder bound to the current settings + store: on each write it re-embeds
|
|
51
65
|
* the row's title+content and stores the vector. Failures are swallowed so a
|
|
52
66
|
* flaky embedding endpoint never breaks memory writes.
|
|
67
|
+
*
|
|
68
|
+
* `vectorIndex` (optional) is the vector_meta fingerprint holder: after any
|
|
69
|
+
* successful embed the model that produced the vectors is recorded, so the
|
|
70
|
+
* index can detect drift and the auto-reindex backfill knows what to rebuild.
|
|
53
71
|
*/
|
|
54
|
-
export function createEmbedder({ store, settings, logger }) {
|
|
72
|
+
export function createEmbedder({ store, settings, logger, vectorIndex }) {
|
|
73
|
+
// Dimension of the most recent successful embed, exposed for fingerprinting.
|
|
74
|
+
let _dimension = 0;
|
|
75
|
+
|
|
76
|
+
/** Record the producing model fingerprint in vector_meta (best-effort). */
|
|
77
|
+
function markModel(cfg, dimension) {
|
|
78
|
+
if (!vectorIndex || typeof vectorIndex.markModel !== "function") return;
|
|
79
|
+
try {
|
|
80
|
+
vectorIndex.markModel(modelHashOf(cfg.model), dimension);
|
|
81
|
+
} catch { /* metadata write is best-effort */ }
|
|
82
|
+
}
|
|
83
|
+
|
|
55
84
|
async function embedFor(id, title, content) {
|
|
56
85
|
const cfg = settings.getVectorConfig();
|
|
57
86
|
if (!cfg?.enabled || !cfg.baseUrl || !cfg.apiKey || !cfg.model) return;
|
|
@@ -59,6 +88,10 @@ export function createEmbedder({ store, settings, logger }) {
|
|
|
59
88
|
const vector = await embedText(cfg, text);
|
|
60
89
|
if (vector) {
|
|
61
90
|
store.setEmbedding(id, vector);
|
|
91
|
+
_dimension = vector.length;
|
|
92
|
+
// Bug3: record which model produced the current vectors so the index can
|
|
93
|
+
// detect drift and skip a redundant backfill when nothing changed.
|
|
94
|
+
markModel(cfg, vector.length);
|
|
62
95
|
logger?.info?.(`[dsh-mneme] embedded memory ${id} (dim=${vector.length})`);
|
|
63
96
|
}
|
|
64
97
|
}
|
|
@@ -74,7 +107,29 @@ export function createEmbedder({ store, settings, logger }) {
|
|
|
74
107
|
async embed(query) {
|
|
75
108
|
const cfg = settings.getVectorConfig();
|
|
76
109
|
if (!cfg?.enabled || !cfg.baseUrl || !cfg.apiKey || !cfg.model) return null;
|
|
77
|
-
|
|
110
|
+
const vector = await embedText(cfg, query);
|
|
111
|
+
if (vector) _dimension = vector.length;
|
|
112
|
+
return vector;
|
|
113
|
+
},
|
|
114
|
+
|
|
115
|
+
// Bug1: single-text adapter. Local/ollama embedders expose embedSingle
|
|
116
|
+
// natively; the legacy OpenAI-compatible client only has embed. This
|
|
117
|
+
// adapter unifies the interface so vector-index rebuildIndex (which guards
|
|
118
|
+
// on `typeof embedder.embedSingle === "function"`) accepts this embedder.
|
|
119
|
+
async embedSingle(text) {
|
|
120
|
+
if (typeof this.embed === "function") return this.embed(text);
|
|
121
|
+
return null;
|
|
122
|
+
},
|
|
123
|
+
|
|
124
|
+
/** Model fingerprint (model#hex), or undefined when not configured. */
|
|
125
|
+
get modelHash() {
|
|
126
|
+
const cfg = settings.getVectorConfig();
|
|
127
|
+
return cfg?.enabled && cfg.model ? modelHashOf(cfg.model) : undefined;
|
|
128
|
+
},
|
|
129
|
+
|
|
130
|
+
/** Dimension of the last successful embed (0 when never embedded). */
|
|
131
|
+
get dimension() {
|
|
132
|
+
return _dimension || undefined;
|
|
78
133
|
},
|
|
79
134
|
|
|
80
135
|
/** Batch re-index rows still missing an embedding. */
|
|
@@ -88,9 +143,11 @@ export function createEmbedder({ store, settings, logger }) {
|
|
|
88
143
|
const vector = await embedText(cfg, text);
|
|
89
144
|
if (vector) {
|
|
90
145
|
store.setEmbedding(row.id, vector);
|
|
146
|
+
_dimension = vector.length;
|
|
91
147
|
indexed++;
|
|
92
148
|
}
|
|
93
149
|
}
|
|
150
|
+
if (indexed > 0) markModel(cfg, _dimension || undefined);
|
|
94
151
|
return { indexed, skipped: rows.length - indexed };
|
|
95
152
|
}
|
|
96
153
|
};
|
package/src/index.js
CHANGED
|
@@ -43,6 +43,15 @@ export const apply = (ctx, config) => {
|
|
|
43
43
|
try {
|
|
44
44
|
store.deleteOldFailures(new Date(Date.now() - 90 * 86400000).toISOString());
|
|
45
45
|
} catch { /* non-fatal */ }
|
|
46
|
+
// Bug8: enforce llm_audit_logs retention on boot (config.llmAudit.retentionDays,
|
|
47
|
+
// default 90). Best-effort like the failure prune — the audit trail is
|
|
48
|
+
// bookkeeping and a failed purge must never block plugin boot.
|
|
49
|
+
try {
|
|
50
|
+
if (cfg.llmAudit?.enabled !== false) {
|
|
51
|
+
const retentionMs = Number.isInteger(cfg.llmAudit?.retentionDays) ? cfg.llmAudit.retentionDays : 90;
|
|
52
|
+
store.deleteOldLlmAudits(new Date(Date.now() - retentionMs * 86400000).toISOString());
|
|
53
|
+
}
|
|
54
|
+
} catch { /* non-fatal */ }
|
|
46
55
|
const mirror = createMirror(memoryDir);
|
|
47
56
|
const service = createService({ store, mirror, config: cfg, logger: ctx.logger });
|
|
48
57
|
|
|
@@ -99,7 +108,9 @@ export const apply = (ctx, config) => {
|
|
|
99
108
|
let embedder = null;
|
|
100
109
|
let reranker = null;
|
|
101
110
|
if (cfg.embedProvider === "openai") {
|
|
102
|
-
|
|
111
|
+
// vectorIndex is passed so the legacy OpenAI embedder records the producing
|
|
112
|
+
// model fingerprint after each successful embed (Bug3).
|
|
113
|
+
embedder = createEmbedder({ store, settings, logger: ctx.logger, vectorIndex });
|
|
103
114
|
service.setEmbedder(embedder);
|
|
104
115
|
// legacy OpenAI embedder is immediately usable
|
|
105
116
|
applyHumanEdits();
|
|
@@ -155,6 +166,62 @@ export const apply = (ctx, config) => {
|
|
|
155
166
|
}
|
|
156
167
|
}
|
|
157
168
|
|
|
169
|
+
// Bug2: lazy auto-backfill of missing embeddings on boot. When the vector API
|
|
170
|
+
// is configured and rows still lack an embedding (e.g. written before vector
|
|
171
|
+
// search was enabled) AND the vector_meta fingerprint is absent or stale, the
|
|
172
|
+
// index is rebuilt in the background after a short delay. Gated on
|
|
173
|
+
// cfg.autoReindexOnBoot; rate-limited in small batches so a large backlog
|
|
174
|
+
// never floods the provider. Failures degrade silently — search stays keyword.
|
|
175
|
+
function scheduleAutoReindex() {
|
|
176
|
+
if (cfg.autoReindexOnBoot === false) return;
|
|
177
|
+
const attempt = (tries) => {
|
|
178
|
+
try {
|
|
179
|
+
if (!embedder || typeof embedder.embedSingle !== "function") return;
|
|
180
|
+
if ("ready" in embedder && embedder.ready !== true) {
|
|
181
|
+
// Local/ollama embedders init asynchronously; give them a moment
|
|
182
|
+
// before giving up on this boot (next boot retries).
|
|
183
|
+
if (tries > 0) setTimeout(() => attempt(tries - 1), 2000);
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
if (!store.needsEmbedding(1).length) return; // nothing to backfill
|
|
187
|
+
// Model fingerprint gate: vectors already produced by the same model
|
|
188
|
+
// mean there is no drift and no rebuild needed.
|
|
189
|
+
const current = embedder.modelHash;
|
|
190
|
+
if (current && vectorIndex.modelHash?.() === current) return;
|
|
191
|
+
const BATCH = 10;
|
|
192
|
+
const MAX_TOTAL = 500; // bound boot-time work
|
|
193
|
+
(async () => {
|
|
194
|
+
let indexed = 0;
|
|
195
|
+
for (let done = 0; done < MAX_TOTAL;) {
|
|
196
|
+
const rows = store.needsEmbedding(BATCH);
|
|
197
|
+
if (!rows.length) break;
|
|
198
|
+
for (const row of rows) {
|
|
199
|
+
try {
|
|
200
|
+
const text = [row.title, row.content].filter(Boolean).join("\n");
|
|
201
|
+
const vector = await embedder.embedSingle(text);
|
|
202
|
+
if (vector?.length) {
|
|
203
|
+
store.setEmbedding(row.id, vector);
|
|
204
|
+
indexed++;
|
|
205
|
+
}
|
|
206
|
+
} catch { /* skip the bad row */ }
|
|
207
|
+
}
|
|
208
|
+
done += rows.length;
|
|
209
|
+
// Rate limit: space out batches so the provider is not hammered.
|
|
210
|
+
if (store.needsEmbedding(1).length) await new Promise((r) => setTimeout(r, 200));
|
|
211
|
+
}
|
|
212
|
+
if (indexed > 0 && current) vectorIndex.markModel?.(current, embedder.dimension);
|
|
213
|
+
ctx.logger?.info?.(`[dsh-mneme] auto-reindex backfilled ${indexed} embeddings on boot`);
|
|
214
|
+
})().catch((error) => {
|
|
215
|
+
ctx.logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
|
|
216
|
+
});
|
|
217
|
+
} catch (error) {
|
|
218
|
+
ctx.logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
|
|
219
|
+
}
|
|
220
|
+
};
|
|
221
|
+
setTimeout(() => attempt(5), 5000);
|
|
222
|
+
}
|
|
223
|
+
scheduleAutoReindex();
|
|
224
|
+
|
|
158
225
|
// Custom commands: register persisted commands into the DSH command registry
|
|
159
226
|
// on boot; add/remove re-register live through the API.
|
|
160
227
|
let commands = null;
|
package/src/inject.js
CHANGED
|
@@ -1,16 +1,88 @@
|
|
|
1
|
+
// Best-effort extraction of the current user's latest message text from the
|
|
2
|
+
// live session, for semantic-first injection (Bug4). The system-prompt
|
|
3
|
+
// interpolator renders synchronously, so this walks the already-materialized
|
|
4
|
+
// session event log (same event shape summarize.js consumes) and returns the
|
|
5
|
+
// most recent human message. Any failure degrades to "" — the injector then
|
|
6
|
+
// falls back to the legacy rule-based pick, never breaking the render.
|
|
7
|
+
function lastUserQuery(ctx) {
|
|
8
|
+
try {
|
|
9
|
+
const events = ctx?.agent?.session?.events;
|
|
10
|
+
if (!Array.isArray(events) || events.length === 0) return "";
|
|
11
|
+
for (let i = events.length - 1; i >= 0; i--) {
|
|
12
|
+
const event = events[i];
|
|
13
|
+
if (event?.type !== "user/message") continue;
|
|
14
|
+
const kind = event.data?.source?.kind;
|
|
15
|
+
if (kind !== undefined && kind !== "user") continue;
|
|
16
|
+
const parts = event.data?.content;
|
|
17
|
+
if (!Array.isArray(parts) || parts.length === 0) continue;
|
|
18
|
+
return parts
|
|
19
|
+
.map((p) => (typeof p === "string" ? p : p?.text ?? ""))
|
|
20
|
+
.filter(Boolean)
|
|
21
|
+
.join("\n")
|
|
22
|
+
.slice(0, 500);
|
|
23
|
+
}
|
|
24
|
+
} catch { /* session internals unavailable: degrade to no query */ }
|
|
25
|
+
return "";
|
|
26
|
+
}
|
|
27
|
+
|
|
1
28
|
export function createInjector(ctx, service, settings, config) {
|
|
2
29
|
const maxItems = config.maxInjectedItems ?? 5;
|
|
3
30
|
const threshold = config.importanceThreshold ?? 3;
|
|
4
31
|
|
|
32
|
+
// Bug6: bound the injected memory block. Each entry's content is truncated to
|
|
33
|
+
// MAX_CONTENT chars (trailing `…`); the whole block gets a MAX_BLOCK budget
|
|
34
|
+
// and an entry that would exceed it collapses to its title only, so a long
|
|
35
|
+
// memory can never push the injected context past a few thousand chars.
|
|
36
|
+
const MAX_CONTENT = 300;
|
|
37
|
+
const MAX_BLOCK = 1500;
|
|
38
|
+
|
|
5
39
|
function render(candidates) {
|
|
6
40
|
if (!candidates.length) return "";
|
|
7
|
-
const
|
|
41
|
+
const header = "[记忆库] 来自 dsh-mneme 的跨会话记忆(用户偏好与高优先级项目/决策):";
|
|
42
|
+
const lines = [header];
|
|
43
|
+
let budget = MAX_BLOCK - header.length;
|
|
8
44
|
for (const m of candidates) {
|
|
9
|
-
|
|
45
|
+
// Epistemic trust (v0.4.5): when enabled, measured observations are
|
|
46
|
+
// flagged so the agent can weigh them above guesses/opinions.
|
|
47
|
+
const verified = config.trustEpistemicWeighting === true && m.epistemic_status === "observation"
|
|
48
|
+
? "[verified] "
|
|
49
|
+
: "";
|
|
50
|
+
const title = `${m.title}(重要性 ${m.importance})`;
|
|
51
|
+
let content = String(m.content ?? "");
|
|
52
|
+
if (content.length > MAX_CONTENT) content = `${content.slice(0, MAX_CONTENT)}…`;
|
|
53
|
+
const full = `- [${m.type}] ${verified}${title}:${content}`;
|
|
54
|
+
if (budget - full.length >= 0) {
|
|
55
|
+
lines.push(full);
|
|
56
|
+
budget -= full.length;
|
|
57
|
+
} else {
|
|
58
|
+
lines.push(`- [${m.type}] ${verified}${title}`);
|
|
59
|
+
}
|
|
10
60
|
}
|
|
11
61
|
return lines.join("\n");
|
|
12
62
|
}
|
|
13
63
|
|
|
64
|
+
// Bug4: the system-prompt render is synchronous, so the semantic query vector
|
|
65
|
+
// must be prefetched asynchronously and cached for the next assembly. The
|
|
66
|
+
// first render after a new user message may still fall back to the rule-based
|
|
67
|
+
// pick; later assemblies in the same session reuse the cached vector. Bounded
|
|
68
|
+
// cache (cap 8, drop oldest) so a long session never grows it unbounded.
|
|
69
|
+
const QUERY_VECTOR_CACHE_MAX = 8;
|
|
70
|
+
const queryVectorCache = new Map();
|
|
71
|
+
let lastPrefetched = "";
|
|
72
|
+
|
|
73
|
+
function prefetchQueryVector(query) {
|
|
74
|
+
if (!query || query === lastPrefetched || queryVectorCache.has(query)) return;
|
|
75
|
+
lastPrefetched = query;
|
|
76
|
+
service.embedQuery(query).then((vec) => {
|
|
77
|
+
if (Array.isArray(vec) && vec.length) {
|
|
78
|
+
queryVectorCache.set(query, vec);
|
|
79
|
+
if (queryVectorCache.size > QUERY_VECTOR_CACHE_MAX) {
|
|
80
|
+
queryVectorCache.delete(queryVectorCache.keys().next().value);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}).catch(() => { /* prefetch is best-effort */ });
|
|
84
|
+
}
|
|
85
|
+
|
|
14
86
|
// User profile + rules: injected ahead of the memory block because they are
|
|
15
87
|
// always-relevant instructions the agent should follow every turn.
|
|
16
88
|
function renderUserSettings() {
|
|
@@ -27,8 +99,15 @@ export function createInjector(ctx, service, settings, config) {
|
|
|
27
99
|
ctx.systemPrompt.context({
|
|
28
100
|
name: "memory",
|
|
29
101
|
order: 90,
|
|
30
|
-
text: () => {
|
|
31
|
-
|
|
102
|
+
text: (ctx) => {
|
|
103
|
+
// Bug4: pass the latest user query so injection prefers semantically
|
|
104
|
+
// relevant memories; lastUserQuery is best-effort (empty → legacy).
|
|
105
|
+
// The query vector is prefetched asynchronously (cached) because the
|
|
106
|
+
// render itself must stay synchronous.
|
|
107
|
+
const query = lastUserQuery(ctx);
|
|
108
|
+
if (query) prefetchQueryVector(query);
|
|
109
|
+
const queryVector = queryVectorCache.get(query);
|
|
110
|
+
const candidates = service.injectCandidates({ query, queryVector, maxItems, threshold });
|
|
32
111
|
return render(candidates);
|
|
33
112
|
}
|
|
34
113
|
}),
|
|
@@ -40,6 +119,7 @@ export function createInjector(ctx, service, settings, config) {
|
|
|
40
119
|
];
|
|
41
120
|
|
|
42
121
|
return () => {
|
|
122
|
+
queryVectorCache.clear();
|
|
43
123
|
for (const dispose of disposers) {
|
|
44
124
|
if (typeof dispose === "function") dispose();
|
|
45
125
|
}
|