@amemhq/core 1.1.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/{chunk-B2NS7WAM.js → chunk-XEMQZNLD.js} +257 -57
- package/dist/chunk-XEMQZNLD.js.map +1 -0
- package/dist/cli-migrate.cjs +259 -52
- package/dist/cli-migrate.cjs.map +1 -1
- package/dist/cli-migrate.d.cts +21 -7
- package/dist/cli-migrate.d.ts +21 -7
- package/dist/cli-migrate.js +128 -43
- package/dist/cli-migrate.js.map +1 -1
- package/dist/index.cjs +256 -55
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +167 -29
- package/dist/index.d.ts +167 -29
- package/dist/index.js +11 -1
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/dist/chunk-B2NS7WAM.js.map +0 -1
package/README.md
CHANGED
|
@@ -17,7 +17,7 @@ Unlike a flat vector store, A-MEM maintains memory as a living, self-evolving se
|
|
|
17
17
|
1. **Note Construction** — an LLM extracts keywords, tags, and a context summary; categorizes the note; and classifies it as `memory` (episodic) or `knowledge` (durable), extracting 1–5 `topics` for knowledge notes.
|
|
18
18
|
2. **Link Generation** — retrieves top-6 candidates; the LLM judges whether to link bidirectionally (similarity > 0.3).
|
|
19
19
|
3. **Memory Evolution** — up to 3 linked notes have their attributes evolved from the new context, possibly triggering further links.
|
|
20
|
-
4. **Hybrid Retrieval** — fuses dense vectors (Transformers.js `
|
|
20
|
+
4. **Hybrid Retrieval** — fuses dense vectors (Transformers.js `bge-m3`, 1024-dim) and BM25 via Reciprocal Rank Fusion (RRF), boosted by retrieval heat.
|
|
21
21
|
5. **2-hop BFS Graph Expansion** — after RRF top-K, BFS walks the link graph up to 2 hops, admitting up to 8 graph-connected notes that pass an embedding relevance gate (cos-sim ≥ 0.25). This is the key advantage over flat vector systems.
|
|
22
22
|
|
|
23
23
|
## Features
|
|
@@ -47,7 +47,7 @@ host (OpenClaw plugin / amem-api / game agent)
|
|
|
47
47
|
▼
|
|
48
48
|
@amemhq/core (TypeScript)
|
|
49
49
|
├── LLM (Anthropic) note construction · link judgment · CRUD · evolution
|
|
50
|
-
├── Transformers.js (ONNX)
|
|
50
|
+
├── Transformers.js (ONNX) local embeddings + Jieba BM25
|
|
51
51
|
└── Qdrant :6333 vector store · owner/readers/writers · agent_id isolation
|
|
52
52
|
```
|
|
53
53
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
// src/auth.ts
|
|
2
|
+
var SYSTEM_ACTOR = "__amem_system__";
|
|
2
3
|
function canWrite(note, callerAgentId) {
|
|
3
4
|
return note.owner === callerAgentId || note.writers.includes(callerAgentId) || note.writers.includes("*");
|
|
4
5
|
}
|
|
@@ -11,9 +12,22 @@ var pipeline = null;
|
|
|
11
12
|
var extractor = null;
|
|
12
13
|
var loadedKey = null;
|
|
13
14
|
var cachedDim = null;
|
|
14
|
-
var DEFAULT_EMBEDDING_MODEL = "Xenova/
|
|
15
|
+
var DEFAULT_EMBEDDING_MODEL = "Xenova/bge-m3";
|
|
16
|
+
var LEGACY_DEFAULT_EMBEDDING_MODEL = "Xenova/paraphrase-multilingual-MiniLM-L12-v2";
|
|
17
|
+
var LEGACY_DEFAULT_DIM = 384;
|
|
18
|
+
var pinnedModel = null;
|
|
19
|
+
function pinEmbeddingModel(model) {
|
|
20
|
+
if (pinnedModel === model) return;
|
|
21
|
+
pinnedModel = model;
|
|
22
|
+
extractor = null;
|
|
23
|
+
loadedKey = null;
|
|
24
|
+
cachedDim = null;
|
|
25
|
+
}
|
|
26
|
+
function getPinnedEmbeddingModel() {
|
|
27
|
+
return pinnedModel;
|
|
28
|
+
}
|
|
15
29
|
function getEmbeddingModel() {
|
|
16
|
-
return process.env.AMEM_EMBED_MODEL?.trim() || DEFAULT_EMBEDDING_MODEL;
|
|
30
|
+
return process.env.AMEM_EMBED_MODEL?.trim() || pinnedModel || DEFAULT_EMBEDDING_MODEL;
|
|
17
31
|
}
|
|
18
32
|
var CLS_POOLED_MODELS = /* @__PURE__ */ new Set([
|
|
19
33
|
"bge-m3",
|
|
@@ -40,6 +54,27 @@ function getEmbeddingDevice() {
|
|
|
40
54
|
function getEmbeddingDtype() {
|
|
41
55
|
return process.env.AMEM_EMBED_DTYPE?.trim() || void 0;
|
|
42
56
|
}
|
|
57
|
+
function applyModelPaths(env) {
|
|
58
|
+
const cache = process.env.AMEM_MODEL_CACHE?.trim();
|
|
59
|
+
if (cache) env.cacheDir = cache;
|
|
60
|
+
const local = process.env.AMEM_MODEL_DIR?.trim();
|
|
61
|
+
if (local) {
|
|
62
|
+
env.localModelPath = local;
|
|
63
|
+
env.allowLocalModels = true;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
function makeProgressReporter() {
|
|
67
|
+
const lastPct = /* @__PURE__ */ new Map();
|
|
68
|
+
return (e) => {
|
|
69
|
+
if (e.status !== "progress" || !e.file || typeof e.progress !== "number") return;
|
|
70
|
+
if (!e.file.endsWith(".onnx") && !e.file.endsWith(".onnx_data")) return;
|
|
71
|
+
const pct = Math.floor(e.progress / 10) * 10;
|
|
72
|
+
if (lastPct.get(e.file) === pct) return;
|
|
73
|
+
lastPct.set(e.file, pct);
|
|
74
|
+
const size = e.total ? ` of ${(e.total / 1e9).toFixed(2)} GB` : "";
|
|
75
|
+
console.log(`[amem] downloading ${e.file}: ${pct}%${size}`);
|
|
76
|
+
};
|
|
77
|
+
}
|
|
43
78
|
function extractorKey() {
|
|
44
79
|
return `${getEmbeddingModel()}|${getEmbeddingDevice() ?? ""}|${getEmbeddingDtype() ?? ""}`;
|
|
45
80
|
}
|
|
@@ -49,11 +84,13 @@ async function getExtractor() {
|
|
|
49
84
|
if (!pipeline) {
|
|
50
85
|
const mod = await import("@huggingface/transformers");
|
|
51
86
|
pipeline = mod.pipeline;
|
|
87
|
+
applyModelPaths(mod.env);
|
|
52
88
|
}
|
|
53
89
|
const device = getEmbeddingDevice();
|
|
54
90
|
const dtype = getEmbeddingDtype();
|
|
55
91
|
extractor = await pipeline("feature-extraction", getEmbeddingModel(), {
|
|
56
92
|
revision: "main",
|
|
93
|
+
progress_callback: makeProgressReporter(),
|
|
57
94
|
// Omitted entirely when unset, so an unconfigured install gets exactly the
|
|
58
95
|
// library defaults it got before these existed.
|
|
59
96
|
...device ? { device } : {},
|
|
@@ -149,12 +186,12 @@ Either set AMEM_EMBED_MODEL back to the model this collection was built with, or
|
|
|
149
186
|
model;
|
|
150
187
|
};
|
|
151
188
|
function migrationHint(collection, targetModel) {
|
|
152
|
-
return `migrate
|
|
189
|
+
return `migrate onto it:
|
|
153
190
|
|
|
154
191
|
AMEM_EMBED_MODEL=${targetModel} \\
|
|
155
|
-
npx --package=@amemhq/core amem-migrate --
|
|
192
|
+
npx --package=@amemhq/core amem-migrate --from-collection ${collection}
|
|
156
193
|
|
|
157
|
-
That
|
|
194
|
+
That only reports; it takes --apply to write anything, and "${collection}" is read either way. The new store ends up behind the name you already use, so there is nothing to change in your config afterwards. See https://amem.owo.lc/reference/embedding-models.`;
|
|
158
195
|
}
|
|
159
196
|
var EmbeddingModelMismatchError = class extends Error {
|
|
160
197
|
constructor(collection, collectionModel, configuredModel) {
|
|
@@ -171,6 +208,25 @@ Either set AMEM_EMBED_MODEL back to "${collectionModel}", or ` + migrationHint(c
|
|
|
171
208
|
collectionModel;
|
|
172
209
|
configuredModel;
|
|
173
210
|
};
|
|
211
|
+
var MixedEmbeddingModelsError = class extends Error {
|
|
212
|
+
constructor(collection, wanted, inUse) {
|
|
213
|
+
super(
|
|
214
|
+
`Collection "${collection}" was built with "${wanted}", but this process is already embedding with "${inUse}" for another collection. One process can only use one model.
|
|
215
|
+
Migrate the remaining collections so they all agree:
|
|
216
|
+
|
|
217
|
+
npx --package=@amemhq/core amem-migrate --from-collection ${collection}
|
|
218
|
+
|
|
219
|
+
Or set AMEM_EMBED_MODEL to pin every collection to one model, which is only correct if they really were all built with it.`
|
|
220
|
+
);
|
|
221
|
+
this.collection = collection;
|
|
222
|
+
this.wanted = wanted;
|
|
223
|
+
this.inUse = inUse;
|
|
224
|
+
this.name = "MixedEmbeddingModelsError";
|
|
225
|
+
}
|
|
226
|
+
collection;
|
|
227
|
+
wanted;
|
|
228
|
+
inUse;
|
|
229
|
+
};
|
|
174
230
|
async function recordCollectionModel(collection, model) {
|
|
175
231
|
try {
|
|
176
232
|
await qdrant("PATCH", `/collections/${collection}`, { metadata: { embedding_model: model } });
|
|
@@ -213,18 +269,38 @@ async function ensureCollection(collectionName) {
|
|
|
213
269
|
}
|
|
214
270
|
if (existing) {
|
|
215
271
|
const collectionDim = existing.config?.params?.vectors?.size;
|
|
272
|
+
const recorded = existing.config?.metadata?.embedding_model;
|
|
273
|
+
const explicit = process.env.AMEM_EMBED_MODEL?.trim();
|
|
274
|
+
const inferLegacy = !explicit && recorded === void 0 && collectionDim === LEGACY_DEFAULT_DIM;
|
|
275
|
+
if (!explicit) {
|
|
276
|
+
const wanted = (
|
|
277
|
+
// The collection says what built it, which outranks whatever the shipped
|
|
278
|
+
// default happens to be today. This is what keeps changing the default
|
|
279
|
+
// from breaking every install that already has data.
|
|
280
|
+
typeof recorded === "string" ? recorded : inferLegacy ? LEGACY_DEFAULT_EMBEDDING_MODEL : DEFAULT_EMBEDDING_MODEL
|
|
281
|
+
);
|
|
282
|
+
const inUse = getPinnedEmbeddingModel();
|
|
283
|
+
if (inUse !== null && inUse !== wanted) throw new MixedEmbeddingModelsError(col, wanted, inUse);
|
|
284
|
+
pinEmbeddingModel(wanted);
|
|
285
|
+
if (wanted === LEGACY_DEFAULT_EMBEDDING_MODEL) {
|
|
286
|
+
console.warn(
|
|
287
|
+
`[amem] "${col}" is on ${LEGACY_DEFAULT_EMBEDDING_MODEL} (${LEGACY_DEFAULT_DIM}-dim).
|
|
288
|
+
[amem] ${DEFAULT_EMBEDDING_MODEL} reads 8192 tokens where that one stops at 128, so anything longer is being truncated before it reaches the vector.
|
|
289
|
+
[amem] To move: npx --package=@amemhq/core amem-migrate --from-collection ${col}`
|
|
290
|
+
);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
216
293
|
if (typeof collectionDim === "number") {
|
|
217
294
|
const modelDim = await getEmbeddingDim();
|
|
218
295
|
if (collectionDim !== modelDim) {
|
|
219
296
|
throw new EmbeddingDimensionMismatchError(col, collectionDim, modelDim, getEmbeddingModel());
|
|
220
297
|
}
|
|
221
298
|
}
|
|
222
|
-
const recorded = existing.config?.metadata?.embedding_model;
|
|
223
299
|
const current = getEmbeddingModel();
|
|
224
300
|
if (typeof recorded === "string" && recorded !== current) {
|
|
225
301
|
throw new EmbeddingModelMismatchError(col, recorded, current);
|
|
226
302
|
}
|
|
227
|
-
if (recorded === void 0) {
|
|
303
|
+
if (recorded === void 0 && !inferLegacy) {
|
|
228
304
|
await recordCollectionModel(col, current);
|
|
229
305
|
}
|
|
230
306
|
markReady();
|
|
@@ -238,7 +314,9 @@ async function ensureCollection(collectionName) {
|
|
|
238
314
|
} catch (err) {
|
|
239
315
|
if (!(err instanceof Error) || !err.message.includes("already exists")) throw err;
|
|
240
316
|
}
|
|
241
|
-
|
|
317
|
+
const created = getEmbeddingModel();
|
|
318
|
+
await recordCollectionModel(col, created);
|
|
319
|
+
if (!process.env.AMEM_EMBED_MODEL?.trim()) pinEmbeddingModel(created);
|
|
242
320
|
await qdrant("PUT", `/collections/${col}/index`, {
|
|
243
321
|
field_name: "agent_id",
|
|
244
322
|
field_schema: "keyword"
|
|
@@ -292,6 +370,47 @@ async function createCollectionRaw(collection, size) {
|
|
|
292
370
|
async function upsertPointsRaw(collection, points) {
|
|
293
371
|
await qdrant("PUT", `/collections/${collection}/points?wait=true`, { points });
|
|
294
372
|
}
|
|
373
|
+
async function scrollIdsRaw(collection, limit = 1e4) {
|
|
374
|
+
const ids = /* @__PURE__ */ new Set();
|
|
375
|
+
let offset = void 0;
|
|
376
|
+
for (; ; ) {
|
|
377
|
+
const body = { with_payload: false, with_vector: false, limit };
|
|
378
|
+
if (offset !== void 0 && offset !== null) body.offset = offset;
|
|
379
|
+
const res = await qdrant("POST", `/collections/${collection}/points/scroll`, body);
|
|
380
|
+
for (const p of res.points) ids.add(String(p.id));
|
|
381
|
+
offset = res.next_page_offset;
|
|
382
|
+
if (offset === void 0 || offset === null || res.points.length === 0) break;
|
|
383
|
+
}
|
|
384
|
+
return ids;
|
|
385
|
+
}
|
|
386
|
+
async function deleteCollectionRaw(collection) {
|
|
387
|
+
await qdrant("DELETE", `/collections/${collection}`);
|
|
388
|
+
}
|
|
389
|
+
async function snapshotCollectionRaw(collection) {
|
|
390
|
+
const r = await qdrant("POST", `/collections/${collection}/snapshots`);
|
|
391
|
+
return { name: r.name, size: r.size ?? 0 };
|
|
392
|
+
}
|
|
393
|
+
async function resolveAliasRaw(alias) {
|
|
394
|
+
try {
|
|
395
|
+
const res = await qdrant("GET", `/aliases`);
|
|
396
|
+
return res.aliases.find((a) => a.alias_name === alias)?.collection_name ?? null;
|
|
397
|
+
} catch {
|
|
398
|
+
return null;
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
async function createAliasRaw(alias, collection) {
|
|
402
|
+
await qdrant("POST", `/collections/aliases`, {
|
|
403
|
+
actions: [{ create_alias: { collection_name: collection, alias_name: alias } }]
|
|
404
|
+
});
|
|
405
|
+
}
|
|
406
|
+
async function setAliasRaw(alias, collection) {
|
|
407
|
+
await qdrant("POST", `/collections/aliases`, {
|
|
408
|
+
actions: [
|
|
409
|
+
{ delete_alias: { alias_name: alias } },
|
|
410
|
+
{ create_alias: { collection_name: collection, alias_name: alias } }
|
|
411
|
+
]
|
|
412
|
+
});
|
|
413
|
+
}
|
|
295
414
|
function noteToPoint(note) {
|
|
296
415
|
return {
|
|
297
416
|
id: note.id,
|
|
@@ -436,12 +555,15 @@ function makeCrud(collectionName, modeBIsolated = false) {
|
|
|
436
555
|
},
|
|
437
556
|
/**
|
|
438
557
|
* Story 36: this is the one read that bypasses the agent filter — it fetches
|
|
439
|
-
* straight by UUID.
|
|
440
|
-
*
|
|
441
|
-
*
|
|
442
|
-
*
|
|
558
|
+
* straight by UUID. An unreadable note comes back as `null`, indistinguishable
|
|
559
|
+
* from missing, so nothing leaks and callers already handle it.
|
|
560
|
+
*
|
|
561
|
+
* `reader` is required. It used to be optional, and omitting it skipped the
|
|
562
|
+
* check — which meant the safe behaviour was the one you had to remember to
|
|
563
|
+
* ask for. Pass `SYSTEM_ACTOR` to read as the engine itself; that reads as a
|
|
564
|
+
* deliberate act at the call site, where an absent argument did not.
|
|
443
565
|
*/
|
|
444
|
-
async getNote(id,
|
|
566
|
+
async getNote(id, reader) {
|
|
445
567
|
await ensureCollection(col);
|
|
446
568
|
try {
|
|
447
569
|
const result = await qdrant("POST", `/collections/${col}/points`, {
|
|
@@ -451,7 +573,7 @@ function makeCrud(collectionName, modeBIsolated = false) {
|
|
|
451
573
|
});
|
|
452
574
|
if (!result.length) return null;
|
|
453
575
|
const note = pointToNote(result[0]);
|
|
454
|
-
if (
|
|
576
|
+
if (reader !== SYSTEM_ACTOR && !canRead(note, reader)) return null;
|
|
455
577
|
return note;
|
|
456
578
|
} catch {
|
|
457
579
|
return null;
|
|
@@ -489,19 +611,22 @@ function makeCrud(collectionName, modeBIsolated = false) {
|
|
|
489
611
|
return pointToNote(result.points[0]);
|
|
490
612
|
},
|
|
491
613
|
/**
|
|
492
|
-
* Story 33:
|
|
493
|
-
*
|
|
494
|
-
*
|
|
495
|
-
*
|
|
496
|
-
*
|
|
497
|
-
*
|
|
614
|
+
* Story 33: enforces the writers policy. Returns false — without writing —
|
|
615
|
+
* when the caller may not write. This fetch-then-check path exists for callers
|
|
616
|
+
* that only have an id (the plugin's CRUD hook); callers already holding the
|
|
617
|
+
* note can check `canWrite` themselves and skip a round trip.
|
|
618
|
+
*
|
|
619
|
+
* `caller` is required. Pass `SYSTEM_ACTOR` for the engine's own maintenance
|
|
620
|
+
* writes. The self-read below is a genuine `SYSTEM_ACTOR` case: it fetches the
|
|
621
|
+
* note in order to decide whether the caller may write it, and gating that
|
|
622
|
+
* fetch on the same policy it exists to evaluate would be circular.
|
|
498
623
|
*/
|
|
499
|
-
async updateNoteContent(id, content, embedding, hash,
|
|
624
|
+
async updateNoteContent(id, content, embedding, hash, caller) {
|
|
500
625
|
await ensureCollection(col);
|
|
501
626
|
let existing = null;
|
|
502
|
-
if (
|
|
503
|
-
existing = await this.getNote(id);
|
|
504
|
-
if (existing && !canWrite(existing,
|
|
627
|
+
if (caller !== SYSTEM_ACTOR) {
|
|
628
|
+
existing = await this.getNote(id, SYSTEM_ACTOR);
|
|
629
|
+
if (existing && !canWrite(existing, caller)) return false;
|
|
505
630
|
}
|
|
506
631
|
await qdrant("PUT", `/collections/${col}/points/vectors?wait=true`, {
|
|
507
632
|
points: [{ id, vector: embedding }]
|
|
@@ -589,11 +714,11 @@ function makeCrud(collectionName, modeBIsolated = false) {
|
|
|
589
714
|
});
|
|
590
715
|
},
|
|
591
716
|
/** Story 33: see `updateNoteContent` — returns false, unwritten, when denied. */
|
|
592
|
-
async invalidateNote(id,
|
|
717
|
+
async invalidateNote(id, caller) {
|
|
593
718
|
await ensureCollection(col);
|
|
594
|
-
if (
|
|
595
|
-
const existing = await this.getNote(id);
|
|
596
|
-
if (existing && !canWrite(existing,
|
|
719
|
+
if (caller !== SYSTEM_ACTOR) {
|
|
720
|
+
const existing = await this.getNote(id, SYSTEM_ACTOR);
|
|
721
|
+
if (existing && !canWrite(existing, caller)) return false;
|
|
597
722
|
}
|
|
598
723
|
await qdrant("POST", `/collections/${col}/points/payload?wait=true`, {
|
|
599
724
|
payload: { is_active: false },
|
|
@@ -659,8 +784,8 @@ function makeCrud(collectionName, modeBIsolated = false) {
|
|
|
659
784
|
function createStorageContext(collectionName, modeBIsolated = false) {
|
|
660
785
|
return makeCrud(collectionName || getCollection(), modeBIsolated);
|
|
661
786
|
}
|
|
662
|
-
async function getNote(id,
|
|
663
|
-
return makeCrud(getCollection()).getNote(id,
|
|
787
|
+
async function getNote(id, reader) {
|
|
788
|
+
return makeCrud(getCollection()).getNote(id, reader);
|
|
664
789
|
}
|
|
665
790
|
async function updateNote(note) {
|
|
666
791
|
return makeCrud(getCollection()).updateNote(note);
|
|
@@ -671,8 +796,8 @@ async function listNotes(agentId, subject) {
|
|
|
671
796
|
async function deleteNote(id) {
|
|
672
797
|
return makeCrud(getCollection()).deleteNote(id);
|
|
673
798
|
}
|
|
674
|
-
async function invalidateNote(id,
|
|
675
|
-
return makeCrud(getCollection()).invalidateNote(id,
|
|
799
|
+
async function invalidateNote(id, caller) {
|
|
800
|
+
return makeCrud(getCollection()).invalidateNote(id, caller);
|
|
676
801
|
}
|
|
677
802
|
async function patchNotePayload(id, fields) {
|
|
678
803
|
return makeCrud(getCollection()).patchNotePayload(id, fields);
|
|
@@ -1441,7 +1566,7 @@ async function addMemory(content, agentId = "main", opts) {
|
|
|
1441
1566
|
if (topMatch.length > 0 && topMatch[0].score >= 0.85) {
|
|
1442
1567
|
if (canWrite(topMatch[0].note, agentId)) {
|
|
1443
1568
|
console.log(`[add] dedup: high-sim match (sim=${topMatch[0].score.toFixed(3)}), updating existing`);
|
|
1444
|
-
await ctx.updateNoteContent(topMatch[0].note.id, content, embedding, hash);
|
|
1569
|
+
await ctx.updateNoteContent(topMatch[0].note.id, content, embedding, hash, agentId);
|
|
1445
1570
|
return topMatch[0].note.id;
|
|
1446
1571
|
}
|
|
1447
1572
|
console.log(
|
|
@@ -1513,7 +1638,7 @@ async function addMemory(content, agentId = "main", opts) {
|
|
|
1513
1638
|
note.links = linkedIds;
|
|
1514
1639
|
await ctx.updateNote(note);
|
|
1515
1640
|
for (const lid of linkedIds) {
|
|
1516
|
-
const linked = await ctx.getNote(lid);
|
|
1641
|
+
const linked = await ctx.getNote(lid, agentId);
|
|
1517
1642
|
if (linked && !linked.links.includes(note.id)) {
|
|
1518
1643
|
if (!canWrite(linked, agentId)) {
|
|
1519
1644
|
console.log(`[link] back-link into ${lid.slice(0, 8)} skipped \u2014 not writable by ${logSafe(agentId)}`);
|
|
@@ -1526,7 +1651,7 @@ async function addMemory(content, agentId = "main", opts) {
|
|
|
1526
1651
|
if (shouldRunEvolution()) {
|
|
1527
1652
|
console.log(` [evo] threshold reached, running evolution for ${Math.min(linkedIds.length, 3)} linked notes`);
|
|
1528
1653
|
for (const lid of linkedIds.slice(0, 3)) {
|
|
1529
|
-
const linked = await ctx.getNote(lid);
|
|
1654
|
+
const linked = await ctx.getNote(lid, agentId);
|
|
1530
1655
|
if (!linked) continue;
|
|
1531
1656
|
if (!canWrite(linked, agentId)) {
|
|
1532
1657
|
console.log(` [evo] skipping ${lid.slice(0, 8)} \u2014 not writable by ${logSafe(agentId)}`);
|
|
@@ -1679,7 +1804,7 @@ async function searchMemory(query, topK = 5, agentId = "main", opts) {
|
|
|
1679
1804
|
const allNotes = await ctx.listNotes(agentId, subject);
|
|
1680
1805
|
const bm25State = buildBM25(allNotes);
|
|
1681
1806
|
const queryTokens = simpleTokenize(query);
|
|
1682
|
-
const bm25Ranked = bm25Score(bm25State, queryTokens).slice(0, n);
|
|
1807
|
+
const bm25Ranked = bm25Score(bm25State, queryTokens).filter(([, score]) => score > 0).slice(0, n);
|
|
1683
1808
|
const merged = rrfMerge(
|
|
1684
1809
|
embResults.map((r) => r.note.id),
|
|
1685
1810
|
bm25Ranked.map((r) => r[0])
|
|
@@ -1702,6 +1827,7 @@ async function searchMemory(query, topK = 5, agentId = "main", opts) {
|
|
|
1702
1827
|
const visitedIds = new Set(topIds);
|
|
1703
1828
|
const bfsQueue = useBfs ? topIds.map((id) => ({ id, hop: 0 })) : [];
|
|
1704
1829
|
const bfsExtra = [];
|
|
1830
|
+
const bfsSimMap = /* @__PURE__ */ new Map();
|
|
1705
1831
|
while (bfsQueue.length > 0 && bfsExtra.length < BFS_MAX_EXPAND) {
|
|
1706
1832
|
const item = bfsQueue.shift();
|
|
1707
1833
|
if (item.hop >= BFS_MAX_HOPS) continue;
|
|
@@ -1712,10 +1838,9 @@ async function searchMemory(query, topK = 5, agentId = "main", opts) {
|
|
|
1712
1838
|
visitedIds.add(linkedId);
|
|
1713
1839
|
const linked = noteMap.get(linkedId);
|
|
1714
1840
|
if (!linked || linked.is_active === false) continue;
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
}
|
|
1841
|
+
const sim = cosineSimilarity(queryEmbedding, linked.embedding);
|
|
1842
|
+
if (bfsSimThreshold > 0 && sim < bfsSimThreshold) continue;
|
|
1843
|
+
bfsSimMap.set(linkedId, sim);
|
|
1719
1844
|
bfsExtra.push(linkedId);
|
|
1720
1845
|
bfsQueue.push({ id: linkedId, hop: item.hop + 1 });
|
|
1721
1846
|
if (bfsExtra.length >= BFS_MAX_EXPAND) break;
|
|
@@ -1731,7 +1856,11 @@ async function searchMemory(query, topK = 5, agentId = "main", opts) {
|
|
|
1731
1856
|
const embSimMap = new Map(embResults.map((r) => [r.note.id, r.score]));
|
|
1732
1857
|
const rrfMap = new Map(boostedMerged.map(([id, score]) => [id, score]));
|
|
1733
1858
|
const results = [];
|
|
1734
|
-
|
|
1859
|
+
const ordered = [
|
|
1860
|
+
...filteredTopIds.map((id) => [id, "match"]),
|
|
1861
|
+
...bfsExtra.map((id) => [id, "link"])
|
|
1862
|
+
];
|
|
1863
|
+
for (const [id, via] of ordered) {
|
|
1735
1864
|
const note = noteMap.get(id);
|
|
1736
1865
|
if (!note) continue;
|
|
1737
1866
|
results.push({
|
|
@@ -1742,8 +1871,9 @@ async function searchMemory(query, topK = 5, agentId = "main", opts) {
|
|
|
1742
1871
|
keywords: note.keywords,
|
|
1743
1872
|
links: note.links,
|
|
1744
1873
|
timestamp: note.timestamp,
|
|
1745
|
-
similarity: embSimMap.get(id) ?? 0,
|
|
1874
|
+
similarity: embSimMap.get(id) ?? bfsSimMap.get(id) ?? 0,
|
|
1746
1875
|
rrf: rrfMap.get(id) ?? 0,
|
|
1876
|
+
via,
|
|
1747
1877
|
topics: note.topics ?? [],
|
|
1748
1878
|
note_type: note.note_type ?? "memory"
|
|
1749
1879
|
});
|
|
@@ -1800,7 +1930,7 @@ async function mergeSimilarNotes(agentId, storageCtx) {
|
|
|
1800
1930
|
const mergedContent = judgment.mergedContent || pendingNote.content;
|
|
1801
1931
|
const newEmbedding = await encode(buildEmbedText({ ...bestNeighbor, content: mergedContent }));
|
|
1802
1932
|
const newHash = createHash("md5").update(mergedContent).digest("hex");
|
|
1803
|
-
await ctx.updateNoteContent(bestNeighbor.id, mergedContent, newEmbedding, newHash);
|
|
1933
|
+
await ctx.updateNoteContent(bestNeighbor.id, mergedContent, newEmbedding, newHash, agentId);
|
|
1804
1934
|
await ctx.patchNotePayload(bestNeighbor.id, {
|
|
1805
1935
|
evolution_history: JSON.stringify(oldHistory),
|
|
1806
1936
|
evolution_type: "EVOLVE"
|
|
@@ -1824,7 +1954,7 @@ async function mergeSimilarNotes(agentId, storageCtx) {
|
|
|
1824
1954
|
const mergedContent = judgment.mergedContent || `${bestNeighbor.content}\uFF1B${pendingNote.content}`;
|
|
1825
1955
|
const newEmbedding = await encode(buildEmbedText({ ...bestNeighbor, content: mergedContent }));
|
|
1826
1956
|
const newHash = createHash("md5").update(mergedContent).digest("hex");
|
|
1827
|
-
await ctx.updateNoteContent(bestNeighbor.id, mergedContent, newEmbedding, newHash);
|
|
1957
|
+
await ctx.updateNoteContent(bestNeighbor.id, mergedContent, newEmbedding, newHash, agentId);
|
|
1828
1958
|
await ctx.patchNotePayload(bestNeighbor.id, {
|
|
1829
1959
|
evolution_history: JSON.stringify(oldHistory),
|
|
1830
1960
|
evolution_type: "EXPAND"
|
|
@@ -1864,7 +1994,7 @@ async function mergeSimilarNotes(agentId, storageCtx) {
|
|
|
1864
1994
|
const [keepNote, dropNote] = noteA.content.length >= noteB.content.length ? [noteA, noteB] : [noteB, noteA];
|
|
1865
1995
|
const newEmbedding = await encode(result.merged);
|
|
1866
1996
|
const newHash = createHash("md5").update(result.merged).digest("hex");
|
|
1867
|
-
await ctx.updateNoteContent(keepNote.id, result.merged, newEmbedding, newHash);
|
|
1997
|
+
await ctx.updateNoteContent(keepNote.id, result.merged, newEmbedding, newHash, agentId);
|
|
1868
1998
|
await ctx.deleteNote(dropNote.id);
|
|
1869
1999
|
deletedIds.add(dropNote.id);
|
|
1870
2000
|
mergedCount++;
|
|
@@ -1973,7 +2103,7 @@ async function consolidateMemories(agentId, logger, storageCtx) {
|
|
|
1973
2103
|
action: "consolidate"
|
|
1974
2104
|
});
|
|
1975
2105
|
await ctx.updateNote(keepNote);
|
|
1976
|
-
await ctx.invalidateNote(dropNote.id);
|
|
2106
|
+
await ctx.invalidateNote(dropNote.id, agentId);
|
|
1977
2107
|
await ctx.replaceLinkReferences(dropNote.id, keepNote.id, agentId);
|
|
1978
2108
|
logMergeToFile(keepNote.id, dropNote.id, keepNote.content);
|
|
1979
2109
|
processedIds.add(keepNote.id);
|
|
@@ -2086,8 +2216,19 @@ async function migrateCollection(opts) {
|
|
|
2086
2216
|
);
|
|
2087
2217
|
if (dryRun) {
|
|
2088
2218
|
log("[migrate] dry run \u2014 nothing written. Pass dryRun: false to apply.");
|
|
2089
|
-
return {
|
|
2219
|
+
return {
|
|
2220
|
+
total: notes.length,
|
|
2221
|
+
missingDerived,
|
|
2222
|
+
refreshed: 0,
|
|
2223
|
+
migrated: 0,
|
|
2224
|
+
skipped: 0,
|
|
2225
|
+
sourceDim,
|
|
2226
|
+
targetDim,
|
|
2227
|
+
model,
|
|
2228
|
+
dryRun: true
|
|
2229
|
+
};
|
|
2090
2230
|
}
|
|
2231
|
+
let alreadyDone = /* @__PURE__ */ new Set();
|
|
2091
2232
|
const existingTargetDim = await collectionDimRaw(to);
|
|
2092
2233
|
if (existingTargetDim === null) {
|
|
2093
2234
|
await createCollectionRaw(to, targetDim);
|
|
@@ -2096,9 +2237,17 @@ async function migrateCollection(opts) {
|
|
|
2096
2237
|
if (existingTargetDim !== targetDim) {
|
|
2097
2238
|
throw new Error(`migrate: target "${to}" exists at ${existingTargetDim}d but the model produces ${targetDim}d`);
|
|
2098
2239
|
}
|
|
2099
|
-
const
|
|
2100
|
-
if (
|
|
2101
|
-
|
|
2240
|
+
const present = await scrollIdsRaw(to);
|
|
2241
|
+
if (present.size > 0) {
|
|
2242
|
+
const sourceIds = new Set(notes.map((n) => n.id));
|
|
2243
|
+
const foreign = [...present].filter((id) => !sourceIds.has(id));
|
|
2244
|
+
if (foreign.length > 0) {
|
|
2245
|
+
throw new Error(
|
|
2246
|
+
`migrate: target "${to}" holds ${foreign.length} point(s) that are not in "${from}" (e.g. ${foreign[0]}). That is not an interrupted migration \u2014 use a different target.`
|
|
2247
|
+
);
|
|
2248
|
+
}
|
|
2249
|
+
alreadyDone = present;
|
|
2250
|
+
log(`[migrate] resuming: ${present.size} of ${notes.length} already in ${to}`);
|
|
2102
2251
|
}
|
|
2103
2252
|
}
|
|
2104
2253
|
let refreshed = 0;
|
|
@@ -2112,6 +2261,7 @@ async function migrateCollection(opts) {
|
|
|
2112
2261
|
buffer = [];
|
|
2113
2262
|
};
|
|
2114
2263
|
for (const note of notes) {
|
|
2264
|
+
if (alreadyDone.has(note.id)) continue;
|
|
2115
2265
|
if (refreshFields && missingDerivedFields(note)) {
|
|
2116
2266
|
try {
|
|
2117
2267
|
const built = await llmConstructNote(note.content);
|
|
@@ -2127,7 +2277,7 @@ async function migrateCollection(opts) {
|
|
|
2127
2277
|
buffer.push(point);
|
|
2128
2278
|
if (buffer.length >= BATCH) {
|
|
2129
2279
|
await flush();
|
|
2130
|
-
log(`[migrate] ${migrated}/${notes.length}`);
|
|
2280
|
+
log(`[migrate] ${alreadyDone.size + migrated}/${notes.length}`);
|
|
2131
2281
|
}
|
|
2132
2282
|
}
|
|
2133
2283
|
await flush();
|
|
@@ -2135,16 +2285,60 @@ async function migrateCollection(opts) {
|
|
|
2135
2285
|
if (finalCount !== notes.length) {
|
|
2136
2286
|
warn(`[migrate] target holds ${finalCount} point(s) but the source had ${notes.length} \u2014 check before switching`);
|
|
2137
2287
|
}
|
|
2138
|
-
log(
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2288
|
+
log(`[migrate] done: ${migrated} written, ${alreadyDone.size} already present, ${refreshed} re-extracted.`);
|
|
2289
|
+
return {
|
|
2290
|
+
total: notes.length,
|
|
2291
|
+
missingDerived,
|
|
2292
|
+
refreshed,
|
|
2293
|
+
migrated,
|
|
2294
|
+
skipped: alreadyDone.size,
|
|
2295
|
+
sourceDim,
|
|
2296
|
+
targetDim,
|
|
2297
|
+
model,
|
|
2298
|
+
dryRun: false
|
|
2299
|
+
};
|
|
2300
|
+
}
|
|
2301
|
+
async function switchToMigrated(opts) {
|
|
2302
|
+
const { name, to } = opts;
|
|
2303
|
+
const log = opts.logger?.info ?? ((m) => console.log(m));
|
|
2304
|
+
if (name === to) throw new Error(`switch: "${name}" and "${to}" are the same collection`);
|
|
2305
|
+
let snap;
|
|
2306
|
+
const already = await resolveAliasRaw(name);
|
|
2307
|
+
if (already === to) {
|
|
2308
|
+
log(`[switch] "${name}" already points at "${to}" \u2014 nothing to do`);
|
|
2309
|
+
return { name, to, moved: await countPointsRaw(to) };
|
|
2310
|
+
}
|
|
2311
|
+
const targetCount = await countPointsRaw(to);
|
|
2312
|
+
if (targetCount === 0) throw new Error(`switch: "${to}" is empty \u2014 migrate into it first`);
|
|
2313
|
+
if (already === null) {
|
|
2314
|
+
const sourceCount = await countPointsRaw(name);
|
|
2315
|
+
if (targetCount < sourceCount) {
|
|
2316
|
+
throw new Error(
|
|
2317
|
+
`switch: "${to}" holds ${targetCount} point(s) but "${name}" still holds ${sourceCount}. The migration is not finished \u2014 run it again before switching.`
|
|
2318
|
+
);
|
|
2319
|
+
}
|
|
2320
|
+
log(`[switch] verified ${targetCount} in "${to}" against ${sourceCount} in "${name}"`);
|
|
2321
|
+
if (opts.snapshot !== false) {
|
|
2322
|
+
snap = await snapshotCollectionRaw(name);
|
|
2323
|
+
log(`[switch] snapshotted "${name}" \u2192 ${snap.name} (${(snap.size / 1e6).toFixed(0)} MB)`);
|
|
2324
|
+
}
|
|
2325
|
+
await deleteCollectionRaw(name);
|
|
2326
|
+
log(`[switch] dropped "${name}"`);
|
|
2327
|
+
await createAliasRaw(name, to);
|
|
2328
|
+
} else {
|
|
2329
|
+
await setAliasRaw(name, to);
|
|
2330
|
+
}
|
|
2331
|
+
log(`[switch] "${name}" now resolves to "${to}"`);
|
|
2332
|
+
return { name, to, moved: targetCount, snapshot: snap };
|
|
2142
2333
|
}
|
|
2143
2334
|
|
|
2144
2335
|
export {
|
|
2336
|
+
SYSTEM_ACTOR,
|
|
2145
2337
|
canWrite,
|
|
2146
2338
|
canRead,
|
|
2147
2339
|
DEFAULT_EMBEDDING_MODEL,
|
|
2340
|
+
LEGACY_DEFAULT_EMBEDDING_MODEL,
|
|
2341
|
+
LEGACY_DEFAULT_DIM,
|
|
2148
2342
|
getEmbeddingModel,
|
|
2149
2343
|
getEmbeddingPooling,
|
|
2150
2344
|
getEmbeddingDevice,
|
|
@@ -2157,8 +2351,13 @@ export {
|
|
|
2157
2351
|
getCollection,
|
|
2158
2352
|
EmbeddingDimensionMismatchError,
|
|
2159
2353
|
EmbeddingModelMismatchError,
|
|
2354
|
+
MixedEmbeddingModelsError,
|
|
2160
2355
|
pingQdrant,
|
|
2161
2356
|
ensureCollection,
|
|
2357
|
+
countPointsRaw,
|
|
2358
|
+
collectionDimRaw,
|
|
2359
|
+
scrollIdsRaw,
|
|
2360
|
+
resolveAliasRaw,
|
|
2162
2361
|
createStorageContext,
|
|
2163
2362
|
getNote,
|
|
2164
2363
|
updateNote,
|
|
@@ -2177,6 +2376,7 @@ export {
|
|
|
2177
2376
|
mergeSimilarNotes,
|
|
2178
2377
|
consolidateMemories,
|
|
2179
2378
|
conflictSweep,
|
|
2180
|
-
migrateCollection
|
|
2379
|
+
migrateCollection,
|
|
2380
|
+
switchToMigrated
|
|
2181
2381
|
};
|
|
2182
|
-
//# sourceMappingURL=chunk-
|
|
2382
|
+
//# sourceMappingURL=chunk-XEMQZNLD.js.map
|