@amemhq/core 1.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/{chunk-B2NS7WAM.js → chunk-K6WZTDM7.js} +229 -58
- package/dist/chunk-K6WZTDM7.js.map +1 -0
- package/dist/cli-migrate.cjs +206 -52
- package/dist/cli-migrate.cjs.map +1 -1
- package/dist/cli-migrate.d.cts +21 -7
- package/dist/cli-migrate.d.ts +21 -7
- package/dist/cli-migrate.js +103 -42
- package/dist/cli-migrate.js.map +1 -1
- package/dist/index.cjs +228 -56
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +145 -29
- package/dist/index.d.ts +145 -29
- package/dist/index.js +11 -1
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/dist/chunk-B2NS7WAM.js.map +0 -1
package/dist/index.cjs
CHANGED
|
@@ -34,6 +34,10 @@ __export(index_exports, {
|
|
|
34
34
|
DEFAULT_EMBEDDING_MODEL: () => DEFAULT_EMBEDDING_MODEL,
|
|
35
35
|
EmbeddingDimensionMismatchError: () => EmbeddingDimensionMismatchError,
|
|
36
36
|
EmbeddingModelMismatchError: () => EmbeddingModelMismatchError,
|
|
37
|
+
LEGACY_DEFAULT_DIM: () => LEGACY_DEFAULT_DIM,
|
|
38
|
+
LEGACY_DEFAULT_EMBEDDING_MODEL: () => LEGACY_DEFAULT_EMBEDDING_MODEL,
|
|
39
|
+
MixedEmbeddingModelsError: () => MixedEmbeddingModelsError,
|
|
40
|
+
SYSTEM_ACTOR: () => SYSTEM_ACTOR,
|
|
37
41
|
addEpisodic: () => addEpisodic,
|
|
38
42
|
addMemory: () => addMemory,
|
|
39
43
|
canRead: () => canRead,
|
|
@@ -68,6 +72,7 @@ __export(index_exports, {
|
|
|
68
72
|
resolveCrudUpdateMinSim: () => resolveCrudUpdateMinSim,
|
|
69
73
|
scanLowQuality: () => scanLowQuality,
|
|
70
74
|
searchMemory: () => searchMemory,
|
|
75
|
+
switchToMigrated: () => switchToMigrated,
|
|
71
76
|
updateNote: () => updateNote
|
|
72
77
|
});
|
|
73
78
|
module.exports = __toCommonJS(index_exports);
|
|
@@ -88,9 +93,23 @@ var pipeline = null;
|
|
|
88
93
|
var extractor = null;
|
|
89
94
|
var loadedKey = null;
|
|
90
95
|
var cachedDim = null;
|
|
91
|
-
var DEFAULT_EMBEDDING_MODEL = "Xenova/
|
|
96
|
+
var DEFAULT_EMBEDDING_MODEL = "Xenova/bge-m3";
|
|
97
|
+
var LEGACY_DEFAULT_EMBEDDING_MODEL = "Xenova/paraphrase-multilingual-MiniLM-L12-v2";
|
|
98
|
+
var LEGACY_DEFAULT_DIM = 384;
|
|
99
|
+
var DEFAULT_MODEL_DTYPE = "fp16";
|
|
100
|
+
var pinnedModel = null;
|
|
101
|
+
function pinEmbeddingModel(model) {
|
|
102
|
+
if (pinnedModel === model) return;
|
|
103
|
+
pinnedModel = model;
|
|
104
|
+
extractor = null;
|
|
105
|
+
loadedKey = null;
|
|
106
|
+
cachedDim = null;
|
|
107
|
+
}
|
|
108
|
+
function getPinnedEmbeddingModel() {
|
|
109
|
+
return pinnedModel;
|
|
110
|
+
}
|
|
92
111
|
function getEmbeddingModel() {
|
|
93
|
-
return process.env.AMEM_EMBED_MODEL?.trim() || DEFAULT_EMBEDDING_MODEL;
|
|
112
|
+
return process.env.AMEM_EMBED_MODEL?.trim() || pinnedModel || DEFAULT_EMBEDDING_MODEL;
|
|
94
113
|
}
|
|
95
114
|
var CLS_POOLED_MODELS = /* @__PURE__ */ new Set([
|
|
96
115
|
"bge-m3",
|
|
@@ -115,7 +134,9 @@ function getEmbeddingDevice() {
|
|
|
115
134
|
return process.env.AMEM_EMBED_DEVICE?.trim() || void 0;
|
|
116
135
|
}
|
|
117
136
|
function getEmbeddingDtype() {
|
|
118
|
-
|
|
137
|
+
const explicit = process.env.AMEM_EMBED_DTYPE?.trim();
|
|
138
|
+
if (explicit) return explicit;
|
|
139
|
+
return getEmbeddingModel() === DEFAULT_EMBEDDING_MODEL ? DEFAULT_MODEL_DTYPE : void 0;
|
|
119
140
|
}
|
|
120
141
|
function extractorKey() {
|
|
121
142
|
return `${getEmbeddingModel()}|${getEmbeddingDevice() ?? ""}|${getEmbeddingDtype() ?? ""}`;
|
|
@@ -212,6 +233,7 @@ var fs2 = __toESM(require("fs"), 1);
|
|
|
212
233
|
var path3 = __toESM(require("path"), 1);
|
|
213
234
|
|
|
214
235
|
// src/auth.ts
|
|
236
|
+
var SYSTEM_ACTOR = "__amem_system__";
|
|
215
237
|
function canWrite(note, callerAgentId) {
|
|
216
238
|
return note.owner === callerAgentId || note.writers.includes(callerAgentId) || note.writers.includes("*");
|
|
217
239
|
}
|
|
@@ -240,12 +262,12 @@ Either set AMEM_EMBED_MODEL back to the model this collection was built with, or
|
|
|
240
262
|
model;
|
|
241
263
|
};
|
|
242
264
|
function migrationHint(collection, targetModel) {
|
|
243
|
-
return `migrate
|
|
265
|
+
return `migrate onto it:
|
|
244
266
|
|
|
245
267
|
AMEM_EMBED_MODEL=${targetModel} \\
|
|
246
|
-
npx --package=@amemhq/core amem-migrate --
|
|
268
|
+
npx --package=@amemhq/core amem-migrate --from-collection ${collection}
|
|
247
269
|
|
|
248
|
-
That
|
|
270
|
+
That only reports; it takes --apply to write anything, and "${collection}" is read either way. The new store ends up behind the name you already use, so there is nothing to change in your config afterwards. See https://amem.owo.lc/reference/embedding-models.`;
|
|
249
271
|
}
|
|
250
272
|
var EmbeddingModelMismatchError = class extends Error {
|
|
251
273
|
constructor(collection, collectionModel, configuredModel) {
|
|
@@ -262,6 +284,25 @@ Either set AMEM_EMBED_MODEL back to "${collectionModel}", or ` + migrationHint(c
|
|
|
262
284
|
collectionModel;
|
|
263
285
|
configuredModel;
|
|
264
286
|
};
|
|
287
|
+
var MixedEmbeddingModelsError = class extends Error {
|
|
288
|
+
constructor(collection, wanted, inUse) {
|
|
289
|
+
super(
|
|
290
|
+
`Collection "${collection}" was built with "${wanted}", but this process is already embedding with "${inUse}" for another collection. One process can only use one model.
|
|
291
|
+
Migrate the remaining collections so they all agree:
|
|
292
|
+
|
|
293
|
+
npx --package=@amemhq/core amem-migrate --from-collection ${collection}
|
|
294
|
+
|
|
295
|
+
Or set AMEM_EMBED_MODEL to pin every collection to one model, which is only correct if they really were all built with it.`
|
|
296
|
+
);
|
|
297
|
+
this.collection = collection;
|
|
298
|
+
this.wanted = wanted;
|
|
299
|
+
this.inUse = inUse;
|
|
300
|
+
this.name = "MixedEmbeddingModelsError";
|
|
301
|
+
}
|
|
302
|
+
collection;
|
|
303
|
+
wanted;
|
|
304
|
+
inUse;
|
|
305
|
+
};
|
|
265
306
|
async function recordCollectionModel(collection, model) {
|
|
266
307
|
try {
|
|
267
308
|
await qdrant("PATCH", `/collections/${collection}`, { metadata: { embedding_model: model } });
|
|
@@ -304,18 +345,38 @@ async function ensureCollection(collectionName) {
|
|
|
304
345
|
}
|
|
305
346
|
if (existing) {
|
|
306
347
|
const collectionDim = existing.config?.params?.vectors?.size;
|
|
348
|
+
const recorded = existing.config?.metadata?.embedding_model;
|
|
349
|
+
const explicit = process.env.AMEM_EMBED_MODEL?.trim();
|
|
350
|
+
const inferLegacy = !explicit && recorded === void 0 && collectionDim === LEGACY_DEFAULT_DIM;
|
|
351
|
+
if (!explicit) {
|
|
352
|
+
const wanted = (
|
|
353
|
+
// The collection says what built it, which outranks whatever the shipped
|
|
354
|
+
// default happens to be today. This is what keeps changing the default
|
|
355
|
+
// from breaking every install that already has data.
|
|
356
|
+
typeof recorded === "string" ? recorded : inferLegacy ? LEGACY_DEFAULT_EMBEDDING_MODEL : DEFAULT_EMBEDDING_MODEL
|
|
357
|
+
);
|
|
358
|
+
const inUse = getPinnedEmbeddingModel();
|
|
359
|
+
if (inUse !== null && inUse !== wanted) throw new MixedEmbeddingModelsError(col, wanted, inUse);
|
|
360
|
+
pinEmbeddingModel(wanted);
|
|
361
|
+
if (wanted === LEGACY_DEFAULT_EMBEDDING_MODEL) {
|
|
362
|
+
console.warn(
|
|
363
|
+
`[amem] "${col}" is on ${LEGACY_DEFAULT_EMBEDDING_MODEL} (${LEGACY_DEFAULT_DIM}-dim).
|
|
364
|
+
[amem] ${DEFAULT_EMBEDDING_MODEL} reads 8192 tokens where that one stops at 128, so anything longer is being truncated before it reaches the vector.
|
|
365
|
+
[amem] To move: npx --package=@amemhq/core amem-migrate --from-collection ${col}`
|
|
366
|
+
);
|
|
367
|
+
}
|
|
368
|
+
}
|
|
307
369
|
if (typeof collectionDim === "number") {
|
|
308
370
|
const modelDim = await getEmbeddingDim();
|
|
309
371
|
if (collectionDim !== modelDim) {
|
|
310
372
|
throw new EmbeddingDimensionMismatchError(col, collectionDim, modelDim, getEmbeddingModel());
|
|
311
373
|
}
|
|
312
374
|
}
|
|
313
|
-
const recorded = existing.config?.metadata?.embedding_model;
|
|
314
375
|
const current = getEmbeddingModel();
|
|
315
376
|
if (typeof recorded === "string" && recorded !== current) {
|
|
316
377
|
throw new EmbeddingModelMismatchError(col, recorded, current);
|
|
317
378
|
}
|
|
318
|
-
if (recorded === void 0) {
|
|
379
|
+
if (recorded === void 0 && !inferLegacy) {
|
|
319
380
|
await recordCollectionModel(col, current);
|
|
320
381
|
}
|
|
321
382
|
markReady();
|
|
@@ -329,7 +390,9 @@ async function ensureCollection(collectionName) {
|
|
|
329
390
|
} catch (err) {
|
|
330
391
|
if (!(err instanceof Error) || !err.message.includes("already exists")) throw err;
|
|
331
392
|
}
|
|
332
|
-
|
|
393
|
+
const created = getEmbeddingModel();
|
|
394
|
+
await recordCollectionModel(col, created);
|
|
395
|
+
if (!process.env.AMEM_EMBED_MODEL?.trim()) pinEmbeddingModel(created);
|
|
333
396
|
await qdrant("PUT", `/collections/${col}/index`, {
|
|
334
397
|
field_name: "agent_id",
|
|
335
398
|
field_schema: "keyword"
|
|
@@ -383,6 +446,43 @@ async function createCollectionRaw(collection, size) {
|
|
|
383
446
|
async function upsertPointsRaw(collection, points) {
|
|
384
447
|
await qdrant("PUT", `/collections/${collection}/points?wait=true`, { points });
|
|
385
448
|
}
|
|
449
|
+
async function scrollIdsRaw(collection, limit = 1e4) {
|
|
450
|
+
const ids = /* @__PURE__ */ new Set();
|
|
451
|
+
let offset = void 0;
|
|
452
|
+
for (; ; ) {
|
|
453
|
+
const body = { with_payload: false, with_vector: false, limit };
|
|
454
|
+
if (offset !== void 0 && offset !== null) body.offset = offset;
|
|
455
|
+
const res = await qdrant("POST", `/collections/${collection}/points/scroll`, body);
|
|
456
|
+
for (const p of res.points) ids.add(String(p.id));
|
|
457
|
+
offset = res.next_page_offset;
|
|
458
|
+
if (offset === void 0 || offset === null || res.points.length === 0) break;
|
|
459
|
+
}
|
|
460
|
+
return ids;
|
|
461
|
+
}
|
|
462
|
+
async function deleteCollectionRaw(collection) {
|
|
463
|
+
await qdrant("DELETE", `/collections/${collection}`);
|
|
464
|
+
}
|
|
465
|
+
async function resolveAliasRaw(alias) {
|
|
466
|
+
try {
|
|
467
|
+
const res = await qdrant("GET", `/aliases`);
|
|
468
|
+
return res.aliases.find((a) => a.alias_name === alias)?.collection_name ?? null;
|
|
469
|
+
} catch {
|
|
470
|
+
return null;
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
async function createAliasRaw(alias, collection) {
|
|
474
|
+
await qdrant("POST", `/collections/aliases`, {
|
|
475
|
+
actions: [{ create_alias: { collection_name: collection, alias_name: alias } }]
|
|
476
|
+
});
|
|
477
|
+
}
|
|
478
|
+
async function setAliasRaw(alias, collection) {
|
|
479
|
+
await qdrant("POST", `/collections/aliases`, {
|
|
480
|
+
actions: [
|
|
481
|
+
{ delete_alias: { alias_name: alias } },
|
|
482
|
+
{ create_alias: { collection_name: collection, alias_name: alias } }
|
|
483
|
+
]
|
|
484
|
+
});
|
|
485
|
+
}
|
|
386
486
|
function noteToPoint(note) {
|
|
387
487
|
return {
|
|
388
488
|
id: note.id,
|
|
@@ -527,12 +627,15 @@ function makeCrud(collectionName, modeBIsolated = false) {
|
|
|
527
627
|
},
|
|
528
628
|
/**
|
|
529
629
|
* Story 36: this is the one read that bypasses the agent filter — it fetches
|
|
530
|
-
* straight by UUID.
|
|
531
|
-
*
|
|
532
|
-
*
|
|
533
|
-
*
|
|
630
|
+
* straight by UUID. An unreadable note comes back as `null`, indistinguishable
|
|
631
|
+
* from missing, so nothing leaks and callers already handle it.
|
|
632
|
+
*
|
|
633
|
+
* `reader` is required. It used to be optional, and omitting it skipped the
|
|
634
|
+
* check — which meant the safe behaviour was the one you had to remember to
|
|
635
|
+
* ask for. Pass `SYSTEM_ACTOR` to read as the engine itself; that reads as a
|
|
636
|
+
* deliberate act at the call site, where an absent argument did not.
|
|
534
637
|
*/
|
|
535
|
-
async getNote(id,
|
|
638
|
+
async getNote(id, reader) {
|
|
536
639
|
await ensureCollection(col);
|
|
537
640
|
try {
|
|
538
641
|
const result = await qdrant("POST", `/collections/${col}/points`, {
|
|
@@ -542,7 +645,7 @@ function makeCrud(collectionName, modeBIsolated = false) {
|
|
|
542
645
|
});
|
|
543
646
|
if (!result.length) return null;
|
|
544
647
|
const note = pointToNote(result[0]);
|
|
545
|
-
if (
|
|
648
|
+
if (reader !== SYSTEM_ACTOR && !canRead(note, reader)) return null;
|
|
546
649
|
return note;
|
|
547
650
|
} catch {
|
|
548
651
|
return null;
|
|
@@ -580,19 +683,22 @@ function makeCrud(collectionName, modeBIsolated = false) {
|
|
|
580
683
|
return pointToNote(result.points[0]);
|
|
581
684
|
},
|
|
582
685
|
/**
|
|
583
|
-
* Story 33:
|
|
584
|
-
*
|
|
585
|
-
*
|
|
586
|
-
*
|
|
587
|
-
*
|
|
588
|
-
*
|
|
686
|
+
* Story 33: enforces the writers policy. Returns false — without writing —
|
|
687
|
+
* when the caller may not write. This fetch-then-check path exists for callers
|
|
688
|
+
* that only have an id (the plugin's CRUD hook); callers already holding the
|
|
689
|
+
* note can check `canWrite` themselves and skip a round trip.
|
|
690
|
+
*
|
|
691
|
+
* `caller` is required. Pass `SYSTEM_ACTOR` for the engine's own maintenance
|
|
692
|
+
* writes. The self-read below is a genuine `SYSTEM_ACTOR` case: it fetches the
|
|
693
|
+
* note in order to decide whether the caller may write it, and gating that
|
|
694
|
+
* fetch on the same policy it exists to evaluate would be circular.
|
|
589
695
|
*/
|
|
590
|
-
async updateNoteContent(id, content, embedding, hash,
|
|
696
|
+
async updateNoteContent(id, content, embedding, hash, caller) {
|
|
591
697
|
await ensureCollection(col);
|
|
592
698
|
let existing = null;
|
|
593
|
-
if (
|
|
594
|
-
existing = await this.getNote(id);
|
|
595
|
-
if (existing && !canWrite(existing,
|
|
699
|
+
if (caller !== SYSTEM_ACTOR) {
|
|
700
|
+
existing = await this.getNote(id, SYSTEM_ACTOR);
|
|
701
|
+
if (existing && !canWrite(existing, caller)) return false;
|
|
596
702
|
}
|
|
597
703
|
await qdrant("PUT", `/collections/${col}/points/vectors?wait=true`, {
|
|
598
704
|
points: [{ id, vector: embedding }]
|
|
@@ -680,11 +786,11 @@ function makeCrud(collectionName, modeBIsolated = false) {
|
|
|
680
786
|
});
|
|
681
787
|
},
|
|
682
788
|
/** Story 33: see `updateNoteContent` — returns false, unwritten, when denied. */
|
|
683
|
-
async invalidateNote(id,
|
|
789
|
+
async invalidateNote(id, caller) {
|
|
684
790
|
await ensureCollection(col);
|
|
685
|
-
if (
|
|
686
|
-
const existing = await this.getNote(id);
|
|
687
|
-
if (existing && !canWrite(existing,
|
|
791
|
+
if (caller !== SYSTEM_ACTOR) {
|
|
792
|
+
const existing = await this.getNote(id, SYSTEM_ACTOR);
|
|
793
|
+
if (existing && !canWrite(existing, caller)) return false;
|
|
688
794
|
}
|
|
689
795
|
await qdrant("POST", `/collections/${col}/points/payload?wait=true`, {
|
|
690
796
|
payload: { is_active: false },
|
|
@@ -750,8 +856,8 @@ function makeCrud(collectionName, modeBIsolated = false) {
|
|
|
750
856
|
function createStorageContext(collectionName, modeBIsolated = false) {
|
|
751
857
|
return makeCrud(collectionName || getCollection(), modeBIsolated);
|
|
752
858
|
}
|
|
753
|
-
async function getNote(id,
|
|
754
|
-
return makeCrud(getCollection()).getNote(id,
|
|
859
|
+
async function getNote(id, reader) {
|
|
860
|
+
return makeCrud(getCollection()).getNote(id, reader);
|
|
755
861
|
}
|
|
756
862
|
async function updateNote(note) {
|
|
757
863
|
return makeCrud(getCollection()).updateNote(note);
|
|
@@ -762,8 +868,8 @@ async function listNotes(agentId, subject) {
|
|
|
762
868
|
async function deleteNote(id) {
|
|
763
869
|
return makeCrud(getCollection()).deleteNote(id);
|
|
764
870
|
}
|
|
765
|
-
async function invalidateNote(id,
|
|
766
|
-
return makeCrud(getCollection()).invalidateNote(id,
|
|
871
|
+
async function invalidateNote(id, caller) {
|
|
872
|
+
return makeCrud(getCollection()).invalidateNote(id, caller);
|
|
767
873
|
}
|
|
768
874
|
async function patchNotePayload(id, fields) {
|
|
769
875
|
return makeCrud(getCollection()).patchNotePayload(id, fields);
|
|
@@ -1515,7 +1621,7 @@ async function addMemory(content, agentId = "main", opts) {
|
|
|
1515
1621
|
if (topMatch.length > 0 && topMatch[0].score >= 0.85) {
|
|
1516
1622
|
if (canWrite(topMatch[0].note, agentId)) {
|
|
1517
1623
|
console.log(`[add] dedup: high-sim match (sim=${topMatch[0].score.toFixed(3)}), updating existing`);
|
|
1518
|
-
await ctx.updateNoteContent(topMatch[0].note.id, content, embedding, hash);
|
|
1624
|
+
await ctx.updateNoteContent(topMatch[0].note.id, content, embedding, hash, agentId);
|
|
1519
1625
|
return topMatch[0].note.id;
|
|
1520
1626
|
}
|
|
1521
1627
|
console.log(
|
|
@@ -1587,7 +1693,7 @@ async function addMemory(content, agentId = "main", opts) {
|
|
|
1587
1693
|
note.links = linkedIds;
|
|
1588
1694
|
await ctx.updateNote(note);
|
|
1589
1695
|
for (const lid of linkedIds) {
|
|
1590
|
-
const linked = await ctx.getNote(lid);
|
|
1696
|
+
const linked = await ctx.getNote(lid, agentId);
|
|
1591
1697
|
if (linked && !linked.links.includes(note.id)) {
|
|
1592
1698
|
if (!canWrite(linked, agentId)) {
|
|
1593
1699
|
console.log(`[link] back-link into ${lid.slice(0, 8)} skipped \u2014 not writable by ${logSafe(agentId)}`);
|
|
@@ -1600,7 +1706,7 @@ async function addMemory(content, agentId = "main", opts) {
|
|
|
1600
1706
|
if (shouldRunEvolution()) {
|
|
1601
1707
|
console.log(` [evo] threshold reached, running evolution for ${Math.min(linkedIds.length, 3)} linked notes`);
|
|
1602
1708
|
for (const lid of linkedIds.slice(0, 3)) {
|
|
1603
|
-
const linked = await ctx.getNote(lid);
|
|
1709
|
+
const linked = await ctx.getNote(lid, agentId);
|
|
1604
1710
|
if (!linked) continue;
|
|
1605
1711
|
if (!canWrite(linked, agentId)) {
|
|
1606
1712
|
console.log(` [evo] skipping ${lid.slice(0, 8)} \u2014 not writable by ${logSafe(agentId)}`);
|
|
@@ -1753,7 +1859,7 @@ async function searchMemory(query, topK = 5, agentId = "main", opts) {
|
|
|
1753
1859
|
const allNotes = await ctx.listNotes(agentId, subject);
|
|
1754
1860
|
const bm25State = buildBM25(allNotes);
|
|
1755
1861
|
const queryTokens = simpleTokenize(query);
|
|
1756
|
-
const bm25Ranked = bm25Score(bm25State, queryTokens).slice(0, n);
|
|
1862
|
+
const bm25Ranked = bm25Score(bm25State, queryTokens).filter(([, score]) => score > 0).slice(0, n);
|
|
1757
1863
|
const merged = rrfMerge(
|
|
1758
1864
|
embResults.map((r) => r.note.id),
|
|
1759
1865
|
bm25Ranked.map((r) => r[0])
|
|
@@ -1776,6 +1882,7 @@ async function searchMemory(query, topK = 5, agentId = "main", opts) {
|
|
|
1776
1882
|
const visitedIds = new Set(topIds);
|
|
1777
1883
|
const bfsQueue = useBfs ? topIds.map((id) => ({ id, hop: 0 })) : [];
|
|
1778
1884
|
const bfsExtra = [];
|
|
1885
|
+
const bfsSimMap = /* @__PURE__ */ new Map();
|
|
1779
1886
|
while (bfsQueue.length > 0 && bfsExtra.length < BFS_MAX_EXPAND) {
|
|
1780
1887
|
const item = bfsQueue.shift();
|
|
1781
1888
|
if (item.hop >= BFS_MAX_HOPS) continue;
|
|
@@ -1786,10 +1893,9 @@ async function searchMemory(query, topK = 5, agentId = "main", opts) {
|
|
|
1786
1893
|
visitedIds.add(linkedId);
|
|
1787
1894
|
const linked = noteMap.get(linkedId);
|
|
1788
1895
|
if (!linked || linked.is_active === false) continue;
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
}
|
|
1896
|
+
const sim = cosineSimilarity(queryEmbedding, linked.embedding);
|
|
1897
|
+
if (bfsSimThreshold > 0 && sim < bfsSimThreshold) continue;
|
|
1898
|
+
bfsSimMap.set(linkedId, sim);
|
|
1793
1899
|
bfsExtra.push(linkedId);
|
|
1794
1900
|
bfsQueue.push({ id: linkedId, hop: item.hop + 1 });
|
|
1795
1901
|
if (bfsExtra.length >= BFS_MAX_EXPAND) break;
|
|
@@ -1805,7 +1911,11 @@ async function searchMemory(query, topK = 5, agentId = "main", opts) {
|
|
|
1805
1911
|
const embSimMap = new Map(embResults.map((r) => [r.note.id, r.score]));
|
|
1806
1912
|
const rrfMap = new Map(boostedMerged.map(([id, score]) => [id, score]));
|
|
1807
1913
|
const results = [];
|
|
1808
|
-
|
|
1914
|
+
const ordered = [
|
|
1915
|
+
...filteredTopIds.map((id) => [id, "match"]),
|
|
1916
|
+
...bfsExtra.map((id) => [id, "link"])
|
|
1917
|
+
];
|
|
1918
|
+
for (const [id, via] of ordered) {
|
|
1809
1919
|
const note = noteMap.get(id);
|
|
1810
1920
|
if (!note) continue;
|
|
1811
1921
|
results.push({
|
|
@@ -1816,8 +1926,9 @@ async function searchMemory(query, topK = 5, agentId = "main", opts) {
|
|
|
1816
1926
|
keywords: note.keywords,
|
|
1817
1927
|
links: note.links,
|
|
1818
1928
|
timestamp: note.timestamp,
|
|
1819
|
-
similarity: embSimMap.get(id) ?? 0,
|
|
1929
|
+
similarity: embSimMap.get(id) ?? bfsSimMap.get(id) ?? 0,
|
|
1820
1930
|
rrf: rrfMap.get(id) ?? 0,
|
|
1931
|
+
via,
|
|
1821
1932
|
topics: note.topics ?? [],
|
|
1822
1933
|
note_type: note.note_type ?? "memory"
|
|
1823
1934
|
});
|
|
@@ -1874,7 +1985,7 @@ async function mergeSimilarNotes(agentId, storageCtx) {
|
|
|
1874
1985
|
const mergedContent = judgment.mergedContent || pendingNote.content;
|
|
1875
1986
|
const newEmbedding = await encode(buildEmbedText({ ...bestNeighbor, content: mergedContent }));
|
|
1876
1987
|
const newHash = (0, import_crypto.createHash)("md5").update(mergedContent).digest("hex");
|
|
1877
|
-
await ctx.updateNoteContent(bestNeighbor.id, mergedContent, newEmbedding, newHash);
|
|
1988
|
+
await ctx.updateNoteContent(bestNeighbor.id, mergedContent, newEmbedding, newHash, agentId);
|
|
1878
1989
|
await ctx.patchNotePayload(bestNeighbor.id, {
|
|
1879
1990
|
evolution_history: JSON.stringify(oldHistory),
|
|
1880
1991
|
evolution_type: "EVOLVE"
|
|
@@ -1898,7 +2009,7 @@ async function mergeSimilarNotes(agentId, storageCtx) {
|
|
|
1898
2009
|
const mergedContent = judgment.mergedContent || `${bestNeighbor.content}\uFF1B${pendingNote.content}`;
|
|
1899
2010
|
const newEmbedding = await encode(buildEmbedText({ ...bestNeighbor, content: mergedContent }));
|
|
1900
2011
|
const newHash = (0, import_crypto.createHash)("md5").update(mergedContent).digest("hex");
|
|
1901
|
-
await ctx.updateNoteContent(bestNeighbor.id, mergedContent, newEmbedding, newHash);
|
|
2012
|
+
await ctx.updateNoteContent(bestNeighbor.id, mergedContent, newEmbedding, newHash, agentId);
|
|
1902
2013
|
await ctx.patchNotePayload(bestNeighbor.id, {
|
|
1903
2014
|
evolution_history: JSON.stringify(oldHistory),
|
|
1904
2015
|
evolution_type: "EXPAND"
|
|
@@ -1938,7 +2049,7 @@ async function mergeSimilarNotes(agentId, storageCtx) {
|
|
|
1938
2049
|
const [keepNote, dropNote] = noteA.content.length >= noteB.content.length ? [noteA, noteB] : [noteB, noteA];
|
|
1939
2050
|
const newEmbedding = await encode(result.merged);
|
|
1940
2051
|
const newHash = (0, import_crypto.createHash)("md5").update(result.merged).digest("hex");
|
|
1941
|
-
await ctx.updateNoteContent(keepNote.id, result.merged, newEmbedding, newHash);
|
|
2052
|
+
await ctx.updateNoteContent(keepNote.id, result.merged, newEmbedding, newHash, agentId);
|
|
1942
2053
|
await ctx.deleteNote(dropNote.id);
|
|
1943
2054
|
deletedIds.add(dropNote.id);
|
|
1944
2055
|
mergedCount++;
|
|
@@ -2047,7 +2158,7 @@ async function consolidateMemories(agentId, logger, storageCtx) {
|
|
|
2047
2158
|
action: "consolidate"
|
|
2048
2159
|
});
|
|
2049
2160
|
await ctx.updateNote(keepNote);
|
|
2050
|
-
await ctx.invalidateNote(dropNote.id);
|
|
2161
|
+
await ctx.invalidateNote(dropNote.id, agentId);
|
|
2051
2162
|
await ctx.replaceLinkReferences(dropNote.id, keepNote.id, agentId);
|
|
2052
2163
|
logMergeToFile(keepNote.id, dropNote.id, keepNote.content);
|
|
2053
2164
|
processedIds.add(keepNote.id);
|
|
@@ -2342,8 +2453,19 @@ async function migrateCollection(opts) {
|
|
|
2342
2453
|
);
|
|
2343
2454
|
if (dryRun) {
|
|
2344
2455
|
log("[migrate] dry run \u2014 nothing written. Pass dryRun: false to apply.");
|
|
2345
|
-
return {
|
|
2456
|
+
return {
|
|
2457
|
+
total: notes.length,
|
|
2458
|
+
missingDerived,
|
|
2459
|
+
refreshed: 0,
|
|
2460
|
+
migrated: 0,
|
|
2461
|
+
skipped: 0,
|
|
2462
|
+
sourceDim,
|
|
2463
|
+
targetDim,
|
|
2464
|
+
model,
|
|
2465
|
+
dryRun: true
|
|
2466
|
+
};
|
|
2346
2467
|
}
|
|
2468
|
+
let alreadyDone = /* @__PURE__ */ new Set();
|
|
2347
2469
|
const existingTargetDim = await collectionDimRaw(to);
|
|
2348
2470
|
if (existingTargetDim === null) {
|
|
2349
2471
|
await createCollectionRaw(to, targetDim);
|
|
@@ -2352,9 +2474,17 @@ async function migrateCollection(opts) {
|
|
|
2352
2474
|
if (existingTargetDim !== targetDim) {
|
|
2353
2475
|
throw new Error(`migrate: target "${to}" exists at ${existingTargetDim}d but the model produces ${targetDim}d`);
|
|
2354
2476
|
}
|
|
2355
|
-
const
|
|
2356
|
-
if (
|
|
2357
|
-
|
|
2477
|
+
const present = await scrollIdsRaw(to);
|
|
2478
|
+
if (present.size > 0) {
|
|
2479
|
+
const sourceIds = new Set(notes.map((n) => n.id));
|
|
2480
|
+
const foreign = [...present].filter((id) => !sourceIds.has(id));
|
|
2481
|
+
if (foreign.length > 0) {
|
|
2482
|
+
throw new Error(
|
|
2483
|
+
`migrate: target "${to}" holds ${foreign.length} point(s) that are not in "${from}" (e.g. ${foreign[0]}). That is not an interrupted migration \u2014 use a different target.`
|
|
2484
|
+
);
|
|
2485
|
+
}
|
|
2486
|
+
alreadyDone = present;
|
|
2487
|
+
log(`[migrate] resuming: ${present.size} of ${notes.length} already in ${to}`);
|
|
2358
2488
|
}
|
|
2359
2489
|
}
|
|
2360
2490
|
let refreshed = 0;
|
|
@@ -2368,6 +2498,7 @@ async function migrateCollection(opts) {
|
|
|
2368
2498
|
buffer = [];
|
|
2369
2499
|
};
|
|
2370
2500
|
for (const note of notes) {
|
|
2501
|
+
if (alreadyDone.has(note.id)) continue;
|
|
2371
2502
|
if (refreshFields && missingDerivedFields(note)) {
|
|
2372
2503
|
try {
|
|
2373
2504
|
const built = await llmConstructNote(note.content);
|
|
@@ -2383,7 +2514,7 @@ async function migrateCollection(opts) {
|
|
|
2383
2514
|
buffer.push(point);
|
|
2384
2515
|
if (buffer.length >= BATCH) {
|
|
2385
2516
|
await flush();
|
|
2386
|
-
log(`[migrate] ${migrated}/${notes.length}`);
|
|
2517
|
+
log(`[migrate] ${alreadyDone.size + migrated}/${notes.length}`);
|
|
2387
2518
|
}
|
|
2388
2519
|
}
|
|
2389
2520
|
await flush();
|
|
@@ -2391,10 +2522,46 @@ async function migrateCollection(opts) {
|
|
|
2391
2522
|
if (finalCount !== notes.length) {
|
|
2392
2523
|
warn(`[migrate] target holds ${finalCount} point(s) but the source had ${notes.length} \u2014 check before switching`);
|
|
2393
2524
|
}
|
|
2394
|
-
log(
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2525
|
+
log(`[migrate] done: ${migrated} written, ${alreadyDone.size} already present, ${refreshed} re-extracted.`);
|
|
2526
|
+
return {
|
|
2527
|
+
total: notes.length,
|
|
2528
|
+
missingDerived,
|
|
2529
|
+
refreshed,
|
|
2530
|
+
migrated,
|
|
2531
|
+
skipped: alreadyDone.size,
|
|
2532
|
+
sourceDim,
|
|
2533
|
+
targetDim,
|
|
2534
|
+
model,
|
|
2535
|
+
dryRun: false
|
|
2536
|
+
};
|
|
2537
|
+
}
|
|
2538
|
+
async function switchToMigrated(opts) {
|
|
2539
|
+
const { name, to } = opts;
|
|
2540
|
+
const log = opts.logger?.info ?? ((m) => console.log(m));
|
|
2541
|
+
if (name === to) throw new Error(`switch: "${name}" and "${to}" are the same collection`);
|
|
2542
|
+
const already = await resolveAliasRaw(name);
|
|
2543
|
+
if (already === to) {
|
|
2544
|
+
log(`[switch] "${name}" already points at "${to}" \u2014 nothing to do`);
|
|
2545
|
+
return { name, to, moved: await countPointsRaw(to) };
|
|
2546
|
+
}
|
|
2547
|
+
const targetCount = await countPointsRaw(to);
|
|
2548
|
+
if (targetCount === 0) throw new Error(`switch: "${to}" is empty \u2014 migrate into it first`);
|
|
2549
|
+
if (already === null) {
|
|
2550
|
+
const sourceCount = await countPointsRaw(name);
|
|
2551
|
+
if (targetCount < sourceCount) {
|
|
2552
|
+
throw new Error(
|
|
2553
|
+
`switch: "${to}" holds ${targetCount} point(s) but "${name}" still holds ${sourceCount}. The migration is not finished \u2014 run it again before switching.`
|
|
2554
|
+
);
|
|
2555
|
+
}
|
|
2556
|
+
log(`[switch] verified ${targetCount} in "${to}" against ${sourceCount} in "${name}"`);
|
|
2557
|
+
await deleteCollectionRaw(name);
|
|
2558
|
+
log(`[switch] dropped "${name}"`);
|
|
2559
|
+
await createAliasRaw(name, to);
|
|
2560
|
+
} else {
|
|
2561
|
+
await setAliasRaw(name, to);
|
|
2562
|
+
}
|
|
2563
|
+
log(`[switch] "${name}" now resolves to "${to}"`);
|
|
2564
|
+
return { name, to, moved: targetCount };
|
|
2398
2565
|
}
|
|
2399
2566
|
|
|
2400
2567
|
// src/crud-guard.ts
|
|
@@ -2416,6 +2583,10 @@ function isPlausibleUpdateTarget(newEmbedding, targetEmbedding, minSimilarity) {
|
|
|
2416
2583
|
DEFAULT_EMBEDDING_MODEL,
|
|
2417
2584
|
EmbeddingDimensionMismatchError,
|
|
2418
2585
|
EmbeddingModelMismatchError,
|
|
2586
|
+
LEGACY_DEFAULT_DIM,
|
|
2587
|
+
LEGACY_DEFAULT_EMBEDDING_MODEL,
|
|
2588
|
+
MixedEmbeddingModelsError,
|
|
2589
|
+
SYSTEM_ACTOR,
|
|
2419
2590
|
addEpisodic,
|
|
2420
2591
|
addMemory,
|
|
2421
2592
|
canRead,
|
|
@@ -2450,6 +2621,7 @@ function isPlausibleUpdateTarget(newEmbedding, targetEmbedding, minSimilarity) {
|
|
|
2450
2621
|
resolveCrudUpdateMinSim,
|
|
2451
2622
|
scanLowQuality,
|
|
2452
2623
|
searchMemory,
|
|
2624
|
+
switchToMigrated,
|
|
2453
2625
|
updateNote
|
|
2454
2626
|
});
|
|
2455
2627
|
//# sourceMappingURL=index.cjs.map
|