@amemhq/core 1.1.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/{chunk-B2NS7WAM.js → chunk-XEMQZNLD.js} +257 -57
- package/dist/chunk-XEMQZNLD.js.map +1 -0
- package/dist/cli-migrate.cjs +259 -52
- package/dist/cli-migrate.cjs.map +1 -1
- package/dist/cli-migrate.d.cts +21 -7
- package/dist/cli-migrate.d.ts +21 -7
- package/dist/cli-migrate.js +128 -43
- package/dist/cli-migrate.js.map +1 -1
- package/dist/index.cjs +256 -55
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +167 -29
- package/dist/index.d.ts +167 -29
- package/dist/index.js +11 -1
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/dist/chunk-B2NS7WAM.js.map +0 -1
package/dist/index.cjs
CHANGED
|
@@ -34,6 +34,10 @@ __export(index_exports, {
|
|
|
34
34
|
DEFAULT_EMBEDDING_MODEL: () => DEFAULT_EMBEDDING_MODEL,
|
|
35
35
|
EmbeddingDimensionMismatchError: () => EmbeddingDimensionMismatchError,
|
|
36
36
|
EmbeddingModelMismatchError: () => EmbeddingModelMismatchError,
|
|
37
|
+
LEGACY_DEFAULT_DIM: () => LEGACY_DEFAULT_DIM,
|
|
38
|
+
LEGACY_DEFAULT_EMBEDDING_MODEL: () => LEGACY_DEFAULT_EMBEDDING_MODEL,
|
|
39
|
+
MixedEmbeddingModelsError: () => MixedEmbeddingModelsError,
|
|
40
|
+
SYSTEM_ACTOR: () => SYSTEM_ACTOR,
|
|
37
41
|
addEpisodic: () => addEpisodic,
|
|
38
42
|
addMemory: () => addMemory,
|
|
39
43
|
canRead: () => canRead,
|
|
@@ -68,6 +72,7 @@ __export(index_exports, {
|
|
|
68
72
|
resolveCrudUpdateMinSim: () => resolveCrudUpdateMinSim,
|
|
69
73
|
scanLowQuality: () => scanLowQuality,
|
|
70
74
|
searchMemory: () => searchMemory,
|
|
75
|
+
switchToMigrated: () => switchToMigrated,
|
|
71
76
|
updateNote: () => updateNote
|
|
72
77
|
});
|
|
73
78
|
module.exports = __toCommonJS(index_exports);
|
|
@@ -88,9 +93,22 @@ var pipeline = null;
|
|
|
88
93
|
var extractor = null;
|
|
89
94
|
var loadedKey = null;
|
|
90
95
|
var cachedDim = null;
|
|
91
|
-
var DEFAULT_EMBEDDING_MODEL = "Xenova/
|
|
96
|
+
var DEFAULT_EMBEDDING_MODEL = "Xenova/bge-m3";
|
|
97
|
+
var LEGACY_DEFAULT_EMBEDDING_MODEL = "Xenova/paraphrase-multilingual-MiniLM-L12-v2";
|
|
98
|
+
var LEGACY_DEFAULT_DIM = 384;
|
|
99
|
+
var pinnedModel = null;
|
|
100
|
+
function pinEmbeddingModel(model) {
|
|
101
|
+
if (pinnedModel === model) return;
|
|
102
|
+
pinnedModel = model;
|
|
103
|
+
extractor = null;
|
|
104
|
+
loadedKey = null;
|
|
105
|
+
cachedDim = null;
|
|
106
|
+
}
|
|
107
|
+
function getPinnedEmbeddingModel() {
|
|
108
|
+
return pinnedModel;
|
|
109
|
+
}
|
|
92
110
|
function getEmbeddingModel() {
|
|
93
|
-
return process.env.AMEM_EMBED_MODEL?.trim() || DEFAULT_EMBEDDING_MODEL;
|
|
111
|
+
return process.env.AMEM_EMBED_MODEL?.trim() || pinnedModel || DEFAULT_EMBEDDING_MODEL;
|
|
94
112
|
}
|
|
95
113
|
var CLS_POOLED_MODELS = /* @__PURE__ */ new Set([
|
|
96
114
|
"bge-m3",
|
|
@@ -117,6 +135,27 @@ function getEmbeddingDevice() {
|
|
|
117
135
|
function getEmbeddingDtype() {
|
|
118
136
|
return process.env.AMEM_EMBED_DTYPE?.trim() || void 0;
|
|
119
137
|
}
|
|
138
|
+
function applyModelPaths(env) {
|
|
139
|
+
const cache = process.env.AMEM_MODEL_CACHE?.trim();
|
|
140
|
+
if (cache) env.cacheDir = cache;
|
|
141
|
+
const local = process.env.AMEM_MODEL_DIR?.trim();
|
|
142
|
+
if (local) {
|
|
143
|
+
env.localModelPath = local;
|
|
144
|
+
env.allowLocalModels = true;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
function makeProgressReporter() {
|
|
148
|
+
const lastPct = /* @__PURE__ */ new Map();
|
|
149
|
+
return (e) => {
|
|
150
|
+
if (e.status !== "progress" || !e.file || typeof e.progress !== "number") return;
|
|
151
|
+
if (!e.file.endsWith(".onnx") && !e.file.endsWith(".onnx_data")) return;
|
|
152
|
+
const pct = Math.floor(e.progress / 10) * 10;
|
|
153
|
+
if (lastPct.get(e.file) === pct) return;
|
|
154
|
+
lastPct.set(e.file, pct);
|
|
155
|
+
const size = e.total ? ` of ${(e.total / 1e9).toFixed(2)} GB` : "";
|
|
156
|
+
console.log(`[amem] downloading ${e.file}: ${pct}%${size}`);
|
|
157
|
+
};
|
|
158
|
+
}
|
|
120
159
|
function extractorKey() {
|
|
121
160
|
return `${getEmbeddingModel()}|${getEmbeddingDevice() ?? ""}|${getEmbeddingDtype() ?? ""}`;
|
|
122
161
|
}
|
|
@@ -126,11 +165,13 @@ async function getExtractor() {
|
|
|
126
165
|
if (!pipeline) {
|
|
127
166
|
const mod = await import("@huggingface/transformers");
|
|
128
167
|
pipeline = mod.pipeline;
|
|
168
|
+
applyModelPaths(mod.env);
|
|
129
169
|
}
|
|
130
170
|
const device = getEmbeddingDevice();
|
|
131
171
|
const dtype = getEmbeddingDtype();
|
|
132
172
|
extractor = await pipeline("feature-extraction", getEmbeddingModel(), {
|
|
133
173
|
revision: "main",
|
|
174
|
+
progress_callback: makeProgressReporter(),
|
|
134
175
|
// Omitted entirely when unset, so an unconfigured install gets exactly the
|
|
135
176
|
// library defaults it got before these existed.
|
|
136
177
|
...device ? { device } : {},
|
|
@@ -212,6 +253,7 @@ var fs2 = __toESM(require("fs"), 1);
|
|
|
212
253
|
var path3 = __toESM(require("path"), 1);
|
|
213
254
|
|
|
214
255
|
// src/auth.ts
|
|
256
|
+
var SYSTEM_ACTOR = "__amem_system__";
|
|
215
257
|
function canWrite(note, callerAgentId) {
|
|
216
258
|
return note.owner === callerAgentId || note.writers.includes(callerAgentId) || note.writers.includes("*");
|
|
217
259
|
}
|
|
@@ -240,12 +282,12 @@ Either set AMEM_EMBED_MODEL back to the model this collection was built with, or
|
|
|
240
282
|
model;
|
|
241
283
|
};
|
|
242
284
|
function migrationHint(collection, targetModel) {
|
|
243
|
-
return `migrate
|
|
285
|
+
return `migrate onto it:
|
|
244
286
|
|
|
245
287
|
AMEM_EMBED_MODEL=${targetModel} \\
|
|
246
|
-
npx --package=@amemhq/core amem-migrate --
|
|
288
|
+
npx --package=@amemhq/core amem-migrate --from-collection ${collection}
|
|
247
289
|
|
|
248
|
-
That
|
|
290
|
+
That only reports; it takes --apply to write anything, and "${collection}" is read either way. The new store ends up behind the name you already use, so there is nothing to change in your config afterwards. See https://amem.owo.lc/reference/embedding-models.`;
|
|
249
291
|
}
|
|
250
292
|
var EmbeddingModelMismatchError = class extends Error {
|
|
251
293
|
constructor(collection, collectionModel, configuredModel) {
|
|
@@ -262,6 +304,25 @@ Either set AMEM_EMBED_MODEL back to "${collectionModel}", or ` + migrationHint(c
|
|
|
262
304
|
collectionModel;
|
|
263
305
|
configuredModel;
|
|
264
306
|
};
|
|
307
|
+
var MixedEmbeddingModelsError = class extends Error {
|
|
308
|
+
constructor(collection, wanted, inUse) {
|
|
309
|
+
super(
|
|
310
|
+
`Collection "${collection}" was built with "${wanted}", but this process is already embedding with "${inUse}" for another collection. One process can only use one model.
|
|
311
|
+
Migrate the remaining collections so they all agree:
|
|
312
|
+
|
|
313
|
+
npx --package=@amemhq/core amem-migrate --from-collection ${collection}
|
|
314
|
+
|
|
315
|
+
Or set AMEM_EMBED_MODEL to pin every collection to one model, which is only correct if they really were all built with it.`
|
|
316
|
+
);
|
|
317
|
+
this.collection = collection;
|
|
318
|
+
this.wanted = wanted;
|
|
319
|
+
this.inUse = inUse;
|
|
320
|
+
this.name = "MixedEmbeddingModelsError";
|
|
321
|
+
}
|
|
322
|
+
collection;
|
|
323
|
+
wanted;
|
|
324
|
+
inUse;
|
|
325
|
+
};
|
|
265
326
|
async function recordCollectionModel(collection, model) {
|
|
266
327
|
try {
|
|
267
328
|
await qdrant("PATCH", `/collections/${collection}`, { metadata: { embedding_model: model } });
|
|
@@ -304,18 +365,38 @@ async function ensureCollection(collectionName) {
|
|
|
304
365
|
}
|
|
305
366
|
if (existing) {
|
|
306
367
|
const collectionDim = existing.config?.params?.vectors?.size;
|
|
368
|
+
const recorded = existing.config?.metadata?.embedding_model;
|
|
369
|
+
const explicit = process.env.AMEM_EMBED_MODEL?.trim();
|
|
370
|
+
const inferLegacy = !explicit && recorded === void 0 && collectionDim === LEGACY_DEFAULT_DIM;
|
|
371
|
+
if (!explicit) {
|
|
372
|
+
const wanted = (
|
|
373
|
+
// The collection says what built it, which outranks whatever the shipped
|
|
374
|
+
// default happens to be today. This is what keeps changing the default
|
|
375
|
+
// from breaking every install that already has data.
|
|
376
|
+
typeof recorded === "string" ? recorded : inferLegacy ? LEGACY_DEFAULT_EMBEDDING_MODEL : DEFAULT_EMBEDDING_MODEL
|
|
377
|
+
);
|
|
378
|
+
const inUse = getPinnedEmbeddingModel();
|
|
379
|
+
if (inUse !== null && inUse !== wanted) throw new MixedEmbeddingModelsError(col, wanted, inUse);
|
|
380
|
+
pinEmbeddingModel(wanted);
|
|
381
|
+
if (wanted === LEGACY_DEFAULT_EMBEDDING_MODEL) {
|
|
382
|
+
console.warn(
|
|
383
|
+
`[amem] "${col}" is on ${LEGACY_DEFAULT_EMBEDDING_MODEL} (${LEGACY_DEFAULT_DIM}-dim).
|
|
384
|
+
[amem] ${DEFAULT_EMBEDDING_MODEL} reads 8192 tokens where that one stops at 128, so anything longer is being truncated before it reaches the vector.
|
|
385
|
+
[amem] To move: npx --package=@amemhq/core amem-migrate --from-collection ${col}`
|
|
386
|
+
);
|
|
387
|
+
}
|
|
388
|
+
}
|
|
307
389
|
if (typeof collectionDim === "number") {
|
|
308
390
|
const modelDim = await getEmbeddingDim();
|
|
309
391
|
if (collectionDim !== modelDim) {
|
|
310
392
|
throw new EmbeddingDimensionMismatchError(col, collectionDim, modelDim, getEmbeddingModel());
|
|
311
393
|
}
|
|
312
394
|
}
|
|
313
|
-
const recorded = existing.config?.metadata?.embedding_model;
|
|
314
395
|
const current = getEmbeddingModel();
|
|
315
396
|
if (typeof recorded === "string" && recorded !== current) {
|
|
316
397
|
throw new EmbeddingModelMismatchError(col, recorded, current);
|
|
317
398
|
}
|
|
318
|
-
if (recorded === void 0) {
|
|
399
|
+
if (recorded === void 0 && !inferLegacy) {
|
|
319
400
|
await recordCollectionModel(col, current);
|
|
320
401
|
}
|
|
321
402
|
markReady();
|
|
@@ -329,7 +410,9 @@ async function ensureCollection(collectionName) {
|
|
|
329
410
|
} catch (err) {
|
|
330
411
|
if (!(err instanceof Error) || !err.message.includes("already exists")) throw err;
|
|
331
412
|
}
|
|
332
|
-
|
|
413
|
+
const created = getEmbeddingModel();
|
|
414
|
+
await recordCollectionModel(col, created);
|
|
415
|
+
if (!process.env.AMEM_EMBED_MODEL?.trim()) pinEmbeddingModel(created);
|
|
333
416
|
await qdrant("PUT", `/collections/${col}/index`, {
|
|
334
417
|
field_name: "agent_id",
|
|
335
418
|
field_schema: "keyword"
|
|
@@ -383,6 +466,47 @@ async function createCollectionRaw(collection, size) {
|
|
|
383
466
|
async function upsertPointsRaw(collection, points) {
|
|
384
467
|
await qdrant("PUT", `/collections/${collection}/points?wait=true`, { points });
|
|
385
468
|
}
|
|
469
|
+
async function scrollIdsRaw(collection, limit = 1e4) {
|
|
470
|
+
const ids = /* @__PURE__ */ new Set();
|
|
471
|
+
let offset = void 0;
|
|
472
|
+
for (; ; ) {
|
|
473
|
+
const body = { with_payload: false, with_vector: false, limit };
|
|
474
|
+
if (offset !== void 0 && offset !== null) body.offset = offset;
|
|
475
|
+
const res = await qdrant("POST", `/collections/${collection}/points/scroll`, body);
|
|
476
|
+
for (const p of res.points) ids.add(String(p.id));
|
|
477
|
+
offset = res.next_page_offset;
|
|
478
|
+
if (offset === void 0 || offset === null || res.points.length === 0) break;
|
|
479
|
+
}
|
|
480
|
+
return ids;
|
|
481
|
+
}
|
|
482
|
+
async function deleteCollectionRaw(collection) {
|
|
483
|
+
await qdrant("DELETE", `/collections/${collection}`);
|
|
484
|
+
}
|
|
485
|
+
async function snapshotCollectionRaw(collection) {
|
|
486
|
+
const r = await qdrant("POST", `/collections/${collection}/snapshots`);
|
|
487
|
+
return { name: r.name, size: r.size ?? 0 };
|
|
488
|
+
}
|
|
489
|
+
async function resolveAliasRaw(alias) {
|
|
490
|
+
try {
|
|
491
|
+
const res = await qdrant("GET", `/aliases`);
|
|
492
|
+
return res.aliases.find((a) => a.alias_name === alias)?.collection_name ?? null;
|
|
493
|
+
} catch {
|
|
494
|
+
return null;
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
async function createAliasRaw(alias, collection) {
|
|
498
|
+
await qdrant("POST", `/collections/aliases`, {
|
|
499
|
+
actions: [{ create_alias: { collection_name: collection, alias_name: alias } }]
|
|
500
|
+
});
|
|
501
|
+
}
|
|
502
|
+
async function setAliasRaw(alias, collection) {
|
|
503
|
+
await qdrant("POST", `/collections/aliases`, {
|
|
504
|
+
actions: [
|
|
505
|
+
{ delete_alias: { alias_name: alias } },
|
|
506
|
+
{ create_alias: { collection_name: collection, alias_name: alias } }
|
|
507
|
+
]
|
|
508
|
+
});
|
|
509
|
+
}
|
|
386
510
|
function noteToPoint(note) {
|
|
387
511
|
return {
|
|
388
512
|
id: note.id,
|
|
@@ -527,12 +651,15 @@ function makeCrud(collectionName, modeBIsolated = false) {
|
|
|
527
651
|
},
|
|
528
652
|
/**
|
|
529
653
|
* Story 36: this is the one read that bypasses the agent filter — it fetches
|
|
530
|
-
* straight by UUID.
|
|
531
|
-
*
|
|
532
|
-
*
|
|
533
|
-
*
|
|
654
|
+
* straight by UUID. An unreadable note comes back as `null`, indistinguishable
|
|
655
|
+
* from missing, so nothing leaks and callers already handle it.
|
|
656
|
+
*
|
|
657
|
+
* `reader` is required. It used to be optional, and omitting it skipped the
|
|
658
|
+
* check — which meant the safe behaviour was the one you had to remember to
|
|
659
|
+
* ask for. Pass `SYSTEM_ACTOR` to read as the engine itself; that reads as a
|
|
660
|
+
* deliberate act at the call site, where an absent argument did not.
|
|
534
661
|
*/
|
|
535
|
-
async getNote(id,
|
|
662
|
+
async getNote(id, reader) {
|
|
536
663
|
await ensureCollection(col);
|
|
537
664
|
try {
|
|
538
665
|
const result = await qdrant("POST", `/collections/${col}/points`, {
|
|
@@ -542,7 +669,7 @@ function makeCrud(collectionName, modeBIsolated = false) {
|
|
|
542
669
|
});
|
|
543
670
|
if (!result.length) return null;
|
|
544
671
|
const note = pointToNote(result[0]);
|
|
545
|
-
if (
|
|
672
|
+
if (reader !== SYSTEM_ACTOR && !canRead(note, reader)) return null;
|
|
546
673
|
return note;
|
|
547
674
|
} catch {
|
|
548
675
|
return null;
|
|
@@ -580,19 +707,22 @@ function makeCrud(collectionName, modeBIsolated = false) {
|
|
|
580
707
|
return pointToNote(result.points[0]);
|
|
581
708
|
},
|
|
582
709
|
/**
|
|
583
|
-
* Story 33:
|
|
584
|
-
*
|
|
585
|
-
*
|
|
586
|
-
*
|
|
587
|
-
*
|
|
588
|
-
*
|
|
710
|
+
* Story 33: enforces the writers policy. Returns false — without writing —
|
|
711
|
+
* when the caller may not write. This fetch-then-check path exists for callers
|
|
712
|
+
* that only have an id (the plugin's CRUD hook); callers already holding the
|
|
713
|
+
* note can check `canWrite` themselves and skip a round trip.
|
|
714
|
+
*
|
|
715
|
+
* `caller` is required. Pass `SYSTEM_ACTOR` for the engine's own maintenance
|
|
716
|
+
* writes. The self-read below is a genuine `SYSTEM_ACTOR` case: it fetches the
|
|
717
|
+
* note in order to decide whether the caller may write it, and gating that
|
|
718
|
+
* fetch on the same policy it exists to evaluate would be circular.
|
|
589
719
|
*/
|
|
590
|
-
async updateNoteContent(id, content, embedding, hash,
|
|
720
|
+
async updateNoteContent(id, content, embedding, hash, caller) {
|
|
591
721
|
await ensureCollection(col);
|
|
592
722
|
let existing = null;
|
|
593
|
-
if (
|
|
594
|
-
existing = await this.getNote(id);
|
|
595
|
-
if (existing && !canWrite(existing,
|
|
723
|
+
if (caller !== SYSTEM_ACTOR) {
|
|
724
|
+
existing = await this.getNote(id, SYSTEM_ACTOR);
|
|
725
|
+
if (existing && !canWrite(existing, caller)) return false;
|
|
596
726
|
}
|
|
597
727
|
await qdrant("PUT", `/collections/${col}/points/vectors?wait=true`, {
|
|
598
728
|
points: [{ id, vector: embedding }]
|
|
@@ -680,11 +810,11 @@ function makeCrud(collectionName, modeBIsolated = false) {
|
|
|
680
810
|
});
|
|
681
811
|
},
|
|
682
812
|
/** Story 33: see `updateNoteContent` — returns false, unwritten, when denied. */
|
|
683
|
-
async invalidateNote(id,
|
|
813
|
+
async invalidateNote(id, caller) {
|
|
684
814
|
await ensureCollection(col);
|
|
685
|
-
if (
|
|
686
|
-
const existing = await this.getNote(id);
|
|
687
|
-
if (existing && !canWrite(existing,
|
|
815
|
+
if (caller !== SYSTEM_ACTOR) {
|
|
816
|
+
const existing = await this.getNote(id, SYSTEM_ACTOR);
|
|
817
|
+
if (existing && !canWrite(existing, caller)) return false;
|
|
688
818
|
}
|
|
689
819
|
await qdrant("POST", `/collections/${col}/points/payload?wait=true`, {
|
|
690
820
|
payload: { is_active: false },
|
|
@@ -750,8 +880,8 @@ function makeCrud(collectionName, modeBIsolated = false) {
|
|
|
750
880
|
function createStorageContext(collectionName, modeBIsolated = false) {
|
|
751
881
|
return makeCrud(collectionName || getCollection(), modeBIsolated);
|
|
752
882
|
}
|
|
753
|
-
async function getNote(id,
|
|
754
|
-
return makeCrud(getCollection()).getNote(id,
|
|
883
|
+
async function getNote(id, reader) {
|
|
884
|
+
return makeCrud(getCollection()).getNote(id, reader);
|
|
755
885
|
}
|
|
756
886
|
async function updateNote(note) {
|
|
757
887
|
return makeCrud(getCollection()).updateNote(note);
|
|
@@ -762,8 +892,8 @@ async function listNotes(agentId, subject) {
|
|
|
762
892
|
async function deleteNote(id) {
|
|
763
893
|
return makeCrud(getCollection()).deleteNote(id);
|
|
764
894
|
}
|
|
765
|
-
async function invalidateNote(id,
|
|
766
|
-
return makeCrud(getCollection()).invalidateNote(id,
|
|
895
|
+
async function invalidateNote(id, caller) {
|
|
896
|
+
return makeCrud(getCollection()).invalidateNote(id, caller);
|
|
767
897
|
}
|
|
768
898
|
async function patchNotePayload(id, fields) {
|
|
769
899
|
return makeCrud(getCollection()).patchNotePayload(id, fields);
|
|
@@ -1515,7 +1645,7 @@ async function addMemory(content, agentId = "main", opts) {
|
|
|
1515
1645
|
if (topMatch.length > 0 && topMatch[0].score >= 0.85) {
|
|
1516
1646
|
if (canWrite(topMatch[0].note, agentId)) {
|
|
1517
1647
|
console.log(`[add] dedup: high-sim match (sim=${topMatch[0].score.toFixed(3)}), updating existing`);
|
|
1518
|
-
await ctx.updateNoteContent(topMatch[0].note.id, content, embedding, hash);
|
|
1648
|
+
await ctx.updateNoteContent(topMatch[0].note.id, content, embedding, hash, agentId);
|
|
1519
1649
|
return topMatch[0].note.id;
|
|
1520
1650
|
}
|
|
1521
1651
|
console.log(
|
|
@@ -1587,7 +1717,7 @@ async function addMemory(content, agentId = "main", opts) {
|
|
|
1587
1717
|
note.links = linkedIds;
|
|
1588
1718
|
await ctx.updateNote(note);
|
|
1589
1719
|
for (const lid of linkedIds) {
|
|
1590
|
-
const linked = await ctx.getNote(lid);
|
|
1720
|
+
const linked = await ctx.getNote(lid, agentId);
|
|
1591
1721
|
if (linked && !linked.links.includes(note.id)) {
|
|
1592
1722
|
if (!canWrite(linked, agentId)) {
|
|
1593
1723
|
console.log(`[link] back-link into ${lid.slice(0, 8)} skipped \u2014 not writable by ${logSafe(agentId)}`);
|
|
@@ -1600,7 +1730,7 @@ async function addMemory(content, agentId = "main", opts) {
|
|
|
1600
1730
|
if (shouldRunEvolution()) {
|
|
1601
1731
|
console.log(` [evo] threshold reached, running evolution for ${Math.min(linkedIds.length, 3)} linked notes`);
|
|
1602
1732
|
for (const lid of linkedIds.slice(0, 3)) {
|
|
1603
|
-
const linked = await ctx.getNote(lid);
|
|
1733
|
+
const linked = await ctx.getNote(lid, agentId);
|
|
1604
1734
|
if (!linked) continue;
|
|
1605
1735
|
if (!canWrite(linked, agentId)) {
|
|
1606
1736
|
console.log(` [evo] skipping ${lid.slice(0, 8)} \u2014 not writable by ${logSafe(agentId)}`);
|
|
@@ -1753,7 +1883,7 @@ async function searchMemory(query, topK = 5, agentId = "main", opts) {
|
|
|
1753
1883
|
const allNotes = await ctx.listNotes(agentId, subject);
|
|
1754
1884
|
const bm25State = buildBM25(allNotes);
|
|
1755
1885
|
const queryTokens = simpleTokenize(query);
|
|
1756
|
-
const bm25Ranked = bm25Score(bm25State, queryTokens).slice(0, n);
|
|
1886
|
+
const bm25Ranked = bm25Score(bm25State, queryTokens).filter(([, score]) => score > 0).slice(0, n);
|
|
1757
1887
|
const merged = rrfMerge(
|
|
1758
1888
|
embResults.map((r) => r.note.id),
|
|
1759
1889
|
bm25Ranked.map((r) => r[0])
|
|
@@ -1776,6 +1906,7 @@ async function searchMemory(query, topK = 5, agentId = "main", opts) {
|
|
|
1776
1906
|
const visitedIds = new Set(topIds);
|
|
1777
1907
|
const bfsQueue = useBfs ? topIds.map((id) => ({ id, hop: 0 })) : [];
|
|
1778
1908
|
const bfsExtra = [];
|
|
1909
|
+
const bfsSimMap = /* @__PURE__ */ new Map();
|
|
1779
1910
|
while (bfsQueue.length > 0 && bfsExtra.length < BFS_MAX_EXPAND) {
|
|
1780
1911
|
const item = bfsQueue.shift();
|
|
1781
1912
|
if (item.hop >= BFS_MAX_HOPS) continue;
|
|
@@ -1786,10 +1917,9 @@ async function searchMemory(query, topK = 5, agentId = "main", opts) {
|
|
|
1786
1917
|
visitedIds.add(linkedId);
|
|
1787
1918
|
const linked = noteMap.get(linkedId);
|
|
1788
1919
|
if (!linked || linked.is_active === false) continue;
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
}
|
|
1920
|
+
const sim = cosineSimilarity(queryEmbedding, linked.embedding);
|
|
1921
|
+
if (bfsSimThreshold > 0 && sim < bfsSimThreshold) continue;
|
|
1922
|
+
bfsSimMap.set(linkedId, sim);
|
|
1793
1923
|
bfsExtra.push(linkedId);
|
|
1794
1924
|
bfsQueue.push({ id: linkedId, hop: item.hop + 1 });
|
|
1795
1925
|
if (bfsExtra.length >= BFS_MAX_EXPAND) break;
|
|
@@ -1805,7 +1935,11 @@ async function searchMemory(query, topK = 5, agentId = "main", opts) {
|
|
|
1805
1935
|
const embSimMap = new Map(embResults.map((r) => [r.note.id, r.score]));
|
|
1806
1936
|
const rrfMap = new Map(boostedMerged.map(([id, score]) => [id, score]));
|
|
1807
1937
|
const results = [];
|
|
1808
|
-
|
|
1938
|
+
const ordered = [
|
|
1939
|
+
...filteredTopIds.map((id) => [id, "match"]),
|
|
1940
|
+
...bfsExtra.map((id) => [id, "link"])
|
|
1941
|
+
];
|
|
1942
|
+
for (const [id, via] of ordered) {
|
|
1809
1943
|
const note = noteMap.get(id);
|
|
1810
1944
|
if (!note) continue;
|
|
1811
1945
|
results.push({
|
|
@@ -1816,8 +1950,9 @@ async function searchMemory(query, topK = 5, agentId = "main", opts) {
|
|
|
1816
1950
|
keywords: note.keywords,
|
|
1817
1951
|
links: note.links,
|
|
1818
1952
|
timestamp: note.timestamp,
|
|
1819
|
-
similarity: embSimMap.get(id) ?? 0,
|
|
1953
|
+
similarity: embSimMap.get(id) ?? bfsSimMap.get(id) ?? 0,
|
|
1820
1954
|
rrf: rrfMap.get(id) ?? 0,
|
|
1955
|
+
via,
|
|
1821
1956
|
topics: note.topics ?? [],
|
|
1822
1957
|
note_type: note.note_type ?? "memory"
|
|
1823
1958
|
});
|
|
@@ -1874,7 +2009,7 @@ async function mergeSimilarNotes(agentId, storageCtx) {
|
|
|
1874
2009
|
const mergedContent = judgment.mergedContent || pendingNote.content;
|
|
1875
2010
|
const newEmbedding = await encode(buildEmbedText({ ...bestNeighbor, content: mergedContent }));
|
|
1876
2011
|
const newHash = (0, import_crypto.createHash)("md5").update(mergedContent).digest("hex");
|
|
1877
|
-
await ctx.updateNoteContent(bestNeighbor.id, mergedContent, newEmbedding, newHash);
|
|
2012
|
+
await ctx.updateNoteContent(bestNeighbor.id, mergedContent, newEmbedding, newHash, agentId);
|
|
1878
2013
|
await ctx.patchNotePayload(bestNeighbor.id, {
|
|
1879
2014
|
evolution_history: JSON.stringify(oldHistory),
|
|
1880
2015
|
evolution_type: "EVOLVE"
|
|
@@ -1898,7 +2033,7 @@ async function mergeSimilarNotes(agentId, storageCtx) {
|
|
|
1898
2033
|
const mergedContent = judgment.mergedContent || `${bestNeighbor.content}\uFF1B${pendingNote.content}`;
|
|
1899
2034
|
const newEmbedding = await encode(buildEmbedText({ ...bestNeighbor, content: mergedContent }));
|
|
1900
2035
|
const newHash = (0, import_crypto.createHash)("md5").update(mergedContent).digest("hex");
|
|
1901
|
-
await ctx.updateNoteContent(bestNeighbor.id, mergedContent, newEmbedding, newHash);
|
|
2036
|
+
await ctx.updateNoteContent(bestNeighbor.id, mergedContent, newEmbedding, newHash, agentId);
|
|
1902
2037
|
await ctx.patchNotePayload(bestNeighbor.id, {
|
|
1903
2038
|
evolution_history: JSON.stringify(oldHistory),
|
|
1904
2039
|
evolution_type: "EXPAND"
|
|
@@ -1938,7 +2073,7 @@ async function mergeSimilarNotes(agentId, storageCtx) {
|
|
|
1938
2073
|
const [keepNote, dropNote] = noteA.content.length >= noteB.content.length ? [noteA, noteB] : [noteB, noteA];
|
|
1939
2074
|
const newEmbedding = await encode(result.merged);
|
|
1940
2075
|
const newHash = (0, import_crypto.createHash)("md5").update(result.merged).digest("hex");
|
|
1941
|
-
await ctx.updateNoteContent(keepNote.id, result.merged, newEmbedding, newHash);
|
|
2076
|
+
await ctx.updateNoteContent(keepNote.id, result.merged, newEmbedding, newHash, agentId);
|
|
1942
2077
|
await ctx.deleteNote(dropNote.id);
|
|
1943
2078
|
deletedIds.add(dropNote.id);
|
|
1944
2079
|
mergedCount++;
|
|
@@ -2047,7 +2182,7 @@ async function consolidateMemories(agentId, logger, storageCtx) {
|
|
|
2047
2182
|
action: "consolidate"
|
|
2048
2183
|
});
|
|
2049
2184
|
await ctx.updateNote(keepNote);
|
|
2050
|
-
await ctx.invalidateNote(dropNote.id);
|
|
2185
|
+
await ctx.invalidateNote(dropNote.id, agentId);
|
|
2051
2186
|
await ctx.replaceLinkReferences(dropNote.id, keepNote.id, agentId);
|
|
2052
2187
|
logMergeToFile(keepNote.id, dropNote.id, keepNote.content);
|
|
2053
2188
|
processedIds.add(keepNote.id);
|
|
@@ -2342,8 +2477,19 @@ async function migrateCollection(opts) {
|
|
|
2342
2477
|
);
|
|
2343
2478
|
if (dryRun) {
|
|
2344
2479
|
log("[migrate] dry run \u2014 nothing written. Pass dryRun: false to apply.");
|
|
2345
|
-
return {
|
|
2480
|
+
return {
|
|
2481
|
+
total: notes.length,
|
|
2482
|
+
missingDerived,
|
|
2483
|
+
refreshed: 0,
|
|
2484
|
+
migrated: 0,
|
|
2485
|
+
skipped: 0,
|
|
2486
|
+
sourceDim,
|
|
2487
|
+
targetDim,
|
|
2488
|
+
model,
|
|
2489
|
+
dryRun: true
|
|
2490
|
+
};
|
|
2346
2491
|
}
|
|
2492
|
+
let alreadyDone = /* @__PURE__ */ new Set();
|
|
2347
2493
|
const existingTargetDim = await collectionDimRaw(to);
|
|
2348
2494
|
if (existingTargetDim === null) {
|
|
2349
2495
|
await createCollectionRaw(to, targetDim);
|
|
@@ -2352,9 +2498,17 @@ async function migrateCollection(opts) {
|
|
|
2352
2498
|
if (existingTargetDim !== targetDim) {
|
|
2353
2499
|
throw new Error(`migrate: target "${to}" exists at ${existingTargetDim}d but the model produces ${targetDim}d`);
|
|
2354
2500
|
}
|
|
2355
|
-
const
|
|
2356
|
-
if (
|
|
2357
|
-
|
|
2501
|
+
const present = await scrollIdsRaw(to);
|
|
2502
|
+
if (present.size > 0) {
|
|
2503
|
+
const sourceIds = new Set(notes.map((n) => n.id));
|
|
2504
|
+
const foreign = [...present].filter((id) => !sourceIds.has(id));
|
|
2505
|
+
if (foreign.length > 0) {
|
|
2506
|
+
throw new Error(
|
|
2507
|
+
`migrate: target "${to}" holds ${foreign.length} point(s) that are not in "${from}" (e.g. ${foreign[0]}). That is not an interrupted migration \u2014 use a different target.`
|
|
2508
|
+
);
|
|
2509
|
+
}
|
|
2510
|
+
alreadyDone = present;
|
|
2511
|
+
log(`[migrate] resuming: ${present.size} of ${notes.length} already in ${to}`);
|
|
2358
2512
|
}
|
|
2359
2513
|
}
|
|
2360
2514
|
let refreshed = 0;
|
|
@@ -2368,6 +2522,7 @@ async function migrateCollection(opts) {
|
|
|
2368
2522
|
buffer = [];
|
|
2369
2523
|
};
|
|
2370
2524
|
for (const note of notes) {
|
|
2525
|
+
if (alreadyDone.has(note.id)) continue;
|
|
2371
2526
|
if (refreshFields && missingDerivedFields(note)) {
|
|
2372
2527
|
try {
|
|
2373
2528
|
const built = await llmConstructNote(note.content);
|
|
@@ -2383,7 +2538,7 @@ async function migrateCollection(opts) {
|
|
|
2383
2538
|
buffer.push(point);
|
|
2384
2539
|
if (buffer.length >= BATCH) {
|
|
2385
2540
|
await flush();
|
|
2386
|
-
log(`[migrate] ${migrated}/${notes.length}`);
|
|
2541
|
+
log(`[migrate] ${alreadyDone.size + migrated}/${notes.length}`);
|
|
2387
2542
|
}
|
|
2388
2543
|
}
|
|
2389
2544
|
await flush();
|
|
@@ -2391,10 +2546,51 @@ async function migrateCollection(opts) {
|
|
|
2391
2546
|
if (finalCount !== notes.length) {
|
|
2392
2547
|
warn(`[migrate] target holds ${finalCount} point(s) but the source had ${notes.length} \u2014 check before switching`);
|
|
2393
2548
|
}
|
|
2394
|
-
log(
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2549
|
+
log(`[migrate] done: ${migrated} written, ${alreadyDone.size} already present, ${refreshed} re-extracted.`);
|
|
2550
|
+
return {
|
|
2551
|
+
total: notes.length,
|
|
2552
|
+
missingDerived,
|
|
2553
|
+
refreshed,
|
|
2554
|
+
migrated,
|
|
2555
|
+
skipped: alreadyDone.size,
|
|
2556
|
+
sourceDim,
|
|
2557
|
+
targetDim,
|
|
2558
|
+
model,
|
|
2559
|
+
dryRun: false
|
|
2560
|
+
};
|
|
2561
|
+
}
|
|
2562
|
+
async function switchToMigrated(opts) {
|
|
2563
|
+
const { name, to } = opts;
|
|
2564
|
+
const log = opts.logger?.info ?? ((m) => console.log(m));
|
|
2565
|
+
if (name === to) throw new Error(`switch: "${name}" and "${to}" are the same collection`);
|
|
2566
|
+
let snap;
|
|
2567
|
+
const already = await resolveAliasRaw(name);
|
|
2568
|
+
if (already === to) {
|
|
2569
|
+
log(`[switch] "${name}" already points at "${to}" \u2014 nothing to do`);
|
|
2570
|
+
return { name, to, moved: await countPointsRaw(to) };
|
|
2571
|
+
}
|
|
2572
|
+
const targetCount = await countPointsRaw(to);
|
|
2573
|
+
if (targetCount === 0) throw new Error(`switch: "${to}" is empty \u2014 migrate into it first`);
|
|
2574
|
+
if (already === null) {
|
|
2575
|
+
const sourceCount = await countPointsRaw(name);
|
|
2576
|
+
if (targetCount < sourceCount) {
|
|
2577
|
+
throw new Error(
|
|
2578
|
+
`switch: "${to}" holds ${targetCount} point(s) but "${name}" still holds ${sourceCount}. The migration is not finished \u2014 run it again before switching.`
|
|
2579
|
+
);
|
|
2580
|
+
}
|
|
2581
|
+
log(`[switch] verified ${targetCount} in "${to}" against ${sourceCount} in "${name}"`);
|
|
2582
|
+
if (opts.snapshot !== false) {
|
|
2583
|
+
snap = await snapshotCollectionRaw(name);
|
|
2584
|
+
log(`[switch] snapshotted "${name}" \u2192 ${snap.name} (${(snap.size / 1e6).toFixed(0)} MB)`);
|
|
2585
|
+
}
|
|
2586
|
+
await deleteCollectionRaw(name);
|
|
2587
|
+
log(`[switch] dropped "${name}"`);
|
|
2588
|
+
await createAliasRaw(name, to);
|
|
2589
|
+
} else {
|
|
2590
|
+
await setAliasRaw(name, to);
|
|
2591
|
+
}
|
|
2592
|
+
log(`[switch] "${name}" now resolves to "${to}"`);
|
|
2593
|
+
return { name, to, moved: targetCount, snapshot: snap };
|
|
2398
2594
|
}
|
|
2399
2595
|
|
|
2400
2596
|
// src/crud-guard.ts
|
|
@@ -2416,6 +2612,10 @@ function isPlausibleUpdateTarget(newEmbedding, targetEmbedding, minSimilarity) {
|
|
|
2416
2612
|
DEFAULT_EMBEDDING_MODEL,
|
|
2417
2613
|
EmbeddingDimensionMismatchError,
|
|
2418
2614
|
EmbeddingModelMismatchError,
|
|
2615
|
+
LEGACY_DEFAULT_DIM,
|
|
2616
|
+
LEGACY_DEFAULT_EMBEDDING_MODEL,
|
|
2617
|
+
MixedEmbeddingModelsError,
|
|
2618
|
+
SYSTEM_ACTOR,
|
|
2419
2619
|
addEpisodic,
|
|
2420
2620
|
addMemory,
|
|
2421
2621
|
canRead,
|
|
@@ -2450,6 +2650,7 @@ function isPlausibleUpdateTarget(newEmbedding, targetEmbedding, minSimilarity) {
|
|
|
2450
2650
|
resolveCrudUpdateMinSim,
|
|
2451
2651
|
scanLowQuality,
|
|
2452
2652
|
searchMemory,
|
|
2653
|
+
switchToMigrated,
|
|
2453
2654
|
updateNote
|
|
2454
2655
|
});
|
|
2455
2656
|
//# sourceMappingURL=index.cjs.map
|