@amemhq/core 1.0.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/chunk-K6WZTDM7.js +2353 -0
- package/dist/chunk-K6WZTDM7.js.map +1 -0
- package/dist/cli-migrate.cjs +1086 -0
- package/dist/cli-migrate.cjs.map +1 -0
- package/dist/cli-migrate.d.cts +30 -0
- package/dist/cli-migrate.d.ts +30 -0
- package/dist/cli-migrate.js +147 -0
- package/dist/cli-migrate.js.map +1 -0
- package/dist/index.cjs +289 -57
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +181 -28
- package/dist/index.d.ts +181 -28
- package/dist/index.js +82 -2120
- package/dist/index.js.map +1 -1
- package/package.json +4 -1
package/dist/index.cjs
CHANGED
|
@@ -33,6 +33,11 @@ __export(index_exports, {
|
|
|
33
33
|
DEFAULT_CRUD_UPDATE_MIN_SIM: () => DEFAULT_CRUD_UPDATE_MIN_SIM,
|
|
34
34
|
DEFAULT_EMBEDDING_MODEL: () => DEFAULT_EMBEDDING_MODEL,
|
|
35
35
|
EmbeddingDimensionMismatchError: () => EmbeddingDimensionMismatchError,
|
|
36
|
+
EmbeddingModelMismatchError: () => EmbeddingModelMismatchError,
|
|
37
|
+
LEGACY_DEFAULT_DIM: () => LEGACY_DEFAULT_DIM,
|
|
38
|
+
LEGACY_DEFAULT_EMBEDDING_MODEL: () => LEGACY_DEFAULT_EMBEDDING_MODEL,
|
|
39
|
+
MixedEmbeddingModelsError: () => MixedEmbeddingModelsError,
|
|
40
|
+
SYSTEM_ACTOR: () => SYSTEM_ACTOR,
|
|
36
41
|
addEpisodic: () => addEpisodic,
|
|
37
42
|
addMemory: () => addMemory,
|
|
38
43
|
canRead: () => canRead,
|
|
@@ -47,7 +52,9 @@ __export(index_exports, {
|
|
|
47
52
|
encode: () => encode,
|
|
48
53
|
ensureCollection: () => ensureCollection,
|
|
49
54
|
generateReviewBatch: () => generateReviewBatch,
|
|
55
|
+
getEmbeddingDevice: () => getEmbeddingDevice,
|
|
50
56
|
getEmbeddingDim: () => getEmbeddingDim,
|
|
57
|
+
getEmbeddingDtype: () => getEmbeddingDtype,
|
|
51
58
|
getEmbeddingModel: () => getEmbeddingModel,
|
|
52
59
|
getEmbeddingPooling: () => getEmbeddingPooling,
|
|
53
60
|
getNote: () => getNote,
|
|
@@ -65,6 +72,7 @@ __export(index_exports, {
|
|
|
65
72
|
resolveCrudUpdateMinSim: () => resolveCrudUpdateMinSim,
|
|
66
73
|
scanLowQuality: () => scanLowQuality,
|
|
67
74
|
searchMemory: () => searchMemory,
|
|
75
|
+
switchToMigrated: () => switchToMigrated,
|
|
68
76
|
updateNote: () => updateNote
|
|
69
77
|
});
|
|
70
78
|
module.exports = __toCommonJS(index_exports);
|
|
@@ -83,11 +91,25 @@ function getDataDir() {
|
|
|
83
91
|
// src/embedding.ts
|
|
84
92
|
var pipeline = null;
|
|
85
93
|
var extractor = null;
|
|
86
|
-
var
|
|
94
|
+
var loadedKey = null;
|
|
87
95
|
var cachedDim = null;
|
|
88
|
-
var DEFAULT_EMBEDDING_MODEL = "Xenova/
|
|
96
|
+
var DEFAULT_EMBEDDING_MODEL = "Xenova/bge-m3";
|
|
97
|
+
var LEGACY_DEFAULT_EMBEDDING_MODEL = "Xenova/paraphrase-multilingual-MiniLM-L12-v2";
|
|
98
|
+
var LEGACY_DEFAULT_DIM = 384;
|
|
99
|
+
var DEFAULT_MODEL_DTYPE = "fp16";
|
|
100
|
+
var pinnedModel = null;
|
|
101
|
+
function pinEmbeddingModel(model) {
|
|
102
|
+
if (pinnedModel === model) return;
|
|
103
|
+
pinnedModel = model;
|
|
104
|
+
extractor = null;
|
|
105
|
+
loadedKey = null;
|
|
106
|
+
cachedDim = null;
|
|
107
|
+
}
|
|
108
|
+
function getPinnedEmbeddingModel() {
|
|
109
|
+
return pinnedModel;
|
|
110
|
+
}
|
|
89
111
|
function getEmbeddingModel() {
|
|
90
|
-
return process.env.AMEM_EMBED_MODEL?.trim() || DEFAULT_EMBEDDING_MODEL;
|
|
112
|
+
return process.env.AMEM_EMBED_MODEL?.trim() || pinnedModel || DEFAULT_EMBEDDING_MODEL;
|
|
91
113
|
}
|
|
92
114
|
var CLS_POOLED_MODELS = /* @__PURE__ */ new Set([
|
|
93
115
|
"bge-m3",
|
|
@@ -108,22 +130,39 @@ function getEmbeddingPooling() {
|
|
|
108
130
|
const basename2 = getEmbeddingModel().split("/").pop()?.toLowerCase() ?? "";
|
|
109
131
|
return CLS_POOLED_MODELS.has(basename2) ? "cls" : "mean";
|
|
110
132
|
}
|
|
133
|
+
function getEmbeddingDevice() {
|
|
134
|
+
return process.env.AMEM_EMBED_DEVICE?.trim() || void 0;
|
|
135
|
+
}
|
|
136
|
+
function getEmbeddingDtype() {
|
|
137
|
+
const explicit = process.env.AMEM_EMBED_DTYPE?.trim();
|
|
138
|
+
if (explicit) return explicit;
|
|
139
|
+
return getEmbeddingModel() === DEFAULT_EMBEDDING_MODEL ? DEFAULT_MODEL_DTYPE : void 0;
|
|
140
|
+
}
|
|
141
|
+
function extractorKey() {
|
|
142
|
+
return `${getEmbeddingModel()}|${getEmbeddingDevice() ?? ""}|${getEmbeddingDtype() ?? ""}`;
|
|
143
|
+
}
|
|
111
144
|
async function getExtractor() {
|
|
112
|
-
const wanted =
|
|
113
|
-
if (extractor &&
|
|
145
|
+
const wanted = extractorKey();
|
|
146
|
+
if (extractor && loadedKey === wanted) return extractor;
|
|
114
147
|
if (!pipeline) {
|
|
115
148
|
const mod = await import("@huggingface/transformers");
|
|
116
149
|
pipeline = mod.pipeline;
|
|
117
150
|
}
|
|
118
|
-
|
|
119
|
-
|
|
151
|
+
const device = getEmbeddingDevice();
|
|
152
|
+
const dtype = getEmbeddingDtype();
|
|
153
|
+
extractor = await pipeline("feature-extraction", getEmbeddingModel(), {
|
|
154
|
+
revision: "main",
|
|
155
|
+
// Omitted entirely when unset, so an unconfigured install gets exactly the
|
|
156
|
+
// library defaults it got before these existed.
|
|
157
|
+
...device ? { device } : {},
|
|
158
|
+
...dtype ? { dtype } : {}
|
|
120
159
|
});
|
|
121
|
-
|
|
160
|
+
loadedKey = wanted;
|
|
122
161
|
cachedDim = null;
|
|
123
162
|
return extractor;
|
|
124
163
|
}
|
|
125
164
|
async function getEmbeddingDim() {
|
|
126
|
-
if (cachedDim !== null &&
|
|
165
|
+
if (cachedDim !== null && loadedKey === extractorKey()) return cachedDim;
|
|
127
166
|
const probe = await encode("dimension probe");
|
|
128
167
|
cachedDim = probe.length;
|
|
129
168
|
return cachedDim;
|
|
@@ -194,6 +233,7 @@ var fs2 = __toESM(require("fs"), 1);
|
|
|
194
233
|
var path3 = __toESM(require("path"), 1);
|
|
195
234
|
|
|
196
235
|
// src/auth.ts
|
|
236
|
+
var SYSTEM_ACTOR = "__amem_system__";
|
|
197
237
|
function canWrite(note, callerAgentId) {
|
|
198
238
|
return note.owner === callerAgentId || note.writers.includes(callerAgentId) || note.writers.includes("*");
|
|
199
239
|
}
|
|
@@ -208,7 +248,7 @@ var EmbeddingDimensionMismatchError = class extends Error {
|
|
|
208
248
|
constructor(collection, collectionDim, modelDim, model) {
|
|
209
249
|
super(
|
|
210
250
|
`Collection "${collection}" stores ${collectionDim}-dimension vectors, but the embedding model "${model}" produces ${modelDim}. Qdrant fixes a collection's vector size at creation and cannot change it, so writes and searches would both fail.
|
|
211
|
-
Either set AMEM_EMBED_MODEL back to the model this collection was built with, or
|
|
251
|
+
Either set AMEM_EMBED_MODEL back to the model this collection was built with, or ${migrationHint(collection, model)}`
|
|
212
252
|
);
|
|
213
253
|
this.collection = collection;
|
|
214
254
|
this.collectionDim = collectionDim;
|
|
@@ -221,6 +261,54 @@ Either set AMEM_EMBED_MODEL back to the model this collection was built with, or
|
|
|
221
261
|
modelDim;
|
|
222
262
|
model;
|
|
223
263
|
};
|
|
264
|
+
function migrationHint(collection, targetModel) {
|
|
265
|
+
return `migrate onto it:
|
|
266
|
+
|
|
267
|
+
AMEM_EMBED_MODEL=${targetModel} \\
|
|
268
|
+
npx --package=@amemhq/core amem-migrate --from-collection ${collection}
|
|
269
|
+
|
|
270
|
+
That only reports; it takes --apply to write anything, and "${collection}" is read either way. The new store ends up behind the name you already use, so there is nothing to change in your config afterwards. See https://amem.owo.lc/reference/embedding-models.`;
|
|
271
|
+
}
|
|
272
|
+
var EmbeddingModelMismatchError = class extends Error {
|
|
273
|
+
constructor(collection, collectionModel, configuredModel) {
|
|
274
|
+
super(
|
|
275
|
+
`Collection "${collection}" was built with the embedding model "${collectionModel}", but this process is configured for "${configuredModel}". Both produce vectors of the same width, so nothing would fail \u2014 searches would just quietly compare vectors from two different models.
|
|
276
|
+
Either set AMEM_EMBED_MODEL back to "${collectionModel}", or ` + migrationHint(collection, configuredModel)
|
|
277
|
+
);
|
|
278
|
+
this.collection = collection;
|
|
279
|
+
this.collectionModel = collectionModel;
|
|
280
|
+
this.configuredModel = configuredModel;
|
|
281
|
+
this.name = "EmbeddingModelMismatchError";
|
|
282
|
+
}
|
|
283
|
+
collection;
|
|
284
|
+
collectionModel;
|
|
285
|
+
configuredModel;
|
|
286
|
+
};
|
|
287
|
+
var MixedEmbeddingModelsError = class extends Error {
|
|
288
|
+
constructor(collection, wanted, inUse) {
|
|
289
|
+
super(
|
|
290
|
+
`Collection "${collection}" was built with "${wanted}", but this process is already embedding with "${inUse}" for another collection. One process can only use one model.
|
|
291
|
+
Migrate the remaining collections so they all agree:
|
|
292
|
+
|
|
293
|
+
npx --package=@amemhq/core amem-migrate --from-collection ${collection}
|
|
294
|
+
|
|
295
|
+
Or set AMEM_EMBED_MODEL to pin every collection to one model, which is only correct if they really were all built with it.`
|
|
296
|
+
);
|
|
297
|
+
this.collection = collection;
|
|
298
|
+
this.wanted = wanted;
|
|
299
|
+
this.inUse = inUse;
|
|
300
|
+
this.name = "MixedEmbeddingModelsError";
|
|
301
|
+
}
|
|
302
|
+
collection;
|
|
303
|
+
wanted;
|
|
304
|
+
inUse;
|
|
305
|
+
};
|
|
306
|
+
async function recordCollectionModel(collection, model) {
|
|
307
|
+
try {
|
|
308
|
+
await qdrant("PATCH", `/collections/${collection}`, { metadata: { embedding_model: model } });
|
|
309
|
+
} catch {
|
|
310
|
+
}
|
|
311
|
+
}
|
|
224
312
|
async function qdrant(method, path5, body) {
|
|
225
313
|
const res = await fetch(`${QDRANT_URL}${path5}`, {
|
|
226
314
|
method,
|
|
@@ -257,12 +345,40 @@ async function ensureCollection(collectionName) {
|
|
|
257
345
|
}
|
|
258
346
|
if (existing) {
|
|
259
347
|
const collectionDim = existing.config?.params?.vectors?.size;
|
|
348
|
+
const recorded = existing.config?.metadata?.embedding_model;
|
|
349
|
+
const explicit = process.env.AMEM_EMBED_MODEL?.trim();
|
|
350
|
+
const inferLegacy = !explicit && recorded === void 0 && collectionDim === LEGACY_DEFAULT_DIM;
|
|
351
|
+
if (!explicit) {
|
|
352
|
+
const wanted = (
|
|
353
|
+
// The collection says what built it, which outranks whatever the shipped
|
|
354
|
+
// default happens to be today. This is what keeps changing the default
|
|
355
|
+
// from breaking every install that already has data.
|
|
356
|
+
typeof recorded === "string" ? recorded : inferLegacy ? LEGACY_DEFAULT_EMBEDDING_MODEL : DEFAULT_EMBEDDING_MODEL
|
|
357
|
+
);
|
|
358
|
+
const inUse = getPinnedEmbeddingModel();
|
|
359
|
+
if (inUse !== null && inUse !== wanted) throw new MixedEmbeddingModelsError(col, wanted, inUse);
|
|
360
|
+
pinEmbeddingModel(wanted);
|
|
361
|
+
if (wanted === LEGACY_DEFAULT_EMBEDDING_MODEL) {
|
|
362
|
+
console.warn(
|
|
363
|
+
`[amem] "${col}" is on ${LEGACY_DEFAULT_EMBEDDING_MODEL} (${LEGACY_DEFAULT_DIM}-dim).
|
|
364
|
+
[amem] ${DEFAULT_EMBEDDING_MODEL} reads 8192 tokens where that one stops at 128, so anything longer is being truncated before it reaches the vector.
|
|
365
|
+
[amem] To move: npx --package=@amemhq/core amem-migrate --from-collection ${col}`
|
|
366
|
+
);
|
|
367
|
+
}
|
|
368
|
+
}
|
|
260
369
|
if (typeof collectionDim === "number") {
|
|
261
370
|
const modelDim = await getEmbeddingDim();
|
|
262
371
|
if (collectionDim !== modelDim) {
|
|
263
372
|
throw new EmbeddingDimensionMismatchError(col, collectionDim, modelDim, getEmbeddingModel());
|
|
264
373
|
}
|
|
265
374
|
}
|
|
375
|
+
const current = getEmbeddingModel();
|
|
376
|
+
if (typeof recorded === "string" && recorded !== current) {
|
|
377
|
+
throw new EmbeddingModelMismatchError(col, recorded, current);
|
|
378
|
+
}
|
|
379
|
+
if (recorded === void 0 && !inferLegacy) {
|
|
380
|
+
await recordCollectionModel(col, current);
|
|
381
|
+
}
|
|
266
382
|
markReady();
|
|
267
383
|
return;
|
|
268
384
|
}
|
|
@@ -274,6 +390,9 @@ async function ensureCollection(collectionName) {
|
|
|
274
390
|
} catch (err) {
|
|
275
391
|
if (!(err instanceof Error) || !err.message.includes("already exists")) throw err;
|
|
276
392
|
}
|
|
393
|
+
const created = getEmbeddingModel();
|
|
394
|
+
await recordCollectionModel(col, created);
|
|
395
|
+
if (!process.env.AMEM_EMBED_MODEL?.trim()) pinEmbeddingModel(created);
|
|
277
396
|
await qdrant("PUT", `/collections/${col}/index`, {
|
|
278
397
|
field_name: "agent_id",
|
|
279
398
|
field_schema: "keyword"
|
|
@@ -319,6 +438,7 @@ async function collectionDimRaw(collection) {
|
|
|
319
438
|
}
|
|
320
439
|
async function createCollectionRaw(collection, size) {
|
|
321
440
|
await qdrant("PUT", `/collections/${collection}`, { vectors: { size, distance: "Cosine" } });
|
|
441
|
+
await recordCollectionModel(collection, getEmbeddingModel());
|
|
322
442
|
for (const field_name of ["agent_id", "hash", "topics", "subjects"]) {
|
|
323
443
|
await qdrant("PUT", `/collections/${collection}/index`, { field_name, field_schema: "keyword" });
|
|
324
444
|
}
|
|
@@ -326,6 +446,43 @@ async function createCollectionRaw(collection, size) {
|
|
|
326
446
|
async function upsertPointsRaw(collection, points) {
|
|
327
447
|
await qdrant("PUT", `/collections/${collection}/points?wait=true`, { points });
|
|
328
448
|
}
|
|
449
|
+
async function scrollIdsRaw(collection, limit = 1e4) {
|
|
450
|
+
const ids = /* @__PURE__ */ new Set();
|
|
451
|
+
let offset = void 0;
|
|
452
|
+
for (; ; ) {
|
|
453
|
+
const body = { with_payload: false, with_vector: false, limit };
|
|
454
|
+
if (offset !== void 0 && offset !== null) body.offset = offset;
|
|
455
|
+
const res = await qdrant("POST", `/collections/${collection}/points/scroll`, body);
|
|
456
|
+
for (const p of res.points) ids.add(String(p.id));
|
|
457
|
+
offset = res.next_page_offset;
|
|
458
|
+
if (offset === void 0 || offset === null || res.points.length === 0) break;
|
|
459
|
+
}
|
|
460
|
+
return ids;
|
|
461
|
+
}
|
|
462
|
+
async function deleteCollectionRaw(collection) {
|
|
463
|
+
await qdrant("DELETE", `/collections/${collection}`);
|
|
464
|
+
}
|
|
465
|
+
async function resolveAliasRaw(alias) {
|
|
466
|
+
try {
|
|
467
|
+
const res = await qdrant("GET", `/aliases`);
|
|
468
|
+
return res.aliases.find((a) => a.alias_name === alias)?.collection_name ?? null;
|
|
469
|
+
} catch {
|
|
470
|
+
return null;
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
async function createAliasRaw(alias, collection) {
|
|
474
|
+
await qdrant("POST", `/collections/aliases`, {
|
|
475
|
+
actions: [{ create_alias: { collection_name: collection, alias_name: alias } }]
|
|
476
|
+
});
|
|
477
|
+
}
|
|
478
|
+
async function setAliasRaw(alias, collection) {
|
|
479
|
+
await qdrant("POST", `/collections/aliases`, {
|
|
480
|
+
actions: [
|
|
481
|
+
{ delete_alias: { alias_name: alias } },
|
|
482
|
+
{ create_alias: { collection_name: collection, alias_name: alias } }
|
|
483
|
+
]
|
|
484
|
+
});
|
|
485
|
+
}
|
|
329
486
|
function noteToPoint(note) {
|
|
330
487
|
return {
|
|
331
488
|
id: note.id,
|
|
@@ -470,12 +627,15 @@ function makeCrud(collectionName, modeBIsolated = false) {
|
|
|
470
627
|
},
|
|
471
628
|
/**
|
|
472
629
|
* Story 36: this is the one read that bypasses the agent filter — it fetches
|
|
473
|
-
* straight by UUID.
|
|
474
|
-
*
|
|
475
|
-
*
|
|
476
|
-
*
|
|
630
|
+
* straight by UUID. An unreadable note comes back as `null`, indistinguishable
|
|
631
|
+
* from missing, so nothing leaks and callers already handle it.
|
|
632
|
+
*
|
|
633
|
+
* `reader` is required. It used to be optional, and omitting it skipped the
|
|
634
|
+
* check — which meant the safe behaviour was the one you had to remember to
|
|
635
|
+
* ask for. Pass `SYSTEM_ACTOR` to read as the engine itself; that reads as a
|
|
636
|
+
* deliberate act at the call site, where an absent argument did not.
|
|
477
637
|
*/
|
|
478
|
-
async getNote(id,
|
|
638
|
+
async getNote(id, reader) {
|
|
479
639
|
await ensureCollection(col);
|
|
480
640
|
try {
|
|
481
641
|
const result = await qdrant("POST", `/collections/${col}/points`, {
|
|
@@ -485,7 +645,7 @@ function makeCrud(collectionName, modeBIsolated = false) {
|
|
|
485
645
|
});
|
|
486
646
|
if (!result.length) return null;
|
|
487
647
|
const note = pointToNote(result[0]);
|
|
488
|
-
if (
|
|
648
|
+
if (reader !== SYSTEM_ACTOR && !canRead(note, reader)) return null;
|
|
489
649
|
return note;
|
|
490
650
|
} catch {
|
|
491
651
|
return null;
|
|
@@ -523,19 +683,22 @@ function makeCrud(collectionName, modeBIsolated = false) {
|
|
|
523
683
|
return pointToNote(result.points[0]);
|
|
524
684
|
},
|
|
525
685
|
/**
|
|
526
|
-
* Story 33:
|
|
527
|
-
*
|
|
528
|
-
*
|
|
529
|
-
*
|
|
530
|
-
*
|
|
531
|
-
*
|
|
686
|
+
* Story 33: enforces the writers policy. Returns false — without writing —
|
|
687
|
+
* when the caller may not write. This fetch-then-check path exists for callers
|
|
688
|
+
* that only have an id (the plugin's CRUD hook); callers already holding the
|
|
689
|
+
* note can check `canWrite` themselves and skip a round trip.
|
|
690
|
+
*
|
|
691
|
+
* `caller` is required. Pass `SYSTEM_ACTOR` for the engine's own maintenance
|
|
692
|
+
* writes. The self-read below is a genuine `SYSTEM_ACTOR` case: it fetches the
|
|
693
|
+
* note in order to decide whether the caller may write it, and gating that
|
|
694
|
+
* fetch on the same policy it exists to evaluate would be circular.
|
|
532
695
|
*/
|
|
533
|
-
async updateNoteContent(id, content, embedding, hash,
|
|
696
|
+
async updateNoteContent(id, content, embedding, hash, caller) {
|
|
534
697
|
await ensureCollection(col);
|
|
535
698
|
let existing = null;
|
|
536
|
-
if (
|
|
537
|
-
existing = await this.getNote(id);
|
|
538
|
-
if (existing && !canWrite(existing,
|
|
699
|
+
if (caller !== SYSTEM_ACTOR) {
|
|
700
|
+
existing = await this.getNote(id, SYSTEM_ACTOR);
|
|
701
|
+
if (existing && !canWrite(existing, caller)) return false;
|
|
539
702
|
}
|
|
540
703
|
await qdrant("PUT", `/collections/${col}/points/vectors?wait=true`, {
|
|
541
704
|
points: [{ id, vector: embedding }]
|
|
@@ -623,11 +786,11 @@ function makeCrud(collectionName, modeBIsolated = false) {
|
|
|
623
786
|
});
|
|
624
787
|
},
|
|
625
788
|
/** Story 33: see `updateNoteContent` — returns false, unwritten, when denied. */
|
|
626
|
-
async invalidateNote(id,
|
|
789
|
+
async invalidateNote(id, caller) {
|
|
627
790
|
await ensureCollection(col);
|
|
628
|
-
if (
|
|
629
|
-
const existing = await this.getNote(id);
|
|
630
|
-
if (existing && !canWrite(existing,
|
|
791
|
+
if (caller !== SYSTEM_ACTOR) {
|
|
792
|
+
const existing = await this.getNote(id, SYSTEM_ACTOR);
|
|
793
|
+
if (existing && !canWrite(existing, caller)) return false;
|
|
631
794
|
}
|
|
632
795
|
await qdrant("POST", `/collections/${col}/points/payload?wait=true`, {
|
|
633
796
|
payload: { is_active: false },
|
|
@@ -693,8 +856,8 @@ function makeCrud(collectionName, modeBIsolated = false) {
|
|
|
693
856
|
function createStorageContext(collectionName, modeBIsolated = false) {
|
|
694
857
|
return makeCrud(collectionName || getCollection(), modeBIsolated);
|
|
695
858
|
}
|
|
696
|
-
async function getNote(id,
|
|
697
|
-
return makeCrud(getCollection()).getNote(id,
|
|
859
|
+
async function getNote(id, reader) {
|
|
860
|
+
return makeCrud(getCollection()).getNote(id, reader);
|
|
698
861
|
}
|
|
699
862
|
async function updateNote(note) {
|
|
700
863
|
return makeCrud(getCollection()).updateNote(note);
|
|
@@ -705,8 +868,8 @@ async function listNotes(agentId, subject) {
|
|
|
705
868
|
async function deleteNote(id) {
|
|
706
869
|
return makeCrud(getCollection()).deleteNote(id);
|
|
707
870
|
}
|
|
708
|
-
async function invalidateNote(id,
|
|
709
|
-
return makeCrud(getCollection()).invalidateNote(id,
|
|
871
|
+
async function invalidateNote(id, caller) {
|
|
872
|
+
return makeCrud(getCollection()).invalidateNote(id, caller);
|
|
710
873
|
}
|
|
711
874
|
async function patchNotePayload(id, fields) {
|
|
712
875
|
return makeCrud(getCollection()).patchNotePayload(id, fields);
|
|
@@ -1458,7 +1621,7 @@ async function addMemory(content, agentId = "main", opts) {
|
|
|
1458
1621
|
if (topMatch.length > 0 && topMatch[0].score >= 0.85) {
|
|
1459
1622
|
if (canWrite(topMatch[0].note, agentId)) {
|
|
1460
1623
|
console.log(`[add] dedup: high-sim match (sim=${topMatch[0].score.toFixed(3)}), updating existing`);
|
|
1461
|
-
await ctx.updateNoteContent(topMatch[0].note.id, content, embedding, hash);
|
|
1624
|
+
await ctx.updateNoteContent(topMatch[0].note.id, content, embedding, hash, agentId);
|
|
1462
1625
|
return topMatch[0].note.id;
|
|
1463
1626
|
}
|
|
1464
1627
|
console.log(
|
|
@@ -1530,7 +1693,7 @@ async function addMemory(content, agentId = "main", opts) {
|
|
|
1530
1693
|
note.links = linkedIds;
|
|
1531
1694
|
await ctx.updateNote(note);
|
|
1532
1695
|
for (const lid of linkedIds) {
|
|
1533
|
-
const linked = await ctx.getNote(lid);
|
|
1696
|
+
const linked = await ctx.getNote(lid, agentId);
|
|
1534
1697
|
if (linked && !linked.links.includes(note.id)) {
|
|
1535
1698
|
if (!canWrite(linked, agentId)) {
|
|
1536
1699
|
console.log(`[link] back-link into ${lid.slice(0, 8)} skipped \u2014 not writable by ${logSafe(agentId)}`);
|
|
@@ -1543,7 +1706,7 @@ async function addMemory(content, agentId = "main", opts) {
|
|
|
1543
1706
|
if (shouldRunEvolution()) {
|
|
1544
1707
|
console.log(` [evo] threshold reached, running evolution for ${Math.min(linkedIds.length, 3)} linked notes`);
|
|
1545
1708
|
for (const lid of linkedIds.slice(0, 3)) {
|
|
1546
|
-
const linked = await ctx.getNote(lid);
|
|
1709
|
+
const linked = await ctx.getNote(lid, agentId);
|
|
1547
1710
|
if (!linked) continue;
|
|
1548
1711
|
if (!canWrite(linked, agentId)) {
|
|
1549
1712
|
console.log(` [evo] skipping ${lid.slice(0, 8)} \u2014 not writable by ${logSafe(agentId)}`);
|
|
@@ -1696,7 +1859,7 @@ async function searchMemory(query, topK = 5, agentId = "main", opts) {
|
|
|
1696
1859
|
const allNotes = await ctx.listNotes(agentId, subject);
|
|
1697
1860
|
const bm25State = buildBM25(allNotes);
|
|
1698
1861
|
const queryTokens = simpleTokenize(query);
|
|
1699
|
-
const bm25Ranked = bm25Score(bm25State, queryTokens).slice(0, n);
|
|
1862
|
+
const bm25Ranked = bm25Score(bm25State, queryTokens).filter(([, score]) => score > 0).slice(0, n);
|
|
1700
1863
|
const merged = rrfMerge(
|
|
1701
1864
|
embResults.map((r) => r.note.id),
|
|
1702
1865
|
bm25Ranked.map((r) => r[0])
|
|
@@ -1719,6 +1882,7 @@ async function searchMemory(query, topK = 5, agentId = "main", opts) {
|
|
|
1719
1882
|
const visitedIds = new Set(topIds);
|
|
1720
1883
|
const bfsQueue = useBfs ? topIds.map((id) => ({ id, hop: 0 })) : [];
|
|
1721
1884
|
const bfsExtra = [];
|
|
1885
|
+
const bfsSimMap = /* @__PURE__ */ new Map();
|
|
1722
1886
|
while (bfsQueue.length > 0 && bfsExtra.length < BFS_MAX_EXPAND) {
|
|
1723
1887
|
const item = bfsQueue.shift();
|
|
1724
1888
|
if (item.hop >= BFS_MAX_HOPS) continue;
|
|
@@ -1729,10 +1893,9 @@ async function searchMemory(query, topK = 5, agentId = "main", opts) {
|
|
|
1729
1893
|
visitedIds.add(linkedId);
|
|
1730
1894
|
const linked = noteMap.get(linkedId);
|
|
1731
1895
|
if (!linked || linked.is_active === false) continue;
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
}
|
|
1896
|
+
const sim = cosineSimilarity(queryEmbedding, linked.embedding);
|
|
1897
|
+
if (bfsSimThreshold > 0 && sim < bfsSimThreshold) continue;
|
|
1898
|
+
bfsSimMap.set(linkedId, sim);
|
|
1736
1899
|
bfsExtra.push(linkedId);
|
|
1737
1900
|
bfsQueue.push({ id: linkedId, hop: item.hop + 1 });
|
|
1738
1901
|
if (bfsExtra.length >= BFS_MAX_EXPAND) break;
|
|
@@ -1748,7 +1911,11 @@ async function searchMemory(query, topK = 5, agentId = "main", opts) {
|
|
|
1748
1911
|
const embSimMap = new Map(embResults.map((r) => [r.note.id, r.score]));
|
|
1749
1912
|
const rrfMap = new Map(boostedMerged.map(([id, score]) => [id, score]));
|
|
1750
1913
|
const results = [];
|
|
1751
|
-
|
|
1914
|
+
const ordered = [
|
|
1915
|
+
...filteredTopIds.map((id) => [id, "match"]),
|
|
1916
|
+
...bfsExtra.map((id) => [id, "link"])
|
|
1917
|
+
];
|
|
1918
|
+
for (const [id, via] of ordered) {
|
|
1752
1919
|
const note = noteMap.get(id);
|
|
1753
1920
|
if (!note) continue;
|
|
1754
1921
|
results.push({
|
|
@@ -1759,8 +1926,9 @@ async function searchMemory(query, topK = 5, agentId = "main", opts) {
|
|
|
1759
1926
|
keywords: note.keywords,
|
|
1760
1927
|
links: note.links,
|
|
1761
1928
|
timestamp: note.timestamp,
|
|
1762
|
-
similarity: embSimMap.get(id) ?? 0,
|
|
1929
|
+
similarity: embSimMap.get(id) ?? bfsSimMap.get(id) ?? 0,
|
|
1763
1930
|
rrf: rrfMap.get(id) ?? 0,
|
|
1931
|
+
via,
|
|
1764
1932
|
topics: note.topics ?? [],
|
|
1765
1933
|
note_type: note.note_type ?? "memory"
|
|
1766
1934
|
});
|
|
@@ -1817,7 +1985,7 @@ async function mergeSimilarNotes(agentId, storageCtx) {
|
|
|
1817
1985
|
const mergedContent = judgment.mergedContent || pendingNote.content;
|
|
1818
1986
|
const newEmbedding = await encode(buildEmbedText({ ...bestNeighbor, content: mergedContent }));
|
|
1819
1987
|
const newHash = (0, import_crypto.createHash)("md5").update(mergedContent).digest("hex");
|
|
1820
|
-
await ctx.updateNoteContent(bestNeighbor.id, mergedContent, newEmbedding, newHash);
|
|
1988
|
+
await ctx.updateNoteContent(bestNeighbor.id, mergedContent, newEmbedding, newHash, agentId);
|
|
1821
1989
|
await ctx.patchNotePayload(bestNeighbor.id, {
|
|
1822
1990
|
evolution_history: JSON.stringify(oldHistory),
|
|
1823
1991
|
evolution_type: "EVOLVE"
|
|
@@ -1841,7 +2009,7 @@ async function mergeSimilarNotes(agentId, storageCtx) {
|
|
|
1841
2009
|
const mergedContent = judgment.mergedContent || `${bestNeighbor.content}\uFF1B${pendingNote.content}`;
|
|
1842
2010
|
const newEmbedding = await encode(buildEmbedText({ ...bestNeighbor, content: mergedContent }));
|
|
1843
2011
|
const newHash = (0, import_crypto.createHash)("md5").update(mergedContent).digest("hex");
|
|
1844
|
-
await ctx.updateNoteContent(bestNeighbor.id, mergedContent, newEmbedding, newHash);
|
|
2012
|
+
await ctx.updateNoteContent(bestNeighbor.id, mergedContent, newEmbedding, newHash, agentId);
|
|
1845
2013
|
await ctx.patchNotePayload(bestNeighbor.id, {
|
|
1846
2014
|
evolution_history: JSON.stringify(oldHistory),
|
|
1847
2015
|
evolution_type: "EXPAND"
|
|
@@ -1881,7 +2049,7 @@ async function mergeSimilarNotes(agentId, storageCtx) {
|
|
|
1881
2049
|
const [keepNote, dropNote] = noteA.content.length >= noteB.content.length ? [noteA, noteB] : [noteB, noteA];
|
|
1882
2050
|
const newEmbedding = await encode(result.merged);
|
|
1883
2051
|
const newHash = (0, import_crypto.createHash)("md5").update(result.merged).digest("hex");
|
|
1884
|
-
await ctx.updateNoteContent(keepNote.id, result.merged, newEmbedding, newHash);
|
|
2052
|
+
await ctx.updateNoteContent(keepNote.id, result.merged, newEmbedding, newHash, agentId);
|
|
1885
2053
|
await ctx.deleteNote(dropNote.id);
|
|
1886
2054
|
deletedIds.add(dropNote.id);
|
|
1887
2055
|
mergedCount++;
|
|
@@ -1990,7 +2158,7 @@ async function consolidateMemories(agentId, logger, storageCtx) {
|
|
|
1990
2158
|
action: "consolidate"
|
|
1991
2159
|
});
|
|
1992
2160
|
await ctx.updateNote(keepNote);
|
|
1993
|
-
await ctx.invalidateNote(dropNote.id);
|
|
2161
|
+
await ctx.invalidateNote(dropNote.id, agentId);
|
|
1994
2162
|
await ctx.replaceLinkReferences(dropNote.id, keepNote.id, agentId);
|
|
1995
2163
|
logMergeToFile(keepNote.id, dropNote.id, keepNote.content);
|
|
1996
2164
|
processedIds.add(keepNote.id);
|
|
@@ -2285,8 +2453,19 @@ async function migrateCollection(opts) {
|
|
|
2285
2453
|
);
|
|
2286
2454
|
if (dryRun) {
|
|
2287
2455
|
log("[migrate] dry run \u2014 nothing written. Pass dryRun: false to apply.");
|
|
2288
|
-
return {
|
|
2456
|
+
return {
|
|
2457
|
+
total: notes.length,
|
|
2458
|
+
missingDerived,
|
|
2459
|
+
refreshed: 0,
|
|
2460
|
+
migrated: 0,
|
|
2461
|
+
skipped: 0,
|
|
2462
|
+
sourceDim,
|
|
2463
|
+
targetDim,
|
|
2464
|
+
model,
|
|
2465
|
+
dryRun: true
|
|
2466
|
+
};
|
|
2289
2467
|
}
|
|
2468
|
+
let alreadyDone = /* @__PURE__ */ new Set();
|
|
2290
2469
|
const existingTargetDim = await collectionDimRaw(to);
|
|
2291
2470
|
if (existingTargetDim === null) {
|
|
2292
2471
|
await createCollectionRaw(to, targetDim);
|
|
@@ -2295,9 +2474,17 @@ async function migrateCollection(opts) {
|
|
|
2295
2474
|
if (existingTargetDim !== targetDim) {
|
|
2296
2475
|
throw new Error(`migrate: target "${to}" exists at ${existingTargetDim}d but the model produces ${targetDim}d`);
|
|
2297
2476
|
}
|
|
2298
|
-
const
|
|
2299
|
-
if (
|
|
2300
|
-
|
|
2477
|
+
const present = await scrollIdsRaw(to);
|
|
2478
|
+
if (present.size > 0) {
|
|
2479
|
+
const sourceIds = new Set(notes.map((n) => n.id));
|
|
2480
|
+
const foreign = [...present].filter((id) => !sourceIds.has(id));
|
|
2481
|
+
if (foreign.length > 0) {
|
|
2482
|
+
throw new Error(
|
|
2483
|
+
`migrate: target "${to}" holds ${foreign.length} point(s) that are not in "${from}" (e.g. ${foreign[0]}). That is not an interrupted migration \u2014 use a different target.`
|
|
2484
|
+
);
|
|
2485
|
+
}
|
|
2486
|
+
alreadyDone = present;
|
|
2487
|
+
log(`[migrate] resuming: ${present.size} of ${notes.length} already in ${to}`);
|
|
2301
2488
|
}
|
|
2302
2489
|
}
|
|
2303
2490
|
let refreshed = 0;
|
|
@@ -2311,6 +2498,7 @@ async function migrateCollection(opts) {
|
|
|
2311
2498
|
buffer = [];
|
|
2312
2499
|
};
|
|
2313
2500
|
for (const note of notes) {
|
|
2501
|
+
if (alreadyDone.has(note.id)) continue;
|
|
2314
2502
|
if (refreshFields && missingDerivedFields(note)) {
|
|
2315
2503
|
try {
|
|
2316
2504
|
const built = await llmConstructNote(note.content);
|
|
@@ -2326,7 +2514,7 @@ async function migrateCollection(opts) {
|
|
|
2326
2514
|
buffer.push(point);
|
|
2327
2515
|
if (buffer.length >= BATCH) {
|
|
2328
2516
|
await flush();
|
|
2329
|
-
log(`[migrate] ${migrated}/${notes.length}`);
|
|
2517
|
+
log(`[migrate] ${alreadyDone.size + migrated}/${notes.length}`);
|
|
2330
2518
|
}
|
|
2331
2519
|
}
|
|
2332
2520
|
await flush();
|
|
@@ -2334,10 +2522,46 @@ async function migrateCollection(opts) {
|
|
|
2334
2522
|
if (finalCount !== notes.length) {
|
|
2335
2523
|
warn(`[migrate] target holds ${finalCount} point(s) but the source had ${notes.length} \u2014 check before switching`);
|
|
2336
2524
|
}
|
|
2337
|
-
log(
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2525
|
+
log(`[migrate] done: ${migrated} written, ${alreadyDone.size} already present, ${refreshed} re-extracted.`);
|
|
2526
|
+
return {
|
|
2527
|
+
total: notes.length,
|
|
2528
|
+
missingDerived,
|
|
2529
|
+
refreshed,
|
|
2530
|
+
migrated,
|
|
2531
|
+
skipped: alreadyDone.size,
|
|
2532
|
+
sourceDim,
|
|
2533
|
+
targetDim,
|
|
2534
|
+
model,
|
|
2535
|
+
dryRun: false
|
|
2536
|
+
};
|
|
2537
|
+
}
|
|
2538
|
+
async function switchToMigrated(opts) {
|
|
2539
|
+
const { name, to } = opts;
|
|
2540
|
+
const log = opts.logger?.info ?? ((m) => console.log(m));
|
|
2541
|
+
if (name === to) throw new Error(`switch: "${name}" and "${to}" are the same collection`);
|
|
2542
|
+
const already = await resolveAliasRaw(name);
|
|
2543
|
+
if (already === to) {
|
|
2544
|
+
log(`[switch] "${name}" already points at "${to}" \u2014 nothing to do`);
|
|
2545
|
+
return { name, to, moved: await countPointsRaw(to) };
|
|
2546
|
+
}
|
|
2547
|
+
const targetCount = await countPointsRaw(to);
|
|
2548
|
+
if (targetCount === 0) throw new Error(`switch: "${to}" is empty \u2014 migrate into it first`);
|
|
2549
|
+
if (already === null) {
|
|
2550
|
+
const sourceCount = await countPointsRaw(name);
|
|
2551
|
+
if (targetCount < sourceCount) {
|
|
2552
|
+
throw new Error(
|
|
2553
|
+
`switch: "${to}" holds ${targetCount} point(s) but "${name}" still holds ${sourceCount}. The migration is not finished \u2014 run it again before switching.`
|
|
2554
|
+
);
|
|
2555
|
+
}
|
|
2556
|
+
log(`[switch] verified ${targetCount} in "${to}" against ${sourceCount} in "${name}"`);
|
|
2557
|
+
await deleteCollectionRaw(name);
|
|
2558
|
+
log(`[switch] dropped "${name}"`);
|
|
2559
|
+
await createAliasRaw(name, to);
|
|
2560
|
+
} else {
|
|
2561
|
+
await setAliasRaw(name, to);
|
|
2562
|
+
}
|
|
2563
|
+
log(`[switch] "${name}" now resolves to "${to}"`);
|
|
2564
|
+
return { name, to, moved: targetCount };
|
|
2341
2565
|
}
|
|
2342
2566
|
|
|
2343
2567
|
// src/crud-guard.ts
|
|
@@ -2358,6 +2582,11 @@ function isPlausibleUpdateTarget(newEmbedding, targetEmbedding, minSimilarity) {
|
|
|
2358
2582
|
DEFAULT_CRUD_UPDATE_MIN_SIM,
|
|
2359
2583
|
DEFAULT_EMBEDDING_MODEL,
|
|
2360
2584
|
EmbeddingDimensionMismatchError,
|
|
2585
|
+
EmbeddingModelMismatchError,
|
|
2586
|
+
LEGACY_DEFAULT_DIM,
|
|
2587
|
+
LEGACY_DEFAULT_EMBEDDING_MODEL,
|
|
2588
|
+
MixedEmbeddingModelsError,
|
|
2589
|
+
SYSTEM_ACTOR,
|
|
2361
2590
|
addEpisodic,
|
|
2362
2591
|
addMemory,
|
|
2363
2592
|
canRead,
|
|
@@ -2372,7 +2601,9 @@ function isPlausibleUpdateTarget(newEmbedding, targetEmbedding, minSimilarity) {
|
|
|
2372
2601
|
encode,
|
|
2373
2602
|
ensureCollection,
|
|
2374
2603
|
generateReviewBatch,
|
|
2604
|
+
getEmbeddingDevice,
|
|
2375
2605
|
getEmbeddingDim,
|
|
2606
|
+
getEmbeddingDtype,
|
|
2376
2607
|
getEmbeddingModel,
|
|
2377
2608
|
getEmbeddingPooling,
|
|
2378
2609
|
getNote,
|
|
@@ -2390,6 +2621,7 @@ function isPlausibleUpdateTarget(newEmbedding, targetEmbedding, minSimilarity) {
|
|
|
2390
2621
|
resolveCrudUpdateMinSim,
|
|
2391
2622
|
scanLowQuality,
|
|
2392
2623
|
searchMemory,
|
|
2624
|
+
switchToMigrated,
|
|
2393
2625
|
updateNote
|
|
2394
2626
|
});
|
|
2395
2627
|
//# sourceMappingURL=index.cjs.map
|