@anuma/sdk 8.2.0 → 9.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/expo/index.cjs +451 -161
- package/dist/expo/index.d.mts +67 -5
- package/dist/expo/index.d.ts +67 -5
- package/dist/expo/index.mjs +448 -161
- package/dist/index.cjs +9 -3
- package/dist/index.mjs +9 -3
- package/dist/react/index.cjs +451 -161
- package/dist/react/index.d.mts +67 -5
- package/dist/react/index.d.ts +67 -5
- package/dist/react/index.mjs +448 -161
- package/dist/server/index.cjs +28 -4
- package/dist/server/index.d.mts +22 -3
- package/dist/server/index.d.ts +22 -3
- package/dist/server/index.mjs +28 -4
- package/dist/tools/selection/index.cjs +9 -3
- package/dist/tools/selection/index.mjs +9 -3
- package/package.json +1 -1
package/dist/expo/index.cjs
CHANGED
|
@@ -5934,11 +5934,17 @@ async function generateEmbeddingsBatch(texts, headers, baseUrl, model, onUsage,
|
|
|
5934
5934
|
}
|
|
5935
5935
|
return response.data.data.map((item) => item.embedding ?? []);
|
|
5936
5936
|
}
|
|
5937
|
-
async function generateEmbeddings(texts, options) {
|
|
5937
|
+
async function generateEmbeddings(texts, options, signal) {
|
|
5938
|
+
signal?.throwIfAborted();
|
|
5938
5939
|
if (texts.length === 0) return [];
|
|
5939
5940
|
const total = options.totalTimeoutMs;
|
|
5940
|
-
if (total === void 0) return embedMany(texts, options,
|
|
5941
|
-
return withDeadline(
|
|
5941
|
+
if (total === void 0) return embedMany(texts, options, signal);
|
|
5942
|
+
return withDeadline(
|
|
5943
|
+
(inner) => embedMany(texts, options, inner),
|
|
5944
|
+
total,
|
|
5945
|
+
"embedding batch",
|
|
5946
|
+
signal
|
|
5947
|
+
);
|
|
5942
5948
|
}
|
|
5943
5949
|
async function embedMany(texts, options, outer) {
|
|
5944
5950
|
const { baseUrl = BASE_URL, model, batchSize, cache } = options;
|
|
@@ -11588,13 +11594,13 @@ function normalizeEntityName(name) {
|
|
|
11588
11594
|
return name.trim().toLowerCase();
|
|
11589
11595
|
}
|
|
11590
11596
|
var TOPIC_SALIENCE_WEIGHTS = { primary: 1, secondary: 0.6, mention: 0.3 };
|
|
11591
|
-
function topicWeight(
|
|
11592
|
-
return
|
|
11597
|
+
function topicWeight(topic2) {
|
|
11598
|
+
return topic2.source === "user" ? 1 : topic2.weight ?? 1;
|
|
11593
11599
|
}
|
|
11594
11600
|
function primaryTopic(topics) {
|
|
11595
11601
|
let best = null;
|
|
11596
|
-
for (const
|
|
11597
|
-
if (best === null || topicWeight(
|
|
11602
|
+
for (const topic2 of topics ?? []) {
|
|
11603
|
+
if (best === null || topicWeight(topic2) > topicWeight(best)) best = topic2;
|
|
11598
11604
|
}
|
|
11599
11605
|
return best;
|
|
11600
11606
|
}
|
|
@@ -13159,7 +13165,7 @@ function resolveTopicsWrite(row, linked, inputs, source, mode) {
|
|
|
13159
13165
|
}
|
|
13160
13166
|
function weightsOf(stored2, inputs) {
|
|
13161
13167
|
const out = /* @__PURE__ */ new Map();
|
|
13162
|
-
for (const
|
|
13168
|
+
for (const topic2 of stored2 ?? []) out.set(normalizeEntityName(topic2.name), topic2.weight);
|
|
13163
13169
|
for (const input of inputs) {
|
|
13164
13170
|
const weight = parseTopicWeight(input.weight);
|
|
13165
13171
|
if (weight === void 0) continue;
|
|
@@ -14043,6 +14049,20 @@ async function getAllVaultMemoryContentsOp(ctx, options) {
|
|
|
14043
14049
|
async function countActiveVaultMemoriesOp(ctx) {
|
|
14044
14050
|
return ctx.vaultMemoryCollection.query(...baseVaultConditions(ctx)).fetchCount();
|
|
14045
14051
|
}
|
|
14052
|
+
async function getVaultMemoryCountsOp(ctx) {
|
|
14053
|
+
const [live, archived, quarantined] = await Promise.all([
|
|
14054
|
+
countActiveVaultMemoriesOp(ctx),
|
|
14055
|
+
ctx.vaultMemoryCollection.query(
|
|
14056
|
+
...baseVaultConditions(ctx, { includeArchived: true }),
|
|
14057
|
+
import_watermelondb21.Q.where("archived_at", import_watermelondb21.Q.notEq(null))
|
|
14058
|
+
).fetchCount(),
|
|
14059
|
+
ctx.vaultMemoryCollection.query(
|
|
14060
|
+
...baseVaultConditions(ctx, { includeQuarantined: true }),
|
|
14061
|
+
import_watermelondb21.Q.where("trust_tier", "quarantined")
|
|
14062
|
+
).fetchCount()
|
|
14063
|
+
]);
|
|
14064
|
+
return { live, archived, quarantined };
|
|
14065
|
+
}
|
|
14046
14066
|
async function getActiveVaultMemoryIdsOp(ctx, ids) {
|
|
14047
14067
|
if (ids.length === 0) return /* @__PURE__ */ new Set();
|
|
14048
14068
|
const rows = await ctx.vaultMemoryCollection.query(...baseVaultConditions(ctx), import_watermelondb21.Q.where("id", import_watermelondb21.Q.oneOf(ids))).unsafeFetchRaw();
|
|
@@ -20750,7 +20770,8 @@ Each memory below is written as "<id>: <text>". Output strict JSON: {"memories":
|
|
|
20750
20770
|
- exactly one element per input memory; copy its id EXACTLY as written (the token before the first ": "), with no brackets, quotes, or extra characters
|
|
20751
20771
|
- "entities" may be empty when a memory mentions no named entities \u2014 most short memories have 0-3
|
|
20752
20772
|
No prose.`;
|
|
20753
|
-
async function extractEntitiesForMemories(memories, options) {
|
|
20773
|
+
async function extractEntitiesForMemories(memories, options, signal) {
|
|
20774
|
+
signal?.throwIfAborted();
|
|
20754
20775
|
const out = /* @__PURE__ */ new Map();
|
|
20755
20776
|
if (memories.length === 0) return out;
|
|
20756
20777
|
const redactor = resolvePiiRedactor(options.piiRedaction);
|
|
@@ -20763,6 +20784,7 @@ When a memory refers to one of these, reuse the EXACT existing name (same spelli
|
|
|
20763
20784
|
|
|
20764
20785
|
` : "";
|
|
20765
20786
|
for (let i = 0; i < memories.length; i += TOPIC_EXTRACTION_BATCH_SIZE) {
|
|
20787
|
+
signal?.throwIfAborted();
|
|
20766
20788
|
const batch = memories.slice(i, i + TOPIC_EXTRACTION_BATCH_SIZE);
|
|
20767
20789
|
const start = Date.now();
|
|
20768
20790
|
let attempts = 0;
|
|
@@ -20810,6 +20832,7 @@ List each memory's named entities.`,
|
|
|
20810
20832
|
...options.totalTimeoutMs !== void 0 && { totalTimeoutMs: options.totalTimeoutMs },
|
|
20811
20833
|
...options.backoffMs && { backoffMs: options.backoffMs }
|
|
20812
20834
|
});
|
|
20835
|
+
signal?.throwIfAborted();
|
|
20813
20836
|
if (parsed === null) {
|
|
20814
20837
|
reportBatch(batch.length);
|
|
20815
20838
|
getLogger().warn(
|
|
@@ -21044,6 +21067,7 @@ function createLocalMemoryStore(options) {
|
|
|
21044
21067
|
backfillTopics: (memoryIds) => backfillMemoryTopicsOp(vaultCtx, memoryIds)
|
|
21045
21068
|
};
|
|
21046
21069
|
return {
|
|
21070
|
+
counts: () => getVaultMemoryCountsOp(vaultCtx),
|
|
21047
21071
|
masksEmbeddingInput: options.embeddingOptions.maskInput !== void 0,
|
|
21048
21072
|
list: (listOptions) => getAllVaultMemoriesOp(vaultCtx, listOptions),
|
|
21049
21073
|
get: (id) => getVaultMemoryOp(vaultCtx, id),
|
|
@@ -21273,6 +21297,137 @@ init_logger();
|
|
|
21273
21297
|
|
|
21274
21298
|
// src/lib/memory/store/remotePersistence.ts
|
|
21275
21299
|
init_portalLlm();
|
|
21300
|
+
|
|
21301
|
+
// src/lib/memory/store/remoteTopics.ts
|
|
21302
|
+
function topic(value) {
|
|
21303
|
+
const t = value;
|
|
21304
|
+
if (!t || typeof t.name !== "string" || !t.name || !Number.isSafeInteger(t.version) || t.version < 1 || typeof t.is_deleted !== "boolean" || typeof t.server_updated_at !== "string" || t.kind !== void 0 && typeof t.kind !== "string")
|
|
21305
|
+
throw new Error("Invalid nearby topic");
|
|
21306
|
+
return t;
|
|
21307
|
+
}
|
|
21308
|
+
function createRemoteTopicPersistence(request) {
|
|
21309
|
+
return {
|
|
21310
|
+
topics: async () => {
|
|
21311
|
+
const out = [];
|
|
21312
|
+
let cursor;
|
|
21313
|
+
do {
|
|
21314
|
+
const page = await request(
|
|
21315
|
+
"/topics?limit=200" + (cursor ? `&cursor=${encodeURIComponent(cursor)}` : ""),
|
|
21316
|
+
{}
|
|
21317
|
+
);
|
|
21318
|
+
if (!Array.isArray(page.items) || page.next_cursor !== void 0 && typeof page.next_cursor !== "string")
|
|
21319
|
+
throw new Error("Invalid nearby topic page");
|
|
21320
|
+
out.push(...page.items.map(topic));
|
|
21321
|
+
cursor = page.next_cursor;
|
|
21322
|
+
} while (cursor);
|
|
21323
|
+
return out;
|
|
21324
|
+
},
|
|
21325
|
+
putTopic: async (value, expectedVersion) => {
|
|
21326
|
+
if (!Number.isSafeInteger(expectedVersion) || expectedVersion < 0)
|
|
21327
|
+
throw new Error("Expected topic version must be a nonnegative integer");
|
|
21328
|
+
return topic(
|
|
21329
|
+
await request("/topics", {
|
|
21330
|
+
method: "PUT",
|
|
21331
|
+
body: JSON.stringify({ topic: value, expected_version: expectedVersion })
|
|
21332
|
+
})
|
|
21333
|
+
);
|
|
21334
|
+
},
|
|
21335
|
+
importTopics: async (topics, signal) => {
|
|
21336
|
+
for (let i = 0; i < topics.length; i += 50) {
|
|
21337
|
+
signal?.throwIfAborted();
|
|
21338
|
+
await request(
|
|
21339
|
+
"/topics/import",
|
|
21340
|
+
{
|
|
21341
|
+
method: "POST",
|
|
21342
|
+
body: JSON.stringify({ topics: topics.slice(i, i + 50) })
|
|
21343
|
+
},
|
|
21344
|
+
signal
|
|
21345
|
+
);
|
|
21346
|
+
}
|
|
21347
|
+
signal?.throwIfAborted();
|
|
21348
|
+
}
|
|
21349
|
+
};
|
|
21350
|
+
}
|
|
21351
|
+
async function createRemoteTopic(persistence, name) {
|
|
21352
|
+
const canonical = normalizeEntityName(name);
|
|
21353
|
+
if (!canonical) return null;
|
|
21354
|
+
const existing = (await persistence.topics()).find((t) => t.name === canonical);
|
|
21355
|
+
if (existing && !existing.is_deleted) return canonical;
|
|
21356
|
+
try {
|
|
21357
|
+
await persistence.putTopic(
|
|
21358
|
+
{ name: canonical, is_deleted: false, ...existing?.kind && { kind: existing.kind } },
|
|
21359
|
+
existing?.version ?? 0
|
|
21360
|
+
);
|
|
21361
|
+
} catch (error) {
|
|
21362
|
+
if (!(error instanceof RemoteMemoryError) || error.code !== "version_conflict") throw error;
|
|
21363
|
+
const current = (await persistence.topics()).find((t) => t.name === canonical);
|
|
21364
|
+
if (!current || current.is_deleted) throw error;
|
|
21365
|
+
}
|
|
21366
|
+
return canonical;
|
|
21367
|
+
}
|
|
21368
|
+
|
|
21369
|
+
// src/lib/memory/store/remoteWork.ts
|
|
21370
|
+
init_logger();
|
|
21371
|
+
function createRemoteWorkPersistence(request) {
|
|
21372
|
+
return {
|
|
21373
|
+
claimWork: async (kind, processorVersion, token) => {
|
|
21374
|
+
const result = await request("/work/claim", {
|
|
21375
|
+
method: "POST",
|
|
21376
|
+
body: JSON.stringify({
|
|
21377
|
+
kind,
|
|
21378
|
+
processor_version: processorVersion,
|
|
21379
|
+
...token && { token }
|
|
21380
|
+
})
|
|
21381
|
+
});
|
|
21382
|
+
if (!result || typeof result !== "object") throw new Error("Invalid nearby work lease");
|
|
21383
|
+
const lease = result.lease;
|
|
21384
|
+
if (lease === void 0) return null;
|
|
21385
|
+
if (!lease || lease.kind !== kind || typeof lease.token !== "string" || !lease.token || lease.processor_version !== processorVersion || typeof lease.expires_at !== "string" || !Number.isFinite(Date.parse(lease.expires_at)))
|
|
21386
|
+
throw new Error("Invalid nearby work lease");
|
|
21387
|
+
return lease;
|
|
21388
|
+
},
|
|
21389
|
+
releaseWork: async (kind, token) => {
|
|
21390
|
+
await request("/work/release", { method: "POST", body: JSON.stringify({ kind, token }) });
|
|
21391
|
+
}
|
|
21392
|
+
};
|
|
21393
|
+
}
|
|
21394
|
+
async function withRemoteWorkLease(persistence, kind, processorVersion, task) {
|
|
21395
|
+
const lease = await persistence.claimWork(kind, processorVersion);
|
|
21396
|
+
if (!lease) return null;
|
|
21397
|
+
const controller = new AbortController();
|
|
21398
|
+
const leaseLostError = new RemoteMemoryError("Maintenance lease lost", 409, "work_lease_expired");
|
|
21399
|
+
const cancelled = new Promise((_resolve, reject) => {
|
|
21400
|
+
controller.signal.addEventListener("abort", () => reject(leaseLostError), {
|
|
21401
|
+
once: true
|
|
21402
|
+
});
|
|
21403
|
+
});
|
|
21404
|
+
let renewing;
|
|
21405
|
+
const timer = setInterval(() => {
|
|
21406
|
+
if (renewing || controller.signal.aborted) return;
|
|
21407
|
+
renewing = persistence.claimWork(kind, processorVersion, lease.token).then((renewed) => {
|
|
21408
|
+
if (!renewed) controller.abort(leaseLostError);
|
|
21409
|
+
}).catch((error) => getLogger().warn("[memory/work] Lease renewal failed:", error)).finally(() => {
|
|
21410
|
+
renewing = void 0;
|
|
21411
|
+
});
|
|
21412
|
+
}, 3e4);
|
|
21413
|
+
try {
|
|
21414
|
+
const result = await Promise.race([task(lease.token, controller.signal), cancelled]);
|
|
21415
|
+
await renewing;
|
|
21416
|
+
return controller.signal.aborted ? null : result;
|
|
21417
|
+
} catch (error) {
|
|
21418
|
+
if (controller.signal.aborted || isRemoteWorkLeaseExpired(error)) return null;
|
|
21419
|
+
throw error;
|
|
21420
|
+
} finally {
|
|
21421
|
+
clearInterval(timer);
|
|
21422
|
+
await renewing;
|
|
21423
|
+
await persistence.releaseWork(kind, lease.token).catch((error) => getLogger().warn("[memory/work] Lease release failed:", error));
|
|
21424
|
+
}
|
|
21425
|
+
}
|
|
21426
|
+
function isRemoteWorkLeaseExpired(error) {
|
|
21427
|
+
return error instanceof RemoteMemoryError && error.code === "work_lease_expired";
|
|
21428
|
+
}
|
|
21429
|
+
|
|
21430
|
+
// src/lib/memory/store/remotePersistence.ts
|
|
21276
21431
|
var RemoteMemoryError = class extends Error {
|
|
21277
21432
|
constructor(message, status, code, retryAfterMs) {
|
|
21278
21433
|
super(message);
|
|
@@ -21651,6 +21806,33 @@ async function createRemoteMemoryPersistence(options) {
|
|
|
21651
21806
|
};
|
|
21652
21807
|
};
|
|
21653
21808
|
return {
|
|
21809
|
+
...createRemoteTopicPersistence(request),
|
|
21810
|
+
...createRemoteWorkPersistence(request),
|
|
21811
|
+
counts: async () => {
|
|
21812
|
+
const result = await request("/counts", {});
|
|
21813
|
+
if (!record(result) || ["live", "archived", "quarantined"].some(
|
|
21814
|
+
(key) => !Number.isSafeInteger(result[key]) || result[key] < 0
|
|
21815
|
+
))
|
|
21816
|
+
throw new Error("Invalid nearby memory counts");
|
|
21817
|
+
return {
|
|
21818
|
+
live: result.live,
|
|
21819
|
+
archived: result.archived,
|
|
21820
|
+
quarantined: result.quarantined
|
|
21821
|
+
};
|
|
21822
|
+
},
|
|
21823
|
+
temporal: async (window2, limit = 100) => {
|
|
21824
|
+
if (!Number.isFinite(window2.start) || !Number.isFinite(window2.end) || window2.end <= window2.start || !Number.isInteger(limit) || limit < 1 || limit > 100)
|
|
21825
|
+
throw new Error("Temporal window must have end > start and limit must be 1\u2013100");
|
|
21826
|
+
const result = await request("/temporal", {
|
|
21827
|
+
method: "POST",
|
|
21828
|
+
body: JSON.stringify({ ...window2, limit })
|
|
21829
|
+
});
|
|
21830
|
+
if (!record(result) || !Array.isArray(result.items) || result.items.some(
|
|
21831
|
+
(item) => !record(item) || typeof item.memory_id !== "string" || !item.memory_id || typeof item.score !== "number" || !Number.isFinite(item.score) || item.score <= 0 || item.score > 1
|
|
21832
|
+
))
|
|
21833
|
+
throw new Error("Invalid nearby temporal candidates");
|
|
21834
|
+
return result.items.map((item) => item.memory_id);
|
|
21835
|
+
},
|
|
21654
21836
|
get: async (id, signal) => {
|
|
21655
21837
|
try {
|
|
21656
21838
|
return await decode(await request(memoryPath(id), {}, signal));
|
|
@@ -21699,7 +21881,7 @@ async function createRemoteMemoryPersistence(options) {
|
|
|
21699
21881
|
write
|
|
21700
21882
|
);
|
|
21701
21883
|
},
|
|
21702
|
-
putMany: async (writes, signal) => {
|
|
21884
|
+
putMany: async (writes, signal, leaseToken) => {
|
|
21703
21885
|
if (writes.length < 1 || writes.length > 50 || new Set(writes.map((w) => w.memory.memory_id)).size !== writes.length)
|
|
21704
21886
|
throw new Error("Batch must contain 1\u201350 distinct memory ids");
|
|
21705
21887
|
const prepared = await Promise.all(
|
|
@@ -21709,7 +21891,11 @@ async function createRemoteMemoryPersistence(options) {
|
|
|
21709
21891
|
"/memories/batch",
|
|
21710
21892
|
{
|
|
21711
21893
|
method: "POST",
|
|
21712
|
-
body: JSON.stringify({
|
|
21894
|
+
body: JSON.stringify({
|
|
21895
|
+
key_id: options.keyId,
|
|
21896
|
+
writes: prepared.map((w) => w.body),
|
|
21897
|
+
...leaseToken && { lease_token: leaseToken }
|
|
21898
|
+
})
|
|
21713
21899
|
},
|
|
21714
21900
|
signal
|
|
21715
21901
|
);
|
|
@@ -22469,6 +22655,12 @@ async function migrateMemoriesToRemote(options) {
|
|
|
22469
22655
|
let claimed = account.key_id === canonical.keyId;
|
|
22470
22656
|
if (!active) await createDecaySweeper({ vaultCtx: options.vaultCtx }).runSweep();
|
|
22471
22657
|
await backfillTopics(options.vaultCtx);
|
|
22658
|
+
const localTopics = options.topics ?? (options.vaultCtx.singleTenant === true ? (await options.vaultCtx.database.get("entity").query().unsafeFetchRaw()).map((t) => ({
|
|
22659
|
+
name: normalizeEntityName(typeof t.canonical_name === "string" ? t.canonical_name : ""),
|
|
22660
|
+
...typeof t.kind === "string" && { kind: t.kind }
|
|
22661
|
+
})).filter((t) => t.name.length > 0) : []);
|
|
22662
|
+
if (localTopics.length)
|
|
22663
|
+
await createRemoteTopicPersistence(request).importTopics(localTopics, signal);
|
|
22472
22664
|
const skipped = /* @__PURE__ */ new Map();
|
|
22473
22665
|
const reembedded = /* @__PURE__ */ new Set();
|
|
22474
22666
|
const unembedded = /* @__PURE__ */ new Set();
|
|
@@ -22596,7 +22788,6 @@ var MAX_QUERY_IDS = 1e3;
|
|
|
22596
22788
|
var MAX_QUERY_TOPICS = 100;
|
|
22597
22789
|
var MAX_QUERY_FILTER = 20;
|
|
22598
22790
|
var GRAPH_LANE_ROWS = 256;
|
|
22599
|
-
var TEMPORAL_LANE_ROWS = 256;
|
|
22600
22791
|
var REPAIR_ROWS = 50;
|
|
22601
22792
|
var REPAIR_SCAN = 2e3;
|
|
22602
22793
|
var DECAY_TRANSITIONS_PER_SWEEP = 200;
|
|
@@ -22667,21 +22858,21 @@ function subscriptionKey({ memory, version }, options) {
|
|
|
22667
22858
|
]);
|
|
22668
22859
|
}
|
|
22669
22860
|
function topicInputs(topics, source) {
|
|
22670
|
-
return topics.flatMap((
|
|
22671
|
-
const name = (typeof
|
|
22672
|
-
const kind = typeof
|
|
22673
|
-
const weight = typeof
|
|
22861
|
+
return topics.flatMap((topic2) => {
|
|
22862
|
+
const name = (typeof topic2 === "string" ? topic2 : topic2.name).trim();
|
|
22863
|
+
const kind = typeof topic2 === "string" ? void 0 : topic2.kind;
|
|
22864
|
+
const weight = typeof topic2 === "string" || source === "user" ? void 0 : parseTopicWeight(topic2.weight);
|
|
22674
22865
|
return name ? [{ name, ...kind && { kind }, source, ...weight && { weight } }] : [];
|
|
22675
22866
|
});
|
|
22676
22867
|
}
|
|
22677
22868
|
function mergeTopics(existing, added) {
|
|
22678
22869
|
const byName = /* @__PURE__ */ new Map();
|
|
22679
|
-
for (const
|
|
22680
|
-
const key = normalizeEntityName(
|
|
22870
|
+
for (const topic2 of [...existing, ...added]) {
|
|
22871
|
+
const key = normalizeEntityName(topic2.name);
|
|
22681
22872
|
const kept = byName.get(key);
|
|
22682
|
-
if (!kept) byName.set(key,
|
|
22683
|
-
else if (kept.weight !== void 0 &&
|
|
22684
|
-
byName.set(key, { ...kept, weight:
|
|
22873
|
+
if (!kept) byName.set(key, topic2);
|
|
22874
|
+
else if (kept.weight !== void 0 && topic2.weight !== void 0 && topic2.weight > kept.weight)
|
|
22875
|
+
byName.set(key, { ...kept, weight: topic2.weight });
|
|
22685
22876
|
}
|
|
22686
22877
|
return [...byName.values()];
|
|
22687
22878
|
}
|
|
@@ -22692,7 +22883,7 @@ function sameTopics(stored2, next) {
|
|
|
22692
22883
|
return next.every((t) => keys.has(key(t)));
|
|
22693
22884
|
}
|
|
22694
22885
|
function canonicalTopics(row) {
|
|
22695
|
-
return new Set((parseTopics(row.topics) ?? []).map((
|
|
22886
|
+
return new Set((parseTopics(row.topics) ?? []).map((topic2) => normalizeEntityName(topic2.name)));
|
|
22696
22887
|
}
|
|
22697
22888
|
function createRemoteMemoryStore(options) {
|
|
22698
22889
|
const { persistence, embeddingOptions } = options;
|
|
@@ -22767,20 +22958,7 @@ function createRemoteMemoryStore(options) {
|
|
|
22767
22958
|
const eventTimeRanking = async (query, now) => {
|
|
22768
22959
|
const window2 = parseQueryTimeWindow(query, now);
|
|
22769
22960
|
if (!window2) return [];
|
|
22770
|
-
|
|
22771
|
-
metadata2,
|
|
22772
|
-
{ event_time_overlaps: { start: window2.start, end: window2.end } },
|
|
22773
|
-
TEMPORAL_LANE_ROWS
|
|
22774
|
-
);
|
|
22775
|
-
return rows.map(({ memory }) => ({
|
|
22776
|
-
memoryId: memory.memory_id,
|
|
22777
|
-
score: scoreEventTimeOverlap(
|
|
22778
|
-
memory.event_time_start ?? null,
|
|
22779
|
-
memory.event_time_end ?? null,
|
|
22780
|
-
memory.event_time_kind ?? null,
|
|
22781
|
-
window2
|
|
22782
|
-
)
|
|
22783
|
-
})).filter((row) => row.score > 0).sort((a, b) => b.score - a.score).map((row) => row.memoryId);
|
|
22961
|
+
return persistence.temporal({ start: window2.start, end: window2.end });
|
|
22784
22962
|
};
|
|
22785
22963
|
const enumerateIds = async (read, ids, filters) => (await Promise.all(
|
|
22786
22964
|
chunks2([...new Set(ids)], MAX_QUERY_IDS).map(
|
|
@@ -22831,22 +23009,28 @@ function createRemoteMemoryStore(options) {
|
|
|
22831
23009
|
writeOptions?.signal?.throwIfAborted();
|
|
22832
23010
|
const next = edit(structuredClone(record2.memory));
|
|
22833
23011
|
if (!next) return null;
|
|
22834
|
-
const saved = writeOptions?.
|
|
23012
|
+
const saved = writeOptions?.leaseToken ? (await persistence.putMany(
|
|
23013
|
+
[{ memory: next, expectedVersion: record2 }],
|
|
23014
|
+
writeOptions.signal,
|
|
23015
|
+
writeOptions.leaseToken
|
|
23016
|
+
))[0] : writeOptions?.signal ? await persistence.put(next, record2, writeOptions.signal) : await persistence.put(next, record2);
|
|
22835
23017
|
notify(typeof change === "function" ? change(record2.memory) : change, [saved]);
|
|
22836
23018
|
return saved;
|
|
22837
23019
|
};
|
|
22838
23020
|
const mutate = (id, change, edit, writeOptions) => serialized(id, () => mutateLocked(id, change, edit, writeOptions));
|
|
22839
23021
|
const serializedAll = (ids, task) => [...new Set(ids)].sort().reduceRight((inner, id) => () => serialized(id, inner), task)();
|
|
22840
|
-
const mutateMany = async (ids, change, edit) => {
|
|
23022
|
+
const mutateMany = async (ids, change, edit, leaseToken, signal) => {
|
|
22841
23023
|
const saved = /* @__PURE__ */ new Map();
|
|
22842
23024
|
const failed = /* @__PURE__ */ new Map();
|
|
22843
23025
|
for (const slice of chunks2([...new Set(ids)], MAX_BATCH)) {
|
|
22844
23026
|
await serializedAll(slice, async () => {
|
|
23027
|
+
signal?.throwIfAborted();
|
|
22845
23028
|
const writes = [];
|
|
22846
23029
|
let read;
|
|
22847
23030
|
try {
|
|
22848
|
-
read = await readEveryState(slice);
|
|
23031
|
+
read = await readEveryState(slice, signal);
|
|
22849
23032
|
} catch (error) {
|
|
23033
|
+
signal?.throwIfAborted();
|
|
22850
23034
|
for (const id of slice) failed.set(id, error);
|
|
22851
23035
|
return;
|
|
22852
23036
|
}
|
|
@@ -22863,7 +23047,9 @@ function createRemoteMemoryStore(options) {
|
|
|
22863
23047
|
if (writes.length === 0) return;
|
|
22864
23048
|
try {
|
|
22865
23049
|
const written = await persistence.putMany(
|
|
22866
|
-
writes.map(({ before, memory }) => ({ memory, expectedVersion: before }))
|
|
23050
|
+
writes.map(({ before, memory }) => ({ memory, expectedVersion: before })),
|
|
23051
|
+
signal,
|
|
23052
|
+
leaseToken
|
|
22867
23053
|
);
|
|
22868
23054
|
const byChange = /* @__PURE__ */ new Map();
|
|
22869
23055
|
written.forEach((record2, i) => {
|
|
@@ -22873,7 +23059,9 @@ function createRemoteMemoryStore(options) {
|
|
|
22873
23059
|
});
|
|
22874
23060
|
for (const [kind, changed] of byChange) notify(kind, changed);
|
|
22875
23061
|
} catch (error) {
|
|
22876
|
-
|
|
23062
|
+
signal?.throwIfAborted();
|
|
23063
|
+
if (isRemoteWorkLeaseExpired(error)) throw error;
|
|
23064
|
+
const perRow = error instanceof RemoteMemoryError && (error.code === "version_conflict" || error.status === 429 && !leaseToken);
|
|
22877
23065
|
for (const { before } of writes) {
|
|
22878
23066
|
const id = before.memory.memory_id;
|
|
22879
23067
|
if (!perRow) {
|
|
@@ -22881,9 +23069,16 @@ function createRemoteMemoryStore(options) {
|
|
|
22881
23069
|
continue;
|
|
22882
23070
|
}
|
|
22883
23071
|
try {
|
|
22884
|
-
const record2 = await mutateLocked(
|
|
23072
|
+
const record2 = await mutateLocked(
|
|
23073
|
+
id,
|
|
23074
|
+
change,
|
|
23075
|
+
edit,
|
|
23076
|
+
leaseToken || signal ? { leaseToken, signal } : void 0
|
|
23077
|
+
);
|
|
22885
23078
|
if (record2) saved.set(id, record2);
|
|
22886
23079
|
} catch (rowError) {
|
|
23080
|
+
signal?.throwIfAborted();
|
|
23081
|
+
if (isRemoteWorkLeaseExpired(rowError)) throw rowError;
|
|
22887
23082
|
failed.set(id, rowError);
|
|
22888
23083
|
}
|
|
22889
23084
|
}
|
|
@@ -22947,7 +23142,7 @@ function createRemoteMemoryStore(options) {
|
|
|
22947
23142
|
return records.map((record2) => stored(record2.memory));
|
|
22948
23143
|
};
|
|
22949
23144
|
const asStored = (record2) => record2 ? stored(record2.memory) : null;
|
|
22950
|
-
const knownKinds = async (topics) => {
|
|
23145
|
+
const knownKinds = async (topics, signal) => {
|
|
22951
23146
|
const unkinded = [
|
|
22952
23147
|
...new Set(topics.filter((t) => !t.kind).map((t) => normalizeEntityName(t.name)))
|
|
22953
23148
|
];
|
|
@@ -22957,21 +23152,22 @@ function createRemoteMemoryStore(options) {
|
|
|
22957
23152
|
topics: names,
|
|
22958
23153
|
include_archived: true,
|
|
22959
23154
|
include_quarantined: true,
|
|
22960
|
-
include_superseded: true
|
|
23155
|
+
include_superseded: true,
|
|
23156
|
+
...signal && { signal }
|
|
22961
23157
|
})) {
|
|
22962
|
-
for (const
|
|
22963
|
-
const name = normalizeEntityName(
|
|
22964
|
-
if (
|
|
23158
|
+
for (const topic2 of parseTopics(memory.topics) ?? []) {
|
|
23159
|
+
const name = normalizeEntityName(topic2.name);
|
|
23160
|
+
if (topic2.kind && !kinds.has(name)) kinds.set(name, topic2.kind);
|
|
22965
23161
|
}
|
|
22966
23162
|
}
|
|
22967
23163
|
}
|
|
22968
|
-
return (
|
|
22969
|
-
const kind =
|
|
22970
|
-
return kind ? { ...
|
|
23164
|
+
return (topic2) => {
|
|
23165
|
+
const kind = topic2.kind ?? kinds.get(normalizeEntityName(topic2.name));
|
|
23166
|
+
return kind ? { ...topic2, kind } : topic2;
|
|
22971
23167
|
};
|
|
22972
23168
|
};
|
|
22973
23169
|
const withKnownKinds = async (topics) => topics.map(await knownKinds(topics));
|
|
22974
|
-
const readEveryState = async (ids) => {
|
|
23170
|
+
const readEveryState = async (ids, signal) => {
|
|
22975
23171
|
const records = /* @__PURE__ */ new Map();
|
|
22976
23172
|
const unreadable = /* @__PURE__ */ new Set();
|
|
22977
23173
|
for (const memory_ids of chunks2(ids, MAX_QUERY_IDS)) {
|
|
@@ -22984,6 +23180,7 @@ function createRemoteMemoryStore(options) {
|
|
|
22984
23180
|
include_quarantined: true,
|
|
22985
23181
|
include_superseded: true,
|
|
22986
23182
|
limit: 200,
|
|
23183
|
+
...signal && { signal },
|
|
22987
23184
|
...cursor2 && { cursor: cursor2 }
|
|
22988
23185
|
});
|
|
22989
23186
|
warnFailed(page.failed);
|
|
@@ -22994,6 +23191,111 @@ function createRemoteMemoryStore(options) {
|
|
|
22994
23191
|
}
|
|
22995
23192
|
return { records, unreadable };
|
|
22996
23193
|
};
|
|
23194
|
+
const extractTopicsUnlocked = async (memoryIds, extractOptions, leaseToken, signal) => {
|
|
23195
|
+
const start = Date.now();
|
|
23196
|
+
const extractedAt = extractOptions.now ?? start;
|
|
23197
|
+
const skippedIds = [];
|
|
23198
|
+
const skippedReasons = /* @__PURE__ */ new Map();
|
|
23199
|
+
const skip = (id, reason) => {
|
|
23200
|
+
if (skippedReasons.has(id)) return;
|
|
23201
|
+
skippedIds.push(id);
|
|
23202
|
+
skippedReasons.set(id, reason);
|
|
23203
|
+
};
|
|
23204
|
+
const ids = [...new Set(memoryIds)];
|
|
23205
|
+
const { records, unreadable } = await readEveryState(ids, signal);
|
|
23206
|
+
signal?.throwIfAborted();
|
|
23207
|
+
const inputs = [];
|
|
23208
|
+
for (const id of ids) {
|
|
23209
|
+
const memory = records.get(id)?.memory;
|
|
23210
|
+
if (unreadable.has(id)) skip(id, "unreadable");
|
|
23211
|
+
else if (!memory) skip(id, "not-found");
|
|
23212
|
+
else if (memory.is_deleted || memory.topics_user_managed) skip(id, "excluded");
|
|
23213
|
+
else inputs.push({ id, content: memory.content });
|
|
23214
|
+
}
|
|
23215
|
+
const batches = [];
|
|
23216
|
+
const llmFailures = /* @__PURE__ */ new Map();
|
|
23217
|
+
const entitiesByMemory = await extractEntitiesForMemories(
|
|
23218
|
+
inputs,
|
|
23219
|
+
{
|
|
23220
|
+
...extractOptions,
|
|
23221
|
+
onBatch: (batch) => {
|
|
23222
|
+
batches.push(batch);
|
|
23223
|
+
if (batch.failure) {
|
|
23224
|
+
for (const id of batch.memoryIds) llmFailures.set(id, batch.failure);
|
|
23225
|
+
}
|
|
23226
|
+
notifyMemoryCallback(extractOptions.onBatch, batch);
|
|
23227
|
+
}
|
|
23228
|
+
},
|
|
23229
|
+
signal
|
|
23230
|
+
);
|
|
23231
|
+
signal?.throwIfAborted();
|
|
23232
|
+
const answered = /* @__PURE__ */ new Map();
|
|
23233
|
+
for (const input of inputs) {
|
|
23234
|
+
const entities = entitiesByMemory.get(input.id);
|
|
23235
|
+
if (entities === void 0) skip(input.id, "llm-unanswered");
|
|
23236
|
+
else
|
|
23237
|
+
answered.set(input.id, { content: input.content, topics: topicInputs(entities, "auto") });
|
|
23238
|
+
}
|
|
23239
|
+
const stampedIds = [];
|
|
23240
|
+
const fail = (id, reason) => {
|
|
23241
|
+
skip(id, reason);
|
|
23242
|
+
entitiesByMemory.delete(id);
|
|
23243
|
+
};
|
|
23244
|
+
try {
|
|
23245
|
+
const kinded = await knownKinds(
|
|
23246
|
+
[...answered.values()].flatMap((a) => a.topics),
|
|
23247
|
+
signal
|
|
23248
|
+
);
|
|
23249
|
+
signal?.throwIfAborted();
|
|
23250
|
+
const topics = new Map(
|
|
23251
|
+
[...answered].map(([id, a]) => [id, mergeTopics([], a.topics.map(kinded))])
|
|
23252
|
+
);
|
|
23253
|
+
const unchanged = (memory) => sameTopics(parseTopics(memory.topics), topics.get(memory.memory_id));
|
|
23254
|
+
const { saved, failed } = await mutateMany(
|
|
23255
|
+
[...answered.keys()],
|
|
23256
|
+
(before) => sameTopics(parseTopics(before.topics) ?? [], topics.get(before.memory_id)) ? "metadata" : "edit",
|
|
23257
|
+
(memory) => {
|
|
23258
|
+
if (memory.topics_user_managed || memory.content !== answered.get(memory.memory_id).content)
|
|
23259
|
+
return null;
|
|
23260
|
+
return {
|
|
23261
|
+
...memory,
|
|
23262
|
+
...!unchanged(memory) && {
|
|
23263
|
+
topics: serializeTopics(topics.get(memory.memory_id)),
|
|
23264
|
+
topics_updated_at: Date.now()
|
|
23265
|
+
},
|
|
23266
|
+
topics_extracted_at: Math.max(extractedAt, memory.updated_at),
|
|
23267
|
+
topics_extracted_version: TOPICS_EXTRACTION_VERSION
|
|
23268
|
+
};
|
|
23269
|
+
},
|
|
23270
|
+
leaseToken,
|
|
23271
|
+
signal
|
|
23272
|
+
);
|
|
23273
|
+
for (const id of answered.keys()) {
|
|
23274
|
+
if (saved.has(id)) stampedIds.push(id);
|
|
23275
|
+
else fail(id, failed.has(id) ? "link-failed" : "link-declined");
|
|
23276
|
+
}
|
|
23277
|
+
if (failed.size)
|
|
23278
|
+
getLogger().warn(
|
|
23279
|
+
"[memory/remote-store] Failed to write extracted topics:",
|
|
23280
|
+
failed.values().next().value
|
|
23281
|
+
);
|
|
23282
|
+
} catch (error) {
|
|
23283
|
+
signal?.throwIfAborted();
|
|
23284
|
+
if (isRemoteWorkLeaseExpired(error)) throw error;
|
|
23285
|
+
getLogger().warn("[memory/remote-store] Failed to write extracted topics:", error);
|
|
23286
|
+
for (const id of answered.keys()) fail(id, "link-failed");
|
|
23287
|
+
}
|
|
23288
|
+
return {
|
|
23289
|
+
entitiesByMemory,
|
|
23290
|
+
stampedIds,
|
|
23291
|
+
skippedIds,
|
|
23292
|
+
skippedReasons,
|
|
23293
|
+
batchCount: batches.length,
|
|
23294
|
+
durationMs: Date.now() - start,
|
|
23295
|
+
batches,
|
|
23296
|
+
llmFailures
|
|
23297
|
+
};
|
|
23298
|
+
};
|
|
22997
23299
|
const topicMaintenance = {
|
|
22998
23300
|
getTopicBacklog: async (backlogOptions = {}) => {
|
|
22999
23301
|
const limit = backlogOptions.limit !== void 0 && backlogOptions.limit > 0 ? backlogOptions.limit : Infinity;
|
|
@@ -23011,97 +23313,12 @@ function createRemoteMemoryStore(options) {
|
|
|
23011
23313
|
topicsBackfill: []
|
|
23012
23314
|
};
|
|
23013
23315
|
},
|
|
23014
|
-
extractTopics: async (
|
|
23015
|
-
|
|
23016
|
-
|
|
23017
|
-
|
|
23018
|
-
|
|
23019
|
-
|
|
23020
|
-
if (skippedReasons.has(id)) return;
|
|
23021
|
-
skippedIds.push(id);
|
|
23022
|
-
skippedReasons.set(id, reason);
|
|
23023
|
-
};
|
|
23024
|
-
const ids = [...new Set(memoryIds)];
|
|
23025
|
-
const { records, unreadable } = await readEveryState(ids);
|
|
23026
|
-
const inputs = [];
|
|
23027
|
-
for (const id of ids) {
|
|
23028
|
-
const memory = records.get(id)?.memory;
|
|
23029
|
-
if (unreadable.has(id)) skip(id, "unreadable");
|
|
23030
|
-
else if (!memory) skip(id, "not-found");
|
|
23031
|
-
else if (memory.is_deleted || memory.topics_user_managed) skip(id, "excluded");
|
|
23032
|
-
else inputs.push({ id, content: memory.content });
|
|
23033
|
-
}
|
|
23034
|
-
const batches = [];
|
|
23035
|
-
const llmFailures = /* @__PURE__ */ new Map();
|
|
23036
|
-
const entitiesByMemory = await extractEntitiesForMemories(inputs, {
|
|
23037
|
-
...extractOptions,
|
|
23038
|
-
onBatch: (batch) => {
|
|
23039
|
-
batches.push(batch);
|
|
23040
|
-
if (batch.failure) {
|
|
23041
|
-
for (const id of batch.memoryIds) llmFailures.set(id, batch.failure);
|
|
23042
|
-
}
|
|
23043
|
-
notifyMemoryCallback(extractOptions.onBatch, batch);
|
|
23044
|
-
}
|
|
23045
|
-
});
|
|
23046
|
-
const answered = /* @__PURE__ */ new Map();
|
|
23047
|
-
for (const input of inputs) {
|
|
23048
|
-
const entities = entitiesByMemory.get(input.id);
|
|
23049
|
-
if (entities === void 0) skip(input.id, "llm-unanswered");
|
|
23050
|
-
else
|
|
23051
|
-
answered.set(input.id, { content: input.content, topics: topicInputs(entities, "auto") });
|
|
23052
|
-
}
|
|
23053
|
-
const stampedIds = [];
|
|
23054
|
-
const fail = (id, reason) => {
|
|
23055
|
-
skip(id, reason);
|
|
23056
|
-
entitiesByMemory.delete(id);
|
|
23057
|
-
};
|
|
23058
|
-
try {
|
|
23059
|
-
const kinded = await knownKinds([...answered.values()].flatMap((a) => a.topics));
|
|
23060
|
-
const topics = new Map(
|
|
23061
|
-
[...answered].map(([id, a]) => [id, mergeTopics([], a.topics.map(kinded))])
|
|
23062
|
-
);
|
|
23063
|
-
const unchanged = (memory) => sameTopics(parseTopics(memory.topics), topics.get(memory.memory_id));
|
|
23064
|
-
const { saved, failed } = await mutateMany(
|
|
23065
|
-
[...answered.keys()],
|
|
23066
|
-
(before) => sameTopics(parseTopics(before.topics) ?? [], topics.get(before.memory_id)) ? "metadata" : "edit",
|
|
23067
|
-
(memory) => {
|
|
23068
|
-
if (memory.topics_user_managed || memory.content !== answered.get(memory.memory_id).content)
|
|
23069
|
-
return null;
|
|
23070
|
-
return {
|
|
23071
|
-
...memory,
|
|
23072
|
-
...!unchanged(memory) && {
|
|
23073
|
-
topics: serializeTopics(topics.get(memory.memory_id)),
|
|
23074
|
-
topics_updated_at: Date.now()
|
|
23075
|
-
},
|
|
23076
|
-
topics_extracted_at: Math.max(extractedAt, memory.updated_at),
|
|
23077
|
-
topics_extracted_version: TOPICS_EXTRACTION_VERSION
|
|
23078
|
-
};
|
|
23079
|
-
}
|
|
23080
|
-
);
|
|
23081
|
-
for (const id of answered.keys()) {
|
|
23082
|
-
if (saved.has(id)) stampedIds.push(id);
|
|
23083
|
-
else fail(id, failed.has(id) ? "link-failed" : "link-declined");
|
|
23084
|
-
}
|
|
23085
|
-
if (failed.size)
|
|
23086
|
-
getLogger().warn(
|
|
23087
|
-
"[memory/remote-store] Failed to write extracted topics:",
|
|
23088
|
-
failed.values().next().value
|
|
23089
|
-
);
|
|
23090
|
-
} catch (error) {
|
|
23091
|
-
getLogger().warn("[memory/remote-store] Failed to write extracted topics:", error);
|
|
23092
|
-
for (const id of answered.keys()) fail(id, "link-failed");
|
|
23093
|
-
}
|
|
23094
|
-
return {
|
|
23095
|
-
entitiesByMemory,
|
|
23096
|
-
stampedIds,
|
|
23097
|
-
skippedIds,
|
|
23098
|
-
skippedReasons,
|
|
23099
|
-
batchCount: batches.length,
|
|
23100
|
-
durationMs: Date.now() - start,
|
|
23101
|
-
batches,
|
|
23102
|
-
llmFailures
|
|
23103
|
-
};
|
|
23104
|
-
},
|
|
23316
|
+
extractTopics: async (ids, extractOptions) => await withRemoteWorkLease(
|
|
23317
|
+
persistence,
|
|
23318
|
+
"topics",
|
|
23319
|
+
String(TOPICS_EXTRACTION_VERSION),
|
|
23320
|
+
(token, signal) => extractTopicsUnlocked(ids, extractOptions, token, signal)
|
|
23321
|
+
) ?? { ...await extractTopicsUnlocked([], extractOptions), deferred: true },
|
|
23105
23322
|
stampTopicsExtracted: async (memoryIds, extractedAt, version = TOPICS_EXTRACTION_VERSION) => {
|
|
23106
23323
|
const ids = [...new Set(memoryIds)];
|
|
23107
23324
|
const { saved, failed } = await mutateMany(
|
|
@@ -23198,14 +23415,29 @@ function createRemoteMemoryStore(options) {
|
|
|
23198
23415
|
let ready = false;
|
|
23199
23416
|
let cursor;
|
|
23200
23417
|
let pending = /* @__PURE__ */ new Set();
|
|
23418
|
+
let topicFingerprint;
|
|
23201
23419
|
let polling = false;
|
|
23202
23420
|
const poll = () => {
|
|
23203
23421
|
if (polling) return;
|
|
23204
23422
|
polling = true;
|
|
23205
23423
|
const current = generation;
|
|
23206
|
-
changesSince(cursor, pending).then(({ records, resume, failed }) => {
|
|
23424
|
+
changesSince(cursor, pending).then(async ({ records, resume, failed }) => {
|
|
23425
|
+
let topicState;
|
|
23426
|
+
if ([...subscribers].some((s) => s.options.topics))
|
|
23427
|
+
try {
|
|
23428
|
+
topicState = JSON.stringify(
|
|
23429
|
+
(await persistence.topics()).map((t) => [t.name, t.version])
|
|
23430
|
+
);
|
|
23431
|
+
} catch (error) {
|
|
23432
|
+
getLogger().warn("[memory/remote-store] Topic subscription poll failed:", error);
|
|
23433
|
+
}
|
|
23207
23434
|
if (current !== generation) return;
|
|
23208
23435
|
const changed = /* @__PURE__ */ new Set();
|
|
23436
|
+
if (topicState !== void 0 && topicFingerprint !== void 0 && topicState !== topicFingerprint) {
|
|
23437
|
+
for (const subscriber of subscribers)
|
|
23438
|
+
if (subscriber.options.topics) changed.add(subscriber);
|
|
23439
|
+
}
|
|
23440
|
+
if (topicState !== void 0) topicFingerprint = topicState;
|
|
23209
23441
|
for (const record2 of records) {
|
|
23210
23442
|
const id = record2.memory.memory_id;
|
|
23211
23443
|
const previous = seen.get(id);
|
|
@@ -23229,11 +23461,11 @@ function createRemoteMemoryStore(options) {
|
|
|
23229
23461
|
};
|
|
23230
23462
|
const unrepairable = /* @__PURE__ */ new Set();
|
|
23231
23463
|
let repairCursor;
|
|
23232
|
-
const
|
|
23464
|
+
const repairEmbeddingsUnlocked = async ({
|
|
23233
23465
|
maxRows = REPAIR_ROWS,
|
|
23234
23466
|
maxScan = REPAIR_SCAN,
|
|
23235
23467
|
cursor: start = repairCursor
|
|
23236
|
-
} = {}) => {
|
|
23468
|
+
} = {}, leaseToken, signal) => {
|
|
23237
23469
|
const scannedRows = [];
|
|
23238
23470
|
let cursor2 = start;
|
|
23239
23471
|
do {
|
|
@@ -23242,6 +23474,7 @@ function createRemoteMemoryStore(options) {
|
|
|
23242
23474
|
include_archived: true,
|
|
23243
23475
|
include_quarantined: true,
|
|
23244
23476
|
include_superseded: true,
|
|
23477
|
+
...signal && { signal },
|
|
23245
23478
|
limit: Math.min(200, Math.max(1, maxScan - scannedRows.length)),
|
|
23246
23479
|
...cursor2 && { cursor: cursor2 }
|
|
23247
23480
|
});
|
|
@@ -23259,6 +23492,7 @@ function createRemoteMemoryStore(options) {
|
|
|
23259
23492
|
skippedConflict: 0
|
|
23260
23493
|
};
|
|
23261
23494
|
const finish = () => {
|
|
23495
|
+
signal?.throwIfAborted();
|
|
23262
23496
|
const settled = allStale.every(
|
|
23263
23497
|
({ memory }) => repaired.has(memory.memory_id) || unrepairable.has(memory.memory_id)
|
|
23264
23498
|
);
|
|
@@ -23267,13 +23501,15 @@ function createRemoteMemoryStore(options) {
|
|
|
23267
23501
|
};
|
|
23268
23502
|
const stale = allStale.slice(0, maxRows).map(({ memory }) => memory.memory_id);
|
|
23269
23503
|
if (stale.length === 0) return finish();
|
|
23270
|
-
const { records, unreadable } = await readEveryState(stale);
|
|
23504
|
+
const { records, unreadable } = await readEveryState(stale, signal);
|
|
23505
|
+
signal?.throwIfAborted();
|
|
23271
23506
|
result.failed += unreadable.size;
|
|
23272
23507
|
for (const id of stale)
|
|
23273
23508
|
if (unreadable.has(id) || !records.get(id)?.memory.content.trim()) unrepairable.add(id);
|
|
23274
23509
|
const targets = stale.filter((id) => !unrepairable.has(id)).map((id) => records.get(id));
|
|
23275
23510
|
if (targets.length === 0) return finish();
|
|
23276
|
-
const vectors = await embedForRepair(targets);
|
|
23511
|
+
const vectors = await embedForRepair(targets, signal);
|
|
23512
|
+
signal?.throwIfAborted();
|
|
23277
23513
|
if (!vectors) {
|
|
23278
23514
|
result.failed += targets.length;
|
|
23279
23515
|
return finish();
|
|
@@ -23285,15 +23521,28 @@ function createRemoteMemoryStore(options) {
|
|
|
23285
23521
|
continue;
|
|
23286
23522
|
}
|
|
23287
23523
|
await serialized(record2.memory.memory_id, async () => {
|
|
23524
|
+
signal?.throwIfAborted();
|
|
23288
23525
|
try {
|
|
23289
|
-
const saved = await persistence.
|
|
23290
|
-
|
|
23291
|
-
|
|
23526
|
+
const [saved] = await persistence.putMany(
|
|
23527
|
+
[
|
|
23528
|
+
{
|
|
23529
|
+
memory: {
|
|
23530
|
+
...record2.memory,
|
|
23531
|
+
embedding: storedVector(vector),
|
|
23532
|
+
embedding_model: model
|
|
23533
|
+
},
|
|
23534
|
+
expectedVersion: record2
|
|
23535
|
+
}
|
|
23536
|
+
],
|
|
23537
|
+
signal,
|
|
23538
|
+
leaseToken
|
|
23292
23539
|
);
|
|
23293
23540
|
notify("embedding", [saved]);
|
|
23294
23541
|
repaired.add(record2.memory.memory_id);
|
|
23295
23542
|
result.reembedded++;
|
|
23296
23543
|
} catch (error) {
|
|
23544
|
+
signal?.throwIfAborted();
|
|
23545
|
+
if (isRemoteWorkLeaseExpired(error)) throw error;
|
|
23297
23546
|
if (error instanceof RemoteMemoryError && error.code === "version_conflict")
|
|
23298
23547
|
result.skippedConflict++;
|
|
23299
23548
|
else result.failed++;
|
|
@@ -23302,19 +23551,28 @@ function createRemoteMemoryStore(options) {
|
|
|
23302
23551
|
}
|
|
23303
23552
|
return finish();
|
|
23304
23553
|
};
|
|
23305
|
-
const
|
|
23554
|
+
const repairEmbeddings = async (repairOptions = {}) => await withRemoteWorkLease(
|
|
23555
|
+
persistence,
|
|
23556
|
+
"embeddings",
|
|
23557
|
+
model,
|
|
23558
|
+
(token, signal) => repairEmbeddingsUnlocked(repairOptions, token, signal)
|
|
23559
|
+
) ?? { scanned: 0, reembedded: 0, failed: 0, skippedConflict: 0, deferred: true };
|
|
23560
|
+
const embedForRepair = async (targets, signal) => {
|
|
23306
23561
|
const texts = targets.map(({ memory }) => memory.content);
|
|
23307
23562
|
try {
|
|
23308
|
-
return await generateEmbeddings(texts, embeddingOptions);
|
|
23563
|
+
return await generateEmbeddings(texts, embeddingOptions, signal);
|
|
23309
23564
|
} catch (error) {
|
|
23565
|
+
signal?.throwIfAborted();
|
|
23310
23566
|
getLogger().warn("[memory/remote-store] Failed to embed memories for repair:", error);
|
|
23311
23567
|
if (!rejectsInput(error)) return null;
|
|
23312
23568
|
}
|
|
23313
23569
|
const vectors = [];
|
|
23314
23570
|
for (const [i, text14] of texts.entries()) {
|
|
23571
|
+
signal?.throwIfAborted();
|
|
23315
23572
|
try {
|
|
23316
|
-
[vectors[i]] = await generateEmbeddings([text14], embeddingOptions);
|
|
23573
|
+
[vectors[i]] = await generateEmbeddings([text14], embeddingOptions, signal);
|
|
23317
23574
|
} catch (error) {
|
|
23575
|
+
signal?.throwIfAborted();
|
|
23318
23576
|
if (!rejectsInput(error)) return null;
|
|
23319
23577
|
unrepairable.add(targets[i].memory.memory_id);
|
|
23320
23578
|
}
|
|
@@ -23322,6 +23580,26 @@ function createRemoteMemoryStore(options) {
|
|
|
23322
23580
|
return vectors;
|
|
23323
23581
|
};
|
|
23324
23582
|
return {
|
|
23583
|
+
createTopic: async (name) => {
|
|
23584
|
+
const canonical = await createRemoteTopic(persistence, name);
|
|
23585
|
+
if (canonical) notify("edit", []);
|
|
23586
|
+
return canonical;
|
|
23587
|
+
},
|
|
23588
|
+
importTopics: async (topics) => {
|
|
23589
|
+
if (!topics.length) return;
|
|
23590
|
+
const existing = new Set((await persistence.topics()).map((topic2) => topic2.name));
|
|
23591
|
+
const missing = topics.flatMap((topic2) => {
|
|
23592
|
+
const name = normalizeEntityName(topic2.name);
|
|
23593
|
+
if (existing.has(name)) return [];
|
|
23594
|
+
existing.add(name);
|
|
23595
|
+
return [{ ...topic2, name }];
|
|
23596
|
+
});
|
|
23597
|
+
if (!missing.length) return;
|
|
23598
|
+
await persistence.importTopics(missing);
|
|
23599
|
+
for (const subscriber of [...subscribers])
|
|
23600
|
+
if (subscriber.options.topics) safely(subscriber.onChange);
|
|
23601
|
+
},
|
|
23602
|
+
counts: () => persistence.counts(),
|
|
23325
23603
|
masksEmbeddingInput: embeddingOptions.maskInput !== void 0,
|
|
23326
23604
|
topicMaintenance,
|
|
23327
23605
|
repairEmbeddings,
|
|
@@ -23338,8 +23616,8 @@ function createRemoteMemoryStore(options) {
|
|
|
23338
23616
|
include_quarantined: true,
|
|
23339
23617
|
include_superseded: true
|
|
23340
23618
|
})) {
|
|
23341
|
-
for (const
|
|
23342
|
-
const name = normalizeEntityName(
|
|
23619
|
+
for (const topic2 of parseTopics(memory.topics) ?? []) {
|
|
23620
|
+
const name = normalizeEntityName(topic2.name);
|
|
23343
23621
|
if (!name) continue;
|
|
23344
23622
|
const entry = topics.get(name) ?? {
|
|
23345
23623
|
name,
|
|
@@ -23347,12 +23625,24 @@ function createRemoteMemoryStore(options) {
|
|
|
23347
23625
|
memoryCount: 0,
|
|
23348
23626
|
members: /* @__PURE__ */ new Set()
|
|
23349
23627
|
};
|
|
23350
|
-
entry.kind ?? (entry.kind =
|
|
23628
|
+
entry.kind ?? (entry.kind = topic2.kind ?? null);
|
|
23351
23629
|
entry.members.add(memory.memory_id);
|
|
23352
23630
|
topics.set(name, entry);
|
|
23353
23631
|
}
|
|
23354
23632
|
}
|
|
23355
|
-
|
|
23633
|
+
for (const topic2 of await persistence.topics()) {
|
|
23634
|
+
if (topic2.is_deleted) continue;
|
|
23635
|
+
const current = topics.get(topic2.name);
|
|
23636
|
+
if (current) current.kind ?? (current.kind = topic2.kind ?? null);
|
|
23637
|
+
else
|
|
23638
|
+
topics.set(topic2.name, {
|
|
23639
|
+
name: topic2.name,
|
|
23640
|
+
kind: topic2.kind ?? null,
|
|
23641
|
+
memoryCount: 0,
|
|
23642
|
+
members: /* @__PURE__ */ new Set()
|
|
23643
|
+
});
|
|
23644
|
+
}
|
|
23645
|
+
return [...topics.values()].map(({ members, ...topic2 }) => ({ ...topic2, memoryCount: members.size })).sort((a, b) => a.name.localeCompare(b.name));
|
|
23356
23646
|
},
|
|
23357
23647
|
get: async (id) => asStored(await live(id)),
|
|
23358
23648
|
listArchived: async () => (await enumerate({ archived_only: true, order: "archived", include_embeddings: true })).map(
|
|
@@ -23497,10 +23787,10 @@ function createRemoteMemoryStore(options) {
|
|
|
23497
23787
|
if (!saved) return [];
|
|
23498
23788
|
const now = /* @__PURE__ */ new Date();
|
|
23499
23789
|
return mergeTopics([], added).map(
|
|
23500
|
-
(
|
|
23501
|
-
uniqueId: normalizeEntityName(
|
|
23502
|
-
canonicalName: normalizeEntityName(
|
|
23503
|
-
kind:
|
|
23790
|
+
(topic2) => ({
|
|
23791
|
+
uniqueId: normalizeEntityName(topic2.name),
|
|
23792
|
+
canonicalName: normalizeEntityName(topic2.name),
|
|
23793
|
+
kind: topic2.kind ?? null,
|
|
23504
23794
|
createdAt: now,
|
|
23505
23795
|
updatedAt: now
|
|
23506
23796
|
})
|