@anuma/sdk 9.1.3 → 9.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/expo/index.cjs +89 -82
- package/dist/expo/index.d.mts +4 -0
- package/dist/expo/index.d.ts +4 -0
- package/dist/expo/index.mjs +89 -82
- package/dist/react/index.cjs +89 -82
- package/dist/react/index.d.mts +4 -0
- package/dist/react/index.d.ts +4 -0
- package/dist/react/index.mjs +89 -82
- package/package.json +1 -1
package/dist/expo/index.cjs
CHANGED
|
@@ -22927,6 +22927,8 @@ async function backfillTopics(ctx) {
|
|
|
22927
22927
|
if (topicless.length > 0) await backfillMemoryTopicsOp(ctx, topicless);
|
|
22928
22928
|
}
|
|
22929
22929
|
async function migrateMemoriesToRemote(options) {
|
|
22930
|
+
if (typeof options.withWriteBarrier !== "function" || typeof options.onActivated !== "function")
|
|
22931
|
+
throw new Error("A write barrier and a remote store handoff are required");
|
|
22930
22932
|
const { canonical, signal } = options;
|
|
22931
22933
|
const rings = [canonical, ...options.fallbacks ?? []];
|
|
22932
22934
|
const model = options.embeddingOptions?.model ?? DEFAULT_API_EMBEDDING_MODEL;
|
|
@@ -22954,93 +22956,98 @@ async function migrateMemoriesToRemote(options) {
|
|
|
22954
22956
|
const placeholders = /* @__PURE__ */ new Set();
|
|
22955
22957
|
const truncated = /* @__PURE__ */ new Set();
|
|
22956
22958
|
let imported = 0;
|
|
22957
|
-
|
|
22958
|
-
|
|
22959
|
-
|
|
22960
|
-
|
|
22961
|
-
|
|
22962
|
-
|
|
22963
|
-
|
|
22964
|
-
|
|
22965
|
-
|
|
22966
|
-
|
|
22967
|
-
|
|
22968
|
-
|
|
22969
|
-
|
|
22970
|
-
);
|
|
22971
|
-
const needsVector = (raw, server) => !!options.embeddingOptions && !embedding(raw, model) && server.embedding_model !== model;
|
|
22972
|
-
for (const raw of candidates) {
|
|
22973
|
-
const server = remote.get(raw.id);
|
|
22974
|
-
if (server && server.updated_at === raw.updated_at && (!sameMetadata2(raw, server, model) || needsVector(raw, server)))
|
|
22975
|
-
pending.set(raw.id, raw.updated_at + 1);
|
|
22976
|
-
}
|
|
22977
|
-
}
|
|
22978
|
-
if (pending.size === 0) {
|
|
22979
|
-
const result = (state) => ({
|
|
22980
|
-
state,
|
|
22981
|
-
total: rows.length,
|
|
22982
|
-
imported,
|
|
22983
|
-
skipped: [...skipped].map(([memoryId, reason]) => ({ memoryId, reason })),
|
|
22984
|
-
reembedded: reembedded.size,
|
|
22985
|
-
unembedded: unembedded.size,
|
|
22986
|
-
placeholderTombstones: placeholders.size,
|
|
22987
|
-
truncatedSourceChunkIds: truncated.size
|
|
22988
|
-
});
|
|
22989
|
-
if (skipped.size > 0 && !options.allowSkipped) return result("migrating");
|
|
22990
|
-
if (!claimed) await importBatch(request, canonical.keyId, [], signal);
|
|
22991
|
-
if (!active)
|
|
22992
|
-
await request(
|
|
22993
|
-
"/account",
|
|
22994
|
-
{ method: "PUT", body: JSON.stringify({ state: "active" }) },
|
|
22959
|
+
const uploadPasses = async (finalizing = false) => {
|
|
22960
|
+
for (let pass = 0; ; pass++) {
|
|
22961
|
+
const rows = await getAllVaultMemoryRowsOp(options.vaultCtx);
|
|
22962
|
+
for (const raw of rows) if (invalid2(raw)) skipped.set(String(raw.id), "invalid");
|
|
22963
|
+
const candidates = rows.filter((raw) => !skipped.has(String(raw.id)));
|
|
22964
|
+
const missing = await missingIds(request, candidates.map(revision), signal);
|
|
22965
|
+
const pending = /* @__PURE__ */ new Map();
|
|
22966
|
+
for (const raw of candidates)
|
|
22967
|
+
if (missing.has(raw.id)) pending.set(raw.id, raw.updated_at);
|
|
22968
|
+
if (pending.size === 0 && !active) {
|
|
22969
|
+
const remote = await remoteMetadata(
|
|
22970
|
+
request,
|
|
22971
|
+
candidates.map((raw) => raw.id),
|
|
22995
22972
|
signal
|
|
22996
22973
|
);
|
|
22997
|
-
|
|
22998
|
-
|
|
22999
|
-
|
|
23000
|
-
|
|
23001
|
-
|
|
23002
|
-
|
|
23003
|
-
|
|
23004
|
-
)
|
|
23005
|
-
|
|
23006
|
-
|
|
23007
|
-
|
|
23008
|
-
|
|
23009
|
-
|
|
23010
|
-
|
|
23011
|
-
|
|
23012
|
-
|
|
23013
|
-
|
|
23014
|
-
|
|
23015
|
-
|
|
23016
|
-
|
|
23017
|
-
|
|
23018
|
-
|
|
23019
|
-
|
|
23020
|
-
|
|
23021
|
-
|
|
23022
|
-
|
|
23023
|
-
|
|
23024
|
-
|
|
23025
|
-
|
|
22974
|
+
const needsVector = (raw, server) => !!options.embeddingOptions && !embedding(raw, model) && server.embedding_model !== model;
|
|
22975
|
+
for (const raw of candidates) {
|
|
22976
|
+
const server = remote.get(raw.id);
|
|
22977
|
+
if (server && server.updated_at === raw.updated_at && (!sameMetadata2(raw, server, model) || needsVector(raw, server)))
|
|
22978
|
+
pending.set(raw.id, raw.updated_at + 1);
|
|
22979
|
+
}
|
|
22980
|
+
}
|
|
22981
|
+
if (pending.size === 0) {
|
|
22982
|
+
const result = (state) => ({
|
|
22983
|
+
state,
|
|
22984
|
+
total: rows.length,
|
|
22985
|
+
imported,
|
|
22986
|
+
skipped: [...skipped].map(([memoryId, reason]) => ({ memoryId, reason })),
|
|
22987
|
+
reembedded: reembedded.size,
|
|
22988
|
+
unembedded: unembedded.size,
|
|
22989
|
+
placeholderTombstones: placeholders.size,
|
|
22990
|
+
truncatedSourceChunkIds: truncated.size
|
|
22991
|
+
});
|
|
22992
|
+
if (skipped.size > 0 && !options.allowSkipped) return result("migrating");
|
|
22993
|
+
if (!finalizing) return options.withWriteBarrier(() => uploadPasses(true));
|
|
22994
|
+
if (!claimed) await importBatch(request, canonical.keyId, [], signal);
|
|
22995
|
+
if (!active)
|
|
22996
|
+
await request(
|
|
22997
|
+
"/account",
|
|
22998
|
+
{ method: "PUT", body: JSON.stringify({ state: "active" }) },
|
|
22999
|
+
signal
|
|
23000
|
+
);
|
|
23001
|
+
await options.onActivated();
|
|
23002
|
+
return result("active");
|
|
23003
|
+
}
|
|
23004
|
+
if (pass >= MAX_UPLOAD_PASSES)
|
|
23005
|
+
throw new RemoteMemoryError(
|
|
23006
|
+
"Nearby still reports memories missing after importing them",
|
|
23007
|
+
409,
|
|
23008
|
+
"migration_incomplete"
|
|
23009
|
+
);
|
|
23010
|
+
const prepared = [];
|
|
23011
|
+
for (const raw of candidates) {
|
|
23012
|
+
const updatedAt = pending.get(raw.id);
|
|
23013
|
+
if (updatedAt === void 0) continue;
|
|
23014
|
+
try {
|
|
23015
|
+
prepared.push(await prepareRow(raw, canonical, rings, model, updatedAt));
|
|
23016
|
+
} catch (error) {
|
|
23017
|
+
if (!(error instanceof MemoryKeyError)) throw error;
|
|
23018
|
+
skipped.set(raw.id, "unreadable");
|
|
23019
|
+
}
|
|
23020
|
+
}
|
|
23021
|
+
const usable = new Set(prepared.filter((p) => p.row.embedding).map((p) => p.row.memory_id));
|
|
23022
|
+
if (options.embeddingOptions)
|
|
23023
|
+
await embedMissing(prepared, options.embeddingOptions, model, signal);
|
|
23024
|
+
const local = new Map(candidates.map((raw) => [raw.id, raw]));
|
|
23025
|
+
const upload = [];
|
|
23026
|
+
for (const { row, plaintext } of prepared) {
|
|
23027
|
+
const id2 = row.memory_id;
|
|
23028
|
+
if (!fitsNearby(row)) {
|
|
23029
|
+
skipped.set(id2, "invalid");
|
|
23030
|
+
continue;
|
|
23031
|
+
}
|
|
23032
|
+
upload.push(row);
|
|
23033
|
+
const track = (set, on) => on ? set.add(id2) : set.delete(id2);
|
|
23034
|
+
track(reembedded, !!row.embedding && !usable.has(id2));
|
|
23035
|
+
track(unembedded, !row.embedding);
|
|
23036
|
+
track(placeholders, plaintext === void 0);
|
|
23037
|
+
track(truncated, row.source_chunk_ids !== text13(local.get(id2).source_chunk_ids));
|
|
23026
23038
|
}
|
|
23027
|
-
|
|
23028
|
-
const track = (set, on) => on ? set.add(id2) : set.delete(id2);
|
|
23029
|
-
track(reembedded, !!row.embedding && !usable.has(id2));
|
|
23030
|
-
track(unembedded, !row.embedding);
|
|
23031
|
-
track(placeholders, plaintext === void 0);
|
|
23032
|
-
track(truncated, row.source_chunk_ids !== text13(local.get(id2).source_chunk_ids));
|
|
23033
|
-
}
|
|
23034
|
-
let sent = 0;
|
|
23035
|
-
options.onProgress?.({ imported: sent, toImport: upload.length });
|
|
23036
|
-
for (const batch of chunks(upload, IMPORT_BATCH)) {
|
|
23037
|
-
await importBatch(request, canonical.keyId, batch, signal);
|
|
23038
|
-
claimed = true;
|
|
23039
|
-
sent += batch.length;
|
|
23039
|
+
let sent = 0;
|
|
23040
23040
|
options.onProgress?.({ imported: sent, toImport: upload.length });
|
|
23041
|
+
for (const batch of chunks(upload, IMPORT_BATCH)) {
|
|
23042
|
+
await importBatch(request, canonical.keyId, batch, signal);
|
|
23043
|
+
claimed = true;
|
|
23044
|
+
sent += batch.length;
|
|
23045
|
+
options.onProgress?.({ imported: sent, toImport: upload.length });
|
|
23046
|
+
}
|
|
23047
|
+
imported += sent;
|
|
23041
23048
|
}
|
|
23042
|
-
|
|
23043
|
-
|
|
23049
|
+
};
|
|
23050
|
+
return uploadPasses();
|
|
23044
23051
|
}
|
|
23045
23052
|
|
|
23046
23053
|
// src/lib/memory/store/remoteStore.ts
|
package/dist/expo/index.d.mts
CHANGED
|
@@ -9931,6 +9931,10 @@ interface MemoryMigrationOptions extends Pick<RemoteMemoryPersistenceOptions, "b
|
|
|
9931
9931
|
name: string;
|
|
9932
9932
|
kind?: string;
|
|
9933
9933
|
}[];
|
|
9934
|
+
/** Drain all local memory writes, block new writes until the task returns, and retain the block if the remote handoff fails. */
|
|
9935
|
+
withWriteBarrier: (task: () => Promise<MemoryMigrationResult>) => Promise<MemoryMigrationResult>;
|
|
9936
|
+
/** Route all subsequent memory writes to the remote store before this callback returns. */
|
|
9937
|
+
onActivated: () => void | Promise<void>;
|
|
9934
9938
|
signal?: AbortSignal;
|
|
9935
9939
|
onProgress?: (progress: MemoryMigrationProgress) => void;
|
|
9936
9940
|
}
|
package/dist/expo/index.d.ts
CHANGED
|
@@ -9931,6 +9931,10 @@ interface MemoryMigrationOptions extends Pick<RemoteMemoryPersistenceOptions, "b
|
|
|
9931
9931
|
name: string;
|
|
9932
9932
|
kind?: string;
|
|
9933
9933
|
}[];
|
|
9934
|
+
/** Drain all local memory writes, block new writes until the task returns, and retain the block if the remote handoff fails. */
|
|
9935
|
+
withWriteBarrier: (task: () => Promise<MemoryMigrationResult>) => Promise<MemoryMigrationResult>;
|
|
9936
|
+
/** Route all subsequent memory writes to the remote store before this callback returns. */
|
|
9937
|
+
onActivated: () => void | Promise<void>;
|
|
9934
9938
|
signal?: AbortSignal;
|
|
9935
9939
|
onProgress?: (progress: MemoryMigrationProgress) => void;
|
|
9936
9940
|
}
|
package/dist/expo/index.mjs
CHANGED
|
@@ -21252,6 +21252,8 @@ async function backfillTopics(ctx) {
|
|
|
21252
21252
|
if (topicless.length > 0) await backfillMemoryTopicsOp(ctx, topicless);
|
|
21253
21253
|
}
|
|
21254
21254
|
async function migrateMemoriesToRemote(options) {
|
|
21255
|
+
if (typeof options.withWriteBarrier !== "function" || typeof options.onActivated !== "function")
|
|
21256
|
+
throw new Error("A write barrier and a remote store handoff are required");
|
|
21255
21257
|
const { canonical, signal } = options;
|
|
21256
21258
|
const rings = [canonical, ...options.fallbacks ?? []];
|
|
21257
21259
|
const model = options.embeddingOptions?.model ?? DEFAULT_API_EMBEDDING_MODEL;
|
|
@@ -21279,93 +21281,98 @@ async function migrateMemoriesToRemote(options) {
|
|
|
21279
21281
|
const placeholders = /* @__PURE__ */ new Set();
|
|
21280
21282
|
const truncated = /* @__PURE__ */ new Set();
|
|
21281
21283
|
let imported = 0;
|
|
21282
|
-
|
|
21283
|
-
|
|
21284
|
-
|
|
21285
|
-
|
|
21286
|
-
|
|
21287
|
-
|
|
21288
|
-
|
|
21289
|
-
|
|
21290
|
-
|
|
21291
|
-
|
|
21292
|
-
|
|
21293
|
-
|
|
21294
|
-
|
|
21295
|
-
);
|
|
21296
|
-
const needsVector = (raw, server) => !!options.embeddingOptions && !embedding(raw, model) && server.embedding_model !== model;
|
|
21297
|
-
for (const raw of candidates) {
|
|
21298
|
-
const server = remote.get(raw.id);
|
|
21299
|
-
if (server && server.updated_at === raw.updated_at && (!sameMetadata2(raw, server, model) || needsVector(raw, server)))
|
|
21300
|
-
pending.set(raw.id, raw.updated_at + 1);
|
|
21301
|
-
}
|
|
21302
|
-
}
|
|
21303
|
-
if (pending.size === 0) {
|
|
21304
|
-
const result = (state) => ({
|
|
21305
|
-
state,
|
|
21306
|
-
total: rows.length,
|
|
21307
|
-
imported,
|
|
21308
|
-
skipped: [...skipped].map(([memoryId, reason]) => ({ memoryId, reason })),
|
|
21309
|
-
reembedded: reembedded.size,
|
|
21310
|
-
unembedded: unembedded.size,
|
|
21311
|
-
placeholderTombstones: placeholders.size,
|
|
21312
|
-
truncatedSourceChunkIds: truncated.size
|
|
21313
|
-
});
|
|
21314
|
-
if (skipped.size > 0 && !options.allowSkipped) return result("migrating");
|
|
21315
|
-
if (!claimed) await importBatch(request, canonical.keyId, [], signal);
|
|
21316
|
-
if (!active)
|
|
21317
|
-
await request(
|
|
21318
|
-
"/account",
|
|
21319
|
-
{ method: "PUT", body: JSON.stringify({ state: "active" }) },
|
|
21284
|
+
const uploadPasses = async (finalizing = false) => {
|
|
21285
|
+
for (let pass = 0; ; pass++) {
|
|
21286
|
+
const rows = await getAllVaultMemoryRowsOp(options.vaultCtx);
|
|
21287
|
+
for (const raw of rows) if (invalid2(raw)) skipped.set(String(raw.id), "invalid");
|
|
21288
|
+
const candidates = rows.filter((raw) => !skipped.has(String(raw.id)));
|
|
21289
|
+
const missing = await missingIds(request, candidates.map(revision), signal);
|
|
21290
|
+
const pending = /* @__PURE__ */ new Map();
|
|
21291
|
+
for (const raw of candidates)
|
|
21292
|
+
if (missing.has(raw.id)) pending.set(raw.id, raw.updated_at);
|
|
21293
|
+
if (pending.size === 0 && !active) {
|
|
21294
|
+
const remote = await remoteMetadata(
|
|
21295
|
+
request,
|
|
21296
|
+
candidates.map((raw) => raw.id),
|
|
21320
21297
|
signal
|
|
21321
21298
|
);
|
|
21322
|
-
|
|
21323
|
-
|
|
21324
|
-
|
|
21325
|
-
|
|
21326
|
-
|
|
21327
|
-
|
|
21328
|
-
"migration_incomplete"
|
|
21329
|
-
);
|
|
21330
|
-
const prepared = [];
|
|
21331
|
-
for (const raw of candidates) {
|
|
21332
|
-
const updatedAt = pending.get(raw.id);
|
|
21333
|
-
if (updatedAt === void 0) continue;
|
|
21334
|
-
try {
|
|
21335
|
-
prepared.push(await prepareRow(raw, canonical, rings, model, updatedAt));
|
|
21336
|
-
} catch (error) {
|
|
21337
|
-
if (!(error instanceof MemoryKeyError)) throw error;
|
|
21338
|
-
skipped.set(raw.id, "unreadable");
|
|
21339
|
-
}
|
|
21340
|
-
}
|
|
21341
|
-
const usable = new Set(prepared.filter((p) => p.row.embedding).map((p) => p.row.memory_id));
|
|
21342
|
-
if (options.embeddingOptions)
|
|
21343
|
-
await embedMissing(prepared, options.embeddingOptions, model, signal);
|
|
21344
|
-
const local = new Map(candidates.map((raw) => [raw.id, raw]));
|
|
21345
|
-
const upload = [];
|
|
21346
|
-
for (const { row, plaintext } of prepared) {
|
|
21347
|
-
const id2 = row.memory_id;
|
|
21348
|
-
if (!fitsNearby(row)) {
|
|
21349
|
-
skipped.set(id2, "invalid");
|
|
21350
|
-
continue;
|
|
21299
|
+
const needsVector = (raw, server) => !!options.embeddingOptions && !embedding(raw, model) && server.embedding_model !== model;
|
|
21300
|
+
for (const raw of candidates) {
|
|
21301
|
+
const server = remote.get(raw.id);
|
|
21302
|
+
if (server && server.updated_at === raw.updated_at && (!sameMetadata2(raw, server, model) || needsVector(raw, server)))
|
|
21303
|
+
pending.set(raw.id, raw.updated_at + 1);
|
|
21304
|
+
}
|
|
21351
21305
|
}
|
|
21352
|
-
|
|
21353
|
-
|
|
21354
|
-
|
|
21355
|
-
|
|
21356
|
-
|
|
21357
|
-
|
|
21358
|
-
|
|
21359
|
-
|
|
21360
|
-
|
|
21361
|
-
|
|
21362
|
-
|
|
21363
|
-
|
|
21364
|
-
|
|
21306
|
+
if (pending.size === 0) {
|
|
21307
|
+
const result = (state) => ({
|
|
21308
|
+
state,
|
|
21309
|
+
total: rows.length,
|
|
21310
|
+
imported,
|
|
21311
|
+
skipped: [...skipped].map(([memoryId, reason]) => ({ memoryId, reason })),
|
|
21312
|
+
reembedded: reembedded.size,
|
|
21313
|
+
unembedded: unembedded.size,
|
|
21314
|
+
placeholderTombstones: placeholders.size,
|
|
21315
|
+
truncatedSourceChunkIds: truncated.size
|
|
21316
|
+
});
|
|
21317
|
+
if (skipped.size > 0 && !options.allowSkipped) return result("migrating");
|
|
21318
|
+
if (!finalizing) return options.withWriteBarrier(() => uploadPasses(true));
|
|
21319
|
+
if (!claimed) await importBatch(request, canonical.keyId, [], signal);
|
|
21320
|
+
if (!active)
|
|
21321
|
+
await request(
|
|
21322
|
+
"/account",
|
|
21323
|
+
{ method: "PUT", body: JSON.stringify({ state: "active" }) },
|
|
21324
|
+
signal
|
|
21325
|
+
);
|
|
21326
|
+
await options.onActivated();
|
|
21327
|
+
return result("active");
|
|
21328
|
+
}
|
|
21329
|
+
if (pass >= MAX_UPLOAD_PASSES)
|
|
21330
|
+
throw new RemoteMemoryError(
|
|
21331
|
+
"Nearby still reports memories missing after importing them",
|
|
21332
|
+
409,
|
|
21333
|
+
"migration_incomplete"
|
|
21334
|
+
);
|
|
21335
|
+
const prepared = [];
|
|
21336
|
+
for (const raw of candidates) {
|
|
21337
|
+
const updatedAt = pending.get(raw.id);
|
|
21338
|
+
if (updatedAt === void 0) continue;
|
|
21339
|
+
try {
|
|
21340
|
+
prepared.push(await prepareRow(raw, canonical, rings, model, updatedAt));
|
|
21341
|
+
} catch (error) {
|
|
21342
|
+
if (!(error instanceof MemoryKeyError)) throw error;
|
|
21343
|
+
skipped.set(raw.id, "unreadable");
|
|
21344
|
+
}
|
|
21345
|
+
}
|
|
21346
|
+
const usable = new Set(prepared.filter((p) => p.row.embedding).map((p) => p.row.memory_id));
|
|
21347
|
+
if (options.embeddingOptions)
|
|
21348
|
+
await embedMissing(prepared, options.embeddingOptions, model, signal);
|
|
21349
|
+
const local = new Map(candidates.map((raw) => [raw.id, raw]));
|
|
21350
|
+
const upload = [];
|
|
21351
|
+
for (const { row, plaintext } of prepared) {
|
|
21352
|
+
const id2 = row.memory_id;
|
|
21353
|
+
if (!fitsNearby(row)) {
|
|
21354
|
+
skipped.set(id2, "invalid");
|
|
21355
|
+
continue;
|
|
21356
|
+
}
|
|
21357
|
+
upload.push(row);
|
|
21358
|
+
const track = (set, on) => on ? set.add(id2) : set.delete(id2);
|
|
21359
|
+
track(reembedded, !!row.embedding && !usable.has(id2));
|
|
21360
|
+
track(unembedded, !row.embedding);
|
|
21361
|
+
track(placeholders, plaintext === void 0);
|
|
21362
|
+
track(truncated, row.source_chunk_ids !== text13(local.get(id2).source_chunk_ids));
|
|
21363
|
+
}
|
|
21364
|
+
let sent = 0;
|
|
21365
21365
|
options.onProgress?.({ imported: sent, toImport: upload.length });
|
|
21366
|
+
for (const batch of chunks(upload, IMPORT_BATCH)) {
|
|
21367
|
+
await importBatch(request, canonical.keyId, batch, signal);
|
|
21368
|
+
claimed = true;
|
|
21369
|
+
sent += batch.length;
|
|
21370
|
+
options.onProgress?.({ imported: sent, toImport: upload.length });
|
|
21371
|
+
}
|
|
21372
|
+
imported += sent;
|
|
21366
21373
|
}
|
|
21367
|
-
|
|
21368
|
-
|
|
21374
|
+
};
|
|
21375
|
+
return uploadPasses();
|
|
21369
21376
|
}
|
|
21370
21377
|
|
|
21371
21378
|
// src/lib/memory/store/remoteStore.ts
|
package/dist/react/index.cjs
CHANGED
|
@@ -25240,6 +25240,8 @@ async function backfillTopics(ctx) {
|
|
|
25240
25240
|
if (topicless.length > 0) await backfillMemoryTopicsOp(ctx, topicless);
|
|
25241
25241
|
}
|
|
25242
25242
|
async function migrateMemoriesToRemote(options) {
|
|
25243
|
+
if (typeof options.withWriteBarrier !== "function" || typeof options.onActivated !== "function")
|
|
25244
|
+
throw new Error("A write barrier and a remote store handoff are required");
|
|
25243
25245
|
const { canonical, signal } = options;
|
|
25244
25246
|
const rings = [canonical, ...options.fallbacks ?? []];
|
|
25245
25247
|
const model = options.embeddingOptions?.model ?? DEFAULT_API_EMBEDDING_MODEL;
|
|
@@ -25267,93 +25269,98 @@ async function migrateMemoriesToRemote(options) {
|
|
|
25267
25269
|
const placeholders = /* @__PURE__ */ new Set();
|
|
25268
25270
|
const truncated = /* @__PURE__ */ new Set();
|
|
25269
25271
|
let imported = 0;
|
|
25270
|
-
|
|
25271
|
-
|
|
25272
|
-
|
|
25273
|
-
|
|
25274
|
-
|
|
25275
|
-
|
|
25276
|
-
|
|
25277
|
-
|
|
25278
|
-
|
|
25279
|
-
|
|
25280
|
-
|
|
25281
|
-
|
|
25282
|
-
|
|
25283
|
-
);
|
|
25284
|
-
const needsVector = (raw, server) => !!options.embeddingOptions && !embedding(raw, model) && server.embedding_model !== model;
|
|
25285
|
-
for (const raw of candidates) {
|
|
25286
|
-
const server = remote.get(raw.id);
|
|
25287
|
-
if (server && server.updated_at === raw.updated_at && (!sameMetadata2(raw, server, model) || needsVector(raw, server)))
|
|
25288
|
-
pending.set(raw.id, raw.updated_at + 1);
|
|
25289
|
-
}
|
|
25290
|
-
}
|
|
25291
|
-
if (pending.size === 0) {
|
|
25292
|
-
const result = (state) => ({
|
|
25293
|
-
state,
|
|
25294
|
-
total: rows.length,
|
|
25295
|
-
imported,
|
|
25296
|
-
skipped: [...skipped].map(([memoryId, reason]) => ({ memoryId, reason })),
|
|
25297
|
-
reembedded: reembedded.size,
|
|
25298
|
-
unembedded: unembedded.size,
|
|
25299
|
-
placeholderTombstones: placeholders.size,
|
|
25300
|
-
truncatedSourceChunkIds: truncated.size
|
|
25301
|
-
});
|
|
25302
|
-
if (skipped.size > 0 && !options.allowSkipped) return result("migrating");
|
|
25303
|
-
if (!claimed) await importBatch(request, canonical.keyId, [], signal);
|
|
25304
|
-
if (!active)
|
|
25305
|
-
await request(
|
|
25306
|
-
"/account",
|
|
25307
|
-
{ method: "PUT", body: JSON.stringify({ state: "active" }) },
|
|
25272
|
+
const uploadPasses = async (finalizing = false) => {
|
|
25273
|
+
for (let pass = 0; ; pass++) {
|
|
25274
|
+
const rows = await getAllVaultMemoryRowsOp(options.vaultCtx);
|
|
25275
|
+
for (const raw of rows) if (invalid2(raw)) skipped.set(String(raw.id), "invalid");
|
|
25276
|
+
const candidates = rows.filter((raw) => !skipped.has(String(raw.id)));
|
|
25277
|
+
const missing = await missingIds(request, candidates.map(revision), signal);
|
|
25278
|
+
const pending = /* @__PURE__ */ new Map();
|
|
25279
|
+
for (const raw of candidates)
|
|
25280
|
+
if (missing.has(raw.id)) pending.set(raw.id, raw.updated_at);
|
|
25281
|
+
if (pending.size === 0 && !active) {
|
|
25282
|
+
const remote = await remoteMetadata(
|
|
25283
|
+
request,
|
|
25284
|
+
candidates.map((raw) => raw.id),
|
|
25308
25285
|
signal
|
|
25309
25286
|
);
|
|
25310
|
-
|
|
25311
|
-
|
|
25312
|
-
|
|
25313
|
-
|
|
25314
|
-
|
|
25315
|
-
|
|
25316
|
-
|
|
25317
|
-
)
|
|
25318
|
-
|
|
25319
|
-
|
|
25320
|
-
|
|
25321
|
-
|
|
25322
|
-
|
|
25323
|
-
|
|
25324
|
-
|
|
25325
|
-
|
|
25326
|
-
|
|
25327
|
-
|
|
25328
|
-
|
|
25329
|
-
|
|
25330
|
-
|
|
25331
|
-
|
|
25332
|
-
|
|
25333
|
-
|
|
25334
|
-
|
|
25335
|
-
|
|
25336
|
-
|
|
25337
|
-
|
|
25338
|
-
|
|
25287
|
+
const needsVector = (raw, server) => !!options.embeddingOptions && !embedding(raw, model) && server.embedding_model !== model;
|
|
25288
|
+
for (const raw of candidates) {
|
|
25289
|
+
const server = remote.get(raw.id);
|
|
25290
|
+
if (server && server.updated_at === raw.updated_at && (!sameMetadata2(raw, server, model) || needsVector(raw, server)))
|
|
25291
|
+
pending.set(raw.id, raw.updated_at + 1);
|
|
25292
|
+
}
|
|
25293
|
+
}
|
|
25294
|
+
if (pending.size === 0) {
|
|
25295
|
+
const result = (state) => ({
|
|
25296
|
+
state,
|
|
25297
|
+
total: rows.length,
|
|
25298
|
+
imported,
|
|
25299
|
+
skipped: [...skipped].map(([memoryId, reason]) => ({ memoryId, reason })),
|
|
25300
|
+
reembedded: reembedded.size,
|
|
25301
|
+
unembedded: unembedded.size,
|
|
25302
|
+
placeholderTombstones: placeholders.size,
|
|
25303
|
+
truncatedSourceChunkIds: truncated.size
|
|
25304
|
+
});
|
|
25305
|
+
if (skipped.size > 0 && !options.allowSkipped) return result("migrating");
|
|
25306
|
+
if (!finalizing) return options.withWriteBarrier(() => uploadPasses(true));
|
|
25307
|
+
if (!claimed) await importBatch(request, canonical.keyId, [], signal);
|
|
25308
|
+
if (!active)
|
|
25309
|
+
await request(
|
|
25310
|
+
"/account",
|
|
25311
|
+
{ method: "PUT", body: JSON.stringify({ state: "active" }) },
|
|
25312
|
+
signal
|
|
25313
|
+
);
|
|
25314
|
+
await options.onActivated();
|
|
25315
|
+
return result("active");
|
|
25316
|
+
}
|
|
25317
|
+
if (pass >= MAX_UPLOAD_PASSES)
|
|
25318
|
+
throw new RemoteMemoryError(
|
|
25319
|
+
"Nearby still reports memories missing after importing them",
|
|
25320
|
+
409,
|
|
25321
|
+
"migration_incomplete"
|
|
25322
|
+
);
|
|
25323
|
+
const prepared = [];
|
|
25324
|
+
for (const raw of candidates) {
|
|
25325
|
+
const updatedAt = pending.get(raw.id);
|
|
25326
|
+
if (updatedAt === void 0) continue;
|
|
25327
|
+
try {
|
|
25328
|
+
prepared.push(await prepareRow(raw, canonical, rings, model, updatedAt));
|
|
25329
|
+
} catch (error) {
|
|
25330
|
+
if (!(error instanceof MemoryKeyError)) throw error;
|
|
25331
|
+
skipped.set(raw.id, "unreadable");
|
|
25332
|
+
}
|
|
25333
|
+
}
|
|
25334
|
+
const usable = new Set(prepared.filter((p) => p.row.embedding).map((p) => p.row.memory_id));
|
|
25335
|
+
if (options.embeddingOptions)
|
|
25336
|
+
await embedMissing(prepared, options.embeddingOptions, model, signal);
|
|
25337
|
+
const local = new Map(candidates.map((raw) => [raw.id, raw]));
|
|
25338
|
+
const upload = [];
|
|
25339
|
+
for (const { row, plaintext } of prepared) {
|
|
25340
|
+
const id2 = row.memory_id;
|
|
25341
|
+
if (!fitsNearby(row)) {
|
|
25342
|
+
skipped.set(id2, "invalid");
|
|
25343
|
+
continue;
|
|
25344
|
+
}
|
|
25345
|
+
upload.push(row);
|
|
25346
|
+
const track = (set, on) => on ? set.add(id2) : set.delete(id2);
|
|
25347
|
+
track(reembedded, !!row.embedding && !usable.has(id2));
|
|
25348
|
+
track(unembedded, !row.embedding);
|
|
25349
|
+
track(placeholders, plaintext === void 0);
|
|
25350
|
+
track(truncated, row.source_chunk_ids !== text13(local.get(id2).source_chunk_ids));
|
|
25339
25351
|
}
|
|
25340
|
-
|
|
25341
|
-
const track = (set, on) => on ? set.add(id2) : set.delete(id2);
|
|
25342
|
-
track(reembedded, !!row.embedding && !usable.has(id2));
|
|
25343
|
-
track(unembedded, !row.embedding);
|
|
25344
|
-
track(placeholders, plaintext === void 0);
|
|
25345
|
-
track(truncated, row.source_chunk_ids !== text13(local.get(id2).source_chunk_ids));
|
|
25346
|
-
}
|
|
25347
|
-
let sent = 0;
|
|
25348
|
-
options.onProgress?.({ imported: sent, toImport: upload.length });
|
|
25349
|
-
for (const batch of chunks(upload, IMPORT_BATCH)) {
|
|
25350
|
-
await importBatch(request, canonical.keyId, batch, signal);
|
|
25351
|
-
claimed = true;
|
|
25352
|
-
sent += batch.length;
|
|
25352
|
+
let sent = 0;
|
|
25353
25353
|
options.onProgress?.({ imported: sent, toImport: upload.length });
|
|
25354
|
+
for (const batch of chunks(upload, IMPORT_BATCH)) {
|
|
25355
|
+
await importBatch(request, canonical.keyId, batch, signal);
|
|
25356
|
+
claimed = true;
|
|
25357
|
+
sent += batch.length;
|
|
25358
|
+
options.onProgress?.({ imported: sent, toImport: upload.length });
|
|
25359
|
+
}
|
|
25360
|
+
imported += sent;
|
|
25354
25361
|
}
|
|
25355
|
-
|
|
25356
|
-
|
|
25362
|
+
};
|
|
25363
|
+
return uploadPasses();
|
|
25357
25364
|
}
|
|
25358
25365
|
|
|
25359
25366
|
// src/lib/memory/store/remoteStore.ts
|
package/dist/react/index.d.mts
CHANGED
|
@@ -11665,6 +11665,10 @@ interface MemoryMigrationOptions extends Pick<RemoteMemoryPersistenceOptions, "b
|
|
|
11665
11665
|
name: string;
|
|
11666
11666
|
kind?: string;
|
|
11667
11667
|
}[];
|
|
11668
|
+
/** Drain all local memory writes, block new writes until the task returns, and retain the block if the remote handoff fails. */
|
|
11669
|
+
withWriteBarrier: (task: () => Promise<MemoryMigrationResult>) => Promise<MemoryMigrationResult>;
|
|
11670
|
+
/** Route all subsequent memory writes to the remote store before this callback returns. */
|
|
11671
|
+
onActivated: () => void | Promise<void>;
|
|
11668
11672
|
signal?: AbortSignal;
|
|
11669
11673
|
onProgress?: (progress: MemoryMigrationProgress) => void;
|
|
11670
11674
|
}
|
package/dist/react/index.d.ts
CHANGED
|
@@ -11665,6 +11665,10 @@ interface MemoryMigrationOptions extends Pick<RemoteMemoryPersistenceOptions, "b
|
|
|
11665
11665
|
name: string;
|
|
11666
11666
|
kind?: string;
|
|
11667
11667
|
}[];
|
|
11668
|
+
/** Drain all local memory writes, block new writes until the task returns, and retain the block if the remote handoff fails. */
|
|
11669
|
+
withWriteBarrier: (task: () => Promise<MemoryMigrationResult>) => Promise<MemoryMigrationResult>;
|
|
11670
|
+
/** Route all subsequent memory writes to the remote store before this callback returns. */
|
|
11671
|
+
onActivated: () => void | Promise<void>;
|
|
11668
11672
|
signal?: AbortSignal;
|
|
11669
11673
|
onProgress?: (progress: MemoryMigrationProgress) => void;
|
|
11670
11674
|
}
|
package/dist/react/index.mjs
CHANGED
|
@@ -23357,6 +23357,8 @@ async function backfillTopics(ctx) {
|
|
|
23357
23357
|
if (topicless.length > 0) await backfillMemoryTopicsOp(ctx, topicless);
|
|
23358
23358
|
}
|
|
23359
23359
|
async function migrateMemoriesToRemote(options) {
|
|
23360
|
+
if (typeof options.withWriteBarrier !== "function" || typeof options.onActivated !== "function")
|
|
23361
|
+
throw new Error("A write barrier and a remote store handoff are required");
|
|
23360
23362
|
const { canonical, signal } = options;
|
|
23361
23363
|
const rings = [canonical, ...options.fallbacks ?? []];
|
|
23362
23364
|
const model = options.embeddingOptions?.model ?? DEFAULT_API_EMBEDDING_MODEL;
|
|
@@ -23384,93 +23386,98 @@ async function migrateMemoriesToRemote(options) {
|
|
|
23384
23386
|
const placeholders = /* @__PURE__ */ new Set();
|
|
23385
23387
|
const truncated = /* @__PURE__ */ new Set();
|
|
23386
23388
|
let imported = 0;
|
|
23387
|
-
|
|
23388
|
-
|
|
23389
|
-
|
|
23390
|
-
|
|
23391
|
-
|
|
23392
|
-
|
|
23393
|
-
|
|
23394
|
-
|
|
23395
|
-
|
|
23396
|
-
|
|
23397
|
-
|
|
23398
|
-
|
|
23399
|
-
|
|
23400
|
-
);
|
|
23401
|
-
const needsVector = (raw, server) => !!options.embeddingOptions && !embedding(raw, model) && server.embedding_model !== model;
|
|
23402
|
-
for (const raw of candidates) {
|
|
23403
|
-
const server = remote.get(raw.id);
|
|
23404
|
-
if (server && server.updated_at === raw.updated_at && (!sameMetadata2(raw, server, model) || needsVector(raw, server)))
|
|
23405
|
-
pending.set(raw.id, raw.updated_at + 1);
|
|
23406
|
-
}
|
|
23407
|
-
}
|
|
23408
|
-
if (pending.size === 0) {
|
|
23409
|
-
const result = (state) => ({
|
|
23410
|
-
state,
|
|
23411
|
-
total: rows.length,
|
|
23412
|
-
imported,
|
|
23413
|
-
skipped: [...skipped].map(([memoryId, reason]) => ({ memoryId, reason })),
|
|
23414
|
-
reembedded: reembedded.size,
|
|
23415
|
-
unembedded: unembedded.size,
|
|
23416
|
-
placeholderTombstones: placeholders.size,
|
|
23417
|
-
truncatedSourceChunkIds: truncated.size
|
|
23418
|
-
});
|
|
23419
|
-
if (skipped.size > 0 && !options.allowSkipped) return result("migrating");
|
|
23420
|
-
if (!claimed) await importBatch(request, canonical.keyId, [], signal);
|
|
23421
|
-
if (!active)
|
|
23422
|
-
await request(
|
|
23423
|
-
"/account",
|
|
23424
|
-
{ method: "PUT", body: JSON.stringify({ state: "active" }) },
|
|
23389
|
+
const uploadPasses = async (finalizing = false) => {
|
|
23390
|
+
for (let pass = 0; ; pass++) {
|
|
23391
|
+
const rows = await getAllVaultMemoryRowsOp(options.vaultCtx);
|
|
23392
|
+
for (const raw of rows) if (invalid2(raw)) skipped.set(String(raw.id), "invalid");
|
|
23393
|
+
const candidates = rows.filter((raw) => !skipped.has(String(raw.id)));
|
|
23394
|
+
const missing = await missingIds(request, candidates.map(revision), signal);
|
|
23395
|
+
const pending = /* @__PURE__ */ new Map();
|
|
23396
|
+
for (const raw of candidates)
|
|
23397
|
+
if (missing.has(raw.id)) pending.set(raw.id, raw.updated_at);
|
|
23398
|
+
if (pending.size === 0 && !active) {
|
|
23399
|
+
const remote = await remoteMetadata(
|
|
23400
|
+
request,
|
|
23401
|
+
candidates.map((raw) => raw.id),
|
|
23425
23402
|
signal
|
|
23426
23403
|
);
|
|
23427
|
-
|
|
23428
|
-
|
|
23429
|
-
|
|
23430
|
-
|
|
23431
|
-
|
|
23432
|
-
|
|
23433
|
-
|
|
23434
|
-
)
|
|
23435
|
-
|
|
23436
|
-
|
|
23437
|
-
|
|
23438
|
-
|
|
23439
|
-
|
|
23440
|
-
|
|
23441
|
-
|
|
23442
|
-
|
|
23443
|
-
|
|
23444
|
-
|
|
23445
|
-
|
|
23446
|
-
|
|
23447
|
-
|
|
23448
|
-
|
|
23449
|
-
|
|
23450
|
-
|
|
23451
|
-
|
|
23452
|
-
|
|
23453
|
-
|
|
23454
|
-
|
|
23455
|
-
|
|
23404
|
+
const needsVector = (raw, server) => !!options.embeddingOptions && !embedding(raw, model) && server.embedding_model !== model;
|
|
23405
|
+
for (const raw of candidates) {
|
|
23406
|
+
const server = remote.get(raw.id);
|
|
23407
|
+
if (server && server.updated_at === raw.updated_at && (!sameMetadata2(raw, server, model) || needsVector(raw, server)))
|
|
23408
|
+
pending.set(raw.id, raw.updated_at + 1);
|
|
23409
|
+
}
|
|
23410
|
+
}
|
|
23411
|
+
if (pending.size === 0) {
|
|
23412
|
+
const result = (state) => ({
|
|
23413
|
+
state,
|
|
23414
|
+
total: rows.length,
|
|
23415
|
+
imported,
|
|
23416
|
+
skipped: [...skipped].map(([memoryId, reason]) => ({ memoryId, reason })),
|
|
23417
|
+
reembedded: reembedded.size,
|
|
23418
|
+
unembedded: unembedded.size,
|
|
23419
|
+
placeholderTombstones: placeholders.size,
|
|
23420
|
+
truncatedSourceChunkIds: truncated.size
|
|
23421
|
+
});
|
|
23422
|
+
if (skipped.size > 0 && !options.allowSkipped) return result("migrating");
|
|
23423
|
+
if (!finalizing) return options.withWriteBarrier(() => uploadPasses(true));
|
|
23424
|
+
if (!claimed) await importBatch(request, canonical.keyId, [], signal);
|
|
23425
|
+
if (!active)
|
|
23426
|
+
await request(
|
|
23427
|
+
"/account",
|
|
23428
|
+
{ method: "PUT", body: JSON.stringify({ state: "active" }) },
|
|
23429
|
+
signal
|
|
23430
|
+
);
|
|
23431
|
+
await options.onActivated();
|
|
23432
|
+
return result("active");
|
|
23433
|
+
}
|
|
23434
|
+
if (pass >= MAX_UPLOAD_PASSES)
|
|
23435
|
+
throw new RemoteMemoryError(
|
|
23436
|
+
"Nearby still reports memories missing after importing them",
|
|
23437
|
+
409,
|
|
23438
|
+
"migration_incomplete"
|
|
23439
|
+
);
|
|
23440
|
+
const prepared = [];
|
|
23441
|
+
for (const raw of candidates) {
|
|
23442
|
+
const updatedAt = pending.get(raw.id);
|
|
23443
|
+
if (updatedAt === void 0) continue;
|
|
23444
|
+
try {
|
|
23445
|
+
prepared.push(await prepareRow(raw, canonical, rings, model, updatedAt));
|
|
23446
|
+
} catch (error) {
|
|
23447
|
+
if (!(error instanceof MemoryKeyError)) throw error;
|
|
23448
|
+
skipped.set(raw.id, "unreadable");
|
|
23449
|
+
}
|
|
23450
|
+
}
|
|
23451
|
+
const usable = new Set(prepared.filter((p) => p.row.embedding).map((p) => p.row.memory_id));
|
|
23452
|
+
if (options.embeddingOptions)
|
|
23453
|
+
await embedMissing(prepared, options.embeddingOptions, model, signal);
|
|
23454
|
+
const local = new Map(candidates.map((raw) => [raw.id, raw]));
|
|
23455
|
+
const upload = [];
|
|
23456
|
+
for (const { row, plaintext } of prepared) {
|
|
23457
|
+
const id2 = row.memory_id;
|
|
23458
|
+
if (!fitsNearby(row)) {
|
|
23459
|
+
skipped.set(id2, "invalid");
|
|
23460
|
+
continue;
|
|
23461
|
+
}
|
|
23462
|
+
upload.push(row);
|
|
23463
|
+
const track = (set, on) => on ? set.add(id2) : set.delete(id2);
|
|
23464
|
+
track(reembedded, !!row.embedding && !usable.has(id2));
|
|
23465
|
+
track(unembedded, !row.embedding);
|
|
23466
|
+
track(placeholders, plaintext === void 0);
|
|
23467
|
+
track(truncated, row.source_chunk_ids !== text13(local.get(id2).source_chunk_ids));
|
|
23456
23468
|
}
|
|
23457
|
-
|
|
23458
|
-
const track = (set, on) => on ? set.add(id2) : set.delete(id2);
|
|
23459
|
-
track(reembedded, !!row.embedding && !usable.has(id2));
|
|
23460
|
-
track(unembedded, !row.embedding);
|
|
23461
|
-
track(placeholders, plaintext === void 0);
|
|
23462
|
-
track(truncated, row.source_chunk_ids !== text13(local.get(id2).source_chunk_ids));
|
|
23463
|
-
}
|
|
23464
|
-
let sent = 0;
|
|
23465
|
-
options.onProgress?.({ imported: sent, toImport: upload.length });
|
|
23466
|
-
for (const batch of chunks(upload, IMPORT_BATCH)) {
|
|
23467
|
-
await importBatch(request, canonical.keyId, batch, signal);
|
|
23468
|
-
claimed = true;
|
|
23469
|
-
sent += batch.length;
|
|
23469
|
+
let sent = 0;
|
|
23470
23470
|
options.onProgress?.({ imported: sent, toImport: upload.length });
|
|
23471
|
+
for (const batch of chunks(upload, IMPORT_BATCH)) {
|
|
23472
|
+
await importBatch(request, canonical.keyId, batch, signal);
|
|
23473
|
+
claimed = true;
|
|
23474
|
+
sent += batch.length;
|
|
23475
|
+
options.onProgress?.({ imported: sent, toImport: upload.length });
|
|
23476
|
+
}
|
|
23477
|
+
imported += sent;
|
|
23471
23478
|
}
|
|
23472
|
-
|
|
23473
|
-
|
|
23479
|
+
};
|
|
23480
|
+
return uploadPasses();
|
|
23474
23481
|
}
|
|
23475
23482
|
|
|
23476
23483
|
// src/lib/memory/store/remoteStore.ts
|