@anuma/sdk 7.0.6 → 7.0.7

package/dist/expo/index.cjs

@@ -13097,6 +13097,17 @@ async function decryptVaultMemoryFields(memory, address, signMessage, embeddedWa
   };
 }
 
+// src/lib/db/memoryVault/sourceChunkIds.ts
+var MAX_SOURCE_CHUNK_IDS_BYTES = 16 << 10;
+var utf8 = new TextEncoder();
+function serializeSourceChunkIds(ids) {
+  const sizes = ids.map((id) => utf8.encode(JSON.stringify(id)).length + 1);
+  let total = sizes.reduce((sum, size) => sum + size, 1);
+  let start = 0;
+  while (total > MAX_SOURCE_CHUNK_IDS_BYTES && start < ids.length) total -= sizes[start++];
+  return JSON.stringify(ids.slice(start));
+}
+
 // src/lib/db/memoryVault/types.ts
 var MEMORY_KINDS = [
   "display_name",
@@ -13244,15 +13255,15 @@ async function mapInBatches(items, fn) {
   return results;
 }
 function vaultMemoryToStoredRaw(memory) {
-  let sourceChunkIds = null;
+  let sourceChunkIds2 = null;
   if (memory.sourceChunkIds) {
     try {
       const parsed = JSON.parse(memory.sourceChunkIds);
       if (Array.isArray(parsed)) {
-        sourceChunkIds = parsed.filter((s) => typeof s === "string");
+        sourceChunkIds2 = parsed.filter((s) => typeof s === "string");
       }
     } catch {
-      sourceChunkIds = null;
+      sourceChunkIds2 = null;
     }
   }
   return {
@@ -13266,7 +13277,7 @@ function vaultMemoryToStoredRaw(memory) {
     userId: memory.userId ?? null,
     embedding: memory.embedding ?? null,
     embeddingModel: memory.embeddingModel ?? null,
-    sourceChunkIds,
+    sourceChunkIds: sourceChunkIds2,
     proofCount: memory.proofCount ?? null,
     source: memory.source ?? null,
     media: parseMedia(memory.media),
@@ -13315,7 +13326,7 @@ function populateNewVaultMemory(record2, ctx, opts, encryptedContent, encryptedK
     record2._setRaw("embedding_model", opts.embeddingModel ?? null);
   }
   if (opts.sourceChunkIds !== void 0) {
-    record2._setRaw("source_chunk_ids", JSON.stringify(opts.sourceChunkIds));
+    record2._setRaw("source_chunk_ids", serializeSourceChunkIds(opts.sourceChunkIds));
   }
   record2._setRaw("proof_count", opts.proofCount ?? 1);
   record2._setRaw("source", opts.source ?? "manual");
@@ -13493,16 +13504,16 @@ async function getVaultMemoryOp(ctx, id) {
   }
 }
 function vaultMemoryRawToStoredRaw(raw) {
-  let sourceChunkIds = null;
+  let sourceChunkIds2 = null;
   const rawChunks = raw.source_chunk_ids;
   if (typeof rawChunks === "string" && rawChunks) {
     try {
       const parsed = JSON.parse(rawChunks);
       if (Array.isArray(parsed)) {
-        sourceChunkIds = parsed.filter((s) => typeof s === "string");
+        sourceChunkIds2 = parsed.filter((s) => typeof s === "string");
       }
     } catch {
-      sourceChunkIds = null;
+      sourceChunkIds2 = null;
     }
   }
   return {
@@ -13516,7 +13527,7 @@ function vaultMemoryRawToStoredRaw(raw) {
     userId: raw.user_id ?? null,
     embedding: raw.embedding ?? null,
     embeddingModel: raw.embedding_model ?? null,
-    sourceChunkIds,
+    sourceChunkIds: sourceChunkIds2,
     proofCount: raw.proof_count ?? null,
     source: raw.source ?? null,
     eventTimeStart: raw.event_time_start ?? null,
@@ -13813,7 +13824,7 @@ async function updateVaultMemoryOp(ctx, id, opts) {
         if (opts.sourceChunkIds !== void 0) {
           r._setRaw(
             "source_chunk_ids",
-            JSON.stringify(
+            serializeSourceChunkIds(
               opts.observationSourceIds?.length ? [.../* @__PURE__ */ new Set([...observedSources, ...opts.sourceChunkIds])] : opts.sourceChunkIds
             )
           );
@@ -20713,13 +20724,20 @@ function memoryCipher(canonical) {
   };
 }
 async function reencryptMemoryField(value, canonical, fallbacks = []) {
-  const { encrypt } = memoryCipher(canonical);
-  if (!value.startsWith("enc:")) return encrypt(value);
+  return memoryCipher(canonical).encrypt(
+    await decryptMemoryField(value, [canonical, ...fallbacks])
+  );
+}
+async function decryptMemoryField(value, rings) {
+  if (!value.startsWith("enc:")) return value;
   if (!isEncrypted(value))
     throw new MemoryKeyError("Field has an encryption prefix but an invalid payload");
-  return encrypt(await decryptWith(value, [canonical, ...fallbacks]));
+  return decryptWith(value, rings);
 }
 
+// src/lib/memory/store/remoteMigration.ts
+init_logger();
+
 // src/lib/memory/store/remotePersistence.ts
 init_portalLlm();
 var RemoteMemoryError = class extends Error {
@@ -21412,7 +21430,7 @@ function createRemoteMemoryPipeline(options) {
           embedding: storedVector(JSON.parse(input.embedding)),
           embedding_model: input.embeddingModel
         },
-        source_chunk_ids: JSON.stringify(input.sourceChunkIds ?? []),
+        source_chunk_ids: serializeSourceChunkIds(input.sourceChunkIds ?? []),
         proof_count: input.proofCount ?? 1,
         source: input.source ?? "manual",
         ...input.eventTime && {
@@ -21471,7 +21489,7 @@ function createRemoteMemoryPipeline(options) {
         const observed = sourceIds(memory.source_chunk_ids);
         const replay = !!patch.observationSourceIds?.length && patch.observationSourceIds.every((id2) => observed.includes(id2));
         if (patch.sourceChunkIds !== void 0)
-          memory.source_chunk_ids = JSON.stringify(
+          memory.source_chunk_ids = serializeSourceChunkIds(
             patch.observationSourceIds?.length ? [.../* @__PURE__ */ new Set([...observed, ...patch.sourceChunkIds])] : patch.sourceChunkIds
           );
         if (!replay) {
@@ -21578,6 +21596,8 @@ function createRemoteMemoryPipeline(options) {
 
 // src/lib/memory/store/remoteMigration.ts
 var IMPORT_BATCH = 50;
+var EMBED_BATCH = 100;
+var LOCAL_VECTOR_DIMS = 4096;
 var VERIFY_BATCH = 1e3;
 var QUERY_IDS = 1e3;
 var QUERY_PAGE = 200;
@@ -21603,7 +21623,7 @@ var SHORT_FIELDS = [
   "visibility",
   "geohash"
 ];
-var utf8 = new TextEncoder();
+var utf82 = new TextEncoder();
 var METADATA_FIELDS = [
   "scope",
   "kind",
@@ -21654,17 +21674,34 @@ function revision(raw) {
     last_observed_at: num(raw.last_observed_at) ?? null
   };
 }
-function embedding(raw) {
+function embedding(raw, model) {
   const value = text13(raw.embedding);
   if (!value) return void 0;
   try {
-    return storedVector(JSON.parse(value));
+    const vector = JSON.parse(value);
+    const stored2 = text13(raw.embedding_model);
+    if (stored2 === void 0 ? vector.length !== LOCAL_VECTOR_DIMS : stored2 !== model)
+      return void 0;
+    return storedVector(vector);
+  } catch {
+    return void 0;
+  }
+}
+function stringArray(value) {
+  try {
+    const parsed = JSON.parse(value);
+    return Array.isArray(parsed) && parsed.every((id) => typeof id === "string") ? parsed : void 0;
   } catch {
     return void 0;
   }
 }
+function sourceChunkIds(raw) {
+  const value = text13(raw.source_chunk_ids);
+  const ids = value && stringArray(value);
+  return ids ? serializeSourceChunkIds(ids) : value;
+}
 function bytes(value) {
-  return value === void 0 ? 0 : utf8.encode(value).length;
+  return value === void 0 ? 0 : utf82.encode(value).length;
 }
 function fitsNearby(row) {
   return bytes(row.memory_id) <= MAX_ID_BYTES && bytes(row.content) <= MAX_CONTENT_BYTES && bytes(row.kind_value) <= MAX_CONTENT_BYTES && TEXT_FIELDS.every((field10) => bytes(row[field10]) <= MAX_TEXT_BYTES) && SHORT_FIELDS.every((field10) => bytes(row[field10]) <= MAX_SHORT_BYTES);
@@ -21673,16 +21710,16 @@ function invalid(raw) {
   const id = raw.id;
   return typeof id !== "string" || id === "" || bytes(id) > MAX_ID_BYTES || !text13(raw.scope) || (num(raw.created_at) ?? 0) <= 0 || (num(raw.updated_at) ?? 0) <= 0;
 }
-function metadata(raw) {
-  const vector = embedding(raw);
+function metadata(raw, model) {
+  const vector = embedding(raw, model);
   return {
     scope: raw.scope,
     kind: text13(raw.kind),
     level: text13(raw.level),
     folder_id: text13(raw.folder_id),
     is_deleted: bool(raw.is_deleted) ?? false,
-    ...vector && { embedding: vector, embedding_model: text13(raw.embedding_model) },
-    source_chunk_ids: text13(raw.source_chunk_ids),
+    ...vector && { embedding: vector, embedding_model: model },
+    source_chunk_ids: sourceChunkIds(raw),
     proof_count: num(raw.proof_count),
     source: text13(raw.source),
     event_time_start: num(raw.event_time_start),
@@ -21711,30 +21748,63 @@ function defined(value) {
     Object.entries(value).filter(([, field10]) => field10 !== void 0)
   );
 }
-async function remoteRow(raw, canonical, fallbacks, updatedAt) {
-  const content = await reencryptMemoryField(
-    typeof raw.content === "string" ? raw.content : "",
-    canonical,
-    fallbacks
-  );
-  const kindValue = text13(raw.kind_value);
-  const encryptedKindValue = kindValue ? await reencryptMemoryField(kindValue, canonical, fallbacks) : void 0;
+async function prepareRow(raw, canonical, rings, model, updatedAt) {
+  const deleted = bool(raw.is_deleted) ?? false;
+  const { encrypt } = memoryCipher(canonical);
+  const localKindValue = text13(raw.kind_value);
+  let plaintext;
+  let kindValue;
+  try {
+    plaintext = await decryptMemoryField(typeof raw.content === "string" ? raw.content : "", rings);
+    kindValue = localKindValue && await decryptMemoryField(localKindValue, rings);
+  } catch (error) {
+    if (!deleted || !(error instanceof MemoryKeyError)) throw error;
+    plaintext = void 0;
+    kindValue = void 0;
+  }
   const row = {
-    ...defined({ ...metadata(raw), kind_value: encryptedKindValue }),
+    ...defined({
+      ...metadata(raw, model),
+      kind_value: kindValue === void 0 ? void 0 : await encrypt(kindValue)
+    }),
     memory_id: raw.id,
-    content,
+    content: await encrypt(plaintext ?? ""),
     scope: raw.scope,
     created_at: raw.created_at,
     updated_at: updatedAt,
-    is_deleted: bool(raw.is_deleted) ?? false
+    is_deleted: deleted
   };
-  return fitsNearby(row) ? row : null;
+  return { row, plaintext };
+}
+async function embedMissing(prepared, options, model, signal) {
+  const needed = prepared.filter((p) => !p.row.embedding && p.plaintext);
+  for (const batch of chunks(needed, EMBED_BATCH)) {
+    if (signal?.aborted) throw aborted(signal);
+    try {
+      const vectors = await generateEmbeddings(
+        batch.map((p) => p.plaintext),
+        { ...options, taskType: "memory_retain_embed" }
+      );
+      if (vectors.length !== batch.length) throw new Error("Incomplete migration embeddings");
+      const stored2 = vectors.map(storedVector);
+      batch.forEach((p, i) => {
+        p.row.embedding = stored2[i];
+        p.row.embedding_model = model;
+      });
+    } catch (error) {
+      getLogger().warn(
+        `[memory/migration] Uploading ${batch.length} row(s) without a vector: embedding failed`,
+        error
+      );
+    }
+  }
 }
-function sameMetadata(local, remote) {
-  const mine = metadata(local);
+function sameMetadata(local, remote, model) {
+  const mine = metadata(local, model);
   return METADATA_FIELDS.every((field10) => {
     const a = mine[field10] ?? null;
     const b = remote[field10] ?? null;
+    if (field10 === "embedding_model" && a === null) return true;
     if (field10 === "visibility") return (a ?? "private") === (b ?? "private");
     if (field10 === "is_deleted" || field10 === "topics_user_managed" || field10 === "twin_opt_in")
       return (a ?? false) === (b ?? false);
@@ -21830,7 +21900,8 @@ async function backfillTopics(ctx) {
 }
 async function migrateMemoriesToRemote(options) {
   const { canonical, signal } = options;
-  const fallbacks = options.fallbacks ?? [];
+  const rings = [canonical, ...options.fallbacks ?? []];
+  const model = options.embeddingOptions?.model ?? DEFAULT_API_EMBEDDING_MODEL;
   const request = createNearbyRequest(options);
   const account = await request("/account", {}, signal);
   if (typeof account.key_id === "string" && account.key_id !== canonical.keyId)
@@ -21841,8 +21912,13 @@ async function migrateMemoriesToRemote(options) {
     );
   const active = account.state === "active";
   let claimed = account.key_id === canonical.keyId;
+  if (!active) await createDecaySweeper({ vaultCtx: options.vaultCtx }).runSweep();
   await backfillTopics(options.vaultCtx);
   const skipped = /* @__PURE__ */ new Map();
+  const reembedded = /* @__PURE__ */ new Set();
+  const unembedded = /* @__PURE__ */ new Set();
+  const placeholders = /* @__PURE__ */ new Set();
+  const truncated = /* @__PURE__ */ new Set();
   let imported = 0;
   for (let pass = 0; ; pass++) {
     const rows = await getAllVaultMemoryRowsOp(options.vaultCtx);
@@ -21858,9 +21934,10 @@ async function migrateMemoriesToRemote(options) {
         candidates.map((raw) => raw.id),
         signal
       );
+      const needsVector = (raw, server) => !!options.embeddingOptions && !embedding(raw, model) && server.embedding_model !== model;
       for (const raw of candidates) {
         const server = remote.get(raw.id);
-        if (server && server.updated_at === raw.updated_at && !sameMetadata(raw, server))
+        if (server && server.updated_at === raw.updated_at && (!sameMetadata(raw, server, model) || needsVector(raw, server)))
           pending.set(raw.id, raw.updated_at + 1);
       }
     }
@@ -21869,7 +21946,11 @@ async function migrateMemoriesToRemote(options) {
         state,
         total: rows.length,
         imported,
-        skipped: [...skipped].map(([memoryId, reason]) => ({ memoryId, reason }))
+        skipped: [...skipped].map(([memoryId, reason]) => ({ memoryId, reason })),
+        reembedded: reembedded.size,
+        unembedded: unembedded.size,
+        placeholderTombstones: placeholders.size,
+        truncatedSourceChunkIds: truncated.size
       });
       if (skipped.size > 0 && !options.allowSkipped) return result("migrating");
       if (!claimed) await importBatch(request, canonical.keyId, [], signal);
@@ -21887,19 +21968,35 @@ async function migrateMemoriesToRemote(options) {
         409,
         "migration_incomplete"
       );
-    const upload = [];
+    const prepared = [];
     for (const raw of candidates) {
       const updatedAt = pending.get(raw.id);
       if (updatedAt === void 0) continue;
       try {
-        const row = await remoteRow(raw, canonical, fallbacks, updatedAt);
-        if (row) upload.push(row);
-        else skipped.set(raw.id, "invalid");
+        prepared.push(await prepareRow(raw, canonical, rings, model, updatedAt));
       } catch (error) {
         if (!(error instanceof MemoryKeyError)) throw error;
         skipped.set(raw.id, "unreadable");
       }
     }
+    const usable = new Set(prepared.filter((p) => p.row.embedding).map((p) => p.row.memory_id));
+    if (options.embeddingOptions)
+      await embedMissing(prepared, options.embeddingOptions, model, signal);
+    const local = new Map(candidates.map((raw) => [raw.id, raw]));
+    const upload = [];
+    for (const { row, plaintext } of prepared) {
+      const id = row.memory_id;
+      if (!fitsNearby(row)) {
+        skipped.set(id, "invalid");
+        continue;
+      }
+      upload.push(row);
+      const track = (set, on) => on ? set.add(id) : set.delete(id);
+      track(reembedded, !!row.embedding && !usable.has(id));
+      track(unembedded, !row.embedding);
+      track(placeholders, plaintext === void 0);
+      track(truncated, row.source_chunk_ids !== text13(local.get(id).source_chunk_ids));
+    }
     let sent = 0;
     options.onProgress?.({ imported: sent, toImport: upload.length });
     for (const batch of chunks(upload, IMPORT_BATCH)) {

package/dist/expo/index.d.mts

@@ -9728,6 +9728,8 @@ interface MemoryMigrationOptions extends Pick<RemoteMemoryPersistenceOptions, "b
     fallbacks?: readonly MemoryKeyRing[];
     /** Activate even when some rows could not be migrated. Default false. */
     allowSkipped?: boolean;
+    /** Re-embeds rows whose local vector nearby could not rank; without it those rows upload unembedded. */
+    embeddingOptions?: EmbeddingOptions;
     signal?: AbortSignal;
     onProgress?: (progress: MemoryMigrationProgress) => void;
 }
@@ -9754,6 +9756,14 @@ interface MemoryMigrationResult {
     total: number;
     imported: number;
     skipped: MemoryMigrationSkip[];
+    /** Rows uploaded with a vector embedded during migration. */
+    reembedded: number;
+    /** Rows uploaded without a vector nearby can rank. */
+    unembedded: number;
+    /** Deleted rows no supplied key decrypts, uploaded with placeholder content. */
+    placeholderTombstones: number;
+    /** Rows whose oldest source ids were dropped to fit nearby's limit. */
+    truncatedSourceChunkIds: number;
 }
 /**
  * Upload this device's local memories to nearby and, once nearby holds every one, make it authoritative.

package/dist/expo/index.d.ts

@@ -9728,6 +9728,8 @@ interface MemoryMigrationOptions extends Pick<RemoteMemoryPersistenceOptions, "b
     fallbacks?: readonly MemoryKeyRing[];
     /** Activate even when some rows could not be migrated. Default false. */
     allowSkipped?: boolean;
+    /** Re-embeds rows whose local vector nearby could not rank; without it those rows upload unembedded. */
+    embeddingOptions?: EmbeddingOptions;
     signal?: AbortSignal;
     onProgress?: (progress: MemoryMigrationProgress) => void;
 }
@@ -9754,6 +9756,14 @@ interface MemoryMigrationResult {
     total: number;
     imported: number;
     skipped: MemoryMigrationSkip[];
+    /** Rows uploaded with a vector embedded during migration. */
+    reembedded: number;
+    /** Rows uploaded without a vector nearby can rank. */
+    unembedded: number;
+    /** Deleted rows no supplied key decrypts, uploaded with placeholder content. */
+    placeholderTombstones: number;
+    /** Rows whose oldest source ids were dropped to fit nearby's limit. */
+    truncatedSourceChunkIds: number;
 }
 /**
  * Upload this device's local memories to nearby and, once nearby holds every one, make it authoritative.