@anuma/sdk 10.3.0 → 10.4.0

package/dist/react/index.mjs

@@ -12423,6 +12423,360 @@ async function migrateChatsToRemote(options) {
   return passes(false);
 }
 
+// src/lib/chatStore/vaultAttachments.ts
+var frameVersion = 1;
+var ivBytes = 12;
+var tagBytes = 16;
+var frameOverhead = 1 + ivBytes + tagBytes;
+var maxCiphertextBytes = 50 * 1024 * 1024;
+var expirySkewMs = 3e4;
+var maxRetryAfterMs = 3e4;
+var maxRefWrites = 5;
+var uploadWindowMs = 16 * 6e4;
+var attachmentIdPattern2 = /^[0-9a-f]{32}$/;
+var maxVaultAttachmentBytes = maxCiphertextBytes - frameOverhead;
+function additionalData(attachmentId) {
+  return new TextEncoder().encode(`anuma-chat-attachment-v1:${attachmentId}`);
+}
+function invalidAttachment() {
+  throw new ChatStoreError("Attachment does not decrypt under this key", 0, "invalid_attachment");
+}
+function sendableHeaders(headers) {
+  return Object.fromEntries(
+    Object.entries(headers).filter(([name]) => name.toLowerCase() !== "content-length")
+  );
+}
+function transientStatus(status) {
+  return status === 429 || status >= 500;
+}
+async function discardBody(response) {
+  await response.body?.cancel().catch(() => void 0);
+}
+function createVaultAttachments(options) {
+  const { store, keyRing, cache } = options;
+  const fetcher = options.fetch ?? globalThis.fetch.bind(globalThis);
+  const retries = options.retries ?? 3;
+  const retryDelayMs = options.retryDelayMs ?? 500;
+  const now = options.now ?? Date.now;
+  const sleep3 = (ms2, signal) => new Promise((resolve, reject) => {
+    signal?.throwIfAborted();
+    const timer = setTimeout(() => {
+      signal?.removeEventListener("abort", onAbort);
+      resolve();
+    }, ms2);
+    const onAbort = () => {
+      clearTimeout(timer);
+      reject(signal?.reason);
+    };
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
+  const pause = (attempt, signal, atLeast = 0) => sleep3(Math.max(atLeast, retryDelayMs * 2 ** attempt), signal);
+  const withRetry = async (call, signal) => {
+    for (let attempt = 0; ; attempt++) {
+      try {
+        return await call();
+      } catch (error) {
+        if (!(error instanceof ChatStoreError) || !transientStatus(error.status) || attempt >= retries || (error.retryAfterMs ?? 0) > maxRetryAfterMs)
+          throw error;
+        await pause(attempt, signal, error.retryAfterMs);
+      }
+    }
+  };
+  const expired = (url) => now() >= Date.parse(url.expiresAt) - expirySkewMs;
+  const seal = async (bytes4, attachmentId) => {
+    const iv = crypto.getRandomValues(new Uint8Array(ivBytes));
+    const sealed2 = new Uint8Array(
+      await crypto.subtle.encrypt(
+        { name: "AES-GCM", iv, additionalData: additionalData(attachmentId) },
+        keyRing.v3,
+        bytes4
+      )
+    );
+    const frame = new Uint8Array(1 + ivBytes + sealed2.length);
+    frame[0] = frameVersion;
+    frame.set(iv, 1);
+    frame.set(sealed2, 1 + ivBytes);
+    return frame;
+  };
+  const open = async (frame, attachmentId) => {
+    if (frame.length < frameOverhead || frame[0] !== frameVersion) invalidAttachment();
+    try {
+      return new Uint8Array(
+        await crypto.subtle.decrypt(
+          {
+            name: "AES-GCM",
+            iv: frame.slice(1, 1 + ivBytes),
+            additionalData: additionalData(attachmentId)
+          },
+          keyRing.v3,
+          frame.slice(1 + ivBytes)
+        )
+      );
+    } catch {
+      return invalidAttachment();
+    }
+  };
+  const put = async (upload, body, signal) => {
+    for (let attempt = 0; ; attempt++) {
+      if (expired(upload)) return "expired";
+      let response;
+      try {
+        response = await fetcher(upload.url, {
+          method: upload.method,
+          headers: sendableHeaders(upload.headers),
+          body,
+          signal
+        });
+      } catch (error) {
+        if (signal?.aborted || attempt >= retries) throw error;
+        await pause(attempt, signal);
+        continue;
+      }
+      await discardBody(response);
+      if (response.ok) return "stored";
+      if (response.status === 403) return "expired";
+      if (!transientStatus(response.status) || attempt >= retries)
+        throw new ChatStoreError(
+          `Attachment upload failed with ${response.status}`,
+          response.status,
+          "attachment_upload_failed"
+        );
+      await pause(attempt, signal);
+    }
+  };
+  const download = async (url, signal) => {
+    for (let attempt = 0; ; attempt++) {
+      if (expired(url)) return "expired";
+      let response;
+      try {
+        response = await fetcher(url.url, {
+          method: url.method,
+          headers: sendableHeaders(url.headers),
+          signal
+        });
+        if (response.ok) return new Uint8Array(await response.arrayBuffer());
+      } catch (error) {
+        if (signal?.aborted || attempt >= retries) throw error;
+        await pause(attempt, signal);
+        continue;
+      }
+      await discardBody(response);
+      if (response.status === 403) return "expired";
+      if (response.status === 404) return "missing";
+      if (!transientStatus(response.status) || attempt >= retries)
+        throw new ChatStoreError(
+          `Attachment download failed with ${response.status}`,
+          response.status,
+          "attachment_download_failed"
+        );
+      await pause(attempt, signal);
+    }
+  };
+  const remove = async (attachmentId, signal) => {
+    try {
+      await withRetry(() => store.attachments.delete(attachmentId, signal), signal);
+    } catch (error) {
+      if (!(error instanceof ChatStoreError) || error.status !== 404) throw error;
+    }
+    await cache?.delete(attachmentId).catch(() => void 0);
+  };
+  const discardPending = async (attachmentId) => {
+    const found = await store.attachments.get(attachmentId);
+    if (found?.attachment.state === "pending") await remove(attachmentId);
+  };
+  return {
+    async upload(bytes4, uploadOptions) {
+      const signal = uploadOptions?.signal;
+      if (!(bytes4 instanceof Uint8Array) || bytes4.length > maxVaultAttachmentBytes)
+        throw new ChatStoreError(
+          "Attachment is larger than vault accepts",
+          413,
+          "attachment_too_large"
+        );
+      const size = bytes4.length + frameOverhead;
+      for (let attempt = 0; ; attempt++) {
+        const { attachment: attachment2, upload } = await withRetry(
+          () => store.attachments.create(size, signal),
+          signal
+        );
+        try {
+          await uploadOptions?.onReserved?.(attachment2.id);
+          const frame = await seal(bytes4, attachment2.id);
+          if (await put(upload, frame, signal) === "stored") return attachment2.id;
+        } catch (error) {
+          await discardPending(attachment2.id).catch(() => void 0);
+          throw error;
+        }
+        await remove(attachment2.id).catch(() => void 0);
+        if (attempt >= retries)
+          throw new ChatStoreError(
+            "Attachment storage rejected the upload URL",
+            403,
+            "attachment_upload_rejected"
+          );
+      }
+    },
+    commit(attachmentId, owner, signal) {
+      return withRetry(() => store.attachments.commit(attachmentId, owner, signal), signal);
+    },
+    async read(attachmentId, signal) {
+      if (!attachmentIdPattern2.test(attachmentId)) invalidAttachment();
+      const cached = await cache?.get(attachmentId).catch(() => null);
+      if (cached) return cached;
+      for (let attempt = 0; ; attempt++) {
+        const found = await withRetry(() => store.attachments.get(attachmentId, signal), signal);
+        if (!found?.download) return null;
+        const frame = await download(found.download, signal);
+        if (frame === "missing") return null;
+        if (frame === "expired") {
+          if (attempt >= retries)
+            throw new ChatStoreError(
+              "Attachment storage rejected the download URL",
+              403,
+              "attachment_download_rejected"
+            );
+          continue;
+        }
+        if (frame.length !== found.attachment.sizeBytes) invalidAttachment();
+        const bytes4 = await open(frame, attachmentId);
+        await cache?.set(attachmentId, bytes4).catch(() => void 0);
+        return bytes4;
+      }
+    },
+    delete: remove
+  };
+}
+function validRef(value) {
+  if (value === null || typeof value !== "object" || Array.isArray(value)) return false;
+  const ref3 = value;
+  return typeof ref3.attachmentId === "string" && attachmentIdPattern2.test(ref3.attachmentId) && typeof ref3.fileId === "string" && ref3.fileId.length > 0 && typeof ref3.name === "string" && typeof ref3.type === "string" && Number.isSafeInteger(ref3.size) && ref3.size >= 0;
+}
+function vaultAttachmentRefs(record2) {
+  const refs = record2.data.vaultAttachments;
+  return Array.isArray(refs) ? refs.filter(validRef) : [];
+}
+function journalKey(messageId, fileId) {
+  return JSON.stringify([messageId, fileId]);
+}
+async function migrateMessageAttachments(options) {
+  const { store, attachments, journal, messageId, signal } = options;
+  const now = options.now ?? Date.now;
+  const owner = { kind: "message", id: messageId };
+  const ownerGone = () => new ChatStoreError("Message is not in vault", 409, "attachment_owner_not_found");
+  let snapshot = await store.get("message", messageId, signal);
+  if (!snapshot || snapshot.record.isDeleted) throw ownerGone();
+  const present = new Map(vaultAttachmentRefs(snapshot.record).map((r) => [r.fileId, r]));
+  const uploadable = (attachment2) => attachment2.state === "pending" && now() < Date.parse(attachment2.createdAt) + uploadWindowMs;
+  const lookup = async (attachmentId) => attachmentId && attachmentIdPattern2.test(attachmentId) ? (await store.attachments.get(attachmentId, signal))?.attachment ?? null : null;
+  const bind = async (found) => {
+    if (!found) return null;
+    if (found.state === "committed")
+      return found.ownerKind === owner.kind && found.ownerId === owner.id ? found : null;
+    try {
+      return await attachments.commit(found.id, owner, signal);
+    } catch (error) {
+      if (!(error instanceof ChatStoreError) || error.code !== "attachment_not_uploaded")
+        throw error;
+      if (uploadable(found)) return "busy";
+      await attachments.delete(found.id, signal);
+      return null;
+    }
+  };
+  const added = [];
+  const settled = [];
+  const missing2 = [];
+  const oversized = [];
+  const deferred = [];
+  const stale = /* @__PURE__ */ new Set();
+  let uploaded = 0;
+  for (const file of options.files) {
+    signal?.throwIfAborted();
+    const key3 = journalKey(messageId, file.fileId);
+    const recorded = present.get(file.fileId);
+    if (recorded) {
+      const bound = await bind(await lookup(recorded.attachmentId));
+      if (bound === "busy") {
+        deferred.push(file.fileId);
+        continue;
+      }
+      if (bound) {
+        const stray = await lookup(await journal.get(key3));
+        if (stray && stray.id !== recorded.attachmentId) {
+          if (uploadable(stray)) continue;
+          await attachments.delete(stray.id, signal);
+        }
+        settled.push(key3);
+        continue;
+      }
+      stale.add(recorded.attachmentId);
+    }
+    const meta = { fileId: file.fileId, name: file.name, type: file.type };
+    const resumed = await bind(await lookup(await journal.get(key3)));
+    if (resumed === "busy") {
+      deferred.push(file.fileId);
+      continue;
+    }
+    if (resumed) {
+      added.push({ attachmentId: resumed.id, ...meta, size: resumed.sizeBytes - frameOverhead });
+      settled.push(key3);
+      continue;
+    }
+    const bytes4 = await file.read();
+    if (!bytes4) {
+      missing2.push(file.fileId);
+      continue;
+    }
+    if (bytes4.length > maxVaultAttachmentBytes) {
+      oversized.push(file.fileId);
+      continue;
+    }
+    const attachmentId = await attachments.upload(bytes4, {
+      signal,
+      onReserved: (id2) => journal.set(key3, id2)
+    });
+    uploaded++;
+    await attachments.commit(attachmentId, owner, signal);
+    added.push({ attachmentId, ...meta, size: bytes4.length });
+    settled.push(key3);
+  }
+  const superseded = [];
+  for (let attempt = 0; added.length > 0; attempt++) {
+    const refs = new Map(vaultAttachmentRefs(snapshot.record).map((r) => [r.fileId, r]));
+    superseded.length = 0;
+    for (const ref3 of added) {
+      const prior = refs.get(ref3.fileId);
+      if (prior && prior.attachmentId !== ref3.attachmentId && !stale.has(prior.attachmentId))
+        superseded.push(ref3.attachmentId);
+      else refs.set(ref3.fileId, ref3);
+    }
+    if (superseded.length === added.length) break;
+    try {
+      [snapshot] = await store.putMany(
+        [
+          {
+            record: {
+              ...snapshot.record,
+              data: { ...snapshot.record.data, ...plain({ vaultAttachments: [...refs.values()] }) }
+            },
+            expectedVersion: snapshot.version
+          }
+        ],
+        signal
+      );
+      break;
+    } catch (error) {
+      if (!(error instanceof ChatStoreError) || error.code !== "version_conflict" || attempt + 1 >= maxRefWrites)
+        throw error;
+      const latest = await store.get("message", messageId, signal);
+      if (!latest || latest.record.isDeleted) throw ownerGone();
+      snapshot = latest;
+    }
+  }
+  for (const id2 of superseded) await attachments.delete(id2, signal);
+  for (const key3 of settled) await journal.set(key3, null);
+  return { refs: vaultAttachmentRefs(snapshot.record), uploaded, missing: missing2, oversized, deferred };
+}
+
 // src/lib/chatStore/vaultBackend.ts
 import { v7 as uuidv73 } from "uuid";
 
@@ -12562,6 +12916,7 @@ var resumableFields = [
   "imageModel",
   "files",
   "fileIds",
+  "vaultAttachments",
   "usage",
   "sources",
   "responseDuration",
@@ -26353,7 +26708,7 @@ function createRemoteMemoryStore(options) {
   };
   const mutate = (id2, change, edit, writeOptions) => serialized(id2, () => mutateLocked(id2, change, edit, writeOptions));
   const serializedAll = (ids, task) => [...new Set(ids)].sort().reduceRight((inner, id2) => () => serialized(id2, inner), task)();
-  const mutateMany = async (ids, change, edit, leaseToken, signal) => {
+  const mutateMany = async (ids, change, edit, leaseToken, signal, includeEmbeddings = false) => {
     const saved = /* @__PURE__ */ new Map();
     const failed = /* @__PURE__ */ new Map();
     for (const slice of chunks2([...new Set(ids)], MAX_BATCH)) {
@@ -26362,7 +26717,7 @@ function createRemoteMemoryStore(options) {
         const writes = [];
         let read;
         try {
-          read = await readEveryState(slice, signal);
+          read = await readEveryState(slice, signal, includeEmbeddings);
         } catch (error) {
           signal?.throwIfAborted();
           for (const id2 of slice) failed.set(id2, error);
@@ -26421,21 +26776,55 @@ function createRemoteMemoryStore(options) {
     }
     return { saved, failed };
   };
-  const embedInBackground = ({ memory: { memory_id: id2, content } }) => {
-    generateEmbedding(content, embeddingOptions).then(
-      (vector2) => serialized(id2, async () => {
-        const current = await live(id2);
-        if (!current || current.memory.content !== content || current.memory.embedding?.length)
-          return;
-        const saved = await persistence.put(
-          { ...current.memory, embedding: storedVector(vector2), embedding_model: model },
-          current
-        );
-        notify("embedding", [saved]);
-      })
-    ).catch((error) => {
-      getLogger().warn("[memory/remote-store] Failed to embed memory:", error);
-    });
+  const pendingEmbeddings = /* @__PURE__ */ new Map();
+  let embeddingActive = false;
+  const drainEmbeddings = async () => {
+    try {
+      while (pendingEmbeddings.size) {
+        const batch = /* @__PURE__ */ new Map();
+        for (const [id2, content] of pendingEmbeddings) {
+          pendingEmbeddings.delete(id2);
+          batch.set(id2, content);
+          if (batch.size === MAX_BATCH) break;
+        }
+        try {
+          const vectors = await generateEmbeddings([...batch.values()], embeddingOptions);
+          const embeddings2 = /* @__PURE__ */ new Map();
+          [...batch.keys()].forEach((id2, i) => {
+            try {
+              embeddings2.set(id2, storedVector(vectors[i]));
+            } catch (error) {
+              getLogger().warn("[memory/remote-store] Failed to embed memory:", error);
+            }
+          });
+          if (!embeddings2.size) continue;
+          const { failed } = await mutateMany(
+            [...embeddings2.keys()],
+            () => "embedding",
+            (memory) => memory.content !== batch.get(memory.memory_id) || memory.embedding?.length ? null : {
+              ...memory,
+              embedding: embeddings2.get(memory.memory_id),
+              embedding_model: model
+            },
+            void 0,
+            void 0,
+            true
+          );
+          for (const error of failed.values())
+            getLogger().warn("[memory/remote-store] Failed to embed memory:", error);
+        } catch (error) {
+          getLogger().warn("[memory/remote-store] Failed to embed memories:", error);
+        }
+      }
+    } finally {
+      embeddingActive = false;
+    }
+  };
+  const embedInBackground = (records) => {
+    for (const { memory } of records) pendingEmbeddings.set(memory.memory_id, memory.content);
+    if (embeddingActive || !pendingEmbeddings.size) return;
+    embeddingActive = true;
+    void drainEmbeddings();
   };
   const newRow = (input) => {
     const now = Date.now();
@@ -26469,9 +26858,7 @@ function createRemoteMemoryStore(options) {
     };
   };
   const created = (records, inputs) => {
-    records.forEach((record2, i) => {
-      if (inputs[i].embedding === void 0) embedInBackground(record2);
-    });
+    embedInBackground(records.filter((_, i) => inputs[i].embedding === void 0));
     notify("membership", records);
     return records.map((record2) => stored(record2.memory));
   };
@@ -26501,7 +26888,7 @@ function createRemoteMemoryStore(options) {
     };
   };
   const withKnownKinds = async (topics) => topics.map(await knownKinds(topics));
-  const readEveryState = async (ids, signal) => {
+  const readEveryState = async (ids, signal, includeEmbeddings = false) => {
     const records = /* @__PURE__ */ new Map();
     const unreadable = /* @__PURE__ */ new Set();
     for (const memory_ids of chunks2(ids, MAX_QUERY_IDS)) {
@@ -26513,6 +26900,7 @@ function createRemoteMemoryStore(options) {
           include_archived: true,
           include_quarantined: true,
           include_superseded: true,
+          ...includeEmbeddings && { include_embeddings: true },
           limit: 200,
           ...signal && { signal },
           ...cursor2 && { cursor: cursor2 }
@@ -27175,7 +27563,7 @@ function createRemoteMemoryStore(options) {
         }
         return memory;
       });
-      if (saved && patch.embedding === void 0) embedInBackground(saved);
+      if (saved && patch.embedding === void 0) embedInBackground([saved]);
       return asStored(saved);
     },
     delete: async (id2) => !!await mutate(id2, "membership", (memory) => ({ ...memory, is_deleted: true })),
@@ -40212,6 +40600,7 @@ export {
   createSelectPaymentWalletTool,
   createServerToolsFilter,
   createStockPricePreProcessor,
+  createVaultAttachments,
   createVaultChatBackend,
   createVaultEmbeddingCache,
   createVaultFolderOp,
@@ -40361,11 +40750,13 @@ export {
   linkMemoryEntitiesOp,
   localChatBackend,
   maskScopedEmbeddingCache,
+  maxVaultAttachmentBytes,
   mediaToStored,
   memoryCipher,
   migrateChatsToRemote,
   migrateDisplayResult,
   migrateMemoriesToRemote,
+  migrateMessageAttachments,
   migrateNotionClientRegistration,
   migrateNotionToken,
   moveMemoriesToFolderOp,
@@ -40474,6 +40865,7 @@ export {
   useVoice,
   useWalletBinding,
   userPreferencesStorageSchema,
+  vaultAttachmentRefs,
   vaultChatBackend,
   verifyMemoriesForPublish,
   walk,

package/dist/server/index.d.mts

@@ -7389,6 +7389,14 @@ interface FileMetadata {
      */
     sourceUrl?: string;
 }
+/** Points a message's local file (`FileMetadata.id` or a `fileIds` media id) at its encrypted copy in vault, so any device can fetch it. */
+interface VaultAttachmentRef {
+    attachmentId: string;
+    fileId: string;
+    name: string;
+    type: string;
+    size: number;
+}
 interface ChatCompletionUsage {
     promptTokens?: number;
     completionTokens?: number;
@@ -7492,6 +7500,7 @@ interface StoredMessage {
     files?: FileMetadata[];
     /** Array of media_id references for direct lookup in media table */
     fileIds?: string[];
+    vaultAttachments?: VaultAttachmentRef[];
     createdAt: Date;
     updatedAt: Date;
     vector?: number[];
@@ -7668,6 +7677,7 @@ interface CreateMessageOptions {
     files?: FileMetadata[];
     /** Array of media_id references for direct lookup in media table */
     fileIds?: string[];
+    vaultAttachments?: VaultAttachmentRef[];
     usage?: ChatCompletionUsage;
     sources?: SearchSource[];
     responseDuration?: number;

package/dist/server/index.d.ts

@@ -7389,6 +7389,14 @@ interface FileMetadata {
      */
     sourceUrl?: string;
 }
+/** Points a message's local file (`FileMetadata.id` or a `fileIds` media id) at its encrypted copy in vault, so any device can fetch it. */
+interface VaultAttachmentRef {
+    attachmentId: string;
+    fileId: string;
+    name: string;
+    type: string;
+    size: number;
+}
 interface ChatCompletionUsage {
     promptTokens?: number;
     completionTokens?: number;
@@ -7492,6 +7500,7 @@ interface StoredMessage {
     files?: FileMetadata[];
     /** Array of media_id references for direct lookup in media table */
     fileIds?: string[];
+    vaultAttachments?: VaultAttachmentRef[];
     createdAt: Date;
     updatedAt: Date;
     vector?: number[];
@@ -7668,6 +7677,7 @@ interface CreateMessageOptions {
     files?: FileMetadata[];
     /** Array of media_id references for direct lookup in media table */
     fileIds?: string[];
+    vaultAttachments?: VaultAttachmentRef[];
     usage?: ChatCompletionUsage;
     sources?: SearchSource[];
     responseDuration?: number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@anuma/sdk",
-  "version": "10.3.0",
+  "version": "10.4.0",
   "description": "",
   "main": "./dist/index.cjs",
   "module": "./dist/index.mjs",