@anuma/sdk 6.4.2 → 6.5.0

package/dist/react/index.mjs

@@ -1,10 +1,12 @@
 import {
+  CHUNKS_DISCARDED_ORIGIN,
   DEFAULT_API_EMBEDDING_MODEL,
   DEFAULT_CHUNK_OVERLAP,
   DEFAULT_CHUNK_SIZE,
   DEFAULT_MIN_CHUNK_SIZE,
   DEFAULT_MIN_CONTENT_LENGTH,
   DEFAULT_VAULT_CACHE_SIZE,
+  DISPLAY_CARD_PLACEHOLDER,
   EMBEDDINGS_DEGRADED_EMPTY,
   ENTITY_FANOUT,
   ENTITY_KINDS,
@@ -15,6 +17,7 @@ import {
   RECALL_MAX_LIMIT,
   RECALL_TOOL_NAME,
   RerankerUnavailableError,
+  TOOL_RESULT_ORIGIN,
   TOPICS_EXTRACTION_VERSION,
   TREND_RECENT_WINDOW_DAYS,
   TREND_STALE_WINDOW_DAYS,
@@ -23,6 +26,8 @@ import {
   assertVaultScopeForSweep,
   backfillMemoryEntityUserIdsOp,
   backfillMemoryTopicsOp,
+  base64ToUint8Array,
+  buildToolResultContent,
   capHopsForDensity,
   chunkAndEmbedAllMessages,
   chunkAndEmbedMessage,
@@ -47,6 +52,8 @@ import {
   createVaultEmbeddingCache,
   createVaultMemoriesBatchOp,
   createVaultMemoryOp,
+  dataUrlToArrayBuffer,
+  decodeChunkVector,
   decomposeQuery,
   decryptData,
   decryptDataBatch,
@@ -62,6 +69,7 @@ import {
   eagerEmbedContent,
   embedAllMessages,
   embedMessage,
+  encodeChunkVector,
   encryptData,
   encryptDataBatch,
   encryptDataBytes,
@@ -134,6 +142,7 @@ import {
   postAuthOauthByProviderRefresh,
   postAuthOauthByProviderRevoke,
   preEmbedVaultMemories,
+  prepareToolResultsForReplay,
   prepareVaultCandidates,
   rankPreparedVaultCandidates,
   recall,
@@ -156,6 +165,7 @@ import {
   summarizeObservationTrends,
   supersedeVaultMemoryOp,
   traverseGraphLane,
+  uint8ArrayToBase64,
   updateConversationPinnedOp,
   updateConversationProjectOp,
   updateConversationTitleOp,
@@ -168,7 +178,7 @@ import {
   updateVaultMemoryOp,
   useEncryption,
   vaultMemoryToStored
-} from "./chunk-IO4ZCPK2.mjs";
+} from "./chunk-OB5HFCLB.mjs";
 import {
   PiiRedactor,
   createStreamingDeAnonymizer,
@@ -5346,153 +5356,6 @@ async function cleanupConversationSummary(database, conversationId) {
   }
 }
 
-// src/lib/chat/toolResultMessage.ts
-var HEADER = "[Tool Execution Results]\n\n";
-var FOOTER = "\n\nBased on these results, continue with the task.";
-var ENTRY_SEPARATOR = "\n\n";
-var TOOL_RESULT_FOOTER_LINE = FOOTER.trim();
-var MAX_PERSISTED_TOOL_RESULT_CHARS = 1e5;
-function buildToolResultContent(results) {
-  const entries = results.map((r) => `Tool "${r.name}" returned: ${JSON.stringify(r.result)}`);
-  const framing = HEADER.length + FOOTER.length + ENTRY_SEPARATOR.length * Math.max(0, entries.length - 1);
-  const summary = capEntries(entries, MAX_PERSISTED_TOOL_RESULT_CHARS - framing).join(
-    ENTRY_SEPARATOR
-  );
-  return `${HEADER}${summary}${FOOTER}`;
-}
-function capToolResultEntries(entries, budget) {
-  return capEntries(entries, budget);
-}
-function capEntries(entries, budget) {
-  const smallestFirst = entries.map((_, index) => index).sort((a, b) => entries[a].length - entries[b].length);
-  const capped = [...entries];
-  let remaining = budget;
-  let unassigned = entries.length;
-  for (const index of smallestFirst) {
-    capped[index] = truncateEntry(entries[index], Math.floor(remaining / unassigned));
-    remaining -= capped[index].length;
-    unassigned--;
-  }
-  return capped;
-}
-function truncateEntry(entry, allowance) {
-  if (entry.length <= allowance) return entry;
-  const keep = allowance - truncationMarker(entry.length).length;
-  if (keep <= 0) return entry.slice(0, Math.max(0, allowance));
-  return `${entry.slice(0, keep)}${truncationMarker(entry.length - keep)}`;
-}
-function truncationMarker(omitted) {
-  return `
-
-... (tool output truncated, ${omitted} characters omitted)`;
-}
-
-// src/lib/chat/toolResults.ts
-var TOOL_RESULTS_PREFIX = "[Tool Execution Results]";
-var DISPLAY_CARD_PLACEHOLDER = "[displayed a card to the user with the tool result]";
-var TOOL_LINE = /^Tool "([^"]+)" returned: (.*)$/;
-var TOOL_RESULT_ORIGIN = "tool_result";
-function isToolResultsRow(row) {
-  if (row.origin === TOOL_RESULT_ORIGIN) return true;
-  if (typeof row.origin === "string") return false;
-  if (row.role !== "user" || !row.content.startsWith(TOOL_RESULTS_PREFIX)) return false;
-  return parseToolResultSegments(row.content).length > 0;
-}
-function parseToolResultSegments(content) {
-  const segments = [];
-  for (const raw of content.split("\n")) {
-    const line = raw.trim();
-    const match = TOOL_LINE.exec(line);
-    if (match) {
-      segments.push({ name: match[1], line });
-      continue;
-    }
-    const current = segments[segments.length - 1];
-    if (!current || line === "" || line === TOOL_RESULT_FOOTER_LINE) continue;
-    if (line === TOOL_RESULTS_PREFIX) continue;
-    current.line = `${current.line}
-${line}`;
-  }
-  return segments;
-}
-function prepareToolResultsForReplay(rows, options) {
-  if (options.fold) {
-    return foldToolResultsRows(rows, {
-      exclude: options.exclude,
-      placeholder: options.placeholder
-    });
-  }
-  return rows.filter((row) => !isToolResultsRow(row));
-}
-function foldToolResultsRows(rows, options) {
-  const exclude = new Set(options?.exclude ?? []);
-  const out = [];
-  const pending = [];
-  const assistantIndexById = /* @__PURE__ */ new Map();
-  for (const row of rows) {
-    if (isToolResultsRow(row)) {
-      pending.push({ row, insertionPoint: out.length });
-      continue;
-    }
-    out.push(row);
-    if (row.role === "assistant" && row.uniqueId) {
-      assistantIndexById.set(row.uniqueId, out.length - 1);
-    }
-  }
-  const byTarget = /* @__PURE__ */ new Map();
-  for (const { row, insertionPoint } of pending) {
-    const target = resolveFoldTarget(out, row, insertionPoint, assistantIndexById);
-    if (target === void 0) continue;
-    const group = byTarget.get(target);
-    if (group) group.push(row);
-    else byTarget.set(target, [row]);
-  }
-  for (const [target, groupedRows] of byTarget) {
-    const assistant = out[target];
-    const kept = groupedRows.flatMap((row) => parseToolResultSegments(row.content)).filter((segment) => !exclude.has(segment.name));
-    if (kept.length > 0) {
-      out[target] = { ...assistant, content: appendToolResults(assistant.content, kept) };
-    } else if (!assistant.content.trim() && options?.placeholder) {
-      out[target] = { ...assistant, content: options.placeholder };
-    }
-  }
-  return out;
-}
-function resolveFoldTarget(out, row, insertionPoint, assistantIndexById) {
-  const linked = row.parentMessageId ? assistantIndexById.get(row.parentMessageId) : void 0;
-  if (linked !== void 0) return linked;
-  let previous;
-  for (let i = insertionPoint - 1; i >= 0; i--) {
-    if (out[i].role === "assistant") {
-      previous = i;
-      break;
-    }
-  }
-  let next;
-  for (let i = insertionPoint; i < out.length; i++) {
-    if (out[i].role === "assistant") {
-      next = i;
-      break;
-    }
-  }
-  if (previous === void 0) return next;
-  if (next === void 0) return previous;
-  return next - insertionPoint < insertionPoint - previous ? next : previous;
-}
-var MAX_FOLDED_APPENDIX_CHARS = 2e4;
-function appendToolResults(assistantContent, kept) {
-  const framing = TOOL_RESULTS_PREFIX.length + 1 + (kept.length - 1);
-  const capped = capToolResultEntries(
-    kept.map((segment) => segment.line),
-    MAX_FOLDED_APPENDIX_CHARS - framing
-  );
-  const appendix = `${TOOL_RESULTS_PREFIX}
-${capped.join("\n")}`;
-  return assistantContent.trim() ? `${assistantContent}
-
-${appendix}` : appendix;
-}
-
 // src/lib/db/chat/lazyDecrypt.ts
 var LRU_CAPACITY = 256;
 var titleCache = /* @__PURE__ */ new Map();
@@ -10456,54 +10319,6 @@ function createMemoryVaultTool(vaultCtx, options, embeddingOptions, cache) {
   };
 }
 
-// src/lib/processors/encoding.ts
-var B64_ENCODE_CHUNK = 32768 * 3;
-async function dataUrlToArrayBuffer(dataUrl) {
-  if (dataUrl.startsWith("blob:") || dataUrl.startsWith("http://") || dataUrl.startsWith("https://")) {
-    const response = await fetch(dataUrl);
-    return response.arrayBuffer();
-  }
-  const base64 = dataUrl.split(",")[1];
-  if (typeof Buffer !== "undefined") {
-    const buf = Buffer.from(base64, "base64");
-    return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
-  }
-  const binary = atob(base64);
-  const buffer = new ArrayBuffer(binary.length);
-  const view = new Uint8Array(buffer);
-  for (let i = 0; i < binary.length; i++) {
-    view[i] = binary.charCodeAt(i);
-  }
-  return buffer;
-}
-function uint8ArrayToBase64(data) {
-  if (typeof Buffer !== "undefined") {
-    return Buffer.from(data).toString("base64");
-  }
-  let base64 = "";
-  for (let i = 0; i < data.length; i += B64_ENCODE_CHUNK) {
-    const slice = data.subarray(i, i + B64_ENCODE_CHUNK);
-    let binary = "";
-    for (let j = 0; j < slice.length; j++) {
-      binary += String.fromCharCode(slice[j]);
-    }
-    base64 += btoa(binary);
-  }
-  return base64;
-}
-function base64ToUint8Array(base64) {
-  if (typeof Buffer !== "undefined") {
-    const buf = Buffer.from(base64, "base64");
-    return new Uint8Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength));
-  }
-  const binary = atob(base64);
-  const bytes = new Uint8Array(binary.length);
-  for (let i = 0; i < binary.length; i++) {
-    bytes[i] = binary.charCodeAt(i);
-  }
-  return bytes;
-}
-
 // src/lib/processors/ExcelProcessor.ts
 if (typeof process !== "undefined" && typeof process.umask !== "function") {
   process.umask = (_mask) => 18;
@@ -25156,6 +24971,7 @@ export {
   BUILT_IN_TOOL_SETS,
   BackupAuthProvider,
   BlobUrlManager,
+  CHUNKS_DISCARDED_ORIGIN,
   ChartCard,
   ChartContainer,
   ChartLegend,
@@ -25232,6 +25048,7 @@ export {
   VaultMemory as StoredVaultMemoryModel,
   StreamExpiredError,
   THEME_ATTRS,
+  TOOL_RESULT_ORIGIN,
   TOPICS_EXTRACTION_VERSION,
   TOPIC_EXTRACTION_BATCH_SIZE,
   TREND_RECENT_WINDOW_DAYS,
@@ -25309,6 +25126,7 @@ export {
   createVaultMemoryOp,
   createWeatherPreProcessor,
   createWebSearchPreProcessor,
+  decodeChunkVector,
   decryptConversationTitle,
   decryptData,
   decryptDataBatch,
@@ -25331,6 +25149,7 @@ export {
   eagerEmbedContent,
   embedAllMessages,
   embedMessage,
+  encodeChunkVector,
   encryptData,
   encryptDataBatch,
   encryptDataBytes,

package/dist/react/{recall-2J2755WC.mjs → recall-DQJINKEY.mjs}

@@ -1,6 +1,6 @@
 import {
   recall
-} from "./chunk-IO4ZCPK2.mjs";
+} from "./chunk-OB5HFCLB.mjs";
 import "./chunk-ILAWDQDE.mjs";
 import "./chunk-GYKJOWEO.mjs";
 export {

package/dist/server/{chunk-EKYGWD5N.mjs → chunk-BZEAYCBV.mjs}

@@ -15115,6 +15115,77 @@ function cosineSimilarity(a, b) {
   return denom === 0 ? 0 : dot / denom;
 }
 
+// src/lib/processors/encoding.ts
+var B64_ENCODE_CHUNK = 32768 * 3;
+async function dataUrlToArrayBuffer(dataUrl) {
+  if (dataUrl.startsWith("blob:") || dataUrl.startsWith("http://") || dataUrl.startsWith("https://")) {
+    const response = await fetch(dataUrl);
+    return response.arrayBuffer();
+  }
+  const base64 = dataUrl.split(",")[1];
+  if (typeof Buffer !== "undefined") {
+    const buf = Buffer.from(base64, "base64");
+    return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
+  }
+  const binary = atob(base64);
+  const buffer = new ArrayBuffer(binary.length);
+  const view = new Uint8Array(buffer);
+  for (let i = 0; i < binary.length; i++) {
+    view[i] = binary.charCodeAt(i);
+  }
+  return buffer;
+}
+function uint8ArrayToBase64(data) {
+  if (typeof Buffer !== "undefined") {
+    return Buffer.from(data).toString("base64");
+  }
+  let base64 = "";
+  for (let i = 0; i < data.length; i += B64_ENCODE_CHUNK) {
+    const slice = data.subarray(i, i + B64_ENCODE_CHUNK);
+    let binary = "";
+    for (let j = 0; j < slice.length; j++) {
+      binary += String.fromCharCode(slice[j]);
+    }
+    base64 += btoa(binary);
+  }
+  return base64;
+}
+function base64ToUint8Array(base64) {
+  if (typeof Buffer !== "undefined") {
+    const buf = Buffer.from(base64, "base64");
+    return new Uint8Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength));
+  }
+  const binary = atob(base64);
+  const bytes = new Uint8Array(binary.length);
+  for (let i = 0; i < binary.length; i++) {
+    bytes[i] = binary.charCodeAt(i);
+  }
+  return bytes;
+}
+
+// src/lib/memoryEngine/vectorEncoding.ts
+var BYTES_PER_FLOAT32 = 4;
+var CANONICAL_BASE64 = /^[A-Za-z0-9+/]+={0,2}$/;
+function decodeChunkVector(vector, onMalformed) {
+  if (!vector || vector.length === 0) return new Float32Array(0);
+  if (typeof vector !== "string") return Float32Array.from(vector);
+  if (vector.length % 4 !== 0 || !CANONICAL_BASE64.test(vector)) {
+    onMalformed?.();
+    return new Float32Array(0);
+  }
+  try {
+    const bytes = base64ToUint8Array(vector);
+    if (bytes.byteLength === 0 || bytes.byteLength % BYTES_PER_FLOAT32 !== 0) {
+      onMalformed?.();
+      return new Float32Array(0);
+    }
+    return new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / BYTES_PER_FLOAT32);
+  } catch {
+    onMalformed?.();
+    return new Float32Array(0);
+  }
+}
+
 // src/lib/db/encryption-utils.ts
 var ENCRYPTION_PREFIX = "enc:v3:";
 function isEncrypted(value) {
@@ -16052,6 +16123,7 @@ async function searchChunksOp(ctx, queryVector, options) {
   const messages = await ctx.messagesCollection.query(...queryConditions).fetch();
   const candidates = [];
   let staleSkipped = 0;
+  let unreadableVectors = 0;
   for (const message of messages) {
     const msgConvId = String(message._getRaw("conversation_id"));
     if (!activeConversationIds.has(msgConvId)) continue;
@@ -16089,7 +16161,9 @@ async function searchChunksOp(ctx, queryVector, options) {
     const chunks = await readJsonField(message, "chunks", ctx.walletAddress);
     if (chunks && chunks.length > 0) {
       const vectors = chunks.map(
-        (chunk) => chunk.vector && chunk.vector.length > 0 ? Float32Array.from(chunk.vector) : new Float32Array(0)
+        (chunk) => decodeChunkVector(chunk.vector, () => {
+          unreadableVectors++;
+        })
       );
       chunkCache?.set(message.id, { version, chunks: vectors });
       for (let ci = 0; ci < chunks.length; ci++) {
@@ -16120,6 +16194,11 @@ async function searchChunksOp(ctx, queryVector, options) {
       `searchChunksOp: skipped ${staleSkipped} messages whose embedding model differs from the current model (${embeddingModel}) \u2014 re-embed via chunkAndEmbedAllMessages`
     );
   }
+  if (unreadableVectors > 0) {
+    getLogger().warn(
+      `searchChunksOp: could not read ${unreadableVectors} chunk vectors and left them out of ranking \u2014 the stored value is corrupt, so those chunks stay unsearchable until the message is re-embedded via chunkAndEmbedAllMessages`
+    );
+  }
   const topK = candidates.sort((a, b) => b.similarity - a.similarity).slice(0, limit);
   const uniqueMessages = /* @__PURE__ */ new Map();
   for (const c of topK) uniqueMessages.set(c.message.id, c.message);
@@ -17695,6 +17774,9 @@ async function updateVaultMemoryEmbeddingOp(ctx, id, embedding, embeddingModel)
 // src/lib/memoryEngine/constants.ts
 var DEFAULT_API_EMBEDDING_MODEL = "qwen/qwen3-embedding-8b";
 
+// src/lib/chat/toolResults.ts
+var TOOL_RESULT_ORIGIN = "tool_result";
+
 // src/lib/memoryEngine/chunking.ts
 var DEFAULT_CHUNK_SIZE = 400;
 var DEFAULT_CHUNK_OVERLAP = 50;
@@ -18820,7 +18902,13 @@ async function generateEmbeddings(texts, options) {
 
 // src/lib/memoryEngine/embeddings.ts
 var DEFAULT_MIN_CONTENT_LENGTH = 10;
-var NON_EMBEDDABLE_ORIGIN = "tool_result";
+var CHUNKS_DISCARDED_ORIGIN = "chunks_discarded";
+var NON_EMBEDDABLE_ORIGINS = /* @__PURE__ */ new Set([TOOL_RESULT_ORIGIN, CHUNKS_DISCARDED_ORIGIN]);
+function isNonEmbeddableOrigin(origin, reembedDiscarded = false) {
+  if (typeof origin !== "string") return false;
+  if (reembedDiscarded && origin === CHUNKS_DISCARDED_ORIGIN) return false;
+  return NON_EMBEDDABLE_ORIGINS.has(origin);
+}
 async function embedMessage(ctx, messageId, options) {
   const message = await getMessageOp(ctx, messageId);
   if (!message) {
@@ -18829,7 +18917,7 @@ async function embedMessage(ctx, messageId, options) {
   if (message.vector && message.vector.length > 0) {
     return message;
   }
-  if (message.origin === NON_EMBEDDABLE_ORIGIN) {
+  if (isNonEmbeddableOrigin(message.origin)) {
     return message;
   }
   if (isEncrypted(message.content)) {
@@ -18865,7 +18953,7 @@ async function embedAllMessages(ctx, options, filter) {
       if (message.role === "system") {
         continue;
       }
-      if (message.origin === NON_EMBEDDABLE_ORIGIN) {
+      if (isNonEmbeddableOrigin(message.origin, filter?.reembedDiscarded)) {
         continue;
       }
       considered++;
@@ -18905,7 +18993,7 @@ async function chunkAndEmbedMessage(ctx, messageId, options) {
   if (message.chunks && message.chunks.length > 0) {
     return message;
   }
-  if (message.origin === NON_EMBEDDABLE_ORIGIN) {
+  if (isNonEmbeddableOrigin(message.origin)) {
     return message;
   }
   if (isEncrypted(message.content)) {
@@ -18953,7 +19041,7 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
       if (hasVector && !filter?.rechunkExisting && !isStale) continue;
       if (filter?.roles && !filter.roles.includes(message.role)) continue;
       if (message.role === "system") continue;
-      if (message.origin === NON_EMBEDDABLE_ORIGIN) continue;
+      if (isNonEmbeddableOrigin(message.origin, filter?.reembedDiscarded)) continue;
       considered++;
       if (isEncrypted(message.content)) {
         stillEncrypted++;
@@ -20757,7 +20845,7 @@ function createMemoryVaultSearchTool(vaultCtx, embeddingOptions, cache, searchOp
             subQueries = decomp.subQueries;
           }
         }
-        const { recall: recall2 } = await import("./recall-IVV6EILK.mjs");
+        const { recall: recall2 } = await import("./recall-CZFWWOVF.mjs");
         let recallDegraded = [];
         const result = await recall2(
           query,
@@ -21746,6 +21834,8 @@ export {
   encryptJsonField,
   decryptJsonField,
   cosineSimilarity,
+  dataUrlToArrayBuffer,
+  uint8ArrayToBase64,
   stripToolCalls,
   generateConversationId,
   conversationToStoredRaw,