@anuma/sdk 7.0.3 → 7.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3731,8 +3731,9 @@ async function updateConversationPinnedOp(ctx, id, pinned) {
3731
3731
  }
3732
3732
  return false;
3733
3733
  }
3734
- async function getMessagesOp(ctx, convId) {
3735
- const results = await ctx.messagesCollection.query(import_watermelondb2.Q.where("conversation_id", convId), import_watermelondb2.Q.sortBy("message_id", import_watermelondb2.Q.asc)).unsafeFetchRaw();
3734
+ async function getMessagesOp(ctx, convId, opts) {
3735
+ const fetched = await ctx.messagesCollection.query(import_watermelondb2.Q.where("conversation_id", convId), import_watermelondb2.Q.sortBy("message_id", import_watermelondb2.Q.asc)).unsafeFetchRaw();
3736
+ const results = opts?.rawFilter ? fetched.filter(opts.rawFilter) : fetched;
3736
3737
  return Promise.all(
3737
3738
  results.map(
3738
3739
  (raw) => messageRawToStored(raw, ctx.walletAddress, ctx.signMessage, ctx.embeddedWalletSigner)
@@ -5885,6 +5886,37 @@ async function chunkAndEmbedMessage(ctx, messageId, options) {
5885
5886
  }));
5886
5887
  return updateMessageChunksOp(ctx, messageId, messageChunks, embeddingModel);
5887
5888
  }
5889
+ var MIN_SEALED_EMBEDDING_HEX = 256;
5890
+ function rawHasEmbedding(value) {
5891
+ if (Array.isArray(value)) return value.length > 0;
5892
+ if (typeof value !== "string" || value.length === 0) return false;
5893
+ if (isEncrypted(value)) {
5894
+ return value.length - value.indexOf(":", 4) - 1 >= MIN_SEALED_EMBEDDING_HEX;
5895
+ }
5896
+ try {
5897
+ const parsed = JSON.parse(value);
5898
+ return Array.isArray(parsed) && parsed.length > 0;
5899
+ } catch {
5900
+ return false;
5901
+ }
5902
+ }
5903
+ function sealedVersion(value) {
5904
+ if (typeof value !== "string" || !isEncrypted(value)) return null;
5905
+ return value.startsWith("enc:v2:") ? "v2" : "v3";
5906
+ }
5907
+ function mayNeedChunkEmbedding(raw, embeddingModel, filter) {
5908
+ const role = raw.role;
5909
+ if (role === "system") return false;
5910
+ if (filter?.roles && !filter.roles.includes(role ?? "")) return false;
5911
+ if (isNonEmbeddableOrigin(raw.origin, filter?.reembedDiscarded)) {
5912
+ return false;
5913
+ }
5914
+ const model = raw.embedding_model;
5915
+ if (typeof model === "string" && model !== embeddingModel) return true;
5916
+ if (rawHasEmbedding(raw.chunks)) return false;
5917
+ if (rawHasEmbedding(raw.vector) && !filter?.rechunkExisting) return false;
5918
+ return true;
5919
+ }
5888
5920
  async function chunkAndEmbedAllMessages(ctx, options, filter) {
5889
5921
  const embeddingModel = options.model ?? DEFAULT_API_EMBEDDING_MODEL;
5890
5922
  const { chunkSize = DEFAULT_CHUNK_SIZE } = options;
@@ -5897,11 +5929,31 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
5897
5929
  let considered = 0;
5898
5930
  let sealedRowsSeen = 0;
5899
5931
  let rowsSeen = 0;
5932
+ const keptSealed = /* @__PURE__ */ new Map();
5933
+ const candidatesStillSealed = /* @__PURE__ */ new Map();
5934
+ const droppedSealed = /* @__PURE__ */ new Map();
5935
+ const rawFilter = (raw) => {
5936
+ rowsSeen++;
5937
+ const keep = mayNeedChunkEmbedding(raw, embeddingModel, filter);
5938
+ const version = sealedVersion(raw.content);
5939
+ if (!version) return keep;
5940
+ if (keep) {
5941
+ keptSealed.set(version, (keptSealed.get(version) ?? 0) + 1);
5942
+ } else {
5943
+ const dropped = droppedSealed.get(version);
5944
+ if (dropped) dropped.count++;
5945
+ else droppedSealed.set(version, { count: 1, probeId: raw.id });
5946
+ }
5947
+ return keep;
5948
+ };
5900
5949
  for (const conv of targetConversations) {
5901
- const messages = await getMessagesOp(ctx, conv.conversationId);
5950
+ const messages = await getMessagesOp(ctx, conv.conversationId, { rawFilter });
5902
5951
  for (const message of messages) {
5903
- rowsSeen++;
5904
- if (isEncrypted(message.content)) sealedRowsSeen++;
5952
+ const stillSealed = sealedVersion(message.content);
5953
+ if (stillSealed) {
5954
+ sealedRowsSeen++;
5955
+ candidatesStillSealed.set(stillSealed, (candidatesStillSealed.get(stillSealed) ?? 0) + 1);
5956
+ }
5905
5957
  const isStale = message.embeddingModel !== void 0 && message.embeddingModel !== null && message.embeddingModel !== embeddingModel;
5906
5958
  if (message.chunks && message.chunks.length > 0 && !isStale) continue;
5907
5959
  const hasVector = message.vector && message.vector.length > 0;
@@ -5914,6 +5966,7 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
5914
5966
  stillEncrypted++;
5915
5967
  continue;
5916
5968
  }
5969
+ if (isToolResultsRow(message)) continue;
5917
5970
  if (message.content.length < minLength) continue;
5918
5971
  if (shouldChunkMessage(message.content, chunkSize)) {
5919
5972
  longMessages.push({
@@ -5925,6 +5978,12 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
5925
5978
  }
5926
5979
  }
5927
5980
  }
5981
+ for (const [version, dropped] of droppedSealed) {
5982
+ const keyOpenedACandidate = (keptSealed.get(version) ?? 0) > (candidatesStillSealed.get(version) ?? 0);
5983
+ if (keyOpenedACandidate) continue;
5984
+ const probe = await getMessageOp(ctx, dropped.probeId);
5985
+ if (!probe || isEncrypted(probe.content)) sealedRowsSeen += dropped.count;
5986
+ }
5928
5987
  if (stillEncrypted > 0 || sealedRowsSeen > 0) {
5929
5988
  getLogger().error(
5930
5989
  "memoryEngine: messages still encrypted (key unavailable?) \u2014 excluded from embedding",
@@ -2158,8 +2158,9 @@ async function updateConversationPinnedOp(ctx, id, pinned) {
2158
2158
  }
2159
2159
  return false;
2160
2160
  }
2161
- async function getMessagesOp(ctx, convId) {
2162
- const results = await ctx.messagesCollection.query(Q.where("conversation_id", convId), Q.sortBy("message_id", Q.asc)).unsafeFetchRaw();
2161
+ async function getMessagesOp(ctx, convId, opts) {
2162
+ const fetched = await ctx.messagesCollection.query(Q.where("conversation_id", convId), Q.sortBy("message_id", Q.asc)).unsafeFetchRaw();
2163
+ const results = opts?.rawFilter ? fetched.filter(opts.rawFilter) : fetched;
2163
2164
  return Promise.all(
2164
2165
  results.map(
2165
2166
  (raw) => messageRawToStored(raw, ctx.walletAddress, ctx.signMessage, ctx.embeddedWalletSigner)
@@ -4306,6 +4307,37 @@ async function chunkAndEmbedMessage(ctx, messageId, options) {
4306
4307
  }));
4307
4308
  return updateMessageChunksOp(ctx, messageId, messageChunks, embeddingModel);
4308
4309
  }
4310
+ var MIN_SEALED_EMBEDDING_HEX = 256;
4311
+ function rawHasEmbedding(value) {
4312
+ if (Array.isArray(value)) return value.length > 0;
4313
+ if (typeof value !== "string" || value.length === 0) return false;
4314
+ if (isEncrypted(value)) {
4315
+ return value.length - value.indexOf(":", 4) - 1 >= MIN_SEALED_EMBEDDING_HEX;
4316
+ }
4317
+ try {
4318
+ const parsed = JSON.parse(value);
4319
+ return Array.isArray(parsed) && parsed.length > 0;
4320
+ } catch {
4321
+ return false;
4322
+ }
4323
+ }
4324
+ function sealedVersion(value) {
4325
+ if (typeof value !== "string" || !isEncrypted(value)) return null;
4326
+ return value.startsWith("enc:v2:") ? "v2" : "v3";
4327
+ }
4328
+ function mayNeedChunkEmbedding(raw, embeddingModel, filter) {
4329
+ const role = raw.role;
4330
+ if (role === "system") return false;
4331
+ if (filter?.roles && !filter.roles.includes(role ?? "")) return false;
4332
+ if (isNonEmbeddableOrigin(raw.origin, filter?.reembedDiscarded)) {
4333
+ return false;
4334
+ }
4335
+ const model = raw.embedding_model;
4336
+ if (typeof model === "string" && model !== embeddingModel) return true;
4337
+ if (rawHasEmbedding(raw.chunks)) return false;
4338
+ if (rawHasEmbedding(raw.vector) && !filter?.rechunkExisting) return false;
4339
+ return true;
4340
+ }
4309
4341
  async function chunkAndEmbedAllMessages(ctx, options, filter) {
4310
4342
  const embeddingModel = options.model ?? DEFAULT_API_EMBEDDING_MODEL;
4311
4343
  const { chunkSize = DEFAULT_CHUNK_SIZE } = options;
@@ -4318,11 +4350,31 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
4318
4350
  let considered = 0;
4319
4351
  let sealedRowsSeen = 0;
4320
4352
  let rowsSeen = 0;
4353
+ const keptSealed = /* @__PURE__ */ new Map();
4354
+ const candidatesStillSealed = /* @__PURE__ */ new Map();
4355
+ const droppedSealed = /* @__PURE__ */ new Map();
4356
+ const rawFilter = (raw) => {
4357
+ rowsSeen++;
4358
+ const keep = mayNeedChunkEmbedding(raw, embeddingModel, filter);
4359
+ const version = sealedVersion(raw.content);
4360
+ if (!version) return keep;
4361
+ if (keep) {
4362
+ keptSealed.set(version, (keptSealed.get(version) ?? 0) + 1);
4363
+ } else {
4364
+ const dropped = droppedSealed.get(version);
4365
+ if (dropped) dropped.count++;
4366
+ else droppedSealed.set(version, { count: 1, probeId: raw.id });
4367
+ }
4368
+ return keep;
4369
+ };
4321
4370
  for (const conv of targetConversations) {
4322
- const messages = await getMessagesOp(ctx, conv.conversationId);
4371
+ const messages = await getMessagesOp(ctx, conv.conversationId, { rawFilter });
4323
4372
  for (const message of messages) {
4324
- rowsSeen++;
4325
- if (isEncrypted(message.content)) sealedRowsSeen++;
4373
+ const stillSealed = sealedVersion(message.content);
4374
+ if (stillSealed) {
4375
+ sealedRowsSeen++;
4376
+ candidatesStillSealed.set(stillSealed, (candidatesStillSealed.get(stillSealed) ?? 0) + 1);
4377
+ }
4326
4378
  const isStale = message.embeddingModel !== void 0 && message.embeddingModel !== null && message.embeddingModel !== embeddingModel;
4327
4379
  if (message.chunks && message.chunks.length > 0 && !isStale) continue;
4328
4380
  const hasVector = message.vector && message.vector.length > 0;
@@ -4335,6 +4387,7 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
4335
4387
  stillEncrypted++;
4336
4388
  continue;
4337
4389
  }
4390
+ if (isToolResultsRow(message)) continue;
4338
4391
  if (message.content.length < minLength) continue;
4339
4392
  if (shouldChunkMessage(message.content, chunkSize)) {
4340
4393
  longMessages.push({
@@ -4346,6 +4399,12 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
4346
4399
  }
4347
4400
  }
4348
4401
  }
4402
+ for (const [version, dropped] of droppedSealed) {
4403
+ const keyOpenedACandidate = (keptSealed.get(version) ?? 0) > (candidatesStillSealed.get(version) ?? 0);
4404
+ if (keyOpenedACandidate) continue;
4405
+ const probe = await getMessageOp(ctx, dropped.probeId);
4406
+ if (!probe || isEncrypted(probe.content)) sealedRowsSeen += dropped.count;
4407
+ }
4349
4408
  if (stillEncrypted > 0 || sealedRowsSeen > 0) {
4350
4409
  getLogger().error(
4351
4410
  "memoryEngine: messages still encrypted (key unavailable?) \u2014 excluded from embedding",
@@ -4456,8 +4456,9 @@ async function getConversationsByProjectOp(ctx, projectId) {
4456
4456
  )
4457
4457
  );
4458
4458
  }
4459
- async function getMessagesOp(ctx, convId) {
4460
- const results = await ctx.messagesCollection.query(import_watermelondb2.Q.where("conversation_id", convId), import_watermelondb2.Q.sortBy("message_id", import_watermelondb2.Q.asc)).unsafeFetchRaw();
4459
+ async function getMessagesOp(ctx, convId, opts) {
4460
+ const fetched = await ctx.messagesCollection.query(import_watermelondb2.Q.where("conversation_id", convId), import_watermelondb2.Q.sortBy("message_id", import_watermelondb2.Q.asc)).unsafeFetchRaw();
4461
+ const results = opts?.rawFilter ? fetched.filter(opts.rawFilter) : fetched;
4461
4462
  return Promise.all(
4462
4463
  results.map(
4463
4464
  (raw) => messageRawToStored(raw, ctx.walletAddress, ctx.signMessage, ctx.embeddedWalletSigner)
@@ -6708,6 +6709,37 @@ async function chunkAndEmbedMessage(ctx, messageId, options) {
6708
6709
  }));
6709
6710
  return updateMessageChunksOp(ctx, messageId, messageChunks, embeddingModel);
6710
6711
  }
6712
+ var MIN_SEALED_EMBEDDING_HEX = 256;
6713
+ function rawHasEmbedding(value) {
6714
+ if (Array.isArray(value)) return value.length > 0;
6715
+ if (typeof value !== "string" || value.length === 0) return false;
6716
+ if (isEncrypted(value)) {
6717
+ return value.length - value.indexOf(":", 4) - 1 >= MIN_SEALED_EMBEDDING_HEX;
6718
+ }
6719
+ try {
6720
+ const parsed = JSON.parse(value);
6721
+ return Array.isArray(parsed) && parsed.length > 0;
6722
+ } catch {
6723
+ return false;
6724
+ }
6725
+ }
6726
+ function sealedVersion(value) {
6727
+ if (typeof value !== "string" || !isEncrypted(value)) return null;
6728
+ return value.startsWith("enc:v2:") ? "v2" : "v3";
6729
+ }
6730
+ function mayNeedChunkEmbedding(raw, embeddingModel, filter) {
6731
+ const role = raw.role;
6732
+ if (role === "system") return false;
6733
+ if (filter?.roles && !filter.roles.includes(role ?? "")) return false;
6734
+ if (isNonEmbeddableOrigin(raw.origin, filter?.reembedDiscarded)) {
6735
+ return false;
6736
+ }
6737
+ const model = raw.embedding_model;
6738
+ if (typeof model === "string" && model !== embeddingModel) return true;
6739
+ if (rawHasEmbedding(raw.chunks)) return false;
6740
+ if (rawHasEmbedding(raw.vector) && !filter?.rechunkExisting) return false;
6741
+ return true;
6742
+ }
6711
6743
  async function chunkAndEmbedAllMessages(ctx, options, filter) {
6712
6744
  const embeddingModel = options.model ?? DEFAULT_API_EMBEDDING_MODEL;
6713
6745
  const { chunkSize = DEFAULT_CHUNK_SIZE } = options;
@@ -6720,11 +6752,31 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
6720
6752
  let considered = 0;
6721
6753
  let sealedRowsSeen = 0;
6722
6754
  let rowsSeen = 0;
6755
+ const keptSealed = /* @__PURE__ */ new Map();
6756
+ const candidatesStillSealed = /* @__PURE__ */ new Map();
6757
+ const droppedSealed = /* @__PURE__ */ new Map();
6758
+ const rawFilter = (raw) => {
6759
+ rowsSeen++;
6760
+ const keep = mayNeedChunkEmbedding(raw, embeddingModel, filter);
6761
+ const version = sealedVersion(raw.content);
6762
+ if (!version) return keep;
6763
+ if (keep) {
6764
+ keptSealed.set(version, (keptSealed.get(version) ?? 0) + 1);
6765
+ } else {
6766
+ const dropped = droppedSealed.get(version);
6767
+ if (dropped) dropped.count++;
6768
+ else droppedSealed.set(version, { count: 1, probeId: raw.id });
6769
+ }
6770
+ return keep;
6771
+ };
6723
6772
  for (const conv of targetConversations) {
6724
- const messages = await getMessagesOp(ctx, conv.conversationId);
6773
+ const messages = await getMessagesOp(ctx, conv.conversationId, { rawFilter });
6725
6774
  for (const message of messages) {
6726
- rowsSeen++;
6727
- if (isEncrypted(message.content)) sealedRowsSeen++;
6775
+ const stillSealed = sealedVersion(message.content);
6776
+ if (stillSealed) {
6777
+ sealedRowsSeen++;
6778
+ candidatesStillSealed.set(stillSealed, (candidatesStillSealed.get(stillSealed) ?? 0) + 1);
6779
+ }
6728
6780
  const isStale = message.embeddingModel !== void 0 && message.embeddingModel !== null && message.embeddingModel !== embeddingModel;
6729
6781
  if (message.chunks && message.chunks.length > 0 && !isStale) continue;
6730
6782
  const hasVector = message.vector && message.vector.length > 0;
@@ -6737,6 +6789,7 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
6737
6789
  stillEncrypted++;
6738
6790
  continue;
6739
6791
  }
6792
+ if (isToolResultsRow(message)) continue;
6740
6793
  if (message.content.length < minLength) continue;
6741
6794
  if (shouldChunkMessage(message.content, chunkSize)) {
6742
6795
  longMessages.push({
@@ -6748,6 +6801,12 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
6748
6801
  }
6749
6802
  }
6750
6803
  }
6804
+ for (const [version, dropped] of droppedSealed) {
6805
+ const keyOpenedACandidate = (keptSealed.get(version) ?? 0) > (candidatesStillSealed.get(version) ?? 0);
6806
+ if (keyOpenedACandidate) continue;
6807
+ const probe = await getMessageOp(ctx, dropped.probeId);
6808
+ if (!probe || isEncrypted(probe.content)) sealedRowsSeen += dropped.count;
6809
+ }
6751
6810
  if (stillEncrypted > 0 || sealedRowsSeen > 0) {
6752
6811
  getLogger().error(
6753
6812
  "memoryEngine: messages still encrypted (key unavailable?) \u2014 excluded from embedding",
@@ -2681,8 +2681,9 @@ async function getConversationsByProjectOp(ctx, projectId) {
2681
2681
  )
2682
2682
  );
2683
2683
  }
2684
- async function getMessagesOp(ctx, convId) {
2685
- const results = await ctx.messagesCollection.query(Q.where("conversation_id", convId), Q.sortBy("message_id", Q.asc)).unsafeFetchRaw();
2684
+ async function getMessagesOp(ctx, convId, opts) {
2685
+ const fetched = await ctx.messagesCollection.query(Q.where("conversation_id", convId), Q.sortBy("message_id", Q.asc)).unsafeFetchRaw();
2686
+ const results = opts?.rawFilter ? fetched.filter(opts.rawFilter) : fetched;
2686
2687
  return Promise.all(
2687
2688
  results.map(
2688
2689
  (raw) => messageRawToStored(raw, ctx.walletAddress, ctx.signMessage, ctx.embeddedWalletSigner)
@@ -4927,6 +4928,37 @@ async function chunkAndEmbedMessage(ctx, messageId, options) {
4927
4928
  }));
4928
4929
  return updateMessageChunksOp(ctx, messageId, messageChunks, embeddingModel);
4929
4930
  }
4931
+ var MIN_SEALED_EMBEDDING_HEX = 256;
4932
+ function rawHasEmbedding(value) {
4933
+ if (Array.isArray(value)) return value.length > 0;
4934
+ if (typeof value !== "string" || value.length === 0) return false;
4935
+ if (isEncrypted(value)) {
4936
+ return value.length - value.indexOf(":", 4) - 1 >= MIN_SEALED_EMBEDDING_HEX;
4937
+ }
4938
+ try {
4939
+ const parsed = JSON.parse(value);
4940
+ return Array.isArray(parsed) && parsed.length > 0;
4941
+ } catch {
4942
+ return false;
4943
+ }
4944
+ }
4945
+ function sealedVersion(value) {
4946
+ if (typeof value !== "string" || !isEncrypted(value)) return null;
4947
+ return value.startsWith("enc:v2:") ? "v2" : "v3";
4948
+ }
4949
+ function mayNeedChunkEmbedding(raw, embeddingModel, filter) {
4950
+ const role = raw.role;
4951
+ if (role === "system") return false;
4952
+ if (filter?.roles && !filter.roles.includes(role ?? "")) return false;
4953
+ if (isNonEmbeddableOrigin(raw.origin, filter?.reembedDiscarded)) {
4954
+ return false;
4955
+ }
4956
+ const model = raw.embedding_model;
4957
+ if (typeof model === "string" && model !== embeddingModel) return true;
4958
+ if (rawHasEmbedding(raw.chunks)) return false;
4959
+ if (rawHasEmbedding(raw.vector) && !filter?.rechunkExisting) return false;
4960
+ return true;
4961
+ }
4930
4962
  async function chunkAndEmbedAllMessages(ctx, options, filter) {
4931
4963
  const embeddingModel = options.model ?? DEFAULT_API_EMBEDDING_MODEL;
4932
4964
  const { chunkSize = DEFAULT_CHUNK_SIZE } = options;
@@ -4939,11 +4971,31 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
4939
4971
  let considered = 0;
4940
4972
  let sealedRowsSeen = 0;
4941
4973
  let rowsSeen = 0;
4974
+ const keptSealed = /* @__PURE__ */ new Map();
4975
+ const candidatesStillSealed = /* @__PURE__ */ new Map();
4976
+ const droppedSealed = /* @__PURE__ */ new Map();
4977
+ const rawFilter = (raw) => {
4978
+ rowsSeen++;
4979
+ const keep = mayNeedChunkEmbedding(raw, embeddingModel, filter);
4980
+ const version = sealedVersion(raw.content);
4981
+ if (!version) return keep;
4982
+ if (keep) {
4983
+ keptSealed.set(version, (keptSealed.get(version) ?? 0) + 1);
4984
+ } else {
4985
+ const dropped = droppedSealed.get(version);
4986
+ if (dropped) dropped.count++;
4987
+ else droppedSealed.set(version, { count: 1, probeId: raw.id });
4988
+ }
4989
+ return keep;
4990
+ };
4942
4991
  for (const conv of targetConversations) {
4943
- const messages = await getMessagesOp(ctx, conv.conversationId);
4992
+ const messages = await getMessagesOp(ctx, conv.conversationId, { rawFilter });
4944
4993
  for (const message of messages) {
4945
- rowsSeen++;
4946
- if (isEncrypted(message.content)) sealedRowsSeen++;
4994
+ const stillSealed = sealedVersion(message.content);
4995
+ if (stillSealed) {
4996
+ sealedRowsSeen++;
4997
+ candidatesStillSealed.set(stillSealed, (candidatesStillSealed.get(stillSealed) ?? 0) + 1);
4998
+ }
4947
4999
  const isStale = message.embeddingModel !== void 0 && message.embeddingModel !== null && message.embeddingModel !== embeddingModel;
4948
5000
  if (message.chunks && message.chunks.length > 0 && !isStale) continue;
4949
5001
  const hasVector = message.vector && message.vector.length > 0;
@@ -4956,6 +5008,7 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
4956
5008
  stillEncrypted++;
4957
5009
  continue;
4958
5010
  }
5011
+ if (isToolResultsRow(message)) continue;
4959
5012
  if (message.content.length < minLength) continue;
4960
5013
  if (shouldChunkMessage(message.content, chunkSize)) {
4961
5014
  longMessages.push({
@@ -4967,6 +5020,12 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
4967
5020
  }
4968
5021
  }
4969
5022
  }
5023
+ for (const [version, dropped] of droppedSealed) {
5024
+ const keyOpenedACandidate = (keptSealed.get(version) ?? 0) > (candidatesStillSealed.get(version) ?? 0);
5025
+ if (keyOpenedACandidate) continue;
5026
+ const probe = await getMessageOp(ctx, dropped.probeId);
5027
+ if (!probe || isEncrypted(probe.content)) sealedRowsSeen += dropped.count;
5028
+ }
4970
5029
  if (stillEncrypted > 0 || sealedRowsSeen > 0) {
4971
5030
  getLogger().error(
4972
5031
  "memoryEngine: messages still encrypted (key unavailable?) \u2014 excluded from embedding",
@@ -16574,6 +16574,10 @@ __export(index_exports, {
16574
16574
  });
16575
16575
  module.exports = __toCommonJS(index_exports);
16576
16576
 
16577
+ // src/lib/chat/toolResultMessage.ts
16578
+ var FOOTER = "\n\nBased on these results, continue with the task.";
16579
+ var TOOL_RESULT_FOOTER_LINE = FOOTER.trim();
16580
+
16577
16581
  // src/lib/tools/toolCatalog.ts
16578
16582
  var TOOL_CATALOG = {
16579
16583
  gmail_search_messages: { label: "Search emails", provider: "gmail", connector: "Gmail" },
@@ -16750,7 +16754,32 @@ ${neutralised}
16750
16754
  }
16751
16755
 
16752
16756
  // src/lib/chat/toolResults.ts
16757
+ var TOOL_RESULTS_PREFIX = "[Tool Execution Results]";
16758
+ var TOOL_LINE = /^Tool "([^"]+)" returned: (.*)$/;
16753
16759
  var TOOL_RESULT_ORIGIN = "tool_result";
16760
+ function isToolResultsRow(row) {
16761
+ if (row.origin === TOOL_RESULT_ORIGIN) return true;
16762
+ if (typeof row.origin === "string") return false;
16763
+ if (row.role !== "user" || !row.content.startsWith(TOOL_RESULTS_PREFIX)) return false;
16764
+ return parseToolResultSegments(row.content).length > 0;
16765
+ }
16766
+ function parseToolResultSegments(content) {
16767
+ const segments = [];
16768
+ for (const raw of content.split("\n")) {
16769
+ const line = raw.trim();
16770
+ const match = TOOL_LINE.exec(line);
16771
+ if (match) {
16772
+ segments.push({ name: match[1], line });
16773
+ continue;
16774
+ }
16775
+ const current = segments[segments.length - 1];
16776
+ if (!current || line === "" || line === TOOL_RESULT_FOOTER_LINE) continue;
16777
+ if (line === TOOL_RESULTS_PREFIX) continue;
16778
+ current.line = `${current.line}
16779
+ ${line}`;
16780
+ }
16781
+ return segments;
16782
+ }
16754
16783
 
16755
16784
  // src/lib/db/chat/operations.ts
16756
16785
  var import_watermelondb2 = __toESM(require_watermelondb());
@@ -18373,8 +18402,9 @@ async function getConversationsByProjectOp(ctx, projectId) {
18373
18402
  )
18374
18403
  );
18375
18404
  }
18376
- async function getMessagesOp(ctx, convId) {
18377
- const results = await ctx.messagesCollection.query(import_watermelondb2.Q.where("conversation_id", convId), import_watermelondb2.Q.sortBy("message_id", import_watermelondb2.Q.asc)).unsafeFetchRaw();
18405
+ async function getMessagesOp(ctx, convId, opts) {
18406
+ const fetched = await ctx.messagesCollection.query(import_watermelondb2.Q.where("conversation_id", convId), import_watermelondb2.Q.sortBy("message_id", import_watermelondb2.Q.asc)).unsafeFetchRaw();
18407
+ const results = opts?.rawFilter ? fetched.filter(opts.rawFilter) : fetched;
18378
18408
  return Promise.all(
18379
18409
  results.map(
18380
18410
  (raw) => messageRawToStored(raw, ctx.walletAddress, ctx.signMessage, ctx.embeddedWalletSigner)
@@ -20498,6 +20528,37 @@ async function chunkAndEmbedMessage(ctx, messageId, options) {
20498
20528
  }));
20499
20529
  return updateMessageChunksOp(ctx, messageId, messageChunks, embeddingModel);
20500
20530
  }
20531
+ var MIN_SEALED_EMBEDDING_HEX = 256;
20532
+ function rawHasEmbedding(value) {
20533
+ if (Array.isArray(value)) return value.length > 0;
20534
+ if (typeof value !== "string" || value.length === 0) return false;
20535
+ if (isEncrypted(value)) {
20536
+ return value.length - value.indexOf(":", 4) - 1 >= MIN_SEALED_EMBEDDING_HEX;
20537
+ }
20538
+ try {
20539
+ const parsed = JSON.parse(value);
20540
+ return Array.isArray(parsed) && parsed.length > 0;
20541
+ } catch {
20542
+ return false;
20543
+ }
20544
+ }
20545
+ function sealedVersion(value) {
20546
+ if (typeof value !== "string" || !isEncrypted(value)) return null;
20547
+ return value.startsWith("enc:v2:") ? "v2" : "v3";
20548
+ }
20549
+ function mayNeedChunkEmbedding(raw, embeddingModel, filter) {
20550
+ const role = raw.role;
20551
+ if (role === "system") return false;
20552
+ if (filter?.roles && !filter.roles.includes(role ?? "")) return false;
20553
+ if (isNonEmbeddableOrigin(raw.origin, filter?.reembedDiscarded)) {
20554
+ return false;
20555
+ }
20556
+ const model = raw.embedding_model;
20557
+ if (typeof model === "string" && model !== embeddingModel) return true;
20558
+ if (rawHasEmbedding(raw.chunks)) return false;
20559
+ if (rawHasEmbedding(raw.vector) && !filter?.rechunkExisting) return false;
20560
+ return true;
20561
+ }
20501
20562
  async function chunkAndEmbedAllMessages(ctx, options, filter) {
20502
20563
  const embeddingModel = options.model ?? DEFAULT_API_EMBEDDING_MODEL;
20503
20564
  const { chunkSize = DEFAULT_CHUNK_SIZE } = options;
@@ -20510,11 +20571,31 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
20510
20571
  let considered = 0;
20511
20572
  let sealedRowsSeen = 0;
20512
20573
  let rowsSeen = 0;
20574
+ const keptSealed = /* @__PURE__ */ new Map();
20575
+ const candidatesStillSealed = /* @__PURE__ */ new Map();
20576
+ const droppedSealed = /* @__PURE__ */ new Map();
20577
+ const rawFilter = (raw) => {
20578
+ rowsSeen++;
20579
+ const keep = mayNeedChunkEmbedding(raw, embeddingModel, filter);
20580
+ const version = sealedVersion(raw.content);
20581
+ if (!version) return keep;
20582
+ if (keep) {
20583
+ keptSealed.set(version, (keptSealed.get(version) ?? 0) + 1);
20584
+ } else {
20585
+ const dropped = droppedSealed.get(version);
20586
+ if (dropped) dropped.count++;
20587
+ else droppedSealed.set(version, { count: 1, probeId: raw.id });
20588
+ }
20589
+ return keep;
20590
+ };
20513
20591
  for (const conv of targetConversations) {
20514
- const messages = await getMessagesOp(ctx, conv.conversationId);
20592
+ const messages = await getMessagesOp(ctx, conv.conversationId, { rawFilter });
20515
20593
  for (const message of messages) {
20516
- rowsSeen++;
20517
- if (isEncrypted(message.content)) sealedRowsSeen++;
20594
+ const stillSealed = sealedVersion(message.content);
20595
+ if (stillSealed) {
20596
+ sealedRowsSeen++;
20597
+ candidatesStillSealed.set(stillSealed, (candidatesStillSealed.get(stillSealed) ?? 0) + 1);
20598
+ }
20518
20599
  const isStale = message.embeddingModel !== void 0 && message.embeddingModel !== null && message.embeddingModel !== embeddingModel;
20519
20600
  if (message.chunks && message.chunks.length > 0 && !isStale) continue;
20520
20601
  const hasVector = message.vector && message.vector.length > 0;
@@ -20527,6 +20608,7 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
20527
20608
  stillEncrypted++;
20528
20609
  continue;
20529
20610
  }
20611
+ if (isToolResultsRow(message)) continue;
20530
20612
  if (message.content.length < minLength) continue;
20531
20613
  if (shouldChunkMessage(message.content, chunkSize)) {
20532
20614
  longMessages.push({
@@ -20538,6 +20620,12 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
20538
20620
  }
20539
20621
  }
20540
20622
  }
20623
+ for (const [version, dropped] of droppedSealed) {
20624
+ const keyOpenedACandidate = (keptSealed.get(version) ?? 0) > (candidatesStillSealed.get(version) ?? 0);
20625
+ if (keyOpenedACandidate) continue;
20626
+ const probe = await getMessageOp(ctx, dropped.probeId);
20627
+ if (!probe || isEncrypted(probe.content)) sealedRowsSeen += dropped.count;
20628
+ }
20541
20629
  if (stillEncrypted > 0 || sealedRowsSeen > 0) {
20542
20630
  getLogger().error(
20543
20631
  "memoryEngine: messages still encrypted (key unavailable?) \u2014 excluded from embedding",
@@ -3756,7 +3756,10 @@ declare function updateConversationProjectOp(ctx: StorageOperationsContext, id:
3756
3756
  * Pass null to get conversations that don't belong to any project.
3757
3757
  */
3758
3758
  declare function getConversationsByProjectOp(ctx: StorageOperationsContext, projectId: string | null): Promise<StoredConversation[]>;
3759
- declare function getMessagesOp(ctx: StorageOperationsContext, convId: string): Promise<StoredMessage[]>;
3759
+ declare function getMessagesOp(ctx: StorageOperationsContext, convId: string, opts?: {
3760
+ /** Keeps only rows whose raw, still-encrypted columns pass; runs before decryption. */
3761
+ rawFilter?: (raw: Record<string, unknown>) => boolean;
3762
+ }): Promise<StoredMessage[]>;
3760
3763
  /**
3761
3764
  * Clear all messages in a conversation.
3762
3765
  * Clears file_ids before deletion.
@@ -3756,7 +3756,10 @@ declare function updateConversationProjectOp(ctx: StorageOperationsContext, id:
3756
3756
  * Pass null to get conversations that don't belong to any project.
3757
3757
  */
3758
3758
  declare function getConversationsByProjectOp(ctx: StorageOperationsContext, projectId: string | null): Promise<StoredConversation[]>;
3759
- declare function getMessagesOp(ctx: StorageOperationsContext, convId: string): Promise<StoredMessage[]>;
3759
+ declare function getMessagesOp(ctx: StorageOperationsContext, convId: string, opts?: {
3760
+ /** Keeps only rows whose raw, still-encrypted columns pass; runs before decryption. */
3761
+ rawFilter?: (raw: Record<string, unknown>) => boolean;
3762
+ }): Promise<StoredMessage[]>;
3760
3763
  /**
3761
3764
  * Clear all messages in a conversation.
3762
3765
  * Clears file_ids before deletion.
@@ -14928,6 +14928,10 @@ var require_stepsForMigration = __commonJS({
14928
14928
  }
14929
14929
  });
14930
14930
 
14931
+ // src/lib/chat/toolResultMessage.ts
14932
+ var FOOTER = "\n\nBased on these results, continue with the task.";
14933
+ var TOOL_RESULT_FOOTER_LINE = FOOTER.trim();
14934
+
14931
14935
  // src/lib/tools/toolCatalog.ts
14932
14936
  var TOOL_CATALOG = {
14933
14937
  gmail_search_messages: { label: "Search emails", provider: "gmail", connector: "Gmail" },
@@ -15104,7 +15108,32 @@ ${neutralised}
15104
15108
  }
15105
15109
 
15106
15110
  // src/lib/chat/toolResults.ts
15111
+ var TOOL_RESULTS_PREFIX = "[Tool Execution Results]";
15112
+ var TOOL_LINE = /^Tool "([^"]+)" returned: (.*)$/;
15107
15113
  var TOOL_RESULT_ORIGIN = "tool_result";
15114
+ function isToolResultsRow(row) {
15115
+ if (row.origin === TOOL_RESULT_ORIGIN) return true;
15116
+ if (typeof row.origin === "string") return false;
15117
+ if (row.role !== "user" || !row.content.startsWith(TOOL_RESULTS_PREFIX)) return false;
15118
+ return parseToolResultSegments(row.content).length > 0;
15119
+ }
15120
+ function parseToolResultSegments(content) {
15121
+ const segments = [];
15122
+ for (const raw of content.split("\n")) {
15123
+ const line = raw.trim();
15124
+ const match = TOOL_LINE.exec(line);
15125
+ if (match) {
15126
+ segments.push({ name: match[1], line });
15127
+ continue;
15128
+ }
15129
+ const current = segments[segments.length - 1];
15130
+ if (!current || line === "" || line === TOOL_RESULT_FOOTER_LINE) continue;
15131
+ if (line === TOOL_RESULTS_PREFIX) continue;
15132
+ current.line = `${current.line}
15133
+ ${line}`;
15134
+ }
15135
+ return segments;
15136
+ }
15108
15137
 
15109
15138
  // src/lib/db/chat/operations.ts
15110
15139
  var import_watermelondb2 = __toESM(require_watermelondb());
@@ -16720,8 +16749,9 @@ async function getConversationsByProjectOp(ctx, projectId) {
16720
16749
  )
16721
16750
  );
16722
16751
  }
16723
- async function getMessagesOp(ctx, convId) {
16724
- const results = await ctx.messagesCollection.query(import_watermelondb2.Q.where("conversation_id", convId), import_watermelondb2.Q.sortBy("message_id", import_watermelondb2.Q.asc)).unsafeFetchRaw();
16752
+ async function getMessagesOp(ctx, convId, opts) {
16753
+ const fetched = await ctx.messagesCollection.query(import_watermelondb2.Q.where("conversation_id", convId), import_watermelondb2.Q.sortBy("message_id", import_watermelondb2.Q.asc)).unsafeFetchRaw();
16754
+ const results = opts?.rawFilter ? fetched.filter(opts.rawFilter) : fetched;
16725
16755
  return Promise.all(
16726
16756
  results.map(
16727
16757
  (raw) => messageRawToStored(raw, ctx.walletAddress, ctx.signMessage, ctx.embeddedWalletSigner)
@@ -18839,6 +18869,37 @@ async function chunkAndEmbedMessage(ctx, messageId, options) {
18839
18869
  }));
18840
18870
  return updateMessageChunksOp(ctx, messageId, messageChunks, embeddingModel);
18841
18871
  }
18872
+ var MIN_SEALED_EMBEDDING_HEX = 256;
18873
+ function rawHasEmbedding(value) {
18874
+ if (Array.isArray(value)) return value.length > 0;
18875
+ if (typeof value !== "string" || value.length === 0) return false;
18876
+ if (isEncrypted(value)) {
18877
+ return value.length - value.indexOf(":", 4) - 1 >= MIN_SEALED_EMBEDDING_HEX;
18878
+ }
18879
+ try {
18880
+ const parsed = JSON.parse(value);
18881
+ return Array.isArray(parsed) && parsed.length > 0;
18882
+ } catch {
18883
+ return false;
18884
+ }
18885
+ }
18886
+ function sealedVersion(value) {
18887
+ if (typeof value !== "string" || !isEncrypted(value)) return null;
18888
+ return value.startsWith("enc:v2:") ? "v2" : "v3";
18889
+ }
18890
+ function mayNeedChunkEmbedding(raw, embeddingModel, filter) {
18891
+ const role = raw.role;
18892
+ if (role === "system") return false;
18893
+ if (filter?.roles && !filter.roles.includes(role ?? "")) return false;
18894
+ if (isNonEmbeddableOrigin(raw.origin, filter?.reembedDiscarded)) {
18895
+ return false;
18896
+ }
18897
+ const model = raw.embedding_model;
18898
+ if (typeof model === "string" && model !== embeddingModel) return true;
18899
+ if (rawHasEmbedding(raw.chunks)) return false;
18900
+ if (rawHasEmbedding(raw.vector) && !filter?.rechunkExisting) return false;
18901
+ return true;
18902
+ }
18842
18903
  async function chunkAndEmbedAllMessages(ctx, options, filter) {
18843
18904
  const embeddingModel = options.model ?? DEFAULT_API_EMBEDDING_MODEL;
18844
18905
  const { chunkSize = DEFAULT_CHUNK_SIZE } = options;
@@ -18851,11 +18912,31 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
18851
18912
  let considered = 0;
18852
18913
  let sealedRowsSeen = 0;
18853
18914
  let rowsSeen = 0;
18915
+ const keptSealed = /* @__PURE__ */ new Map();
18916
+ const candidatesStillSealed = /* @__PURE__ */ new Map();
18917
+ const droppedSealed = /* @__PURE__ */ new Map();
18918
+ const rawFilter = (raw) => {
18919
+ rowsSeen++;
18920
+ const keep = mayNeedChunkEmbedding(raw, embeddingModel, filter);
18921
+ const version = sealedVersion(raw.content);
18922
+ if (!version) return keep;
18923
+ if (keep) {
18924
+ keptSealed.set(version, (keptSealed.get(version) ?? 0) + 1);
18925
+ } else {
18926
+ const dropped = droppedSealed.get(version);
18927
+ if (dropped) dropped.count++;
18928
+ else droppedSealed.set(version, { count: 1, probeId: raw.id });
18929
+ }
18930
+ return keep;
18931
+ };
18854
18932
  for (const conv of targetConversations) {
18855
- const messages = await getMessagesOp(ctx, conv.conversationId);
18933
+ const messages = await getMessagesOp(ctx, conv.conversationId, { rawFilter });
18856
18934
  for (const message of messages) {
18857
- rowsSeen++;
18858
- if (isEncrypted(message.content)) sealedRowsSeen++;
18935
+ const stillSealed = sealedVersion(message.content);
18936
+ if (stillSealed) {
18937
+ sealedRowsSeen++;
18938
+ candidatesStillSealed.set(stillSealed, (candidatesStillSealed.get(stillSealed) ?? 0) + 1);
18939
+ }
18859
18940
  const isStale = message.embeddingModel !== void 0 && message.embeddingModel !== null && message.embeddingModel !== embeddingModel;
18860
18941
  if (message.chunks && message.chunks.length > 0 && !isStale) continue;
18861
18942
  const hasVector = message.vector && message.vector.length > 0;
@@ -18868,6 +18949,7 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
18868
18949
  stillEncrypted++;
18869
18950
  continue;
18870
18951
  }
18952
+ if (isToolResultsRow(message)) continue;
18871
18953
  if (message.content.length < minLength) continue;
18872
18954
  if (shouldChunkMessage(message.content, chunkSize)) {
18873
18955
  longMessages.push({
@@ -18879,6 +18961,12 @@ async function chunkAndEmbedAllMessages(ctx, options, filter) {
18879
18961
  }
18880
18962
  }
18881
18963
  }
18964
+ for (const [version, dropped] of droppedSealed) {
18965
+ const keyOpenedACandidate = (keptSealed.get(version) ?? 0) > (candidatesStillSealed.get(version) ?? 0);
18966
+ if (keyOpenedACandidate) continue;
18967
+ const probe = await getMessageOp(ctx, dropped.probeId);
18968
+ if (!probe || isEncrypted(probe.content)) sealedRowsSeen += dropped.count;
18969
+ }
18882
18970
  if (stillEncrypted > 0 || sealedRowsSeen > 0) {
18883
18971
  getLogger().error(
18884
18972
  "memoryEngine: messages still encrypted (key unavailable?) \u2014 excluded from embedding",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@anuma/sdk",
3
- "version": "7.0.3",
3
+ "version": "7.0.4",
4
4
  "description": "",
5
5
  "main": "./dist/index.cjs",
6
6
  "module": "./dist/index.mjs",