@anuma/sdk 7.1.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/dist/expo/index.cjs +59 -11
- package/dist/expo/index.d.mts +13 -0
- package/dist/expo/index.d.ts +13 -0
- package/dist/expo/index.mjs +59 -11
- package/dist/memory/context.d.mts +5 -0
- package/dist/memory/context.d.ts +5 -0
- package/dist/react/index.cjs +96 -27
- package/dist/react/index.d.mts +16 -0
- package/dist/react/index.d.ts +16 -0
- package/dist/react/index.mjs +96 -27
- package/dist/server/index.cjs +59 -11
- package/dist/server/index.d.mts +13 -0
- package/dist/server/index.d.ts +13 -0
- package/dist/server/index.mjs +59 -11
- package/dist/telemetry/index.d.mts +5 -0
- package/dist/telemetry/index.d.ts +5 -0
- package/package.json +1 -1
package/dist/react/index.mjs
CHANGED
|
@@ -17169,10 +17169,18 @@ function admitVaultProjections(queryEmbedding, vectored, k) {
|
|
|
17169
17169
|
}
|
|
17170
17170
|
async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, queryOpts, opts) {
|
|
17171
17171
|
let rowsDecrypted = 0;
|
|
17172
|
+
const keysStart = nowMs();
|
|
17172
17173
|
const keys = await getVaultCandidateKeysOp(
|
|
17173
17174
|
vaultCtx,
|
|
17174
17175
|
Object.keys(queryOpts).length > 0 ? queryOpts : void 0
|
|
17175
17176
|
);
|
|
17177
|
+
const corpusStats = {
|
|
17178
|
+
keysMs: nowMs() - keysStart,
|
|
17179
|
+
embeddingsFetchMs: 0,
|
|
17180
|
+
rowsFetchMs: 0,
|
|
17181
|
+
cacheHits: 0,
|
|
17182
|
+
cacheMisses: 0
|
|
17183
|
+
};
|
|
17176
17184
|
const hydrateOpts = queryOpts.includeArchived ? { includeArchived: true } : void 0;
|
|
17177
17185
|
const vaultSize = keys.length;
|
|
17178
17186
|
if (keys.length === 0) {
|
|
@@ -17182,7 +17190,8 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
17182
17190
|
queryEmbedding: [],
|
|
17183
17191
|
vaultSize: 0,
|
|
17184
17192
|
laneEmbedFailed: false,
|
|
17185
|
-
rowsDecrypted: 0
|
|
17193
|
+
rowsDecrypted: 0,
|
|
17194
|
+
corpusStats
|
|
17186
17195
|
};
|
|
17187
17196
|
}
|
|
17188
17197
|
const queryEmbedStart = nowMs();
|
|
@@ -17204,16 +17213,20 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
17204
17213
|
for (const k2 of keys) {
|
|
17205
17214
|
const cached = cachedRowVector(cache, k2.uniqueId, k2.updatedAt);
|
|
17206
17215
|
if (cached && cached.length === queryEmbedding.length) {
|
|
17216
|
+
corpusStats.cacheHits++;
|
|
17207
17217
|
vectored.push({ uniqueId: k2.uniqueId, embedding: cached, updatedAt: k2.updatedAt });
|
|
17208
17218
|
continue;
|
|
17209
17219
|
}
|
|
17220
|
+
corpusStats.cacheMisses++;
|
|
17210
17221
|
const modelCompatible = (k2.embeddingModel ?? currentModel) === currentModel;
|
|
17211
17222
|
if (modelCompatible) missIds.push(k2.uniqueId);
|
|
17212
17223
|
else noVectorIds.push(k2.uniqueId);
|
|
17213
17224
|
}
|
|
17214
17225
|
}
|
|
17215
17226
|
if (missIds.length > 0) {
|
|
17227
|
+
const embeddingsStart = nowMs();
|
|
17216
17228
|
const rows = await getVaultEmbeddingsByIdsOp(vaultCtx, missIds, hydrateOpts);
|
|
17229
|
+
corpusStats.embeddingsFetchMs += nowMs() - embeddingsStart;
|
|
17217
17230
|
const gotVector = /* @__PURE__ */ new Set();
|
|
17218
17231
|
for (const r of rows) {
|
|
17219
17232
|
if (!r.embedding) continue;
|
|
@@ -17241,7 +17254,9 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
17241
17254
|
`memoryVault: projected search un-embedded lane capped at ${laneIds.length}/${noVectorIds.length}`
|
|
17242
17255
|
);
|
|
17243
17256
|
}
|
|
17257
|
+
const laneStart = nowMs();
|
|
17244
17258
|
const laneFetched = await getVaultMemoriesByIdsOp(vaultCtx, laneIds, hydrateOpts);
|
|
17259
|
+
corpusStats.rowsFetchMs += nowMs() - laneStart;
|
|
17245
17260
|
rowsDecrypted += laneFetched.length;
|
|
17246
17261
|
for (const row of laneFetched) laneById.set(row.uniqueId, row);
|
|
17247
17262
|
const laneRows = laneFetched.filter((m) => !isEncrypted(m.content));
|
|
@@ -17293,7 +17308,9 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
17293
17308
|
}
|
|
17294
17309
|
}
|
|
17295
17310
|
const idsToFetch = [...admissionSet].filter((id) => !laneById.has(id));
|
|
17311
|
+
const admittedStart = nowMs();
|
|
17296
17312
|
const fetched = idsToFetch.length > 0 ? await getVaultMemoriesByIdsOp(vaultCtx, idsToFetch, hydrateOpts) : [];
|
|
17313
|
+
corpusStats.rowsFetchMs += nowMs() - admittedStart;
|
|
17297
17314
|
rowsDecrypted += fetched.length;
|
|
17298
17315
|
const admittedRows = [
|
|
17299
17316
|
...fetched,
|
|
@@ -17322,7 +17339,10 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
17322
17339
|
}
|
|
17323
17340
|
if (staleIds.length > 0) {
|
|
17324
17341
|
const contentById = new Map(memories.map((m) => [m.uniqueId, m]));
|
|
17325
|
-
|
|
17342
|
+
const staleStart = nowMs();
|
|
17343
|
+
const staleRows = await getVaultEmbeddingsByIdsOp(vaultCtx, staleIds, hydrateOpts);
|
|
17344
|
+
corpusStats.embeddingsFetchMs += nowMs() - staleStart;
|
|
17345
|
+
for (const r of staleRows) {
|
|
17326
17346
|
if (!r.embedding) continue;
|
|
17327
17347
|
try {
|
|
17328
17348
|
const parsed = JSON.parse(r.embedding);
|
|
@@ -17354,7 +17374,15 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
17354
17374
|
eventTimeKind: normalizeEventTimeKind(m.eventTimeKind),
|
|
17355
17375
|
factType: m.factType
|
|
17356
17376
|
}));
|
|
17357
|
-
return {
|
|
17377
|
+
return {
|
|
17378
|
+
memories,
|
|
17379
|
+
embeddedItems,
|
|
17380
|
+
queryEmbedding,
|
|
17381
|
+
vaultSize,
|
|
17382
|
+
laneEmbedFailed,
|
|
17383
|
+
rowsDecrypted,
|
|
17384
|
+
corpusStats
|
|
17385
|
+
};
|
|
17358
17386
|
}
|
|
17359
17387
|
async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache, searchOptions) {
|
|
17360
17388
|
if (!query || typeof query !== "string") {
|
|
@@ -17395,6 +17423,7 @@ async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache,
|
|
|
17395
17423
|
embeddingFailure = true;
|
|
17396
17424
|
};
|
|
17397
17425
|
const embedStats = { queryMs: 0, rowsEmbedded: 0 };
|
|
17426
|
+
let corpusStats;
|
|
17398
17427
|
if (searchOptions?.decryptLast) {
|
|
17399
17428
|
const forceIncludeIds = [
|
|
17400
17429
|
...searchOptions.entityRanking ?? [],
|
|
@@ -17415,6 +17444,7 @@ async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache,
|
|
|
17415
17444
|
onEmbeddingDegraded,
|
|
17416
17445
|
embedStats
|
|
17417
17446
|
});
|
|
17447
|
+
corpusStats = corpus.corpusStats;
|
|
17418
17448
|
if (corpus.vaultSize === 0) {
|
|
17419
17449
|
return {
|
|
17420
17450
|
memories: [],
|
|
@@ -17426,7 +17456,8 @@ async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache,
|
|
|
17426
17456
|
decryptLast: true,
|
|
17427
17457
|
rowsDecrypted: corpus.rowsDecrypted,
|
|
17428
17458
|
queryEmbedMs: embedStats.queryMs,
|
|
17429
|
-
rowsEmbedded: embedStats.rowsEmbedded
|
|
17459
|
+
rowsEmbedded: embedStats.rowsEmbedded,
|
|
17460
|
+
corpusStats
|
|
17430
17461
|
};
|
|
17431
17462
|
}
|
|
17432
17463
|
({ memories, embeddedItems, queryEmbedding, vaultSize } = corpus);
|
|
@@ -17443,7 +17474,8 @@ async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache,
|
|
|
17443
17474
|
decryptLast: true,
|
|
17444
17475
|
rowsDecrypted: corpus.rowsDecrypted,
|
|
17445
17476
|
queryEmbedMs: embedStats.queryMs,
|
|
17446
|
-
rowsEmbedded: embedStats.rowsEmbedded
|
|
17477
|
+
rowsEmbedded: embedStats.rowsEmbedded,
|
|
17478
|
+
corpusStats
|
|
17447
17479
|
};
|
|
17448
17480
|
}
|
|
17449
17481
|
} else {
|
|
@@ -17576,7 +17608,8 @@ async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache,
|
|
|
17576
17608
|
embeddingsUnavailable,
|
|
17577
17609
|
embeddingFailure,
|
|
17578
17610
|
decryptLast: !!searchOptions?.decryptLast,
|
|
17579
|
-
rowsDecrypted
|
|
17611
|
+
rowsDecrypted,
|
|
17612
|
+
...corpusStats && { corpusStats }
|
|
17580
17613
|
};
|
|
17581
17614
|
}
|
|
17582
17615
|
async function rankPreparedVaultCandidates(query, prepared, embeddingOptions, searchOptions) {
|
|
@@ -17793,7 +17826,8 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
17793
17826
|
decryptLast: prepared.decryptLast,
|
|
17794
17827
|
rowsDecrypted: prepared.rowsDecrypted,
|
|
17795
17828
|
queryEmbedMs: prepared.queryEmbedMs,
|
|
17796
|
-
rowsEmbedded: prepared.rowsEmbedded
|
|
17829
|
+
rowsEmbedded: prepared.rowsEmbedded,
|
|
17830
|
+
...prepared.corpusStats && { corpusStats: prepared.corpusStats }
|
|
17797
17831
|
};
|
|
17798
17832
|
}
|
|
17799
17833
|
if (prepared.memories.length === 0) {
|
|
@@ -17808,7 +17842,8 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
17808
17842
|
decryptLast: prepared.decryptLast,
|
|
17809
17843
|
rowsDecrypted: prepared.rowsDecrypted,
|
|
17810
17844
|
queryEmbedMs: prepared.queryEmbedMs,
|
|
17811
|
-
rowsEmbedded: prepared.rowsEmbedded
|
|
17845
|
+
rowsEmbedded: prepared.rowsEmbedded,
|
|
17846
|
+
...prepared.corpusStats && { corpusStats: prepared.corpusStats }
|
|
17812
17847
|
};
|
|
17813
17848
|
}
|
|
17814
17849
|
const ranked = await rankPreparedVaultCandidates(
|
|
@@ -17823,7 +17858,8 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
17823
17858
|
decryptLast: prepared.decryptLast,
|
|
17824
17859
|
rowsDecrypted: prepared.rowsDecrypted,
|
|
17825
17860
|
queryEmbedMs: prepared.queryEmbedMs,
|
|
17826
|
-
rowsEmbedded: prepared.rowsEmbedded
|
|
17861
|
+
rowsEmbedded: prepared.rowsEmbedded,
|
|
17862
|
+
...prepared.corpusStats && { corpusStats: prepared.corpusStats }
|
|
17827
17863
|
};
|
|
17828
17864
|
}
|
|
17829
17865
|
async function searchVaultMemories(query, vaultCtx, embeddingOptions, cache, searchOptions) {
|
|
@@ -19824,6 +19860,7 @@ async function recall(query, ctx, options = {}) {
|
|
|
19824
19860
|
let factLaneRankedOnCosine = false;
|
|
19825
19861
|
let decryptLastRan;
|
|
19826
19862
|
let vaultRowsDecrypted;
|
|
19863
|
+
let corpusStats;
|
|
19827
19864
|
let graphLaneCount = 0;
|
|
19828
19865
|
let temporalLaneCount = 0;
|
|
19829
19866
|
let graphLaneFailed = false;
|
|
@@ -19864,6 +19901,10 @@ async function recall(query, ctx, options = {}) {
|
|
|
19864
19901
|
...decryptLastRan !== void 0 && { decryptLast: decryptLastRan },
|
|
19865
19902
|
...vaultRowsDecrypted !== void 0 && { vaultRowsDecrypted },
|
|
19866
19903
|
...vaultRowsEmbedded !== void 0 && { vaultRowsEmbedded },
|
|
19904
|
+
...corpusStats && {
|
|
19905
|
+
vaultCacheHits: corpusStats.cacheHits,
|
|
19906
|
+
vaultCacheMisses: corpusStats.cacheMisses
|
|
19907
|
+
},
|
|
19867
19908
|
...chunkStats && {
|
|
19868
19909
|
chunkRowsScanned: chunkStats.rowsScanned,
|
|
19869
19910
|
chunkRowsFetched: chunkStats.rowsFetched,
|
|
@@ -19886,7 +19927,12 @@ async function recall(query, ctx, options = {}) {
|
|
|
19886
19927
|
rerank: rerankMs,
|
|
19887
19928
|
queryEmbed: queryEmbedMs,
|
|
19888
19929
|
chunkLane: chunkLaneMs,
|
|
19889
|
-
fuse: fuseMs
|
|
19930
|
+
fuse: fuseMs,
|
|
19931
|
+
...corpusStats && {
|
|
19932
|
+
corpusKeys: corpusStats.keysMs,
|
|
19933
|
+
corpusEmbeddingsFetch: corpusStats.embeddingsFetchMs,
|
|
19934
|
+
corpusRowsFetch: corpusStats.rowsFetchMs
|
|
19935
|
+
}
|
|
19890
19936
|
},
|
|
19891
19937
|
degraded
|
|
19892
19938
|
};
|
|
@@ -19960,7 +20006,8 @@ async function recall(query, ctx, options = {}) {
|
|
|
19960
20006
|
embeddingsUnavailable: factEmbeddingsUnavailable,
|
|
19961
20007
|
rankedOnCosine: factRankedOnCosine,
|
|
19962
20008
|
decryptLast: factDecryptLast,
|
|
19963
|
-
rowsDecrypted: factRowsDecrypted
|
|
20009
|
+
rowsDecrypted: factRowsDecrypted,
|
|
20010
|
+
corpusStats: factCorpusStats
|
|
19964
20011
|
} = await (ctx.factSource?.search.bind(ctx.factSource) ?? ((searchQuery, searchOptions) => searchVaultMemoriesWithSize(
|
|
19965
20012
|
searchQuery,
|
|
19966
20013
|
ctx.vaultCtx,
|
|
@@ -20026,6 +20073,7 @@ async function recall(query, ctx, options = {}) {
|
|
|
20026
20073
|
factLaneRankedOnCosine = factRankedOnCosine;
|
|
20027
20074
|
decryptLastRan = factDecryptLast;
|
|
20028
20075
|
vaultRowsDecrypted = factRowsDecrypted;
|
|
20076
|
+
corpusStats = factCorpusStats;
|
|
20029
20077
|
factLaneMs = nowMs() - factStart;
|
|
20030
20078
|
}
|
|
20031
20079
|
if (chunkEmbedFailed && !factLaneRankedOnCosine) embeddingsUnavailable = true;
|
|
@@ -29112,7 +29160,7 @@ async function requestICloudSignIn() {
|
|
|
29112
29160
|
}
|
|
29113
29161
|
return container.whenUserSignsIn();
|
|
29114
29162
|
}
|
|
29115
|
-
async function uploadFileToICloud(filename, content) {
|
|
29163
|
+
async function uploadFileToICloud(filename, content, backupRevision) {
|
|
29116
29164
|
const container = await getContainer();
|
|
29117
29165
|
const database = container.privateCloudDatabase;
|
|
29118
29166
|
const recordName = `backup_${filename.replace(/[^a-zA-Z0-9]/g, "_")}`;
|
|
@@ -29125,7 +29173,8 @@ async function uploadFileToICloud(filename, content) {
|
|
|
29125
29173
|
filename: { value: filename },
|
|
29126
29174
|
data: { value: base64Data },
|
|
29127
29175
|
size: { value: content.size },
|
|
29128
|
-
contentType: { value: content.type || "application/json" }
|
|
29176
|
+
contentType: { value: content.type || "application/json" },
|
|
29177
|
+
...backupRevision ? { backupRevision: { value: backupRevision } } : {}
|
|
29129
29178
|
}
|
|
29130
29179
|
};
|
|
29131
29180
|
const response = await database.saveRecords(record2);
|
|
@@ -29137,7 +29186,8 @@ async function uploadFileToICloud(filename, content) {
|
|
|
29137
29186
|
recordName: savedRecord.recordName,
|
|
29138
29187
|
filename,
|
|
29139
29188
|
modifiedAt: new Date(savedRecord.modified?.timestamp ?? Date.now()),
|
|
29140
|
-
size: content.size
|
|
29189
|
+
size: content.size,
|
|
29190
|
+
backupRevision: savedRecord.fields.backupRevision?.value
|
|
29141
29191
|
};
|
|
29142
29192
|
}
|
|
29143
29193
|
async function listICloudFiles() {
|
|
@@ -29158,6 +29208,7 @@ async function listICloudFiles() {
|
|
|
29158
29208
|
recordName: record2.recordName,
|
|
29159
29209
|
filename: record2.fields.filename?.value ?? "",
|
|
29160
29210
|
modifiedAt: new Date(record2.modified?.timestamp ?? Date.now()),
|
|
29211
|
+
backupRevision: record2.fields.backupRevision?.value,
|
|
29161
29212
|
size: typeof record2.fields.data?.value === "object" && record2.fields.data?.value !== null ? record2.fields.data.value.size : 0
|
|
29162
29213
|
}));
|
|
29163
29214
|
return files.sort((a, b) => b.modifiedAt.getTime() - a.modifiedAt.getTime());
|
|
@@ -29211,35 +29262,53 @@ async function findICloudFile(filename) {
|
|
|
29211
29262
|
recordName: record2.recordName,
|
|
29212
29263
|
filename: record2.fields.filename?.value ?? "",
|
|
29213
29264
|
modifiedAt: new Date(record2.modified?.timestamp ?? Date.now()),
|
|
29265
|
+
backupRevision: record2.fields.backupRevision?.value,
|
|
29214
29266
|
size: typeof record2.fields.data?.value === "object" && record2.fields.data?.value !== null ? record2.fields.data.value.size : 0
|
|
29215
29267
|
};
|
|
29216
29268
|
}
|
|
29217
29269
|
|
|
29218
29270
|
// src/lib/backup/icloud/backup.ts
|
|
29219
29271
|
var isAuthError = (err) => err instanceof Error && (err.message.includes("AUTHENTICATION") || err.message.includes("NOT_AUTHENTICATED") || err.message.includes("sign in"));
|
|
29272
|
+
var UNREVISIONED_COLUMNS = /* @__PURE__ */ new Set([
|
|
29273
|
+
"updated_at",
|
|
29274
|
+
"vector",
|
|
29275
|
+
"embedding_model",
|
|
29276
|
+
"chunks",
|
|
29277
|
+
"origin"
|
|
29278
|
+
]);
|
|
29279
|
+
async function getBackupRevision(database, conversationId) {
|
|
29280
|
+
const serialized = await database.read(async () => {
|
|
29281
|
+
const { Q: Q20 } = await import("@nozbe/watermelondb");
|
|
29282
|
+
const tables = await Promise.all(
|
|
29283
|
+
["conversations", "history"].map(
|
|
29284
|
+
(table) => database.get(table).query(Q20.where("conversation_id", conversationId), Q20.sortBy("id", Q20.asc)).unsafeFetchRaw()
|
|
29285
|
+
)
|
|
29286
|
+
);
|
|
29287
|
+
return JSON.stringify(
|
|
29288
|
+
tables.map(
|
|
29289
|
+
(rows) => rows.map(
|
|
29290
|
+
(row) => Object.entries(row).filter(([key]) => !key.startsWith("_") && !UNREVISIONED_COLUMNS.has(key)).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0)
|
|
29291
|
+
)
|
|
29292
|
+
)
|
|
29293
|
+
);
|
|
29294
|
+
});
|
|
29295
|
+
const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(serialized));
|
|
29296
|
+
return "v1:" + Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
29297
|
+
}
|
|
29220
29298
|
async function pushConversationToICloud(database, conversationId, userAddress, deps, _retried = false) {
|
|
29221
29299
|
try {
|
|
29222
29300
|
await deps.requestEncryptionKey(userAddress);
|
|
29223
29301
|
const filename = `${conversationId}.json`;
|
|
29224
29302
|
const existingFile = await findICloudFile(filename);
|
|
29225
|
-
|
|
29226
|
-
|
|
29227
|
-
|
|
29228
|
-
const records = await conversationsCollection.query(Q20.where("conversation_id", conversationId)).fetch();
|
|
29229
|
-
if (records.length > 0) {
|
|
29230
|
-
const conversation = conversationToStoredRaw(records[0]);
|
|
29231
|
-
const localUpdated = conversation.updatedAt.getTime();
|
|
29232
|
-
const remoteModified = existingFile.modifiedAt.getTime();
|
|
29233
|
-
if (localUpdated <= remoteModified) {
|
|
29234
|
-
return "skipped";
|
|
29235
|
-
}
|
|
29236
|
-
}
|
|
29303
|
+
const backupRevision = await getBackupRevision(database, conversationId);
|
|
29304
|
+
if (existingFile?.backupRevision === backupRevision) {
|
|
29305
|
+
return "skipped";
|
|
29237
29306
|
}
|
|
29238
29307
|
const exportResult = await deps.exportConversation(conversationId, userAddress);
|
|
29239
29308
|
if (!exportResult.success || !exportResult.blob) {
|
|
29240
29309
|
return "failed";
|
|
29241
29310
|
}
|
|
29242
|
-
await uploadFileToICloud(filename, exportResult.blob);
|
|
29311
|
+
await uploadFileToICloud(filename, exportResult.blob, backupRevision);
|
|
29243
29312
|
return "uploaded";
|
|
29244
29313
|
} catch (err) {
|
|
29245
29314
|
if (isAuthError(err) && !_retried) {
|
package/dist/server/index.cjs
CHANGED
|
@@ -31957,10 +31957,18 @@ function admitVaultProjections(queryEmbedding, vectored, k) {
|
|
|
31957
31957
|
}
|
|
31958
31958
|
async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, queryOpts, opts) {
|
|
31959
31959
|
let rowsDecrypted = 0;
|
|
31960
|
+
const keysStart = nowMs();
|
|
31960
31961
|
const keys = await getVaultCandidateKeysOp(
|
|
31961
31962
|
vaultCtx,
|
|
31962
31963
|
Object.keys(queryOpts).length > 0 ? queryOpts : void 0
|
|
31963
31964
|
);
|
|
31965
|
+
const corpusStats = {
|
|
31966
|
+
keysMs: nowMs() - keysStart,
|
|
31967
|
+
embeddingsFetchMs: 0,
|
|
31968
|
+
rowsFetchMs: 0,
|
|
31969
|
+
cacheHits: 0,
|
|
31970
|
+
cacheMisses: 0
|
|
31971
|
+
};
|
|
31964
31972
|
const hydrateOpts = queryOpts.includeArchived ? { includeArchived: true } : void 0;
|
|
31965
31973
|
const vaultSize = keys.length;
|
|
31966
31974
|
if (keys.length === 0) {
|
|
@@ -31970,7 +31978,8 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
31970
31978
|
queryEmbedding: [],
|
|
31971
31979
|
vaultSize: 0,
|
|
31972
31980
|
laneEmbedFailed: false,
|
|
31973
|
-
rowsDecrypted: 0
|
|
31981
|
+
rowsDecrypted: 0,
|
|
31982
|
+
corpusStats
|
|
31974
31983
|
};
|
|
31975
31984
|
}
|
|
31976
31985
|
const queryEmbedStart = nowMs();
|
|
@@ -31992,16 +32001,20 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
31992
32001
|
for (const k2 of keys) {
|
|
31993
32002
|
const cached = cachedRowVector(cache, k2.uniqueId, k2.updatedAt);
|
|
31994
32003
|
if (cached && cached.length === queryEmbedding.length) {
|
|
32004
|
+
corpusStats.cacheHits++;
|
|
31995
32005
|
vectored.push({ uniqueId: k2.uniqueId, embedding: cached, updatedAt: k2.updatedAt });
|
|
31996
32006
|
continue;
|
|
31997
32007
|
}
|
|
32008
|
+
corpusStats.cacheMisses++;
|
|
31998
32009
|
const modelCompatible = (k2.embeddingModel ?? currentModel) === currentModel;
|
|
31999
32010
|
if (modelCompatible) missIds.push(k2.uniqueId);
|
|
32000
32011
|
else noVectorIds.push(k2.uniqueId);
|
|
32001
32012
|
}
|
|
32002
32013
|
}
|
|
32003
32014
|
if (missIds.length > 0) {
|
|
32015
|
+
const embeddingsStart = nowMs();
|
|
32004
32016
|
const rows = await getVaultEmbeddingsByIdsOp(vaultCtx, missIds, hydrateOpts);
|
|
32017
|
+
corpusStats.embeddingsFetchMs += nowMs() - embeddingsStart;
|
|
32005
32018
|
const gotVector = /* @__PURE__ */ new Set();
|
|
32006
32019
|
for (const r of rows) {
|
|
32007
32020
|
if (!r.embedding) continue;
|
|
@@ -32029,7 +32042,9 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
32029
32042
|
`memoryVault: projected search un-embedded lane capped at ${laneIds.length}/${noVectorIds.length}`
|
|
32030
32043
|
);
|
|
32031
32044
|
}
|
|
32045
|
+
const laneStart = nowMs();
|
|
32032
32046
|
const laneFetched = await getVaultMemoriesByIdsOp(vaultCtx, laneIds, hydrateOpts);
|
|
32047
|
+
corpusStats.rowsFetchMs += nowMs() - laneStart;
|
|
32033
32048
|
rowsDecrypted += laneFetched.length;
|
|
32034
32049
|
for (const row of laneFetched) laneById.set(row.uniqueId, row);
|
|
32035
32050
|
const laneRows = laneFetched.filter((m) => !isEncrypted(m.content));
|
|
@@ -32081,7 +32096,9 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
32081
32096
|
}
|
|
32082
32097
|
}
|
|
32083
32098
|
const idsToFetch = [...admissionSet].filter((id) => !laneById.has(id));
|
|
32099
|
+
const admittedStart = nowMs();
|
|
32084
32100
|
const fetched = idsToFetch.length > 0 ? await getVaultMemoriesByIdsOp(vaultCtx, idsToFetch, hydrateOpts) : [];
|
|
32101
|
+
corpusStats.rowsFetchMs += nowMs() - admittedStart;
|
|
32085
32102
|
rowsDecrypted += fetched.length;
|
|
32086
32103
|
const admittedRows = [
|
|
32087
32104
|
...fetched,
|
|
@@ -32110,7 +32127,10 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
32110
32127
|
}
|
|
32111
32128
|
if (staleIds.length > 0) {
|
|
32112
32129
|
const contentById = new Map(memories.map((m) => [m.uniqueId, m]));
|
|
32113
|
-
|
|
32130
|
+
const staleStart = nowMs();
|
|
32131
|
+
const staleRows = await getVaultEmbeddingsByIdsOp(vaultCtx, staleIds, hydrateOpts);
|
|
32132
|
+
corpusStats.embeddingsFetchMs += nowMs() - staleStart;
|
|
32133
|
+
for (const r of staleRows) {
|
|
32114
32134
|
if (!r.embedding) continue;
|
|
32115
32135
|
try {
|
|
32116
32136
|
const parsed = JSON.parse(r.embedding);
|
|
@@ -32142,7 +32162,15 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
32142
32162
|
eventTimeKind: normalizeEventTimeKind(m.eventTimeKind),
|
|
32143
32163
|
factType: m.factType
|
|
32144
32164
|
}));
|
|
32145
|
-
return {
|
|
32165
|
+
return {
|
|
32166
|
+
memories,
|
|
32167
|
+
embeddedItems,
|
|
32168
|
+
queryEmbedding,
|
|
32169
|
+
vaultSize,
|
|
32170
|
+
laneEmbedFailed,
|
|
32171
|
+
rowsDecrypted,
|
|
32172
|
+
corpusStats
|
|
32173
|
+
};
|
|
32146
32174
|
}
|
|
32147
32175
|
async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache, searchOptions) {
|
|
32148
32176
|
if (!query || typeof query !== "string") {
|
|
@@ -32183,6 +32211,7 @@ async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache,
|
|
|
32183
32211
|
embeddingFailure = true;
|
|
32184
32212
|
};
|
|
32185
32213
|
const embedStats = { queryMs: 0, rowsEmbedded: 0 };
|
|
32214
|
+
let corpusStats;
|
|
32186
32215
|
if (searchOptions?.decryptLast) {
|
|
32187
32216
|
const forceIncludeIds = [
|
|
32188
32217
|
...searchOptions.entityRanking ?? [],
|
|
@@ -32203,6 +32232,7 @@ async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache,
|
|
|
32203
32232
|
onEmbeddingDegraded,
|
|
32204
32233
|
embedStats
|
|
32205
32234
|
});
|
|
32235
|
+
corpusStats = corpus.corpusStats;
|
|
32206
32236
|
if (corpus.vaultSize === 0) {
|
|
32207
32237
|
return {
|
|
32208
32238
|
memories: [],
|
|
@@ -32214,7 +32244,8 @@ async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache,
|
|
|
32214
32244
|
decryptLast: true,
|
|
32215
32245
|
rowsDecrypted: corpus.rowsDecrypted,
|
|
32216
32246
|
queryEmbedMs: embedStats.queryMs,
|
|
32217
|
-
rowsEmbedded: embedStats.rowsEmbedded
|
|
32247
|
+
rowsEmbedded: embedStats.rowsEmbedded,
|
|
32248
|
+
corpusStats
|
|
32218
32249
|
};
|
|
32219
32250
|
}
|
|
32220
32251
|
({ memories, embeddedItems, queryEmbedding, vaultSize } = corpus);
|
|
@@ -32231,7 +32262,8 @@ async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache,
|
|
|
32231
32262
|
decryptLast: true,
|
|
32232
32263
|
rowsDecrypted: corpus.rowsDecrypted,
|
|
32233
32264
|
queryEmbedMs: embedStats.queryMs,
|
|
32234
|
-
rowsEmbedded: embedStats.rowsEmbedded
|
|
32265
|
+
rowsEmbedded: embedStats.rowsEmbedded,
|
|
32266
|
+
corpusStats
|
|
32235
32267
|
};
|
|
32236
32268
|
}
|
|
32237
32269
|
} else {
|
|
@@ -32364,7 +32396,8 @@ async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache,
|
|
|
32364
32396
|
embeddingsUnavailable,
|
|
32365
32397
|
embeddingFailure,
|
|
32366
32398
|
decryptLast: !!searchOptions?.decryptLast,
|
|
32367
|
-
rowsDecrypted
|
|
32399
|
+
rowsDecrypted,
|
|
32400
|
+
...corpusStats && { corpusStats }
|
|
32368
32401
|
};
|
|
32369
32402
|
}
|
|
32370
32403
|
async function rankPreparedVaultCandidates(query, prepared, embeddingOptions, searchOptions) {
|
|
@@ -32581,7 +32614,8 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
32581
32614
|
decryptLast: prepared.decryptLast,
|
|
32582
32615
|
rowsDecrypted: prepared.rowsDecrypted,
|
|
32583
32616
|
queryEmbedMs: prepared.queryEmbedMs,
|
|
32584
|
-
rowsEmbedded: prepared.rowsEmbedded
|
|
32617
|
+
rowsEmbedded: prepared.rowsEmbedded,
|
|
32618
|
+
...prepared.corpusStats && { corpusStats: prepared.corpusStats }
|
|
32585
32619
|
};
|
|
32586
32620
|
}
|
|
32587
32621
|
if (prepared.memories.length === 0) {
|
|
@@ -32596,7 +32630,8 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
32596
32630
|
decryptLast: prepared.decryptLast,
|
|
32597
32631
|
rowsDecrypted: prepared.rowsDecrypted,
|
|
32598
32632
|
queryEmbedMs: prepared.queryEmbedMs,
|
|
32599
|
-
rowsEmbedded: prepared.rowsEmbedded
|
|
32633
|
+
rowsEmbedded: prepared.rowsEmbedded,
|
|
32634
|
+
...prepared.corpusStats && { corpusStats: prepared.corpusStats }
|
|
32600
32635
|
};
|
|
32601
32636
|
}
|
|
32602
32637
|
const ranked = await rankPreparedVaultCandidates(
|
|
@@ -32611,7 +32646,8 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
32611
32646
|
decryptLast: prepared.decryptLast,
|
|
32612
32647
|
rowsDecrypted: prepared.rowsDecrypted,
|
|
32613
32648
|
queryEmbedMs: prepared.queryEmbedMs,
|
|
32614
|
-
rowsEmbedded: prepared.rowsEmbedded
|
|
32649
|
+
rowsEmbedded: prepared.rowsEmbedded,
|
|
32650
|
+
...prepared.corpusStats && { corpusStats: prepared.corpusStats }
|
|
32615
32651
|
};
|
|
32616
32652
|
}
|
|
32617
32653
|
async function searchVaultMemories(query, vaultCtx, embeddingOptions, cache, searchOptions) {
|
|
@@ -34626,6 +34662,7 @@ async function recall(query, ctx, options = {}) {
|
|
|
34626
34662
|
let factLaneRankedOnCosine = false;
|
|
34627
34663
|
let decryptLastRan;
|
|
34628
34664
|
let vaultRowsDecrypted;
|
|
34665
|
+
let corpusStats;
|
|
34629
34666
|
let graphLaneCount = 0;
|
|
34630
34667
|
let temporalLaneCount = 0;
|
|
34631
34668
|
let graphLaneFailed = false;
|
|
@@ -34666,6 +34703,10 @@ async function recall(query, ctx, options = {}) {
|
|
|
34666
34703
|
...decryptLastRan !== void 0 && { decryptLast: decryptLastRan },
|
|
34667
34704
|
...vaultRowsDecrypted !== void 0 && { vaultRowsDecrypted },
|
|
34668
34705
|
...vaultRowsEmbedded !== void 0 && { vaultRowsEmbedded },
|
|
34706
|
+
...corpusStats && {
|
|
34707
|
+
vaultCacheHits: corpusStats.cacheHits,
|
|
34708
|
+
vaultCacheMisses: corpusStats.cacheMisses
|
|
34709
|
+
},
|
|
34669
34710
|
...chunkStats && {
|
|
34670
34711
|
chunkRowsScanned: chunkStats.rowsScanned,
|
|
34671
34712
|
chunkRowsFetched: chunkStats.rowsFetched,
|
|
@@ -34688,7 +34729,12 @@ async function recall(query, ctx, options = {}) {
|
|
|
34688
34729
|
rerank: rerankMs,
|
|
34689
34730
|
queryEmbed: queryEmbedMs,
|
|
34690
34731
|
chunkLane: chunkLaneMs,
|
|
34691
|
-
fuse: fuseMs
|
|
34732
|
+
fuse: fuseMs,
|
|
34733
|
+
...corpusStats && {
|
|
34734
|
+
corpusKeys: corpusStats.keysMs,
|
|
34735
|
+
corpusEmbeddingsFetch: corpusStats.embeddingsFetchMs,
|
|
34736
|
+
corpusRowsFetch: corpusStats.rowsFetchMs
|
|
34737
|
+
}
|
|
34692
34738
|
},
|
|
34693
34739
|
degraded
|
|
34694
34740
|
};
|
|
@@ -34762,7 +34808,8 @@ async function recall(query, ctx, options = {}) {
|
|
|
34762
34808
|
embeddingsUnavailable: factEmbeddingsUnavailable,
|
|
34763
34809
|
rankedOnCosine: factRankedOnCosine,
|
|
34764
34810
|
decryptLast: factDecryptLast,
|
|
34765
|
-
rowsDecrypted: factRowsDecrypted
|
|
34811
|
+
rowsDecrypted: factRowsDecrypted,
|
|
34812
|
+
corpusStats: factCorpusStats
|
|
34766
34813
|
} = await (ctx.factSource?.search.bind(ctx.factSource) ?? ((searchQuery, searchOptions) => searchVaultMemoriesWithSize(
|
|
34767
34814
|
searchQuery,
|
|
34768
34815
|
ctx.vaultCtx,
|
|
@@ -34828,6 +34875,7 @@ async function recall(query, ctx, options = {}) {
|
|
|
34828
34875
|
factLaneRankedOnCosine = factRankedOnCosine;
|
|
34829
34876
|
decryptLastRan = factDecryptLast;
|
|
34830
34877
|
vaultRowsDecrypted = factRowsDecrypted;
|
|
34878
|
+
corpusStats = factCorpusStats;
|
|
34831
34879
|
factLaneMs = nowMs() - factStart;
|
|
34832
34880
|
}
|
|
34833
34881
|
if (chunkEmbedFailed && !factLaneRankedOnCosine) embeddingsUnavailable = true;
|
package/dist/server/index.d.mts
CHANGED
|
@@ -3538,6 +3538,13 @@ declare function eagerEmbedContent(content: string, embeddingOptions: EmbeddingO
|
|
|
3538
3538
|
* trusting it (see `vectorVersion.ts`).
|
|
3539
3539
|
*/
|
|
3540
3540
|
updatedAt?: Date): Promise<void>;
|
|
3541
|
+
interface CorpusStats {
|
|
3542
|
+
keysMs: number;
|
|
3543
|
+
embeddingsFetchMs: number;
|
|
3544
|
+
rowsFetchMs: number;
|
|
3545
|
+
cacheHits: number;
|
|
3546
|
+
cacheMisses: number;
|
|
3547
|
+
}
|
|
3541
3548
|
/**
|
|
3542
3549
|
* A single vault search result with its similarity score.
|
|
3543
3550
|
*/
|
|
@@ -3616,6 +3623,7 @@ declare function searchVaultMemoriesWithSize(query: string, vaultCtx: VaultMemor
|
|
|
3616
3623
|
*/
|
|
3617
3624
|
queryEmbedMs: number;
|
|
3618
3625
|
rowsEmbedded: number;
|
|
3626
|
+
corpusStats?: CorpusStats;
|
|
3619
3627
|
}>;
|
|
3620
3628
|
/**
|
|
3621
3629
|
* Search vault memories by semantic similarity. Returns structured results
|
|
@@ -4335,6 +4343,8 @@ interface RecallDiagnostics {
|
|
|
4335
4343
|
* LEGACY read path this batch is uncapped, so it can be the whole vault.
|
|
4336
4344
|
*/
|
|
4337
4345
|
vaultRowsEmbedded?: number;
|
|
4346
|
+
vaultCacheHits?: number;
|
|
4347
|
+
vaultCacheMisses?: number;
|
|
4338
4348
|
/** Embedded messages the chunk lane considered; absent when the lane did not run. */
|
|
4339
4349
|
chunkRowsScanned?: number;
|
|
4340
4350
|
/** Messages whose vectors the chunk lane read from storage on a cache miss. */
|
|
@@ -4437,6 +4447,9 @@ interface RecallDiagnostics {
|
|
|
4437
4447
|
chunkLane: number;
|
|
4438
4448
|
/** Cross-lane RRF fusion + provenance dedup after both lanes. */
|
|
4439
4449
|
fuse: number;
|
|
4450
|
+
corpusKeys?: number;
|
|
4451
|
+
corpusEmbeddingsFetch?: number;
|
|
4452
|
+
corpusRowsFetch?: number;
|
|
4440
4453
|
};
|
|
4441
4454
|
/** Soft-degradation signals that fired this call (empty when clean). */
|
|
4442
4455
|
degraded: RecallDegradation[];
|
package/dist/server/index.d.ts
CHANGED
|
@@ -3538,6 +3538,13 @@ declare function eagerEmbedContent(content: string, embeddingOptions: EmbeddingO
|
|
|
3538
3538
|
* trusting it (see `vectorVersion.ts`).
|
|
3539
3539
|
*/
|
|
3540
3540
|
updatedAt?: Date): Promise<void>;
|
|
3541
|
+
interface CorpusStats {
|
|
3542
|
+
keysMs: number;
|
|
3543
|
+
embeddingsFetchMs: number;
|
|
3544
|
+
rowsFetchMs: number;
|
|
3545
|
+
cacheHits: number;
|
|
3546
|
+
cacheMisses: number;
|
|
3547
|
+
}
|
|
3541
3548
|
/**
|
|
3542
3549
|
* A single vault search result with its similarity score.
|
|
3543
3550
|
*/
|
|
@@ -3616,6 +3623,7 @@ declare function searchVaultMemoriesWithSize(query: string, vaultCtx: VaultMemor
|
|
|
3616
3623
|
*/
|
|
3617
3624
|
queryEmbedMs: number;
|
|
3618
3625
|
rowsEmbedded: number;
|
|
3626
|
+
corpusStats?: CorpusStats;
|
|
3619
3627
|
}>;
|
|
3620
3628
|
/**
|
|
3621
3629
|
* Search vault memories by semantic similarity. Returns structured results
|
|
@@ -4335,6 +4343,8 @@ interface RecallDiagnostics {
|
|
|
4335
4343
|
* LEGACY read path this batch is uncapped, so it can be the whole vault.
|
|
4336
4344
|
*/
|
|
4337
4345
|
vaultRowsEmbedded?: number;
|
|
4346
|
+
vaultCacheHits?: number;
|
|
4347
|
+
vaultCacheMisses?: number;
|
|
4338
4348
|
/** Embedded messages the chunk lane considered; absent when the lane did not run. */
|
|
4339
4349
|
chunkRowsScanned?: number;
|
|
4340
4350
|
/** Messages whose vectors the chunk lane read from storage on a cache miss. */
|
|
@@ -4437,6 +4447,9 @@ interface RecallDiagnostics {
|
|
|
4437
4447
|
chunkLane: number;
|
|
4438
4448
|
/** Cross-lane RRF fusion + provenance dedup after both lanes. */
|
|
4439
4449
|
fuse: number;
|
|
4450
|
+
corpusKeys?: number;
|
|
4451
|
+
corpusEmbeddingsFetch?: number;
|
|
4452
|
+
corpusRowsFetch?: number;
|
|
4440
4453
|
};
|
|
4441
4454
|
/** Soft-degradation signals that fired this call (empty when clean). */
|
|
4442
4455
|
degraded: RecallDegradation[];
|