@anuma/sdk 7.1.1 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/expo/index.cjs +59 -11
- package/dist/expo/index.d.mts +13 -0
- package/dist/expo/index.d.ts +13 -0
- package/dist/expo/index.mjs +59 -11
- package/dist/memory/context.d.mts +5 -0
- package/dist/memory/context.d.ts +5 -0
- package/dist/react/index.cjs +59 -11
- package/dist/react/index.d.mts +13 -0
- package/dist/react/index.d.ts +13 -0
- package/dist/react/index.mjs +59 -11
- package/dist/server/index.cjs +59 -11
- package/dist/server/index.d.mts +13 -0
- package/dist/server/index.d.ts +13 -0
- package/dist/server/index.mjs +59 -11
- package/dist/telemetry/index.d.mts +5 -0
- package/dist/telemetry/index.d.ts +5 -0
- package/package.json +1 -1
package/dist/react/index.cjs
CHANGED
|
@@ -18992,10 +18992,18 @@ function admitVaultProjections(queryEmbedding, vectored, k) {
|
|
|
18992
18992
|
}
|
|
18993
18993
|
async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, queryOpts, opts) {
|
|
18994
18994
|
let rowsDecrypted = 0;
|
|
18995
|
+
const keysStart = nowMs();
|
|
18995
18996
|
const keys = await getVaultCandidateKeysOp(
|
|
18996
18997
|
vaultCtx,
|
|
18997
18998
|
Object.keys(queryOpts).length > 0 ? queryOpts : void 0
|
|
18998
18999
|
);
|
|
19000
|
+
const corpusStats = {
|
|
19001
|
+
keysMs: nowMs() - keysStart,
|
|
19002
|
+
embeddingsFetchMs: 0,
|
|
19003
|
+
rowsFetchMs: 0,
|
|
19004
|
+
cacheHits: 0,
|
|
19005
|
+
cacheMisses: 0
|
|
19006
|
+
};
|
|
18999
19007
|
const hydrateOpts = queryOpts.includeArchived ? { includeArchived: true } : void 0;
|
|
19000
19008
|
const vaultSize = keys.length;
|
|
19001
19009
|
if (keys.length === 0) {
|
|
@@ -19005,7 +19013,8 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
19005
19013
|
queryEmbedding: [],
|
|
19006
19014
|
vaultSize: 0,
|
|
19007
19015
|
laneEmbedFailed: false,
|
|
19008
|
-
rowsDecrypted: 0
|
|
19016
|
+
rowsDecrypted: 0,
|
|
19017
|
+
corpusStats
|
|
19009
19018
|
};
|
|
19010
19019
|
}
|
|
19011
19020
|
const queryEmbedStart = nowMs();
|
|
@@ -19027,16 +19036,20 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
19027
19036
|
for (const k2 of keys) {
|
|
19028
19037
|
const cached = cachedRowVector(cache, k2.uniqueId, k2.updatedAt);
|
|
19029
19038
|
if (cached && cached.length === queryEmbedding.length) {
|
|
19039
|
+
corpusStats.cacheHits++;
|
|
19030
19040
|
vectored.push({ uniqueId: k2.uniqueId, embedding: cached, updatedAt: k2.updatedAt });
|
|
19031
19041
|
continue;
|
|
19032
19042
|
}
|
|
19043
|
+
corpusStats.cacheMisses++;
|
|
19033
19044
|
const modelCompatible = (k2.embeddingModel ?? currentModel) === currentModel;
|
|
19034
19045
|
if (modelCompatible) missIds.push(k2.uniqueId);
|
|
19035
19046
|
else noVectorIds.push(k2.uniqueId);
|
|
19036
19047
|
}
|
|
19037
19048
|
}
|
|
19038
19049
|
if (missIds.length > 0) {
|
|
19050
|
+
const embeddingsStart = nowMs();
|
|
19039
19051
|
const rows = await getVaultEmbeddingsByIdsOp(vaultCtx, missIds, hydrateOpts);
|
|
19052
|
+
corpusStats.embeddingsFetchMs += nowMs() - embeddingsStart;
|
|
19040
19053
|
const gotVector = /* @__PURE__ */ new Set();
|
|
19041
19054
|
for (const r of rows) {
|
|
19042
19055
|
if (!r.embedding) continue;
|
|
@@ -19064,7 +19077,9 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
19064
19077
|
`memoryVault: projected search un-embedded lane capped at ${laneIds.length}/${noVectorIds.length}`
|
|
19065
19078
|
);
|
|
19066
19079
|
}
|
|
19080
|
+
const laneStart = nowMs();
|
|
19067
19081
|
const laneFetched = await getVaultMemoriesByIdsOp(vaultCtx, laneIds, hydrateOpts);
|
|
19082
|
+
corpusStats.rowsFetchMs += nowMs() - laneStart;
|
|
19068
19083
|
rowsDecrypted += laneFetched.length;
|
|
19069
19084
|
for (const row of laneFetched) laneById.set(row.uniqueId, row);
|
|
19070
19085
|
const laneRows = laneFetched.filter((m) => !isEncrypted(m.content));
|
|
@@ -19116,7 +19131,9 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
19116
19131
|
}
|
|
19117
19132
|
}
|
|
19118
19133
|
const idsToFetch = [...admissionSet].filter((id) => !laneById.has(id));
|
|
19134
|
+
const admittedStart = nowMs();
|
|
19119
19135
|
const fetched = idsToFetch.length > 0 ? await getVaultMemoriesByIdsOp(vaultCtx, idsToFetch, hydrateOpts) : [];
|
|
19136
|
+
corpusStats.rowsFetchMs += nowMs() - admittedStart;
|
|
19120
19137
|
rowsDecrypted += fetched.length;
|
|
19121
19138
|
const admittedRows = [
|
|
19122
19139
|
...fetched,
|
|
@@ -19145,7 +19162,10 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
19145
19162
|
}
|
|
19146
19163
|
if (staleIds.length > 0) {
|
|
19147
19164
|
const contentById = new Map(memories.map((m) => [m.uniqueId, m]));
|
|
19148
|
-
|
|
19165
|
+
const staleStart = nowMs();
|
|
19166
|
+
const staleRows = await getVaultEmbeddingsByIdsOp(vaultCtx, staleIds, hydrateOpts);
|
|
19167
|
+
corpusStats.embeddingsFetchMs += nowMs() - staleStart;
|
|
19168
|
+
for (const r of staleRows) {
|
|
19149
19169
|
if (!r.embedding) continue;
|
|
19150
19170
|
try {
|
|
19151
19171
|
const parsed = JSON.parse(r.embedding);
|
|
@@ -19177,7 +19197,15 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
19177
19197
|
eventTimeKind: normalizeEventTimeKind(m.eventTimeKind),
|
|
19178
19198
|
factType: m.factType
|
|
19179
19199
|
}));
|
|
19180
|
-
return {
|
|
19200
|
+
return {
|
|
19201
|
+
memories,
|
|
19202
|
+
embeddedItems,
|
|
19203
|
+
queryEmbedding,
|
|
19204
|
+
vaultSize,
|
|
19205
|
+
laneEmbedFailed,
|
|
19206
|
+
rowsDecrypted,
|
|
19207
|
+
corpusStats
|
|
19208
|
+
};
|
|
19181
19209
|
}
|
|
19182
19210
|
async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache, searchOptions) {
|
|
19183
19211
|
if (!query || typeof query !== "string") {
|
|
@@ -19218,6 +19246,7 @@ async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache,
|
|
|
19218
19246
|
embeddingFailure = true;
|
|
19219
19247
|
};
|
|
19220
19248
|
const embedStats = { queryMs: 0, rowsEmbedded: 0 };
|
|
19249
|
+
let corpusStats;
|
|
19221
19250
|
if (searchOptions?.decryptLast) {
|
|
19222
19251
|
const forceIncludeIds = [
|
|
19223
19252
|
...searchOptions.entityRanking ?? [],
|
|
@@ -19238,6 +19267,7 @@ async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache,
|
|
|
19238
19267
|
onEmbeddingDegraded,
|
|
19239
19268
|
embedStats
|
|
19240
19269
|
});
|
|
19270
|
+
corpusStats = corpus.corpusStats;
|
|
19241
19271
|
if (corpus.vaultSize === 0) {
|
|
19242
19272
|
return {
|
|
19243
19273
|
memories: [],
|
|
@@ -19249,7 +19279,8 @@ async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache,
|
|
|
19249
19279
|
decryptLast: true,
|
|
19250
19280
|
rowsDecrypted: corpus.rowsDecrypted,
|
|
19251
19281
|
queryEmbedMs: embedStats.queryMs,
|
|
19252
|
-
rowsEmbedded: embedStats.rowsEmbedded
|
|
19282
|
+
rowsEmbedded: embedStats.rowsEmbedded,
|
|
19283
|
+
corpusStats
|
|
19253
19284
|
};
|
|
19254
19285
|
}
|
|
19255
19286
|
({ memories, embeddedItems, queryEmbedding, vaultSize } = corpus);
|
|
@@ -19266,7 +19297,8 @@ async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache,
|
|
|
19266
19297
|
decryptLast: true,
|
|
19267
19298
|
rowsDecrypted: corpus.rowsDecrypted,
|
|
19268
19299
|
queryEmbedMs: embedStats.queryMs,
|
|
19269
|
-
rowsEmbedded: embedStats.rowsEmbedded
|
|
19300
|
+
rowsEmbedded: embedStats.rowsEmbedded,
|
|
19301
|
+
corpusStats
|
|
19270
19302
|
};
|
|
19271
19303
|
}
|
|
19272
19304
|
} else {
|
|
@@ -19399,7 +19431,8 @@ async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache,
|
|
|
19399
19431
|
embeddingsUnavailable,
|
|
19400
19432
|
embeddingFailure,
|
|
19401
19433
|
decryptLast: !!searchOptions?.decryptLast,
|
|
19402
|
-
rowsDecrypted
|
|
19434
|
+
rowsDecrypted,
|
|
19435
|
+
...corpusStats && { corpusStats }
|
|
19403
19436
|
};
|
|
19404
19437
|
}
|
|
19405
19438
|
async function rankPreparedVaultCandidates(query, prepared, embeddingOptions, searchOptions) {
|
|
@@ -19616,7 +19649,8 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
19616
19649
|
decryptLast: prepared.decryptLast,
|
|
19617
19650
|
rowsDecrypted: prepared.rowsDecrypted,
|
|
19618
19651
|
queryEmbedMs: prepared.queryEmbedMs,
|
|
19619
|
-
rowsEmbedded: prepared.rowsEmbedded
|
|
19652
|
+
rowsEmbedded: prepared.rowsEmbedded,
|
|
19653
|
+
...prepared.corpusStats && { corpusStats: prepared.corpusStats }
|
|
19620
19654
|
};
|
|
19621
19655
|
}
|
|
19622
19656
|
if (prepared.memories.length === 0) {
|
|
@@ -19631,7 +19665,8 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
19631
19665
|
decryptLast: prepared.decryptLast,
|
|
19632
19666
|
rowsDecrypted: prepared.rowsDecrypted,
|
|
19633
19667
|
queryEmbedMs: prepared.queryEmbedMs,
|
|
19634
|
-
rowsEmbedded: prepared.rowsEmbedded
|
|
19668
|
+
rowsEmbedded: prepared.rowsEmbedded,
|
|
19669
|
+
...prepared.corpusStats && { corpusStats: prepared.corpusStats }
|
|
19635
19670
|
};
|
|
19636
19671
|
}
|
|
19637
19672
|
const ranked = await rankPreparedVaultCandidates(
|
|
@@ -19646,7 +19681,8 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
19646
19681
|
decryptLast: prepared.decryptLast,
|
|
19647
19682
|
rowsDecrypted: prepared.rowsDecrypted,
|
|
19648
19683
|
queryEmbedMs: prepared.queryEmbedMs,
|
|
19649
|
-
rowsEmbedded: prepared.rowsEmbedded
|
|
19684
|
+
rowsEmbedded: prepared.rowsEmbedded,
|
|
19685
|
+
...prepared.corpusStats && { corpusStats: prepared.corpusStats }
|
|
19650
19686
|
};
|
|
19651
19687
|
}
|
|
19652
19688
|
async function searchVaultMemories(query, vaultCtx, embeddingOptions, cache, searchOptions) {
|
|
@@ -21661,6 +21697,7 @@ async function recall(query, ctx, options = {}) {
|
|
|
21661
21697
|
let factLaneRankedOnCosine = false;
|
|
21662
21698
|
let decryptLastRan;
|
|
21663
21699
|
let vaultRowsDecrypted;
|
|
21700
|
+
let corpusStats;
|
|
21664
21701
|
let graphLaneCount = 0;
|
|
21665
21702
|
let temporalLaneCount = 0;
|
|
21666
21703
|
let graphLaneFailed = false;
|
|
@@ -21701,6 +21738,10 @@ async function recall(query, ctx, options = {}) {
|
|
|
21701
21738
|
...decryptLastRan !== void 0 && { decryptLast: decryptLastRan },
|
|
21702
21739
|
...vaultRowsDecrypted !== void 0 && { vaultRowsDecrypted },
|
|
21703
21740
|
...vaultRowsEmbedded !== void 0 && { vaultRowsEmbedded },
|
|
21741
|
+
...corpusStats && {
|
|
21742
|
+
vaultCacheHits: corpusStats.cacheHits,
|
|
21743
|
+
vaultCacheMisses: corpusStats.cacheMisses
|
|
21744
|
+
},
|
|
21704
21745
|
...chunkStats && {
|
|
21705
21746
|
chunkRowsScanned: chunkStats.rowsScanned,
|
|
21706
21747
|
chunkRowsFetched: chunkStats.rowsFetched,
|
|
@@ -21723,7 +21764,12 @@ async function recall(query, ctx, options = {}) {
|
|
|
21723
21764
|
rerank: rerankMs,
|
|
21724
21765
|
queryEmbed: queryEmbedMs,
|
|
21725
21766
|
chunkLane: chunkLaneMs,
|
|
21726
|
-
fuse: fuseMs
|
|
21767
|
+
fuse: fuseMs,
|
|
21768
|
+
...corpusStats && {
|
|
21769
|
+
corpusKeys: corpusStats.keysMs,
|
|
21770
|
+
corpusEmbeddingsFetch: corpusStats.embeddingsFetchMs,
|
|
21771
|
+
corpusRowsFetch: corpusStats.rowsFetchMs
|
|
21772
|
+
}
|
|
21727
21773
|
},
|
|
21728
21774
|
degraded
|
|
21729
21775
|
};
|
|
@@ -21797,7 +21843,8 @@ async function recall(query, ctx, options = {}) {
|
|
|
21797
21843
|
embeddingsUnavailable: factEmbeddingsUnavailable,
|
|
21798
21844
|
rankedOnCosine: factRankedOnCosine,
|
|
21799
21845
|
decryptLast: factDecryptLast,
|
|
21800
|
-
rowsDecrypted: factRowsDecrypted
|
|
21846
|
+
rowsDecrypted: factRowsDecrypted,
|
|
21847
|
+
corpusStats: factCorpusStats
|
|
21801
21848
|
} = await (ctx.factSource?.search.bind(ctx.factSource) ?? ((searchQuery, searchOptions) => searchVaultMemoriesWithSize(
|
|
21802
21849
|
searchQuery,
|
|
21803
21850
|
ctx.vaultCtx,
|
|
@@ -21863,6 +21910,7 @@ async function recall(query, ctx, options = {}) {
|
|
|
21863
21910
|
factLaneRankedOnCosine = factRankedOnCosine;
|
|
21864
21911
|
decryptLastRan = factDecryptLast;
|
|
21865
21912
|
vaultRowsDecrypted = factRowsDecrypted;
|
|
21913
|
+
corpusStats = factCorpusStats;
|
|
21866
21914
|
factLaneMs = nowMs() - factStart;
|
|
21867
21915
|
}
|
|
21868
21916
|
if (chunkEmbedFailed && !factLaneRankedOnCosine) embeddingsUnavailable = true;
|
package/dist/react/index.d.mts
CHANGED
|
@@ -4511,6 +4511,13 @@ declare function eagerEmbedContent(content: string, embeddingOptions: EmbeddingO
|
|
|
4511
4511
|
* trusting it (see `vectorVersion.ts`).
|
|
4512
4512
|
*/
|
|
4513
4513
|
updatedAt?: Date): Promise<void>;
|
|
4514
|
+
interface CorpusStats {
|
|
4515
|
+
keysMs: number;
|
|
4516
|
+
embeddingsFetchMs: number;
|
|
4517
|
+
rowsFetchMs: number;
|
|
4518
|
+
cacheHits: number;
|
|
4519
|
+
cacheMisses: number;
|
|
4520
|
+
}
|
|
4514
4521
|
/**
|
|
4515
4522
|
* A single vault search result with its similarity score.
|
|
4516
4523
|
*/
|
|
@@ -4589,6 +4596,7 @@ declare function searchVaultMemoriesWithSize(query: string, vaultCtx: VaultMemor
|
|
|
4589
4596
|
*/
|
|
4590
4597
|
queryEmbedMs: number;
|
|
4591
4598
|
rowsEmbedded: number;
|
|
4599
|
+
corpusStats?: CorpusStats;
|
|
4592
4600
|
}>;
|
|
4593
4601
|
/**
|
|
4594
4602
|
* Search vault memories by semantic similarity. Returns structured results
|
|
@@ -5169,6 +5177,8 @@ interface RecallDiagnostics {
|
|
|
5169
5177
|
* LEGACY read path this batch is uncapped, so it can be the whole vault.
|
|
5170
5178
|
*/
|
|
5171
5179
|
vaultRowsEmbedded?: number;
|
|
5180
|
+
vaultCacheHits?: number;
|
|
5181
|
+
vaultCacheMisses?: number;
|
|
5172
5182
|
/** Embedded messages the chunk lane considered; absent when the lane did not run. */
|
|
5173
5183
|
chunkRowsScanned?: number;
|
|
5174
5184
|
/** Messages whose vectors the chunk lane read from storage on a cache miss. */
|
|
@@ -5271,6 +5281,9 @@ interface RecallDiagnostics {
|
|
|
5271
5281
|
chunkLane: number;
|
|
5272
5282
|
/** Cross-lane RRF fusion + provenance dedup after both lanes. */
|
|
5273
5283
|
fuse: number;
|
|
5284
|
+
corpusKeys?: number;
|
|
5285
|
+
corpusEmbeddingsFetch?: number;
|
|
5286
|
+
corpusRowsFetch?: number;
|
|
5274
5287
|
};
|
|
5275
5288
|
/** Soft-degradation signals that fired this call (empty when clean). */
|
|
5276
5289
|
degraded: RecallDegradation[];
|
package/dist/react/index.d.ts
CHANGED
|
@@ -4511,6 +4511,13 @@ declare function eagerEmbedContent(content: string, embeddingOptions: EmbeddingO
|
|
|
4511
4511
|
* trusting it (see `vectorVersion.ts`).
|
|
4512
4512
|
*/
|
|
4513
4513
|
updatedAt?: Date): Promise<void>;
|
|
4514
|
+
interface CorpusStats {
|
|
4515
|
+
keysMs: number;
|
|
4516
|
+
embeddingsFetchMs: number;
|
|
4517
|
+
rowsFetchMs: number;
|
|
4518
|
+
cacheHits: number;
|
|
4519
|
+
cacheMisses: number;
|
|
4520
|
+
}
|
|
4514
4521
|
/**
|
|
4515
4522
|
* A single vault search result with its similarity score.
|
|
4516
4523
|
*/
|
|
@@ -4589,6 +4596,7 @@ declare function searchVaultMemoriesWithSize(query: string, vaultCtx: VaultMemor
|
|
|
4589
4596
|
*/
|
|
4590
4597
|
queryEmbedMs: number;
|
|
4591
4598
|
rowsEmbedded: number;
|
|
4599
|
+
corpusStats?: CorpusStats;
|
|
4592
4600
|
}>;
|
|
4593
4601
|
/**
|
|
4594
4602
|
* Search vault memories by semantic similarity. Returns structured results
|
|
@@ -5169,6 +5177,8 @@ interface RecallDiagnostics {
|
|
|
5169
5177
|
* LEGACY read path this batch is uncapped, so it can be the whole vault.
|
|
5170
5178
|
*/
|
|
5171
5179
|
vaultRowsEmbedded?: number;
|
|
5180
|
+
vaultCacheHits?: number;
|
|
5181
|
+
vaultCacheMisses?: number;
|
|
5172
5182
|
/** Embedded messages the chunk lane considered; absent when the lane did not run. */
|
|
5173
5183
|
chunkRowsScanned?: number;
|
|
5174
5184
|
/** Messages whose vectors the chunk lane read from storage on a cache miss. */
|
|
@@ -5271,6 +5281,9 @@ interface RecallDiagnostics {
|
|
|
5271
5281
|
chunkLane: number;
|
|
5272
5282
|
/** Cross-lane RRF fusion + provenance dedup after both lanes. */
|
|
5273
5283
|
fuse: number;
|
|
5284
|
+
corpusKeys?: number;
|
|
5285
|
+
corpusEmbeddingsFetch?: number;
|
|
5286
|
+
corpusRowsFetch?: number;
|
|
5274
5287
|
};
|
|
5275
5288
|
/** Soft-degradation signals that fired this call (empty when clean). */
|
|
5276
5289
|
degraded: RecallDegradation[];
|
package/dist/react/index.mjs
CHANGED
|
@@ -17169,10 +17169,18 @@ function admitVaultProjections(queryEmbedding, vectored, k) {
|
|
|
17169
17169
|
}
|
|
17170
17170
|
async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, queryOpts, opts) {
|
|
17171
17171
|
let rowsDecrypted = 0;
|
|
17172
|
+
const keysStart = nowMs();
|
|
17172
17173
|
const keys = await getVaultCandidateKeysOp(
|
|
17173
17174
|
vaultCtx,
|
|
17174
17175
|
Object.keys(queryOpts).length > 0 ? queryOpts : void 0
|
|
17175
17176
|
);
|
|
17177
|
+
const corpusStats = {
|
|
17178
|
+
keysMs: nowMs() - keysStart,
|
|
17179
|
+
embeddingsFetchMs: 0,
|
|
17180
|
+
rowsFetchMs: 0,
|
|
17181
|
+
cacheHits: 0,
|
|
17182
|
+
cacheMisses: 0
|
|
17183
|
+
};
|
|
17176
17184
|
const hydrateOpts = queryOpts.includeArchived ? { includeArchived: true } : void 0;
|
|
17177
17185
|
const vaultSize = keys.length;
|
|
17178
17186
|
if (keys.length === 0) {
|
|
@@ -17182,7 +17190,8 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
17182
17190
|
queryEmbedding: [],
|
|
17183
17191
|
vaultSize: 0,
|
|
17184
17192
|
laneEmbedFailed: false,
|
|
17185
|
-
rowsDecrypted: 0
|
|
17193
|
+
rowsDecrypted: 0,
|
|
17194
|
+
corpusStats
|
|
17186
17195
|
};
|
|
17187
17196
|
}
|
|
17188
17197
|
const queryEmbedStart = nowMs();
|
|
@@ -17204,16 +17213,20 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
17204
17213
|
for (const k2 of keys) {
|
|
17205
17214
|
const cached = cachedRowVector(cache, k2.uniqueId, k2.updatedAt);
|
|
17206
17215
|
if (cached && cached.length === queryEmbedding.length) {
|
|
17216
|
+
corpusStats.cacheHits++;
|
|
17207
17217
|
vectored.push({ uniqueId: k2.uniqueId, embedding: cached, updatedAt: k2.updatedAt });
|
|
17208
17218
|
continue;
|
|
17209
17219
|
}
|
|
17220
|
+
corpusStats.cacheMisses++;
|
|
17210
17221
|
const modelCompatible = (k2.embeddingModel ?? currentModel) === currentModel;
|
|
17211
17222
|
if (modelCompatible) missIds.push(k2.uniqueId);
|
|
17212
17223
|
else noVectorIds.push(k2.uniqueId);
|
|
17213
17224
|
}
|
|
17214
17225
|
}
|
|
17215
17226
|
if (missIds.length > 0) {
|
|
17227
|
+
const embeddingsStart = nowMs();
|
|
17216
17228
|
const rows = await getVaultEmbeddingsByIdsOp(vaultCtx, missIds, hydrateOpts);
|
|
17229
|
+
corpusStats.embeddingsFetchMs += nowMs() - embeddingsStart;
|
|
17217
17230
|
const gotVector = /* @__PURE__ */ new Set();
|
|
17218
17231
|
for (const r of rows) {
|
|
17219
17232
|
if (!r.embedding) continue;
|
|
@@ -17241,7 +17254,9 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
17241
17254
|
`memoryVault: projected search un-embedded lane capped at ${laneIds.length}/${noVectorIds.length}`
|
|
17242
17255
|
);
|
|
17243
17256
|
}
|
|
17257
|
+
const laneStart = nowMs();
|
|
17244
17258
|
const laneFetched = await getVaultMemoriesByIdsOp(vaultCtx, laneIds, hydrateOpts);
|
|
17259
|
+
corpusStats.rowsFetchMs += nowMs() - laneStart;
|
|
17245
17260
|
rowsDecrypted += laneFetched.length;
|
|
17246
17261
|
for (const row of laneFetched) laneById.set(row.uniqueId, row);
|
|
17247
17262
|
const laneRows = laneFetched.filter((m) => !isEncrypted(m.content));
|
|
@@ -17293,7 +17308,9 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
17293
17308
|
}
|
|
17294
17309
|
}
|
|
17295
17310
|
const idsToFetch = [...admissionSet].filter((id) => !laneById.has(id));
|
|
17311
|
+
const admittedStart = nowMs();
|
|
17296
17312
|
const fetched = idsToFetch.length > 0 ? await getVaultMemoriesByIdsOp(vaultCtx, idsToFetch, hydrateOpts) : [];
|
|
17313
|
+
corpusStats.rowsFetchMs += nowMs() - admittedStart;
|
|
17297
17314
|
rowsDecrypted += fetched.length;
|
|
17298
17315
|
const admittedRows = [
|
|
17299
17316
|
...fetched,
|
|
@@ -17322,7 +17339,10 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
17322
17339
|
}
|
|
17323
17340
|
if (staleIds.length > 0) {
|
|
17324
17341
|
const contentById = new Map(memories.map((m) => [m.uniqueId, m]));
|
|
17325
|
-
|
|
17342
|
+
const staleStart = nowMs();
|
|
17343
|
+
const staleRows = await getVaultEmbeddingsByIdsOp(vaultCtx, staleIds, hydrateOpts);
|
|
17344
|
+
corpusStats.embeddingsFetchMs += nowMs() - staleStart;
|
|
17345
|
+
for (const r of staleRows) {
|
|
17326
17346
|
if (!r.embedding) continue;
|
|
17327
17347
|
try {
|
|
17328
17348
|
const parsed = JSON.parse(r.embedding);
|
|
@@ -17354,7 +17374,15 @@ async function buildProjectedCorpus(query, vaultCtx, embeddingOptions, cache, qu
|
|
|
17354
17374
|
eventTimeKind: normalizeEventTimeKind(m.eventTimeKind),
|
|
17355
17375
|
factType: m.factType
|
|
17356
17376
|
}));
|
|
17357
|
-
return {
|
|
17377
|
+
return {
|
|
17378
|
+
memories,
|
|
17379
|
+
embeddedItems,
|
|
17380
|
+
queryEmbedding,
|
|
17381
|
+
vaultSize,
|
|
17382
|
+
laneEmbedFailed,
|
|
17383
|
+
rowsDecrypted,
|
|
17384
|
+
corpusStats
|
|
17385
|
+
};
|
|
17358
17386
|
}
|
|
17359
17387
|
async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache, searchOptions) {
|
|
17360
17388
|
if (!query || typeof query !== "string") {
|
|
@@ -17395,6 +17423,7 @@ async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache,
|
|
|
17395
17423
|
embeddingFailure = true;
|
|
17396
17424
|
};
|
|
17397
17425
|
const embedStats = { queryMs: 0, rowsEmbedded: 0 };
|
|
17426
|
+
let corpusStats;
|
|
17398
17427
|
if (searchOptions?.decryptLast) {
|
|
17399
17428
|
const forceIncludeIds = [
|
|
17400
17429
|
...searchOptions.entityRanking ?? [],
|
|
@@ -17415,6 +17444,7 @@ async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache,
|
|
|
17415
17444
|
onEmbeddingDegraded,
|
|
17416
17445
|
embedStats
|
|
17417
17446
|
});
|
|
17447
|
+
corpusStats = corpus.corpusStats;
|
|
17418
17448
|
if (corpus.vaultSize === 0) {
|
|
17419
17449
|
return {
|
|
17420
17450
|
memories: [],
|
|
@@ -17426,7 +17456,8 @@ async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache,
|
|
|
17426
17456
|
decryptLast: true,
|
|
17427
17457
|
rowsDecrypted: corpus.rowsDecrypted,
|
|
17428
17458
|
queryEmbedMs: embedStats.queryMs,
|
|
17429
|
-
rowsEmbedded: embedStats.rowsEmbedded
|
|
17459
|
+
rowsEmbedded: embedStats.rowsEmbedded,
|
|
17460
|
+
corpusStats
|
|
17430
17461
|
};
|
|
17431
17462
|
}
|
|
17432
17463
|
({ memories, embeddedItems, queryEmbedding, vaultSize } = corpus);
|
|
@@ -17443,7 +17474,8 @@ async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache,
|
|
|
17443
17474
|
decryptLast: true,
|
|
17444
17475
|
rowsDecrypted: corpus.rowsDecrypted,
|
|
17445
17476
|
queryEmbedMs: embedStats.queryMs,
|
|
17446
|
-
rowsEmbedded: embedStats.rowsEmbedded
|
|
17477
|
+
rowsEmbedded: embedStats.rowsEmbedded,
|
|
17478
|
+
corpusStats
|
|
17447
17479
|
};
|
|
17448
17480
|
}
|
|
17449
17481
|
} else {
|
|
@@ -17576,7 +17608,8 @@ async function prepareVaultCandidates(query, vaultCtx, embeddingOptions, cache,
|
|
|
17576
17608
|
embeddingsUnavailable,
|
|
17577
17609
|
embeddingFailure,
|
|
17578
17610
|
decryptLast: !!searchOptions?.decryptLast,
|
|
17579
|
-
rowsDecrypted
|
|
17611
|
+
rowsDecrypted,
|
|
17612
|
+
...corpusStats && { corpusStats }
|
|
17580
17613
|
};
|
|
17581
17614
|
}
|
|
17582
17615
|
async function rankPreparedVaultCandidates(query, prepared, embeddingOptions, searchOptions) {
|
|
@@ -17793,7 +17826,8 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
17793
17826
|
decryptLast: prepared.decryptLast,
|
|
17794
17827
|
rowsDecrypted: prepared.rowsDecrypted,
|
|
17795
17828
|
queryEmbedMs: prepared.queryEmbedMs,
|
|
17796
|
-
rowsEmbedded: prepared.rowsEmbedded
|
|
17829
|
+
rowsEmbedded: prepared.rowsEmbedded,
|
|
17830
|
+
...prepared.corpusStats && { corpusStats: prepared.corpusStats }
|
|
17797
17831
|
};
|
|
17798
17832
|
}
|
|
17799
17833
|
if (prepared.memories.length === 0) {
|
|
@@ -17808,7 +17842,8 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
17808
17842
|
decryptLast: prepared.decryptLast,
|
|
17809
17843
|
rowsDecrypted: prepared.rowsDecrypted,
|
|
17810
17844
|
queryEmbedMs: prepared.queryEmbedMs,
|
|
17811
|
-
rowsEmbedded: prepared.rowsEmbedded
|
|
17845
|
+
rowsEmbedded: prepared.rowsEmbedded,
|
|
17846
|
+
...prepared.corpusStats && { corpusStats: prepared.corpusStats }
|
|
17812
17847
|
};
|
|
17813
17848
|
}
|
|
17814
17849
|
const ranked = await rankPreparedVaultCandidates(
|
|
@@ -17823,7 +17858,8 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
17823
17858
|
decryptLast: prepared.decryptLast,
|
|
17824
17859
|
rowsDecrypted: prepared.rowsDecrypted,
|
|
17825
17860
|
queryEmbedMs: prepared.queryEmbedMs,
|
|
17826
|
-
rowsEmbedded: prepared.rowsEmbedded
|
|
17861
|
+
rowsEmbedded: prepared.rowsEmbedded,
|
|
17862
|
+
...prepared.corpusStats && { corpusStats: prepared.corpusStats }
|
|
17827
17863
|
};
|
|
17828
17864
|
}
|
|
17829
17865
|
async function searchVaultMemories(query, vaultCtx, embeddingOptions, cache, searchOptions) {
|
|
@@ -19824,6 +19860,7 @@ async function recall(query, ctx, options = {}) {
|
|
|
19824
19860
|
let factLaneRankedOnCosine = false;
|
|
19825
19861
|
let decryptLastRan;
|
|
19826
19862
|
let vaultRowsDecrypted;
|
|
19863
|
+
let corpusStats;
|
|
19827
19864
|
let graphLaneCount = 0;
|
|
19828
19865
|
let temporalLaneCount = 0;
|
|
19829
19866
|
let graphLaneFailed = false;
|
|
@@ -19864,6 +19901,10 @@ async function recall(query, ctx, options = {}) {
|
|
|
19864
19901
|
...decryptLastRan !== void 0 && { decryptLast: decryptLastRan },
|
|
19865
19902
|
...vaultRowsDecrypted !== void 0 && { vaultRowsDecrypted },
|
|
19866
19903
|
...vaultRowsEmbedded !== void 0 && { vaultRowsEmbedded },
|
|
19904
|
+
...corpusStats && {
|
|
19905
|
+
vaultCacheHits: corpusStats.cacheHits,
|
|
19906
|
+
vaultCacheMisses: corpusStats.cacheMisses
|
|
19907
|
+
},
|
|
19867
19908
|
...chunkStats && {
|
|
19868
19909
|
chunkRowsScanned: chunkStats.rowsScanned,
|
|
19869
19910
|
chunkRowsFetched: chunkStats.rowsFetched,
|
|
@@ -19886,7 +19927,12 @@ async function recall(query, ctx, options = {}) {
|
|
|
19886
19927
|
rerank: rerankMs,
|
|
19887
19928
|
queryEmbed: queryEmbedMs,
|
|
19888
19929
|
chunkLane: chunkLaneMs,
|
|
19889
|
-
fuse: fuseMs
|
|
19930
|
+
fuse: fuseMs,
|
|
19931
|
+
...corpusStats && {
|
|
19932
|
+
corpusKeys: corpusStats.keysMs,
|
|
19933
|
+
corpusEmbeddingsFetch: corpusStats.embeddingsFetchMs,
|
|
19934
|
+
corpusRowsFetch: corpusStats.rowsFetchMs
|
|
19935
|
+
}
|
|
19890
19936
|
},
|
|
19891
19937
|
degraded
|
|
19892
19938
|
};
|
|
@@ -19960,7 +20006,8 @@ async function recall(query, ctx, options = {}) {
|
|
|
19960
20006
|
embeddingsUnavailable: factEmbeddingsUnavailable,
|
|
19961
20007
|
rankedOnCosine: factRankedOnCosine,
|
|
19962
20008
|
decryptLast: factDecryptLast,
|
|
19963
|
-
rowsDecrypted: factRowsDecrypted
|
|
20009
|
+
rowsDecrypted: factRowsDecrypted,
|
|
20010
|
+
corpusStats: factCorpusStats
|
|
19964
20011
|
} = await (ctx.factSource?.search.bind(ctx.factSource) ?? ((searchQuery, searchOptions) => searchVaultMemoriesWithSize(
|
|
19965
20012
|
searchQuery,
|
|
19966
20013
|
ctx.vaultCtx,
|
|
@@ -20026,6 +20073,7 @@ async function recall(query, ctx, options = {}) {
|
|
|
20026
20073
|
factLaneRankedOnCosine = factRankedOnCosine;
|
|
20027
20074
|
decryptLastRan = factDecryptLast;
|
|
20028
20075
|
vaultRowsDecrypted = factRowsDecrypted;
|
|
20076
|
+
corpusStats = factCorpusStats;
|
|
20029
20077
|
factLaneMs = nowMs() - factStart;
|
|
20030
20078
|
}
|
|
20031
20079
|
if (chunkEmbedFailed && !factLaneRankedOnCosine) embeddingsUnavailable = true;
|