@anuma/sdk 2.4.0 → 2.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/index.cjs +8 -0
- package/dist/client/index.d.mts +44 -1
- package/dist/client/index.d.ts +44 -1
- package/dist/client/index.mjs +7 -0
- package/dist/expo/{chunk-SF5AYTXH.mjs → chunk-UOUJUZM4.mjs} +229 -35
- package/dist/expo/index.cjs +256 -38
- package/dist/expo/index.d.mts +146 -1
- package/dist/expo/index.d.ts +146 -1
- package/dist/expo/index.mjs +25 -3
- package/dist/expo/{recall-PCKDVF3U.mjs → recall-HI7VO3VM.mjs} +1 -1
- package/dist/react/{chunk-JUDJVX66.mjs → chunk-IYDQBFS2.mjs} +229 -35
- package/dist/react/index.cjs +252 -37
- package/dist/react/index.d.mts +153 -1
- package/dist/react/index.d.ts +153 -1
- package/dist/react/index.mjs +21 -2
- package/dist/react/{recall-DSJSBDR4.mjs → recall-G3CS444F.mjs} +1 -1
- package/dist/server/{chunk-3AWYZRTS.mjs → chunk-3MLG7SUH.mjs} +229 -35
- package/dist/server/index.cjs +245 -36
- package/dist/server/index.d.mts +153 -1
- package/dist/server/index.d.ts +153 -1
- package/dist/server/index.mjs +14 -1
- package/dist/server/{recall-EWMI64SA.mjs → recall-MJHGXMHZ.mjs} +1 -1
- package/package.json +2 -2
package/dist/server/index.cjs
CHANGED
|
@@ -16385,7 +16385,13 @@ async function searchMessagesOp(ctx, queryVector, options) {
|
|
|
16385
16385
|
);
|
|
16386
16386
|
}
|
|
16387
16387
|
async function searchChunksOp(ctx, queryVector, options) {
|
|
16388
|
-
const {
|
|
16388
|
+
const {
|
|
16389
|
+
limit = 10,
|
|
16390
|
+
minSimilarity = 0.5,
|
|
16391
|
+
conversationId,
|
|
16392
|
+
embeddingModel,
|
|
16393
|
+
chunkCache
|
|
16394
|
+
} = options || {};
|
|
16389
16395
|
const activeConversations = await ctx.conversationsCollection.query(import_watermelondb14.Q.where("is_deleted", false)).fetch();
|
|
16390
16396
|
const activeConversationIds = new Set(
|
|
16391
16397
|
// Use _getRaw for reliable raw column access
|
|
@@ -16405,23 +16411,54 @@ async function searchChunksOp(ctx, queryVector, options) {
|
|
|
16405
16411
|
continue;
|
|
16406
16412
|
}
|
|
16407
16413
|
}
|
|
16414
|
+
const version = Number(message._getRaw("updated_at")) || 0;
|
|
16415
|
+
const cached = chunkCache?.get(message.id);
|
|
16416
|
+
if (cached && cached.version === version) {
|
|
16417
|
+
if (cached.chunks.length > 0) {
|
|
16418
|
+
for (let ci = 0; ci < cached.chunks.length; ci++) {
|
|
16419
|
+
const vec = cached.chunks[ci];
|
|
16420
|
+
if (vec.length === 0) continue;
|
|
16421
|
+
const similarity = cosineSimilarity(queryVector, vec);
|
|
16422
|
+
if (similarity >= minSimilarity) {
|
|
16423
|
+
candidates.push({
|
|
16424
|
+
message,
|
|
16425
|
+
similarity,
|
|
16426
|
+
chunkTextSource: { kind: "chunk-cached", chunkIndex: ci, version }
|
|
16427
|
+
});
|
|
16428
|
+
}
|
|
16429
|
+
}
|
|
16430
|
+
} else if (cached.fallback) {
|
|
16431
|
+
const similarity = cosineSimilarity(queryVector, cached.fallback);
|
|
16432
|
+
if (similarity >= minSimilarity) {
|
|
16433
|
+
candidates.push({ message, similarity, chunkTextSource: { kind: "message" } });
|
|
16434
|
+
}
|
|
16435
|
+
}
|
|
16436
|
+
continue;
|
|
16437
|
+
}
|
|
16408
16438
|
const chunks = await readJsonField(message, "chunks", ctx.walletAddress);
|
|
16409
16439
|
if (chunks && chunks.length > 0) {
|
|
16410
|
-
|
|
16411
|
-
|
|
16412
|
-
|
|
16440
|
+
const vectors = chunks.map(
|
|
16441
|
+
(chunk) => chunk.vector && chunk.vector.length > 0 ? Float32Array.from(chunk.vector) : new Float32Array(0)
|
|
16442
|
+
);
|
|
16443
|
+
chunkCache?.set(message.id, { version, chunks: vectors });
|
|
16444
|
+
for (let ci = 0; ci < chunks.length; ci++) {
|
|
16445
|
+
const vec = vectors[ci];
|
|
16446
|
+
if (vec.length === 0) continue;
|
|
16447
|
+
const similarity = cosineSimilarity(queryVector, vec);
|
|
16413
16448
|
if (similarity >= minSimilarity) {
|
|
16414
16449
|
candidates.push({
|
|
16415
16450
|
message,
|
|
16416
16451
|
similarity,
|
|
16417
|
-
chunkTextSource: { kind: "chunk", text:
|
|
16452
|
+
chunkTextSource: { kind: "chunk", text: chunks[ci].text }
|
|
16418
16453
|
});
|
|
16419
16454
|
}
|
|
16420
16455
|
}
|
|
16421
16456
|
} else {
|
|
16422
16457
|
const messageVector = await readJsonField(message, "vector", ctx.walletAddress);
|
|
16423
16458
|
if (!messageVector || messageVector.length === 0) continue;
|
|
16424
|
-
const
|
|
16459
|
+
const fallback = Float32Array.from(messageVector);
|
|
16460
|
+
chunkCache?.set(message.id, { version, chunks: [], fallback });
|
|
16461
|
+
const similarity = cosineSimilarity(queryVector, fallback);
|
|
16425
16462
|
if (similarity >= minSimilarity) {
|
|
16426
16463
|
candidates.push({ message, similarity, chunkTextSource: { kind: "message" } });
|
|
16427
16464
|
}
|
|
@@ -16448,9 +16485,31 @@ async function searchChunksOp(ctx, queryVector, options) {
|
|
|
16448
16485
|
})
|
|
16449
16486
|
)
|
|
16450
16487
|
);
|
|
16488
|
+
const chunkStateByMsgId = /* @__PURE__ */ new Map();
|
|
16489
|
+
const needsChunkText = /* @__PURE__ */ new Set();
|
|
16490
|
+
for (const c of topK) {
|
|
16491
|
+
if (c.chunkTextSource.kind === "chunk-cached") needsChunkText.add(c.message.id);
|
|
16492
|
+
}
|
|
16493
|
+
await Promise.all(
|
|
16494
|
+
Array.from(needsChunkText, async (id) => {
|
|
16495
|
+
const m = uniqueMessages.get(id);
|
|
16496
|
+
const parsed = await readJsonField(m, "chunks", ctx.walletAddress);
|
|
16497
|
+
const version = Number(m._getRaw("updated_at")) || 0;
|
|
16498
|
+
if (parsed) chunkStateByMsgId.set(id, { version, chunks: parsed });
|
|
16499
|
+
})
|
|
16500
|
+
);
|
|
16451
16501
|
return topK.map(({ message, similarity, chunkTextSource }) => {
|
|
16452
16502
|
const stored = { ...storedById.get(message.id) };
|
|
16453
|
-
|
|
16503
|
+
let chunkText2;
|
|
16504
|
+
if (chunkTextSource.kind === "chunk") {
|
|
16505
|
+
chunkText2 = chunkTextSource.text;
|
|
16506
|
+
} else if (chunkTextSource.kind === "chunk-cached") {
|
|
16507
|
+
const state = chunkStateByMsgId.get(message.id);
|
|
16508
|
+
const chunk = state && state.version === chunkTextSource.version ? state.chunks[chunkTextSource.chunkIndex] : void 0;
|
|
16509
|
+
chunkText2 = chunk?.text ?? stored.content;
|
|
16510
|
+
} else {
|
|
16511
|
+
chunkText2 = stored.content;
|
|
16512
|
+
}
|
|
16454
16513
|
return { chunkText: chunkText2, message: stored, similarity };
|
|
16455
16514
|
});
|
|
16456
16515
|
}
|
|
@@ -18765,9 +18824,9 @@ function recencyMultiplier(updatedAt, opts = {}) {
|
|
|
18765
18824
|
const updatedMs = updatedAt.getTime();
|
|
18766
18825
|
if (!Number.isFinite(updatedMs)) return noDate;
|
|
18767
18826
|
const now = opts.now ?? /* @__PURE__ */ new Date();
|
|
18768
|
-
const
|
|
18769
|
-
if (!Number.isFinite(
|
|
18770
|
-
const ageDays = Math.max(0, (
|
|
18827
|
+
const nowMs2 = now.getTime();
|
|
18828
|
+
if (!Number.isFinite(nowMs2)) return noDate;
|
|
18829
|
+
const ageDays = Math.max(0, (nowMs2 - updatedMs) / DAY_MS);
|
|
18771
18830
|
const perYearDecay = opts.perYearDecay ?? DEFAULT_PER_YEAR_DECAY;
|
|
18772
18831
|
const floor = opts.floor ?? DEFAULT_FLOOR;
|
|
18773
18832
|
const decay = perYearDecay * (ageDays / 365);
|
|
@@ -18785,14 +18844,42 @@ var init_recency = __esm({
|
|
|
18785
18844
|
});
|
|
18786
18845
|
|
|
18787
18846
|
// src/lib/memory/reranker.ts
|
|
18847
|
+
function isRerankerAvailable() {
|
|
18848
|
+
return available;
|
|
18849
|
+
}
|
|
18850
|
+
function isModuleMissing(err) {
|
|
18851
|
+
let e = err;
|
|
18852
|
+
for (let depth = 0; depth < 5 && e !== null && e !== void 0; depth++) {
|
|
18853
|
+
const code = e.code;
|
|
18854
|
+
if (code === "ERR_MODULE_NOT_FOUND" || code === "MODULE_NOT_FOUND") return true;
|
|
18855
|
+
if (e instanceof Error && /cannot find (module|package)|failed to resolve|unable to resolve module|requiring unknown module/i.test(
|
|
18856
|
+
e.message
|
|
18857
|
+
)) {
|
|
18858
|
+
return true;
|
|
18859
|
+
}
|
|
18860
|
+
e = e.cause;
|
|
18861
|
+
}
|
|
18862
|
+
return false;
|
|
18863
|
+
}
|
|
18788
18864
|
async function getModel() {
|
|
18865
|
+
if (available === false) throw new RerankerUnavailableError(void 0);
|
|
18789
18866
|
if (!modelPromise) {
|
|
18790
18867
|
const load = (async () => {
|
|
18791
|
-
|
|
18868
|
+
let transformers;
|
|
18869
|
+
try {
|
|
18870
|
+
transformers = await import("@huggingface/transformers");
|
|
18871
|
+
} catch (err) {
|
|
18872
|
+
if (isModuleMissing(err)) {
|
|
18873
|
+
available = false;
|
|
18874
|
+
throw new RerankerUnavailableError(err);
|
|
18875
|
+
}
|
|
18876
|
+
throw err;
|
|
18877
|
+
}
|
|
18792
18878
|
const [tokenizer, model] = await Promise.all([
|
|
18793
18879
|
transformers.AutoTokenizer.from_pretrained(MODEL_ID),
|
|
18794
18880
|
transformers.AutoModelForSequenceClassification.from_pretrained(MODEL_ID)
|
|
18795
18881
|
]);
|
|
18882
|
+
available = true;
|
|
18796
18883
|
return { tokenizer, model };
|
|
18797
18884
|
})();
|
|
18798
18885
|
modelPromise = load;
|
|
@@ -18834,11 +18921,19 @@ async function rerankPairs(query, items) {
|
|
|
18834
18921
|
}
|
|
18835
18922
|
return items.map((item, i) => ({ id: item.id, content: item.content, score: scores[i] })).sort((a, b) => b.score - a.score);
|
|
18836
18923
|
}
|
|
18837
|
-
var MODEL_ID, modelPromise;
|
|
18924
|
+
var MODEL_ID, RerankerUnavailableError, available, modelPromise;
|
|
18838
18925
|
var init_reranker = __esm({
|
|
18839
18926
|
"src/lib/memory/reranker.ts"() {
|
|
18840
18927
|
"use strict";
|
|
18841
18928
|
MODEL_ID = "Xenova/ms-marco-MiniLM-L-6-v2";
|
|
18929
|
+
RerankerUnavailableError = class extends Error {
|
|
18930
|
+
constructor(reason) {
|
|
18931
|
+
super("cross-encoder reranker unavailable (@huggingface/transformers not installed)");
|
|
18932
|
+
this.name = "RerankerUnavailableError";
|
|
18933
|
+
this.reason = reason;
|
|
18934
|
+
}
|
|
18935
|
+
};
|
|
18936
|
+
available = void 0;
|
|
18842
18937
|
modelPromise = null;
|
|
18843
18938
|
}
|
|
18844
18939
|
});
|
|
@@ -19722,22 +19817,67 @@ async function recall(query, ctx, options = {}) {
|
|
|
19722
19817
|
const flags = flagsForBudget(requestedBudget);
|
|
19723
19818
|
const decomposeAvailable = flags.decompose && !!options.decomposeOptions;
|
|
19724
19819
|
const usedBudget = requestedBudget === "high" && !decomposeAvailable ? "mid" : requestedBudget;
|
|
19820
|
+
const t0 = nowMs();
|
|
19821
|
+
let prepMs = 0;
|
|
19822
|
+
let factLaneMs = 0;
|
|
19823
|
+
let chunkLaneMs = 0;
|
|
19824
|
+
let fuseMs = 0;
|
|
19825
|
+
const factResults = [];
|
|
19826
|
+
const chunkResults = [];
|
|
19827
|
+
let vaultSize;
|
|
19828
|
+
let didRerank = false;
|
|
19829
|
+
let hadV2Head = false;
|
|
19830
|
+
const emitDiagnostics = (candidateCount) => {
|
|
19831
|
+
const cb = options.onDiagnostics;
|
|
19832
|
+
if (!cb) return;
|
|
19833
|
+
const degraded = [];
|
|
19834
|
+
if (flags.rerank && factResults.length > 0 && hadV2Head && !didRerank) {
|
|
19835
|
+
degraded.push("rerank-unavailable");
|
|
19836
|
+
}
|
|
19837
|
+
if (flags.decompose && !decomposeAvailable) degraded.push("decompose-unavailable");
|
|
19838
|
+
const diagnostics = {
|
|
19839
|
+
usedBudget,
|
|
19840
|
+
reranked: didRerank,
|
|
19841
|
+
candidateCount,
|
|
19842
|
+
...vaultSize !== void 0 && { vaultSize },
|
|
19843
|
+
factCount: factResults.length,
|
|
19844
|
+
chunkCount: chunkResults.length,
|
|
19845
|
+
timings: {
|
|
19846
|
+
total: nowMs() - t0,
|
|
19847
|
+
prep: prepMs,
|
|
19848
|
+
factLane: factLaneMs,
|
|
19849
|
+
chunkLane: chunkLaneMs,
|
|
19850
|
+
fuse: fuseMs
|
|
19851
|
+
},
|
|
19852
|
+
degraded
|
|
19853
|
+
};
|
|
19854
|
+
try {
|
|
19855
|
+
cb(diagnostics);
|
|
19856
|
+
} catch {
|
|
19857
|
+
}
|
|
19858
|
+
};
|
|
19725
19859
|
if (!query || typeof query !== "string" || query.trim().length === 0) {
|
|
19860
|
+
emitDiagnostics(0);
|
|
19726
19861
|
return { memories: [], usedBudget, reranked: false, candidateCount: 0 };
|
|
19727
19862
|
}
|
|
19728
19863
|
const needsChunkEmbedding = types.includes("chunk") && ctx.storageCtx;
|
|
19729
19864
|
const wantsTemporal = types.includes("fact") && ctx.vaultCtx;
|
|
19865
|
+
const prepStart = nowMs();
|
|
19730
19866
|
const [queryEmbedding, entityRanking, temporalRanking] = await Promise.all([
|
|
19731
19867
|
needsChunkEmbedding ? generateEmbedding(query, ctx.embeddingOptions) : Promise.resolve(void 0),
|
|
19732
19868
|
buildGraphLaneRanking(query, ctx),
|
|
19733
19869
|
wantsTemporal ? buildTemporalLaneRanking(query, ctx.vaultCtx, options.now) : Promise.resolve([])
|
|
19734
19870
|
]);
|
|
19735
|
-
|
|
19736
|
-
const chunkResults = [];
|
|
19737
|
-
let vaultSize;
|
|
19871
|
+
prepMs = nowMs() - prepStart;
|
|
19738
19872
|
if (types.includes("fact") && ctx.vaultCtx && ctx.vaultCache) {
|
|
19873
|
+
const factStart = nowMs();
|
|
19739
19874
|
const vaultMinScore = options.minScore ?? DEFAULT_FACT_MIN_SCORE;
|
|
19740
|
-
const {
|
|
19875
|
+
const {
|
|
19876
|
+
results,
|
|
19877
|
+
vaultSize: size,
|
|
19878
|
+
reranked,
|
|
19879
|
+
hadV2Head: v2Head
|
|
19880
|
+
} = await searchVaultMemoriesWithSize(
|
|
19741
19881
|
query,
|
|
19742
19882
|
ctx.vaultCtx,
|
|
19743
19883
|
ctx.embeddingOptions,
|
|
@@ -19788,14 +19928,19 @@ async function recall(query, ctx, options = {}) {
|
|
|
19788
19928
|
)
|
|
19789
19929
|
);
|
|
19790
19930
|
vaultSize = size;
|
|
19931
|
+
didRerank = reranked;
|
|
19932
|
+
hadV2Head = v2Head;
|
|
19933
|
+
factLaneMs = nowMs() - factStart;
|
|
19791
19934
|
}
|
|
19792
19935
|
if (types.includes("chunk") && ctx.storageCtx && queryEmbedding) {
|
|
19936
|
+
const chunkStart = nowMs();
|
|
19793
19937
|
const chunkMinScore = options.minScore ?? DEFAULT_CHUNK_MIN_SCORE;
|
|
19794
19938
|
const results = await searchChunksOp(ctx.storageCtx, queryEmbedding, {
|
|
19795
19939
|
limit: types.includes("fact") ? Math.max(limit * 2, 16) : limit,
|
|
19796
19940
|
minSimilarity: chunkMinScore,
|
|
19797
19941
|
embeddingModel: ctx.embeddingOptions.model ?? DEFAULT_API_EMBEDDING_MODEL,
|
|
19798
|
-
...options.conversationId && { conversationId: options.conversationId }
|
|
19942
|
+
...options.conversationId && { conversationId: options.conversationId },
|
|
19943
|
+
...ctx.chunkCache && { chunkCache: ctx.chunkCache }
|
|
19799
19944
|
});
|
|
19800
19945
|
chunkResults.push(
|
|
19801
19946
|
...dedupeBy(
|
|
@@ -19805,18 +19950,23 @@ async function recall(query, ctx, options = {}) {
|
|
|
19805
19950
|
(r) => r.chunkText.trim()
|
|
19806
19951
|
)
|
|
19807
19952
|
);
|
|
19953
|
+
chunkLaneMs = nowMs() - chunkStart;
|
|
19808
19954
|
}
|
|
19955
|
+
const fuseStart = nowMs();
|
|
19809
19956
|
if (types.length === 1 || factResults.length === 0 || chunkResults.length === 0) {
|
|
19810
19957
|
const memories2 = [
|
|
19811
19958
|
...factResults.map(toFactMemory),
|
|
19812
19959
|
...chunkResults.map(toChunkMemory)
|
|
19813
19960
|
];
|
|
19814
19961
|
memories2.sort((a, b) => b.score - a.score);
|
|
19962
|
+
const candidateCount = factResults.length + chunkResults.length;
|
|
19963
|
+
fuseMs = nowMs() - fuseStart;
|
|
19964
|
+
emitDiagnostics(candidateCount);
|
|
19815
19965
|
return {
|
|
19816
19966
|
memories: memories2.slice(0, limit),
|
|
19817
19967
|
usedBudget,
|
|
19818
|
-
reranked:
|
|
19819
|
-
candidateCount
|
|
19968
|
+
reranked: didRerank,
|
|
19969
|
+
candidateCount,
|
|
19820
19970
|
...vaultSize !== void 0 && { vaultSize }
|
|
19821
19971
|
};
|
|
19822
19972
|
}
|
|
@@ -19865,10 +20015,12 @@ async function recall(query, ctx, options = {}) {
|
|
|
19865
20015
|
suppressed = next;
|
|
19866
20016
|
memories = selectWith(suppressed);
|
|
19867
20017
|
}
|
|
20018
|
+
fuseMs = nowMs() - fuseStart;
|
|
20019
|
+
emitDiagnostics(byId.size);
|
|
19868
20020
|
return {
|
|
19869
20021
|
memories,
|
|
19870
20022
|
usedBudget,
|
|
19871
|
-
reranked:
|
|
20023
|
+
reranked: didRerank,
|
|
19872
20024
|
candidateCount: byId.size,
|
|
19873
20025
|
...vaultSize !== void 0 && { vaultSize }
|
|
19874
20026
|
};
|
|
@@ -19932,7 +20084,7 @@ function toChunkMemory(r) {
|
|
|
19932
20084
|
updatedAt: r.message.createdAt
|
|
19933
20085
|
};
|
|
19934
20086
|
}
|
|
19935
|
-
var DEFAULT_LIMIT, DEFAULT_BUDGET, DEFAULT_FACT_MIN_SCORE, DEFAULT_CHUNK_MIN_SCORE;
|
|
20087
|
+
var DEFAULT_LIMIT, DEFAULT_BUDGET, DEFAULT_FACT_MIN_SCORE, nowMs, DEFAULT_CHUNK_MIN_SCORE;
|
|
19936
20088
|
var init_recall = __esm({
|
|
19937
20089
|
"src/lib/memory/recall.ts"() {
|
|
19938
20090
|
"use strict";
|
|
@@ -19948,6 +20100,7 @@ var init_recall = __esm({
|
|
|
19948
20100
|
DEFAULT_LIMIT = 8;
|
|
19949
20101
|
DEFAULT_BUDGET = "low";
|
|
19950
20102
|
DEFAULT_FACT_MIN_SCORE = 0.1;
|
|
20103
|
+
nowMs = () => typeof performance !== "undefined" && typeof performance.now === "function" ? performance.now() : Date.now();
|
|
19951
20104
|
DEFAULT_CHUNK_MIN_SCORE = 0.5;
|
|
19952
20105
|
}
|
|
19953
20106
|
});
|
|
@@ -20147,6 +20300,9 @@ async function rankFusedVaultMemoriesAsync(query, queryEmbedding, items, options
|
|
|
20147
20300
|
bm25AdmissionDivisor: options?.bm25AdmissionDivisor,
|
|
20148
20301
|
rrfK: options?.rrfK
|
|
20149
20302
|
});
|
|
20303
|
+
if (options?.v2HeadStats) {
|
|
20304
|
+
options.v2HeadStats.hadResults = v2Ranked.length > 0;
|
|
20305
|
+
}
|
|
20150
20306
|
let combined;
|
|
20151
20307
|
let tailSlice = [];
|
|
20152
20308
|
if (options?.rerank) {
|
|
@@ -20167,8 +20323,13 @@ async function rankFusedVaultMemoriesAsync(query, queryEmbedding, items, options
|
|
|
20167
20323
|
return { ...r, similarity: v2 * (1 + ceWeight * ce) };
|
|
20168
20324
|
});
|
|
20169
20325
|
combined.sort((a, b) => b.similarity - a.similarity);
|
|
20326
|
+
if (headSlice.length > 0 && options.rerankStats) options.rerankStats.applied = true;
|
|
20170
20327
|
} catch (err) {
|
|
20171
|
-
|
|
20328
|
+
if (err instanceof RerankerUnavailableError) {
|
|
20329
|
+
getLogger().debug("[memory/search] cross-encoder unavailable; using V2 ranking");
|
|
20330
|
+
} else {
|
|
20331
|
+
getLogger().warn("[memory/search] cross-encoder rerank failed; using V2 ranking", err);
|
|
20332
|
+
}
|
|
20172
20333
|
combined = v2Ranked;
|
|
20173
20334
|
}
|
|
20174
20335
|
} else {
|
|
@@ -20302,6 +20463,9 @@ async function rankComposite(originalQuery, originalQueryEmbedding, subQueries,
|
|
|
20302
20463
|
(r) => r !== null
|
|
20303
20464
|
);
|
|
20304
20465
|
combined.sort((a, b) => b.similarity - a.similarity);
|
|
20466
|
+
if (options?.v2HeadStats) {
|
|
20467
|
+
options.v2HeadStats.hadResults = combined.length > 0;
|
|
20468
|
+
}
|
|
20305
20469
|
if (options?.includeUnrankedTail) {
|
|
20306
20470
|
const fusedIds = new Set(combined.map((r) => r.uniqueId));
|
|
20307
20471
|
for (const item of items) {
|
|
@@ -20337,11 +20501,16 @@ async function rankComposite(originalQuery, originalQueryEmbedding, subQueries,
|
|
|
20337
20501
|
}));
|
|
20338
20502
|
head.sort((a, b) => b.similarity - a.similarity);
|
|
20339
20503
|
combined = [...head, ...tailSlice];
|
|
20504
|
+
if (headSlice.length > 0 && options.rerankStats) options.rerankStats.applied = true;
|
|
20340
20505
|
} catch (err) {
|
|
20341
|
-
|
|
20342
|
-
"[memory/search]
|
|
20343
|
-
|
|
20344
|
-
|
|
20506
|
+
if (err instanceof RerankerUnavailableError) {
|
|
20507
|
+
getLogger().debug("[memory/search] cross-encoder unavailable; using fused ranking");
|
|
20508
|
+
} else {
|
|
20509
|
+
getLogger().warn(
|
|
20510
|
+
"[memory/search] composite cross-encoder rerank failed; using fused ranking",
|
|
20511
|
+
err
|
|
20512
|
+
);
|
|
20513
|
+
}
|
|
20345
20514
|
}
|
|
20346
20515
|
}
|
|
20347
20516
|
if (options?.mmr) {
|
|
@@ -20433,7 +20602,7 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
20433
20602
|
const minSimilarity = searchOptions?.minSimilarity ?? 0.1;
|
|
20434
20603
|
const scopes = searchOptions?.scopes;
|
|
20435
20604
|
if (!query || typeof query !== "string") {
|
|
20436
|
-
return { results: [], vaultSize: 0 };
|
|
20605
|
+
return { results: [], vaultSize: 0, reranked: false, hadV2Head: false };
|
|
20437
20606
|
}
|
|
20438
20607
|
const folderId = searchOptions?.folderId;
|
|
20439
20608
|
const queryOpts = {};
|
|
@@ -20450,7 +20619,7 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
20450
20619
|
);
|
|
20451
20620
|
}
|
|
20452
20621
|
if (memories.length === 0) {
|
|
20453
|
-
return { results: [], vaultSize: loaded.length };
|
|
20622
|
+
return { results: [], vaultSize: loaded.length, reranked: false, hadV2Head: false };
|
|
20454
20623
|
}
|
|
20455
20624
|
const queryEmbedding = await generateEmbedding(query, embeddingOptions);
|
|
20456
20625
|
const currentModel = embeddingOptions.model ?? DEFAULT_API_EMBEDDING_MODEL;
|
|
@@ -20526,12 +20695,18 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
20526
20695
|
memories.map((m) => [m.uniqueId, { createdAt: m.createdAt, updatedAt: m.updatedAt }])
|
|
20527
20696
|
);
|
|
20528
20697
|
const stampTimestamps = (out) => {
|
|
20529
|
-
|
|
20698
|
+
const results2 = out.results.map((r) => {
|
|
20530
20699
|
const ts = timestampById.get(r.uniqueId);
|
|
20531
20700
|
return ts ? { ...r, createdAt: ts.createdAt, updatedAt: ts.updatedAt } : r;
|
|
20532
20701
|
});
|
|
20533
|
-
return
|
|
20702
|
+
return {
|
|
20703
|
+
results: results2,
|
|
20704
|
+
vaultSize: out.vaultSize,
|
|
20705
|
+
reranked: out.reranked ?? false,
|
|
20706
|
+
hadV2Head: out.hadV2Head ?? false
|
|
20707
|
+
};
|
|
20534
20708
|
};
|
|
20709
|
+
const rerankStats = { applied: false };
|
|
20535
20710
|
const useFusion = searchOptions?.useFusion ?? true;
|
|
20536
20711
|
const tuning = {
|
|
20537
20712
|
...searchOptions?.recencyAlpha !== void 0 && { recencyAlpha: searchOptions.recencyAlpha },
|
|
@@ -20558,6 +20733,7 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
20558
20733
|
query: sq,
|
|
20559
20734
|
embedding: subEmbeddings[i]
|
|
20560
20735
|
}));
|
|
20736
|
+
const v2HeadStats = { hadResults: false };
|
|
20561
20737
|
const composite = await rankComposite(query, queryEmbedding, subQueries, embeddedItems, {
|
|
20562
20738
|
limit,
|
|
20563
20739
|
minSimilarity,
|
|
@@ -20569,12 +20745,20 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
20569
20745
|
...searchOptions.mmr !== void 0 && { mmr: searchOptions.mmr },
|
|
20570
20746
|
...tuning,
|
|
20571
20747
|
...searchOptions.entityRanking && { entityRanking: searchOptions.entityRanking },
|
|
20572
|
-
...searchOptions.temporalRanking && { temporalRanking: searchOptions.temporalRanking }
|
|
20748
|
+
...searchOptions.temporalRanking && { temporalRanking: searchOptions.temporalRanking },
|
|
20749
|
+
rerankStats,
|
|
20750
|
+
v2HeadStats
|
|
20751
|
+
});
|
|
20752
|
+
return stampTimestamps({
|
|
20753
|
+
results: composite,
|
|
20754
|
+
vaultSize: loaded.length,
|
|
20755
|
+
reranked: rerankStats.applied,
|
|
20756
|
+
hadV2Head: v2HeadStats.hadResults
|
|
20573
20757
|
});
|
|
20574
|
-
return stampTimestamps({ results: composite, vaultSize: loaded.length });
|
|
20575
20758
|
}
|
|
20576
20759
|
}
|
|
20577
20760
|
if (useFusion && searchOptions?.rerank) {
|
|
20761
|
+
const v2HeadStats = { hadResults: false };
|
|
20578
20762
|
const results2 = await rankFusedVaultMemoriesAsync(query, queryEmbedding, embeddedItems, {
|
|
20579
20763
|
limit,
|
|
20580
20764
|
minSimilarity,
|
|
@@ -20586,9 +20770,16 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
20586
20770
|
...searchOptions.mmr !== void 0 && { mmr: searchOptions.mmr },
|
|
20587
20771
|
...tuning,
|
|
20588
20772
|
...searchOptions.entityRanking && { entityRanking: searchOptions.entityRanking },
|
|
20589
|
-
...searchOptions.temporalRanking && { temporalRanking: searchOptions.temporalRanking }
|
|
20773
|
+
...searchOptions.temporalRanking && { temporalRanking: searchOptions.temporalRanking },
|
|
20774
|
+
rerankStats,
|
|
20775
|
+
v2HeadStats
|
|
20776
|
+
});
|
|
20777
|
+
return stampTimestamps({
|
|
20778
|
+
results: results2,
|
|
20779
|
+
vaultSize: loaded.length,
|
|
20780
|
+
reranked: rerankStats.applied,
|
|
20781
|
+
hadV2Head: v2HeadStats.hadResults
|
|
20590
20782
|
});
|
|
20591
|
-
return stampTimestamps({ results: results2, vaultSize: loaded.length });
|
|
20592
20783
|
}
|
|
20593
20784
|
if (useFusion) {
|
|
20594
20785
|
const results2 = rankFusedVaultMemories(query, queryEmbedding, embeddedItems, {
|
|
@@ -20598,7 +20789,7 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
20598
20789
|
...searchOptions?.entityRanking && { entityRanking: searchOptions.entityRanking },
|
|
20599
20790
|
...searchOptions?.temporalRanking && { temporalRanking: searchOptions.temporalRanking }
|
|
20600
20791
|
});
|
|
20601
|
-
return stampTimestamps({ results: results2, vaultSize: loaded.length });
|
|
20792
|
+
return stampTimestamps({ results: results2, vaultSize: loaded.length, hadV2Head: results2.length > 0 });
|
|
20602
20793
|
}
|
|
20603
20794
|
const results = rankVaultMemories(query, queryEmbedding, embeddedItems, {
|
|
20604
20795
|
limit,
|
|
@@ -20610,7 +20801,7 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
20610
20801
|
supersessionWindow: searchOptions.supersessionWindow
|
|
20611
20802
|
}
|
|
20612
20803
|
});
|
|
20613
|
-
return stampTimestamps({ results, vaultSize: loaded.length });
|
|
20804
|
+
return stampTimestamps({ results, vaultSize: loaded.length, hadV2Head: results.length > 0 });
|
|
20614
20805
|
}
|
|
20615
20806
|
async function searchVaultMemories(query, vaultCtx, embeddingOptions, cache, searchOptions) {
|
|
20616
20807
|
const { results } = await searchVaultMemoriesWithSize(
|
|
@@ -21560,6 +21751,7 @@ __export(index_exports, {
|
|
|
21560
21751
|
DEFAULT_ANALYSIS_PROMPT: () => DEFAULT_ANALYSIS_PROMPT,
|
|
21561
21752
|
DEFAULT_ANALYZE_AFTER_MESSAGES: () => DEFAULT_ANALYZE_AFTER_MESSAGES,
|
|
21562
21753
|
DEFAULT_CACHE_EXPIRATION_MS: () => DEFAULT_CACHE_EXPIRATION_MS,
|
|
21754
|
+
DEFAULT_CHUNK_CACHE_SIZE: () => DEFAULT_CHUNK_CACHE_SIZE,
|
|
21563
21755
|
DEFAULT_CHUNK_OVERLAP: () => DEFAULT_CHUNK_OVERLAP,
|
|
21564
21756
|
DEFAULT_CHUNK_SIZE: () => DEFAULT_CHUNK_SIZE,
|
|
21565
21757
|
DEFAULT_DOCUMENT_PATTERN: () => DEFAULT_DOCUMENT_PATTERN,
|
|
@@ -21590,6 +21782,7 @@ __export(index_exports, {
|
|
|
21590
21782
|
QueueManager: () => QueueManager,
|
|
21591
21783
|
RECALL_MAX_LIMIT: () => RECALL_MAX_LIMIT,
|
|
21592
21784
|
RECALL_TOOL_NAME: () => RECALL_TOOL_NAME,
|
|
21785
|
+
RerankerUnavailableError: () => RerankerUnavailableError,
|
|
21593
21786
|
SDK_SCHEMA_VERSION: () => SDK_SCHEMA_VERSION,
|
|
21594
21787
|
SERVER_TOOL_DEPENDENCY_SETS: () => SERVER_TOOL_DEPENDENCY_SETS,
|
|
21595
21788
|
SLIDER_CONFIG: () => SLIDER_CONFIG,
|
|
@@ -21631,6 +21824,7 @@ __export(index_exports, {
|
|
|
21631
21824
|
clearServerToolsCache: () => clearServerToolsCache,
|
|
21632
21825
|
composeHooks: () => composeHooks,
|
|
21633
21826
|
createAutoExtractor: () => createAutoExtractor,
|
|
21827
|
+
createChunkVectorCache: () => createChunkVectorCache,
|
|
21634
21828
|
createConversationOp: () => createConversationOp,
|
|
21635
21829
|
createCryptoPricePreProcessor: () => createCryptoPricePreProcessor,
|
|
21636
21830
|
createMediaBatchOp: () => createMediaBatchOp,
|
|
@@ -21736,6 +21930,7 @@ __export(index_exports, {
|
|
|
21736
21930
|
hardDeleteMediaOp: () => hardDeleteMediaOp,
|
|
21737
21931
|
hasEncryptionKey: () => hasEncryptionKey,
|
|
21738
21932
|
hasKeyPair: () => hasKeyPair,
|
|
21933
|
+
isRerankerAvailable: () => isRerankerAvailable,
|
|
21739
21934
|
isSupportedFile: () => isSupportedFile,
|
|
21740
21935
|
isSupportedMediaType: () => isSupportedMediaType,
|
|
21741
21936
|
makeSyntheticStoredConversation: () => makeSyntheticStoredConversation,
|
|
@@ -26484,6 +26679,13 @@ function createAutoExtractor(options) {
|
|
|
26484
26679
|
};
|
|
26485
26680
|
}
|
|
26486
26681
|
|
|
26682
|
+
// src/lib/memory/chunkVectorCache.ts
|
|
26683
|
+
init_lruCache();
|
|
26684
|
+
var DEFAULT_CHUNK_CACHE_SIZE = 5e3;
|
|
26685
|
+
function createChunkVectorCache(maxSize = DEFAULT_CHUNK_CACHE_SIZE) {
|
|
26686
|
+
return new LRUCache(maxSize);
|
|
26687
|
+
}
|
|
26688
|
+
|
|
26487
26689
|
// src/lib/memory/index.ts
|
|
26488
26690
|
init_recall();
|
|
26489
26691
|
|
|
@@ -26748,6 +26950,9 @@ function parseAnswer(body, base, parseSchema) {
|
|
|
26748
26950
|
};
|
|
26749
26951
|
}
|
|
26750
26952
|
|
|
26953
|
+
// src/lib/memory/index.ts
|
|
26954
|
+
init_reranker();
|
|
26955
|
+
|
|
26751
26956
|
// src/lib/memory/topicExtract.ts
|
|
26752
26957
|
init_operations2();
|
|
26753
26958
|
init_operations3();
|
|
@@ -32459,6 +32664,7 @@ function buildSystemPrompt(ctx, options) {
|
|
|
32459
32664
|
DEFAULT_ANALYSIS_PROMPT,
|
|
32460
32665
|
DEFAULT_ANALYZE_AFTER_MESSAGES,
|
|
32461
32666
|
DEFAULT_CACHE_EXPIRATION_MS,
|
|
32667
|
+
DEFAULT_CHUNK_CACHE_SIZE,
|
|
32462
32668
|
DEFAULT_CHUNK_OVERLAP,
|
|
32463
32669
|
DEFAULT_CHUNK_SIZE,
|
|
32464
32670
|
DEFAULT_DOCUMENT_PATTERN,
|
|
@@ -32489,6 +32695,7 @@ function buildSystemPrompt(ctx, options) {
|
|
|
32489
32695
|
QueueManager,
|
|
32490
32696
|
RECALL_MAX_LIMIT,
|
|
32491
32697
|
RECALL_TOOL_NAME,
|
|
32698
|
+
RerankerUnavailableError,
|
|
32492
32699
|
SDK_SCHEMA_VERSION,
|
|
32493
32700
|
SERVER_TOOL_DEPENDENCY_SETS,
|
|
32494
32701
|
SLIDER_CONFIG,
|
|
@@ -32530,6 +32737,7 @@ function buildSystemPrompt(ctx, options) {
|
|
|
32530
32737
|
clearServerToolsCache,
|
|
32531
32738
|
composeHooks,
|
|
32532
32739
|
createAutoExtractor,
|
|
32740
|
+
createChunkVectorCache,
|
|
32533
32741
|
createConversationOp,
|
|
32534
32742
|
createCryptoPricePreProcessor,
|
|
32535
32743
|
createMediaBatchOp,
|
|
@@ -32635,6 +32843,7 @@ function buildSystemPrompt(ctx, options) {
|
|
|
32635
32843
|
hardDeleteMediaOp,
|
|
32636
32844
|
hasEncryptionKey,
|
|
32637
32845
|
hasKeyPair,
|
|
32846
|
+
isRerankerAvailable,
|
|
32638
32847
|
isSupportedFile,
|
|
32639
32848
|
isSupportedMediaType,
|
|
32640
32849
|
makeSyntheticStoredConversation,
|