@anuma/sdk 2.3.1 → 2.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/expo/{chunk-SF5AYTXH.mjs → chunk-UOUJUZM4.mjs} +229 -35
- package/dist/expo/index.cjs +310 -44
- package/dist/expo/index.d.mts +212 -10
- package/dist/expo/index.d.ts +212 -10
- package/dist/expo/index.mjs +78 -9
- package/dist/expo/{recall-PCKDVF3U.mjs → recall-HI7VO3VM.mjs} +1 -1
- package/dist/react/{chunk-JUDJVX66.mjs → chunk-IYDQBFS2.mjs} +229 -35
- package/dist/react/index.cjs +306 -43
- package/dist/react/index.d.mts +219 -10
- package/dist/react/index.d.ts +219 -10
- package/dist/react/index.mjs +74 -8
- package/dist/react/{recall-DSJSBDR4.mjs → recall-G3CS444F.mjs} +1 -1
- package/dist/server/{chunk-3AWYZRTS.mjs → chunk-3MLG7SUH.mjs} +229 -35
- package/dist/server/index.cjs +299 -42
- package/dist/server/index.d.mts +219 -10
- package/dist/server/index.d.ts +219 -10
- package/dist/server/index.mjs +67 -7
- package/dist/server/{recall-EWMI64SA.mjs → recall-MJHGXMHZ.mjs} +1 -1
- package/package.json +1 -1
package/dist/server/index.cjs
CHANGED
|
@@ -16385,7 +16385,13 @@ async function searchMessagesOp(ctx, queryVector, options) {
|
|
|
16385
16385
|
);
|
|
16386
16386
|
}
|
|
16387
16387
|
async function searchChunksOp(ctx, queryVector, options) {
|
|
16388
|
-
const {
|
|
16388
|
+
const {
|
|
16389
|
+
limit = 10,
|
|
16390
|
+
minSimilarity = 0.5,
|
|
16391
|
+
conversationId,
|
|
16392
|
+
embeddingModel,
|
|
16393
|
+
chunkCache
|
|
16394
|
+
} = options || {};
|
|
16389
16395
|
const activeConversations = await ctx.conversationsCollection.query(import_watermelondb14.Q.where("is_deleted", false)).fetch();
|
|
16390
16396
|
const activeConversationIds = new Set(
|
|
16391
16397
|
// Use _getRaw for reliable raw column access
|
|
@@ -16405,23 +16411,54 @@ async function searchChunksOp(ctx, queryVector, options) {
|
|
|
16405
16411
|
continue;
|
|
16406
16412
|
}
|
|
16407
16413
|
}
|
|
16414
|
+
const version = Number(message._getRaw("updated_at")) || 0;
|
|
16415
|
+
const cached = chunkCache?.get(message.id);
|
|
16416
|
+
if (cached && cached.version === version) {
|
|
16417
|
+
if (cached.chunks.length > 0) {
|
|
16418
|
+
for (let ci = 0; ci < cached.chunks.length; ci++) {
|
|
16419
|
+
const vec = cached.chunks[ci];
|
|
16420
|
+
if (vec.length === 0) continue;
|
|
16421
|
+
const similarity = cosineSimilarity(queryVector, vec);
|
|
16422
|
+
if (similarity >= minSimilarity) {
|
|
16423
|
+
candidates.push({
|
|
16424
|
+
message,
|
|
16425
|
+
similarity,
|
|
16426
|
+
chunkTextSource: { kind: "chunk-cached", chunkIndex: ci, version }
|
|
16427
|
+
});
|
|
16428
|
+
}
|
|
16429
|
+
}
|
|
16430
|
+
} else if (cached.fallback) {
|
|
16431
|
+
const similarity = cosineSimilarity(queryVector, cached.fallback);
|
|
16432
|
+
if (similarity >= minSimilarity) {
|
|
16433
|
+
candidates.push({ message, similarity, chunkTextSource: { kind: "message" } });
|
|
16434
|
+
}
|
|
16435
|
+
}
|
|
16436
|
+
continue;
|
|
16437
|
+
}
|
|
16408
16438
|
const chunks = await readJsonField(message, "chunks", ctx.walletAddress);
|
|
16409
16439
|
if (chunks && chunks.length > 0) {
|
|
16410
|
-
|
|
16411
|
-
|
|
16412
|
-
|
|
16440
|
+
const vectors = chunks.map(
|
|
16441
|
+
(chunk) => chunk.vector && chunk.vector.length > 0 ? Float32Array.from(chunk.vector) : new Float32Array(0)
|
|
16442
|
+
);
|
|
16443
|
+
chunkCache?.set(message.id, { version, chunks: vectors });
|
|
16444
|
+
for (let ci = 0; ci < chunks.length; ci++) {
|
|
16445
|
+
const vec = vectors[ci];
|
|
16446
|
+
if (vec.length === 0) continue;
|
|
16447
|
+
const similarity = cosineSimilarity(queryVector, vec);
|
|
16413
16448
|
if (similarity >= minSimilarity) {
|
|
16414
16449
|
candidates.push({
|
|
16415
16450
|
message,
|
|
16416
16451
|
similarity,
|
|
16417
|
-
chunkTextSource: { kind: "chunk", text:
|
|
16452
|
+
chunkTextSource: { kind: "chunk", text: chunks[ci].text }
|
|
16418
16453
|
});
|
|
16419
16454
|
}
|
|
16420
16455
|
}
|
|
16421
16456
|
} else {
|
|
16422
16457
|
const messageVector = await readJsonField(message, "vector", ctx.walletAddress);
|
|
16423
16458
|
if (!messageVector || messageVector.length === 0) continue;
|
|
16424
|
-
const
|
|
16459
|
+
const fallback = Float32Array.from(messageVector);
|
|
16460
|
+
chunkCache?.set(message.id, { version, chunks: [], fallback });
|
|
16461
|
+
const similarity = cosineSimilarity(queryVector, fallback);
|
|
16425
16462
|
if (similarity >= minSimilarity) {
|
|
16426
16463
|
candidates.push({ message, similarity, chunkTextSource: { kind: "message" } });
|
|
16427
16464
|
}
|
|
@@ -16448,9 +16485,31 @@ async function searchChunksOp(ctx, queryVector, options) {
|
|
|
16448
16485
|
})
|
|
16449
16486
|
)
|
|
16450
16487
|
);
|
|
16488
|
+
const chunkStateByMsgId = /* @__PURE__ */ new Map();
|
|
16489
|
+
const needsChunkText = /* @__PURE__ */ new Set();
|
|
16490
|
+
for (const c of topK) {
|
|
16491
|
+
if (c.chunkTextSource.kind === "chunk-cached") needsChunkText.add(c.message.id);
|
|
16492
|
+
}
|
|
16493
|
+
await Promise.all(
|
|
16494
|
+
Array.from(needsChunkText, async (id) => {
|
|
16495
|
+
const m = uniqueMessages.get(id);
|
|
16496
|
+
const parsed = await readJsonField(m, "chunks", ctx.walletAddress);
|
|
16497
|
+
const version = Number(m._getRaw("updated_at")) || 0;
|
|
16498
|
+
if (parsed) chunkStateByMsgId.set(id, { version, chunks: parsed });
|
|
16499
|
+
})
|
|
16500
|
+
);
|
|
16451
16501
|
return topK.map(({ message, similarity, chunkTextSource }) => {
|
|
16452
16502
|
const stored = { ...storedById.get(message.id) };
|
|
16453
|
-
|
|
16503
|
+
let chunkText2;
|
|
16504
|
+
if (chunkTextSource.kind === "chunk") {
|
|
16505
|
+
chunkText2 = chunkTextSource.text;
|
|
16506
|
+
} else if (chunkTextSource.kind === "chunk-cached") {
|
|
16507
|
+
const state = chunkStateByMsgId.get(message.id);
|
|
16508
|
+
const chunk = state && state.version === chunkTextSource.version ? state.chunks[chunkTextSource.chunkIndex] : void 0;
|
|
16509
|
+
chunkText2 = chunk?.text ?? stored.content;
|
|
16510
|
+
} else {
|
|
16511
|
+
chunkText2 = stored.content;
|
|
16512
|
+
}
|
|
16454
16513
|
return { chunkText: chunkText2, message: stored, similarity };
|
|
16455
16514
|
});
|
|
16456
16515
|
}
|
|
@@ -18765,9 +18824,9 @@ function recencyMultiplier(updatedAt, opts = {}) {
|
|
|
18765
18824
|
const updatedMs = updatedAt.getTime();
|
|
18766
18825
|
if (!Number.isFinite(updatedMs)) return noDate;
|
|
18767
18826
|
const now = opts.now ?? /* @__PURE__ */ new Date();
|
|
18768
|
-
const
|
|
18769
|
-
if (!Number.isFinite(
|
|
18770
|
-
const ageDays = Math.max(0, (
|
|
18827
|
+
const nowMs2 = now.getTime();
|
|
18828
|
+
if (!Number.isFinite(nowMs2)) return noDate;
|
|
18829
|
+
const ageDays = Math.max(0, (nowMs2 - updatedMs) / DAY_MS);
|
|
18771
18830
|
const perYearDecay = opts.perYearDecay ?? DEFAULT_PER_YEAR_DECAY;
|
|
18772
18831
|
const floor = opts.floor ?? DEFAULT_FLOOR;
|
|
18773
18832
|
const decay = perYearDecay * (ageDays / 365);
|
|
@@ -18785,14 +18844,42 @@ var init_recency = __esm({
|
|
|
18785
18844
|
});
|
|
18786
18845
|
|
|
18787
18846
|
// src/lib/memory/reranker.ts
|
|
18847
|
+
function isRerankerAvailable() {
|
|
18848
|
+
return available;
|
|
18849
|
+
}
|
|
18850
|
+
function isModuleMissing(err) {
|
|
18851
|
+
let e = err;
|
|
18852
|
+
for (let depth = 0; depth < 5 && e !== null && e !== void 0; depth++) {
|
|
18853
|
+
const code = e.code;
|
|
18854
|
+
if (code === "ERR_MODULE_NOT_FOUND" || code === "MODULE_NOT_FOUND") return true;
|
|
18855
|
+
if (e instanceof Error && /cannot find (module|package)|failed to resolve|unable to resolve module|requiring unknown module/i.test(
|
|
18856
|
+
e.message
|
|
18857
|
+
)) {
|
|
18858
|
+
return true;
|
|
18859
|
+
}
|
|
18860
|
+
e = e.cause;
|
|
18861
|
+
}
|
|
18862
|
+
return false;
|
|
18863
|
+
}
|
|
18788
18864
|
async function getModel() {
|
|
18865
|
+
if (available === false) throw new RerankerUnavailableError(void 0);
|
|
18789
18866
|
if (!modelPromise) {
|
|
18790
18867
|
const load = (async () => {
|
|
18791
|
-
|
|
18868
|
+
let transformers;
|
|
18869
|
+
try {
|
|
18870
|
+
transformers = await import("@huggingface/transformers");
|
|
18871
|
+
} catch (err) {
|
|
18872
|
+
if (isModuleMissing(err)) {
|
|
18873
|
+
available = false;
|
|
18874
|
+
throw new RerankerUnavailableError(err);
|
|
18875
|
+
}
|
|
18876
|
+
throw err;
|
|
18877
|
+
}
|
|
18792
18878
|
const [tokenizer, model] = await Promise.all([
|
|
18793
18879
|
transformers.AutoTokenizer.from_pretrained(MODEL_ID),
|
|
18794
18880
|
transformers.AutoModelForSequenceClassification.from_pretrained(MODEL_ID)
|
|
18795
18881
|
]);
|
|
18882
|
+
available = true;
|
|
18796
18883
|
return { tokenizer, model };
|
|
18797
18884
|
})();
|
|
18798
18885
|
modelPromise = load;
|
|
@@ -18834,11 +18921,19 @@ async function rerankPairs(query, items) {
|
|
|
18834
18921
|
}
|
|
18835
18922
|
return items.map((item, i) => ({ id: item.id, content: item.content, score: scores[i] })).sort((a, b) => b.score - a.score);
|
|
18836
18923
|
}
|
|
18837
|
-
var MODEL_ID, modelPromise;
|
|
18924
|
+
var MODEL_ID, RerankerUnavailableError, available, modelPromise;
|
|
18838
18925
|
var init_reranker = __esm({
|
|
18839
18926
|
"src/lib/memory/reranker.ts"() {
|
|
18840
18927
|
"use strict";
|
|
18841
18928
|
MODEL_ID = "Xenova/ms-marco-MiniLM-L-6-v2";
|
|
18929
|
+
RerankerUnavailableError = class extends Error {
|
|
18930
|
+
constructor(reason) {
|
|
18931
|
+
super("cross-encoder reranker unavailable (@huggingface/transformers not installed)");
|
|
18932
|
+
this.name = "RerankerUnavailableError";
|
|
18933
|
+
this.reason = reason;
|
|
18934
|
+
}
|
|
18935
|
+
};
|
|
18936
|
+
available = void 0;
|
|
18842
18937
|
modelPromise = null;
|
|
18843
18938
|
}
|
|
18844
18939
|
});
|
|
@@ -19722,22 +19817,67 @@ async function recall(query, ctx, options = {}) {
|
|
|
19722
19817
|
const flags = flagsForBudget(requestedBudget);
|
|
19723
19818
|
const decomposeAvailable = flags.decompose && !!options.decomposeOptions;
|
|
19724
19819
|
const usedBudget = requestedBudget === "high" && !decomposeAvailable ? "mid" : requestedBudget;
|
|
19820
|
+
const t0 = nowMs();
|
|
19821
|
+
let prepMs = 0;
|
|
19822
|
+
let factLaneMs = 0;
|
|
19823
|
+
let chunkLaneMs = 0;
|
|
19824
|
+
let fuseMs = 0;
|
|
19825
|
+
const factResults = [];
|
|
19826
|
+
const chunkResults = [];
|
|
19827
|
+
let vaultSize;
|
|
19828
|
+
let didRerank = false;
|
|
19829
|
+
let hadV2Head = false;
|
|
19830
|
+
const emitDiagnostics = (candidateCount) => {
|
|
19831
|
+
const cb = options.onDiagnostics;
|
|
19832
|
+
if (!cb) return;
|
|
19833
|
+
const degraded = [];
|
|
19834
|
+
if (flags.rerank && factResults.length > 0 && hadV2Head && !didRerank) {
|
|
19835
|
+
degraded.push("rerank-unavailable");
|
|
19836
|
+
}
|
|
19837
|
+
if (flags.decompose && !decomposeAvailable) degraded.push("decompose-unavailable");
|
|
19838
|
+
const diagnostics = {
|
|
19839
|
+
usedBudget,
|
|
19840
|
+
reranked: didRerank,
|
|
19841
|
+
candidateCount,
|
|
19842
|
+
...vaultSize !== void 0 && { vaultSize },
|
|
19843
|
+
factCount: factResults.length,
|
|
19844
|
+
chunkCount: chunkResults.length,
|
|
19845
|
+
timings: {
|
|
19846
|
+
total: nowMs() - t0,
|
|
19847
|
+
prep: prepMs,
|
|
19848
|
+
factLane: factLaneMs,
|
|
19849
|
+
chunkLane: chunkLaneMs,
|
|
19850
|
+
fuse: fuseMs
|
|
19851
|
+
},
|
|
19852
|
+
degraded
|
|
19853
|
+
};
|
|
19854
|
+
try {
|
|
19855
|
+
cb(diagnostics);
|
|
19856
|
+
} catch {
|
|
19857
|
+
}
|
|
19858
|
+
};
|
|
19725
19859
|
if (!query || typeof query !== "string" || query.trim().length === 0) {
|
|
19860
|
+
emitDiagnostics(0);
|
|
19726
19861
|
return { memories: [], usedBudget, reranked: false, candidateCount: 0 };
|
|
19727
19862
|
}
|
|
19728
19863
|
const needsChunkEmbedding = types.includes("chunk") && ctx.storageCtx;
|
|
19729
19864
|
const wantsTemporal = types.includes("fact") && ctx.vaultCtx;
|
|
19865
|
+
const prepStart = nowMs();
|
|
19730
19866
|
const [queryEmbedding, entityRanking, temporalRanking] = await Promise.all([
|
|
19731
19867
|
needsChunkEmbedding ? generateEmbedding(query, ctx.embeddingOptions) : Promise.resolve(void 0),
|
|
19732
19868
|
buildGraphLaneRanking(query, ctx),
|
|
19733
19869
|
wantsTemporal ? buildTemporalLaneRanking(query, ctx.vaultCtx, options.now) : Promise.resolve([])
|
|
19734
19870
|
]);
|
|
19735
|
-
|
|
19736
|
-
const chunkResults = [];
|
|
19737
|
-
let vaultSize;
|
|
19871
|
+
prepMs = nowMs() - prepStart;
|
|
19738
19872
|
if (types.includes("fact") && ctx.vaultCtx && ctx.vaultCache) {
|
|
19873
|
+
const factStart = nowMs();
|
|
19739
19874
|
const vaultMinScore = options.minScore ?? DEFAULT_FACT_MIN_SCORE;
|
|
19740
|
-
const {
|
|
19875
|
+
const {
|
|
19876
|
+
results,
|
|
19877
|
+
vaultSize: size,
|
|
19878
|
+
reranked,
|
|
19879
|
+
hadV2Head: v2Head
|
|
19880
|
+
} = await searchVaultMemoriesWithSize(
|
|
19741
19881
|
query,
|
|
19742
19882
|
ctx.vaultCtx,
|
|
19743
19883
|
ctx.embeddingOptions,
|
|
@@ -19788,14 +19928,19 @@ async function recall(query, ctx, options = {}) {
|
|
|
19788
19928
|
)
|
|
19789
19929
|
);
|
|
19790
19930
|
vaultSize = size;
|
|
19931
|
+
didRerank = reranked;
|
|
19932
|
+
hadV2Head = v2Head;
|
|
19933
|
+
factLaneMs = nowMs() - factStart;
|
|
19791
19934
|
}
|
|
19792
19935
|
if (types.includes("chunk") && ctx.storageCtx && queryEmbedding) {
|
|
19936
|
+
const chunkStart = nowMs();
|
|
19793
19937
|
const chunkMinScore = options.minScore ?? DEFAULT_CHUNK_MIN_SCORE;
|
|
19794
19938
|
const results = await searchChunksOp(ctx.storageCtx, queryEmbedding, {
|
|
19795
19939
|
limit: types.includes("fact") ? Math.max(limit * 2, 16) : limit,
|
|
19796
19940
|
minSimilarity: chunkMinScore,
|
|
19797
19941
|
embeddingModel: ctx.embeddingOptions.model ?? DEFAULT_API_EMBEDDING_MODEL,
|
|
19798
|
-
...options.conversationId && { conversationId: options.conversationId }
|
|
19942
|
+
...options.conversationId && { conversationId: options.conversationId },
|
|
19943
|
+
...ctx.chunkCache && { chunkCache: ctx.chunkCache }
|
|
19799
19944
|
});
|
|
19800
19945
|
chunkResults.push(
|
|
19801
19946
|
...dedupeBy(
|
|
@@ -19805,18 +19950,23 @@ async function recall(query, ctx, options = {}) {
|
|
|
19805
19950
|
(r) => r.chunkText.trim()
|
|
19806
19951
|
)
|
|
19807
19952
|
);
|
|
19953
|
+
chunkLaneMs = nowMs() - chunkStart;
|
|
19808
19954
|
}
|
|
19955
|
+
const fuseStart = nowMs();
|
|
19809
19956
|
if (types.length === 1 || factResults.length === 0 || chunkResults.length === 0) {
|
|
19810
19957
|
const memories2 = [
|
|
19811
19958
|
...factResults.map(toFactMemory),
|
|
19812
19959
|
...chunkResults.map(toChunkMemory)
|
|
19813
19960
|
];
|
|
19814
19961
|
memories2.sort((a, b) => b.score - a.score);
|
|
19962
|
+
const candidateCount = factResults.length + chunkResults.length;
|
|
19963
|
+
fuseMs = nowMs() - fuseStart;
|
|
19964
|
+
emitDiagnostics(candidateCount);
|
|
19815
19965
|
return {
|
|
19816
19966
|
memories: memories2.slice(0, limit),
|
|
19817
19967
|
usedBudget,
|
|
19818
|
-
reranked:
|
|
19819
|
-
candidateCount
|
|
19968
|
+
reranked: didRerank,
|
|
19969
|
+
candidateCount,
|
|
19820
19970
|
...vaultSize !== void 0 && { vaultSize }
|
|
19821
19971
|
};
|
|
19822
19972
|
}
|
|
@@ -19865,10 +20015,12 @@ async function recall(query, ctx, options = {}) {
|
|
|
19865
20015
|
suppressed = next;
|
|
19866
20016
|
memories = selectWith(suppressed);
|
|
19867
20017
|
}
|
|
20018
|
+
fuseMs = nowMs() - fuseStart;
|
|
20019
|
+
emitDiagnostics(byId.size);
|
|
19868
20020
|
return {
|
|
19869
20021
|
memories,
|
|
19870
20022
|
usedBudget,
|
|
19871
|
-
reranked:
|
|
20023
|
+
reranked: didRerank,
|
|
19872
20024
|
candidateCount: byId.size,
|
|
19873
20025
|
...vaultSize !== void 0 && { vaultSize }
|
|
19874
20026
|
};
|
|
@@ -19932,7 +20084,7 @@ function toChunkMemory(r) {
|
|
|
19932
20084
|
updatedAt: r.message.createdAt
|
|
19933
20085
|
};
|
|
19934
20086
|
}
|
|
19935
|
-
var DEFAULT_LIMIT, DEFAULT_BUDGET, DEFAULT_FACT_MIN_SCORE, DEFAULT_CHUNK_MIN_SCORE;
|
|
20087
|
+
var DEFAULT_LIMIT, DEFAULT_BUDGET, DEFAULT_FACT_MIN_SCORE, nowMs, DEFAULT_CHUNK_MIN_SCORE;
|
|
19936
20088
|
var init_recall = __esm({
|
|
19937
20089
|
"src/lib/memory/recall.ts"() {
|
|
19938
20090
|
"use strict";
|
|
@@ -19948,6 +20100,7 @@ var init_recall = __esm({
|
|
|
19948
20100
|
DEFAULT_LIMIT = 8;
|
|
19949
20101
|
DEFAULT_BUDGET = "low";
|
|
19950
20102
|
DEFAULT_FACT_MIN_SCORE = 0.1;
|
|
20103
|
+
nowMs = () => typeof performance !== "undefined" && typeof performance.now === "function" ? performance.now() : Date.now();
|
|
19951
20104
|
DEFAULT_CHUNK_MIN_SCORE = 0.5;
|
|
19952
20105
|
}
|
|
19953
20106
|
});
|
|
@@ -20147,6 +20300,9 @@ async function rankFusedVaultMemoriesAsync(query, queryEmbedding, items, options
|
|
|
20147
20300
|
bm25AdmissionDivisor: options?.bm25AdmissionDivisor,
|
|
20148
20301
|
rrfK: options?.rrfK
|
|
20149
20302
|
});
|
|
20303
|
+
if (options?.v2HeadStats) {
|
|
20304
|
+
options.v2HeadStats.hadResults = v2Ranked.length > 0;
|
|
20305
|
+
}
|
|
20150
20306
|
let combined;
|
|
20151
20307
|
let tailSlice = [];
|
|
20152
20308
|
if (options?.rerank) {
|
|
@@ -20167,8 +20323,13 @@ async function rankFusedVaultMemoriesAsync(query, queryEmbedding, items, options
|
|
|
20167
20323
|
return { ...r, similarity: v2 * (1 + ceWeight * ce) };
|
|
20168
20324
|
});
|
|
20169
20325
|
combined.sort((a, b) => b.similarity - a.similarity);
|
|
20326
|
+
if (headSlice.length > 0 && options.rerankStats) options.rerankStats.applied = true;
|
|
20170
20327
|
} catch (err) {
|
|
20171
|
-
|
|
20328
|
+
if (err instanceof RerankerUnavailableError) {
|
|
20329
|
+
getLogger().debug("[memory/search] cross-encoder unavailable; using V2 ranking");
|
|
20330
|
+
} else {
|
|
20331
|
+
getLogger().warn("[memory/search] cross-encoder rerank failed; using V2 ranking", err);
|
|
20332
|
+
}
|
|
20172
20333
|
combined = v2Ranked;
|
|
20173
20334
|
}
|
|
20174
20335
|
} else {
|
|
@@ -20302,6 +20463,9 @@ async function rankComposite(originalQuery, originalQueryEmbedding, subQueries,
|
|
|
20302
20463
|
(r) => r !== null
|
|
20303
20464
|
);
|
|
20304
20465
|
combined.sort((a, b) => b.similarity - a.similarity);
|
|
20466
|
+
if (options?.v2HeadStats) {
|
|
20467
|
+
options.v2HeadStats.hadResults = combined.length > 0;
|
|
20468
|
+
}
|
|
20305
20469
|
if (options?.includeUnrankedTail) {
|
|
20306
20470
|
const fusedIds = new Set(combined.map((r) => r.uniqueId));
|
|
20307
20471
|
for (const item of items) {
|
|
@@ -20337,11 +20501,16 @@ async function rankComposite(originalQuery, originalQueryEmbedding, subQueries,
|
|
|
20337
20501
|
}));
|
|
20338
20502
|
head.sort((a, b) => b.similarity - a.similarity);
|
|
20339
20503
|
combined = [...head, ...tailSlice];
|
|
20504
|
+
if (headSlice.length > 0 && options.rerankStats) options.rerankStats.applied = true;
|
|
20340
20505
|
} catch (err) {
|
|
20341
|
-
|
|
20342
|
-
"[memory/search]
|
|
20343
|
-
|
|
20344
|
-
|
|
20506
|
+
if (err instanceof RerankerUnavailableError) {
|
|
20507
|
+
getLogger().debug("[memory/search] cross-encoder unavailable; using fused ranking");
|
|
20508
|
+
} else {
|
|
20509
|
+
getLogger().warn(
|
|
20510
|
+
"[memory/search] composite cross-encoder rerank failed; using fused ranking",
|
|
20511
|
+
err
|
|
20512
|
+
);
|
|
20513
|
+
}
|
|
20345
20514
|
}
|
|
20346
20515
|
}
|
|
20347
20516
|
if (options?.mmr) {
|
|
@@ -20433,7 +20602,7 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
20433
20602
|
const minSimilarity = searchOptions?.minSimilarity ?? 0.1;
|
|
20434
20603
|
const scopes = searchOptions?.scopes;
|
|
20435
20604
|
if (!query || typeof query !== "string") {
|
|
20436
|
-
return { results: [], vaultSize: 0 };
|
|
20605
|
+
return { results: [], vaultSize: 0, reranked: false, hadV2Head: false };
|
|
20437
20606
|
}
|
|
20438
20607
|
const folderId = searchOptions?.folderId;
|
|
20439
20608
|
const queryOpts = {};
|
|
@@ -20450,7 +20619,7 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
20450
20619
|
);
|
|
20451
20620
|
}
|
|
20452
20621
|
if (memories.length === 0) {
|
|
20453
|
-
return { results: [], vaultSize: loaded.length };
|
|
20622
|
+
return { results: [], vaultSize: loaded.length, reranked: false, hadV2Head: false };
|
|
20454
20623
|
}
|
|
20455
20624
|
const queryEmbedding = await generateEmbedding(query, embeddingOptions);
|
|
20456
20625
|
const currentModel = embeddingOptions.model ?? DEFAULT_API_EMBEDDING_MODEL;
|
|
@@ -20526,12 +20695,18 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
20526
20695
|
memories.map((m) => [m.uniqueId, { createdAt: m.createdAt, updatedAt: m.updatedAt }])
|
|
20527
20696
|
);
|
|
20528
20697
|
const stampTimestamps = (out) => {
|
|
20529
|
-
|
|
20698
|
+
const results2 = out.results.map((r) => {
|
|
20530
20699
|
const ts = timestampById.get(r.uniqueId);
|
|
20531
20700
|
return ts ? { ...r, createdAt: ts.createdAt, updatedAt: ts.updatedAt } : r;
|
|
20532
20701
|
});
|
|
20533
|
-
return
|
|
20702
|
+
return {
|
|
20703
|
+
results: results2,
|
|
20704
|
+
vaultSize: out.vaultSize,
|
|
20705
|
+
reranked: out.reranked ?? false,
|
|
20706
|
+
hadV2Head: out.hadV2Head ?? false
|
|
20707
|
+
};
|
|
20534
20708
|
};
|
|
20709
|
+
const rerankStats = { applied: false };
|
|
20535
20710
|
const useFusion = searchOptions?.useFusion ?? true;
|
|
20536
20711
|
const tuning = {
|
|
20537
20712
|
...searchOptions?.recencyAlpha !== void 0 && { recencyAlpha: searchOptions.recencyAlpha },
|
|
@@ -20558,6 +20733,7 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
20558
20733
|
query: sq,
|
|
20559
20734
|
embedding: subEmbeddings[i]
|
|
20560
20735
|
}));
|
|
20736
|
+
const v2HeadStats = { hadResults: false };
|
|
20561
20737
|
const composite = await rankComposite(query, queryEmbedding, subQueries, embeddedItems, {
|
|
20562
20738
|
limit,
|
|
20563
20739
|
minSimilarity,
|
|
@@ -20569,12 +20745,20 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
20569
20745
|
...searchOptions.mmr !== void 0 && { mmr: searchOptions.mmr },
|
|
20570
20746
|
...tuning,
|
|
20571
20747
|
...searchOptions.entityRanking && { entityRanking: searchOptions.entityRanking },
|
|
20572
|
-
...searchOptions.temporalRanking && { temporalRanking: searchOptions.temporalRanking }
|
|
20748
|
+
...searchOptions.temporalRanking && { temporalRanking: searchOptions.temporalRanking },
|
|
20749
|
+
rerankStats,
|
|
20750
|
+
v2HeadStats
|
|
20751
|
+
});
|
|
20752
|
+
return stampTimestamps({
|
|
20753
|
+
results: composite,
|
|
20754
|
+
vaultSize: loaded.length,
|
|
20755
|
+
reranked: rerankStats.applied,
|
|
20756
|
+
hadV2Head: v2HeadStats.hadResults
|
|
20573
20757
|
});
|
|
20574
|
-
return stampTimestamps({ results: composite, vaultSize: loaded.length });
|
|
20575
20758
|
}
|
|
20576
20759
|
}
|
|
20577
20760
|
if (useFusion && searchOptions?.rerank) {
|
|
20761
|
+
const v2HeadStats = { hadResults: false };
|
|
20578
20762
|
const results2 = await rankFusedVaultMemoriesAsync(query, queryEmbedding, embeddedItems, {
|
|
20579
20763
|
limit,
|
|
20580
20764
|
minSimilarity,
|
|
@@ -20586,9 +20770,16 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
20586
20770
|
...searchOptions.mmr !== void 0 && { mmr: searchOptions.mmr },
|
|
20587
20771
|
...tuning,
|
|
20588
20772
|
...searchOptions.entityRanking && { entityRanking: searchOptions.entityRanking },
|
|
20589
|
-
...searchOptions.temporalRanking && { temporalRanking: searchOptions.temporalRanking }
|
|
20773
|
+
...searchOptions.temporalRanking && { temporalRanking: searchOptions.temporalRanking },
|
|
20774
|
+
rerankStats,
|
|
20775
|
+
v2HeadStats
|
|
20776
|
+
});
|
|
20777
|
+
return stampTimestamps({
|
|
20778
|
+
results: results2,
|
|
20779
|
+
vaultSize: loaded.length,
|
|
20780
|
+
reranked: rerankStats.applied,
|
|
20781
|
+
hadV2Head: v2HeadStats.hadResults
|
|
20590
20782
|
});
|
|
20591
|
-
return stampTimestamps({ results: results2, vaultSize: loaded.length });
|
|
20592
20783
|
}
|
|
20593
20784
|
if (useFusion) {
|
|
20594
20785
|
const results2 = rankFusedVaultMemories(query, queryEmbedding, embeddedItems, {
|
|
@@ -20598,7 +20789,7 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
20598
20789
|
...searchOptions?.entityRanking && { entityRanking: searchOptions.entityRanking },
|
|
20599
20790
|
...searchOptions?.temporalRanking && { temporalRanking: searchOptions.temporalRanking }
|
|
20600
20791
|
});
|
|
20601
|
-
return stampTimestamps({ results: results2, vaultSize: loaded.length });
|
|
20792
|
+
return stampTimestamps({ results: results2, vaultSize: loaded.length, hadV2Head: results2.length > 0 });
|
|
20602
20793
|
}
|
|
20603
20794
|
const results = rankVaultMemories(query, queryEmbedding, embeddedItems, {
|
|
20604
20795
|
limit,
|
|
@@ -20610,7 +20801,7 @@ async function searchVaultMemoriesWithSize(query, vaultCtx, embeddingOptions, ca
|
|
|
20610
20801
|
supersessionWindow: searchOptions.supersessionWindow
|
|
20611
20802
|
}
|
|
20612
20803
|
});
|
|
20613
|
-
return stampTimestamps({ results, vaultSize: loaded.length });
|
|
20804
|
+
return stampTimestamps({ results, vaultSize: loaded.length, hadV2Head: results.length > 0 });
|
|
20614
20805
|
}
|
|
20615
20806
|
async function searchVaultMemories(query, vaultCtx, embeddingOptions, cache, searchOptions) {
|
|
20616
20807
|
const { results } = await searchVaultMemoriesWithSize(
|
|
@@ -21560,6 +21751,7 @@ __export(index_exports, {
|
|
|
21560
21751
|
DEFAULT_ANALYSIS_PROMPT: () => DEFAULT_ANALYSIS_PROMPT,
|
|
21561
21752
|
DEFAULT_ANALYZE_AFTER_MESSAGES: () => DEFAULT_ANALYZE_AFTER_MESSAGES,
|
|
21562
21753
|
DEFAULT_CACHE_EXPIRATION_MS: () => DEFAULT_CACHE_EXPIRATION_MS,
|
|
21754
|
+
DEFAULT_CHUNK_CACHE_SIZE: () => DEFAULT_CHUNK_CACHE_SIZE,
|
|
21563
21755
|
DEFAULT_CHUNK_OVERLAP: () => DEFAULT_CHUNK_OVERLAP,
|
|
21564
21756
|
DEFAULT_CHUNK_SIZE: () => DEFAULT_CHUNK_SIZE,
|
|
21565
21757
|
DEFAULT_DOCUMENT_PATTERN: () => DEFAULT_DOCUMENT_PATTERN,
|
|
@@ -21590,6 +21782,7 @@ __export(index_exports, {
|
|
|
21590
21782
|
QueueManager: () => QueueManager,
|
|
21591
21783
|
RECALL_MAX_LIMIT: () => RECALL_MAX_LIMIT,
|
|
21592
21784
|
RECALL_TOOL_NAME: () => RECALL_TOOL_NAME,
|
|
21785
|
+
RerankerUnavailableError: () => RerankerUnavailableError,
|
|
21593
21786
|
SDK_SCHEMA_VERSION: () => SDK_SCHEMA_VERSION,
|
|
21594
21787
|
SERVER_TOOL_DEPENDENCY_SETS: () => SERVER_TOOL_DEPENDENCY_SETS,
|
|
21595
21788
|
SLIDER_CONFIG: () => SLIDER_CONFIG,
|
|
@@ -21631,6 +21824,7 @@ __export(index_exports, {
|
|
|
21631
21824
|
clearServerToolsCache: () => clearServerToolsCache,
|
|
21632
21825
|
composeHooks: () => composeHooks,
|
|
21633
21826
|
createAutoExtractor: () => createAutoExtractor,
|
|
21827
|
+
createChunkVectorCache: () => createChunkVectorCache,
|
|
21634
21828
|
createConversationOp: () => createConversationOp,
|
|
21635
21829
|
createCryptoPricePreProcessor: () => createCryptoPricePreProcessor,
|
|
21636
21830
|
createMediaBatchOp: () => createMediaBatchOp,
|
|
@@ -21639,6 +21833,7 @@ __export(index_exports, {
|
|
|
21639
21833
|
createMemoryVaultSearchTool: () => createMemoryVaultSearchTool,
|
|
21640
21834
|
createMemoryVaultTool: () => createMemoryVaultTool,
|
|
21641
21835
|
createMessageOp: () => createMessageOp,
|
|
21836
|
+
createPlatformCursorStore: () => createPlatformCursorStore,
|
|
21642
21837
|
createProjectOp: () => createProjectOp,
|
|
21643
21838
|
createRecallTool: () => createRecallTool,
|
|
21644
21839
|
createServerToolsFilter: () => createServerToolsFilter,
|
|
@@ -21735,6 +21930,7 @@ __export(index_exports, {
|
|
|
21735
21930
|
hardDeleteMediaOp: () => hardDeleteMediaOp,
|
|
21736
21931
|
hasEncryptionKey: () => hasEncryptionKey,
|
|
21737
21932
|
hasKeyPair: () => hasKeyPair,
|
|
21933
|
+
isRerankerAvailable: () => isRerankerAvailable,
|
|
21738
21934
|
isSupportedFile: () => isSupportedFile,
|
|
21739
21935
|
isSupportedMediaType: () => isSupportedMediaType,
|
|
21740
21936
|
makeSyntheticStoredConversation: () => makeSyntheticStoredConversation,
|
|
@@ -26269,6 +26465,23 @@ function parseLocalCalendarDay2(year, month, day) {
|
|
|
26269
26465
|
// src/lib/memory/autoExtractWorker.ts
|
|
26270
26466
|
init_logger();
|
|
26271
26467
|
init_redactor();
|
|
26468
|
+
function createPlatformCursorStore(storage, keyPrefix = "anuma:mem:xcursor:") {
|
|
26469
|
+
return {
|
|
26470
|
+
get(conversationId) {
|
|
26471
|
+
try {
|
|
26472
|
+
return storage.getItem(keyPrefix + conversationId) ?? void 0;
|
|
26473
|
+
} catch {
|
|
26474
|
+
return void 0;
|
|
26475
|
+
}
|
|
26476
|
+
},
|
|
26477
|
+
set(conversationId, messageId) {
|
|
26478
|
+
try {
|
|
26479
|
+
storage.setItem(keyPrefix + conversationId, messageId);
|
|
26480
|
+
} catch {
|
|
26481
|
+
}
|
|
26482
|
+
}
|
|
26483
|
+
};
|
|
26484
|
+
}
|
|
26272
26485
|
var DEFAULT_WINDOW_SIZE = 6;
|
|
26273
26486
|
var DEFAULT_MAX_WINDOW_SIZE = 20;
|
|
26274
26487
|
var CONTEXT_OVERLAP = 2;
|
|
@@ -26288,12 +26501,24 @@ function createAutoExtractor(options) {
|
|
|
26288
26501
|
let inflight = 0;
|
|
26289
26502
|
let disposed = false;
|
|
26290
26503
|
const conversations = /* @__PURE__ */ new Map();
|
|
26504
|
+
const cursorStore = options.cursorStore;
|
|
26505
|
+
const hydrate = (state, conversationId) => {
|
|
26506
|
+
if (state.hydrated) return;
|
|
26507
|
+
state.hydrated = true;
|
|
26508
|
+
if (!cursorStore || conversationId === void 0 || state.watermark !== void 0) return;
|
|
26509
|
+
try {
|
|
26510
|
+
const persisted = cursorStore.get(conversationId);
|
|
26511
|
+
if (persisted) state.watermark = persisted;
|
|
26512
|
+
} catch {
|
|
26513
|
+
}
|
|
26514
|
+
};
|
|
26291
26515
|
const stateFor = (conversationId) => {
|
|
26292
26516
|
let state = conversations.get(conversationId);
|
|
26293
26517
|
if (!state) {
|
|
26294
26518
|
if (conversations.size >= maxTrackedConversations) evictOldestIdle(conversationId);
|
|
26295
26519
|
state = {};
|
|
26296
26520
|
conversations.set(conversationId, state);
|
|
26521
|
+
hydrate(state, conversationId);
|
|
26297
26522
|
}
|
|
26298
26523
|
return state;
|
|
26299
26524
|
};
|
|
@@ -26306,10 +26531,12 @@ function createAutoExtractor(options) {
|
|
|
26306
26531
|
}
|
|
26307
26532
|
};
|
|
26308
26533
|
function computeWindow(messages, conversationId) {
|
|
26309
|
-
const lastId =
|
|
26534
|
+
const lastId = stateFor(conversationId).watermark;
|
|
26310
26535
|
const idx = lastId === void 0 ? -1 : messages.findIndex((m) => m.id === lastId);
|
|
26311
|
-
if (idx === -1)
|
|
26312
|
-
|
|
26536
|
+
if (idx === -1) {
|
|
26537
|
+
return { window: messages.slice(-windowSize), contiguous: messages.length <= windowSize };
|
|
26538
|
+
}
|
|
26539
|
+
if (idx >= messages.length - 1) return { window: [], contiguous: true };
|
|
26313
26540
|
const start = Math.max(0, idx + 1 - CONTEXT_OVERLAP);
|
|
26314
26541
|
let window2 = messages.slice(start);
|
|
26315
26542
|
if (window2.length > maxWindowSize) {
|
|
@@ -26318,8 +26545,21 @@ function createAutoExtractor(options) {
|
|
|
26318
26545
|
);
|
|
26319
26546
|
window2 = window2.slice(0, maxWindowSize);
|
|
26320
26547
|
}
|
|
26321
|
-
return window2;
|
|
26548
|
+
return { window: window2, contiguous: true };
|
|
26322
26549
|
}
|
|
26550
|
+
const persistCursor = (conversationId, messages, advancedToId, contiguous) => {
|
|
26551
|
+
if (!cursorStore || conversationId === void 0 || !contiguous) return;
|
|
26552
|
+
try {
|
|
26553
|
+
const stored = cursorStore.get(conversationId);
|
|
26554
|
+
if (stored && stored !== advancedToId) {
|
|
26555
|
+
const storedIdx = messages.findIndex((m) => m.id === stored);
|
|
26556
|
+
const newIdx = messages.findIndex((m) => m.id === advancedToId);
|
|
26557
|
+
if (storedIdx !== -1 && newIdx !== -1 && storedIdx >= newIdx) return;
|
|
26558
|
+
}
|
|
26559
|
+
cursorStore.set(conversationId, advancedToId);
|
|
26560
|
+
} catch {
|
|
26561
|
+
}
|
|
26562
|
+
};
|
|
26323
26563
|
const consolidateBaseUrl = options.consolidate?.baseUrl ?? options.extract.baseUrl;
|
|
26324
26564
|
const consolidateOptions = options.consolidate ? {
|
|
26325
26565
|
...options.extract.apiKey !== void 0 && { apiKey: options.extract.apiKey },
|
|
@@ -26365,7 +26605,7 @@ function createAutoExtractor(options) {
|
|
|
26365
26605
|
return dispatch(messages, conversationId);
|
|
26366
26606
|
}
|
|
26367
26607
|
function dispatch(messages, conversationId) {
|
|
26368
|
-
const window2 = computeWindow(messages, conversationId);
|
|
26608
|
+
const { window: window2, contiguous } = computeWindow(messages, conversationId);
|
|
26369
26609
|
if (window2.length === 0) {
|
|
26370
26610
|
options.onSkipped?.({ reason: "no-new-content", conversationId });
|
|
26371
26611
|
drainPending();
|
|
@@ -26390,7 +26630,9 @@ function createAutoExtractor(options) {
|
|
|
26390
26630
|
}
|
|
26391
26631
|
}
|
|
26392
26632
|
);
|
|
26393
|
-
|
|
26633
|
+
const advancedTo = window2[window2.length - 1].id;
|
|
26634
|
+
stateFor(conversationId).watermark = advancedTo;
|
|
26635
|
+
persistCursor(conversationId, messages, advancedTo, contiguous);
|
|
26394
26636
|
for (let i = 0; i < results.length; i++) {
|
|
26395
26637
|
if (results[i].action === "suppressed") continue;
|
|
26396
26638
|
options.onMemoryExtracted?.({
|
|
@@ -26437,6 +26679,13 @@ function createAutoExtractor(options) {
|
|
|
26437
26679
|
};
|
|
26438
26680
|
}
|
|
26439
26681
|
|
|
26682
|
+
// src/lib/memory/chunkVectorCache.ts
|
|
26683
|
+
init_lruCache();
|
|
26684
|
+
var DEFAULT_CHUNK_CACHE_SIZE = 5e3;
|
|
26685
|
+
function createChunkVectorCache(maxSize = DEFAULT_CHUNK_CACHE_SIZE) {
|
|
26686
|
+
return new LRUCache(maxSize);
|
|
26687
|
+
}
|
|
26688
|
+
|
|
26440
26689
|
// src/lib/memory/index.ts
|
|
26441
26690
|
init_recall();
|
|
26442
26691
|
|
|
@@ -26701,6 +26950,9 @@ function parseAnswer(body, base, parseSchema) {
|
|
|
26701
26950
|
};
|
|
26702
26951
|
}
|
|
26703
26952
|
|
|
26953
|
+
// src/lib/memory/index.ts
|
|
26954
|
+
init_reranker();
|
|
26955
|
+
|
|
26704
26956
|
// src/lib/memory/topicExtract.ts
|
|
26705
26957
|
init_operations2();
|
|
26706
26958
|
init_operations3();
|
|
@@ -32412,6 +32664,7 @@ function buildSystemPrompt(ctx, options) {
|
|
|
32412
32664
|
DEFAULT_ANALYSIS_PROMPT,
|
|
32413
32665
|
DEFAULT_ANALYZE_AFTER_MESSAGES,
|
|
32414
32666
|
DEFAULT_CACHE_EXPIRATION_MS,
|
|
32667
|
+
DEFAULT_CHUNK_CACHE_SIZE,
|
|
32415
32668
|
DEFAULT_CHUNK_OVERLAP,
|
|
32416
32669
|
DEFAULT_CHUNK_SIZE,
|
|
32417
32670
|
DEFAULT_DOCUMENT_PATTERN,
|
|
@@ -32442,6 +32695,7 @@ function buildSystemPrompt(ctx, options) {
|
|
|
32442
32695
|
QueueManager,
|
|
32443
32696
|
RECALL_MAX_LIMIT,
|
|
32444
32697
|
RECALL_TOOL_NAME,
|
|
32698
|
+
RerankerUnavailableError,
|
|
32445
32699
|
SDK_SCHEMA_VERSION,
|
|
32446
32700
|
SERVER_TOOL_DEPENDENCY_SETS,
|
|
32447
32701
|
SLIDER_CONFIG,
|
|
@@ -32483,6 +32737,7 @@ function buildSystemPrompt(ctx, options) {
|
|
|
32483
32737
|
clearServerToolsCache,
|
|
32484
32738
|
composeHooks,
|
|
32485
32739
|
createAutoExtractor,
|
|
32740
|
+
createChunkVectorCache,
|
|
32486
32741
|
createConversationOp,
|
|
32487
32742
|
createCryptoPricePreProcessor,
|
|
32488
32743
|
createMediaBatchOp,
|
|
@@ -32491,6 +32746,7 @@ function buildSystemPrompt(ctx, options) {
|
|
|
32491
32746
|
createMemoryVaultSearchTool,
|
|
32492
32747
|
createMemoryVaultTool,
|
|
32493
32748
|
createMessageOp,
|
|
32749
|
+
createPlatformCursorStore,
|
|
32494
32750
|
createProjectOp,
|
|
32495
32751
|
createRecallTool,
|
|
32496
32752
|
createServerToolsFilter,
|
|
@@ -32587,6 +32843,7 @@ function buildSystemPrompt(ctx, options) {
|
|
|
32587
32843
|
hardDeleteMediaOp,
|
|
32588
32844
|
hasEncryptionKey,
|
|
32589
32845
|
hasKeyPair,
|
|
32846
|
+
isRerankerAvailable,
|
|
32590
32847
|
isSupportedFile,
|
|
32591
32848
|
isSupportedMediaType,
|
|
32592
32849
|
makeSyntheticStoredConversation,
|