@memtensor/memos-local-openclaw-plugin 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +11 -0
- package/README.md +251 -0
- package/SKILL.md +43 -0
- package/dist/capture/index.d.ts +16 -0
- package/dist/capture/index.d.ts.map +1 -0
- package/dist/capture/index.js +80 -0
- package/dist/capture/index.js.map +1 -0
- package/dist/config.d.ts +4 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +96 -0
- package/dist/config.js.map +1 -0
- package/dist/embedding/index.d.ts +12 -0
- package/dist/embedding/index.d.ts.map +1 -0
- package/dist/embedding/index.js +75 -0
- package/dist/embedding/index.js.map +1 -0
- package/dist/embedding/local.d.ts +3 -0
- package/dist/embedding/local.d.ts.map +1 -0
- package/dist/embedding/local.js +65 -0
- package/dist/embedding/local.js.map +1 -0
- package/dist/embedding/providers/cohere.d.ts +4 -0
- package/dist/embedding/providers/cohere.d.ts.map +1 -0
- package/dist/embedding/providers/cohere.js +57 -0
- package/dist/embedding/providers/cohere.js.map +1 -0
- package/dist/embedding/providers/gemini.d.ts +3 -0
- package/dist/embedding/providers/gemini.d.ts.map +1 -0
- package/dist/embedding/providers/gemini.js +31 -0
- package/dist/embedding/providers/gemini.js.map +1 -0
- package/dist/embedding/providers/mistral.d.ts +3 -0
- package/dist/embedding/providers/mistral.d.ts.map +1 -0
- package/dist/embedding/providers/mistral.js +25 -0
- package/dist/embedding/providers/mistral.js.map +1 -0
- package/dist/embedding/providers/openai.d.ts +3 -0
- package/dist/embedding/providers/openai.d.ts.map +1 -0
- package/dist/embedding/providers/openai.js +35 -0
- package/dist/embedding/providers/openai.js.map +1 -0
- package/dist/embedding/providers/voyage.d.ts +3 -0
- package/dist/embedding/providers/voyage.d.ts.map +1 -0
- package/dist/embedding/providers/voyage.js +25 -0
- package/dist/embedding/providers/voyage.js.map +1 -0
- package/dist/index.d.ts +44 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +75 -0
- package/dist/index.js.map +1 -0
- package/dist/ingest/chunker.d.ts +15 -0
- package/dist/ingest/chunker.d.ts.map +1 -0
- package/dist/ingest/chunker.js +193 -0
- package/dist/ingest/chunker.js.map +1 -0
- package/dist/ingest/dedup.d.ts +11 -0
- package/dist/ingest/dedup.d.ts.map +1 -0
- package/dist/ingest/dedup.js +29 -0
- package/dist/ingest/dedup.js.map +1 -0
- package/dist/ingest/providers/anthropic.d.ts +3 -0
- package/dist/ingest/providers/anthropic.d.ts.map +1 -0
- package/dist/ingest/providers/anthropic.js +33 -0
- package/dist/ingest/providers/anthropic.js.map +1 -0
- package/dist/ingest/providers/bedrock.d.ts +8 -0
- package/dist/ingest/providers/bedrock.d.ts.map +1 -0
- package/dist/ingest/providers/bedrock.js +41 -0
- package/dist/ingest/providers/bedrock.js.map +1 -0
- package/dist/ingest/providers/gemini.d.ts +3 -0
- package/dist/ingest/providers/gemini.d.ts.map +1 -0
- package/dist/ingest/providers/gemini.js +31 -0
- package/dist/ingest/providers/gemini.js.map +1 -0
- package/dist/ingest/providers/index.d.ts +9 -0
- package/dist/ingest/providers/index.d.ts.map +1 -0
- package/dist/ingest/providers/index.js +68 -0
- package/dist/ingest/providers/index.js.map +1 -0
- package/dist/ingest/providers/openai.d.ts +3 -0
- package/dist/ingest/providers/openai.d.ts.map +1 -0
- package/dist/ingest/providers/openai.js +41 -0
- package/dist/ingest/providers/openai.js.map +1 -0
- package/dist/ingest/worker.d.ts +21 -0
- package/dist/ingest/worker.d.ts.map +1 -0
- package/dist/ingest/worker.js +111 -0
- package/dist/ingest/worker.js.map +1 -0
- package/dist/recall/engine.d.ts +23 -0
- package/dist/recall/engine.d.ts.map +1 -0
- package/dist/recall/engine.js +153 -0
- package/dist/recall/engine.js.map +1 -0
- package/dist/recall/mmr.d.ts +17 -0
- package/dist/recall/mmr.d.ts.map +1 -0
- package/dist/recall/mmr.js +51 -0
- package/dist/recall/mmr.js.map +1 -0
- package/dist/recall/recency.d.ts +20 -0
- package/dist/recall/recency.d.ts.map +1 -0
- package/dist/recall/recency.js +26 -0
- package/dist/recall/recency.js.map +1 -0
- package/dist/recall/rrf.d.ts +16 -0
- package/dist/recall/rrf.d.ts.map +1 -0
- package/dist/recall/rrf.js +15 -0
- package/dist/recall/rrf.js.map +1 -0
- package/dist/storage/sqlite.d.ts +34 -0
- package/dist/storage/sqlite.d.ts.map +1 -0
- package/dist/storage/sqlite.js +274 -0
- package/dist/storage/sqlite.js.map +1 -0
- package/dist/storage/vector.d.ts +13 -0
- package/dist/storage/vector.d.ts.map +1 -0
- package/dist/storage/vector.js +33 -0
- package/dist/storage/vector.js.map +1 -0
- package/dist/tools/index.d.ts +4 -0
- package/dist/tools/index.d.ts.map +1 -0
- package/dist/tools/index.js +10 -0
- package/dist/tools/index.js.map +1 -0
- package/dist/tools/memory-get.d.ts +4 -0
- package/dist/tools/memory-get.d.ts.map +1 -0
- package/dist/tools/memory-get.js +59 -0
- package/dist/tools/memory-get.js.map +1 -0
- package/dist/tools/memory-search.d.ts +4 -0
- package/dist/tools/memory-search.d.ts.map +1 -0
- package/dist/tools/memory-search.js +36 -0
- package/dist/tools/memory-search.js.map +1 -0
- package/dist/tools/memory-timeline.d.ts +4 -0
- package/dist/tools/memory-timeline.d.ts.map +1 -0
- package/dist/tools/memory-timeline.js +64 -0
- package/dist/tools/memory-timeline.js.map +1 -0
- package/dist/types.d.ts +158 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +25 -0
- package/dist/types.js.map +1 -0
- package/dist/viewer/html.d.ts +2 -0
- package/dist/viewer/html.d.ts.map +1 -0
- package/dist/viewer/html.js +686 -0
- package/dist/viewer/html.js.map +1 -0
- package/dist/viewer/server.d.ts +48 -0
- package/dist/viewer/server.d.ts.map +1 -0
- package/dist/viewer/server.js +470 -0
- package/dist/viewer/server.js.map +1 -0
- package/index.ts +357 -0
- package/openclaw.plugin.json +57 -0
- package/package.json +57 -0
- package/src/capture/index.ts +92 -0
- package/src/config.ts +67 -0
- package/src/embedding/index.ts +76 -0
- package/src/embedding/local.ts +35 -0
- package/src/embedding/providers/cohere.ts +69 -0
- package/src/embedding/providers/gemini.ts +41 -0
- package/src/embedding/providers/mistral.ts +32 -0
- package/src/embedding/providers/openai.ts +42 -0
- package/src/embedding/providers/voyage.ts +32 -0
- package/src/index.ts +106 -0
- package/src/ingest/chunker.ts +217 -0
- package/src/ingest/dedup.ts +37 -0
- package/src/ingest/providers/anthropic.ts +41 -0
- package/src/ingest/providers/bedrock.ts +50 -0
- package/src/ingest/providers/gemini.ts +41 -0
- package/src/ingest/providers/index.ts +67 -0
- package/src/ingest/providers/openai.ts +48 -0
- package/src/ingest/worker.ts +130 -0
- package/src/recall/engine.ts +182 -0
- package/src/recall/mmr.ts +60 -0
- package/src/recall/recency.ts +27 -0
- package/src/recall/rrf.ts +31 -0
- package/src/storage/sqlite.ts +305 -0
- package/src/storage/vector.ts +39 -0
- package/src/tools/index.ts +3 -0
- package/src/tools/memory-get.ts +68 -0
- package/src/tools/memory-search.ts +36 -0
- package/src/tools/memory-timeline.ts +73 -0
- package/src/types.ts +214 -0
- package/src/viewer/html.ts +682 -0
- package/src/viewer/server.ts +464 -0
- package/www/index.html +606 -0
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.IngestWorker = void 0;
|
|
4
|
+
const uuid_1 = require("uuid");
|
|
5
|
+
const providers_1 = require("./providers");
|
|
6
|
+
const chunker_1 = require("./chunker");
|
|
7
|
+
const dedup_1 = require("./dedup");
|
|
8
|
+
class IngestWorker {
|
|
9
|
+
store;
|
|
10
|
+
embedder;
|
|
11
|
+
ctx;
|
|
12
|
+
summarizer;
|
|
13
|
+
queue = [];
|
|
14
|
+
processing = false;
|
|
15
|
+
flushResolvers = [];
|
|
16
|
+
constructor(store, embedder, ctx) {
|
|
17
|
+
this.store = store;
|
|
18
|
+
this.embedder = embedder;
|
|
19
|
+
this.ctx = ctx;
|
|
20
|
+
this.summarizer = new providers_1.Summarizer(ctx.config.summarizer, ctx.log);
|
|
21
|
+
}
|
|
22
|
+
enqueue(messages) {
|
|
23
|
+
this.queue.push(...messages);
|
|
24
|
+
if (!this.processing) {
|
|
25
|
+
this.processQueue().catch((err) => {
|
|
26
|
+
this.ctx.log.error(`Ingest worker error: ${err}`);
|
|
27
|
+
this.processing = false;
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
/** Wait until all queued messages have been processed. */
|
|
32
|
+
async flush() {
|
|
33
|
+
if (this.queue.length === 0 && !this.processing)
|
|
34
|
+
return;
|
|
35
|
+
return new Promise((resolve) => {
|
|
36
|
+
this.flushResolvers.push(resolve);
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
async processQueue() {
|
|
40
|
+
this.processing = true;
|
|
41
|
+
while (this.queue.length > 0) {
|
|
42
|
+
const msg = this.queue.shift();
|
|
43
|
+
try {
|
|
44
|
+
await this.ingestMessage(msg);
|
|
45
|
+
}
|
|
46
|
+
catch (err) {
|
|
47
|
+
this.ctx.log.error(`Failed to ingest message turn=${msg.turnId}: ${err}`);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
this.processing = false;
|
|
51
|
+
for (const resolve of this.flushResolvers)
|
|
52
|
+
resolve();
|
|
53
|
+
this.flushResolvers = [];
|
|
54
|
+
}
|
|
55
|
+
async ingestMessage(msg) {
|
|
56
|
+
if (msg.role === "tool") {
|
|
57
|
+
await this.ingestToolResult(msg);
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
const rawChunks = (0, chunker_1.chunkText)(msg.content);
|
|
61
|
+
this.ctx.log.debug(`Chunked turn=${msg.turnId} into ${rawChunks.length} chunks`);
|
|
62
|
+
for (let seq = 0; seq < rawChunks.length; seq++) {
|
|
63
|
+
const raw = rawChunks[seq];
|
|
64
|
+
await this.storeChunk(msg, raw.content, raw.kind, seq);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
async ingestToolResult(msg) {
|
|
68
|
+
this.ctx.log.debug(`Ingesting tool result turn=${msg.turnId} tool=${msg.toolName ?? "unknown"} len=${msg.content.length}`);
|
|
69
|
+
await this.storeChunk(msg, msg.content, "tool_result", 0);
|
|
70
|
+
}
|
|
71
|
+
async storeChunk(msg, content, kind, seq) {
|
|
72
|
+
const chunkId = (0, uuid_1.v4)();
|
|
73
|
+
const summary = await this.summarizer.summarize(content);
|
|
74
|
+
let embedding = null;
|
|
75
|
+
try {
|
|
76
|
+
[embedding] = await this.embedder.embed([summary]);
|
|
77
|
+
}
|
|
78
|
+
catch (err) {
|
|
79
|
+
this.ctx.log.warn(`Embedding failed for chunk=${chunkId}, storing without vector: ${err}`);
|
|
80
|
+
}
|
|
81
|
+
if (embedding) {
|
|
82
|
+
const dupId = (0, dedup_1.findDuplicate)(this.store, embedding, this.ctx.config.dedup?.similarityThreshold ?? 0.93, this.ctx.log);
|
|
83
|
+
if (dupId) {
|
|
84
|
+
this.store.updateSummary(dupId, summary);
|
|
85
|
+
this.store.upsertEmbedding(dupId, embedding);
|
|
86
|
+
this.ctx.log.debug(`Dedup-merged into existing chunk=${dupId}`);
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
const chunk = {
|
|
91
|
+
id: chunkId,
|
|
92
|
+
sessionKey: msg.sessionKey,
|
|
93
|
+
turnId: msg.turnId,
|
|
94
|
+
seq,
|
|
95
|
+
role: msg.role,
|
|
96
|
+
content,
|
|
97
|
+
kind,
|
|
98
|
+
summary,
|
|
99
|
+
embedding: null,
|
|
100
|
+
createdAt: msg.timestamp,
|
|
101
|
+
updatedAt: msg.timestamp,
|
|
102
|
+
};
|
|
103
|
+
this.store.insertChunk(chunk);
|
|
104
|
+
if (embedding) {
|
|
105
|
+
this.store.upsertEmbedding(chunkId, embedding);
|
|
106
|
+
}
|
|
107
|
+
this.ctx.log.debug(`Stored chunk=${chunkId} kind=${kind} role=${msg.role} len=${content.length} hasVec=${!!embedding}`);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
exports.IngestWorker = IngestWorker;
|
|
111
|
+
//# sourceMappingURL=worker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../src/ingest/worker.ts"],"names":[],"mappings":";;;AAAA,+BAAkC;AAIlC,2CAAyC;AACzC,uCAAsC;AACtC,mCAAwC;AAExC,MAAa,YAAY;IAOb;IACA;IACA;IARF,UAAU,CAAa;IACvB,KAAK,GAA0B,EAAE,CAAC;IAClC,UAAU,GAAG,KAAK,CAAC;IACnB,cAAc,GAAsB,EAAE,CAAC;IAE/C,YACU,KAAkB,EAClB,QAAkB,EAClB,GAAkB;QAFlB,UAAK,GAAL,KAAK,CAAa;QAClB,aAAQ,GAAR,QAAQ,CAAU;QAClB,QAAG,GAAH,GAAG,CAAe;QAE1B,IAAI,CAAC,UAAU,GAAG,IAAI,sBAAU,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACnE,CAAC;IAED,OAAO,CAAC,QAA+B;QACrC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;QAC7B,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACrB,IAAI,CAAC,YAAY,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBAChC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,GAAG,EAAE,CAAC,CAAC;gBAClD,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;YAC1B,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,0DAA0D;IAC1D,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QACxD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QAEvB,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAG,CAAC;YAChC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YAChC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,iCAAiC,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC;YAC5E,CAAC;QACH,CAAC;QAED,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO,EAAE,CAAC;QACrD,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,GAAwB;QAClD,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACjC,OAAO;QACT,CAAC;QAED,MAAM,SAAS,GAAG,IAAA,mBAAS,EAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,gBAAgB,GAAG,CAAC,MAAM,SAAS,SAAS,CAAC,MAAM,SAAS,CAAC,CAAC;QAEjF,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,SAAS,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;YAChD,MAAM,GAAG,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;YAC3B,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,GAAwB;QACrD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,8BAA8B,GAAG,CAAC,MAAM,SAAS,GAAG,CAAC,QAAQ,IAAI,SAAS,QAAQ,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3H,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC;IAC5D,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,GAAwB,EACxB,OAAe,EACf,IAAmB,EACnB,GAAW;QAEX,MAAM,OAAO,GAAG,IAAA,SAAI,GAAE,CAAC;QACvB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAEzD,IAAI,SAAS,GAAoB,IAAI,CAAC;QACtC,IAAI,CAAC;YACH,CAAC,SAAS,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;QACrD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,8BAA8B,OAAO,6BAA6B,GAAG,EAAE,CAAC,CAAC;QAC7F,CAAC;QAED,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,KAAK,GAAG,IAAA,qBAAa,EACzB,IAAI,CAAC,KAAK,EACV,SAAS,EACT,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,mBAAmB,IAAI,IAAI,EAClD,IAAI,CAAC,GAAG,CAAC,GAAG,CACb,CAAC;YAEF,IAAI,KAAK,EAAE,CAAC;gBACV,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;gBACzC,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;gBAC7C,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,oCAAoC,KAAK,EAAE,CAAC,CAAC;gBAChE,OAAO;YACT,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAU;YACnB,EAAE,EAAE,OAAO;YACX,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,GAAG;YACH,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,OAAO;YACP,IAAI;YACJ,OAAO;YACP,SAAS,EAAE,IAAI;YACf,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,SAAS,EAAE,GAAG,CAAC,SAAS;SACzB,CAAC;QAEF,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAC9B,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QACjD,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,gBAAgB,OAAO,SAAS,IAAI,SAAS,GAAG,CAAC,IAAI,QAAQ,OAAO,CAAC,MAAM,WAAW,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;IAC1H,CAAC;CACF;AAzHD,oCAyHC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { SqliteStore } from "../storage/sqlite";
|
|
2
|
+
import type { Embedder } from "../embedding";
|
|
3
|
+
import type { PluginContext, SearchResult } from "../types";
|
|
4
|
+
export interface RecallOptions {
|
|
5
|
+
query?: string;
|
|
6
|
+
maxResults?: number;
|
|
7
|
+
minScore?: number;
|
|
8
|
+
}
|
|
9
|
+
export declare class RecallEngine {
|
|
10
|
+
private store;
|
|
11
|
+
private embedder;
|
|
12
|
+
private ctx;
|
|
13
|
+
private recentQueries;
|
|
14
|
+
constructor(store: SqliteStore, embedder: Embedder, ctx: PluginContext);
|
|
15
|
+
search(opts: RecallOptions): Promise<SearchResult>;
|
|
16
|
+
/**
|
|
17
|
+
* PRD §6.1: Detect repeated identical/similar queries and produce a
|
|
18
|
+
* warning note so the model knows to vary its approach.
|
|
19
|
+
*/
|
|
20
|
+
private checkRepeat;
|
|
21
|
+
private recordQuery;
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=engine.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/recall/engine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,KAAK,EAAE,aAAa,EAAa,YAAY,EAAE,MAAM,UAAU,CAAC;AAMvE,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAID,qBAAa,YAAY;IAIrB,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,GAAG;IALb,OAAO,CAAC,aAAa,CAAwF;gBAGnG,KAAK,EAAE,WAAW,EAClB,QAAQ,EAAE,QAAQ,EAClB,GAAG,EAAE,aAAa;IAGtB,MAAM,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IA6GxD;;;OAGG;IACH,OAAO,CAAC,WAAW;IAkBnB,OAAO,CAAC,WAAW;CAapB"}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RecallEngine = void 0;
|
|
4
|
+
const vector_1 = require("../storage/vector");
|
|
5
|
+
const rrf_1 = require("./rrf");
|
|
6
|
+
const mmr_1 = require("./mmr");
|
|
7
|
+
const recency_1 = require("./recency");
|
|
8
|
+
const MAX_RECENT_QUERIES = 20;
|
|
9
|
+
class RecallEngine {
|
|
10
|
+
store;
|
|
11
|
+
embedder;
|
|
12
|
+
ctx;
|
|
13
|
+
recentQueries = [];
|
|
14
|
+
constructor(store, embedder, ctx) {
|
|
15
|
+
this.store = store;
|
|
16
|
+
this.embedder = embedder;
|
|
17
|
+
this.ctx = ctx;
|
|
18
|
+
}
|
|
19
|
+
async search(opts) {
|
|
20
|
+
const recallCfg = this.ctx.config.recall;
|
|
21
|
+
const maxResults = Math.min(opts.maxResults ?? recallCfg.maxResultsDefault, recallCfg.maxResultsMax);
|
|
22
|
+
const minScore = opts.minScore ?? recallCfg.minScoreDefault;
|
|
23
|
+
const query = opts.query ?? "";
|
|
24
|
+
const repeatNote = this.checkRepeat(query, maxResults, minScore);
|
|
25
|
+
const candidatePool = maxResults * 5;
|
|
26
|
+
// Step 1: Gather candidates from both FTS and vector search
|
|
27
|
+
const ftsCandidates = query
|
|
28
|
+
? this.store.ftsSearch(query, candidatePool)
|
|
29
|
+
: [];
|
|
30
|
+
let vecCandidates = [];
|
|
31
|
+
if (query) {
|
|
32
|
+
try {
|
|
33
|
+
const queryVec = await this.embedder.embedQuery(query);
|
|
34
|
+
vecCandidates = (0, vector_1.vectorSearch)(this.store, queryVec, candidatePool);
|
|
35
|
+
}
|
|
36
|
+
catch (err) {
|
|
37
|
+
this.ctx.log.warn(`Vector search failed, using FTS only: ${err}`);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
// Step 2: RRF fusion
|
|
41
|
+
const ftsRanked = ftsCandidates.map((c) => ({ id: c.chunkId, score: c.score }));
|
|
42
|
+
const vecRanked = vecCandidates.map((c) => ({ id: c.chunkId, score: c.score }));
|
|
43
|
+
const rrfScores = (0, rrf_1.rrfFuse)([ftsRanked, vecRanked], recallCfg.rrfK);
|
|
44
|
+
if (rrfScores.size === 0) {
|
|
45
|
+
this.recordQuery(query, maxResults, minScore, 0);
|
|
46
|
+
return {
|
|
47
|
+
hits: [],
|
|
48
|
+
meta: {
|
|
49
|
+
usedMinScore: minScore,
|
|
50
|
+
usedMaxResults: maxResults,
|
|
51
|
+
totalCandidates: 0,
|
|
52
|
+
note: repeatNote ?? "No candidates found for the given query.",
|
|
53
|
+
},
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
// Step 3: MMR re-ranking
|
|
57
|
+
const rrfList = [...rrfScores.entries()]
|
|
58
|
+
.map(([id, score]) => ({ id, score }))
|
|
59
|
+
.sort((a, b) => b.score - a.score);
|
|
60
|
+
const mmrResults = (0, mmr_1.mmrRerank)(rrfList, this.store, recallCfg.mmrLambda, maxResults * 2);
|
|
61
|
+
// Step 4: Time decay
|
|
62
|
+
const withTs = mmrResults.map((r) => {
|
|
63
|
+
const chunk = this.store.getChunk(r.id);
|
|
64
|
+
return { ...r, createdAt: chunk?.createdAt ?? 0 };
|
|
65
|
+
});
|
|
66
|
+
const decayed = (0, recency_1.applyRecencyDecay)(withTs, recallCfg.recencyHalfLifeDays);
|
|
67
|
+
// Step 5: Normalize scores to [0,1]
|
|
68
|
+
const maxScore = Math.max(...decayed.map((d) => d.score), 1e-10);
|
|
69
|
+
const normalized = decayed.map((d) => ({
|
|
70
|
+
...d,
|
|
71
|
+
score: d.score / maxScore,
|
|
72
|
+
}));
|
|
73
|
+
// Step 6: Filter by minScore and limit
|
|
74
|
+
const filtered = normalized
|
|
75
|
+
.filter((d) => d.score >= minScore)
|
|
76
|
+
.sort((a, b) => b.score - a.score)
|
|
77
|
+
.slice(0, maxResults);
|
|
78
|
+
// Step 7: Build hits
|
|
79
|
+
const hits = [];
|
|
80
|
+
for (const candidate of filtered) {
|
|
81
|
+
const chunk = this.store.getChunk(candidate.id);
|
|
82
|
+
if (!chunk)
|
|
83
|
+
continue;
|
|
84
|
+
hits.push({
|
|
85
|
+
summary: chunk.summary,
|
|
86
|
+
original_excerpt: makeExcerpt(chunk.content),
|
|
87
|
+
ref: {
|
|
88
|
+
sessionKey: chunk.sessionKey,
|
|
89
|
+
chunkId: chunk.id,
|
|
90
|
+
turnId: chunk.turnId,
|
|
91
|
+
seq: chunk.seq,
|
|
92
|
+
},
|
|
93
|
+
score: Math.round(candidate.score * 1000) / 1000,
|
|
94
|
+
source: {
|
|
95
|
+
ts: chunk.createdAt,
|
|
96
|
+
role: chunk.role,
|
|
97
|
+
sessionKey: chunk.sessionKey,
|
|
98
|
+
},
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
this.recordQuery(query, maxResults, minScore, hits.length);
|
|
102
|
+
return {
|
|
103
|
+
hits,
|
|
104
|
+
meta: {
|
|
105
|
+
usedMinScore: minScore,
|
|
106
|
+
usedMaxResults: maxResults,
|
|
107
|
+
totalCandidates: rrfScores.size,
|
|
108
|
+
...(repeatNote ? { note: repeatNote } : {}),
|
|
109
|
+
},
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* PRD §6.1: Detect repeated identical/similar queries and produce a
|
|
114
|
+
* warning note so the model knows to vary its approach.
|
|
115
|
+
*/
|
|
116
|
+
checkRepeat(query, maxResults, minScore) {
|
|
117
|
+
const normalized = query.toLowerCase().trim();
|
|
118
|
+
if (!normalized)
|
|
119
|
+
return undefined;
|
|
120
|
+
const dup = this.recentQueries.find((q) => q.query === normalized && q.maxResults === maxResults && q.minScore === minScore);
|
|
121
|
+
if (dup) {
|
|
122
|
+
if (dup.hitCount === 0) {
|
|
123
|
+
return "This exact query with the same parameters was already tried and returned 0 results. Try rephrasing with different keywords, or adjust maxResults/minScore.";
|
|
124
|
+
}
|
|
125
|
+
return "This exact query with the same parameters was already executed. Consider varying the query or expanding parameters to get different results.";
|
|
126
|
+
}
|
|
127
|
+
return undefined;
|
|
128
|
+
}
|
|
129
|
+
recordQuery(query, maxResults, minScore, hitCount) {
|
|
130
|
+
const normalized = query.toLowerCase().trim();
|
|
131
|
+
if (!normalized)
|
|
132
|
+
return;
|
|
133
|
+
this.recentQueries = this.recentQueries.filter((q) => !(q.query === normalized && q.maxResults === maxResults && q.minScore === minScore));
|
|
134
|
+
this.recentQueries.push({ query: normalized, maxResults, minScore, hitCount });
|
|
135
|
+
if (this.recentQueries.length > MAX_RECENT_QUERIES) {
|
|
136
|
+
this.recentQueries.shift();
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
exports.RecallEngine = RecallEngine;
|
|
141
|
+
function makeExcerpt(content) {
|
|
142
|
+
const min = 200;
|
|
143
|
+
const max = 500;
|
|
144
|
+
if (content.length <= max)
|
|
145
|
+
return content;
|
|
146
|
+
let cut = content.lastIndexOf(".", max);
|
|
147
|
+
if (cut < min)
|
|
148
|
+
cut = content.lastIndexOf(" ", max);
|
|
149
|
+
if (cut < min)
|
|
150
|
+
cut = max;
|
|
151
|
+
return content.slice(0, cut) + "…";
|
|
152
|
+
}
|
|
153
|
+
//# sourceMappingURL=engine.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"engine.js","sourceRoot":"","sources":["../../src/recall/engine.ts"],"names":[],"mappings":";;;AAGA,8CAAiD;AACjD,+BAAgC;AAChC,+BAAkC;AAClC,uCAA8C;AAQ9C,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAE9B,MAAa,YAAY;IAIb;IACA;IACA;IALF,aAAa,GAAqF,EAAE,CAAC;IAE7G,YACU,KAAkB,EAClB,QAAkB,EAClB,GAAkB;QAFlB,UAAK,GAAL,KAAK,CAAa;QAClB,aAAQ,GAAR,QAAQ,CAAU;QAClB,QAAG,GAAH,GAAG,CAAe;IACzB,CAAC;IAEJ,KAAK,CAAC,MAAM,CAAC,IAAmB;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAO,CAAC;QAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CACzB,IAAI,CAAC,UAAU,IAAI,SAAS,CAAC,iBAAkB,EAC/C,SAAS,CAAC,aAAc,CACzB,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,SAAS,CAAC,eAAgB,CAAC;QAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QAE/B,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;QACjE,MAAM,aAAa,GAAG,UAAU,GAAG,CAAC,CAAC;QAErC,4DAA4D;QAC5D,MAAM,aAAa,GAAG,KAAK;YACzB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,aAAa,CAAC;YAC5C,CAAC,CAAC,EAAE,CAAC;QAEP,IAAI,aAAa,GAA8C,EAAE,CAAC;QAClE,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;gBACvD,aAAa,GAAG,IAAA,qBAAY,EAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;YACpE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,yCAAyC,GAAG,EAAE,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QAED,qBAAqB;QACrB,MAAM,SAAS,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAChF,MAAM,SAAS,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAChF,MAAM,SAAS,GAAG,IAAA,aAAO,EAAC,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;QAElE,IAAI,SAAS,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACzB,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;YACjD,OAAO;gBACL,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,YAAY,EAAE,QAAQ;oBACtB,cAAc,EAAE,UAAU;oBAC1B,eAAe,EAAE,CAAC;oBAClB,IAAI,EAAE,UAAU,IAAI,0CAA0C;iBAC/D;aACF,CAAC;QACJ,CAAC;QAED,yBAAyB;QACzB,MAAM,OAAO,GAAG,CAAC,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;aACrC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;aACrC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAErC,MAAM,UAAU,GAAG,IAAA,eAAS,EAAC,OAAO,EAAE,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,SAAS,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC;QAEvF,qBAAqB;QACrB,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAClC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACxC,OAAO,EAAE,GAAG,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,IAAI,CAAC,EAAE,CAAC;QACpD,CAAC,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,IAAA,2BAAiB,EAAC,MAAM,EAAE,SAAS,CAAC,mBAAmB,CAAC,CAAC;QAEzE,oCAAoC;QACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;QACjE,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACrC,GAAG,CAAC;YACJ,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,QAAQ;SAC1B,CAAC,CAAC,CAAC;QAEJ,uCAAuC;QACvC,MAAM,QAAQ,GAAG,UAAU;aACxB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,QAAQ,CAAC;aAClC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;aACjC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QAExB,qBAAqB;QACrB,MAAM,IAAI,GAAgB,EAAE,CAAC;QAC7B,KAAK,MAAM,SAAS,IAAI,QAAQ,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YAChD,IAAI,CAAC,KAAK;gBAAE,SAAS;YAErB,IAAI,CAAC,IAAI,CAAC;gBACR,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,gBAAgB,EAAE,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC;gBAC5C,GAAG,EAAE;oBACH,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,OAAO,EAAE,KAAK,CAAC,EAAE;oBACjB,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;iBACf;gBACD,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,IAAI;gBAChD,MAAM,EAAE;oBACN,EAAE,EAAE,KAAK,CAAC,SAAS;oBACnB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,UAAU,EAAE,KAAK,CAAC,UAAU;iBAC7B;aACF,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAE3D,OAAO;YACL,IAAI;YACJ,IAAI,EAAE;gBACJ,YAAY,EAAE,QAAQ;gBACtB,cAAc,EAAE,UAAU;gBAC1B,eAAe,EAAE,SAAS,CAAC,IAAI;gBAC/B,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC5C;SACF,CAAC;IACJ,CAAC;IAED;;;OAGG;IACK,WAAW,CAAC,KAAa,EAAE,UAAkB,EAAE,QAAgB;QACrE,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QAC9C,IAAI,CAAC,UAAU;YAAE,OAAO,SAAS,CAAC;QAElC,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CACjC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,UAAU,IAAI,CAAC,CAAC,UAAU,KAAK,UAAU,IAAI,CAAC,CAAC,QAAQ,KAAK,QAAQ,CACxF,CAAC;QAEF,IAAI,GAAG,EAAE,CAAC;YACR,IAAI,GAAG,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;gBACvB,OAAO,4JAA4J,CAAC;YACtK,CAAC;YACD,OAAO,8IAA8I,CAAC;QACxJ,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,WAAW,CAAC,KAAa,EAAE,UAAkB,EAAE,QAAgB,EAAE,QAAgB;QACvF,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QAC9C,IAAI,CAAC,UAAU;YAAE,OAAO;QAExB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAC5C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,UAAU,IAAI,CAAC,CAAC,UAAU,KAAK,UAAU,IAAI,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAC3F,CAAC;QACF,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;QAE/E,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;YACnD,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;CACF;AAzJD,oCAyJC;AAED,SAAS,WAAW,CAAC,OAAe;IAClC,MAAM,GAAG,GAAG,GAAG,CAAC;IAChB,MAAM,GAAG,GAAG,GAAG,CAAC;IAChB,IAAI,OAAO,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,OAAO,CAAC;IAE1C,IAAI,GAAG,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACxC,IAAI,GAAG,GAAG,GAAG;QAAE,GAAG,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACnD,IAAI,GAAG,GAAG,GAAG;QAAE,GAAG,GAAG,GAAG,CAAC;IAEzB,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,GAAG,CAAC;AACrC,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { SqliteStore } from "../storage/sqlite";
|
|
2
|
+
/**
|
|
3
|
+
* Maximal Marginal Relevance (PRD §5.3)
|
|
4
|
+
*
|
|
5
|
+
* Re-ranks candidates to balance relevance with diversity,
|
|
6
|
+
* preventing top-K results from being too similar.
|
|
7
|
+
*
|
|
8
|
+
* MMR = λ · sim(q, d) - (1-λ) · max(sim(d, d_selected))
|
|
9
|
+
*/
|
|
10
|
+
export declare function mmrRerank(candidates: Array<{
|
|
11
|
+
id: string;
|
|
12
|
+
score: number;
|
|
13
|
+
}>, store: SqliteStore, lambda?: number, topK?: number): Array<{
|
|
14
|
+
id: string;
|
|
15
|
+
score: number;
|
|
16
|
+
}>;
|
|
17
|
+
//# sourceMappingURL=mmr.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mmr.d.ts","sourceRoot":"","sources":["../../src/recall/mmr.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD;;;;;;;GAOG;AACH,wBAAgB,SAAS,CACvB,UAAU,EAAE,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,EAChD,KAAK,EAAE,WAAW,EAClB,MAAM,GAAE,MAAY,EACpB,IAAI,GAAE,MAAW,GAChB,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA2CtC"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.mmrRerank = mmrRerank;
|
|
4
|
+
const vector_1 = require("../storage/vector");
|
|
5
|
+
/**
|
|
6
|
+
* Maximal Marginal Relevance (PRD §5.3)
|
|
7
|
+
*
|
|
8
|
+
* Re-ranks candidates to balance relevance with diversity,
|
|
9
|
+
* preventing top-K results from being too similar.
|
|
10
|
+
*
|
|
11
|
+
* MMR = λ · sim(q, d) - (1-λ) · max(sim(d, d_selected))
|
|
12
|
+
*/
|
|
13
|
+
function mmrRerank(candidates, store, lambda = 0.7, topK = 20) {
|
|
14
|
+
if (candidates.length <= 1)
|
|
15
|
+
return candidates;
|
|
16
|
+
const embeddings = new Map();
|
|
17
|
+
for (const c of candidates) {
|
|
18
|
+
const vec = store.getEmbedding(c.id);
|
|
19
|
+
if (vec)
|
|
20
|
+
embeddings.set(c.id, vec);
|
|
21
|
+
}
|
|
22
|
+
const selected = [];
|
|
23
|
+
const remaining = [...candidates];
|
|
24
|
+
while (selected.length < topK && remaining.length > 0) {
|
|
25
|
+
let bestIdx = 0;
|
|
26
|
+
let bestMmr = -Infinity;
|
|
27
|
+
for (let i = 0; i < remaining.length; i++) {
|
|
28
|
+
const cand = remaining[i];
|
|
29
|
+
const candVec = embeddings.get(cand.id);
|
|
30
|
+
let maxSimToSelected = 0;
|
|
31
|
+
if (candVec && selected.length > 0) {
|
|
32
|
+
for (const s of selected) {
|
|
33
|
+
const sVec = embeddings.get(s.id);
|
|
34
|
+
if (sVec) {
|
|
35
|
+
const sim = (0, vector_1.cosineSimilarity)(candVec, sVec);
|
|
36
|
+
maxSimToSelected = Math.max(maxSimToSelected, sim);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
const mmrScore = lambda * cand.score - (1 - lambda) * maxSimToSelected;
|
|
41
|
+
if (mmrScore > bestMmr) {
|
|
42
|
+
bestMmr = mmrScore;
|
|
43
|
+
bestIdx = i;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
const chosen = remaining.splice(bestIdx, 1)[0];
|
|
47
|
+
selected.push({ id: chosen.id, score: bestMmr });
|
|
48
|
+
}
|
|
49
|
+
return selected;
|
|
50
|
+
}
|
|
51
|
+
//# sourceMappingURL=mmr.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mmr.js","sourceRoot":"","sources":["../../src/recall/mmr.ts"],"names":[],"mappings":";;AAWA,8BAgDC;AA3DD,8CAAqD;AAGrD;;;;;;;GAOG;AACH,SAAgB,SAAS,CACvB,UAAgD,EAChD,KAAkB,EAClB,SAAiB,GAAG,EACpB,OAAe,EAAE;IAEjB,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,UAAU,CAAC;IAE9C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACrC,IAAI,GAAG;YAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACrC,CAAC;IAED,MAAM,QAAQ,GAAyC,EAAE,CAAC;IAC1D,MAAM,SAAS,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC;IAElC,OAAO,QAAQ,CAAC,MAAM,GAAG,IAAI,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,OAAO,GAAG,CAAC,QAAQ,CAAC;QAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YAC1B,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAExC,IAAI,gBAAgB,GAAG,CAAC,CAAC;YACzB,IAAI,OAAO,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACnC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;oBACzB,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;oBAClC,IAAI,IAAI,EAAE,CAAC;wBACT,MAAM,GAAG,GAAG,IAAA,yBAAgB,EAAC,OAAO,EAAE,IAAI,CAAC,CAAC;wBAC5C,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC;oBACrD,CAAC;gBACH,CAAC;YACH,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,gBAAgB,CAAC;YACvE,IAAI,QAAQ,GAAG,OAAO,EAAE,CAAC;gBACvB,OAAO,GAAG,QAAQ,CAAC;gBACnB,OAAO,GAAG,CAAC,CAAC;YACd,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/C,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Time decay scoring (PRD §5.3)
|
|
3
|
+
*
|
|
4
|
+
* Applies exponential decay based on document age, biasing towards
|
|
5
|
+
* more recent memories. Uses configurable half-life (default 14 days).
|
|
6
|
+
*
|
|
7
|
+
* decay(t) = 0.5 ^ (age_days / half_life)
|
|
8
|
+
* final = base_score * (alpha + (1-alpha) * decay)
|
|
9
|
+
*
|
|
10
|
+
* alpha=0.3 ensures old but highly relevant results are not zeroed out.
|
|
11
|
+
*/
|
|
12
|
+
export declare function applyRecencyDecay(candidates: Array<{
|
|
13
|
+
id: string;
|
|
14
|
+
score: number;
|
|
15
|
+
createdAt: number;
|
|
16
|
+
}>, halfLifeDays?: number, now?: number): Array<{
|
|
17
|
+
id: string;
|
|
18
|
+
score: number;
|
|
19
|
+
}>;
|
|
20
|
+
//# sourceMappingURL=recency.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recency.d.ts","sourceRoot":"","sources":["../../src/recall/recency.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAC/B,UAAU,EAAE,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,EACnE,YAAY,GAAE,MAAW,EACzB,GAAG,CAAC,EAAE,MAAM,GACX,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAWtC"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.applyRecencyDecay = applyRecencyDecay;
|
|
4
|
+
/**
|
|
5
|
+
* Time decay scoring (PRD §5.3)
|
|
6
|
+
*
|
|
7
|
+
* Applies exponential decay based on document age, biasing towards
|
|
8
|
+
* more recent memories. Uses configurable half-life (default 14 days).
|
|
9
|
+
*
|
|
10
|
+
* decay(t) = 0.5 ^ (age_days / half_life)
|
|
11
|
+
* final = base_score * (alpha + (1-alpha) * decay)
|
|
12
|
+
*
|
|
13
|
+
* alpha=0.3 ensures old but highly relevant results are not zeroed out.
|
|
14
|
+
*/
|
|
15
|
+
function applyRecencyDecay(candidates, halfLifeDays = 14, now) {
|
|
16
|
+
const currentTime = now ?? Date.now();
|
|
17
|
+
const halfLifeMs = halfLifeDays * 24 * 60 * 60 * 1000;
|
|
18
|
+
const alpha = 0.3;
|
|
19
|
+
return candidates.map((c) => {
|
|
20
|
+
const ageMs = Math.max(0, currentTime - c.createdAt);
|
|
21
|
+
const decay = Math.pow(0.5, ageMs / halfLifeMs);
|
|
22
|
+
const adjustedScore = c.score * (alpha + (1 - alpha) * decay);
|
|
23
|
+
return { id: c.id, score: adjustedScore };
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=recency.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recency.js","sourceRoot":"","sources":["../../src/recall/recency.ts"],"names":[],"mappings":";;AAWA,8CAeC;AA1BD;;;;;;;;;;GAUG;AACH,SAAgB,iBAAiB,CAC/B,UAAmE,EACnE,eAAuB,EAAE,EACzB,GAAY;IAEZ,MAAM,WAAW,GAAG,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;IACtC,MAAM,UAAU,GAAG,YAAY,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IACtD,MAAM,KAAK,GAAG,GAAG,CAAC;IAElB,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC;QACrD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,GAAG,UAAU,CAAC,CAAC;QAChD,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC;QAC9D,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC;IAC5C,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reciprocal Rank Fusion (PRD §5.2)
|
|
3
|
+
*
|
|
4
|
+
* Merges ranked lists from different retrieval sources (FTS, vector)
|
|
5
|
+
* into a single ranking. Handles score scale mismatch between BM25
|
|
6
|
+
* and cosine similarity.
|
|
7
|
+
*
|
|
8
|
+
* RRF(d) = Σ 1 / (k + rank_i(d))
|
|
9
|
+
* where k is a constant (default 60) and rank_i is the rank in list i.
|
|
10
|
+
*/
|
|
11
|
+
export interface RankedItem {
|
|
12
|
+
id: string;
|
|
13
|
+
score: number;
|
|
14
|
+
}
|
|
15
|
+
export declare function rrfFuse(lists: RankedItem[][], k?: number): Map<string, number>;
|
|
16
|
+
//# sourceMappingURL=rrf.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rrf.d.ts","sourceRoot":"","sources":["../../src/recall/rrf.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACf;AAED,wBAAgB,OAAO,CACrB,KAAK,EAAE,UAAU,EAAE,EAAE,EACrB,CAAC,GAAE,MAAW,GACb,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAYrB"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.rrfFuse = rrfFuse;
|
|
4
|
+
function rrfFuse(lists, k = 60) {
|
|
5
|
+
const scores = new Map();
|
|
6
|
+
for (const list of lists) {
|
|
7
|
+
for (let rank = 0; rank < list.length; rank++) {
|
|
8
|
+
const item = list[rank];
|
|
9
|
+
const prev = scores.get(item.id) ?? 0;
|
|
10
|
+
scores.set(item.id, prev + 1 / (k + rank + 1));
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
return scores;
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=rrf.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rrf.js","sourceRoot":"","sources":["../../src/recall/rrf.ts"],"names":[],"mappings":";;AAeA,0BAeC;AAfD,SAAgB,OAAO,CACrB,KAAqB,EACrB,IAAY,EAAE;IAEd,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEzC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;YAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;YACtC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { Chunk, ChunkRef, Logger } from "../types";
|
|
2
|
+
export declare class SqliteStore {
|
|
3
|
+
private log;
|
|
4
|
+
private db;
|
|
5
|
+
constructor(dbPath: string, log: Logger);
|
|
6
|
+
private migrate;
|
|
7
|
+
insertChunk(chunk: Chunk): void;
|
|
8
|
+
updateSummary(chunkId: string, summary: string): void;
|
|
9
|
+
upsertEmbedding(chunkId: string, vector: number[]): void;
|
|
10
|
+
getChunk(chunkId: string): Chunk | null;
|
|
11
|
+
getChunksByRef(ref: ChunkRef): Chunk | null;
|
|
12
|
+
getNeighborChunks(sessionKey: string, turnId: string, seq: number, window: number): Chunk[];
|
|
13
|
+
ftsSearch(query: string, limit: number): Array<{
|
|
14
|
+
chunkId: string;
|
|
15
|
+
score: number;
|
|
16
|
+
}>;
|
|
17
|
+
getAllEmbeddings(): Array<{
|
|
18
|
+
chunkId: string;
|
|
19
|
+
vector: number[];
|
|
20
|
+
}>;
|
|
21
|
+
getEmbedding(chunkId: string): number[] | null;
|
|
22
|
+
updateChunk(chunkId: string, fields: {
|
|
23
|
+
summary?: string;
|
|
24
|
+
content?: string;
|
|
25
|
+
role?: string;
|
|
26
|
+
kind?: string;
|
|
27
|
+
}): boolean;
|
|
28
|
+
deleteChunk(chunkId: string): boolean;
|
|
29
|
+
deleteSession(sessionKey: string): number;
|
|
30
|
+
deleteAll(): number;
|
|
31
|
+
getRecentChunkIds(limit: number): string[];
|
|
32
|
+
close(): void;
|
|
33
|
+
}
|
|
34
|
+
//# sourceMappingURL=sqlite.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sqlite.d.ts","sourceRoot":"","sources":["../../src/storage/sqlite.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAExD,qBAAa,WAAW;IAGM,OAAO,CAAC,GAAG;IAFvC,OAAO,CAAC,EAAE,CAAoB;gBAElB,MAAM,EAAE,MAAM,EAAU,GAAG,EAAE,MAAM;IAU/C,OAAO,CAAC,OAAO;IA2Df,WAAW,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;IAmB/B,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI;IAQrD,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI;IAUxD,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,KAAK,GAAG,IAAI;IAKvC,cAAc,CAAC,GAAG,EAAE,QAAQ,GAAG,KAAK,GAAG,IAAI;IAI3C,iBAAiB,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,KAAK,EAAE;IAoB3F,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IA4BlF,gBAAgB,IAAI,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAWhE,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,IAAI;IAU9C,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO;IAkCnH,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO;IAKrC,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM;IAKzC,SAAS,IAAI,MAAM;IAOnB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE;IAO1C,KAAK,IAAI,IAAI;CAGd"}
|