@framers/agentos 0.1.247 → 0.1.249
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/memory/CognitiveMemoryManager.d.ts +47 -0
- package/dist/memory/CognitiveMemoryManager.d.ts.map +1 -1
- package/dist/memory/CognitiveMemoryManager.js +54 -0
- package/dist/memory/CognitiveMemoryManager.js.map +1 -1
- package/dist/memory/core/types.d.ts +31 -0
- package/dist/memory/core/types.d.ts.map +1 -1
- package/dist/memory/index.d.ts +13 -1
- package/dist/memory/index.d.ts.map +1 -1
- package/dist/memory/index.js +24 -0
- package/dist/memory/index.js.map +1 -1
- package/dist/memory/ingest/SessionSummarizer.d.ts +155 -0
- package/dist/memory/ingest/SessionSummarizer.d.ts.map +1 -0
- package/dist/memory/ingest/SessionSummarizer.js +178 -0
- package/dist/memory/ingest/SessionSummarizer.js.map +1 -0
- package/dist/memory/retrieval/fact-supersession/FactSupersession.d.ts +82 -0
- package/dist/memory/retrieval/fact-supersession/FactSupersession.d.ts.map +1 -0
- package/dist/memory/retrieval/fact-supersession/FactSupersession.js +159 -0
- package/dist/memory/retrieval/fact-supersession/FactSupersession.js.map +1 -0
- package/dist/memory/retrieval/fact-supersession/index.d.ts +10 -0
- package/dist/memory/retrieval/fact-supersession/index.d.ts.map +1 -0
- package/dist/memory/retrieval/fact-supersession/index.js +9 -0
- package/dist/memory/retrieval/fact-supersession/index.js.map +1 -0
- package/dist/memory/retrieval/hybrid/HybridRetriever.d.ts +151 -0
- package/dist/memory/retrieval/hybrid/HybridRetriever.d.ts.map +1 -0
- package/dist/memory/retrieval/hybrid/HybridRetriever.js +287 -0
- package/dist/memory/retrieval/hybrid/HybridRetriever.js.map +1 -0
- package/dist/memory/retrieval/hybrid/index.d.ts +12 -0
- package/dist/memory/retrieval/hybrid/index.d.ts.map +1 -0
- package/dist/memory/retrieval/hybrid/index.js +10 -0
- package/dist/memory/retrieval/hybrid/index.js.map +1 -0
- package/dist/memory/retrieval/hybrid/reciprocalRankFusion.d.ts +87 -0
- package/dist/memory/retrieval/hybrid/reciprocalRankFusion.d.ts.map +1 -0
- package/dist/memory/retrieval/hybrid/reciprocalRankFusion.js +88 -0
- package/dist/memory/retrieval/hybrid/reciprocalRankFusion.js.map +1 -0
- package/dist/memory/retrieval/session/SessionRetriever.d.ts +123 -0
- package/dist/memory/retrieval/session/SessionRetriever.d.ts.map +1 -0
- package/dist/memory/retrieval/session/SessionRetriever.js +220 -0
- package/dist/memory/retrieval/session/SessionRetriever.js.map +1 -0
- package/dist/memory/retrieval/session/SessionSummaryStore.d.ts +122 -0
- package/dist/memory/retrieval/session/SessionSummaryStore.d.ts.map +1 -0
- package/dist/memory/retrieval/session/SessionSummaryStore.js +142 -0
- package/dist/memory/retrieval/session/SessionSummaryStore.js.map +1 -0
- package/dist/memory/retrieval/session/index.d.ts +12 -0
- package/dist/memory/retrieval/session/index.d.ts.map +1 -0
- package/dist/memory/retrieval/session/index.js +10 -0
- package/dist/memory/retrieval/session/index.js.map +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file HybridRetriever.ts
|
|
3
|
+
* @description Hybrid BM25 + dense retrieval for memory-domain traces.
|
|
4
|
+
* Dense side uses {@link MemoryStore} (preserves 6-signal cognitive
|
|
5
|
+
* scoring). Sparse side uses a per-instance {@link BM25Index}. RRF
|
|
6
|
+
* merges by rank. Optional {@link RerankerService} runs over the
|
|
7
|
+
* merged pool.
|
|
8
|
+
*
|
|
9
|
+
* ## What this does
|
|
10
|
+
*
|
|
11
|
+
* Given a query, runs dense retrieval through `MemoryStore.query`
|
|
12
|
+
* (cognitive-scored traces) and sparse retrieval through an owned
|
|
13
|
+
* `BM25Index` (keyword-matched trace content). Fuses the two ranked
|
|
14
|
+
* lists via Reciprocal Rank Fusion, optionally reranks the merged
|
|
15
|
+
* pool with a neural cross-encoder, and returns a standard
|
|
16
|
+
* `CognitiveRetrievalResult` so downstream consumers (prompt
|
|
17
|
+
* assembly, bench adapters) don't change shape.
|
|
18
|
+
*
|
|
19
|
+
* ## Why a separate class, not a `CognitiveMemoryManager` option
|
|
20
|
+
*
|
|
21
|
+
* Keeps the existing manager retrieval path untouched (same reason
|
|
22
|
+
* as SessionRetriever in Step 2). MVP ships as opt-in.
|
|
23
|
+
*
|
|
24
|
+
* ## Rerank integration is mandatory-wired from the bench
|
|
25
|
+
*
|
|
26
|
+
* Per the Step 2 post-mortem (rerank-skip was the root cause of
|
|
27
|
+
* that step's RED verdict), Step 3 threads rerank from day 1 when
|
|
28
|
+
* the bench is configured with `--rerank cohere`. Callers outside
|
|
29
|
+
* the bench can pass `undefined` for `rerankerService` to skip
|
|
30
|
+
* rerank explicitly.
|
|
31
|
+
*
|
|
32
|
+
* ## Sparse-only documents are skipped in MVP
|
|
33
|
+
*
|
|
34
|
+
* A document that appears in `bm25.search` results but NOT in the
|
|
35
|
+
* dense over-fetch pool is skipped. Rationale: at the default
|
|
36
|
+
* over-fetch=3 and K=10 (30 dense candidates), a doc ranked top-30
|
|
37
|
+
* on sparse is very likely in dense's top-30 on any coherent query.
|
|
38
|
+
* Measured impact is expected to be negligible. If Tier A surfaces
|
|
39
|
+
* a meaningful drop rate, the fix is to add a
|
|
40
|
+
* `memoryStore.getTrace(id)` hydration path.
|
|
41
|
+
*
|
|
42
|
+
* @module agentos/memory/retrieval/hybrid/HybridRetriever
|
|
43
|
+
*/
|
|
44
|
+
import { BM25Index, type BM25Config } from '../../../rag/search/BM25Index.js';
|
|
45
|
+
import type { MemoryStore } from '../store/MemoryStore.js';
|
|
46
|
+
import type { RerankerService } from '../../../rag/reranking/RerankerService.js';
|
|
47
|
+
import type { HydeRetriever } from '../../../rag/HydeRetriever.js';
|
|
48
|
+
import type { CognitiveRetrievalResult, MemoryScope } from '../../core/types.js';
|
|
49
|
+
import type { PADState } from '../../core/config.js';
|
|
50
|
+
/**
|
|
51
|
+
* Options for constructing a {@link HybridRetriever}.
|
|
52
|
+
*/
|
|
53
|
+
export interface HybridRetrieverOptions {
|
|
54
|
+
memoryStore: MemoryStore;
|
|
55
|
+
/** BM25 config (k1, b, optional tokenizer). Defaults match BM25Index. */
|
|
56
|
+
bm25Config?: BM25Config;
|
|
57
|
+
/**
|
|
58
|
+
* Optional neural reranker. When provided, the merged pool is
|
|
59
|
+
* reranked before truncation. Passing the same reranker the
|
|
60
|
+
* baseline uses is the matched-ablation path.
|
|
61
|
+
*/
|
|
62
|
+
rerankerService?: RerankerService;
|
|
63
|
+
/**
|
|
64
|
+
* Optional HyDE retriever for query expansion (Step 4). When set,
|
|
65
|
+
* each `retrieve()` call generates a hypothesis and uses it as the
|
|
66
|
+
* query for BOTH dense (`memoryStore.query`) and sparse
|
|
67
|
+
* (`bm25.search`). The reranker continues to use the ORIGINAL user
|
|
68
|
+
* query so it scores documents against real user intent, not the
|
|
69
|
+
* hypothesis. HyDE generation is non-critical — errors fall back
|
|
70
|
+
* to the raw query without aborting retrieval.
|
|
71
|
+
*/
|
|
72
|
+
hydeRetriever?: HydeRetriever;
|
|
73
|
+
/**
|
|
74
|
+
* Step-6: enable split-on-ambiguous rerank refinement. When set to a
|
|
75
|
+
* value in (0, 1], the bottom fraction of traces by first-pass
|
|
76
|
+
* rerank score are split at sentence boundaries, rescored with a
|
|
77
|
+
* second rerank call (same query), and replaced by their better
|
|
78
|
+
* half ONLY IF the better half outscores the original. Monotonic.
|
|
79
|
+
*
|
|
80
|
+
* Default: undefined (no split, Step 3 behavior preserved).
|
|
81
|
+
*/
|
|
82
|
+
splitAmbiguousThreshold?: number;
|
|
83
|
+
/** Default dense weight in RRF. @default 0.7 */
|
|
84
|
+
defaultDenseWeight?: number;
|
|
85
|
+
/** Default sparse weight in RRF. @default 0.3 */
|
|
86
|
+
defaultSparseWeight?: number;
|
|
87
|
+
/** Default RRF constant. @default 60 */
|
|
88
|
+
defaultRrfK?: number;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Per-call options for {@link HybridRetriever.retrieve}.
|
|
92
|
+
*/
|
|
93
|
+
export interface HybridRetrieveOptions {
|
|
94
|
+
/** Final truncation after merge + rerank. @default 10 */
|
|
95
|
+
recallTopK?: number;
|
|
96
|
+
/** Over-fetch multiplier for each side before merge. @default 3 */
|
|
97
|
+
overFetchMultiplier?: number;
|
|
98
|
+
denseWeight?: number;
|
|
99
|
+
sparseWeight?: number;
|
|
100
|
+
rrfK?: number;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Hybrid BM25 + dense retriever.
|
|
104
|
+
*
|
|
105
|
+
* @example
|
|
106
|
+
* ```ts
|
|
107
|
+
* const hybrid = new HybridRetriever({ memoryStore, rerankerService });
|
|
108
|
+
* // At ingest:
|
|
109
|
+
* hybrid.bm25.addDocument(trace.id, trace.content, { tag: 'bench-session:s-1' });
|
|
110
|
+
* // At query time:
|
|
111
|
+
* const result = await hybrid.retrieve(
|
|
112
|
+
* 'What did the user say about their mortgage?',
|
|
113
|
+
* { valence: 0, arousal: 0, dominance: 0 },
|
|
114
|
+
* { scope: 'user', scopeId: 'u1' },
|
|
115
|
+
* { recallTopK: 10 },
|
|
116
|
+
* );
|
|
117
|
+
* ```
|
|
118
|
+
*/
|
|
119
|
+
export declare class HybridRetriever {
|
|
120
|
+
readonly bm25: BM25Index;
|
|
121
|
+
private readonly memoryStore;
|
|
122
|
+
private readonly rerankerService?;
|
|
123
|
+
private readonly hydeRetriever?;
|
|
124
|
+
private readonly splitAmbiguousThreshold?;
|
|
125
|
+
private readonly defaultDenseWeight;
|
|
126
|
+
private readonly defaultSparseWeight;
|
|
127
|
+
private readonly defaultRrfK;
|
|
128
|
+
constructor(opts: HybridRetrieverOptions);
|
|
129
|
+
retrieve(query: string, mood: PADState, scope: {
|
|
130
|
+
scope: MemoryScope;
|
|
131
|
+
scopeId: string;
|
|
132
|
+
}, options?: HybridRetrieveOptions): Promise<CognitiveRetrievalResult>;
|
|
133
|
+
/** Assemble the CognitiveRetrievalResult shape. */
|
|
134
|
+
private buildResult;
|
|
135
|
+
/**
|
|
136
|
+
* Step-6: split bottom-fraction traces by neural score, rescore the
|
|
137
|
+
* halves, replace a trace's content with its better half IFF the
|
|
138
|
+
* better half's neural score outranks the original's. Monotonic.
|
|
139
|
+
*
|
|
140
|
+
* Modifies `hydrated` in place: `trace.content` and `trace.retrievalScore`
|
|
141
|
+
* are updated for replaced traces. Returns a diagnostic summary.
|
|
142
|
+
*/
|
|
143
|
+
private refineAmbiguous;
|
|
144
|
+
/**
|
|
145
|
+
* Split a string at the sentence boundary nearest its midpoint.
|
|
146
|
+
* Returns [firstHalf, secondHalf] or null if the string is too short
|
|
147
|
+
* or no valid boundary is found.
|
|
148
|
+
*/
|
|
149
|
+
private splitAtMidpointSentence;
|
|
150
|
+
}
|
|
151
|
+
//# sourceMappingURL=HybridRetriever.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"HybridRetriever.d.ts","sourceRoot":"","sources":["../../../../src/memory/retrieval/hybrid/HybridRetriever.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AAEH,OAAO,EAAE,SAAS,EAAE,KAAK,UAAU,EAAE,MAAM,kCAAkC,CAAC;AAE9E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,KAAK,EACV,wBAAwB,EACxB,WAAW,EAEZ,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAErD;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,WAAW,EAAE,WAAW,CAAC;IACzB,yEAAyE;IACzE,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB;;;;OAIG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;;;OAQG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B;;;;;;;;OAQG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,gDAAgD;IAChD,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,iDAAiD;IACjD,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,wCAAwC;IACxC,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,yDAAyD;IACzD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,mEAAmE;IACnE,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,eAAe;IAC1B,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAEzB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAgB;IAC/C,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAS;IAClD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;gBAEzB,IAAI,EAAE,sBAAsB;IAWlC,QAAQ,CACZ,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,QAAQ,EACd,KAAK,EAAE;QAAE,KAAK,EAAE,WAAW,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,EAC9C,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,wBAAwB,CAAC;IAkIpC,mDAAmD;IACnD,OAAO,CAAC,WAAW;IA2BnB;;;;;;;OAOG;YACW,eAAe;IAiE7B;;;;OAIG;IACH,OAAO,CAAC,uBAAuB;CAwBhC"}
|
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file HybridRetriever.ts
|
|
3
|
+
* @description Hybrid BM25 + dense retrieval for memory-domain traces.
|
|
4
|
+
* Dense side uses {@link MemoryStore} (preserves 6-signal cognitive
|
|
5
|
+
* scoring). Sparse side uses a per-instance {@link BM25Index}. RRF
|
|
6
|
+
* merges by rank. Optional {@link RerankerService} runs over the
|
|
7
|
+
* merged pool.
|
|
8
|
+
*
|
|
9
|
+
* ## What this does
|
|
10
|
+
*
|
|
11
|
+
* Given a query, runs dense retrieval through `MemoryStore.query`
|
|
12
|
+
* (cognitive-scored traces) and sparse retrieval through an owned
|
|
13
|
+
* `BM25Index` (keyword-matched trace content). Fuses the two ranked
|
|
14
|
+
* lists via Reciprocal Rank Fusion, optionally reranks the merged
|
|
15
|
+
* pool with a neural cross-encoder, and returns a standard
|
|
16
|
+
* `CognitiveRetrievalResult` so downstream consumers (prompt
|
|
17
|
+
* assembly, bench adapters) don't change shape.
|
|
18
|
+
*
|
|
19
|
+
* ## Why a separate class, not a `CognitiveMemoryManager` option
|
|
20
|
+
*
|
|
21
|
+
* Keeps the existing manager retrieval path untouched (same reason
|
|
22
|
+
* as SessionRetriever in Step 2). MVP ships as opt-in.
|
|
23
|
+
*
|
|
24
|
+
* ## Rerank integration is mandatory-wired from the bench
|
|
25
|
+
*
|
|
26
|
+
* Per the Step 2 post-mortem (rerank-skip was the root cause of
|
|
27
|
+
* that step's RED verdict), Step 3 threads rerank from day 1 when
|
|
28
|
+
* the bench is configured with `--rerank cohere`. Callers outside
|
|
29
|
+
* the bench can pass `undefined` for `rerankerService` to skip
|
|
30
|
+
* rerank explicitly.
|
|
31
|
+
*
|
|
32
|
+
* ## Sparse-only documents are skipped in MVP
|
|
33
|
+
*
|
|
34
|
+
* A document that appears in `bm25.search` results but NOT in the
|
|
35
|
+
* dense over-fetch pool is skipped. Rationale: at the default
|
|
36
|
+
* over-fetch=3 and K=10 (30 dense candidates), a doc ranked top-30
|
|
37
|
+
* on sparse is very likely in dense's top-30 on any coherent query.
|
|
38
|
+
* Measured impact is expected to be negligible. If Tier A surfaces
|
|
39
|
+
* a meaningful drop rate, the fix is to add a
|
|
40
|
+
* `memoryStore.getTrace(id)` hydration path.
|
|
41
|
+
*
|
|
42
|
+
* @module agentos/memory/retrieval/hybrid/HybridRetriever
|
|
43
|
+
*/
|
|
44
|
+
import { BM25Index } from '../../../rag/search/BM25Index.js';
|
|
45
|
+
import { reciprocalRankFusion } from './reciprocalRankFusion.js';
|
|
46
|
+
/**
|
|
47
|
+
* Hybrid BM25 + dense retriever.
|
|
48
|
+
*
|
|
49
|
+
* @example
|
|
50
|
+
* ```ts
|
|
51
|
+
* const hybrid = new HybridRetriever({ memoryStore, rerankerService });
|
|
52
|
+
* // At ingest:
|
|
53
|
+
* hybrid.bm25.addDocument(trace.id, trace.content, { tag: 'bench-session:s-1' });
|
|
54
|
+
* // At query time:
|
|
55
|
+
* const result = await hybrid.retrieve(
|
|
56
|
+
* 'What did the user say about their mortgage?',
|
|
57
|
+
* { valence: 0, arousal: 0, dominance: 0 },
|
|
58
|
+
* { scope: 'user', scopeId: 'u1' },
|
|
59
|
+
* { recallTopK: 10 },
|
|
60
|
+
* );
|
|
61
|
+
* ```
|
|
62
|
+
*/
|
|
63
|
+
export class HybridRetriever {
|
|
64
|
+
constructor(opts) {
|
|
65
|
+
this.memoryStore = opts.memoryStore;
|
|
66
|
+
this.bm25 = new BM25Index(opts.bm25Config);
|
|
67
|
+
this.rerankerService = opts.rerankerService;
|
|
68
|
+
this.hydeRetriever = opts.hydeRetriever;
|
|
69
|
+
this.splitAmbiguousThreshold = opts.splitAmbiguousThreshold;
|
|
70
|
+
this.defaultDenseWeight = opts.defaultDenseWeight ?? 0.7;
|
|
71
|
+
this.defaultSparseWeight = opts.defaultSparseWeight ?? 0.3;
|
|
72
|
+
this.defaultRrfK = opts.defaultRrfK ?? 60;
|
|
73
|
+
}
|
|
74
|
+
async retrieve(query, mood, scope, options = {}) {
|
|
75
|
+
const startTime = Date.now();
|
|
76
|
+
const recallTopK = options.recallTopK ?? 10;
|
|
77
|
+
const overFetchMultiplier = options.overFetchMultiplier ?? 3;
|
|
78
|
+
const overFetchTopK = recallTopK * overFetchMultiplier;
|
|
79
|
+
const wDense = options.denseWeight ?? this.defaultDenseWeight;
|
|
80
|
+
const wSparse = options.sparseWeight ?? this.defaultSparseWeight;
|
|
81
|
+
const rrfK = options.rrfK ?? this.defaultRrfK;
|
|
82
|
+
// HyDE expansion (Step 4): when a hydeRetriever is attached,
|
|
83
|
+
// generate a hypothetical answer and use it as the query for BOTH
|
|
84
|
+
// dense and sparse sides. The reranker (below) keeps the ORIGINAL
|
|
85
|
+
// query so it scores documents against the user's real intent,
|
|
86
|
+
// not the hypothesis. Errors are non-critical — fall back to raw.
|
|
87
|
+
let effectiveQuery = query;
|
|
88
|
+
let hypothesisDiagnostic;
|
|
89
|
+
if (this.hydeRetriever) {
|
|
90
|
+
try {
|
|
91
|
+
const hypo = await this.hydeRetriever.generateHypothesis(query);
|
|
92
|
+
if (hypo.hypothesis && hypo.hypothesis.trim().length > 0) {
|
|
93
|
+
effectiveQuery = hypo.hypothesis;
|
|
94
|
+
hypothesisDiagnostic = hypo.hypothesis.slice(0, 120);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
catch {
|
|
98
|
+
// HyDE generation failed — raw query fallback.
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
// Dense side: use MemoryStore.query so we keep the 6-signal
|
|
102
|
+
// cognitive scoring (strength, recency, etc.) — matches baseline.
|
|
103
|
+
const { scored: denseScored, timings: denseTimings } = await this.memoryStore.query(effectiveQuery, mood, { topK: overFetchTopK, scopes: [scope] });
|
|
104
|
+
// Sparse side: BM25 over the per-instance index.
|
|
105
|
+
const sparseResults = this.bm25.search(effectiveQuery, overFetchTopK);
|
|
106
|
+
// Fallback: empty BM25 index or zero sparse hits => dense-only
|
|
107
|
+
// with explicit escalation diagnostic.
|
|
108
|
+
if (sparseResults.length === 0) {
|
|
109
|
+
return this.buildResult(denseScored.slice(0, recallTopK), {
|
|
110
|
+
escalations: ['hybrid-retriever:sparse-empty'],
|
|
111
|
+
candidatesScanned: denseScored.length,
|
|
112
|
+
vectorSearchMs: denseTimings.vectorSearchMs,
|
|
113
|
+
scoringMs: denseTimings.scoringMs,
|
|
114
|
+
totalMs: Date.now() - startTime,
|
|
115
|
+
hypothesis: hypothesisDiagnostic,
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
// Build 1-based ranked lists for RRF.
|
|
119
|
+
const denseRanked = denseScored.map((t, i) => ({ id: t.id, rank: i + 1 }));
|
|
120
|
+
const sparseRanked = sparseResults.map((r, i) => ({ id: r.id, rank: i + 1 }));
|
|
121
|
+
const merged = reciprocalRankFusion(denseRanked, sparseRanked, {
|
|
122
|
+
denseWeight: wDense,
|
|
123
|
+
sparseWeight: wSparse,
|
|
124
|
+
k: rrfK,
|
|
125
|
+
});
|
|
126
|
+
// Hydrate: resolve each RRFResult.id to the ScoredMemoryTrace from
|
|
127
|
+
// the dense side. Skip sparse-only docs (see file docstring).
|
|
128
|
+
const denseById = new Map(denseScored.map((t) => [t.id, t]));
|
|
129
|
+
const hydrated = [];
|
|
130
|
+
for (const m of merged) {
|
|
131
|
+
const trace = denseById.get(m.id);
|
|
132
|
+
if (trace) {
|
|
133
|
+
hydrated.push(trace);
|
|
134
|
+
}
|
|
135
|
+
// MVP: sparse-only docs (not in denseById) are skipped.
|
|
136
|
+
}
|
|
137
|
+
// Optional rerank: same 0.7 cognitive + 0.3 neural blend as baseline.
|
|
138
|
+
let splitDiagnostic;
|
|
139
|
+
if (this.rerankerService && hydrated.length > 0) {
|
|
140
|
+
try {
|
|
141
|
+
const rerankerOutput = await this.rerankerService.rerank({
|
|
142
|
+
query,
|
|
143
|
+
documents: hydrated.map((t) => ({
|
|
144
|
+
id: t.id,
|
|
145
|
+
content: t.content,
|
|
146
|
+
originalScore: t.retrievalScore,
|
|
147
|
+
})),
|
|
148
|
+
}, { topN: hydrated.length });
|
|
149
|
+
const neuralScores = new Map(rerankerOutput.results.map((r) => [r.id, r.relevanceScore]));
|
|
150
|
+
for (const trace of hydrated) {
|
|
151
|
+
const neural = neuralScores.get(trace.id);
|
|
152
|
+
if (neural !== undefined) {
|
|
153
|
+
trace.retrievalScore = 0.7 * trace.retrievalScore + 0.3 * neural;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
// Step-6: split-on-ambiguous refinement.
|
|
157
|
+
if (this.splitAmbiguousThreshold !== undefined &&
|
|
158
|
+
this.splitAmbiguousThreshold > 0 &&
|
|
159
|
+
hydrated.length > 0) {
|
|
160
|
+
splitDiagnostic = await this.refineAmbiguous(hydrated, neuralScores, query, this.splitAmbiguousThreshold);
|
|
161
|
+
}
|
|
162
|
+
hydrated.sort((a, b) => b.retrievalScore - a.retrievalScore);
|
|
163
|
+
}
|
|
164
|
+
catch {
|
|
165
|
+
// Reranker errors are non-critical: keep RRF ordering.
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
// Truncate to recallTopK.
|
|
169
|
+
const truncated = hydrated.slice(0, recallTopK);
|
|
170
|
+
return this.buildResult(truncated, {
|
|
171
|
+
candidatesScanned: denseScored.length + sparseResults.length,
|
|
172
|
+
vectorSearchMs: denseTimings.vectorSearchMs,
|
|
173
|
+
scoringMs: denseTimings.scoringMs,
|
|
174
|
+
totalMs: Date.now() - startTime,
|
|
175
|
+
hypothesis: hypothesisDiagnostic,
|
|
176
|
+
splitOnAmbiguous: splitDiagnostic,
|
|
177
|
+
});
|
|
178
|
+
}
|
|
179
|
+
/** Assemble the CognitiveRetrievalResult shape. */
|
|
180
|
+
buildResult(retrieved, d) {
|
|
181
|
+
return {
|
|
182
|
+
retrieved,
|
|
183
|
+
partiallyRetrieved: [],
|
|
184
|
+
diagnostics: {
|
|
185
|
+
candidatesScanned: d.candidatesScanned,
|
|
186
|
+
vectorSearchTimeMs: d.vectorSearchMs,
|
|
187
|
+
scoringTimeMs: d.scoringMs,
|
|
188
|
+
totalTimeMs: d.totalMs,
|
|
189
|
+
...(d.escalations ? { escalations: d.escalations } : {}),
|
|
190
|
+
...(d.hypothesis ? { hyde: { hypothesis: d.hypothesis } } : {}),
|
|
191
|
+
...(d.splitOnAmbiguous ? { splitOnAmbiguous: d.splitOnAmbiguous } : {}),
|
|
192
|
+
},
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Step-6: split bottom-fraction traces by neural score, rescore the
|
|
197
|
+
* halves, replace a trace's content with its better half IFF the
|
|
198
|
+
* better half's neural score outranks the original's. Monotonic.
|
|
199
|
+
*
|
|
200
|
+
* Modifies `hydrated` in place: `trace.content` and `trace.retrievalScore`
|
|
201
|
+
* are updated for replaced traces. Returns a diagnostic summary.
|
|
202
|
+
*/
|
|
203
|
+
async refineAmbiguous(hydrated, neuralScores, query, threshold) {
|
|
204
|
+
const replacedIds = [];
|
|
205
|
+
const sortedByNeural = hydrated
|
|
206
|
+
.map((t) => ({ trace: t, neural: neuralScores.get(t.id) ?? 0 }))
|
|
207
|
+
.sort((a, b) => a.neural - b.neural);
|
|
208
|
+
const candidateCount = Math.ceil(hydrated.length * threshold);
|
|
209
|
+
const candidates = sortedByNeural.slice(0, candidateCount);
|
|
210
|
+
const splits = [];
|
|
211
|
+
for (const { trace, neural } of candidates) {
|
|
212
|
+
const halves = this.splitAtMidpointSentence(trace.content);
|
|
213
|
+
if (!halves)
|
|
214
|
+
continue;
|
|
215
|
+
splits.push({
|
|
216
|
+
traceId: trace.id,
|
|
217
|
+
halfAId: `${trace.id}::a`,
|
|
218
|
+
halfBId: `${trace.id}::b`,
|
|
219
|
+
halfA: halves[0],
|
|
220
|
+
halfB: halves[1],
|
|
221
|
+
originalNeural: neural,
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
if (splits.length === 0) {
|
|
225
|
+
return { threshold, candidateCount, replacedIds };
|
|
226
|
+
}
|
|
227
|
+
const halfDocs = splits.flatMap((s) => [
|
|
228
|
+
{ id: s.halfAId, content: s.halfA },
|
|
229
|
+
{ id: s.halfBId, content: s.halfB },
|
|
230
|
+
]);
|
|
231
|
+
let halfScores;
|
|
232
|
+
try {
|
|
233
|
+
const halfOut = await this.rerankerService.rerank({ query, documents: halfDocs }, { topN: halfDocs.length });
|
|
234
|
+
halfScores = new Map(halfOut.results.map((r) => [r.id, r.relevanceScore]));
|
|
235
|
+
}
|
|
236
|
+
catch {
|
|
237
|
+
return { threshold, candidateCount, replacedIds };
|
|
238
|
+
}
|
|
239
|
+
const traceById = new Map(hydrated.map((t) => [t.id, t]));
|
|
240
|
+
for (const s of splits) {
|
|
241
|
+
const a = halfScores.get(s.halfAId) ?? -Infinity;
|
|
242
|
+
const b = halfScores.get(s.halfBId) ?? -Infinity;
|
|
243
|
+
const winningScore = Math.max(a, b);
|
|
244
|
+
if (winningScore <= s.originalNeural)
|
|
245
|
+
continue;
|
|
246
|
+
const winningText = a >= b ? s.halfA : s.halfB;
|
|
247
|
+
const trace = traceById.get(s.traceId);
|
|
248
|
+
if (!trace)
|
|
249
|
+
continue;
|
|
250
|
+
trace.content = winningText;
|
|
251
|
+
trace.retrievalScore += 0.3 * (winningScore - s.originalNeural);
|
|
252
|
+
replacedIds.push(s.traceId);
|
|
253
|
+
}
|
|
254
|
+
return { threshold, candidateCount, replacedIds };
|
|
255
|
+
}
|
|
256
|
+
/**
|
|
257
|
+
* Split a string at the sentence boundary nearest its midpoint.
|
|
258
|
+
* Returns [firstHalf, secondHalf] or null if the string is too short
|
|
259
|
+
* or no valid boundary is found.
|
|
260
|
+
*/
|
|
261
|
+
splitAtMidpointSentence(text) {
|
|
262
|
+
if (text.length < 50)
|
|
263
|
+
return null;
|
|
264
|
+
const mid = Math.floor(text.length / 2);
|
|
265
|
+
const window = Math.floor(text.length * 0.4);
|
|
266
|
+
const lo = Math.max(0, mid - window);
|
|
267
|
+
const hi = Math.min(text.length, mid + window);
|
|
268
|
+
for (let offset = 0; offset <= window; offset++) {
|
|
269
|
+
for (const sign of [-1, 1]) {
|
|
270
|
+
const i = mid + sign * offset;
|
|
271
|
+
if (i < lo || i > hi)
|
|
272
|
+
continue;
|
|
273
|
+
if (i > 0 &&
|
|
274
|
+
i < text.length - 1 &&
|
|
275
|
+
/[.!?]/.test(text[i]) &&
|
|
276
|
+
/\s/.test(text[i + 1])) {
|
|
277
|
+
return [text.slice(0, i + 1).trim(), text.slice(i + 1).trim()];
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
const spaceIdx = text.indexOf(' ', mid);
|
|
282
|
+
if (spaceIdx === -1 || spaceIdx >= text.length - 1)
|
|
283
|
+
return null;
|
|
284
|
+
return [text.slice(0, spaceIdx).trim(), text.slice(spaceIdx + 1).trim()];
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
//# sourceMappingURL=HybridRetriever.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"HybridRetriever.js","sourceRoot":"","sources":["../../../../src/memory/retrieval/hybrid/HybridRetriever.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AAEH,OAAO,EAAE,SAAS,EAAmB,MAAM,kCAAkC,CAAC;AAC9E,OAAO,EAAE,oBAAoB,EAAkB,MAAM,2BAA2B,CAAC;AAiEjF;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,OAAO,eAAe;IAW1B,YAAY,IAA4B;QACtC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;QACpC,IAAI,CAAC,IAAI,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC3C,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC;QAC5C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC;QACxC,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,uBAAuB,CAAC;QAC5D,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,IAAI,GAAG,CAAC;QACzD,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,IAAI,GAAG,CAAC;QAC3D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,QAAQ,CACZ,KAAa,EACb,IAAc,EACd,KAA8C,EAC9C,UAAiC,EAAE;QAEnC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC;QAC5C,MAAM,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,CAAC,CAAC;QAC7D,MAAM,aAAa,GAAG,UAAU,GAAG,mBAAmB,CAAC;QACvD,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,kBAAkB,CAAC;QAC9D,MAAM,OAAO,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,mBAAmB,CAAC;QACjE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC;QAE9C,6DAA6D;QAC7D,kEAAkE;QAClE,kEAAkE;QAClE,+DAA+D;QAC/D,kEAAkE;QAClE,IAAI,cAAc,GAAG,KAAK,CAAC;QAC3B,IAAI,oBAAwC,CAAC;QAC7C,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;gBAChE,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzD,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC;oBACjC,oBAAoB,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBACvD,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,+CAA+C;YACjD,CAAC;QACH,CAAC;QAED,4DAA4D;QAC5D,kEAAkE;QAClE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CACjF,cAAc,EACd,IAAI,EACJ,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,EAAE,CACzC,CAAC;QAEF,iDAAiD;QACjD,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC;QAEtE,+DAA+D;QAC/D,uCAAuC;QACvC,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE;gBACxD,WAAW,EAAE,CAAC,+BAA+B,CAAC;gBAC9C,iBAAiB,EAAE,WAAW,CAAC,MAAM;gBACrC,cAAc,EAAE,YAAY,CAAC,cAAc;gBAC3C,SAAS,EAAE,YAAY,CAAC,SAAS;gBACjC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAC/B,UAAU,EAAE,oBAAoB;aACjC,CAAC,CAAC;QACL,CAAC;QAED,sCAAsC;QACtC,MAAM,WAAW,GAAgB,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QACxF,MAAM,YAAY,GAAgB,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAC3F,MAAM,MAAM,GAAG,oBAAoB,CAAC,WAAW,EAAE,YAAY,EAAE;YAC7D,WAAW,EAAE,MAAM;YACnB,YAAY,EAAE,OAAO;YACrB,CAAC,EAAE,IAAI;SACR,CAAC,CAAC;QAEH,mEAAmE;QACnE,8DAA8D;QAC9D,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7D,MAAM,QAAQ,GAAwB,EAAE,CAAC;QACzC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAClC,IAAI,KAAK,EAAE,CAAC;gBACV,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC;YACD,wDAAwD;QAC1D,CAAC;QAED,sEAAsE;QACtE,IAAI,eAAiG,CAAC;QACtG,IAAI,IAAI,CAAC,eAAe,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChD,IAAI,CAAC;gBACH,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,CACtD;oBACE,KAAK;oBACL,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBAC9B,EAAE,EAAE,CAAC,CAAC,EAAE;wBACR,OAAO,EAAE,CAAC,CAAC,OAAO;wBAClB,aAAa,EAAE,CAAC,CAAC,cAAc;qBAChC,CAAC,CAAC;iBACJ,EACD,EAAE,IAAI,EAAE,QAAQ,CAAC,MAAM,EAAE,CAC1B,CAAC;gBACF,MAAM,YAAY,GAAG,IAAI,GAAG,CAC1B,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,CAAC,CAC5D,CAAC;gBACF,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;oBAC7B,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBAC1C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;wBACzB,KAAK,CAAC,cAAc,GAAG,GAAG,GAAG,KAAK,CAAC,cAAc,GAAG,GAAG,GAAG,MAAM,CAAC;oBACnE,CAAC;gBACH,CAAC;gBAED,yCAAyC;gBACzC,IACE,IAAI,CAAC,uBAAuB,KAAK,SAAS;oBAC1C,IAAI,CAAC,uBAAuB,GAAG,CAAC;oBAChC,QAAQ,CAAC,MAAM,GAAG,CAAC,EACnB,CAAC;oBACD,eAAe,GAAG,MAAM,IAAI,CAAC,eAAe,CAC1C,QAAQ,EACR,YAAY,EACZ,KAAK,EACL,IAAI,CAAC,uBAAuB,CAC7B,CAAC;gBACJ,CAAC;gBAED,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC;YAC/D,CAAC;YAAC,MAAM,CAAC;gBACP,uDAAuD;YACzD,CAAC;QACH,CAAC;QAED,0BAA0B;QAC1B,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QAChD,OAAO,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE;YACjC,iBAAiB,EAAE,WAAW,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM;YAC5D,cAAc,EAAE,YAAY,CAAC,cAAc;YAC3C,SAAS,EAAE,YAAY,CAAC,SAAS;YACjC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAC/B,UAAU,EAAE,oBAAoB;YAChC,gBAAgB,EAAE,eAAe;SAClC,CAAC,CAAC;IACL,CAAC;IAED,mDAAmD;IAC3C,WAAW,CACjB,SAA8B,EAC9B,CAQC;QAED,OAAO;YACL,SAAS;YACT,kBAAkB,EAAE,EAAE;YACtB,WAAW,EAAE;gBACX,iBAAiB,EAAE,CAAC,CAAC,iBAAiB;gBACtC,kBAAkB,EAAE,CAAC,CAAC,cAAc;gBACpC,aAAa,EAAE,CAAC,CAAC,SAAS;gBAC1B,WAAW,EAAE,CAAC,CAAC,OAAO;gBACtB,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACxD,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/D,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,CAAC,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACxE;SACF,CAAC;IACJ,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,eAAe,CAC3B,QAA6B,EAC7B,YAAiC,EACjC,KAAa,EACb,SAAiB;QAEjB,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,MAAM,cAAc,GAAG,QAAQ;aAC5B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;aAC/D,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;QACvC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;QAC9D,MAAM,UAAU,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAG3D,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,KAAK,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,UAAU,EAAE,CAAC;YAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC3D,IAAI,CAAC,MAAM;gBAAE,SAAS;YACtB,MAAM,CAAC,IAAI,CAAC;gBACV,OAAO,EAAE,KAAK,CAAC,EAAE;gBACjB,OAAO,EAAE,GAAG,KAAK,CAAC,EAAE,KAAK;gBACzB,OAAO,EAAE,GAAG,KAAK,CAAC,EAAE,KAAK;gBACzB,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;gBAChB,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;gBAChB,cAAc,EAAE,MAAM;aACvB,CAAC,CAAC;QACL,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,CAAC;QACpD,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YACrC,EAAE,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,KAAK,EAAE;YACnC,EAAE,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,KAAK,EAAE;SACpC,CAAC,CAAC;QACH,IAAI,UAA+B,CAAC;QACpC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,eAAgB,CAAC,MAAM,CAChD,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,EAC9B,EAAE,IAAI,EAAE,QAAQ,CAAC,MAAM,EAAE,CAC1B,CAAC;YACF,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;QAC7E,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,CAAC;QACpD,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1D,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,MAAM,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC;YACjD,MAAM,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC;YACjD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACpC,IAAI,YAAY,IAAI,CAAC,CAAC,cAAc;gBAAE,SAAS;YAC/C,MAAM,WAAW,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;YAC/C,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACvC,IAAI,CAAC,KAAK;gBAAE,SAAS;YACrB,KAAK,CAAC,OAAO,GAAG,WAAW,CAAC;YAC5B,KAAK,CAAC,cAAc,IAAI,GAAG,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC;YAChE,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC9B,CAAC;QAED,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,CAAC;IACpD,CAAC;IAED;;;;OAIG;IACK,uBAAuB,CAAC,IAAY;QAC1C,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE;YAAE,OAAO,IAAI,CAAC;QAClC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;QAC7C,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,MAAM,CAAC,CAAC;QACrC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,MAAM,CAAC,CAAC;QAC/C,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,IAAI,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;YAChD,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAU,EAAE,CAAC;gBACpC,MAAM,CAAC,GAAG,GAAG,GAAG,IAAI,GAAG,MAAM,CAAC;gBAC9B,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE;oBAAE,SAAS;gBAC/B,IACE,CAAC,GAAG,CAAC;oBACL,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC;oBACnB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;oBACrB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EACtB,CAAC;oBACD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBACjE,CAAC;YACH,CAAC;QACH,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACxC,IAAI,QAAQ,KAAK,CAAC,CAAC,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QAChE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3E,CAAC;CACF"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file index.ts
|
|
3
|
+
* @description Barrel exports for hybrid BM25 + dense retrieval
|
|
4
|
+
* (Step 3 of the RAG stack sequenced rollout).
|
|
5
|
+
*
|
|
6
|
+
* @module agentos/memory/retrieval/hybrid
|
|
7
|
+
*/
|
|
8
|
+
export { HybridRetriever } from './HybridRetriever.js';
|
|
9
|
+
export type { HybridRetrieverOptions, HybridRetrieveOptions, } from './HybridRetriever.js';
|
|
10
|
+
export { reciprocalRankFusion } from './reciprocalRankFusion.js';
|
|
11
|
+
export type { RankedDoc, RRFOptions, RRFResult, } from './reciprocalRankFusion.js';
|
|
12
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/memory/retrieval/hybrid/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,YAAY,EACV,sBAAsB,EACtB,qBAAqB,GACtB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACjE,YAAY,EACV,SAAS,EACT,UAAU,EACV,SAAS,GACV,MAAM,2BAA2B,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file index.ts
|
|
3
|
+
* @description Barrel exports for hybrid BM25 + dense retrieval
|
|
4
|
+
* (Step 3 of the RAG stack sequenced rollout).
|
|
5
|
+
*
|
|
6
|
+
* @module agentos/memory/retrieval/hybrid
|
|
7
|
+
*/
|
|
8
|
+
export { HybridRetriever } from './HybridRetriever.js';
|
|
9
|
+
export { reciprocalRankFusion } from './reciprocalRankFusion.js';
|
|
10
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/memory/retrieval/hybrid/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAMvD,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC"}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file reciprocalRankFusion.ts
|
|
3
|
+
* @description Rank-based fusion of two ranked document lists using
|
|
4
|
+
* the Reciprocal Rank Fusion (RRF) algorithm (Cormack et al. 2009).
|
|
5
|
+
*
|
|
6
|
+
* ## What this does
|
|
7
|
+
*
|
|
8
|
+
* Takes two 1-based ranked lists of document ids (one from dense
|
|
9
|
+
* retrieval, one from sparse retrieval) and merges them into a single
|
|
10
|
+
* ranked list via:
|
|
11
|
+
*
|
|
12
|
+
* ```
|
|
13
|
+
* score(d) = w_dense / (k + rank_dense(d)) + w_sparse / (k + rank_sparse(d))
|
|
14
|
+
* ```
|
|
15
|
+
*
|
|
16
|
+
* Missing ranks (a doc appearing on only one side) contribute zero
|
|
17
|
+
* from the other side.
|
|
18
|
+
*
|
|
19
|
+
* ## Why rank-based (not score-based)
|
|
20
|
+
*
|
|
21
|
+
* Dense scorers (cosine similarity, cognitive composites) and sparse
|
|
22
|
+
* scorers (BM25) produce values on different scales with different
|
|
23
|
+
* distributions. Score-weighted fusion requires calibration; rank
|
|
24
|
+
* fusion sidesteps this entirely. Published RAG literature
|
|
25
|
+
* consistently prefers RRF over weighted-sum for heterogeneous
|
|
26
|
+
* retrievers.
|
|
27
|
+
*
|
|
28
|
+
* ## Stable ordering
|
|
29
|
+
*
|
|
30
|
+
* When two documents have identical RRF scores, they are ordered by
|
|
31
|
+
* id ascending. Deterministic across process restarts.
|
|
32
|
+
*
|
|
33
|
+
* @module agentos/memory/retrieval/hybrid/reciprocalRankFusion
|
|
34
|
+
*/
|
|
35
|
+
/**
|
|
36
|
+
* One document's position in a ranked retrieval result.
|
|
37
|
+
*/
|
|
38
|
+
export interface RankedDoc {
|
|
39
|
+
id: string;
|
|
40
|
+
/** 1-based rank. First result has rank 1. */
|
|
41
|
+
rank: number;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Options for {@link reciprocalRankFusion}.
|
|
45
|
+
*/
|
|
46
|
+
export interface RRFOptions {
|
|
47
|
+
/** Weight on the dense-side rank contribution. Default 0.7. */
|
|
48
|
+
denseWeight?: number;
|
|
49
|
+
/** Weight on the sparse-side rank contribution. Default 0.3. */
|
|
50
|
+
sparseWeight?: number;
|
|
51
|
+
/**
|
|
52
|
+
* RRF smoothing constant. Larger k flattens rank differences.
|
|
53
|
+
* Default 60 per Cormack et al. 2009.
|
|
54
|
+
*/
|
|
55
|
+
k?: number;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* One merged result from {@link reciprocalRankFusion}.
|
|
59
|
+
*/
|
|
60
|
+
export interface RRFResult {
|
|
61
|
+
id: string;
|
|
62
|
+
/** Fused score; higher = more relevant. */
|
|
63
|
+
score: number;
|
|
64
|
+
/** Rank in the dense list (undefined if doc was sparse-only). */
|
|
65
|
+
denseRank?: number;
|
|
66
|
+
/** Rank in the sparse list (undefined if doc was dense-only). */
|
|
67
|
+
sparseRank?: number;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Merge two ranked retrieval results via Reciprocal Rank Fusion.
|
|
71
|
+
*
|
|
72
|
+
* @param denseRanked - 1-based ranked list from dense retrieval.
|
|
73
|
+
* @param sparseRanked - 1-based ranked list from sparse retrieval.
|
|
74
|
+
* @param options - {@link RRFOptions}; defaults to w_dense=0.7, w_sparse=0.3, k=60.
|
|
75
|
+
* @returns Merged results sorted by fused score descending, stable
|
|
76
|
+
* tiebreak by id ascending.
|
|
77
|
+
*
|
|
78
|
+
* @example
|
|
79
|
+
* ```ts
|
|
80
|
+
* const dense = [{ id: 'a', rank: 1 }, { id: 'b', rank: 2 }];
|
|
81
|
+
* const sparse = [{ id: 'b', rank: 1 }, { id: 'c', rank: 2 }];
|
|
82
|
+
* const merged = reciprocalRankFusion(dense, sparse);
|
|
83
|
+
* // => [{ id: 'b', score: 0.0162, denseRank: 2, sparseRank: 1 }, ...]
|
|
84
|
+
* ```
|
|
85
|
+
*/
|
|
86
|
+
export declare function reciprocalRankFusion(denseRanked: RankedDoc[], sparseRanked: RankedDoc[], options?: RRFOptions): RRFResult[];
|
|
87
|
+
//# sourceMappingURL=reciprocalRankFusion.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reciprocalRankFusion.d.ts","sourceRoot":"","sources":["../../../../src/memory/retrieval/hybrid/reciprocalRankFusion.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,6CAA6C;IAC7C,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gEAAgE;IAChE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,CAAC,CAAC,EAAE,MAAM,CAAC;CACZ;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,2CAA2C;IAC3C,KAAK,EAAE,MAAM,CAAC;IACd,iEAAiE;IACjE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,SAAS,EAAE,EACxB,YAAY,EAAE,SAAS,EAAE,EACzB,OAAO,GAAE,UAAe,GACvB,SAAS,EAAE,CAoCb"}
|