@graphorin/memory 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +90 -1
- package/README.md +31 -31
- package/dist/conflict/index.d.ts +3 -1
- package/dist/conflict/index.js +1 -1
- package/dist/conflict/locale-packs/en.d.ts +13 -0
- package/dist/conflict/locale-packs/en.d.ts.map +1 -0
- package/dist/conflict/locale-packs/en.js +1 -1
- package/dist/conflict/locale-packs/en.js.map +1 -1
- package/dist/conflict/pipeline.d.ts +2 -2
- package/dist/conflict/pipeline.js +3 -3
- package/dist/conflict/pipeline.js.map +1 -1
- package/dist/conflict/stages/helpers.js +1 -1
- package/dist/conflict/stages/helpers.js.map +1 -1
- package/dist/conflict/stages/stage1-exact-dedup.js.map +1 -1
- package/dist/conflict/stages/stage2-embedding-three-zone.js.map +1 -1
- package/dist/conflict/stages/stage3-heuristic-regex.js +1 -1
- package/dist/conflict/stages/stage3-heuristic-regex.js.map +1 -1
- package/dist/conflict/stages/stage4-subject-predicate.js.map +1 -1
- package/dist/conflict/stages/stage5-defer-to-deep.js.map +1 -1
- package/dist/conflict/types.d.ts +7 -7
- package/dist/conflict/types.js +1 -1
- package/dist/conflict/types.js.map +1 -1
- package/dist/consolidator/budget.js +16 -3
- package/dist/consolidator/budget.js.map +1 -1
- package/dist/consolidator/decay.d.ts +13 -2
- package/dist/consolidator/decay.js +17 -5
- package/dist/consolidator/decay.js.map +1 -1
- package/dist/consolidator/dlq.js +1 -1
- package/dist/consolidator/dlq.js.map +1 -1
- package/dist/consolidator/errors.js +2 -2
- package/dist/consolidator/errors.js.map +1 -1
- package/dist/consolidator/index.js +8 -6
- package/dist/consolidator/index.js.map +1 -1
- package/dist/consolidator/lock.js +1 -1
- package/dist/consolidator/lock.js.map +1 -1
- package/dist/consolidator/noise-filter.js.map +1 -1
- package/dist/consolidator/phases/deep.js +6 -2
- package/dist/consolidator/phases/deep.js.map +1 -1
- package/dist/consolidator/phases/induce.d.ts +10 -10
- package/dist/consolidator/phases/induce.js +6 -6
- package/dist/consolidator/phases/induce.js.map +1 -1
- package/dist/consolidator/phases/learned-context.js +129 -0
- package/dist/consolidator/phases/learned-context.js.map +1 -0
- package/dist/consolidator/phases/light.js +2 -1
- package/dist/consolidator/phases/light.js.map +1 -1
- package/dist/consolidator/phases/reflect.js +5 -4
- package/dist/consolidator/phases/reflect.js.map +1 -1
- package/dist/consolidator/phases/standard.js +37 -8
- package/dist/consolidator/phases/standard.js.map +1 -1
- package/dist/consolidator/reconcile.js +1 -1
- package/dist/consolidator/reconcile.js.map +1 -1
- package/dist/consolidator/runtime.d.ts +4 -4
- package/dist/consolidator/runtime.d.ts.map +1 -1
- package/dist/consolidator/runtime.js +73 -16
- package/dist/consolidator/runtime.js.map +1 -1
- package/dist/consolidator/scheduler.d.ts +5 -5
- package/dist/consolidator/scheduler.js +1 -1
- package/dist/consolidator/scheduler.js.map +1 -1
- package/dist/consolidator/triggers.js +7 -7
- package/dist/consolidator/triggers.js.map +1 -1
- package/dist/consolidator/types.d.ts +68 -20
- package/dist/consolidator/types.d.ts.map +1 -1
- package/dist/consolidator/types.js +16 -6
- package/dist/consolidator/types.js.map +1 -1
- package/dist/context-engine/annotations.d.ts +9 -9
- package/dist/context-engine/annotations.js +3 -3
- package/dist/context-engine/annotations.js.map +1 -1
- package/dist/context-engine/auto-recall.js.map +1 -1
- package/dist/context-engine/compaction/clear-tool-results.d.ts +18 -3
- package/dist/context-engine/compaction/clear-tool-results.d.ts.map +1 -1
- package/dist/context-engine/compaction/clear-tool-results.js +26 -5
- package/dist/context-engine/compaction/clear-tool-results.js.map +1 -1
- package/dist/context-engine/compaction/compactor.js +19 -6
- package/dist/context-engine/compaction/compactor.js.map +1 -1
- package/dist/context-engine/compaction/hooks/reanchor-persona-block.js +4 -0
- package/dist/context-engine/compaction/hooks/reanchor-persona-block.js.map +1 -1
- package/dist/context-engine/compaction/hooks/reanchor-pinned-facts.js +1 -0
- package/dist/context-engine/compaction/hooks/reanchor-pinned-facts.js.map +1 -1
- package/dist/context-engine/compaction/hooks/reanchor-project-rules.js +2 -1
- package/dist/context-engine/compaction/hooks/reanchor-project-rules.js.map +1 -1
- package/dist/context-engine/compaction/hooks/reanchor-recent-results.d.ts +28 -0
- package/dist/context-engine/compaction/hooks/reanchor-recent-results.d.ts.map +1 -0
- package/dist/context-engine/compaction/hooks/reanchor-recent-results.js +71 -0
- package/dist/context-engine/compaction/hooks/reanchor-recent-results.js.map +1 -0
- package/dist/context-engine/compaction/hooks/types.d.ts +15 -3
- package/dist/context-engine/compaction/hooks/types.d.ts.map +1 -1
- package/dist/context-engine/compaction/index.d.ts +1 -0
- package/dist/context-engine/compaction/index.js +2 -1
- package/dist/context-engine/compaction/templates/summary-9-section.d.ts +10 -3
- package/dist/context-engine/compaction/templates/summary-9-section.d.ts.map +1 -1
- package/dist/context-engine/compaction/templates/summary-9-section.js +42 -5
- package/dist/context-engine/compaction/templates/summary-9-section.js.map +1 -1
- package/dist/context-engine/compaction/thresholds.js.map +1 -1
- package/dist/context-engine/compaction/types.d.ts +47 -7
- package/dist/context-engine/compaction/types.d.ts.map +1 -1
- package/dist/context-engine/engine.d.ts +23 -4
- package/dist/context-engine/engine.d.ts.map +1 -1
- package/dist/context-engine/engine.js +69 -15
- package/dist/context-engine/engine.js.map +1 -1
- package/dist/context-engine/index.d.ts +1 -0
- package/dist/context-engine/index.d.ts.map +1 -1
- package/dist/context-engine/index.js +2 -1
- package/dist/context-engine/locale-packs/en.js +14 -13
- package/dist/context-engine/locale-packs/en.js.map +1 -1
- package/dist/context-engine/locale-packs/types.d.ts +10 -10
- package/dist/context-engine/locale-packs/types.d.ts.map +1 -1
- package/dist/context-engine/locale-packs/types.js.map +1 -1
- package/dist/context-engine/metadata.d.ts +1 -1
- package/dist/context-engine/metadata.js +2 -2
- package/dist/context-engine/metadata.js.map +1 -1
- package/dist/context-engine/preambles/inbound-en.d.ts +1 -1
- package/dist/context-engine/preambles/inbound-en.js +1 -1
- package/dist/context-engine/preambles/inbound-en.js.map +1 -1
- package/dist/context-engine/privacy-filter.d.ts +2 -2
- package/dist/context-engine/privacy-filter.js +1 -1
- package/dist/context-engine/privacy-filter.js.map +1 -1
- package/dist/context-engine/templates/base-en.d.ts +1 -1
- package/dist/context-engine/templates/base-en.js +1 -1
- package/dist/context-engine/templates/base-en.js.map +1 -1
- package/dist/context-engine/templates/composer.d.ts +1 -1
- package/dist/context-engine/templates/composer.js.map +1 -1
- package/dist/context-engine/token-budget.d.ts +1 -1
- package/dist/context-engine/token-budget.js +1 -1
- package/dist/context-engine/token-budget.js.map +1 -1
- package/dist/context-engine/token-counter.d.ts +16 -7
- package/dist/context-engine/token-counter.d.ts.map +1 -1
- package/dist/context-engine/token-counter.js +31 -9
- package/dist/context-engine/token-counter.js.map +1 -1
- package/dist/errors/index.d.ts +2 -2
- package/dist/errors/index.js +4 -4
- package/dist/errors/index.js.map +1 -1
- package/dist/facade.d.ts +44 -15
- package/dist/facade.d.ts.map +1 -1
- package/dist/facade.js +16 -8
- package/dist/facade.js.map +1 -1
- package/dist/graph/entity-resolver.d.ts +2 -2
- package/dist/graph/entity-resolver.js +2 -2
- package/dist/graph/entity-resolver.js.map +1 -1
- package/dist/index.d.ts +9 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -4
- package/dist/index.js.map +1 -1
- package/dist/internal/contextualize.js +2 -2
- package/dist/internal/contextualize.js.map +1 -1
- package/dist/internal/embedder-binding.js +1 -1
- package/dist/internal/embedder-binding.js.map +1 -1
- package/dist/internal/id.js +1 -1
- package/dist/internal/id.js.map +1 -1
- package/dist/internal/injection-heuristics.js +25 -2
- package/dist/internal/injection-heuristics.js.map +1 -1
- package/dist/internal/spans.js +1 -1
- package/dist/internal/spans.js.map +1 -1
- package/dist/internal/storage-adapter.d.ts +71 -22
- package/dist/internal/storage-adapter.d.ts.map +1 -1
- package/dist/migration/embedder-migration.d.ts +7 -7
- package/dist/migration/embedder-migration.js.map +1 -1
- package/dist/search/explain.d.ts +7 -4
- package/dist/search/explain.d.ts.map +1 -1
- package/dist/search/explain.js +1 -1
- package/dist/search/explain.js.map +1 -1
- package/dist/search/fit-weights.d.ts +48 -0
- package/dist/search/fit-weights.d.ts.map +1 -0
- package/dist/search/fit-weights.js +70 -0
- package/dist/search/fit-weights.js.map +1 -0
- package/dist/search/graph-ppr.d.ts +36 -0
- package/dist/search/graph-ppr.d.ts.map +1 -0
- package/dist/search/graph-ppr.js +38 -0
- package/dist/search/graph-ppr.js.map +1 -0
- package/dist/search/index.d.ts +4 -1
- package/dist/search/index.js +4 -1
- package/dist/search/iterative.d.ts +15 -8
- package/dist/search/iterative.d.ts.map +1 -1
- package/dist/search/iterative.js +10 -9
- package/dist/search/iterative.js.map +1 -1
- package/dist/search/query-transform.d.ts +6 -6
- package/dist/search/query-transform.js +6 -6
- package/dist/search/query-transform.js.map +1 -1
- package/dist/search/rrf.d.ts +4 -4
- package/dist/search/rrf.js +4 -4
- package/dist/search/rrf.js.map +1 -1
- package/dist/search/trust.d.ts +30 -0
- package/dist/search/trust.d.ts.map +1 -0
- package/dist/search/trust.js +50 -0
- package/dist/search/trust.js.map +1 -0
- package/dist/search/types.d.ts +1 -1
- package/dist/tiers/episodic-memory.d.ts +6 -4
- package/dist/tiers/episodic-memory.d.ts.map +1 -1
- package/dist/tiers/episodic-memory.js +3 -2
- package/dist/tiers/episodic-memory.js.map +1 -1
- package/dist/tiers/index.d.ts +1 -1
- package/dist/tiers/index.js +1 -1
- package/dist/tiers/insight-memory.d.ts +4 -4
- package/dist/tiers/insight-memory.js +3 -3
- package/dist/tiers/insight-memory.js.map +1 -1
- package/dist/tiers/procedural-memory.d.ts +29 -7
- package/dist/tiers/procedural-memory.d.ts.map +1 -1
- package/dist/tiers/procedural-memory.js +66 -6
- package/dist/tiers/procedural-memory.js.map +1 -1
- package/dist/tiers/semantic-memory.d.ts +111 -30
- package/dist/tiers/semantic-memory.d.ts.map +1 -1
- package/dist/tiers/semantic-memory.js +120 -27
- package/dist/tiers/semantic-memory.js.map +1 -1
- package/dist/tiers/session-memory.d.ts +7 -7
- package/dist/tiers/session-memory.js +7 -7
- package/dist/tiers/session-memory.js.map +1 -1
- package/dist/tiers/shared-memory.d.ts +1 -1
- package/dist/tiers/shared-memory.js +1 -1
- package/dist/tiers/shared-memory.js.map +1 -1
- package/dist/tiers/working-memory.d.ts +3 -3
- package/dist/tiers/working-memory.js +2 -2
- package/dist/tiers/working-memory.js.map +1 -1
- package/dist/tools/block-tools.d.ts +9 -9
- package/dist/tools/block-tools.d.ts.map +1 -1
- package/dist/tools/block-tools.js +3 -3
- package/dist/tools/block-tools.js.map +1 -1
- package/dist/tools/fact-tools.d.ts +48 -31
- package/dist/tools/fact-tools.d.ts.map +1 -1
- package/dist/tools/fact-tools.js +30 -18
- package/dist/tools/fact-tools.js.map +1 -1
- package/dist/tools/index.d.ts +9 -2
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +4 -2
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/recall-tools.d.ts +40 -20
- package/dist/tools/recall-tools.d.ts.map +1 -1
- package/dist/tools/recall-tools.js +29 -10
- package/dist/tools/recall-tools.js.map +1 -1
- package/dist/tools/runbook-tools.d.ts +76 -0
- package/dist/tools/runbook-tools.d.ts.map +1 -0
- package/dist/tools/runbook-tools.js +53 -0
- package/dist/tools/runbook-tools.js.map +1 -0
- package/dist/tools/taint.js +33 -0
- package/dist/tools/taint.js.map +1 -0
- package/dist/tools/types.d.ts +1 -1
- package/package.json +8 -8
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"graph-ppr.d.ts","names":[],"sources":["../../src/search/graph-ppr.ts"],"sourcesContent":[],"mappings":";;AAkBA;AAaA;;;;;;;;;;;;;;;cAba,mBAAA;;;;;;;;;;;;iBAaG,aAAA,aACF"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
//#region src/search/graph-ppr.ts
|
|
2
|
+
/**
|
|
3
|
+
* PPR-lite graph scoring (D5) - damped spreading activation over the
|
|
4
|
+
* entity-graph neighbourhood, generalizing the flat one-hop expansion.
|
|
5
|
+
* Pure: given the seed facts (the lexical/vector candidates retrieved so
|
|
6
|
+
* far) and the graded neighbours (`{ fact, depth }` from
|
|
7
|
+
* `expandActivation`), it assigns each neighbour an activation score
|
|
8
|
+
* that decays with hop distance, so a fact two hops from a strong seed
|
|
9
|
+
* ranks below a direct neighbour instead of tying at a flat `1`.
|
|
10
|
+
*
|
|
11
|
+
* This is HippoRAG-style personalized-PageRank, "lite": a single damped
|
|
12
|
+
* BFS pass seeded from retrieval rather than an iterative power method.
|
|
13
|
+
* Seeding from query-matched entities (rather than retrieved candidates)
|
|
14
|
+
* is the eval-gated extension the roadmap defers until numbers justify.
|
|
15
|
+
*
|
|
16
|
+
* @packageDocumentation
|
|
17
|
+
*/
|
|
18
|
+
/** Default damping applied per hop (activation × damping^depth). */
|
|
19
|
+
const DEFAULT_PPR_DAMPING = .5;
|
|
20
|
+
/**
|
|
21
|
+
* Compute PPR-lite activation for graded neighbours. `damping` in
|
|
22
|
+
* `(0, 1]`; a neighbour at `depth` d contributes `damping^d` per seed
|
|
23
|
+
* that reaches it. Since `expandActivation` already returns the MINIMUM
|
|
24
|
+
* depth per neighbour (dedup across seeds), the score here is a pure
|
|
25
|
+
* function of that min-depth - `damping^depth`, clamped to `[0, 1]`.
|
|
26
|
+
* A neighbour list produced with `maxHops: 1` reproduces a flat-ish
|
|
27
|
+
* score of `damping` at every node; `maxHops: 2` separates the tiers.
|
|
28
|
+
*
|
|
29
|
+
* @stable
|
|
30
|
+
*/
|
|
31
|
+
function pprActivation(neighbours, damping = DEFAULT_PPR_DAMPING) {
|
|
32
|
+
const d = damping <= 0 || damping > 1 ? DEFAULT_PPR_DAMPING : damping;
|
|
33
|
+
return neighbours.map((n) => d ** Math.max(1, n.depth));
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
//#endregion
|
|
37
|
+
export { DEFAULT_PPR_DAMPING, pprActivation };
|
|
38
|
+
//# sourceMappingURL=graph-ppr.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"graph-ppr.js","names":[],"sources":["../../src/search/graph-ppr.ts"],"sourcesContent":["/**\n * PPR-lite graph scoring (D5) - damped spreading activation over the\n * entity-graph neighbourhood, generalizing the flat one-hop expansion.\n * Pure: given the seed facts (the lexical/vector candidates retrieved so\n * far) and the graded neighbours (`{ fact, depth }` from\n * `expandActivation`), it assigns each neighbour an activation score\n * that decays with hop distance, so a fact two hops from a strong seed\n * ranks below a direct neighbour instead of tying at a flat `1`.\n *\n * This is HippoRAG-style personalized-PageRank, \"lite\": a single damped\n * BFS pass seeded from retrieval rather than an iterative power method.\n * Seeding from query-matched entities (rather than retrieved candidates)\n * is the eval-gated extension the roadmap defers until numbers justify.\n *\n * @packageDocumentation\n */\n\n/** Default damping applied per hop (activation × damping^depth). */\nexport const DEFAULT_PPR_DAMPING = 0.5;\n\n/**\n * Compute PPR-lite activation for graded neighbours. `damping` in\n * `(0, 1]`; a neighbour at `depth` d contributes `damping^d` per seed\n * that reaches it. Since `expandActivation` already returns the MINIMUM\n * depth per neighbour (dedup across seeds), the score here is a pure\n * function of that min-depth - `damping^depth`, clamped to `[0, 1]`.\n * A neighbour list produced with `maxHops: 1` reproduces a flat-ish\n * score of `damping` at every node; `maxHops: 2` separates the tiers.\n *\n * @stable\n */\nexport function pprActivation(\n neighbours: ReadonlyArray<{ readonly depth: number }>,\n damping = DEFAULT_PPR_DAMPING,\n): number[] {\n const d = damping <= 0 || damping > 1 ? DEFAULT_PPR_DAMPING : damping;\n return neighbours.map((n) => d ** Math.max(1, n.depth));\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAkBA,MAAa,sBAAsB;;;;;;;;;;;;AAanC,SAAgB,cACd,YACA,UAAU,qBACA;CACV,MAAM,IAAI,WAAW,KAAK,UAAU,IAAI,sBAAsB;AAC9D,QAAO,WAAW,KAAK,MAAM,KAAK,KAAK,IAAI,GAAG,EAAE,MAAM,CAAC"}
|
package/dist/search/index.d.ts
CHANGED
|
@@ -2,5 +2,8 @@ import { DEFAULT_DIFFICULTY_THRESHOLD, DEFAULT_MAX_ITERATIONS, DifficultyAssessm
|
|
|
2
2
|
import { DEFAULT_MAX_QUERY_VARIANTS, HYDE_SYSTEM_PROMPT, QUERY_EXPANSION_SYSTEM_PROMPT, QueryTransformOptions, QueryTransformer, buildExpansionRequest, buildHydeRequest, createProviderQueryTransformer, parseHypothetical, parseQueryVariants } from "./query-transform.js";
|
|
3
3
|
import { ReRankOptions, ReRanker } from "./types.js";
|
|
4
4
|
import { RecallExplanation, RecalledMemoryExplanation, explainRecall, formatRecallExplanation } from "./explain.js";
|
|
5
|
+
import { FitFusionCase, FitFusionOptions, FitFusionResult, fitFusionWeights, ndcgAtK } from "./fit-weights.js";
|
|
6
|
+
import { DEFAULT_PPR_DAMPING, pprActivation } from "./graph-ppr.js";
|
|
5
7
|
import { RRFReranker, RRF_DEFAULT_K, WeightedRRFReranker, fuseRrf, fuseWeighted } from "./rrf.js";
|
|
6
|
-
|
|
8
|
+
import { isForeignProvenance, trustDiscount } from "./trust.js";
|
|
9
|
+
export { DEFAULT_DIFFICULTY_THRESHOLD, DEFAULT_MAX_ITERATIONS, DEFAULT_MAX_QUERY_VARIANTS, DEFAULT_PPR_DAMPING, type DifficultyAssessment, type DifficultyGateOptions, type FitFusionCase, type FitFusionOptions, type FitFusionResult, HYDE_SYSTEM_PROMPT, type IterativeRetrievalDeps, type IterativeRetrievalOptions, type IterativeRetrievalResult, MAX_ITERATIONS_CEILING, QUERY_EXPANSION_SYSTEM_PROMPT, type QueryTransformOptions, type QueryTransformer, RETRIEVAL_GRADE_SYSTEM_PROMPT, RRFReranker, RRF_DEFAULT_K, type ReRankOptions, type ReRanker, type RecallExplanation, type RecalledMemoryExplanation, type RetrievalGrade, type RetrievalGradeOptions, type RetrievalGrader, WeightedRRFReranker, assessQueryDifficulty, buildExpansionRequest, buildGradeRequest, buildHydeRequest, createProviderQueryTransformer, createProviderRetrievalGrader, explainRecall, fitFusionWeights, formatRecallExplanation, fuseRrf, fuseWeighted, isForeignProvenance, ndcgAtK, parseGrade, parseHypothetical, parseQueryVariants, pprActivation, runIterativeRetrieval, trustDiscount };
|
package/dist/search/index.js
CHANGED
|
@@ -2,5 +2,8 @@ import { DEFAULT_DIFFICULTY_THRESHOLD, DEFAULT_MAX_ITERATIONS, MAX_ITERATIONS_CE
|
|
|
2
2
|
import { DEFAULT_MAX_QUERY_VARIANTS, HYDE_SYSTEM_PROMPT, QUERY_EXPANSION_SYSTEM_PROMPT, buildExpansionRequest, buildHydeRequest, createProviderQueryTransformer, parseHypothetical, parseQueryVariants } from "./query-transform.js";
|
|
3
3
|
import { RRFReranker, RRF_DEFAULT_K, WeightedRRFReranker, fuseRrf, fuseWeighted } from "./rrf.js";
|
|
4
4
|
import { explainRecall, formatRecallExplanation } from "./explain.js";
|
|
5
|
+
import { DEFAULT_PPR_DAMPING, pprActivation } from "./graph-ppr.js";
|
|
6
|
+
import { isForeignProvenance, trustDiscount } from "./trust.js";
|
|
7
|
+
import { fitFusionWeights, ndcgAtK } from "./fit-weights.js";
|
|
5
8
|
|
|
6
|
-
export { DEFAULT_DIFFICULTY_THRESHOLD, DEFAULT_MAX_ITERATIONS, DEFAULT_MAX_QUERY_VARIANTS, HYDE_SYSTEM_PROMPT, MAX_ITERATIONS_CEILING, QUERY_EXPANSION_SYSTEM_PROMPT, RETRIEVAL_GRADE_SYSTEM_PROMPT, RRFReranker, RRF_DEFAULT_K, WeightedRRFReranker, assessQueryDifficulty, buildExpansionRequest, buildGradeRequest, buildHydeRequest, createProviderQueryTransformer, createProviderRetrievalGrader, explainRecall, formatRecallExplanation, fuseRrf, fuseWeighted, parseGrade, parseHypothetical, parseQueryVariants, runIterativeRetrieval };
|
|
9
|
+
export { DEFAULT_DIFFICULTY_THRESHOLD, DEFAULT_MAX_ITERATIONS, DEFAULT_MAX_QUERY_VARIANTS, DEFAULT_PPR_DAMPING, HYDE_SYSTEM_PROMPT, MAX_ITERATIONS_CEILING, QUERY_EXPANSION_SYSTEM_PROMPT, RETRIEVAL_GRADE_SYSTEM_PROMPT, RRFReranker, RRF_DEFAULT_K, WeightedRRFReranker, assessQueryDifficulty, buildExpansionRequest, buildGradeRequest, buildHydeRequest, createProviderQueryTransformer, createProviderRetrievalGrader, explainRecall, fitFusionWeights, formatRecallExplanation, fuseRrf, fuseWeighted, isForeignProvenance, ndcgAtK, parseGrade, parseHypothetical, parseQueryVariants, pprActivation, runIterativeRetrieval, trustDiscount };
|
|
@@ -6,7 +6,7 @@ import { Provider, ProviderRequest } from "@graphorin/core";
|
|
|
6
6
|
declare const DEFAULT_MAX_ITERATIONS = 3;
|
|
7
7
|
/**
|
|
8
8
|
* Absolute clamp on total retrieval passes, applied regardless of the
|
|
9
|
-
* caller's `maxIterations`
|
|
9
|
+
* caller's `maxIterations` - a latency guardrail so a mis-configured
|
|
10
10
|
* caller can never unbound the loop.
|
|
11
11
|
*/
|
|
12
12
|
declare const MAX_ITERATIONS_CEILING = 5;
|
|
@@ -37,7 +37,7 @@ interface DifficultyGateOptions {
|
|
|
37
37
|
}
|
|
38
38
|
/**
|
|
39
39
|
* Score a query's retrieval difficulty with cheap, deterministic, fully
|
|
40
|
-
* **local** heuristics (no I/O). Conservative by design
|
|
40
|
+
* **local** heuristics (no I/O). Conservative by design - it prefers to
|
|
41
41
|
* leave a query single-shot (`hard: false`) unless several
|
|
42
42
|
* multi-hop / temporal / comparison signals stack up, so the gate adds
|
|
43
43
|
* passes only where they are likely to help. Used by
|
|
@@ -74,7 +74,7 @@ interface RetrievalGradeOptions {
|
|
|
74
74
|
readonly signal?: AbortSignal;
|
|
75
75
|
/**
|
|
76
76
|
* Reformulations already attempted (MRET-11). Surfaced to the grader
|
|
77
|
-
* as context so it can propose something genuinely new
|
|
77
|
+
* as context so it can propose something genuinely new - the grade
|
|
78
78
|
* itself is ALWAYS judged against the original question, never a
|
|
79
79
|
* reformulation (a narrowed sub-query must not be declared
|
|
80
80
|
* "sufficient" while the original multi-hop question is not).
|
|
@@ -87,8 +87,8 @@ interface RetrievalGradeOptions {
|
|
|
87
87
|
* {@link createProviderRetrievalGrader}; advanced callers can supply a
|
|
88
88
|
* bespoke grader (e.g. a deterministic heuristic).
|
|
89
89
|
*
|
|
90
|
-
* Implementations MUST degrade gracefully
|
|
91
|
-
* than throw
|
|
90
|
+
* Implementations MUST degrade gracefully - return a "stop" grade rather
|
|
91
|
+
* than throw - so a grader failure never breaks recall.
|
|
92
92
|
*
|
|
93
93
|
* @stable
|
|
94
94
|
*/
|
|
@@ -103,7 +103,7 @@ interface RetrievalGrader {
|
|
|
103
103
|
*/
|
|
104
104
|
declare const RETRIEVAL_GRADE_SYSTEM_PROMPT: string;
|
|
105
105
|
/**
|
|
106
|
-
* Build the grade request. Pure
|
|
106
|
+
* Build the grade request. Pure - no I/O. Temperature 0 so the verdict
|
|
107
107
|
* is as stable as the model allows.
|
|
108
108
|
*
|
|
109
109
|
* @stable
|
|
@@ -154,16 +154,23 @@ interface IterativeRetrievalResult<H> {
|
|
|
154
154
|
readonly sufficient: boolean;
|
|
155
155
|
/**
|
|
156
156
|
* `true` when the loop exhausted its cap / ran out of reformulations
|
|
157
|
-
* while still judged insufficient
|
|
157
|
+
* while still judged insufficient - the caller should abstain rather
|
|
158
158
|
* than answer from `hits`.
|
|
159
159
|
*/
|
|
160
160
|
readonly abstained: boolean;
|
|
161
|
+
/**
|
|
162
|
+
* `true` when the grader actually judged the result at least once
|
|
163
|
+
* (memory-retrieval-02). `false` on the single-shot path (gate not
|
|
164
|
+
* hard, or no grader configured) - there `sufficient: true` is a
|
|
165
|
+
* DEFAULT, not a verdict, and consumers must not read it as one.
|
|
166
|
+
*/
|
|
167
|
+
readonly graded: boolean;
|
|
161
168
|
/** The sequence of queries tried (original first). */
|
|
162
169
|
readonly queries: ReadonlyArray<string>;
|
|
163
170
|
}
|
|
164
171
|
/**
|
|
165
172
|
* Dependencies injected into {@link runIterativeRetrieval}. The loop does
|
|
166
|
-
* no I/O of its own
|
|
173
|
+
* no I/O of its own - `retrieve` and `grader` own all side effects.
|
|
167
174
|
*
|
|
168
175
|
* @stable
|
|
169
176
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"iterative.d.ts","names":[],"sources":["../../src/search/iterative.ts"],"sourcesContent":[],"mappings":";;;;;
|
|
1
|
+
{"version":3,"file":"iterative.d.ts","names":[],"sources":["../../src/search/iterative.ts"],"sourcesContent":[],"mappings":";;;;;AA0XiE,cArVpD,sBAAA,GAqVoD,CAAA;;;;;;AAclD,cA5VF,sBAAA,GA4VE,CAAA;;AAAkC,cAzVpC,4BAAA,GAyVoC,GAAA;;AAQjD;AAwBA;;;;;AAIW,UA1WM,oBAAA,CA0WN;EAAR,SAAA,IAAA,EAAA,OAAA;EAAO;;;oBArWU;;;;;;;UAQH,qBAAA;;;;;;;;;;;;;;iBAqCD,qBAAA,0BAEL,wBACR;;;;;;UA8Cc,cAAA;;;;;;;;;;;;;;;;;;UAmBA,qBAAA;;oBAEG;;;;;;;;0BAQM;;;;;;;;;;;;;UAcT,eAAA;iCAGH,iCACA,wBACT,QAAQ;;;;;;;;cASA;;;;;;;iBAiBG,iBAAA,0BAEJ;;oBAGU;0BACM;IAEzB;;;;;;;;;;;;;;iBAkCa,UAAA,4BAAsC;;;;;;;;;iBA4BtC,6BAAA,WACJ;;IAET;;;;;;UA4Bc;;iBAEA,cAAc;;;;;;;;;;;;;;;;;;;;;oBAqBX;;;;;;;;UASH;;;;;;mDAMkC,cAAc,QAAQ,cAAc;;iBAEtE;;YAEL;;UAEF;;;;;;;;eAQK,cAAc,cAAc,MAAM,cAAc;;;;;;;UAQ9C,yBAAA;;;;wBAIO;;;;;;;oBAOJ;;;;;;;;;;;;iBAaE,8CAEd,uBAAuB,cACpB,4BACR,QAAQ,yBAAyB"}
|
package/dist/search/iterative.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
const DEFAULT_MAX_ITERATIONS = 3;
|
|
4
4
|
/**
|
|
5
5
|
* Absolute clamp on total retrieval passes, applied regardless of the
|
|
6
|
-
* caller's `maxIterations`
|
|
6
|
+
* caller's `maxIterations` - a latency guardrail so a mis-configured
|
|
7
7
|
* caller can never unbound the loop.
|
|
8
8
|
*/
|
|
9
9
|
const MAX_ITERATIONS_CEILING = 5;
|
|
@@ -32,7 +32,7 @@ const W_LONG = .15;
|
|
|
32
32
|
const LONG_WORD_COUNT = 12;
|
|
33
33
|
/**
|
|
34
34
|
* Score a query's retrieval difficulty with cheap, deterministic, fully
|
|
35
|
-
* **local** heuristics (no I/O). Conservative by design
|
|
35
|
+
* **local** heuristics (no I/O). Conservative by design - it prefers to
|
|
36
36
|
* leave a query single-shot (`hard: false`) unless several
|
|
37
37
|
* multi-hop / temporal / comparison signals stack up, so the gate adds
|
|
38
38
|
* passes only where they are likely to help. Used by
|
|
@@ -78,9 +78,9 @@ function assessQueryDifficulty(query, options = {}) {
|
|
|
78
78
|
*
|
|
79
79
|
* @internal
|
|
80
80
|
*/
|
|
81
|
-
const RETRIEVAL_GRADE_SYSTEM_PROMPT = "You are a retrieval grader for a personal memory store. Given a question and a numbered list of retrieved memories, judge whether they are SUFFICIENT to answer the question correctly and completely. Respond with ONLY a JSON object: {\"sufficient\": boolean, \"confidence\": number between 0 and 1, \"reformulation\": string or null, \"reason\": string}. Set \"sufficient\" to true only if the memories actually contain the answer
|
|
81
|
+
const RETRIEVAL_GRADE_SYSTEM_PROMPT = "You are a retrieval grader for a personal memory store. Given a question and a numbered list of retrieved memories, judge whether they are SUFFICIENT to answer the question correctly and completely. Respond with ONLY a JSON object: {\"sufficient\": boolean, \"confidence\": number between 0 and 1, \"reformulation\": string or null, \"reason\": string}. Set \"sufficient\" to true only if the memories actually contain the answer - do not guess. When insufficient, set \"reformulation\" to a single alternative search query likely to retrieve the missing information (vary entities, time frame, or specificity); otherwise null. Output only the JSON object - no prose, no markdown.";
|
|
82
82
|
/**
|
|
83
|
-
* Build the grade request. Pure
|
|
83
|
+
* Build the grade request. Pure - no I/O. Temperature 0 so the verdict
|
|
84
84
|
* is as stable as the model allows.
|
|
85
85
|
*
|
|
86
86
|
* @stable
|
|
@@ -188,7 +188,7 @@ async function runIterativeRetrieval(query, deps, options = {}) {
|
|
|
188
188
|
const tried = new Set([query.trim().toLowerCase()]);
|
|
189
189
|
accumulate(await deps.retrieve(query, false, signal));
|
|
190
190
|
const grader = deps.grader;
|
|
191
|
-
if (!gate.hard || grader === null) return finalize(deps, passLists, 1, gate.hard, true, false, queries, options.maxResults);
|
|
191
|
+
if (!gate.hard || grader === null) return finalize(deps, passLists, 1, gate.hard, true, false, false, queries, options.maxResults);
|
|
192
192
|
let passes = 1;
|
|
193
193
|
for (;;) {
|
|
194
194
|
const window = interleave(passLists, maxSnippets, deps.idOf).map(deps.snippetOf);
|
|
@@ -196,7 +196,7 @@ async function runIterativeRetrieval(query, deps, options = {}) {
|
|
|
196
196
|
...signal !== void 0 ? { signal } : {},
|
|
197
197
|
...queries.length > 1 ? { triedQueries: queries.slice(1) } : {}
|
|
198
198
|
});
|
|
199
|
-
if (grade.sufficient) return finalize(deps, passLists, passes, true, true, false, queries, options.maxResults);
|
|
199
|
+
if (grade.sufficient) return finalize(deps, passLists, passes, true, true, false, true, queries, options.maxResults);
|
|
200
200
|
if (passes >= cap) break;
|
|
201
201
|
const next = normalizeReformulation(grade.reformulation, tried);
|
|
202
202
|
if (next === null) break;
|
|
@@ -205,11 +205,11 @@ async function runIterativeRetrieval(query, deps, options = {}) {
|
|
|
205
205
|
tried.add(next.toLowerCase());
|
|
206
206
|
accumulate(await deps.retrieve(next, true, signal));
|
|
207
207
|
}
|
|
208
|
-
return finalize(deps, passLists, passes, true, false, true, queries, options.maxResults);
|
|
208
|
+
return finalize(deps, passLists, passes, true, false, true, true, queries, options.maxResults);
|
|
209
209
|
}
|
|
210
210
|
/**
|
|
211
211
|
* Round-robin interleave of per-pass lists, deduped by id: hit 1 of
|
|
212
|
-
* every pass, then hit 2 of every pass, …
|
|
212
|
+
* every pass, then hit 2 of every pass, … - guarantees the latest
|
|
213
213
|
* pass's top hits enter a window of any size.
|
|
214
214
|
*/
|
|
215
215
|
function interleave(lists, limit, idOf) {
|
|
@@ -227,7 +227,7 @@ function interleave(lists, limit, idOf) {
|
|
|
227
227
|
}
|
|
228
228
|
return out;
|
|
229
229
|
}
|
|
230
|
-
function finalize(deps, passLists, iterations, gateHard, sufficient, abstained, queries, maxResults) {
|
|
230
|
+
function finalize(deps, passLists, iterations, gateHard, sufficient, abstained, graded, queries, maxResults) {
|
|
231
231
|
const ranked = typeof deps.fuse === "function" ? dedupBy(deps.fuse(passLists), deps.idOf) : interleave(passLists, Number.POSITIVE_INFINITY, deps.idOf);
|
|
232
232
|
return {
|
|
233
233
|
hits: maxResults !== void 0 && maxResults >= 0 ? ranked.slice(0, maxResults) : ranked.slice(),
|
|
@@ -235,6 +235,7 @@ function finalize(deps, passLists, iterations, gateHard, sufficient, abstained,
|
|
|
235
235
|
gateHard,
|
|
236
236
|
sufficient,
|
|
237
237
|
abstained,
|
|
238
|
+
graded,
|
|
238
239
|
queries: [...queries]
|
|
239
240
|
};
|
|
240
241
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"iterative.js","names":["signals: string[]","stop: RetrievalGrade","passLists: Array<ReadonlyArray<H>>","queries: string[]","out: H[]"],"sources":["../../src/search/iterative.ts"],"sourcesContent":["/**\n * Agentic / iterative retrieval for hard queries (P2-4) — a **gated**,\n * CRAG/Self-RAG-style grade-then-reformulate loop.\n *\n * Some multi-hop / temporal questions can't be answered from a single\n * retrieval pass; the naive system returns a confident-but-wrong answer\n * instead of retrieving again. This module adds a small loop: grade the\n * retrieved memories for sufficiency and, when weak, reformulate the\n * query and retrieve once more (optionally widening to the P2-1 graph) —\n * up to a hard iteration cap, then **abstain** rather than confabulate.\n *\n * Both CRAG (arXiv:2401.15884) and Self-RAG (arXiv:2310.11511) add\n * latency/cost, so the literature's advice — and this module's design —\n * is to *gate* the loop, not make it default:\n *\n * 1. A cheap, **local** {@link assessQueryDifficulty} heuristic decides\n * whether a query is even a candidate; simple lookups never enter\n * the loop (no provider call) and stay single-shot.\n * 2. The grade/reformulate step needs a {@link RetrievalGrader}; with\n * none configured the loop degrades to a single retrieval pass. The\n * only provider-backed grader, {@link createProviderRetrievalGrader},\n * is **resilient**: any provider error or unparseable output degrades\n * to a \"stop\" grade rather than throwing or looping forever.\n * 3. A mandatory iteration cap bounds the worst case regardless of the\n * grader.\n *\n * The module is provider-agnostic — it imports only `@graphorin/core`\n * types and never performs I/O itself; {@link runIterativeRetrieval} is a\n * pure orchestrator over an injected `retrieve` function + grader, so the\n * loop is unit-testable without a store or a model.\n *\n * @packageDocumentation\n */\n\nimport type { Provider, ProviderRequest } from '@graphorin/core';\n\n/** Default hard ceiling on total retrieval passes (gate + cap mandatory). */\nexport const DEFAULT_MAX_ITERATIONS = 3;\n\n/**\n * Absolute clamp on total retrieval passes, applied regardless of the\n * caller's `maxIterations` — a latency guardrail so a mis-configured\n * caller can never unbound the loop.\n */\nexport const MAX_ITERATIONS_CEILING = 5;\n\n/** Default score (0..1) at or above which a query is classified as hard. */\nexport const DEFAULT_DIFFICULTY_THRESHOLD = 0.5;\n\n/** Default output-token ceiling for a single grade call. */\nconst DEFAULT_GRADE_MAX_TOKENS = 256;\n\n/** Default number of retrieved snippets shown to the grader per pass. */\nconst DEFAULT_MAX_GRADE_SNIPPETS = 8;\n\n// ---------------------------------------------------------------------------\n// Difficulty gate (pure, local — no I/O, no provider)\n// ---------------------------------------------------------------------------\n\n/**\n * Outcome of the heuristic difficulty gate. `hard` is the gating\n * decision (`score >= threshold`); `signals` lists which heuristic\n * categories fired, for explainability / tracing.\n *\n * @stable\n */\nexport interface DifficultyAssessment {\n readonly hard: boolean;\n /** Aggregate difficulty score, clamped to `[0, 1]`. */\n readonly score: number;\n /** Heuristic categories that fired (`'multi-hop'`, `'temporal'`, …). */\n readonly signals: ReadonlyArray<string>;\n}\n\n/**\n * Tuning for {@link assessQueryDifficulty}.\n *\n * @stable\n */\nexport interface DifficultyGateOptions {\n /** Minimum score (0..1) to classify the query as hard. Default 0.5. */\n readonly threshold?: number;\n}\n\n/** Relational / referential markers implying a hop to another entity. */\nconst RE_REL_PRONOUN = /\\b(?:who|whom|whose|which)\\b/iu;\nconst RE_OF_CHAIN = /\\bthe\\s+\\w+\\s+of\\b/iu;\nconst RE_POSSESSIVE = /\\b[\\w-]+'s\\b/giu;\nconst RE_DESC_REF =\n /\\bthe\\s+(?:person|people|place|thing|one|company|team|book|restaurant|project|guy|woman|man)\\b/iu;\n/** Temporal ordering / relative-time markers. */\nconst RE_TEMPORAL =\n /\\b(?:before|after|when|while|during|since|until|first|last|earliest|latest|previously|originally|ago|prior|then|recently|used\\s+to)\\b/iu;\n/** Comparison / superlative markers. */\nconst RE_COMPARISON =\n /\\b(?:most|least|more|less|fewer|greater|biggest|largest|smallest|oldest|newest|best|worst|highest|lowest|compared?|versus|vs\\.?|between)\\b/iu;\n/** Clause-joining conjunctions used to count clauses. */\nconst RE_CONJUNCTION = /\\b(?:and|or|but|then)\\b/giu;\n\nconst W_MULTI_HOP = 0.4;\nconst W_TEMPORAL = 0.3;\nconst W_COMPARISON = 0.3;\nconst W_MULTI_CLAUSE = 0.2;\nconst W_LONG = 0.15;\nconst LONG_WORD_COUNT = 12;\n\n/**\n * Score a query's retrieval difficulty with cheap, deterministic, fully\n * **local** heuristics (no I/O). Conservative by design — it prefers to\n * leave a query single-shot (`hard: false`) unless several\n * multi-hop / temporal / comparison signals stack up, so the gate adds\n * passes only where they are likely to help. Used by\n * {@link runIterativeRetrieval} to decide whether to enter the loop.\n *\n * @stable\n */\nexport function assessQueryDifficulty(\n query: string,\n options: DifficultyGateOptions = {},\n): DifficultyAssessment {\n const threshold = options.threshold ?? DEFAULT_DIFFICULTY_THRESHOLD;\n const signals: string[] = [];\n let score = 0;\n\n const possessiveCount = (query.match(RE_POSSESSIVE) ?? []).length;\n const multiHop =\n RE_REL_PRONOUN.test(query) ||\n RE_OF_CHAIN.test(query) ||\n RE_DESC_REF.test(query) ||\n possessiveCount >= 2;\n if (multiHop) {\n signals.push('multi-hop');\n score += W_MULTI_HOP;\n }\n if (RE_TEMPORAL.test(query)) {\n signals.push('temporal');\n score += W_TEMPORAL;\n }\n if (RE_COMPARISON.test(query)) {\n signals.push('comparison');\n score += W_COMPARISON;\n }\n const clauseSignals = (query.match(RE_CONJUNCTION) ?? []).length + countCommas(query);\n if (clauseSignals >= 2 || questionMarkCount(query) >= 2) {\n signals.push('multi-clause');\n score += W_MULTI_CLAUSE;\n }\n if (wordCount(query) >= LONG_WORD_COUNT) {\n signals.push('long');\n score += W_LONG;\n }\n\n const clamped = Math.min(1, score);\n return { hard: clamped >= threshold, score: clamped, signals };\n}\n\n// ---------------------------------------------------------------------------\n// Retrieval grader seam (provider-agnostic)\n// ---------------------------------------------------------------------------\n\n/**\n * Verdict from grading a retrieved candidate set against a query.\n *\n * @stable\n */\nexport interface RetrievalGrade {\n /** Whether the retrieved memories suffice to answer the query. */\n readonly sufficient: boolean;\n /** Grader confidence, clamped to `[0, 1]`. */\n readonly confidence: number;\n /**\n * A single better search query to try next, or `null` when the grader\n * proposes none (the loop then stops / abstains).\n */\n readonly reformulation: string | null;\n /** Optional short rationale (never surfaced in spans). */\n readonly reason?: string;\n}\n\n/**\n * Per-call options for a {@link RetrievalGrader}.\n *\n * @stable\n */\nexport interface RetrievalGradeOptions {\n /** Cancellation signal forwarded to the underlying provider call. */\n readonly signal?: AbortSignal;\n /**\n * Reformulations already attempted (MRET-11). Surfaced to the grader\n * as context so it can propose something genuinely new — the grade\n * itself is ALWAYS judged against the original question, never a\n * reformulation (a narrowed sub-query must not be declared\n * \"sufficient\" while the original multi-hop question is not).\n */\n readonly triedQueries?: ReadonlyArray<string>;\n}\n\n/**\n * Pluggable retrieval-grader seam consumed by the iterative loop. The\n * built-in provider-backed implementation lives in\n * {@link createProviderRetrievalGrader}; advanced callers can supply a\n * bespoke grader (e.g. a deterministic heuristic).\n *\n * Implementations MUST degrade gracefully — return a \"stop\" grade rather\n * than throw — so a grader failure never breaks recall.\n *\n * @stable\n */\nexport interface RetrievalGrader {\n grade(\n query: string,\n snippets: ReadonlyArray<string>,\n options?: RetrievalGradeOptions,\n ): Promise<RetrievalGrade>;\n}\n\n/**\n * System prompt for the retrieval grader. Asks for a bare JSON verdict;\n * {@link parseGrade} also tolerates a chatty model.\n *\n * @internal\n */\nexport const RETRIEVAL_GRADE_SYSTEM_PROMPT =\n 'You are a retrieval grader for a personal memory store. Given a question and a ' +\n 'numbered list of retrieved memories, judge whether they are SUFFICIENT to answer ' +\n 'the question correctly and completely. Respond with ONLY a JSON object: ' +\n '{\"sufficient\": boolean, \"confidence\": number between 0 and 1, \"reformulation\": ' +\n 'string or null, \"reason\": string}. Set \"sufficient\" to true only if the memories ' +\n 'actually contain the answer — do not guess. When insufficient, set \"reformulation\" ' +\n 'to a single alternative search query likely to retrieve the missing information ' +\n '(vary entities, time frame, or specificity); otherwise null. Output only the JSON ' +\n 'object — no prose, no markdown.';\n\n/**\n * Build the grade request. Pure — no I/O. Temperature 0 so the verdict\n * is as stable as the model allows.\n *\n * @stable\n */\nexport function buildGradeRequest(\n query: string,\n snippets: ReadonlyArray<string>,\n options: {\n readonly maxTokens?: number;\n readonly signal?: AbortSignal;\n readonly triedQueries?: ReadonlyArray<string>;\n } = {},\n): ProviderRequest {\n const body =\n snippets.length === 0 ? '(none)' : snippets.map((s, i) => `[${i + 1}] ${s}`).join('\\n');\n const tried =\n options.triedQueries !== undefined && options.triedQueries.length > 0\n ? `\\n\\nQueries already tried (do not repeat them): ${options.triedQueries.join(' | ')}`\n : '';\n const content =\n `Question: ${query}\\n\\nRetrieved memories:\\n${body}${tried}\\n\\n` +\n 'Grade whether these memories are sufficient to answer the question. If not, ' +\n 'propose a single better search query.';\n return {\n messages: [{ role: 'user', content }],\n systemMessage: RETRIEVAL_GRADE_SYSTEM_PROMPT,\n temperature: 0,\n maxTokens: options.maxTokens ?? DEFAULT_GRADE_MAX_TOKENS,\n ...(options.signal !== undefined ? { signal: options.signal } : {}),\n outputType: { kind: 'structured' },\n };\n}\n\n/**\n * Parse a grader model output into a {@link RetrievalGrade}. Tolerates a\n * bare JSON object, a fenced block, and chatty text around the object.\n *\n * **Fail-safe = stop.** When the verdict can't be read (undefined / empty\n * / unparseable / missing `sufficient`), returns\n * `{ sufficient: true, confidence: 0, reformulation: null }` so a broken\n * grader degrades to a single pass rather than looping or falsely\n * abstaining. A *parsed* `sufficient: false` (a real insufficiency\n * verdict) is preserved.\n *\n * @stable\n */\nexport function parseGrade(text: string | undefined): RetrievalGrade {\n const stop: RetrievalGrade = { sufficient: true, confidence: 0, reformulation: null };\n if (text === undefined) return stop;\n const stripped = stripFence(text).trim();\n if (stripped.length === 0) return stop;\n const obj = tryParseObject(stripped);\n if (obj === null) return stop;\n const sufficient = coerceBool(obj.sufficient);\n if (sufficient === null) return stop;\n const confidence = coerceConfidence(obj.confidence, sufficient);\n const reformulation = sufficient ? null : coerceReformulation(obj.reformulation);\n const reason = coerceReason(obj.reason);\n return {\n sufficient,\n confidence,\n reformulation,\n ...(reason !== undefined ? { reason } : {}),\n };\n}\n\n/**\n * Wrap a {@link Provider} as a {@link RetrievalGrader}. **Resilient**: a\n * provider error or unparseable response degrades to the\n * {@link parseGrade} \"stop\" grade so grading never throws into the loop\n * (and a flaky grader can't drive endless reformulation).\n *\n * @stable\n */\nexport function createProviderRetrievalGrader(\n provider: Provider,\n options: { readonly maxTokens?: number } = {},\n): RetrievalGrader {\n const maxTokens = options.maxTokens ?? DEFAULT_GRADE_MAX_TOKENS;\n return {\n async grade(query, snippets, opts = {}): Promise<RetrievalGrade> {\n try {\n const request = buildGradeRequest(query, snippets, {\n maxTokens,\n ...(opts.signal !== undefined ? { signal: opts.signal } : {}),\n ...(opts.triedQueries !== undefined ? { triedQueries: opts.triedQueries } : {}),\n });\n const response = await provider.generate(request);\n return parseGrade(response.text);\n } catch {\n return { sufficient: true, confidence: 0, reformulation: null };\n }\n },\n };\n}\n\n// ---------------------------------------------------------------------------\n// The bounded loop (pure orchestration over injected retrieve + grader)\n// ---------------------------------------------------------------------------\n\n/**\n * Result of an iterative retrieval run.\n *\n * @stable\n */\nexport interface IterativeRetrievalResult<H> {\n /** Accumulated hits across all passes (deduped, in discovery order). */\n readonly hits: ReadonlyArray<H>;\n /** Number of retrieval passes performed (≥ 1). */\n readonly iterations: number;\n /** Difficulty-gate verdict (whether the loop was eligible to run). */\n readonly gateHard: boolean;\n /** Final sufficiency verdict. */\n readonly sufficient: boolean;\n /**\n * `true` when the loop exhausted its cap / ran out of reformulations\n * while still judged insufficient — the caller should abstain rather\n * than answer from `hits`.\n */\n readonly abstained: boolean;\n /** The sequence of queries tried (original first). */\n readonly queries: ReadonlyArray<string>;\n}\n\n/**\n * Dependencies injected into {@link runIterativeRetrieval}. The loop does\n * no I/O of its own — `retrieve` and `grader` own all side effects.\n *\n * @stable\n */\nexport interface IterativeRetrievalDeps<H> {\n /**\n * Run one retrieval pass for `query`. `widen` is `true` on\n * reformulation passes so the caller can broaden recall (e.g. enable\n * P2-1 one-hop graph expansion).\n */\n retrieve(query: string, widen: boolean, signal?: AbortSignal): Promise<ReadonlyArray<H>>;\n /** Snippet shown to the grader for a hit. */\n snippetOf(hit: H): string;\n /** Stable id used to dedup hits across passes. */\n idOf(hit: H): string;\n /** Grader; `null` ⇒ single-shot (no grading, no provider call). */\n grader: RetrievalGrader | null;\n /**\n * Re-fuse the per-pass hit lists into one ranked list (MRET-2).\n * Receives one list per pass in pass order; the result feeds the\n * final `maxResults` cut so a pass-2 find can outrank pass-1 noise.\n * Absent ⇒ the loop falls back to round-robin interleaving (still\n * strictly better than the old discovery-order cut).\n */\n fuse?(lists: ReadonlyArray<ReadonlyArray<H>>): ReadonlyArray<H>;\n}\n\n/**\n * Options for {@link runIterativeRetrieval}.\n *\n * @stable\n */\nexport interface IterativeRetrievalOptions {\n /** Total-pass cap; clamped to `[1, {@link MAX_ITERATIONS_CEILING}]`. */\n readonly maxIterations?: number;\n /** Difficulty-gate tuning. */\n readonly difficulty?: DifficultyGateOptions;\n /** Skip the heuristic gate and force the loop (still capped). */\n readonly forceHard?: boolean;\n /** Max snippets passed to the grader per pass. Default 8. */\n readonly maxGradeSnippets?: number;\n /** Cap on the returned hit count (omitted ⇒ all accumulated). */\n readonly maxResults?: number;\n readonly signal?: AbortSignal;\n}\n\n/**\n * Run the gated grade-then-reformulate loop.\n *\n * Flow: assess difficulty → retrieve (pass 1) → if not hard *or* no\n * grader, return single-shot → else grade; if sufficient, return; if\n * weak and a reformulation is offered and the cap is not hit, retrieve\n * again (widened) and re-grade; otherwise **abstain**.\n *\n * @stable\n */\nexport async function runIterativeRetrieval<H>(\n query: string,\n deps: IterativeRetrievalDeps<H>,\n options: IterativeRetrievalOptions = {},\n): Promise<IterativeRetrievalResult<H>> {\n const gate =\n options.forceHard === true\n ? { hard: true, score: 1, signals: ['forced'] as ReadonlyArray<string> }\n : assessQueryDifficulty(query, options.difficulty ?? {});\n const cap = clampIterations(options.maxIterations);\n const maxSnippets = Math.max(1, options.maxGradeSnippets ?? DEFAULT_MAX_GRADE_SNIPPETS);\n const signal = options.signal;\n\n // MRET-2: hits are tracked PER PASS — RAW, including cross-pass\n // repeats, so rank fusion can reward consensus between passes. The\n // grade window and the final list dedup by id at read time. The old\n // flat append meant pass 1 saturated the first `maxSnippets` forever\n // and the final cut ran in discovery order.\n const passLists: Array<ReadonlyArray<H>> = [];\n const accumulate = (hits: ReadonlyArray<H>): void => {\n passLists.push([...hits]);\n };\n const queries: string[] = [query];\n const tried = new Set<string>([query.trim().toLowerCase()]);\n\n // Pass 1 — always single-shot, no widening.\n accumulate(await deps.retrieve(query, false, signal));\n\n // Gate: the loop is eligible only when the query is judged hard *and* a\n // grader exists. Otherwise stay single-shot — we did not grade, so we\n // make no abstention claim.\n const grader = deps.grader;\n if (!gate.hard || grader === null) {\n return finalize(deps, passLists, 1, gate.hard, true, false, queries, options.maxResults);\n }\n\n let passes = 1;\n for (;;) {\n // MRET-2: the grade window interleaves the TOP hits of every pass\n // (latest pass included) instead of replaying pass 1's head.\n // MRET-11: sufficiency is always judged against the ORIGINAL\n // question; reformulations ride along as already-tried context.\n const window = interleave(passLists, maxSnippets, deps.idOf).map(deps.snippetOf);\n const grade = await grader.grade(query, window, {\n ...(signal !== undefined ? { signal } : {}),\n ...(queries.length > 1 ? { triedQueries: queries.slice(1) } : {}),\n });\n if (grade.sufficient) {\n return finalize(deps, passLists, passes, true, true, false, queries, options.maxResults);\n }\n if (passes >= cap) break;\n const next = normalizeReformulation(grade.reformulation, tried);\n if (next === null) break;\n passes += 1;\n queries.push(next);\n tried.add(next.toLowerCase());\n accumulate(await deps.retrieve(next, true, signal));\n }\n // Cap reached or no further reformulation while still insufficient ⇒ abstain.\n return finalize(deps, passLists, passes, true, false, true, queries, options.maxResults);\n}\n\n/**\n * Round-robin interleave of per-pass lists, deduped by id: hit 1 of\n * every pass, then hit 2 of every pass, … — guarantees the latest\n * pass's top hits enter a window of any size.\n */\nfunction interleave<H>(\n lists: ReadonlyArray<ReadonlyArray<H>>,\n limit: number,\n idOf: (hit: H) => string,\n): H[] {\n const out: H[] = [];\n const seen = new Set<string>();\n const longest = lists.reduce((max, l) => Math.max(max, l.length), 0);\n for (let rank = 0; rank < longest && out.length < limit; rank += 1) {\n for (const list of lists) {\n const hit = list[rank];\n if (hit === undefined) continue;\n const id = idOf(hit);\n if (seen.has(id)) continue;\n seen.add(id);\n out.push(hit);\n if (out.length >= limit) break;\n }\n }\n return out;\n}\n\nfunction finalize<H>(\n deps: IterativeRetrievalDeps<H>,\n passLists: ReadonlyArray<ReadonlyArray<H>>,\n iterations: number,\n gateHard: boolean,\n sufficient: boolean,\n abstained: boolean,\n queries: ReadonlyArray<string>,\n maxResults: number | undefined,\n): IterativeRetrievalResult<H> {\n // MRET-2: re-rank ACROSS passes before the cut — discovery order let\n // pass-1 noise crowd out the pass-2 hit that actually answers.\n const ranked =\n typeof deps.fuse === 'function'\n ? dedupBy(deps.fuse(passLists), deps.idOf)\n : interleave(passLists, Number.POSITIVE_INFINITY, deps.idOf);\n const capped =\n maxResults !== undefined && maxResults >= 0 ? ranked.slice(0, maxResults) : ranked.slice();\n return { hits: capped, iterations, gateHard, sufficient, abstained, queries: [...queries] };\n}\n\nfunction dedupBy<H>(hits: ReadonlyArray<H>, idOf: (hit: H) => string): H[] {\n const seen = new Set<string>();\n const out: H[] = [];\n for (const hit of hits) {\n const id = idOf(hit);\n if (seen.has(id)) continue;\n seen.add(id);\n out.push(hit);\n }\n return out;\n}\n\nfunction clampIterations(value: number | undefined): number {\n const n = value ?? DEFAULT_MAX_ITERATIONS;\n if (!Number.isFinite(n)) return DEFAULT_MAX_ITERATIONS;\n return Math.min(MAX_ITERATIONS_CEILING, Math.max(1, Math.floor(n)));\n}\n\n/**\n * Normalise a proposed reformulation: trim, drop empties, and reject a\n * query already tried (case-insensitive) so the loop never re-treads and\n * cannot spin.\n */\nfunction normalizeReformulation(raw: string | null, tried: ReadonlySet<string>): string | null {\n if (raw === null) return null;\n const trimmed = raw.trim();\n if (trimmed.length === 0) return null;\n if (tried.has(trimmed.toLowerCase())) return null;\n return trimmed;\n}\n\n// ---------------------------------------------------------------------------\n// Helpers\n// ---------------------------------------------------------------------------\n\nfunction wordCount(text: string): number {\n const trimmed = text.trim();\n return trimmed.length === 0 ? 0 : trimmed.split(/\\s+/u).length;\n}\n\nfunction countCommas(text: string): number {\n return (text.match(/,/gu) ?? []).length;\n}\n\nfunction questionMarkCount(text: string): number {\n return (text.match(/\\?/gu) ?? []).length;\n}\n\nfunction coerceBool(value: unknown): boolean | null {\n if (typeof value === 'boolean') return value;\n if (typeof value === 'string') {\n const v = value.trim().toLowerCase();\n if (v === 'true' || v === 'yes' || v === 'sufficient') return true;\n if (v === 'false' || v === 'no' || v === 'insufficient') return false;\n }\n return null;\n}\n\nfunction coerceConfidence(value: unknown, sufficient: boolean): number {\n if (typeof value === 'number' && Number.isFinite(value)) {\n return Math.min(1, Math.max(0, value));\n }\n return sufficient ? 0.8 : 0.3;\n}\n\nfunction coerceReformulation(value: unknown): string | null {\n if (typeof value !== 'string') return null;\n const trimmed = value.trim();\n if (trimmed.length === 0) return null;\n return trimmed.length > 1024 ? trimmed.slice(0, 1024) : trimmed;\n}\n\nfunction coerceReason(value: unknown): string | undefined {\n if (typeof value !== 'string') return undefined;\n const trimmed = value.trim();\n return trimmed.length === 0 ? undefined : trimmed;\n}\n\nfunction tryParseObject(text: string): Record<string, unknown> | null {\n const direct = tryJson(text);\n if (isRecord(direct)) return direct;\n const start = text.indexOf('{');\n const end = text.lastIndexOf('}');\n if (start >= 0 && end > start) {\n const sliced = tryJson(text.slice(start, end + 1));\n if (isRecord(sliced)) return sliced;\n }\n return null;\n}\n\nfunction tryJson(text: string): unknown {\n try {\n return JSON.parse(text);\n } catch {\n return undefined;\n }\n}\n\nfunction stripFence(text: string): string {\n const match = /^```[^\\n]*\\n([\\s\\S]*?)\\n```/u.exec(text.trim());\n return match?.[1] ?? text;\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n return value !== null && typeof value === 'object' && !Array.isArray(value);\n}\n"],"mappings":";;AAqCA,MAAa,yBAAyB;;;;;;AAOtC,MAAa,yBAAyB;;AAGtC,MAAa,+BAA+B;;AAG5C,MAAM,2BAA2B;;AAGjC,MAAM,6BAA6B;;AAgCnC,MAAM,iBAAiB;AACvB,MAAM,cAAc;AACpB,MAAM,gBAAgB;AACtB,MAAM,cACJ;;AAEF,MAAM,cACJ;;AAEF,MAAM,gBACJ;;AAEF,MAAM,iBAAiB;AAEvB,MAAM,cAAc;AACpB,MAAM,aAAa;AACnB,MAAM,eAAe;AACrB,MAAM,iBAAiB;AACvB,MAAM,SAAS;AACf,MAAM,kBAAkB;;;;;;;;;;;AAYxB,SAAgB,sBACd,OACA,UAAiC,EAAE,EACb;CACtB,MAAM,YAAY,QAAQ,aAAa;CACvC,MAAMA,UAAoB,EAAE;CAC5B,IAAI,QAAQ;CAEZ,MAAM,mBAAmB,MAAM,MAAM,cAAc,IAAI,EAAE,EAAE;AAM3D,KAJE,eAAe,KAAK,MAAM,IAC1B,YAAY,KAAK,MAAM,IACvB,YAAY,KAAK,MAAM,IACvB,mBAAmB,GACP;AACZ,UAAQ,KAAK,YAAY;AACzB,WAAS;;AAEX,KAAI,YAAY,KAAK,MAAM,EAAE;AAC3B,UAAQ,KAAK,WAAW;AACxB,WAAS;;AAEX,KAAI,cAAc,KAAK,MAAM,EAAE;AAC7B,UAAQ,KAAK,aAAa;AAC1B,WAAS;;AAGX,MADuB,MAAM,MAAM,eAAe,IAAI,EAAE,EAAE,SAAS,YAAY,MAAM,IAChE,KAAK,kBAAkB,MAAM,IAAI,GAAG;AACvD,UAAQ,KAAK,eAAe;AAC5B,WAAS;;AAEX,KAAI,UAAU,MAAM,IAAI,iBAAiB;AACvC,UAAQ,KAAK,OAAO;AACpB,WAAS;;CAGX,MAAM,UAAU,KAAK,IAAI,GAAG,MAAM;AAClC,QAAO;EAAE,MAAM,WAAW;EAAW,OAAO;EAAS;EAAS;;;;;;;;AAqEhE,MAAa,gCACX;;;;;;;AAgBF,SAAgB,kBACd,OACA,UACA,UAII,EAAE,EACW;AAWjB,QAAO;EACL,UAAU,CAAC;GAAE,MAAM;GAAQ,SAJ3B,aAAa,MAAM,2BANnB,SAAS,WAAW,IAAI,WAAW,SAAS,KAAK,GAAG,MAAM,IAAI,IAAI,EAAE,IAAI,IAAI,CAAC,KAAK,KAAK,GAEvF,QAAQ,iBAAiB,UAAa,QAAQ,aAAa,SAAS,IAChE,mDAAmD,QAAQ,aAAa,KAAK,MAAM,KACnF,GAEuD;GAIvB,CAAC;EACrC,eAAe;EACf,aAAa;EACb,WAAW,QAAQ,aAAa;EAChC,GAAI,QAAQ,WAAW,SAAY,EAAE,QAAQ,QAAQ,QAAQ,GAAG,EAAE;EAClE,YAAY,EAAE,MAAM,cAAc;EACnC;;;;;;;;;;;;;;;AAgBH,SAAgB,WAAW,MAA0C;CACnE,MAAMC,OAAuB;EAAE,YAAY;EAAM,YAAY;EAAG,eAAe;EAAM;AACrF,KAAI,SAAS,OAAW,QAAO;CAC/B,MAAM,WAAW,WAAW,KAAK,CAAC,MAAM;AACxC,KAAI,SAAS,WAAW,EAAG,QAAO;CAClC,MAAM,MAAM,eAAe,SAAS;AACpC,KAAI,QAAQ,KAAM,QAAO;CACzB,MAAM,aAAa,WAAW,IAAI,WAAW;AAC7C,KAAI,eAAe,KAAM,QAAO;CAChC,MAAM,aAAa,iBAAiB,IAAI,YAAY,WAAW;CAC/D,MAAM,gBAAgB,aAAa,OAAO,oBAAoB,IAAI,cAAc;CAChF,MAAM,SAAS,aAAa,IAAI,OAAO;AACvC,QAAO;EACL;EACA;EACA;EACA,GAAI,WAAW,SAAY,EAAE,QAAQ,GAAG,EAAE;EAC3C;;;;;;;;;;AAWH,SAAgB,8BACd,UACA,UAA2C,EAAE,EAC5B;CACjB,MAAM,YAAY,QAAQ,aAAa;AACvC,QAAO,EACL,MAAM,MAAM,OAAO,UAAU,OAAO,EAAE,EAA2B;AAC/D,MAAI;GACF,MAAM,UAAU,kBAAkB,OAAO,UAAU;IACjD;IACA,GAAI,KAAK,WAAW,SAAY,EAAE,QAAQ,KAAK,QAAQ,GAAG,EAAE;IAC5D,GAAI,KAAK,iBAAiB,SAAY,EAAE,cAAc,KAAK,cAAc,GAAG,EAAE;IAC/E,CAAC;AAEF,UAAO,YADU,MAAM,SAAS,SAAS,QAAQ,EACtB,KAAK;UAC1B;AACN,UAAO;IAAE,YAAY;IAAM,YAAY;IAAG,eAAe;IAAM;;IAGpE;;;;;;;;;;;;AAyFH,eAAsB,sBACpB,OACA,MACA,UAAqC,EAAE,EACD;CACtC,MAAM,OACJ,QAAQ,cAAc,OAClB;EAAE,MAAM;EAAM,OAAO;EAAG,SAAS,CAAC,SAAS;EAA2B,GACtE,sBAAsB,OAAO,QAAQ,cAAc,EAAE,CAAC;CAC5D,MAAM,MAAM,gBAAgB,QAAQ,cAAc;CAClD,MAAM,cAAc,KAAK,IAAI,GAAG,QAAQ,oBAAoB,2BAA2B;CACvF,MAAM,SAAS,QAAQ;CAOvB,MAAMC,YAAqC,EAAE;CAC7C,MAAM,cAAc,SAAiC;AACnD,YAAU,KAAK,CAAC,GAAG,KAAK,CAAC;;CAE3B,MAAMC,UAAoB,CAAC,MAAM;CACjC,MAAM,QAAQ,IAAI,IAAY,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;AAG3D,YAAW,MAAM,KAAK,SAAS,OAAO,OAAO,OAAO,CAAC;CAKrD,MAAM,SAAS,KAAK;AACpB,KAAI,CAAC,KAAK,QAAQ,WAAW,KAC3B,QAAO,SAAS,MAAM,WAAW,GAAG,KAAK,MAAM,MAAM,OAAO,SAAS,QAAQ,WAAW;CAG1F,IAAI,SAAS;AACb,UAAS;EAKP,MAAM,SAAS,WAAW,WAAW,aAAa,KAAK,KAAK,CAAC,IAAI,KAAK,UAAU;EAChF,MAAM,QAAQ,MAAM,OAAO,MAAM,OAAO,QAAQ;GAC9C,GAAI,WAAW,SAAY,EAAE,QAAQ,GAAG,EAAE;GAC1C,GAAI,QAAQ,SAAS,IAAI,EAAE,cAAc,QAAQ,MAAM,EAAE,EAAE,GAAG,EAAE;GACjE,CAAC;AACF,MAAI,MAAM,WACR,QAAO,SAAS,MAAM,WAAW,QAAQ,MAAM,MAAM,OAAO,SAAS,QAAQ,WAAW;AAE1F,MAAI,UAAU,IAAK;EACnB,MAAM,OAAO,uBAAuB,MAAM,eAAe,MAAM;AAC/D,MAAI,SAAS,KAAM;AACnB,YAAU;AACV,UAAQ,KAAK,KAAK;AAClB,QAAM,IAAI,KAAK,aAAa,CAAC;AAC7B,aAAW,MAAM,KAAK,SAAS,MAAM,MAAM,OAAO,CAAC;;AAGrD,QAAO,SAAS,MAAM,WAAW,QAAQ,MAAM,OAAO,MAAM,SAAS,QAAQ,WAAW;;;;;;;AAQ1F,SAAS,WACP,OACA,OACA,MACK;CACL,MAAMC,MAAW,EAAE;CACnB,MAAM,uBAAO,IAAI,KAAa;CAC9B,MAAM,UAAU,MAAM,QAAQ,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,OAAO,EAAE,EAAE;AACpE,MAAK,IAAI,OAAO,GAAG,OAAO,WAAW,IAAI,SAAS,OAAO,QAAQ,EAC/D,MAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,MAAM,KAAK;AACjB,MAAI,QAAQ,OAAW;EACvB,MAAM,KAAK,KAAK,IAAI;AACpB,MAAI,KAAK,IAAI,GAAG,CAAE;AAClB,OAAK,IAAI,GAAG;AACZ,MAAI,KAAK,IAAI;AACb,MAAI,IAAI,UAAU,MAAO;;AAG7B,QAAO;;AAGT,SAAS,SACP,MACA,WACA,YACA,UACA,YACA,WACA,SACA,YAC6B;CAG7B,MAAM,SACJ,OAAO,KAAK,SAAS,aACjB,QAAQ,KAAK,KAAK,UAAU,EAAE,KAAK,KAAK,GACxC,WAAW,WAAW,OAAO,mBAAmB,KAAK,KAAK;AAGhE,QAAO;EAAE,MADP,eAAe,UAAa,cAAc,IAAI,OAAO,MAAM,GAAG,WAAW,GAAG,OAAO,OAAO;EACrE;EAAY;EAAU;EAAY;EAAW,SAAS,CAAC,GAAG,QAAQ;EAAE;;AAG7F,SAAS,QAAW,MAAwB,MAA+B;CACzE,MAAM,uBAAO,IAAI,KAAa;CAC9B,MAAMA,MAAW,EAAE;AACnB,MAAK,MAAM,OAAO,MAAM;EACtB,MAAM,KAAK,KAAK,IAAI;AACpB,MAAI,KAAK,IAAI,GAAG,CAAE;AAClB,OAAK,IAAI,GAAG;AACZ,MAAI,KAAK,IAAI;;AAEf,QAAO;;AAGT,SAAS,gBAAgB,OAAmC;CAC1D,MAAM,IAAI,SAAS;AACnB,KAAI,CAAC,OAAO,SAAS,EAAE,CAAE,QAAO;AAChC,QAAO,KAAK,IAAI,wBAAwB,KAAK,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC,CAAC;;;;;;;AAQrE,SAAS,uBAAuB,KAAoB,OAA2C;AAC7F,KAAI,QAAQ,KAAM,QAAO;CACzB,MAAM,UAAU,IAAI,MAAM;AAC1B,KAAI,QAAQ,WAAW,EAAG,QAAO;AACjC,KAAI,MAAM,IAAI,QAAQ,aAAa,CAAC,CAAE,QAAO;AAC7C,QAAO;;AAOT,SAAS,UAAU,MAAsB;CACvC,MAAM,UAAU,KAAK,MAAM;AAC3B,QAAO,QAAQ,WAAW,IAAI,IAAI,QAAQ,MAAM,OAAO,CAAC;;AAG1D,SAAS,YAAY,MAAsB;AACzC,SAAQ,KAAK,MAAM,MAAM,IAAI,EAAE,EAAE;;AAGnC,SAAS,kBAAkB,MAAsB;AAC/C,SAAQ,KAAK,MAAM,OAAO,IAAI,EAAE,EAAE;;AAGpC,SAAS,WAAW,OAAgC;AAClD,KAAI,OAAO,UAAU,UAAW,QAAO;AACvC,KAAI,OAAO,UAAU,UAAU;EAC7B,MAAM,IAAI,MAAM,MAAM,CAAC,aAAa;AACpC,MAAI,MAAM,UAAU,MAAM,SAAS,MAAM,aAAc,QAAO;AAC9D,MAAI,MAAM,WAAW,MAAM,QAAQ,MAAM,eAAgB,QAAO;;AAElE,QAAO;;AAGT,SAAS,iBAAiB,OAAgB,YAA6B;AACrE,KAAI,OAAO,UAAU,YAAY,OAAO,SAAS,MAAM,CACrD,QAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,MAAM,CAAC;AAExC,QAAO,aAAa,KAAM;;AAG5B,SAAS,oBAAoB,OAA+B;AAC1D,KAAI,OAAO,UAAU,SAAU,QAAO;CACtC,MAAM,UAAU,MAAM,MAAM;AAC5B,KAAI,QAAQ,WAAW,EAAG,QAAO;AACjC,QAAO,QAAQ,SAAS,OAAO,QAAQ,MAAM,GAAG,KAAK,GAAG;;AAG1D,SAAS,aAAa,OAAoC;AACxD,KAAI,OAAO,UAAU,SAAU,QAAO;CACtC,MAAM,UAAU,MAAM,MAAM;AAC5B,QAAO,QAAQ,WAAW,IAAI,SAAY;;AAG5C,SAAS,eAAe,MAA8C;CACpE,MAAM,SAAS,QAAQ,KAAK;AAC5B,KAAI,SAAS,OAAO,CAAE,QAAO;CAC7B,MAAM,QAAQ,KAAK,QAAQ,IAAI;CAC/B,MAAM,MAAM,KAAK,YAAY,IAAI;AACjC,KAAI,SAAS,KAAK,MAAM,OAAO;EAC7B,MAAM,SAAS,QAAQ,KAAK,MAAM,OAAO,MAAM,EAAE,CAAC;AAClD,MAAI,SAAS,OAAO,CAAE,QAAO;;AAE/B,QAAO;;AAGT,SAAS,QAAQ,MAAuB;AACtC,KAAI;AACF,SAAO,KAAK,MAAM,KAAK;SACjB;AACN;;;AAIJ,SAAS,WAAW,MAAsB;AAExC,QADc,+BAA+B,KAAK,KAAK,MAAM,CAAC,GAC/C,MAAM;;AAGvB,SAAS,SAAS,OAAkD;AAClE,QAAO,UAAU,QAAQ,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,MAAM"}
|
|
1
|
+
{"version":3,"file":"iterative.js","names":["signals: string[]","stop: RetrievalGrade","passLists: Array<ReadonlyArray<H>>","queries: string[]","out: H[]"],"sources":["../../src/search/iterative.ts"],"sourcesContent":["/**\n * Agentic / iterative retrieval for hard queries (P2-4) - a **gated**,\n * CRAG/Self-RAG-style grade-then-reformulate loop.\n *\n * Some multi-hop / temporal questions can't be answered from a single\n * retrieval pass; the naive system returns a confident-but-wrong answer\n * instead of retrieving again. This module adds a small loop: grade the\n * retrieved memories for sufficiency and, when weak, reformulate the\n * query and retrieve once more (optionally widening to the P2-1 graph) -\n * up to a hard iteration cap, then **abstain** rather than confabulate.\n *\n * Both CRAG (arXiv:2401.15884) and Self-RAG (arXiv:2310.11511) add\n * latency/cost, so the literature's advice - and this module's design -\n * is to *gate* the loop, not make it default:\n *\n * 1. A cheap, **local** {@link assessQueryDifficulty} heuristic decides\n * whether a query is even a candidate; simple lookups never enter\n * the loop (no provider call) and stay single-shot.\n * 2. The grade/reformulate step needs a {@link RetrievalGrader}; with\n * none configured the loop degrades to a single retrieval pass. The\n * only provider-backed grader, {@link createProviderRetrievalGrader},\n * is **resilient**: any provider error or unparseable output degrades\n * to a \"stop\" grade rather than throwing or looping forever.\n * 3. A mandatory iteration cap bounds the worst case regardless of the\n * grader.\n *\n * The module is provider-agnostic - it imports only `@graphorin/core`\n * types and never performs I/O itself; {@link runIterativeRetrieval} is a\n * pure orchestrator over an injected `retrieve` function + grader, so the\n * loop is unit-testable without a store or a model.\n *\n * @packageDocumentation\n */\n\nimport type { Provider, ProviderRequest } from '@graphorin/core';\n\n/** Default hard ceiling on total retrieval passes (gate + cap mandatory). */\nexport const DEFAULT_MAX_ITERATIONS = 3;\n\n/**\n * Absolute clamp on total retrieval passes, applied regardless of the\n * caller's `maxIterations` - a latency guardrail so a mis-configured\n * caller can never unbound the loop.\n */\nexport const MAX_ITERATIONS_CEILING = 5;\n\n/** Default score (0..1) at or above which a query is classified as hard. */\nexport const DEFAULT_DIFFICULTY_THRESHOLD = 0.5;\n\n/** Default output-token ceiling for a single grade call. */\nconst DEFAULT_GRADE_MAX_TOKENS = 256;\n\n/** Default number of retrieved snippets shown to the grader per pass. */\nconst DEFAULT_MAX_GRADE_SNIPPETS = 8;\n\n// ---------------------------------------------------------------------------\n// Difficulty gate (pure, local - no I/O, no provider)\n// ---------------------------------------------------------------------------\n\n/**\n * Outcome of the heuristic difficulty gate. `hard` is the gating\n * decision (`score >= threshold`); `signals` lists which heuristic\n * categories fired, for explainability / tracing.\n *\n * @stable\n */\nexport interface DifficultyAssessment {\n readonly hard: boolean;\n /** Aggregate difficulty score, clamped to `[0, 1]`. */\n readonly score: number;\n /** Heuristic categories that fired (`'multi-hop'`, `'temporal'`, …). */\n readonly signals: ReadonlyArray<string>;\n}\n\n/**\n * Tuning for {@link assessQueryDifficulty}.\n *\n * @stable\n */\nexport interface DifficultyGateOptions {\n /** Minimum score (0..1) to classify the query as hard. Default 0.5. */\n readonly threshold?: number;\n}\n\n/** Relational / referential markers implying a hop to another entity. */\nconst RE_REL_PRONOUN = /\\b(?:who|whom|whose|which)\\b/iu;\nconst RE_OF_CHAIN = /\\bthe\\s+\\w+\\s+of\\b/iu;\nconst RE_POSSESSIVE = /\\b[\\w-]+'s\\b/giu;\nconst RE_DESC_REF =\n /\\bthe\\s+(?:person|people|place|thing|one|company|team|book|restaurant|project|guy|woman|man)\\b/iu;\n/** Temporal ordering / relative-time markers. */\nconst RE_TEMPORAL =\n /\\b(?:before|after|when|while|during|since|until|first|last|earliest|latest|previously|originally|ago|prior|then|recently|used\\s+to)\\b/iu;\n/** Comparison / superlative markers. */\nconst RE_COMPARISON =\n /\\b(?:most|least|more|less|fewer|greater|biggest|largest|smallest|oldest|newest|best|worst|highest|lowest|compared?|versus|vs\\.?|between)\\b/iu;\n/** Clause-joining conjunctions used to count clauses. */\nconst RE_CONJUNCTION = /\\b(?:and|or|but|then)\\b/giu;\n\nconst W_MULTI_HOP = 0.4;\nconst W_TEMPORAL = 0.3;\nconst W_COMPARISON = 0.3;\nconst W_MULTI_CLAUSE = 0.2;\nconst W_LONG = 0.15;\nconst LONG_WORD_COUNT = 12;\n\n/**\n * Score a query's retrieval difficulty with cheap, deterministic, fully\n * **local** heuristics (no I/O). Conservative by design - it prefers to\n * leave a query single-shot (`hard: false`) unless several\n * multi-hop / temporal / comparison signals stack up, so the gate adds\n * passes only where they are likely to help. Used by\n * {@link runIterativeRetrieval} to decide whether to enter the loop.\n *\n * @stable\n */\nexport function assessQueryDifficulty(\n query: string,\n options: DifficultyGateOptions = {},\n): DifficultyAssessment {\n const threshold = options.threshold ?? DEFAULT_DIFFICULTY_THRESHOLD;\n const signals: string[] = [];\n let score = 0;\n\n const possessiveCount = (query.match(RE_POSSESSIVE) ?? []).length;\n const multiHop =\n RE_REL_PRONOUN.test(query) ||\n RE_OF_CHAIN.test(query) ||\n RE_DESC_REF.test(query) ||\n possessiveCount >= 2;\n if (multiHop) {\n signals.push('multi-hop');\n score += W_MULTI_HOP;\n }\n if (RE_TEMPORAL.test(query)) {\n signals.push('temporal');\n score += W_TEMPORAL;\n }\n if (RE_COMPARISON.test(query)) {\n signals.push('comparison');\n score += W_COMPARISON;\n }\n const clauseSignals = (query.match(RE_CONJUNCTION) ?? []).length + countCommas(query);\n if (clauseSignals >= 2 || questionMarkCount(query) >= 2) {\n signals.push('multi-clause');\n score += W_MULTI_CLAUSE;\n }\n if (wordCount(query) >= LONG_WORD_COUNT) {\n signals.push('long');\n score += W_LONG;\n }\n\n const clamped = Math.min(1, score);\n return { hard: clamped >= threshold, score: clamped, signals };\n}\n\n// ---------------------------------------------------------------------------\n// Retrieval grader seam (provider-agnostic)\n// ---------------------------------------------------------------------------\n\n/**\n * Verdict from grading a retrieved candidate set against a query.\n *\n * @stable\n */\nexport interface RetrievalGrade {\n /** Whether the retrieved memories suffice to answer the query. */\n readonly sufficient: boolean;\n /** Grader confidence, clamped to `[0, 1]`. */\n readonly confidence: number;\n /**\n * A single better search query to try next, or `null` when the grader\n * proposes none (the loop then stops / abstains).\n */\n readonly reformulation: string | null;\n /** Optional short rationale (never surfaced in spans). */\n readonly reason?: string;\n}\n\n/**\n * Per-call options for a {@link RetrievalGrader}.\n *\n * @stable\n */\nexport interface RetrievalGradeOptions {\n /** Cancellation signal forwarded to the underlying provider call. */\n readonly signal?: AbortSignal;\n /**\n * Reformulations already attempted (MRET-11). Surfaced to the grader\n * as context so it can propose something genuinely new - the grade\n * itself is ALWAYS judged against the original question, never a\n * reformulation (a narrowed sub-query must not be declared\n * \"sufficient\" while the original multi-hop question is not).\n */\n readonly triedQueries?: ReadonlyArray<string>;\n}\n\n/**\n * Pluggable retrieval-grader seam consumed by the iterative loop. The\n * built-in provider-backed implementation lives in\n * {@link createProviderRetrievalGrader}; advanced callers can supply a\n * bespoke grader (e.g. a deterministic heuristic).\n *\n * Implementations MUST degrade gracefully - return a \"stop\" grade rather\n * than throw - so a grader failure never breaks recall.\n *\n * @stable\n */\nexport interface RetrievalGrader {\n grade(\n query: string,\n snippets: ReadonlyArray<string>,\n options?: RetrievalGradeOptions,\n ): Promise<RetrievalGrade>;\n}\n\n/**\n * System prompt for the retrieval grader. Asks for a bare JSON verdict;\n * {@link parseGrade} also tolerates a chatty model.\n *\n * @internal\n */\nexport const RETRIEVAL_GRADE_SYSTEM_PROMPT =\n 'You are a retrieval grader for a personal memory store. Given a question and a ' +\n 'numbered list of retrieved memories, judge whether they are SUFFICIENT to answer ' +\n 'the question correctly and completely. Respond with ONLY a JSON object: ' +\n '{\"sufficient\": boolean, \"confidence\": number between 0 and 1, \"reformulation\": ' +\n 'string or null, \"reason\": string}. Set \"sufficient\" to true only if the memories ' +\n 'actually contain the answer - do not guess. When insufficient, set \"reformulation\" ' +\n 'to a single alternative search query likely to retrieve the missing information ' +\n '(vary entities, time frame, or specificity); otherwise null. Output only the JSON ' +\n 'object - no prose, no markdown.';\n\n/**\n * Build the grade request. Pure - no I/O. Temperature 0 so the verdict\n * is as stable as the model allows.\n *\n * @stable\n */\nexport function buildGradeRequest(\n query: string,\n snippets: ReadonlyArray<string>,\n options: {\n readonly maxTokens?: number;\n readonly signal?: AbortSignal;\n readonly triedQueries?: ReadonlyArray<string>;\n } = {},\n): ProviderRequest {\n const body =\n snippets.length === 0 ? '(none)' : snippets.map((s, i) => `[${i + 1}] ${s}`).join('\\n');\n const tried =\n options.triedQueries !== undefined && options.triedQueries.length > 0\n ? `\\n\\nQueries already tried (do not repeat them): ${options.triedQueries.join(' | ')}`\n : '';\n const content =\n `Question: ${query}\\n\\nRetrieved memories:\\n${body}${tried}\\n\\n` +\n 'Grade whether these memories are sufficient to answer the question. If not, ' +\n 'propose a single better search query.';\n return {\n messages: [{ role: 'user', content }],\n systemMessage: RETRIEVAL_GRADE_SYSTEM_PROMPT,\n temperature: 0,\n maxTokens: options.maxTokens ?? DEFAULT_GRADE_MAX_TOKENS,\n ...(options.signal !== undefined ? { signal: options.signal } : {}),\n outputType: { kind: 'structured' },\n };\n}\n\n/**\n * Parse a grader model output into a {@link RetrievalGrade}. Tolerates a\n * bare JSON object, a fenced block, and chatty text around the object.\n *\n * **Fail-safe = stop.** When the verdict can't be read (undefined / empty\n * / unparseable / missing `sufficient`), returns\n * `{ sufficient: true, confidence: 0, reformulation: null }` so a broken\n * grader degrades to a single pass rather than looping or falsely\n * abstaining. A *parsed* `sufficient: false` (a real insufficiency\n * verdict) is preserved.\n *\n * @stable\n */\nexport function parseGrade(text: string | undefined): RetrievalGrade {\n const stop: RetrievalGrade = { sufficient: true, confidence: 0, reformulation: null };\n if (text === undefined) return stop;\n const stripped = stripFence(text).trim();\n if (stripped.length === 0) return stop;\n const obj = tryParseObject(stripped);\n if (obj === null) return stop;\n const sufficient = coerceBool(obj.sufficient);\n if (sufficient === null) return stop;\n const confidence = coerceConfidence(obj.confidence, sufficient);\n const reformulation = sufficient ? null : coerceReformulation(obj.reformulation);\n const reason = coerceReason(obj.reason);\n return {\n sufficient,\n confidence,\n reformulation,\n ...(reason !== undefined ? { reason } : {}),\n };\n}\n\n/**\n * Wrap a {@link Provider} as a {@link RetrievalGrader}. **Resilient**: a\n * provider error or unparseable response degrades to the\n * {@link parseGrade} \"stop\" grade so grading never throws into the loop\n * (and a flaky grader can't drive endless reformulation).\n *\n * @stable\n */\nexport function createProviderRetrievalGrader(\n provider: Provider,\n options: { readonly maxTokens?: number } = {},\n): RetrievalGrader {\n const maxTokens = options.maxTokens ?? DEFAULT_GRADE_MAX_TOKENS;\n return {\n async grade(query, snippets, opts = {}): Promise<RetrievalGrade> {\n try {\n const request = buildGradeRequest(query, snippets, {\n maxTokens,\n ...(opts.signal !== undefined ? { signal: opts.signal } : {}),\n ...(opts.triedQueries !== undefined ? { triedQueries: opts.triedQueries } : {}),\n });\n const response = await provider.generate(request);\n return parseGrade(response.text);\n } catch {\n return { sufficient: true, confidence: 0, reformulation: null };\n }\n },\n };\n}\n\n// ---------------------------------------------------------------------------\n// The bounded loop (pure orchestration over injected retrieve + grader)\n// ---------------------------------------------------------------------------\n\n/**\n * Result of an iterative retrieval run.\n *\n * @stable\n */\nexport interface IterativeRetrievalResult<H> {\n /** Accumulated hits across all passes (deduped, in discovery order). */\n readonly hits: ReadonlyArray<H>;\n /** Number of retrieval passes performed (≥ 1). */\n readonly iterations: number;\n /** Difficulty-gate verdict (whether the loop was eligible to run). */\n readonly gateHard: boolean;\n /** Final sufficiency verdict. */\n readonly sufficient: boolean;\n /**\n * `true` when the loop exhausted its cap / ran out of reformulations\n * while still judged insufficient - the caller should abstain rather\n * than answer from `hits`.\n */\n readonly abstained: boolean;\n /**\n * `true` when the grader actually judged the result at least once\n * (memory-retrieval-02). `false` on the single-shot path (gate not\n * hard, or no grader configured) - there `sufficient: true` is a\n * DEFAULT, not a verdict, and consumers must not read it as one.\n */\n readonly graded: boolean;\n /** The sequence of queries tried (original first). */\n readonly queries: ReadonlyArray<string>;\n}\n\n/**\n * Dependencies injected into {@link runIterativeRetrieval}. The loop does\n * no I/O of its own - `retrieve` and `grader` own all side effects.\n *\n * @stable\n */\nexport interface IterativeRetrievalDeps<H> {\n /**\n * Run one retrieval pass for `query`. `widen` is `true` on\n * reformulation passes so the caller can broaden recall (e.g. enable\n * P2-1 one-hop graph expansion).\n */\n retrieve(query: string, widen: boolean, signal?: AbortSignal): Promise<ReadonlyArray<H>>;\n /** Snippet shown to the grader for a hit. */\n snippetOf(hit: H): string;\n /** Stable id used to dedup hits across passes. */\n idOf(hit: H): string;\n /** Grader; `null` ⇒ single-shot (no grading, no provider call). */\n grader: RetrievalGrader | null;\n /**\n * Re-fuse the per-pass hit lists into one ranked list (MRET-2).\n * Receives one list per pass in pass order; the result feeds the\n * final `maxResults` cut so a pass-2 find can outrank pass-1 noise.\n * Absent ⇒ the loop falls back to round-robin interleaving (still\n * strictly better than the old discovery-order cut).\n */\n fuse?(lists: ReadonlyArray<ReadonlyArray<H>>): ReadonlyArray<H>;\n}\n\n/**\n * Options for {@link runIterativeRetrieval}.\n *\n * @stable\n */\nexport interface IterativeRetrievalOptions {\n /** Total-pass cap; clamped to `[1, {@link MAX_ITERATIONS_CEILING}]`. */\n readonly maxIterations?: number;\n /** Difficulty-gate tuning. */\n readonly difficulty?: DifficultyGateOptions;\n /** Skip the heuristic gate and force the loop (still capped). */\n readonly forceHard?: boolean;\n /** Max snippets passed to the grader per pass. Default 8. */\n readonly maxGradeSnippets?: number;\n /** Cap on the returned hit count (omitted ⇒ all accumulated). */\n readonly maxResults?: number;\n readonly signal?: AbortSignal;\n}\n\n/**\n * Run the gated grade-then-reformulate loop.\n *\n * Flow: assess difficulty → retrieve (pass 1) → if not hard *or* no\n * grader, return single-shot → else grade; if sufficient, return; if\n * weak and a reformulation is offered and the cap is not hit, retrieve\n * again (widened) and re-grade; otherwise **abstain**.\n *\n * @stable\n */\nexport async function runIterativeRetrieval<H>(\n query: string,\n deps: IterativeRetrievalDeps<H>,\n options: IterativeRetrievalOptions = {},\n): Promise<IterativeRetrievalResult<H>> {\n const gate =\n options.forceHard === true\n ? { hard: true, score: 1, signals: ['forced'] as ReadonlyArray<string> }\n : assessQueryDifficulty(query, options.difficulty ?? {});\n const cap = clampIterations(options.maxIterations);\n const maxSnippets = Math.max(1, options.maxGradeSnippets ?? DEFAULT_MAX_GRADE_SNIPPETS);\n const signal = options.signal;\n\n // MRET-2: hits are tracked PER PASS - RAW, including cross-pass\n // repeats, so rank fusion can reward consensus between passes. The\n // grade window and the final list dedup by id at read time. The old\n // flat append meant pass 1 saturated the first `maxSnippets` forever\n // and the final cut ran in discovery order.\n const passLists: Array<ReadonlyArray<H>> = [];\n const accumulate = (hits: ReadonlyArray<H>): void => {\n passLists.push([...hits]);\n };\n const queries: string[] = [query];\n const tried = new Set<string>([query.trim().toLowerCase()]);\n\n // Pass 1 - always single-shot, no widening.\n accumulate(await deps.retrieve(query, false, signal));\n\n // Gate: the loop is eligible only when the query is judged hard *and* a\n // grader exists. Otherwise stay single-shot - we did not grade, so we\n // make no abstention claim.\n const grader = deps.grader;\n if (!gate.hard || grader === null) {\n // Single-shot: nothing was graded, so `sufficient: true` is a\n // default - `graded: false` tells consumers not to trust it as a\n // verdict (memory-retrieval-02).\n return finalize(deps, passLists, 1, gate.hard, true, false, false, queries, options.maxResults);\n }\n\n let passes = 1;\n for (;;) {\n // MRET-2: the grade window interleaves the TOP hits of every pass\n // (latest pass included) instead of replaying pass 1's head.\n // MRET-11: sufficiency is always judged against the ORIGINAL\n // question; reformulations ride along as already-tried context.\n const window = interleave(passLists, maxSnippets, deps.idOf).map(deps.snippetOf);\n const grade = await grader.grade(query, window, {\n ...(signal !== undefined ? { signal } : {}),\n ...(queries.length > 1 ? { triedQueries: queries.slice(1) } : {}),\n });\n if (grade.sufficient) {\n return finalize(\n deps,\n passLists,\n passes,\n true,\n true,\n false,\n true,\n queries,\n options.maxResults,\n );\n }\n if (passes >= cap) break;\n const next = normalizeReformulation(grade.reformulation, tried);\n if (next === null) break;\n passes += 1;\n queries.push(next);\n tried.add(next.toLowerCase());\n accumulate(await deps.retrieve(next, true, signal));\n }\n // Cap reached or no further reformulation while still insufficient ⇒ abstain.\n return finalize(deps, passLists, passes, true, false, true, true, queries, options.maxResults);\n}\n\n/**\n * Round-robin interleave of per-pass lists, deduped by id: hit 1 of\n * every pass, then hit 2 of every pass, … - guarantees the latest\n * pass's top hits enter a window of any size.\n */\nfunction interleave<H>(\n lists: ReadonlyArray<ReadonlyArray<H>>,\n limit: number,\n idOf: (hit: H) => string,\n): H[] {\n const out: H[] = [];\n const seen = new Set<string>();\n const longest = lists.reduce((max, l) => Math.max(max, l.length), 0);\n for (let rank = 0; rank < longest && out.length < limit; rank += 1) {\n for (const list of lists) {\n const hit = list[rank];\n if (hit === undefined) continue;\n const id = idOf(hit);\n if (seen.has(id)) continue;\n seen.add(id);\n out.push(hit);\n if (out.length >= limit) break;\n }\n }\n return out;\n}\n\nfunction finalize<H>(\n deps: IterativeRetrievalDeps<H>,\n passLists: ReadonlyArray<ReadonlyArray<H>>,\n iterations: number,\n gateHard: boolean,\n sufficient: boolean,\n abstained: boolean,\n graded: boolean,\n queries: ReadonlyArray<string>,\n maxResults: number | undefined,\n): IterativeRetrievalResult<H> {\n // MRET-2: re-rank ACROSS passes before the cut - discovery order let\n // pass-1 noise crowd out the pass-2 hit that actually answers.\n const ranked =\n typeof deps.fuse === 'function'\n ? dedupBy(deps.fuse(passLists), deps.idOf)\n : interleave(passLists, Number.POSITIVE_INFINITY, deps.idOf);\n const capped =\n maxResults !== undefined && maxResults >= 0 ? ranked.slice(0, maxResults) : ranked.slice();\n return {\n hits: capped,\n iterations,\n gateHard,\n sufficient,\n abstained,\n graded,\n queries: [...queries],\n };\n}\n\nfunction dedupBy<H>(hits: ReadonlyArray<H>, idOf: (hit: H) => string): H[] {\n const seen = new Set<string>();\n const out: H[] = [];\n for (const hit of hits) {\n const id = idOf(hit);\n if (seen.has(id)) continue;\n seen.add(id);\n out.push(hit);\n }\n return out;\n}\n\nfunction clampIterations(value: number | undefined): number {\n const n = value ?? DEFAULT_MAX_ITERATIONS;\n if (!Number.isFinite(n)) return DEFAULT_MAX_ITERATIONS;\n return Math.min(MAX_ITERATIONS_CEILING, Math.max(1, Math.floor(n)));\n}\n\n/**\n * Normalise a proposed reformulation: trim, drop empties, and reject a\n * query already tried (case-insensitive) so the loop never re-treads and\n * cannot spin.\n */\nfunction normalizeReformulation(raw: string | null, tried: ReadonlySet<string>): string | null {\n if (raw === null) return null;\n const trimmed = raw.trim();\n if (trimmed.length === 0) return null;\n if (tried.has(trimmed.toLowerCase())) return null;\n return trimmed;\n}\n\n// ---------------------------------------------------------------------------\n// Helpers\n// ---------------------------------------------------------------------------\n\nfunction wordCount(text: string): number {\n const trimmed = text.trim();\n return trimmed.length === 0 ? 0 : trimmed.split(/\\s+/u).length;\n}\n\nfunction countCommas(text: string): number {\n return (text.match(/,/gu) ?? []).length;\n}\n\nfunction questionMarkCount(text: string): number {\n return (text.match(/\\?/gu) ?? []).length;\n}\n\nfunction coerceBool(value: unknown): boolean | null {\n if (typeof value === 'boolean') return value;\n if (typeof value === 'string') {\n const v = value.trim().toLowerCase();\n if (v === 'true' || v === 'yes' || v === 'sufficient') return true;\n if (v === 'false' || v === 'no' || v === 'insufficient') return false;\n }\n return null;\n}\n\nfunction coerceConfidence(value: unknown, sufficient: boolean): number {\n if (typeof value === 'number' && Number.isFinite(value)) {\n return Math.min(1, Math.max(0, value));\n }\n return sufficient ? 0.8 : 0.3;\n}\n\nfunction coerceReformulation(value: unknown): string | null {\n if (typeof value !== 'string') return null;\n const trimmed = value.trim();\n if (trimmed.length === 0) return null;\n return trimmed.length > 1024 ? trimmed.slice(0, 1024) : trimmed;\n}\n\nfunction coerceReason(value: unknown): string | undefined {\n if (typeof value !== 'string') return undefined;\n const trimmed = value.trim();\n return trimmed.length === 0 ? undefined : trimmed;\n}\n\nfunction tryParseObject(text: string): Record<string, unknown> | null {\n const direct = tryJson(text);\n if (isRecord(direct)) return direct;\n const start = text.indexOf('{');\n const end = text.lastIndexOf('}');\n if (start >= 0 && end > start) {\n const sliced = tryJson(text.slice(start, end + 1));\n if (isRecord(sliced)) return sliced;\n }\n return null;\n}\n\nfunction tryJson(text: string): unknown {\n try {\n return JSON.parse(text);\n } catch {\n return undefined;\n }\n}\n\nfunction stripFence(text: string): string {\n const match = /^```[^\\n]*\\n([\\s\\S]*?)\\n```/u.exec(text.trim());\n return match?.[1] ?? text;\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n return value !== null && typeof value === 'object' && !Array.isArray(value);\n}\n"],"mappings":";;AAqCA,MAAa,yBAAyB;;;;;;AAOtC,MAAa,yBAAyB;;AAGtC,MAAa,+BAA+B;;AAG5C,MAAM,2BAA2B;;AAGjC,MAAM,6BAA6B;;AAgCnC,MAAM,iBAAiB;AACvB,MAAM,cAAc;AACpB,MAAM,gBAAgB;AACtB,MAAM,cACJ;;AAEF,MAAM,cACJ;;AAEF,MAAM,gBACJ;;AAEF,MAAM,iBAAiB;AAEvB,MAAM,cAAc;AACpB,MAAM,aAAa;AACnB,MAAM,eAAe;AACrB,MAAM,iBAAiB;AACvB,MAAM,SAAS;AACf,MAAM,kBAAkB;;;;;;;;;;;AAYxB,SAAgB,sBACd,OACA,UAAiC,EAAE,EACb;CACtB,MAAM,YAAY,QAAQ,aAAa;CACvC,MAAMA,UAAoB,EAAE;CAC5B,IAAI,QAAQ;CAEZ,MAAM,mBAAmB,MAAM,MAAM,cAAc,IAAI,EAAE,EAAE;AAM3D,KAJE,eAAe,KAAK,MAAM,IAC1B,YAAY,KAAK,MAAM,IACvB,YAAY,KAAK,MAAM,IACvB,mBAAmB,GACP;AACZ,UAAQ,KAAK,YAAY;AACzB,WAAS;;AAEX,KAAI,YAAY,KAAK,MAAM,EAAE;AAC3B,UAAQ,KAAK,WAAW;AACxB,WAAS;;AAEX,KAAI,cAAc,KAAK,MAAM,EAAE;AAC7B,UAAQ,KAAK,aAAa;AAC1B,WAAS;;AAGX,MADuB,MAAM,MAAM,eAAe,IAAI,EAAE,EAAE,SAAS,YAAY,MAAM,IAChE,KAAK,kBAAkB,MAAM,IAAI,GAAG;AACvD,UAAQ,KAAK,eAAe;AAC5B,WAAS;;AAEX,KAAI,UAAU,MAAM,IAAI,iBAAiB;AACvC,UAAQ,KAAK,OAAO;AACpB,WAAS;;CAGX,MAAM,UAAU,KAAK,IAAI,GAAG,MAAM;AAClC,QAAO;EAAE,MAAM,WAAW;EAAW,OAAO;EAAS;EAAS;;;;;;;;AAqEhE,MAAa,gCACX;;;;;;;AAgBF,SAAgB,kBACd,OACA,UACA,UAII,EAAE,EACW;AAWjB,QAAO;EACL,UAAU,CAAC;GAAE,MAAM;GAAQ,SAJ3B,aAAa,MAAM,2BANnB,SAAS,WAAW,IAAI,WAAW,SAAS,KAAK,GAAG,MAAM,IAAI,IAAI,EAAE,IAAI,IAAI,CAAC,KAAK,KAAK,GAEvF,QAAQ,iBAAiB,UAAa,QAAQ,aAAa,SAAS,IAChE,mDAAmD,QAAQ,aAAa,KAAK,MAAM,KACnF,GAEuD;GAIvB,CAAC;EACrC,eAAe;EACf,aAAa;EACb,WAAW,QAAQ,aAAa;EAChC,GAAI,QAAQ,WAAW,SAAY,EAAE,QAAQ,QAAQ,QAAQ,GAAG,EAAE;EAClE,YAAY,EAAE,MAAM,cAAc;EACnC;;;;;;;;;;;;;;;AAgBH,SAAgB,WAAW,MAA0C;CACnE,MAAMC,OAAuB;EAAE,YAAY;EAAM,YAAY;EAAG,eAAe;EAAM;AACrF,KAAI,SAAS,OAAW,QAAO;CAC/B,MAAM,WAAW,WAAW,KAAK,CAAC,MAAM;AACxC,KAAI,SAAS,WAAW,EAAG,QAAO;CAClC,MAAM,MAAM,eAAe,SAAS;AACpC,KAAI,QAAQ,KAAM,QAAO;CACzB,MAAM,aAAa,WAAW,IAAI,WAAW;AAC7C,KAAI,eAAe,KAAM,QAAO;CAChC,MAAM,aAAa,iBAAiB,IAAI,YAAY,WAAW;CAC/D,MAAM,gBAAgB,aAAa,OAAO,oBAAoB,IAAI,cAAc;CAChF,MAAM,SAAS,aAAa,IAAI,OAAO;AACvC,QAAO;EACL;EACA;EACA;EACA,GAAI,WAAW,SAAY,EAAE,QAAQ,GAAG,EAAE;EAC3C;;;;;;;;;;AAWH,SAAgB,8BACd,UACA,UAA2C,EAAE,EAC5B;CACjB,MAAM,YAAY,QAAQ,aAAa;AACvC,QAAO,EACL,MAAM,MAAM,OAAO,UAAU,OAAO,EAAE,EAA2B;AAC/D,MAAI;GACF,MAAM,UAAU,kBAAkB,OAAO,UAAU;IACjD;IACA,GAAI,KAAK,WAAW,SAAY,EAAE,QAAQ,KAAK,QAAQ,GAAG,EAAE;IAC5D,GAAI,KAAK,iBAAiB,SAAY,EAAE,cAAc,KAAK,cAAc,GAAG,EAAE;IAC/E,CAAC;AAEF,UAAO,YADU,MAAM,SAAS,SAAS,QAAQ,EACtB,KAAK;UAC1B;AACN,UAAO;IAAE,YAAY;IAAM,YAAY;IAAG,eAAe;IAAM;;IAGpE;;;;;;;;;;;;AAgGH,eAAsB,sBACpB,OACA,MACA,UAAqC,EAAE,EACD;CACtC,MAAM,OACJ,QAAQ,cAAc,OAClB;EAAE,MAAM;EAAM,OAAO;EAAG,SAAS,CAAC,SAAS;EAA2B,GACtE,sBAAsB,OAAO,QAAQ,cAAc,EAAE,CAAC;CAC5D,MAAM,MAAM,gBAAgB,QAAQ,cAAc;CAClD,MAAM,cAAc,KAAK,IAAI,GAAG,QAAQ,oBAAoB,2BAA2B;CACvF,MAAM,SAAS,QAAQ;CAOvB,MAAMC,YAAqC,EAAE;CAC7C,MAAM,cAAc,SAAiC;AACnD,YAAU,KAAK,CAAC,GAAG,KAAK,CAAC;;CAE3B,MAAMC,UAAoB,CAAC,MAAM;CACjC,MAAM,QAAQ,IAAI,IAAY,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;AAG3D,YAAW,MAAM,KAAK,SAAS,OAAO,OAAO,OAAO,CAAC;CAKrD,MAAM,SAAS,KAAK;AACpB,KAAI,CAAC,KAAK,QAAQ,WAAW,KAI3B,QAAO,SAAS,MAAM,WAAW,GAAG,KAAK,MAAM,MAAM,OAAO,OAAO,SAAS,QAAQ,WAAW;CAGjG,IAAI,SAAS;AACb,UAAS;EAKP,MAAM,SAAS,WAAW,WAAW,aAAa,KAAK,KAAK,CAAC,IAAI,KAAK,UAAU;EAChF,MAAM,QAAQ,MAAM,OAAO,MAAM,OAAO,QAAQ;GAC9C,GAAI,WAAW,SAAY,EAAE,QAAQ,GAAG,EAAE;GAC1C,GAAI,QAAQ,SAAS,IAAI,EAAE,cAAc,QAAQ,MAAM,EAAE,EAAE,GAAG,EAAE;GACjE,CAAC;AACF,MAAI,MAAM,WACR,QAAO,SACL,MACA,WACA,QACA,MACA,MACA,OACA,MACA,SACA,QAAQ,WACT;AAEH,MAAI,UAAU,IAAK;EACnB,MAAM,OAAO,uBAAuB,MAAM,eAAe,MAAM;AAC/D,MAAI,SAAS,KAAM;AACnB,YAAU;AACV,UAAQ,KAAK,KAAK;AAClB,QAAM,IAAI,KAAK,aAAa,CAAC;AAC7B,aAAW,MAAM,KAAK,SAAS,MAAM,MAAM,OAAO,CAAC;;AAGrD,QAAO,SAAS,MAAM,WAAW,QAAQ,MAAM,OAAO,MAAM,MAAM,SAAS,QAAQ,WAAW;;;;;;;AAQhG,SAAS,WACP,OACA,OACA,MACK;CACL,MAAMC,MAAW,EAAE;CACnB,MAAM,uBAAO,IAAI,KAAa;CAC9B,MAAM,UAAU,MAAM,QAAQ,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,OAAO,EAAE,EAAE;AACpE,MAAK,IAAI,OAAO,GAAG,OAAO,WAAW,IAAI,SAAS,OAAO,QAAQ,EAC/D,MAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,MAAM,KAAK;AACjB,MAAI,QAAQ,OAAW;EACvB,MAAM,KAAK,KAAK,IAAI;AACpB,MAAI,KAAK,IAAI,GAAG,CAAE;AAClB,OAAK,IAAI,GAAG;AACZ,MAAI,KAAK,IAAI;AACb,MAAI,IAAI,UAAU,MAAO;;AAG7B,QAAO;;AAGT,SAAS,SACP,MACA,WACA,YACA,UACA,YACA,WACA,QACA,SACA,YAC6B;CAG7B,MAAM,SACJ,OAAO,KAAK,SAAS,aACjB,QAAQ,KAAK,KAAK,UAAU,EAAE,KAAK,KAAK,GACxC,WAAW,WAAW,OAAO,mBAAmB,KAAK,KAAK;AAGhE,QAAO;EACL,MAFA,eAAe,UAAa,cAAc,IAAI,OAAO,MAAM,GAAG,WAAW,GAAG,OAAO,OAAO;EAG1F;EACA;EACA;EACA;EACA;EACA,SAAS,CAAC,GAAG,QAAQ;EACtB;;AAGH,SAAS,QAAW,MAAwB,MAA+B;CACzE,MAAM,uBAAO,IAAI,KAAa;CAC9B,MAAMA,MAAW,EAAE;AACnB,MAAK,MAAM,OAAO,MAAM;EACtB,MAAM,KAAK,KAAK,IAAI;AACpB,MAAI,KAAK,IAAI,GAAG,CAAE;AAClB,OAAK,IAAI,GAAG;AACZ,MAAI,KAAK,IAAI;;AAEf,QAAO;;AAGT,SAAS,gBAAgB,OAAmC;CAC1D,MAAM,IAAI,SAAS;AACnB,KAAI,CAAC,OAAO,SAAS,EAAE,CAAE,QAAO;AAChC,QAAO,KAAK,IAAI,wBAAwB,KAAK,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC,CAAC;;;;;;;AAQrE,SAAS,uBAAuB,KAAoB,OAA2C;AAC7F,KAAI,QAAQ,KAAM,QAAO;CACzB,MAAM,UAAU,IAAI,MAAM;AAC1B,KAAI,QAAQ,WAAW,EAAG,QAAO;AACjC,KAAI,MAAM,IAAI,QAAQ,aAAa,CAAC,CAAE,QAAO;AAC7C,QAAO;;AAOT,SAAS,UAAU,MAAsB;CACvC,MAAM,UAAU,KAAK,MAAM;AAC3B,QAAO,QAAQ,WAAW,IAAI,IAAI,QAAQ,MAAM,OAAO,CAAC;;AAG1D,SAAS,YAAY,MAAsB;AACzC,SAAQ,KAAK,MAAM,MAAM,IAAI,EAAE,EAAE;;AAGnC,SAAS,kBAAkB,MAAsB;AAC/C,SAAQ,KAAK,MAAM,OAAO,IAAI,EAAE,EAAE;;AAGpC,SAAS,WAAW,OAAgC;AAClD,KAAI,OAAO,UAAU,UAAW,QAAO;AACvC,KAAI,OAAO,UAAU,UAAU;EAC7B,MAAM,IAAI,MAAM,MAAM,CAAC,aAAa;AACpC,MAAI,MAAM,UAAU,MAAM,SAAS,MAAM,aAAc,QAAO;AAC9D,MAAI,MAAM,WAAW,MAAM,QAAQ,MAAM,eAAgB,QAAO;;AAElE,QAAO;;AAGT,SAAS,iBAAiB,OAAgB,YAA6B;AACrE,KAAI,OAAO,UAAU,YAAY,OAAO,SAAS,MAAM,CACrD,QAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,MAAM,CAAC;AAExC,QAAO,aAAa,KAAM;;AAG5B,SAAS,oBAAoB,OAA+B;AAC1D,KAAI,OAAO,UAAU,SAAU,QAAO;CACtC,MAAM,UAAU,MAAM,MAAM;AAC5B,KAAI,QAAQ,WAAW,EAAG,QAAO;AACjC,QAAO,QAAQ,SAAS,OAAO,QAAQ,MAAM,GAAG,KAAK,GAAG;;AAG1D,SAAS,aAAa,OAAoC;AACxD,KAAI,OAAO,UAAU,SAAU,QAAO;CACtC,MAAM,UAAU,MAAM,MAAM;AAC5B,QAAO,QAAQ,WAAW,IAAI,SAAY;;AAG5C,SAAS,eAAe,MAA8C;CACpE,MAAM,SAAS,QAAQ,KAAK;AAC5B,KAAI,SAAS,OAAO,CAAE,QAAO;CAC7B,MAAM,QAAQ,KAAK,QAAQ,IAAI;CAC/B,MAAM,MAAM,KAAK,YAAY,IAAI;AACjC,KAAI,SAAS,KAAK,MAAM,OAAO;EAC7B,MAAM,SAAS,QAAQ,KAAK,MAAM,OAAO,MAAM,EAAE,CAAC;AAClD,MAAI,SAAS,OAAO,CAAE,QAAO;;AAE/B,QAAO;;AAGT,SAAS,QAAQ,MAAuB;AACtC,KAAI;AACF,SAAO,KAAK,MAAM,KAAK;SACjB;AACN;;;AAIJ,SAAS,WAAW,MAAsB;AAExC,QADc,+BAA+B,KAAK,KAAK,MAAM,CAAC,GAC/C,MAAM;;AAGvB,SAAS,SAAS,OAAkD;AAClE,QAAO,UAAU,QAAQ,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,MAAM"}
|
|
@@ -12,8 +12,8 @@ declare const DEFAULT_MAX_QUERY_VARIANTS = 5;
|
|
|
12
12
|
* bespoke one (e.g. a deterministic synonym expander) to
|
|
13
13
|
* `new SemanticMemory({ queryTransformer })`.
|
|
14
14
|
*
|
|
15
|
-
* Implementations MUST degrade gracefully
|
|
16
|
-
* than throw
|
|
15
|
+
* Implementations MUST degrade gracefully - return `[]` / `null` rather
|
|
16
|
+
* than throw - so a transform failure never breaks recall.
|
|
17
17
|
*
|
|
18
18
|
* @stable
|
|
19
19
|
*/
|
|
@@ -43,7 +43,7 @@ interface QueryTransformOptions {
|
|
|
43
43
|
/**
|
|
44
44
|
* System prompt for multi-query variant generation. Asks for a bare
|
|
45
45
|
* JSON array of standalone rephrasings that preserve the original
|
|
46
|
-
* intent
|
|
46
|
+
* intent - {@link parseQueryVariants} also tolerates a chatty model.
|
|
47
47
|
*
|
|
48
48
|
* @internal
|
|
49
49
|
*/
|
|
@@ -57,7 +57,7 @@ declare const QUERY_EXPANSION_SYSTEM_PROMPT: string;
|
|
|
57
57
|
*/
|
|
58
58
|
declare const HYDE_SYSTEM_PROMPT: string;
|
|
59
59
|
/**
|
|
60
|
-
* Build the multi-query expansion request. Pure
|
|
60
|
+
* Build the multi-query expansion request. Pure - no I/O. A higher
|
|
61
61
|
* temperature is used deliberately so the variants diverge (the
|
|
62
62
|
* downstream retrieval + RRF fusion stays deterministic).
|
|
63
63
|
*
|
|
@@ -68,7 +68,7 @@ declare function buildExpansionRequest(query: string, count: number, options?: {
|
|
|
68
68
|
readonly signal?: AbortSignal;
|
|
69
69
|
}): ProviderRequest;
|
|
70
70
|
/**
|
|
71
|
-
* Build the HyDE pseudo-document request. Pure
|
|
71
|
+
* Build the HyDE pseudo-document request. Pure - no I/O.
|
|
72
72
|
*
|
|
73
73
|
* @stable
|
|
74
74
|
*/
|
|
@@ -80,7 +80,7 @@ declare function buildHydeRequest(query: string, options?: {
|
|
|
80
80
|
* Parse the variant-generation model output into a deduped, capped list
|
|
81
81
|
* of reworded queries. Tolerates a JSON array, a `{ "variants": [...] }`
|
|
82
82
|
* / `{ "queries": [...] }` wrapper, fenced blocks, and (as a last
|
|
83
|
-
* resort) a newline / numbered list
|
|
83
|
+
* resort) a newline / numbered list - so a chatty model never breaks
|
|
84
84
|
* recall. Empty strings and case-insensitive duplicates are dropped;
|
|
85
85
|
* the result is capped at `max`.
|
|
86
86
|
*
|
|
@@ -6,11 +6,11 @@ const DEFAULT_TRANSFORM_MAX_TOKENS = 256;
|
|
|
6
6
|
/**
|
|
7
7
|
* System prompt for multi-query variant generation. Asks for a bare
|
|
8
8
|
* JSON array of standalone rephrasings that preserve the original
|
|
9
|
-
* intent
|
|
9
|
+
* intent - {@link parseQueryVariants} also tolerates a chatty model.
|
|
10
10
|
*
|
|
11
11
|
* @internal
|
|
12
12
|
*/
|
|
13
|
-
const QUERY_EXPANSION_SYSTEM_PROMPT = "You rewrite a search query into alternative phrasings to improve recall over a personal memory store. Preserve the original intent; vary vocabulary, specificity, and perspective so different wordings of the same stored memory are matched. Return ONLY a JSON array of strings
|
|
13
|
+
const QUERY_EXPANSION_SYSTEM_PROMPT = "You rewrite a search query into alternative phrasings to improve recall over a personal memory store. Preserve the original intent; vary vocabulary, specificity, and perspective so different wordings of the same stored memory are matched. Return ONLY a JSON array of strings - each a standalone query, no numbering, no commentary, no duplicates of the original.";
|
|
14
14
|
/**
|
|
15
15
|
* System prompt for HyDE. Asks for a short, plausible hypothetical
|
|
16
16
|
* answer (not a question rephrase) whose embedding sits near the
|
|
@@ -18,9 +18,9 @@ const QUERY_EXPANSION_SYSTEM_PROMPT = "You rewrite a search query into alternati
|
|
|
18
18
|
*
|
|
19
19
|
* @internal
|
|
20
20
|
*/
|
|
21
|
-
const HYDE_SYSTEM_PROMPT = "Write a short, plausible hypothetical answer to the question, phrased as a first-person statement of fact as it might be recorded in a memory store. One to three sentences. Output only the passage
|
|
21
|
+
const HYDE_SYSTEM_PROMPT = "Write a short, plausible hypothetical answer to the question, phrased as a first-person statement of fact as it might be recorded in a memory store. One to three sentences. Output only the passage - no preamble, no caveats, no markdown.";
|
|
22
22
|
/**
|
|
23
|
-
* Build the multi-query expansion request. Pure
|
|
23
|
+
* Build the multi-query expansion request. Pure - no I/O. A higher
|
|
24
24
|
* temperature is used deliberately so the variants diverge (the
|
|
25
25
|
* downstream retrieval + RRF fusion stays deterministic).
|
|
26
26
|
*
|
|
@@ -40,7 +40,7 @@ function buildExpansionRequest(query, count, options = {}) {
|
|
|
40
40
|
};
|
|
41
41
|
}
|
|
42
42
|
/**
|
|
43
|
-
* Build the HyDE pseudo-document request. Pure
|
|
43
|
+
* Build the HyDE pseudo-document request. Pure - no I/O.
|
|
44
44
|
*
|
|
45
45
|
* @stable
|
|
46
46
|
*/
|
|
@@ -60,7 +60,7 @@ function buildHydeRequest(query, options = {}) {
|
|
|
60
60
|
* Parse the variant-generation model output into a deduped, capped list
|
|
61
61
|
* of reworded queries. Tolerates a JSON array, a `{ "variants": [...] }`
|
|
62
62
|
* / `{ "queries": [...] }` wrapper, fenced blocks, and (as a last
|
|
63
|
-
* resort) a newline / numbered list
|
|
63
|
+
* resort) a newline / numbered list - so a chatty model never breaks
|
|
64
64
|
* recall. Empty strings and case-insensitive duplicates are dropped;
|
|
65
65
|
* the result is capped at `max`.
|
|
66
66
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"query-transform.js","names":["out: string[]"],"sources":["../../src/search/query-transform.ts"],"sourcesContent":["/**\n * Query transformation for hybrid retrieval (P2-3) — **multi-query /\n * RAG-Fusion** and optional **HyDE**.\n *\n * Auto-recall is a regex trigger and the default search is single-shot:\n * the user's phrasing rarely matches how a memory was stored, so a\n * relevant fact is missed at the retrieval step. This module fans a\n * query into a few reworded variants (one cheap LLM call) and/or writes\n * a hypothetical answer passage to embed (HyDE, arXiv:2212.10496); the\n * caller retrieves each and fuses the lists through the existing RRF\n * reranker. Variant generation is the only thing that touches a\n * provider, so it is **opt-in**: with no transformer configured the\n * default search path stays fully offline and single-shot.\n *\n * The module is provider-agnostic — it imports only `@graphorin/core`\n * types and never performs I/O itself. The concrete `Provider` is\n * injected by the caller via {@link createProviderQueryTransformer},\n * which is resilient: any provider error or unparseable output degrades\n * to \"no variants\" / \"no hypothetical\" rather than throwing into the\n * search hot path.\n *\n * @packageDocumentation\n */\n\nimport type { Provider, ProviderRequest } from '@graphorin/core';\n\n/** Default ceiling on reworded variants a provider-backed transformer requests. */\nexport const DEFAULT_MAX_QUERY_VARIANTS = 5;\n\n/** Default output-token ceiling for a single transform call. */\nconst DEFAULT_TRANSFORM_MAX_TOKENS = 256;\n\n/**\n * Pluggable query-transformation seam consumed by\n * `SemanticMemory.search(..., { multiQuery, hyde })`. The built-in\n * provider-backed implementation lives in\n * {@link createProviderQueryTransformer}; advanced callers can supply a\n * bespoke one (e.g. a deterministic synonym expander) to\n * `new SemanticMemory({ queryTransformer })`.\n *\n * Implementations MUST degrade gracefully — return `[]` / `null` rather\n * than throw — so a transform failure never breaks recall.\n *\n * @stable\n */\nexport interface QueryTransformer {\n /**\n * Rewrite a query into up to `count` **additional** reworded variants\n * (the original query is retained by the caller). Returns `[]` when\n * transformation is unavailable or the model returns nothing usable.\n */\n expand(\n query: string,\n count: number,\n options?: QueryTransformOptions,\n ): Promise<ReadonlyArray<string>>;\n /**\n * Generate a single short hypothetical-answer passage to embed (HyDE),\n * or `null` when unavailable. The caller embeds the passage and fuses\n * its vector neighbours into the result.\n */\n hypothetical(query: string, options?: QueryTransformOptions): Promise<string | null>;\n}\n\n/**\n * Per-call options for a {@link QueryTransformer}.\n *\n * @stable\n */\nexport interface QueryTransformOptions {\n /** Cancellation signal forwarded to the underlying provider call. */\n readonly signal?: AbortSignal;\n}\n\n/**\n * System prompt for multi-query variant generation. Asks for a bare\n * JSON array of standalone rephrasings that preserve the original\n * intent — {@link parseQueryVariants} also tolerates a chatty model.\n *\n * @internal\n */\nexport const QUERY_EXPANSION_SYSTEM_PROMPT =\n 'You rewrite a search query into alternative phrasings to improve recall over a ' +\n 'personal memory store. Preserve the original intent; vary vocabulary, specificity, ' +\n 'and perspective so different wordings of the same stored memory are matched. ' +\n 'Return ONLY a JSON array of strings — each a standalone query, no numbering, no ' +\n 'commentary, no duplicates of the original.';\n\n/**\n * System prompt for HyDE. Asks for a short, plausible hypothetical\n * answer (not a question rephrase) whose embedding sits near the\n * passage that would actually answer the query.\n *\n * @internal\n */\nexport const HYDE_SYSTEM_PROMPT =\n 'Write a short, plausible hypothetical answer to the question, phrased as a ' +\n 'first-person statement of fact as it might be recorded in a memory store. One to ' +\n 'three sentences. Output only the passage — no preamble, no caveats, no markdown.';\n\n/**\n * Build the multi-query expansion request. Pure — no I/O. A higher\n * temperature is used deliberately so the variants diverge (the\n * downstream retrieval + RRF fusion stays deterministic).\n *\n * @stable\n */\nexport function buildExpansionRequest(\n query: string,\n count: number,\n options: { readonly maxTokens?: number; readonly signal?: AbortSignal } = {},\n): ProviderRequest {\n return {\n messages: [\n {\n role: 'user',\n content: `Original query: ${query}\\nReturn up to ${count} alternative phrasings as a JSON array of strings.`,\n },\n ],\n systemMessage: QUERY_EXPANSION_SYSTEM_PROMPT,\n temperature: 0.7,\n maxTokens: options.maxTokens ?? DEFAULT_TRANSFORM_MAX_TOKENS,\n ...(options.signal !== undefined ? { signal: options.signal } : {}),\n outputType: { kind: 'structured' },\n };\n}\n\n/**\n * Build the HyDE pseudo-document request. Pure — no I/O.\n *\n * @stable\n */\nexport function buildHydeRequest(\n query: string,\n options: { readonly maxTokens?: number; readonly signal?: AbortSignal } = {},\n): ProviderRequest {\n return {\n messages: [{ role: 'user', content: query }],\n systemMessage: HYDE_SYSTEM_PROMPT,\n temperature: 0.3,\n maxTokens: options.maxTokens ?? DEFAULT_TRANSFORM_MAX_TOKENS,\n ...(options.signal !== undefined ? { signal: options.signal } : {}),\n };\n}\n\n/**\n * Parse the variant-generation model output into a deduped, capped list\n * of reworded queries. Tolerates a JSON array, a `{ \"variants\": [...] }`\n * / `{ \"queries\": [...] }` wrapper, fenced blocks, and (as a last\n * resort) a newline / numbered list — so a chatty model never breaks\n * recall. Empty strings and case-insensitive duplicates are dropped;\n * the result is capped at `max`.\n *\n * @stable\n */\nexport function parseQueryVariants(text: string | undefined, max: number): ReadonlyArray<string> {\n if (max <= 0 || text === undefined) return [];\n const candidate = stripFence(text).trim();\n if (candidate.length === 0) return [];\n const raw = extractVariantStrings(candidate);\n const seen = new Set<string>();\n const out: string[] = [];\n for (const item of raw) {\n const trimmed = item.trim();\n if (trimmed.length === 0) continue;\n const key = trimmed.toLowerCase();\n if (seen.has(key)) continue;\n seen.add(key);\n out.push(trimmed);\n if (out.length >= max) break;\n }\n return out;\n}\n\n/**\n * Parse the HyDE model output into a single passage, or `null` when the\n * model returned nothing usable. Strips a fenced block and trims.\n *\n * @stable\n */\nexport function parseHypothetical(text: string | undefined): string | null {\n if (text === undefined) return null;\n const trimmed = stripFence(text).trim();\n return trimmed.length === 0 ? null : trimmed;\n}\n\n/**\n * Wrap a {@link Provider} as a {@link QueryTransformer}. Both methods\n * are **resilient**: a provider error or unparseable response degrades\n * to `[]` / `null` so transformation never throws into `search`. The\n * `maxVariants` ceiling caps the variants requested regardless of the\n * caller's `multiQuery` value (a latency guardrail).\n *\n * @stable\n */\nexport function createProviderQueryTransformer(\n provider: Provider,\n options: { readonly maxVariants?: number; readonly maxTokens?: number } = {},\n): QueryTransformer {\n const maxVariants = Math.max(0, options.maxVariants ?? DEFAULT_MAX_QUERY_VARIANTS);\n const maxTokens = options.maxTokens ?? DEFAULT_TRANSFORM_MAX_TOKENS;\n return {\n async expand(query, count, opts = {}): Promise<ReadonlyArray<string>> {\n const want = Math.min(Math.max(0, count), maxVariants);\n if (want === 0 || query.trim().length === 0) return [];\n try {\n const request = buildExpansionRequest(query, want, {\n maxTokens,\n ...(opts.signal !== undefined ? { signal: opts.signal } : {}),\n });\n const response = await provider.generate(request);\n return parseQueryVariants(response.text, want);\n } catch {\n return [];\n }\n },\n async hypothetical(query, opts = {}): Promise<string | null> {\n if (query.trim().length === 0) return null;\n try {\n const request = buildHydeRequest(query, {\n maxTokens,\n ...(opts.signal !== undefined ? { signal: opts.signal } : {}),\n });\n const response = await provider.generate(request);\n return parseHypothetical(response.text);\n } catch {\n return null;\n }\n },\n };\n}\n\n/** Extract the raw (pre-dedup) variant strings from tolerant model output. */\nfunction extractVariantStrings(candidate: string): ReadonlyArray<string> {\n const parsed = tryParseJson(candidate);\n if (parsed !== undefined) {\n const arr = Array.isArray(parsed) ? parsed : isRecord(parsed) ? pickArray(parsed) : null;\n if (arr !== null) {\n return arr.filter((v): v is string => typeof v === 'string');\n }\n }\n // Fallback — split a newline / numbered list and strip list markers.\n return candidate.split('\\n').map((line) =>\n line\n .replace(/^\\s*(?:[-*•]|\\d+[.)])\\s*/u, '')\n .replace(/^[\"']|[\"']$/gu, '')\n .trim(),\n );\n}\n\nfunction pickArray(record: Record<string, unknown>): ReadonlyArray<unknown> | null {\n if (Array.isArray(record.variants)) return record.variants;\n if (Array.isArray(record.queries)) return record.queries;\n return null;\n}\n\nfunction tryParseJson(text: string): unknown {\n try {\n return JSON.parse(text);\n } catch {\n const slice = sliceJson(text);\n if (slice === null) return undefined;\n try {\n return JSON.parse(slice);\n } catch {\n return undefined;\n }\n }\n}\n\n/** Slice the widest `[...]` array or `{...}` object out of chatty text. */\nfunction sliceJson(text: string): string | null {\n const arrStart = text.indexOf('[');\n const arrEnd = text.lastIndexOf(']');\n if (arrStart >= 0 && arrEnd > arrStart) return text.slice(arrStart, arrEnd + 1);\n const objStart = text.indexOf('{');\n const objEnd = text.lastIndexOf('}');\n if (objStart >= 0 && objEnd > objStart) return text.slice(objStart, objEnd + 1);\n return null;\n}\n\nfunction stripFence(text: string): string {\n const match = /^```[^\\n]*\\n([\\s\\S]*?)\\n```/u.exec(text.trim());\n return match?.[1] ?? text;\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n return value !== null && typeof value === 'object';\n}\n"],"mappings":";;AA2BA,MAAa,6BAA6B;;AAG1C,MAAM,+BAA+B;;;;;;;;AAmDrC,MAAa,gCACX;;;;;;;;AAaF,MAAa,qBACX;;;;;;;;AAWF,SAAgB,sBACd,OACA,OACA,UAA0E,EAAE,EAC3D;AACjB,QAAO;EACL,UAAU,CACR;GACE,MAAM;GACN,SAAS,mBAAmB,MAAM,iBAAiB,MAAM;GAC1D,CACF;EACD,eAAe;EACf,aAAa;EACb,WAAW,QAAQ,aAAa;EAChC,GAAI,QAAQ,WAAW,SAAY,EAAE,QAAQ,QAAQ,QAAQ,GAAG,EAAE;EAClE,YAAY,EAAE,MAAM,cAAc;EACnC;;;;;;;AAQH,SAAgB,iBACd,OACA,UAA0E,EAAE,EAC3D;AACjB,QAAO;EACL,UAAU,CAAC;GAAE,MAAM;GAAQ,SAAS;GAAO,CAAC;EAC5C,eAAe;EACf,aAAa;EACb,WAAW,QAAQ,aAAa;EAChC,GAAI,QAAQ,WAAW,SAAY,EAAE,QAAQ,QAAQ,QAAQ,GAAG,EAAE;EACnE;;;;;;;;;;;;AAaH,SAAgB,mBAAmB,MAA0B,KAAoC;AAC/F,KAAI,OAAO,KAAK,SAAS,OAAW,QAAO,EAAE;CAC7C,MAAM,YAAY,WAAW,KAAK,CAAC,MAAM;AACzC,KAAI,UAAU,WAAW,EAAG,QAAO,EAAE;CACrC,MAAM,MAAM,sBAAsB,UAAU;CAC5C,MAAM,uBAAO,IAAI,KAAa;CAC9B,MAAMA,MAAgB,EAAE;AACxB,MAAK,MAAM,QAAQ,KAAK;EACtB,MAAM,UAAU,KAAK,MAAM;AAC3B,MAAI,QAAQ,WAAW,EAAG;EAC1B,MAAM,MAAM,QAAQ,aAAa;AACjC,MAAI,KAAK,IAAI,IAAI,CAAE;AACnB,OAAK,IAAI,IAAI;AACb,MAAI,KAAK,QAAQ;AACjB,MAAI,IAAI,UAAU,IAAK;;AAEzB,QAAO;;;;;;;;AAST,SAAgB,kBAAkB,MAAyC;AACzE,KAAI,SAAS,OAAW,QAAO;CAC/B,MAAM,UAAU,WAAW,KAAK,CAAC,MAAM;AACvC,QAAO,QAAQ,WAAW,IAAI,OAAO;;;;;;;;;;;AAYvC,SAAgB,+BACd,UACA,UAA0E,EAAE,EAC1D;CAClB,MAAM,cAAc,KAAK,IAAI,GAAG,QAAQ,eAAe,2BAA2B;CAClF,MAAM,YAAY,QAAQ,aAAa;AACvC,QAAO;EACL,MAAM,OAAO,OAAO,OAAO,OAAO,EAAE,EAAkC;GACpE,MAAM,OAAO,KAAK,IAAI,KAAK,IAAI,GAAG,MAAM,EAAE,YAAY;AACtD,OAAI,SAAS,KAAK,MAAM,MAAM,CAAC,WAAW,EAAG,QAAO,EAAE;AACtD,OAAI;IACF,MAAM,UAAU,sBAAsB,OAAO,MAAM;KACjD;KACA,GAAI,KAAK,WAAW,SAAY,EAAE,QAAQ,KAAK,QAAQ,GAAG,EAAE;KAC7D,CAAC;AAEF,WAAO,oBADU,MAAM,SAAS,SAAS,QAAQ,EACd,MAAM,KAAK;WACxC;AACN,WAAO,EAAE;;;EAGb,MAAM,aAAa,OAAO,OAAO,EAAE,EAA0B;AAC3D,OAAI,MAAM,MAAM,CAAC,WAAW,EAAG,QAAO;AACtC,OAAI;IACF,MAAM,UAAU,iBAAiB,OAAO;KACtC;KACA,GAAI,KAAK,WAAW,SAAY,EAAE,QAAQ,KAAK,QAAQ,GAAG,EAAE;KAC7D,CAAC;AAEF,WAAO,mBADU,MAAM,SAAS,SAAS,QAAQ,EACf,KAAK;WACjC;AACN,WAAO;;;EAGZ;;;AAIH,SAAS,sBAAsB,WAA0C;CACvE,MAAM,SAAS,aAAa,UAAU;AACtC,KAAI,WAAW,QAAW;EACxB,MAAM,MAAM,MAAM,QAAQ,OAAO,GAAG,SAAS,SAAS,OAAO,GAAG,UAAU,OAAO,GAAG;AACpF,MAAI,QAAQ,KACV,QAAO,IAAI,QAAQ,MAAmB,OAAO,MAAM,SAAS;;AAIhE,QAAO,UAAU,MAAM,KAAK,CAAC,KAAK,SAChC,KACG,QAAQ,6BAA6B,GAAG,CACxC,QAAQ,iBAAiB,GAAG,CAC5B,MAAM,CACV;;AAGH,SAAS,UAAU,QAAgE;AACjF,KAAI,MAAM,QAAQ,OAAO,SAAS,CAAE,QAAO,OAAO;AAClD,KAAI,MAAM,QAAQ,OAAO,QAAQ,CAAE,QAAO,OAAO;AACjD,QAAO;;AAGT,SAAS,aAAa,MAAuB;AAC3C,KAAI;AACF,SAAO,KAAK,MAAM,KAAK;SACjB;EACN,MAAM,QAAQ,UAAU,KAAK;AAC7B,MAAI,UAAU,KAAM,QAAO;AAC3B,MAAI;AACF,UAAO,KAAK,MAAM,MAAM;UAClB;AACN;;;;;AAMN,SAAS,UAAU,MAA6B;CAC9C,MAAM,WAAW,KAAK,QAAQ,IAAI;CAClC,MAAM,SAAS,KAAK,YAAY,IAAI;AACpC,KAAI,YAAY,KAAK,SAAS,SAAU,QAAO,KAAK,MAAM,UAAU,SAAS,EAAE;CAC/E,MAAM,WAAW,KAAK,QAAQ,IAAI;CAClC,MAAM,SAAS,KAAK,YAAY,IAAI;AACpC,KAAI,YAAY,KAAK,SAAS,SAAU,QAAO,KAAK,MAAM,UAAU,SAAS,EAAE;AAC/E,QAAO;;AAGT,SAAS,WAAW,MAAsB;AAExC,QADc,+BAA+B,KAAK,KAAK,MAAM,CAAC,GAC/C,MAAM;;AAGvB,SAAS,SAAS,OAAkD;AAClE,QAAO,UAAU,QAAQ,OAAO,UAAU"}
|
|
1
|
+
{"version":3,"file":"query-transform.js","names":["out: string[]"],"sources":["../../src/search/query-transform.ts"],"sourcesContent":["/**\n * Query transformation for hybrid retrieval (P2-3) - **multi-query /\n * RAG-Fusion** and optional **HyDE**.\n *\n * Auto-recall is a regex trigger and the default search is single-shot:\n * the user's phrasing rarely matches how a memory was stored, so a\n * relevant fact is missed at the retrieval step. This module fans a\n * query into a few reworded variants (one cheap LLM call) and/or writes\n * a hypothetical answer passage to embed (HyDE, arXiv:2212.10496); the\n * caller retrieves each and fuses the lists through the existing RRF\n * reranker. Variant generation is the only thing that touches a\n * provider, so it is **opt-in**: with no transformer configured the\n * default search path stays fully offline and single-shot.\n *\n * The module is provider-agnostic - it imports only `@graphorin/core`\n * types and never performs I/O itself. The concrete `Provider` is\n * injected by the caller via {@link createProviderQueryTransformer},\n * which is resilient: any provider error or unparseable output degrades\n * to \"no variants\" / \"no hypothetical\" rather than throwing into the\n * search hot path.\n *\n * @packageDocumentation\n */\n\nimport type { Provider, ProviderRequest } from '@graphorin/core';\n\n/** Default ceiling on reworded variants a provider-backed transformer requests. */\nexport const DEFAULT_MAX_QUERY_VARIANTS = 5;\n\n/** Default output-token ceiling for a single transform call. */\nconst DEFAULT_TRANSFORM_MAX_TOKENS = 256;\n\n/**\n * Pluggable query-transformation seam consumed by\n * `SemanticMemory.search(..., { multiQuery, hyde })`. The built-in\n * provider-backed implementation lives in\n * {@link createProviderQueryTransformer}; advanced callers can supply a\n * bespoke one (e.g. a deterministic synonym expander) to\n * `new SemanticMemory({ queryTransformer })`.\n *\n * Implementations MUST degrade gracefully - return `[]` / `null` rather\n * than throw - so a transform failure never breaks recall.\n *\n * @stable\n */\nexport interface QueryTransformer {\n /**\n * Rewrite a query into up to `count` **additional** reworded variants\n * (the original query is retained by the caller). Returns `[]` when\n * transformation is unavailable or the model returns nothing usable.\n */\n expand(\n query: string,\n count: number,\n options?: QueryTransformOptions,\n ): Promise<ReadonlyArray<string>>;\n /**\n * Generate a single short hypothetical-answer passage to embed (HyDE),\n * or `null` when unavailable. The caller embeds the passage and fuses\n * its vector neighbours into the result.\n */\n hypothetical(query: string, options?: QueryTransformOptions): Promise<string | null>;\n}\n\n/**\n * Per-call options for a {@link QueryTransformer}.\n *\n * @stable\n */\nexport interface QueryTransformOptions {\n /** Cancellation signal forwarded to the underlying provider call. */\n readonly signal?: AbortSignal;\n}\n\n/**\n * System prompt for multi-query variant generation. Asks for a bare\n * JSON array of standalone rephrasings that preserve the original\n * intent - {@link parseQueryVariants} also tolerates a chatty model.\n *\n * @internal\n */\nexport const QUERY_EXPANSION_SYSTEM_PROMPT =\n 'You rewrite a search query into alternative phrasings to improve recall over a ' +\n 'personal memory store. Preserve the original intent; vary vocabulary, specificity, ' +\n 'and perspective so different wordings of the same stored memory are matched. ' +\n 'Return ONLY a JSON array of strings - each a standalone query, no numbering, no ' +\n 'commentary, no duplicates of the original.';\n\n/**\n * System prompt for HyDE. Asks for a short, plausible hypothetical\n * answer (not a question rephrase) whose embedding sits near the\n * passage that would actually answer the query.\n *\n * @internal\n */\nexport const HYDE_SYSTEM_PROMPT =\n 'Write a short, plausible hypothetical answer to the question, phrased as a ' +\n 'first-person statement of fact as it might be recorded in a memory store. One to ' +\n 'three sentences. Output only the passage - no preamble, no caveats, no markdown.';\n\n/**\n * Build the multi-query expansion request. Pure - no I/O. A higher\n * temperature is used deliberately so the variants diverge (the\n * downstream retrieval + RRF fusion stays deterministic).\n *\n * @stable\n */\nexport function buildExpansionRequest(\n query: string,\n count: number,\n options: { readonly maxTokens?: number; readonly signal?: AbortSignal } = {},\n): ProviderRequest {\n return {\n messages: [\n {\n role: 'user',\n content: `Original query: ${query}\\nReturn up to ${count} alternative phrasings as a JSON array of strings.`,\n },\n ],\n systemMessage: QUERY_EXPANSION_SYSTEM_PROMPT,\n temperature: 0.7,\n maxTokens: options.maxTokens ?? DEFAULT_TRANSFORM_MAX_TOKENS,\n ...(options.signal !== undefined ? { signal: options.signal } : {}),\n outputType: { kind: 'structured' },\n };\n}\n\n/**\n * Build the HyDE pseudo-document request. Pure - no I/O.\n *\n * @stable\n */\nexport function buildHydeRequest(\n query: string,\n options: { readonly maxTokens?: number; readonly signal?: AbortSignal } = {},\n): ProviderRequest {\n return {\n messages: [{ role: 'user', content: query }],\n systemMessage: HYDE_SYSTEM_PROMPT,\n temperature: 0.3,\n maxTokens: options.maxTokens ?? DEFAULT_TRANSFORM_MAX_TOKENS,\n ...(options.signal !== undefined ? { signal: options.signal } : {}),\n };\n}\n\n/**\n * Parse the variant-generation model output into a deduped, capped list\n * of reworded queries. Tolerates a JSON array, a `{ \"variants\": [...] }`\n * / `{ \"queries\": [...] }` wrapper, fenced blocks, and (as a last\n * resort) a newline / numbered list - so a chatty model never breaks\n * recall. Empty strings and case-insensitive duplicates are dropped;\n * the result is capped at `max`.\n *\n * @stable\n */\nexport function parseQueryVariants(text: string | undefined, max: number): ReadonlyArray<string> {\n if (max <= 0 || text === undefined) return [];\n const candidate = stripFence(text).trim();\n if (candidate.length === 0) return [];\n const raw = extractVariantStrings(candidate);\n const seen = new Set<string>();\n const out: string[] = [];\n for (const item of raw) {\n const trimmed = item.trim();\n if (trimmed.length === 0) continue;\n const key = trimmed.toLowerCase();\n if (seen.has(key)) continue;\n seen.add(key);\n out.push(trimmed);\n if (out.length >= max) break;\n }\n return out;\n}\n\n/**\n * Parse the HyDE model output into a single passage, or `null` when the\n * model returned nothing usable. Strips a fenced block and trims.\n *\n * @stable\n */\nexport function parseHypothetical(text: string | undefined): string | null {\n if (text === undefined) return null;\n const trimmed = stripFence(text).trim();\n return trimmed.length === 0 ? null : trimmed;\n}\n\n/**\n * Wrap a {@link Provider} as a {@link QueryTransformer}. Both methods\n * are **resilient**: a provider error or unparseable response degrades\n * to `[]` / `null` so transformation never throws into `search`. The\n * `maxVariants` ceiling caps the variants requested regardless of the\n * caller's `multiQuery` value (a latency guardrail).\n *\n * @stable\n */\nexport function createProviderQueryTransformer(\n provider: Provider,\n options: { readonly maxVariants?: number; readonly maxTokens?: number } = {},\n): QueryTransformer {\n const maxVariants = Math.max(0, options.maxVariants ?? DEFAULT_MAX_QUERY_VARIANTS);\n const maxTokens = options.maxTokens ?? DEFAULT_TRANSFORM_MAX_TOKENS;\n return {\n async expand(query, count, opts = {}): Promise<ReadonlyArray<string>> {\n const want = Math.min(Math.max(0, count), maxVariants);\n if (want === 0 || query.trim().length === 0) return [];\n try {\n const request = buildExpansionRequest(query, want, {\n maxTokens,\n ...(opts.signal !== undefined ? { signal: opts.signal } : {}),\n });\n const response = await provider.generate(request);\n return parseQueryVariants(response.text, want);\n } catch {\n return [];\n }\n },\n async hypothetical(query, opts = {}): Promise<string | null> {\n if (query.trim().length === 0) return null;\n try {\n const request = buildHydeRequest(query, {\n maxTokens,\n ...(opts.signal !== undefined ? { signal: opts.signal } : {}),\n });\n const response = await provider.generate(request);\n return parseHypothetical(response.text);\n } catch {\n return null;\n }\n },\n };\n}\n\n/** Extract the raw (pre-dedup) variant strings from tolerant model output. */\nfunction extractVariantStrings(candidate: string): ReadonlyArray<string> {\n const parsed = tryParseJson(candidate);\n if (parsed !== undefined) {\n const arr = Array.isArray(parsed) ? parsed : isRecord(parsed) ? pickArray(parsed) : null;\n if (arr !== null) {\n return arr.filter((v): v is string => typeof v === 'string');\n }\n }\n // Fallback - split a newline / numbered list and strip list markers.\n return candidate.split('\\n').map((line) =>\n line\n .replace(/^\\s*(?:[-*•]|\\d+[.)])\\s*/u, '')\n .replace(/^[\"']|[\"']$/gu, '')\n .trim(),\n );\n}\n\nfunction pickArray(record: Record<string, unknown>): ReadonlyArray<unknown> | null {\n if (Array.isArray(record.variants)) return record.variants;\n if (Array.isArray(record.queries)) return record.queries;\n return null;\n}\n\nfunction tryParseJson(text: string): unknown {\n try {\n return JSON.parse(text);\n } catch {\n const slice = sliceJson(text);\n if (slice === null) return undefined;\n try {\n return JSON.parse(slice);\n } catch {\n return undefined;\n }\n }\n}\n\n/** Slice the widest `[...]` array or `{...}` object out of chatty text. */\nfunction sliceJson(text: string): string | null {\n const arrStart = text.indexOf('[');\n const arrEnd = text.lastIndexOf(']');\n if (arrStart >= 0 && arrEnd > arrStart) return text.slice(arrStart, arrEnd + 1);\n const objStart = text.indexOf('{');\n const objEnd = text.lastIndexOf('}');\n if (objStart >= 0 && objEnd > objStart) return text.slice(objStart, objEnd + 1);\n return null;\n}\n\nfunction stripFence(text: string): string {\n const match = /^```[^\\n]*\\n([\\s\\S]*?)\\n```/u.exec(text.trim());\n return match?.[1] ?? text;\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n return value !== null && typeof value === 'object';\n}\n"],"mappings":";;AA2BA,MAAa,6BAA6B;;AAG1C,MAAM,+BAA+B;;;;;;;;AAmDrC,MAAa,gCACX;;;;;;;;AAaF,MAAa,qBACX;;;;;;;;AAWF,SAAgB,sBACd,OACA,OACA,UAA0E,EAAE,EAC3D;AACjB,QAAO;EACL,UAAU,CACR;GACE,MAAM;GACN,SAAS,mBAAmB,MAAM,iBAAiB,MAAM;GAC1D,CACF;EACD,eAAe;EACf,aAAa;EACb,WAAW,QAAQ,aAAa;EAChC,GAAI,QAAQ,WAAW,SAAY,EAAE,QAAQ,QAAQ,QAAQ,GAAG,EAAE;EAClE,YAAY,EAAE,MAAM,cAAc;EACnC;;;;;;;AAQH,SAAgB,iBACd,OACA,UAA0E,EAAE,EAC3D;AACjB,QAAO;EACL,UAAU,CAAC;GAAE,MAAM;GAAQ,SAAS;GAAO,CAAC;EAC5C,eAAe;EACf,aAAa;EACb,WAAW,QAAQ,aAAa;EAChC,GAAI,QAAQ,WAAW,SAAY,EAAE,QAAQ,QAAQ,QAAQ,GAAG,EAAE;EACnE;;;;;;;;;;;;AAaH,SAAgB,mBAAmB,MAA0B,KAAoC;AAC/F,KAAI,OAAO,KAAK,SAAS,OAAW,QAAO,EAAE;CAC7C,MAAM,YAAY,WAAW,KAAK,CAAC,MAAM;AACzC,KAAI,UAAU,WAAW,EAAG,QAAO,EAAE;CACrC,MAAM,MAAM,sBAAsB,UAAU;CAC5C,MAAM,uBAAO,IAAI,KAAa;CAC9B,MAAMA,MAAgB,EAAE;AACxB,MAAK,MAAM,QAAQ,KAAK;EACtB,MAAM,UAAU,KAAK,MAAM;AAC3B,MAAI,QAAQ,WAAW,EAAG;EAC1B,MAAM,MAAM,QAAQ,aAAa;AACjC,MAAI,KAAK,IAAI,IAAI,CAAE;AACnB,OAAK,IAAI,IAAI;AACb,MAAI,KAAK,QAAQ;AACjB,MAAI,IAAI,UAAU,IAAK;;AAEzB,QAAO;;;;;;;;AAST,SAAgB,kBAAkB,MAAyC;AACzE,KAAI,SAAS,OAAW,QAAO;CAC/B,MAAM,UAAU,WAAW,KAAK,CAAC,MAAM;AACvC,QAAO,QAAQ,WAAW,IAAI,OAAO;;;;;;;;;;;AAYvC,SAAgB,+BACd,UACA,UAA0E,EAAE,EAC1D;CAClB,MAAM,cAAc,KAAK,IAAI,GAAG,QAAQ,eAAe,2BAA2B;CAClF,MAAM,YAAY,QAAQ,aAAa;AACvC,QAAO;EACL,MAAM,OAAO,OAAO,OAAO,OAAO,EAAE,EAAkC;GACpE,MAAM,OAAO,KAAK,IAAI,KAAK,IAAI,GAAG,MAAM,EAAE,YAAY;AACtD,OAAI,SAAS,KAAK,MAAM,MAAM,CAAC,WAAW,EAAG,QAAO,EAAE;AACtD,OAAI;IACF,MAAM,UAAU,sBAAsB,OAAO,MAAM;KACjD;KACA,GAAI,KAAK,WAAW,SAAY,EAAE,QAAQ,KAAK,QAAQ,GAAG,EAAE;KAC7D,CAAC;AAEF,WAAO,oBADU,MAAM,SAAS,SAAS,QAAQ,EACd,MAAM,KAAK;WACxC;AACN,WAAO,EAAE;;;EAGb,MAAM,aAAa,OAAO,OAAO,EAAE,EAA0B;AAC3D,OAAI,MAAM,MAAM,CAAC,WAAW,EAAG,QAAO;AACtC,OAAI;IACF,MAAM,UAAU,iBAAiB,OAAO;KACtC;KACA,GAAI,KAAK,WAAW,SAAY,EAAE,QAAQ,KAAK,QAAQ,GAAG,EAAE;KAC7D,CAAC;AAEF,WAAO,mBADU,MAAM,SAAS,SAAS,QAAQ,EACf,KAAK;WACjC;AACN,WAAO;;;EAGZ;;;AAIH,SAAS,sBAAsB,WAA0C;CACvE,MAAM,SAAS,aAAa,UAAU;AACtC,KAAI,WAAW,QAAW;EACxB,MAAM,MAAM,MAAM,QAAQ,OAAO,GAAG,SAAS,SAAS,OAAO,GAAG,UAAU,OAAO,GAAG;AACpF,MAAI,QAAQ,KACV,QAAO,IAAI,QAAQ,MAAmB,OAAO,MAAM,SAAS;;AAIhE,QAAO,UAAU,MAAM,KAAK,CAAC,KAAK,SAChC,KACG,QAAQ,6BAA6B,GAAG,CACxC,QAAQ,iBAAiB,GAAG,CAC5B,MAAM,CACV;;AAGH,SAAS,UAAU,QAAgE;AACjF,KAAI,MAAM,QAAQ,OAAO,SAAS,CAAE,QAAO,OAAO;AAClD,KAAI,MAAM,QAAQ,OAAO,QAAQ,CAAE,QAAO,OAAO;AACjD,QAAO;;AAGT,SAAS,aAAa,MAAuB;AAC3C,KAAI;AACF,SAAO,KAAK,MAAM,KAAK;SACjB;EACN,MAAM,QAAQ,UAAU,KAAK;AAC7B,MAAI,UAAU,KAAM,QAAO;AAC3B,MAAI;AACF,UAAO,KAAK,MAAM,MAAM;UAClB;AACN;;;;;AAMN,SAAS,UAAU,MAA6B;CAC9C,MAAM,WAAW,KAAK,QAAQ,IAAI;CAClC,MAAM,SAAS,KAAK,YAAY,IAAI;AACpC,KAAI,YAAY,KAAK,SAAS,SAAU,QAAO,KAAK,MAAM,UAAU,SAAS,EAAE;CAC/E,MAAM,WAAW,KAAK,QAAQ,IAAI;CAClC,MAAM,SAAS,KAAK,YAAY,IAAI;AACpC,KAAI,YAAY,KAAK,SAAS,SAAU,QAAO,KAAK,MAAM,UAAU,SAAS,EAAE;AAC/E,QAAO;;AAGT,SAAS,WAAW,MAAsB;AAExC,QADc,+BAA+B,KAAK,KAAK,MAAM,CAAC,GAC/C,MAAM;;AAGvB,SAAS,SAAS,OAAkD;AAClE,QAAO,UAAU,QAAQ,OAAO,UAAU"}
|
package/dist/search/rrf.d.ts
CHANGED
|
@@ -38,7 +38,7 @@ declare class RRFReranker implements ReRanker {
|
|
|
38
38
|
* Weighted-RRF reranker (X-2). Fuses parallel ranked lists through
|
|
39
39
|
* {@link fuseWeighted}, applying a per-list `weights[i]` so a caller who
|
|
40
40
|
* has calibrated list reliability against labels (the P0-1 eval harness)
|
|
41
|
-
* can trust one retriever over another
|
|
41
|
+
* can trust one retriever over another - e.g. up-weight dense vector
|
|
42
42
|
* hits over lexical FTS hits. `weights` is **positional**: aligned to the
|
|
43
43
|
* input `lists` order (the built-in hybrid search passes FTS first, then
|
|
44
44
|
* vector). At equal weights it is identical to {@link RRFReranker}; RRF
|
|
@@ -60,7 +60,7 @@ declare class WeightedRRFReranker implements ReRanker {
|
|
|
60
60
|
rerank<TRecord extends MemoryRecord>(query: string, lists: ReadonlyArray<ReadonlyArray<MemoryHit<TRecord>>>, options?: ReRankOptions): Promise<ReadonlyArray<MemoryHit<TRecord>>>;
|
|
61
61
|
}
|
|
62
62
|
/**
|
|
63
|
-
* Pure functional core of the RRF reranker
|
|
63
|
+
* Pure functional core of the RRF reranker - the equal-weight case of
|
|
64
64
|
* {@link fuseWeighted}. Exported separately so the test suite (and the
|
|
65
65
|
* property-based fuzzer) can exercise the math without the `Promise<…>`
|
|
66
66
|
* wrapping of the public surface.
|
|
@@ -81,9 +81,9 @@ declare function fuseRrf<TRecord extends MemoryRecord>(lists: ReadonlyArray<Read
|
|
|
81
81
|
*
|
|
82
82
|
* Like {@link fuseRrf} the fusion is deterministic, tie-broken by
|
|
83
83
|
* first-seen order, and preserves any upstream `signals` (FTS `bm25`,
|
|
84
|
-
* vector similarity) the hits carried in
|
|
84
|
+
* vector similarity) the hits carried in - the `rrf` / `rrf.list_N`
|
|
85
85
|
* signals it records are the *weighted* contributions, so the fused
|
|
86
|
-
* `score` still equals their sum. Note that
|
|
86
|
+
* `score` still equals their sum. Note that - unlike RRF - the result
|
|
87
87
|
* depends on input list *order*, because each weight is bound to a list
|
|
88
88
|
* position.
|
|
89
89
|
*
|
package/dist/search/rrf.js
CHANGED
|
@@ -42,7 +42,7 @@ var RRFReranker = class {
|
|
|
42
42
|
* Weighted-RRF reranker (X-2). Fuses parallel ranked lists through
|
|
43
43
|
* {@link fuseWeighted}, applying a per-list `weights[i]` so a caller who
|
|
44
44
|
* has calibrated list reliability against labels (the P0-1 eval harness)
|
|
45
|
-
* can trust one retriever over another
|
|
45
|
+
* can trust one retriever over another - e.g. up-weight dense vector
|
|
46
46
|
* hits over lexical FTS hits. `weights` is **positional**: aligned to the
|
|
47
47
|
* input `lists` order (the built-in hybrid search passes FTS first, then
|
|
48
48
|
* vector). At equal weights it is identical to {@link RRFReranker}; RRF
|
|
@@ -72,7 +72,7 @@ var WeightedRRFReranker = class {
|
|
|
72
72
|
}
|
|
73
73
|
};
|
|
74
74
|
/**
|
|
75
|
-
* Pure functional core of the RRF reranker
|
|
75
|
+
* Pure functional core of the RRF reranker - the equal-weight case of
|
|
76
76
|
* {@link fuseWeighted}. Exported separately so the test suite (and the
|
|
77
77
|
* property-based fuzzer) can exercise the math without the `Promise<…>`
|
|
78
78
|
* wrapping of the public surface.
|
|
@@ -95,9 +95,9 @@ function fuseRrf(lists, k, labels) {
|
|
|
95
95
|
*
|
|
96
96
|
* Like {@link fuseRrf} the fusion is deterministic, tie-broken by
|
|
97
97
|
* first-seen order, and preserves any upstream `signals` (FTS `bm25`,
|
|
98
|
-
* vector similarity) the hits carried in
|
|
98
|
+
* vector similarity) the hits carried in - the `rrf` / `rrf.list_N`
|
|
99
99
|
* signals it records are the *weighted* contributions, so the fused
|
|
100
|
-
* `score` still equals their sum. Note that
|
|
100
|
+
* `score` still equals their sum. Note that - unlike RRF - the result
|
|
101
101
|
* depends on input list *order*, because each weight is bound to a list
|
|
102
102
|
* position.
|
|
103
103
|
*
|