@graphorin/memory 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +83 -0
- package/README.md +24 -7
- package/dist/conflict/index.js +0 -1
- package/dist/conflict/pipeline.js +0 -1
- package/dist/conflict/pipeline.js.map +1 -1
- package/dist/conflict/stages/stage3-heuristic-regex.js +0 -1
- package/dist/conflict/stages/stage3-heuristic-regex.js.map +1 -1
- package/dist/conflict/types.d.ts +1 -1
- package/dist/conflict/types.js.map +1 -1
- package/dist/consolidator/index.js +2 -0
- package/dist/consolidator/index.js.map +1 -1
- package/dist/consolidator/lock.js +7 -1
- package/dist/consolidator/lock.js.map +1 -1
- package/dist/consolidator/phases/deep.js +19 -12
- package/dist/consolidator/phases/deep.js.map +1 -1
- package/dist/consolidator/phases/induce.d.ts.map +1 -1
- package/dist/consolidator/phases/induce.js +2 -3
- package/dist/consolidator/phases/induce.js.map +1 -1
- package/dist/consolidator/phases/reflect.js +35 -8
- package/dist/consolidator/phases/reflect.js.map +1 -1
- package/dist/consolidator/phases/standard.js +199 -28
- package/dist/consolidator/phases/standard.js.map +1 -1
- package/dist/consolidator/reconcile.js +16 -15
- package/dist/consolidator/reconcile.js.map +1 -1
- package/dist/consolidator/runtime.d.ts.map +1 -1
- package/dist/consolidator/runtime.js +129 -47
- package/dist/consolidator/runtime.js.map +1 -1
- package/dist/consolidator/scheduler.d.ts +17 -3
- package/dist/consolidator/scheduler.d.ts.map +1 -1
- package/dist/consolidator/scheduler.js.map +1 -1
- package/dist/consolidator/types.d.ts +30 -1
- package/dist/consolidator/types.d.ts.map +1 -1
- package/dist/consolidator/types.js +10 -0
- package/dist/consolidator/types.js.map +1 -1
- package/dist/context-engine/compaction/clear-tool-results.d.ts +1 -1
- package/dist/context-engine/compaction/clear-tool-results.js +1 -1
- package/dist/context-engine/compaction/clear-tool-results.js.map +1 -1
- package/dist/context-engine/compaction/compactor.d.ts +1 -0
- package/dist/context-engine/compaction/compactor.d.ts.map +1 -1
- package/dist/context-engine/compaction/compactor.js +32 -8
- package/dist/context-engine/compaction/compactor.js.map +1 -1
- package/dist/context-engine/compaction/hooks/types.d.ts +1 -1
- package/dist/context-engine/compaction/index.d.ts +1 -0
- package/dist/context-engine/compaction/templates/marker.d.ts +33 -0
- package/dist/context-engine/compaction/templates/marker.d.ts.map +1 -0
- package/dist/context-engine/compaction/templates/marker.js +34 -0
- package/dist/context-engine/compaction/templates/marker.js.map +1 -0
- package/dist/context-engine/compaction/templates/summary-9-section.d.ts +1 -1
- package/dist/context-engine/compaction/templates/summary-9-section.d.ts.map +1 -1
- package/dist/context-engine/compaction/templates/summary-9-section.js +4 -3
- package/dist/context-engine/compaction/templates/summary-9-section.js.map +1 -1
- package/dist/context-engine/engine.d.ts +7 -176
- package/dist/context-engine/engine.d.ts.map +1 -1
- package/dist/context-engine/engine.js +0 -2
- package/dist/context-engine/engine.js.map +1 -1
- package/dist/context-engine/index.d.ts +8 -49
- package/dist/context-engine/io-types.d.ts +144 -0
- package/dist/context-engine/io-types.d.ts.map +1 -0
- package/dist/context-engine/locale-packs/types.d.ts +1 -1
- package/dist/context-engine/locale-packs/types.js.map +1 -1
- package/dist/context-engine/privacy-filter.d.ts +1 -2
- package/dist/context-engine/privacy-filter.d.ts.map +1 -1
- package/dist/context-engine/privacy-filter.js +1 -2
- package/dist/context-engine/privacy-filter.js.map +1 -1
- package/dist/errors/index.d.ts +2 -2
- package/dist/errors/index.js +2 -2
- package/dist/errors/index.js.map +1 -1
- package/dist/facade.d.ts +43 -52
- package/dist/facade.d.ts.map +1 -1
- package/dist/facade.js +5 -3
- package/dist/facade.js.map +1 -1
- package/dist/graph/entity-resolver.js +1 -1
- package/dist/graph/entity-resolver.js.map +1 -1
- package/dist/index.d.ts +17 -15
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -7
- package/dist/index.js.map +1 -1
- package/dist/internal/envelope.js +57 -0
- package/dist/internal/envelope.js.map +1 -0
- package/dist/internal/injection-heuristics.js +0 -0
- package/dist/internal/injection-heuristics.js.map +1 -1
- package/dist/internal/llm-json.js +40 -0
- package/dist/internal/llm-json.js.map +1 -0
- package/dist/internal/storage-adapter.d.ts +23 -10
- package/dist/internal/storage-adapter.d.ts.map +1 -1
- package/dist/memory-interface.d.ts +131 -0
- package/dist/memory-interface.d.ts.map +1 -0
- package/dist/package.js +6 -0
- package/dist/package.js.map +1 -0
- package/dist/search/iterative.d.ts.map +1 -1
- package/dist/search/iterative.js +2 -3
- package/dist/search/iterative.js.map +1 -1
- package/dist/search/query-transform.d.ts.map +1 -1
- package/dist/search/query-transform.js +2 -3
- package/dist/search/query-transform.js.map +1 -1
- package/dist/tiers/episodic-memory.js +3 -3
- package/dist/tiers/episodic-memory.js.map +1 -1
- package/dist/tiers/index.d.ts +2 -2
- package/dist/tiers/insight-memory.js +1 -1
- package/dist/tiers/insight-memory.js.map +1 -1
- package/dist/tiers/procedural-memory.d.ts +1 -1
- package/dist/tiers/procedural-memory.js +2 -2
- package/dist/tiers/procedural-memory.js.map +1 -1
- package/dist/tiers/semantic-memory.d.ts +58 -4
- package/dist/tiers/semantic-memory.d.ts.map +1 -1
- package/dist/tiers/semantic-memory.js +103 -20
- package/dist/tiers/semantic-memory.js.map +1 -1
- package/dist/tools/block-tools.d.ts +19 -57
- package/dist/tools/block-tools.d.ts.map +1 -1
- package/dist/tools/block-tools.js.map +1 -1
- package/dist/tools/fact-tools.d.ts +53 -214
- package/dist/tools/fact-tools.d.ts.map +1 -1
- package/dist/tools/fact-tools.js +3 -2
- package/dist/tools/fact-tools.js.map +1 -1
- package/dist/tools/recall-tools.d.ts +34 -175
- package/dist/tools/recall-tools.d.ts.map +1 -1
- package/dist/tools/recall-tools.js +5 -4
- package/dist/tools/recall-tools.js.map +1 -1
- package/dist/tools/runbook-tools.d.ts +15 -58
- package/dist/tools/runbook-tools.d.ts.map +1 -1
- package/dist/tools/runbook-tools.js.map +1 -1
- package/dist/tools/types.d.ts +1 -1
- package/package.json +18 -17
- package/src/conflict/index.ts +29 -0
- package/src/conflict/locale-packs/en.ts +142 -0
- package/src/conflict/locale-packs/index.ts +17 -0
- package/src/conflict/locale-packs/types.ts +138 -0
- package/src/conflict/pipeline.ts +371 -0
- package/src/conflict/stages/helpers.ts +65 -0
- package/src/conflict/stages/stage1-exact-dedup.ts +40 -0
- package/src/conflict/stages/stage2-embedding-three-zone.ts +54 -0
- package/src/conflict/stages/stage3-heuristic-regex.ts +51 -0
- package/src/conflict/stages/stage4-subject-predicate.ts +46 -0
- package/src/conflict/stages/stage5-defer-to-deep.ts +34 -0
- package/src/conflict/types.ts +263 -0
- package/src/consolidator/budget.ts +312 -0
- package/src/consolidator/decay.ts +246 -0
- package/src/consolidator/dlq.ts +61 -0
- package/src/consolidator/errors.ts +87 -0
- package/src/consolidator/idempotency.ts +27 -0
- package/src/consolidator/index.ts +269 -0
- package/src/consolidator/lock.ts +141 -0
- package/src/consolidator/noise-filter.ts +235 -0
- package/src/consolidator/phases/deep.ts +308 -0
- package/src/consolidator/phases/induce.ts +423 -0
- package/src/consolidator/phases/learned-context.ts +233 -0
- package/src/consolidator/phases/light.ts +166 -0
- package/src/consolidator/phases/reflect.ts +504 -0
- package/src/consolidator/phases/standard.ts +1002 -0
- package/src/consolidator/reconcile.ts +273 -0
- package/src/consolidator/runtime.ts +1049 -0
- package/src/consolidator/scheduler.ts +182 -0
- package/src/consolidator/triggers.ts +134 -0
- package/src/consolidator/types.ts +622 -0
- package/src/context-engine/annotations.ts +161 -0
- package/src/context-engine/auto-recall.ts +92 -0
- package/src/context-engine/compaction/clear-tool-results.ts +232 -0
- package/src/context-engine/compaction/compactor.ts +403 -0
- package/src/context-engine/compaction/hooks/reanchor-persona-block.ts +55 -0
- package/src/context-engine/compaction/hooks/reanchor-pinned-facts.ts +66 -0
- package/src/context-engine/compaction/hooks/reanchor-project-rules.ts +57 -0
- package/src/context-engine/compaction/hooks/reanchor-recent-results.ts +116 -0
- package/src/context-engine/compaction/hooks/types.ts +68 -0
- package/src/context-engine/compaction/index.ts +60 -0
- package/src/context-engine/compaction/templates/marker.ts +33 -0
- package/src/context-engine/compaction/templates/summary-9-section.ts +223 -0
- package/src/context-engine/compaction/thresholds.ts +107 -0
- package/src/context-engine/compaction/types.ts +255 -0
- package/src/context-engine/engine.ts +970 -0
- package/src/context-engine/index.ts +118 -0
- package/src/context-engine/io-types.ts +162 -0
- package/src/context-engine/locale-packs/en.ts +107 -0
- package/src/context-engine/locale-packs/index.ts +19 -0
- package/src/context-engine/locale-packs/resolver.ts +197 -0
- package/src/context-engine/locale-packs/types.ts +177 -0
- package/src/context-engine/metadata.ts +114 -0
- package/src/context-engine/preambles/inbound-en.ts +19 -0
- package/src/context-engine/privacy-filter.ts +190 -0
- package/src/context-engine/templates/base-en.ts +19 -0
- package/src/context-engine/templates/composer.ts +91 -0
- package/src/context-engine/token-budget.ts +319 -0
- package/src/context-engine/token-counter.ts +157 -0
- package/src/errors/index.ts +273 -0
- package/src/facade.ts +750 -0
- package/src/graph/entity-resolver.ts +364 -0
- package/src/graph/index.ts +24 -0
- package/src/index.ts +259 -0
- package/src/internal/contextualize.ts +173 -0
- package/src/internal/embedder-binding.ts +53 -0
- package/src/internal/envelope.ts +70 -0
- package/src/internal/id.ts +23 -0
- package/src/internal/injection-heuristics.ts +0 -0
- package/src/internal/llm-json.ts +38 -0
- package/src/internal/spans.ts +35 -0
- package/src/internal/storage-adapter.ts +963 -0
- package/src/memory-interface.ts +158 -0
- package/src/migration/embedder-migration.ts +237 -0
- package/src/migration/index.ts +13 -0
- package/src/search/explain.ts +123 -0
- package/src/search/fit-weights.ts +109 -0
- package/src/search/graph-ppr.ts +38 -0
- package/src/search/index.ts +55 -0
- package/src/search/iterative.ts +658 -0
- package/src/search/query-transform.ts +285 -0
- package/src/search/rrf.ts +225 -0
- package/src/search/trust.ts +55 -0
- package/src/search/types.ts +50 -0
- package/src/tiers/episodic-memory.ts +463 -0
- package/src/tiers/index.ts +50 -0
- package/src/tiers/insight-memory.ts +208 -0
- package/src/tiers/procedural-memory.ts +482 -0
- package/src/tiers/semantic-memory.ts +1729 -0
- package/src/tiers/session-memory.ts +271 -0
- package/src/tiers/shared-memory.ts +62 -0
- package/src/tiers/working-memory.ts +368 -0
- package/src/tools/block-tools.ts +172 -0
- package/src/tools/fact-tools.ts +473 -0
- package/src/tools/index.ts +109 -0
- package/src/tools/recall-tools.ts +303 -0
- package/src/tools/runbook-tools.ts +107 -0
- package/src/tools/taint.ts +39 -0
- package/src/tools/types.ts +37 -0
- package/dist/conflict/locale-packs/index.js +0 -4
- package/dist/context-engine/compaction/index.js +0 -10
- package/dist/context-engine/index.d.ts.map +0 -1
- package/dist/context-engine/index.js +0 -25
- package/dist/context-engine/locale-packs/index.js +0 -4
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The mutually-recursive `Memory` + `ContextEngine` interface pair
|
|
3
|
+
* (issue #22). `ContextEngine.assemble` / `compactNow` consume the
|
|
4
|
+
* full `Memory` facade, and `Memory` exposes its `contextEngine` -
|
|
5
|
+
* splitting the two across `facade.ts` and `context-engine/engine.ts`
|
|
6
|
+
* put a type-only cycle into the module graph (facade -> engine ->
|
|
7
|
+
* facade, plus every compaction hook that names `Memory`). Co-locating
|
|
8
|
+
* the pair in this leaf module (importing only IO types, tier
|
|
9
|
+
* surfaces, and other leaves) makes the graph acyclic.
|
|
10
|
+
*
|
|
11
|
+
* Both interfaces are re-exported from their original homes
|
|
12
|
+
* (`facade.ts` and the context-engine barrel) - no public import path
|
|
13
|
+
* changed.
|
|
14
|
+
*
|
|
15
|
+
* @packageDocumentation
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import type {
|
|
19
|
+
EmbedderProvider,
|
|
20
|
+
MemoryMetadata,
|
|
21
|
+
Message,
|
|
22
|
+
MessageContent,
|
|
23
|
+
SessionScope,
|
|
24
|
+
Tool,
|
|
25
|
+
} from '@graphorin/core';
|
|
26
|
+
import type { ConflictPipeline } from './conflict/index.js';
|
|
27
|
+
import type { Consolidator } from './consolidator/runtime.js';
|
|
28
|
+
import type {
|
|
29
|
+
CompactionResult,
|
|
30
|
+
CompactionSource,
|
|
31
|
+
CompactionSummarizer,
|
|
32
|
+
} from './context-engine/compaction/types.js';
|
|
33
|
+
import type {
|
|
34
|
+
AssembledPrompt,
|
|
35
|
+
AssembleInput,
|
|
36
|
+
CompileOptions,
|
|
37
|
+
CompileScope,
|
|
38
|
+
MemoryContextBlocks,
|
|
39
|
+
ResolvedContextEngineConfig,
|
|
40
|
+
} from './context-engine/io-types.js';
|
|
41
|
+
import type { EpisodicMemory } from './tiers/episodic-memory.js';
|
|
42
|
+
import type { InsightMemory } from './tiers/insight-memory.js';
|
|
43
|
+
import type { ProceduralMemory } from './tiers/procedural-memory.js';
|
|
44
|
+
import type { SemanticMemory } from './tiers/semantic-memory.js';
|
|
45
|
+
import type { SessionMemory } from './tiers/session-memory.js';
|
|
46
|
+
import type { SharedMemory } from './tiers/shared-memory.js';
|
|
47
|
+
import type { WorkingMemory } from './tiers/working-memory.js';
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Public surface of the {@link ContextEngine} instance returned by
|
|
51
|
+
* {@link createContextEngine}.
|
|
52
|
+
*
|
|
53
|
+
* @stable
|
|
54
|
+
*/
|
|
55
|
+
export interface ContextEngine {
|
|
56
|
+
/** Assemble the layered system prompt for a single step. */
|
|
57
|
+
assemble(memory: Memory, input: AssembleInput): Promise<AssembledPrompt>;
|
|
58
|
+
/**
|
|
59
|
+
* Trigger evaluation primitive used by Phase 12 (agent runtime)
|
|
60
|
+
* at the top of every step. Returns `true` when the in-flight
|
|
61
|
+
* buffer's token count crosses the per-provider trigger
|
|
62
|
+
* threshold. Pass `precomputedTokens` to amortize the count
|
|
63
|
+
* via the per-message cache surfaced by
|
|
64
|
+
* `SessionMemoryStoreExt.totalCachedTokens(scope)` (DEC-131) -
|
|
65
|
+
* the production hot path is an O(1) comparison when the cache
|
|
66
|
+
* is warm.
|
|
67
|
+
*/
|
|
68
|
+
shouldCompact(
|
|
69
|
+
messages: ReadonlyArray<Message>,
|
|
70
|
+
options?: {
|
|
71
|
+
readonly precomputedTokens?: number;
|
|
72
|
+
/**
|
|
73
|
+
* Index of the first COMPACTABLE message (context-engine-04): the
|
|
74
|
+
* caller's pinned, never-compacted system prefix ends here. The
|
|
75
|
+
* SOTA-4 reclaim floor counts only `messages.slice(from)` older
|
|
76
|
+
* turns as reclaimable - without it a large system prompt is
|
|
77
|
+
* counted as reclaimable and the floor fires the summarizer for
|
|
78
|
+
* near-zero real reclaim. Default `0` (everything compactable).
|
|
79
|
+
*/
|
|
80
|
+
readonly compactableFromIndex?: number;
|
|
81
|
+
},
|
|
82
|
+
): Promise<boolean>;
|
|
83
|
+
/**
|
|
84
|
+
* Run a compaction call. Phase 12 calls this when the trigger
|
|
85
|
+
* fires (`source: 'auto-trigger'`) or the operator invokes
|
|
86
|
+
* `agent.compact(...)` (`source: 'manual'`).
|
|
87
|
+
*/
|
|
88
|
+
compactNow(input: {
|
|
89
|
+
readonly scope: SessionScope;
|
|
90
|
+
readonly runId: string;
|
|
91
|
+
readonly sessionId: string;
|
|
92
|
+
readonly agentId: string;
|
|
93
|
+
readonly source: CompactionSource;
|
|
94
|
+
readonly messages: ReadonlyArray<Message>;
|
|
95
|
+
readonly memory: Memory;
|
|
96
|
+
readonly summarizer?: CompactionSummarizer;
|
|
97
|
+
/** Per-call override of the strategy's preserve-recent count (CE-3). */
|
|
98
|
+
readonly preserveRecentTurns?: number;
|
|
99
|
+
/** Topic/tags narrowing for the procedural-rules re-anchor hook (CE-6). */
|
|
100
|
+
readonly procedural?: { readonly topic?: string; readonly tags?: ReadonlyArray<string> };
|
|
101
|
+
/**
|
|
102
|
+
* The caller's pinned system prefix - the messages EXCLUDED from
|
|
103
|
+
* `messages` before this call (context-engine-04). Used only for
|
|
104
|
+
* accounting: the anti-thrash guard and the "still above threshold"
|
|
105
|
+
* warning must compare against the FULL post-splice context
|
|
106
|
+
* (prefix + summary + preserved + essentials), or a real system
|
|
107
|
+
* prompt defeats the guard and a summarizer call fires every step
|
|
108
|
+
* at the context edge. Never compacted, never returned.
|
|
109
|
+
*/
|
|
110
|
+
readonly prefixMessages?: ReadonlyArray<Message>;
|
|
111
|
+
readonly signal?: AbortSignal;
|
|
112
|
+
}): Promise<{
|
|
113
|
+
readonly result: CompactionResult;
|
|
114
|
+
readonly extraContent: ReadonlyArray<MessageContent>;
|
|
115
|
+
readonly hookFailures: ReadonlyArray<{ readonly hookName: string; readonly reason: string }>;
|
|
116
|
+
}>;
|
|
117
|
+
/** Resolved configuration snapshot. */
|
|
118
|
+
config(): ResolvedContextEngineConfig;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* The facade returned by {@link createMemory}.
|
|
123
|
+
*
|
|
124
|
+
* @stable
|
|
125
|
+
*/
|
|
126
|
+
export interface Memory {
|
|
127
|
+
readonly working: WorkingMemory;
|
|
128
|
+
readonly session: SessionMemory;
|
|
129
|
+
readonly episodic: EpisodicMemory;
|
|
130
|
+
readonly semantic: SemanticMemory;
|
|
131
|
+
readonly procedural: ProceduralMemory;
|
|
132
|
+
readonly shared: SharedMemory;
|
|
133
|
+
/**
|
|
134
|
+
* Read surface over reflection insights (P1-1). A no-op (returns
|
|
135
|
+
* empty) when the storage adapter does not expose the optional
|
|
136
|
+
* insight surface.
|
|
137
|
+
*/
|
|
138
|
+
readonly insights: InsightMemory;
|
|
139
|
+
readonly tools: ReadonlyArray<Tool>;
|
|
140
|
+
readonly consolidator: Consolidator;
|
|
141
|
+
/** The configured conflict pipeline. Surfaced for tests + CLI tooling. */
|
|
142
|
+
readonly conflictPipeline: ConflictPipeline;
|
|
143
|
+
/** The configured context engine (Phase 10d). */
|
|
144
|
+
readonly contextEngine: ContextEngine;
|
|
145
|
+
/** The active embedder, when configured. `null` otherwise. */
|
|
146
|
+
readonly embedder: EmbedderProvider | null;
|
|
147
|
+
/** The canonical id of the active embedder, when configured. */
|
|
148
|
+
embedderId(): string | null;
|
|
149
|
+
/**
|
|
150
|
+
* Compile a system-prompt block bundle. The bundle carries the
|
|
151
|
+
* static fragments per memory tier; the agent runtime consumes
|
|
152
|
+
* the {@link ContextEngine} surface (`memory.contextEngine`)
|
|
153
|
+
* directly for the full six-layer assembly.
|
|
154
|
+
*/
|
|
155
|
+
compile(scope: CompileScope, options?: CompileOptions): Promise<MemoryContextBlocks>;
|
|
156
|
+
/** Counter snapshot consumed by Phase 10d's metadata layer. */
|
|
157
|
+
metadata(scope: SessionScope): Promise<MemoryMetadata>;
|
|
158
|
+
}
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
import type { EmbedderProvider } from '@graphorin/core';
|
|
2
|
+
import {
|
|
3
|
+
EmbedderMigrationAbortedError,
|
|
4
|
+
EmbedderMigrationLockedError,
|
|
5
|
+
EmbedderMigrationStateError,
|
|
6
|
+
} from '../errors/index.js';
|
|
7
|
+
import { bindEmbedder } from '../internal/embedder-binding.js';
|
|
8
|
+
import type { EmbeddingMetaRegistryLike } from '../internal/storage-adapter.js';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Coexistence policy for embedder swaps.
|
|
12
|
+
*
|
|
13
|
+
* - `'lock-on-first'` (default) - refuses to register a second active
|
|
14
|
+
* embedder; surfaces an actionable error pointing at the planned
|
|
15
|
+
* migration runner.
|
|
16
|
+
* - `'multi-active'` - keeps both embedders alive (read union, write
|
|
17
|
+
* to active); used while a long migration is in flight.
|
|
18
|
+
* - `'auto-migrate'` - re-embeds existing rows in resumable batches
|
|
19
|
+
* until the source embedder has zero rows, then retires it.
|
|
20
|
+
*
|
|
21
|
+
* @stable
|
|
22
|
+
*/
|
|
23
|
+
export type EmbedderMigrationStrategy = 'lock-on-first' | 'multi-active' | 'auto-migrate';
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Per-iteration progress snapshot yielded by {@link migrateEmbedder}.
|
|
27
|
+
*
|
|
28
|
+
* @stable
|
|
29
|
+
*/
|
|
30
|
+
export interface MigrationProgress {
|
|
31
|
+
/** `'fact'`, `'episode'`, or `'message'` - which entity is being migrated. */
|
|
32
|
+
readonly kind: 'fact' | 'episode' | 'message';
|
|
33
|
+
/** Number of records processed so far. */
|
|
34
|
+
readonly processed: number;
|
|
35
|
+
/** Total records expected for this entity (when known). */
|
|
36
|
+
readonly total: number;
|
|
37
|
+
/** Identifier of the source embedder (`'<adapter>:<model>@<dim>'`). */
|
|
38
|
+
readonly source: string;
|
|
39
|
+
/** Identifier of the target embedder. */
|
|
40
|
+
readonly target: string;
|
|
41
|
+
/**
|
|
42
|
+
* Identifier for this migration run. MST-12: this is an in-memory id for
|
|
43
|
+
* the current run - there is no persisted `migration_state` cursor today, so
|
|
44
|
+
* a migration does not resume across processes.
|
|
45
|
+
*/
|
|
46
|
+
readonly migrationId: string;
|
|
47
|
+
/** Phase discriminator. */
|
|
48
|
+
readonly phase: 'planning' | 'running' | 'paused' | 'committed' | 'aborted';
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Options accepted by {@link migrateEmbedder}.
|
|
53
|
+
*
|
|
54
|
+
* @stable
|
|
55
|
+
*/
|
|
56
|
+
export interface MigrateEmbedderOptions {
|
|
57
|
+
/** Source embedder (currently active). */
|
|
58
|
+
readonly source: EmbedderProvider;
|
|
59
|
+
/** Target embedder (becomes active when migration commits). */
|
|
60
|
+
readonly target: EmbedderProvider;
|
|
61
|
+
/** Storage layer's embedder registry. */
|
|
62
|
+
readonly embeddings: EmbeddingMetaRegistryLike;
|
|
63
|
+
/** Strategy applied per `embedding_meta` row. Default `'lock-on-first'`. */
|
|
64
|
+
readonly strategy?: EmbedderMigrationStrategy;
|
|
65
|
+
/**
|
|
66
|
+
* Threshold for `auto-migrate`. The runner streams source rows in
|
|
67
|
+
* batches of `batchSize` (default `512`) and yields progress after
|
|
68
|
+
* each batch.
|
|
69
|
+
*/
|
|
70
|
+
readonly batchSize?: number;
|
|
71
|
+
/** Optional cap on the number of rows to migrate per kind. */
|
|
72
|
+
readonly maxRecordsPerKind?: number;
|
|
73
|
+
/**
|
|
74
|
+
* Hook that returns the next batch of rows to re-embed for a given kind.
|
|
75
|
+
* MST-12: this is **caller-supplied** - there is no store-side helper that
|
|
76
|
+
* auto-wires it today, and `auto-migrate` throws without it. Pass a paging
|
|
77
|
+
* function over your source rows to drive the migration.
|
|
78
|
+
*/
|
|
79
|
+
readonly nextBatch?: NextBatchHook;
|
|
80
|
+
/** Optional abort signal - aborting yields one final progress event. */
|
|
81
|
+
readonly signal?: AbortSignal;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Per-batch loader. Returns up to `batchSize` rows for the supplied
|
|
86
|
+
* `kind` whose `embedder_id` is the source embedder. Returning an
|
|
87
|
+
* empty array signals end-of-stream.
|
|
88
|
+
*
|
|
89
|
+
* @stable
|
|
90
|
+
*/
|
|
91
|
+
export type NextBatchHook = (args: {
|
|
92
|
+
readonly kind: 'fact' | 'episode' | 'message';
|
|
93
|
+
readonly source: string;
|
|
94
|
+
readonly target: string;
|
|
95
|
+
readonly batchSize: number;
|
|
96
|
+
readonly cursor: string | null;
|
|
97
|
+
}) => Promise<{ readonly rows: ReadonlyArray<MigrationRow>; readonly nextCursor: string | null }>;
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Single row exposed to the migration runner. The runner re-embeds
|
|
101
|
+
* `text` with the target embedder; the storage adapter is responsible
|
|
102
|
+
* for committing the new vector + updating `embedder_id`.
|
|
103
|
+
*
|
|
104
|
+
* @stable
|
|
105
|
+
*/
|
|
106
|
+
export interface MigrationRow {
|
|
107
|
+
readonly id: string;
|
|
108
|
+
readonly text: string;
|
|
109
|
+
readonly write: (vector: Float32Array) => Promise<void>;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Stream embedder migrations as `AsyncIterable<MigrationProgress>`.
|
|
114
|
+
*
|
|
115
|
+
* The function is the universal entry point for every migration
|
|
116
|
+
* strategy:
|
|
117
|
+
*
|
|
118
|
+
* - `'lock-on-first'`: surfaces the locked error eagerly so the
|
|
119
|
+
* operator can act before any work happens.
|
|
120
|
+
* - `'multi-active'`: registers the target alongside the source and
|
|
121
|
+
* yields a single `'committed'` progress event.
|
|
122
|
+
* - `'auto-migrate'`: streams batches until the source is fully
|
|
123
|
+
* drained, then retires the source row.
|
|
124
|
+
*
|
|
125
|
+
* @stable
|
|
126
|
+
*/
|
|
127
|
+
export async function* migrateEmbedder(
|
|
128
|
+
options: MigrateEmbedderOptions,
|
|
129
|
+
): AsyncGenerator<MigrationProgress, void, unknown> {
|
|
130
|
+
const { source, target, embeddings } = options;
|
|
131
|
+
const strategy = options.strategy ?? 'lock-on-first';
|
|
132
|
+
const sourceId = source.id();
|
|
133
|
+
const targetId = target.id();
|
|
134
|
+
const migrationId = `migration_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 7)}`;
|
|
135
|
+
|
|
136
|
+
if (sourceId === targetId) {
|
|
137
|
+
throw new EmbedderMigrationStateError(
|
|
138
|
+
`source and target embedders must differ; both resolved to '${sourceId}'.`,
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const sourceMeta = embeddings.get(sourceId);
|
|
143
|
+
if (sourceMeta === null) {
|
|
144
|
+
throw new EmbedderMigrationStateError(
|
|
145
|
+
`source embedder '${sourceId}' is not registered. Run createMemory({ embedder: source }) first.`,
|
|
146
|
+
);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
if (strategy === 'lock-on-first') {
|
|
150
|
+
throw new EmbedderMigrationLockedError(sourceId, targetId);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
bindEmbedder(target, embeddings);
|
|
154
|
+
|
|
155
|
+
if (strategy === 'multi-active') {
|
|
156
|
+
yield {
|
|
157
|
+
kind: 'fact',
|
|
158
|
+
processed: 0,
|
|
159
|
+
total: 0,
|
|
160
|
+
source: sourceId,
|
|
161
|
+
target: targetId,
|
|
162
|
+
migrationId,
|
|
163
|
+
phase: 'committed',
|
|
164
|
+
};
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// strategy === 'auto-migrate'
|
|
169
|
+
if (options.nextBatch === undefined) {
|
|
170
|
+
throw new EmbedderMigrationStateError(
|
|
171
|
+
"'auto-migrate' requires a `nextBatch` hook supplied by the storage adapter.",
|
|
172
|
+
);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
const batchSize = options.batchSize ?? 512;
|
|
176
|
+
const cap = options.maxRecordsPerKind ?? Number.POSITIVE_INFINITY;
|
|
177
|
+
|
|
178
|
+
for (const kind of ['fact', 'episode', 'message'] as const) {
|
|
179
|
+
let cursor: string | null = null;
|
|
180
|
+
let processed = 0;
|
|
181
|
+
while (processed < cap) {
|
|
182
|
+
if (options.signal?.aborted === true) {
|
|
183
|
+
yield {
|
|
184
|
+
kind,
|
|
185
|
+
processed,
|
|
186
|
+
total: processed,
|
|
187
|
+
source: sourceId,
|
|
188
|
+
target: targetId,
|
|
189
|
+
migrationId,
|
|
190
|
+
phase: 'aborted',
|
|
191
|
+
};
|
|
192
|
+
throw new EmbedderMigrationAbortedError(migrationId);
|
|
193
|
+
}
|
|
194
|
+
const batch = await options.nextBatch({
|
|
195
|
+
kind,
|
|
196
|
+
source: sourceId,
|
|
197
|
+
target: targetId,
|
|
198
|
+
batchSize,
|
|
199
|
+
cursor,
|
|
200
|
+
});
|
|
201
|
+
if (batch.rows.length === 0) break;
|
|
202
|
+
const texts = batch.rows.map((r) => r.text);
|
|
203
|
+
const vectors = await target.embed(texts, {
|
|
204
|
+
...(options.signal !== undefined ? { signal: options.signal } : {}),
|
|
205
|
+
});
|
|
206
|
+
for (let i = 0; i < batch.rows.length; i++) {
|
|
207
|
+
const row = batch.rows[i];
|
|
208
|
+
const vector = vectors[i];
|
|
209
|
+
if (row === undefined || vector === undefined) continue;
|
|
210
|
+
await row.write(vector);
|
|
211
|
+
}
|
|
212
|
+
processed += batch.rows.length;
|
|
213
|
+
cursor = batch.nextCursor;
|
|
214
|
+
yield {
|
|
215
|
+
kind,
|
|
216
|
+
processed,
|
|
217
|
+
total: processed,
|
|
218
|
+
source: sourceId,
|
|
219
|
+
target: targetId,
|
|
220
|
+
migrationId,
|
|
221
|
+
phase: 'running',
|
|
222
|
+
};
|
|
223
|
+
if (cursor === null) break;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
embeddings.retire(sourceId);
|
|
228
|
+
yield {
|
|
229
|
+
kind: 'fact',
|
|
230
|
+
processed: 0,
|
|
231
|
+
total: 0,
|
|
232
|
+
source: sourceId,
|
|
233
|
+
target: targetId,
|
|
234
|
+
migrationId,
|
|
235
|
+
phase: 'committed',
|
|
236
|
+
};
|
|
237
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedder migration runner - one of the three deliverables that turns
|
|
3
|
+
* `@graphorin/memory` into a long-running, embedder-aware system.
|
|
4
|
+
*
|
|
5
|
+
* @packageDocumentation
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
export type {
|
|
9
|
+
EmbedderMigrationStrategy,
|
|
10
|
+
MigrateEmbedderOptions,
|
|
11
|
+
MigrationProgress,
|
|
12
|
+
} from './embedder-migration.js';
|
|
13
|
+
export { migrateEmbedder } from './embedder-migration.js';
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Recall explainability for `@graphorin/memory` - "why was this
|
|
3
|
+
* recalled?" (X-3). Turns the per-signal scores the hybrid pipeline
|
|
4
|
+
* already records on each {@link MemoryHit} (FTS `bm25`, vector
|
|
5
|
+
* similarity, the fused reciprocal-rank term, the decay multiplier,
|
|
6
|
+
* future graph-hop terms) into a structured, renderable explanation.
|
|
7
|
+
*
|
|
8
|
+
* Pure data: the builder re-uses whatever {@link MemoryHit.signals} the
|
|
9
|
+
* pipeline left behind rather than re-deriving them, so the explanation
|
|
10
|
+
* always matches the ranking that actually happened. The renderer
|
|
11
|
+
* returns a string and performs no I/O, so the explanation can be
|
|
12
|
+
* attached to a span, logged, or surfaced by operator tooling.
|
|
13
|
+
*
|
|
14
|
+
* @packageDocumentation
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import type { MemoryHit, MemoryKind, MemoryRecord } from '@graphorin/core';
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Per-memory contribution breakdown for one recalled record: the
|
|
21
|
+
* decomposed signals the hybrid pipeline summed into the final score,
|
|
22
|
+
* plus the record's final rank. Surfaces *why* a single memory was
|
|
23
|
+
* recalled and how it ranked relative to its siblings.
|
|
24
|
+
*
|
|
25
|
+
* @stable
|
|
26
|
+
*/
|
|
27
|
+
export interface RecalledMemoryExplanation {
|
|
28
|
+
/** Record id (fact / episode / insight / …). */
|
|
29
|
+
readonly id: string;
|
|
30
|
+
/** Memory kind of the recalled record. */
|
|
31
|
+
readonly kind: MemoryKind;
|
|
32
|
+
/** 1-based position in the final fused + decayed ranking. */
|
|
33
|
+
readonly rank: number;
|
|
34
|
+
/** Final score after fusion and (optional) decay. */
|
|
35
|
+
readonly score: number;
|
|
36
|
+
/**
|
|
37
|
+
* Raw per-signal scores contributing to `score`, exactly as the
|
|
38
|
+
* pipeline recorded them on the hit: FTS (`bm25`), vector similarity
|
|
39
|
+
* (`vector`), the fused reciprocal-rank terms (`rrf` plus one
|
|
40
|
+
* `rrf.<label>` entry per fused list keyed by the retriever kind,
|
|
41
|
+
* e.g. `rrf.fts` / `rrf.vector`, with `rrf.list_N` as the fallback
|
|
42
|
+
* when a list carries no label), and the decay multiplier (`decay`)
|
|
43
|
+
* when decay ran.
|
|
44
|
+
*/
|
|
45
|
+
readonly signals: Readonly<Record<string, number>>;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Structured "why was this recalled?" explanation for one
|
|
50
|
+
* {@link SemanticMemory.search} call: the query, the reranker that
|
|
51
|
+
* fused the candidate lists, and the per-memory signal breakdown in
|
|
52
|
+
* final-rank order. Pure data - safe to attach to a span, log, or
|
|
53
|
+
* render. X-3.
|
|
54
|
+
*
|
|
55
|
+
* @stable
|
|
56
|
+
*/
|
|
57
|
+
export interface RecallExplanation {
|
|
58
|
+
/** The query the recall ran for. */
|
|
59
|
+
readonly query: string;
|
|
60
|
+
/** Stable id of the reranker that fused the candidate lists. */
|
|
61
|
+
readonly rerankerId: string;
|
|
62
|
+
/** Number of recalled memories explained. */
|
|
63
|
+
readonly count: number;
|
|
64
|
+
/** Per-memory breakdown, in final-rank order (rank 1 first). */
|
|
65
|
+
readonly results: ReadonlyArray<RecalledMemoryExplanation>;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Build a {@link RecallExplanation} from a ranked hit list. Pure and
|
|
70
|
+
* deterministic: it surfaces whatever per-signal scores the hybrid
|
|
71
|
+
* pipeline left on each {@link MemoryHit.signals} (FTS, vector, RRF,
|
|
72
|
+
* decay, future graph-hop terms) without re-deriving them, so the
|
|
73
|
+
* explanation always matches what actually drove the ranking. Hits are
|
|
74
|
+
* assumed to be in final-rank order (as returned by `search`).
|
|
75
|
+
*
|
|
76
|
+
* @stable
|
|
77
|
+
*/
|
|
78
|
+
export function explainRecall<TRecord extends MemoryRecord>(
|
|
79
|
+
hits: ReadonlyArray<MemoryHit<TRecord>>,
|
|
80
|
+
opts: { readonly query: string; readonly rerankerId: string },
|
|
81
|
+
): RecallExplanation {
|
|
82
|
+
const results = hits.map(
|
|
83
|
+
(hit, idx): RecalledMemoryExplanation =>
|
|
84
|
+
Object.freeze({
|
|
85
|
+
id: hit.record.id,
|
|
86
|
+
kind: hit.record.kind,
|
|
87
|
+
rank: idx + 1,
|
|
88
|
+
score: hit.score,
|
|
89
|
+
signals: Object.freeze({ ...(hit.signals ?? {}) }),
|
|
90
|
+
}),
|
|
91
|
+
);
|
|
92
|
+
return Object.freeze({
|
|
93
|
+
query: opts.query,
|
|
94
|
+
rerankerId: opts.rerankerId,
|
|
95
|
+
count: results.length,
|
|
96
|
+
results: Object.freeze(results),
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Render a {@link RecallExplanation} as a compact ASCII block - a
|
|
102
|
+
* header line plus one line per recalled memory with its signal
|
|
103
|
+
* breakdown. Pure (returns a string, no I/O); used by operator tooling
|
|
104
|
+
* and trace inspectors.
|
|
105
|
+
*
|
|
106
|
+
* @stable
|
|
107
|
+
*/
|
|
108
|
+
export function formatRecallExplanation(explanation: RecallExplanation): string {
|
|
109
|
+
const header = `recall ${JSON.stringify(explanation.query)} via ${explanation.rerankerId} -> ${explanation.count} result(s)`;
|
|
110
|
+
if (explanation.results.length === 0) return header;
|
|
111
|
+
const lines = explanation.results.map((r) => {
|
|
112
|
+
const signals = Object.entries(r.signals)
|
|
113
|
+
.map(([key, value]) => `${key}=${formatScore(value)}`)
|
|
114
|
+
.join(' ');
|
|
115
|
+
const suffix = signals === '' ? '' : ` (${signals})`;
|
|
116
|
+
return ` #${r.rank} ${r.id} [${r.kind}] score=${formatScore(r.score)}${suffix}`;
|
|
117
|
+
});
|
|
118
|
+
return [header, ...lines].join('\n');
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function formatScore(value: number): string {
|
|
122
|
+
return Number.isFinite(value) ? value.toFixed(4) : String(value);
|
|
123
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* C5: offline fusion-weight fitting. Tuned convex/weighted fusion beats
|
|
3
|
+
* RRF with a handful of labelled queries (Bruch et al., TOIS 2023) - this
|
|
4
|
+
* module turns that into a small, pure routine: grid-search
|
|
5
|
+
* {@link FusionWeights} over labelled cases and return the setting that
|
|
6
|
+
* maximizes mean nDCG@k, together with the plain-RRF baseline so the
|
|
7
|
+
* caller can see whether tuning actually paid.
|
|
8
|
+
*
|
|
9
|
+
* Pure and I/O-free: the caller supplies per-kind candidate lists (run
|
|
10
|
+
* your FTS and vector retrievers once per query, offline) plus the ids
|
|
11
|
+
* judged relevant. Feed the winning weights to
|
|
12
|
+
* `search(..., { fusion: { strategy: 'weighted', weights } })` or a
|
|
13
|
+
* process-default `WeightedRRFReranker`.
|
|
14
|
+
*
|
|
15
|
+
* @packageDocumentation
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import type { MemoryHit, MemoryRecord } from '@graphorin/core';
|
|
19
|
+
import { fuseWeighted } from './rrf.js';
|
|
20
|
+
|
|
21
|
+
/** One labelled query: per-kind candidate lists + the relevant ids. */
|
|
22
|
+
export interface FitFusionCase<TRecord extends MemoryRecord = MemoryRecord> {
|
|
23
|
+
/** FTS candidate list, best-first (as the store returns it). */
|
|
24
|
+
readonly fts: ReadonlyArray<MemoryHit<TRecord>>;
|
|
25
|
+
/** Vector candidate list, best-first. */
|
|
26
|
+
readonly vector: ReadonlyArray<MemoryHit<TRecord>>;
|
|
27
|
+
/** Ids judged relevant for this query (binary gain). */
|
|
28
|
+
readonly relevantIds: ReadonlyArray<string>;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Options for {@link fitFusionWeights}. */
|
|
32
|
+
export interface FitFusionOptions {
|
|
33
|
+
/** Candidate weight values per axis. Default `[0.25, 0.5, 1, 2, 4]`. */
|
|
34
|
+
readonly grid?: ReadonlyArray<number>;
|
|
35
|
+
/** nDCG cutoff. Default `10`. */
|
|
36
|
+
readonly k?: number;
|
|
37
|
+
/** RRF constant forwarded to the fuse. Default `60`. */
|
|
38
|
+
readonly rrfK?: number;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Result of a fit: the winning weights + its score and the RRF baseline. */
|
|
42
|
+
export interface FitFusionResult {
|
|
43
|
+
readonly weights: { readonly fts: number; readonly vector: number };
|
|
44
|
+
/** Mean nDCG@k of the winning weights across the cases. */
|
|
45
|
+
readonly score: number;
|
|
46
|
+
/** Mean nDCG@k of unit weights (plain RRF) - compare before adopting. */
|
|
47
|
+
readonly baseline: number;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Binary-gain nDCG@k over a ranked id list.
|
|
52
|
+
*
|
|
53
|
+
* @stable
|
|
54
|
+
*/
|
|
55
|
+
export function ndcgAtK(
|
|
56
|
+
rankedIds: ReadonlyArray<string>,
|
|
57
|
+
relevant: ReadonlySet<string>,
|
|
58
|
+
k: number,
|
|
59
|
+
): number {
|
|
60
|
+
const cutoff = Math.min(k, rankedIds.length);
|
|
61
|
+
let dcg = 0;
|
|
62
|
+
for (let i = 0; i < cutoff; i++) {
|
|
63
|
+
const id = rankedIds[i];
|
|
64
|
+
if (id !== undefined && relevant.has(id)) dcg += 1 / Math.log2(i + 2);
|
|
65
|
+
}
|
|
66
|
+
const idealHits = Math.min(relevant.size, k);
|
|
67
|
+
let idcg = 0;
|
|
68
|
+
for (let i = 0; i < idealHits; i++) idcg += 1 / Math.log2(i + 2);
|
|
69
|
+
return idcg === 0 ? 0 : dcg / idcg;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Grid-search fusion weights against labelled cases (C5).
|
|
74
|
+
*
|
|
75
|
+
* @stable
|
|
76
|
+
*/
|
|
77
|
+
export function fitFusionWeights<TRecord extends MemoryRecord = MemoryRecord>(
|
|
78
|
+
cases: ReadonlyArray<FitFusionCase<TRecord>>,
|
|
79
|
+
options: FitFusionOptions = {},
|
|
80
|
+
): FitFusionResult {
|
|
81
|
+
if (cases.length === 0) {
|
|
82
|
+
throw new TypeError('[graphorin/memory] fitFusionWeights: at least one labelled case required');
|
|
83
|
+
}
|
|
84
|
+
const grid = options.grid ?? [0.25, 0.5, 1, 2, 4];
|
|
85
|
+
const k = options.k ?? 10;
|
|
86
|
+
const rrfK = options.rrfK ?? 60;
|
|
87
|
+
|
|
88
|
+
const meanNdcg = (wFts: number, wVector: number): number => {
|
|
89
|
+
let total = 0;
|
|
90
|
+
for (const c of cases) {
|
|
91
|
+
const fused = fuseWeighted([c.fts, c.vector], [wFts, wVector], rrfK);
|
|
92
|
+
const rankedIds = fused.map((h) => h.record.id);
|
|
93
|
+
total += ndcgAtK(rankedIds, new Set(c.relevantIds), k);
|
|
94
|
+
}
|
|
95
|
+
return total / cases.length;
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
const baseline = meanNdcg(1, 1);
|
|
99
|
+
let best: FitFusionResult = { weights: { fts: 1, vector: 1 }, score: baseline, baseline };
|
|
100
|
+
for (const wFts of grid) {
|
|
101
|
+
for (const wVector of grid) {
|
|
102
|
+
const score = meanNdcg(wFts, wVector);
|
|
103
|
+
if (score > best.score + 1e-12) {
|
|
104
|
+
best = { weights: { fts: wFts, vector: wVector }, score, baseline };
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return best;
|
|
109
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PPR-lite graph scoring (D5) - damped spreading activation over the
|
|
3
|
+
* entity-graph neighbourhood, generalizing the flat one-hop expansion.
|
|
4
|
+
* Pure: given the seed facts (the lexical/vector candidates retrieved so
|
|
5
|
+
* far) and the graded neighbours (`{ fact, depth }` from
|
|
6
|
+
* `expandActivation`), it assigns each neighbour an activation score
|
|
7
|
+
* that decays with hop distance, so a fact two hops from a strong seed
|
|
8
|
+
* ranks below a direct neighbour instead of tying at a flat `1`.
|
|
9
|
+
*
|
|
10
|
+
* This is HippoRAG-style personalized-PageRank, "lite": a single damped
|
|
11
|
+
* BFS pass seeded from retrieval rather than an iterative power method.
|
|
12
|
+
* Seeding from query-matched entities (rather than retrieved candidates)
|
|
13
|
+
* is the eval-gated extension the roadmap defers until numbers justify.
|
|
14
|
+
*
|
|
15
|
+
* @packageDocumentation
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/** Default damping applied per hop (activation × damping^depth). */
|
|
19
|
+
export const DEFAULT_PPR_DAMPING = 0.5;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Compute PPR-lite activation for graded neighbours. `damping` in
|
|
23
|
+
* `(0, 1]`; a neighbour at `depth` d contributes `damping^d` per seed
|
|
24
|
+
* that reaches it. Since `expandActivation` already returns the MINIMUM
|
|
25
|
+
* depth per neighbour (dedup across seeds), the score here is a pure
|
|
26
|
+
* function of that min-depth - `damping^depth`, clamped to `[0, 1]`.
|
|
27
|
+
* A neighbour list produced with `maxHops: 1` reproduces a flat-ish
|
|
28
|
+
* score of `damping` at every node; `maxHops: 2` separates the tiers.
|
|
29
|
+
*
|
|
30
|
+
* @stable
|
|
31
|
+
*/
|
|
32
|
+
export function pprActivation(
|
|
33
|
+
neighbours: ReadonlyArray<{ readonly depth: number }>,
|
|
34
|
+
damping = DEFAULT_PPR_DAMPING,
|
|
35
|
+
): number[] {
|
|
36
|
+
const d = damping <= 0 || damping > 1 ? DEFAULT_PPR_DAMPING : damping;
|
|
37
|
+
return neighbours.map((n) => d ** Math.max(1, n.depth));
|
|
38
|
+
}
|