@noir-ai/context 1.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +21 -0
- package/dist/index.d.ts +763 -0
- package/dist/index.js +1182 -0
- package/dist/index.js.map +1 -0
- package/package.json +58 -0
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,763 @@
|
|
|
1
|
+
import { Store, EmbedFn, FtsHit, VecHit } from '@noir-ai/store';
|
|
2
|
+
export { EmbedFn, FtsHit, IndexDoc, Store, VecHit } from '@noir-ai/store';
|
|
3
|
+
import { ProjectId } from '@noir-ai/core';
|
|
4
|
+
export { ProjectId } from '@noir-ai/core';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Logical source buckets a chunk can belong to. Reused by S7 (memory) —
|
|
8
|
+
* `'memory'` is reserved for that slice. The store treats `source` as a
|
|
9
|
+
* free-form string; this enum is the context package's contract for what a
|
|
10
|
+
* well-formed `source` value is.
|
|
11
|
+
*/
|
|
12
|
+
declare const SOURCES: readonly ["codebase", "docs", "spec", "memory"];
|
|
13
|
+
type SourceKind = (typeof SOURCES)[number];
|
|
14
|
+
/** Discriminant for {@link EmbedderConfig}. */
|
|
15
|
+
type EmbedderKind = 'local' | 'remote' | 'ollama' | 'none';
|
|
16
|
+
/**
|
|
17
|
+
* Local in-process embeddings via `@huggingface/transformers`
|
|
18
|
+
* (default `Xenova/all-MiniLM-L6-v2`, 384-dim → zero vec0 migration).
|
|
19
|
+
* The model is loaded lazily on first `embed()` call, never at import time.
|
|
20
|
+
*/
|
|
21
|
+
interface LocalEmbedderConfig {
|
|
22
|
+
kind: 'local';
|
|
23
|
+
/** HF repo id (defaults to `Xenova/all-MiniLM-L6-v2` in the factory). */
|
|
24
|
+
model?: string;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Opt-in remote embeddings (OpenAI / Voyage / Cohere). Provider-explicit:
|
|
28
|
+
* sends source text to a cloud endpoint. Vectors are L2-normalized and
|
|
29
|
+
* Matryoshka-truncated to `dim` client-side to stay vec0-compatible.
|
|
30
|
+
* NEVER the default (blueprint D6 — no silent paid calls).
|
|
31
|
+
*/
|
|
32
|
+
interface RemoteEmbedderConfig {
|
|
33
|
+
kind: 'remote';
|
|
34
|
+
/** Provider key, e.g. `'openai'` | `'voyage'` | `'cohere'`. */
|
|
35
|
+
provider: string;
|
|
36
|
+
/** API key. If absent the factory throws a clear "not configured" error. */
|
|
37
|
+
apiKey?: string;
|
|
38
|
+
/** Provider-specific model id. */
|
|
39
|
+
model: string;
|
|
40
|
+
/** Target dimensionality (must be 384 to match the vec0 table). */
|
|
41
|
+
dim: number;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Opt-in Ollama embeddings via a user-supplied base URL. Provider-explicit;
|
|
45
|
+
* never the default.
|
|
46
|
+
*/
|
|
47
|
+
interface OllamaEmbedderConfig {
|
|
48
|
+
kind: 'ollama';
|
|
49
|
+
/** Base URL of the Ollama server, e.g. `http://localhost:11434`. */
|
|
50
|
+
baseURL: string;
|
|
51
|
+
/** Ollama model tag. */
|
|
52
|
+
model: string;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Escape hatch: disable vectors entirely. `search` degrades to BM25-only
|
|
56
|
+
* (`degraded:true, mode:'bm25-only'`).
|
|
57
|
+
*/
|
|
58
|
+
interface NoneEmbedderConfig {
|
|
59
|
+
kind: 'none';
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Discriminated configuration for the embedder factory (`createEmbedFn`,
|
|
63
|
+
* task t3). The core zod schema maps onto this via `resolveEmbedderConfig`
|
|
64
|
+
* (task t10) — that mapper is the only bridge between the two layers.
|
|
65
|
+
*/
|
|
66
|
+
type EmbedderConfig = LocalEmbedderConfig | RemoteEmbedderConfig | OllamaEmbedderConfig | NoneEmbedderConfig;
|
|
67
|
+
/**
|
|
68
|
+
* Description of the active embedder, surfaced by `context_status` and
|
|
69
|
+
* recorded in store KV (`ctx:embedder`) so a model swap is detectable.
|
|
70
|
+
* For `kind:'none'`, `dim` is `0` and `model` is `undefined`.
|
|
71
|
+
*/
|
|
72
|
+
interface EmbedderInfo {
|
|
73
|
+
kind: EmbedderKind;
|
|
74
|
+
model?: string;
|
|
75
|
+
dim: number;
|
|
76
|
+
}
|
|
77
|
+
/** Metadata attached to every chunk (stored as `docs.meta` / `vec.meta`). */
|
|
78
|
+
interface ChunkMeta {
|
|
79
|
+
/** Repo-relative or absolute path of the source file. */
|
|
80
|
+
path: string;
|
|
81
|
+
/**
|
|
82
|
+
* `sha256(path)` — stable parent-document id. The budget packer collapses
|
|
83
|
+
* duplicate parent-docs on this field so one file can't flood the results.
|
|
84
|
+
*/
|
|
85
|
+
parentDocId: string;
|
|
86
|
+
/** 0-based position of this chunk within its parent file. */
|
|
87
|
+
chunkIndex: number;
|
|
88
|
+
/** Detected language / extension hint (e.g. `'typescript'`, `'markdown'`). */
|
|
89
|
+
language: string;
|
|
90
|
+
/** SHA-256 of this chunk's (post-identifier-explosion) content. */
|
|
91
|
+
sha256: string;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* A unit of indexed content. One chunk → one `docs` row (FTS-synced) AND one
|
|
95
|
+
* `vec0` row under the SAME `id`, so RRF can join BM25 and kNN hits.
|
|
96
|
+
* `id` is `<sha256(path)>#chunk-<n>` (stable across re-indexing).
|
|
97
|
+
*/
|
|
98
|
+
interface Chunk {
|
|
99
|
+
id: string;
|
|
100
|
+
source: SourceKind;
|
|
101
|
+
content: string;
|
|
102
|
+
meta: ChunkMeta;
|
|
103
|
+
}
|
|
104
|
+
/** Pair of non-negative weights `[bm25Weight, kNNWeight]`; defaults `[0.5, 0.5]`. */
|
|
105
|
+
type RRFWeights = [number, number];
|
|
106
|
+
/**
|
|
107
|
+
* Retrieval mode actually used for a `search` call. `'hybrid'` = BM25 ∪ kNN
|
|
108
|
+
* fused by RRF; `'bm25-only'` = degraded fallback when the embedder is
|
|
109
|
+
* unavailable (`kind:'none'` or `embed()` threw).
|
|
110
|
+
*/
|
|
111
|
+
type SearchMode = 'hybrid' | 'bm25-only';
|
|
112
|
+
/**
|
|
113
|
+
* Secondary metadata carried on a retriever hit. All optional: a BM25 hit
|
|
114
|
+
* reaches the retriever already enriched, while a kNN-only hit may only
|
|
115
|
+
* carry `id` + `source` until the retriever backfills the rest. The primary
|
|
116
|
+
* `path` / `parentDocId` fields live top-level on {@link RetrieverHit}
|
|
117
|
+
* (spec F6); this holds the remaining enrichment.
|
|
118
|
+
*/
|
|
119
|
+
interface RetrieverMeta {
|
|
120
|
+
language?: string;
|
|
121
|
+
sha256?: string;
|
|
122
|
+
chunkIndex?: number;
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* A single ranked result. `snippet` is FTS5 window-extracted
|
|
126
|
+
* (`snippet(docs_fts,0,'<<','>>','…',16)`) for BM25 hits, or a window around
|
|
127
|
+
* the chunk's first N tokens for kNN-only hits — never truncated mid-token.
|
|
128
|
+
*
|
|
129
|
+
* `path` and `parentDocId` are top-level (spec F6 enumerates the item as
|
|
130
|
+
* `{id, source, score, snippet, path, parentDocId}`): the retriever
|
|
131
|
+
* backfills them from the chunk's `meta` before the hit leaves `search()`,
|
|
132
|
+
* so they are always present at the public/MCP boundary.
|
|
133
|
+
*/
|
|
134
|
+
interface RetrieverHit {
|
|
135
|
+
id: string;
|
|
136
|
+
source: SourceKind;
|
|
137
|
+
/** RRF-fused rank score (rank-based; not a normalized similarity). */
|
|
138
|
+
score: number;
|
|
139
|
+
snippet: string;
|
|
140
|
+
/** Repo-relative or absolute path of the source file (backfilled from meta). */
|
|
141
|
+
path: string;
|
|
142
|
+
/** `sha256(path)` — stable parent-document id (backfilled from meta). */
|
|
143
|
+
parentDocId: string;
|
|
144
|
+
/** Secondary enrichment (language, sha256, chunkIndex). */
|
|
145
|
+
meta: RetrieverMeta;
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Return value of `search`. `results` is packed greedily into `budgetTokens`
|
|
149
|
+
* (default ~4k): when the budget is exhausted before the ranked list,
|
|
150
|
+
* `truncated` is `true` and `consumedTokens` ~= `budgetTokens`.
|
|
151
|
+
*/
|
|
152
|
+
interface SearchResult {
|
|
153
|
+
results: RetrieverHit[];
|
|
154
|
+
/** Tokens accumulated across the returned snippets. */
|
|
155
|
+
consumedTokens: number;
|
|
156
|
+
/** `true` if the budget was hit before exhausting the ranked list. */
|
|
157
|
+
truncated: boolean;
|
|
158
|
+
/** `true` when the embedder was unavailable (BM25-only fallback, F8). */
|
|
159
|
+
degraded: boolean;
|
|
160
|
+
/** Retrieval mode actually used. */
|
|
161
|
+
mode: SearchMode;
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Return value of `indexPaths`. Counts are mutually consistent: a chunk is
|
|
165
|
+
* either newly `indexed`, `skipped` (content-hash unchanged in KV), or part
|
|
166
|
+
* of a `deleted`/`failed` file. `totalChunks` is the running total tracked
|
|
167
|
+
* for the indexed paths after this call. `degraded` mirrors the search path
|
|
168
|
+
* (spec F10: `context_index -> {indexed, skipped, deleted, degraded}`).
|
|
169
|
+
*/
|
|
170
|
+
interface IndexResult {
|
|
171
|
+
/** Chunks newly written to `docs` + `vec0` this run. */
|
|
172
|
+
indexed: number;
|
|
173
|
+
/** Chunks whose SHA-256 matched KV (unchanged since last index). */
|
|
174
|
+
skipped: number;
|
|
175
|
+
/** Files removed since last index — their chunks + vectors were deleted. */
|
|
176
|
+
deleted: number;
|
|
177
|
+
/**
|
|
178
|
+
* True when the embedder was unavailable this run (docs indexed without
|
|
179
|
+
* vectors). Mirrors SearchResult.degraded / StoreStatus.degraded.
|
|
180
|
+
*/
|
|
181
|
+
degraded: boolean;
|
|
182
|
+
/** Files that could not be read/parsed (binary, IO error, encoding). */
|
|
183
|
+
failed: number;
|
|
184
|
+
/** Total chunks now tracked for the indexed paths. */
|
|
185
|
+
totalChunks: number;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/** Default maximum estimated tokens per non-markdown chunk (spec DS-5). */
|
|
189
|
+
declare const DEFAULT_CHUNK_MAX_TOKENS = 512;
|
|
190
|
+
/** Default token overlap between consecutive non-markdown chunks (spec DS-5). */
|
|
191
|
+
declare const DEFAULT_CHUNK_OVERLAP = 64;
|
|
192
|
+
/**
|
|
193
|
+
* Cheap token estimate factor: ~1.3 tokens per whitespace-separated word (the
|
|
194
|
+
* canonical "1 word ≈ 1.3 tokens" heuristic). Good enough for windowing/budget
|
|
195
|
+
* packing without pulling in a tokenizer dep.
|
|
196
|
+
*/
|
|
197
|
+
declare const TOKEN_ESTIMATE_FACTOR = 1.3;
|
|
198
|
+
/**
|
|
199
|
+
* Input to {@link chunkFile}. `path` + `content` are required; everything else
|
|
200
|
+
* has a sensible default. `maxTokens`/`overlap` flow from the resolved
|
|
201
|
+
* `context.chunk` config (task t10); `language`/`source` are inferred from the
|
|
202
|
+
* path when omitted so a bare `chunkFile({path, content})` call just works.
|
|
203
|
+
*/
|
|
204
|
+
interface ChunkOptions {
|
|
205
|
+
/** Repo-relative or absolute path of the source file. */
|
|
206
|
+
path: string;
|
|
207
|
+
/** Full text content of the file (the caller reads it; the chunker never does I/O). */
|
|
208
|
+
content: string;
|
|
209
|
+
/**
|
|
210
|
+
* Language hint (e.g. `'typescript'`, `'markdown'`). Inferred from the path
|
|
211
|
+
* extension when absent (see {@link inferLanguage}).
|
|
212
|
+
*/
|
|
213
|
+
language?: string;
|
|
214
|
+
/**
|
|
215
|
+
* Override the inferred {@link SourceKind}. Defaults to `'docs'` for
|
|
216
|
+
* markdown and `'codebase'` otherwise; pass `'spec'`/`'memory'` when indexing
|
|
217
|
+
* those trees explicitly.
|
|
218
|
+
*/
|
|
219
|
+
source?: SourceKind;
|
|
220
|
+
/** Max estimated tokens per code chunk (default {@link DEFAULT_CHUNK_MAX_TOKENS}). */
|
|
221
|
+
maxTokens?: number;
|
|
222
|
+
/** Token overlap between consecutive code chunks (default {@link DEFAULT_CHUNK_OVERLAP}). */
|
|
223
|
+
overlap?: number;
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* Split the identifiers in `text` into lowercase tokens, expanding
|
|
227
|
+
* camelCase / PascalCase / snake_case / kebab-case / digit boundaries.
|
|
228
|
+
*
|
|
229
|
+
* Examples:
|
|
230
|
+
* `contextEngine` → `context engine`
|
|
231
|
+
* `ContextEngine` → `context engine`
|
|
232
|
+
* `XMLHttpRequest` → `xml http request`
|
|
233
|
+
* `myHTTPSConnection` → `my https connection`
|
|
234
|
+
* `snake_case`/`kebab-case` → `snake case` / `kebab case`
|
|
235
|
+
* `ctx:file:<path>` → `ctx file path`
|
|
236
|
+
*
|
|
237
|
+
* The stream is APPENDED to chunk content at index time (see
|
|
238
|
+
* {@link withIdentifierExplosion}) so identifier queries match under
|
|
239
|
+
* `porter unicode61` without a tokenizer migration. Pure/deterministic.
|
|
240
|
+
*/
|
|
241
|
+
declare function explodeIdentifiers(text: string): string;
|
|
242
|
+
/**
|
|
243
|
+
* The canonical index-time content for a chunk: its clean text followed by the
|
|
244
|
+
* identifier-exploded token stream on a trailing line. Both the FTS row and
|
|
245
|
+
* the embedding are produced from this exact string (so identifier queries get
|
|
246
|
+
* a BM25 AND a semantic signal), and {@link chunkSha256} hashes this form to
|
|
247
|
+
* match the `ChunkMeta.sha256` contract ("post-identifier-explosion").
|
|
248
|
+
*
|
|
249
|
+
* `chunk.content` itself stays CLEAN (no explosion) so FTS5 window-extracted
|
|
250
|
+
* snippets read naturally — explosion is index-time-only (spec §7, DS-7).
|
|
251
|
+
*/
|
|
252
|
+
declare function withIdentifierExplosion(content: string): string;
|
|
253
|
+
/**
|
|
254
|
+
* Cheap token-count proxy: ~1.3 tokens per whitespace-separated word (see
|
|
255
|
+
* {@link TOKEN_ESTIMATE_FACTOR}). Empty/whitespace-only text → 0. Used both
|
|
256
|
+
* for chunk windowing and (exported) for the retriever's budget packing.
|
|
257
|
+
*/
|
|
258
|
+
declare function estimateTokens(text: string): number;
|
|
259
|
+
/**
|
|
260
|
+
* Best-effort language tag from the path extension (e.g. `.ts` → `'typescript'`).
|
|
261
|
+
* Unknown extensions fall back to `'text'`. Used as the `language` default and
|
|
262
|
+
* recorded per-file in the indexer's KV.
|
|
263
|
+
*/
|
|
264
|
+
declare function inferLanguage(path: string): string;
|
|
265
|
+
/**
|
|
266
|
+
* Split a file's content into {@link Chunk}s.
|
|
267
|
+
*
|
|
268
|
+
* Markdown → heading-delimited sections; everything else → overlapping
|
|
269
|
+
* line/token windows. Each chunk's `id` is `<sha256(path)>#chunk-<n>` and
|
|
270
|
+
* `meta.parentDocId` is `sha256(path)` (stable across re-indexing), with
|
|
271
|
+
* `meta.sha256` over the post-identifier-explosion content (see
|
|
272
|
+
* {@link withIdentifierExplosion}). Empty content yields no chunks.
|
|
273
|
+
*/
|
|
274
|
+
declare function chunkFile(opts: ChunkOptions): Chunk[];
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* User-facing context config shape — mirrors `NoirConfig['context']` (the zod
|
|
278
|
+
* block @noir-ai/core ships, slice S6). Declared locally with every field
|
|
279
|
+
* optional so this module type-checks WITHOUT a forward dependency on a core
|
|
280
|
+
* type (core never imports context — no cycle), AND so a config with no
|
|
281
|
+
* `context:` block (or a partial one) parses cleanly (AC-7 / NFR-6). The fully
|
|
282
|
+
* resolved zod output is structurally assignable to this permissive shape, so
|
|
283
|
+
* the mapper accepts `NoirConfig['context']` directly.
|
|
284
|
+
*/
|
|
285
|
+
interface ContextUserConfig {
|
|
286
|
+
embedder?: {
|
|
287
|
+
/** Default `'local'` (in-process transformers.js — offline, free, private). */
|
|
288
|
+
kind?: 'local' | 'remote' | 'ollama' | 'none';
|
|
289
|
+
/** Local HF repo id / remote provider model id / Ollama model tag. */
|
|
290
|
+
model?: string;
|
|
291
|
+
/** Remote provider key, e.g. `'openai'` (only meaningful when `kind:'remote'`). */
|
|
292
|
+
provider?: string;
|
|
293
|
+
/** Ollama base URL, e.g. `http://localhost:11434` (only when `kind:'ollama'`). */
|
|
294
|
+
baseURL?: string;
|
|
295
|
+
/** Target dimensionality (must be 384 to match the existing vec0 table). */
|
|
296
|
+
dim?: number;
|
|
297
|
+
};
|
|
298
|
+
/** Configured index roots (informational here; the daemon/indexer consume them). */
|
|
299
|
+
roots?: string[];
|
|
300
|
+
/** Default token budget for `search` (informational here; consumed by the retriever). */
|
|
301
|
+
budgetTokens?: number;
|
|
302
|
+
}
|
|
303
|
+
/**
|
|
304
|
+
* Resolve a user-facing {@link ContextUserConfig} into the discriminated
|
|
305
|
+
* {@link EmbedderConfig} the factory (`createEmbedFn`) consumes.
|
|
306
|
+
*
|
|
307
|
+
* - `undefined` / missing block ⇒ `{kind:'local'}` (the safe default — a config
|
|
308
|
+
* with no `context:` block stays local-embedder-attempted, AC-7 / NFR-6).
|
|
309
|
+
* - `kind:'local'` ⇒ in-process transformers.js (model optional; factory
|
|
310
|
+
* defaults to `Xenova/all-MiniLM-L6-v2`).
|
|
311
|
+
* - `kind:'none'` ⇒ vectors disabled; `search` degrades to BM25-only.
|
|
312
|
+
* - `kind:'remote'` / `'ollama'` ⇒ provider-explicit; the API key (remote) /
|
|
313
|
+
* base URL (ollama) are resolved from env when absent in config. A missing
|
|
314
|
+
* key is NOT an error here — the built embedder throws from `embed()` so the
|
|
315
|
+
* engine degrades to BM25-only (F8) instead of crashing at construction.
|
|
316
|
+
*
|
|
317
|
+
* NEVER returns a remote/ollama config unless `kind` is explicitly that value —
|
|
318
|
+
* there is no path from the default to a silent paid call (blueprint D6).
|
|
319
|
+
*/
|
|
320
|
+
declare function resolveEmbedderConfig(ctx?: ContextUserConfig): EmbedderConfig;
|
|
321
|
+
|
|
322
|
+
/** KV key holding the sorted list of indexed path keys. */
|
|
323
|
+
declare const CTX_REGISTRY_KEY = "ctx:registry";
|
|
324
|
+
/** KV key holding the recorded {@link EmbedderInfo} (model-swap detection). */
|
|
325
|
+
declare const CTX_EMBEDDER_KEY = "ctx:embedder";
|
|
326
|
+
/** Per-file record key prefix; the value is a {@link FileRecord} (or `null` tombstone). */
|
|
327
|
+
declare const CTX_FILE_PREFIX = "ctx:file:";
|
|
328
|
+
/**
|
|
329
|
+
* Per-file tracking record stored under {@link ctxFileKey}. `sha256` is the
|
|
330
|
+
* content-hash skip key; `chunkIds` lets a change/removal delete exactly the
|
|
331
|
+
* rows that belong to this file; `language` mirrors the chunker's inference.
|
|
332
|
+
*/
|
|
333
|
+
interface FileRecord {
|
|
334
|
+
/** SHA-256 of the file's UTF-8 content — the content-hash skip key (DS-4). */
|
|
335
|
+
sha256: string;
|
|
336
|
+
/** Chunk ids belonging to this file (for exact delete on change/removal). */
|
|
337
|
+
chunkIds: string[];
|
|
338
|
+
/** Detected language tag (mirrors the chunker's inference). */
|
|
339
|
+
language: string;
|
|
340
|
+
}
|
|
341
|
+
/** Build a `ctx:file:<key>` KV key. */
|
|
342
|
+
declare function ctxFileKey(pathKey: string): string;
|
|
343
|
+
/**
|
|
344
|
+
* Directory names never descended into during a walk (spec F1). Covers the
|
|
345
|
+
* common `.gitignore` entries without needing a gitignore parser — full
|
|
346
|
+
* `.gitignore` intersection (and `git diff --name-only` for incremental scope)
|
|
347
|
+
* is deferred v0 debt (grounds §10 indexing trigger).
|
|
348
|
+
*/
|
|
349
|
+
declare const SKIP_DIRS: Set<string>;
|
|
350
|
+
/** True if the path's extension is a known binary type. */
|
|
351
|
+
declare function isBinaryExt(pathOrName: string): boolean;
|
|
352
|
+
/**
|
|
353
|
+
* True if `name` is a sensitive file that must NEVER be chunked, embedded, or
|
|
354
|
+
* indexed. Accepts a bare basename (`id_rsa`, `.env`, `cert.pem`) OR a
|
|
355
|
+
* `/`-separated relative path (`deploy/id_rsa`, `.aws/credentials`); the
|
|
356
|
+
* basename is extracted internally so prefix/suffix checks anchor on the file
|
|
357
|
+
* name rather than a path segment. Wired into the walk's skip decision
|
|
358
|
+
* alongside {@link isBinaryExt} so secrets never reach `indexDoc` / `upsertVec`.
|
|
359
|
+
*/
|
|
360
|
+
declare function isSensitive(name: string): boolean;
|
|
361
|
+
/** Construction options for {@link createIndexer}. */
|
|
362
|
+
interface IndexerOptions {
|
|
363
|
+
/** The daemon's store handle — the ONLY storage surface used (single writer). */
|
|
364
|
+
store: Store;
|
|
365
|
+
/** The resolved embedder. Failures here degrade the run (F8-style), never crash. */
|
|
366
|
+
embed: EmbedFn;
|
|
367
|
+
/** Active embedder description; recorded in KV for model-swap detection. */
|
|
368
|
+
info: EmbedderInfo;
|
|
369
|
+
/**
|
|
370
|
+
* Optional repository root. When provided, paths passed to
|
|
371
|
+
* {@link Indexer.indexPaths} are resolved against it AND stored repo-relative
|
|
372
|
+
* (portable `meta.path`, stable across checkouts). When omitted, paths resolve
|
|
373
|
+
* against `process.cwd()` and are stored absolute.
|
|
374
|
+
*/
|
|
375
|
+
root?: string;
|
|
376
|
+
}
|
|
377
|
+
/** Per-call options for {@link Indexer.indexPaths}. */
|
|
378
|
+
interface IndexPathOptions {
|
|
379
|
+
/** Chunk size override (flows from `context.chunk.maxTokens`). */
|
|
380
|
+
maxTokens?: number;
|
|
381
|
+
/** Chunk overlap override (flows from `context.chunk.overlap`). */
|
|
382
|
+
overlap?: number;
|
|
383
|
+
/**
|
|
384
|
+
* Override the source bucket for every file in this call (otherwise the
|
|
385
|
+
* chunker infers `'docs'` for markdown and `'codebase'` for everything else).
|
|
386
|
+
*/
|
|
387
|
+
source?: SourceKind;
|
|
388
|
+
}
|
|
389
|
+
/** Return value of {@link Indexer.forget}. */
|
|
390
|
+
interface ForgetResult {
|
|
391
|
+
/** Files removed from the index (their chunks + vectors were deleted). */
|
|
392
|
+
deleted: number;
|
|
393
|
+
/** Total chunks now tracked. */
|
|
394
|
+
totalChunks: number;
|
|
395
|
+
}
|
|
396
|
+
/** The indexer surface returned by {@link createIndexer}. */
|
|
397
|
+
interface Indexer {
|
|
398
|
+
/**
|
|
399
|
+
* Walk + incrementally index `paths` (files or directories). Removed files
|
|
400
|
+
* under a re-scanned root are reconciled (deleted). Returns chunk/file counts
|
|
401
|
+
* + the `degraded` flag (spec F1/F3/F4, AC-1).
|
|
402
|
+
*/
|
|
403
|
+
indexPaths(paths: string[], opts?: IndexPathOptions): Promise<IndexResult>;
|
|
404
|
+
/** Remove `paths` (files or dirs) from the index; deletes their chunks + vectors. */
|
|
405
|
+
forget(paths: string[]): Promise<ForgetResult>;
|
|
406
|
+
/** Drop every indexed chunk + vector, then re-index the registered roots from scratch. */
|
|
407
|
+
reindex(): Promise<IndexResult>;
|
|
408
|
+
}
|
|
409
|
+
/**
|
|
410
|
+
* Build an {@link Indexer} bound to a single store handle. Construction does no
|
|
411
|
+
* I/O and touches no KV — all work happens inside `indexPaths`/`forget`/`reindex`.
|
|
412
|
+
*/
|
|
413
|
+
declare function createIndexer(opts: IndexerOptions): Indexer;
|
|
414
|
+
|
|
415
|
+
/**
|
|
416
|
+
* Default per-query hit cap applied to BOTH the BM25 and kNN legs so RRF has
|
|
417
|
+
* equal-depth lists to fuse (matches `Store.searchFt`'s default of 10; the
|
|
418
|
+
* store's `knn` default of 5 is intentionally raised here for fusion parity).
|
|
419
|
+
*/
|
|
420
|
+
declare const DEFAULT_SEARCH_LIMIT = 10;
|
|
421
|
+
/** Default token budget for the greedy packer (spec §6 / config `budgetTokens`). */
|
|
422
|
+
declare const DEFAULT_BUDGET_TOKENS = 4096;
|
|
423
|
+
/**
|
|
424
|
+
* Default snippet window for kNN-only hydration, in (estimated) tokens. Mirrors
|
|
425
|
+
* the FTS5 `snippet(..., 16)` 16-token window used by `Store.searchFt` so a
|
|
426
|
+
* kNN-only hit's fallback window is the same width as a BM25 hit's window.
|
|
427
|
+
*/
|
|
428
|
+
declare const DEFAULT_SNIPPET_WINDOW_TOKENS = 16;
|
|
429
|
+
/**
|
|
430
|
+
* Content + meta returned by the optional {@link RetrieverOptions.readDoc}
|
|
431
|
+
* hydrator. This is the shape needed to window-extract a snippet for a kNN-only
|
|
432
|
+
* hit (vec0 returns no meta, and the `Store` interface exposes no read-by-id).
|
|
433
|
+
*/
|
|
434
|
+
interface ChunkDoc {
|
|
435
|
+
/** Full chunk text (the hydrator decides the source: KV, a future getDoc, …). */
|
|
436
|
+
content: string;
|
|
437
|
+
/** The chunk's stored metadata, if available. */
|
|
438
|
+
meta?: ChunkMeta;
|
|
439
|
+
}
|
|
440
|
+
/** Tunables held for the life of the retriever (RRF + budget + hydration). */
|
|
441
|
+
interface RetrieverOptions {
|
|
442
|
+
/** RRF constant `k` (defaults to {@link DEFAULT_RRF_K} = 60). */
|
|
443
|
+
k?: number;
|
|
444
|
+
/** RRF per-retriever weights `[bm25Weight, kNNWeight]` (default `[0.5, 0.5]`). */
|
|
445
|
+
weights?: RRFWeights;
|
|
446
|
+
/** Default token budget (overridable per `search()` call). */
|
|
447
|
+
budgetTokens?: number;
|
|
448
|
+
/** kNN-only snippet window width in tokens (default 16). */
|
|
449
|
+
snippetWindowTokens?: number;
|
|
450
|
+
/**
|
|
451
|
+
* Optional content/meta lookup for kNN-only hits. The store's `knn` returns
|
|
452
|
+
* only `{id, source, score}` (vec0 has no meta column) and the `Store`
|
|
453
|
+
* interface exposes no read-by-id, so without a hydrator a purely-semantic
|
|
454
|
+
* hit cannot be windowed. When provided and it hits, the chunk's content is
|
|
455
|
+
* prefix-windowed with `<<query-term>>` highlights (mirroring FTS5). When
|
|
456
|
+
* omitted or it misses, the hit is emitted with an empty snippet — degraded
|
|
457
|
+
* but ranked (F8). The engine (t8) wires this when a content source exists.
|
|
458
|
+
*/
|
|
459
|
+
readDoc?: (id: string) => ChunkDoc | null;
|
|
460
|
+
}
|
|
461
|
+
/** Per-call options for {@link Retriever.search}. */
|
|
462
|
+
interface SearchOptions {
|
|
463
|
+
/** Max hits requested from EACH leg before fusion (default 10). */
|
|
464
|
+
limit?: number;
|
|
465
|
+
/** Token budget for this call (defaults to the retriever's `budgetTokens`). */
|
|
466
|
+
budgetTokens?: number;
|
|
467
|
+
/** Restrict both legs to a single source bucket (passes through to the store). */
|
|
468
|
+
source?: string;
|
|
469
|
+
}
|
|
470
|
+
/** Constructor dependencies for {@link createRetriever}. */
|
|
471
|
+
interface RetrieverDeps {
|
|
472
|
+
/** The store handle (the daemon's single-writer handle, or a read-only one). */
|
|
473
|
+
store: Store;
|
|
474
|
+
/** Query/chunk embedder. A throw on `embed(query)` ⇒ BM25-only fallback (F8). */
|
|
475
|
+
embed: EmbedFn;
|
|
476
|
+
/** Tunables + the optional kNN-only hydrator. */
|
|
477
|
+
opts?: RetrieverOptions;
|
|
478
|
+
}
|
|
479
|
+
/** The retriever surface returned by {@link createRetriever}. */
|
|
480
|
+
interface Retriever {
|
|
481
|
+
search(query: string, opts?: SearchOptions): Promise<SearchResult>;
|
|
482
|
+
}
|
|
483
|
+
/**
|
|
484
|
+
* Build a best-effort prefix snippet for a kNN-only hit: the first
|
|
485
|
+
* `windowTokens` whitespace-separated words of `content`, with any word whose
|
|
486
|
+
* lowercase form is in the query-term set wrapped in `<<…>>` (mirroring FTS5's
|
|
487
|
+
* `<<match>>` marker convention so BM25 and kNN snippets read the same way).
|
|
488
|
+
*
|
|
489
|
+
* Unlike FTS5's `snippet()` this is a PREFIX window (FTS5 picks the densest
|
|
490
|
+
* window around match positions; vec0 gives us no match positions to center
|
|
491
|
+
* on). Content shorter than the window is returned whole (highlighted) — it is
|
|
492
|
+
* never padded or mid-word split.
|
|
493
|
+
*/
|
|
494
|
+
declare function windowSnippet(content: string, query: string, windowTokens: number): string;
|
|
495
|
+
/**
|
|
496
|
+
* Build a hybrid retriever over an injected store + embedder.
|
|
497
|
+
*
|
|
498
|
+
* The retriever owns no state beyond its configured tunables; every `search`
|
|
499
|
+
* call is independent and side-effect-free (read-only against the store). It is
|
|
500
|
+
* the ONLY read path for context — `context_search` (t9) delegates here. Write
|
|
501
|
+
* (`indexDoc`/`upsertVec`) stays with the indexer/engine; the retriever never
|
|
502
|
+
* mutates the store.
|
|
503
|
+
*/
|
|
504
|
+
declare function createRetriever(deps: RetrieverDeps): Retriever;
|
|
505
|
+
|
|
506
|
+
/**
|
|
507
|
+
* JSON returned by the `context_status` MCP tool (spec F11). Mirrors
|
|
508
|
+
* `StoreStatus` and adds the active embedder description + the indexed-file
|
|
509
|
+
* count (the size of the `ctx:registry` KV list the indexer maintains).
|
|
510
|
+
*/
|
|
511
|
+
interface ContextStatus {
|
|
512
|
+
ok: boolean;
|
|
513
|
+
/** Canonical project id (never a filesystem path — blueprint D6). */
|
|
514
|
+
projectId: string;
|
|
515
|
+
/** Rows in `docs` (live read off the single writer handle — no cache). */
|
|
516
|
+
docCount: number;
|
|
517
|
+
/** Rows in `vec0` (live read off the single writer handle — no cache). */
|
|
518
|
+
vecCount: number;
|
|
519
|
+
/** Number of files currently tracked in `ctx:registry`. */
|
|
520
|
+
indexedFiles: number;
|
|
521
|
+
/** Active embedder description (`kind:'none'` ⇒ `{kind:'none', dim:0}`). */
|
|
522
|
+
embedder: EmbedderInfo;
|
|
523
|
+
/**
|
|
524
|
+
* Persistent degradation flag: the store handle is read-only OR the embedder
|
|
525
|
+
* is disabled (`kind:'none'`). Per-query degradation lives on SearchResult.
|
|
526
|
+
*/
|
|
527
|
+
degraded: boolean;
|
|
528
|
+
}
|
|
529
|
+
/** Construction options for {@link ContextEngine}. */
|
|
530
|
+
interface ContextEngineOptions {
|
|
531
|
+
/**
|
|
532
|
+
* The daemon's store handle — the ONLY storage surface used (single writer).
|
|
533
|
+
* May be a read-only fallback handle; pass {@link storeDegraded} so `status()`
|
|
534
|
+
* can report it.
|
|
535
|
+
*/
|
|
536
|
+
store: Store;
|
|
537
|
+
/**
|
|
538
|
+
* Project root. Paths passed to {@link ContextEngine.indexPaths} resolve
|
|
539
|
+
* against it and (because the indexer is root-bound) are stored repo-relative
|
|
540
|
+
* for portable `meta.path` across checkouts.
|
|
541
|
+
*/
|
|
542
|
+
root: string;
|
|
543
|
+
/** Canonical project identifier (NEVER a filesystem path). */
|
|
544
|
+
projectId: ProjectId;
|
|
545
|
+
/**
|
|
546
|
+
* Resolved embedder config (from `resolveEmbedderConfig`). Construction never
|
|
547
|
+
* touches the network or native runtime — even `local` defers its dynamic
|
|
548
|
+
* import to the first `embed()` call.
|
|
549
|
+
*/
|
|
550
|
+
embedderCfg: EmbedderConfig;
|
|
551
|
+
/**
|
|
552
|
+
* True when `store` was opened read-only (the daemon-down fallback). Threads
|
|
553
|
+
* the store's degraded story into the engine's persistent `degraded` flag so
|
|
554
|
+
* `status()` reports it and callers can branch on it.
|
|
555
|
+
*/
|
|
556
|
+
storeDegraded?: boolean;
|
|
557
|
+
}
|
|
558
|
+
/**
|
|
559
|
+
* Noir's embedded hybrid retrieval engine — the `ctx.context` service.
|
|
560
|
+
*
|
|
561
|
+
* Constructed once per serve lifecycle (mirror `buildWorkflowEngine`) from the
|
|
562
|
+
* daemon's store handle + a resolved {@link EmbedderConfig}. Resolves the
|
|
563
|
+
* embedder, then owns the indexer (the only context writer) and the retriever
|
|
564
|
+
* (the only context reader). Public surface: {@link indexPaths}, {@link search},
|
|
565
|
+
* {@link status} — the three operations the `context_{index,search,status}` MCP
|
|
566
|
+
* tools (task t9) delegate to.
|
|
567
|
+
*/
|
|
568
|
+
declare class ContextEngine {
|
|
569
|
+
/** The daemon's single-writer store handle (possibly read-only). */
|
|
570
|
+
readonly store: Store;
|
|
571
|
+
/** Project root (paths resolve against this). */
|
|
572
|
+
readonly root: string;
|
|
573
|
+
/** Canonical project identifier. */
|
|
574
|
+
readonly projectId: ProjectId;
|
|
575
|
+
/** Description of the active embedder (surfaced by `status()`). */
|
|
576
|
+
readonly embedder: EmbedderInfo;
|
|
577
|
+
/**
|
|
578
|
+
* Persistent degradation flag (read-only store OR `kind:'none'`). Per-query
|
|
579
|
+
* degradation lives on {@link SearchResult}.
|
|
580
|
+
*/
|
|
581
|
+
readonly degraded: boolean;
|
|
582
|
+
private readonly indexer;
|
|
583
|
+
private readonly retriever;
|
|
584
|
+
constructor(opts: ContextEngineOptions);
|
|
585
|
+
/**
|
|
586
|
+
* Incrementally index `paths` (files or directories) into the store. Delegates
|
|
587
|
+
* to the indexer (spec F1/F3/F4). The engine — through the indexer — is the
|
|
588
|
+
* ONLY context writer; the daemon stays the single writer via this handle.
|
|
589
|
+
*/
|
|
590
|
+
indexPaths(paths: string[], opts?: IndexPathOptions): Promise<IndexResult>;
|
|
591
|
+
/**
|
|
592
|
+
* Hybrid search: BM25 ∪ cosine-kNN fused by RRF (k=60), collapsed by
|
|
593
|
+
* parent-doc, packed to a token budget with window-extracted snippets (spec
|
|
594
|
+
* F6/F7). Delegates to the retriever. The per-call `degraded`/`mode` on the
|
|
595
|
+
* returned {@link SearchResult} reflect THIS query's outcome (independent of
|
|
596
|
+
* the engine's persistent {@link degraded}).
|
|
597
|
+
*/
|
|
598
|
+
search(query: string, opts?: SearchOptions): Promise<SearchResult>;
|
|
599
|
+
/**
|
|
600
|
+
* Snapshot the engine's state (spec F11; mirrors `buildStoreStatus`).
|
|
601
|
+
*
|
|
602
|
+
* `docCount`/`vecCount` are live reads off the single writer handle (no
|
|
603
|
+
* cache); `indexedFiles` is the size of the `ctx:registry` KV list maintained
|
|
604
|
+
* by the indexer; `embedder` describes the active provider; `degraded` is the
|
|
605
|
+
* persistent flag (read-only store OR `kind:'none'`).
|
|
606
|
+
*/
|
|
607
|
+
status(): ContextStatus;
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
/**
|
|
611
|
+
* Build a deterministic embedder. The returned function maps each input string
|
|
612
|
+
* to a stable, unit-norm `EMBED_DIM`-wide `Float32Array` derived from a SHA-256
|
|
613
|
+
* hash of the text. Suitable for unit tests and as a BM25-companion fallback
|
|
614
|
+
* where a real vector signal is unavailable but the shape must be valid.
|
|
615
|
+
*
|
|
616
|
+
* @param dim output width (defaults to {@link EMBED_DIM} = 384)
|
|
617
|
+
*/
|
|
618
|
+
declare function fakeEmbedFn(dim?: number): EmbedFn;
|
|
619
|
+
|
|
620
|
+
/** Default HF repo id — 384-dim, matches the vec0 table with zero migration. */
|
|
621
|
+
declare const DEFAULT_LOCAL_MODEL = "Xenova/all-MiniLM-L6-v2";
|
|
622
|
+
/**
|
|
623
|
+
* `~/.noir/models/` — user-global cache for downloaded ONNX weights.
|
|
624
|
+
* HOME-relative (NOT project `.noir/`) so projects stay portable. Centralized
|
|
625
|
+
* via `@noir-ai/core`'s `modelsDir()`; re-exported here so existing imports of
|
|
626
|
+
* `MODELS_DIR` keep resolving to the same path.
|
|
627
|
+
*/
|
|
628
|
+
declare const MODELS_DIR: string;
|
|
629
|
+
interface LocalEmbedderOptions {
|
|
630
|
+
/** HF repo id (defaults to {@link DEFAULT_LOCAL_MODEL}). */
|
|
631
|
+
model?: string;
|
|
632
|
+
}
|
|
633
|
+
interface LocalEmbedder {
|
|
634
|
+
/** Embed function — loads the model lazily on first invocation. */
|
|
635
|
+
embed: EmbedFn;
|
|
636
|
+
/** Resolved model id. */
|
|
637
|
+
model: string;
|
|
638
|
+
}
|
|
639
|
+
/**
|
|
640
|
+
* Build a local in-process embedder. Construction is synchronous and never
|
|
641
|
+
* touches the network or the native runtime — the dynamic `import()` happens
|
|
642
|
+
* inside `embed()`, so `localEmbedder()` is safe to call at startup. Load
|
|
643
|
+
* failures surface as rejections from `embed()`; callers (retriever/engine)
|
|
644
|
+
* catch them and degrade to BM25-only (F8).
|
|
645
|
+
*/
|
|
646
|
+
declare function localEmbedder(opts?: LocalEmbedderOptions): LocalEmbedder;
|
|
647
|
+
|
|
648
|
+
/**
|
|
649
|
+
* Fixed embedding width. Matches the S1 `vec0(float[384])` virtual table and
|
|
650
|
+
* the default `all-MiniLM-L6-v2` model. Remote/ollama vectors are truncated or
|
|
651
|
+
* required to be >= this width (see `remote.ts` / `ollama.ts`).
|
|
652
|
+
*/
|
|
653
|
+
declare const EMBED_DIM = 384;
|
|
654
|
+
/**
|
|
655
|
+
* Return a unit-norm copy of `vec` (L2 normalization). The input is never
|
|
656
|
+
* mutated. Length is preserved: callers responsible for ensuring the vector is
|
|
657
|
+
* already `EMBED_DIM`-wide (truncate BEFORE normalizing — Matryoshka order
|
|
658
|
+
* matters; normalizing then truncating would distort directions).
|
|
659
|
+
*
|
|
660
|
+
* A zero-length or all-zero / non-finite input cannot be normalized; for those
|
|
661
|
+
* degenerate cases a fresh zeroed vector of the same length is returned rather
|
|
662
|
+
* than inventing directional signal or producing NaN. Real embedders never
|
|
663
|
+
* emit such vectors; this guard exists purely for robustness.
|
|
664
|
+
*/
|
|
665
|
+
declare function l2normalize(vec: Float32Array): Float32Array;
|
|
666
|
+
|
|
667
|
+
interface OllamaEmbedderOptions {
|
|
668
|
+
/** Base URL of the Ollama server, e.g. `http://localhost:11434`. */
|
|
669
|
+
baseURL: string;
|
|
670
|
+
/** Ollama model tag, e.g. `nomic-embed-text`. */
|
|
671
|
+
model: string;
|
|
672
|
+
/** Target dimensionality (defaults to {@link EMBED_DIM}). */
|
|
673
|
+
dim?: number;
|
|
674
|
+
}
|
|
675
|
+
/**
|
|
676
|
+
* Build an Ollama embedder. Construction is synchronous and makes no network
|
|
677
|
+
* calls. Each `embed()` POSTs the source text to `<baseURL>/api/embeddings`.
|
|
678
|
+
*/
|
|
679
|
+
declare function ollamaEmbedder(opts: OllamaEmbedderOptions): EmbedFn;
|
|
680
|
+
|
|
681
|
+
interface RemoteEmbedderOptions {
|
|
682
|
+
/** Provider key: `'openai'` | `'voyage'` | `'cohere'` (others fall back to the OpenAI shape). */
|
|
683
|
+
provider: string;
|
|
684
|
+
/** API key. If absent, `embed()` throws a clear "not configured" error. */
|
|
685
|
+
apiKey?: string;
|
|
686
|
+
/** Provider-specific model id. */
|
|
687
|
+
model: string;
|
|
688
|
+
/** Target dimensionality (must be 384 to match the vec0 table). */
|
|
689
|
+
dim?: number;
|
|
690
|
+
}
|
|
691
|
+
/**
|
|
692
|
+
* Build a remote (cloud) embedder. Construction is synchronous and makes no
|
|
693
|
+
* network calls. Each `embed()` call POSTs the source text to the provider.
|
|
694
|
+
*/
|
|
695
|
+
declare function remoteEmbedder(opts: RemoteEmbedderOptions): EmbedFn;
|
|
696
|
+
|
|
697
|
+
/** Result of resolving an embedder config: a callable + describing metadata. */
|
|
698
|
+
interface ResolvedEmbedder {
|
|
699
|
+
/** The `EmbedFn` the engine/retriever/indexer call. May throw on load/config failure. */
|
|
700
|
+
embed: EmbedFn;
|
|
701
|
+
/** Surfaced by `context_status` and recorded in store KV (`ctx:embedder`). */
|
|
702
|
+
info: EmbedderInfo;
|
|
703
|
+
}
|
|
704
|
+
/**
|
|
705
|
+
* Build an `{ embed, info }` pair from a config. Construction never touches the
|
|
706
|
+
* network or the native runtime — even `local` defers its dynamic import to the
|
|
707
|
+
* first `embed()` call. `kind:'none'` (and any load failure the caller lets
|
|
708
|
+
* propagate) yields BM25-only retrieval downstream (F8).
|
|
709
|
+
*/
|
|
710
|
+
declare function createEmbedFn(cfg: EmbedderConfig): ResolvedEmbedder;
|
|
711
|
+
|
|
712
|
+
/**
|
|
713
|
+
* Canonical RRF constant (Cormack SIGIR'09). Larger values dampen the
|
|
714
|
+
* advantage of the top ranks.
|
|
715
|
+
*/
|
|
716
|
+
declare const DEFAULT_RRF_K = 60;
|
|
717
|
+
/** Default per-retriever weights `[bm25Weight, kNNWeight]` (DS-3). */
|
|
718
|
+
declare const DEFAULT_RRF_WEIGHTS: readonly [number, number];
|
|
719
|
+
/** Options for {@link fuseRrf}. */
|
|
720
|
+
interface FuseRrfOptions {
|
|
721
|
+
/**
|
|
722
|
+
* RRF constant `k`. Larger values flatten the top-rank advantage.
|
|
723
|
+
* Defaults to {@link DEFAULT_RRF_K} (60 — the canonical value).
|
|
724
|
+
*/
|
|
725
|
+
k?: number;
|
|
726
|
+
/**
|
|
727
|
+
* Per-retriever weights `[bm25Weight, kNNWeight]`. Need not sum to 1: RRF is
|
|
728
|
+
* rank-based, not a normalized blend. Defaults to `[0.5, 0.5]`.
|
|
729
|
+
*/
|
|
730
|
+
weights?: [number, number];
|
|
731
|
+
}
|
|
732
|
+
/** A fused result row from {@link fuseRrf}. */
|
|
733
|
+
interface RrfResult {
|
|
734
|
+
/** Chunk id (the same id was indexed into both `docs` and `vec0`). */
|
|
735
|
+
id: string;
|
|
736
|
+
/** Source bucket from the originating hit (bm25's, else kNN's). */
|
|
737
|
+
source: string;
|
|
738
|
+
/**
|
|
739
|
+
* Fused RRF score `Σ w_i/(k+rank_i)`. Rank-based — NOT a normalized
|
|
740
|
+
* similarity; only meaningful for ordering within a single `fuseRrf` call.
|
|
741
|
+
*/
|
|
742
|
+
score: number;
|
|
743
|
+
}
|
|
744
|
+
/**
|
|
745
|
+
* Fuse two pre-ranked retrieval lists via Reciprocal Rank Fusion.
|
|
746
|
+
*
|
|
747
|
+
* The input lists MUST be ordered best-first (the store returns them that way:
|
|
748
|
+
* `searchFt` ranks by BM25 relevance, `knn` by ascending distance). Position in
|
|
749
|
+
* the array IS the rank — the hits' raw `score` fields are intentionally
|
|
750
|
+
* ignored (DS-3: never sum raw BM25 + cosine).
|
|
751
|
+
*
|
|
752
|
+
* For each unique id: `score = w_bm25/(k + rank_bm25) + w_knn/(k + rank_knn)`,
|
|
753
|
+
* dropping a term when the id is absent from that list (no penalty, no
|
|
754
|
+
* normalization). Results sort by score descending; ties break by the doc's
|
|
755
|
+
* best (minimum) rank across both lists, then by first-seen insertion order
|
|
756
|
+
* (bm25 list first, then kNN-only docs) — all deterministic, so identical
|
|
757
|
+
* inputs always yield identical ordering (NFR-5).
|
|
758
|
+
*
|
|
759
|
+
* Pure and side-effect-free.
|
|
760
|
+
*/
|
|
761
|
+
declare function fuseRrf(bm25: ReadonlyArray<FtsHit>, knn: ReadonlyArray<VecHit>, opts?: FuseRrfOptions): RrfResult[];
|
|
762
|
+
|
|
763
|
+
export { CTX_EMBEDDER_KEY, CTX_FILE_PREFIX, CTX_REGISTRY_KEY, type Chunk, type ChunkDoc, type ChunkMeta, type ChunkOptions, ContextEngine, type ContextEngineOptions, type ContextStatus, type ContextUserConfig, DEFAULT_BUDGET_TOKENS, DEFAULT_CHUNK_MAX_TOKENS, DEFAULT_CHUNK_OVERLAP, DEFAULT_LOCAL_MODEL, DEFAULT_RRF_K, DEFAULT_RRF_WEIGHTS, DEFAULT_SEARCH_LIMIT, DEFAULT_SNIPPET_WINDOW_TOKENS, EMBED_DIM, type EmbedderConfig, type EmbedderInfo, type EmbedderKind, type FileRecord, type ForgetResult, type FuseRrfOptions, type IndexPathOptions, type IndexResult, type Indexer, type IndexerOptions, type LocalEmbedder, type LocalEmbedderConfig, type LocalEmbedderOptions, MODELS_DIR, type NoneEmbedderConfig, type OllamaEmbedderConfig, type OllamaEmbedderOptions, type RRFWeights, type RemoteEmbedderConfig, type RemoteEmbedderOptions, type ResolvedEmbedder, type Retriever, type RetrieverDeps, type RetrieverHit, type RetrieverMeta, type RetrieverOptions, type RrfResult, SKIP_DIRS, SOURCES, type SearchMode, type SearchOptions, type SearchResult, type SourceKind, TOKEN_ESTIMATE_FACTOR, chunkFile, createEmbedFn, createIndexer, createRetriever, ctxFileKey, estimateTokens, explodeIdentifiers, fakeEmbedFn, fuseRrf, inferLanguage, isBinaryExt, isSensitive, l2normalize, localEmbedder, ollamaEmbedder, remoteEmbedder, resolveEmbedderConfig, windowSnippet, withIdentifierExplosion };
|