ko-qmd 2.8.3-ko.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1937 -0
- package/LICENSE +21 -0
- package/README.en.md +184 -0
- package/README.md +292 -0
- package/bin/qmd +192 -0
- package/dist/ast.d.ts +65 -0
- package/dist/ast.js +51 -0
- package/dist/bench/bench.d.ts +41 -0
- package/dist/bench/bench.js +12 -0
- package/dist/bench/score.d.ts +33 -0
- package/dist/bench/score.js +1 -0
- package/dist/bench/types.d.ts +101 -0
- package/dist/bench/types.js +1 -0
- package/dist/cli/build-info.json +4 -0
- package/dist/cli/embed-lock.d.ts +24 -0
- package/dist/cli/embed-lock.js +2 -0
- package/dist/cli/formatter.d.ts +120 -0
- package/dist/cli/formatter.js +60 -0
- package/dist/cli/mcp-pid.d.ts +33 -0
- package/dist/cli/mcp-pid.js +1 -0
- package/dist/cli/qmd.d.ts +101 -0
- package/dist/cli/qmd.js +165 -0
- package/dist/cli/version.d.ts +42 -0
- package/dist/cli/version.js +1 -0
- package/dist/collections.d.ts +162 -0
- package/dist/collections.js +1 -0
- package/dist/db.d.ts +63 -0
- package/dist/db.js +2 -0
- package/dist/filters.d.ts +46 -0
- package/dist/filters.js +1 -0
- package/dist/grep.d.ts +85 -0
- package/dist/grep.js +9 -0
- package/dist/hangul.d.ts +102 -0
- package/dist/hangul.js +1 -0
- package/dist/index.d.ts +270 -0
- package/dist/index.js +1 -0
- package/dist/llm.d.ts +584 -0
- package/dist/llm.js +40 -0
- package/dist/maintenance.d.ts +33 -0
- package/dist/maintenance.js +1 -0
- package/dist/mcp/daemon-health.d.ts +17 -0
- package/dist/mcp/daemon-health.js +1 -0
- package/dist/mcp/origin-guard.d.ts +67 -0
- package/dist/mcp/origin-guard.js +1 -0
- package/dist/mcp/server.d.ts +131 -0
- package/dist/mcp/server.js +35 -0
- package/dist/memory.d.ts +74 -0
- package/dist/memory.js +13 -0
- package/dist/metadata-filter.d.ts +74 -0
- package/dist/metadata-filter.js +1 -0
- package/dist/metadata-store.d.ts +45 -0
- package/dist/metadata-store.js +68 -0
- package/dist/metadata.d.ts +61 -0
- package/dist/metadata.js +1 -0
- package/dist/paths.d.ts +1 -0
- package/dist/paths.js +1 -0
- package/dist/query-log.d.ts +156 -0
- package/dist/query-log.js +2 -0
- package/dist/refresh.d.ts +60 -0
- package/dist/refresh.js +1 -0
- package/dist/rerank-input.d.ts +46 -0
- package/dist/rerank-input.js +5 -0
- package/dist/store.d.ts +1267 -0
- package/dist/store.js +421 -0
- package/dist/trust.d.ts +152 -0
- package/dist/trust.js +2 -0
- package/package.json +124 -0
- package/scripts/build.mjs +130 -0
- package/scripts/check-package-grammars.mjs +29 -0
- package/scripts/package-smoke.mjs +65 -0
- package/scripts/test-all.mjs +42 -0
- package/skills/qmd/SKILL.md +333 -0
- package/skills/qmd/references/mcp-setup.md +122 -0
- package/skills/release/SKILL.md +141 -0
- package/skills/release/scripts/install-hooks.sh +38 -0
- package/skills/release/scripts/release-context.sh +129 -0
package/dist/store.d.ts
ADDED
|
@@ -0,0 +1,1267 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* QMD Store - Core data access and retrieval functions
|
|
3
|
+
*
|
|
4
|
+
* This module provides all database operations, search functions, and document
|
|
5
|
+
* retrieval for QMD. It returns raw data structures that can be formatted by
|
|
6
|
+
* CLI or MCP consumers.
|
|
7
|
+
*
|
|
8
|
+
* Usage:
|
|
9
|
+
* const store = createStore("/path/to/db.sqlite");
|
|
10
|
+
* // or use default path:
|
|
11
|
+
* const store = createStore();
|
|
12
|
+
*/
|
|
13
|
+
import type { Database } from "./db.js";
|
|
14
|
+
import { type DocumentFilter } from "./filters.js";
|
|
15
|
+
import { INTENT_WEIGHT_CHUNK } from "./rerank-input.js";
|
|
16
|
+
import { LlamaCpp, formatQueryForEmbedding, formatDocForEmbedding, type ILLMSession } from "./llm.js";
|
|
17
|
+
import type { NamedCollection, Collection, CollectionConfig } from "./collections.js";
|
|
18
|
+
import { type DocumentMetadata } from "./metadata.js";
|
|
19
|
+
import { type MetadataFilter } from "./metadata-filter.js";
|
|
20
|
+
export declare const DEFAULT_EMBED_MODEL = "hf:Qwen/Qwen3-Embedding-0.6B-GGUF/Qwen3-Embedding-0.6B-Q8_0.gguf";
|
|
21
|
+
export declare const DEFAULT_RERANK_MODEL = "hf:ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF/qwen3-reranker-0.6b-q8_0.gguf";
|
|
22
|
+
export declare const DEFAULT_QUERY_MODEL = "hf:tobil/qmd-query-expansion-1.7B-gguf/qmd-query-expansion-1.7B-q4_k_m.gguf";
|
|
23
|
+
export declare const DEFAULT_GLOB = "**/*.md";
|
|
24
|
+
/**
|
|
25
|
+
* Split a collection glob mask into fast-glob patterns.
|
|
26
|
+
*
|
|
27
|
+
* `--mask "a.md,*.txt"` is a comma-separated union (issue #557), but
|
|
28
|
+
* fast-glob treats a comma outside `{...}` as a literal character, so
|
|
29
|
+
* the joined string matches nothing. Brace form `{a.md,*.txt}` is
|
|
30
|
+
* already valid glob syntax and is left intact.
|
|
31
|
+
*
|
|
32
|
+
* Commas inside `{...}` or `[...]` are not separators. Empty segments
|
|
33
|
+
* after the split are dropped.
|
|
34
|
+
*/
|
|
35
|
+
export declare function splitGlobMask(mask: string): string[];
|
|
36
|
+
export declare const DEFAULT_MULTI_GET_MAX_BYTES: number;
|
|
37
|
+
export declare const DEFAULT_EMBED_MAX_DOCS_PER_BATCH = 64;
|
|
38
|
+
export declare const DEFAULT_EMBED_MAX_BATCH_BYTES: number;
|
|
39
|
+
export declare const DEFAULT_EMBED_MAX_DURATION_MS: number;
|
|
40
|
+
export declare const CHUNK_SIZE_TOKENS = 900;
|
|
41
|
+
export declare const CHUNK_OVERLAP_TOKENS: number;
|
|
42
|
+
export declare const CHUNK_SIZE_CHARS: number;
|
|
43
|
+
export declare const CHUNK_OVERLAP_CHARS: number;
|
|
44
|
+
export declare const CHUNK_WINDOW_TOKENS = 200;
|
|
45
|
+
export declare const CHUNK_WINDOW_CHARS: number;
|
|
46
|
+
export declare function getEmbeddingFingerprint(model?: string): string;
|
|
47
|
+
/**
|
|
48
|
+
* A potential break point in the document with a base score indicating quality.
|
|
49
|
+
*/
|
|
50
|
+
export interface BreakPoint {
|
|
51
|
+
pos: number;
|
|
52
|
+
score: number;
|
|
53
|
+
type: string;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* A region where a code fence exists (between ``` markers).
|
|
57
|
+
* We should never split inside a code fence.
|
|
58
|
+
*/
|
|
59
|
+
export interface CodeFenceRegion {
|
|
60
|
+
start: number;
|
|
61
|
+
end: number;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Patterns for detecting break points in markdown documents.
|
|
65
|
+
* Higher scores indicate better places to split.
|
|
66
|
+
* Scores are spread wide so headings decisively beat lower-quality breaks.
|
|
67
|
+
* Order matters for scoring - more specific patterns first.
|
|
68
|
+
*/
|
|
69
|
+
export declare const BREAK_PATTERNS: [RegExp, number, string][];
|
|
70
|
+
/**
|
|
71
|
+
* Scan text for all potential break points.
|
|
72
|
+
* Returns sorted array of break points with higher-scoring patterns taking precedence
|
|
73
|
+
* when multiple patterns match the same position.
|
|
74
|
+
*/
|
|
75
|
+
export declare function scanBreakPoints(text: string): BreakPoint[];
|
|
76
|
+
/**
|
|
77
|
+
* Find all code fence regions in the text.
|
|
78
|
+
* Code fences are delimited by ``` and we should never split inside them.
|
|
79
|
+
*/
|
|
80
|
+
export declare function findCodeFences(text: string): CodeFenceRegion[];
|
|
81
|
+
/**
|
|
82
|
+
* Check if a position is inside a code fence region.
|
|
83
|
+
*/
|
|
84
|
+
export declare function isInsideCodeFence(pos: number, fences: CodeFenceRegion[]): boolean;
|
|
85
|
+
/**
|
|
86
|
+
* Find the best cut position using scored break points with distance decay.
|
|
87
|
+
*
|
|
88
|
+
* Uses squared distance for gentler early decay - headings far back still win
|
|
89
|
+
* over low-quality breaks near the target.
|
|
90
|
+
*
|
|
91
|
+
* @param breakPoints - Pre-scanned break points from scanBreakPoints()
|
|
92
|
+
* @param targetCharPos - The ideal cut position (e.g., maxChars boundary)
|
|
93
|
+
* @param windowChars - How far back to search for break points (default ~200 tokens)
|
|
94
|
+
* @param decayFactor - How much to penalize distance (0.7 = 30% score at window edge)
|
|
95
|
+
* @param codeFences - Code fence regions to avoid splitting inside
|
|
96
|
+
* @returns The best position to cut at
|
|
97
|
+
*/
|
|
98
|
+
export declare function findBestCutoff(breakPoints: BreakPoint[], targetCharPos: number, windowChars?: number, decayFactor?: number, codeFences?: CodeFenceRegion[]): number;
|
|
99
|
+
export type ChunkStrategy = "auto" | "regex";
|
|
100
|
+
/**
|
|
101
|
+
* Merge two sets of break points (e.g. regex + AST), keeping the highest
|
|
102
|
+
* score at each position. Result is sorted by position.
|
|
103
|
+
*/
|
|
104
|
+
export declare function mergeBreakPoints(a: BreakPoint[], b: BreakPoint[]): BreakPoint[];
|
|
105
|
+
/**
|
|
106
|
+
* Core chunk algorithm that operates on precomputed break points and code fences.
|
|
107
|
+
* This is the shared implementation used by both regex-only and AST-aware chunking.
|
|
108
|
+
*/
|
|
109
|
+
export declare function chunkDocumentWithBreakPoints(content: string, breakPoints: BreakPoint[], codeFences: CodeFenceRegion[], maxChars?: number, overlapChars?: number, windowChars?: number): {
|
|
110
|
+
text: string;
|
|
111
|
+
pos: number;
|
|
112
|
+
}[];
|
|
113
|
+
export declare const STRONG_SIGNAL_MIN_SCORE = 0.85;
|
|
114
|
+
export declare const STRONG_SIGNAL_MIN_GAP = 0.15;
|
|
115
|
+
export declare const RERANK_CANDIDATE_LIMIT = 40;
|
|
116
|
+
/**
|
|
117
|
+
* A typed query expansion result. Decoupled from llm.ts internal Queryable —
|
|
118
|
+
* same shape, but store.ts owns its own public API type.
|
|
119
|
+
*
|
|
120
|
+
* - lex: keyword variant → routes to FTS only
|
|
121
|
+
* - vec: semantic variant → routes to vector only
|
|
122
|
+
* - hyde: hypothetical document → routes to vector only
|
|
123
|
+
*/
|
|
124
|
+
export type ExpandedQuery = {
|
|
125
|
+
type: 'lex' | 'vec' | 'hyde';
|
|
126
|
+
query: string;
|
|
127
|
+
/** Optional line number for error reporting (CLI parser) */
|
|
128
|
+
line?: number;
|
|
129
|
+
};
|
|
130
|
+
export declare function homedir(): string;
|
|
131
|
+
/**
|
|
132
|
+
* Check if a path is absolute.
|
|
133
|
+
* Supports:
|
|
134
|
+
* - Unix paths: /path/to/file
|
|
135
|
+
* - Windows native: C:\path or C:/path
|
|
136
|
+
* - Git Bash: /c/path or /C/path (C-Z drives, excluding A/B floppy drives)
|
|
137
|
+
*
|
|
138
|
+
* Note: /c without trailing slash is treated as Unix path (directory named "c"),
|
|
139
|
+
* while /c/ or /c/path are treated as Git Bash paths (C: drive).
|
|
140
|
+
*/
|
|
141
|
+
export declare function isAbsolutePath(path: string): boolean;
|
|
142
|
+
/**
|
|
143
|
+
* Normalize path separators to forward slashes.
|
|
144
|
+
* Converts Windows backslashes to forward slashes.
|
|
145
|
+
*/
|
|
146
|
+
export declare function normalizePathSeparators(path: string): string;
|
|
147
|
+
/**
|
|
148
|
+
* Get the relative path from a prefix.
|
|
149
|
+
* Returns null if path is not under prefix.
|
|
150
|
+
* Returns empty string if path equals prefix.
|
|
151
|
+
*/
|
|
152
|
+
export declare function getRelativePathFromPrefix(path: string, prefix: string): string | null;
|
|
153
|
+
export declare function resolve(...paths: string[]): string;
|
|
154
|
+
export declare function enableProductionMode(): void;
|
|
155
|
+
/** Reset production mode flag — only for testing. */
|
|
156
|
+
export declare function _resetProductionModeForTesting(): void;
|
|
157
|
+
export declare function getDefaultDbPath(indexName?: string): string;
|
|
158
|
+
export declare function getPwd(): string;
|
|
159
|
+
export declare function getRealPath(path: string): string;
|
|
160
|
+
/**
|
|
161
|
+
* True if `target` is `dir` or a descendant, after resolving symlinks.
|
|
162
|
+
* Used to keep indexing and qmd:// filesystem resolution inside a collection.
|
|
163
|
+
*/
|
|
164
|
+
export declare function isPathInsideDir(dir: string, target: string): boolean;
|
|
165
|
+
export type VirtualPath = {
|
|
166
|
+
collectionName: string;
|
|
167
|
+
path: string;
|
|
168
|
+
indexName?: string;
|
|
169
|
+
};
|
|
170
|
+
/**
|
|
171
|
+
* Normalize explicit virtual path formats to standard qmd:// format.
|
|
172
|
+
* Only handles paths that are already explicitly virtual:
|
|
173
|
+
* - qmd://collection/path.md (already normalized)
|
|
174
|
+
* - qmd:////collection/path.md (extra slashes - normalize)
|
|
175
|
+
* - //collection/path.md (missing qmd: prefix - add it)
|
|
176
|
+
*
|
|
177
|
+
* Does NOT handle:
|
|
178
|
+
* - collection/path.md (bare paths - could be filesystem relative)
|
|
179
|
+
* - :linenum suffix (should be parsed separately before calling this)
|
|
180
|
+
*/
|
|
181
|
+
export declare function normalizeVirtualPath(input: string): string;
|
|
182
|
+
/**
|
|
183
|
+
* Parse a virtual path like "qmd://collection-name/path/to/file.md"
|
|
184
|
+
* into its components.
|
|
185
|
+
* Also supports collection root: "qmd://collection-name/" or "qmd://collection-name"
|
|
186
|
+
*/
|
|
187
|
+
export declare function parseVirtualPath(virtualPath: string): VirtualPath | null;
|
|
188
|
+
/**
|
|
189
|
+
* Build a virtual path from collection name and relative path.
|
|
190
|
+
*/
|
|
191
|
+
export declare function buildVirtualPath(collectionName: string, path: string, indexName?: string): string;
|
|
192
|
+
/**
|
|
193
|
+
* Check if a path is explicitly a virtual path.
|
|
194
|
+
* Only recognizes explicit virtual path formats:
|
|
195
|
+
* - qmd://collection/path.md
|
|
196
|
+
* - //collection/path.md
|
|
197
|
+
*
|
|
198
|
+
* Does NOT consider bare collection/path.md as virtual - that should be
|
|
199
|
+
* handled separately by checking if the first component is a collection name.
|
|
200
|
+
*/
|
|
201
|
+
export declare function isVirtualPath(path: string): boolean;
|
|
202
|
+
/**
|
|
203
|
+
* Resolve a virtual path to absolute filesystem path.
|
|
204
|
+
*/
|
|
205
|
+
export declare function resolveVirtualPath(db: Database, virtualPath: string): string | null;
|
|
206
|
+
/**
|
|
207
|
+
* Convert an absolute filesystem path to a virtual path.
|
|
208
|
+
* Returns null if the file is not in any indexed collection.
|
|
209
|
+
*/
|
|
210
|
+
export declare function toVirtualPath(db: Database, absolutePath: string): string | null;
|
|
211
|
+
export declare function verifySqliteVecLoaded(db: Database): void;
|
|
212
|
+
/**
|
|
213
|
+
* FTS5's unicode61 tokenizer does not segment CJK text into searchable words.
|
|
214
|
+
* Normalize CJK runs by spacing every character so exact CJK queries can be
|
|
215
|
+
* translated into phrase queries while Latin text keeps the default tokenizer.
|
|
216
|
+
*/
|
|
217
|
+
export declare function normalizeCjkForFTS(text: string): string;
|
|
218
|
+
export declare function getStoreCollections(db: Database): NamedCollection[];
|
|
219
|
+
export declare function getStoreCollection(db: Database, name: string): NamedCollection | null;
|
|
220
|
+
export declare function getStoreGlobalContext(db: Database): string | undefined;
|
|
221
|
+
export declare function getStoreContexts(db: Database): Array<{
|
|
222
|
+
collection: string;
|
|
223
|
+
path: string;
|
|
224
|
+
context: string;
|
|
225
|
+
}>;
|
|
226
|
+
export declare function upsertStoreCollection(db: Database, name: string, collection: Omit<Collection, 'pattern'> & {
|
|
227
|
+
pattern?: string;
|
|
228
|
+
}): void;
|
|
229
|
+
export declare function deleteStoreCollection(db: Database, name: string): boolean;
|
|
230
|
+
export declare function renameStoreCollection(db: Database, oldName: string, newName: string): boolean;
|
|
231
|
+
export declare function updateStoreContext(db: Database, collectionName: string, path: string, text: string): boolean;
|
|
232
|
+
export declare function removeStoreContext(db: Database, collectionName: string, path: string): boolean;
|
|
233
|
+
export declare function setStoreGlobalContext(db: Database, value: string | undefined): void;
|
|
234
|
+
/**
|
|
235
|
+
* Sync external config (YAML/inline) into SQLite store_collections.
|
|
236
|
+
* External config always wins. Skips sync if config hash hasn't changed.
|
|
237
|
+
*/
|
|
238
|
+
export declare function syncConfigToDb(db: Database, config: CollectionConfig): void;
|
|
239
|
+
export declare function isSqliteVecAvailable(): boolean;
|
|
240
|
+
export type Store = {
|
|
241
|
+
db: Database;
|
|
242
|
+
dbPath: string;
|
|
243
|
+
/** Optional LlamaCpp instance for this store (overrides the global singleton) */
|
|
244
|
+
llm?: LlamaCpp;
|
|
245
|
+
close: () => void;
|
|
246
|
+
ensureVecTable: (dimensions: number) => void;
|
|
247
|
+
getHashesNeedingEmbedding: (model?: string) => number;
|
|
248
|
+
getIndexHealth: (model?: string) => IndexHealthInfo;
|
|
249
|
+
getStatus: (model?: string) => IndexStatus;
|
|
250
|
+
getCacheKey: typeof getCacheKey;
|
|
251
|
+
getCachedResult: (cacheKey: string) => string | null;
|
|
252
|
+
setCachedResult: (cacheKey: string, result: string) => void;
|
|
253
|
+
clearCache: () => void;
|
|
254
|
+
deleteLLMCache: () => number;
|
|
255
|
+
deleteInactiveDocuments: () => number;
|
|
256
|
+
cleanupOrphanedContent: () => number;
|
|
257
|
+
cleanupOrphanedVectors: () => number;
|
|
258
|
+
vacuumDatabase: () => void;
|
|
259
|
+
getContextForFile: (filepath: string) => string | null;
|
|
260
|
+
getContextForPath: (collectionName: string, path: string) => string | null;
|
|
261
|
+
getCollectionByName: (name: string) => {
|
|
262
|
+
name: string;
|
|
263
|
+
pwd: string;
|
|
264
|
+
glob_pattern: string;
|
|
265
|
+
} | null;
|
|
266
|
+
getCollectionsWithoutContext: () => {
|
|
267
|
+
name: string;
|
|
268
|
+
pwd: string;
|
|
269
|
+
doc_count: number;
|
|
270
|
+
}[];
|
|
271
|
+
getTopLevelPathsWithoutContext: (collectionName: string) => string[];
|
|
272
|
+
parseVirtualPath: typeof parseVirtualPath;
|
|
273
|
+
buildVirtualPath: typeof buildVirtualPath;
|
|
274
|
+
isVirtualPath: typeof isVirtualPath;
|
|
275
|
+
resolveVirtualPath: (virtualPath: string) => string | null;
|
|
276
|
+
toVirtualPath: (absolutePath: string) => string | null;
|
|
277
|
+
searchFTS: (query: string, limit?: number, collectionName?: string | readonly string[], filter?: MetadataFilter, scope?: DocumentFilter, onScanCapped?: () => void) => SearchResult[];
|
|
278
|
+
searchVec: (query: string, model: string, limit?: number, collectionName?: string | readonly string[], session?: ILLMSession, precomputedEmbedding?: number[], filter?: MetadataFilter, scope?: DocumentFilter) => Promise<SearchResult[]>;
|
|
279
|
+
/** `onModelReady` fires once the generation model is loaded (not at all on a cache hit). */
|
|
280
|
+
expandQuery: (query: string, model?: string, options?: {
|
|
281
|
+
onModelReady?: () => void;
|
|
282
|
+
}) => Promise<ExpandedQuery[]>;
|
|
283
|
+
/** Drop the cached expansion for a query so the next call regenerates. */
|
|
284
|
+
invalidateExpansionCache: (query: string) => void;
|
|
285
|
+
rerank: (query: string, documents: {
|
|
286
|
+
file: string;
|
|
287
|
+
text: string;
|
|
288
|
+
}[], model?: string, intent?: string, options?: StoreRerankOptions) => Promise<{
|
|
289
|
+
file: string;
|
|
290
|
+
score: number;
|
|
291
|
+
}[]>;
|
|
292
|
+
findDocument: (filename: string, options?: {
|
|
293
|
+
includeBody?: boolean;
|
|
294
|
+
}) => DocumentResult | DocumentLookupError;
|
|
295
|
+
getDocumentBody: (doc: DocumentResult | {
|
|
296
|
+
filepath: string;
|
|
297
|
+
}, fromLine?: number, maxLines?: number) => string | null;
|
|
298
|
+
findDocuments: (pattern: string, options?: {
|
|
299
|
+
includeBody?: boolean;
|
|
300
|
+
maxBytes?: number;
|
|
301
|
+
}) => {
|
|
302
|
+
docs: MultiGetResult[];
|
|
303
|
+
errors: string[];
|
|
304
|
+
};
|
|
305
|
+
findSimilarFiles: (query: string, maxDistance?: number, limit?: number) => string[];
|
|
306
|
+
matchFilesByGlob: (pattern: string) => {
|
|
307
|
+
filepath: string;
|
|
308
|
+
displayPath: string;
|
|
309
|
+
bodyLength: number;
|
|
310
|
+
}[];
|
|
311
|
+
findDocumentByDocid: (docid: string) => {
|
|
312
|
+
filepath: string;
|
|
313
|
+
hash: string;
|
|
314
|
+
} | null;
|
|
315
|
+
insertContent: (hash: string, content: string, createdAt: string) => void;
|
|
316
|
+
insertDocument: (collectionName: string, path: string, title: string, hash: string, createdAt: string, modifiedAt: string) => number;
|
|
317
|
+
findActiveDocument: (collectionName: string, path: string) => {
|
|
318
|
+
id: number;
|
|
319
|
+
hash: string;
|
|
320
|
+
title: string;
|
|
321
|
+
} | null;
|
|
322
|
+
findOrMigrateLegacyDocument: (collectionName: string, path: string) => {
|
|
323
|
+
id: number;
|
|
324
|
+
hash: string;
|
|
325
|
+
title: string;
|
|
326
|
+
} | null;
|
|
327
|
+
updateDocumentTitle: (documentId: number, title: string, modifiedAt: string) => void;
|
|
328
|
+
updateDocument: (documentId: number, title: string, hash: string, modifiedAt: string) => void;
|
|
329
|
+
deactivateDocument: (collectionName: string, path: string) => void;
|
|
330
|
+
getActiveDocumentPaths: (collectionName: string) => string[];
|
|
331
|
+
getHashesForEmbedding: () => {
|
|
332
|
+
hash: string;
|
|
333
|
+
body: string;
|
|
334
|
+
path: string;
|
|
335
|
+
}[];
|
|
336
|
+
clearAllEmbeddings: () => void;
|
|
337
|
+
insertEmbedding: (hash: string, seq: number, pos: number, embedding: Float32Array, model: string, embeddedAt: string, totalChunks?: number, fingerprint?: string) => void;
|
|
338
|
+
};
|
|
339
|
+
export type ReindexProgress = {
|
|
340
|
+
file: string;
|
|
341
|
+
current: number;
|
|
342
|
+
total: number;
|
|
343
|
+
};
|
|
344
|
+
export type ReindexSkippedFile = {
|
|
345
|
+
file: string;
|
|
346
|
+
code: string;
|
|
347
|
+
};
|
|
348
|
+
export type ReindexResult = {
|
|
349
|
+
indexed: number;
|
|
350
|
+
updated: number;
|
|
351
|
+
unchanged: number;
|
|
352
|
+
removed: number;
|
|
353
|
+
orphanedCleaned: number;
|
|
354
|
+
skipped: number;
|
|
355
|
+
skippedFiles: ReindexSkippedFile[];
|
|
356
|
+
/** Documents whose qmd.metadata frontmatter failed extraction this pass. */
|
|
357
|
+
metadataErrors: number;
|
|
358
|
+
};
|
|
359
|
+
/**
|
|
360
|
+
* The files a collection is made of, relative to its root: the glob, minus dependency and
|
|
361
|
+
* build directories, minus the collection's own ignore patterns, minus anything hidden.
|
|
362
|
+
* One definition, because the daemon's refresh has to see exactly the set that
|
|
363
|
+
* reindexCollection indexes or it would decide staleness about different files.
|
|
364
|
+
*/
|
|
365
|
+
export declare function listCollectionFiles(collectionPath: string, globPattern: string, ignorePatterns?: string[]): Promise<string[]>;
|
|
366
|
+
/**
|
|
367
|
+
* Re-index a single collection by scanning the filesystem and updating the database.
|
|
368
|
+
* Pure function — no console output, no db lifecycle management.
|
|
369
|
+
*/
|
|
370
|
+
export declare function reindexCollection(store: Store, collectionPath: string, globPattern: string, collectionName: string, options?: {
|
|
371
|
+
ignorePatterns?: string[];
|
|
372
|
+
onProgress?: (info: ReindexProgress) => void;
|
|
373
|
+
}): Promise<ReindexResult>;
|
|
374
|
+
export type EmbedFailure = {
|
|
375
|
+
path: string;
|
|
376
|
+
hash: string;
|
|
377
|
+
seq: number;
|
|
378
|
+
attempts: number;
|
|
379
|
+
reason: string;
|
|
380
|
+
};
|
|
381
|
+
export type EmbedProgress = {
|
|
382
|
+
chunksEmbedded: number;
|
|
383
|
+
totalChunks: number;
|
|
384
|
+
bytesProcessed: number;
|
|
385
|
+
totalBytes: number;
|
|
386
|
+
/** Active failed chunks still awaiting a successful retry. */
|
|
387
|
+
errors: number;
|
|
388
|
+
failures?: EmbedFailure[];
|
|
389
|
+
};
|
|
390
|
+
export type EmbedResult = {
|
|
391
|
+
docsProcessed: number;
|
|
392
|
+
chunksEmbedded: number;
|
|
393
|
+
/** Active failed chunks that did not recover after retries. */
|
|
394
|
+
errors: number;
|
|
395
|
+
failures?: EmbedFailure[];
|
|
396
|
+
durationMs: number;
|
|
397
|
+
};
|
|
398
|
+
export type EmbedOptions = {
|
|
399
|
+
force?: boolean;
|
|
400
|
+
model?: string;
|
|
401
|
+
/**
|
|
402
|
+
* Restrict embedding to documents in a single collection.
|
|
403
|
+
* When omitted, all pending documents across every collection are embedded.
|
|
404
|
+
*/
|
|
405
|
+
collection?: string;
|
|
406
|
+
maxDocsPerBatch?: number;
|
|
407
|
+
maxBatchBytes?: number;
|
|
408
|
+
chunkStrategy?: ChunkStrategy;
|
|
409
|
+
/**
|
|
410
|
+
* Max wall-clock duration for the whole embed session, in milliseconds. When the
|
|
411
|
+
* cap is reached, remaining document batches are skipped (re-run `qmd embed` to
|
|
412
|
+
* continue). A value <= 0 disables the cap. Defaults to
|
|
413
|
+
* {@link DEFAULT_EMBED_MAX_DURATION_MS} (30 minutes).
|
|
414
|
+
*/
|
|
415
|
+
maxDurationMs?: number;
|
|
416
|
+
onProgress?: (info: EmbedProgress) => void;
|
|
417
|
+
};
|
|
418
|
+
/**
|
|
419
|
+
* Generate vector embeddings for documents that need them.
|
|
420
|
+
* Pure function — no console output, no db lifecycle management.
|
|
421
|
+
* Uses the store's LlamaCpp instance if set, otherwise the global singleton.
|
|
422
|
+
*/
|
|
423
|
+
export declare function generateEmbeddings(store: Store, options?: EmbedOptions): Promise<EmbedResult>;
|
|
424
|
+
/**
|
|
425
|
+
* Create a new store instance with the given database path.
|
|
426
|
+
* If no path is provided, uses the default path (~/.cache/qmd/index.sqlite).
|
|
427
|
+
*
|
|
428
|
+
* @param dbPath - Path to the SQLite database file
|
|
429
|
+
* @returns Store instance with all methods bound to the database
|
|
430
|
+
*/
|
|
431
|
+
export declare function createStore(dbPath?: string): Store;
|
|
432
|
+
/**
|
|
433
|
+
* Unified document result type with all metadata.
|
|
434
|
+
* Body is optional - use getDocumentBody() to load it separately if needed.
|
|
435
|
+
*/
|
|
436
|
+
export type DocumentResult = {
|
|
437
|
+
filepath: string;
|
|
438
|
+
displayPath: string;
|
|
439
|
+
title: string;
|
|
440
|
+
context: string | null;
|
|
441
|
+
hash: string;
|
|
442
|
+
docid: string;
|
|
443
|
+
collectionName: string;
|
|
444
|
+
modifiedAt: string;
|
|
445
|
+
bodyLength: number;
|
|
446
|
+
body?: string;
|
|
447
|
+
};
|
|
448
|
+
/**
|
|
449
|
+
* Extract short docid from a full hash (first 6 characters).
|
|
450
|
+
*/
|
|
451
|
+
export declare function getDocid(hash: string): string;
|
|
452
|
+
export declare function handelize(path: string): string;
|
|
453
|
+
/**
|
|
454
|
+
* Search result extends DocumentResult with score and source info
|
|
455
|
+
*/
|
|
456
|
+
export type SearchResult = DocumentResult & {
|
|
457
|
+
metadata: DocumentMetadata;
|
|
458
|
+
score: number;
|
|
459
|
+
source: "fts" | "vec";
|
|
460
|
+
matchMode?: "all" | "any";
|
|
461
|
+
chunkPos?: number;
|
|
462
|
+
};
|
|
463
|
+
/**
|
|
464
|
+
* Ranked result for RRF fusion (simplified, used internally)
|
|
465
|
+
*/
|
|
466
|
+
export type RankedResult = {
|
|
467
|
+
file: string;
|
|
468
|
+
displayPath: string;
|
|
469
|
+
title: string;
|
|
470
|
+
body: string;
|
|
471
|
+
score: number;
|
|
472
|
+
};
|
|
473
|
+
export type RRFContributionTrace = {
|
|
474
|
+
listIndex: number;
|
|
475
|
+
source: "fts" | "vec";
|
|
476
|
+
queryType: "original" | "lex" | "vec" | "hyde";
|
|
477
|
+
query: string;
|
|
478
|
+
rank: number;
|
|
479
|
+
weight: number;
|
|
480
|
+
multiplier?: number;
|
|
481
|
+
backendScore: number;
|
|
482
|
+
rrfContribution: number;
|
|
483
|
+
};
|
|
484
|
+
export type RRFScoreTrace = {
|
|
485
|
+
contributions: RRFContributionTrace[];
|
|
486
|
+
baseScore: number;
|
|
487
|
+
topRank: number;
|
|
488
|
+
topRankBonus: number;
|
|
489
|
+
totalScore: number;
|
|
490
|
+
};
|
|
491
|
+
export type HybridQueryExplain = {
|
|
492
|
+
ftsScores: number[];
|
|
493
|
+
vectorScores: number[];
|
|
494
|
+
rrf: {
|
|
495
|
+
rank: number;
|
|
496
|
+
positionScore: number;
|
|
497
|
+
weight: number;
|
|
498
|
+
baseScore: number;
|
|
499
|
+
topRankBonus: number;
|
|
500
|
+
totalScore: number;
|
|
501
|
+
contributions: RRFContributionTrace[];
|
|
502
|
+
};
|
|
503
|
+
rerankScore: number;
|
|
504
|
+
blendedScore: number;
|
|
505
|
+
};
|
|
506
|
+
/**
|
|
507
|
+
* Error result when document is not found
|
|
508
|
+
*/
|
|
509
|
+
export type DocumentNotFound = {
|
|
510
|
+
error: "not_found";
|
|
511
|
+
query: string;
|
|
512
|
+
similarFiles: string[];
|
|
513
|
+
};
|
|
514
|
+
export type DocumentExcludedByIgnore = {
|
|
515
|
+
error: "excluded_by_ignore";
|
|
516
|
+
query: string;
|
|
517
|
+
collection: string;
|
|
518
|
+
path: string;
|
|
519
|
+
rule: string;
|
|
520
|
+
};
|
|
521
|
+
export type DocumentLookupError = DocumentNotFound | DocumentExcludedByIgnore;
|
|
522
|
+
/**
|
|
523
|
+
* Result from multi-get operations
|
|
524
|
+
*/
|
|
525
|
+
export type MultiGetResult = {
|
|
526
|
+
doc: DocumentResult;
|
|
527
|
+
skipped: false;
|
|
528
|
+
} | {
|
|
529
|
+
doc: Pick<DocumentResult, "filepath" | "displayPath">;
|
|
530
|
+
skipped: true;
|
|
531
|
+
skipReason: string;
|
|
532
|
+
};
|
|
533
|
+
export type CollectionInfo = {
|
|
534
|
+
name: string;
|
|
535
|
+
path: string | null;
|
|
536
|
+
pattern: string | null;
|
|
537
|
+
documents: number;
|
|
538
|
+
lastUpdated: string;
|
|
539
|
+
};
|
|
540
|
+
export type IndexStatus = {
|
|
541
|
+
totalDocuments: number;
|
|
542
|
+
needsEmbedding: number;
|
|
543
|
+
hasVectorIndex: boolean;
|
|
544
|
+
/** Active documents without current, error-free metadata extraction. */
|
|
545
|
+
pendingMetadata: number;
|
|
546
|
+
collections: CollectionInfo[];
|
|
547
|
+
};
|
|
548
|
+
export declare function getHashesNeedingEmbedding(db: Database, collection?: string, model?: string): number;
|
|
549
|
+
export type IndexHealthInfo = {
|
|
550
|
+
needsEmbedding: number;
|
|
551
|
+
totalDocs: number;
|
|
552
|
+
daysStale: number | null;
|
|
553
|
+
};
|
|
554
|
+
export type LegacyFingerprintAdoptionResult = {
|
|
555
|
+
checked: boolean;
|
|
556
|
+
adopted: number;
|
|
557
|
+
reason: string;
|
|
558
|
+
};
|
|
559
|
+
export declare function maybeAdoptLegacyEmbeddingFingerprint(store: Store, model?: string): Promise<LegacyFingerprintAdoptionResult>;
|
|
560
|
+
export declare function getIndexHealth(db: Database, model?: string): IndexHealthInfo;
|
|
561
|
+
export type CacheKeyBody = {
|
|
562
|
+
query?: string;
|
|
563
|
+
model?: string;
|
|
564
|
+
chunk?: string;
|
|
565
|
+
file?: string;
|
|
566
|
+
};
|
|
567
|
+
export declare function getCacheKey(url: string, body: CacheKeyBody): string;
|
|
568
|
+
export declare function getCachedResult(db: Database, cacheKey: string): string | null;
|
|
569
|
+
export declare function setCachedResult(db: Database, cacheKey: string, result: string): void;
|
|
570
|
+
export declare function clearCache(db: Database): void;
|
|
571
|
+
/**
|
|
572
|
+
* Delete cached LLM API responses.
|
|
573
|
+
* Returns the number of cached responses deleted.
|
|
574
|
+
*/
|
|
575
|
+
export declare function deleteLLMCache(db: Database): number;
|
|
576
|
+
/**
|
|
577
|
+
* Remove inactive document records (active = 0).
|
|
578
|
+
* Returns the number of inactive documents deleted.
|
|
579
|
+
*/
|
|
580
|
+
export declare function deleteInactiveDocuments(db: Database): number;
|
|
581
|
+
/**
|
|
582
|
+
* Remove orphaned content hashes that are not referenced by any document.
|
|
583
|
+
* Inactive documents are soft-deleted tombstones, so their content rows must
|
|
584
|
+
* remain referenced until deleteInactiveDocuments() hard-deletes them.
|
|
585
|
+
* Returns the number of orphaned content hashes deleted.
|
|
586
|
+
*/
|
|
587
|
+
export declare function cleanupOrphanedContent(db: Database): number;
|
|
588
|
+
/**
|
|
589
|
+
* Count content hashes that would be unreferenced after inactive documents
|
|
590
|
+
* are hard-deleted. Shared hashes still used by an active document are kept.
|
|
591
|
+
*/
|
|
592
|
+
export declare function countOrphanedContent(db: Database): number;
|
|
593
|
+
/**
|
|
594
|
+
* Count embedding chunks whose hash is not referenced by any active document.
|
|
595
|
+
* Reads `content_vectors` only, so this works even when sqlite-vec is unavailable (#768).
|
|
596
|
+
*/
|
|
597
|
+
export declare function countOrphanedVectors(db: Database): number;
|
|
598
|
+
/**
|
|
599
|
+
* Remove orphaned vector embeddings that are not referenced by any active document.
|
|
600
|
+
* Returns the number of orphaned embedding chunks deleted.
|
|
601
|
+
*/
|
|
602
|
+
export declare function cleanupOrphanedVectors(db: Database): number;
|
|
603
|
+
/**
|
|
604
|
+
* Run VACUUM to reclaim unused space in the database.
|
|
605
|
+
* This operation rebuilds the database file to eliminate fragmentation.
|
|
606
|
+
*/
|
|
607
|
+
export declare function vacuumDatabase(db: Database): void;
|
|
608
|
+
/**
|
|
609
|
+
* Merge FTS5 b-trees so deleted rows (deactivated / hard-deleted documents)
|
|
610
|
+
* leave `documents_fts_data`. VACUUM alone does not compact FTS5 (#550).
|
|
611
|
+
*/
|
|
612
|
+
export declare function optimizeDocumentsFts(db: Database): void;
|
|
613
|
+
export type CleanupStats = {
|
|
614
|
+
cacheCount: number;
|
|
615
|
+
orphanedVectors: number;
|
|
616
|
+
inactiveDocs: number;
|
|
617
|
+
orphanedContent: number;
|
|
618
|
+
};
|
|
619
|
+
/** Counts what `runCleanup` would remove, including content only held by inactive docs. */
|
|
620
|
+
export declare function previewCleanup(db: Database): CleanupStats;
|
|
621
|
+
/**
|
|
622
|
+
* Full `qmd cleanup` sequence: drop cache, orphaned vectors, inactive document
|
|
623
|
+
* rows, then the content those rows were pinning, compact FTS5, vacuum.
|
|
624
|
+
*/
|
|
625
|
+
export declare function runCleanup(db: Database): CleanupStats;
|
|
626
|
+
export declare function hashContent(content: string): Promise<string>;
|
|
627
|
+
export declare function extractTitle(content: string, filename: string): string;
|
|
628
|
+
/**
|
|
629
|
+
* Insert content into the content table (content-addressable storage).
|
|
630
|
+
* Uses INSERT OR IGNORE so duplicate hashes are skipped.
|
|
631
|
+
*/
|
|
632
|
+
export declare function insertContent(db: Database, hash: string, content: string, createdAt: string): void;
|
|
633
|
+
/**
|
|
634
|
+
* Insert a new document into the documents table.
|
|
635
|
+
* Returns the document's id so callers can attach document-scoped state.
|
|
636
|
+
*/
|
|
637
|
+
export declare function insertDocument(db: Database, collectionName: string, path: string, title: string, hash: string, createdAt: string, modifiedAt: string): number;
|
|
638
|
+
/**
|
|
639
|
+
* Find an active document by collection name and path.
|
|
640
|
+
*/
|
|
641
|
+
export declare function findActiveDocument(db: Database, collectionName: string, path: string): {
|
|
642
|
+
id: number;
|
|
643
|
+
hash: string;
|
|
644
|
+
title: string;
|
|
645
|
+
} | null;
|
|
646
|
+
/**
|
|
647
|
+
* Find an active document, falling back to a legacy handalized-path match.
|
|
648
|
+
* If found under the pre-2.6 slug, renames it in-place and rebuilds the
|
|
649
|
+
* FTS entry. Embeddings are keyed by content hash, so the rename is
|
|
650
|
+
* safe — no re-embedding required.
|
|
651
|
+
*
|
|
652
|
+
* `livePaths`, when given, is the set of literal paths of every file in the
|
|
653
|
+
* current scan of this collection. A legacy row whose path is in that set
|
|
654
|
+
* belongs to a *different* file that still exists on disk, so it must never be
|
|
655
|
+
* adopted — renaming it would evict that file from the index (#717).
|
|
656
|
+
*
|
|
657
|
+
* @internal Used by reindexCollection and indexFiles during qmd update.
|
|
658
|
+
* Returns null if the document does not exist under either path.
|
|
659
|
+
*/
|
|
660
|
+
export declare function findOrMigrateLegacyDocument(db: Database, collectionName: string, path: string, livePaths?: ReadonlySet<string>): {
|
|
661
|
+
id: number;
|
|
662
|
+
hash: string;
|
|
663
|
+
title: string;
|
|
664
|
+
} | null;
|
|
665
|
+
/**
|
|
666
|
+
* Update the title and modified_at timestamp for a document.
|
|
667
|
+
*
|
|
668
|
+
* Title-only fast path: callers invoke this only when the content hash is
|
|
669
|
+
* unchanged, so the body needs no re-tokenization. The documents_au sync
|
|
670
|
+
* trigger is scoped to (collection, path, hash, active) and does not fire
|
|
671
|
+
* for this write, so a single FTS title UPDATE preserves the existing
|
|
672
|
+
* normalized filepath/body — skipping both the body re-normalization and
|
|
673
|
+
* the redundant raw DELETE+INSERT rewrite (~5ms per 105KB document).
|
|
674
|
+
* When no FTS row exists, fall back to a full rebuild so the title is still
|
|
675
|
+
* indexed (and any orphan row is cleaned up, as before).
|
|
676
|
+
*/
|
|
677
|
+
export declare function updateDocumentTitle(db: Database, documentId: number, title: string, modifiedAt: string): void;
|
|
678
|
+
/**
|
|
679
|
+
* Update an existing document's hash, title, and modified_at timestamp.
|
|
680
|
+
* Used when content changes but the file path stays the same.
|
|
681
|
+
*/
|
|
682
|
+
export declare function updateDocument(db: Database, documentId: number, title: string, hash: string, modifiedAt: string): void;
|
|
683
|
+
/**
|
|
684
|
+
* Deactivate a document (mark as inactive but don't delete).
|
|
685
|
+
*/
|
|
686
|
+
export declare function deactivateDocument(db: Database, collectionName: string, path: string): void;
|
|
687
|
+
/**
|
|
688
|
+
* Get all active document paths for a collection.
|
|
689
|
+
*/
|
|
690
|
+
export declare function getActiveDocumentPaths(db: Database, collectionName: string): string[];
|
|
691
|
+
export { formatQueryForEmbedding, formatDocForEmbedding };
|
|
692
|
+
/**
|
|
693
|
+
* Chunk a document using regex-only break point detection.
|
|
694
|
+
* This is the sync, backward-compatible API used by tests and legacy callers.
|
|
695
|
+
*/
|
|
696
|
+
export declare function chunkDocument(content: string, maxChars?: number, overlapChars?: number, windowChars?: number): {
|
|
697
|
+
text: string;
|
|
698
|
+
pos: number;
|
|
699
|
+
}[];
|
|
700
|
+
/**
|
|
701
|
+
* Async AST-aware chunking. Detects language from filepath, computes AST
|
|
702
|
+
* break points for supported code files, merges with regex break points,
|
|
703
|
+
* and delegates to the shared chunk algorithm.
|
|
704
|
+
*
|
|
705
|
+
* Falls back to regex-only when strategy is "regex", filepath is absent,
|
|
706
|
+
* or language is unsupported.
|
|
707
|
+
*/
|
|
708
|
+
export declare function chunkDocumentAsync(content: string, maxChars?: number, overlapChars?: number, windowChars?: number, filepath?: string, chunkStrategy?: ChunkStrategy): Promise<{
|
|
709
|
+
text: string;
|
|
710
|
+
pos: number;
|
|
711
|
+
}[]>;
|
|
712
|
+
export declare function chunkDocumentByTokens(content: string, maxTokens?: number, overlapTokens?: number, windowTokens?: number, filepath?: string, chunkStrategy?: ChunkStrategy, signal?: AbortSignal): Promise<{
|
|
713
|
+
text: string;
|
|
714
|
+
pos: number;
|
|
715
|
+
tokens: number;
|
|
716
|
+
}[]>;
|
|
717
|
+
/**
|
|
718
|
+
* Normalize a docid input by stripping surrounding quotes and leading #.
|
|
719
|
+
* Handles: "#abc123", 'abc123', "abc123", #abc123, abc123
|
|
720
|
+
* Returns the bare hex string.
|
|
721
|
+
*/
|
|
722
|
+
export declare function normalizeDocid(docid: string): string;
|
|
723
|
+
/**
|
|
724
|
+
* Check if a string looks like a docid reference.
|
|
725
|
+
* Accepts: #abc123, abc123, "#abc123", "abc123", '#abc123', 'abc123'
|
|
726
|
+
* Returns true if the normalized form is a valid hex string of 6+ chars.
|
|
727
|
+
*/
|
|
728
|
+
export declare function isDocid(input: string): boolean;
|
|
729
|
+
/**
|
|
730
|
+
* Find a document by its short docid (first 6 characters of hash).
|
|
731
|
+
* Returns the document's virtual path if found, null otherwise.
|
|
732
|
+
* If multiple documents match the same short hash (collision), returns the first one.
|
|
733
|
+
*
|
|
734
|
+
* Accepts lenient input: #abc123, abc123, "#abc123", "abc123"
|
|
735
|
+
*/
|
|
736
|
+
export declare function findDocumentByDocid(db: Database, docid: string): {
|
|
737
|
+
filepath: string;
|
|
738
|
+
hash: string;
|
|
739
|
+
} | null;
|
|
740
|
+
export declare function findSimilarFiles(db: Database, query: string, maxDistance?: number, limit?: number): string[];
|
|
741
|
+
export declare function matchFilesByGlob(db: Database, pattern: string): {
|
|
742
|
+
filepath: string;
|
|
743
|
+
displayPath: string;
|
|
744
|
+
bodyLength: number;
|
|
745
|
+
}[];
|
|
746
|
+
/**
|
|
747
|
+
* Get context for a file path using hierarchical inheritance.
|
|
748
|
+
* Contexts are collection-scoped and inherit from parent directories.
|
|
749
|
+
* For example, context at "/talks" applies to "/talks/2024/keynote.md".
|
|
750
|
+
*
|
|
751
|
+
* @param db Database instance (unused - kept for compatibility)
|
|
752
|
+
* @param collectionName Collection name
|
|
753
|
+
* @param path Relative path within the collection
|
|
754
|
+
* @returns Context string or null if no context is defined
|
|
755
|
+
*/
|
|
756
|
+
export declare function getContextForPath(db: Database, collectionName: string, path: string): string | null;
|
|
757
|
+
/**
|
|
758
|
+
* Get context for a file path (virtual or filesystem).
|
|
759
|
+
* Resolves the collection and relative path from the DB store_collections table.
|
|
760
|
+
*/
|
|
761
|
+
export declare function getContextForFile(db: Database, filepath: string): string | null;
|
|
762
|
+
/**
|
|
763
|
+
* Batch version of {@link getContextForFile}: one collections fetch, one
|
|
764
|
+
* global-context fetch and one document-existence query for the whole batch
|
|
765
|
+
* instead of three per file. Context assembly per file is identical.
|
|
766
|
+
*/
|
|
767
|
+
export declare function getContextsForFiles(db: Database, filepaths: string[]): Map<string, string | null>;
|
|
768
|
+
/**
|
|
769
|
+
* Get collection by name from DB store_collections table.
|
|
770
|
+
*/
|
|
771
|
+
export declare function getCollectionByName(db: Database, name: string): {
|
|
772
|
+
name: string;
|
|
773
|
+
pwd: string;
|
|
774
|
+
glob_pattern: string;
|
|
775
|
+
} | null;
|
|
776
|
+
/**
|
|
777
|
+
* List all collections with document counts from database.
|
|
778
|
+
* Merges store_collections config with database statistics.
|
|
779
|
+
*/
|
|
780
|
+
export declare function listCollections(db: Database): {
|
|
781
|
+
name: string;
|
|
782
|
+
pwd: string;
|
|
783
|
+
glob_pattern: string;
|
|
784
|
+
doc_count: number;
|
|
785
|
+
active_count: number;
|
|
786
|
+
last_modified: string | null;
|
|
787
|
+
includeByDefault: boolean;
|
|
788
|
+
}[];
|
|
789
|
+
/**
|
|
790
|
+
* Remove a collection and clean up its documents.
|
|
791
|
+
* Uses collections.ts to remove from YAML config and cleans up database.
|
|
792
|
+
*/
|
|
793
|
+
export declare function removeCollection(db: Database, collectionName: string): {
|
|
794
|
+
deletedDocs: number;
|
|
795
|
+
cleanedHashes: number;
|
|
796
|
+
};
|
|
797
|
+
/**
|
|
798
|
+
* Rename a collection.
|
|
799
|
+
* Updates both YAML config and database documents table.
|
|
800
|
+
*/
|
|
801
|
+
export declare function renameCollection(db: Database, oldName: string, newName: string): void;
|
|
802
|
+
/**
|
|
803
|
+
* Insert or update a context for a specific collection and path prefix.
|
|
804
|
+
*
|
|
805
|
+
* `store_collections` is keyed by name (`TEXT PRIMARY KEY`), not an integer id.
|
|
806
|
+
* #754 retargeted this query from the dropped `collections` table but left
|
|
807
|
+
* `WHERE id = ?`, which throws `no such column: id`.
|
|
808
|
+
*/
|
|
809
|
+
export declare function insertContext(db: Database, collectionName: string, pathPrefix: string, context: string): void;
|
|
810
|
+
/**
|
|
811
|
+
* Check which collections don't have any context defined.
|
|
812
|
+
* Returns collections that have no context entries at all (not even root context).
|
|
813
|
+
*/
|
|
814
|
+
export declare function getCollectionsWithoutContext(db: Database): {
|
|
815
|
+
name: string;
|
|
816
|
+
pwd: string;
|
|
817
|
+
doc_count: number;
|
|
818
|
+
}[];
|
|
819
|
+
/**
|
|
820
|
+
* Get top-level directories in a collection that don't have context.
|
|
821
|
+
* Useful for suggesting where context might be needed.
|
|
822
|
+
*/
|
|
823
|
+
export declare function getTopLevelPathsWithoutContext(db: Database, collectionName: string): string[];
|
|
824
|
+
export declare function sanitizeFTS5Term(term: string): string;
|
|
825
|
+
/**
|
|
826
|
+
* Validate that a vec/hyde query doesn't use lex-only syntax.
|
|
827
|
+
* Returns error message if invalid, null if valid.
|
|
828
|
+
*/
|
|
829
|
+
export declare function validateSemanticQuery(query: string): string | null;
|
|
830
|
+
export declare function validateLexQuery(query: string): string | null;
|
|
831
|
+
/** One collection, several (OR), or all (undefined). */
|
|
832
|
+
export type CollectionScope = string | readonly string[] | undefined;
|
|
833
|
+
/**
|
|
834
|
+
* Candidate ceiling for a filtered FTS scan. High enough that a real query's whole match set
|
|
835
|
+
* gets through — which is what stops a narrow filter from starving — and low enough that a
|
|
836
|
+
* query matching most of a large index still terminates.
|
|
837
|
+
*
|
|
838
|
+
* A caller asking for more than this (today: `--all` at 100k) with a filter/scope in play
|
|
839
|
+
* gets at most the cap's worth of candidates, and the excess is dropped before narrowing —
|
|
840
|
+
* silently, unless the caller passes `onScanCapped` to `searchFTS`. Never raise this to
|
|
841
|
+
* satisfy `--all`; it is a perf guard, not a result-size knob.
|
|
842
|
+
*/
|
|
843
|
+
export declare const FILTERED_FTS_SCAN_MAX = 50000;
|
|
844
|
+
/**
|
|
845
|
+
* How many active documents the filter leaves standing, and how many there were.
|
|
846
|
+
*
|
|
847
|
+
* An empty result list cannot tell a user whether nothing matched their words or whether
|
|
848
|
+
* their filter removed the corpus first. Callers ask this only when they got nothing back,
|
|
849
|
+
* so the scan costs nothing on the path that found something.
|
|
850
|
+
*/
|
|
851
|
+
export declare function countFilteredDocuments(db: Database, filter: DocumentFilter, collectionName?: string | readonly string[]): {
|
|
852
|
+
kept: number;
|
|
853
|
+
total: number;
|
|
854
|
+
};
|
|
855
|
+
export declare function searchFTS(db: Database, query: string, limit?: number, collectionName?: string | readonly string[], filter?: MetadataFilter, scope?: DocumentFilter, onScanCapped?: () => void): SearchResult[];
|
|
856
|
+
export declare function searchVec(db: Database, query: string, model: string, limit?: number, collectionName?: string | readonly string[], session?: ILLMSession, precomputedEmbedding?: number[], llm?: LlamaCpp, filter?: MetadataFilter, scope?: DocumentFilter): Promise<SearchResult[]>;
|
|
857
|
+
/**
|
|
858
|
+
* Get all unique content hashes that need embeddings (from active documents).
|
|
859
|
+
* Returns hash, document body, and a sample path for display purposes.
|
|
860
|
+
*/
|
|
861
|
+
export declare function getHashesForEmbedding(db: Database, model?: string): {
|
|
862
|
+
hash: string;
|
|
863
|
+
body: string;
|
|
864
|
+
path: string;
|
|
865
|
+
}[];
|
|
866
|
+
/**
|
|
867
|
+
* Clear embeddings for the whole index, or just for one collection.
|
|
868
|
+
*
|
|
869
|
+
* When `collection` is omitted the entire content_vectors table is emptied and
|
|
870
|
+
* the vectors_vec virtual table is dropped (it is recreated with the right
|
|
871
|
+
* dimensions on the next embed run).
|
|
872
|
+
*
|
|
873
|
+
* When `collection` is provided, only vectors whose hash is referenced
|
|
874
|
+
* exclusively by active documents in that collection are removed. Hashes
|
|
875
|
+
* shared with active documents in other collections are left in place so
|
|
876
|
+
* vector search keeps working there (content_vectors is keyed globally by
|
|
877
|
+
* content hash; identical document bodies across collections share a row).
|
|
878
|
+
* vectors_vec is preserved so other collections keep working unless the scoped
|
|
879
|
+
* clear empties content_vectors entirely, in which case it is dropped so the
|
|
880
|
+
* next embed can recreate the table with the current dimensions.
|
|
881
|
+
*/
|
|
882
|
+
export declare function clearAllEmbeddings(db: Database, collection?: string): void;
|
|
883
|
+
/**
|
|
884
|
+
* Insert a single embedding into both content_vectors and vectors_vec tables.
|
|
885
|
+
* The hash_seq key is formatted as "hash_seq" for the vectors_vec table.
|
|
886
|
+
*
|
|
887
|
+
* The content_vectors INSERT and the vectors_vec DELETE + INSERT run inside
|
|
888
|
+
* one IMMEDIATE transaction, so a crash or error between them can never leave
|
|
889
|
+
* a phantom-complete row (content_vectors claiming a chunk whose vector is
|
|
890
|
+
* missing — pending checks read content_vectors only and would skip the hash
|
|
891
|
+
* forever). Either both tables show the chunk or neither does; a killed batch
|
|
892
|
+
* leaves a partial hash that the next embed run re-selects and converges.
|
|
893
|
+
*
|
|
894
|
+
* IMMEDIATE (not deferred): the txn writes after reading nothing, but the
|
|
895
|
+
* reserved lock must be taken up front — upgrading a deferred read snapshot
|
|
896
|
+
* under a concurrent WAL writer fails with SQLITE_BUSY_SNAPSHOT instead of
|
|
897
|
+
* honoring busy_timeout. Readers still proceed under WAL; writers queue via
|
|
898
|
+
* busy_timeout. Never EXCLUSIVE. The migration wrapper stays outside: a
|
|
899
|
+
* column repair (ALTER TABLE) must not run inside the chunk txn — on a
|
|
900
|
+
* column-miss the txn body throws, rolls back, the repair runs, and the retry
|
|
901
|
+
* re-enters a fresh txn.
|
|
902
|
+
*
|
|
903
|
+
* vectors_vec uses DELETE + INSERT instead of INSERT OR REPLACE because sqlite-vec's
|
|
904
|
+
* vec0 virtual tables silently ignore the OR REPLACE conflict clause.
|
|
905
|
+
*/
|
|
906
|
+
export declare function insertEmbedding(db: Database, hash: string, seq: number, pos: number, embedding: Float32Array, model: string, embeddedAt: string, totalChunks?: number, fingerprint?: string): void;
|
|
907
|
+
export declare function expandQuery(query: string, model: string | undefined, db: Database, llmOverride?: LlamaCpp, options?: {
|
|
908
|
+
onModelReady?: () => void;
|
|
909
|
+
}): Promise<ExpandedQuery[]>;
|
|
910
|
+
/**
|
|
911
|
+
* Delete the cached expansion for a query. hybridQuery() calls this when an
|
|
912
|
+
* expansion's sub-queries all came back empty — left in place, the dud entry
|
|
913
|
+
* would replay the same misses on every warm repeat of the query.
|
|
914
|
+
*/
|
|
915
|
+
export declare function deleteExpansionCacheEntry(db: Database, query: string, model?: string): void;
|
|
916
|
+
/** Per-request rerank options. A 5th positional on Store.rerank keeps existing SDK callers working. */
|
|
917
|
+
export type StoreRerankOptions = {
|
|
918
|
+
/** Doc token cap for this request; overrides QMD_RERANK_MAX_DOC_TOKENS. */
|
|
919
|
+
maxDocTokens?: number;
|
|
920
|
+
/** Called once the rerank model is loaded (not at all when every chunk is cached). */
|
|
921
|
+
onModelReady?: () => void;
|
|
922
|
+
};
|
|
923
|
+
export declare function rerank(query: string, documents: {
|
|
924
|
+
file: string;
|
|
925
|
+
text: string;
|
|
926
|
+
}[], model: string | undefined, db: Database, intent?: string, llmOverride?: LlamaCpp, options?: StoreRerankOptions): Promise<{
|
|
927
|
+
file: string;
|
|
928
|
+
score: number;
|
|
929
|
+
}[]>;
|
|
930
|
+
/**
|
|
931
|
+
* Fraction of the query's units present in the document body, used to weight a
|
|
932
|
+
* relaxed (any-word) FTS list per document instead of one global constant
|
|
933
|
+
* (BASELINE.md 2026-10-09 follow-up: a global weight cannot split the mirrored
|
|
934
|
+
* tie of `hmu-05` vs `sem-08`).
|
|
935
|
+
*
|
|
936
|
+
* Units mirror the rerank chunk scorer: one per Hangul run of two or more
|
|
937
|
+
* syllables (a run of three or more also matches on any shared syllable
|
|
938
|
+
* bigram, so a particle-suffixed word meets its stem), one per Latin/digit run
|
|
939
|
+
* longer than two characters. Single syllables and short tokens carry no
|
|
940
|
+
* signal and are skipped. A query with no units covers everything (1.0) so it
|
|
941
|
+
* never punishes.
|
|
942
|
+
*/
|
|
943
|
+
export declare function relaxedDocCoverage(query: string, body: string): number;
|
|
944
|
+
/**
|
|
945
|
+
* Per-document RRF multipliers, aligned with `rankedLists`: a relaxed list
|
|
946
|
+
* scales each document by `0.5 + 0.5 × coverage`, every other list is untouched
|
|
947
|
+
* (`undefined` entry = multiplier 1.0). The 0.5 floor keeps a zero-coverage
|
|
948
|
+
* candidate in the pool instead of erasing it.
|
|
949
|
+
*/
|
|
950
|
+
export declare function rrfDocMultipliers(rankedLists: RankedResult[][], rankedListMeta: RankedListMeta[]): Array<Map<string, number> | undefined>;
|
|
951
|
+
export declare function reciprocalRankFusion(resultLists: RankedResult[][], weights?: number[], k?: number, docMultipliers?: Array<Map<string, number> | undefined>): RankedResult[];
|
|
952
|
+
/**
|
|
953
|
+
* Build per-document RRF contribution traces for explain/debug output.
|
|
954
|
+
*/
|
|
955
|
+
export declare function buildRrfTrace(resultLists: RankedResult[][], weights?: number[], listMeta?: RankedListMeta[], k?: number, docMultipliers?: Array<Map<string, number> | undefined>): Map<string, RRFScoreTrace>;
|
|
956
|
+
/**
|
|
957
|
+
* Find a document by filename/path, docid (#hash), or path-boundary suffix.
|
|
958
|
+
* Returns document metadata without body by default.
|
|
959
|
+
*
|
|
960
|
+
* Supports:
|
|
961
|
+
* - Virtual paths: qmd://collection/path/to/file.md
|
|
962
|
+
* - Absolute paths: /path/to/file.md
|
|
963
|
+
* - Relative paths: path/to/file.md
|
|
964
|
+
* - Short docid: #abc123 (first 6 chars of hash)
|
|
965
|
+
*/
|
|
966
|
+
export declare function findDocument(db: Database, filename: string, options?: {
|
|
967
|
+
includeBody?: boolean;
|
|
968
|
+
}): DocumentResult | DocumentLookupError;
|
|
969
|
+
/**
|
|
970
|
+
* Get the body content for a document
|
|
971
|
+
* Optionally slice by line range
|
|
972
|
+
*/
|
|
973
|
+
export declare function getDocumentBody(db: Database, doc: DocumentResult | {
|
|
974
|
+
filepath: string;
|
|
975
|
+
}, fromLine?: number, maxLines?: number): string | null;
|
|
976
|
+
/**
|
|
977
|
+
* Escape a user-supplied string so it is matched literally by SQLite LIKE.
|
|
978
|
+
* Uses '#' as the ESCAPE character (avoids quote-escaping pitfalls with '\\').
|
|
979
|
+
*/
|
|
980
|
+
export declare function escapeLikePattern(value: string): string;
|
|
981
|
+
export type CommaListMatch = {
|
|
982
|
+
collection: string;
|
|
983
|
+
path: string;
|
|
984
|
+
virtualPath: string;
|
|
985
|
+
bodyLength: number;
|
|
986
|
+
};
|
|
987
|
+
export type CommaListResolve = {
|
|
988
|
+
ok: true;
|
|
989
|
+
match: CommaListMatch;
|
|
990
|
+
} | {
|
|
991
|
+
ok: false;
|
|
992
|
+
error: string;
|
|
993
|
+
};
|
|
994
|
+
/**
|
|
995
|
+
* Resolve one comma-list name for multi-get (shared by CLI and SDK/MCP).
|
|
996
|
+
*
|
|
997
|
+
* Match order: docid / qmd:// URI (exact only), then exact collection-prefixed
|
|
998
|
+
* path, then exact document path, then a path-boundary suffix (`.../name`).
|
|
999
|
+
* Unanchored LIKE is never used, so a fragment like `NTAX.md` cannot silently
|
|
1000
|
+
* fetch `SYNTAX.md`. Multiple hits at the same tier error with the candidate
|
|
1001
|
+
* list instead of `LIMIT 1` (#759).
|
|
1002
|
+
*/
|
|
1003
|
+
export declare function resolveCommaListName(db: Database, name: string): CommaListResolve;
|
|
1004
|
+
/**
|
|
1005
|
+
* Find multiple documents by glob pattern or comma-separated list
|
|
1006
|
+
* Returns documents without body by default (use getDocumentBody to load)
|
|
1007
|
+
*/
|
|
1008
|
+
export declare function findDocuments(db: Database, pattern: string, options?: {
|
|
1009
|
+
includeBody?: boolean;
|
|
1010
|
+
maxBytes?: number;
|
|
1011
|
+
}): {
|
|
1012
|
+
docs: MultiGetResult[];
|
|
1013
|
+
errors: string[];
|
|
1014
|
+
};
|
|
1015
|
+
export declare function getStatus(db: Database, model?: string): IndexStatus;
|
|
1016
|
+
export type SnippetResult = {
|
|
1017
|
+
line: number;
|
|
1018
|
+
snippet: string;
|
|
1019
|
+
linesBefore: number;
|
|
1020
|
+
linesAfter: number;
|
|
1021
|
+
snippetLines: number;
|
|
1022
|
+
};
|
|
1023
|
+
/** Weight for intent terms relative to query terms (1.0) in snippet scoring */
|
|
1024
|
+
export declare const INTENT_WEIGHT_SNIPPET = 0.3;
|
|
1025
|
+
export { INTENT_WEIGHT_CHUNK };
|
|
1026
|
+
/**
|
|
1027
|
+
* Extract meaningful terms from an intent string, filtering stop words and punctuation.
|
|
1028
|
+
* Uses Unicode-aware punctuation stripping so domain terms like "API" survive.
|
|
1029
|
+
* Returns lowercase terms suitable for text matching.
|
|
1030
|
+
*/
|
|
1031
|
+
export declare function extractIntentTerms(intent: string): string[];
|
|
1032
|
+
export declare function extractSnippet(body: string, query: string, maxLen?: number, chunkPos?: number, chunkLen?: number, intent?: string): SnippetResult;
|
|
1033
|
+
/**
|
|
1034
|
+
* Add line numbers to text content.
|
|
1035
|
+
* Each line becomes: "{lineNum}: {content}"
|
|
1036
|
+
*/
|
|
1037
|
+
export declare function addLineNumbers(text: string, startLine?: number): string;
|
|
1038
|
+
/**
|
|
1039
|
+
* Optional progress hooks for search orchestration.
|
|
1040
|
+
* CLI wires these to stderr for user feedback; MCP leaves them unset.
|
|
1041
|
+
*/
|
|
1042
|
+
export interface SearchHooks {
|
|
1043
|
+
/** BM25 probe found strong signal — expansion will be skipped */
|
|
1044
|
+
onStrongSignal?: (topScore: number) => void;
|
|
1045
|
+
/** Query expansion starting */
|
|
1046
|
+
onExpandStart?: () => void;
|
|
1047
|
+
/** Query expansion complete. Empty array = strong signal skip. elapsedMs = time taken. */
|
|
1048
|
+
onExpand?: (original: string, expanded: ExpandedQuery[], elapsedMs: number) => void;
|
|
1049
|
+
/** Embedding starting (vec/hyde queries) */
|
|
1050
|
+
onEmbedStart?: (count: number) => void;
|
|
1051
|
+
/** Embedding complete */
|
|
1052
|
+
onEmbedDone?: (elapsedMs: number) => void;
|
|
1053
|
+
/** Every BM25 lookup of the search finished; elapsedMs is their total */
|
|
1054
|
+
onFtsDone?: (elapsedMs: number) => void;
|
|
1055
|
+
/** Every sqlite-vec lookup of the search finished (embedding excluded); elapsedMs is their total */
|
|
1056
|
+
onVecDone?: (elapsedMs: number) => void;
|
|
1057
|
+
/** Candidates were chunked and each one's best chunk picked (runs with or without rerank) */
|
|
1058
|
+
onChunkDone?: (elapsedMs: number) => void;
|
|
1059
|
+
/**
|
|
1060
|
+
* A stage went without its LLM (expansion: original query only, rerank: RRF order). elapsedMs is
|
|
1061
|
+
* how long it was waited on after its model loaded; undefined when it was skipped outright because
|
|
1062
|
+
* an earlier call of that stage, abandoned at its deadline, is still running.
|
|
1063
|
+
*/
|
|
1064
|
+
onDeadline?: (stage: DeadlineStage, elapsedMs?: number) => void;
|
|
1065
|
+
/** The stage's model finished loading (cold start or reload after idle unload); elapsedMs is the wait */
|
|
1066
|
+
onModelLoad?: (stage: DeadlineStage, elapsedMs: number) => void;
|
|
1067
|
+
/** Reranking is about to start */
|
|
1068
|
+
onRerankStart?: (chunkCount: number) => void;
|
|
1069
|
+
/** Reranking finished */
|
|
1070
|
+
onRerankDone?: (elapsedMs: number) => void;
|
|
1071
|
+
}
|
|
1072
|
+
export interface HybridQueryOptions {
|
|
1073
|
+
collection?: string | readonly string[];
|
|
1074
|
+
filter?: MetadataFilter;
|
|
1075
|
+
limit?: number;
|
|
1076
|
+
minScore?: number;
|
|
1077
|
+
candidateLimit?: number;
|
|
1078
|
+
rerankMaxDocTokens?: number;
|
|
1079
|
+
explain?: boolean;
|
|
1080
|
+
intent?: string;
|
|
1081
|
+
skipRerank?: boolean;
|
|
1082
|
+
rerankDeadlineMs?: number;
|
|
1083
|
+
expandDeadlineMs?: number;
|
|
1084
|
+
chunkStrategy?: ChunkStrategy;
|
|
1085
|
+
scope?: DocumentFilter;
|
|
1086
|
+
hooks?: SearchHooks;
|
|
1087
|
+
}
|
|
1088
|
+
export interface HybridQueryResult {
|
|
1089
|
+
file: string;
|
|
1090
|
+
displayPath: string;
|
|
1091
|
+
title: string;
|
|
1092
|
+
body: string;
|
|
1093
|
+
bestChunk: string;
|
|
1094
|
+
bestChunkPos: number;
|
|
1095
|
+
score: number;
|
|
1096
|
+
context: string | null;
|
|
1097
|
+
docid: string;
|
|
1098
|
+
metadata: DocumentMetadata;
|
|
1099
|
+
explain?: HybridQueryExplain;
|
|
1100
|
+
}
|
|
1101
|
+
export type RankedListMeta = {
|
|
1102
|
+
source: "fts" | "vec";
|
|
1103
|
+
queryType: "original" | "lex" | "vec" | "hyde";
|
|
1104
|
+
query: string;
|
|
1105
|
+
relaxed?: boolean;
|
|
1106
|
+
searchIndex?: number;
|
|
1107
|
+
};
|
|
1108
|
+
/**
|
|
1109
|
+
* RRF list weights for hybridQuery.
|
|
1110
|
+
*
|
|
1111
|
+
* Original-query retrieval paths are the primary evidence and get 2x weight:
|
|
1112
|
+
* - original FTS
|
|
1113
|
+
* - original vector search
|
|
1114
|
+
*
|
|
1115
|
+
* Expansion-derived lists (lex/vec/hyde) stay at 1x regardless of list order,
|
|
1116
|
+
* so a lex expansion inserted before original vector search cannot steal the
|
|
1117
|
+
* original vector boost.
|
|
1118
|
+
*/
|
|
1119
|
+
export declare function getHybridRrfWeights(rankedListMeta: RankedListMeta[]): number[];
|
|
1120
|
+
export declare function getStructuredRrfWeights(rankedListMeta: RankedListMeta[]): number[];
|
|
1121
|
+
/**
|
|
1122
|
+
* Blend a reranker score with a linear retrieval-position tiebreak.
|
|
1123
|
+
*
|
|
1124
|
+
* The position term used to be 1/rrfRank under rank-gated weights (0.75/0.60/0.40).
|
|
1125
|
+
* That score drops 50% between rank 1 and rank 2 — a gap of 0.375 at weight 0.75,
|
|
1126
|
+
* larger than the 0.25 the whole reranker term could ever contribute — so no
|
|
1127
|
+
* reranker score could displace the top retrieval hit, and the reranker was inert
|
|
1128
|
+
* exactly where it decided the answer. A linear decay over the candidate window
|
|
1129
|
+
* separates adjacent ranks by a constant the reranker can overcome.
|
|
1130
|
+
*
|
|
1131
|
+
* Measured 2026-09-18 on one fixed candidate dump per goldset, so every formula was
|
|
1132
|
+
* scored against identical retrieval (scripts/bench-dump-candidates.ts feeding
|
|
1133
|
+
* scripts/bench-blend-sweep.mjs):
|
|
1134
|
+
*
|
|
1135
|
+
* alias (n=223) paraphrase (n=30)
|
|
1136
|
+
* 1/rrfRank r@1 .713 mrr .808 r@1 .467 mrr .664
|
|
1137
|
+
* this r@1 .874 mrr .927 r@1 .867 mrr .903
|
|
1138
|
+
*
|
|
1139
|
+
* On both goldsets the old blend's r@1 equalled retrieval-only r@1 to three
|
|
1140
|
+
* decimals, which is what "the reranker cannot move rank 1" looks like as a number.
|
|
1141
|
+
*/
|
|
1142
|
+
export declare function blendRerankScore(rrfRank: number, rerankScore: number, candidateLimit: number): number;
|
|
1143
|
+
export type DeadlineStage = "expand" | "rerank";
|
|
1144
|
+
/**
|
|
1145
|
+
* Bound on the model-load wait inside runWithDeadline: a load that never
|
|
1146
|
+
* reports ready must not hang the query. QMD_LLM_LOAD_TIMEOUT_MS sets it
|
|
1147
|
+
* (positive ms, 0 = wait forever, anything else = the default).
|
|
1148
|
+
*/
|
|
1149
|
+
export declare const DEFAULT_LOAD_TIMEOUT_MS = 120000;
|
|
1150
|
+
/** Load-timeout resolver for runWithDeadline; exported for tests. */
|
|
1151
|
+
export declare function resolveLoadTimeoutMs(): number;
|
|
1152
|
+
/** Rerank `chunks`, or null when the reranker misses `deadlineMs` (the caller then answers in RRF order). */
|
|
1153
|
+
export declare const RERANK_CASCADE_FIRST_BATCH = 8;
|
|
1154
|
+
/** Skip the tail past the first batch when the top-2 rerank margin reaches this. */
|
|
1155
|
+
export declare const RERANK_CASCADE_MARGIN = 0.08;
|
|
1156
|
+
/** First-batch size, overridable for tuning via `QMD_RERANK_FIRST_BATCH`. */
|
|
1157
|
+
export declare function rerankCascadeFirstBatch(): number;
|
|
1158
|
+
/** Tail-skip margin, overridable for tuning via `QMD_RERANK_MARGIN`. */
|
|
1159
|
+
export declare function rerankCascadeMargin(): number;
|
|
1160
|
+
/**
|
|
1161
|
+
* QMD_RERANK_CASCADE=0 (or false/off/no) restores single-batch rerank;
|
|
1162
|
+
* anything else (including unset) cascades. Kill switch for measurement and
|
|
1163
|
+
* operations; exported for tests.
|
|
1164
|
+
*/
|
|
1165
|
+
export declare function isRerankCascadeEnabled(): boolean;
|
|
1166
|
+
/**
|
|
1167
|
+
* True when the first cascade batch's top-2 margin is too tight to skip the
|
|
1168
|
+
* tail: a contested top means an unscored document could still win the blend.
|
|
1169
|
+
* A batch of fewer than two holds nothing to contest.
|
|
1170
|
+
*/
|
|
1171
|
+
export declare function cascadeNeedsSecondBatch(sortedScores: number[], margin?: number): boolean;
|
|
1172
|
+
export declare function rerankWithDeadline(store: Store, query: string, chunks: {
|
|
1173
|
+
file: string;
|
|
1174
|
+
text: string;
|
|
1175
|
+
}[], intent: string | undefined, options: {
|
|
1176
|
+
rerankMaxDocTokens?: number;
|
|
1177
|
+
rerankDeadlineMs?: number;
|
|
1178
|
+
hooks?: SearchHooks;
|
|
1179
|
+
} | undefined): Promise<{
|
|
1180
|
+
file: string;
|
|
1181
|
+
score: number;
|
|
1182
|
+
unscored?: boolean;
|
|
1183
|
+
}[] | null>;
|
|
1184
|
+
/**
|
|
1185
|
+
* Hybrid search: BM25 + vector + query expansion + RRF + chunked reranking.
|
|
1186
|
+
*
|
|
1187
|
+
* Pipeline:
|
|
1188
|
+
* 1. BM25 probe → skip expansion if strong signal
|
|
1189
|
+
* 2. expandQuery() → typed query variants (lex/vec/hyde)
|
|
1190
|
+
* 3. Type-routed search: original→vector, lex→FTS, vec/hyde→vector
|
|
1191
|
+
* 4. RRF fusion → slice to candidateLimit
|
|
1192
|
+
* 5. chunkDocument() + keyword-best-chunk selection
|
|
1193
|
+
* 6. rerank on chunks (NOT full bodies — O(tokens) trap)
|
|
1194
|
+
* 7. Position-aware score blending (RRF rank × reranker score)
|
|
1195
|
+
* 8. Dedup by file, filter by minScore, slice to limit
|
|
1196
|
+
*/
|
|
1197
|
+
export declare function hybridQuery(store: Store, rawQuery: string, options?: HybridQueryOptions): Promise<HybridQueryResult[]>;
|
|
1198
|
+
export interface VectorSearchOptions {
|
|
1199
|
+
collection?: string | readonly string[];
|
|
1200
|
+
filter?: MetadataFilter;
|
|
1201
|
+
limit?: number;
|
|
1202
|
+
minScore?: number;
|
|
1203
|
+
intent?: string;
|
|
1204
|
+
scope?: DocumentFilter;
|
|
1205
|
+
hooks?: Pick<SearchHooks, 'onExpand'>;
|
|
1206
|
+
}
|
|
1207
|
+
export interface VectorSearchResult {
|
|
1208
|
+
file: string;
|
|
1209
|
+
displayPath: string;
|
|
1210
|
+
title: string;
|
|
1211
|
+
body: string;
|
|
1212
|
+
score: number;
|
|
1213
|
+
context: string | null;
|
|
1214
|
+
docid: string;
|
|
1215
|
+
metadata: DocumentMetadata;
|
|
1216
|
+
}
|
|
1217
|
+
/**
|
|
1218
|
+
* Vector-only semantic search with query expansion.
|
|
1219
|
+
*
|
|
1220
|
+
* Pipeline:
|
|
1221
|
+
* 1. expandQuery() → typed variants, filter to vec/hyde only (lex irrelevant here)
|
|
1222
|
+
* 2. searchVec() for original + vec/hyde variants (sequential — node-llama-cpp embed limitation)
|
|
1223
|
+
* 3. Dedup by filepath (keep max score)
|
|
1224
|
+
* 4. Sort by score descending, filter by minScore, slice to limit
|
|
1225
|
+
*/
|
|
1226
|
+
export declare function vectorSearchQuery(store: Store, query: string, options?: VectorSearchOptions): Promise<VectorSearchResult[]>;
|
|
1227
|
+
/**
|
|
1228
|
+
* A single sub-search in a structured search request.
|
|
1229
|
+
* Matches the format used in QMD training data.
|
|
1230
|
+
*/
|
|
1231
|
+
export interface StructuredSearchOptions {
|
|
1232
|
+
collections?: string[];
|
|
1233
|
+
filter?: MetadataFilter;
|
|
1234
|
+
limit?: number;
|
|
1235
|
+
minScore?: number;
|
|
1236
|
+
candidateLimit?: number;
|
|
1237
|
+
rerankMaxDocTokens?: number;
|
|
1238
|
+
explain?: boolean;
|
|
1239
|
+
/** Domain intent hint for disambiguation — steers reranking and chunk selection */
|
|
1240
|
+
intent?: string;
|
|
1241
|
+
/** Skip LLM reranking, use only RRF scores */
|
|
1242
|
+
skipRerank?: boolean;
|
|
1243
|
+
rerankDeadlineMs?: number;
|
|
1244
|
+
chunkStrategy?: ChunkStrategy;
|
|
1245
|
+
/** Narrow the corpus by path glob and modified_at before anything is ranked */
|
|
1246
|
+
scope?: DocumentFilter;
|
|
1247
|
+
hooks?: SearchHooks;
|
|
1248
|
+
}
|
|
1249
|
+
/**
|
|
1250
|
+
* Structured search: execute pre-expanded queries without LLM query expansion.
|
|
1251
|
+
*
|
|
1252
|
+
* Designed for LLM callers (MCP/HTTP) that generate their own query expansions.
|
|
1253
|
+
* Skips the internal expandQuery() step — goes directly to:
|
|
1254
|
+
*
|
|
1255
|
+
* Pipeline:
|
|
1256
|
+
* 1. Route searches: lex→FTS, vec/hyde→vector (batch embed)
|
|
1257
|
+
* 2. RRF fusion across all result lists
|
|
1258
|
+
* 3. Chunk documents + keyword-best-chunk selection
|
|
1259
|
+
* 4. Rerank on chunks
|
|
1260
|
+
* 5. Position-aware score blending
|
|
1261
|
+
* 6. Dedup, filter, slice
|
|
1262
|
+
*
|
|
1263
|
+
* This is the recommended endpoint for capable LLMs — they can generate
|
|
1264
|
+
* better query variations than our small local model, especially for
|
|
1265
|
+
* domain-specific or nuanced queries.
|
|
1266
|
+
*/
|
|
1267
|
+
export declare function structuredSearch(store: Store, rawSearches: ExpandedQuery[], options?: StructuredSearchOptions): Promise<HybridQueryResult[]>;
|