@tobilu/qmd 1.0.0 → 1.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/store.ts DELETED
@@ -1,3057 +0,0 @@
1
- /**
2
- * QMD Store - Core data access and retrieval functions
3
- *
4
- * This module provides all database operations, search functions, and document
5
- * retrieval for QMD. It returns raw data structures that can be formatted by
6
- * CLI or MCP consumers.
7
- *
8
- * Usage:
9
- * const store = createStore("/path/to/db.sqlite");
10
- * // or use default path:
11
- * const store = createStore();
12
- */
13
-
14
- import { openDatabase, loadSqliteVec } from "./db.js";
15
- import type { Database } from "./db.js";
16
- import picomatch from "picomatch";
17
- import { createHash } from "crypto";
18
- import { realpathSync, statSync, mkdirSync } from "node:fs";
19
- import {
20
- LlamaCpp,
21
- getDefaultLlamaCpp,
22
- formatQueryForEmbedding,
23
- formatDocForEmbedding,
24
- type RerankDocument,
25
- type ILLMSession,
26
- } from "./llm";
27
- import {
28
- findContextForPath as collectionsFindContextForPath,
29
- addContext as collectionsAddContext,
30
- removeContext as collectionsRemoveContext,
31
- listAllContexts as collectionsListAllContexts,
32
- getCollection,
33
- listCollections as collectionsListCollections,
34
- addCollection as collectionsAddCollection,
35
- removeCollection as collectionsRemoveCollection,
36
- renameCollection as collectionsRenameCollection,
37
- setGlobalContext,
38
- loadConfig as collectionsLoadConfig,
39
- type NamedCollection,
40
- } from "./collections";
41
-
42
- // =============================================================================
43
- // Configuration
44
- // =============================================================================
45
-
46
- const HOME = process.env.HOME || "/tmp";
47
- export const DEFAULT_EMBED_MODEL = "embeddinggemma";
48
- export const DEFAULT_RERANK_MODEL = "ExpedientFalcon/qwen3-reranker:0.6b-q8_0";
49
- export const DEFAULT_QUERY_MODEL = "Qwen/Qwen3-1.7B";
50
- export const DEFAULT_GLOB = "**/*.md";
51
- export const DEFAULT_MULTI_GET_MAX_BYTES = 10 * 1024; // 10KB
52
-
53
- // Chunking: 900 tokens per chunk with 15% overlap
54
- // Increased from 800 to accommodate smart chunking finding natural break points
55
- export const CHUNK_SIZE_TOKENS = 900;
56
- export const CHUNK_OVERLAP_TOKENS = Math.floor(CHUNK_SIZE_TOKENS * 0.15); // 135 tokens (15% overlap)
57
- // Fallback char-based approximation for sync chunking (~4 chars per token)
58
- export const CHUNK_SIZE_CHARS = CHUNK_SIZE_TOKENS * 4; // 3600 chars
59
- export const CHUNK_OVERLAP_CHARS = CHUNK_OVERLAP_TOKENS * 4; // 540 chars
60
- // Search window for finding optimal break points (in tokens, ~200 tokens)
61
- export const CHUNK_WINDOW_TOKENS = 200;
62
- export const CHUNK_WINDOW_CHARS = CHUNK_WINDOW_TOKENS * 4; // 800 chars
63
-
64
- // =============================================================================
65
- // Smart Chunking - Break Point Detection
66
- // =============================================================================
67
-
68
- /**
69
- * A potential break point in the document with a base score indicating quality.
70
- */
71
- export interface BreakPoint {
72
- pos: number; // character position
73
- score: number; // base score (higher = better break point)
74
- type: string; // for debugging: 'h1', 'h2', 'blank', etc.
75
- }
76
-
77
- /**
78
- * A region where a code fence exists (between ``` markers).
79
- * We should never split inside a code fence.
80
- */
81
- export interface CodeFenceRegion {
82
- start: number; // position of opening ```
83
- end: number; // position of closing ``` (or document end if unclosed)
84
- }
85
-
86
- /**
87
- * Patterns for detecting break points in markdown documents.
88
- * Higher scores indicate better places to split.
89
- * Scores are spread wide so headings decisively beat lower-quality breaks.
90
- * Order matters for scoring - more specific patterns first.
91
- */
92
- export const BREAK_PATTERNS: [RegExp, number, string][] = [
93
- [/\n#{1}(?!#)/g, 100, 'h1'], // # but not ##
94
- [/\n#{2}(?!#)/g, 90, 'h2'], // ## but not ###
95
- [/\n#{3}(?!#)/g, 80, 'h3'], // ### but not ####
96
- [/\n#{4}(?!#)/g, 70, 'h4'], // #### but not #####
97
- [/\n#{5}(?!#)/g, 60, 'h5'], // ##### but not ######
98
- [/\n#{6}(?!#)/g, 50, 'h6'], // ######
99
- [/\n```/g, 80, 'codeblock'], // code block boundary (same as h3)
100
- [/\n(?:---|\*\*\*|___)\s*\n/g, 60, 'hr'], // horizontal rule
101
- [/\n\n+/g, 20, 'blank'], // paragraph boundary
102
- [/\n[-*]\s/g, 5, 'list'], // unordered list item
103
- [/\n\d+\.\s/g, 5, 'numlist'], // ordered list item
104
- [/\n/g, 1, 'newline'], // minimal break
105
- ];
106
-
107
- /**
108
- * Scan text for all potential break points.
109
- * Returns sorted array of break points with higher-scoring patterns taking precedence
110
- * when multiple patterns match the same position.
111
- */
112
- export function scanBreakPoints(text: string): BreakPoint[] {
113
- const points: BreakPoint[] = [];
114
- const seen = new Map<number, BreakPoint>(); // pos -> best break point at that pos
115
-
116
- for (const [pattern, score, type] of BREAK_PATTERNS) {
117
- for (const match of text.matchAll(pattern)) {
118
- const pos = match.index!;
119
- const existing = seen.get(pos);
120
- // Keep higher score if position already seen
121
- if (!existing || score > existing.score) {
122
- const bp = { pos, score, type };
123
- seen.set(pos, bp);
124
- }
125
- }
126
- }
127
-
128
- // Convert to array and sort by position
129
- for (const bp of seen.values()) {
130
- points.push(bp);
131
- }
132
- return points.sort((a, b) => a.pos - b.pos);
133
- }
134
-
135
- /**
136
- * Find all code fence regions in the text.
137
- * Code fences are delimited by ``` and we should never split inside them.
138
- */
139
- export function findCodeFences(text: string): CodeFenceRegion[] {
140
- const regions: CodeFenceRegion[] = [];
141
- const fencePattern = /\n```/g;
142
- let inFence = false;
143
- let fenceStart = 0;
144
-
145
- for (const match of text.matchAll(fencePattern)) {
146
- if (!inFence) {
147
- fenceStart = match.index!;
148
- inFence = true;
149
- } else {
150
- regions.push({ start: fenceStart, end: match.index! + match[0].length });
151
- inFence = false;
152
- }
153
- }
154
-
155
- // Handle unclosed fence - extends to end of document
156
- if (inFence) {
157
- regions.push({ start: fenceStart, end: text.length });
158
- }
159
-
160
- return regions;
161
- }
162
-
163
- /**
164
- * Check if a position is inside a code fence region.
165
- */
166
- export function isInsideCodeFence(pos: number, fences: CodeFenceRegion[]): boolean {
167
- return fences.some(f => pos > f.start && pos < f.end);
168
- }
169
-
170
- /**
171
- * Find the best cut position using scored break points with distance decay.
172
- *
173
- * Uses squared distance for gentler early decay - headings far back still win
174
- * over low-quality breaks near the target.
175
- *
176
- * @param breakPoints - Pre-scanned break points from scanBreakPoints()
177
- * @param targetCharPos - The ideal cut position (e.g., maxChars boundary)
178
- * @param windowChars - How far back to search for break points (default ~200 tokens)
179
- * @param decayFactor - How much to penalize distance (0.7 = 30% score at window edge)
180
- * @param codeFences - Code fence regions to avoid splitting inside
181
- * @returns The best position to cut at
182
- */
183
- export function findBestCutoff(
184
- breakPoints: BreakPoint[],
185
- targetCharPos: number,
186
- windowChars: number = CHUNK_WINDOW_CHARS,
187
- decayFactor: number = 0.7,
188
- codeFences: CodeFenceRegion[] = []
189
- ): number {
190
- const windowStart = targetCharPos - windowChars;
191
- let bestScore = -1;
192
- let bestPos = targetCharPos;
193
-
194
- for (const bp of breakPoints) {
195
- if (bp.pos < windowStart) continue;
196
- if (bp.pos > targetCharPos) break; // sorted, so we can stop
197
-
198
- // Skip break points inside code fences
199
- if (isInsideCodeFence(bp.pos, codeFences)) continue;
200
-
201
- const distance = targetCharPos - bp.pos;
202
- // Squared distance decay: gentle early, steep late
203
- // At target: multiplier = 1.0
204
- // At 25% back: multiplier = 0.956
205
- // At 50% back: multiplier = 0.825
206
- // At 75% back: multiplier = 0.606
207
- // At window edge: multiplier = 0.3
208
- const normalizedDist = distance / windowChars;
209
- const multiplier = 1.0 - (normalizedDist * normalizedDist) * decayFactor;
210
- const finalScore = bp.score * multiplier;
211
-
212
- if (finalScore > bestScore) {
213
- bestScore = finalScore;
214
- bestPos = bp.pos;
215
- }
216
- }
217
-
218
- return bestPos;
219
- }
220
-
221
- // Hybrid query: strong BM25 signal detection thresholds
222
- // Skip expensive LLM expansion when top result is strong AND clearly separated from runner-up
223
- export const STRONG_SIGNAL_MIN_SCORE = 0.85;
224
- export const STRONG_SIGNAL_MIN_GAP = 0.15;
225
- // Max candidates to pass to reranker — balances quality vs latency.
226
- // 40 keeps rank 31-40 visible to the reranker (matters for recall on broad queries).
227
- export const RERANK_CANDIDATE_LIMIT = 40;
228
-
229
- /**
230
- * A typed query expansion result. Decoupled from llm.ts internal Queryable —
231
- * same shape, but store.ts owns its own public API type.
232
- *
233
- * - lex: keyword variant → routes to FTS only
234
- * - vec: semantic variant → routes to vector only
235
- * - hyde: hypothetical document → routes to vector only
236
- */
237
- export type ExpandedQuery = {
238
- type: 'lex' | 'vec' | 'hyde';
239
- text: string;
240
- };
241
-
242
- // =============================================================================
243
- // Path utilities
244
- // =============================================================================
245
-
246
- export function homedir(): string {
247
- return HOME;
248
- }
249
-
250
- /**
251
- * Check if a path is absolute.
252
- * Supports:
253
- * - Unix paths: /path/to/file
254
- * - Windows native: C:\path or C:/path
255
- * - Git Bash: /c/path or /C/path (C-Z drives, excluding A/B floppy drives)
256
- *
257
- * Note: /c without trailing slash is treated as Unix path (directory named "c"),
258
- * while /c/ or /c/path are treated as Git Bash paths (C: drive).
259
- */
260
- export function isAbsolutePath(path: string): boolean {
261
- if (!path) return false;
262
-
263
- // Unix absolute path
264
- if (path.startsWith('/')) {
265
- // Check if it's a Git Bash style path like /c/ or /c/Users (C-Z only, not A or B)
266
- // Requires path[2] === '/' to distinguish from Unix paths like /c or /cache
267
- if (path.length >= 3 && path[2] === '/') {
268
- const driveLetter = path[1];
269
- if (driveLetter && /[c-zC-Z]/.test(driveLetter)) {
270
- return true;
271
- }
272
- }
273
- // Any other path starting with / is Unix absolute
274
- return true;
275
- }
276
-
277
- // Windows native path: C:\ or C:/ (any letter A-Z)
278
- if (path.length >= 2 && /[a-zA-Z]/.test(path[0]!) && path[1] === ':') {
279
- return true;
280
- }
281
-
282
- return false;
283
- }
284
-
285
- /**
286
- * Normalize path separators to forward slashes.
287
- * Converts Windows backslashes to forward slashes.
288
- */
289
- export function normalizePathSeparators(path: string): string {
290
- return path.replace(/\\/g, '/');
291
- }
292
-
293
- /**
294
- * Get the relative path from a prefix.
295
- * Returns null if path is not under prefix.
296
- * Returns empty string if path equals prefix.
297
- */
298
- export function getRelativePathFromPrefix(path: string, prefix: string): string | null {
299
- // Empty prefix is invalid
300
- if (!prefix) {
301
- return null;
302
- }
303
-
304
- const normalizedPath = normalizePathSeparators(path);
305
- const normalizedPrefix = normalizePathSeparators(prefix);
306
-
307
- // Ensure prefix ends with / for proper matching
308
- const prefixWithSlash = !normalizedPrefix.endsWith('/')
309
- ? normalizedPrefix + '/'
310
- : normalizedPrefix;
311
-
312
- // Exact match
313
- if (normalizedPath === normalizedPrefix) {
314
- return '';
315
- }
316
-
317
- // Check if path starts with prefix
318
- if (normalizedPath.startsWith(prefixWithSlash)) {
319
- return normalizedPath.slice(prefixWithSlash.length);
320
- }
321
-
322
- return null;
323
- }
324
-
325
- export function resolve(...paths: string[]): string {
326
- if (paths.length === 0) {
327
- throw new Error("resolve: at least one path segment is required");
328
- }
329
-
330
- // Normalize all paths to use forward slashes
331
- const normalizedPaths = paths.map(normalizePathSeparators);
332
-
333
- let result = '';
334
- let windowsDrive = '';
335
-
336
- // Check if first path is absolute
337
- const firstPath = normalizedPaths[0]!;
338
- if (isAbsolutePath(firstPath)) {
339
- result = firstPath;
340
-
341
- // Extract Windows drive letter if present
342
- if (firstPath.length >= 2 && /[a-zA-Z]/.test(firstPath[0]!) && firstPath[1] === ':') {
343
- windowsDrive = firstPath.slice(0, 2);
344
- result = firstPath.slice(2);
345
- } else if (firstPath.startsWith('/') && firstPath.length >= 3 && firstPath[2] === '/') {
346
- // Git Bash style: /c/ -> C: (C-Z drives only, not A or B)
347
- const driveLetter = firstPath[1];
348
- if (driveLetter && /[c-zC-Z]/.test(driveLetter)) {
349
- windowsDrive = driveLetter.toUpperCase() + ':';
350
- result = firstPath.slice(2);
351
- }
352
- }
353
- } else {
354
- // Start with PWD or cwd, then append the first relative path
355
- const pwd = normalizePathSeparators(process.env.PWD || process.cwd());
356
-
357
- // Extract Windows drive from PWD if present
358
- if (pwd.length >= 2 && /[a-zA-Z]/.test(pwd[0]!) && pwd[1] === ':') {
359
- windowsDrive = pwd.slice(0, 2);
360
- result = pwd.slice(2) + '/' + firstPath;
361
- } else {
362
- result = pwd + '/' + firstPath;
363
- }
364
- }
365
-
366
- // Process remaining paths
367
- for (let i = 1; i < normalizedPaths.length; i++) {
368
- const p = normalizedPaths[i]!;
369
- if (isAbsolutePath(p)) {
370
- // Absolute path replaces everything
371
- result = p;
372
-
373
- // Update Windows drive if present
374
- if (p.length >= 2 && /[a-zA-Z]/.test(p[0]!) && p[1] === ':') {
375
- windowsDrive = p.slice(0, 2);
376
- result = p.slice(2);
377
- } else if (p.startsWith('/') && p.length >= 3 && p[2] === '/') {
378
- // Git Bash style (C-Z drives only, not A or B)
379
- const driveLetter = p[1];
380
- if (driveLetter && /[c-zC-Z]/.test(driveLetter)) {
381
- windowsDrive = driveLetter.toUpperCase() + ':';
382
- result = p.slice(2);
383
- } else {
384
- windowsDrive = '';
385
- }
386
- } else {
387
- windowsDrive = '';
388
- }
389
- } else {
390
- // Relative path - append
391
- result = result + '/' + p;
392
- }
393
- }
394
-
395
- // Normalize . and .. components
396
- const parts = result.split('/').filter(Boolean);
397
- const normalized: string[] = [];
398
- for (const part of parts) {
399
- if (part === '..') {
400
- normalized.pop();
401
- } else if (part !== '.') {
402
- normalized.push(part);
403
- }
404
- }
405
-
406
- // Build final path
407
- const finalPath = '/' + normalized.join('/');
408
-
409
- // Prepend Windows drive if present
410
- if (windowsDrive) {
411
- return windowsDrive + finalPath;
412
- }
413
-
414
- return finalPath;
415
- }
416
-
417
- // Flag to indicate production mode (set by qmd.ts at startup)
418
- let _productionMode = false;
419
-
420
- export function enableProductionMode(): void {
421
- _productionMode = true;
422
- }
423
-
424
- export function getDefaultDbPath(indexName: string = "index"): string {
425
- // Always allow override via INDEX_PATH (for testing)
426
- if (process.env.INDEX_PATH) {
427
- return process.env.INDEX_PATH;
428
- }
429
-
430
- // In non-production mode (tests), require explicit path
431
- if (!_productionMode) {
432
- throw new Error(
433
- "Database path not set. Tests must set INDEX_PATH env var or use createStore() with explicit path. " +
434
- "This prevents tests from accidentally writing to the global index."
435
- );
436
- }
437
-
438
- const cacheDir = process.env.XDG_CACHE_HOME || resolve(homedir(), ".cache");
439
- const qmdCacheDir = resolve(cacheDir, "qmd");
440
- try { mkdirSync(qmdCacheDir, { recursive: true }); } catch { }
441
- return resolve(qmdCacheDir, `${indexName}.sqlite`);
442
- }
443
-
444
- export function getPwd(): string {
445
- return process.env.PWD || process.cwd();
446
- }
447
-
448
- export function getRealPath(path: string): string {
449
- try {
450
- return realpathSync(path);
451
- } catch {
452
- return resolve(path);
453
- }
454
- }
455
-
456
- // =============================================================================
457
- // Virtual Path Utilities (qmd://)
458
- // =============================================================================
459
-
460
- export type VirtualPath = {
461
- collectionName: string;
462
- path: string; // relative path within collection
463
- };
464
-
465
- /**
466
- * Normalize explicit virtual path formats to standard qmd:// format.
467
- * Only handles paths that are already explicitly virtual:
468
- * - qmd://collection/path.md (already normalized)
469
- * - qmd:////collection/path.md (extra slashes - normalize)
470
- * - //collection/path.md (missing qmd: prefix - add it)
471
- *
472
- * Does NOT handle:
473
- * - collection/path.md (bare paths - could be filesystem relative)
474
- * - :linenum suffix (should be parsed separately before calling this)
475
- */
476
- export function normalizeVirtualPath(input: string): string {
477
- let path = input.trim();
478
-
479
- // Handle qmd:// with extra slashes: qmd:////collection/path -> qmd://collection/path
480
- if (path.startsWith('qmd:')) {
481
- // Remove qmd: prefix and normalize slashes
482
- path = path.slice(4);
483
- // Remove leading slashes and re-add exactly two
484
- path = path.replace(/^\/+/, '');
485
- return `qmd://${path}`;
486
- }
487
-
488
- // Handle //collection/path (missing qmd: prefix)
489
- if (path.startsWith('//')) {
490
- path = path.replace(/^\/+/, '');
491
- return `qmd://${path}`;
492
- }
493
-
494
- // Return as-is for other cases (filesystem paths, docids, bare collection/path, etc.)
495
- return path;
496
- }
497
-
498
- /**
499
- * Parse a virtual path like "qmd://collection-name/path/to/file.md"
500
- * into its components.
501
- * Also supports collection root: "qmd://collection-name/" or "qmd://collection-name"
502
- */
503
- export function parseVirtualPath(virtualPath: string): VirtualPath | null {
504
- // Normalize the path first
505
- const normalized = normalizeVirtualPath(virtualPath);
506
-
507
- // Match: qmd://collection-name[/optional-path]
508
- // Allows: qmd://name, qmd://name/, qmd://name/path
509
- const match = normalized.match(/^qmd:\/\/([^\/]+)\/?(.*)$/);
510
- if (!match?.[1]) return null;
511
- return {
512
- collectionName: match[1],
513
- path: match[2] ?? '', // Empty string for collection root
514
- };
515
- }
516
-
517
- /**
518
- * Build a virtual path from collection name and relative path.
519
- */
520
- export function buildVirtualPath(collectionName: string, path: string): string {
521
- return `qmd://${collectionName}/${path}`;
522
- }
523
-
524
- /**
525
- * Check if a path is explicitly a virtual path.
526
- * Only recognizes explicit virtual path formats:
527
- * - qmd://collection/path.md
528
- * - //collection/path.md
529
- *
530
- * Does NOT consider bare collection/path.md as virtual - that should be
531
- * handled separately by checking if the first component is a collection name.
532
- */
533
- export function isVirtualPath(path: string): boolean {
534
- const trimmed = path.trim();
535
-
536
- // Explicit qmd:// prefix (with any number of slashes)
537
- if (trimmed.startsWith('qmd:')) return true;
538
-
539
- // //collection/path format (missing qmd: prefix)
540
- if (trimmed.startsWith('//')) return true;
541
-
542
- return false;
543
- }
544
-
545
- /**
546
- * Resolve a virtual path to absolute filesystem path.
547
- */
548
- export function resolveVirtualPath(db: Database, virtualPath: string): string | null {
549
- const parsed = parseVirtualPath(virtualPath);
550
- if (!parsed) return null;
551
-
552
- const coll = getCollectionByName(db, parsed.collectionName);
553
- if (!coll) return null;
554
-
555
- return resolve(coll.pwd, parsed.path);
556
- }
557
-
558
- /**
559
- * Convert an absolute filesystem path to a virtual path.
560
- * Returns null if the file is not in any indexed collection.
561
- */
562
- export function toVirtualPath(db: Database, absolutePath: string): string | null {
563
- // Get all collections from YAML config
564
- const collections = collectionsListCollections();
565
-
566
- // Find which collection this absolute path belongs to
567
- for (const coll of collections) {
568
- if (absolutePath.startsWith(coll.path + '/') || absolutePath === coll.path) {
569
- // Extract relative path
570
- const relativePath = absolutePath.startsWith(coll.path + '/')
571
- ? absolutePath.slice(coll.path.length + 1)
572
- : '';
573
-
574
- // Verify this document exists in the database
575
- const doc = db.prepare(`
576
- SELECT d.path
577
- FROM documents d
578
- WHERE d.collection = ? AND d.path = ? AND d.active = 1
579
- LIMIT 1
580
- `).get(coll.name, relativePath) as { path: string } | null;
581
-
582
- if (doc) {
583
- return buildVirtualPath(coll.name, relativePath);
584
- }
585
- }
586
- }
587
-
588
- return null;
589
- }
590
-
591
- // =============================================================================
592
- // Database initialization
593
- // =============================================================================
594
-
595
-
596
- function createSqliteVecUnavailableError(reason: string): Error {
597
- return new Error(
598
- "sqlite-vec extension is unavailable. " +
599
- `${reason}. ` +
600
- "Install Homebrew SQLite so the sqlite-vec extension can be loaded, " +
601
- "and set BREW_PREFIX if Homebrew is installed in a non-standard location."
602
- );
603
- }
604
-
605
- function getErrorMessage(err: unknown): string {
606
- return err instanceof Error ? err.message : String(err);
607
- }
608
-
609
- export function verifySqliteVecLoaded(db: Database): void {
610
- try {
611
- const row = db.prepare(`SELECT vec_version() AS version`).get() as { version?: string } | null;
612
- if (!row?.version || typeof row.version !== "string") {
613
- throw new Error("vec_version() returned no version");
614
- }
615
- } catch (err) {
616
- const message = getErrorMessage(err);
617
- throw createSqliteVecUnavailableError(`sqlite-vec probe failed (${message})`);
618
- }
619
- }
620
-
621
- let _sqliteVecAvailable: boolean | null = null;
622
-
623
- function initializeDatabase(db: Database): void {
624
- try {
625
- loadSqliteVec(db);
626
- verifySqliteVecLoaded(db);
627
- _sqliteVecAvailable = true;
628
- } catch {
629
- // sqlite-vec is optional — vector search won't work but FTS is fine
630
- _sqliteVecAvailable = false;
631
- }
632
- db.exec("PRAGMA journal_mode = WAL");
633
- db.exec("PRAGMA foreign_keys = ON");
634
-
635
- // Drop legacy tables that are now managed in YAML
636
- db.exec(`DROP TABLE IF EXISTS path_contexts`);
637
- db.exec(`DROP TABLE IF EXISTS collections`);
638
-
639
- // Content-addressable storage - the source of truth for document content
640
- db.exec(`
641
- CREATE TABLE IF NOT EXISTS content (
642
- hash TEXT PRIMARY KEY,
643
- doc TEXT NOT NULL,
644
- created_at TEXT NOT NULL
645
- )
646
- `);
647
-
648
- // Documents table - file system layer mapping virtual paths to content hashes
649
- // Collections are now managed in ~/.config/qmd/index.yml
650
- db.exec(`
651
- CREATE TABLE IF NOT EXISTS documents (
652
- id INTEGER PRIMARY KEY AUTOINCREMENT,
653
- collection TEXT NOT NULL,
654
- path TEXT NOT NULL,
655
- title TEXT NOT NULL,
656
- hash TEXT NOT NULL,
657
- created_at TEXT NOT NULL,
658
- modified_at TEXT NOT NULL,
659
- active INTEGER NOT NULL DEFAULT 1,
660
- FOREIGN KEY (hash) REFERENCES content(hash) ON DELETE CASCADE,
661
- UNIQUE(collection, path)
662
- )
663
- `);
664
-
665
- db.exec(`CREATE INDEX IF NOT EXISTS idx_documents_collection ON documents(collection, active)`);
666
- db.exec(`CREATE INDEX IF NOT EXISTS idx_documents_hash ON documents(hash)`);
667
- db.exec(`CREATE INDEX IF NOT EXISTS idx_documents_path ON documents(path, active)`);
668
-
669
- // Cache table for LLM API calls
670
- db.exec(`
671
- CREATE TABLE IF NOT EXISTS llm_cache (
672
- hash TEXT PRIMARY KEY,
673
- result TEXT NOT NULL,
674
- created_at TEXT NOT NULL
675
- )
676
- `);
677
-
678
- // Content vectors
679
- const cvInfo = db.prepare(`PRAGMA table_info(content_vectors)`).all() as { name: string }[];
680
- const hasSeqColumn = cvInfo.some(col => col.name === 'seq');
681
- if (cvInfo.length > 0 && !hasSeqColumn) {
682
- db.exec(`DROP TABLE IF EXISTS content_vectors`);
683
- db.exec(`DROP TABLE IF EXISTS vectors_vec`);
684
- }
685
- db.exec(`
686
- CREATE TABLE IF NOT EXISTS content_vectors (
687
- hash TEXT NOT NULL,
688
- seq INTEGER NOT NULL DEFAULT 0,
689
- pos INTEGER NOT NULL DEFAULT 0,
690
- model TEXT NOT NULL,
691
- embedded_at TEXT NOT NULL,
692
- PRIMARY KEY (hash, seq)
693
- )
694
- `);
695
-
696
- // FTS - index filepath (collection/path), title, and content
697
- db.exec(`
698
- CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5(
699
- filepath, title, body,
700
- tokenize='porter unicode61'
701
- )
702
- `);
703
-
704
- // Triggers to keep FTS in sync
705
- db.exec(`
706
- CREATE TRIGGER IF NOT EXISTS documents_ai AFTER INSERT ON documents
707
- WHEN new.active = 1
708
- BEGIN
709
- INSERT INTO documents_fts(rowid, filepath, title, body)
710
- SELECT
711
- new.id,
712
- new.collection || '/' || new.path,
713
- new.title,
714
- (SELECT doc FROM content WHERE hash = new.hash)
715
- WHERE new.active = 1;
716
- END
717
- `);
718
-
719
- db.exec(`
720
- CREATE TRIGGER IF NOT EXISTS documents_ad AFTER DELETE ON documents BEGIN
721
- DELETE FROM documents_fts WHERE rowid = old.id;
722
- END
723
- `);
724
-
725
- db.exec(`
726
- CREATE TRIGGER IF NOT EXISTS documents_au AFTER UPDATE ON documents
727
- BEGIN
728
- -- Delete from FTS if no longer active
729
- DELETE FROM documents_fts WHERE rowid = old.id AND new.active = 0;
730
-
731
- -- Update FTS if still/newly active
732
- INSERT OR REPLACE INTO documents_fts(rowid, filepath, title, body)
733
- SELECT
734
- new.id,
735
- new.collection || '/' || new.path,
736
- new.title,
737
- (SELECT doc FROM content WHERE hash = new.hash)
738
- WHERE new.active = 1;
739
- END
740
- `);
741
- }
742
-
743
-
744
- export function isSqliteVecAvailable(): boolean {
745
- return _sqliteVecAvailable === true;
746
- }
747
-
748
- function ensureVecTableInternal(db: Database, dimensions: number): void {
749
- if (!_sqliteVecAvailable) {
750
- throw new Error("sqlite-vec is not available. Vector operations require a SQLite build with extension loading support.");
751
- }
752
- const tableInfo = db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get() as { sql: string } | null;
753
- if (tableInfo) {
754
- const match = tableInfo.sql.match(/float\[(\d+)\]/);
755
- const hasHashSeq = tableInfo.sql.includes('hash_seq');
756
- const hasCosine = tableInfo.sql.includes('distance_metric=cosine');
757
- const existingDims = match?.[1] ? parseInt(match[1], 10) : null;
758
- if (existingDims === dimensions && hasHashSeq && hasCosine) return;
759
- // Table exists but wrong schema - need to rebuild
760
- db.exec("DROP TABLE IF EXISTS vectors_vec");
761
- }
762
- db.exec(`CREATE VIRTUAL TABLE vectors_vec USING vec0(hash_seq TEXT PRIMARY KEY, embedding float[${dimensions}] distance_metric=cosine)`);
763
- }
764
-
765
- // =============================================================================
766
- // Store Factory
767
- // =============================================================================
768
-
769
- export type Store = {
770
- db: Database;
771
- dbPath: string;
772
- close: () => void;
773
- ensureVecTable: (dimensions: number) => void;
774
-
775
- // Index health
776
- getHashesNeedingEmbedding: () => number;
777
- getIndexHealth: () => IndexHealthInfo;
778
- getStatus: () => IndexStatus;
779
-
780
- // Caching
781
- getCacheKey: typeof getCacheKey;
782
- getCachedResult: (cacheKey: string) => string | null;
783
- setCachedResult: (cacheKey: string, result: string) => void;
784
- clearCache: () => void;
785
-
786
- // Cleanup and maintenance
787
- deleteLLMCache: () => number;
788
- deleteInactiveDocuments: () => number;
789
- cleanupOrphanedContent: () => number;
790
- cleanupOrphanedVectors: () => number;
791
- vacuumDatabase: () => void;
792
-
793
- // Context
794
- getContextForFile: (filepath: string) => string | null;
795
- getContextForPath: (collectionName: string, path: string) => string | null;
796
- getCollectionByName: (name: string) => { name: string; pwd: string; glob_pattern: string } | null;
797
- getCollectionsWithoutContext: () => { name: string; pwd: string; doc_count: number }[];
798
- getTopLevelPathsWithoutContext: (collectionName: string) => string[];
799
-
800
- // Virtual paths
801
- parseVirtualPath: typeof parseVirtualPath;
802
- buildVirtualPath: typeof buildVirtualPath;
803
- isVirtualPath: typeof isVirtualPath;
804
- resolveVirtualPath: (virtualPath: string) => string | null;
805
- toVirtualPath: (absolutePath: string) => string | null;
806
-
807
- // Search
808
- searchFTS: (query: string, limit?: number, collectionName?: string) => SearchResult[];
809
- searchVec: (query: string, model: string, limit?: number, collectionName?: string, session?: ILLMSession, precomputedEmbedding?: number[]) => Promise<SearchResult[]>;
810
-
811
- // Query expansion & reranking
812
- expandQuery: (query: string, model?: string) => Promise<ExpandedQuery[]>;
813
- rerank: (query: string, documents: { file: string; text: string }[], model?: string) => Promise<{ file: string; score: number }[]>;
814
-
815
- // Document retrieval
816
- findDocument: (filename: string, options?: { includeBody?: boolean }) => DocumentResult | DocumentNotFound;
817
- getDocumentBody: (doc: DocumentResult | { filepath: string }, fromLine?: number, maxLines?: number) => string | null;
818
- findDocuments: (pattern: string, options?: { includeBody?: boolean; maxBytes?: number }) => { docs: MultiGetResult[]; errors: string[] };
819
-
820
- // Fuzzy matching and docid lookup
821
- findSimilarFiles: (query: string, maxDistance?: number, limit?: number) => string[];
822
- matchFilesByGlob: (pattern: string) => { filepath: string; displayPath: string; bodyLength: number }[];
823
- findDocumentByDocid: (docid: string) => { filepath: string; hash: string } | null;
824
-
825
- // Document indexing operations
826
- insertContent: (hash: string, content: string, createdAt: string) => void;
827
- insertDocument: (collectionName: string, path: string, title: string, hash: string, createdAt: string, modifiedAt: string) => void;
828
- findActiveDocument: (collectionName: string, path: string) => { id: number; hash: string; title: string } | null;
829
- updateDocumentTitle: (documentId: number, title: string, modifiedAt: string) => void;
830
- updateDocument: (documentId: number, title: string, hash: string, modifiedAt: string) => void;
831
- deactivateDocument: (collectionName: string, path: string) => void;
832
- getActiveDocumentPaths: (collectionName: string) => string[];
833
-
834
- // Vector/embedding operations
835
- getHashesForEmbedding: () => { hash: string; body: string; path: string }[];
836
- clearAllEmbeddings: () => void;
837
- insertEmbedding: (hash: string, seq: number, pos: number, embedding: Float32Array, model: string, embeddedAt: string) => void;
838
- };
839
-
840
- /**
841
- * Create a new store instance with the given database path.
842
- * If no path is provided, uses the default path (~/.cache/qmd/index.sqlite).
843
- *
844
- * @param dbPath - Path to the SQLite database file
845
- * @returns Store instance with all methods bound to the database
846
- */
847
- export function createStore(dbPath?: string): Store {
848
- const resolvedPath = dbPath || getDefaultDbPath();
849
- const db = openDatabase(resolvedPath);
850
- initializeDatabase(db);
851
-
852
- return {
853
- db,
854
- dbPath: resolvedPath,
855
- close: () => db.close(),
856
- ensureVecTable: (dimensions: number) => ensureVecTableInternal(db, dimensions),
857
-
858
- // Index health
859
- getHashesNeedingEmbedding: () => getHashesNeedingEmbedding(db),
860
- getIndexHealth: () => getIndexHealth(db),
861
- getStatus: () => getStatus(db),
862
-
863
- // Caching
864
- getCacheKey,
865
- getCachedResult: (cacheKey: string) => getCachedResult(db, cacheKey),
866
- setCachedResult: (cacheKey: string, result: string) => setCachedResult(db, cacheKey, result),
867
- clearCache: () => clearCache(db),
868
-
869
- // Cleanup and maintenance
870
- deleteLLMCache: () => deleteLLMCache(db),
871
- deleteInactiveDocuments: () => deleteInactiveDocuments(db),
872
- cleanupOrphanedContent: () => cleanupOrphanedContent(db),
873
- cleanupOrphanedVectors: () => cleanupOrphanedVectors(db),
874
- vacuumDatabase: () => vacuumDatabase(db),
875
-
876
- // Context
877
- getContextForFile: (filepath: string) => getContextForFile(db, filepath),
878
- getContextForPath: (collectionName: string, path: string) => getContextForPath(db, collectionName, path),
879
- getCollectionByName: (name: string) => getCollectionByName(db, name),
880
- getCollectionsWithoutContext: () => getCollectionsWithoutContext(db),
881
- getTopLevelPathsWithoutContext: (collectionName: string) => getTopLevelPathsWithoutContext(db, collectionName),
882
-
883
- // Virtual paths
884
- parseVirtualPath,
885
- buildVirtualPath,
886
- isVirtualPath,
887
- resolveVirtualPath: (virtualPath: string) => resolveVirtualPath(db, virtualPath),
888
- toVirtualPath: (absolutePath: string) => toVirtualPath(db, absolutePath),
889
-
890
- // Search
891
- searchFTS: (query: string, limit?: number, collectionName?: string) => searchFTS(db, query, limit, collectionName),
892
- searchVec: (query: string, model: string, limit?: number, collectionName?: string, session?: ILLMSession, precomputedEmbedding?: number[]) => searchVec(db, query, model, limit, collectionName, session, precomputedEmbedding),
893
-
894
- // Query expansion & reranking
895
- expandQuery: (query: string, model?: string) => expandQuery(query, model, db),
896
- rerank: (query: string, documents: { file: string; text: string }[], model?: string) => rerank(query, documents, model, db),
897
-
898
- // Document retrieval
899
- findDocument: (filename: string, options?: { includeBody?: boolean }) => findDocument(db, filename, options),
900
- getDocumentBody: (doc: DocumentResult | { filepath: string }, fromLine?: number, maxLines?: number) => getDocumentBody(db, doc, fromLine, maxLines),
901
- findDocuments: (pattern: string, options?: { includeBody?: boolean; maxBytes?: number }) => findDocuments(db, pattern, options),
902
-
903
- // Fuzzy matching and docid lookup
904
- findSimilarFiles: (query: string, maxDistance?: number, limit?: number) => findSimilarFiles(db, query, maxDistance, limit),
905
- matchFilesByGlob: (pattern: string) => matchFilesByGlob(db, pattern),
906
- findDocumentByDocid: (docid: string) => findDocumentByDocid(db, docid),
907
-
908
- // Document indexing operations
909
- insertContent: (hash: string, content: string, createdAt: string) => insertContent(db, hash, content, createdAt),
910
- insertDocument: (collectionName: string, path: string, title: string, hash: string, createdAt: string, modifiedAt: string) => insertDocument(db, collectionName, path, title, hash, createdAt, modifiedAt),
911
- findActiveDocument: (collectionName: string, path: string) => findActiveDocument(db, collectionName, path),
912
- updateDocumentTitle: (documentId: number, title: string, modifiedAt: string) => updateDocumentTitle(db, documentId, title, modifiedAt),
913
- updateDocument: (documentId: number, title: string, hash: string, modifiedAt: string) => updateDocument(db, documentId, title, hash, modifiedAt),
914
- deactivateDocument: (collectionName: string, path: string) => deactivateDocument(db, collectionName, path),
915
- getActiveDocumentPaths: (collectionName: string) => getActiveDocumentPaths(db, collectionName),
916
-
917
- // Vector/embedding operations
918
- getHashesForEmbedding: () => getHashesForEmbedding(db),
919
- clearAllEmbeddings: () => clearAllEmbeddings(db),
920
- insertEmbedding: (hash: string, seq: number, pos: number, embedding: Float32Array, model: string, embeddedAt: string) => insertEmbedding(db, hash, seq, pos, embedding, model, embeddedAt),
921
- };
922
- }
923
-
924
- // =============================================================================
925
- // Core Document Type
926
- // =============================================================================
927
-
928
- /**
929
- * Unified document result type with all metadata.
930
- * Body is optional - use getDocumentBody() to load it separately if needed.
931
- */
932
- export type DocumentResult = {
933
- filepath: string; // Full filesystem path
934
- displayPath: string; // Short display path (e.g., "docs/readme.md")
935
- title: string; // Document title (from first heading or filename)
936
- context: string | null; // Folder context description if configured
937
- hash: string; // Content hash for caching/change detection
938
- docid: string; // Short docid (first 6 chars of hash) for quick reference
939
- collectionName: string; // Parent collection name
940
- modifiedAt: string; // Last modification timestamp
941
- bodyLength: number; // Body length in bytes (useful before loading)
942
- body?: string; // Document body (optional, load with getDocumentBody)
943
- };
944
-
945
- /**
946
- * Extract short docid from a full hash (first 6 characters).
947
- */
948
- export function getDocid(hash: string): string {
949
- return hash.slice(0, 6);
950
- }
951
-
952
- /**
953
- * Handelize a filename to be more token-friendly.
954
- * - Convert triple underscore `___` to `/` (folder separator)
955
- * - Convert to lowercase
956
- * - Replace sequences of non-word chars (except /) with single dash
957
- * - Remove leading/trailing dashes from path segments
958
- * - Preserve folder structure (a/b/c/d.md stays structured)
959
- * - Preserve file extension
960
- */
961
- export function handelize(path: string): string {
962
- if (!path || path.trim() === '') {
963
- throw new Error('handelize: path cannot be empty');
964
- }
965
-
966
- // Allow route-style "$" filenames while still rejecting paths with no usable content.
967
- const segments = path.split('/').filter(Boolean);
968
- const lastSegment = segments[segments.length - 1] || '';
969
- const filenameWithoutExt = lastSegment.replace(/\.[^.]+$/, '');
970
- const hasValidContent = /[\p{L}\p{N}$]/u.test(filenameWithoutExt);
971
- if (!hasValidContent) {
972
- throw new Error(`handelize: path "${path}" has no valid filename content`);
973
- }
974
-
975
- const result = path
976
- .replace(/___/g, '/') // Triple underscore becomes folder separator
977
- .toLowerCase()
978
- .split('/')
979
- .map((segment, idx, arr) => {
980
- const isLastSegment = idx === arr.length - 1;
981
-
982
- if (isLastSegment) {
983
- // For the filename (last segment), preserve the extension
984
- const extMatch = segment.match(/(\.[a-z0-9]+)$/i);
985
- const ext = extMatch ? extMatch[1] : '';
986
- const nameWithoutExt = ext ? segment.slice(0, -ext.length) : segment;
987
-
988
- const cleanedName = nameWithoutExt
989
- .replace(/[^\p{L}\p{N}$]+/gu, '-') // Keep route marker "$", dash-separate other chars
990
- .replace(/^-+|-+$/g, ''); // Remove leading/trailing dashes
991
-
992
- return cleanedName + ext;
993
- } else {
994
- // For directories, just clean normally
995
- return segment
996
- .replace(/[^\p{L}\p{N}$]+/gu, '-')
997
- .replace(/^-+|-+$/g, '');
998
- }
999
- })
1000
- .filter(Boolean)
1001
- .join('/');
1002
-
1003
- if (!result) {
1004
- throw new Error(`handelize: path "${path}" resulted in empty string after processing`);
1005
- }
1006
-
1007
- return result;
1008
- }
1009
-
1010
- /**
1011
- * Search result extends DocumentResult with score and source info
1012
- */
1013
- export type SearchResult = DocumentResult & {
1014
- score: number; // Relevance score (0-1)
1015
- source: "fts" | "vec"; // Search source (full-text or vector)
1016
- chunkPos?: number; // Character position of matching chunk (for vector search)
1017
- };
1018
-
1019
- /**
1020
- * Ranked result for RRF fusion (simplified, used internally)
1021
- */
1022
- export type RankedResult = {
1023
- file: string;
1024
- displayPath: string;
1025
- title: string;
1026
- body: string;
1027
- score: number;
1028
- };
1029
-
1030
- /**
1031
- * Error result when document is not found
1032
- */
1033
- export type DocumentNotFound = {
1034
- error: "not_found";
1035
- query: string;
1036
- similarFiles: string[];
1037
- };
1038
-
1039
- /**
1040
- * Result from multi-get operations
1041
- */
1042
- export type MultiGetResult = {
1043
- doc: DocumentResult;
1044
- skipped: false;
1045
- } | {
1046
- doc: Pick<DocumentResult, "filepath" | "displayPath">;
1047
- skipped: true;
1048
- skipReason: string;
1049
- };
1050
-
1051
- export type CollectionInfo = {
1052
- name: string;
1053
- path: string;
1054
- pattern: string;
1055
- documents: number;
1056
- lastUpdated: string;
1057
- };
1058
-
1059
- export type IndexStatus = {
1060
- totalDocuments: number;
1061
- needsEmbedding: number;
1062
- hasVectorIndex: boolean;
1063
- collections: CollectionInfo[];
1064
- };
1065
-
1066
- // =============================================================================
1067
- // Index health
1068
- // =============================================================================
1069
-
1070
- export function getHashesNeedingEmbedding(db: Database): number {
1071
- const result = db.prepare(`
1072
- SELECT COUNT(DISTINCT d.hash) as count
1073
- FROM documents d
1074
- LEFT JOIN content_vectors v ON d.hash = v.hash AND v.seq = 0
1075
- WHERE d.active = 1 AND v.hash IS NULL
1076
- `).get() as { count: number };
1077
- return result.count;
1078
- }
1079
-
1080
- export type IndexHealthInfo = {
1081
- needsEmbedding: number;
1082
- totalDocs: number;
1083
- daysStale: number | null;
1084
- };
1085
-
1086
- export function getIndexHealth(db: Database): IndexHealthInfo {
1087
- const needsEmbedding = getHashesNeedingEmbedding(db);
1088
- const totalDocs = (db.prepare(`SELECT COUNT(*) as count FROM documents WHERE active = 1`).get() as { count: number }).count;
1089
-
1090
- const mostRecent = db.prepare(`SELECT MAX(modified_at) as latest FROM documents WHERE active = 1`).get() as { latest: string | null };
1091
- let daysStale: number | null = null;
1092
- if (mostRecent?.latest) {
1093
- const lastUpdate = new Date(mostRecent.latest);
1094
- daysStale = Math.floor((Date.now() - lastUpdate.getTime()) / (24 * 60 * 60 * 1000));
1095
- }
1096
-
1097
- return { needsEmbedding, totalDocs, daysStale };
1098
- }
1099
-
1100
- // =============================================================================
1101
- // Caching
1102
- // =============================================================================
1103
-
1104
- export function getCacheKey(url: string, body: object): string {
1105
- const hash = createHash("sha256");
1106
- hash.update(url);
1107
- hash.update(JSON.stringify(body));
1108
- return hash.digest("hex");
1109
- }
1110
-
1111
- export function getCachedResult(db: Database, cacheKey: string): string | null {
1112
- const row = db.prepare(`SELECT result FROM llm_cache WHERE hash = ?`).get(cacheKey) as { result: string } | null;
1113
- return row?.result || null;
1114
- }
1115
-
1116
- export function setCachedResult(db: Database, cacheKey: string, result: string): void {
1117
- const now = new Date().toISOString();
1118
- db.prepare(`INSERT OR REPLACE INTO llm_cache (hash, result, created_at) VALUES (?, ?, ?)`).run(cacheKey, result, now);
1119
- if (Math.random() < 0.01) {
1120
- db.exec(`DELETE FROM llm_cache WHERE hash NOT IN (SELECT hash FROM llm_cache ORDER BY created_at DESC LIMIT 1000)`);
1121
- }
1122
- }
1123
-
1124
- export function clearCache(db: Database): void {
1125
- db.exec(`DELETE FROM llm_cache`);
1126
- }
1127
-
1128
- // =============================================================================
1129
- // Cleanup and maintenance operations
1130
- // =============================================================================
1131
-
1132
- /**
1133
- * Delete cached LLM API responses.
1134
- * Returns the number of cached responses deleted.
1135
- */
1136
- export function deleteLLMCache(db: Database): number {
1137
- const result = db.prepare(`DELETE FROM llm_cache`).run();
1138
- return result.changes;
1139
- }
1140
-
1141
- /**
1142
- * Remove inactive document records (active = 0).
1143
- * Returns the number of inactive documents deleted.
1144
- */
1145
- export function deleteInactiveDocuments(db: Database): number {
1146
- const result = db.prepare(`DELETE FROM documents WHERE active = 0`).run();
1147
- return result.changes;
1148
- }
1149
-
1150
- /**
1151
- * Remove orphaned content hashes that are not referenced by any active document.
1152
- * Returns the number of orphaned content hashes deleted.
1153
- */
1154
- export function cleanupOrphanedContent(db: Database): number {
1155
- const result = db.prepare(`
1156
- DELETE FROM content
1157
- WHERE hash NOT IN (SELECT DISTINCT hash FROM documents WHERE active = 1)
1158
- `).run();
1159
- return result.changes;
1160
- }
1161
-
1162
- /**
1163
- * Remove orphaned vector embeddings that are not referenced by any active document.
1164
- * Returns the number of orphaned embedding chunks deleted.
1165
- */
1166
- export function cleanupOrphanedVectors(db: Database): number {
1167
- // Check if vectors_vec table exists
1168
- const tableExists = db.prepare(`
1169
- SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'
1170
- `).get();
1171
-
1172
- if (!tableExists) {
1173
- return 0;
1174
- }
1175
-
1176
- // Count orphaned vectors first
1177
- const countResult = db.prepare(`
1178
- SELECT COUNT(*) as c FROM content_vectors cv
1179
- WHERE NOT EXISTS (
1180
- SELECT 1 FROM documents d WHERE d.hash = cv.hash AND d.active = 1
1181
- )
1182
- `).get() as { c: number };
1183
-
1184
- if (countResult.c === 0) {
1185
- return 0;
1186
- }
1187
-
1188
- // Delete from vectors_vec first
1189
- db.exec(`
1190
- DELETE FROM vectors_vec WHERE hash_seq IN (
1191
- SELECT cv.hash || '_' || cv.seq FROM content_vectors cv
1192
- WHERE NOT EXISTS (
1193
- SELECT 1 FROM documents d WHERE d.hash = cv.hash AND d.active = 1
1194
- )
1195
- )
1196
- `);
1197
-
1198
- // Delete from content_vectors
1199
- db.exec(`
1200
- DELETE FROM content_vectors WHERE hash NOT IN (
1201
- SELECT hash FROM documents WHERE active = 1
1202
- )
1203
- `);
1204
-
1205
- return countResult.c;
1206
- }
1207
-
1208
- /**
1209
- * Run VACUUM to reclaim unused space in the database.
1210
- * This operation rebuilds the database file to eliminate fragmentation.
1211
- */
1212
- export function vacuumDatabase(db: Database): void {
1213
- db.exec(`VACUUM`);
1214
- }
1215
-
1216
- // =============================================================================
1217
- // Document helpers
1218
- // =============================================================================
1219
-
1220
- export async function hashContent(content: string): Promise<string> {
1221
- const hash = createHash("sha256");
1222
- hash.update(content);
1223
- return hash.digest("hex");
1224
- }
1225
-
1226
- const titleExtractors: Record<string, (content: string) => string | null> = {
1227
- '.md': (content) => {
1228
- const match = content.match(/^##?\s+(.+)$/m);
1229
- if (match) {
1230
- const title = (match[1] ?? "").trim();
1231
- if (title === "📝 Notes" || title === "Notes") {
1232
- const nextMatch = content.match(/^##\s+(.+)$/m);
1233
- if (nextMatch?.[1]) return nextMatch[1].trim();
1234
- }
1235
- return title;
1236
- }
1237
- return null;
1238
- },
1239
- '.org': (content) => {
1240
- const titleProp = content.match(/^#\+TITLE:\s*(.+)$/im);
1241
- if (titleProp?.[1]) return titleProp[1].trim();
1242
- const heading = content.match(/^\*+\s+(.+)$/m);
1243
- if (heading?.[1]) return heading[1].trim();
1244
- return null;
1245
- },
1246
- };
1247
-
1248
- export function extractTitle(content: string, filename: string): string {
1249
- const ext = filename.slice(filename.lastIndexOf('.')).toLowerCase();
1250
- const extractor = titleExtractors[ext];
1251
- if (extractor) {
1252
- const title = extractor(content);
1253
- if (title) return title;
1254
- }
1255
- return filename.replace(/\.[^.]+$/, "").split("/").pop() || filename;
1256
- }
1257
-
1258
- // =============================================================================
1259
- // Document indexing operations
1260
- // =============================================================================
1261
-
1262
- /**
1263
- * Insert content into the content table (content-addressable storage).
1264
- * Uses INSERT OR IGNORE so duplicate hashes are skipped.
1265
- */
1266
- export function insertContent(db: Database, hash: string, content: string, createdAt: string): void {
1267
- db.prepare(`INSERT OR IGNORE INTO content (hash, doc, created_at) VALUES (?, ?, ?)`)
1268
- .run(hash, content, createdAt);
1269
- }
1270
-
1271
- /**
1272
- * Insert a new document into the documents table.
1273
- */
1274
- export function insertDocument(
1275
- db: Database,
1276
- collectionName: string,
1277
- path: string,
1278
- title: string,
1279
- hash: string,
1280
- createdAt: string,
1281
- modifiedAt: string
1282
- ): void {
1283
- db.prepare(`
1284
- INSERT INTO documents (collection, path, title, hash, created_at, modified_at, active)
1285
- VALUES (?, ?, ?, ?, ?, ?, 1)
1286
- ON CONFLICT(collection, path) DO UPDATE SET
1287
- title = excluded.title,
1288
- hash = excluded.hash,
1289
- modified_at = excluded.modified_at,
1290
- active = 1
1291
- `).run(collectionName, path, title, hash, createdAt, modifiedAt);
1292
- }
1293
-
1294
- /**
1295
- * Find an active document by collection name and path.
1296
- */
1297
- export function findActiveDocument(
1298
- db: Database,
1299
- collectionName: string,
1300
- path: string
1301
- ): { id: number; hash: string; title: string } | null {
1302
- const row = db.prepare(`
1303
- SELECT id, hash, title FROM documents
1304
- WHERE collection = ? AND path = ? AND active = 1
1305
- `).get(collectionName, path) as { id: number; hash: string; title: string } | undefined;
1306
- return row ?? null;
1307
- }
1308
-
1309
- /**
1310
- * Update the title and modified_at timestamp for a document.
1311
- */
1312
- export function updateDocumentTitle(
1313
- db: Database,
1314
- documentId: number,
1315
- title: string,
1316
- modifiedAt: string
1317
- ): void {
1318
- db.prepare(`UPDATE documents SET title = ?, modified_at = ? WHERE id = ?`)
1319
- .run(title, modifiedAt, documentId);
1320
- }
1321
-
1322
- /**
1323
- * Update an existing document's hash, title, and modified_at timestamp.
1324
- * Used when content changes but the file path stays the same.
1325
- */
1326
- export function updateDocument(
1327
- db: Database,
1328
- documentId: number,
1329
- title: string,
1330
- hash: string,
1331
- modifiedAt: string
1332
- ): void {
1333
- db.prepare(`UPDATE documents SET title = ?, hash = ?, modified_at = ? WHERE id = ?`)
1334
- .run(title, hash, modifiedAt, documentId);
1335
- }
1336
-
1337
- /**
1338
- * Deactivate a document (mark as inactive but don't delete).
1339
- */
1340
- export function deactivateDocument(db: Database, collectionName: string, path: string): void {
1341
- db.prepare(`UPDATE documents SET active = 0 WHERE collection = ? AND path = ? AND active = 1`)
1342
- .run(collectionName, path);
1343
- }
1344
-
1345
- /**
1346
- * Get all active document paths for a collection.
1347
- */
1348
- export function getActiveDocumentPaths(db: Database, collectionName: string): string[] {
1349
- const rows = db.prepare(`
1350
- SELECT path FROM documents WHERE collection = ? AND active = 1
1351
- `).all(collectionName) as { path: string }[];
1352
- return rows.map(r => r.path);
1353
- }
1354
-
1355
- export { formatQueryForEmbedding, formatDocForEmbedding };
1356
-
1357
- export function chunkDocument(
1358
- content: string,
1359
- maxChars: number = CHUNK_SIZE_CHARS,
1360
- overlapChars: number = CHUNK_OVERLAP_CHARS,
1361
- windowChars: number = CHUNK_WINDOW_CHARS
1362
- ): { text: string; pos: number }[] {
1363
- if (content.length <= maxChars) {
1364
- return [{ text: content, pos: 0 }];
1365
- }
1366
-
1367
- // Pre-scan all break points and code fences once
1368
- const breakPoints = scanBreakPoints(content);
1369
- const codeFences = findCodeFences(content);
1370
-
1371
- const chunks: { text: string; pos: number }[] = [];
1372
- let charPos = 0;
1373
-
1374
- while (charPos < content.length) {
1375
- // Calculate target end position for this chunk
1376
- const targetEndPos = Math.min(charPos + maxChars, content.length);
1377
-
1378
- let endPos = targetEndPos;
1379
-
1380
- // If not at the end, find the best break point
1381
- if (endPos < content.length) {
1382
- // Find best cutoff using scored algorithm
1383
- const bestCutoff = findBestCutoff(
1384
- breakPoints,
1385
- targetEndPos,
1386
- windowChars,
1387
- 0.7,
1388
- codeFences
1389
- );
1390
-
1391
- // Only use the cutoff if it's within our current chunk
1392
- if (bestCutoff > charPos && bestCutoff <= targetEndPos) {
1393
- endPos = bestCutoff;
1394
- }
1395
- }
1396
-
1397
- // Ensure we make progress
1398
- if (endPos <= charPos) {
1399
- endPos = Math.min(charPos + maxChars, content.length);
1400
- }
1401
-
1402
- chunks.push({ text: content.slice(charPos, endPos), pos: charPos });
1403
-
1404
- // Move forward, but overlap with previous chunk
1405
- // For last chunk, don't overlap (just go to the end)
1406
- if (endPos >= content.length) {
1407
- break;
1408
- }
1409
- charPos = endPos - overlapChars;
1410
- const lastChunkPos = chunks.at(-1)!.pos;
1411
- if (charPos <= lastChunkPos) {
1412
- // Prevent infinite loop - move forward at least a bit
1413
- charPos = endPos;
1414
- }
1415
- }
1416
-
1417
- return chunks;
1418
- }
1419
-
1420
- /**
1421
- * Chunk a document by actual token count using the LLM tokenizer.
1422
- * More accurate than character-based chunking but requires async.
1423
- */
1424
- export async function chunkDocumentByTokens(
1425
- content: string,
1426
- maxTokens: number = CHUNK_SIZE_TOKENS,
1427
- overlapTokens: number = CHUNK_OVERLAP_TOKENS,
1428
- windowTokens: number = CHUNK_WINDOW_TOKENS
1429
- ): Promise<{ text: string; pos: number; tokens: number }[]> {
1430
- const llm = getDefaultLlamaCpp();
1431
-
1432
- // Use moderate chars/token estimate (prose ~4, code ~2, mixed ~3)
1433
- // If chunks exceed limit, they'll be re-split with actual ratio
1434
- const avgCharsPerToken = 3;
1435
- const maxChars = maxTokens * avgCharsPerToken;
1436
- const overlapChars = overlapTokens * avgCharsPerToken;
1437
- const windowChars = windowTokens * avgCharsPerToken;
1438
-
1439
- // Chunk in character space with conservative estimate
1440
- let charChunks = chunkDocument(content, maxChars, overlapChars, windowChars);
1441
-
1442
- // Tokenize and split any chunks that still exceed limit
1443
- const results: { text: string; pos: number; tokens: number }[] = [];
1444
-
1445
- for (const chunk of charChunks) {
1446
- const tokens = await llm.tokenize(chunk.text);
1447
-
1448
- if (tokens.length <= maxTokens) {
1449
- results.push({ text: chunk.text, pos: chunk.pos, tokens: tokens.length });
1450
- } else {
1451
- // Chunk is still too large - split it further
1452
- // Use actual token count to estimate better char limit
1453
- const actualCharsPerToken = chunk.text.length / tokens.length;
1454
- const safeMaxChars = Math.floor(maxTokens * actualCharsPerToken * 0.95); // 5% safety margin
1455
-
1456
- const subChunks = chunkDocument(chunk.text, safeMaxChars, Math.floor(overlapChars * actualCharsPerToken / 2), Math.floor(windowChars * actualCharsPerToken / 2));
1457
-
1458
- for (const subChunk of subChunks) {
1459
- const subTokens = await llm.tokenize(subChunk.text);
1460
- results.push({
1461
- text: subChunk.text,
1462
- pos: chunk.pos + subChunk.pos,
1463
- tokens: subTokens.length,
1464
- });
1465
- }
1466
- }
1467
- }
1468
-
1469
- return results;
1470
- }
1471
-
1472
- // =============================================================================
1473
- // Fuzzy matching
1474
- // =============================================================================
1475
-
1476
- function levenshtein(a: string, b: string): number {
1477
- const m = a.length, n = b.length;
1478
- if (m === 0) return n;
1479
- if (n === 0) return m;
1480
- const dp: number[][] = Array.from({ length: m + 1 }, () => Array(n + 1).fill(0));
1481
- for (let i = 0; i <= m; i++) dp[i]![0] = i;
1482
- for (let j = 0; j <= n; j++) dp[0]![j] = j;
1483
- for (let i = 1; i <= m; i++) {
1484
- for (let j = 1; j <= n; j++) {
1485
- const cost = a[i - 1] === b[j - 1] ? 0 : 1;
1486
- dp[i]![j] = Math.min(
1487
- dp[i - 1]![j]! + 1,
1488
- dp[i]![j - 1]! + 1,
1489
- dp[i - 1]![j - 1]! + cost
1490
- );
1491
- }
1492
- }
1493
- return dp[m]![n]!;
1494
- }
1495
-
1496
- /**
1497
- * Normalize a docid input by stripping surrounding quotes and leading #.
1498
- * Handles: "#abc123", 'abc123', "abc123", #abc123, abc123
1499
- * Returns the bare hex string.
1500
- */
1501
- export function normalizeDocid(docid: string): string {
1502
- let normalized = docid.trim();
1503
-
1504
- // Strip surrounding quotes (single or double)
1505
- if ((normalized.startsWith('"') && normalized.endsWith('"')) ||
1506
- (normalized.startsWith("'") && normalized.endsWith("'"))) {
1507
- normalized = normalized.slice(1, -1);
1508
- }
1509
-
1510
- // Strip leading # if present
1511
- if (normalized.startsWith('#')) {
1512
- normalized = normalized.slice(1);
1513
- }
1514
-
1515
- return normalized;
1516
- }
1517
-
1518
- /**
1519
- * Check if a string looks like a docid reference.
1520
- * Accepts: #abc123, abc123, "#abc123", "abc123", '#abc123', 'abc123'
1521
- * Returns true if the normalized form is a valid hex string of 6+ chars.
1522
- */
1523
- export function isDocid(input: string): boolean {
1524
- const normalized = normalizeDocid(input);
1525
- // Must be at least 6 hex characters
1526
- return normalized.length >= 6 && /^[a-f0-9]+$/i.test(normalized);
1527
- }
1528
-
1529
- /**
1530
- * Find a document by its short docid (first 6 characters of hash).
1531
- * Returns the document's virtual path if found, null otherwise.
1532
- * If multiple documents match the same short hash (collision), returns the first one.
1533
- *
1534
- * Accepts lenient input: #abc123, abc123, "#abc123", "abc123"
1535
- */
1536
- export function findDocumentByDocid(db: Database, docid: string): { filepath: string; hash: string } | null {
1537
- const shortHash = normalizeDocid(docid);
1538
-
1539
- if (shortHash.length < 1) return null;
1540
-
1541
- // Look up documents where hash starts with the short hash
1542
- const doc = db.prepare(`
1543
- SELECT 'qmd://' || d.collection || '/' || d.path as filepath, d.hash
1544
- FROM documents d
1545
- WHERE d.hash LIKE ? AND d.active = 1
1546
- LIMIT 1
1547
- `).get(`${shortHash}%`) as { filepath: string; hash: string } | null;
1548
-
1549
- return doc;
1550
- }
1551
-
1552
- export function findSimilarFiles(db: Database, query: string, maxDistance: number = 3, limit: number = 5): string[] {
1553
- const allFiles = db.prepare(`
1554
- SELECT d.path
1555
- FROM documents d
1556
- WHERE d.active = 1
1557
- `).all() as { path: string }[];
1558
- const queryLower = query.toLowerCase();
1559
- const scored = allFiles
1560
- .map(f => ({ path: f.path, dist: levenshtein(f.path.toLowerCase(), queryLower) }))
1561
- .filter(f => f.dist <= maxDistance)
1562
- .sort((a, b) => a.dist - b.dist)
1563
- .slice(0, limit);
1564
- return scored.map(f => f.path);
1565
- }
1566
-
1567
- export function matchFilesByGlob(db: Database, pattern: string): { filepath: string; displayPath: string; bodyLength: number }[] {
1568
- const allFiles = db.prepare(`
1569
- SELECT
1570
- 'qmd://' || d.collection || '/' || d.path as virtual_path,
1571
- LENGTH(content.doc) as body_length,
1572
- d.path,
1573
- d.collection
1574
- FROM documents d
1575
- JOIN content ON content.hash = d.hash
1576
- WHERE d.active = 1
1577
- `).all() as { virtual_path: string; body_length: number; path: string; collection: string }[];
1578
-
1579
- const isMatch = picomatch(pattern);
1580
- return allFiles
1581
- .filter(f => isMatch(f.virtual_path) || isMatch(f.path))
1582
- .map(f => ({
1583
- filepath: f.virtual_path, // Virtual path for precise lookup
1584
- displayPath: f.path, // Relative path for display
1585
- bodyLength: f.body_length
1586
- }));
1587
- }
1588
-
1589
- // =============================================================================
1590
- // Context
1591
- // =============================================================================
1592
-
1593
- /**
1594
- * Get context for a file path using hierarchical inheritance.
1595
- * Contexts are collection-scoped and inherit from parent directories.
1596
- * For example, context at "/talks" applies to "/talks/2024/keynote.md".
1597
- *
1598
- * @param db Database instance (unused - kept for compatibility)
1599
- * @param collectionName Collection name
1600
- * @param path Relative path within the collection
1601
- * @returns Context string or null if no context is defined
1602
- */
1603
- export function getContextForPath(db: Database, collectionName: string, path: string): string | null {
1604
- const config = collectionsLoadConfig();
1605
- const coll = getCollection(collectionName);
1606
-
1607
- if (!coll) return null;
1608
-
1609
- // Collect ALL matching contexts (global + all path prefixes)
1610
- const contexts: string[] = [];
1611
-
1612
- // Add global context if present
1613
- if (config.global_context) {
1614
- contexts.push(config.global_context);
1615
- }
1616
-
1617
- // Add all matching path contexts (from most general to most specific)
1618
- if (coll.context) {
1619
- const normalizedPath = path.startsWith("/") ? path : `/${path}`;
1620
-
1621
- // Collect all matching prefixes
1622
- const matchingContexts: { prefix: string; context: string }[] = [];
1623
- for (const [prefix, context] of Object.entries(coll.context)) {
1624
- const normalizedPrefix = prefix.startsWith("/") ? prefix : `/${prefix}`;
1625
- if (normalizedPath.startsWith(normalizedPrefix)) {
1626
- matchingContexts.push({ prefix: normalizedPrefix, context });
1627
- }
1628
- }
1629
-
1630
- // Sort by prefix length (shortest/most general first)
1631
- matchingContexts.sort((a, b) => a.prefix.length - b.prefix.length);
1632
-
1633
- // Add all matching contexts
1634
- for (const match of matchingContexts) {
1635
- contexts.push(match.context);
1636
- }
1637
- }
1638
-
1639
- // Join all contexts with double newline
1640
- return contexts.length > 0 ? contexts.join('\n\n') : null;
1641
- }
1642
-
1643
- /**
1644
- * Get context for a file path (virtual or filesystem).
1645
- * Resolves the collection and relative path using the YAML collections config.
1646
- */
1647
- export function getContextForFile(db: Database, filepath: string): string | null {
1648
- // Handle undefined or null filepath
1649
- if (!filepath) return null;
1650
-
1651
- // Get all collections from YAML config
1652
- const collections = collectionsListCollections();
1653
- const config = collectionsLoadConfig();
1654
-
1655
- // Parse virtual path format: qmd://collection/path
1656
- let collectionName: string | null = null;
1657
- let relativePath: string | null = null;
1658
-
1659
- const parsedVirtual = filepath.startsWith('qmd://') ? parseVirtualPath(filepath) : null;
1660
- if (parsedVirtual) {
1661
- collectionName = parsedVirtual.collectionName;
1662
- relativePath = parsedVirtual.path;
1663
- } else {
1664
- // Filesystem path: find which collection this absolute path belongs to
1665
- for (const coll of collections) {
1666
- // Skip collections with missing paths
1667
- if (!coll || !coll.path) continue;
1668
-
1669
- if (filepath.startsWith(coll.path + '/') || filepath === coll.path) {
1670
- collectionName = coll.name;
1671
- // Extract relative path
1672
- relativePath = filepath.startsWith(coll.path + '/')
1673
- ? filepath.slice(coll.path.length + 1)
1674
- : '';
1675
- break;
1676
- }
1677
- }
1678
-
1679
- if (!collectionName || relativePath === null) return null;
1680
- }
1681
-
1682
- // Get the collection from config
1683
- const coll = getCollection(collectionName);
1684
- if (!coll) return null;
1685
-
1686
- // Verify this document exists in the database
1687
- const doc = db.prepare(`
1688
- SELECT d.path
1689
- FROM documents d
1690
- WHERE d.collection = ? AND d.path = ? AND d.active = 1
1691
- LIMIT 1
1692
- `).get(collectionName, relativePath) as { path: string } | null;
1693
-
1694
- if (!doc) return null;
1695
-
1696
- // Collect ALL matching contexts (global + all path prefixes)
1697
- const contexts: string[] = [];
1698
-
1699
- // Add global context if present
1700
- if (config.global_context) {
1701
- contexts.push(config.global_context);
1702
- }
1703
-
1704
- // Add all matching path contexts (from most general to most specific)
1705
- if (coll.context) {
1706
- const normalizedPath = relativePath.startsWith("/") ? relativePath : `/${relativePath}`;
1707
-
1708
- // Collect all matching prefixes
1709
- const matchingContexts: { prefix: string; context: string }[] = [];
1710
- for (const [prefix, context] of Object.entries(coll.context)) {
1711
- const normalizedPrefix = prefix.startsWith("/") ? prefix : `/${prefix}`;
1712
- if (normalizedPath.startsWith(normalizedPrefix)) {
1713
- matchingContexts.push({ prefix: normalizedPrefix, context });
1714
- }
1715
- }
1716
-
1717
- // Sort by prefix length (shortest/most general first)
1718
- matchingContexts.sort((a, b) => a.prefix.length - b.prefix.length);
1719
-
1720
- // Add all matching contexts
1721
- for (const match of matchingContexts) {
1722
- contexts.push(match.context);
1723
- }
1724
- }
1725
-
1726
- // Join all contexts with double newline
1727
- return contexts.length > 0 ? contexts.join('\n\n') : null;
1728
- }
1729
-
1730
- /**
1731
- * Get collection by name from YAML config.
1732
- * Returns collection metadata from ~/.config/qmd/index.yml
1733
- */
1734
- export function getCollectionByName(db: Database, name: string): { name: string; pwd: string; glob_pattern: string } | null {
1735
- const collection = getCollection(name);
1736
- if (!collection) return null;
1737
-
1738
- return {
1739
- name: collection.name,
1740
- pwd: collection.path,
1741
- glob_pattern: collection.pattern,
1742
- };
1743
- }
1744
-
1745
- /**
1746
- * List all collections with document counts from database.
1747
- * Merges YAML config with database statistics.
1748
- */
1749
- export function listCollections(db: Database): { name: string; pwd: string; glob_pattern: string; doc_count: number; active_count: number; last_modified: string | null }[] {
1750
- const collections = collectionsListCollections();
1751
-
1752
- // Get document counts from database for each collection
1753
- const result = collections.map(coll => {
1754
- const stats = db.prepare(`
1755
- SELECT
1756
- COUNT(d.id) as doc_count,
1757
- SUM(CASE WHEN d.active = 1 THEN 1 ELSE 0 END) as active_count,
1758
- MAX(d.modified_at) as last_modified
1759
- FROM documents d
1760
- WHERE d.collection = ?
1761
- `).get(coll.name) as { doc_count: number; active_count: number; last_modified: string | null } | null;
1762
-
1763
- return {
1764
- name: coll.name,
1765
- pwd: coll.path,
1766
- glob_pattern: coll.pattern,
1767
- doc_count: stats?.doc_count || 0,
1768
- active_count: stats?.active_count || 0,
1769
- last_modified: stats?.last_modified || null,
1770
- };
1771
- });
1772
-
1773
- return result;
1774
- }
1775
-
1776
- /**
1777
- * Remove a collection and clean up its documents.
1778
- * Uses collections.ts to remove from YAML config and cleans up database.
1779
- */
1780
- export function removeCollection(db: Database, collectionName: string): { deletedDocs: number; cleanedHashes: number } {
1781
- // Delete documents from database
1782
- const docResult = db.prepare(`DELETE FROM documents WHERE collection = ?`).run(collectionName);
1783
-
1784
- // Clean up orphaned content hashes
1785
- const cleanupResult = db.prepare(`
1786
- DELETE FROM content
1787
- WHERE hash NOT IN (SELECT DISTINCT hash FROM documents WHERE active = 1)
1788
- `).run();
1789
-
1790
- // Remove from YAML config (returns true if found and removed)
1791
- collectionsRemoveCollection(collectionName);
1792
-
1793
- return {
1794
- deletedDocs: docResult.changes,
1795
- cleanedHashes: cleanupResult.changes
1796
- };
1797
- }
1798
-
1799
- /**
1800
- * Rename a collection.
1801
- * Updates both YAML config and database documents table.
1802
- */
1803
- export function renameCollection(db: Database, oldName: string, newName: string): void {
1804
- // Update all documents with the new collection name in database
1805
- db.prepare(`UPDATE documents SET collection = ? WHERE collection = ?`)
1806
- .run(newName, oldName);
1807
-
1808
- // Rename in YAML config
1809
- collectionsRenameCollection(oldName, newName);
1810
- }
1811
-
1812
- // =============================================================================
1813
- // Context Management Operations
1814
- // =============================================================================
1815
-
1816
- /**
1817
- * Insert or update a context for a specific collection and path prefix.
1818
- */
1819
- export function insertContext(db: Database, collectionId: number, pathPrefix: string, context: string): void {
1820
- // Get collection name from ID
1821
- const coll = db.prepare(`SELECT name FROM collections WHERE id = ?`).get(collectionId) as { name: string } | null;
1822
- if (!coll) {
1823
- throw new Error(`Collection with id ${collectionId} not found`);
1824
- }
1825
-
1826
- // Use collections.ts to add context
1827
- collectionsAddContext(coll.name, pathPrefix, context);
1828
- }
1829
-
1830
- /**
1831
- * Delete a context for a specific collection and path prefix.
1832
- * Returns the number of contexts deleted.
1833
- */
1834
- export function deleteContext(db: Database, collectionName: string, pathPrefix: string): number {
1835
- // Use collections.ts to remove context
1836
- const success = collectionsRemoveContext(collectionName, pathPrefix);
1837
- return success ? 1 : 0;
1838
- }
1839
-
1840
- /**
1841
- * Delete all global contexts (contexts with empty path_prefix).
1842
- * Returns the number of contexts deleted.
1843
- */
1844
- export function deleteGlobalContexts(db: Database): number {
1845
- let deletedCount = 0;
1846
-
1847
- // Remove global context
1848
- setGlobalContext(undefined);
1849
- deletedCount++;
1850
-
1851
- // Remove root context (empty string) from all collections
1852
- const collections = collectionsListCollections();
1853
- for (const coll of collections) {
1854
- const success = collectionsRemoveContext(coll.name, '');
1855
- if (success) {
1856
- deletedCount++;
1857
- }
1858
- }
1859
-
1860
- return deletedCount;
1861
- }
1862
-
1863
- /**
1864
- * List all contexts, grouped by collection.
1865
- * Returns contexts ordered by collection name, then by path prefix length (longest first).
1866
- */
1867
- export function listPathContexts(db: Database): { collection_name: string; path_prefix: string; context: string }[] {
1868
- const allContexts = collectionsListAllContexts();
1869
-
1870
- // Convert to expected format and sort
1871
- return allContexts.map(ctx => ({
1872
- collection_name: ctx.collection,
1873
- path_prefix: ctx.path,
1874
- context: ctx.context,
1875
- })).sort((a, b) => {
1876
- // Sort by collection name first
1877
- if (a.collection_name !== b.collection_name) {
1878
- return a.collection_name.localeCompare(b.collection_name);
1879
- }
1880
- // Then by path prefix length (longest first)
1881
- if (a.path_prefix.length !== b.path_prefix.length) {
1882
- return b.path_prefix.length - a.path_prefix.length;
1883
- }
1884
- // Then alphabetically
1885
- return a.path_prefix.localeCompare(b.path_prefix);
1886
- });
1887
- }
1888
-
1889
- /**
1890
- * Get all collections (name only - from YAML config).
1891
- */
1892
- export function getAllCollections(db: Database): { name: string }[] {
1893
- const collections = collectionsListCollections();
1894
- return collections.map(c => ({ name: c.name }));
1895
- }
1896
-
1897
- /**
1898
- * Check which collections don't have any context defined.
1899
- * Returns collections that have no context entries at all (not even root context).
1900
- */
1901
- export function getCollectionsWithoutContext(db: Database): { name: string; pwd: string; doc_count: number }[] {
1902
- // Get all collections from YAML config
1903
- const yamlCollections = collectionsListCollections();
1904
-
1905
- // Filter to those without context
1906
- const collectionsWithoutContext: { name: string; pwd: string; doc_count: number }[] = [];
1907
-
1908
- for (const coll of yamlCollections) {
1909
- // Check if collection has any context
1910
- if (!coll.context || Object.keys(coll.context).length === 0) {
1911
- // Get doc count from database
1912
- const stats = db.prepare(`
1913
- SELECT COUNT(d.id) as doc_count
1914
- FROM documents d
1915
- WHERE d.collection = ? AND d.active = 1
1916
- `).get(coll.name) as { doc_count: number } | null;
1917
-
1918
- collectionsWithoutContext.push({
1919
- name: coll.name,
1920
- pwd: coll.path,
1921
- doc_count: stats?.doc_count || 0,
1922
- });
1923
- }
1924
- }
1925
-
1926
- return collectionsWithoutContext.sort((a, b) => a.name.localeCompare(b.name));
1927
- }
1928
-
1929
- /**
1930
- * Get top-level directories in a collection that don't have context.
1931
- * Useful for suggesting where context might be needed.
1932
- */
1933
- export function getTopLevelPathsWithoutContext(db: Database, collectionName: string): string[] {
1934
- // Get all paths in the collection from database
1935
- const paths = db.prepare(`
1936
- SELECT DISTINCT path FROM documents
1937
- WHERE collection = ? AND active = 1
1938
- `).all(collectionName) as { path: string }[];
1939
-
1940
- // Get existing contexts for this collection from YAML
1941
- const yamlColl = getCollection(collectionName);
1942
- if (!yamlColl) return [];
1943
-
1944
- const contextPrefixes = new Set<string>();
1945
- if (yamlColl.context) {
1946
- for (const prefix of Object.keys(yamlColl.context)) {
1947
- contextPrefixes.add(prefix);
1948
- }
1949
- }
1950
-
1951
- // Extract top-level directories (first path component)
1952
- const topLevelDirs = new Set<string>();
1953
- for (const { path } of paths) {
1954
- const parts = path.split('/').filter(Boolean);
1955
- if (parts.length > 1) {
1956
- const dir = parts[0];
1957
- if (dir) topLevelDirs.add(dir);
1958
- }
1959
- }
1960
-
1961
- // Filter out directories that already have context (exact or parent)
1962
- const missing: string[] = [];
1963
- for (const dir of topLevelDirs) {
1964
- let hasContext = false;
1965
-
1966
- // Check if this dir or any parent has context
1967
- for (const prefix of contextPrefixes) {
1968
- if (prefix === '' || prefix === dir || dir.startsWith(prefix + '/')) {
1969
- hasContext = true;
1970
- break;
1971
- }
1972
- }
1973
-
1974
- if (!hasContext) {
1975
- missing.push(dir);
1976
- }
1977
- }
1978
-
1979
- return missing.sort();
1980
- }
1981
-
1982
- // =============================================================================
1983
- // FTS Search
1984
- // =============================================================================
1985
-
1986
- function sanitizeFTS5Term(term: string): string {
1987
- return term.replace(/[^\p{L}\p{N}']/gu, '').toLowerCase();
1988
- }
1989
-
1990
- function buildFTS5Query(query: string): string | null {
1991
- const terms = query.split(/\s+/)
1992
- .map(t => sanitizeFTS5Term(t))
1993
- .filter(t => t.length > 0);
1994
- if (terms.length === 0) return null;
1995
- if (terms.length === 1) return `"${terms[0]}"*`;
1996
- return terms.map(t => `"${t}"*`).join(' AND ');
1997
- }
1998
-
1999
- export function searchFTS(db: Database, query: string, limit: number = 20, collectionName?: string): SearchResult[] {
2000
- const ftsQuery = buildFTS5Query(query);
2001
- if (!ftsQuery) return [];
2002
-
2003
- let sql = `
2004
- SELECT
2005
- 'qmd://' || d.collection || '/' || d.path as filepath,
2006
- d.collection || '/' || d.path as display_path,
2007
- d.title,
2008
- content.doc as body,
2009
- d.hash,
2010
- bm25(documents_fts, 10.0, 1.0) as bm25_score
2011
- FROM documents_fts f
2012
- JOIN documents d ON d.id = f.rowid
2013
- JOIN content ON content.hash = d.hash
2014
- WHERE documents_fts MATCH ? AND d.active = 1
2015
- `;
2016
- const params: (string | number)[] = [ftsQuery];
2017
-
2018
- if (collectionName) {
2019
- sql += ` AND d.collection = ?`;
2020
- params.push(String(collectionName));
2021
- }
2022
-
2023
- // bm25 lower is better; sort ascending.
2024
- sql += ` ORDER BY bm25_score ASC LIMIT ?`;
2025
- params.push(limit);
2026
-
2027
- const rows = db.prepare(sql).all(...params) as { filepath: string; display_path: string; title: string; body: string; hash: string; bm25_score: number }[];
2028
- return rows.map(row => {
2029
- const collectionName = row.filepath.split('//')[1]?.split('/')[0] || "";
2030
- // Convert bm25 (negative, lower is better) into a stable [0..1) score where higher is better.
2031
- // FTS5 BM25 scores are negative (e.g., -10 is strong, -2 is weak).
2032
- // |x| / (1 + |x|) maps: strong(-10)→0.91, medium(-2)→0.67, weak(-0.5)→0.33, none(0)→0.
2033
- // Monotonic and query-independent — no per-query normalization needed.
2034
- const score = Math.abs(row.bm25_score) / (1 + Math.abs(row.bm25_score));
2035
- return {
2036
- filepath: row.filepath,
2037
- displayPath: row.display_path,
2038
- title: row.title,
2039
- hash: row.hash,
2040
- docid: getDocid(row.hash),
2041
- collectionName,
2042
- modifiedAt: "", // Not available in FTS query
2043
- bodyLength: row.body.length,
2044
- body: row.body,
2045
- context: getContextForFile(db, row.filepath),
2046
- score,
2047
- source: "fts" as const,
2048
- };
2049
- });
2050
- }
2051
-
2052
- // =============================================================================
2053
- // Vector Search
2054
- // =============================================================================
2055
-
2056
- export async function searchVec(db: Database, query: string, model: string, limit: number = 20, collectionName?: string, session?: ILLMSession, precomputedEmbedding?: number[]): Promise<SearchResult[]> {
2057
- const tableExists = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get();
2058
- if (!tableExists) return [];
2059
-
2060
- const embedding = precomputedEmbedding ?? await getEmbedding(query, model, true, session);
2061
- if (!embedding) return [];
2062
-
2063
- // IMPORTANT: We use a two-step query approach here because sqlite-vec virtual tables
2064
- // hang indefinitely when combined with JOINs in the same query. Do NOT try to
2065
- // "optimize" this by combining into a single query with JOINs - it will break.
2066
- // See: https://github.com/tobi/qmd/pull/23
2067
-
2068
- // Step 1: Get vector matches from sqlite-vec (no JOINs allowed)
2069
- const vecResults = db.prepare(`
2070
- SELECT hash_seq, distance
2071
- FROM vectors_vec
2072
- WHERE embedding MATCH ? AND k = ?
2073
- `).all(new Float32Array(embedding), limit * 3) as { hash_seq: string; distance: number }[];
2074
-
2075
- if (vecResults.length === 0) return [];
2076
-
2077
- // Step 2: Get chunk info and document data
2078
- const hashSeqs = vecResults.map(r => r.hash_seq);
2079
- const distanceMap = new Map(vecResults.map(r => [r.hash_seq, r.distance]));
2080
-
2081
- // Build query for document lookup
2082
- const placeholders = hashSeqs.map(() => '?').join(',');
2083
- let docSql = `
2084
- SELECT
2085
- cv.hash || '_' || cv.seq as hash_seq,
2086
- cv.hash,
2087
- cv.pos,
2088
- 'qmd://' || d.collection || '/' || d.path as filepath,
2089
- d.collection || '/' || d.path as display_path,
2090
- d.title,
2091
- content.doc as body
2092
- FROM content_vectors cv
2093
- JOIN documents d ON d.hash = cv.hash AND d.active = 1
2094
- JOIN content ON content.hash = d.hash
2095
- WHERE cv.hash || '_' || cv.seq IN (${placeholders})
2096
- `;
2097
- const params: string[] = [...hashSeqs];
2098
-
2099
- if (collectionName) {
2100
- docSql += ` AND d.collection = ?`;
2101
- params.push(collectionName);
2102
- }
2103
-
2104
- const docRows = db.prepare(docSql).all(...params) as {
2105
- hash_seq: string; hash: string; pos: number; filepath: string;
2106
- display_path: string; title: string; body: string;
2107
- }[];
2108
-
2109
- // Combine with distances and dedupe by filepath
2110
- const seen = new Map<string, { row: typeof docRows[0]; bestDist: number }>();
2111
- for (const row of docRows) {
2112
- const distance = distanceMap.get(row.hash_seq) ?? 1;
2113
- const existing = seen.get(row.filepath);
2114
- if (!existing || distance < existing.bestDist) {
2115
- seen.set(row.filepath, { row, bestDist: distance });
2116
- }
2117
- }
2118
-
2119
- return Array.from(seen.values())
2120
- .sort((a, b) => a.bestDist - b.bestDist)
2121
- .slice(0, limit)
2122
- .map(({ row, bestDist }) => {
2123
- const collectionName = row.filepath.split('//')[1]?.split('/')[0] || "";
2124
- return {
2125
- filepath: row.filepath,
2126
- displayPath: row.display_path,
2127
- title: row.title,
2128
- hash: row.hash,
2129
- docid: getDocid(row.hash),
2130
- collectionName,
2131
- modifiedAt: "", // Not available in vec query
2132
- bodyLength: row.body.length,
2133
- body: row.body,
2134
- context: getContextForFile(db, row.filepath),
2135
- score: 1 - bestDist, // Cosine similarity = 1 - cosine distance
2136
- source: "vec" as const,
2137
- chunkPos: row.pos,
2138
- };
2139
- });
2140
- }
2141
-
2142
- // =============================================================================
2143
- // Embeddings
2144
- // =============================================================================
2145
-
2146
- async function getEmbedding(text: string, model: string, isQuery: boolean, session?: ILLMSession): Promise<number[] | null> {
2147
- // Format text using the appropriate prompt template
2148
- const formattedText = isQuery ? formatQueryForEmbedding(text) : formatDocForEmbedding(text);
2149
- const result = session
2150
- ? await session.embed(formattedText, { model, isQuery })
2151
- : await getDefaultLlamaCpp().embed(formattedText, { model, isQuery });
2152
- return result?.embedding || null;
2153
- }
2154
-
2155
- /**
2156
- * Get all unique content hashes that need embeddings (from active documents).
2157
- * Returns hash, document body, and a sample path for display purposes.
2158
- */
2159
- export function getHashesForEmbedding(db: Database): { hash: string; body: string; path: string }[] {
2160
- return db.prepare(`
2161
- SELECT d.hash, c.doc as body, MIN(d.path) as path
2162
- FROM documents d
2163
- JOIN content c ON d.hash = c.hash
2164
- LEFT JOIN content_vectors v ON d.hash = v.hash AND v.seq = 0
2165
- WHERE d.active = 1 AND v.hash IS NULL
2166
- GROUP BY d.hash
2167
- `).all() as { hash: string; body: string; path: string }[];
2168
- }
2169
-
2170
- /**
2171
- * Clear all embeddings from the database (force re-index).
2172
- * Deletes all rows from content_vectors and drops the vectors_vec table.
2173
- */
2174
- export function clearAllEmbeddings(db: Database): void {
2175
- db.exec(`DELETE FROM content_vectors`);
2176
- db.exec(`DROP TABLE IF EXISTS vectors_vec`);
2177
- }
2178
-
2179
- /**
2180
- * Insert a single embedding into both content_vectors and vectors_vec tables.
2181
- * The hash_seq key is formatted as "hash_seq" for the vectors_vec table.
2182
- */
2183
- export function insertEmbedding(
2184
- db: Database,
2185
- hash: string,
2186
- seq: number,
2187
- pos: number,
2188
- embedding: Float32Array,
2189
- model: string,
2190
- embeddedAt: string
2191
- ): void {
2192
- const hashSeq = `${hash}_${seq}`;
2193
- const insertVecStmt = db.prepare(`INSERT OR REPLACE INTO vectors_vec (hash_seq, embedding) VALUES (?, ?)`);
2194
- const insertContentVectorStmt = db.prepare(`INSERT OR REPLACE INTO content_vectors (hash, seq, pos, model, embedded_at) VALUES (?, ?, ?, ?, ?)`);
2195
-
2196
- insertVecStmt.run(hashSeq, embedding);
2197
- insertContentVectorStmt.run(hash, seq, pos, model, embeddedAt);
2198
- }
2199
-
2200
- // =============================================================================
2201
- // Query expansion
2202
- // =============================================================================
2203
-
2204
- export async function expandQuery(query: string, model: string = DEFAULT_QUERY_MODEL, db: Database): Promise<ExpandedQuery[]> {
2205
- // Check cache first — stored as JSON preserving types
2206
- const cacheKey = getCacheKey("expandQuery", { query, model });
2207
- const cached = getCachedResult(db, cacheKey);
2208
- if (cached) {
2209
- try {
2210
- return JSON.parse(cached) as ExpandedQuery[];
2211
- } catch {
2212
- // Old cache format (pre-typed, newline-separated text) — re-expand
2213
- }
2214
- }
2215
-
2216
- const llm = getDefaultLlamaCpp();
2217
- // Note: LlamaCpp uses hardcoded model, model parameter is ignored
2218
- const results = await llm.expandQuery(query);
2219
-
2220
- // Map Queryable[] → ExpandedQuery[] (same shape, decoupled from llm.ts internals).
2221
- // Filter out entries that duplicate the original query text.
2222
- const expanded: ExpandedQuery[] = results
2223
- .filter(r => r.text !== query)
2224
- .map(r => ({ type: r.type, text: r.text }));
2225
-
2226
- if (expanded.length > 0) {
2227
- setCachedResult(db, cacheKey, JSON.stringify(expanded));
2228
- }
2229
-
2230
- return expanded;
2231
- }
2232
-
2233
- // =============================================================================
2234
- // Reranking
2235
- // =============================================================================
2236
-
2237
- export async function rerank(query: string, documents: { file: string; text: string }[], model: string = DEFAULT_RERANK_MODEL, db: Database): Promise<{ file: string; score: number }[]> {
2238
- const cachedResults: Map<string, number> = new Map();
2239
- const uncachedDocs: RerankDocument[] = [];
2240
-
2241
- // Check cache for each document
2242
- // Cache key includes chunk text — different queries can select different chunks
2243
- // from the same file, and the reranker score depends on which chunk was sent.
2244
- for (const doc of documents) {
2245
- const cacheKey = getCacheKey("rerank", { query, file: doc.file, model, chunk: doc.text });
2246
- const cached = getCachedResult(db, cacheKey);
2247
- if (cached !== null) {
2248
- cachedResults.set(doc.file, parseFloat(cached));
2249
- } else {
2250
- uncachedDocs.push({ file: doc.file, text: doc.text });
2251
- }
2252
- }
2253
-
2254
- // Rerank uncached documents using LlamaCpp
2255
- if (uncachedDocs.length > 0) {
2256
- const llm = getDefaultLlamaCpp();
2257
- const rerankResult = await llm.rerank(query, uncachedDocs, { model });
2258
-
2259
- // Cache results — use original doc.text for cache key (result.file lacks chunk text)
2260
- const textByFile = new Map(documents.map(d => [d.file, d.text]));
2261
- for (const result of rerankResult.results) {
2262
- const cacheKey = getCacheKey("rerank", { query, file: result.file, model, chunk: textByFile.get(result.file) || "" });
2263
- setCachedResult(db, cacheKey, result.score.toString());
2264
- cachedResults.set(result.file, result.score);
2265
- }
2266
- }
2267
-
2268
- // Return all results sorted by score
2269
- return documents
2270
- .map(doc => ({ file: doc.file, score: cachedResults.get(doc.file) || 0 }))
2271
- .sort((a, b) => b.score - a.score);
2272
- }
2273
-
2274
- // =============================================================================
2275
- // Reciprocal Rank Fusion
2276
- // =============================================================================
2277
-
2278
- export function reciprocalRankFusion(
2279
- resultLists: RankedResult[][],
2280
- weights: number[] = [],
2281
- k: number = 60
2282
- ): RankedResult[] {
2283
- const scores = new Map<string, { result: RankedResult; rrfScore: number; topRank: number }>();
2284
-
2285
- for (let listIdx = 0; listIdx < resultLists.length; listIdx++) {
2286
- const list = resultLists[listIdx];
2287
- if (!list) continue;
2288
- const weight = weights[listIdx] ?? 1.0;
2289
-
2290
- for (let rank = 0; rank < list.length; rank++) {
2291
- const result = list[rank];
2292
- if (!result) continue;
2293
- const rrfContribution = weight / (k + rank + 1);
2294
- const existing = scores.get(result.file);
2295
-
2296
- if (existing) {
2297
- existing.rrfScore += rrfContribution;
2298
- existing.topRank = Math.min(existing.topRank, rank);
2299
- } else {
2300
- scores.set(result.file, {
2301
- result,
2302
- rrfScore: rrfContribution,
2303
- topRank: rank,
2304
- });
2305
- }
2306
- }
2307
- }
2308
-
2309
- // Top-rank bonus
2310
- for (const entry of scores.values()) {
2311
- if (entry.topRank === 0) {
2312
- entry.rrfScore += 0.05;
2313
- } else if (entry.topRank <= 2) {
2314
- entry.rrfScore += 0.02;
2315
- }
2316
- }
2317
-
2318
- return Array.from(scores.values())
2319
- .sort((a, b) => b.rrfScore - a.rrfScore)
2320
- .map(e => ({ ...e.result, score: e.rrfScore }));
2321
- }
2322
-
2323
- // =============================================================================
2324
- // Document retrieval
2325
- // =============================================================================
2326
-
2327
- type DbDocRow = {
2328
- virtual_path: string;
2329
- display_path: string;
2330
- title: string;
2331
- hash: string;
2332
- collection: string;
2333
- path: string;
2334
- modified_at: string;
2335
- body_length: number;
2336
- body?: string;
2337
- };
2338
-
2339
- /**
2340
- * Find a document by filename/path, docid (#hash), or with fuzzy matching.
2341
- * Returns document metadata without body by default.
2342
- *
2343
- * Supports:
2344
- * - Virtual paths: qmd://collection/path/to/file.md
2345
- * - Absolute paths: /path/to/file.md
2346
- * - Relative paths: path/to/file.md
2347
- * - Short docid: #abc123 (first 6 chars of hash)
2348
- */
2349
- export function findDocument(db: Database, filename: string, options: { includeBody?: boolean } = {}): DocumentResult | DocumentNotFound {
2350
- let filepath = filename;
2351
- const colonMatch = filepath.match(/:(\d+)$/);
2352
- if (colonMatch) {
2353
- filepath = filepath.slice(0, -colonMatch[0].length);
2354
- }
2355
-
2356
- // Check if this is a docid lookup (#abc123, abc123, "#abc123", "abc123", etc.)
2357
- if (isDocid(filepath)) {
2358
- const docidMatch = findDocumentByDocid(db, filepath);
2359
- if (docidMatch) {
2360
- filepath = docidMatch.filepath;
2361
- } else {
2362
- return { error: "not_found", query: filename, similarFiles: [] };
2363
- }
2364
- }
2365
-
2366
- if (filepath.startsWith('~/')) {
2367
- filepath = homedir() + filepath.slice(1);
2368
- }
2369
-
2370
- const bodyCol = options.includeBody ? `, content.doc as body` : ``;
2371
-
2372
- // Build computed columns
2373
- // Note: absoluteFilepath is computed from YAML collections after query
2374
- const selectCols = `
2375
- 'qmd://' || d.collection || '/' || d.path as virtual_path,
2376
- d.collection || '/' || d.path as display_path,
2377
- d.title,
2378
- d.hash,
2379
- d.collection,
2380
- d.modified_at,
2381
- LENGTH(content.doc) as body_length
2382
- ${bodyCol}
2383
- `;
2384
-
2385
- // Try to match by virtual path first
2386
- let doc = db.prepare(`
2387
- SELECT ${selectCols}
2388
- FROM documents d
2389
- JOIN content ON content.hash = d.hash
2390
- WHERE 'qmd://' || d.collection || '/' || d.path = ? AND d.active = 1
2391
- `).get(filepath) as DbDocRow | null;
2392
-
2393
- // Try fuzzy match by virtual path
2394
- if (!doc) {
2395
- doc = db.prepare(`
2396
- SELECT ${selectCols}
2397
- FROM documents d
2398
- JOIN content ON content.hash = d.hash
2399
- WHERE 'qmd://' || d.collection || '/' || d.path LIKE ? AND d.active = 1
2400
- LIMIT 1
2401
- `).get(`%${filepath}`) as DbDocRow | null;
2402
- }
2403
-
2404
- // Try to match by absolute path (requires looking up collection paths from YAML)
2405
- if (!doc && !filepath.startsWith('qmd://')) {
2406
- const collections = collectionsListCollections();
2407
- for (const coll of collections) {
2408
- let relativePath: string | null = null;
2409
-
2410
- // If filepath is absolute and starts with collection path, extract relative part
2411
- if (filepath.startsWith(coll.path + '/')) {
2412
- relativePath = filepath.slice(coll.path.length + 1);
2413
- }
2414
- // Otherwise treat filepath as relative to collection
2415
- else if (!filepath.startsWith('/')) {
2416
- relativePath = filepath;
2417
- }
2418
-
2419
- if (relativePath) {
2420
- doc = db.prepare(`
2421
- SELECT ${selectCols}
2422
- FROM documents d
2423
- JOIN content ON content.hash = d.hash
2424
- WHERE d.collection = ? AND d.path = ? AND d.active = 1
2425
- `).get(coll.name, relativePath) as DbDocRow | null;
2426
- if (doc) break;
2427
- }
2428
- }
2429
- }
2430
-
2431
- if (!doc) {
2432
- const similar = findSimilarFiles(db, filepath, 5, 5);
2433
- return { error: "not_found", query: filename, similarFiles: similar };
2434
- }
2435
-
2436
- // Get context using virtual path
2437
- const virtualPath = doc.virtual_path || `qmd://${doc.collection}/${doc.display_path}`;
2438
- const context = getContextForFile(db, virtualPath);
2439
-
2440
- return {
2441
- filepath: virtualPath,
2442
- displayPath: doc.display_path,
2443
- title: doc.title,
2444
- context,
2445
- hash: doc.hash,
2446
- docid: getDocid(doc.hash),
2447
- collectionName: doc.collection,
2448
- modifiedAt: doc.modified_at,
2449
- bodyLength: doc.body_length,
2450
- ...(options.includeBody && doc.body !== undefined && { body: doc.body }),
2451
- };
2452
- }
2453
-
2454
- /**
2455
- * Get the body content for a document
2456
- * Optionally slice by line range
2457
- */
2458
- export function getDocumentBody(db: Database, doc: DocumentResult | { filepath: string }, fromLine?: number, maxLines?: number): string | null {
2459
- const filepath = doc.filepath;
2460
-
2461
- // Try to resolve document by filepath (absolute or virtual)
2462
- let row: { body: string } | null = null;
2463
-
2464
- // Try virtual path first
2465
- if (filepath.startsWith('qmd://')) {
2466
- row = db.prepare(`
2467
- SELECT content.doc as body
2468
- FROM documents d
2469
- JOIN content ON content.hash = d.hash
2470
- WHERE 'qmd://' || d.collection || '/' || d.path = ? AND d.active = 1
2471
- `).get(filepath) as { body: string } | null;
2472
- }
2473
-
2474
- // Try absolute path by looking up in YAML collections
2475
- if (!row) {
2476
- const collections = collectionsListCollections();
2477
- for (const coll of collections) {
2478
- if (filepath.startsWith(coll.path + '/')) {
2479
- const relativePath = filepath.slice(coll.path.length + 1);
2480
- row = db.prepare(`
2481
- SELECT content.doc as body
2482
- FROM documents d
2483
- JOIN content ON content.hash = d.hash
2484
- WHERE d.collection = ? AND d.path = ? AND d.active = 1
2485
- `).get(coll.name, relativePath) as { body: string } | null;
2486
- if (row) break;
2487
- }
2488
- }
2489
- }
2490
-
2491
- if (!row) return null;
2492
-
2493
- let body = row.body;
2494
- if (fromLine !== undefined || maxLines !== undefined) {
2495
- const lines = body.split('\n');
2496
- const start = (fromLine || 1) - 1;
2497
- const end = maxLines !== undefined ? start + maxLines : lines.length;
2498
- body = lines.slice(start, end).join('\n');
2499
- }
2500
-
2501
- return body;
2502
- }
2503
-
2504
- /**
2505
- * Find multiple documents by glob pattern or comma-separated list
2506
- * Returns documents without body by default (use getDocumentBody to load)
2507
- */
2508
- export function findDocuments(
2509
- db: Database,
2510
- pattern: string,
2511
- options: { includeBody?: boolean; maxBytes?: number } = {}
2512
- ): { docs: MultiGetResult[]; errors: string[] } {
2513
- const isCommaSeparated = pattern.includes(',') && !pattern.includes('*') && !pattern.includes('?');
2514
- const errors: string[] = [];
2515
- const maxBytes = options.maxBytes ?? DEFAULT_MULTI_GET_MAX_BYTES;
2516
-
2517
- const bodyCol = options.includeBody ? `, content.doc as body` : ``;
2518
- const selectCols = `
2519
- 'qmd://' || d.collection || '/' || d.path as virtual_path,
2520
- d.collection || '/' || d.path as display_path,
2521
- d.title,
2522
- d.hash,
2523
- d.collection,
2524
- d.modified_at,
2525
- LENGTH(content.doc) as body_length
2526
- ${bodyCol}
2527
- `;
2528
-
2529
- let fileRows: DbDocRow[];
2530
-
2531
- if (isCommaSeparated) {
2532
- const names = pattern.split(',').map(s => s.trim()).filter(Boolean);
2533
- fileRows = [];
2534
- for (const name of names) {
2535
- let doc = db.prepare(`
2536
- SELECT ${selectCols}
2537
- FROM documents d
2538
- JOIN content ON content.hash = d.hash
2539
- WHERE 'qmd://' || d.collection || '/' || d.path = ? AND d.active = 1
2540
- `).get(name) as DbDocRow | null;
2541
- if (!doc) {
2542
- doc = db.prepare(`
2543
- SELECT ${selectCols}
2544
- FROM documents d
2545
- JOIN content ON content.hash = d.hash
2546
- WHERE 'qmd://' || d.collection || '/' || d.path LIKE ? AND d.active = 1
2547
- LIMIT 1
2548
- `).get(`%${name}`) as DbDocRow | null;
2549
- }
2550
- if (doc) {
2551
- fileRows.push(doc);
2552
- } else {
2553
- const similar = findSimilarFiles(db, name, 5, 3);
2554
- let msg = `File not found: ${name}`;
2555
- if (similar.length > 0) {
2556
- msg += ` (did you mean: ${similar.join(', ')}?)`;
2557
- }
2558
- errors.push(msg);
2559
- }
2560
- }
2561
- } else {
2562
- // Glob pattern match
2563
- const matched = matchFilesByGlob(db, pattern);
2564
- if (matched.length === 0) {
2565
- errors.push(`No files matched pattern: ${pattern}`);
2566
- return { docs: [], errors };
2567
- }
2568
- const virtualPaths = matched.map(m => m.filepath);
2569
- const placeholders = virtualPaths.map(() => '?').join(',');
2570
- fileRows = db.prepare(`
2571
- SELECT ${selectCols}
2572
- FROM documents d
2573
- JOIN content ON content.hash = d.hash
2574
- WHERE 'qmd://' || d.collection || '/' || d.path IN (${placeholders}) AND d.active = 1
2575
- `).all(...virtualPaths) as DbDocRow[];
2576
- }
2577
-
2578
- const results: MultiGetResult[] = [];
2579
-
2580
- for (const row of fileRows) {
2581
- // Get context using virtual path
2582
- const virtualPath = row.virtual_path || `qmd://${row.collection}/${row.display_path}`;
2583
- const context = getContextForFile(db, virtualPath);
2584
-
2585
- if (row.body_length > maxBytes) {
2586
- results.push({
2587
- doc: { filepath: virtualPath, displayPath: row.display_path },
2588
- skipped: true,
2589
- skipReason: `File too large (${Math.round(row.body_length / 1024)}KB > ${Math.round(maxBytes / 1024)}KB)`,
2590
- });
2591
- continue;
2592
- }
2593
-
2594
- results.push({
2595
- doc: {
2596
- filepath: virtualPath,
2597
- displayPath: row.display_path,
2598
- title: row.title || row.display_path.split('/').pop() || row.display_path,
2599
- context,
2600
- hash: row.hash,
2601
- docid: getDocid(row.hash),
2602
- collectionName: row.collection,
2603
- modifiedAt: row.modified_at,
2604
- bodyLength: row.body_length,
2605
- ...(options.includeBody && row.body !== undefined && { body: row.body }),
2606
- },
2607
- skipped: false,
2608
- });
2609
- }
2610
-
2611
- return { docs: results, errors };
2612
- }
2613
-
2614
- // =============================================================================
2615
- // Status
2616
- // =============================================================================
2617
-
2618
- export function getStatus(db: Database): IndexStatus {
2619
- // Load collections from YAML
2620
- const yamlCollections = collectionsListCollections();
2621
-
2622
- // Get document counts and last update times for each collection
2623
- const collections = yamlCollections.map(col => {
2624
- const stats = db.prepare(`
2625
- SELECT
2626
- COUNT(*) as active_count,
2627
- MAX(modified_at) as last_doc_update
2628
- FROM documents
2629
- WHERE collection = ? AND active = 1
2630
- `).get(col.name) as { active_count: number; last_doc_update: string | null };
2631
-
2632
- return {
2633
- name: col.name,
2634
- path: col.path,
2635
- pattern: col.pattern,
2636
- documents: stats.active_count,
2637
- lastUpdated: stats.last_doc_update || new Date().toISOString(),
2638
- };
2639
- });
2640
-
2641
- // Sort by last update time (most recent first)
2642
- collections.sort((a, b) => {
2643
- if (!a.lastUpdated) return 1;
2644
- if (!b.lastUpdated) return -1;
2645
- return new Date(b.lastUpdated).getTime() - new Date(a.lastUpdated).getTime();
2646
- });
2647
-
2648
- const totalDocs = (db.prepare(`SELECT COUNT(*) as c FROM documents WHERE active = 1`).get() as { c: number }).c;
2649
- const needsEmbedding = getHashesNeedingEmbedding(db);
2650
- const hasVectors = !!db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get();
2651
-
2652
- return {
2653
- totalDocuments: totalDocs,
2654
- needsEmbedding,
2655
- hasVectorIndex: hasVectors,
2656
- collections,
2657
- };
2658
- }
2659
-
2660
- // =============================================================================
2661
- // Snippet extraction
2662
- // =============================================================================
2663
-
2664
- export type SnippetResult = {
2665
- line: number; // 1-indexed line number of best match
2666
- snippet: string; // The snippet text with diff-style header
2667
- linesBefore: number; // Lines in document before snippet
2668
- linesAfter: number; // Lines in document after snippet
2669
- snippetLines: number; // Number of lines in snippet
2670
- };
2671
-
2672
- export function extractSnippet(body: string, query: string, maxLen = 500, chunkPos?: number, chunkLen?: number): SnippetResult {
2673
- const totalLines = body.split('\n').length;
2674
- let searchBody = body;
2675
- let lineOffset = 0;
2676
-
2677
- if (chunkPos && chunkPos > 0) {
2678
- // Search within the chunk region, with some padding for context
2679
- // Use provided chunkLen or fall back to max chunk size (covers variable-length chunks)
2680
- const searchLen = chunkLen || CHUNK_SIZE_CHARS;
2681
- const contextStart = Math.max(0, chunkPos - 100);
2682
- const contextEnd = Math.min(body.length, chunkPos + searchLen + 100);
2683
- searchBody = body.slice(contextStart, contextEnd);
2684
- if (contextStart > 0) {
2685
- lineOffset = body.slice(0, contextStart).split('\n').length - 1;
2686
- }
2687
- }
2688
-
2689
- const lines = searchBody.split('\n');
2690
- const queryTerms = query.toLowerCase().split(/\s+/).filter(t => t.length > 0);
2691
- let bestLine = 0, bestScore = -1;
2692
-
2693
- for (let i = 0; i < lines.length; i++) {
2694
- const lineLower = (lines[i] ?? "").toLowerCase();
2695
- let score = 0;
2696
- for (const term of queryTerms) {
2697
- if (lineLower.includes(term)) score++;
2698
- }
2699
- if (score > bestScore) {
2700
- bestScore = score;
2701
- bestLine = i;
2702
- }
2703
- }
2704
-
2705
- const start = Math.max(0, bestLine - 1);
2706
- const end = Math.min(lines.length, bestLine + 3);
2707
- const snippetLines = lines.slice(start, end);
2708
- let snippetText = snippetLines.join('\n');
2709
-
2710
- // If we focused on a chunk window and it produced an empty/whitespace-only snippet,
2711
- // fall back to a full-document snippet so we always show something useful.
2712
- if (chunkPos && chunkPos > 0 && snippetText.trim().length === 0) {
2713
- return extractSnippet(body, query, maxLen, undefined);
2714
- }
2715
-
2716
- if (snippetText.length > maxLen) snippetText = snippetText.substring(0, maxLen - 3) + "...";
2717
-
2718
- const absoluteStart = lineOffset + start + 1; // 1-indexed
2719
- const snippetLineCount = snippetLines.length;
2720
- const linesBefore = absoluteStart - 1;
2721
- const linesAfter = totalLines - (absoluteStart + snippetLineCount - 1);
2722
-
2723
- // Format with diff-style header: @@ -start,count @@ (linesBefore before, linesAfter after)
2724
- const header = `@@ -${absoluteStart},${snippetLineCount} @@ (${linesBefore} before, ${linesAfter} after)`;
2725
- const snippet = `${header}\n${snippetText}`;
2726
-
2727
- return {
2728
- line: lineOffset + bestLine + 1,
2729
- snippet,
2730
- linesBefore,
2731
- linesAfter,
2732
- snippetLines: snippetLineCount,
2733
- };
2734
- }
2735
-
2736
- // =============================================================================
2737
- // Shared helpers (used by both CLI and MCP)
2738
- // =============================================================================
2739
-
2740
- /**
2741
- * Add line numbers to text content.
2742
- * Each line becomes: "{lineNum}: {content}"
2743
- */
2744
- export function addLineNumbers(text: string, startLine: number = 1): string {
2745
- const lines = text.split('\n');
2746
- return lines.map((line, i) => `${startLine + i}: ${line}`).join('\n');
2747
- }
2748
-
2749
- // =============================================================================
2750
- // Shared search orchestration
2751
- //
2752
- // hybridQuery() and vectorSearchQuery() are standalone functions (not Store
2753
- // methods) because they are orchestration over primitives — same rationale as
2754
- // reciprocalRankFusion(). They take a Store as first argument so both CLI
2755
- // and MCP can share the identical pipeline.
2756
- // =============================================================================
2757
-
2758
- /**
2759
- * Optional progress hooks for search orchestration.
2760
- * CLI wires these to stderr for user feedback; MCP leaves them unset.
2761
- */
2762
- export interface SearchHooks {
2763
- /** BM25 probe found strong signal — expansion will be skipped */
2764
- onStrongSignal?: (topScore: number) => void;
2765
- /** Query expansion complete. Empty array = strong signal skip (no expansion). */
2766
- onExpand?: (original: string, expanded: ExpandedQuery[]) => void;
2767
- /** Reranking is about to start */
2768
- onRerankStart?: (chunkCount: number) => void;
2769
- /** Reranking finished */
2770
- onRerankDone?: () => void;
2771
- }
2772
-
2773
- export interface HybridQueryOptions {
2774
- collection?: string;
2775
- limit?: number; // default 10
2776
- minScore?: number; // default 0
2777
- candidateLimit?: number; // default RERANK_CANDIDATE_LIMIT
2778
- hooks?: SearchHooks;
2779
- }
2780
-
2781
- export interface HybridQueryResult {
2782
- file: string; // internal filepath (qmd://collection/path)
2783
- displayPath: string;
2784
- title: string;
2785
- body: string; // full document body (for snippet extraction)
2786
- bestChunk: string; // best chunk text
2787
- bestChunkPos: number; // char offset of best chunk in body
2788
- score: number; // blended score (full precision)
2789
- context: string | null; // user-set context
2790
- docid: string; // content hash prefix (6 chars)
2791
- }
2792
-
2793
- /**
2794
- * Hybrid search: BM25 + vector + query expansion + RRF + chunked reranking.
2795
- *
2796
- * Pipeline:
2797
- * 1. BM25 probe → skip expansion if strong signal
2798
- * 2. expandQuery() → typed query variants (lex/vec/hyde)
2799
- * 3. Type-routed search: original→vector, lex→FTS, vec/hyde→vector
2800
- * 4. RRF fusion → slice to candidateLimit
2801
- * 5. chunkDocument() + keyword-best-chunk selection
2802
- * 6. rerank on chunks (NOT full bodies — O(tokens) trap)
2803
- * 7. Position-aware score blending (RRF rank × reranker score)
2804
- * 8. Dedup by file, filter by minScore, slice to limit
2805
- */
2806
- export async function hybridQuery(
2807
- store: Store,
2808
- query: string,
2809
- options?: HybridQueryOptions
2810
- ): Promise<HybridQueryResult[]> {
2811
- const limit = options?.limit ?? 10;
2812
- const minScore = options?.minScore ?? 0;
2813
- const candidateLimit = options?.candidateLimit ?? RERANK_CANDIDATE_LIMIT;
2814
- const collection = options?.collection;
2815
- const hooks = options?.hooks;
2816
-
2817
- const rankedLists: RankedResult[][] = [];
2818
- const docidMap = new Map<string, string>(); // filepath -> docid
2819
- const hasVectors = !!store.db.prepare(
2820
- `SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'`
2821
- ).get();
2822
-
2823
- // Step 1: BM25 probe — strong signal skips expensive LLM expansion
2824
- // Pass collection directly into FTS query (filter at SQL level, not post-hoc)
2825
- const initialFts = store.searchFTS(query, 20, collection);
2826
- const topScore = initialFts[0]?.score ?? 0;
2827
- const secondScore = initialFts[1]?.score ?? 0;
2828
- const hasStrongSignal = initialFts.length > 0
2829
- && topScore >= STRONG_SIGNAL_MIN_SCORE
2830
- && (topScore - secondScore) >= STRONG_SIGNAL_MIN_GAP;
2831
-
2832
- if (hasStrongSignal) hooks?.onStrongSignal?.(topScore);
2833
-
2834
- // Step 2: Expand query (or skip if strong signal)
2835
- const expanded = hasStrongSignal
2836
- ? []
2837
- : await store.expandQuery(query);
2838
-
2839
- hooks?.onExpand?.(query, expanded);
2840
-
2841
- // Seed with initial FTS results (avoid re-running original query FTS)
2842
- if (initialFts.length > 0) {
2843
- for (const r of initialFts) docidMap.set(r.filepath, r.docid);
2844
- rankedLists.push(initialFts.map(r => ({
2845
- file: r.filepath, displayPath: r.displayPath,
2846
- title: r.title, body: r.body || "", score: r.score,
2847
- })));
2848
- }
2849
-
2850
- // Step 3: Route searches by query type
2851
- //
2852
- // Strategy: run all FTS queries immediately (they're sync/instant), then
2853
- // batch-embed all vector queries in one embedBatch() call, then run
2854
- // sqlite-vec lookups with pre-computed embeddings.
2855
-
2856
- // 3a: Run FTS for all lex expansions right away (no LLM needed)
2857
- for (const q of expanded) {
2858
- if (q.type === 'lex') {
2859
- const ftsResults = store.searchFTS(q.text, 20, collection);
2860
- if (ftsResults.length > 0) {
2861
- for (const r of ftsResults) docidMap.set(r.filepath, r.docid);
2862
- rankedLists.push(ftsResults.map(r => ({
2863
- file: r.filepath, displayPath: r.displayPath,
2864
- title: r.title, body: r.body || "", score: r.score,
2865
- })));
2866
- }
2867
- }
2868
- }
2869
-
2870
- // 3b: Collect all texts that need vector search (original query + vec/hyde expansions)
2871
- if (hasVectors) {
2872
- const vecQueries: { text: string; isOriginal: boolean }[] = [
2873
- { text: query, isOriginal: true },
2874
- ];
2875
- for (const q of expanded) {
2876
- if (q.type === 'vec' || q.type === 'hyde') {
2877
- vecQueries.push({ text: q.text, isOriginal: false });
2878
- }
2879
- }
2880
-
2881
- // Batch embed all vector queries in a single call
2882
- const llm = getDefaultLlamaCpp();
2883
- const textsToEmbed = vecQueries.map(q => formatQueryForEmbedding(q.text));
2884
- const embeddings = await llm.embedBatch(textsToEmbed);
2885
-
2886
- // Run sqlite-vec lookups with pre-computed embeddings
2887
- for (let i = 0; i < vecQueries.length; i++) {
2888
- const embedding = embeddings[i]?.embedding;
2889
- if (!embedding) continue;
2890
-
2891
- const vecResults = await store.searchVec(
2892
- vecQueries[i]!.text, DEFAULT_EMBED_MODEL, 20, collection,
2893
- undefined, embedding
2894
- );
2895
- if (vecResults.length > 0) {
2896
- for (const r of vecResults) docidMap.set(r.filepath, r.docid);
2897
- rankedLists.push(vecResults.map(r => ({
2898
- file: r.filepath, displayPath: r.displayPath,
2899
- title: r.title, body: r.body || "", score: r.score,
2900
- })));
2901
- }
2902
- }
2903
- }
2904
-
2905
- // Step 4: RRF fusion — first 2 lists (original FTS + first vec) get 2x weight
2906
- const weights = rankedLists.map((_, i) => i < 2 ? 2.0 : 1.0);
2907
- const fused = reciprocalRankFusion(rankedLists, weights);
2908
- const candidates = fused.slice(0, candidateLimit);
2909
-
2910
- if (candidates.length === 0) return [];
2911
-
2912
- // Step 5: Chunk documents, pick best chunk per doc for reranking.
2913
- // Reranking full bodies is O(tokens) — the critical perf lesson that motivated this refactor.
2914
- const queryTerms = query.toLowerCase().split(/\s+/).filter(t => t.length > 2);
2915
- const chunksToRerank: { file: string; text: string }[] = [];
2916
- const docChunkMap = new Map<string, { chunks: { text: string; pos: number }[]; bestIdx: number }>();
2917
-
2918
- for (const cand of candidates) {
2919
- const chunks = chunkDocument(cand.body);
2920
- if (chunks.length === 0) continue;
2921
-
2922
- // Pick chunk with most keyword overlap (fallback: first chunk)
2923
- let bestIdx = 0;
2924
- let bestScore = -1;
2925
- for (let i = 0; i < chunks.length; i++) {
2926
- const chunkLower = chunks[i]!.text.toLowerCase();
2927
- const score = queryTerms.reduce((acc, term) => acc + (chunkLower.includes(term) ? 1 : 0), 0);
2928
- if (score > bestScore) { bestScore = score; bestIdx = i; }
2929
- }
2930
-
2931
- chunksToRerank.push({ file: cand.file, text: chunks[bestIdx]!.text });
2932
- docChunkMap.set(cand.file, { chunks, bestIdx });
2933
- }
2934
-
2935
- // Step 6: Rerank chunks (NOT full bodies)
2936
- hooks?.onRerankStart?.(chunksToRerank.length);
2937
- const reranked = await store.rerank(query, chunksToRerank);
2938
- hooks?.onRerankDone?.();
2939
-
2940
- // Step 7: Blend RRF position score with reranker score
2941
- // Position-aware weights: top retrieval results get more protection from reranker disagreement
2942
- const candidateMap = new Map(candidates.map(c => [c.file, {
2943
- displayPath: c.displayPath, title: c.title, body: c.body,
2944
- }]));
2945
- const rrfRankMap = new Map(candidates.map((c, i) => [c.file, i + 1]));
2946
-
2947
- const blended = reranked.map(r => {
2948
- const rrfRank = rrfRankMap.get(r.file) || candidateLimit;
2949
- let rrfWeight: number;
2950
- if (rrfRank <= 3) rrfWeight = 0.75;
2951
- else if (rrfRank <= 10) rrfWeight = 0.60;
2952
- else rrfWeight = 0.40;
2953
- const rrfScore = 1 / rrfRank;
2954
- const blendedScore = rrfWeight * rrfScore + (1 - rrfWeight) * r.score;
2955
-
2956
- const candidate = candidateMap.get(r.file);
2957
- const chunkInfo = docChunkMap.get(r.file);
2958
- const bestIdx = chunkInfo?.bestIdx ?? 0;
2959
- const bestChunk = chunkInfo?.chunks[bestIdx]?.text || candidate?.body || "";
2960
- const bestChunkPos = chunkInfo?.chunks[bestIdx]?.pos || 0;
2961
-
2962
- return {
2963
- file: r.file,
2964
- displayPath: candidate?.displayPath || "",
2965
- title: candidate?.title || "",
2966
- body: candidate?.body || "",
2967
- bestChunk,
2968
- bestChunkPos,
2969
- score: blendedScore,
2970
- context: store.getContextForFile(r.file),
2971
- docid: docidMap.get(r.file) || "",
2972
- };
2973
- }).sort((a, b) => b.score - a.score);
2974
-
2975
- // Step 8: Dedup by file (safety net — prevents duplicate output)
2976
- const seenFiles = new Set<string>();
2977
- return blended
2978
- .filter(r => {
2979
- if (seenFiles.has(r.file)) return false;
2980
- seenFiles.add(r.file);
2981
- return true;
2982
- })
2983
- .filter(r => r.score >= minScore)
2984
- .slice(0, limit);
2985
- }
2986
-
2987
- export interface VectorSearchOptions {
2988
- collection?: string;
2989
- limit?: number; // default 10
2990
- minScore?: number; // default 0.3
2991
- hooks?: Pick<SearchHooks, 'onExpand'>;
2992
- }
2993
-
2994
- export interface VectorSearchResult {
2995
- file: string;
2996
- displayPath: string;
2997
- title: string;
2998
- body: string;
2999
- score: number;
3000
- context: string | null;
3001
- docid: string;
3002
- }
3003
-
3004
- /**
3005
- * Vector-only semantic search with query expansion.
3006
- *
3007
- * Pipeline:
3008
- * 1. expandQuery() → typed variants, filter to vec/hyde only (lex irrelevant here)
3009
- * 2. searchVec() for original + vec/hyde variants (sequential — node-llama-cpp embed limitation)
3010
- * 3. Dedup by filepath (keep max score)
3011
- * 4. Sort by score descending, filter by minScore, slice to limit
3012
- */
3013
- export async function vectorSearchQuery(
3014
- store: Store,
3015
- query: string,
3016
- options?: VectorSearchOptions
3017
- ): Promise<VectorSearchResult[]> {
3018
- const limit = options?.limit ?? 10;
3019
- const minScore = options?.minScore ?? 0.3;
3020
- const collection = options?.collection;
3021
-
3022
- const hasVectors = !!store.db.prepare(
3023
- `SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'`
3024
- ).get();
3025
- if (!hasVectors) return [];
3026
-
3027
- // Expand query — filter to vec/hyde only (lex queries target FTS, not vector)
3028
- const allExpanded = await store.expandQuery(query);
3029
- const vecExpanded = allExpanded.filter(q => q.type !== 'lex');
3030
- options?.hooks?.onExpand?.(query, vecExpanded);
3031
-
3032
- // Run original + vec/hyde expanded through vector, sequentially — concurrent embed() hangs
3033
- const queryTexts = [query, ...vecExpanded.map(q => q.text)];
3034
- const allResults = new Map<string, VectorSearchResult>();
3035
- for (const q of queryTexts) {
3036
- const vecResults = await store.searchVec(q, DEFAULT_EMBED_MODEL, limit, collection);
3037
- for (const r of vecResults) {
3038
- const existing = allResults.get(r.filepath);
3039
- if (!existing || r.score > existing.score) {
3040
- allResults.set(r.filepath, {
3041
- file: r.filepath,
3042
- displayPath: r.displayPath,
3043
- title: r.title,
3044
- body: r.body || "",
3045
- score: r.score,
3046
- context: store.getContextForFile(r.filepath),
3047
- docid: r.docid,
3048
- });
3049
- }
3050
- }
3051
- }
3052
-
3053
- return Array.from(allResults.values())
3054
- .sort((a, b) => b.score - a.score)
3055
- .filter(r => r.score >= minScore)
3056
- .slice(0, limit);
3057
- }