dsh-locator 0.8.2 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +47 -0
  2. package/README-es.md +30 -5
  3. package/README-hi.md +31 -5
  4. package/README-pt.md +31 -5
  5. package/README.en.md +31 -5
  6. package/README.md +31 -5
  7. package/bin/darwin-arm64/indexd +0 -0
  8. package/bin/darwin-x64/indexd +0 -0
  9. package/bin/linux-arm64/indexd +0 -0
  10. package/bin/linux-x64/indexd +0 -0
  11. package/bin/win32-arm64/indexd.exe +0 -0
  12. package/bin/win32-x64/indexd.exe +0 -0
  13. package/client.js +1198 -117
  14. package/lib/combined.d.ts +20 -3
  15. package/lib/combined.d.ts.map +1 -1
  16. package/lib/combined.js +85 -15
  17. package/lib/combined.js.map +1 -1
  18. package/lib/content-search-worker.d.ts +8 -3
  19. package/lib/content-search-worker.d.ts.map +1 -1
  20. package/lib/content-search-worker.js +54 -27
  21. package/lib/content-search-worker.js.map +1 -1
  22. package/lib/content-search.d.ts +127 -7
  23. package/lib/content-search.d.ts.map +1 -1
  24. package/lib/content-search.js +571 -36
  25. package/lib/content-search.js.map +1 -1
  26. package/lib/content.d.ts +83 -5
  27. package/lib/content.d.ts.map +1 -1
  28. package/lib/content.js +351 -87
  29. package/lib/content.js.map +1 -1
  30. package/lib/export.d.ts +61 -0
  31. package/lib/export.d.ts.map +1 -1
  32. package/lib/export.js +217 -52
  33. package/lib/export.js.map +1 -1
  34. package/lib/find.d.ts.map +1 -1
  35. package/lib/find.js +73 -16
  36. package/lib/find.js.map +1 -1
  37. package/lib/http.d.ts +38 -2
  38. package/lib/http.d.ts.map +1 -1
  39. package/lib/http.js +279 -82
  40. package/lib/http.js.map +1 -1
  41. package/lib/reveal.d.ts +69 -3
  42. package/lib/reveal.d.ts.map +1 -1
  43. package/lib/reveal.js +140 -11
  44. package/lib/reveal.js.map +1 -1
  45. package/lib/tools.d.ts +30 -0
  46. package/lib/tools.d.ts.map +1 -1
  47. package/lib/tools.js +65 -3
  48. package/lib/tools.js.map +1 -1
  49. package/package.json +1 -1
  50. package/src/combined.ts +107 -17
  51. package/src/content-search-worker.ts +52 -24
  52. package/src/content-search.ts +710 -36
  53. package/src/content.ts +349 -73
  54. package/src/export.ts +240 -50
  55. package/src/find.ts +83 -16
  56. package/src/http.ts +271 -83
  57. package/src/reveal.ts +148 -10
  58. package/src/tools.ts +83 -6
@@ -1,48 +1,722 @@
1
1
  /**
2
- * Content FTS5 query shared by the host thread and the search worker.
2
+ * Content FTS5 query engine: normal substring search, Notepad++-style extended
3
+ * escaping, case-sensitive GLOB matching and whole-word post-filtering, with
4
+ * exact or bounded counts.
5
+ *
6
+ * Shared by the host thread, the search worker and the combined panel search.
7
+ * The page path is deliberately two-phase: rank the rowids first, then read the
8
+ * few bodies and build the snippet in JS. Putting `snippet()` in the same
9
+ * `ORDER BY rank` statement made a SQLite sorter evaluate it for every match
10
+ * (measured 2.7 s for "the" on a 1 GB corpus; 60 ms with this split).
11
+ *
12
+ * Every unbounded-looking path has an explicit budget: queries shorter than a
13
+ * trigram are rejected instead of degrading into a full-table LIKE/GLOB scan,
14
+ * and the whole-word post-filter stops after a fixed number of rows, body bytes
15
+ * or milliseconds and reports a bounded count.
3
16
  * @module dsh-locator/content-search
4
17
  */
5
18
 
19
+ import { sep } from 'node:path'
6
20
  import type { DatabaseSync } from 'node:sqlite'
7
21
  import type { ContentHit } from './content.ts'
8
22
 
23
+ /** How the query text is interpreted. */
24
+ export type ContentMatchMode = 'normal' | 'extended' | 'regex'
25
+
26
+ /** How much work the caller wants the count to spend. */
27
+ export type ContentCountMode = 'exact' | 'bounded' | 'none'
28
+
29
+ /** A match count, with its confidence. `bounded` means "at least value". */
30
+ export interface ContentCount {
31
+ value: number
32
+ kind: 'exact' | 'bounded' | 'unknown'
33
+ limit?: number
34
+ }
35
+
36
+ /**
37
+ * Hard budget for the whole-word post-filter, which has to read candidate
38
+ * bodies in JS. Exceeding any cap stops the scan and keeps the hits found so
39
+ * far; the count then becomes bounded. Exposed mainly so embedders and tests
40
+ * can shrink the caps.
41
+ */
42
+ export interface ContentScanBudget {
43
+ maxRows?: number
44
+ maxBytes?: number
45
+ maxMs?: number
46
+ }
47
+
48
+ /** Options accepted by {@link searchContent} and the content search worker. */
49
+ export interface ContentSearchOptions {
50
+ matchMode?: ContentMatchMode
51
+ caseSensitive?: boolean
52
+ wholeWord?: boolean
53
+ /** Resolved by ContentIndex from the request's sessionId. */
54
+ sessionId?: string
55
+ /** Session log path; set by ContentIndex after resolving sessionId. */
56
+ sessionPath?: string | null
57
+ count?: ContentCountMode
58
+ countLimit?: number
59
+ /**
60
+ * Keep only file/session documents whose stored path lives under this
61
+ * directory prefix (the locator_find `scope: workspace` filter). Session
62
+ * documents are unaffected: they are scoped by `sessionPath`/`sessionId`.
63
+ */
64
+ pathPrefix?: string | null
65
+ /** Override for the whole-word scan caps; production uses the defaults. */
66
+ scanBudget?: ContentScanBudget
67
+ }
68
+
69
+ /** Result of one content query. */
70
+ export interface ContentSearchResult {
71
+ hits: ContentHit[]
72
+ count: ContentCount
73
+ warnings: string[]
74
+ error?: string
75
+ }
76
+
77
+ /** Shortest text a trigram tokenizer can index. */
78
+ const TRIGRAM_MIN = 3
79
+ /** Query entry limits: no unbounded parsing, no unbounded SQL. */
80
+ const MAX_QUERY_CODEPOINTS = 256
81
+ const MAX_QUERY_TERMS = 32
82
+ const MAX_LIMIT = 100
83
+ const DEFAULT_COUNT_LIMIT = 1000
84
+ const MAX_COUNT_LIMIT = 5000
85
+ const SNIPPET_BEFORE = 60
86
+ const SNIPPET_AFTER = 120
87
+ const SNIPPET_MAX = 400
88
+
89
+ /** Whole-word post-filter defaults (rows, body bytes, wall clock). */
90
+ const SCAN_MAX_ROWS = 5_000
91
+ const SCAN_MAX_BYTES = 64 * 1024 * 1024
92
+ const SCAN_MAX_MS = 800
93
+
94
+ /** Bytes that count as "inside a word", matching the name engine. */
95
+ const WORD_BYTE = /[A-Za-z0-9_]/
96
+ /** Control bytes: file names cannot contain them, so name patterns drop them. */
97
+ const CONTROL_CHARS = /[\u0000-\u001f]/
98
+ /**
99
+ * Control bytes rejected in a content query. Tab/LF/CR stay allowed: they are
100
+ * ordinary term separators in normal mode and explicit escapes in extended
101
+ * mode. NUL and the other C0 bytes must never reach SQLite.
102
+ */
103
+ const QUERY_CONTROL_CHARS = /[\u0000-\u0008\u000b\u000c\u000e-\u001f]/
104
+
105
+ /** Text escaped for an FTS5 quoted phrase (`"` doubled). */
106
+ export function quoteFts(text: string): string {
107
+ return '"' + text.replace(/"/g, '""') + '"'
108
+ }
109
+
110
+ /** Escape a literal for SQLite GLOB (`*`, `?`, `[`, `]`). */
111
+ export function escapeGlob(text: string): string {
112
+ return text.replace(/[*?[\]]/g, ch => (ch === ']' ? '[]]' : '[' + ch + ']'))
113
+ }
114
+
115
+ /** Escape a literal for SQL LIKE with `ESCAPE '\'`. */
116
+ export function escapeLike(text: string): string {
117
+ return text.replace(/[\\%_]/g, ch => '\\' + ch)
118
+ }
119
+
120
+ /** Decoded extended-mode query text. */
121
+ export interface DecodedExtended {
122
+ ok: boolean
123
+ text: string
124
+ warnings: string[]
125
+ /** `invalid-escape` when `\\xNN` is malformed. */
126
+ error?: string
127
+ }
128
+
129
+ /**
130
+ * Decode Notepad++-style extended escapes: `\n`, `\r`, `\t`, `\0`, `\\`,
131
+ * `\xNN`. Unknown escapes are kept literally with a warning.
132
+ */
133
+ export function decodeExtended(input: string): DecodedExtended {
134
+ const warnings = new Set<string>()
135
+ let text = ''
136
+ for (let i = 0; i < input.length; i++) {
137
+ const ch = input[i]!
138
+ if (ch !== '\\') {
139
+ text += ch
140
+ continue
141
+ }
142
+ const next = input[i + 1]
143
+ if (next === undefined) {
144
+ warnings.add('extended-trailing-backslash')
145
+ text += '\\'
146
+ continue
147
+ }
148
+ switch (next) {
149
+ case 'n':
150
+ text += '\n'
151
+ i++
152
+ continue
153
+ case 'r':
154
+ text += '\r'
155
+ i++
156
+ continue
157
+ case 't':
158
+ text += '\t'
159
+ i++
160
+ continue
161
+ case '0':
162
+ text += '\0'
163
+ warnings.add('extended-control-char')
164
+ i++
165
+ continue
166
+ case '\\':
167
+ text += '\\'
168
+ i++
169
+ continue
170
+ case 'x': {
171
+ const hex = input.slice(i + 2, i + 4)
172
+ if (!/^[0-9a-fA-F]{2}$/.test(hex)) {
173
+ return { ok: false, text: '', warnings: [...warnings], error: 'invalid-escape' }
174
+ }
175
+ const code = Number.parseInt(hex, 16)
176
+ text += String.fromCharCode(code)
177
+ if (code < 0x20 && code !== 0x09 && code !== 0x0a && code !== 0x0d) warnings.add('extended-control-char')
178
+ i += 3
179
+ continue
180
+ }
181
+ default:
182
+ warnings.add('extended-unknown-escape')
183
+ text += '\\' + next
184
+ i++
185
+ }
186
+ }
187
+ return { ok: true, text, warnings: [...warnings] }
188
+ }
189
+
190
+ /**
191
+ * Extended decoding for file names: names cannot contain control characters, so
192
+ * they are dropped from the pattern and reported instead of silently matching
193
+ * nothing.
194
+ */
195
+ export function decodeExtendedForNames(input: string): DecodedExtended {
196
+ const decoded = decodeExtended(input)
197
+ if (!decoded.ok) return decoded
198
+ const kept = decoded.text.replace(CONTROL_CHARS, '')
199
+ if (kept !== decoded.text) decoded.warnings.push('name-extended-control-char')
200
+ return { ...decoded, text: kept }
201
+ }
202
+
203
+ /** Length in Unicode code points, so emoji count as one character. */
204
+ function codePointCount(text: string): number {
205
+ return Array.from(text).length
206
+ }
207
+
208
+ /** Whether text still contains a byte the content query must reject. */
209
+ function hasControlChars(text: string): boolean {
210
+ return QUERY_CONTROL_CHARS.test(text)
211
+ }
212
+
213
+ /** Split a normal-mode query on whitespace, dropping empty terms. */
214
+ function splitTerms(text: string): string[] {
215
+ return text.split(/\s+/).map(part => part.replace(/^"|"$/g, '')).filter(part => part !== '')
216
+ }
217
+
218
+ /** Split decoded extended text into words for the entry checks. */
219
+ function splitWords(text: string): string[] {
220
+ return text.split(/\s+/).filter(part => part !== '')
221
+ }
222
+
223
+ /** Result of the query-entry checks shared by searchContent and the panel. */
224
+ export interface ContentQueryCheck {
225
+ ok: boolean
226
+ /** Literal terms for normal mode; one phrase for extended mode. */
227
+ terms: string[]
228
+ phrase: boolean
229
+ warnings: string[]
230
+ error?: string
231
+ }
232
+
9
233
  /**
10
- * Run one content query against an open SQLite handle.
11
- * @param db - open database (read-only in the worker).
12
- * @param query - user query; shorter than three characters falls back to LIKE.
13
- * @param limit - maximum hits.
14
- * @returns hits, ordered by relevance.
234
+ * Validate a content query before it reaches SQL.
235
+ *
236
+ * Content search only supports substrings the FTS5 trigram tokenizer can
237
+ * serve. A 1–2 character term used to fall back to a whole-corpus `LIKE` scan,
238
+ * which is both slow and unbounded; it is now a hard, visible error instead.
239
+ * The same limits apply after extended-mode escapes are decoded, and lengths
240
+ * are measured in Unicode code points so emoji are not mistaken for long text.
241
+ * @param query - raw query text.
242
+ * @param matchMode - `normal` (whitespace-separated terms) or `extended`.
243
+ * @returns normalized terms, or the first entry error.
15
244
  */
16
- export function searchContent(db: DatabaseSync, query: string, limit: number): ContentHit[] {
245
+ export function checkContentQuery(query: string, matchMode: ContentMatchMode): ContentQueryCheck {
246
+ const warnings: string[] = []
17
247
  const text = query.trim()
18
- if (text === '') return []
19
- const capped = Math.max(1, Math.min(100, limit))
20
- let rows: Array<Record<string, unknown>>
21
- if ([...text].length < 3) {
22
- // A one- or two-character query cannot use the trigram index, so bound the
23
- // payload as well as the row count: never ship a whole megabyte body.
24
- rows = db.prepare(`SELECT kind, path, title, session_id, message_id,
25
- (SELECT node_key FROM docs_meta WHERE docs_meta.rowid = docs.rowid) AS node_key,
26
- substr(body, 1, 400) AS snippet, 0 AS score
27
- FROM docs WHERE body LIKE '%' || ? || '%' LIMIT ?`).all(text, capped) as Array<Record<string, unknown>>
248
+ if (text === '') return { ok: true, terms: [], phrase: false, warnings }
249
+ if (codePointCount(text) > MAX_QUERY_CODEPOINTS) {
250
+ return { ok: false, terms: [], phrase: false, warnings, error: 'content-query-too-long' }
251
+ }
252
+ if (hasControlChars(text)) {
253
+ // A NUL (or other C0 byte) in the query must never be passed to SQLite:
254
+ // some string paths would stop at it and turn the query into "match
255
+ // everything". Reject it explicitly instead.
256
+ return { ok: false, terms: [], phrase: false, warnings, error: 'invalid-control-char' }
257
+ }
258
+ if (matchMode === 'extended') {
259
+ const decoded = decodeExtended(text)
260
+ warnings.push(...decoded.warnings)
261
+ if (!decoded.ok) {
262
+ return { ok: false, terms: [], phrase: false, warnings, error: decoded.error ?? 'invalid-escape' }
263
+ }
264
+ if (hasControlChars(decoded.text)) {
265
+ return { ok: false, terms: [], phrase: false, warnings, error: 'invalid-control-char' }
266
+ }
267
+ const words = splitWords(decoded.text)
268
+ if (words.length === 0) return { ok: true, terms: [], phrase: true, warnings }
269
+ if (words.length > MAX_QUERY_TERMS) {
270
+ return { ok: false, terms: [], phrase: true, warnings, error: 'content-query-too-many-terms' }
271
+ }
272
+ if (words.some(word => codePointCount(word) < TRIGRAM_MIN)) {
273
+ return { ok: false, terms: [], phrase: true, warnings, error: 'content-query-too-short' }
274
+ }
275
+ return { ok: true, terms: [decoded.text], phrase: true, warnings }
276
+ }
277
+ const terms = splitTerms(text)
278
+ if (terms.length > MAX_QUERY_TERMS) {
279
+ return { ok: false, terms: [], phrase: false, warnings, error: 'content-query-too-many-terms' }
280
+ }
281
+ if (terms.some(term => codePointCount(term) < TRIGRAM_MIN)) {
282
+ return { ok: false, terms: [], phrase: false, warnings, error: 'content-query-too-short' }
283
+ }
284
+ return { ok: true, terms, phrase: false, warnings }
285
+ }
286
+
287
+ /** Whether `index` starts a whole-word occurrence of `length` bytes. */
288
+ export function isWholeWord(hay: string, index: number, length: number): boolean {
289
+ const left = index === 0 || !WORD_BYTE.test(hay[index - 1] ?? '')
290
+ const end = index + length
291
+ const right = end >= hay.length || !WORD_BYTE.test(hay[end] ?? '')
292
+ return left && right
293
+ }
294
+
295
+ /** One located match: byte offset and length. */
296
+ interface Located {
297
+ index: number
298
+ length: number
299
+ }
300
+
301
+ /** First whole-word (or plain) occurrence of any term, or null. */
302
+ export function locateMatch(
303
+ body: string,
304
+ terms: readonly string[],
305
+ caseSensitive: boolean,
306
+ wholeWord: boolean,
307
+ ): Located | null {
308
+ const hay = caseSensitive ? body : body.toLowerCase()
309
+ let best: Located | null = null
310
+ for (const term of terms) {
311
+ if (term === '') continue
312
+ const needle = caseSensitive ? term : term.toLowerCase()
313
+ let from = 0
314
+ while (from <= hay.length) {
315
+ const index = hay.indexOf(needle, from)
316
+ if (index < 0) break
317
+ if (!wholeWord || isWholeWord(hay, index, needle.length)) {
318
+ if (best === null || index < best.index) best = { index, length: needle.length }
319
+ break
320
+ }
321
+ from = index + 1
322
+ }
323
+ }
324
+ return best
325
+ }
326
+
327
+ /** Every term must appear as a whole word (normal mode) or the literal must
328
+ * have non-word edges (extended mode is a single "term"). */
329
+ function matchesWholeWord(body: string, terms: readonly string[], caseSensitive: boolean): boolean {
330
+ for (const term of terms) {
331
+ if (term === '') continue
332
+ if (locateMatch(body, [term], caseSensitive, true) === null) return false
333
+ }
334
+ return true
335
+ }
336
+
337
+ /** 1-based line number of a byte offset. */
338
+ export function lineOf(body: string, index: number): number {
339
+ let line = 1
340
+ let from = 0
341
+ for (;;) {
342
+ const next = body.indexOf('\n', from)
343
+ if (next < 0 || next >= index) return line
344
+ line++
345
+ from = next + 1
346
+ }
347
+ }
348
+
349
+ /** Bounded, single-line snippet around the first match. */
350
+ function makeSnippet(body: string, located: Located | null): string {
351
+ if (located === null) {
352
+ const flat = body.replace(/\s+/g, ' ').trim()
353
+ return flat.length > SNIPPET_MAX ? flat.slice(0, SNIPPET_MAX - 1) + '…' : flat
354
+ }
355
+ const start = Math.max(0, located.index - SNIPPET_BEFORE)
356
+ const end = Math.min(body.length, located.index + located.length + SNIPPET_AFTER)
357
+ const flat = body.slice(start, end).replace(/\s+/g, ' ')
358
+ const text = (start > 0 ? '…' : '') + flat + (end < body.length ? '…' : '')
359
+ return text.length > SNIPPET_MAX ? text.slice(0, SNIPPET_MAX - 1) + '…' : text
360
+ }
361
+
362
+ function emptyCount(mode: ContentCountMode): ContentCount {
363
+ if (mode === 'none') return { value: 0, kind: 'unknown' }
364
+ return { value: 0, kind: 'exact' }
365
+ }
366
+
367
+ function clampLimit(limit: number): number {
368
+ return Math.max(1, Math.min(MAX_LIMIT, Number.isFinite(limit) ? Math.trunc(limit) : MAX_LIMIT))
369
+ }
370
+
371
+ function clampCountLimit(limit: number | undefined): number {
372
+ if (limit === undefined || !Number.isFinite(limit)) return DEFAULT_COUNT_LIMIT
373
+ return Math.max(1, Math.min(MAX_COUNT_LIMIT, Math.trunc(limit)))
374
+ }
375
+
376
+ /** A positive integer override, or the default when unset/invalid. */
377
+ function budgetValue(value: number | undefined, fallback: number): number {
378
+ if (value === undefined || !Number.isFinite(value) || value < 0) return fallback
379
+ return Math.trunc(value)
380
+ }
381
+
382
+ function resolveScanBudget(budget: ContentScanBudget | undefined): { maxRows: number; maxBytes: number; maxMs: number } {
383
+ return {
384
+ maxRows: budgetValue(budget?.maxRows, SCAN_MAX_ROWS),
385
+ maxBytes: budgetValue(budget?.maxBytes, SCAN_MAX_BYTES),
386
+ maxMs: budgetValue(budget?.maxMs, SCAN_MAX_MS),
387
+ }
388
+ }
389
+
390
+ /** Normalize the optional path-prefix filter; keep the platform separator. */
391
+ function normalizePathPrefix(prefix: string | null | undefined): string | null {
392
+ if (typeof prefix !== 'string') return null
393
+ const trimmed = prefix.trim()
394
+ if (trimmed === '') return null
395
+ if (trimmed.endsWith('/') || trimmed.endsWith('\\')) return trimmed
396
+ return trimmed + sep
397
+ }
398
+
399
+ /** One query plan: the SQL WHERE fragment plus how to page/count it. */
400
+ interface QueryPlan {
401
+ where: string
402
+ params: Array<string | number>
403
+ /** Literal terms used for whole-word checks and snippet location. */
404
+ terms: string[]
405
+ caseSensitive: boolean
406
+ wholeWord: boolean
407
+ /** Match the whole phrase as one literal (extended mode). */
408
+ phrase: boolean
409
+ rankOrder: boolean
410
+ exactCount: boolean
411
+ /** GLOB counting is exact only below countLimit; above it is bounded. */
412
+ boundedCount: boolean
413
+ }
414
+
415
+ function buildPlan(terms: readonly string[], options: {
416
+ caseSensitive: boolean
417
+ wholeWord: boolean
418
+ phrase: boolean
419
+ sessionPath: string | null
420
+ pathPrefix: string | null
421
+ }): QueryPlan | null {
422
+ const { caseSensitive, wholeWord, phrase, sessionPath, pathPrefix } = options
423
+ if (terms.length === 0) return null
424
+ // Only files live inside the workspace prefix. Session rows belong to the
425
+ // session scope and are filtered by sessionPath/sessionId instead.
426
+ const suffix: string[] = []
427
+ const suffixParams: Array<string | number> = []
428
+ if (sessionPath !== null) {
429
+ suffix.push(' AND rowid IN (SELECT rowid FROM docs_meta WHERE path = ?)')
430
+ suffixParams.push(sessionPath)
431
+ }
432
+ if (pathPrefix !== null) {
433
+ suffix.push(" AND (kind = 'session' OR rowid IN (SELECT rowid FROM docs_meta WHERE path LIKE ? ESCAPE '\\'))")
434
+ suffixParams.push(escapeLike(pathPrefix) + '%')
435
+ }
436
+ const sessionClause = suffix.join('')
437
+
438
+ if (phrase) {
439
+ const literal = terms[0]!
440
+ if (caseSensitive) {
441
+ return {
442
+ where: 'body GLOB ?' + sessionClause,
443
+ params: ['*' + escapeGlob(literal) + '*', ...suffixParams],
444
+ terms: [literal],
445
+ caseSensitive,
446
+ wholeWord,
447
+ phrase,
448
+ rankOrder: false,
449
+ exactCount: true,
450
+ boundedCount: true,
451
+ }
452
+ }
453
+ return {
454
+ where: 'docs MATCH ?' + sessionClause,
455
+ params: [quoteFts(literal), ...suffixParams],
456
+ terms: [literal],
457
+ caseSensitive,
458
+ wholeWord,
459
+ phrase,
460
+ rankOrder: true,
461
+ exactCount: true,
462
+ boundedCount: false,
463
+ }
464
+ }
465
+
466
+ if (caseSensitive) {
467
+ return {
468
+ where: terms.map(() => 'body GLOB ?').join(' AND ') + sessionClause,
469
+ params: [...terms.map(term => '*' + escapeGlob(term) + '*'), ...suffixParams],
470
+ terms: [...terms],
471
+ caseSensitive,
472
+ wholeWord,
473
+ phrase,
474
+ rankOrder: false,
475
+ exactCount: true,
476
+ boundedCount: true,
477
+ }
478
+ }
479
+ return {
480
+ where: 'docs MATCH ?' + sessionClause,
481
+ params: [terms.map(quoteFts).join(' AND '), ...suffixParams],
482
+ terms: [...terms],
483
+ caseSensitive,
484
+ wholeWord,
485
+ phrase,
486
+ rankOrder: true,
487
+ exactCount: true,
488
+ boundedCount: false,
489
+ }
490
+ }
491
+
492
+ /** One document row fetched for a hit. */
493
+ interface DocRow {
494
+ kind?: string
495
+ path?: string
496
+ title?: string
497
+ session_id?: string | null
498
+ message_id?: string | null
499
+ body?: string
500
+ node_key?: string | null
501
+ score?: number
502
+ }
503
+
504
+ const DOC_SQL = `SELECT d.kind AS kind, d.path AS path, d.title AS title,
505
+ d.session_id AS session_id, d.message_id AS message_id, d.body AS body,
506
+ m.node_key AS node_key
507
+ FROM docs d LEFT JOIN docs_meta m ON m.rowid = d.rowid
508
+ WHERE d.rowid = ?`
509
+
510
+ /** Build hits from rowids: metadata, snippet, first-match line and score. */
511
+ function buildHits(db: DatabaseSync, plan: QueryPlan, rowids: readonly number[], scores: ReadonlyMap<number, number>): ContentHit[] {
512
+ const statement = db.prepare(DOC_SQL)
513
+ const hits: ContentHit[] = []
514
+ for (const rowid of rowids) {
515
+ const row = statement.get(rowid) as DocRow | undefined
516
+ if (row === undefined || typeof row.body !== 'string') continue
517
+ const located = locateMatch(row.body, plan.terms, plan.caseSensitive, plan.wholeWord)
518
+ const line = located === null ? undefined : lineOf(row.body, located.index)
519
+ const score = scores.get(rowid)
520
+ const hit: ContentHit = {
521
+ kind: row.kind === 'session' ? 'session' : 'file',
522
+ path: typeof row.path === 'string' ? row.path : '',
523
+ title: typeof row.title === 'string' ? row.title : '',
524
+ snippet: makeSnippet(row.body, located),
525
+ score: score === undefined ? 0 : score,
526
+ ...(typeof row.session_id === 'string' && row.session_id !== '' ? { sessionId: row.session_id } : {}),
527
+ ...(typeof row.message_id === 'string' && row.message_id !== '' ? { messageId: row.message_id } : {}),
528
+ ...(typeof row.node_key === 'string' && row.node_key !== '' ? { nodeKey: row.node_key } : {}),
529
+ ...(line === undefined ? {} : { line }),
530
+ }
531
+ hits.push(hit)
532
+ }
533
+ return hits
534
+ }
535
+
536
+ /** Candidate rowids for one page (no post-filter). */
537
+ function pageRowids(db: DatabaseSync, plan: QueryPlan, limit: number): { rowids: number[]; scores: Map<number, number> } {
538
+ const scores = new Map<number, number>()
539
+ const sql = plan.rankOrder
540
+ ? `SELECT rowid AS rowid, rank AS score FROM docs WHERE ${plan.where} ORDER BY rank LIMIT ?`
541
+ : `SELECT rowid AS rowid FROM docs WHERE ${plan.where} LIMIT ?`
542
+ const rows = db.prepare(sql).all(...plan.params, limit) as Array<{ rowid?: number; score?: number }>
543
+ const rowids: number[] = []
544
+ for (const row of rows) {
545
+ if (typeof row.rowid !== 'number') continue
546
+ rowids.push(row.rowid)
547
+ if (typeof row.score === 'number') scores.set(row.rowid, row.score)
548
+ }
549
+ return { rowids, scores }
550
+ }
551
+
552
+ /** Exact COUNT over the plan's WHERE fragment. */
553
+ function exactCount(db: DatabaseSync, plan: QueryPlan): number {
554
+ const row = db.prepare(`SELECT count(*) AS n FROM docs WHERE ${plan.where}`).get(...plan.params) as { n?: number } | undefined
555
+ return Number(row?.n ?? 0)
556
+ }
557
+
558
+ /** Count up to `limit + 1` rows: exact below the cap, bounded at the cap. */
559
+ function boundedCount(db: DatabaseSync, plan: QueryPlan, limit: number): ContentCount {
560
+ const row = db
561
+ .prepare(`SELECT count(*) AS n FROM (SELECT rowid FROM docs WHERE ${plan.where} LIMIT ?)`)
562
+ .get(...plan.params, limit + 1) as { n?: number } | undefined
563
+ const value = Number(row?.n ?? 0)
564
+ if (value > limit) return { value: limit, kind: 'bounded', limit }
565
+ return { value, kind: 'exact' }
566
+ }
567
+
568
+ /** A plan with a whole-word post-filter. */
569
+ interface WholeWordScan {
570
+ rowids: number[]
571
+ count: ContentCount
572
+ warnings: string[]
573
+ }
574
+
575
+ /**
576
+ * Scan candidates for whole-word matches under a hard budget.
577
+ *
578
+ * Reading every candidate body in JS is the only correct way to apply the
579
+ * ASCII word-boundary rule, but a common word in a multi-GB corpus would keep
580
+ * the worker (and the query) busy for minutes. Rows, body bytes and wall clock
581
+ * are all capped: on any cap the scan stops, keeps the rowids found so far and
582
+ * reports a bounded count plus a `scan-capped` warning. Session-scoped scans go
583
+ * through the same budget, so a huge session cannot run unbounded either.
584
+ */
585
+ function scanWholeWord(
586
+ db: DatabaseSync,
587
+ plan: QueryPlan,
588
+ limit: number,
589
+ countMode: ContentCountMode,
590
+ countLimit: number,
591
+ sessionScoped: boolean,
592
+ budget: { maxRows: number; maxBytes: number; maxMs: number },
593
+ ): WholeWordScan {
594
+ const warnings = new Set<string>()
595
+ const rowids: number[] = []
596
+ const sql = plan.rankOrder
597
+ ? `SELECT rowid AS rowid FROM docs WHERE ${plan.where} ORDER BY rank`
598
+ : `SELECT rowid AS rowid FROM docs WHERE ${plan.where}`
599
+ const iterator = db.prepare(sql).iterate(...plan.params)
600
+ const bodyStatement = db.prepare('SELECT body FROM docs WHERE rowid = ?')
601
+ const deadline = performance.now() + budget.maxMs
602
+ let count = 0
603
+ let scannedRows = 0
604
+ let scannedBytes = 0
605
+ let capped = false
606
+ let exhausted = true
607
+ for (const raw of iterator) {
608
+ scannedRows++
609
+ if (scannedRows > budget.maxRows || scannedBytes >= budget.maxBytes || performance.now() > deadline) {
610
+ capped = true
611
+ exhausted = false
612
+ break
613
+ }
614
+ const rowid = (raw as { rowid?: number }).rowid
615
+ if (typeof rowid !== 'number') continue
616
+ const row = bodyStatement.get(rowid) as { body?: string } | undefined
617
+ if (row === undefined || typeof row.body !== 'string') continue
618
+ scannedBytes += row.body.length
619
+ if (!matchesWholeWord(row.body, plan.terms, plan.caseSensitive)) continue
620
+ count++
621
+ if (rowids.length < limit) rowids.push(rowid)
622
+ if (countMode === 'none') {
623
+ if (rowids.length >= limit) {
624
+ // The page is full; no count was requested, so stopping is safe.
625
+ exhausted = false
626
+ break
627
+ }
628
+ } else if (!sessionScoped && count > countLimit) {
629
+ exhausted = false
630
+ break
631
+ }
632
+ }
633
+ if (capped) warnings.add('scan-capped')
634
+ let result: ContentCount
635
+ if (countMode === 'none') {
636
+ result = { value: 0, kind: 'unknown' }
637
+ } else if (count > countLimit) {
638
+ // Above the cap: report the known floor, never an exact count we did not
639
+ // finish. Global whole-word counts are deliberately bounded.
640
+ warnings.add('bounded-count')
641
+ result = { value: countLimit, kind: 'bounded', limit: countLimit }
642
+ } else if (capped || !exhausted) {
643
+ result = { value: count, kind: 'bounded', limit: countLimit }
28
644
  } else {
29
- const match = text.split(/\s+/).filter(Boolean).map(term => '"' + term.replace(/"/g, '') + '"').join(' AND ')
30
- // Keep MATCH in this statement: FTS5 snippet() and bm25() are only defined
31
- // inside a matching query (a rowid subquery silently degrades both). The
32
- // query runs in the search worker, so its cost stays off the host thread.
33
- rows = db.prepare(`SELECT kind, path, title, session_id, message_id,
34
- (SELECT node_key FROM docs_meta WHERE docs_meta.rowid = docs.rowid) AS node_key,
35
- snippet(docs, 0, char(1), char(2), '…', 16) AS snippet, bm25(docs) AS score
36
- FROM docs WHERE docs MATCH ? ORDER BY score LIMIT ?`).all(match, capped) as Array<Record<string, unknown>>
37
- }
38
- return rows.map(row => ({
39
- kind: row.kind === 'session' ? 'session' : 'file',
40
- path: String(row.path ?? ''),
41
- title: String(row.title ?? ''),
42
- snippet: String(row.snippet ?? row.body ?? '').replace(/[\u0001\u0002]/g, ''),
43
- ...(row.session_id ? { sessionId: String(row.session_id) } : {}),
44
- ...(row.message_id ? { messageId: String(row.message_id) } : {}),
45
- ...(row.node_key ? { nodeKey: String(row.node_key) } : {}),
46
- score: Number(row.score ?? 0),
47
- }))
645
+ result = { value: count, kind: 'exact' }
646
+ }
647
+ return { rowids, count: result, warnings: [...warnings] }
648
+ }
649
+
650
+ /**
651
+ * Run one content query.
652
+ *
653
+ * Modes: `normal` (whitespace-separated substring terms), `extended`
654
+ * (Notepad++ escapes, one literal phrase), `regex` (not executed server-side in
655
+ * 0.9.0: returns `content-regex-not-supported`).
656
+ */
657
+ export function searchContent(
658
+ db: DatabaseSync,
659
+ query: string,
660
+ limit: number,
661
+ options: ContentSearchOptions = {},
662
+ ): ContentSearchResult {
663
+ const capped = clampLimit(limit)
664
+ const matchMode = options.matchMode ?? 'normal'
665
+ const countMode = options.count ?? 'exact'
666
+ const countLimit = clampCountLimit(options.countLimit)
667
+
668
+ if (matchMode === 'regex') {
669
+ return {
670
+ hits: [],
671
+ count: { value: 0, kind: 'unknown' },
672
+ warnings: ['content-regex-paused'],
673
+ error: 'content-regex-not-supported',
674
+ }
675
+ }
676
+
677
+ const check = checkContentQuery(query, matchMode)
678
+ const warnings = [...check.warnings]
679
+ if (!check.ok) {
680
+ return {
681
+ hits: [],
682
+ count: { value: 0, kind: 'unknown' },
683
+ warnings,
684
+ error: check.error ?? 'content-query-invalid',
685
+ }
686
+ }
687
+ if (check.terms.length === 0) return { hits: [], count: emptyCount(countMode), warnings }
688
+
689
+ const plan = buildPlan(check.terms, {
690
+ caseSensitive: options.caseSensitive === true,
691
+ wholeWord: options.wholeWord === true,
692
+ phrase: check.phrase,
693
+ sessionPath: options.sessionPath ?? null,
694
+ pathPrefix: normalizePathPrefix(options.pathPrefix),
695
+ })
696
+ if (plan === null) return { hits: [], count: emptyCount(countMode), warnings }
697
+
698
+ if (plan.wholeWord) {
699
+ const scanned = scanWholeWord(db, plan, capped, countMode, countLimit, options.sessionPath != null, resolveScanBudget(options.scanBudget))
700
+ for (const warning of scanned.warnings) warnings.push(warning)
701
+ const hits = buildHits(db, plan, scanned.rowids, new Map())
702
+ return { hits, count: scanned.count, warnings: [...new Set(warnings)] }
703
+ }
704
+
705
+ let count: ContentCount
706
+ if (countMode === 'none') {
707
+ count = { value: 0, kind: 'unknown' }
708
+ } else if (countMode === 'bounded') {
709
+ count = boundedCount(db, plan, countLimit)
710
+ if (count.kind === 'bounded') warnings.push('bounded-count')
711
+ } else if (plan.exactCount && !plan.boundedCount) {
712
+ count = { value: exactCount(db, plan), kind: 'exact' }
713
+ } else {
714
+ // GLOB patterns cannot count cheaply: cap the work and report the known
715
+ // floor plus a warning.
716
+ count = boundedCount(db, plan, countLimit)
717
+ if (count.kind === 'bounded') warnings.push('bounded-count')
718
+ }
719
+ const page = pageRowids(db, plan, capped)
720
+ const hits = buildHits(db, plan, page.rowids, page.scores)
721
+ return { hits, count, warnings: [...new Set(warnings)] }
48
722
  }