dsh-locator 0.8.2 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +47 -0
  2. package/README-es.md +30 -5
  3. package/README-hi.md +31 -5
  4. package/README-pt.md +31 -5
  5. package/README.en.md +31 -5
  6. package/README.md +31 -5
  7. package/bin/darwin-arm64/indexd +0 -0
  8. package/bin/darwin-x64/indexd +0 -0
  9. package/bin/linux-arm64/indexd +0 -0
  10. package/bin/linux-x64/indexd +0 -0
  11. package/bin/win32-arm64/indexd.exe +0 -0
  12. package/bin/win32-x64/indexd.exe +0 -0
  13. package/client.js +1198 -117
  14. package/lib/combined.d.ts +20 -3
  15. package/lib/combined.d.ts.map +1 -1
  16. package/lib/combined.js +85 -15
  17. package/lib/combined.js.map +1 -1
  18. package/lib/content-search-worker.d.ts +8 -3
  19. package/lib/content-search-worker.d.ts.map +1 -1
  20. package/lib/content-search-worker.js +54 -27
  21. package/lib/content-search-worker.js.map +1 -1
  22. package/lib/content-search.d.ts +127 -7
  23. package/lib/content-search.d.ts.map +1 -1
  24. package/lib/content-search.js +571 -36
  25. package/lib/content-search.js.map +1 -1
  26. package/lib/content.d.ts +83 -5
  27. package/lib/content.d.ts.map +1 -1
  28. package/lib/content.js +351 -87
  29. package/lib/content.js.map +1 -1
  30. package/lib/export.d.ts +61 -0
  31. package/lib/export.d.ts.map +1 -1
  32. package/lib/export.js +217 -52
  33. package/lib/export.js.map +1 -1
  34. package/lib/find.d.ts.map +1 -1
  35. package/lib/find.js +73 -16
  36. package/lib/find.js.map +1 -1
  37. package/lib/http.d.ts +38 -2
  38. package/lib/http.d.ts.map +1 -1
  39. package/lib/http.js +279 -82
  40. package/lib/http.js.map +1 -1
  41. package/lib/reveal.d.ts +69 -3
  42. package/lib/reveal.d.ts.map +1 -1
  43. package/lib/reveal.js +140 -11
  44. package/lib/reveal.js.map +1 -1
  45. package/lib/tools.d.ts +30 -0
  46. package/lib/tools.d.ts.map +1 -1
  47. package/lib/tools.js +65 -3
  48. package/lib/tools.js.map +1 -1
  49. package/package.json +1 -1
  50. package/src/combined.ts +107 -17
  51. package/src/content-search-worker.ts +52 -24
  52. package/src/content-search.ts +710 -36
  53. package/src/content.ts +349 -73
  54. package/src/export.ts +240 -50
  55. package/src/find.ts +83 -16
  56. package/src/http.ts +271 -83
  57. package/src/reveal.ts +148 -10
  58. package/src/tools.ts +83 -6
package/src/content.ts CHANGED
@@ -16,7 +16,7 @@
16
16
  * @module dsh-locator/content
17
17
  */
18
18
 
19
- import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
19
+ import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'
20
20
  import { readdir, readFile, stat } from 'node:fs/promises'
21
21
  import { homedir } from 'node:os'
22
22
  import { basename, dirname, extname, join } from 'node:path'
@@ -27,7 +27,7 @@ import { DatabaseSync } from 'node:sqlite'
27
27
  import { randomUUID } from 'node:crypto'
28
28
  import { fileURLToPath } from 'node:url'
29
29
  import { Worker } from 'node:worker_threads'
30
- import { searchContent } from './content-search.ts'
30
+ import { searchContent, type ContentSearchOptions, type ContentSearchResult } from './content-search.ts'
31
31
  import type { LocatorRuntime } from './runtime.ts'
32
32
 
33
33
  const SCHEMA_VERSION = 3
@@ -67,6 +67,8 @@ export interface ContentHit {
67
67
  messageId?: string
68
68
  /** Exact `data-chat-node-key` of the rendered row, when the host has one. */
69
69
  nodeKey?: string
70
+ /** 1-based line of the first match inside the stored body, for file hits. */
71
+ line?: number
70
72
  score: number
71
73
  }
72
74
 
@@ -77,6 +79,11 @@ export interface ContentState {
77
79
  messages: number
78
80
  truncated: boolean
79
81
  error: string
82
+ /**
83
+ * Non-fatal problems from the last build/refresh: unreadable session logs,
84
+ * a quarantined database, and so on. Empty when everything was read.
85
+ */
86
+ warnings: string[]
80
87
  updatedAt: number
81
88
  }
82
89
 
@@ -102,6 +109,16 @@ export interface ContentOptions {
102
109
  * single-threaded.
103
110
  */
104
111
  searchWorker?: boolean
112
+ /**
113
+ * How long one worker query may run before it is reported as
114
+ * `content-search-timeout`. The query is never re-run on the host thread.
115
+ */
116
+ searchWorkerTimeoutMs?: number
117
+ /**
118
+ * Test seam: build the search worker instead of spawning the compiled one.
119
+ * Production never sets it.
120
+ */
121
+ searchWorkerFactory?: (workerPath: string) => Worker
105
122
  }
106
123
 
107
124
  interface DocRow {
@@ -118,7 +135,7 @@ interface DocRow {
118
135
 
119
136
  export class ContentIndex {
120
137
  private db: DatabaseSync | null = null
121
- private readonly state: ContentState = { building: false, files: 0, sessions: 0, messages: 0, truncated: false, error: '', updatedAt: 0 }
138
+ private readonly state: ContentState = { building: false, files: 0, sessions: 0, messages: 0, truncated: false, error: '', warnings: [], updatedAt: 0 }
122
139
  private generation = 0
123
140
  private totalBytes = 0
124
141
  private sliceDeadline = 0
@@ -128,13 +145,15 @@ export class ContentIndex {
128
145
  private workerSeq = 0
129
146
  private pendingStart = false
130
147
  private dbIdentity: { dev: number; ino: number } | null = null
131
- private readonly pending = new Map<number, { resolve: (hits: ContentHit[]) => void; reject: (error: Error) => void }>()
148
+ /** Session id → indexed session log path, for session-scoped content search. */
149
+ private readonly sessionPaths = new Map<string, string>()
150
+ private readonly pending = new Map<number, (result: ContentSearchResult) => void>()
132
151
 
133
152
  constructor(private readonly rt: LocatorRuntime, private readonly options: ContentOptions) {}
134
153
 
135
154
  status(): ContentState {
136
155
  if (this.db !== null && !this.state.building && !this.databaseIsCurrent()) this.recoverExternal()
137
- return { ...this.state, building: this.state.building || this.pendingStart }
156
+ return { ...this.state, warnings: [...this.state.warnings], building: this.state.building || this.pendingStart }
138
157
  }
139
158
 
140
159
  /**
@@ -176,6 +195,7 @@ export class ContentIndex {
176
195
  this.releaseLock()
177
196
  this.pendingStart = false
178
197
  this.dbIdentity = null
198
+ this.sessionPaths.clear()
179
199
  }
180
200
 
181
201
  /** Build everything from scratch. */
@@ -251,6 +271,7 @@ export class ContentIndex {
251
271
  this.state.building = true
252
272
  this.state.error = ''
253
273
  this.state.truncated = false
274
+ this.state.warnings = []
254
275
  const run = mode === 'full' ? this.full(generation) : this.incremental(generation)
255
276
  void run.catch(error => {
256
277
  if (generation !== this.generation) return
@@ -303,8 +324,20 @@ export class ContentIndex {
303
324
  }
304
325
  const dbPath = join(this.options.stateDir, 'content.db')
305
326
  const existed = existsSync(dbPath)
306
- if (existed && !this.ownedDatabase(dbPath)) {
307
- throw new Error(`content.db is not a dsh-locator database, refusing to touch it: ${dbPath}`)
327
+ if (existed) {
328
+ const verdict = this.classifyDatabase(dbPath)
329
+ if (verdict === 'foreign') {
330
+ throw new Error(`content.db is not a dsh-locator database, refusing to touch it: ${dbPath}`)
331
+ }
332
+ if (verdict !== 'ok') {
333
+ // A corrupt, empty or older-schema database must not disable content
334
+ // search for the rest of the session. Keep the bad file for diagnosis
335
+ // (rename, never delete bytes we did not create) and rebuild fresh.
336
+ // During an incremental open this throws so the caller falls back to a
337
+ // full rebuild; a full open simply continues with a clean file.
338
+ this.quarantineDatabase(dbPath, verdict)
339
+ if (!fresh) throw new Error('content database is incompatible; rebuilding')
340
+ }
308
341
  }
309
342
  let oldFileRemained = false
310
343
  if (fresh) {
@@ -347,52 +380,135 @@ export class ContentIndex {
347
380
  this.db.exec("INSERT OR REPLACE INTO content_meta (key, value) VALUES ('complete', 0)")
348
381
  }
349
382
  if (fresh) this.db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`)
383
+ this.refreshSessionPaths()
350
384
  this.reopenWorker()
351
385
  }
352
386
 
387
+ /**
388
+ * Decide what an existing `content.db` is before touching it:
389
+ * - `foreign`: no dsh-locator table at all — refuse to modify it.
390
+ * - `corrupt`: zero bytes, unreadable, or not a SQLite database.
391
+ * - `incompatible`: has our tables but a different schema version/shape.
392
+ * - `ok`: safe to reuse.
393
+ */
394
+ private classifyDatabase(path: string): 'ok' | 'foreign' | 'corrupt' | 'incompatible' {
395
+ let size = 0
396
+ try { size = statSync(path).size } catch { return 'corrupt' }
397
+ if (size === 0) return 'corrupt'
398
+ let probe: DatabaseSync | undefined
399
+ try {
400
+ probe = new DatabaseSync(path, { readOnly: true })
401
+ const names = new Set(
402
+ (probe.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").all() as Array<{ name?: string }>)
403
+ .map(row => row.name)
404
+ .filter((name): name is string => typeof name === 'string'),
405
+ )
406
+ const owned = ['docs', 'docs_meta', 'sources', 'content_meta']
407
+ if (!owned.some(name => names.has(name))) return 'foreign'
408
+ if (!names.has('docs') || !names.has('docs_meta') || !names.has('sources') || !names.has('content_meta')) return 'incompatible'
409
+ const version = (probe.prepare('PRAGMA user_version').get() as { user_version?: number } | undefined)?.user_version ?? 0
410
+ if (Number(version) !== SCHEMA_VERSION) return 'incompatible'
411
+ return 'ok'
412
+ } catch {
413
+ return 'corrupt'
414
+ } finally {
415
+ try { probe?.close() } catch { /* ignore */ }
416
+ }
417
+ }
418
+
419
+ /**
420
+ * Move a broken/outdated database aside instead of destroying it. The
421
+ * timestamped name keeps the bytes around for inspection; if the rename
422
+ * fails (another process still holds it on Windows) a fresh file is still
423
+ * created so content search recovers.
424
+ */
425
+ private quarantineDatabase(dbPath: string, verdict: 'corrupt' | 'incompatible'): void {
426
+ this.stopWorker()
427
+ try { this.db?.close() } catch { /* ignore */ }
428
+ this.db = null
429
+ const target = `${dbPath}.corrupt-${Date.now()}`
430
+ try {
431
+ renameSync(dbPath, target)
432
+ this.addWarning(verdict === 'corrupt' ? 'content-db-corrupt' : 'content-db-incompatible')
433
+ } catch {
434
+ try { rmSync(dbPath, { force: true }) } catch { /* ignore */ }
435
+ this.addWarning(verdict === 'corrupt' ? 'content-db-corrupt' : 'content-db-incompatible')
436
+ }
437
+ for (const suffix of ['-wal', '-shm']) {
438
+ try { rmSync(dbPath + suffix, { force: true }) } catch { /* ignore */ }
439
+ }
440
+ }
441
+
442
+ /** Record one non-fatal build warning (deduplicated, bounded). */
443
+ private addWarning(warning: string): void {
444
+ if (this.state.warnings.includes(warning)) return
445
+ if (this.state.warnings.length >= 20) return
446
+ this.state.warnings.push(warning)
447
+ }
448
+
353
449
  private reopenWorker(): void {
354
450
  if (this.worker === null) return
355
- this.worker.postMessage({ id: ++this.workerSeq, open: join(this.options.stateDir, 'content.db') })
451
+ try {
452
+ this.worker.postMessage({ id: ++this.workerSeq, open: join(this.options.stateDir, 'content.db') })
453
+ } catch { /* a dying worker is replaced on the next query */ }
356
454
  }
357
455
 
358
456
  private stopWorker(): void {
359
457
  const worker = this.worker
360
458
  this.worker = null
361
- for (const entry of this.pending.values()) entry.resolve([])
459
+ for (const resolve of this.pending.values()) resolve(workerFailureResult('content-search-worker-failed'))
362
460
  this.pending.clear()
363
461
  if (worker !== null) void worker.terminate()
364
462
  }
365
463
 
366
- private ensureWorker(): void {
367
- if (this.options.searchWorker !== true || this.worker !== null) return
464
+ /**
465
+ * Whether a usable search worker exists, creating one when needed.
466
+ * @returns false when the worker cannot be created at all; that is the only
467
+ * case in which an in-process (sync) query is allowed.
468
+ */
469
+ private ensureWorker(): boolean {
470
+ if (this.options.searchWorker !== true) return false
471
+ if (this.worker !== null) return true
368
472
  // In a TypeScript test run the compiled worker does not exist next to the
369
- // source; skip spawning and answer queries in-process instead.
473
+ // source; answer queries in-process instead (budgeted, see searchSync).
370
474
  const workerPath = fileURLToPath(new URL('./content-search-worker.js', import.meta.url))
371
- if (!existsSync(workerPath)) return
475
+ const factory = this.options.searchWorkerFactory
476
+ if (factory === undefined && !existsSync(workerPath)) return false
372
477
  try {
373
- const worker = new Worker(workerPath, {
374
- workerData: { dbPath: join(this.options.stateDir, 'content.db') },
375
- })
376
- worker.on('message', (message: { id: number; hits?: ContentHit[] }) => {
377
- const entry = this.pending.get(message.id)
378
- if (entry === undefined) return
379
- this.pending.delete(message.id)
380
- entry.resolve(message.hits ?? [])
381
- })
478
+ const worker = factory !== undefined
479
+ ? factory(workerPath)
480
+ : new Worker(workerPath, { workerData: { dbPath: join(this.options.stateDir, 'content.db') } })
481
+ worker.on('message', (message: unknown) => { this.onWorkerMessage(message) })
382
482
  const fail = (): void => {
383
483
  if (this.worker !== worker) return
384
484
  this.worker = null
385
- for (const entry of this.pending.values()) entry.reject(new Error('content search worker failed'))
485
+ for (const resolve of this.pending.values()) resolve(workerFailureResult('content-search-worker-failed'))
386
486
  this.pending.clear()
387
487
  }
388
488
  worker.on('error', fail)
389
489
  worker.on('exit', fail)
390
490
  this.worker = worker
491
+ return true
391
492
  } catch {
392
493
  this.worker = null
494
+ return false
393
495
  }
394
496
  }
395
497
 
498
+ /**
499
+ * Resolve one worker reply. A malformed reply is reported as a protocol
500
+ * error; the query is never retried on the host thread.
501
+ */
502
+ private onWorkerMessage(message: unknown): void {
503
+ if (typeof message !== 'object' || message === null) return
504
+ const record = message as { id?: unknown; result?: unknown }
505
+ if (typeof record.id !== 'number') return
506
+ const resolve = this.pending.get(record.id)
507
+ if (resolve === undefined) return
508
+ this.pending.delete(record.id)
509
+ resolve(isContentSearchResult(record.result) ? record.result : workerFailureResult('content-search-worker-protocol-error'))
510
+ }
511
+
396
512
  private identityOf(path: string): { dev: number; ino: number } | null {
397
513
  try {
398
514
  const info = statSync(path)
@@ -424,6 +540,7 @@ export class ContentIndex {
424
540
  this.dbIdentity = null
425
541
  this.stopWorker()
426
542
  this.totalBytes = 0
543
+ this.sessionPaths.clear()
427
544
  this.state.updatedAt = 0
428
545
  this.state.error = ''
429
546
  this.state.truncated = false
@@ -439,20 +556,6 @@ export class ContentIndex {
439
556
  return scanning !== undefined && scanning !== null
440
557
  }
441
558
 
442
- /** Whether `path` is a SQLite database this plugin created (has docs_meta). */
443
- private ownedDatabase(path: string): boolean {
444
- let probe: DatabaseSync | undefined
445
- try {
446
- probe = new DatabaseSync(path, { readOnly: true })
447
- const row = probe.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'docs_meta'").get()
448
- return row !== undefined
449
- } catch {
450
- return false
451
- } finally {
452
- try { probe?.close() } catch { /* ignore */ }
453
- }
454
- }
455
-
456
559
  private readFlag(key: string): number | undefined {
457
560
  const row = this.db!.prepare('SELECT value FROM content_meta WHERE key = ?').get(key) as { value?: number } | undefined
458
561
  return row?.value === undefined ? undefined : Number(row.value)
@@ -502,12 +605,39 @@ export class ContentIndex {
502
605
  this.state.updatedAt = Date.now()
503
606
  this.writeTotalBytes()
504
607
  this.writeFlag('complete', 1)
608
+ this.refreshSessionPaths()
505
609
  this.reopenWorker()
506
610
  }
507
611
 
612
+ /**
613
+ * Rebuild the session id → session log path map from the persisted sources.
614
+ * Runs after `open()` and after every build/refresh, so session-scoped
615
+ * searches work against an existing database before the first refresh ends.
616
+ */
617
+ private refreshSessionPaths(): void {
618
+ this.sessionPaths.clear()
619
+ if (this.db === null) return
620
+ try {
621
+ const rows = this.db.prepare('SELECT path FROM sources').all() as Array<{ path?: string }>
622
+ for (const row of rows) {
623
+ const path = row.path
624
+ if (typeof path !== 'string' || path === '') continue
625
+ const sessionId = basename(dirname(path))
626
+ if (sessionId !== '') this.sessionPaths.set(sessionId, path)
627
+ }
628
+ } catch {
629
+ // A database from an older schema cannot answer session-scoped queries
630
+ // until its next rebuild; leave the map empty instead of failing boot.
631
+ }
632
+ }
633
+
508
634
  private addDoc(row: DocRow): boolean {
509
635
  const cap = this.options.maxTotalBytes ?? 0
510
- if (cap > 0 && this.totalBytes + row.body.length > cap) {
636
+ // Replacing an existing document frees its old body first: an unchanged
637
+ // file rewritten at refresh time must not be rejected because its stale
638
+ // copy still occupies the budget.
639
+ const previous = this.storedBytes(row.key)
640
+ if (cap > 0 && this.totalBytes - previous + row.body.length > cap) {
511
641
  this.state.truncated = true
512
642
  return false
513
643
  }
@@ -519,6 +649,13 @@ export class ContentIndex {
519
649
  return true
520
650
  }
521
651
 
652
+ /** Body bytes stored for one document key (0 when it is not indexed). */
653
+ private storedBytes(key: string): number {
654
+ const row = this.db!.prepare(`SELECT COALESCE(length(d.body), 0) AS n
655
+ FROM docs_meta m JOIN docs d ON d.rowid = m.rowid WHERE m.doc_key = ?`).get(key) as { n?: number } | undefined
656
+ return Number(row?.n ?? 0)
657
+ }
658
+
522
659
  private deleteDoc(key: string): void {
523
660
  const row = this.db!.prepare('SELECT rowid FROM docs_meta WHERE doc_key = ?').get(key) as { rowid: number } | undefined
524
661
  if (row === undefined) return
@@ -544,6 +681,7 @@ export class ContentIndex {
544
681
  let after = 0
545
682
  let seen = 0
546
683
  let scanned = 0
684
+ let capped = false
547
685
  for (;;) {
548
686
  if (generation !== this.generation) return
549
687
  if (this.engineRescanning()) throw new Error('engine re-scan started during content build')
@@ -553,13 +691,14 @@ export class ContentIndex {
553
691
  if (generation !== this.generation) return
554
692
  if (entry.dir || entry.size <= 0 || entry.size > this.options.maxFileBytes) continue
555
693
  if (!TEXT_EXTENSIONS.has(extname(entry.name).toLowerCase()) || contentSkipPath(entry.path, this.options.excludes ?? [])) continue
556
- if (this.options.maxFiles > 0 && seen >= this.options.maxFiles) break
694
+ if (this.options.maxFiles > 0 && seen >= this.options.maxFiles) { capped = true; break }
557
695
  const row = await this.readFileDoc(entry.path, entry.name, entry.mtime)
558
696
  if (row === null) continue
559
697
  seen++
560
698
  if (!this.addDoc(row)) return
561
699
  await this.breathe()
562
700
  }
701
+ if (capped) break
563
702
  after = page.next
564
703
  if (page.done || page.entries.length === 0) break
565
704
  }
@@ -571,8 +710,13 @@ export class ContentIndex {
571
710
  const seen = new Set<string>()
572
711
  const deferred: Array<{ path: string; name: string; mtime: number; size: number }> = []
573
712
  let after = 0
713
+ // `maxFiles` counts documents that are actually stored. A file whose read
714
+ // failed is not stored, so it must not consume a slot: counting it here used
715
+ // to stop the loop early (and the deletion pass then removed every trailing
716
+ // document that was never reached). `indexed` only moves on success.
574
717
  let indexed = 0
575
718
  let scanned = 0
719
+ let capped = false
576
720
  for (;;) {
577
721
  if (generation !== this.generation) return
578
722
  if (this.engineRescanning()) throw new Error('engine re-scan started during content build')
@@ -582,22 +726,30 @@ export class ContentIndex {
582
726
  if (generation !== this.generation) return
583
727
  if (entry.dir || entry.size <= 0 || entry.size > this.options.maxFileBytes) continue
584
728
  if (!TEXT_EXTENSIONS.has(extname(entry.name).toLowerCase()) || contentSkipPath(entry.path, this.options.excludes ?? [])) continue
585
- if (this.options.maxFiles > 0 && indexed >= this.options.maxFiles) break
729
+ if (this.options.maxFiles > 0 && indexed >= this.options.maxFiles) { capped = true; break }
586
730
  const key = `file:${entry.path}`
587
731
  seen.add(key)
588
- indexed++
589
- if (meta.get(key) === entry.mtime) continue
590
- if (this.wouldExceed(entry.size)) {
732
+ if (meta.get(key) === entry.mtime) {
733
+ // Unchanged and already stored: it still occupies one of the slots.
734
+ indexed++
735
+ continue
736
+ }
737
+ if (this.wouldExceed(entry.size, key)) {
591
738
  deferred.push({ path: entry.path, name: entry.name, mtime: entry.mtime, size: entry.size })
592
739
  continue
593
740
  }
594
- this.deleteDoc(key)
741
+ // Read first, replace only on success. A transient read failure must
742
+ // keep the document indexed by the previous refresh, not delete it.
595
743
  const row = await this.readFileDoc(entry.path, entry.name, entry.mtime)
596
- if (row !== null && !this.addDoc(row)) {
744
+ if (row === null) continue
745
+ if (!this.addDoc(row)) {
597
746
  deferred.push({ path: entry.path, name: entry.name, mtime: entry.mtime, size: entry.size })
747
+ continue
598
748
  }
749
+ indexed++
599
750
  await this.breathe()
600
751
  }
752
+ if (capped) break
601
753
  after = page.next
602
754
  if (page.done || page.entries.length === 0) break
603
755
  }
@@ -606,19 +758,20 @@ export class ContentIndex {
606
758
  for (const key of meta.keys()) if (!seen.has(key)) this.deleteDoc(key)
607
759
  for (const entry of deferred) {
608
760
  if (generation !== this.generation) return
609
- if (this.wouldExceed(entry.size)) { this.state.truncated = true; continue }
610
- this.deleteDoc(`file:${entry.path}`)
761
+ if (this.wouldExceed(entry.size, `file:${entry.path}`)) { this.state.truncated = true; continue }
611
762
  const row = await this.readFileDoc(entry.path, entry.name, entry.mtime)
612
- if (row === null || !this.addDoc(row)) { this.state.truncated = true; continue }
763
+ if (row === null) continue
764
+ if (!this.addDoc(row)) { this.state.truncated = true; continue }
613
765
  await this.breathe()
614
766
  }
615
767
  }
616
768
 
617
769
  /** Would storing a file of this size overflow the configured body budget? */
618
- private wouldExceed(size: number): boolean {
770
+ private wouldExceed(size: number, docKey?: string): boolean {
619
771
  const cap = this.options.maxTotalBytes ?? 0
620
772
  if (cap <= 0) return false
621
- return this.totalBytes + Math.min(size, this.options.maxFileBytes) > cap
773
+ const previous = docKey === undefined ? 0 : this.storedBytes(docKey)
774
+ return this.totalBytes - previous + Math.min(size, this.options.maxFileBytes) > cap
622
775
  }
623
776
 
624
777
  private async readFileDoc(path: string, name: string, mtime: number): Promise<DocRow | null> {
@@ -633,8 +786,15 @@ export class ContentIndex {
633
786
  const files = (await this.sessionCandidates()).filter(file => this.settled(file.mtime))
634
787
  for (const file of files) {
635
788
  if (generation !== this.generation) return
789
+ const read = await this.readSessionText(file)
790
+ if (!read.ok) {
791
+ // A log that cannot be read or decompressed does not contribute rows;
792
+ // the build continues with the remaining sessions.
793
+ this.addWarning('content-session-log-unreadable')
794
+ continue
795
+ }
636
796
  this.deletePath(file.path)
637
- if (!(await this.indexSessionFile(file, generation))) return
797
+ if (!(await this.indexSessionText(file, read.text, generation))) return
638
798
  this.db!.prepare('INSERT OR REPLACE INTO sources (path, mtime) VALUES (?, ?)').run(file.path, file.mtime)
639
799
  await this.breathe()
640
800
  }
@@ -652,8 +812,16 @@ export class ContentIndex {
652
812
  // its existing rows instead of re-reading the whole growing log.
653
813
  if (!this.settled(file.mtime)) continue
654
814
  if (known.get(file.path) === file.mtime) continue
815
+ // Decode the whole log before touching the old rows. On failure keep the
816
+ // previous messages and the previous sources.mtime, so the next refresh
817
+ // retries the same file instead of treating it as empty.
818
+ const read = await this.readSessionText(file)
819
+ if (!read.ok) {
820
+ this.addWarning('content-session-log-unreadable')
821
+ continue
822
+ }
655
823
  this.deletePath(file.path)
656
- if (!(await this.indexSessionFile(file, generation))) return
824
+ if (!(await this.indexSessionText(file, read.text, generation))) return
657
825
  this.db!.prepare('INSERT OR REPLACE INTO sources (path, mtime) VALUES (?, ?)').run(file.path, file.mtime)
658
826
  await this.breathe()
659
827
  }
@@ -664,11 +832,24 @@ export class ContentIndex {
664
832
  }
665
833
  }
666
834
 
667
- private async indexSessionFile(file: { path: string; mtime: number }, generation: number): Promise<boolean> {
835
+ /** Read and decompress one session log completely; no side effects on rows. */
836
+ private async readSessionText(file: { path: string; mtime: number }): Promise<
837
+ { ok: true; text: string } | { ok: false; error: string }
838
+ > {
668
839
  let text: string
669
- try { text = (await zstdDecompressAsync(await readFile(file.path))).toString('utf8') } catch { return true }
840
+ try {
841
+ text = (await zstdDecompressAsync(await readFile(file.path))).toString('utf8')
842
+ } catch (error) {
843
+ return { ok: false, error: messageOf(error) }
844
+ }
845
+ return { ok: true, text }
846
+ }
847
+
848
+ /** Index one already-decoded session log; the caller owns row replacement. */
849
+ private async indexSessionText(file: { path: string; mtime: number }, text: string, generation: number): Promise<boolean> {
670
850
  if (generation !== this.generation) return false
671
851
  const sessionId = basename(dirname(file.path))
852
+ this.sessionPaths.set(sessionId, file.path)
672
853
  let title = ''
673
854
  let parsed = 0
674
855
  for (const line of text.split('\n')) {
@@ -710,49 +891,144 @@ export class ContentIndex {
710
891
 
711
892
  /**
712
893
  * Search file bodies and session messages. With `searchWorker` enabled the
713
- * query itself runs off the host thread; a failed or wedged worker falls back
714
- * to the in-process query so a result is still produced.
894
+ * query itself runs off the host thread.
895
+ *
896
+ * A worker timeout, crash or malformed reply is a hard error result: the
897
+ * same query is never re-run on the host thread, where a heavy query (for
898
+ * example a whole-word scan) would stall the DSH event loop. Only when the
899
+ * worker cannot be created at all does the query run in-process, and that
900
+ * fallback is budgeted (bounded counts).
901
+ *
902
+ * The two-argument form returns the hit array (legacy callers). The
903
+ * three-argument form returns hits plus count, warnings and an error code,
904
+ * and understands the advanced match options.
715
905
  * @param query - words or a phrase to find.
716
906
  * @param limit - maximum hits.
907
+ * @param options - match mode, case/whole-word flags, session scope, count mode.
717
908
  * @returns matching documents ordered by relevance.
718
909
  */
719
- async search(query: string, limit: number): Promise<ContentHit[]> {
720
- if (this.db === null) return []
910
+ async search(query: string, limit: number): Promise<ContentHit[]>
911
+ async search(query: string, limit: number, options: ContentSearchOptions): Promise<ContentSearchResult>
912
+ async search(query: string, limit: number, options?: ContentSearchOptions): Promise<ContentHit[] | ContentSearchResult> {
913
+ const result = await this.searchDetailed(query, limit, options ?? {})
914
+ return options === undefined ? result.hits : result
915
+ }
916
+
917
+ private async searchDetailed(query: string, limit: number, options: ContentSearchOptions): Promise<ContentSearchResult> {
918
+ const resolved = this.resolveSession(options)
919
+ if (resolved === null) return emptySearchResult(['unknown-session'])
920
+ if (this.db === null) return emptySearchResult()
721
921
  if (!this.databaseIsCurrent()) {
722
922
  this.recoverExternal()
723
- return []
923
+ return emptySearchResult()
724
924
  }
725
- if (this.options.searchWorker !== true) return this.searchSync(query, limit)
726
- this.ensureWorker()
727
- if (this.worker === null) return this.searchSync(query, limit)
728
- const id = ++this.workerSeq
925
+ // Direct embedders run in-process by design (searchWorker is off).
926
+ if (this.options.searchWorker !== true) return this.searchSync(query, limit, resolved, false)
927
+ if (!this.ensureWorker()) {
928
+ // The worker could not be created at all: the one allowed sync path.
929
+ return this.searchSync(query, limit, resolved, true)
930
+ }
931
+ return await this.searchInWorker(query, limit, resolved)
932
+ }
933
+
934
+ /** Run one query on the worker; timeout/protocol errors are final results. */
935
+ private async searchInWorker(query: string, limit: number, options: ContentSearchOptions): Promise<ContentSearchResult> {
729
936
  const worker = this.worker
730
- const response = new Promise<ContentHit[]>((resolve, reject) => { this.pending.set(id, { resolve, reject }) })
731
- worker.postMessage({ id, query, limit })
937
+ if (worker === null) return workerFailureResult('content-search-worker-failed')
938
+ const id = ++this.workerSeq
939
+ const response = new Promise<ContentSearchResult>(resolve => { this.pending.set(id, resolve) })
940
+ const timeoutMs = Math.max(1, this.options.searchWorkerTimeoutMs ?? WORKER_TIMEOUT_MS)
732
941
  let timer: NodeJS.Timeout | undefined
733
- const timeout = new Promise<ContentHit[]>((_resolve, reject) => {
734
- timer = setTimeout(() => { this.pending.delete(id); reject(new Error('content search worker timed out')) }, 5_000)
942
+ const timeout = new Promise<ContentSearchResult>(resolve => {
943
+ timer = setTimeout(() => {
944
+ if (!this.pending.delete(id)) return
945
+ // A wedged worker is thrown away; the next query gets a fresh one. The
946
+ // current query is reported as timed out, never re-run here.
947
+ if (this.worker === worker) {
948
+ this.worker = null
949
+ void worker.terminate()
950
+ }
951
+ resolve(workerFailureResult('content-search-timeout'))
952
+ }, timeoutMs)
735
953
  })
736
954
  try {
737
- return await Promise.race([response, timeout])
955
+ worker.postMessage({ id, query, limit, options })
738
956
  } catch {
739
- return this.searchSync(query, limit)
957
+ if (this.pending.delete(id)) {
958
+ if (timer !== undefined) clearTimeout(timer)
959
+ return workerFailureResult('content-search-worker-failed')
960
+ }
961
+ }
962
+ try {
963
+ return await Promise.race([response, timeout])
740
964
  } finally {
741
965
  if (timer !== undefined) clearTimeout(timer)
742
966
  }
743
967
  }
744
968
 
745
- private searchSync(query: string, limit: number): ContentHit[] {
746
- if (this.db === null) return []
969
+ /** Map a request's sessionId onto the indexed session log path. */
970
+ private resolveSession(options: ContentSearchOptions): ContentSearchOptions | null {
971
+ if (options.sessionPath !== undefined && options.sessionPath !== null) return options
972
+ if (options.sessionId === undefined || options.sessionId === '') return { ...options, sessionPath: null }
973
+ const path = this.sessionPaths.get(options.sessionId)
974
+ if (path === undefined) return null
975
+ return { ...options, sessionPath: path }
976
+ }
977
+
978
+ /**
979
+ * In-process query path. When `degraded` is set (worker creation failed) the
980
+ * options are tightened so this rare path cannot spend an unbounded amount of
981
+ * host-thread time counting matches.
982
+ */
983
+ private searchSync(query: string, limit: number, options: ContentSearchOptions, degraded: boolean): ContentSearchResult {
984
+ if (this.db === null) return emptySearchResult()
985
+ const effective = degraded ? budgetedSearchOptions(options) : options
747
986
  try {
748
- return searchContent(this.db, query, limit)
987
+ const result = searchContent(this.db, query, limit, effective)
988
+ if (!degraded) return result
989
+ return { ...result, warnings: [...new Set(['content-search-in-process', ...result.warnings])] }
749
990
  } catch (error) {
750
991
  this.state.error = messageOf(error)
751
- return []
992
+ return emptySearchResult()
752
993
  }
753
994
  }
754
995
  }
755
996
 
997
+ /** Default deadline for one worker query. */
998
+ const WORKER_TIMEOUT_MS = 5_000
999
+ /** Counting cap used by the rare in-process fallback. */
1000
+ const IN_PROCESS_COUNT_LIMIT = 200
1001
+
1002
+ /** Tighten a query for the in-process fallback: bounded counting only. */
1003
+ function budgetedSearchOptions(options: ContentSearchOptions): ContentSearchOptions {
1004
+ return {
1005
+ ...options,
1006
+ count: options.count === 'none' ? 'none' : 'bounded',
1007
+ countLimit: Math.min(options.countLimit ?? IN_PROCESS_COUNT_LIMIT, IN_PROCESS_COUNT_LIMIT),
1008
+ }
1009
+ }
1010
+
1011
+ /** Whether a value looks like a complete worker search result. */
1012
+ function isContentSearchResult(value: unknown): value is ContentSearchResult {
1013
+ if (typeof value !== 'object' || value === null) return false
1014
+ const record = value as Record<string, unknown>
1015
+ if (!Array.isArray(record.hits) || !Array.isArray(record.warnings)) return false
1016
+ const count = record.count
1017
+ if (typeof count !== 'object' || count === null) return false
1018
+ const kind = (count as { kind?: unknown }).kind
1019
+ return kind === 'exact' || kind === 'bounded' || kind === 'unknown'
1020
+ }
1021
+
1022
+ /** Result for a failed worker query; carries an explicit error code. */
1023
+ function workerFailureResult(error: string): ContentSearchResult {
1024
+ return { hits: [], count: { value: 0, kind: 'unknown' }, warnings: [error], error }
1025
+ }
1026
+
1027
+ /** Result used when a content query could not run at all. */
1028
+ function emptySearchResult(warnings: string[] = []): ContentSearchResult {
1029
+ return { hits: [], count: { value: 0, kind: 'unknown' }, warnings }
1030
+ }
1031
+
756
1032
  function dshHome(): string {
757
1033
  return process.env.DSH_HOME || join(homedir(), '.dsh')
758
1034
  }