dsh-locator 0.7.1 → 0.7.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/CHANGELOG.md +37 -0
  2. package/README-es.md +37 -5
  3. package/README-hi.md +35 -5
  4. package/README-pt.md +33 -5
  5. package/README.en.md +34 -4
  6. package/README.md +27 -5
  7. package/client.js +117 -17
  8. package/cordis.patch.yml +13 -0
  9. package/lib/config.d.ts +20 -0
  10. package/lib/config.d.ts.map +1 -1
  11. package/lib/config.js +38 -4
  12. package/lib/config.js.map +1 -1
  13. package/lib/content-search-worker.d.ts +8 -0
  14. package/lib/content-search-worker.d.ts.map +1 -0
  15. package/lib/content-search-worker.js +54 -0
  16. package/lib/content-search-worker.js.map +1 -0
  17. package/lib/content-search.d.ts +15 -0
  18. package/lib/content-search.d.ts.map +1 -0
  19. package/lib/content-search.js +44 -0
  20. package/lib/content-search.js.map +1 -0
  21. package/lib/content.d.ts +83 -4
  22. package/lib/content.d.ts.map +1 -1
  23. package/lib/content.js +496 -77
  24. package/lib/content.js.map +1 -1
  25. package/lib/find.d.ts +1 -1
  26. package/lib/find.d.ts.map +1 -1
  27. package/lib/find.js +8 -3
  28. package/lib/find.js.map +1 -1
  29. package/lib/http.d.ts +1 -1
  30. package/lib/http.d.ts.map +1 -1
  31. package/lib/http.js +4 -2
  32. package/lib/http.js.map +1 -1
  33. package/lib/index.d.ts +5 -1
  34. package/lib/index.d.ts.map +1 -1
  35. package/lib/index.js +106 -27
  36. package/lib/index.js.map +1 -1
  37. package/lib/reindex.d.ts +1 -1
  38. package/lib/reindex.d.ts.map +1 -1
  39. package/lib/reindex.js +2 -1
  40. package/lib/reindex.js.map +1 -1
  41. package/package.json +1 -1
  42. package/src/config.ts +58 -4
  43. package/src/content-search-worker.ts +52 -0
  44. package/src/content-search.ts +45 -0
  45. package/src/content.ts +456 -71
  46. package/src/find.ts +8 -3
  47. package/src/http.ts +4 -2
  48. package/src/index.ts +106 -25
  49. package/src/reindex.ts +2 -1
package/src/content.ts CHANGED
@@ -6,16 +6,49 @@
6
6
  * `session.v4.jsonl.zstd`, and SQLite's trigram tokenizer gives substring and
7
7
  * CJK search. A meta table stores each source's mtime so refresh() only
8
8
  * re-reads what changed; rebuild() drops and rebuilds everything.
9
+ *
10
+ * Everything here runs inside the DSH host process, so the build must never
11
+ * monopolise its event loop: file reads and zstd decoding are asynchronous and
12
+ * the loops yield to the scheduler every ~8 ms of work. Session logs are read
13
+ * only once they have settled (an actively written session is skipped until it
14
+ * has been quiet for `activeSessionDelayMs`), and archived sessions are left
15
+ * alone unless `includeArchived` is set.
9
16
  * @module dsh-locator/content
10
17
  */
11
18
 
12
- import { existsSync, mkdirSync, readdirSync, readFileSync, statSync } from 'node:fs'
19
+ import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
20
+ import { readdir, readFile, stat } from 'node:fs/promises'
13
21
  import { homedir } from 'node:os'
14
22
  import { basename, dirname, extname, join } from 'node:path'
15
- import { zstdDecompressSync } from 'node:zlib'
23
+ import { promisify } from 'node:util'
24
+ import { zstdDecompress } from 'node:zlib'
25
+ import { scheduler } from 'node:timers/promises'
16
26
  import { DatabaseSync } from 'node:sqlite'
27
+ import { randomUUID } from 'node:crypto'
28
+ import { fileURLToPath } from 'node:url'
29
+ import { Worker } from 'node:worker_threads'
30
+ import { searchContent } from './content-search.ts'
17
31
  import type { LocatorRuntime } from './runtime.ts'
18
32
 
33
+ const SCHEMA_VERSION = 2
34
+
35
+ /** Work budget between two event-loop yields; keeps host latency low. */
36
+ const SLICE_MS = 8
37
+
38
+ /** Wall-clock start of this process; distinguishes a reused pid from a live sibling. */
39
+ const PROCESS_START = Math.round(Date.now() - process.uptime() * 1000)
40
+
41
+ /** Session bodies are short: a message, not a whole file. */
42
+ const SESSION_BODY_BYTES = 8_000
43
+
44
+ const zstdDecompressAsync = promisify(zstdDecompress)
45
+
46
+ // `scheduler.yield` is newer than the oldest supported Node line; fall back to
47
+ // setImmediate so a build on Node 22 still yields instead of throwing.
48
+ const yieldToLoop: () => Promise<void> = typeof (scheduler as { yield?: unknown }).yield === 'function'
49
+ ? () => scheduler.yield()
50
+ : () => new Promise<void>(resolve => { setImmediate(resolve) })
51
+
19
52
  const TEXT_EXTENSIONS = new Set([
20
53
  '.txt', '.md', '.markdown', '.rst', '.log', '.csv', '.tsv', '.json', '.jsonl',
21
54
  '.yaml', '.yml', '.toml', '.ini', '.cfg', '.conf', '.xml', '.html', '.htm',
@@ -48,10 +81,25 @@ export interface ContentState {
48
81
  export interface ContentOptions {
49
82
  stateDir: string
50
83
  sessionDirs?: string[] | undefined
84
+ /** Path segments or substrings never read into the content index. */
85
+ excludes?: string[]
86
+ /** Index archived sessions too; false (the default) leaves them out. */
87
+ includeArchived?: boolean
88
+ /**
89
+ * Skip session logs modified more recently than this many milliseconds, so
90
+ * the session being written right now is not re-read on every refresh.
91
+ */
92
+ activeSessionDelayMs?: number
51
93
  maxFileBytes: number
52
94
  maxTotalBytes?: number
53
95
  maxFiles: number
54
96
  maxSessions: number
97
+ /**
98
+ * Run content queries in a worker thread so a slow search never blocks the
99
+ * DSH host event loop. Off by default so direct embedders and tests stay
100
+ * single-threaded.
101
+ */
102
+ searchWorker?: boolean
55
103
  }
56
104
 
57
105
  interface DocRow {
@@ -70,16 +118,34 @@ export class ContentIndex {
70
118
  private readonly state: ContentState = { building: false, files: 0, sessions: 0, messages: 0, truncated: false, error: '', updatedAt: 0 }
71
119
  private generation = 0
72
120
  private totalBytes = 0
73
- private timer: NodeJS.Timeout | undefined
121
+ private sliceDeadline = 0
122
+ private worker: Worker | null = null
123
+ private lockHeld = false
124
+ private readonly lockToken = randomUUID()
125
+ private workerSeq = 0
126
+ private pendingStart = false
127
+ private readonly pending = new Map<number, { resolve: (hits: ContentHit[]) => void; reject: (error: Error) => void }>()
74
128
 
75
129
  constructor(private readonly rt: LocatorRuntime, private readonly options: ContentOptions) {}
76
130
 
77
- status(): ContentState { return { ...this.state } }
131
+ status(): ContentState {
132
+ return { ...this.state, building: this.state.building || this.pendingStart }
133
+ }
134
+
135
+ /**
136
+ * Report a lazy build as in progress before the asynchronous start reaches
137
+ * `boot()`, so the panel/agent sees "building" on the very first query
138
+ * instead of an empty result with no hint.
139
+ */
140
+ markPending(): void {
141
+ if (this.state.building) return
142
+ this.state.error = ''
143
+ this.pendingStart = true
144
+ }
78
145
 
79
- /** Start periodic incremental refreshes; 0 disables. */
80
- startAutoRefresh(ms: number): void {
81
- if (ms <= 0 || this.timer !== undefined) return
82
- this.timer = setInterval(() => this.refresh(), ms)
146
+ /** Drop the pending flag when the lazy start failed before it began. */
147
+ clearPending(): void {
148
+ this.pendingStart = false
83
149
  }
84
150
 
85
151
  /** Queue a full rebuild; false when one is already running. */
@@ -97,21 +163,84 @@ export class ContentIndex {
97
163
  }
98
164
 
99
165
  close(): void {
100
- if (this.timer !== undefined) { clearInterval(this.timer); this.timer = undefined }
101
166
  this.generation++
102
167
  this.db?.close()
103
168
  this.db = null
104
169
  this.state.building = false
170
+ this.stopWorker()
171
+ this.releaseLock()
172
+ this.pendingStart = false
105
173
  }
106
174
 
107
175
  /** Build everything from scratch. */
108
176
  start(): void { this.begin('full') }
109
177
 
178
+ /**
179
+ * Start at plugin boot: reuse an existing database and refresh it
180
+ * incrementally, so a restart does not re-read and re-index gigabytes of
181
+ * text. A missing or incompatible database falls back to a full build.
182
+ */
183
+ boot(): void {
184
+ // A content operation that ran before the deferred boot may have built the
185
+ // database already; do not start a second pass behind it.
186
+ if (this.state.updatedAt > 0) return
187
+ this.begin(existsSync(join(this.options.stateDir, 'content.db')) ? 'incremental' : 'full')
188
+ }
189
+
110
190
  /** Re-read changed files/sessions; open the DB when this is the first run. */
111
191
  refresh(): void { this.begin(this.db === null ? 'full' : 'incremental') }
112
192
 
193
+ /**
194
+ * Serialize content builds across plugin instances and DSH processes that
195
+ * share a state directory: a stale lock (dead pid or older than 30 minutes)
196
+ * is taken over, a live one keeps this instance from writing concurrently.
197
+ * @returns whether this instance owns the build lock.
198
+ */
199
+ private acquireLock(): boolean {
200
+ if (this.lockHeld) return true
201
+ mkdirSync(this.options.stateDir, { recursive: true, mode: 0o700 })
202
+ const path = join(this.options.stateDir, 'content.lock')
203
+ for (let attempt = 0; attempt < 2; attempt++) {
204
+ try {
205
+ const fd = openSync(path, 'wx', 0o600)
206
+ writeFileSync(fd, JSON.stringify({ pid: process.pid, startedAt: Date.now(), token: this.lockToken, procStart: PROCESS_START }))
207
+ closeSync(fd)
208
+ this.lockHeld = true
209
+ return true
210
+ } catch {
211
+ let holder: { pid?: number; startedAt?: number; token?: string; procStart?: number } = {}
212
+ try { holder = JSON.parse(readFileSync(path, 'utf8')) as typeof holder } catch { /* unreadable: treat as stale */ }
213
+ const age = Date.now() - Number(holder.startedAt ?? 0)
214
+ let alive = false
215
+ if (typeof holder.pid === 'number' && age < 30 * 60_000) {
216
+ try { process.kill(holder.pid, 0); alive = true } catch (error) {
217
+ alive = (error as NodeJS.ErrnoException).code === 'EPERM'
218
+ }
219
+ }
220
+ // A live sibling instance (for example during profile HMR) blocks even
221
+ // when it shares our pid; the per-instance token tells them apart. A
222
+ // reused pid from an earlier process incarnation is stale, not live.
223
+ const sameProcess = holder.procStart === undefined || Math.abs(Number(holder.procStart) - PROCESS_START) < 5_000
224
+ if (alive && sameProcess && holder.token !== this.lockToken) return false
225
+ try { rmSync(path, { force: true }) } catch { return false }
226
+ }
227
+ }
228
+ return false
229
+ }
230
+
231
+ private releaseLock(): void {
232
+ if (!this.lockHeld) return
233
+ this.lockHeld = false
234
+ try { rmSync(join(this.options.stateDir, 'content.lock'), { force: true }) } catch { /* ignore */ }
235
+ }
236
+
113
237
  private begin(mode: 'full' | 'incremental'): void {
114
238
  if (this.state.building) return
239
+ this.pendingStart = false
240
+ if (!this.acquireLock()) {
241
+ this.state.error = 'another dsh-locator content build is running'
242
+ return
243
+ }
115
244
  const generation = ++this.generation
116
245
  this.state.building = true
117
246
  this.state.error = ''
@@ -121,6 +250,8 @@ export class ContentIndex {
121
250
  if (generation !== this.generation) return
122
251
  this.state.error = messageOf(error)
123
252
  this.state.building = false
253
+ }).finally(() => {
254
+ this.releaseLock()
124
255
  })
125
256
  }
126
257
 
@@ -135,22 +266,60 @@ export class ContentIndex {
135
266
  }
136
267
 
137
268
  private async incremental(generation: number): Promise<void> {
138
- this.open(false)
139
- this.totalBytes = Number((this.db!.prepare('SELECT COALESCE(SUM(length(body)), 0) AS n FROM docs').get() as { n: number }).n)
140
- await this.refreshFiles(generation)
141
- if (generation !== this.generation) return
142
- await this.refreshSessions(generation)
143
- if (generation !== this.generation) return
144
- this.finish()
269
+ try {
270
+ this.open(false)
271
+ const stored = (this.db!.prepare('PRAGMA user_version').get() as { user_version?: number } | undefined)?.user_version ?? 0
272
+ if (Number(stored) !== SCHEMA_VERSION) return await this.full(generation)
273
+ // A database whose last build never finished (crash, update, forced
274
+ // reload) has a stale byte counter; rebuild instead of trusting it.
275
+ if (this.readFlag('complete') !== 1) return await this.full(generation)
276
+ this.totalBytes = this.readTotalBytes()
277
+ await this.refreshFiles(generation)
278
+ if (generation !== this.generation) return
279
+ await this.refreshSessions(generation)
280
+ if (generation !== this.generation) return
281
+ this.finish()
282
+ } catch {
283
+ if (generation !== this.generation) return
284
+ // A corrupt or outdated database must not disable content search for the
285
+ // rest of the session: rebuild it from scratch instead.
286
+ this.db?.close()
287
+ this.db = null
288
+ await this.full(generation)
289
+ }
145
290
  }
146
291
 
147
292
  private open(fresh: boolean): void {
148
- mkdirSync(this.options.stateDir, { recursive: true })
293
+ const existedDir = existsSync(this.options.stateDir)
294
+ mkdirSync(this.options.stateDir, { recursive: true, mode: 0o700 })
295
+ if (!existedDir) {
296
+ try { chmodSync(this.options.stateDir, 0o700) } catch { /* Windows or a read-only profile */ }
297
+ }
298
+ const dbPath = join(this.options.stateDir, 'content.db')
299
+ const existed = existsSync(dbPath)
300
+ if (existed && !this.ownedDatabase(dbPath)) {
301
+ throw new Error(`content.db is not a dsh-locator database, refusing to touch it: ${dbPath}`)
302
+ }
303
+ let oldFileRemained = false
304
+ if (fresh) {
305
+ // A full rebuild throws every row away. Dropping the FTS tables in place
306
+ // on a multi-GB database blocks the host for about a second, so recreate
307
+ // the file instead: the worker is stopped and the sidecars go with it.
308
+ this.stopWorker()
309
+ try { this.db?.close() } catch { /* ignore */ }
310
+ this.db = null
311
+ for (const suffix of ['', '-wal', '-shm']) {
312
+ try { rmSync(dbPath + suffix, { force: true }) } catch { /* Windows keeps a busy file; DROP below still works */ }
313
+ }
314
+ oldFileRemained = existsSync(dbPath)
315
+ }
149
316
  this.db?.close()
150
- this.db = new DatabaseSync(join(this.options.stateDir, 'content.db'))
317
+ this.db = new DatabaseSync(dbPath)
318
+ try { chmodSync(dbPath, 0o600) } catch { /* Windows or a read-only profile */ }
151
319
  this.db.exec('PRAGMA journal_mode = WAL')
152
320
  this.db.exec('PRAGMA synchronous = OFF')
153
- if (fresh) {
321
+ if (fresh && oldFileRemained) {
322
+ // Fallback for the rare recreate failure above: still start from empty.
154
323
  this.db.exec('DROP TABLE IF EXISTS docs')
155
324
  this.db.exec('DROP TABLE IF EXISTS docs_meta')
156
325
  this.db.exec('DROP TABLE IF EXISTS sources')
@@ -161,6 +330,114 @@ export class ContentIndex {
161
330
  )`)
162
331
  this.db.exec('CREATE TABLE IF NOT EXISTS docs_meta (rowid INTEGER PRIMARY KEY, doc_key TEXT UNIQUE, path TEXT, mtime INTEGER, kind TEXT)')
163
332
  this.db.exec('CREATE TABLE IF NOT EXISTS sources (path TEXT PRIMARY KEY, mtime INTEGER)')
333
+ this.db.exec('CREATE INDEX IF NOT EXISTS docs_meta_path ON docs_meta(path)')
334
+ if (fresh) {
335
+ this.db.exec('DROP TABLE IF EXISTS content_meta')
336
+ }
337
+ this.db.exec('CREATE TABLE IF NOT EXISTS content_meta (key TEXT PRIMARY KEY, value INTEGER)')
338
+ if (fresh) {
339
+ this.db.exec("INSERT OR REPLACE INTO content_meta (key, value) VALUES ('total_bytes', 0)")
340
+ this.db.exec("INSERT OR REPLACE INTO content_meta (key, value) VALUES ('complete', 0)")
341
+ }
342
+ if (fresh) this.db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`)
343
+ this.reopenWorker()
344
+ }
345
+
346
+ private reopenWorker(): void {
347
+ if (this.worker === null) return
348
+ this.worker.postMessage({ id: ++this.workerSeq, open: join(this.options.stateDir, 'content.db') })
349
+ }
350
+
351
+ private stopWorker(): void {
352
+ const worker = this.worker
353
+ this.worker = null
354
+ for (const entry of this.pending.values()) entry.resolve([])
355
+ this.pending.clear()
356
+ if (worker !== null) void worker.terminate()
357
+ }
358
+
359
+ private ensureWorker(): void {
360
+ if (this.options.searchWorker !== true || this.worker !== null) return
361
+ // In a TypeScript test run the compiled worker does not exist next to the
362
+ // source; skip spawning and answer queries in-process instead.
363
+ const workerPath = fileURLToPath(new URL('./content-search-worker.js', import.meta.url))
364
+ if (!existsSync(workerPath)) return
365
+ try {
366
+ const worker = new Worker(workerPath, {
367
+ workerData: { dbPath: join(this.options.stateDir, 'content.db') },
368
+ })
369
+ worker.on('message', (message: { id: number; hits?: ContentHit[] }) => {
370
+ const entry = this.pending.get(message.id)
371
+ if (entry === undefined) return
372
+ this.pending.delete(message.id)
373
+ entry.resolve(message.hits ?? [])
374
+ })
375
+ const fail = (): void => {
376
+ if (this.worker !== worker) return
377
+ this.worker = null
378
+ for (const entry of this.pending.values()) entry.reject(new Error('content search worker failed'))
379
+ this.pending.clear()
380
+ }
381
+ worker.on('error', fail)
382
+ worker.on('exit', fail)
383
+ this.worker = worker
384
+ } catch {
385
+ this.worker = null
386
+ }
387
+ }
388
+
389
+ /** Whether `path` is a SQLite database this plugin created (has docs_meta). */
390
+ private ownedDatabase(path: string): boolean {
391
+ let probe: DatabaseSync | undefined
392
+ try {
393
+ probe = new DatabaseSync(path, { readOnly: true })
394
+ const row = probe.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'docs_meta'").get()
395
+ return row !== undefined
396
+ } catch {
397
+ return false
398
+ } finally {
399
+ try { probe?.close() } catch { /* ignore */ }
400
+ }
401
+ }
402
+
403
+ private readFlag(key: string): number | undefined {
404
+ const row = this.db!.prepare('SELECT value FROM content_meta WHERE key = ?').get(key) as { value?: number } | undefined
405
+ return row?.value === undefined ? undefined : Number(row.value)
406
+ }
407
+
408
+ private writeFlag(key: string, value: number): void {
409
+ this.db!.prepare('INSERT OR REPLACE INTO content_meta (key, value) VALUES (?, ?)').run(key, value)
410
+ }
411
+
412
+ /**
413
+ * Body bytes already stored. Reading a persisted counter avoids a full scan
414
+ * of every FTS body (measured ~0.6 s for 1 GB) at the start of each refresh.
415
+ * A database written before the counter existed falls back to one scan.
416
+ */
417
+ private readTotalBytes(): number {
418
+ const row = this.db!.prepare("SELECT value FROM content_meta WHERE key = 'total_bytes'").get() as { value?: number } | undefined
419
+ if (row?.value !== undefined) return Number(row.value)
420
+ const scanned = Number((this.db!.prepare('SELECT COALESCE(SUM(length(body)), 0) AS n FROM docs').get() as { n: number }).n)
421
+ this.db!.prepare("INSERT OR REPLACE INTO content_meta (key, value) VALUES ('total_bytes', ?)").run(scanned)
422
+ return scanned
423
+ }
424
+
425
+ private writeTotalBytes(): void {
426
+ this.db!.prepare("INSERT OR REPLACE INTO content_meta (key, value) VALUES ('total_bytes', ?)").run(Math.max(0, Math.round(this.totalBytes)))
427
+ }
428
+
429
+ /** Yield to the host event loop once the current ~8 ms work slice is spent. */
430
+ private async breathe(): Promise<void> {
431
+ const now = performance.now()
432
+ if (now < this.sliceDeadline) return
433
+ this.sliceDeadline = now + SLICE_MS
434
+ await yieldToLoop()
435
+ }
436
+
437
+ /** True when a session log has been quiet long enough to be read safely. */
438
+ private settled(mtime: number): boolean {
439
+ const delay = this.options.activeSessionDelayMs ?? 0
440
+ return delay <= 0 || Date.now() - mtime >= delay
164
441
  }
165
442
 
166
443
  private finish(): void {
@@ -170,6 +447,9 @@ export class ContentIndex {
170
447
  this.state.sessions = Number((this.db!.prepare('SELECT COUNT(*) AS n FROM sources').get() as { n: number }).n)
171
448
  this.state.building = false
172
449
  this.state.updatedAt = Date.now()
450
+ this.writeTotalBytes()
451
+ this.writeFlag('complete', 1)
452
+ this.reopenWorker()
173
453
  }
174
454
 
175
455
  private addDoc(row: DocRow): boolean {
@@ -189,31 +469,42 @@ export class ContentIndex {
189
469
  private deleteDoc(key: string): void {
190
470
  const row = this.db!.prepare('SELECT rowid FROM docs_meta WHERE doc_key = ?').get(key) as { rowid: number } | undefined
191
471
  if (row === undefined) return
472
+ const body = this.db!.prepare('SELECT length(body) AS n FROM docs WHERE rowid = ?').get(row.rowid) as { n: number } | undefined
473
+ if (body !== undefined) this.totalBytes = Math.max(0, this.totalBytes - Number(body.n))
192
474
  this.db!.prepare('DELETE FROM docs WHERE rowid = ?').run(row.rowid)
193
475
  this.db!.prepare('DELETE FROM docs_meta WHERE rowid = ?').run(row.rowid)
194
476
  }
195
477
 
196
478
  private deletePath(path: string): void {
197
- const rows = this.db!.prepare('SELECT rowid FROM docs_meta WHERE path = ?').all(path) as Array<{ rowid: number }>
479
+ const rows = this.db!.prepare(`SELECT m.rowid AS rowid, length(d.body) AS n
480
+ FROM docs_meta m JOIN docs d ON d.rowid = m.rowid WHERE m.path = ?`).all(path) as Array<{ rowid: number; n: number }>
198
481
  const delDoc = this.db!.prepare('DELETE FROM docs WHERE rowid = ?')
199
482
  const delMeta = this.db!.prepare('DELETE FROM docs_meta WHERE rowid = ?')
200
- for (const row of rows) { delDoc.run(row.rowid); delMeta.run(row.rowid) }
483
+ for (const row of rows) {
484
+ this.totalBytes = Math.max(0, this.totalBytes - Number(row.n))
485
+ delDoc.run(row.rowid)
486
+ delMeta.run(row.rowid)
487
+ }
201
488
  }
202
489
 
203
490
  private async indexFiles(generation: number): Promise<void> {
204
491
  let after = 0
205
492
  let seen = 0
493
+ let scanned = 0
206
494
  for (;;) {
207
495
  if (generation !== this.generation) return
208
496
  const page = await this.rt.dump(after, 2_000)
209
497
  for (const entry of page.entries) {
498
+ if ((++scanned & 0xff) === 0) await this.breathe()
499
+ if (generation !== this.generation) return
210
500
  if (entry.dir || entry.size <= 0 || entry.size > this.options.maxFileBytes) continue
211
- if (!TEXT_EXTENSIONS.has(extname(entry.name).toLowerCase()) || contentSkipPath(entry.path)) continue
501
+ if (!TEXT_EXTENSIONS.has(extname(entry.name).toLowerCase()) || contentSkipPath(entry.path, this.options.excludes ?? [])) continue
212
502
  if (this.options.maxFiles > 0 && seen >= this.options.maxFiles) break
213
- const row = this.readFileDoc(entry.path, entry.name, entry.mtime)
503
+ const row = await this.readFileDoc(entry.path, entry.name, entry.mtime)
214
504
  if (row === null) continue
215
505
  seen++
216
506
  if (!this.addDoc(row)) return
507
+ await this.breathe()
217
508
  }
218
509
  after = page.next
219
510
  if (page.done || page.entries.length === 0) break
@@ -224,58 +515,92 @@ export class ContentIndex {
224
515
  const meta = new Map<string, number>()
225
516
  for (const row of this.db!.prepare("SELECT doc_key, mtime FROM docs_meta WHERE kind = 'file'").all() as Array<{ doc_key: string; mtime: number }>) meta.set(row.doc_key, row.mtime)
226
517
  const seen = new Set<string>()
518
+ const deferred: Array<{ path: string; name: string; mtime: number; size: number }> = []
227
519
  let after = 0
228
520
  let indexed = 0
521
+ let scanned = 0
229
522
  for (;;) {
230
523
  if (generation !== this.generation) return
231
524
  const page = await this.rt.dump(after, 2_000)
232
525
  for (const entry of page.entries) {
526
+ if ((++scanned & 0xff) === 0) await this.breathe()
527
+ if (generation !== this.generation) return
233
528
  if (entry.dir || entry.size <= 0 || entry.size > this.options.maxFileBytes) continue
234
- if (!TEXT_EXTENSIONS.has(extname(entry.name).toLowerCase()) || contentSkipPath(entry.path)) continue
529
+ if (!TEXT_EXTENSIONS.has(extname(entry.name).toLowerCase()) || contentSkipPath(entry.path, this.options.excludes ?? [])) continue
235
530
  if (this.options.maxFiles > 0 && indexed >= this.options.maxFiles) break
236
531
  const key = `file:${entry.path}`
237
532
  seen.add(key)
238
533
  indexed++
239
534
  if (meta.get(key) === entry.mtime) continue
535
+ if (this.wouldExceed(entry.size)) {
536
+ deferred.push({ path: entry.path, name: entry.name, mtime: entry.mtime, size: entry.size })
537
+ continue
538
+ }
240
539
  this.deleteDoc(key)
241
- const row = this.readFileDoc(entry.path, entry.name, entry.mtime)
242
- if (row !== null && !this.addDoc(row)) return
540
+ const row = await this.readFileDoc(entry.path, entry.name, entry.mtime)
541
+ if (row !== null && !this.addDoc(row)) {
542
+ deferred.push({ path: entry.path, name: entry.name, mtime: entry.mtime, size: entry.size })
543
+ }
544
+ await this.breathe()
243
545
  }
244
546
  after = page.next
245
547
  if (page.done || page.entries.length === 0) break
246
548
  }
549
+ // Removing rows for files that disappeared frees budget before the deferred
550
+ // files are retried; otherwise a stale large entry would block new ones.
247
551
  for (const key of meta.keys()) if (!seen.has(key)) this.deleteDoc(key)
552
+ for (const entry of deferred) {
553
+ if (generation !== this.generation) return
554
+ if (this.wouldExceed(entry.size)) { this.state.truncated = true; continue }
555
+ this.deleteDoc(`file:${entry.path}`)
556
+ const row = await this.readFileDoc(entry.path, entry.name, entry.mtime)
557
+ if (row === null || !this.addDoc(row)) { this.state.truncated = true; continue }
558
+ await this.breathe()
559
+ }
248
560
  }
249
561
 
250
- private readFileDoc(path: string, name: string, mtime: number): DocRow | null {
251
- let raw: string
252
- try { raw = readFileSync(path, 'utf8') } catch { return null }
253
- if (raw.includes('\u0000')) return null
254
- return { key: `file:${path}`, body: raw.slice(0, this.options.maxFileBytes), kind: 'file', path, title: name, sessionId: null, messageId: null, mtime }
562
+ /** Would storing a file of this size overflow the configured body budget? */
563
+ private wouldExceed(size: number): boolean {
564
+ const cap = this.options.maxTotalBytes ?? 0
565
+ if (cap <= 0) return false
566
+ return this.totalBytes + Math.min(size, this.options.maxFileBytes) > cap
567
+ }
568
+
569
+ private async readFileDoc(path: string, name: string, mtime: number): Promise<DocRow | null> {
570
+ let buffer: Buffer
571
+ try { buffer = await readFile(path) } catch { return null }
572
+ if (buffer.includes(0)) return null
573
+ const body = buffer.toString('utf8').slice(0, this.options.maxFileBytes)
574
+ return { key: `file:${path}`, body, kind: 'file', path, title: name, sessionId: null, messageId: null, mtime }
255
575
  }
256
576
 
257
577
  private async indexSessions(generation: number): Promise<void> {
258
- const files = sessionFiles(this.sessionRoots()).slice(0, this.options.maxSessions > 0 ? this.options.maxSessions : undefined)
578
+ const files = (await this.sessionCandidates()).filter(file => this.settled(file.mtime))
259
579
  for (const file of files) {
260
580
  if (generation !== this.generation) return
261
581
  this.deletePath(file.path)
262
- if (!this.indexSessionFile(file)) return
582
+ if (!(await this.indexSessionFile(file, generation))) return
263
583
  this.db!.prepare('INSERT OR REPLACE INTO sources (path, mtime) VALUES (?, ?)').run(file.path, file.mtime)
584
+ await this.breathe()
264
585
  }
265
586
  }
266
587
 
267
588
  private async refreshSessions(generation: number): Promise<void> {
268
589
  const known = new Map<string, number>()
269
590
  for (const row of this.db!.prepare('SELECT path, mtime FROM sources').all() as Array<{ path: string; mtime: number }>) known.set(row.path, row.mtime)
270
- const files = sessionFiles(this.sessionRoots()).slice(0, this.options.maxSessions > 0 ? this.options.maxSessions : undefined)
591
+ const files = await this.sessionCandidates()
271
592
  const present = new Set<string>()
272
593
  for (const file of files) {
273
594
  if (generation !== this.generation) return
274
595
  present.add(file.path)
596
+ // The session being written right now is left for a later refresh: keep
597
+ // its existing rows instead of re-reading the whole growing log.
598
+ if (!this.settled(file.mtime)) continue
275
599
  if (known.get(file.path) === file.mtime) continue
276
600
  this.deletePath(file.path)
277
- if (!this.indexSessionFile(file)) return
601
+ if (!(await this.indexSessionFile(file, generation))) return
278
602
  this.db!.prepare('INSERT OR REPLACE INTO sources (path, mtime) VALUES (?, ?)').run(file.path, file.mtime)
603
+ await this.breathe()
279
604
  }
280
605
  for (const path of known.keys()) {
281
606
  if (present.has(path)) continue
@@ -284,55 +609,84 @@ export class ContentIndex {
284
609
  }
285
610
  }
286
611
 
287
- private indexSessionFile(file: { path: string; mtime: number }): boolean {
612
+ private async indexSessionFile(file: { path: string; mtime: number }, generation: number): Promise<boolean> {
288
613
  let text: string
289
- try { text = zstdDecompressSync(readFileSync(file.path)).toString('utf8') } catch { return true }
614
+ try { text = (await zstdDecompressAsync(await readFile(file.path))).toString('utf8') } catch { return true }
615
+ if (generation !== this.generation) return false
290
616
  const sessionId = basename(dirname(file.path))
291
617
  let title = ''
618
+ let parsed = 0
292
619
  for (const line of text.split('\n')) {
293
620
  if (!line) continue
621
+ if ((++parsed & 0xff) === 0) {
622
+ await this.breathe()
623
+ if (generation !== this.generation) return false
624
+ }
294
625
  let event: { type?: string; data?: Record<string, unknown> }
295
626
  try { event = JSON.parse(line) } catch { continue }
296
627
  const extracted = messageText(event)
297
628
  if (extracted === undefined) continue
298
629
  if (title === '') title = extracted.text.replace(/\s+/g, ' ').slice(0, 90)
299
- const body = extracted.text.slice(0, 8_000)
630
+ const body = extracted.text.slice(0, SESSION_BODY_BYTES)
300
631
  const row: DocRow = { key: `session:${sessionId}:${extracted.id}`, body, kind: 'session', path: file.path, title: title || sessionId, sessionId, messageId: extracted.id, mtime: file.mtime }
301
632
  if (!this.addDoc(row)) return false
302
633
  }
303
634
  return true
304
635
  }
305
636
 
637
+ /** Session logs to consider, newest first, minus archived ones by default. */
638
+ private async sessionCandidates(): Promise<Array<{ path: string; mtime: number }>> {
639
+ const files = await sessionFiles(this.sessionRoots())
640
+ const archived = this.options.includeArchived === true ? new Set<string>() : await archivedSessionIds()
641
+ const max = this.options.maxSessions > 0 ? this.options.maxSessions : Number.POSITIVE_INFINITY
642
+ const out: Array<{ path: string; mtime: number }> = []
643
+ for (const file of files) {
644
+ if (out.length >= max) break
645
+ if (archived.has(basename(dirname(file.path)))) continue
646
+ out.push(file)
647
+ }
648
+ return out
649
+ }
650
+
306
651
  sessionRoots(): string[] {
307
652
  if (this.options.sessionDirs !== undefined) return this.options.sessionDirs
308
- return [join(process.env.DSH_HOME || join(homedir(), '.dsh'), 'sessions')]
653
+ return [join(dshHome(), 'sessions')]
309
654
  }
310
655
 
311
- search(query: string, limit: number): ContentHit[] {
656
+ /**
657
+ * Search file bodies and session messages. With `searchWorker` enabled the
658
+ * query itself runs off the host thread; a failed or wedged worker falls back
659
+ * to the in-process query so a result is still produced.
660
+ * @param query - words or a phrase to find.
661
+ * @param limit - maximum hits.
662
+ * @returns matching documents ordered by relevance.
663
+ */
664
+ async search(query: string, limit: number): Promise<ContentHit[]> {
312
665
  if (this.db === null) return []
313
- const text = query.trim()
314
- if (text === '') return []
315
- const capped = Math.max(1, Math.min(100, limit))
666
+ if (this.options.searchWorker !== true) return this.searchSync(query, limit)
667
+ this.ensureWorker()
668
+ if (this.worker === null) return this.searchSync(query, limit)
669
+ const id = ++this.workerSeq
670
+ const worker = this.worker
671
+ const response = new Promise<ContentHit[]>((resolve, reject) => { this.pending.set(id, { resolve, reject }) })
672
+ worker.postMessage({ id, query, limit })
673
+ let timer: NodeJS.Timeout | undefined
674
+ const timeout = new Promise<ContentHit[]>((_resolve, reject) => {
675
+ timer = setTimeout(() => { this.pending.delete(id); reject(new Error('content search worker timed out')) }, 5_000)
676
+ })
316
677
  try {
317
- let rows: Array<Record<string, unknown>>
318
- if ([...text].length < 3) {
319
- rows = this.db.prepare(`SELECT kind, path, title, session_id, message_id, body, 0 AS score
320
- FROM docs WHERE body LIKE '%' || ? || '%' LIMIT ?`).all(text, capped) as Array<Record<string, unknown>>
321
- } else {
322
- const match = text.split(/\s+/).filter(Boolean).map(term => '"' + term.replace(/"/g, '') + '"').join(' AND ')
323
- rows = this.db.prepare(`SELECT kind, path, title, session_id, message_id,
324
- snippet(docs, 0, char(1), char(2), '…', 16) AS snippet, bm25(docs) AS score
325
- FROM docs WHERE docs MATCH ? ORDER BY score LIMIT ?`).all(match, capped) as Array<Record<string, unknown>>
326
- }
327
- return rows.map(row => ({
328
- kind: row.kind === 'session' ? 'session' : 'file',
329
- path: String(row.path ?? ''),
330
- title: String(row.title ?? ''),
331
- snippet: String(row.snippet ?? row.body ?? '').replace(/[\u0001\u0002]/g, ''),
332
- ...(row.session_id ? { sessionId: String(row.session_id) } : {}),
333
- ...(row.message_id ? { messageId: String(row.message_id) } : {}),
334
- score: Number(row.score ?? 0),
335
- }))
678
+ return await Promise.race([response, timeout])
679
+ } catch {
680
+ return this.searchSync(query, limit)
681
+ } finally {
682
+ if (timer !== undefined) clearTimeout(timer)
683
+ }
684
+ }
685
+
686
+ private searchSync(query: string, limit: number): ContentHit[] {
687
+ if (this.db === null) return []
688
+ try {
689
+ return searchContent(this.db, query, limit)
336
690
  } catch (error) {
337
691
  this.state.error = messageOf(error)
338
692
  return []
@@ -340,6 +694,22 @@ export class ContentIndex {
340
694
  }
341
695
  }
342
696
 
697
+ function dshHome(): string {
698
+ return process.env.DSH_HOME || join(homedir(), '.dsh')
699
+ }
700
+
701
+ /** Archived session ids recorded by the Workspace registry; empty when absent. */
702
+ async function archivedSessionIds(): Promise<Set<string>> {
703
+ try {
704
+ const raw = await readFile(join(dshHome(), 'storages', 'workspace.json'), 'utf8')
705
+ const parsed = JSON.parse(raw) as { global?: { archivedSessionIds?: unknown } }
706
+ const ids = parsed.global?.archivedSessionIds
707
+ return new Set(Array.isArray(ids) ? ids.filter((id): id is string => typeof id === 'string') : [])
708
+ } catch {
709
+ return new Set()
710
+ }
711
+ }
712
+
343
713
  function messageText(event: { type?: string; data?: Record<string, unknown> }): { text: string; id: string } | undefined {
344
714
  if (event.type !== 'user/message' && event.type !== 'assistant/message') return undefined
345
715
  const data = event.data ?? {}
@@ -360,26 +730,41 @@ function messageText(event: { type?: string; data?: Record<string, unknown> }):
360
730
  return { text, id: id || `seq-${String(data.seq ?? '')}` }
361
731
  }
362
732
 
363
- function sessionFiles(roots: string[]): Array<{ path: string; mtime: number }> {
733
+ async function sessionFiles(roots: string[]): Promise<Array<{ path: string; mtime: number }>> {
364
734
  const out: Array<{ path: string; mtime: number }> = []
365
- const walk = (root: string, depth: number): void => {
366
- if (depth > 4 || !existsSync(root)) return
735
+ const walk = async (root: string, depth: number): Promise<void> => {
736
+ if (depth > 4) return
367
737
  let entries
368
- try { entries = readdirSync(root, { withFileTypes: true }) } catch { return }
738
+ try { entries = await readdir(root, { withFileTypes: true }) } catch { return }
369
739
  for (const entry of entries) {
370
740
  const path = join(root, entry.name)
371
- if (entry.isDirectory()) { walk(path, depth + 1); continue }
741
+ if (entry.isDirectory()) { await walk(path, depth + 1); continue }
372
742
  if (!entry.isFile() || !entry.name.endsWith('.v4.jsonl.zstd')) continue
373
- try { out.push({ path, mtime: statSync(path).mtimeMs }) } catch { /* ignore */ }
743
+ try { out.push({ path, mtime: (await stat(path)).mtimeMs }) } catch { /* ignore */ }
374
744
  }
375
745
  }
376
- for (const root of roots) walk(root, 0)
746
+ for (const root of roots) await walk(root, 0)
377
747
  out.sort((a, b) => b.mtime - a.mtime)
378
748
  return out
379
749
  }
380
750
 
381
- function contentSkipPath(path: string): boolean {
382
- return path.includes('/.dsh/') || path.includes('\\.dsh\\')
751
+ function contentSkipPath(path: string, excludes: string[]): boolean {
752
+ const normalized = path.replace(/\\/g, '/')
753
+ // The plugin's own session/config tree is never content.
754
+ if (normalized === '/.dsh' || normalized.includes('/.dsh/')) return true
755
+ if (excludes.length === 0) return false
756
+ const lower = normalized.toLowerCase()
757
+ const segments = lower.split('/')
758
+ for (const raw of excludes) {
759
+ const needle = raw.trim().toLowerCase()
760
+ if (needle === '') continue
761
+ if (needle.includes('/')) {
762
+ if (lower.includes(needle)) return true
763
+ continue
764
+ }
765
+ if (segments.includes(needle)) return true
766
+ }
767
+ return false
383
768
  }
384
769
 
385
770
  function messageOf(error: unknown): string {