dsh-locator 0.8.2 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README-es.md +30 -5
- package/README-hi.md +31 -5
- package/README-pt.md +31 -5
- package/README.en.md +31 -5
- package/README.md +31 -5
- package/bin/darwin-arm64/indexd +0 -0
- package/bin/darwin-x64/indexd +0 -0
- package/bin/linux-arm64/indexd +0 -0
- package/bin/linux-x64/indexd +0 -0
- package/bin/win32-arm64/indexd.exe +0 -0
- package/bin/win32-x64/indexd.exe +0 -0
- package/client.js +1198 -117
- package/lib/combined.d.ts +20 -3
- package/lib/combined.d.ts.map +1 -1
- package/lib/combined.js +85 -15
- package/lib/combined.js.map +1 -1
- package/lib/content-search-worker.d.ts +8 -3
- package/lib/content-search-worker.d.ts.map +1 -1
- package/lib/content-search-worker.js +54 -27
- package/lib/content-search-worker.js.map +1 -1
- package/lib/content-search.d.ts +127 -7
- package/lib/content-search.d.ts.map +1 -1
- package/lib/content-search.js +571 -36
- package/lib/content-search.js.map +1 -1
- package/lib/content.d.ts +83 -5
- package/lib/content.d.ts.map +1 -1
- package/lib/content.js +351 -87
- package/lib/content.js.map +1 -1
- package/lib/export.d.ts +61 -0
- package/lib/export.d.ts.map +1 -1
- package/lib/export.js +217 -52
- package/lib/export.js.map +1 -1
- package/lib/find.d.ts.map +1 -1
- package/lib/find.js +73 -16
- package/lib/find.js.map +1 -1
- package/lib/http.d.ts +38 -2
- package/lib/http.d.ts.map +1 -1
- package/lib/http.js +279 -82
- package/lib/http.js.map +1 -1
- package/lib/reveal.d.ts +69 -3
- package/lib/reveal.d.ts.map +1 -1
- package/lib/reveal.js +140 -11
- package/lib/reveal.js.map +1 -1
- package/lib/tools.d.ts +30 -0
- package/lib/tools.d.ts.map +1 -1
- package/lib/tools.js +65 -3
- package/lib/tools.js.map +1 -1
- package/package.json +1 -1
- package/src/combined.ts +107 -17
- package/src/content-search-worker.ts +52 -24
- package/src/content-search.ts +710 -36
- package/src/content.ts +349 -73
- package/src/export.ts +240 -50
- package/src/find.ts +83 -16
- package/src/http.ts +271 -83
- package/src/reveal.ts +148 -10
- package/src/tools.ts +83 -6
package/src/content.ts
CHANGED
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
* @module dsh-locator/content
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
|
-
import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
|
19
|
+
import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
|
20
20
|
import { readdir, readFile, stat } from 'node:fs/promises'
|
|
21
21
|
import { homedir } from 'node:os'
|
|
22
22
|
import { basename, dirname, extname, join } from 'node:path'
|
|
@@ -27,7 +27,7 @@ import { DatabaseSync } from 'node:sqlite'
|
|
|
27
27
|
import { randomUUID } from 'node:crypto'
|
|
28
28
|
import { fileURLToPath } from 'node:url'
|
|
29
29
|
import { Worker } from 'node:worker_threads'
|
|
30
|
-
import { searchContent } from './content-search.ts'
|
|
30
|
+
import { searchContent, type ContentSearchOptions, type ContentSearchResult } from './content-search.ts'
|
|
31
31
|
import type { LocatorRuntime } from './runtime.ts'
|
|
32
32
|
|
|
33
33
|
const SCHEMA_VERSION = 3
|
|
@@ -67,6 +67,8 @@ export interface ContentHit {
|
|
|
67
67
|
messageId?: string
|
|
68
68
|
/** Exact `data-chat-node-key` of the rendered row, when the host has one. */
|
|
69
69
|
nodeKey?: string
|
|
70
|
+
/** 1-based line of the first match inside the stored body, for file hits. */
|
|
71
|
+
line?: number
|
|
70
72
|
score: number
|
|
71
73
|
}
|
|
72
74
|
|
|
@@ -77,6 +79,11 @@ export interface ContentState {
|
|
|
77
79
|
messages: number
|
|
78
80
|
truncated: boolean
|
|
79
81
|
error: string
|
|
82
|
+
/**
|
|
83
|
+
* Non-fatal problems from the last build/refresh: unreadable session logs,
|
|
84
|
+
* a quarantined database, and so on. Empty when everything was read.
|
|
85
|
+
*/
|
|
86
|
+
warnings: string[]
|
|
80
87
|
updatedAt: number
|
|
81
88
|
}
|
|
82
89
|
|
|
@@ -102,6 +109,16 @@ export interface ContentOptions {
|
|
|
102
109
|
* single-threaded.
|
|
103
110
|
*/
|
|
104
111
|
searchWorker?: boolean
|
|
112
|
+
/**
|
|
113
|
+
* How long one worker query may run before it is reported as
|
|
114
|
+
* `content-search-timeout`. The query is never re-run on the host thread.
|
|
115
|
+
*/
|
|
116
|
+
searchWorkerTimeoutMs?: number
|
|
117
|
+
/**
|
|
118
|
+
* Test seam: build the search worker instead of spawning the compiled one.
|
|
119
|
+
* Production never sets it.
|
|
120
|
+
*/
|
|
121
|
+
searchWorkerFactory?: (workerPath: string) => Worker
|
|
105
122
|
}
|
|
106
123
|
|
|
107
124
|
interface DocRow {
|
|
@@ -118,7 +135,7 @@ interface DocRow {
|
|
|
118
135
|
|
|
119
136
|
export class ContentIndex {
|
|
120
137
|
private db: DatabaseSync | null = null
|
|
121
|
-
private readonly state: ContentState = { building: false, files: 0, sessions: 0, messages: 0, truncated: false, error: '', updatedAt: 0 }
|
|
138
|
+
private readonly state: ContentState = { building: false, files: 0, sessions: 0, messages: 0, truncated: false, error: '', warnings: [], updatedAt: 0 }
|
|
122
139
|
private generation = 0
|
|
123
140
|
private totalBytes = 0
|
|
124
141
|
private sliceDeadline = 0
|
|
@@ -128,13 +145,15 @@ export class ContentIndex {
|
|
|
128
145
|
private workerSeq = 0
|
|
129
146
|
private pendingStart = false
|
|
130
147
|
private dbIdentity: { dev: number; ino: number } | null = null
|
|
131
|
-
|
|
148
|
+
/** Session id → indexed session log path, for session-scoped content search. */
|
|
149
|
+
private readonly sessionPaths = new Map<string, string>()
|
|
150
|
+
private readonly pending = new Map<number, (result: ContentSearchResult) => void>()
|
|
132
151
|
|
|
133
152
|
constructor(private readonly rt: LocatorRuntime, private readonly options: ContentOptions) {}
|
|
134
153
|
|
|
135
154
|
status(): ContentState {
|
|
136
155
|
if (this.db !== null && !this.state.building && !this.databaseIsCurrent()) this.recoverExternal()
|
|
137
|
-
return { ...this.state, building: this.state.building || this.pendingStart }
|
|
156
|
+
return { ...this.state, warnings: [...this.state.warnings], building: this.state.building || this.pendingStart }
|
|
138
157
|
}
|
|
139
158
|
|
|
140
159
|
/**
|
|
@@ -176,6 +195,7 @@ export class ContentIndex {
|
|
|
176
195
|
this.releaseLock()
|
|
177
196
|
this.pendingStart = false
|
|
178
197
|
this.dbIdentity = null
|
|
198
|
+
this.sessionPaths.clear()
|
|
179
199
|
}
|
|
180
200
|
|
|
181
201
|
/** Build everything from scratch. */
|
|
@@ -251,6 +271,7 @@ export class ContentIndex {
|
|
|
251
271
|
this.state.building = true
|
|
252
272
|
this.state.error = ''
|
|
253
273
|
this.state.truncated = false
|
|
274
|
+
this.state.warnings = []
|
|
254
275
|
const run = mode === 'full' ? this.full(generation) : this.incremental(generation)
|
|
255
276
|
void run.catch(error => {
|
|
256
277
|
if (generation !== this.generation) return
|
|
@@ -303,8 +324,20 @@ export class ContentIndex {
|
|
|
303
324
|
}
|
|
304
325
|
const dbPath = join(this.options.stateDir, 'content.db')
|
|
305
326
|
const existed = existsSync(dbPath)
|
|
306
|
-
if (existed
|
|
307
|
-
|
|
327
|
+
if (existed) {
|
|
328
|
+
const verdict = this.classifyDatabase(dbPath)
|
|
329
|
+
if (verdict === 'foreign') {
|
|
330
|
+
throw new Error(`content.db is not a dsh-locator database, refusing to touch it: ${dbPath}`)
|
|
331
|
+
}
|
|
332
|
+
if (verdict !== 'ok') {
|
|
333
|
+
// A corrupt, empty or older-schema database must not disable content
|
|
334
|
+
// search for the rest of the session. Keep the bad file for diagnosis
|
|
335
|
+
// (rename, never delete bytes we did not create) and rebuild fresh.
|
|
336
|
+
// During an incremental open this throws so the caller falls back to a
|
|
337
|
+
// full rebuild; a full open simply continues with a clean file.
|
|
338
|
+
this.quarantineDatabase(dbPath, verdict)
|
|
339
|
+
if (!fresh) throw new Error('content database is incompatible; rebuilding')
|
|
340
|
+
}
|
|
308
341
|
}
|
|
309
342
|
let oldFileRemained = false
|
|
310
343
|
if (fresh) {
|
|
@@ -347,52 +380,135 @@ export class ContentIndex {
|
|
|
347
380
|
this.db.exec("INSERT OR REPLACE INTO content_meta (key, value) VALUES ('complete', 0)")
|
|
348
381
|
}
|
|
349
382
|
if (fresh) this.db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`)
|
|
383
|
+
this.refreshSessionPaths()
|
|
350
384
|
this.reopenWorker()
|
|
351
385
|
}
|
|
352
386
|
|
|
387
|
+
/**
|
|
388
|
+
* Decide what an existing `content.db` is before touching it:
|
|
389
|
+
* - `foreign`: no dsh-locator table at all — refuse to modify it.
|
|
390
|
+
* - `corrupt`: zero bytes, unreadable, or not a SQLite database.
|
|
391
|
+
* - `incompatible`: has our tables but a different schema version/shape.
|
|
392
|
+
* - `ok`: safe to reuse.
|
|
393
|
+
*/
|
|
394
|
+
private classifyDatabase(path: string): 'ok' | 'foreign' | 'corrupt' | 'incompatible' {
|
|
395
|
+
let size = 0
|
|
396
|
+
try { size = statSync(path).size } catch { return 'corrupt' }
|
|
397
|
+
if (size === 0) return 'corrupt'
|
|
398
|
+
let probe: DatabaseSync | undefined
|
|
399
|
+
try {
|
|
400
|
+
probe = new DatabaseSync(path, { readOnly: true })
|
|
401
|
+
const names = new Set(
|
|
402
|
+
(probe.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").all() as Array<{ name?: string }>)
|
|
403
|
+
.map(row => row.name)
|
|
404
|
+
.filter((name): name is string => typeof name === 'string'),
|
|
405
|
+
)
|
|
406
|
+
const owned = ['docs', 'docs_meta', 'sources', 'content_meta']
|
|
407
|
+
if (!owned.some(name => names.has(name))) return 'foreign'
|
|
408
|
+
if (!names.has('docs') || !names.has('docs_meta') || !names.has('sources') || !names.has('content_meta')) return 'incompatible'
|
|
409
|
+
const version = (probe.prepare('PRAGMA user_version').get() as { user_version?: number } | undefined)?.user_version ?? 0
|
|
410
|
+
if (Number(version) !== SCHEMA_VERSION) return 'incompatible'
|
|
411
|
+
return 'ok'
|
|
412
|
+
} catch {
|
|
413
|
+
return 'corrupt'
|
|
414
|
+
} finally {
|
|
415
|
+
try { probe?.close() } catch { /* ignore */ }
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
/**
|
|
420
|
+
* Move a broken/outdated database aside instead of destroying it. The
|
|
421
|
+
* timestamped name keeps the bytes around for inspection; if the rename
|
|
422
|
+
* fails (another process still holds it on Windows) a fresh file is still
|
|
423
|
+
* created so content search recovers.
|
|
424
|
+
*/
|
|
425
|
+
private quarantineDatabase(dbPath: string, verdict: 'corrupt' | 'incompatible'): void {
|
|
426
|
+
this.stopWorker()
|
|
427
|
+
try { this.db?.close() } catch { /* ignore */ }
|
|
428
|
+
this.db = null
|
|
429
|
+
const target = `${dbPath}.corrupt-${Date.now()}`
|
|
430
|
+
try {
|
|
431
|
+
renameSync(dbPath, target)
|
|
432
|
+
this.addWarning(verdict === 'corrupt' ? 'content-db-corrupt' : 'content-db-incompatible')
|
|
433
|
+
} catch {
|
|
434
|
+
try { rmSync(dbPath, { force: true }) } catch { /* ignore */ }
|
|
435
|
+
this.addWarning(verdict === 'corrupt' ? 'content-db-corrupt' : 'content-db-incompatible')
|
|
436
|
+
}
|
|
437
|
+
for (const suffix of ['-wal', '-shm']) {
|
|
438
|
+
try { rmSync(dbPath + suffix, { force: true }) } catch { /* ignore */ }
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
/** Record one non-fatal build warning (deduplicated, bounded). */
|
|
443
|
+
private addWarning(warning: string): void {
|
|
444
|
+
if (this.state.warnings.includes(warning)) return
|
|
445
|
+
if (this.state.warnings.length >= 20) return
|
|
446
|
+
this.state.warnings.push(warning)
|
|
447
|
+
}
|
|
448
|
+
|
|
353
449
|
private reopenWorker(): void {
|
|
354
450
|
if (this.worker === null) return
|
|
355
|
-
|
|
451
|
+
try {
|
|
452
|
+
this.worker.postMessage({ id: ++this.workerSeq, open: join(this.options.stateDir, 'content.db') })
|
|
453
|
+
} catch { /* a dying worker is replaced on the next query */ }
|
|
356
454
|
}
|
|
357
455
|
|
|
358
456
|
private stopWorker(): void {
|
|
359
457
|
const worker = this.worker
|
|
360
458
|
this.worker = null
|
|
361
|
-
for (const
|
|
459
|
+
for (const resolve of this.pending.values()) resolve(workerFailureResult('content-search-worker-failed'))
|
|
362
460
|
this.pending.clear()
|
|
363
461
|
if (worker !== null) void worker.terminate()
|
|
364
462
|
}
|
|
365
463
|
|
|
366
|
-
|
|
367
|
-
|
|
464
|
+
/**
|
|
465
|
+
* Whether a usable search worker exists, creating one when needed.
|
|
466
|
+
* @returns false when the worker cannot be created at all; that is the only
|
|
467
|
+
* case in which an in-process (sync) query is allowed.
|
|
468
|
+
*/
|
|
469
|
+
private ensureWorker(): boolean {
|
|
470
|
+
if (this.options.searchWorker !== true) return false
|
|
471
|
+
if (this.worker !== null) return true
|
|
368
472
|
// In a TypeScript test run the compiled worker does not exist next to the
|
|
369
|
-
// source;
|
|
473
|
+
// source; answer queries in-process instead (budgeted, see searchSync).
|
|
370
474
|
const workerPath = fileURLToPath(new URL('./content-search-worker.js', import.meta.url))
|
|
371
|
-
|
|
475
|
+
const factory = this.options.searchWorkerFactory
|
|
476
|
+
if (factory === undefined && !existsSync(workerPath)) return false
|
|
372
477
|
try {
|
|
373
|
-
const worker =
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
worker.on('message', (message:
|
|
377
|
-
const entry = this.pending.get(message.id)
|
|
378
|
-
if (entry === undefined) return
|
|
379
|
-
this.pending.delete(message.id)
|
|
380
|
-
entry.resolve(message.hits ?? [])
|
|
381
|
-
})
|
|
478
|
+
const worker = factory !== undefined
|
|
479
|
+
? factory(workerPath)
|
|
480
|
+
: new Worker(workerPath, { workerData: { dbPath: join(this.options.stateDir, 'content.db') } })
|
|
481
|
+
worker.on('message', (message: unknown) => { this.onWorkerMessage(message) })
|
|
382
482
|
const fail = (): void => {
|
|
383
483
|
if (this.worker !== worker) return
|
|
384
484
|
this.worker = null
|
|
385
|
-
for (const
|
|
485
|
+
for (const resolve of this.pending.values()) resolve(workerFailureResult('content-search-worker-failed'))
|
|
386
486
|
this.pending.clear()
|
|
387
487
|
}
|
|
388
488
|
worker.on('error', fail)
|
|
389
489
|
worker.on('exit', fail)
|
|
390
490
|
this.worker = worker
|
|
491
|
+
return true
|
|
391
492
|
} catch {
|
|
392
493
|
this.worker = null
|
|
494
|
+
return false
|
|
393
495
|
}
|
|
394
496
|
}
|
|
395
497
|
|
|
498
|
+
/**
|
|
499
|
+
* Resolve one worker reply. A malformed reply is reported as a protocol
|
|
500
|
+
* error; the query is never retried on the host thread.
|
|
501
|
+
*/
|
|
502
|
+
private onWorkerMessage(message: unknown): void {
|
|
503
|
+
if (typeof message !== 'object' || message === null) return
|
|
504
|
+
const record = message as { id?: unknown; result?: unknown }
|
|
505
|
+
if (typeof record.id !== 'number') return
|
|
506
|
+
const resolve = this.pending.get(record.id)
|
|
507
|
+
if (resolve === undefined) return
|
|
508
|
+
this.pending.delete(record.id)
|
|
509
|
+
resolve(isContentSearchResult(record.result) ? record.result : workerFailureResult('content-search-worker-protocol-error'))
|
|
510
|
+
}
|
|
511
|
+
|
|
396
512
|
private identityOf(path: string): { dev: number; ino: number } | null {
|
|
397
513
|
try {
|
|
398
514
|
const info = statSync(path)
|
|
@@ -424,6 +540,7 @@ export class ContentIndex {
|
|
|
424
540
|
this.dbIdentity = null
|
|
425
541
|
this.stopWorker()
|
|
426
542
|
this.totalBytes = 0
|
|
543
|
+
this.sessionPaths.clear()
|
|
427
544
|
this.state.updatedAt = 0
|
|
428
545
|
this.state.error = ''
|
|
429
546
|
this.state.truncated = false
|
|
@@ -439,20 +556,6 @@ export class ContentIndex {
|
|
|
439
556
|
return scanning !== undefined && scanning !== null
|
|
440
557
|
}
|
|
441
558
|
|
|
442
|
-
/** Whether `path` is a SQLite database this plugin created (has docs_meta). */
|
|
443
|
-
private ownedDatabase(path: string): boolean {
|
|
444
|
-
let probe: DatabaseSync | undefined
|
|
445
|
-
try {
|
|
446
|
-
probe = new DatabaseSync(path, { readOnly: true })
|
|
447
|
-
const row = probe.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'docs_meta'").get()
|
|
448
|
-
return row !== undefined
|
|
449
|
-
} catch {
|
|
450
|
-
return false
|
|
451
|
-
} finally {
|
|
452
|
-
try { probe?.close() } catch { /* ignore */ }
|
|
453
|
-
}
|
|
454
|
-
}
|
|
455
|
-
|
|
456
559
|
private readFlag(key: string): number | undefined {
|
|
457
560
|
const row = this.db!.prepare('SELECT value FROM content_meta WHERE key = ?').get(key) as { value?: number } | undefined
|
|
458
561
|
return row?.value === undefined ? undefined : Number(row.value)
|
|
@@ -502,12 +605,39 @@ export class ContentIndex {
|
|
|
502
605
|
this.state.updatedAt = Date.now()
|
|
503
606
|
this.writeTotalBytes()
|
|
504
607
|
this.writeFlag('complete', 1)
|
|
608
|
+
this.refreshSessionPaths()
|
|
505
609
|
this.reopenWorker()
|
|
506
610
|
}
|
|
507
611
|
|
|
612
|
+
/**
|
|
613
|
+
* Rebuild the session id → session log path map from the persisted sources.
|
|
614
|
+
* Runs after `open()` and after every build/refresh, so session-scoped
|
|
615
|
+
* searches work against an existing database before the first refresh ends.
|
|
616
|
+
*/
|
|
617
|
+
private refreshSessionPaths(): void {
|
|
618
|
+
this.sessionPaths.clear()
|
|
619
|
+
if (this.db === null) return
|
|
620
|
+
try {
|
|
621
|
+
const rows = this.db.prepare('SELECT path FROM sources').all() as Array<{ path?: string }>
|
|
622
|
+
for (const row of rows) {
|
|
623
|
+
const path = row.path
|
|
624
|
+
if (typeof path !== 'string' || path === '') continue
|
|
625
|
+
const sessionId = basename(dirname(path))
|
|
626
|
+
if (sessionId !== '') this.sessionPaths.set(sessionId, path)
|
|
627
|
+
}
|
|
628
|
+
} catch {
|
|
629
|
+
// A database from an older schema cannot answer session-scoped queries
|
|
630
|
+
// until its next rebuild; leave the map empty instead of failing boot.
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
|
|
508
634
|
private addDoc(row: DocRow): boolean {
|
|
509
635
|
const cap = this.options.maxTotalBytes ?? 0
|
|
510
|
-
|
|
636
|
+
// Replacing an existing document frees its old body first: an unchanged
|
|
637
|
+
// file rewritten at refresh time must not be rejected because its stale
|
|
638
|
+
// copy still occupies the budget.
|
|
639
|
+
const previous = this.storedBytes(row.key)
|
|
640
|
+
if (cap > 0 && this.totalBytes - previous + row.body.length > cap) {
|
|
511
641
|
this.state.truncated = true
|
|
512
642
|
return false
|
|
513
643
|
}
|
|
@@ -519,6 +649,13 @@ export class ContentIndex {
|
|
|
519
649
|
return true
|
|
520
650
|
}
|
|
521
651
|
|
|
652
|
+
/** Body bytes stored for one document key (0 when it is not indexed). */
|
|
653
|
+
private storedBytes(key: string): number {
|
|
654
|
+
const row = this.db!.prepare(`SELECT COALESCE(length(d.body), 0) AS n
|
|
655
|
+
FROM docs_meta m JOIN docs d ON d.rowid = m.rowid WHERE m.doc_key = ?`).get(key) as { n?: number } | undefined
|
|
656
|
+
return Number(row?.n ?? 0)
|
|
657
|
+
}
|
|
658
|
+
|
|
522
659
|
private deleteDoc(key: string): void {
|
|
523
660
|
const row = this.db!.prepare('SELECT rowid FROM docs_meta WHERE doc_key = ?').get(key) as { rowid: number } | undefined
|
|
524
661
|
if (row === undefined) return
|
|
@@ -544,6 +681,7 @@ export class ContentIndex {
|
|
|
544
681
|
let after = 0
|
|
545
682
|
let seen = 0
|
|
546
683
|
let scanned = 0
|
|
684
|
+
let capped = false
|
|
547
685
|
for (;;) {
|
|
548
686
|
if (generation !== this.generation) return
|
|
549
687
|
if (this.engineRescanning()) throw new Error('engine re-scan started during content build')
|
|
@@ -553,13 +691,14 @@ export class ContentIndex {
|
|
|
553
691
|
if (generation !== this.generation) return
|
|
554
692
|
if (entry.dir || entry.size <= 0 || entry.size > this.options.maxFileBytes) continue
|
|
555
693
|
if (!TEXT_EXTENSIONS.has(extname(entry.name).toLowerCase()) || contentSkipPath(entry.path, this.options.excludes ?? [])) continue
|
|
556
|
-
if (this.options.maxFiles > 0 && seen >= this.options.maxFiles) break
|
|
694
|
+
if (this.options.maxFiles > 0 && seen >= this.options.maxFiles) { capped = true; break }
|
|
557
695
|
const row = await this.readFileDoc(entry.path, entry.name, entry.mtime)
|
|
558
696
|
if (row === null) continue
|
|
559
697
|
seen++
|
|
560
698
|
if (!this.addDoc(row)) return
|
|
561
699
|
await this.breathe()
|
|
562
700
|
}
|
|
701
|
+
if (capped) break
|
|
563
702
|
after = page.next
|
|
564
703
|
if (page.done || page.entries.length === 0) break
|
|
565
704
|
}
|
|
@@ -571,8 +710,13 @@ export class ContentIndex {
|
|
|
571
710
|
const seen = new Set<string>()
|
|
572
711
|
const deferred: Array<{ path: string; name: string; mtime: number; size: number }> = []
|
|
573
712
|
let after = 0
|
|
713
|
+
// `maxFiles` counts documents that are actually stored. A file whose read
|
|
714
|
+
// failed is not stored, so it must not consume a slot: counting it here used
|
|
715
|
+
// to stop the loop early (and the deletion pass then removed every trailing
|
|
716
|
+
// document that was never reached). `indexed` only moves on success.
|
|
574
717
|
let indexed = 0
|
|
575
718
|
let scanned = 0
|
|
719
|
+
let capped = false
|
|
576
720
|
for (;;) {
|
|
577
721
|
if (generation !== this.generation) return
|
|
578
722
|
if (this.engineRescanning()) throw new Error('engine re-scan started during content build')
|
|
@@ -582,22 +726,30 @@ export class ContentIndex {
|
|
|
582
726
|
if (generation !== this.generation) return
|
|
583
727
|
if (entry.dir || entry.size <= 0 || entry.size > this.options.maxFileBytes) continue
|
|
584
728
|
if (!TEXT_EXTENSIONS.has(extname(entry.name).toLowerCase()) || contentSkipPath(entry.path, this.options.excludes ?? [])) continue
|
|
585
|
-
if (this.options.maxFiles > 0 && indexed >= this.options.maxFiles) break
|
|
729
|
+
if (this.options.maxFiles > 0 && indexed >= this.options.maxFiles) { capped = true; break }
|
|
586
730
|
const key = `file:${entry.path}`
|
|
587
731
|
seen.add(key)
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
732
|
+
if (meta.get(key) === entry.mtime) {
|
|
733
|
+
// Unchanged and already stored: it still occupies one of the slots.
|
|
734
|
+
indexed++
|
|
735
|
+
continue
|
|
736
|
+
}
|
|
737
|
+
if (this.wouldExceed(entry.size, key)) {
|
|
591
738
|
deferred.push({ path: entry.path, name: entry.name, mtime: entry.mtime, size: entry.size })
|
|
592
739
|
continue
|
|
593
740
|
}
|
|
594
|
-
|
|
741
|
+
// Read first, replace only on success. A transient read failure must
|
|
742
|
+
// keep the document indexed by the previous refresh, not delete it.
|
|
595
743
|
const row = await this.readFileDoc(entry.path, entry.name, entry.mtime)
|
|
596
|
-
if (row
|
|
744
|
+
if (row === null) continue
|
|
745
|
+
if (!this.addDoc(row)) {
|
|
597
746
|
deferred.push({ path: entry.path, name: entry.name, mtime: entry.mtime, size: entry.size })
|
|
747
|
+
continue
|
|
598
748
|
}
|
|
749
|
+
indexed++
|
|
599
750
|
await this.breathe()
|
|
600
751
|
}
|
|
752
|
+
if (capped) break
|
|
601
753
|
after = page.next
|
|
602
754
|
if (page.done || page.entries.length === 0) break
|
|
603
755
|
}
|
|
@@ -606,19 +758,20 @@ export class ContentIndex {
|
|
|
606
758
|
for (const key of meta.keys()) if (!seen.has(key)) this.deleteDoc(key)
|
|
607
759
|
for (const entry of deferred) {
|
|
608
760
|
if (generation !== this.generation) return
|
|
609
|
-
if (this.wouldExceed(entry.size)) { this.state.truncated = true; continue }
|
|
610
|
-
this.deleteDoc(`file:${entry.path}`)
|
|
761
|
+
if (this.wouldExceed(entry.size, `file:${entry.path}`)) { this.state.truncated = true; continue }
|
|
611
762
|
const row = await this.readFileDoc(entry.path, entry.name, entry.mtime)
|
|
612
|
-
if (row === null
|
|
763
|
+
if (row === null) continue
|
|
764
|
+
if (!this.addDoc(row)) { this.state.truncated = true; continue }
|
|
613
765
|
await this.breathe()
|
|
614
766
|
}
|
|
615
767
|
}
|
|
616
768
|
|
|
617
769
|
/** Would storing a file of this size overflow the configured body budget? */
|
|
618
|
-
private wouldExceed(size: number): boolean {
|
|
770
|
+
private wouldExceed(size: number, docKey?: string): boolean {
|
|
619
771
|
const cap = this.options.maxTotalBytes ?? 0
|
|
620
772
|
if (cap <= 0) return false
|
|
621
|
-
|
|
773
|
+
const previous = docKey === undefined ? 0 : this.storedBytes(docKey)
|
|
774
|
+
return this.totalBytes - previous + Math.min(size, this.options.maxFileBytes) > cap
|
|
622
775
|
}
|
|
623
776
|
|
|
624
777
|
private async readFileDoc(path: string, name: string, mtime: number): Promise<DocRow | null> {
|
|
@@ -633,8 +786,15 @@ export class ContentIndex {
|
|
|
633
786
|
const files = (await this.sessionCandidates()).filter(file => this.settled(file.mtime))
|
|
634
787
|
for (const file of files) {
|
|
635
788
|
if (generation !== this.generation) return
|
|
789
|
+
const read = await this.readSessionText(file)
|
|
790
|
+
if (!read.ok) {
|
|
791
|
+
// A log that cannot be read or decompressed does not contribute rows;
|
|
792
|
+
// the build continues with the remaining sessions.
|
|
793
|
+
this.addWarning('content-session-log-unreadable')
|
|
794
|
+
continue
|
|
795
|
+
}
|
|
636
796
|
this.deletePath(file.path)
|
|
637
|
-
if (!(await this.
|
|
797
|
+
if (!(await this.indexSessionText(file, read.text, generation))) return
|
|
638
798
|
this.db!.prepare('INSERT OR REPLACE INTO sources (path, mtime) VALUES (?, ?)').run(file.path, file.mtime)
|
|
639
799
|
await this.breathe()
|
|
640
800
|
}
|
|
@@ -652,8 +812,16 @@ export class ContentIndex {
|
|
|
652
812
|
// its existing rows instead of re-reading the whole growing log.
|
|
653
813
|
if (!this.settled(file.mtime)) continue
|
|
654
814
|
if (known.get(file.path) === file.mtime) continue
|
|
815
|
+
// Decode the whole log before touching the old rows. On failure keep the
|
|
816
|
+
// previous messages and the previous sources.mtime, so the next refresh
|
|
817
|
+
// retries the same file instead of treating it as empty.
|
|
818
|
+
const read = await this.readSessionText(file)
|
|
819
|
+
if (!read.ok) {
|
|
820
|
+
this.addWarning('content-session-log-unreadable')
|
|
821
|
+
continue
|
|
822
|
+
}
|
|
655
823
|
this.deletePath(file.path)
|
|
656
|
-
if (!(await this.
|
|
824
|
+
if (!(await this.indexSessionText(file, read.text, generation))) return
|
|
657
825
|
this.db!.prepare('INSERT OR REPLACE INTO sources (path, mtime) VALUES (?, ?)').run(file.path, file.mtime)
|
|
658
826
|
await this.breathe()
|
|
659
827
|
}
|
|
@@ -664,11 +832,24 @@ export class ContentIndex {
|
|
|
664
832
|
}
|
|
665
833
|
}
|
|
666
834
|
|
|
667
|
-
|
|
835
|
+
/** Read and decompress one session log completely; no side effects on rows. */
|
|
836
|
+
private async readSessionText(file: { path: string; mtime: number }): Promise<
|
|
837
|
+
{ ok: true; text: string } | { ok: false; error: string }
|
|
838
|
+
> {
|
|
668
839
|
let text: string
|
|
669
|
-
try {
|
|
840
|
+
try {
|
|
841
|
+
text = (await zstdDecompressAsync(await readFile(file.path))).toString('utf8')
|
|
842
|
+
} catch (error) {
|
|
843
|
+
return { ok: false, error: messageOf(error) }
|
|
844
|
+
}
|
|
845
|
+
return { ok: true, text }
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
/** Index one already-decoded session log; the caller owns row replacement. */
|
|
849
|
+
private async indexSessionText(file: { path: string; mtime: number }, text: string, generation: number): Promise<boolean> {
|
|
670
850
|
if (generation !== this.generation) return false
|
|
671
851
|
const sessionId = basename(dirname(file.path))
|
|
852
|
+
this.sessionPaths.set(sessionId, file.path)
|
|
672
853
|
let title = ''
|
|
673
854
|
let parsed = 0
|
|
674
855
|
for (const line of text.split('\n')) {
|
|
@@ -710,49 +891,144 @@ export class ContentIndex {
|
|
|
710
891
|
|
|
711
892
|
/**
|
|
712
893
|
* Search file bodies and session messages. With `searchWorker` enabled the
|
|
713
|
-
* query itself runs off the host thread
|
|
714
|
-
*
|
|
894
|
+
* query itself runs off the host thread.
|
|
895
|
+
*
|
|
896
|
+
* A worker timeout, crash or malformed reply is a hard error result: the
|
|
897
|
+
* same query is never re-run on the host thread, where a heavy query (for
|
|
898
|
+
* example a whole-word scan) would stall the DSH event loop. Only when the
|
|
899
|
+
* worker cannot be created at all does the query run in-process, and that
|
|
900
|
+
* fallback is budgeted (bounded counts).
|
|
901
|
+
*
|
|
902
|
+
* The two-argument form returns the hit array (legacy callers). The
|
|
903
|
+
* three-argument form returns hits plus count, warnings and an error code,
|
|
904
|
+
* and understands the advanced match options.
|
|
715
905
|
* @param query - words or a phrase to find.
|
|
716
906
|
* @param limit - maximum hits.
|
|
907
|
+
* @param options - match mode, case/whole-word flags, session scope, count mode.
|
|
717
908
|
* @returns matching documents ordered by relevance.
|
|
718
909
|
*/
|
|
719
|
-
async search(query: string, limit: number): Promise<ContentHit[]>
|
|
720
|
-
|
|
910
|
+
async search(query: string, limit: number): Promise<ContentHit[]>
|
|
911
|
+
async search(query: string, limit: number, options: ContentSearchOptions): Promise<ContentSearchResult>
|
|
912
|
+
async search(query: string, limit: number, options?: ContentSearchOptions): Promise<ContentHit[] | ContentSearchResult> {
|
|
913
|
+
const result = await this.searchDetailed(query, limit, options ?? {})
|
|
914
|
+
return options === undefined ? result.hits : result
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
private async searchDetailed(query: string, limit: number, options: ContentSearchOptions): Promise<ContentSearchResult> {
|
|
918
|
+
const resolved = this.resolveSession(options)
|
|
919
|
+
if (resolved === null) return emptySearchResult(['unknown-session'])
|
|
920
|
+
if (this.db === null) return emptySearchResult()
|
|
721
921
|
if (!this.databaseIsCurrent()) {
|
|
722
922
|
this.recoverExternal()
|
|
723
|
-
return
|
|
923
|
+
return emptySearchResult()
|
|
724
924
|
}
|
|
725
|
-
|
|
726
|
-
this.
|
|
727
|
-
if (this.
|
|
728
|
-
|
|
925
|
+
// Direct embedders run in-process by design (searchWorker is off).
|
|
926
|
+
if (this.options.searchWorker !== true) return this.searchSync(query, limit, resolved, false)
|
|
927
|
+
if (!this.ensureWorker()) {
|
|
928
|
+
// The worker could not be created at all: the one allowed sync path.
|
|
929
|
+
return this.searchSync(query, limit, resolved, true)
|
|
930
|
+
}
|
|
931
|
+
return await this.searchInWorker(query, limit, resolved)
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
/** Run one query on the worker; timeout/protocol errors are final results. */
|
|
935
|
+
private async searchInWorker(query: string, limit: number, options: ContentSearchOptions): Promise<ContentSearchResult> {
|
|
729
936
|
const worker = this.worker
|
|
730
|
-
|
|
731
|
-
|
|
937
|
+
if (worker === null) return workerFailureResult('content-search-worker-failed')
|
|
938
|
+
const id = ++this.workerSeq
|
|
939
|
+
const response = new Promise<ContentSearchResult>(resolve => { this.pending.set(id, resolve) })
|
|
940
|
+
const timeoutMs = Math.max(1, this.options.searchWorkerTimeoutMs ?? WORKER_TIMEOUT_MS)
|
|
732
941
|
let timer: NodeJS.Timeout | undefined
|
|
733
|
-
const timeout = new Promise<
|
|
734
|
-
timer = setTimeout(() => {
|
|
942
|
+
const timeout = new Promise<ContentSearchResult>(resolve => {
|
|
943
|
+
timer = setTimeout(() => {
|
|
944
|
+
if (!this.pending.delete(id)) return
|
|
945
|
+
// A wedged worker is thrown away; the next query gets a fresh one. The
|
|
946
|
+
// current query is reported as timed out, never re-run here.
|
|
947
|
+
if (this.worker === worker) {
|
|
948
|
+
this.worker = null
|
|
949
|
+
void worker.terminate()
|
|
950
|
+
}
|
|
951
|
+
resolve(workerFailureResult('content-search-timeout'))
|
|
952
|
+
}, timeoutMs)
|
|
735
953
|
})
|
|
736
954
|
try {
|
|
737
|
-
|
|
955
|
+
worker.postMessage({ id, query, limit, options })
|
|
738
956
|
} catch {
|
|
739
|
-
|
|
957
|
+
if (this.pending.delete(id)) {
|
|
958
|
+
if (timer !== undefined) clearTimeout(timer)
|
|
959
|
+
return workerFailureResult('content-search-worker-failed')
|
|
960
|
+
}
|
|
961
|
+
}
|
|
962
|
+
try {
|
|
963
|
+
return await Promise.race([response, timeout])
|
|
740
964
|
} finally {
|
|
741
965
|
if (timer !== undefined) clearTimeout(timer)
|
|
742
966
|
}
|
|
743
967
|
}
|
|
744
968
|
|
|
745
|
-
|
|
746
|
-
|
|
969
|
+
/** Map a request's sessionId onto the indexed session log path. */
|
|
970
|
+
private resolveSession(options: ContentSearchOptions): ContentSearchOptions | null {
|
|
971
|
+
if (options.sessionPath !== undefined && options.sessionPath !== null) return options
|
|
972
|
+
if (options.sessionId === undefined || options.sessionId === '') return { ...options, sessionPath: null }
|
|
973
|
+
const path = this.sessionPaths.get(options.sessionId)
|
|
974
|
+
if (path === undefined) return null
|
|
975
|
+
return { ...options, sessionPath: path }
|
|
976
|
+
}
|
|
977
|
+
|
|
978
|
+
/**
|
|
979
|
+
* In-process query path. When `degraded` is set (worker creation failed) the
|
|
980
|
+
* options are tightened so this rare path cannot spend an unbounded amount of
|
|
981
|
+
* host-thread time counting matches.
|
|
982
|
+
*/
|
|
983
|
+
private searchSync(query: string, limit: number, options: ContentSearchOptions, degraded: boolean): ContentSearchResult {
|
|
984
|
+
if (this.db === null) return emptySearchResult()
|
|
985
|
+
const effective = degraded ? budgetedSearchOptions(options) : options
|
|
747
986
|
try {
|
|
748
|
-
|
|
987
|
+
const result = searchContent(this.db, query, limit, effective)
|
|
988
|
+
if (!degraded) return result
|
|
989
|
+
return { ...result, warnings: [...new Set(['content-search-in-process', ...result.warnings])] }
|
|
749
990
|
} catch (error) {
|
|
750
991
|
this.state.error = messageOf(error)
|
|
751
|
-
return
|
|
992
|
+
return emptySearchResult()
|
|
752
993
|
}
|
|
753
994
|
}
|
|
754
995
|
}
|
|
755
996
|
|
|
997
|
+
/** Default deadline for one worker query. */
|
|
998
|
+
const WORKER_TIMEOUT_MS = 5_000
|
|
999
|
+
/** Counting cap used by the rare in-process fallback. */
|
|
1000
|
+
const IN_PROCESS_COUNT_LIMIT = 200
|
|
1001
|
+
|
|
1002
|
+
/** Tighten a query for the in-process fallback: bounded counting only. */
|
|
1003
|
+
function budgetedSearchOptions(options: ContentSearchOptions): ContentSearchOptions {
|
|
1004
|
+
return {
|
|
1005
|
+
...options,
|
|
1006
|
+
count: options.count === 'none' ? 'none' : 'bounded',
|
|
1007
|
+
countLimit: Math.min(options.countLimit ?? IN_PROCESS_COUNT_LIMIT, IN_PROCESS_COUNT_LIMIT),
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
/** Whether a value looks like a complete worker search result. */
|
|
1012
|
+
function isContentSearchResult(value: unknown): value is ContentSearchResult {
|
|
1013
|
+
if (typeof value !== 'object' || value === null) return false
|
|
1014
|
+
const record = value as Record<string, unknown>
|
|
1015
|
+
if (!Array.isArray(record.hits) || !Array.isArray(record.warnings)) return false
|
|
1016
|
+
const count = record.count
|
|
1017
|
+
if (typeof count !== 'object' || count === null) return false
|
|
1018
|
+
const kind = (count as { kind?: unknown }).kind
|
|
1019
|
+
return kind === 'exact' || kind === 'bounded' || kind === 'unknown'
|
|
1020
|
+
}
|
|
1021
|
+
|
|
1022
|
+
/** Result for a failed worker query; carries an explicit error code. */
|
|
1023
|
+
function workerFailureResult(error: string): ContentSearchResult {
|
|
1024
|
+
return { hits: [], count: { value: 0, kind: 'unknown' }, warnings: [error], error }
|
|
1025
|
+
}
|
|
1026
|
+
|
|
1027
|
+
/** Result used when a content query could not run at all. */
|
|
1028
|
+
function emptySearchResult(warnings: string[] = []): ContentSearchResult {
|
|
1029
|
+
return { hits: [], count: { value: 0, kind: 'unknown' }, warnings }
|
|
1030
|
+
}
|
|
1031
|
+
|
|
756
1032
|
function dshHome(): string {
|
|
757
1033
|
return process.env.DSH_HOME || join(homedir(), '.dsh')
|
|
758
1034
|
}
|