dsh-locator 0.7.1 → 0.7.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/README-es.md +37 -5
- package/README-hi.md +35 -5
- package/README-pt.md +33 -5
- package/README.en.md +34 -4
- package/README.md +27 -5
- package/client.js +117 -17
- package/cordis.patch.yml +13 -0
- package/lib/config.d.ts +20 -0
- package/lib/config.d.ts.map +1 -1
- package/lib/config.js +38 -4
- package/lib/config.js.map +1 -1
- package/lib/content-search-worker.d.ts +8 -0
- package/lib/content-search-worker.d.ts.map +1 -0
- package/lib/content-search-worker.js +54 -0
- package/lib/content-search-worker.js.map +1 -0
- package/lib/content-search.d.ts +15 -0
- package/lib/content-search.d.ts.map +1 -0
- package/lib/content-search.js +44 -0
- package/lib/content-search.js.map +1 -0
- package/lib/content.d.ts +83 -4
- package/lib/content.d.ts.map +1 -1
- package/lib/content.js +496 -77
- package/lib/content.js.map +1 -1
- package/lib/find.d.ts +1 -1
- package/lib/find.d.ts.map +1 -1
- package/lib/find.js +8 -3
- package/lib/find.js.map +1 -1
- package/lib/http.d.ts +1 -1
- package/lib/http.d.ts.map +1 -1
- package/lib/http.js +4 -2
- package/lib/http.js.map +1 -1
- package/lib/index.d.ts +5 -1
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +106 -27
- package/lib/index.js.map +1 -1
- package/lib/reindex.d.ts +1 -1
- package/lib/reindex.d.ts.map +1 -1
- package/lib/reindex.js +2 -1
- package/lib/reindex.js.map +1 -1
- package/package.json +1 -1
- package/src/config.ts +58 -4
- package/src/content-search-worker.ts +52 -0
- package/src/content-search.ts +45 -0
- package/src/content.ts +456 -71
- package/src/find.ts +8 -3
- package/src/http.ts +4 -2
- package/src/index.ts +106 -25
- package/src/reindex.ts +2 -1
package/src/content.ts
CHANGED
|
@@ -6,16 +6,49 @@
|
|
|
6
6
|
* `session.v4.jsonl.zstd`, and SQLite's trigram tokenizer gives substring and
|
|
7
7
|
* CJK search. A meta table stores each source's mtime so refresh() only
|
|
8
8
|
* re-reads what changed; rebuild() drops and rebuilds everything.
|
|
9
|
+
*
|
|
10
|
+
* Everything here runs inside the DSH host process, so the build must never
|
|
11
|
+
* monopolise its event loop: file reads and zstd decoding are asynchronous and
|
|
12
|
+
* the loops yield to the scheduler every ~8 ms of work. Session logs are read
|
|
13
|
+
* only once they have settled (an actively written session is skipped until it
|
|
14
|
+
* has been quiet for `activeSessionDelayMs`), and archived sessions are left
|
|
15
|
+
* alone unless `includeArchived` is set.
|
|
9
16
|
* @module dsh-locator/content
|
|
10
17
|
*/
|
|
11
18
|
|
|
12
|
-
import { existsSync, mkdirSync,
|
|
19
|
+
import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
|
20
|
+
import { readdir, readFile, stat } from 'node:fs/promises'
|
|
13
21
|
import { homedir } from 'node:os'
|
|
14
22
|
import { basename, dirname, extname, join } from 'node:path'
|
|
15
|
-
import {
|
|
23
|
+
import { promisify } from 'node:util'
|
|
24
|
+
import { zstdDecompress } from 'node:zlib'
|
|
25
|
+
import { scheduler } from 'node:timers/promises'
|
|
16
26
|
import { DatabaseSync } from 'node:sqlite'
|
|
27
|
+
import { randomUUID } from 'node:crypto'
|
|
28
|
+
import { fileURLToPath } from 'node:url'
|
|
29
|
+
import { Worker } from 'node:worker_threads'
|
|
30
|
+
import { searchContent } from './content-search.ts'
|
|
17
31
|
import type { LocatorRuntime } from './runtime.ts'
|
|
18
32
|
|
|
33
|
+
const SCHEMA_VERSION = 2
|
|
34
|
+
|
|
35
|
+
/** Work budget between two event-loop yields; keeps host latency low. */
|
|
36
|
+
const SLICE_MS = 8
|
|
37
|
+
|
|
38
|
+
/** Wall-clock start of this process; distinguishes a reused pid from a live sibling. */
|
|
39
|
+
const PROCESS_START = Math.round(Date.now() - process.uptime() * 1000)
|
|
40
|
+
|
|
41
|
+
/** Session bodies are short: a message, not a whole file. */
|
|
42
|
+
const SESSION_BODY_BYTES = 8_000
|
|
43
|
+
|
|
44
|
+
const zstdDecompressAsync = promisify(zstdDecompress)
|
|
45
|
+
|
|
46
|
+
// `scheduler.yield` is newer than the oldest supported Node line; fall back to
|
|
47
|
+
// setImmediate so a build on Node 22 still yields instead of throwing.
|
|
48
|
+
const yieldToLoop: () => Promise<void> = typeof (scheduler as { yield?: unknown }).yield === 'function'
|
|
49
|
+
? () => scheduler.yield()
|
|
50
|
+
: () => new Promise<void>(resolve => { setImmediate(resolve) })
|
|
51
|
+
|
|
19
52
|
const TEXT_EXTENSIONS = new Set([
|
|
20
53
|
'.txt', '.md', '.markdown', '.rst', '.log', '.csv', '.tsv', '.json', '.jsonl',
|
|
21
54
|
'.yaml', '.yml', '.toml', '.ini', '.cfg', '.conf', '.xml', '.html', '.htm',
|
|
@@ -48,10 +81,25 @@ export interface ContentState {
|
|
|
48
81
|
export interface ContentOptions {
|
|
49
82
|
stateDir: string
|
|
50
83
|
sessionDirs?: string[] | undefined
|
|
84
|
+
/** Path segments or substrings never read into the content index. */
|
|
85
|
+
excludes?: string[]
|
|
86
|
+
/** Index archived sessions too; false (the default) leaves them out. */
|
|
87
|
+
includeArchived?: boolean
|
|
88
|
+
/**
|
|
89
|
+
* Skip session logs modified more recently than this many milliseconds, so
|
|
90
|
+
* the session being written right now is not re-read on every refresh.
|
|
91
|
+
*/
|
|
92
|
+
activeSessionDelayMs?: number
|
|
51
93
|
maxFileBytes: number
|
|
52
94
|
maxTotalBytes?: number
|
|
53
95
|
maxFiles: number
|
|
54
96
|
maxSessions: number
|
|
97
|
+
/**
|
|
98
|
+
* Run content queries in a worker thread so a slow search never blocks the
|
|
99
|
+
* DSH host event loop. Off by default so direct embedders and tests stay
|
|
100
|
+
* single-threaded.
|
|
101
|
+
*/
|
|
102
|
+
searchWorker?: boolean
|
|
55
103
|
}
|
|
56
104
|
|
|
57
105
|
interface DocRow {
|
|
@@ -70,16 +118,34 @@ export class ContentIndex {
|
|
|
70
118
|
private readonly state: ContentState = { building: false, files: 0, sessions: 0, messages: 0, truncated: false, error: '', updatedAt: 0 }
|
|
71
119
|
private generation = 0
|
|
72
120
|
private totalBytes = 0
|
|
73
|
-
private
|
|
121
|
+
private sliceDeadline = 0
|
|
122
|
+
private worker: Worker | null = null
|
|
123
|
+
private lockHeld = false
|
|
124
|
+
private readonly lockToken = randomUUID()
|
|
125
|
+
private workerSeq = 0
|
|
126
|
+
private pendingStart = false
|
|
127
|
+
private readonly pending = new Map<number, { resolve: (hits: ContentHit[]) => void; reject: (error: Error) => void }>()
|
|
74
128
|
|
|
75
129
|
constructor(private readonly rt: LocatorRuntime, private readonly options: ContentOptions) {}
|
|
76
130
|
|
|
77
|
-
status(): ContentState {
|
|
131
|
+
status(): ContentState {
|
|
132
|
+
return { ...this.state, building: this.state.building || this.pendingStart }
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Report a lazy build as in progress before the asynchronous start reaches
|
|
137
|
+
* `boot()`, so the panel/agent sees "building" on the very first query
|
|
138
|
+
* instead of an empty result with no hint.
|
|
139
|
+
*/
|
|
140
|
+
markPending(): void {
|
|
141
|
+
if (this.state.building) return
|
|
142
|
+
this.state.error = ''
|
|
143
|
+
this.pendingStart = true
|
|
144
|
+
}
|
|
78
145
|
|
|
79
|
-
/**
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
this.timer = setInterval(() => this.refresh(), ms)
|
|
146
|
+
/** Drop the pending flag when the lazy start failed before it began. */
|
|
147
|
+
clearPending(): void {
|
|
148
|
+
this.pendingStart = false
|
|
83
149
|
}
|
|
84
150
|
|
|
85
151
|
/** Queue a full rebuild; false when one is already running. */
|
|
@@ -97,21 +163,84 @@ export class ContentIndex {
|
|
|
97
163
|
}
|
|
98
164
|
|
|
99
165
|
close(): void {
|
|
100
|
-
if (this.timer !== undefined) { clearInterval(this.timer); this.timer = undefined }
|
|
101
166
|
this.generation++
|
|
102
167
|
this.db?.close()
|
|
103
168
|
this.db = null
|
|
104
169
|
this.state.building = false
|
|
170
|
+
this.stopWorker()
|
|
171
|
+
this.releaseLock()
|
|
172
|
+
this.pendingStart = false
|
|
105
173
|
}
|
|
106
174
|
|
|
107
175
|
/** Build everything from scratch. */
|
|
108
176
|
start(): void { this.begin('full') }
|
|
109
177
|
|
|
178
|
+
/**
|
|
179
|
+
* Start at plugin boot: reuse an existing database and refresh it
|
|
180
|
+
* incrementally, so a restart does not re-read and re-index gigabytes of
|
|
181
|
+
* text. A missing or incompatible database falls back to a full build.
|
|
182
|
+
*/
|
|
183
|
+
boot(): void {
|
|
184
|
+
// A content operation that ran before the deferred boot may have built the
|
|
185
|
+
// database already; do not start a second pass behind it.
|
|
186
|
+
if (this.state.updatedAt > 0) return
|
|
187
|
+
this.begin(existsSync(join(this.options.stateDir, 'content.db')) ? 'incremental' : 'full')
|
|
188
|
+
}
|
|
189
|
+
|
|
110
190
|
/** Re-read changed files/sessions; open the DB when this is the first run. */
|
|
111
191
|
refresh(): void { this.begin(this.db === null ? 'full' : 'incremental') }
|
|
112
192
|
|
|
193
|
+
/**
|
|
194
|
+
* Serialize content builds across plugin instances and DSH processes that
|
|
195
|
+
* share a state directory: a stale lock (dead pid or older than 30 minutes)
|
|
196
|
+
* is taken over, a live one keeps this instance from writing concurrently.
|
|
197
|
+
* @returns whether this instance owns the build lock.
|
|
198
|
+
*/
|
|
199
|
+
private acquireLock(): boolean {
|
|
200
|
+
if (this.lockHeld) return true
|
|
201
|
+
mkdirSync(this.options.stateDir, { recursive: true, mode: 0o700 })
|
|
202
|
+
const path = join(this.options.stateDir, 'content.lock')
|
|
203
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
204
|
+
try {
|
|
205
|
+
const fd = openSync(path, 'wx', 0o600)
|
|
206
|
+
writeFileSync(fd, JSON.stringify({ pid: process.pid, startedAt: Date.now(), token: this.lockToken, procStart: PROCESS_START }))
|
|
207
|
+
closeSync(fd)
|
|
208
|
+
this.lockHeld = true
|
|
209
|
+
return true
|
|
210
|
+
} catch {
|
|
211
|
+
let holder: { pid?: number; startedAt?: number; token?: string; procStart?: number } = {}
|
|
212
|
+
try { holder = JSON.parse(readFileSync(path, 'utf8')) as typeof holder } catch { /* unreadable: treat as stale */ }
|
|
213
|
+
const age = Date.now() - Number(holder.startedAt ?? 0)
|
|
214
|
+
let alive = false
|
|
215
|
+
if (typeof holder.pid === 'number' && age < 30 * 60_000) {
|
|
216
|
+
try { process.kill(holder.pid, 0); alive = true } catch (error) {
|
|
217
|
+
alive = (error as NodeJS.ErrnoException).code === 'EPERM'
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
// A live sibling instance (for example during profile HMR) blocks even
|
|
221
|
+
// when it shares our pid; the per-instance token tells them apart. A
|
|
222
|
+
// reused pid from an earlier process incarnation is stale, not live.
|
|
223
|
+
const sameProcess = holder.procStart === undefined || Math.abs(Number(holder.procStart) - PROCESS_START) < 5_000
|
|
224
|
+
if (alive && sameProcess && holder.token !== this.lockToken) return false
|
|
225
|
+
try { rmSync(path, { force: true }) } catch { return false }
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
return false
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
private releaseLock(): void {
|
|
232
|
+
if (!this.lockHeld) return
|
|
233
|
+
this.lockHeld = false
|
|
234
|
+
try { rmSync(join(this.options.stateDir, 'content.lock'), { force: true }) } catch { /* ignore */ }
|
|
235
|
+
}
|
|
236
|
+
|
|
113
237
|
private begin(mode: 'full' | 'incremental'): void {
|
|
114
238
|
if (this.state.building) return
|
|
239
|
+
this.pendingStart = false
|
|
240
|
+
if (!this.acquireLock()) {
|
|
241
|
+
this.state.error = 'another dsh-locator content build is running'
|
|
242
|
+
return
|
|
243
|
+
}
|
|
115
244
|
const generation = ++this.generation
|
|
116
245
|
this.state.building = true
|
|
117
246
|
this.state.error = ''
|
|
@@ -121,6 +250,8 @@ export class ContentIndex {
|
|
|
121
250
|
if (generation !== this.generation) return
|
|
122
251
|
this.state.error = messageOf(error)
|
|
123
252
|
this.state.building = false
|
|
253
|
+
}).finally(() => {
|
|
254
|
+
this.releaseLock()
|
|
124
255
|
})
|
|
125
256
|
}
|
|
126
257
|
|
|
@@ -135,22 +266,60 @@ export class ContentIndex {
|
|
|
135
266
|
}
|
|
136
267
|
|
|
137
268
|
private async incremental(generation: number): Promise<void> {
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
269
|
+
try {
|
|
270
|
+
this.open(false)
|
|
271
|
+
const stored = (this.db!.prepare('PRAGMA user_version').get() as { user_version?: number } | undefined)?.user_version ?? 0
|
|
272
|
+
if (Number(stored) !== SCHEMA_VERSION) return await this.full(generation)
|
|
273
|
+
// A database whose last build never finished (crash, update, forced
|
|
274
|
+
// reload) has a stale byte counter; rebuild instead of trusting it.
|
|
275
|
+
if (this.readFlag('complete') !== 1) return await this.full(generation)
|
|
276
|
+
this.totalBytes = this.readTotalBytes()
|
|
277
|
+
await this.refreshFiles(generation)
|
|
278
|
+
if (generation !== this.generation) return
|
|
279
|
+
await this.refreshSessions(generation)
|
|
280
|
+
if (generation !== this.generation) return
|
|
281
|
+
this.finish()
|
|
282
|
+
} catch {
|
|
283
|
+
if (generation !== this.generation) return
|
|
284
|
+
// A corrupt or outdated database must not disable content search for the
|
|
285
|
+
// rest of the session: rebuild it from scratch instead.
|
|
286
|
+
this.db?.close()
|
|
287
|
+
this.db = null
|
|
288
|
+
await this.full(generation)
|
|
289
|
+
}
|
|
145
290
|
}
|
|
146
291
|
|
|
147
292
|
private open(fresh: boolean): void {
|
|
148
|
-
|
|
293
|
+
const existedDir = existsSync(this.options.stateDir)
|
|
294
|
+
mkdirSync(this.options.stateDir, { recursive: true, mode: 0o700 })
|
|
295
|
+
if (!existedDir) {
|
|
296
|
+
try { chmodSync(this.options.stateDir, 0o700) } catch { /* Windows or a read-only profile */ }
|
|
297
|
+
}
|
|
298
|
+
const dbPath = join(this.options.stateDir, 'content.db')
|
|
299
|
+
const existed = existsSync(dbPath)
|
|
300
|
+
if (existed && !this.ownedDatabase(dbPath)) {
|
|
301
|
+
throw new Error(`content.db is not a dsh-locator database, refusing to touch it: ${dbPath}`)
|
|
302
|
+
}
|
|
303
|
+
let oldFileRemained = false
|
|
304
|
+
if (fresh) {
|
|
305
|
+
// A full rebuild throws every row away. Dropping the FTS tables in place
|
|
306
|
+
// on a multi-GB database blocks the host for about a second, so recreate
|
|
307
|
+
// the file instead: the worker is stopped and the sidecars go with it.
|
|
308
|
+
this.stopWorker()
|
|
309
|
+
try { this.db?.close() } catch { /* ignore */ }
|
|
310
|
+
this.db = null
|
|
311
|
+
for (const suffix of ['', '-wal', '-shm']) {
|
|
312
|
+
try { rmSync(dbPath + suffix, { force: true }) } catch { /* Windows keeps a busy file; DROP below still works */ }
|
|
313
|
+
}
|
|
314
|
+
oldFileRemained = existsSync(dbPath)
|
|
315
|
+
}
|
|
149
316
|
this.db?.close()
|
|
150
|
-
this.db = new DatabaseSync(
|
|
317
|
+
this.db = new DatabaseSync(dbPath)
|
|
318
|
+
try { chmodSync(dbPath, 0o600) } catch { /* Windows or a read-only profile */ }
|
|
151
319
|
this.db.exec('PRAGMA journal_mode = WAL')
|
|
152
320
|
this.db.exec('PRAGMA synchronous = OFF')
|
|
153
|
-
if (fresh) {
|
|
321
|
+
if (fresh && oldFileRemained) {
|
|
322
|
+
// Fallback for the rare recreate failure above: still start from empty.
|
|
154
323
|
this.db.exec('DROP TABLE IF EXISTS docs')
|
|
155
324
|
this.db.exec('DROP TABLE IF EXISTS docs_meta')
|
|
156
325
|
this.db.exec('DROP TABLE IF EXISTS sources')
|
|
@@ -161,6 +330,114 @@ export class ContentIndex {
|
|
|
161
330
|
)`)
|
|
162
331
|
this.db.exec('CREATE TABLE IF NOT EXISTS docs_meta (rowid INTEGER PRIMARY KEY, doc_key TEXT UNIQUE, path TEXT, mtime INTEGER, kind TEXT)')
|
|
163
332
|
this.db.exec('CREATE TABLE IF NOT EXISTS sources (path TEXT PRIMARY KEY, mtime INTEGER)')
|
|
333
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS docs_meta_path ON docs_meta(path)')
|
|
334
|
+
if (fresh) {
|
|
335
|
+
this.db.exec('DROP TABLE IF EXISTS content_meta')
|
|
336
|
+
}
|
|
337
|
+
this.db.exec('CREATE TABLE IF NOT EXISTS content_meta (key TEXT PRIMARY KEY, value INTEGER)')
|
|
338
|
+
if (fresh) {
|
|
339
|
+
this.db.exec("INSERT OR REPLACE INTO content_meta (key, value) VALUES ('total_bytes', 0)")
|
|
340
|
+
this.db.exec("INSERT OR REPLACE INTO content_meta (key, value) VALUES ('complete', 0)")
|
|
341
|
+
}
|
|
342
|
+
if (fresh) this.db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`)
|
|
343
|
+
this.reopenWorker()
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
private reopenWorker(): void {
|
|
347
|
+
if (this.worker === null) return
|
|
348
|
+
this.worker.postMessage({ id: ++this.workerSeq, open: join(this.options.stateDir, 'content.db') })
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
private stopWorker(): void {
|
|
352
|
+
const worker = this.worker
|
|
353
|
+
this.worker = null
|
|
354
|
+
for (const entry of this.pending.values()) entry.resolve([])
|
|
355
|
+
this.pending.clear()
|
|
356
|
+
if (worker !== null) void worker.terminate()
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
private ensureWorker(): void {
|
|
360
|
+
if (this.options.searchWorker !== true || this.worker !== null) return
|
|
361
|
+
// In a TypeScript test run the compiled worker does not exist next to the
|
|
362
|
+
// source; skip spawning and answer queries in-process instead.
|
|
363
|
+
const workerPath = fileURLToPath(new URL('./content-search-worker.js', import.meta.url))
|
|
364
|
+
if (!existsSync(workerPath)) return
|
|
365
|
+
try {
|
|
366
|
+
const worker = new Worker(workerPath, {
|
|
367
|
+
workerData: { dbPath: join(this.options.stateDir, 'content.db') },
|
|
368
|
+
})
|
|
369
|
+
worker.on('message', (message: { id: number; hits?: ContentHit[] }) => {
|
|
370
|
+
const entry = this.pending.get(message.id)
|
|
371
|
+
if (entry === undefined) return
|
|
372
|
+
this.pending.delete(message.id)
|
|
373
|
+
entry.resolve(message.hits ?? [])
|
|
374
|
+
})
|
|
375
|
+
const fail = (): void => {
|
|
376
|
+
if (this.worker !== worker) return
|
|
377
|
+
this.worker = null
|
|
378
|
+
for (const entry of this.pending.values()) entry.reject(new Error('content search worker failed'))
|
|
379
|
+
this.pending.clear()
|
|
380
|
+
}
|
|
381
|
+
worker.on('error', fail)
|
|
382
|
+
worker.on('exit', fail)
|
|
383
|
+
this.worker = worker
|
|
384
|
+
} catch {
|
|
385
|
+
this.worker = null
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
/** Whether `path` is a SQLite database this plugin created (has docs_meta). */
|
|
390
|
+
private ownedDatabase(path: string): boolean {
|
|
391
|
+
let probe: DatabaseSync | undefined
|
|
392
|
+
try {
|
|
393
|
+
probe = new DatabaseSync(path, { readOnly: true })
|
|
394
|
+
const row = probe.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'docs_meta'").get()
|
|
395
|
+
return row !== undefined
|
|
396
|
+
} catch {
|
|
397
|
+
return false
|
|
398
|
+
} finally {
|
|
399
|
+
try { probe?.close() } catch { /* ignore */ }
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
private readFlag(key: string): number | undefined {
|
|
404
|
+
const row = this.db!.prepare('SELECT value FROM content_meta WHERE key = ?').get(key) as { value?: number } | undefined
|
|
405
|
+
return row?.value === undefined ? undefined : Number(row.value)
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
private writeFlag(key: string, value: number): void {
|
|
409
|
+
this.db!.prepare('INSERT OR REPLACE INTO content_meta (key, value) VALUES (?, ?)').run(key, value)
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
/**
|
|
413
|
+
* Body bytes already stored. Reading a persisted counter avoids a full scan
|
|
414
|
+
* of every FTS body (measured ~0.6 s for 1 GB) at the start of each refresh.
|
|
415
|
+
* A database written before the counter existed falls back to one scan.
|
|
416
|
+
*/
|
|
417
|
+
private readTotalBytes(): number {
|
|
418
|
+
const row = this.db!.prepare("SELECT value FROM content_meta WHERE key = 'total_bytes'").get() as { value?: number } | undefined
|
|
419
|
+
if (row?.value !== undefined) return Number(row.value)
|
|
420
|
+
const scanned = Number((this.db!.prepare('SELECT COALESCE(SUM(length(body)), 0) AS n FROM docs').get() as { n: number }).n)
|
|
421
|
+
this.db!.prepare("INSERT OR REPLACE INTO content_meta (key, value) VALUES ('total_bytes', ?)").run(scanned)
|
|
422
|
+
return scanned
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
private writeTotalBytes(): void {
|
|
426
|
+
this.db!.prepare("INSERT OR REPLACE INTO content_meta (key, value) VALUES ('total_bytes', ?)").run(Math.max(0, Math.round(this.totalBytes)))
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
/** Yield to the host event loop once the current ~8 ms work slice is spent. */
|
|
430
|
+
private async breathe(): Promise<void> {
|
|
431
|
+
const now = performance.now()
|
|
432
|
+
if (now < this.sliceDeadline) return
|
|
433
|
+
this.sliceDeadline = now + SLICE_MS
|
|
434
|
+
await yieldToLoop()
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
/** True when a session log has been quiet long enough to be read safely. */
|
|
438
|
+
private settled(mtime: number): boolean {
|
|
439
|
+
const delay = this.options.activeSessionDelayMs ?? 0
|
|
440
|
+
return delay <= 0 || Date.now() - mtime >= delay
|
|
164
441
|
}
|
|
165
442
|
|
|
166
443
|
private finish(): void {
|
|
@@ -170,6 +447,9 @@ export class ContentIndex {
|
|
|
170
447
|
this.state.sessions = Number((this.db!.prepare('SELECT COUNT(*) AS n FROM sources').get() as { n: number }).n)
|
|
171
448
|
this.state.building = false
|
|
172
449
|
this.state.updatedAt = Date.now()
|
|
450
|
+
this.writeTotalBytes()
|
|
451
|
+
this.writeFlag('complete', 1)
|
|
452
|
+
this.reopenWorker()
|
|
173
453
|
}
|
|
174
454
|
|
|
175
455
|
private addDoc(row: DocRow): boolean {
|
|
@@ -189,31 +469,42 @@ export class ContentIndex {
|
|
|
189
469
|
private deleteDoc(key: string): void {
|
|
190
470
|
const row = this.db!.prepare('SELECT rowid FROM docs_meta WHERE doc_key = ?').get(key) as { rowid: number } | undefined
|
|
191
471
|
if (row === undefined) return
|
|
472
|
+
const body = this.db!.prepare('SELECT length(body) AS n FROM docs WHERE rowid = ?').get(row.rowid) as { n: number } | undefined
|
|
473
|
+
if (body !== undefined) this.totalBytes = Math.max(0, this.totalBytes - Number(body.n))
|
|
192
474
|
this.db!.prepare('DELETE FROM docs WHERE rowid = ?').run(row.rowid)
|
|
193
475
|
this.db!.prepare('DELETE FROM docs_meta WHERE rowid = ?').run(row.rowid)
|
|
194
476
|
}
|
|
195
477
|
|
|
196
478
|
private deletePath(path: string): void {
|
|
197
|
-
const rows = this.db!.prepare(
|
|
479
|
+
const rows = this.db!.prepare(`SELECT m.rowid AS rowid, length(d.body) AS n
|
|
480
|
+
FROM docs_meta m JOIN docs d ON d.rowid = m.rowid WHERE m.path = ?`).all(path) as Array<{ rowid: number; n: number }>
|
|
198
481
|
const delDoc = this.db!.prepare('DELETE FROM docs WHERE rowid = ?')
|
|
199
482
|
const delMeta = this.db!.prepare('DELETE FROM docs_meta WHERE rowid = ?')
|
|
200
|
-
for (const row of rows) {
|
|
483
|
+
for (const row of rows) {
|
|
484
|
+
this.totalBytes = Math.max(0, this.totalBytes - Number(row.n))
|
|
485
|
+
delDoc.run(row.rowid)
|
|
486
|
+
delMeta.run(row.rowid)
|
|
487
|
+
}
|
|
201
488
|
}
|
|
202
489
|
|
|
203
490
|
private async indexFiles(generation: number): Promise<void> {
|
|
204
491
|
let after = 0
|
|
205
492
|
let seen = 0
|
|
493
|
+
let scanned = 0
|
|
206
494
|
for (;;) {
|
|
207
495
|
if (generation !== this.generation) return
|
|
208
496
|
const page = await this.rt.dump(after, 2_000)
|
|
209
497
|
for (const entry of page.entries) {
|
|
498
|
+
if ((++scanned & 0xff) === 0) await this.breathe()
|
|
499
|
+
if (generation !== this.generation) return
|
|
210
500
|
if (entry.dir || entry.size <= 0 || entry.size > this.options.maxFileBytes) continue
|
|
211
|
-
if (!TEXT_EXTENSIONS.has(extname(entry.name).toLowerCase()) || contentSkipPath(entry.path)) continue
|
|
501
|
+
if (!TEXT_EXTENSIONS.has(extname(entry.name).toLowerCase()) || contentSkipPath(entry.path, this.options.excludes ?? [])) continue
|
|
212
502
|
if (this.options.maxFiles > 0 && seen >= this.options.maxFiles) break
|
|
213
|
-
const row = this.readFileDoc(entry.path, entry.name, entry.mtime)
|
|
503
|
+
const row = await this.readFileDoc(entry.path, entry.name, entry.mtime)
|
|
214
504
|
if (row === null) continue
|
|
215
505
|
seen++
|
|
216
506
|
if (!this.addDoc(row)) return
|
|
507
|
+
await this.breathe()
|
|
217
508
|
}
|
|
218
509
|
after = page.next
|
|
219
510
|
if (page.done || page.entries.length === 0) break
|
|
@@ -224,58 +515,92 @@ export class ContentIndex {
|
|
|
224
515
|
const meta = new Map<string, number>()
|
|
225
516
|
for (const row of this.db!.prepare("SELECT doc_key, mtime FROM docs_meta WHERE kind = 'file'").all() as Array<{ doc_key: string; mtime: number }>) meta.set(row.doc_key, row.mtime)
|
|
226
517
|
const seen = new Set<string>()
|
|
518
|
+
const deferred: Array<{ path: string; name: string; mtime: number; size: number }> = []
|
|
227
519
|
let after = 0
|
|
228
520
|
let indexed = 0
|
|
521
|
+
let scanned = 0
|
|
229
522
|
for (;;) {
|
|
230
523
|
if (generation !== this.generation) return
|
|
231
524
|
const page = await this.rt.dump(after, 2_000)
|
|
232
525
|
for (const entry of page.entries) {
|
|
526
|
+
if ((++scanned & 0xff) === 0) await this.breathe()
|
|
527
|
+
if (generation !== this.generation) return
|
|
233
528
|
if (entry.dir || entry.size <= 0 || entry.size > this.options.maxFileBytes) continue
|
|
234
|
-
if (!TEXT_EXTENSIONS.has(extname(entry.name).toLowerCase()) || contentSkipPath(entry.path)) continue
|
|
529
|
+
if (!TEXT_EXTENSIONS.has(extname(entry.name).toLowerCase()) || contentSkipPath(entry.path, this.options.excludes ?? [])) continue
|
|
235
530
|
if (this.options.maxFiles > 0 && indexed >= this.options.maxFiles) break
|
|
236
531
|
const key = `file:${entry.path}`
|
|
237
532
|
seen.add(key)
|
|
238
533
|
indexed++
|
|
239
534
|
if (meta.get(key) === entry.mtime) continue
|
|
535
|
+
if (this.wouldExceed(entry.size)) {
|
|
536
|
+
deferred.push({ path: entry.path, name: entry.name, mtime: entry.mtime, size: entry.size })
|
|
537
|
+
continue
|
|
538
|
+
}
|
|
240
539
|
this.deleteDoc(key)
|
|
241
|
-
const row = this.readFileDoc(entry.path, entry.name, entry.mtime)
|
|
242
|
-
if (row !== null && !this.addDoc(row))
|
|
540
|
+
const row = await this.readFileDoc(entry.path, entry.name, entry.mtime)
|
|
541
|
+
if (row !== null && !this.addDoc(row)) {
|
|
542
|
+
deferred.push({ path: entry.path, name: entry.name, mtime: entry.mtime, size: entry.size })
|
|
543
|
+
}
|
|
544
|
+
await this.breathe()
|
|
243
545
|
}
|
|
244
546
|
after = page.next
|
|
245
547
|
if (page.done || page.entries.length === 0) break
|
|
246
548
|
}
|
|
549
|
+
// Removing rows for files that disappeared frees budget before the deferred
|
|
550
|
+
// files are retried; otherwise a stale large entry would block new ones.
|
|
247
551
|
for (const key of meta.keys()) if (!seen.has(key)) this.deleteDoc(key)
|
|
552
|
+
for (const entry of deferred) {
|
|
553
|
+
if (generation !== this.generation) return
|
|
554
|
+
if (this.wouldExceed(entry.size)) { this.state.truncated = true; continue }
|
|
555
|
+
this.deleteDoc(`file:${entry.path}`)
|
|
556
|
+
const row = await this.readFileDoc(entry.path, entry.name, entry.mtime)
|
|
557
|
+
if (row === null || !this.addDoc(row)) { this.state.truncated = true; continue }
|
|
558
|
+
await this.breathe()
|
|
559
|
+
}
|
|
248
560
|
}
|
|
249
561
|
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
if (
|
|
254
|
-
return
|
|
562
|
+
/** Would storing a file of this size overflow the configured body budget? */
|
|
563
|
+
private wouldExceed(size: number): boolean {
|
|
564
|
+
const cap = this.options.maxTotalBytes ?? 0
|
|
565
|
+
if (cap <= 0) return false
|
|
566
|
+
return this.totalBytes + Math.min(size, this.options.maxFileBytes) > cap
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
private async readFileDoc(path: string, name: string, mtime: number): Promise<DocRow | null> {
|
|
570
|
+
let buffer: Buffer
|
|
571
|
+
try { buffer = await readFile(path) } catch { return null }
|
|
572
|
+
if (buffer.includes(0)) return null
|
|
573
|
+
const body = buffer.toString('utf8').slice(0, this.options.maxFileBytes)
|
|
574
|
+
return { key: `file:${path}`, body, kind: 'file', path, title: name, sessionId: null, messageId: null, mtime }
|
|
255
575
|
}
|
|
256
576
|
|
|
257
577
|
private async indexSessions(generation: number): Promise<void> {
|
|
258
|
-
const files =
|
|
578
|
+
const files = (await this.sessionCandidates()).filter(file => this.settled(file.mtime))
|
|
259
579
|
for (const file of files) {
|
|
260
580
|
if (generation !== this.generation) return
|
|
261
581
|
this.deletePath(file.path)
|
|
262
|
-
if (!this.indexSessionFile(file)) return
|
|
582
|
+
if (!(await this.indexSessionFile(file, generation))) return
|
|
263
583
|
this.db!.prepare('INSERT OR REPLACE INTO sources (path, mtime) VALUES (?, ?)').run(file.path, file.mtime)
|
|
584
|
+
await this.breathe()
|
|
264
585
|
}
|
|
265
586
|
}
|
|
266
587
|
|
|
267
588
|
private async refreshSessions(generation: number): Promise<void> {
|
|
268
589
|
const known = new Map<string, number>()
|
|
269
590
|
for (const row of this.db!.prepare('SELECT path, mtime FROM sources').all() as Array<{ path: string; mtime: number }>) known.set(row.path, row.mtime)
|
|
270
|
-
const files =
|
|
591
|
+
const files = await this.sessionCandidates()
|
|
271
592
|
const present = new Set<string>()
|
|
272
593
|
for (const file of files) {
|
|
273
594
|
if (generation !== this.generation) return
|
|
274
595
|
present.add(file.path)
|
|
596
|
+
// The session being written right now is left for a later refresh: keep
|
|
597
|
+
// its existing rows instead of re-reading the whole growing log.
|
|
598
|
+
if (!this.settled(file.mtime)) continue
|
|
275
599
|
if (known.get(file.path) === file.mtime) continue
|
|
276
600
|
this.deletePath(file.path)
|
|
277
|
-
if (!this.indexSessionFile(file)) return
|
|
601
|
+
if (!(await this.indexSessionFile(file, generation))) return
|
|
278
602
|
this.db!.prepare('INSERT OR REPLACE INTO sources (path, mtime) VALUES (?, ?)').run(file.path, file.mtime)
|
|
603
|
+
await this.breathe()
|
|
279
604
|
}
|
|
280
605
|
for (const path of known.keys()) {
|
|
281
606
|
if (present.has(path)) continue
|
|
@@ -284,55 +609,84 @@ export class ContentIndex {
|
|
|
284
609
|
}
|
|
285
610
|
}
|
|
286
611
|
|
|
287
|
-
private indexSessionFile(file: { path: string; mtime: number }): boolean {
|
|
612
|
+
private async indexSessionFile(file: { path: string; mtime: number }, generation: number): Promise<boolean> {
|
|
288
613
|
let text: string
|
|
289
|
-
try { text =
|
|
614
|
+
try { text = (await zstdDecompressAsync(await readFile(file.path))).toString('utf8') } catch { return true }
|
|
615
|
+
if (generation !== this.generation) return false
|
|
290
616
|
const sessionId = basename(dirname(file.path))
|
|
291
617
|
let title = ''
|
|
618
|
+
let parsed = 0
|
|
292
619
|
for (const line of text.split('\n')) {
|
|
293
620
|
if (!line) continue
|
|
621
|
+
if ((++parsed & 0xff) === 0) {
|
|
622
|
+
await this.breathe()
|
|
623
|
+
if (generation !== this.generation) return false
|
|
624
|
+
}
|
|
294
625
|
let event: { type?: string; data?: Record<string, unknown> }
|
|
295
626
|
try { event = JSON.parse(line) } catch { continue }
|
|
296
627
|
const extracted = messageText(event)
|
|
297
628
|
if (extracted === undefined) continue
|
|
298
629
|
if (title === '') title = extracted.text.replace(/\s+/g, ' ').slice(0, 90)
|
|
299
|
-
const body = extracted.text.slice(0,
|
|
630
|
+
const body = extracted.text.slice(0, SESSION_BODY_BYTES)
|
|
300
631
|
const row: DocRow = { key: `session:${sessionId}:${extracted.id}`, body, kind: 'session', path: file.path, title: title || sessionId, sessionId, messageId: extracted.id, mtime: file.mtime }
|
|
301
632
|
if (!this.addDoc(row)) return false
|
|
302
633
|
}
|
|
303
634
|
return true
|
|
304
635
|
}
|
|
305
636
|
|
|
637
|
+
/** Session logs to consider, newest first, minus archived ones by default. */
|
|
638
|
+
private async sessionCandidates(): Promise<Array<{ path: string; mtime: number }>> {
|
|
639
|
+
const files = await sessionFiles(this.sessionRoots())
|
|
640
|
+
const archived = this.options.includeArchived === true ? new Set<string>() : await archivedSessionIds()
|
|
641
|
+
const max = this.options.maxSessions > 0 ? this.options.maxSessions : Number.POSITIVE_INFINITY
|
|
642
|
+
const out: Array<{ path: string; mtime: number }> = []
|
|
643
|
+
for (const file of files) {
|
|
644
|
+
if (out.length >= max) break
|
|
645
|
+
if (archived.has(basename(dirname(file.path)))) continue
|
|
646
|
+
out.push(file)
|
|
647
|
+
}
|
|
648
|
+
return out
|
|
649
|
+
}
|
|
650
|
+
|
|
306
651
|
sessionRoots(): string[] {
|
|
307
652
|
if (this.options.sessionDirs !== undefined) return this.options.sessionDirs
|
|
308
|
-
return [join(
|
|
653
|
+
return [join(dshHome(), 'sessions')]
|
|
309
654
|
}
|
|
310
655
|
|
|
311
|
-
|
|
656
|
+
/**
|
|
657
|
+
* Search file bodies and session messages. With `searchWorker` enabled the
|
|
658
|
+
* query itself runs off the host thread; a failed or wedged worker falls back
|
|
659
|
+
* to the in-process query so a result is still produced.
|
|
660
|
+
* @param query - words or a phrase to find.
|
|
661
|
+
* @param limit - maximum hits.
|
|
662
|
+
* @returns matching documents ordered by relevance.
|
|
663
|
+
*/
|
|
664
|
+
async search(query: string, limit: number): Promise<ContentHit[]> {
|
|
312
665
|
if (this.db === null) return []
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
666
|
+
if (this.options.searchWorker !== true) return this.searchSync(query, limit)
|
|
667
|
+
this.ensureWorker()
|
|
668
|
+
if (this.worker === null) return this.searchSync(query, limit)
|
|
669
|
+
const id = ++this.workerSeq
|
|
670
|
+
const worker = this.worker
|
|
671
|
+
const response = new Promise<ContentHit[]>((resolve, reject) => { this.pending.set(id, { resolve, reject }) })
|
|
672
|
+
worker.postMessage({ id, query, limit })
|
|
673
|
+
let timer: NodeJS.Timeout | undefined
|
|
674
|
+
const timeout = new Promise<ContentHit[]>((_resolve, reject) => {
|
|
675
|
+
timer = setTimeout(() => { this.pending.delete(id); reject(new Error('content search worker timed out')) }, 5_000)
|
|
676
|
+
})
|
|
316
677
|
try {
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
path: String(row.path ?? ''),
|
|
330
|
-
title: String(row.title ?? ''),
|
|
331
|
-
snippet: String(row.snippet ?? row.body ?? '').replace(/[\u0001\u0002]/g, ''),
|
|
332
|
-
...(row.session_id ? { sessionId: String(row.session_id) } : {}),
|
|
333
|
-
...(row.message_id ? { messageId: String(row.message_id) } : {}),
|
|
334
|
-
score: Number(row.score ?? 0),
|
|
335
|
-
}))
|
|
678
|
+
return await Promise.race([response, timeout])
|
|
679
|
+
} catch {
|
|
680
|
+
return this.searchSync(query, limit)
|
|
681
|
+
} finally {
|
|
682
|
+
if (timer !== undefined) clearTimeout(timer)
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
private searchSync(query: string, limit: number): ContentHit[] {
|
|
687
|
+
if (this.db === null) return []
|
|
688
|
+
try {
|
|
689
|
+
return searchContent(this.db, query, limit)
|
|
336
690
|
} catch (error) {
|
|
337
691
|
this.state.error = messageOf(error)
|
|
338
692
|
return []
|
|
@@ -340,6 +694,22 @@ export class ContentIndex {
|
|
|
340
694
|
}
|
|
341
695
|
}
|
|
342
696
|
|
|
697
|
+
function dshHome(): string {
|
|
698
|
+
return process.env.DSH_HOME || join(homedir(), '.dsh')
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
/** Archived session ids recorded by the Workspace registry; empty when absent. */
|
|
702
|
+
async function archivedSessionIds(): Promise<Set<string>> {
|
|
703
|
+
try {
|
|
704
|
+
const raw = await readFile(join(dshHome(), 'storages', 'workspace.json'), 'utf8')
|
|
705
|
+
const parsed = JSON.parse(raw) as { global?: { archivedSessionIds?: unknown } }
|
|
706
|
+
const ids = parsed.global?.archivedSessionIds
|
|
707
|
+
return new Set(Array.isArray(ids) ? ids.filter((id): id is string => typeof id === 'string') : [])
|
|
708
|
+
} catch {
|
|
709
|
+
return new Set()
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
|
|
343
713
|
function messageText(event: { type?: string; data?: Record<string, unknown> }): { text: string; id: string } | undefined {
|
|
344
714
|
if (event.type !== 'user/message' && event.type !== 'assistant/message') return undefined
|
|
345
715
|
const data = event.data ?? {}
|
|
@@ -360,26 +730,41 @@ function messageText(event: { type?: string; data?: Record<string, unknown> }):
|
|
|
360
730
|
return { text, id: id || `seq-${String(data.seq ?? '')}` }
|
|
361
731
|
}
|
|
362
732
|
|
|
363
|
-
function sessionFiles(roots: string[]): Array<{ path: string; mtime: number }
|
|
733
|
+
async function sessionFiles(roots: string[]): Promise<Array<{ path: string; mtime: number }>> {
|
|
364
734
|
const out: Array<{ path: string; mtime: number }> = []
|
|
365
|
-
const walk = (root: string, depth: number): void => {
|
|
366
|
-
if (depth > 4
|
|
735
|
+
const walk = async (root: string, depth: number): Promise<void> => {
|
|
736
|
+
if (depth > 4) return
|
|
367
737
|
let entries
|
|
368
|
-
try { entries =
|
|
738
|
+
try { entries = await readdir(root, { withFileTypes: true }) } catch { return }
|
|
369
739
|
for (const entry of entries) {
|
|
370
740
|
const path = join(root, entry.name)
|
|
371
|
-
if (entry.isDirectory()) { walk(path, depth + 1); continue }
|
|
741
|
+
if (entry.isDirectory()) { await walk(path, depth + 1); continue }
|
|
372
742
|
if (!entry.isFile() || !entry.name.endsWith('.v4.jsonl.zstd')) continue
|
|
373
|
-
try { out.push({ path, mtime:
|
|
743
|
+
try { out.push({ path, mtime: (await stat(path)).mtimeMs }) } catch { /* ignore */ }
|
|
374
744
|
}
|
|
375
745
|
}
|
|
376
|
-
for (const root of roots) walk(root, 0)
|
|
746
|
+
for (const root of roots) await walk(root, 0)
|
|
377
747
|
out.sort((a, b) => b.mtime - a.mtime)
|
|
378
748
|
return out
|
|
379
749
|
}
|
|
380
750
|
|
|
381
|
-
function contentSkipPath(path: string): boolean {
|
|
382
|
-
|
|
751
|
+
function contentSkipPath(path: string, excludes: string[]): boolean {
|
|
752
|
+
const normalized = path.replace(/\\/g, '/')
|
|
753
|
+
// The plugin's own session/config tree is never content.
|
|
754
|
+
if (normalized === '/.dsh' || normalized.includes('/.dsh/')) return true
|
|
755
|
+
if (excludes.length === 0) return false
|
|
756
|
+
const lower = normalized.toLowerCase()
|
|
757
|
+
const segments = lower.split('/')
|
|
758
|
+
for (const raw of excludes) {
|
|
759
|
+
const needle = raw.trim().toLowerCase()
|
|
760
|
+
if (needle === '') continue
|
|
761
|
+
if (needle.includes('/')) {
|
|
762
|
+
if (lower.includes(needle)) return true
|
|
763
|
+
continue
|
|
764
|
+
}
|
|
765
|
+
if (segments.includes(needle)) return true
|
|
766
|
+
}
|
|
767
|
+
return false
|
|
383
768
|
}
|
|
384
769
|
|
|
385
770
|
function messageOf(error: unknown): string {
|