dsh-locator 0.8.2 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README-es.md +30 -5
- package/README-hi.md +31 -5
- package/README-pt.md +31 -5
- package/README.en.md +31 -5
- package/README.md +31 -5
- package/bin/darwin-arm64/indexd +0 -0
- package/bin/darwin-x64/indexd +0 -0
- package/bin/linux-arm64/indexd +0 -0
- package/bin/linux-x64/indexd +0 -0
- package/bin/win32-arm64/indexd.exe +0 -0
- package/bin/win32-x64/indexd.exe +0 -0
- package/client.js +1198 -117
- package/lib/combined.d.ts +20 -3
- package/lib/combined.d.ts.map +1 -1
- package/lib/combined.js +85 -15
- package/lib/combined.js.map +1 -1
- package/lib/content-search-worker.d.ts +8 -3
- package/lib/content-search-worker.d.ts.map +1 -1
- package/lib/content-search-worker.js +54 -27
- package/lib/content-search-worker.js.map +1 -1
- package/lib/content-search.d.ts +127 -7
- package/lib/content-search.d.ts.map +1 -1
- package/lib/content-search.js +571 -36
- package/lib/content-search.js.map +1 -1
- package/lib/content.d.ts +83 -5
- package/lib/content.d.ts.map +1 -1
- package/lib/content.js +351 -87
- package/lib/content.js.map +1 -1
- package/lib/export.d.ts +61 -0
- package/lib/export.d.ts.map +1 -1
- package/lib/export.js +217 -52
- package/lib/export.js.map +1 -1
- package/lib/find.d.ts.map +1 -1
- package/lib/find.js +73 -16
- package/lib/find.js.map +1 -1
- package/lib/http.d.ts +38 -2
- package/lib/http.d.ts.map +1 -1
- package/lib/http.js +279 -82
- package/lib/http.js.map +1 -1
- package/lib/reveal.d.ts +69 -3
- package/lib/reveal.d.ts.map +1 -1
- package/lib/reveal.js +140 -11
- package/lib/reveal.js.map +1 -1
- package/lib/tools.d.ts +30 -0
- package/lib/tools.d.ts.map +1 -1
- package/lib/tools.js +65 -3
- package/lib/tools.js.map +1 -1
- package/package.json +1 -1
- package/src/combined.ts +107 -17
- package/src/content-search-worker.ts +52 -24
- package/src/content-search.ts +710 -36
- package/src/content.ts +349 -73
- package/src/export.ts +240 -50
- package/src/find.ts +83 -16
- package/src/http.ts +271 -83
- package/src/reveal.ts +148 -10
- package/src/tools.ts +83 -6
package/lib/content.js
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* alone unless `includeArchived` is set.
|
|
16
16
|
* @module dsh-locator/content
|
|
17
17
|
*/
|
|
18
|
-
import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
18
|
+
import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
19
19
|
import { readdir, readFile, stat } from 'node:fs/promises';
|
|
20
20
|
import { homedir } from 'node:os';
|
|
21
21
|
import { basename, dirname, extname, join } from 'node:path';
|
|
@@ -52,7 +52,7 @@ export class ContentIndex {
|
|
|
52
52
|
rt;
|
|
53
53
|
options;
|
|
54
54
|
db = null;
|
|
55
|
-
state = { building: false, files: 0, sessions: 0, messages: 0, truncated: false, error: '', updatedAt: 0 };
|
|
55
|
+
state = { building: false, files: 0, sessions: 0, messages: 0, truncated: false, error: '', warnings: [], updatedAt: 0 };
|
|
56
56
|
generation = 0;
|
|
57
57
|
totalBytes = 0;
|
|
58
58
|
sliceDeadline = 0;
|
|
@@ -62,6 +62,8 @@ export class ContentIndex {
|
|
|
62
62
|
workerSeq = 0;
|
|
63
63
|
pendingStart = false;
|
|
64
64
|
dbIdentity = null;
|
|
65
|
+
/** Session id → indexed session log path, for session-scoped content search. */
|
|
66
|
+
sessionPaths = new Map();
|
|
65
67
|
pending = new Map();
|
|
66
68
|
constructor(rt, options) {
|
|
67
69
|
this.rt = rt;
|
|
@@ -70,7 +72,7 @@ export class ContentIndex {
|
|
|
70
72
|
status() {
|
|
71
73
|
if (this.db !== null && !this.state.building && !this.databaseIsCurrent())
|
|
72
74
|
this.recoverExternal();
|
|
73
|
-
return { ...this.state, building: this.state.building || this.pendingStart };
|
|
75
|
+
return { ...this.state, warnings: [...this.state.warnings], building: this.state.building || this.pendingStart };
|
|
74
76
|
}
|
|
75
77
|
/**
|
|
76
78
|
* Report a lazy build as in progress before the asynchronous start reaches
|
|
@@ -110,6 +112,7 @@ export class ContentIndex {
|
|
|
110
112
|
this.releaseLock();
|
|
111
113
|
this.pendingStart = false;
|
|
112
114
|
this.dbIdentity = null;
|
|
115
|
+
this.sessionPaths.clear();
|
|
113
116
|
}
|
|
114
117
|
/** Build everything from scratch. */
|
|
115
118
|
start() { this.begin('full'); }
|
|
@@ -200,6 +203,7 @@ export class ContentIndex {
|
|
|
200
203
|
this.state.building = true;
|
|
201
204
|
this.state.error = '';
|
|
202
205
|
this.state.truncated = false;
|
|
206
|
+
this.state.warnings = [];
|
|
203
207
|
const run = mode === 'full' ? this.full(generation) : this.incremental(generation);
|
|
204
208
|
void run.catch(error => {
|
|
205
209
|
if (generation !== this.generation)
|
|
@@ -261,8 +265,21 @@ export class ContentIndex {
|
|
|
261
265
|
}
|
|
262
266
|
const dbPath = join(this.options.stateDir, 'content.db');
|
|
263
267
|
const existed = existsSync(dbPath);
|
|
264
|
-
if (existed
|
|
265
|
-
|
|
268
|
+
if (existed) {
|
|
269
|
+
const verdict = this.classifyDatabase(dbPath);
|
|
270
|
+
if (verdict === 'foreign') {
|
|
271
|
+
throw new Error(`content.db is not a dsh-locator database, refusing to touch it: ${dbPath}`);
|
|
272
|
+
}
|
|
273
|
+
if (verdict !== 'ok') {
|
|
274
|
+
// A corrupt, empty or older-schema database must not disable content
|
|
275
|
+
// search for the rest of the session. Keep the bad file for diagnosis
|
|
276
|
+
// (rename, never delete bytes we did not create) and rebuild fresh.
|
|
277
|
+
// During an incremental open this throws so the caller falls back to a
|
|
278
|
+
// full rebuild; a full open simply continues with a clean file.
|
|
279
|
+
this.quarantineDatabase(dbPath, verdict);
|
|
280
|
+
if (!fresh)
|
|
281
|
+
throw new Error('content database is incompatible; rebuilding');
|
|
282
|
+
}
|
|
266
283
|
}
|
|
267
284
|
let oldFileRemained = false;
|
|
268
285
|
if (fresh) {
|
|
@@ -315,57 +332,164 @@ export class ContentIndex {
|
|
|
315
332
|
}
|
|
316
333
|
if (fresh)
|
|
317
334
|
this.db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`);
|
|
335
|
+
this.refreshSessionPaths();
|
|
318
336
|
this.reopenWorker();
|
|
319
337
|
}
|
|
338
|
+
/**
|
|
339
|
+
* Decide what an existing `content.db` is before touching it:
|
|
340
|
+
* - `foreign`: no dsh-locator table at all — refuse to modify it.
|
|
341
|
+
* - `corrupt`: zero bytes, unreadable, or not a SQLite database.
|
|
342
|
+
* - `incompatible`: has our tables but a different schema version/shape.
|
|
343
|
+
* - `ok`: safe to reuse.
|
|
344
|
+
*/
|
|
345
|
+
classifyDatabase(path) {
|
|
346
|
+
let size = 0;
|
|
347
|
+
try {
|
|
348
|
+
size = statSync(path).size;
|
|
349
|
+
}
|
|
350
|
+
catch {
|
|
351
|
+
return 'corrupt';
|
|
352
|
+
}
|
|
353
|
+
if (size === 0)
|
|
354
|
+
return 'corrupt';
|
|
355
|
+
let probe;
|
|
356
|
+
try {
|
|
357
|
+
probe = new DatabaseSync(path, { readOnly: true });
|
|
358
|
+
const names = new Set(probe.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").all()
|
|
359
|
+
.map(row => row.name)
|
|
360
|
+
.filter((name) => typeof name === 'string'));
|
|
361
|
+
const owned = ['docs', 'docs_meta', 'sources', 'content_meta'];
|
|
362
|
+
if (!owned.some(name => names.has(name)))
|
|
363
|
+
return 'foreign';
|
|
364
|
+
if (!names.has('docs') || !names.has('docs_meta') || !names.has('sources') || !names.has('content_meta'))
|
|
365
|
+
return 'incompatible';
|
|
366
|
+
const version = probe.prepare('PRAGMA user_version').get()?.user_version ?? 0;
|
|
367
|
+
if (Number(version) !== SCHEMA_VERSION)
|
|
368
|
+
return 'incompatible';
|
|
369
|
+
return 'ok';
|
|
370
|
+
}
|
|
371
|
+
catch {
|
|
372
|
+
return 'corrupt';
|
|
373
|
+
}
|
|
374
|
+
finally {
|
|
375
|
+
try {
|
|
376
|
+
probe?.close();
|
|
377
|
+
}
|
|
378
|
+
catch { /* ignore */ }
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
/**
|
|
382
|
+
* Move a broken/outdated database aside instead of destroying it. The
|
|
383
|
+
* timestamped name keeps the bytes around for inspection; if the rename
|
|
384
|
+
* fails (another process still holds it on Windows) a fresh file is still
|
|
385
|
+
* created so content search recovers.
|
|
386
|
+
*/
|
|
387
|
+
quarantineDatabase(dbPath, verdict) {
|
|
388
|
+
this.stopWorker();
|
|
389
|
+
try {
|
|
390
|
+
this.db?.close();
|
|
391
|
+
}
|
|
392
|
+
catch { /* ignore */ }
|
|
393
|
+
this.db = null;
|
|
394
|
+
const target = `${dbPath}.corrupt-${Date.now()}`;
|
|
395
|
+
try {
|
|
396
|
+
renameSync(dbPath, target);
|
|
397
|
+
this.addWarning(verdict === 'corrupt' ? 'content-db-corrupt' : 'content-db-incompatible');
|
|
398
|
+
}
|
|
399
|
+
catch {
|
|
400
|
+
try {
|
|
401
|
+
rmSync(dbPath, { force: true });
|
|
402
|
+
}
|
|
403
|
+
catch { /* ignore */ }
|
|
404
|
+
this.addWarning(verdict === 'corrupt' ? 'content-db-corrupt' : 'content-db-incompatible');
|
|
405
|
+
}
|
|
406
|
+
for (const suffix of ['-wal', '-shm']) {
|
|
407
|
+
try {
|
|
408
|
+
rmSync(dbPath + suffix, { force: true });
|
|
409
|
+
}
|
|
410
|
+
catch { /* ignore */ }
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
/** Record one non-fatal build warning (deduplicated, bounded). */
|
|
414
|
+
addWarning(warning) {
|
|
415
|
+
if (this.state.warnings.includes(warning))
|
|
416
|
+
return;
|
|
417
|
+
if (this.state.warnings.length >= 20)
|
|
418
|
+
return;
|
|
419
|
+
this.state.warnings.push(warning);
|
|
420
|
+
}
|
|
320
421
|
reopenWorker() {
|
|
321
422
|
if (this.worker === null)
|
|
322
423
|
return;
|
|
323
|
-
|
|
424
|
+
try {
|
|
425
|
+
this.worker.postMessage({ id: ++this.workerSeq, open: join(this.options.stateDir, 'content.db') });
|
|
426
|
+
}
|
|
427
|
+
catch { /* a dying worker is replaced on the next query */ }
|
|
324
428
|
}
|
|
325
429
|
stopWorker() {
|
|
326
430
|
const worker = this.worker;
|
|
327
431
|
this.worker = null;
|
|
328
|
-
for (const
|
|
329
|
-
|
|
432
|
+
for (const resolve of this.pending.values())
|
|
433
|
+
resolve(workerFailureResult('content-search-worker-failed'));
|
|
330
434
|
this.pending.clear();
|
|
331
435
|
if (worker !== null)
|
|
332
436
|
void worker.terminate();
|
|
333
437
|
}
|
|
438
|
+
/**
|
|
439
|
+
* Whether a usable search worker exists, creating one when needed.
|
|
440
|
+
* @returns false when the worker cannot be created at all; that is the only
|
|
441
|
+
* case in which an in-process (sync) query is allowed.
|
|
442
|
+
*/
|
|
334
443
|
ensureWorker() {
|
|
335
|
-
if (this.options.searchWorker !== true
|
|
336
|
-
return;
|
|
444
|
+
if (this.options.searchWorker !== true)
|
|
445
|
+
return false;
|
|
446
|
+
if (this.worker !== null)
|
|
447
|
+
return true;
|
|
337
448
|
// In a TypeScript test run the compiled worker does not exist next to the
|
|
338
|
-
// source;
|
|
449
|
+
// source; answer queries in-process instead (budgeted, see searchSync).
|
|
339
450
|
const workerPath = fileURLToPath(new URL('./content-search-worker.js', import.meta.url));
|
|
340
|
-
|
|
341
|
-
|
|
451
|
+
const factory = this.options.searchWorkerFactory;
|
|
452
|
+
if (factory === undefined && !existsSync(workerPath))
|
|
453
|
+
return false;
|
|
342
454
|
try {
|
|
343
|
-
const worker =
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
worker.on('message', (message) => {
|
|
347
|
-
const entry = this.pending.get(message.id);
|
|
348
|
-
if (entry === undefined)
|
|
349
|
-
return;
|
|
350
|
-
this.pending.delete(message.id);
|
|
351
|
-
entry.resolve(message.hits ?? []);
|
|
352
|
-
});
|
|
455
|
+
const worker = factory !== undefined
|
|
456
|
+
? factory(workerPath)
|
|
457
|
+
: new Worker(workerPath, { workerData: { dbPath: join(this.options.stateDir, 'content.db') } });
|
|
458
|
+
worker.on('message', (message) => { this.onWorkerMessage(message); });
|
|
353
459
|
const fail = () => {
|
|
354
460
|
if (this.worker !== worker)
|
|
355
461
|
return;
|
|
356
462
|
this.worker = null;
|
|
357
|
-
for (const
|
|
358
|
-
|
|
463
|
+
for (const resolve of this.pending.values())
|
|
464
|
+
resolve(workerFailureResult('content-search-worker-failed'));
|
|
359
465
|
this.pending.clear();
|
|
360
466
|
};
|
|
361
467
|
worker.on('error', fail);
|
|
362
468
|
worker.on('exit', fail);
|
|
363
469
|
this.worker = worker;
|
|
470
|
+
return true;
|
|
364
471
|
}
|
|
365
472
|
catch {
|
|
366
473
|
this.worker = null;
|
|
474
|
+
return false;
|
|
367
475
|
}
|
|
368
476
|
}
|
|
477
|
+
/**
|
|
478
|
+
* Resolve one worker reply. A malformed reply is reported as a protocol
|
|
479
|
+
* error; the query is never retried on the host thread.
|
|
480
|
+
*/
|
|
481
|
+
onWorkerMessage(message) {
|
|
482
|
+
if (typeof message !== 'object' || message === null)
|
|
483
|
+
return;
|
|
484
|
+
const record = message;
|
|
485
|
+
if (typeof record.id !== 'number')
|
|
486
|
+
return;
|
|
487
|
+
const resolve = this.pending.get(record.id);
|
|
488
|
+
if (resolve === undefined)
|
|
489
|
+
return;
|
|
490
|
+
this.pending.delete(record.id);
|
|
491
|
+
resolve(isContentSearchResult(record.result) ? record.result : workerFailureResult('content-search-worker-protocol-error'));
|
|
492
|
+
}
|
|
369
493
|
identityOf(path) {
|
|
370
494
|
try {
|
|
371
495
|
const info = statSync(path);
|
|
@@ -401,6 +525,7 @@ export class ContentIndex {
|
|
|
401
525
|
this.dbIdentity = null;
|
|
402
526
|
this.stopWorker();
|
|
403
527
|
this.totalBytes = 0;
|
|
528
|
+
this.sessionPaths.clear();
|
|
404
529
|
this.state.updatedAt = 0;
|
|
405
530
|
this.state.error = '';
|
|
406
531
|
this.state.truncated = false;
|
|
@@ -414,24 +539,6 @@ export class ContentIndex {
|
|
|
414
539
|
const scanning = this.rt.status?.scanning;
|
|
415
540
|
return scanning !== undefined && scanning !== null;
|
|
416
541
|
}
|
|
417
|
-
/** Whether `path` is a SQLite database this plugin created (has docs_meta). */
|
|
418
|
-
ownedDatabase(path) {
|
|
419
|
-
let probe;
|
|
420
|
-
try {
|
|
421
|
-
probe = new DatabaseSync(path, { readOnly: true });
|
|
422
|
-
const row = probe.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'docs_meta'").get();
|
|
423
|
-
return row !== undefined;
|
|
424
|
-
}
|
|
425
|
-
catch {
|
|
426
|
-
return false;
|
|
427
|
-
}
|
|
428
|
-
finally {
|
|
429
|
-
try {
|
|
430
|
-
probe?.close();
|
|
431
|
-
}
|
|
432
|
-
catch { /* ignore */ }
|
|
433
|
-
}
|
|
434
|
-
}
|
|
435
542
|
readFlag(key) {
|
|
436
543
|
const row = this.db.prepare('SELECT value FROM content_meta WHERE key = ?').get(key);
|
|
437
544
|
return row?.value === undefined ? undefined : Number(row.value);
|
|
@@ -477,11 +584,41 @@ export class ContentIndex {
|
|
|
477
584
|
this.state.updatedAt = Date.now();
|
|
478
585
|
this.writeTotalBytes();
|
|
479
586
|
this.writeFlag('complete', 1);
|
|
587
|
+
this.refreshSessionPaths();
|
|
480
588
|
this.reopenWorker();
|
|
481
589
|
}
|
|
590
|
+
/**
|
|
591
|
+
* Rebuild the session id → session log path map from the persisted sources.
|
|
592
|
+
* Runs after `open()` and after every build/refresh, so session-scoped
|
|
593
|
+
* searches work against an existing database before the first refresh ends.
|
|
594
|
+
*/
|
|
595
|
+
refreshSessionPaths() {
|
|
596
|
+
this.sessionPaths.clear();
|
|
597
|
+
if (this.db === null)
|
|
598
|
+
return;
|
|
599
|
+
try {
|
|
600
|
+
const rows = this.db.prepare('SELECT path FROM sources').all();
|
|
601
|
+
for (const row of rows) {
|
|
602
|
+
const path = row.path;
|
|
603
|
+
if (typeof path !== 'string' || path === '')
|
|
604
|
+
continue;
|
|
605
|
+
const sessionId = basename(dirname(path));
|
|
606
|
+
if (sessionId !== '')
|
|
607
|
+
this.sessionPaths.set(sessionId, path);
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
catch {
|
|
611
|
+
// A database from an older schema cannot answer session-scoped queries
|
|
612
|
+
// until its next rebuild; leave the map empty instead of failing boot.
|
|
613
|
+
}
|
|
614
|
+
}
|
|
482
615
|
addDoc(row) {
|
|
483
616
|
const cap = this.options.maxTotalBytes ?? 0;
|
|
484
|
-
|
|
617
|
+
// Replacing an existing document frees its old body first: an unchanged
|
|
618
|
+
// file rewritten at refresh time must not be rejected because its stale
|
|
619
|
+
// copy still occupies the budget.
|
|
620
|
+
const previous = this.storedBytes(row.key);
|
|
621
|
+
if (cap > 0 && this.totalBytes - previous + row.body.length > cap) {
|
|
485
622
|
this.state.truncated = true;
|
|
486
623
|
return false;
|
|
487
624
|
}
|
|
@@ -492,6 +629,12 @@ export class ContentIndex {
|
|
|
492
629
|
this.totalBytes += row.body.length;
|
|
493
630
|
return true;
|
|
494
631
|
}
|
|
632
|
+
/** Body bytes stored for one document key (0 when it is not indexed). */
|
|
633
|
+
storedBytes(key) {
|
|
634
|
+
const row = this.db.prepare(`SELECT COALESCE(length(d.body), 0) AS n
|
|
635
|
+
FROM docs_meta m JOIN docs d ON d.rowid = m.rowid WHERE m.doc_key = ?`).get(key);
|
|
636
|
+
return Number(row?.n ?? 0);
|
|
637
|
+
}
|
|
495
638
|
deleteDoc(key) {
|
|
496
639
|
const row = this.db.prepare('SELECT rowid FROM docs_meta WHERE doc_key = ?').get(key);
|
|
497
640
|
if (row === undefined)
|
|
@@ -517,6 +660,7 @@ export class ContentIndex {
|
|
|
517
660
|
let after = 0;
|
|
518
661
|
let seen = 0;
|
|
519
662
|
let scanned = 0;
|
|
663
|
+
let capped = false;
|
|
520
664
|
for (;;) {
|
|
521
665
|
if (generation !== this.generation)
|
|
522
666
|
return;
|
|
@@ -532,8 +676,10 @@ export class ContentIndex {
|
|
|
532
676
|
continue;
|
|
533
677
|
if (!TEXT_EXTENSIONS.has(extname(entry.name).toLowerCase()) || contentSkipPath(entry.path, this.options.excludes ?? []))
|
|
534
678
|
continue;
|
|
535
|
-
if (this.options.maxFiles > 0 && seen >= this.options.maxFiles)
|
|
679
|
+
if (this.options.maxFiles > 0 && seen >= this.options.maxFiles) {
|
|
680
|
+
capped = true;
|
|
536
681
|
break;
|
|
682
|
+
}
|
|
537
683
|
const row = await this.readFileDoc(entry.path, entry.name, entry.mtime);
|
|
538
684
|
if (row === null)
|
|
539
685
|
continue;
|
|
@@ -542,6 +688,8 @@ export class ContentIndex {
|
|
|
542
688
|
return;
|
|
543
689
|
await this.breathe();
|
|
544
690
|
}
|
|
691
|
+
if (capped)
|
|
692
|
+
break;
|
|
545
693
|
after = page.next;
|
|
546
694
|
if (page.done || page.entries.length === 0)
|
|
547
695
|
break;
|
|
@@ -554,8 +702,13 @@ export class ContentIndex {
|
|
|
554
702
|
const seen = new Set();
|
|
555
703
|
const deferred = [];
|
|
556
704
|
let after = 0;
|
|
705
|
+
// `maxFiles` counts documents that are actually stored. A file whose read
|
|
706
|
+
// failed is not stored, so it must not consume a slot: counting it here used
|
|
707
|
+
// to stop the loop early (and the deletion pass then removed every trailing
|
|
708
|
+
// document that was never reached). `indexed` only moves on success.
|
|
557
709
|
let indexed = 0;
|
|
558
710
|
let scanned = 0;
|
|
711
|
+
let capped = false;
|
|
559
712
|
for (;;) {
|
|
560
713
|
if (generation !== this.generation)
|
|
561
714
|
return;
|
|
@@ -571,24 +724,35 @@ export class ContentIndex {
|
|
|
571
724
|
continue;
|
|
572
725
|
if (!TEXT_EXTENSIONS.has(extname(entry.name).toLowerCase()) || contentSkipPath(entry.path, this.options.excludes ?? []))
|
|
573
726
|
continue;
|
|
574
|
-
if (this.options.maxFiles > 0 && indexed >= this.options.maxFiles)
|
|
727
|
+
if (this.options.maxFiles > 0 && indexed >= this.options.maxFiles) {
|
|
728
|
+
capped = true;
|
|
575
729
|
break;
|
|
730
|
+
}
|
|
576
731
|
const key = `file:${entry.path}`;
|
|
577
732
|
seen.add(key);
|
|
578
|
-
|
|
579
|
-
|
|
733
|
+
if (meta.get(key) === entry.mtime) {
|
|
734
|
+
// Unchanged and already stored: it still occupies one of the slots.
|
|
735
|
+
indexed++;
|
|
580
736
|
continue;
|
|
581
|
-
|
|
737
|
+
}
|
|
738
|
+
if (this.wouldExceed(entry.size, key)) {
|
|
582
739
|
deferred.push({ path: entry.path, name: entry.name, mtime: entry.mtime, size: entry.size });
|
|
583
740
|
continue;
|
|
584
741
|
}
|
|
585
|
-
|
|
742
|
+
// Read first, replace only on success. A transient read failure must
|
|
743
|
+
// keep the document indexed by the previous refresh, not delete it.
|
|
586
744
|
const row = await this.readFileDoc(entry.path, entry.name, entry.mtime);
|
|
587
|
-
if (row
|
|
745
|
+
if (row === null)
|
|
746
|
+
continue;
|
|
747
|
+
if (!this.addDoc(row)) {
|
|
588
748
|
deferred.push({ path: entry.path, name: entry.name, mtime: entry.mtime, size: entry.size });
|
|
749
|
+
continue;
|
|
589
750
|
}
|
|
751
|
+
indexed++;
|
|
590
752
|
await this.breathe();
|
|
591
753
|
}
|
|
754
|
+
if (capped)
|
|
755
|
+
break;
|
|
592
756
|
after = page.next;
|
|
593
757
|
if (page.done || page.entries.length === 0)
|
|
594
758
|
break;
|
|
@@ -601,13 +765,14 @@ export class ContentIndex {
|
|
|
601
765
|
for (const entry of deferred) {
|
|
602
766
|
if (generation !== this.generation)
|
|
603
767
|
return;
|
|
604
|
-
if (this.wouldExceed(entry.size)) {
|
|
768
|
+
if (this.wouldExceed(entry.size, `file:${entry.path}`)) {
|
|
605
769
|
this.state.truncated = true;
|
|
606
770
|
continue;
|
|
607
771
|
}
|
|
608
|
-
this.deleteDoc(`file:${entry.path}`);
|
|
609
772
|
const row = await this.readFileDoc(entry.path, entry.name, entry.mtime);
|
|
610
|
-
if (row === null
|
|
773
|
+
if (row === null)
|
|
774
|
+
continue;
|
|
775
|
+
if (!this.addDoc(row)) {
|
|
611
776
|
this.state.truncated = true;
|
|
612
777
|
continue;
|
|
613
778
|
}
|
|
@@ -615,11 +780,12 @@ export class ContentIndex {
|
|
|
615
780
|
}
|
|
616
781
|
}
|
|
617
782
|
/** Would storing a file of this size overflow the configured body budget? */
|
|
618
|
-
wouldExceed(size) {
|
|
783
|
+
wouldExceed(size, docKey) {
|
|
619
784
|
const cap = this.options.maxTotalBytes ?? 0;
|
|
620
785
|
if (cap <= 0)
|
|
621
786
|
return false;
|
|
622
|
-
|
|
787
|
+
const previous = docKey === undefined ? 0 : this.storedBytes(docKey);
|
|
788
|
+
return this.totalBytes - previous + Math.min(size, this.options.maxFileBytes) > cap;
|
|
623
789
|
}
|
|
624
790
|
async readFileDoc(path, name, mtime) {
|
|
625
791
|
let buffer;
|
|
@@ -639,8 +805,15 @@ export class ContentIndex {
|
|
|
639
805
|
for (const file of files) {
|
|
640
806
|
if (generation !== this.generation)
|
|
641
807
|
return;
|
|
808
|
+
const read = await this.readSessionText(file);
|
|
809
|
+
if (!read.ok) {
|
|
810
|
+
// A log that cannot be read or decompressed does not contribute rows;
|
|
811
|
+
// the build continues with the remaining sessions.
|
|
812
|
+
this.addWarning('content-session-log-unreadable');
|
|
813
|
+
continue;
|
|
814
|
+
}
|
|
642
815
|
this.deletePath(file.path);
|
|
643
|
-
if (!(await this.
|
|
816
|
+
if (!(await this.indexSessionText(file, read.text, generation)))
|
|
644
817
|
return;
|
|
645
818
|
this.db.prepare('INSERT OR REPLACE INTO sources (path, mtime) VALUES (?, ?)').run(file.path, file.mtime);
|
|
646
819
|
await this.breathe();
|
|
@@ -662,8 +835,16 @@ export class ContentIndex {
|
|
|
662
835
|
continue;
|
|
663
836
|
if (known.get(file.path) === file.mtime)
|
|
664
837
|
continue;
|
|
838
|
+
// Decode the whole log before touching the old rows. On failure keep the
|
|
839
|
+
// previous messages and the previous sources.mtime, so the next refresh
|
|
840
|
+
// retries the same file instead of treating it as empty.
|
|
841
|
+
const read = await this.readSessionText(file);
|
|
842
|
+
if (!read.ok) {
|
|
843
|
+
this.addWarning('content-session-log-unreadable');
|
|
844
|
+
continue;
|
|
845
|
+
}
|
|
665
846
|
this.deletePath(file.path);
|
|
666
|
-
if (!(await this.
|
|
847
|
+
if (!(await this.indexSessionText(file, read.text, generation)))
|
|
667
848
|
return;
|
|
668
849
|
this.db.prepare('INSERT OR REPLACE INTO sources (path, mtime) VALUES (?, ?)').run(file.path, file.mtime);
|
|
669
850
|
await this.breathe();
|
|
@@ -675,17 +856,23 @@ export class ContentIndex {
|
|
|
675
856
|
this.db.prepare('DELETE FROM sources WHERE path = ?').run(path);
|
|
676
857
|
}
|
|
677
858
|
}
|
|
678
|
-
|
|
859
|
+
/** Read and decompress one session log completely; no side effects on rows. */
|
|
860
|
+
async readSessionText(file) {
|
|
679
861
|
let text;
|
|
680
862
|
try {
|
|
681
863
|
text = (await zstdDecompressAsync(await readFile(file.path))).toString('utf8');
|
|
682
864
|
}
|
|
683
|
-
catch {
|
|
684
|
-
return
|
|
865
|
+
catch (error) {
|
|
866
|
+
return { ok: false, error: messageOf(error) };
|
|
685
867
|
}
|
|
868
|
+
return { ok: true, text };
|
|
869
|
+
}
|
|
870
|
+
/** Index one already-decoded session log; the caller owns row replacement. */
|
|
871
|
+
async indexSessionText(file, text, generation) {
|
|
686
872
|
if (generation !== this.generation)
|
|
687
873
|
return false;
|
|
688
874
|
const sessionId = basename(dirname(file.path));
|
|
875
|
+
this.sessionPaths.set(sessionId, file.path);
|
|
689
876
|
let title = '';
|
|
690
877
|
let parsed = 0;
|
|
691
878
|
for (const line of text.split('\n')) {
|
|
@@ -735,57 +922,134 @@ export class ContentIndex {
|
|
|
735
922
|
return this.options.sessionDirs;
|
|
736
923
|
return [join(dshHome(), 'sessions')];
|
|
737
924
|
}
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
async search(query, limit) {
|
|
925
|
+
async search(query, limit, options) {
|
|
926
|
+
const result = await this.searchDetailed(query, limit, options ?? {});
|
|
927
|
+
return options === undefined ? result.hits : result;
|
|
928
|
+
}
|
|
929
|
+
async searchDetailed(query, limit, options) {
|
|
930
|
+
const resolved = this.resolveSession(options);
|
|
931
|
+
if (resolved === null)
|
|
932
|
+
return emptySearchResult(['unknown-session']);
|
|
747
933
|
if (this.db === null)
|
|
748
|
-
return
|
|
934
|
+
return emptySearchResult();
|
|
749
935
|
if (!this.databaseIsCurrent()) {
|
|
750
936
|
this.recoverExternal();
|
|
751
|
-
return
|
|
937
|
+
return emptySearchResult();
|
|
752
938
|
}
|
|
939
|
+
// Direct embedders run in-process by design (searchWorker is off).
|
|
753
940
|
if (this.options.searchWorker !== true)
|
|
754
|
-
return this.searchSync(query, limit);
|
|
755
|
-
this.ensureWorker()
|
|
756
|
-
|
|
757
|
-
return this.searchSync(query, limit);
|
|
758
|
-
|
|
941
|
+
return this.searchSync(query, limit, resolved, false);
|
|
942
|
+
if (!this.ensureWorker()) {
|
|
943
|
+
// The worker could not be created at all: the one allowed sync path.
|
|
944
|
+
return this.searchSync(query, limit, resolved, true);
|
|
945
|
+
}
|
|
946
|
+
return await this.searchInWorker(query, limit, resolved);
|
|
947
|
+
}
|
|
948
|
+
/** Run one query on the worker; timeout/protocol errors are final results. */
|
|
949
|
+
async searchInWorker(query, limit, options) {
|
|
759
950
|
const worker = this.worker;
|
|
760
|
-
|
|
761
|
-
|
|
951
|
+
if (worker === null)
|
|
952
|
+
return workerFailureResult('content-search-worker-failed');
|
|
953
|
+
const id = ++this.workerSeq;
|
|
954
|
+
const response = new Promise(resolve => { this.pending.set(id, resolve); });
|
|
955
|
+
const timeoutMs = Math.max(1, this.options.searchWorkerTimeoutMs ?? WORKER_TIMEOUT_MS);
|
|
762
956
|
let timer;
|
|
763
|
-
const timeout = new Promise(
|
|
764
|
-
timer = setTimeout(() => {
|
|
957
|
+
const timeout = new Promise(resolve => {
|
|
958
|
+
timer = setTimeout(() => {
|
|
959
|
+
if (!this.pending.delete(id))
|
|
960
|
+
return;
|
|
961
|
+
// A wedged worker is thrown away; the next query gets a fresh one. The
|
|
962
|
+
// current query is reported as timed out, never re-run here.
|
|
963
|
+
if (this.worker === worker) {
|
|
964
|
+
this.worker = null;
|
|
965
|
+
void worker.terminate();
|
|
966
|
+
}
|
|
967
|
+
resolve(workerFailureResult('content-search-timeout'));
|
|
968
|
+
}, timeoutMs);
|
|
765
969
|
});
|
|
766
970
|
try {
|
|
767
|
-
|
|
971
|
+
worker.postMessage({ id, query, limit, options });
|
|
768
972
|
}
|
|
769
973
|
catch {
|
|
770
|
-
|
|
974
|
+
if (this.pending.delete(id)) {
|
|
975
|
+
if (timer !== undefined)
|
|
976
|
+
clearTimeout(timer);
|
|
977
|
+
return workerFailureResult('content-search-worker-failed');
|
|
978
|
+
}
|
|
979
|
+
}
|
|
980
|
+
try {
|
|
981
|
+
return await Promise.race([response, timeout]);
|
|
771
982
|
}
|
|
772
983
|
finally {
|
|
773
984
|
if (timer !== undefined)
|
|
774
985
|
clearTimeout(timer);
|
|
775
986
|
}
|
|
776
987
|
}
|
|
777
|
-
|
|
988
|
+
/** Map a request's sessionId onto the indexed session log path. */
|
|
989
|
+
resolveSession(options) {
|
|
990
|
+
if (options.sessionPath !== undefined && options.sessionPath !== null)
|
|
991
|
+
return options;
|
|
992
|
+
if (options.sessionId === undefined || options.sessionId === '')
|
|
993
|
+
return { ...options, sessionPath: null };
|
|
994
|
+
const path = this.sessionPaths.get(options.sessionId);
|
|
995
|
+
if (path === undefined)
|
|
996
|
+
return null;
|
|
997
|
+
return { ...options, sessionPath: path };
|
|
998
|
+
}
|
|
999
|
+
/**
|
|
1000
|
+
* In-process query path. When `degraded` is set (worker creation failed) the
|
|
1001
|
+
* options are tightened so this rare path cannot spend an unbounded amount of
|
|
1002
|
+
* host-thread time counting matches.
|
|
1003
|
+
*/
|
|
1004
|
+
searchSync(query, limit, options, degraded) {
|
|
778
1005
|
if (this.db === null)
|
|
779
|
-
return
|
|
1006
|
+
return emptySearchResult();
|
|
1007
|
+
const effective = degraded ? budgetedSearchOptions(options) : options;
|
|
780
1008
|
try {
|
|
781
|
-
|
|
1009
|
+
const result = searchContent(this.db, query, limit, effective);
|
|
1010
|
+
if (!degraded)
|
|
1011
|
+
return result;
|
|
1012
|
+
return { ...result, warnings: [...new Set(['content-search-in-process', ...result.warnings])] };
|
|
782
1013
|
}
|
|
783
1014
|
catch (error) {
|
|
784
1015
|
this.state.error = messageOf(error);
|
|
785
|
-
return
|
|
1016
|
+
return emptySearchResult();
|
|
786
1017
|
}
|
|
787
1018
|
}
|
|
788
1019
|
}
|
|
1020
|
+
/** Default deadline for one worker query. */
|
|
1021
|
+
const WORKER_TIMEOUT_MS = 5_000;
|
|
1022
|
+
/** Counting cap used by the rare in-process fallback. */
|
|
1023
|
+
const IN_PROCESS_COUNT_LIMIT = 200;
|
|
1024
|
+
/** Tighten a query for the in-process fallback: bounded counting only. */
|
|
1025
|
+
function budgetedSearchOptions(options) {
|
|
1026
|
+
return {
|
|
1027
|
+
...options,
|
|
1028
|
+
count: options.count === 'none' ? 'none' : 'bounded',
|
|
1029
|
+
countLimit: Math.min(options.countLimit ?? IN_PROCESS_COUNT_LIMIT, IN_PROCESS_COUNT_LIMIT),
|
|
1030
|
+
};
|
|
1031
|
+
}
|
|
1032
|
+
/** Whether a value looks like a complete worker search result. */
|
|
1033
|
+
function isContentSearchResult(value) {
|
|
1034
|
+
if (typeof value !== 'object' || value === null)
|
|
1035
|
+
return false;
|
|
1036
|
+
const record = value;
|
|
1037
|
+
if (!Array.isArray(record.hits) || !Array.isArray(record.warnings))
|
|
1038
|
+
return false;
|
|
1039
|
+
const count = record.count;
|
|
1040
|
+
if (typeof count !== 'object' || count === null)
|
|
1041
|
+
return false;
|
|
1042
|
+
const kind = count.kind;
|
|
1043
|
+
return kind === 'exact' || kind === 'bounded' || kind === 'unknown';
|
|
1044
|
+
}
|
|
1045
|
+
/** Result for a failed worker query; carries an explicit error code. */
|
|
1046
|
+
function workerFailureResult(error) {
|
|
1047
|
+
return { hits: [], count: { value: 0, kind: 'unknown' }, warnings: [error], error };
|
|
1048
|
+
}
|
|
1049
|
+
/** Result used when a content query could not run at all. */
|
|
1050
|
+
function emptySearchResult(warnings = []) {
|
|
1051
|
+
return { hits: [], count: { value: 0, kind: 'unknown' }, warnings };
|
|
1052
|
+
}
|
|
789
1053
|
function dshHome() {
|
|
790
1054
|
return process.env.DSH_HOME || join(homedir(), '.dsh');
|
|
791
1055
|
}
|