@lincoln504/pi-research 1.1.2 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/README.md +3 -3
  2. package/dist/cli.mjs +1287 -540
  3. package/dist/thread-worker.mjs +30 -11
  4. package/docs/AGENT-SKILL.md +11 -4
  5. package/docs/ARCHITECTURE.md +19 -9
  6. package/docs/CHANGELOG.md +81 -1
  7. package/docs/CONFIGURATION.md +9 -7
  8. package/docs/KNOWLEDGE-STORE.md +4 -3
  9. package/docs/PI-EXTENSION.md +1 -1
  10. package/docs/SDK.md +55 -0
  11. package/docs/THIRD-PARTY-NOTICES.md +1 -1
  12. package/package.json +5 -4
  13. package/scripts/cleanup.cjs +50 -10
  14. package/scripts/prepare.cjs +15 -5
  15. package/scripts/setup.cjs +24 -4
  16. package/skills/pi-research/README.md +9 -2
  17. package/skills/pi-research/SKILL.md +6 -3
  18. package/skills/pi-research/references/configuration.md +6 -2
  19. package/skills/pi-research/scripts/run.mjs +24 -2
  20. package/src/cli.ts +149 -21
  21. package/src/config.ts +56 -15
  22. package/src/constants.ts +31 -0
  23. package/src/core/interfaces/knowledge-interfaces.ts +6 -1
  24. package/src/core/interfaces/state-manager-interfaces.ts +1 -1
  25. package/src/core/pi-version.ts +136 -0
  26. package/src/core/service-registry.ts +117 -8
  27. package/src/index.ts +11 -18
  28. package/src/infrastructure/browser/browser-client.ts +16 -4
  29. package/src/infrastructure/browser/browser-error-utils.ts +6 -1
  30. package/src/infrastructure/browser/browser-server.ts +17 -5
  31. package/src/infrastructure/browser/browser-task-scheduler.ts +101 -23
  32. package/src/infrastructure/browser/cleanup-utils.ts +45 -7
  33. package/src/infrastructure/browser/client-agent.ts +4 -3
  34. package/src/infrastructure/browser/priority-task-queue.ts +10 -3
  35. package/src/infrastructure/browser/scheduler-factory.ts +26 -1
  36. package/src/infrastructure/browser/task-execution-service.ts +92 -28
  37. package/src/infrastructure/browser/thread-worker-lifecycle.ts +50 -13
  38. package/src/infrastructure/browser/thread-worker.mjs +30 -11
  39. package/src/infrastructure/browser/worker-pool-manager.ts +11 -3
  40. package/src/infrastructure/embedding/embedding-client.ts +22 -8
  41. package/src/infrastructure/embedding/embedding-factory.ts +28 -11
  42. package/src/infrastructure/embedding/embedding-server.ts +65 -4
  43. package/src/infrastructure/file-lock-service.ts +223 -26
  44. package/src/infrastructure/process-lifecycle-service.ts +108 -31
  45. package/src/infrastructure/state/state-backup-manager.ts +5 -7
  46. package/src/infrastructure/state/state-manager-service.ts +1 -1
  47. package/src/infrastructure/state/state-manager.ts +19 -20
  48. package/src/infrastructure/types/state-types.ts +9 -0
  49. package/src/knowledge/embedder-init.ts +1 -1
  50. package/src/knowledge/embedder.ts +52 -3
  51. package/src/knowledge/index.ts +1 -1
  52. package/src/knowledge/store.ts +35 -9
  53. package/src/knowledge/webgpu-viability.ts +9 -4
  54. package/src/knowledge/writer-queue.ts +88 -46
  55. package/src/orchestration/deep-research-orchestrator.ts +13 -2
  56. package/src/orchestration/quick-research-orchestrator.ts +2 -2
  57. package/src/orchestration/research-orchestration-service.ts +2 -4
  58. package/src/orchestration/research-synthesis-service.ts +33 -11
  59. package/src/security/cisa-kev.ts +10 -3
  60. package/src/security/github-advisories.ts +18 -7
  61. package/src/security/nvd-client.ts +17 -6
  62. package/src/security/osv-client.ts +14 -4
  63. package/src/skill-install/skill-installer.ts +135 -9
  64. package/src/stackexchange/rest-client.ts +6 -2
  65. package/src/tools/research-knowledge-search.ts +15 -0
  66. package/src/tools/search.ts +6 -1
  67. package/src/utils/atomic-replace.ts +101 -0
  68. package/src/utils/citation-utils.ts +3 -1
  69. package/src/utils/http-body.ts +176 -0
  70. package/src/web-research/browser-search.ts +50 -4
  71. package/src/web-research/search.ts +20 -5
  72. package/src/web-research/types.ts +17 -4
  73. package/src/web-research/web-scraper.ts +7 -50
  74. package/src/youtube/transcript-client.ts +5 -1
@@ -17,6 +17,7 @@ import { normalizeEcosystem } from './ecosystem.ts';
17
17
  import { createTimeoutSignal, retryWithBackoff, isTransientError } from '../web-research/retry-utils.ts';
18
18
  import { CircuitBreaker } from '../utils/circuit-breaker.ts';
19
19
  import { metrics } from '../utils/metrics.ts';
20
+ import { readJsonCapped, BodyTooLargeError } from '../utils/http-body.ts';
20
21
  import { logger } from '../logger.ts';
21
22
  import {
22
23
  isGitHubAdvisoryRaw,
@@ -153,10 +154,16 @@ export async function searchGitHubAdvisories(
153
154
 
154
155
  let data: unknown;
155
156
  try {
156
- data = await response.json();
157
- } catch {
157
+ data = await readJsonCapped(response);
158
+ } catch (err) {
159
+ // Size failures keep their own identity rather than being reported as a
160
+ // syntax error, which would point diagnosis at the wrong thing entirely.
161
+ if (err instanceof BodyTooLargeError) {
162
+ metrics.increment('github_search_errors_total', 1, { error_type: 'body_too_large' });
163
+ throw err;
164
+ }
158
165
  metrics.increment('github_search_errors_total', 1, { error_type: 'malformed_json' });
159
- throw new Error('GitHub API returned a malformed JSON response (repo advisories).');
166
+ throw new Error('GitHub API returned a malformed JSON response (repo advisories).', { cause: err });
160
167
  }
161
168
  let repoAdvisories: readonly GitHubAdvisoryRaw[] = [];
162
169
 
@@ -244,10 +251,14 @@ export async function searchGitHubAdvisories(
244
251
  // A 404 on a direct GHSA/CVE lookup means "no such advisory", not failure.
245
252
  let data: unknown;
246
253
  try {
247
- data = await response.json();
248
- } catch {
254
+ data = await readJsonCapped(response);
255
+ } catch (err) {
256
+ if (err instanceof BodyTooLargeError) {
257
+ metrics.increment('github_search_errors_total', 1, { error_type: 'body_too_large', endpoint: endpointType });
258
+ throw err;
259
+ }
249
260
  metrics.increment('github_search_errors_total', 1, { error_type: 'malformed_json', endpoint: endpointType });
250
- throw new Error(`GitHub API returned a malformed JSON response for term "${term}".`);
261
+ throw new Error(`GitHub API returned a malformed JSON response for term "${term}".`, { cause: err });
251
262
  }
252
263
  let items: readonly GitHubAdvisoryRaw[] = [];
253
264
 
@@ -259,7 +270,7 @@ export async function searchGitHubAdvisories(
259
270
  items = data.items.filter(isGitHubAdvisoryRaw);
260
271
  }
261
272
 
262
- metrics.increment(items.length > 0 ? 'github_cache_hits_total' : 'github_cache_misses_total', 1, { term, endpoint: endpointType });
273
+ metrics.increment(items.length > 0 ? 'github_cache_hits_total' : 'github_cache_misses_total', 1, { endpoint: endpointType });
263
274
 
264
275
  return items.map(mapGitHubAdvisory);
265
276
  };
@@ -24,6 +24,7 @@ import { createTimeoutSignal, retryWithBackoff, isTransientError } from '../web-
24
24
  import { CircuitBreaker } from '../utils/circuit-breaker.ts';
25
25
  import { safeUnref } from '../utils/safe-unref.ts';
26
26
  import { metrics } from '../utils/metrics.ts';
27
+ import { readJsonCapped, BodyTooLargeError, MAX_BULK_JSON_BODY_BYTES } from '../utils/http-body.ts';
27
28
  import {
28
29
  isWeaknessDescription,
29
30
  isWeakness,
@@ -396,11 +397,17 @@ async function fetchPaginated(
396
397
  const response = await fetchWithRetry(url, signal);
397
398
  let data: unknown;
398
399
  try {
399
- data = await response.json();
400
- } catch {
400
+ // A page can legitimately carry up to MAX_RESULTS_PER_PAGE (2000) full CVE
401
+ // records, so pagination reads against the bulk cap rather than the default.
402
+ data = await readJsonCapped(response, MAX_BULK_JSON_BODY_BYTES);
403
+ } catch (err) {
401
404
  // A 200 with a truncated/HTML body (proxy or CDN error page) would otherwise
402
- // throw and discard this whole term. Stop paginating this term instead.
403
- metrics.increment('nvd_search_errors_total', 1, { error_type: 'malformed_json' });
405
+ // throw and discard this whole term. Stop paginating this term instead —
406
+ // the pages already gathered are kept. An over-cap body degrades the same
407
+ // way but is counted separately so the cause stays diagnosable.
408
+ metrics.increment('nvd_search_errors_total', 1, {
409
+ error_type: err instanceof BodyTooLargeError ? 'body_too_large' : 'malformed_json',
410
+ });
404
411
  break;
405
412
  }
406
413
 
@@ -412,10 +419,14 @@ async function fetchPaginated(
412
419
  const admitted = rawEntries.filter(admitEntry).map((entry) => parseNVDEntry(entry, options));
413
420
 
414
421
  if (rawEntries.length === 0) {
415
- metrics.increment('nvd_cache_misses_total', 1, { term });
422
+ // No `term` label: the metrics registry keys counters by label set and is only
423
+ // cleared by an explicit clearSession(), so labelling by the raw search term made
424
+ // every distinct security query add a permanent entry for the life of a pi host
425
+ // session — unbounded growth driven straight by user input.
426
+ metrics.increment('nvd_cache_misses_total', 1);
416
427
  break;
417
428
  }
418
- metrics.increment('nvd_cache_hits_total', 1, { term });
429
+ metrics.increment('nvd_cache_hits_total', 1);
419
430
 
420
431
  allVulnerabilities.push(...admitted);
421
432
 
@@ -14,6 +14,7 @@ import { logger } from '../logger.ts';
14
14
  import { OSV_TIMEOUT_MS, DEFAULT_MAX_RETRIES, DEFAULT_INITIAL_DELAY_MS, DEFAULT_MAX_DELAY_MS } from '../constants.ts';
15
15
  import { CircuitBreaker } from '../utils/circuit-breaker.ts';
16
16
  import { metrics } from '../utils/metrics.ts';
17
+ import { readJsonCapped, BodyTooLargeError } from '../utils/http-body.ts';
17
18
  import {
18
19
  isOsvVulnerability,
19
20
  isOsvQueryResponse,
@@ -137,10 +138,17 @@ export async function searchOSV(
137
138
 
138
139
  let data: unknown;
139
140
  try {
140
- data = await response.json();
141
- } catch {
141
+ data = await readJsonCapped(response);
142
+ } catch (err) {
143
+ // An over-cap body is a size failure, not a syntax one. Rethrowing it
144
+ // unwrapped keeps the diagnosis honest instead of reporting a 40MB reply
145
+ // as malformed JSON.
146
+ if (err instanceof BodyTooLargeError) {
147
+ metrics.increment('osv_search_errors_total', 1, { error_type: 'body_too_large' });
148
+ throw err;
149
+ }
142
150
  metrics.increment('osv_search_errors_total', 1, { error_type: 'malformed_json' });
143
- throw new Error(`OSV returned a malformed JSON response for "${term}".`);
151
+ throw new Error(`OSV returned a malformed JSON response for "${term}".`, { cause: err });
144
152
  }
145
153
 
146
154
  let items: OsvVulnerability[];
@@ -154,7 +162,9 @@ export async function searchOSV(
154
162
  }
155
163
 
156
164
  const endpoint = isIdLookup ? 'vulns_by_id' : 'query';
157
- metrics.increment(items.length > 0 ? 'osv_cache_hits_total' : 'osv_cache_misses_total', 1, { term, endpoint });
165
+ // `endpoint` only — see the nvd-client note: a raw-term label grows the counter
166
+ // registry without bound across a session.
167
+ metrics.increment(items.length > 0 ? 'osv_cache_hits_total' : 'osv_cache_misses_total', 1, { endpoint });
158
168
 
159
169
  const vulns: Vulnerability[] = [];
160
170
  for (const item of items) {
@@ -18,6 +18,7 @@
18
18
  * only written on explicit opt-in (callers must pass them deliberately).
19
19
  */
20
20
 
21
+ import * as crypto from 'node:crypto';
21
22
  import * as fs from 'node:fs';
22
23
  import * as os from 'node:os';
23
24
  import * as path from 'node:path';
@@ -57,6 +58,24 @@ export const HARNESSES: readonly HarnessDef[] = [
57
58
  const SKILL_NAME = 'pi-research';
58
59
  const PACKAGE_NAME = '@lincoln504/pi-research';
59
60
 
61
+ /**
62
+ * This package's version, used to stamp manifest entries so a stale COPY can be
63
+ * detected and refreshed. Read from the SKILL.md we are about to install rather
64
+ * than from package.json: that file is the copied artifact, its metadata line is
65
+ * kept in sync by the `version` npm script, and it is present in every layout
66
+ * (repo, published tarball, installed copy). Returns null if unreadable, which
67
+ * simply means "cannot tell" — the entry is then always treated as stale, which
68
+ * refreshes rather than skips.
69
+ */
70
+ function readSkillVersion(sourceDir: string): string | null {
71
+ try {
72
+ const md = fs.readFileSync(path.join(sourceDir, 'SKILL.md'), 'utf-8');
73
+ return /"version"\s*:\s*"([^"]+)"/.exec(md)?.[1] ?? null;
74
+ } catch {
75
+ return null;
76
+ }
77
+ }
78
+
60
79
  /**
61
80
  * The coding agents the in-app installer (the /research-config TUI) targets:
62
81
  * Claude, Codex, and OpenClaw. Each loads skills from a home-directory
@@ -81,6 +100,22 @@ export interface ManifestEntry {
81
100
  /** Absolute path of the skill source this was linked/copied from. */
82
101
  source: string;
83
102
  createdAt: string;
103
+ /**
104
+ * Package version that produced this entry.
105
+ *
106
+ * Only load-bearing for `copy` entries: a symlink re-reads the package on every
107
+ * use and so is always current, but a COPY is a point-in-time snapshot that
108
+ * nothing ever refreshed — reconcile skipped non-symlinks entirely, and the
109
+ * entry recorded no version, so drift was undetectable by construction. Copies
110
+ * are not rare: they happen on Windows without symlink privilege, cross-device,
111
+ * with `--copy`, and via the documented OpenClaw install route. Those users kept
112
+ * the SKILL.md their engine shipped with *forever* — including its exit-code
113
+ * contract — while the engine underneath them changed.
114
+ *
115
+ * Optional so manifests written before this field are still valid; an entry with
116
+ * no version simply reads as "not current" and is refreshed once.
117
+ */
118
+ version?: string;
84
119
  }
85
120
 
86
121
  interface Manifest {
@@ -293,6 +328,7 @@ export function installSkill(toolIds: string[], opts: InstallOptions = {}): Inst
293
328
  const manifest = readManifest(opts);
294
329
  const results: InstallResult[] = [];
295
330
  const now = new Date().toISOString();
331
+ const skillVersion = readSkillVersion(source);
296
332
 
297
333
  for (const id of toolIds) {
298
334
  const def = HARNESSES.find(h => h.id === id);
@@ -315,16 +351,22 @@ export function installSkill(toolIds: string[], opts: InstallOptions = {}): Inst
315
351
  if (lst.isSymbolicLink()) {
316
352
  const dest = path.resolve(path.dirname(absSkillPath), fs.readlinkSync(absSkillPath));
317
353
  if (dest !== path.resolve(source)) {
354
+ // dryRun means "plan only, touch nothing" — the re-point is a real
355
+ // unlink+symlink and must honour it like the fresh-install path does.
356
+ if (opts.dryRun) {
357
+ results.push({ tool: id, path: absSkillPath, type: 'symlink', status: 'planned', message: 'would re-point stale symlink to current source' });
358
+ continue;
359
+ }
318
360
  fs.unlinkSync(absSkillPath);
319
361
  fs.symlinkSync(source, absSkillPath, process.platform === 'win32' ? 'junction' : 'dir');
320
- upsertEntry(manifest, { tool: id, path: absSkillPath, type: 'symlink', source, createdAt: now });
362
+ upsertEntry(manifest, { tool: id, path: absSkillPath, type: 'symlink', source, createdAt: now, ...(skillVersion ? { version: skillVersion } : {}) });
321
363
  results.push({ tool: id, path: absSkillPath, type: 'symlink', status: 'installed', message: 're-pointed stale symlink to current source' });
322
364
  continue;
323
365
  }
324
366
  }
325
367
  } catch { /* fall through to already-installed below */ }
326
368
  results.push({ tool: id, path: absSkillPath, type, status: 'already-installed' });
327
- upsertEntry(manifest, { tool: id, path: absSkillPath, type: isSymlinkPresent(absSkillPath) ? 'symlink' : 'copy', source, createdAt: now });
369
+ upsertEntry(manifest, { tool: id, path: absSkillPath, type: isSymlinkPresent(absSkillPath) ? 'symlink' : 'copy', source, createdAt: now, ...(skillVersion ? { version: skillVersion } : {}) });
328
370
  continue;
329
371
  }
330
372
  if (fs.existsSync(absSkillPath) || isSymlinkPresent(absSkillPath)) {
@@ -344,14 +386,14 @@ export function installSkill(toolIds: string[], opts: InstallOptions = {}): Inst
344
386
  } catch (err: any) {
345
387
  // Windows without privilege, or cross-device: fall back to copy.
346
388
  copyDir(source, absSkillPath);
347
- upsertEntry(manifest, { tool: id, path: absSkillPath, type: 'copy', source, createdAt: now });
389
+ upsertEntry(manifest, { tool: id, path: absSkillPath, type: 'copy', source, createdAt: now, ...(skillVersion ? { version: skillVersion } : {}) });
348
390
  results.push({ tool: id, path: absSkillPath, type: 'copy', status: 'installed', message: `symlink unavailable (${err?.code ?? err?.message}); copied instead` });
349
391
  continue;
350
392
  }
351
393
  } else {
352
394
  copyDir(source, absSkillPath);
353
395
  }
354
- upsertEntry(manifest, { tool: id, path: absSkillPath, type, source, createdAt: now });
396
+ upsertEntry(manifest, { tool: id, path: absSkillPath, type, source, createdAt: now, ...(skillVersion ? { version: skillVersion } : {}) });
355
397
  results.push({ tool: id, path: absSkillPath, type, status: 'installed' });
356
398
  } catch (err: any) {
357
399
  results.push({ tool: id, path: absSkillPath, type, status: 'error', message: err?.message ?? String(err) });
@@ -412,7 +454,7 @@ export function uninstallSkill(toolIds?: string[], opts: InstallOptions = {}): U
412
454
  }
413
455
  try {
414
456
  if (isSymlinkPresent(skillPath)) fs.unlinkSync(skillPath);
415
- else fs.rmSync(skillPath, { recursive: true, force: true });
457
+ else fs.rmSync(skillPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
416
458
  results.push({ tool, path: skillPath, status: 'removed' });
417
459
  } catch (err: any) {
418
460
  results.push({ tool, path: skillPath, status: 'error', message: err?.message ?? String(err) });
@@ -441,6 +483,12 @@ export interface ReconcileResult {
441
483
  pruned: string[];
442
484
  /** Stale/dangling owned symlinks re-pointed to the current package source. */
443
485
  repointed: string[];
486
+ /**
487
+ * Owned COPIES refreshed because the package version moved on. A symlink tracks
488
+ * the package automatically; a copy does not, so without this an upgrade left
489
+ * copy-installed users on the previous SKILL.md indefinitely.
490
+ */
491
+ refreshed: string[];
444
492
  }
445
493
 
446
494
  /**
@@ -460,17 +508,77 @@ export interface ReconcileResult {
460
508
  * Foreign (non-owned) paths and self-contained copies are never touched.
461
509
  */
462
510
  export function reconcileSkillInstalls(opts: InstallOptions = {}): ReconcileResult {
463
- const result: ReconcileResult = { pruned: [], repointed: [] };
511
+ const result: ReconcileResult = { pruned: [], repointed: [], refreshed: [] };
464
512
  const manifest = readManifest(opts);
465
513
  if (manifest.entries.length === 0) return result;
466
514
 
467
515
  let source: string | null;
468
516
  try { source = resolveSkillSourceDir(opts.skillSourceDir); } catch { source = null; }
469
517
 
518
+ const currentVersion = source !== null ? readSkillVersion(source) : null;
519
+
470
520
  const keep: ManifestEntry[] = [];
471
521
  for (const e of manifest.entries) {
472
- // Copies are self-contained (no target to dangle); leave them as-is.
473
- if (e.type !== 'symlink') { keep.push(e); continue; }
522
+ // A COPY has no target to dangle, but it is a point-in-time snapshot: unlike a
523
+ // symlink it does not track the package, so an engine upgrade leaves it serving
524
+ // the OLD SKILL.md — including its exit-code contract — indefinitely. Refresh it
525
+ // when the stamped version differs from what we now ship. An entry written
526
+ // before versions were stamped has none, so it reads as stale and is refreshed
527
+ // once. Ownership is re-checked first: a foreign directory at this path is
528
+ // never overwritten.
529
+ if (e.type !== 'symlink') {
530
+ const stale = currentVersion !== null && e.version !== currentVersion;
531
+ if (stale && source !== null && fs.existsSync(e.path) && isOwnedCopy(e.path)) {
532
+ // Build-then-swap, never rm-then-copy. The old shape (rmSync, then
533
+ // copyDir) destroyed the install permanently when the copy failed
534
+ // (ENOSPC, a locked file): the retry guard requires the path to exist,
535
+ // so a vanished install was never retried either. The staging names are
536
+ // dot-prefixed so a crash mid-swap cannot leave a directory a skill
537
+ // scanner would pick up as a duplicate skill.
538
+ const dir = path.dirname(e.path);
539
+ const base = path.basename(e.path);
540
+ // Sweep leftovers from a previous crashed swap before starting a new one.
541
+ try {
542
+ for (const name of fs.readdirSync(dir)) {
543
+ if (name.startsWith(`.${base}.new-`) || name.startsWith(`.${base}.old-`)) {
544
+ fs.rmSync(path.join(dir, name), { recursive: true, force: true });
545
+ }
546
+ }
547
+ } catch { /* hygiene only */ }
548
+ const suffix = crypto.randomBytes(4).toString('hex');
549
+ const fresh = path.join(dir, `.${base}.new-${suffix}`);
550
+ const retired = path.join(dir, `.${base}.old-${suffix}`);
551
+ try {
552
+ copyDir(source, fresh);
553
+ } catch {
554
+ try { fs.rmSync(fresh, { recursive: true, force: true }); } catch { /* best effort */ }
555
+ keep.push(e); // source unreadable / disk full — install intact, retry next startup
556
+ continue;
557
+ }
558
+ try {
559
+ fs.renameSync(e.path, retired);
560
+ } catch {
561
+ try { fs.rmSync(fresh, { recursive: true, force: true }); } catch { /* best effort */ }
562
+ keep.push(e); // could not move the old copy aside (locked on Windows) — intact
563
+ continue;
564
+ }
565
+ try {
566
+ fs.renameSync(fresh, e.path);
567
+ } catch {
568
+ // Put the old copy back — an old skill is strictly better than none.
569
+ try { fs.renameSync(retired, e.path); } catch { /* skills dir itself broken */ }
570
+ try { fs.rmSync(fresh, { recursive: true, force: true }); } catch { /* best effort */ }
571
+ keep.push(e);
572
+ continue;
573
+ }
574
+ try { fs.rmSync(retired, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 }); } catch { /* leaked dot-dir; swept next reconcile */ }
575
+ keep.push({ ...e, source, version: currentVersion });
576
+ result.refreshed.push(e.path);
577
+ } else {
578
+ keep.push(e);
579
+ }
580
+ continue;
581
+ }
474
582
 
475
583
  // Link the user deleted by hand — nothing on disk; drop the stale entry.
476
584
  if (!isSymlinkPresent(e.path)) { result.pruned.push(e.path); continue; }
@@ -514,6 +622,24 @@ export function reconcileSkillInstalls(opts: InstallOptions = {}): ReconcileResu
514
622
  // Helpers
515
623
  // ---------------------------------------------------------------------------
516
624
 
625
+ /**
626
+ * Recursive copy at the JS level — deliberately NOT fs.cpSync. cpSync's native
627
+ * implementation (std::filesystem) ABORTS the whole process on an unreadable
628
+ * directory instead of throwing a catchable error (observed on Node 25:
629
+ * "terminate called after throwing an instance of
630
+ * 'std::filesystem::__cxx11::filesystem_error'"). This runs inside
631
+ * reconcileSkillInstalls at every startup, where a copy failure must be caught
632
+ * and rolled back — never take the host down. statSync/copyFileSync follow
633
+ * symlinks, matching the dereference behaviour cpSync was given.
634
+ */
517
635
  function copyDir(src: string, dest: string): void {
518
- fs.cpSync(src, dest, { recursive: true, dereference: true });
636
+ const st = fs.statSync(src);
637
+ if (st.isDirectory()) {
638
+ fs.mkdirSync(dest, { recursive: true });
639
+ for (const name of fs.readdirSync(src)) {
640
+ copyDir(path.join(src, name), path.join(dest, name));
641
+ }
642
+ } else {
643
+ fs.copyFileSync(src, dest);
644
+ }
519
645
  }
@@ -7,6 +7,7 @@ import type { StackExchangeWrapper } from './types.ts';
7
7
  import { logger } from '../logger.ts';
8
8
  import { CircuitBreaker } from '../utils/circuit-breaker.ts';
9
9
  import { metrics } from '../utils/metrics.ts';
10
+ import { readJsonCapped, BodyTooLargeError } from '../utils/http-body.ts';
10
11
 
11
12
  const API_BASE = 'https://api.stackexchange.com/2.3';
12
13
 
@@ -99,7 +100,7 @@ export class StackExchangeClient {
99
100
 
100
101
  // NB: the timeout is cleared in `finally`, not here. Clearing it now —
101
102
  // when only the headers have arrived — would leave the body read
102
- // (response.json() below) with no timeout, so a stalled body could hang
103
+ // (readJsonCapped below) with no timeout, so a stalled body could hang
103
104
  // for the whole process lifetime.
104
105
 
105
106
  // The SE API returns a structured JSON wrapper (with error_id) even for
@@ -109,8 +110,11 @@ export class StackExchangeClient {
109
110
  // "Unexpected token <" SyntaxError that hides the real cause.
110
111
  let data: StackExchangeWrapper<T>;
111
112
  try {
112
- data = await response.json() as StackExchangeWrapper<T>;
113
+ data = await readJsonCapped<StackExchangeWrapper<T>>(response);
113
114
  } catch (parseErr) {
115
+ // An over-cap body is a size failure, not a malformed-JSON one: rethrow
116
+ // it unwrapped so it can't be relabelled as an HTTP-status error below.
117
+ if (parseErr instanceof BodyTooLargeError) throw parseErr;
114
118
  if (!response.ok) throw new Error(`HTTP ${response.status} from Stack Exchange API`, { cause: parseErr });
115
119
  throw parseErr;
116
120
  }
@@ -962,6 +962,21 @@ export function createResearchKnowledgeSearchTool(iface?: ConfigInterface): Tool
962
962
  return buildSteeringResult(result, urls);
963
963
  } catch (error) {
964
964
  const durationMs = Date.now() - startTime;
965
+ const msg = error instanceof Error ? error.message : String(error);
966
+
967
+ // A CANCELLED search is not an empty store. Reporting the miss string here
968
+ // told the calling agent, definitively, that the knowledge store has
969
+ // nothing on this topic — so it would go straight to full live research,
970
+ // the very work the user just interrupted. Propagate the abort instead;
971
+ // the run is ending either way. (Same misclassification as the search
972
+ // layer's "browser workers may be unavailable" on Ctrl-C.)
973
+ if (signal?.aborted || (error instanceof Error && error.name === 'AbortError') || /^(aborted|research (aborted|cancelled))$/i.test(msg)) {
974
+ metrics.observe('research_knowledge_search_duration_ms', durationMs, { status: 'cancelled' });
975
+ metrics.increment('research_knowledge_search_total', 1, { status: 'cancelled' });
976
+ logger.debug('[research-knowledge-search] Cancelled mid-search — propagating rather than reporting an empty store.');
977
+ throw error;
978
+ }
979
+
965
980
  metrics.observe('research_knowledge_search_duration_ms', durationMs, { status: 'error' });
966
981
  metrics.increment('research_knowledge_search_total', 1, { status: 'error' });
967
982
 
@@ -106,7 +106,12 @@ export function createSearchTool(options: {
106
106
  results.forEach((r, i) => {
107
107
  markdown += `## Query ${i + 1}: ${r.query}\n`;
108
108
  if (r.results.length === 0) {
109
- markdown += `*No results found.*\n\n`;
109
+ // Render the attributed failure, not a bare "no results": a timeout or
110
+ // dead worker says nothing about the query, and presenting it as an
111
+ // empty result sent the model off rewriting perfectly good queries —
112
+ // the misdirection QueryFailure exists to prevent. This is the only
113
+ // surface the researcher agent actually reads.
114
+ markdown += r.error ? `*${r.error.message}*\n\n` : `*No results found.*\n\n`;
110
115
  } else {
111
116
  r.results.forEach((item, j) => {
112
117
  markdown += `[${j + 1}] **${item.title}**\n${item.url}\n${item.content}\n\n`;
@@ -0,0 +1,101 @@
1
+ /**
2
+ * Atomic file replacement, with the Windows caveat handled honestly.
3
+ *
4
+ * The temp-file + fsync + rename dance exists so a reader never observes a partial
5
+ * file and a crash never leaves one: rename is atomic, so the destination is either
6
+ * entirely the old content or entirely the new.
7
+ *
8
+ * On Windows that guarantee is easy to lose. The long-standing comment at these call
9
+ * sites — "fs.rename fails on Windows if target exists (NTFS)" — is not true: Node
10
+ * calls `MoveFileExW` with `MOVEFILE_REPLACE_EXISTING`, so replacing an existing file
11
+ * is fine. What actually fails is a rename whose TARGET is held open by another
12
+ * process without delete sharing (an AV scanner mid-scan, a backup agent), which
13
+ * surfaces as EPERM/EBUSY/EACCES.
14
+ *
15
+ * The old remedy — fall straight to `copyFile` — traded a transient, retryable error
16
+ * for a permanent hazard: `copyFile` truncates the destination and rewrites it in
17
+ * place, so a concurrent reader or a crash mid-copy sees a torn or zero-length file.
18
+ * For `state.json` (leader election, server auth secrets) and `config.env` (API keys)
19
+ * that is a much worse outcome than the error it was avoiding.
20
+ *
21
+ * So: retry the rename first. An AV lock clears in tens of milliseconds, and a retried
22
+ * rename stays atomic. Only a lock that outlives every retry falls back to copying,
23
+ * which is reported to the caller so it can be logged as the degraded path it is.
24
+ */
25
+
26
+ import * as fs from 'node:fs/promises';
27
+ import * as fsSync from 'node:fs';
28
+
29
+ /** Transient Windows sharing/locking failures — worth retrying, unlike ENOENT/ENOSPC. */
30
+ const RETRYABLE_CODES = new Set(['EPERM', 'EBUSY', 'EACCES']);
31
+
32
+ const MAX_RENAME_ATTEMPTS = 5;
33
+ const RENAME_RETRY_DELAY_MS = 25;
34
+
35
+ export type ReplaceOutcome = 'renamed' | 'copied';
36
+
37
+ function isRetryable(err: unknown): boolean {
38
+ const code = (err as NodeJS.ErrnoException | undefined)?.code;
39
+ return code !== undefined && RETRYABLE_CODES.has(code);
40
+ }
41
+
42
+ /**
43
+ * Replace `destPath` with `tmpPath` atomically where the platform allows it.
44
+ *
45
+ * Returns `'renamed'` for the atomic path and `'copied'` for the degraded Windows
46
+ * fallback. Rethrows on every non-Windows failure, and on Windows failures that are
47
+ * not lock-related.
48
+ */
49
+ export async function replaceFile(tmpPath: string, destPath: string): Promise<ReplaceOutcome> {
50
+ let lastError: unknown;
51
+ for (let attempt = 0; attempt < MAX_RENAME_ATTEMPTS; attempt++) {
52
+ try {
53
+ await fs.rename(tmpPath, destPath);
54
+ return 'renamed';
55
+ } catch (err) {
56
+ lastError = err;
57
+ // Only a Windows sharing conflict is worth another try; anything else (and any
58
+ // failure on a POSIX platform, where rename-over-existing always works) is real.
59
+ if (process.platform !== 'win32' || !isRetryable(err)) throw err;
60
+ if (attempt < MAX_RENAME_ATTEMPTS - 1) {
61
+ await new Promise((r) => setTimeout(r, RENAME_RETRY_DELAY_MS * (attempt + 1)));
62
+ }
63
+ }
64
+ }
65
+
66
+ // Every retry lost to the same lock. Copying is not atomic — a reader can catch the
67
+ // destination mid-write — but it is the only remaining way to persist the data.
68
+ try {
69
+ await fs.copyFile(tmpPath, destPath);
70
+ return 'copied';
71
+ } catch {
72
+ throw lastError;
73
+ }
74
+ }
75
+
76
+ /** Synchronous {@link replaceFile}, for the config writer, which is sync throughout. */
77
+ export function replaceFileSync(tmpPath: string, destPath: string): ReplaceOutcome {
78
+ let lastError: unknown;
79
+ for (let attempt = 0; attempt < MAX_RENAME_ATTEMPTS; attempt++) {
80
+ try {
81
+ fsSync.renameSync(tmpPath, destPath);
82
+ return 'renamed';
83
+ } catch (err) {
84
+ lastError = err;
85
+ if (process.platform !== 'win32' || !isRetryable(err)) throw err;
86
+ if (attempt < MAX_RENAME_ATTEMPTS - 1) {
87
+ // Sync busy-wait: this path is rare, bounded (~250ms worst case) and the
88
+ // caller is already blocking on a synchronous write.
89
+ const until = Date.now() + RENAME_RETRY_DELAY_MS * (attempt + 1);
90
+ while (Date.now() < until) { /* spin */ }
91
+ }
92
+ }
93
+ }
94
+
95
+ try {
96
+ fsSync.copyFileSync(tmpPath, destPath);
97
+ return 'copied';
98
+ } catch {
99
+ throw lastError;
100
+ }
101
+ }
@@ -16,7 +16,9 @@ export interface GlobalCitation extends Citation {
16
16
  * @param reports - Map of researcher ID to report content
17
17
  * @returns Object containing normalized reports and the global citation list
18
18
  */
19
- export function normalizeCitations(reports: Map<string, string>): {
19
+ // ReadonlyMap: this only reads `reports` (it returns a NEW normalizedReports map),
20
+ // and callers now pass a read-only view to avoid create-on-read side effects.
21
+ export function normalizeCitations(reports: ReadonlyMap<string, string>): {
20
22
  normalizedReports: Map<string, string>;
21
23
  globalCitations: GlobalCitation[];
22
24
  } {