@lincoln504/pi-research 1.1.2 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/README.md +3 -3
  2. package/dist/cli.mjs +2599 -1268
  3. package/dist/thread-worker.mjs +224 -147
  4. package/docs/AGENT-SKILL.md +11 -4
  5. package/docs/ARCHITECTURE.md +20 -10
  6. package/docs/CHANGELOG.md +195 -17
  7. package/docs/CONFIGURATION.md +9 -7
  8. package/docs/KNOWLEDGE-STORE.md +4 -3
  9. package/docs/PI-EXTENSION.md +1 -1
  10. package/docs/SDK.md +55 -0
  11. package/docs/THIRD-PARTY-NOTICES.md +1 -1
  12. package/package.json +5 -4
  13. package/scripts/cleanup.cjs +114 -12
  14. package/scripts/prepare.cjs +15 -5
  15. package/scripts/setup.cjs +24 -4
  16. package/skills/pi-research/README.md +9 -2
  17. package/skills/pi-research/SKILL.md +6 -3
  18. package/skills/pi-research/references/configuration.md +6 -2
  19. package/skills/pi-research/scripts/run.mjs +24 -2
  20. package/src/cleanup/research-cleanup.ts +12 -4
  21. package/src/cli.ts +372 -86
  22. package/src/config.ts +134 -17
  23. package/src/constants.ts +31 -0
  24. package/src/core/interfaces/health-check-interfaces.ts +3 -39
  25. package/src/core/interfaces/knowledge-interfaces.ts +6 -1
  26. package/src/core/interfaces/planning-interfaces.ts +8 -0
  27. package/src/core/interfaces/service-names.ts +0 -1
  28. package/src/core/interfaces/state-manager-interfaces.ts +11 -3
  29. package/src/core/llm/agentic-repair.ts +14 -0
  30. package/src/core/pi-version.ts +136 -0
  31. package/src/core/planning-service.ts +43 -1
  32. package/src/core/service-initialization.ts +6 -14
  33. package/src/core/service-interfaces.ts +1 -1
  34. package/src/core/service-registry.ts +117 -8
  35. package/src/healthcheck/index.ts +7 -2
  36. package/src/index.ts +11 -18
  37. package/src/infrastructure/browser/browser-client.ts +80 -13
  38. package/src/infrastructure/browser/browser-error-utils.ts +6 -1
  39. package/src/infrastructure/browser/browser-server.ts +41 -12
  40. package/src/infrastructure/browser/browser-task-scheduler.ts +159 -35
  41. package/src/infrastructure/browser/cleanup-utils.ts +95 -9
  42. package/src/infrastructure/browser/client-agent.ts +6 -5
  43. package/src/infrastructure/browser/config.ts +14 -3
  44. package/src/infrastructure/browser/priority-task-queue.ts +10 -3
  45. package/src/infrastructure/browser/scheduler-factory.ts +44 -4
  46. package/src/infrastructure/browser/task-execution-service.ts +92 -28
  47. package/src/infrastructure/browser/thread-worker-lifecycle.ts +50 -13
  48. package/src/infrastructure/browser/thread-worker-messaging.ts +54 -2
  49. package/src/infrastructure/browser/thread-worker-types.ts +3 -1
  50. package/src/infrastructure/browser/thread-worker.mjs +224 -147
  51. package/src/infrastructure/browser/worker-pool-manager.ts +18 -5
  52. package/src/infrastructure/embedding/embedding-client.ts +22 -8
  53. package/src/infrastructure/embedding/embedding-factory.ts +61 -15
  54. package/src/infrastructure/embedding/embedding-server.ts +105 -11
  55. package/src/infrastructure/file-lock-service.ts +295 -28
  56. package/src/infrastructure/knowledge-store-service.ts +48 -3
  57. package/src/infrastructure/process-lifecycle-service.ts +134 -34
  58. package/src/infrastructure/research-run-semaphore.ts +37 -3
  59. package/src/infrastructure/state/state-backup-manager.ts +5 -7
  60. package/src/infrastructure/state/state-browser-api.ts +5 -2
  61. package/src/infrastructure/state/state-browser-manager.ts +15 -1
  62. package/src/infrastructure/state/state-manager-service.ts +5 -5
  63. package/src/infrastructure/state/state-manager.ts +34 -23
  64. package/src/infrastructure/types/state-types.ts +9 -0
  65. package/src/knowledge/chunker.ts +88 -1
  66. package/src/knowledge/embedder-init.ts +1 -1
  67. package/src/knowledge/embedder.ts +188 -46
  68. package/src/knowledge/index.ts +1 -1
  69. package/src/knowledge/store.ts +101 -21
  70. package/src/knowledge/webgpu-viability.ts +9 -4
  71. package/src/knowledge/writer-queue.ts +88 -46
  72. package/src/logger.ts +5 -1
  73. package/src/observers/research-observer-impl.ts +31 -0
  74. package/src/orchestration/abort-utils.ts +60 -0
  75. package/src/orchestration/deep-research-orchestrator.ts +25 -4
  76. package/src/orchestration/headless-observer.ts +8 -0
  77. package/src/orchestration/quick-research-orchestrator.ts +2 -2
  78. package/src/orchestration/research-orchestration-service.ts +65 -15
  79. package/src/orchestration/research-session-service.ts +37 -11
  80. package/src/orchestration/research-synthesis-service.ts +33 -11
  81. package/src/orchestration/researcher-executor.ts +100 -20
  82. package/src/orchestration/researcher.ts +39 -20
  83. package/src/orchestration/session-state.ts +11 -0
  84. package/src/sdk.ts +125 -19
  85. package/src/security/cisa-kev.ts +10 -3
  86. package/src/security/github-advisories.ts +18 -7
  87. package/src/security/index.ts +51 -30
  88. package/src/security/nvd-client.ts +24 -8
  89. package/src/security/osv-client.ts +14 -4
  90. package/src/skill-install/skill-installer.ts +153 -11
  91. package/src/stackexchange/index.ts +20 -5
  92. package/src/stackexchange/output/table.ts +7 -2
  93. package/src/stackexchange/rest-client.ts +41 -3
  94. package/src/tools/research-knowledge-search.ts +15 -0
  95. package/src/tools/research-tool-definition.ts +8 -2
  96. package/src/tools/scrape.ts +48 -25
  97. package/src/tools/search.ts +16 -2
  98. package/src/tools/security.ts +68 -7
  99. package/src/tui/research-health.ts +18 -2
  100. package/src/utils/atomic-replace.ts +101 -0
  101. package/src/utils/circuit-breaker.ts +9 -2
  102. package/src/utils/citation-utils.ts +3 -1
  103. package/src/utils/disk-space-checker.ts +11 -6
  104. package/src/utils/http-body.ts +176 -0
  105. package/src/utils/json-utils.ts +116 -76
  106. package/src/utils/llm-usage.ts +17 -1
  107. package/src/utils/log-rotation.ts +42 -10
  108. package/src/utils/log-utils.ts +40 -0
  109. package/src/utils/metrics-summary.ts +27 -9
  110. package/src/utils/stdio-capture.ts +63 -22
  111. package/src/utils/text-utils.ts +8 -1
  112. package/src/utils/tool-usage-tracker.ts +9 -2
  113. package/src/web-research/browser-search.ts +70 -5
  114. package/src/web-research/scraper-utils.ts +13 -0
  115. package/src/web-research/search.ts +25 -6
  116. package/src/web-research/types.ts +19 -4
  117. package/src/web-research/web-scraper.ts +72 -54
  118. package/src/youtube/transcript-client.ts +5 -1
  119. package/src/core/health-check-service.ts +0 -265
@@ -17,6 +17,7 @@ import { normalizeEcosystem } from './ecosystem.ts';
17
17
  import { createTimeoutSignal, retryWithBackoff, isTransientError } from '../web-research/retry-utils.ts';
18
18
  import { CircuitBreaker } from '../utils/circuit-breaker.ts';
19
19
  import { metrics } from '../utils/metrics.ts';
20
+ import { readJsonCapped, BodyTooLargeError } from '../utils/http-body.ts';
20
21
  import { logger } from '../logger.ts';
21
22
  import {
22
23
  isGitHubAdvisoryRaw,
@@ -153,10 +154,16 @@ export async function searchGitHubAdvisories(
153
154
 
154
155
  let data: unknown;
155
156
  try {
156
- data = await response.json();
157
- } catch {
157
+ data = await readJsonCapped(response);
158
+ } catch (err) {
159
+ // Size failures keep their own identity rather than being reported as a
160
+ // syntax error, which would point diagnosis at the wrong thing entirely.
161
+ if (err instanceof BodyTooLargeError) {
162
+ metrics.increment('github_search_errors_total', 1, { error_type: 'body_too_large' });
163
+ throw err;
164
+ }
158
165
  metrics.increment('github_search_errors_total', 1, { error_type: 'malformed_json' });
159
- throw new Error('GitHub API returned a malformed JSON response (repo advisories).');
166
+ throw new Error('GitHub API returned a malformed JSON response (repo advisories).', { cause: err });
160
167
  }
161
168
  let repoAdvisories: readonly GitHubAdvisoryRaw[] = [];
162
169
 
@@ -244,10 +251,14 @@ export async function searchGitHubAdvisories(
244
251
  // A 404 on a direct GHSA/CVE lookup means "no such advisory", not failure.
245
252
  let data: unknown;
246
253
  try {
247
- data = await response.json();
248
- } catch {
254
+ data = await readJsonCapped(response);
255
+ } catch (err) {
256
+ if (err instanceof BodyTooLargeError) {
257
+ metrics.increment('github_search_errors_total', 1, { error_type: 'body_too_large', endpoint: endpointType });
258
+ throw err;
259
+ }
249
260
  metrics.increment('github_search_errors_total', 1, { error_type: 'malformed_json', endpoint: endpointType });
250
- throw new Error(`GitHub API returned a malformed JSON response for term "${term}".`);
261
+ throw new Error(`GitHub API returned a malformed JSON response for term "${term}".`, { cause: err });
251
262
  }
252
263
  let items: readonly GitHubAdvisoryRaw[] = [];
253
264
 
@@ -259,7 +270,7 @@ export async function searchGitHubAdvisories(
259
270
  items = data.items.filter(isGitHubAdvisoryRaw);
260
271
  }
261
272
 
262
- metrics.increment(items.length > 0 ? 'github_cache_hits_total' : 'github_cache_misses_total', 1, { term, endpoint: endpointType });
273
+ metrics.increment(items.length > 0 ? 'github_cache_hits_total' : 'github_cache_misses_total', 1, { endpoint: endpointType });
263
274
 
264
275
  return items.map(mapGitHubAdvisory);
265
276
  };
@@ -22,12 +22,11 @@ import type {
22
22
  GitHubResult,
23
23
  OSVResult,
24
24
  } from './types.ts';
25
- import { REQUEST_DELAY_MS_NVD, REQUEST_DELAY_MS_OTHER } from '../constants.ts';
26
25
  import { searchNVD } from './nvd-client.ts';
27
26
  import { searchCisaKev } from './cisa-kev.ts';
28
27
  import { searchGitHubAdvisories } from './github-advisories.ts';
29
28
  import { searchOSV } from './osv-client.ts';
30
- import { safeUnref } from '../utils/safe-unref.ts';
29
+ import { logger } from '../logger.ts';
31
30
 
32
31
  // ============================================================================
33
32
  // Type Definitions
@@ -35,6 +34,9 @@ import { safeUnref } from '../utils/safe-unref.ts';
35
34
 
36
35
  type Severity = 'LOW' | 'MEDIUM' | 'HIGH' | 'CRITICAL';
37
36
 
37
+ type KnownDatabase = 'nvd' | 'cisa_kev' | 'github' | 'osv';
38
+ const KNOWN_DATABASES: readonly KnownDatabase[] = ['nvd', 'cisa_kev', 'github', 'osv'];
39
+
38
40
  interface DatabaseInfo {
39
41
  readonly databases: readonly DatabaseInfoEntry[];
40
42
  }
@@ -58,9 +60,22 @@ function isValidSeverity(value: unknown): value is Severity {
58
60
  );
59
61
  }
60
62
 
63
+ // Called ONCE per search(), before dispatch, and the result is shared by every
64
+ // requested backend (NVD, GitHub, OSV) — see the call site in search(). Do not
65
+ // call this per-leg: that previously let NVD normalize while GitHub/OSV read
66
+ // params.severity raw, so the same unrecognized value ran NVD unfiltered while
67
+ // silently zeroing GitHub/OSV's strict filters.
61
68
  function getSeverityParam(params: SecuritySearchParams): Severity | undefined {
62
69
  if (params.severity === undefined) return undefined;
63
- return isValidSeverity(params.severity) ? params.severity : undefined;
70
+ // NVD requires the exact uppercase enum. GitHub/OSV normalize case and the
71
+ // MODERATE↔MEDIUM synonym in their own clients (github-advisories.ts,
72
+ // osv-client.ts) as well, so applying the same normalization here keeps all
73
+ // three backends in agreement on what severity:"high" or "Moderate" means.
74
+ const upper = params.severity.trim().toUpperCase();
75
+ const normalized = upper === 'MODERATE' ? 'MEDIUM' : upper;
76
+ if (isValidSeverity(normalized)) return normalized;
77
+ logger.warn(`[security] Unknown severity filter "${params.severity}" (valid: LOW, MEDIUM, HIGH, CRITICAL) — no database will be severity-filtered`);
78
+ return undefined;
64
79
  }
65
80
 
66
81
  function getErrorMessage(error: unknown): string {
@@ -77,7 +92,6 @@ export interface SecuritySearcherConfig {
77
92
  readonly cisaKevClient?: ICisaKevClient;
78
93
  readonly githubAdvisoriesClient?: IGitHubAdvisoriesClient;
79
94
  readonly osvClient?: IOSVClient;
80
- readonly requestDelay?: number; // Override for testing
81
95
  }
82
96
 
83
97
  const DEFAULT_CONFIG: SecuritySearcherConfig = {
@@ -85,7 +99,6 @@ const DEFAULT_CONFIG: SecuritySearcherConfig = {
85
99
  cisaKevClient: undefined,
86
100
  githubAdvisoriesClient: undefined,
87
101
  osvClient: undefined,
88
- requestDelay: -1, // Use database-specific logic by default
89
102
  };
90
103
 
91
104
  export class SecuritySearcher {
@@ -101,13 +114,35 @@ export class SecuritySearcher {
101
114
  const errors: string[] = [];
102
115
  let totalVulnerabilities = 0;
103
116
 
117
+ // Normalize severity ONCE, before dispatch, so every backend that receives a
118
+ // severity filter sees the exact same value. Previously this call sat inline
119
+ // in the NVD branch only; GitHub and OSV read the raw, unvalidated
120
+ // params.severity directly, so an unrecognized value ran NVD unfiltered while
121
+ // silently zeroing GitHub/OSV (or vice versa depending on each client's own
122
+ // parsing) — the single-chokepoint fix has to normalize before the fan-out,
123
+ // not duplicate the call at each leg.
124
+ const severity = getSeverityParam(params);
125
+
126
+ // Normalize database names before dispatch: callers (agents, SDK users) write
127
+ // "NVD" / " osv " etc. An unrecognized name must surface in `errors` rather than
128
+ // silently contributing an authoritative-looking "0 vulnerabilities" for a
129
+ // database that was never queried.
130
+ const databases: string[] = [];
131
+ for (const raw of params.databases) {
132
+ const db = String(raw).trim().toLowerCase();
133
+ databases.push(db);
134
+ if (!(KNOWN_DATABASES as readonly string[]).includes(db)) {
135
+ errors.push(`Unknown database "${raw}" — valid databases: ${KNOWN_DATABASES.join(', ')}`);
136
+ }
137
+ }
138
+
104
139
  const searchPromises: Promise<void>[] = [];
105
140
 
106
- if (params.databases.includes('nvd')) {
141
+ if (databases.includes('nvd')) {
107
142
  searchPromises.push((async () => {
108
143
  try {
109
144
  const nvdResult = await this.searchNVD(params.terms, {
110
- severity: getSeverityParam(params),
145
+ severity,
111
146
  maxResults: params.maxResults,
112
147
  includeExploited: params.includeExploited,
113
148
  signal,
@@ -120,7 +155,7 @@ export class SecuritySearcher {
120
155
  })());
121
156
  }
122
157
 
123
- if (params.databases.includes('cisa_kev')) {
158
+ if (databases.includes('cisa_kev')) {
124
159
  searchPromises.push((async () => {
125
160
  try {
126
161
  const cisaResult = await this.searchCisaKev(params.terms, { maxResults: params.maxResults, signal });
@@ -132,12 +167,12 @@ export class SecuritySearcher {
132
167
  })());
133
168
  }
134
169
 
135
- if (params.databases.includes('github')) {
170
+ if (databases.includes('github')) {
136
171
  searchPromises.push((async () => {
137
172
  try {
138
173
  const githubResult = await this.searchGitHub(params.terms, {
139
174
  ecosystem: params.ecosystem,
140
- severity: params.severity,
175
+ severity,
141
176
  maxResults: params.maxResults,
142
177
  repo: params.githubRepo,
143
178
  signal,
@@ -150,12 +185,12 @@ export class SecuritySearcher {
150
185
  })());
151
186
  }
152
187
 
153
- if (params.databases.includes('osv')) {
188
+ if (databases.includes('osv')) {
154
189
  searchPromises.push((async () => {
155
190
  try {
156
191
  const osvResult = await this.searchOSV(params.terms, {
157
192
  ecosystem: params.ecosystem,
158
- severity: params.severity,
193
+ severity,
159
194
  maxResults: params.maxResults,
160
195
  signal,
161
196
  });
@@ -169,24 +204,10 @@ export class SecuritySearcher {
169
204
 
170
205
  await Promise.all(searchPromises);
171
206
 
172
- // Apply delay
173
- let delay = this.config.requestDelay ?? -1;
174
- if (delay === -1) {
175
- delay = params.databases.includes('nvd') ? REQUEST_DELAY_MS_NVD : REQUEST_DELAY_MS_OTHER;
176
- }
177
-
178
- if (delay > 0) {
179
- // Abortable: a caller cancel must not sit through the post-sweep delay.
180
- await new Promise<void>((resolve) => {
181
- const timeoutId = setTimeout(() => {
182
- signal?.removeEventListener('abort', onAbort);
183
- resolve();
184
- }, delay);
185
- safeUnref(timeoutId);
186
- const onAbort = () => { clearTimeout(timeoutId); resolve(); };
187
- if (signal) signal.addEventListener('abort', onAbort, { once: true });
188
- });
189
- }
207
+ // No post-sweep delay: NVD inter-request spacing is enforced by the module-level
208
+ // NVDRateLimiter in nvd-client.ts, whose state persists across searches and tool
209
+ // calls, so sleeping here after results were already in hand only added dead
210
+ // researcher time.
190
211
 
191
212
  return {
192
213
  results,
@@ -24,6 +24,7 @@ import { createTimeoutSignal, retryWithBackoff, isTransientError } from '../web-
24
24
  import { CircuitBreaker } from '../utils/circuit-breaker.ts';
25
25
  import { safeUnref } from '../utils/safe-unref.ts';
26
26
  import { metrics } from '../utils/metrics.ts';
27
+ import { readJsonCapped, BodyTooLargeError, MAX_BULK_JSON_BODY_BYTES } from '../utils/http-body.ts';
27
28
  import {
28
29
  isWeaknessDescription,
29
30
  isWeakness,
@@ -352,6 +353,11 @@ function fetchWithRetry(url: string, signal?: AbortSignal): Promise<Response> {
352
353
 
353
354
  return nvdCircuitBreaker.execute(async () => {
354
355
  return retryWithBackoff(async () => {
356
+ // Every attempt — the first AND each 2s/4s retry — must hold a rate-limiter
357
+ // slot: the limiter's spacing is the ONLY mechanism enforcing NVD's ~6s
358
+ // inter-request interval (the post-sweep sleep was removed), and an acquire
359
+ // outside this closure let 429/5xx retries re-hit the API faster than it.
360
+ await nvdRateLimiter.acquire(signal);
355
361
  let response: Response;
356
362
  try {
357
363
  response = await fetch(url, createFetchOptions(signal));
@@ -391,16 +397,22 @@ async function fetchPaginated(
391
397
  if (signal?.aborted) break; // stop paginating promptly on cancel
392
398
  const url = buildURL(term, options, pageSize, startIndex, severityVersion);
393
399
 
394
- await nvdRateLimiter.acquire(signal);
395
-
400
+ // No acquire here: fetchWithRetry acquires a rate-limiter slot per ATTEMPT,
401
+ // so retries are spaced too — a second acquire would double-charge the page.
396
402
  const response = await fetchWithRetry(url, signal);
397
403
  let data: unknown;
398
404
  try {
399
- data = await response.json();
400
- } catch {
405
+ // A page can legitimately carry up to MAX_RESULTS_PER_PAGE (2000) full CVE
406
+ // records, so pagination reads against the bulk cap rather than the default.
407
+ data = await readJsonCapped(response, MAX_BULK_JSON_BODY_BYTES);
408
+ } catch (err) {
401
409
  // A 200 with a truncated/HTML body (proxy or CDN error page) would otherwise
402
- // throw and discard this whole term. Stop paginating this term instead.
403
- metrics.increment('nvd_search_errors_total', 1, { error_type: 'malformed_json' });
410
+ // throw and discard this whole term. Stop paginating this term instead —
411
+ // the pages already gathered are kept. An over-cap body degrades the same
412
+ // way but is counted separately so the cause stays diagnosable.
413
+ metrics.increment('nvd_search_errors_total', 1, {
414
+ error_type: err instanceof BodyTooLargeError ? 'body_too_large' : 'malformed_json',
415
+ });
404
416
  break;
405
417
  }
406
418
 
@@ -412,10 +424,14 @@ async function fetchPaginated(
412
424
  const admitted = rawEntries.filter(admitEntry).map((entry) => parseNVDEntry(entry, options));
413
425
 
414
426
  if (rawEntries.length === 0) {
415
- metrics.increment('nvd_cache_misses_total', 1, { term });
427
+ // No `term` label: the metrics registry keys counters by label set and is only
428
+ // cleared by an explicit clearSession(), so labelling by the raw search term made
429
+ // every distinct security query add a permanent entry for the life of a pi host
430
+ // session — unbounded growth driven straight by user input.
431
+ metrics.increment('nvd_cache_misses_total', 1);
416
432
  break;
417
433
  }
418
- metrics.increment('nvd_cache_hits_total', 1, { term });
434
+ metrics.increment('nvd_cache_hits_total', 1);
419
435
 
420
436
  allVulnerabilities.push(...admitted);
421
437
 
@@ -14,6 +14,7 @@ import { logger } from '../logger.ts';
14
14
  import { OSV_TIMEOUT_MS, DEFAULT_MAX_RETRIES, DEFAULT_INITIAL_DELAY_MS, DEFAULT_MAX_DELAY_MS } from '../constants.ts';
15
15
  import { CircuitBreaker } from '../utils/circuit-breaker.ts';
16
16
  import { metrics } from '../utils/metrics.ts';
17
+ import { readJsonCapped, BodyTooLargeError } from '../utils/http-body.ts';
17
18
  import {
18
19
  isOsvVulnerability,
19
20
  isOsvQueryResponse,
@@ -137,10 +138,17 @@ export async function searchOSV(
137
138
 
138
139
  let data: unknown;
139
140
  try {
140
- data = await response.json();
141
- } catch {
141
+ data = await readJsonCapped(response);
142
+ } catch (err) {
143
+ // An over-cap body is a size failure, not a syntax one. Rethrowing it
144
+ // unwrapped keeps the diagnosis honest instead of reporting a 40MB reply
145
+ // as malformed JSON.
146
+ if (err instanceof BodyTooLargeError) {
147
+ metrics.increment('osv_search_errors_total', 1, { error_type: 'body_too_large' });
148
+ throw err;
149
+ }
142
150
  metrics.increment('osv_search_errors_total', 1, { error_type: 'malformed_json' });
143
- throw new Error(`OSV returned a malformed JSON response for "${term}".`);
151
+ throw new Error(`OSV returned a malformed JSON response for "${term}".`, { cause: err });
144
152
  }
145
153
 
146
154
  let items: OsvVulnerability[];
@@ -154,7 +162,9 @@ export async function searchOSV(
154
162
  }
155
163
 
156
164
  const endpoint = isIdLookup ? 'vulns_by_id' : 'query';
157
- metrics.increment(items.length > 0 ? 'osv_cache_hits_total' : 'osv_cache_misses_total', 1, { term, endpoint });
165
+ // `endpoint` only — see the nvd-client note: a raw-term label grows the counter
166
+ // registry without bound across a session.
167
+ metrics.increment(items.length > 0 ? 'osv_cache_hits_total' : 'osv_cache_misses_total', 1, { endpoint });
158
168
 
159
169
  const vulns: Vulnerability[] = [];
160
170
  for (const item of items) {
@@ -18,6 +18,7 @@
18
18
  * only written on explicit opt-in (callers must pass them deliberately).
19
19
  */
20
20
 
21
+ import * as crypto from 'node:crypto';
21
22
  import * as fs from 'node:fs';
22
23
  import * as os from 'node:os';
23
24
  import * as path from 'node:path';
@@ -57,6 +58,24 @@ export const HARNESSES: readonly HarnessDef[] = [
57
58
  const SKILL_NAME = 'pi-research';
58
59
  const PACKAGE_NAME = '@lincoln504/pi-research';
59
60
 
61
+ /**
62
+ * This package's version, used to stamp manifest entries so a stale COPY can be
63
+ * detected and refreshed. Read from the SKILL.md we are about to install rather
64
+ * than from package.json: that file is the copied artifact, its metadata line is
65
+ * kept in sync by the `version` npm script, and it is present in every layout
66
+ * (repo, published tarball, installed copy). Returns null if unreadable, which
67
+ * simply means "cannot tell" — the entry is then always treated as stale, which
68
+ * refreshes rather than skips.
69
+ */
70
+ function readSkillVersion(sourceDir: string): string | null {
71
+ try {
72
+ const md = fs.readFileSync(path.join(sourceDir, 'SKILL.md'), 'utf-8');
73
+ return /"version"\s*:\s*"([^"]+)"/.exec(md)?.[1] ?? null;
74
+ } catch {
75
+ return null;
76
+ }
77
+ }
78
+
60
79
  /**
61
80
  * The coding agents the in-app installer (the /research-config TUI) targets:
62
81
  * Claude, Codex, and OpenClaw. Each loads skills from a home-directory
@@ -81,6 +100,22 @@ export interface ManifestEntry {
81
100
  /** Absolute path of the skill source this was linked/copied from. */
82
101
  source: string;
83
102
  createdAt: string;
103
+ /**
104
+ * Package version that produced this entry.
105
+ *
106
+ * Only load-bearing for `copy` entries: a symlink re-reads the package on every
107
+ * use and so is always current, but a COPY is a point-in-time snapshot that
108
+ * nothing ever refreshed — reconcile skipped non-symlinks entirely, and the
109
+ * entry recorded no version, so drift was undetectable by construction. Copies
110
+ * are not rare: they happen on Windows without symlink privilege, cross-device,
111
+ * with `--copy`, and via the documented OpenClaw install route. Those users kept
112
+ * the SKILL.md their engine shipped with *forever* — including its exit-code
113
+ * contract — while the engine underneath them changed.
114
+ *
115
+ * Optional so manifests written before this field are still valid; an entry with
116
+ * no version simply reads as "not current" and is refreshed once.
117
+ */
118
+ version?: string;
84
119
  }
85
120
 
86
121
  interface Manifest {
@@ -182,12 +217,28 @@ export function readManifest(opts?: { home?: string }): Manifest {
182
217
  return { version: 1, package: PACKAGE_NAME, entries: [] };
183
218
  }
184
219
 
220
+ /** Monotonic per-process suffix for manifest tmp files (see writeManifest). */
221
+ let manifestTmpSeq = 0;
222
+
185
223
  function writeManifest(manifest: Manifest, opts?: { home?: string }): void {
186
224
  const p = getManifestPath(opts);
187
225
  fs.mkdirSync(path.dirname(p), { recursive: true });
188
- const tmp = `${p}.tmp`;
226
+ // Unique tmp name (pid + timestamp + counter, mirroring config.ts's registry
227
+ // writer): a FIXED `${p}.tmp` was a shared collision surface — concurrent
228
+ // runs' reconciles (the run cap allows 3 live runs) staged through the same
229
+ // path, the winner renamed it away, and the loser's renameSync threw ENOENT,
230
+ // surfacing as a spurious skill-install error. The read-modify-write around
231
+ // this stays deliberately UNLOCKED: a lost update is self-healing churn (the
232
+ // next reconcile re-derives the same state from disk), unlike the hard
233
+ // failure a tmp-name collision caused.
234
+ const tmp = `${p}.tmp.${process.pid}.${Date.now()}.${manifestTmpSeq++}`;
189
235
  fs.writeFileSync(tmp, JSON.stringify(manifest, null, 2) + '\n', 'utf-8');
190
- fs.renameSync(tmp, p);
236
+ try {
237
+ fs.renameSync(tmp, p);
238
+ } catch (err) {
239
+ try { fs.unlinkSync(tmp); } catch { /* best effort */ }
240
+ throw err;
241
+ }
191
242
  }
192
243
 
193
244
  function upsertEntry(manifest: Manifest, entry: ManifestEntry): void {
@@ -293,6 +344,7 @@ export function installSkill(toolIds: string[], opts: InstallOptions = {}): Inst
293
344
  const manifest = readManifest(opts);
294
345
  const results: InstallResult[] = [];
295
346
  const now = new Date().toISOString();
347
+ const skillVersion = readSkillVersion(source);
296
348
 
297
349
  for (const id of toolIds) {
298
350
  const def = HARNESSES.find(h => h.id === id);
@@ -315,16 +367,22 @@ export function installSkill(toolIds: string[], opts: InstallOptions = {}): Inst
315
367
  if (lst.isSymbolicLink()) {
316
368
  const dest = path.resolve(path.dirname(absSkillPath), fs.readlinkSync(absSkillPath));
317
369
  if (dest !== path.resolve(source)) {
370
+ // dryRun means "plan only, touch nothing" — the re-point is a real
371
+ // unlink+symlink and must honour it like the fresh-install path does.
372
+ if (opts.dryRun) {
373
+ results.push({ tool: id, path: absSkillPath, type: 'symlink', status: 'planned', message: 'would re-point stale symlink to current source' });
374
+ continue;
375
+ }
318
376
  fs.unlinkSync(absSkillPath);
319
377
  fs.symlinkSync(source, absSkillPath, process.platform === 'win32' ? 'junction' : 'dir');
320
- upsertEntry(manifest, { tool: id, path: absSkillPath, type: 'symlink', source, createdAt: now });
378
+ upsertEntry(manifest, { tool: id, path: absSkillPath, type: 'symlink', source, createdAt: now, ...(skillVersion ? { version: skillVersion } : {}) });
321
379
  results.push({ tool: id, path: absSkillPath, type: 'symlink', status: 'installed', message: 're-pointed stale symlink to current source' });
322
380
  continue;
323
381
  }
324
382
  }
325
383
  } catch { /* fall through to already-installed below */ }
326
384
  results.push({ tool: id, path: absSkillPath, type, status: 'already-installed' });
327
- upsertEntry(manifest, { tool: id, path: absSkillPath, type: isSymlinkPresent(absSkillPath) ? 'symlink' : 'copy', source, createdAt: now });
385
+ upsertEntry(manifest, { tool: id, path: absSkillPath, type: isSymlinkPresent(absSkillPath) ? 'symlink' : 'copy', source, createdAt: now, ...(skillVersion ? { version: skillVersion } : {}) });
328
386
  continue;
329
387
  }
330
388
  if (fs.existsSync(absSkillPath) || isSymlinkPresent(absSkillPath)) {
@@ -344,14 +402,14 @@ export function installSkill(toolIds: string[], opts: InstallOptions = {}): Inst
344
402
  } catch (err: any) {
345
403
  // Windows without privilege, or cross-device: fall back to copy.
346
404
  copyDir(source, absSkillPath);
347
- upsertEntry(manifest, { tool: id, path: absSkillPath, type: 'copy', source, createdAt: now });
405
+ upsertEntry(manifest, { tool: id, path: absSkillPath, type: 'copy', source, createdAt: now, ...(skillVersion ? { version: skillVersion } : {}) });
348
406
  results.push({ tool: id, path: absSkillPath, type: 'copy', status: 'installed', message: `symlink unavailable (${err?.code ?? err?.message}); copied instead` });
349
407
  continue;
350
408
  }
351
409
  } else {
352
410
  copyDir(source, absSkillPath);
353
411
  }
354
- upsertEntry(manifest, { tool: id, path: absSkillPath, type, source, createdAt: now });
412
+ upsertEntry(manifest, { tool: id, path: absSkillPath, type, source, createdAt: now, ...(skillVersion ? { version: skillVersion } : {}) });
355
413
  results.push({ tool: id, path: absSkillPath, type, status: 'installed' });
356
414
  } catch (err: any) {
357
415
  results.push({ tool: id, path: absSkillPath, type, status: 'error', message: err?.message ?? String(err) });
@@ -412,7 +470,7 @@ export function uninstallSkill(toolIds?: string[], opts: InstallOptions = {}): U
412
470
  }
413
471
  try {
414
472
  if (isSymlinkPresent(skillPath)) fs.unlinkSync(skillPath);
415
- else fs.rmSync(skillPath, { recursive: true, force: true });
473
+ else fs.rmSync(skillPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
416
474
  results.push({ tool, path: skillPath, status: 'removed' });
417
475
  } catch (err: any) {
418
476
  results.push({ tool, path: skillPath, status: 'error', message: err?.message ?? String(err) });
@@ -441,6 +499,12 @@ export interface ReconcileResult {
441
499
  pruned: string[];
442
500
  /** Stale/dangling owned symlinks re-pointed to the current package source. */
443
501
  repointed: string[];
502
+ /**
503
+ * Owned COPIES refreshed because the package version moved on. A symlink tracks
504
+ * the package automatically; a copy does not, so without this an upgrade left
505
+ * copy-installed users on the previous SKILL.md indefinitely.
506
+ */
507
+ refreshed: string[];
444
508
  }
445
509
 
446
510
  /**
@@ -460,17 +524,77 @@ export interface ReconcileResult {
460
524
  * Foreign (non-owned) paths and self-contained copies are never touched.
461
525
  */
462
526
  export function reconcileSkillInstalls(opts: InstallOptions = {}): ReconcileResult {
463
- const result: ReconcileResult = { pruned: [], repointed: [] };
527
+ const result: ReconcileResult = { pruned: [], repointed: [], refreshed: [] };
464
528
  const manifest = readManifest(opts);
465
529
  if (manifest.entries.length === 0) return result;
466
530
 
467
531
  let source: string | null;
468
532
  try { source = resolveSkillSourceDir(opts.skillSourceDir); } catch { source = null; }
469
533
 
534
+ const currentVersion = source !== null ? readSkillVersion(source) : null;
535
+
470
536
  const keep: ManifestEntry[] = [];
471
537
  for (const e of manifest.entries) {
472
- // Copies are self-contained (no target to dangle); leave them as-is.
473
- if (e.type !== 'symlink') { keep.push(e); continue; }
538
+ // A COPY has no target to dangle, but it is a point-in-time snapshot: unlike a
539
+ // symlink it does not track the package, so an engine upgrade leaves it serving
540
+ // the OLD SKILL.md — including its exit-code contract — indefinitely. Refresh it
541
+ // when the stamped version differs from what we now ship. An entry written
542
+ // before versions were stamped has none, so it reads as stale and is refreshed
543
+ // once. Ownership is re-checked first: a foreign directory at this path is
544
+ // never overwritten.
545
+ if (e.type !== 'symlink') {
546
+ const stale = currentVersion !== null && e.version !== currentVersion;
547
+ if (stale && source !== null && fs.existsSync(e.path) && isOwnedCopy(e.path)) {
548
+ // Build-then-swap, never rm-then-copy. The old shape (rmSync, then
549
+ // copyDir) destroyed the install permanently when the copy failed
550
+ // (ENOSPC, a locked file): the retry guard requires the path to exist,
551
+ // so a vanished install was never retried either. The staging names are
552
+ // dot-prefixed so a crash mid-swap cannot leave a directory a skill
553
+ // scanner would pick up as a duplicate skill.
554
+ const dir = path.dirname(e.path);
555
+ const base = path.basename(e.path);
556
+ // Sweep leftovers from a previous crashed swap before starting a new one.
557
+ try {
558
+ for (const name of fs.readdirSync(dir)) {
559
+ if (name.startsWith(`.${base}.new-`) || name.startsWith(`.${base}.old-`)) {
560
+ fs.rmSync(path.join(dir, name), { recursive: true, force: true });
561
+ }
562
+ }
563
+ } catch { /* hygiene only */ }
564
+ const suffix = crypto.randomBytes(4).toString('hex');
565
+ const fresh = path.join(dir, `.${base}.new-${suffix}`);
566
+ const retired = path.join(dir, `.${base}.old-${suffix}`);
567
+ try {
568
+ copyDir(source, fresh);
569
+ } catch {
570
+ try { fs.rmSync(fresh, { recursive: true, force: true }); } catch { /* best effort */ }
571
+ keep.push(e); // source unreadable / disk full — install intact, retry next startup
572
+ continue;
573
+ }
574
+ try {
575
+ fs.renameSync(e.path, retired);
576
+ } catch {
577
+ try { fs.rmSync(fresh, { recursive: true, force: true }); } catch { /* best effort */ }
578
+ keep.push(e); // could not move the old copy aside (locked on Windows) — intact
579
+ continue;
580
+ }
581
+ try {
582
+ fs.renameSync(fresh, e.path);
583
+ } catch {
584
+ // Put the old copy back — an old skill is strictly better than none.
585
+ try { fs.renameSync(retired, e.path); } catch { /* skills dir itself broken */ }
586
+ try { fs.rmSync(fresh, { recursive: true, force: true }); } catch { /* best effort */ }
587
+ keep.push(e);
588
+ continue;
589
+ }
590
+ try { fs.rmSync(retired, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 }); } catch { /* leaked dot-dir; swept next reconcile */ }
591
+ keep.push({ ...e, source, version: currentVersion });
592
+ result.refreshed.push(e.path);
593
+ } else {
594
+ keep.push(e);
595
+ }
596
+ continue;
597
+ }
474
598
 
475
599
  // Link the user deleted by hand — nothing on disk; drop the stale entry.
476
600
  if (!isSymlinkPresent(e.path)) { result.pruned.push(e.path); continue; }
@@ -514,6 +638,24 @@ export function reconcileSkillInstalls(opts: InstallOptions = {}): ReconcileResu
514
638
  // Helpers
515
639
  // ---------------------------------------------------------------------------
516
640
 
641
+ /**
642
+ * Recursive copy at the JS level — deliberately NOT fs.cpSync. cpSync's native
643
+ * implementation (std::filesystem) ABORTS the whole process on an unreadable
644
+ * directory instead of throwing a catchable error (observed on Node 25:
645
+ * "terminate called after throwing an instance of
646
+ * 'std::filesystem::__cxx11::filesystem_error'"). This runs inside
647
+ * reconcileSkillInstalls at every startup, where a copy failure must be caught
648
+ * and rolled back — never take the host down. statSync/copyFileSync follow
649
+ * symlinks, matching the dereference behaviour cpSync was given.
650
+ */
517
651
  function copyDir(src: string, dest: string): void {
518
- fs.cpSync(src, dest, { recursive: true, dereference: true });
652
+ const st = fs.statSync(src);
653
+ if (st.isDirectory()) {
654
+ fs.mkdirSync(dest, { recursive: true });
655
+ for (const name of fs.readdirSync(src)) {
656
+ copyDir(path.join(src, name), path.join(dest, name));
657
+ }
658
+ } else {
659
+ fs.copyFileSync(src, dest);
660
+ }
519
661
  }
@@ -45,6 +45,24 @@ function loadConfig(): StackExchangeConfig {
45
45
  };
46
46
  }
47
47
 
48
+ // The client instance carries the resilience state the SE API requires callers to
49
+ // honor across requests: the `backoff` directive (rest-client lastBackoff), quota
50
+ // tracking, and the 5-failure circuit breaker. A per-call instance resets all of
51
+ // that on every tool call — the exact throttle-violation pattern SE punishes with
52
+ // IP bans — so the client is process-wide, rebuilt only when the API key changes
53
+ // (the sole constructor input that can vary between calls; requestTimeout is a
54
+ // constant in loadConfig).
55
+ let sharedClient: StackExchangeClient | null = null;
56
+ let sharedClientApiKey: string | null = null;
57
+
58
+ function getClient(config: StackExchangeConfig): StackExchangeClient {
59
+ if (sharedClient === null || sharedClientApiKey !== config.apiKey) {
60
+ sharedClient = new StackExchangeClient(config.apiKey, config.requestTimeout);
61
+ sharedClientApiKey = config.apiKey;
62
+ }
63
+ return sharedClient;
64
+ }
65
+
48
66
  /**
49
67
  * Main command handler for Stack Exchange operations
50
68
  */
@@ -60,11 +78,8 @@ export async function stackexchangeCommand(options: {
60
78
  // Load config
61
79
  const config = loadConfig();
62
80
 
63
- // Initialize client
64
- const client = new StackExchangeClient(
65
- config.apiKey,
66
- config.requestTimeout,
67
- );
81
+ // Shared client — resilience state (backoff, quota, breaker) persists across calls
82
+ const client = getClient(config);
68
83
 
69
84
  try {
70
85
  // Check quota before making requests
@@ -56,8 +56,13 @@ export function formatAnswersTable(answers: Answer[]): string {
56
56
  output += `- **Question ID:** ${a.question_id}\n`;
57
57
  output += `- **Created:** ${new Date(a.creation_date * 1000).toLocaleString()}\n`;
58
58
 
59
- if (a.body && a.body.length < 2000) {
60
- output += `\n### Answer Body\n\n${a.body}\n`;
59
+ if (a.body) {
60
+ // Truncate long bodies to a preview rather than dropping them entirely — the
61
+ // previous `length < 2000` guard omitted the body for exactly the longest
62
+ // answers (same class as the question-body fix above; answers get a more
63
+ // generous bound since the body is the whole point of an answer).
64
+ const body = a.body.length > 2000 ? `${a.body.substring(0, 2000)}...` : a.body;
65
+ output += `\n### Answer Body\n\n${body}\n`;
61
66
  }
62
67
 
63
68
  output += '\n---\n\n';