@lincoln504/pi-research 1.1.2 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/cli.mjs +1287 -540
- package/dist/thread-worker.mjs +30 -11
- package/docs/AGENT-SKILL.md +11 -4
- package/docs/ARCHITECTURE.md +19 -9
- package/docs/CHANGELOG.md +81 -1
- package/docs/CONFIGURATION.md +9 -7
- package/docs/KNOWLEDGE-STORE.md +4 -3
- package/docs/PI-EXTENSION.md +1 -1
- package/docs/SDK.md +55 -0
- package/docs/THIRD-PARTY-NOTICES.md +1 -1
- package/package.json +5 -4
- package/scripts/cleanup.cjs +50 -10
- package/scripts/prepare.cjs +15 -5
- package/scripts/setup.cjs +24 -4
- package/skills/pi-research/README.md +9 -2
- package/skills/pi-research/SKILL.md +6 -3
- package/skills/pi-research/references/configuration.md +6 -2
- package/skills/pi-research/scripts/run.mjs +24 -2
- package/src/cli.ts +149 -21
- package/src/config.ts +56 -15
- package/src/constants.ts +31 -0
- package/src/core/interfaces/knowledge-interfaces.ts +6 -1
- package/src/core/interfaces/state-manager-interfaces.ts +1 -1
- package/src/core/pi-version.ts +136 -0
- package/src/core/service-registry.ts +117 -8
- package/src/index.ts +11 -18
- package/src/infrastructure/browser/browser-client.ts +16 -4
- package/src/infrastructure/browser/browser-error-utils.ts +6 -1
- package/src/infrastructure/browser/browser-server.ts +17 -5
- package/src/infrastructure/browser/browser-task-scheduler.ts +101 -23
- package/src/infrastructure/browser/cleanup-utils.ts +45 -7
- package/src/infrastructure/browser/client-agent.ts +4 -3
- package/src/infrastructure/browser/priority-task-queue.ts +10 -3
- package/src/infrastructure/browser/scheduler-factory.ts +26 -1
- package/src/infrastructure/browser/task-execution-service.ts +92 -28
- package/src/infrastructure/browser/thread-worker-lifecycle.ts +50 -13
- package/src/infrastructure/browser/thread-worker.mjs +30 -11
- package/src/infrastructure/browser/worker-pool-manager.ts +11 -3
- package/src/infrastructure/embedding/embedding-client.ts +22 -8
- package/src/infrastructure/embedding/embedding-factory.ts +28 -11
- package/src/infrastructure/embedding/embedding-server.ts +65 -4
- package/src/infrastructure/file-lock-service.ts +223 -26
- package/src/infrastructure/process-lifecycle-service.ts +108 -31
- package/src/infrastructure/state/state-backup-manager.ts +5 -7
- package/src/infrastructure/state/state-manager-service.ts +1 -1
- package/src/infrastructure/state/state-manager.ts +19 -20
- package/src/infrastructure/types/state-types.ts +9 -0
- package/src/knowledge/embedder-init.ts +1 -1
- package/src/knowledge/embedder.ts +52 -3
- package/src/knowledge/index.ts +1 -1
- package/src/knowledge/store.ts +35 -9
- package/src/knowledge/webgpu-viability.ts +9 -4
- package/src/knowledge/writer-queue.ts +88 -46
- package/src/orchestration/deep-research-orchestrator.ts +13 -2
- package/src/orchestration/quick-research-orchestrator.ts +2 -2
- package/src/orchestration/research-orchestration-service.ts +2 -4
- package/src/orchestration/research-synthesis-service.ts +33 -11
- package/src/security/cisa-kev.ts +10 -3
- package/src/security/github-advisories.ts +18 -7
- package/src/security/nvd-client.ts +17 -6
- package/src/security/osv-client.ts +14 -4
- package/src/skill-install/skill-installer.ts +135 -9
- package/src/stackexchange/rest-client.ts +6 -2
- package/src/tools/research-knowledge-search.ts +15 -0
- package/src/tools/search.ts +6 -1
- package/src/utils/atomic-replace.ts +101 -0
- package/src/utils/citation-utils.ts +3 -1
- package/src/utils/http-body.ts +176 -0
- package/src/web-research/browser-search.ts +50 -4
- package/src/web-research/search.ts +20 -5
- package/src/web-research/types.ts +17 -4
- package/src/web-research/web-scraper.ts +7 -50
- package/src/youtube/transcript-client.ts +5 -1
|
@@ -17,6 +17,7 @@ import { normalizeEcosystem } from './ecosystem.ts';
|
|
|
17
17
|
import { createTimeoutSignal, retryWithBackoff, isTransientError } from '../web-research/retry-utils.ts';
|
|
18
18
|
import { CircuitBreaker } from '../utils/circuit-breaker.ts';
|
|
19
19
|
import { metrics } from '../utils/metrics.ts';
|
|
20
|
+
import { readJsonCapped, BodyTooLargeError } from '../utils/http-body.ts';
|
|
20
21
|
import { logger } from '../logger.ts';
|
|
21
22
|
import {
|
|
22
23
|
isGitHubAdvisoryRaw,
|
|
@@ -153,10 +154,16 @@ export async function searchGitHubAdvisories(
|
|
|
153
154
|
|
|
154
155
|
let data: unknown;
|
|
155
156
|
try {
|
|
156
|
-
data = await response
|
|
157
|
-
} catch {
|
|
157
|
+
data = await readJsonCapped(response);
|
|
158
|
+
} catch (err) {
|
|
159
|
+
// Size failures keep their own identity rather than being reported as a
|
|
160
|
+
// syntax error, which would point diagnosis at the wrong thing entirely.
|
|
161
|
+
if (err instanceof BodyTooLargeError) {
|
|
162
|
+
metrics.increment('github_search_errors_total', 1, { error_type: 'body_too_large' });
|
|
163
|
+
throw err;
|
|
164
|
+
}
|
|
158
165
|
metrics.increment('github_search_errors_total', 1, { error_type: 'malformed_json' });
|
|
159
|
-
throw new Error('GitHub API returned a malformed JSON response (repo advisories).');
|
|
166
|
+
throw new Error('GitHub API returned a malformed JSON response (repo advisories).', { cause: err });
|
|
160
167
|
}
|
|
161
168
|
let repoAdvisories: readonly GitHubAdvisoryRaw[] = [];
|
|
162
169
|
|
|
@@ -244,10 +251,14 @@ export async function searchGitHubAdvisories(
|
|
|
244
251
|
// A 404 on a direct GHSA/CVE lookup means "no such advisory", not failure.
|
|
245
252
|
let data: unknown;
|
|
246
253
|
try {
|
|
247
|
-
data = await response
|
|
248
|
-
} catch {
|
|
254
|
+
data = await readJsonCapped(response);
|
|
255
|
+
} catch (err) {
|
|
256
|
+
if (err instanceof BodyTooLargeError) {
|
|
257
|
+
metrics.increment('github_search_errors_total', 1, { error_type: 'body_too_large', endpoint: endpointType });
|
|
258
|
+
throw err;
|
|
259
|
+
}
|
|
249
260
|
metrics.increment('github_search_errors_total', 1, { error_type: 'malformed_json', endpoint: endpointType });
|
|
250
|
-
throw new Error(`GitHub API returned a malformed JSON response for term "${term}"
|
|
261
|
+
throw new Error(`GitHub API returned a malformed JSON response for term "${term}".`, { cause: err });
|
|
251
262
|
}
|
|
252
263
|
let items: readonly GitHubAdvisoryRaw[] = [];
|
|
253
264
|
|
|
@@ -259,7 +270,7 @@ export async function searchGitHubAdvisories(
|
|
|
259
270
|
items = data.items.filter(isGitHubAdvisoryRaw);
|
|
260
271
|
}
|
|
261
272
|
|
|
262
|
-
metrics.increment(items.length > 0 ? 'github_cache_hits_total' : 'github_cache_misses_total', 1, {
|
|
273
|
+
metrics.increment(items.length > 0 ? 'github_cache_hits_total' : 'github_cache_misses_total', 1, { endpoint: endpointType });
|
|
263
274
|
|
|
264
275
|
return items.map(mapGitHubAdvisory);
|
|
265
276
|
};
|
|
@@ -24,6 +24,7 @@ import { createTimeoutSignal, retryWithBackoff, isTransientError } from '../web-
|
|
|
24
24
|
import { CircuitBreaker } from '../utils/circuit-breaker.ts';
|
|
25
25
|
import { safeUnref } from '../utils/safe-unref.ts';
|
|
26
26
|
import { metrics } from '../utils/metrics.ts';
|
|
27
|
+
import { readJsonCapped, BodyTooLargeError, MAX_BULK_JSON_BODY_BYTES } from '../utils/http-body.ts';
|
|
27
28
|
import {
|
|
28
29
|
isWeaknessDescription,
|
|
29
30
|
isWeakness,
|
|
@@ -396,11 +397,17 @@ async function fetchPaginated(
|
|
|
396
397
|
const response = await fetchWithRetry(url, signal);
|
|
397
398
|
let data: unknown;
|
|
398
399
|
try {
|
|
399
|
-
|
|
400
|
-
|
|
400
|
+
// A page can legitimately carry up to MAX_RESULTS_PER_PAGE (2000) full CVE
|
|
401
|
+
// records, so pagination reads against the bulk cap rather than the default.
|
|
402
|
+
data = await readJsonCapped(response, MAX_BULK_JSON_BODY_BYTES);
|
|
403
|
+
} catch (err) {
|
|
401
404
|
// A 200 with a truncated/HTML body (proxy or CDN error page) would otherwise
|
|
402
|
-
// throw and discard this whole term. Stop paginating this term instead
|
|
403
|
-
|
|
405
|
+
// throw and discard this whole term. Stop paginating this term instead —
|
|
406
|
+
// the pages already gathered are kept. An over-cap body degrades the same
|
|
407
|
+
// way but is counted separately so the cause stays diagnosable.
|
|
408
|
+
metrics.increment('nvd_search_errors_total', 1, {
|
|
409
|
+
error_type: err instanceof BodyTooLargeError ? 'body_too_large' : 'malformed_json',
|
|
410
|
+
});
|
|
404
411
|
break;
|
|
405
412
|
}
|
|
406
413
|
|
|
@@ -412,10 +419,14 @@ async function fetchPaginated(
|
|
|
412
419
|
const admitted = rawEntries.filter(admitEntry).map((entry) => parseNVDEntry(entry, options));
|
|
413
420
|
|
|
414
421
|
if (rawEntries.length === 0) {
|
|
415
|
-
metrics
|
|
422
|
+
// No `term` label: the metrics registry keys counters by label set and is only
|
|
423
|
+
// cleared by an explicit clearSession(), so labelling by the raw search term made
|
|
424
|
+
// every distinct security query add a permanent entry for the life of a pi host
|
|
425
|
+
// session — unbounded growth driven straight by user input.
|
|
426
|
+
metrics.increment('nvd_cache_misses_total', 1);
|
|
416
427
|
break;
|
|
417
428
|
}
|
|
418
|
-
metrics.increment('nvd_cache_hits_total', 1
|
|
429
|
+
metrics.increment('nvd_cache_hits_total', 1);
|
|
419
430
|
|
|
420
431
|
allVulnerabilities.push(...admitted);
|
|
421
432
|
|
|
@@ -14,6 +14,7 @@ import { logger } from '../logger.ts';
|
|
|
14
14
|
import { OSV_TIMEOUT_MS, DEFAULT_MAX_RETRIES, DEFAULT_INITIAL_DELAY_MS, DEFAULT_MAX_DELAY_MS } from '../constants.ts';
|
|
15
15
|
import { CircuitBreaker } from '../utils/circuit-breaker.ts';
|
|
16
16
|
import { metrics } from '../utils/metrics.ts';
|
|
17
|
+
import { readJsonCapped, BodyTooLargeError } from '../utils/http-body.ts';
|
|
17
18
|
import {
|
|
18
19
|
isOsvVulnerability,
|
|
19
20
|
isOsvQueryResponse,
|
|
@@ -137,10 +138,17 @@ export async function searchOSV(
|
|
|
137
138
|
|
|
138
139
|
let data: unknown;
|
|
139
140
|
try {
|
|
140
|
-
data = await response
|
|
141
|
-
} catch {
|
|
141
|
+
data = await readJsonCapped(response);
|
|
142
|
+
} catch (err) {
|
|
143
|
+
// An over-cap body is a size failure, not a syntax one. Rethrowing it
|
|
144
|
+
// unwrapped keeps the diagnosis honest instead of reporting a 40MB reply
|
|
145
|
+
// as malformed JSON.
|
|
146
|
+
if (err instanceof BodyTooLargeError) {
|
|
147
|
+
metrics.increment('osv_search_errors_total', 1, { error_type: 'body_too_large' });
|
|
148
|
+
throw err;
|
|
149
|
+
}
|
|
142
150
|
metrics.increment('osv_search_errors_total', 1, { error_type: 'malformed_json' });
|
|
143
|
-
throw new Error(`OSV returned a malformed JSON response for "${term}"
|
|
151
|
+
throw new Error(`OSV returned a malformed JSON response for "${term}".`, { cause: err });
|
|
144
152
|
}
|
|
145
153
|
|
|
146
154
|
let items: OsvVulnerability[];
|
|
@@ -154,7 +162,9 @@ export async function searchOSV(
|
|
|
154
162
|
}
|
|
155
163
|
|
|
156
164
|
const endpoint = isIdLookup ? 'vulns_by_id' : 'query';
|
|
157
|
-
|
|
165
|
+
// `endpoint` only — see the nvd-client note: a raw-term label grows the counter
|
|
166
|
+
// registry without bound across a session.
|
|
167
|
+
metrics.increment(items.length > 0 ? 'osv_cache_hits_total' : 'osv_cache_misses_total', 1, { endpoint });
|
|
158
168
|
|
|
159
169
|
const vulns: Vulnerability[] = [];
|
|
160
170
|
for (const item of items) {
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
* only written on explicit opt-in (callers must pass them deliberately).
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
|
+
import * as crypto from 'node:crypto';
|
|
21
22
|
import * as fs from 'node:fs';
|
|
22
23
|
import * as os from 'node:os';
|
|
23
24
|
import * as path from 'node:path';
|
|
@@ -57,6 +58,24 @@ export const HARNESSES: readonly HarnessDef[] = [
|
|
|
57
58
|
const SKILL_NAME = 'pi-research';
|
|
58
59
|
const PACKAGE_NAME = '@lincoln504/pi-research';
|
|
59
60
|
|
|
61
|
+
/**
|
|
62
|
+
* This package's version, used to stamp manifest entries so a stale COPY can be
|
|
63
|
+
* detected and refreshed. Read from the SKILL.md we are about to install rather
|
|
64
|
+
* than from package.json: that file is the copied artifact, its metadata line is
|
|
65
|
+
* kept in sync by the `version` npm script, and it is present in every layout
|
|
66
|
+
* (repo, published tarball, installed copy). Returns null if unreadable, which
|
|
67
|
+
* simply means "cannot tell" — the entry is then always treated as stale, which
|
|
68
|
+
* refreshes rather than skips.
|
|
69
|
+
*/
|
|
70
|
+
function readSkillVersion(sourceDir: string): string | null {
|
|
71
|
+
try {
|
|
72
|
+
const md = fs.readFileSync(path.join(sourceDir, 'SKILL.md'), 'utf-8');
|
|
73
|
+
return /"version"\s*:\s*"([^"]+)"/.exec(md)?.[1] ?? null;
|
|
74
|
+
} catch {
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
60
79
|
/**
|
|
61
80
|
* The coding agents the in-app installer (the /research-config TUI) targets:
|
|
62
81
|
* Claude, Codex, and OpenClaw. Each loads skills from a home-directory
|
|
@@ -81,6 +100,22 @@ export interface ManifestEntry {
|
|
|
81
100
|
/** Absolute path of the skill source this was linked/copied from. */
|
|
82
101
|
source: string;
|
|
83
102
|
createdAt: string;
|
|
103
|
+
/**
|
|
104
|
+
* Package version that produced this entry.
|
|
105
|
+
*
|
|
106
|
+
* Only load-bearing for `copy` entries: a symlink re-reads the package on every
|
|
107
|
+
* use and so is always current, but a COPY is a point-in-time snapshot that
|
|
108
|
+
* nothing ever refreshed — reconcile skipped non-symlinks entirely, and the
|
|
109
|
+
* entry recorded no version, so drift was undetectable by construction. Copies
|
|
110
|
+
* are not rare: they happen on Windows without symlink privilege, cross-device,
|
|
111
|
+
* with `--copy`, and via the documented OpenClaw install route. Those users kept
|
|
112
|
+
* the SKILL.md their engine shipped with *forever* — including its exit-code
|
|
113
|
+
* contract — while the engine underneath them changed.
|
|
114
|
+
*
|
|
115
|
+
* Optional so manifests written before this field are still valid; an entry with
|
|
116
|
+
* no version simply reads as "not current" and is refreshed once.
|
|
117
|
+
*/
|
|
118
|
+
version?: string;
|
|
84
119
|
}
|
|
85
120
|
|
|
86
121
|
interface Manifest {
|
|
@@ -293,6 +328,7 @@ export function installSkill(toolIds: string[], opts: InstallOptions = {}): Inst
|
|
|
293
328
|
const manifest = readManifest(opts);
|
|
294
329
|
const results: InstallResult[] = [];
|
|
295
330
|
const now = new Date().toISOString();
|
|
331
|
+
const skillVersion = readSkillVersion(source);
|
|
296
332
|
|
|
297
333
|
for (const id of toolIds) {
|
|
298
334
|
const def = HARNESSES.find(h => h.id === id);
|
|
@@ -315,16 +351,22 @@ export function installSkill(toolIds: string[], opts: InstallOptions = {}): Inst
|
|
|
315
351
|
if (lst.isSymbolicLink()) {
|
|
316
352
|
const dest = path.resolve(path.dirname(absSkillPath), fs.readlinkSync(absSkillPath));
|
|
317
353
|
if (dest !== path.resolve(source)) {
|
|
354
|
+
// dryRun means "plan only, touch nothing" — the re-point is a real
|
|
355
|
+
// unlink+symlink and must honour it like the fresh-install path does.
|
|
356
|
+
if (opts.dryRun) {
|
|
357
|
+
results.push({ tool: id, path: absSkillPath, type: 'symlink', status: 'planned', message: 'would re-point stale symlink to current source' });
|
|
358
|
+
continue;
|
|
359
|
+
}
|
|
318
360
|
fs.unlinkSync(absSkillPath);
|
|
319
361
|
fs.symlinkSync(source, absSkillPath, process.platform === 'win32' ? 'junction' : 'dir');
|
|
320
|
-
upsertEntry(manifest, { tool: id, path: absSkillPath, type: 'symlink', source, createdAt: now });
|
|
362
|
+
upsertEntry(manifest, { tool: id, path: absSkillPath, type: 'symlink', source, createdAt: now, ...(skillVersion ? { version: skillVersion } : {}) });
|
|
321
363
|
results.push({ tool: id, path: absSkillPath, type: 'symlink', status: 'installed', message: 're-pointed stale symlink to current source' });
|
|
322
364
|
continue;
|
|
323
365
|
}
|
|
324
366
|
}
|
|
325
367
|
} catch { /* fall through to already-installed below */ }
|
|
326
368
|
results.push({ tool: id, path: absSkillPath, type, status: 'already-installed' });
|
|
327
|
-
upsertEntry(manifest, { tool: id, path: absSkillPath, type: isSymlinkPresent(absSkillPath) ? 'symlink' : 'copy', source, createdAt: now });
|
|
369
|
+
upsertEntry(manifest, { tool: id, path: absSkillPath, type: isSymlinkPresent(absSkillPath) ? 'symlink' : 'copy', source, createdAt: now, ...(skillVersion ? { version: skillVersion } : {}) });
|
|
328
370
|
continue;
|
|
329
371
|
}
|
|
330
372
|
if (fs.existsSync(absSkillPath) || isSymlinkPresent(absSkillPath)) {
|
|
@@ -344,14 +386,14 @@ export function installSkill(toolIds: string[], opts: InstallOptions = {}): Inst
|
|
|
344
386
|
} catch (err: any) {
|
|
345
387
|
// Windows without privilege, or cross-device: fall back to copy.
|
|
346
388
|
copyDir(source, absSkillPath);
|
|
347
|
-
upsertEntry(manifest, { tool: id, path: absSkillPath, type: 'copy', source, createdAt: now });
|
|
389
|
+
upsertEntry(manifest, { tool: id, path: absSkillPath, type: 'copy', source, createdAt: now, ...(skillVersion ? { version: skillVersion } : {}) });
|
|
348
390
|
results.push({ tool: id, path: absSkillPath, type: 'copy', status: 'installed', message: `symlink unavailable (${err?.code ?? err?.message}); copied instead` });
|
|
349
391
|
continue;
|
|
350
392
|
}
|
|
351
393
|
} else {
|
|
352
394
|
copyDir(source, absSkillPath);
|
|
353
395
|
}
|
|
354
|
-
upsertEntry(manifest, { tool: id, path: absSkillPath, type, source, createdAt: now });
|
|
396
|
+
upsertEntry(manifest, { tool: id, path: absSkillPath, type, source, createdAt: now, ...(skillVersion ? { version: skillVersion } : {}) });
|
|
355
397
|
results.push({ tool: id, path: absSkillPath, type, status: 'installed' });
|
|
356
398
|
} catch (err: any) {
|
|
357
399
|
results.push({ tool: id, path: absSkillPath, type, status: 'error', message: err?.message ?? String(err) });
|
|
@@ -412,7 +454,7 @@ export function uninstallSkill(toolIds?: string[], opts: InstallOptions = {}): U
|
|
|
412
454
|
}
|
|
413
455
|
try {
|
|
414
456
|
if (isSymlinkPresent(skillPath)) fs.unlinkSync(skillPath);
|
|
415
|
-
else fs.rmSync(skillPath, { recursive: true, force: true });
|
|
457
|
+
else fs.rmSync(skillPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
|
|
416
458
|
results.push({ tool, path: skillPath, status: 'removed' });
|
|
417
459
|
} catch (err: any) {
|
|
418
460
|
results.push({ tool, path: skillPath, status: 'error', message: err?.message ?? String(err) });
|
|
@@ -441,6 +483,12 @@ export interface ReconcileResult {
|
|
|
441
483
|
pruned: string[];
|
|
442
484
|
/** Stale/dangling owned symlinks re-pointed to the current package source. */
|
|
443
485
|
repointed: string[];
|
|
486
|
+
/**
|
|
487
|
+
* Owned COPIES refreshed because the package version moved on. A symlink tracks
|
|
488
|
+
* the package automatically; a copy does not, so without this an upgrade left
|
|
489
|
+
* copy-installed users on the previous SKILL.md indefinitely.
|
|
490
|
+
*/
|
|
491
|
+
refreshed: string[];
|
|
444
492
|
}
|
|
445
493
|
|
|
446
494
|
/**
|
|
@@ -460,17 +508,77 @@ export interface ReconcileResult {
|
|
|
460
508
|
* Foreign (non-owned) paths and self-contained copies are never touched.
|
|
461
509
|
*/
|
|
462
510
|
export function reconcileSkillInstalls(opts: InstallOptions = {}): ReconcileResult {
|
|
463
|
-
const result: ReconcileResult = { pruned: [], repointed: [] };
|
|
511
|
+
const result: ReconcileResult = { pruned: [], repointed: [], refreshed: [] };
|
|
464
512
|
const manifest = readManifest(opts);
|
|
465
513
|
if (manifest.entries.length === 0) return result;
|
|
466
514
|
|
|
467
515
|
let source: string | null;
|
|
468
516
|
try { source = resolveSkillSourceDir(opts.skillSourceDir); } catch { source = null; }
|
|
469
517
|
|
|
518
|
+
const currentVersion = source !== null ? readSkillVersion(source) : null;
|
|
519
|
+
|
|
470
520
|
const keep: ManifestEntry[] = [];
|
|
471
521
|
for (const e of manifest.entries) {
|
|
472
|
-
//
|
|
473
|
-
|
|
522
|
+
// A COPY has no target to dangle, but it is a point-in-time snapshot: unlike a
|
|
523
|
+
// symlink it does not track the package, so an engine upgrade leaves it serving
|
|
524
|
+
// the OLD SKILL.md — including its exit-code contract — indefinitely. Refresh it
|
|
525
|
+
// when the stamped version differs from what we now ship. An entry written
|
|
526
|
+
// before versions were stamped has none, so it reads as stale and is refreshed
|
|
527
|
+
// once. Ownership is re-checked first: a foreign directory at this path is
|
|
528
|
+
// never overwritten.
|
|
529
|
+
if (e.type !== 'symlink') {
|
|
530
|
+
const stale = currentVersion !== null && e.version !== currentVersion;
|
|
531
|
+
if (stale && source !== null && fs.existsSync(e.path) && isOwnedCopy(e.path)) {
|
|
532
|
+
// Build-then-swap, never rm-then-copy. The old shape (rmSync, then
|
|
533
|
+
// copyDir) destroyed the install permanently when the copy failed
|
|
534
|
+
// (ENOSPC, a locked file): the retry guard requires the path to exist,
|
|
535
|
+
// so a vanished install was never retried either. The staging names are
|
|
536
|
+
// dot-prefixed so a crash mid-swap cannot leave a directory a skill
|
|
537
|
+
// scanner would pick up as a duplicate skill.
|
|
538
|
+
const dir = path.dirname(e.path);
|
|
539
|
+
const base = path.basename(e.path);
|
|
540
|
+
// Sweep leftovers from a previous crashed swap before starting a new one.
|
|
541
|
+
try {
|
|
542
|
+
for (const name of fs.readdirSync(dir)) {
|
|
543
|
+
if (name.startsWith(`.${base}.new-`) || name.startsWith(`.${base}.old-`)) {
|
|
544
|
+
fs.rmSync(path.join(dir, name), { recursive: true, force: true });
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
} catch { /* hygiene only */ }
|
|
548
|
+
const suffix = crypto.randomBytes(4).toString('hex');
|
|
549
|
+
const fresh = path.join(dir, `.${base}.new-${suffix}`);
|
|
550
|
+
const retired = path.join(dir, `.${base}.old-${suffix}`);
|
|
551
|
+
try {
|
|
552
|
+
copyDir(source, fresh);
|
|
553
|
+
} catch {
|
|
554
|
+
try { fs.rmSync(fresh, { recursive: true, force: true }); } catch { /* best effort */ }
|
|
555
|
+
keep.push(e); // source unreadable / disk full — install intact, retry next startup
|
|
556
|
+
continue;
|
|
557
|
+
}
|
|
558
|
+
try {
|
|
559
|
+
fs.renameSync(e.path, retired);
|
|
560
|
+
} catch {
|
|
561
|
+
try { fs.rmSync(fresh, { recursive: true, force: true }); } catch { /* best effort */ }
|
|
562
|
+
keep.push(e); // could not move the old copy aside (locked on Windows) — intact
|
|
563
|
+
continue;
|
|
564
|
+
}
|
|
565
|
+
try {
|
|
566
|
+
fs.renameSync(fresh, e.path);
|
|
567
|
+
} catch {
|
|
568
|
+
// Put the old copy back — an old skill is strictly better than none.
|
|
569
|
+
try { fs.renameSync(retired, e.path); } catch { /* skills dir itself broken */ }
|
|
570
|
+
try { fs.rmSync(fresh, { recursive: true, force: true }); } catch { /* best effort */ }
|
|
571
|
+
keep.push(e);
|
|
572
|
+
continue;
|
|
573
|
+
}
|
|
574
|
+
try { fs.rmSync(retired, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 }); } catch { /* leaked dot-dir; swept next reconcile */ }
|
|
575
|
+
keep.push({ ...e, source, version: currentVersion });
|
|
576
|
+
result.refreshed.push(e.path);
|
|
577
|
+
} else {
|
|
578
|
+
keep.push(e);
|
|
579
|
+
}
|
|
580
|
+
continue;
|
|
581
|
+
}
|
|
474
582
|
|
|
475
583
|
// Link the user deleted by hand — nothing on disk; drop the stale entry.
|
|
476
584
|
if (!isSymlinkPresent(e.path)) { result.pruned.push(e.path); continue; }
|
|
@@ -514,6 +622,24 @@ export function reconcileSkillInstalls(opts: InstallOptions = {}): ReconcileResu
|
|
|
514
622
|
// Helpers
|
|
515
623
|
// ---------------------------------------------------------------------------
|
|
516
624
|
|
|
625
|
+
/**
|
|
626
|
+
* Recursive copy at the JS level — deliberately NOT fs.cpSync. cpSync's native
|
|
627
|
+
* implementation (std::filesystem) ABORTS the whole process on an unreadable
|
|
628
|
+
* directory instead of throwing a catchable error (observed on Node 25:
|
|
629
|
+
* "terminate called after throwing an instance of
|
|
630
|
+
* 'std::filesystem::__cxx11::filesystem_error'"). This runs inside
|
|
631
|
+
* reconcileSkillInstalls at every startup, where a copy failure must be caught
|
|
632
|
+
* and rolled back — never take the host down. statSync/copyFileSync follow
|
|
633
|
+
* symlinks, matching the dereference behaviour cpSync was given.
|
|
634
|
+
*/
|
|
517
635
|
function copyDir(src: string, dest: string): void {
|
|
518
|
-
fs.
|
|
636
|
+
const st = fs.statSync(src);
|
|
637
|
+
if (st.isDirectory()) {
|
|
638
|
+
fs.mkdirSync(dest, { recursive: true });
|
|
639
|
+
for (const name of fs.readdirSync(src)) {
|
|
640
|
+
copyDir(path.join(src, name), path.join(dest, name));
|
|
641
|
+
}
|
|
642
|
+
} else {
|
|
643
|
+
fs.copyFileSync(src, dest);
|
|
644
|
+
}
|
|
519
645
|
}
|
|
@@ -7,6 +7,7 @@ import type { StackExchangeWrapper } from './types.ts';
|
|
|
7
7
|
import { logger } from '../logger.ts';
|
|
8
8
|
import { CircuitBreaker } from '../utils/circuit-breaker.ts';
|
|
9
9
|
import { metrics } from '../utils/metrics.ts';
|
|
10
|
+
import { readJsonCapped, BodyTooLargeError } from '../utils/http-body.ts';
|
|
10
11
|
|
|
11
12
|
const API_BASE = 'https://api.stackexchange.com/2.3';
|
|
12
13
|
|
|
@@ -99,7 +100,7 @@ export class StackExchangeClient {
|
|
|
99
100
|
|
|
100
101
|
// NB: the timeout is cleared in `finally`, not here. Clearing it now —
|
|
101
102
|
// when only the headers have arrived — would leave the body read
|
|
102
|
-
// (
|
|
103
|
+
// (readJsonCapped below) with no timeout, so a stalled body could hang
|
|
103
104
|
// for the whole process lifetime.
|
|
104
105
|
|
|
105
106
|
// The SE API returns a structured JSON wrapper (with error_id) even for
|
|
@@ -109,8 +110,11 @@ export class StackExchangeClient {
|
|
|
109
110
|
// "Unexpected token <" SyntaxError that hides the real cause.
|
|
110
111
|
let data: StackExchangeWrapper<T>;
|
|
111
112
|
try {
|
|
112
|
-
data = await
|
|
113
|
+
data = await readJsonCapped<StackExchangeWrapper<T>>(response);
|
|
113
114
|
} catch (parseErr) {
|
|
115
|
+
// An over-cap body is a size failure, not a malformed-JSON one: rethrow
|
|
116
|
+
// it unwrapped so it can't be relabelled as an HTTP-status error below.
|
|
117
|
+
if (parseErr instanceof BodyTooLargeError) throw parseErr;
|
|
114
118
|
if (!response.ok) throw new Error(`HTTP ${response.status} from Stack Exchange API`, { cause: parseErr });
|
|
115
119
|
throw parseErr;
|
|
116
120
|
}
|
|
@@ -962,6 +962,21 @@ export function createResearchKnowledgeSearchTool(iface?: ConfigInterface): Tool
|
|
|
962
962
|
return buildSteeringResult(result, urls);
|
|
963
963
|
} catch (error) {
|
|
964
964
|
const durationMs = Date.now() - startTime;
|
|
965
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
966
|
+
|
|
967
|
+
// A CANCELLED search is not an empty store. Reporting the miss string here
|
|
968
|
+
// told the calling agent, definitively, that the knowledge store has
|
|
969
|
+
// nothing on this topic — so it would go straight to full live research,
|
|
970
|
+
// the very work the user just interrupted. Propagate the abort instead;
|
|
971
|
+
// the run is ending either way. (Same misclassification as the search
|
|
972
|
+
// layer's "browser workers may be unavailable" on Ctrl-C.)
|
|
973
|
+
if (signal?.aborted || (error instanceof Error && error.name === 'AbortError') || /^(aborted|research (aborted|cancelled))$/i.test(msg)) {
|
|
974
|
+
metrics.observe('research_knowledge_search_duration_ms', durationMs, { status: 'cancelled' });
|
|
975
|
+
metrics.increment('research_knowledge_search_total', 1, { status: 'cancelled' });
|
|
976
|
+
logger.debug('[research-knowledge-search] Cancelled mid-search — propagating rather than reporting an empty store.');
|
|
977
|
+
throw error;
|
|
978
|
+
}
|
|
979
|
+
|
|
965
980
|
metrics.observe('research_knowledge_search_duration_ms', durationMs, { status: 'error' });
|
|
966
981
|
metrics.increment('research_knowledge_search_total', 1, { status: 'error' });
|
|
967
982
|
|
package/src/tools/search.ts
CHANGED
|
@@ -106,7 +106,12 @@ export function createSearchTool(options: {
|
|
|
106
106
|
results.forEach((r, i) => {
|
|
107
107
|
markdown += `## Query ${i + 1}: ${r.query}\n`;
|
|
108
108
|
if (r.results.length === 0) {
|
|
109
|
-
|
|
109
|
+
// Render the attributed failure, not a bare "no results": a timeout or
|
|
110
|
+
// dead worker says nothing about the query, and presenting it as an
|
|
111
|
+
// empty result sent the model off rewriting perfectly good queries —
|
|
112
|
+
// the misdirection QueryFailure exists to prevent. This is the only
|
|
113
|
+
// surface the researcher agent actually reads.
|
|
114
|
+
markdown += r.error ? `*${r.error.message}*\n\n` : `*No results found.*\n\n`;
|
|
110
115
|
} else {
|
|
111
116
|
r.results.forEach((item, j) => {
|
|
112
117
|
markdown += `[${j + 1}] **${item.title}**\n${item.url}\n${item.content}\n\n`;
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Atomic file replacement, with the Windows caveat handled honestly.
|
|
3
|
+
*
|
|
4
|
+
* The temp-file + fsync + rename dance exists so a reader never observes a partial
|
|
5
|
+
* file and a crash never leaves one: rename is atomic, so the destination is either
|
|
6
|
+
* entirely the old content or entirely the new.
|
|
7
|
+
*
|
|
8
|
+
* On Windows that guarantee is easy to lose. The long-standing comment at these call
|
|
9
|
+
* sites — "fs.rename fails on Windows if target exists (NTFS)" — is not true: Node
|
|
10
|
+
* calls `MoveFileExW` with `MOVEFILE_REPLACE_EXISTING`, so replacing an existing file
|
|
11
|
+
* is fine. What actually fails is a rename whose TARGET is held open by another
|
|
12
|
+
* process without delete sharing (an AV scanner mid-scan, a backup agent), which
|
|
13
|
+
* surfaces as EPERM/EBUSY/EACCES.
|
|
14
|
+
*
|
|
15
|
+
* The old remedy — fall straight to `copyFile` — traded a transient, retryable error
|
|
16
|
+
* for a permanent hazard: `copyFile` truncates the destination and rewrites it in
|
|
17
|
+
* place, so a concurrent reader or a crash mid-copy sees a torn or zero-length file.
|
|
18
|
+
* For `state.json` (leader election, server auth secrets) and `config.env` (API keys)
|
|
19
|
+
* that is a much worse outcome than the error it was avoiding.
|
|
20
|
+
*
|
|
21
|
+
* So: retry the rename first. An AV lock clears in tens of milliseconds, and a retried
|
|
22
|
+
* rename stays atomic. Only a lock that outlives every retry falls back to copying,
|
|
23
|
+
* which is reported to the caller so it can be logged as the degraded path it is.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import * as fs from 'node:fs/promises';
|
|
27
|
+
import * as fsSync from 'node:fs';
|
|
28
|
+
|
|
29
|
+
/** Transient Windows sharing/locking failures — worth retrying, unlike ENOENT/ENOSPC. */
|
|
30
|
+
const RETRYABLE_CODES = new Set(['EPERM', 'EBUSY', 'EACCES']);
|
|
31
|
+
|
|
32
|
+
const MAX_RENAME_ATTEMPTS = 5;
|
|
33
|
+
const RENAME_RETRY_DELAY_MS = 25;
|
|
34
|
+
|
|
35
|
+
export type ReplaceOutcome = 'renamed' | 'copied';
|
|
36
|
+
|
|
37
|
+
function isRetryable(err: unknown): boolean {
|
|
38
|
+
const code = (err as NodeJS.ErrnoException | undefined)?.code;
|
|
39
|
+
return code !== undefined && RETRYABLE_CODES.has(code);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Replace `destPath` with `tmpPath` atomically where the platform allows it.
|
|
44
|
+
*
|
|
45
|
+
* Returns `'renamed'` for the atomic path and `'copied'` for the degraded Windows
|
|
46
|
+
* fallback. Rethrows on every non-Windows failure, and on Windows failures that are
|
|
47
|
+
* not lock-related.
|
|
48
|
+
*/
|
|
49
|
+
export async function replaceFile(tmpPath: string, destPath: string): Promise<ReplaceOutcome> {
|
|
50
|
+
let lastError: unknown;
|
|
51
|
+
for (let attempt = 0; attempt < MAX_RENAME_ATTEMPTS; attempt++) {
|
|
52
|
+
try {
|
|
53
|
+
await fs.rename(tmpPath, destPath);
|
|
54
|
+
return 'renamed';
|
|
55
|
+
} catch (err) {
|
|
56
|
+
lastError = err;
|
|
57
|
+
// Only a Windows sharing conflict is worth another try; anything else (and any
|
|
58
|
+
// failure on a POSIX platform, where rename-over-existing always works) is real.
|
|
59
|
+
if (process.platform !== 'win32' || !isRetryable(err)) throw err;
|
|
60
|
+
if (attempt < MAX_RENAME_ATTEMPTS - 1) {
|
|
61
|
+
await new Promise((r) => setTimeout(r, RENAME_RETRY_DELAY_MS * (attempt + 1)));
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// Every retry lost to the same lock. Copying is not atomic — a reader can catch the
|
|
67
|
+
// destination mid-write — but it is the only remaining way to persist the data.
|
|
68
|
+
try {
|
|
69
|
+
await fs.copyFile(tmpPath, destPath);
|
|
70
|
+
return 'copied';
|
|
71
|
+
} catch {
|
|
72
|
+
throw lastError;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Synchronous {@link replaceFile}, for the config writer, which is sync throughout. */
|
|
77
|
+
export function replaceFileSync(tmpPath: string, destPath: string): ReplaceOutcome {
|
|
78
|
+
let lastError: unknown;
|
|
79
|
+
for (let attempt = 0; attempt < MAX_RENAME_ATTEMPTS; attempt++) {
|
|
80
|
+
try {
|
|
81
|
+
fsSync.renameSync(tmpPath, destPath);
|
|
82
|
+
return 'renamed';
|
|
83
|
+
} catch (err) {
|
|
84
|
+
lastError = err;
|
|
85
|
+
if (process.platform !== 'win32' || !isRetryable(err)) throw err;
|
|
86
|
+
if (attempt < MAX_RENAME_ATTEMPTS - 1) {
|
|
87
|
+
// Sync busy-wait: this path is rare, bounded (~250ms worst case) and the
|
|
88
|
+
// caller is already blocking on a synchronous write.
|
|
89
|
+
const until = Date.now() + RENAME_RETRY_DELAY_MS * (attempt + 1);
|
|
90
|
+
while (Date.now() < until) { /* spin */ }
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
try {
|
|
96
|
+
fsSync.copyFileSync(tmpPath, destPath);
|
|
97
|
+
return 'copied';
|
|
98
|
+
} catch {
|
|
99
|
+
throw lastError;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
@@ -16,7 +16,9 @@ export interface GlobalCitation extends Citation {
|
|
|
16
16
|
* @param reports - Map of researcher ID to report content
|
|
17
17
|
* @returns Object containing normalized reports and the global citation list
|
|
18
18
|
*/
|
|
19
|
-
|
|
19
|
+
// ReadonlyMap: this only reads `reports` (it returns a NEW normalizedReports map),
|
|
20
|
+
// and callers now pass a read-only view to avoid create-on-read side effects.
|
|
21
|
+
export function normalizeCitations(reports: ReadonlyMap<string, string>): {
|
|
20
22
|
normalizedReports: Map<string, string>;
|
|
21
23
|
globalCitations: GlobalCitation[];
|
|
22
24
|
} {
|