@lincoln504/pi-research 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.mjs +1334 -750
- package/dist/thread-worker.mjs +194 -136
- package/docs/ARCHITECTURE.md +1 -1
- package/docs/CHANGELOG.md +114 -16
- package/package.json +1 -1
- package/scripts/cleanup.cjs +64 -2
- package/skills/pi-research/SKILL.md +1 -1
- package/src/cleanup/research-cleanup.ts +12 -4
- package/src/cli.ts +228 -70
- package/src/config.ts +78 -2
- package/src/core/interfaces/health-check-interfaces.ts +3 -39
- package/src/core/interfaces/planning-interfaces.ts +8 -0
- package/src/core/interfaces/service-names.ts +0 -1
- package/src/core/interfaces/state-manager-interfaces.ts +10 -2
- package/src/core/llm/agentic-repair.ts +14 -0
- package/src/core/planning-service.ts +43 -1
- package/src/core/service-initialization.ts +6 -14
- package/src/core/service-interfaces.ts +1 -1
- package/src/healthcheck/index.ts +7 -2
- package/src/infrastructure/browser/browser-client.ts +64 -9
- package/src/infrastructure/browser/browser-server.ts +24 -7
- package/src/infrastructure/browser/browser-task-scheduler.ts +58 -12
- package/src/infrastructure/browser/cleanup-utils.ts +50 -2
- package/src/infrastructure/browser/client-agent.ts +6 -6
- package/src/infrastructure/browser/config.ts +14 -3
- package/src/infrastructure/browser/scheduler-factory.ts +18 -3
- package/src/infrastructure/browser/thread-worker-messaging.ts +54 -2
- package/src/infrastructure/browser/thread-worker-types.ts +3 -1
- package/src/infrastructure/browser/thread-worker.mjs +194 -136
- package/src/infrastructure/browser/worker-pool-manager.ts +7 -2
- package/src/infrastructure/embedding/embedding-factory.ts +35 -6
- package/src/infrastructure/embedding/embedding-server.ts +40 -7
- package/src/infrastructure/file-lock-service.ts +72 -2
- package/src/infrastructure/knowledge-store-service.ts +48 -3
- package/src/infrastructure/process-lifecycle-service.ts +28 -5
- package/src/infrastructure/research-run-semaphore.ts +37 -3
- package/src/infrastructure/state/state-browser-api.ts +5 -2
- package/src/infrastructure/state/state-browser-manager.ts +15 -1
- package/src/infrastructure/state/state-manager-service.ts +4 -4
- package/src/infrastructure/state/state-manager.ts +15 -3
- package/src/knowledge/chunker.ts +88 -1
- package/src/knowledge/embedder.ts +136 -43
- package/src/knowledge/store.ts +66 -12
- package/src/logger.ts +5 -1
- package/src/observers/research-observer-impl.ts +31 -0
- package/src/orchestration/abort-utils.ts +60 -0
- package/src/orchestration/deep-research-orchestrator.ts +12 -2
- package/src/orchestration/headless-observer.ts +8 -0
- package/src/orchestration/research-orchestration-service.ts +63 -11
- package/src/orchestration/research-session-service.ts +37 -11
- package/src/orchestration/researcher-executor.ts +100 -20
- package/src/orchestration/researcher.ts +39 -20
- package/src/orchestration/session-state.ts +11 -0
- package/src/sdk.ts +125 -19
- package/src/security/index.ts +51 -30
- package/src/security/nvd-client.ts +7 -2
- package/src/skill-install/skill-installer.ts +18 -2
- package/src/stackexchange/index.ts +20 -5
- package/src/stackexchange/output/table.ts +7 -2
- package/src/stackexchange/rest-client.ts +35 -1
- package/src/tools/research-tool-definition.ts +8 -2
- package/src/tools/scrape.ts +48 -25
- package/src/tools/search.ts +10 -1
- package/src/tools/security.ts +68 -7
- package/src/tui/research-health.ts +18 -2
- package/src/utils/circuit-breaker.ts +9 -2
- package/src/utils/disk-space-checker.ts +11 -6
- package/src/utils/json-utils.ts +116 -76
- package/src/utils/llm-usage.ts +17 -1
- package/src/utils/log-rotation.ts +42 -10
- package/src/utils/log-utils.ts +40 -0
- package/src/utils/metrics-summary.ts +27 -9
- package/src/utils/stdio-capture.ts +63 -22
- package/src/utils/text-utils.ts +8 -1
- package/src/utils/tool-usage-tracker.ts +9 -2
- package/src/web-research/browser-search.ts +26 -7
- package/src/web-research/scraper-utils.ts +13 -0
- package/src/web-research/search.ts +6 -2
- package/src/web-research/types.ts +4 -2
- package/src/web-research/web-scraper.ts +65 -4
- package/src/core/health-check-service.ts +0 -265
package/src/sdk.ts
CHANGED
|
@@ -28,6 +28,9 @@ import type {
|
|
|
28
28
|
ResearchOptions
|
|
29
29
|
} from './core/interfaces/orchestration-interfaces.ts';
|
|
30
30
|
import type { IPlanningService } from './core/interfaces/planning-interfaces.ts';
|
|
31
|
+
import type { ResearchObserver, HeadlessObserverOptions } from './core/interfaces/observer-interfaces.ts';
|
|
32
|
+
import type { ResearchPlan } from './core/interfaces/research-plan-types.ts';
|
|
33
|
+
import { HeadlessObserver } from './orchestration/headless-observer.ts';
|
|
31
34
|
import type { Model } from '@earendil-works/pi-ai';
|
|
32
35
|
import { type ExtensionContext, ModelRegistry } from '@earendil-works/pi-coding-agent';
|
|
33
36
|
import { logger, createLogger, setLogger } from './logger.ts';
|
|
@@ -94,6 +97,13 @@ export interface ResearcherOutcome {
|
|
|
94
97
|
failureReasons: Record<string, string>;
|
|
95
98
|
}
|
|
96
99
|
let _lastResearcherOutcome: ResearcherOutcome | null = null;
|
|
100
|
+
// Monotonic SDK-lifetime counter, bumped by _doShutdown. A run left in flight across
|
|
101
|
+
// shutdownResearchSDK() settles AFTER the shutdown wiped the `_last*` accessors and
|
|
102
|
+
// cleared session metrics; its catch/finally would then repopulate them, so a
|
|
103
|
+
// re-initialized SDK reported a phantom run from the previous lifetime. Each run
|
|
104
|
+
// captures the generation at start and every post-run telemetry write is gated on it
|
|
105
|
+
// still matching.
|
|
106
|
+
let _sdkGeneration = 0;
|
|
97
107
|
|
|
98
108
|
// Signal handler state — registered once, removed on clean shutdown.
|
|
99
109
|
let _onSignal: ((signal: string) => void) | null = null;
|
|
@@ -489,6 +499,18 @@ export async function runDeepResearch(
|
|
|
489
499
|
// here instead of vanishing into the never-read global tracker.
|
|
490
500
|
const runTracker = new ErrorTracker();
|
|
491
501
|
|
|
502
|
+
// Stamp the run with the current shutdown generation: if shutdownResearchSDK()
|
|
503
|
+
// completes while this run is still in flight, _doShutdown bumps the generation
|
|
504
|
+
// and every telemetry write below is skipped — the shutdown wipe must win over
|
|
505
|
+
// an older-generation run settling late (see _sdkGeneration).
|
|
506
|
+
const runGeneration = _sdkGeneration;
|
|
507
|
+
|
|
508
|
+
// Planned-researcher count, captured DURING the run via the observer wrapper —
|
|
509
|
+
// it is unreadable from the planning service once runResearch() settles (the deep
|
|
510
|
+
// orchestrator clears that per-run state in its own finally first).
|
|
511
|
+
let plannedResearchers = 0;
|
|
512
|
+
const observer = wrapObserverForOutcomeCapture(options.observer, (count) => { plannedResearchers += count; });
|
|
513
|
+
|
|
492
514
|
try {
|
|
493
515
|
// Resolve the orchestrator INSIDE the try: getService() can throw (container
|
|
494
516
|
// disposal race with shutdown, or an unregistered service), and if that throw
|
|
@@ -506,6 +528,9 @@ export async function runDeepResearch(
|
|
|
506
528
|
depth: depth as ResearchDepth,
|
|
507
529
|
complexity: complexity as 1 | 2 | 3,
|
|
508
530
|
initialLinks: options.initialLinks,
|
|
531
|
+
// The caller's observer, wrapped so the SDK captures the planned-researcher
|
|
532
|
+
// events for getLastResearcherOutcome() (must land AFTER the options spread).
|
|
533
|
+
observer,
|
|
509
534
|
// Forward the SDK's resolved config so initResearchSDK({ config }) overrides
|
|
510
535
|
// actually reach the orchestrator and every downstream service. A per-call
|
|
511
536
|
// options.config (if provided) takes precedence over the SDK-global config.
|
|
@@ -522,27 +547,40 @@ export async function runDeepResearch(
|
|
|
522
547
|
// unbounded across calls (a long-lived SDK/benchmark process). runRegistry.observe() lands
|
|
523
548
|
// it in this run's snapshot, matching the tool path's run-scoped emission.
|
|
524
549
|
runRegistry.observe('research_manager_latency_ms', completedAt - researchStart, { depth: depthLabel, status: 'success', source: 'sdk' });
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
550
|
+
// Generation guard: skip every module-level telemetry write if a shutdown
|
|
551
|
+
// completed mid-run — the wipe must not be resurrected by a stale run.
|
|
552
|
+
if (runGeneration === _sdkGeneration) {
|
|
553
|
+
_lastRunSummary = {
|
|
554
|
+
runId: researchId, startedAt: researchStart, completedAt,
|
|
555
|
+
durationMs: completedAt - researchStart, status: 'success', snapshot: runRegistry.getSnapshot(),
|
|
556
|
+
};
|
|
557
|
+
metrics.recordRunSummary(_lastRunSummary);
|
|
558
|
+
_lastErrorReport = runTracker.getReport();
|
|
559
|
+
logRunErrorSummary(_lastErrorReport, depthLabel, 'success');
|
|
560
|
+
// Re-check the generation AFTER the await: the gate above was evaluated
|
|
561
|
+
// before it, so a shutdown completing during the capture would otherwise
|
|
562
|
+
// land this assignment after the wipe — a phantom outcome from a dead run.
|
|
563
|
+
const outcome = await captureResearcherOutcome(sessionId, researchId, _lastRunSummary.snapshot, plannedResearchers);
|
|
564
|
+
if (runGeneration === _sdkGeneration) _lastResearcherOutcome = outcome;
|
|
565
|
+
}
|
|
533
566
|
return result;
|
|
534
567
|
} catch (err) {
|
|
535
568
|
const completedAt = Date.now();
|
|
536
569
|
// Per-run scope (see the success path above) — avoids unbounded session-scope accumulation.
|
|
537
570
|
runRegistry.observe('research_manager_latency_ms', completedAt - researchStart, { depth: depthLabel, status: 'error', source: 'sdk' });
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
571
|
+
// Generation guard — see the success path above.
|
|
572
|
+
if (runGeneration === _sdkGeneration) {
|
|
573
|
+
_lastRunSummary = {
|
|
574
|
+
runId: researchId, startedAt: researchStart, completedAt,
|
|
575
|
+
durationMs: completedAt - researchStart, status: 'error', snapshot: runRegistry.getSnapshot(),
|
|
576
|
+
};
|
|
577
|
+
metrics.recordRunSummary(_lastRunSummary);
|
|
578
|
+
_lastErrorReport = runTracker.getReport();
|
|
579
|
+
logRunErrorSummary(_lastErrorReport, depthLabel, 'error');
|
|
580
|
+
// Re-check after the await — see the success path above.
|
|
581
|
+
const outcome = await captureResearcherOutcome(sessionId, researchId, _lastRunSummary.snapshot, plannedResearchers);
|
|
582
|
+
if (runGeneration === _sdkGeneration) _lastResearcherOutcome = outcome;
|
|
583
|
+
}
|
|
546
584
|
throw err;
|
|
547
585
|
} finally {
|
|
548
586
|
// Free the per-run PiSessionState that shouldStopResearch()/getFailedResearchers()
|
|
@@ -603,16 +641,73 @@ function logRunErrorSummary(report: ErrorReport | null, depthLabel: string, stat
|
|
|
603
641
|
logger.warn(`[SDK] run ${status} with ${report.totalErrors} tracked error(s) across ${report.uniquePatterns} pattern(s) [${depthLabel}]${top ? ` — top: ${top}` : ''}`);
|
|
604
642
|
}
|
|
605
643
|
|
|
644
|
+
/**
|
|
645
|
+
* Wrap the caller's observer so the SDK sees the planned-researcher events the deep
|
|
646
|
+
* orchestrator emits DURING the run. The count cannot be read back afterwards: the
|
|
647
|
+
* orchestrator's own finally runs cleanupResearchServices → clearPlanningState(researchId)
|
|
648
|
+
* BEFORE runResearch() resolves for the SDK, so a post-run getTotalResearchersPlanned()
|
|
649
|
+
* always sees 0. onPlanningSuccess (round 1) and onEvaluationDecision('delegate', …)
|
|
650
|
+
* (later rounds) fire at exactly the points the orchestrator increments that counter —
|
|
651
|
+
* it suppresses the delegate decision for a round-cap delegation it discards, so the
|
|
652
|
+
* captured total only counts researchers that were actually going to run.
|
|
653
|
+
*
|
|
654
|
+
* An { onProgress } options bag is normalized into a HeadlessObserver HERE rather than
|
|
655
|
+
* left to the orchestrator (which uses the same shape check): if the orchestrator wrapped
|
|
656
|
+
* the bag itself, the planning events would dispatch on ITS HeadlessObserver instance and
|
|
657
|
+
* never reach this capture layer. Mirrors makeSafeObserver's empty-target Proxy so a
|
|
658
|
+
* frozen caller observer can never trip the Proxy own-property invariant.
|
|
659
|
+
*/
|
|
660
|
+
function wrapObserverForOutcomeCapture(
|
|
661
|
+
observer: ResearchObserver | HeadlessObserverOptions | undefined,
|
|
662
|
+
onPlanned: (count: number) => void,
|
|
663
|
+
): ResearchObserver {
|
|
664
|
+
const base: ResearchObserver | undefined =
|
|
665
|
+
observer && typeof (observer as HeadlessObserverOptions).onProgress === 'function' && !(observer instanceof HeadlessObserver)
|
|
666
|
+
? new HeadlessObserver(observer as HeadlessObserverOptions)
|
|
667
|
+
: (observer as ResearchObserver | undefined);
|
|
668
|
+
|
|
669
|
+
return new Proxy({} as ResearchObserver, {
|
|
670
|
+
get(_target, prop) {
|
|
671
|
+
if (prop === 'onPlanningSuccess') {
|
|
672
|
+
return (plan: ResearchPlan) => {
|
|
673
|
+
onPlanned(plan?.researchers?.length ?? 0);
|
|
674
|
+
base?.onPlanningSuccess?.(plan);
|
|
675
|
+
};
|
|
676
|
+
}
|
|
677
|
+
if (prop === 'onEvaluationDecision') {
|
|
678
|
+
return (action: 'synthesize' | 'delegate', plan?: ResearchPlan, round?: number) => {
|
|
679
|
+
if (action === 'delegate') onPlanned(plan?.researchers?.length ?? 0);
|
|
680
|
+
base?.onEvaluationDecision?.(action, plan, round);
|
|
681
|
+
};
|
|
682
|
+
}
|
|
683
|
+
if (!base) return undefined;
|
|
684
|
+
const value = Reflect.get(base, prop, base);
|
|
685
|
+
return typeof value === 'function' ? (value as (...args: unknown[]) => unknown).bind(base) : value;
|
|
686
|
+
},
|
|
687
|
+
has(_target, prop) {
|
|
688
|
+
if (prop === 'onPlanningSuccess' || prop === 'onEvaluationDecision') return true;
|
|
689
|
+
return base ? Reflect.has(base, prop) : false;
|
|
690
|
+
},
|
|
691
|
+
});
|
|
692
|
+
}
|
|
693
|
+
|
|
606
694
|
/**
|
|
607
695
|
* Capture how many researchers were planned/launched/failed for this run, BEFORE
|
|
608
696
|
* the caller's `finally` block frees the PiSessionState (endResearchSession) that
|
|
609
697
|
* getFailedResearchers()/getResearcherFailureReasons() read from. Best-effort:
|
|
610
698
|
* this is instrumentation only and must never affect the run's own result.
|
|
611
699
|
*/
|
|
612
|
-
async function captureResearcherOutcome(sessionId: string, researchId: string, snapshot: IMetricsSnapshot): Promise<ResearcherOutcome | null> {
|
|
700
|
+
async function captureResearcherOutcome(sessionId: string, researchId: string, snapshot: IMetricsSnapshot, plannedFromObserver: number): Promise<ResearcherOutcome | null> {
|
|
613
701
|
try {
|
|
614
|
-
|
|
615
|
-
|
|
702
|
+
// Deep runs: the observer-captured count is authoritative — the orchestrator's
|
|
703
|
+
// finally already cleared the planning service's per-run entry by the time the SDK
|
|
704
|
+
// regains control (see wrapObserverForOutcomeCapture). The service read stays as a
|
|
705
|
+
// fallback for any path that never emits the planning events.
|
|
706
|
+
let planned = plannedFromObserver;
|
|
707
|
+
if (planned === 0) {
|
|
708
|
+
const planningService = await getService<IPlanningService>(ServiceNames.PLANNING, undefined, globalContainer!);
|
|
709
|
+
planned = planningService.getTotalResearchersPlanned(researchId);
|
|
710
|
+
}
|
|
616
711
|
const launched = extractRunStats(snapshot)?.researchersLaunched ?? 0;
|
|
617
712
|
const failed = getFailedResearchers(sessionId, researchId);
|
|
618
713
|
return {
|
|
@@ -825,6 +920,14 @@ export async function shutdownResearchSDK(): Promise<void> {
|
|
|
825
920
|
async function _doShutdown(): Promise<void> {
|
|
826
921
|
logger.log('[SDK] Shutting down Research SDK...');
|
|
827
922
|
|
|
923
|
+
// Invalidate the pending telemetry writes of any run still in flight FIRST —
|
|
924
|
+
// before runCleanup and the other long awaits below. The invariant: a run
|
|
925
|
+
// settling at ANY point after shutdown begins fails the generation gate. Bumping
|
|
926
|
+
// at the end instead left the whole teardown (cleanup, clearSession, service
|
|
927
|
+
// disposal) as a window in which a settling run passed the gate and wrote a
|
|
928
|
+
// phantom run-summary into the just-cleared session metrics.
|
|
929
|
+
_sdkGeneration++;
|
|
930
|
+
|
|
828
931
|
const errors: Error[] = [];
|
|
829
932
|
|
|
830
933
|
try {
|
|
@@ -886,6 +989,9 @@ async function _doShutdown(): Promise<void> {
|
|
|
886
989
|
_lastRunSummary = null;
|
|
887
990
|
_lastErrorReport = null;
|
|
888
991
|
_lastResearcherOutcome = null;
|
|
992
|
+
// (No generation bump here — it happened at the TOP of this function, so any
|
|
993
|
+
// run settling during the teardown above already failed the gate and could not
|
|
994
|
+
// repopulate the `_last*` accessors or session metrics being wiped.)
|
|
889
995
|
// The module-level config cache survives the globals above. Without this, a
|
|
890
996
|
// shutdown → env/config-file change → re-init cycle in one process silently
|
|
891
997
|
// reuses the first init's config. (The CLI/skill are unaffected — they pass
|
package/src/security/index.ts
CHANGED
|
@@ -22,12 +22,11 @@ import type {
|
|
|
22
22
|
GitHubResult,
|
|
23
23
|
OSVResult,
|
|
24
24
|
} from './types.ts';
|
|
25
|
-
import { REQUEST_DELAY_MS_NVD, REQUEST_DELAY_MS_OTHER } from '../constants.ts';
|
|
26
25
|
import { searchNVD } from './nvd-client.ts';
|
|
27
26
|
import { searchCisaKev } from './cisa-kev.ts';
|
|
28
27
|
import { searchGitHubAdvisories } from './github-advisories.ts';
|
|
29
28
|
import { searchOSV } from './osv-client.ts';
|
|
30
|
-
import {
|
|
29
|
+
import { logger } from '../logger.ts';
|
|
31
30
|
|
|
32
31
|
// ============================================================================
|
|
33
32
|
// Type Definitions
|
|
@@ -35,6 +34,9 @@ import { safeUnref } from '../utils/safe-unref.ts';
|
|
|
35
34
|
|
|
36
35
|
type Severity = 'LOW' | 'MEDIUM' | 'HIGH' | 'CRITICAL';
|
|
37
36
|
|
|
37
|
+
type KnownDatabase = 'nvd' | 'cisa_kev' | 'github' | 'osv';
|
|
38
|
+
const KNOWN_DATABASES: readonly KnownDatabase[] = ['nvd', 'cisa_kev', 'github', 'osv'];
|
|
39
|
+
|
|
38
40
|
interface DatabaseInfo {
|
|
39
41
|
readonly databases: readonly DatabaseInfoEntry[];
|
|
40
42
|
}
|
|
@@ -58,9 +60,22 @@ function isValidSeverity(value: unknown): value is Severity {
|
|
|
58
60
|
);
|
|
59
61
|
}
|
|
60
62
|
|
|
63
|
+
// Called ONCE per search(), before dispatch, and the result is shared by every
|
|
64
|
+
// requested backend (NVD, GitHub, OSV) — see the call site in search(). Do not
|
|
65
|
+
// call this per-leg: that previously let NVD normalize while GitHub/OSV read
|
|
66
|
+
// params.severity raw, so the same unrecognized value ran NVD unfiltered while
|
|
67
|
+
// silently zeroing GitHub/OSV's strict filters.
|
|
61
68
|
function getSeverityParam(params: SecuritySearchParams): Severity | undefined {
|
|
62
69
|
if (params.severity === undefined) return undefined;
|
|
63
|
-
|
|
70
|
+
// NVD requires the exact uppercase enum. GitHub/OSV normalize case and the
|
|
71
|
+
// MODERATE↔MEDIUM synonym in their own clients (github-advisories.ts,
|
|
72
|
+
// osv-client.ts) as well, so applying the same normalization here keeps all
|
|
73
|
+
// three backends in agreement on what severity:"high" or "Moderate" means.
|
|
74
|
+
const upper = params.severity.trim().toUpperCase();
|
|
75
|
+
const normalized = upper === 'MODERATE' ? 'MEDIUM' : upper;
|
|
76
|
+
if (isValidSeverity(normalized)) return normalized;
|
|
77
|
+
logger.warn(`[security] Unknown severity filter "${params.severity}" (valid: LOW, MEDIUM, HIGH, CRITICAL) — no database will be severity-filtered`);
|
|
78
|
+
return undefined;
|
|
64
79
|
}
|
|
65
80
|
|
|
66
81
|
function getErrorMessage(error: unknown): string {
|
|
@@ -77,7 +92,6 @@ export interface SecuritySearcherConfig {
|
|
|
77
92
|
readonly cisaKevClient?: ICisaKevClient;
|
|
78
93
|
readonly githubAdvisoriesClient?: IGitHubAdvisoriesClient;
|
|
79
94
|
readonly osvClient?: IOSVClient;
|
|
80
|
-
readonly requestDelay?: number; // Override for testing
|
|
81
95
|
}
|
|
82
96
|
|
|
83
97
|
const DEFAULT_CONFIG: SecuritySearcherConfig = {
|
|
@@ -85,7 +99,6 @@ const DEFAULT_CONFIG: SecuritySearcherConfig = {
|
|
|
85
99
|
cisaKevClient: undefined,
|
|
86
100
|
githubAdvisoriesClient: undefined,
|
|
87
101
|
osvClient: undefined,
|
|
88
|
-
requestDelay: -1, // Use database-specific logic by default
|
|
89
102
|
};
|
|
90
103
|
|
|
91
104
|
export class SecuritySearcher {
|
|
@@ -101,13 +114,35 @@ export class SecuritySearcher {
|
|
|
101
114
|
const errors: string[] = [];
|
|
102
115
|
let totalVulnerabilities = 0;
|
|
103
116
|
|
|
117
|
+
// Normalize severity ONCE, before dispatch, so every backend that receives a
|
|
118
|
+
// severity filter sees the exact same value. Previously this call sat inline
|
|
119
|
+
// in the NVD branch only; GitHub and OSV read the raw, unvalidated
|
|
120
|
+
// params.severity directly, so an unrecognized value ran NVD unfiltered while
|
|
121
|
+
// silently zeroing GitHub/OSV (or vice versa depending on each client's own
|
|
122
|
+
// parsing) — the single-chokepoint fix has to normalize before the fan-out,
|
|
123
|
+
// not duplicate the call at each leg.
|
|
124
|
+
const severity = getSeverityParam(params);
|
|
125
|
+
|
|
126
|
+
// Normalize database names before dispatch: callers (agents, SDK users) write
|
|
127
|
+
// "NVD" / " osv " etc. An unrecognized name must surface in `errors` rather than
|
|
128
|
+
// silently contributing an authoritative-looking "0 vulnerabilities" for a
|
|
129
|
+
// database that was never queried.
|
|
130
|
+
const databases: string[] = [];
|
|
131
|
+
for (const raw of params.databases) {
|
|
132
|
+
const db = String(raw).trim().toLowerCase();
|
|
133
|
+
databases.push(db);
|
|
134
|
+
if (!(KNOWN_DATABASES as readonly string[]).includes(db)) {
|
|
135
|
+
errors.push(`Unknown database "${raw}" — valid databases: ${KNOWN_DATABASES.join(', ')}`);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
104
139
|
const searchPromises: Promise<void>[] = [];
|
|
105
140
|
|
|
106
|
-
if (
|
|
141
|
+
if (databases.includes('nvd')) {
|
|
107
142
|
searchPromises.push((async () => {
|
|
108
143
|
try {
|
|
109
144
|
const nvdResult = await this.searchNVD(params.terms, {
|
|
110
|
-
severity
|
|
145
|
+
severity,
|
|
111
146
|
maxResults: params.maxResults,
|
|
112
147
|
includeExploited: params.includeExploited,
|
|
113
148
|
signal,
|
|
@@ -120,7 +155,7 @@ export class SecuritySearcher {
|
|
|
120
155
|
})());
|
|
121
156
|
}
|
|
122
157
|
|
|
123
|
-
if (
|
|
158
|
+
if (databases.includes('cisa_kev')) {
|
|
124
159
|
searchPromises.push((async () => {
|
|
125
160
|
try {
|
|
126
161
|
const cisaResult = await this.searchCisaKev(params.terms, { maxResults: params.maxResults, signal });
|
|
@@ -132,12 +167,12 @@ export class SecuritySearcher {
|
|
|
132
167
|
})());
|
|
133
168
|
}
|
|
134
169
|
|
|
135
|
-
if (
|
|
170
|
+
if (databases.includes('github')) {
|
|
136
171
|
searchPromises.push((async () => {
|
|
137
172
|
try {
|
|
138
173
|
const githubResult = await this.searchGitHub(params.terms, {
|
|
139
174
|
ecosystem: params.ecosystem,
|
|
140
|
-
severity
|
|
175
|
+
severity,
|
|
141
176
|
maxResults: params.maxResults,
|
|
142
177
|
repo: params.githubRepo,
|
|
143
178
|
signal,
|
|
@@ -150,12 +185,12 @@ export class SecuritySearcher {
|
|
|
150
185
|
})());
|
|
151
186
|
}
|
|
152
187
|
|
|
153
|
-
if (
|
|
188
|
+
if (databases.includes('osv')) {
|
|
154
189
|
searchPromises.push((async () => {
|
|
155
190
|
try {
|
|
156
191
|
const osvResult = await this.searchOSV(params.terms, {
|
|
157
192
|
ecosystem: params.ecosystem,
|
|
158
|
-
severity
|
|
193
|
+
severity,
|
|
159
194
|
maxResults: params.maxResults,
|
|
160
195
|
signal,
|
|
161
196
|
});
|
|
@@ -169,24 +204,10 @@ export class SecuritySearcher {
|
|
|
169
204
|
|
|
170
205
|
await Promise.all(searchPromises);
|
|
171
206
|
|
|
172
|
-
//
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
if (delay > 0) {
|
|
179
|
-
// Abortable: a caller cancel must not sit through the post-sweep delay.
|
|
180
|
-
await new Promise<void>((resolve) => {
|
|
181
|
-
const timeoutId = setTimeout(() => {
|
|
182
|
-
signal?.removeEventListener('abort', onAbort);
|
|
183
|
-
resolve();
|
|
184
|
-
}, delay);
|
|
185
|
-
safeUnref(timeoutId);
|
|
186
|
-
const onAbort = () => { clearTimeout(timeoutId); resolve(); };
|
|
187
|
-
if (signal) signal.addEventListener('abort', onAbort, { once: true });
|
|
188
|
-
});
|
|
189
|
-
}
|
|
207
|
+
// No post-sweep delay: NVD inter-request spacing is enforced by the module-level
|
|
208
|
+
// NVDRateLimiter in nvd-client.ts, whose state persists across searches and tool
|
|
209
|
+
// calls, so sleeping here after results were already in hand only added dead
|
|
210
|
+
// researcher time.
|
|
190
211
|
|
|
191
212
|
return {
|
|
192
213
|
results,
|
|
@@ -353,6 +353,11 @@ function fetchWithRetry(url: string, signal?: AbortSignal): Promise<Response> {
|
|
|
353
353
|
|
|
354
354
|
return nvdCircuitBreaker.execute(async () => {
|
|
355
355
|
return retryWithBackoff(async () => {
|
|
356
|
+
// Every attempt — the first AND each 2s/4s retry — must hold a rate-limiter
|
|
357
|
+
// slot: the limiter's spacing is the ONLY mechanism enforcing NVD's ~6s
|
|
358
|
+
// inter-request interval (the post-sweep sleep was removed), and an acquire
|
|
359
|
+
// outside this closure let 429/5xx retries re-hit the API faster than it.
|
|
360
|
+
await nvdRateLimiter.acquire(signal);
|
|
356
361
|
let response: Response;
|
|
357
362
|
try {
|
|
358
363
|
response = await fetch(url, createFetchOptions(signal));
|
|
@@ -392,8 +397,8 @@ async function fetchPaginated(
|
|
|
392
397
|
if (signal?.aborted) break; // stop paginating promptly on cancel
|
|
393
398
|
const url = buildURL(term, options, pageSize, startIndex, severityVersion);
|
|
394
399
|
|
|
395
|
-
|
|
396
|
-
|
|
400
|
+
// No acquire here: fetchWithRetry acquires a rate-limiter slot per ATTEMPT,
|
|
401
|
+
// so retries are spaced too — a second acquire would double-charge the page.
|
|
397
402
|
const response = await fetchWithRetry(url, signal);
|
|
398
403
|
let data: unknown;
|
|
399
404
|
try {
|
|
@@ -217,12 +217,28 @@ export function readManifest(opts?: { home?: string }): Manifest {
|
|
|
217
217
|
return { version: 1, package: PACKAGE_NAME, entries: [] };
|
|
218
218
|
}
|
|
219
219
|
|
|
220
|
+
/** Monotonic per-process suffix for manifest tmp files (see writeManifest). */
|
|
221
|
+
let manifestTmpSeq = 0;
|
|
222
|
+
|
|
220
223
|
function writeManifest(manifest: Manifest, opts?: { home?: string }): void {
|
|
221
224
|
const p = getManifestPath(opts);
|
|
222
225
|
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
223
|
-
|
|
226
|
+
// Unique tmp name (pid + timestamp + counter, mirroring config.ts's registry
|
|
227
|
+
// writer): a FIXED `${p}.tmp` was a shared collision surface — concurrent
|
|
228
|
+
// runs' reconciles (the run cap allows 3 live runs) staged through the same
|
|
229
|
+
// path, the winner renamed it away, and the loser's renameSync threw ENOENT,
|
|
230
|
+
// surfacing as a spurious skill-install error. The read-modify-write around
|
|
231
|
+
// this stays deliberately UNLOCKED: a lost update is self-healing churn (the
|
|
232
|
+
// next reconcile re-derives the same state from disk), unlike the hard
|
|
233
|
+
// failure a tmp-name collision caused.
|
|
234
|
+
const tmp = `${p}.tmp.${process.pid}.${Date.now()}.${manifestTmpSeq++}`;
|
|
224
235
|
fs.writeFileSync(tmp, JSON.stringify(manifest, null, 2) + '\n', 'utf-8');
|
|
225
|
-
|
|
236
|
+
try {
|
|
237
|
+
fs.renameSync(tmp, p);
|
|
238
|
+
} catch (err) {
|
|
239
|
+
try { fs.unlinkSync(tmp); } catch { /* best effort */ }
|
|
240
|
+
throw err;
|
|
241
|
+
}
|
|
226
242
|
}
|
|
227
243
|
|
|
228
244
|
function upsertEntry(manifest: Manifest, entry: ManifestEntry): void {
|
|
@@ -45,6 +45,24 @@ function loadConfig(): StackExchangeConfig {
|
|
|
45
45
|
};
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
+
// The client instance carries the resilience state the SE API requires callers to
|
|
49
|
+
// honor across requests: the `backoff` directive (rest-client lastBackoff), quota
|
|
50
|
+
// tracking, and the 5-failure circuit breaker. A per-call instance resets all of
|
|
51
|
+
// that on every tool call — the exact throttle-violation pattern SE punishes with
|
|
52
|
+
// IP bans — so the client is process-wide, rebuilt only when the API key changes
|
|
53
|
+
// (the sole constructor input that can vary between calls; requestTimeout is a
|
|
54
|
+
// constant in loadConfig).
|
|
55
|
+
let sharedClient: StackExchangeClient | null = null;
|
|
56
|
+
let sharedClientApiKey: string | null = null;
|
|
57
|
+
|
|
58
|
+
function getClient(config: StackExchangeConfig): StackExchangeClient {
|
|
59
|
+
if (sharedClient === null || sharedClientApiKey !== config.apiKey) {
|
|
60
|
+
sharedClient = new StackExchangeClient(config.apiKey, config.requestTimeout);
|
|
61
|
+
sharedClientApiKey = config.apiKey;
|
|
62
|
+
}
|
|
63
|
+
return sharedClient;
|
|
64
|
+
}
|
|
65
|
+
|
|
48
66
|
/**
|
|
49
67
|
* Main command handler for Stack Exchange operations
|
|
50
68
|
*/
|
|
@@ -60,11 +78,8 @@ export async function stackexchangeCommand(options: {
|
|
|
60
78
|
// Load config
|
|
61
79
|
const config = loadConfig();
|
|
62
80
|
|
|
63
|
-
//
|
|
64
|
-
const client =
|
|
65
|
-
config.apiKey,
|
|
66
|
-
config.requestTimeout,
|
|
67
|
-
);
|
|
81
|
+
// Shared client — resilience state (backoff, quota, breaker) persists across calls
|
|
82
|
+
const client = getClient(config);
|
|
68
83
|
|
|
69
84
|
try {
|
|
70
85
|
// Check quota before making requests
|
|
@@ -56,8 +56,13 @@ export function formatAnswersTable(answers: Answer[]): string {
|
|
|
56
56
|
output += `- **Question ID:** ${a.question_id}\n`;
|
|
57
57
|
output += `- **Created:** ${new Date(a.creation_date * 1000).toLocaleString()}\n`;
|
|
58
58
|
|
|
59
|
-
if (a.body
|
|
60
|
-
|
|
59
|
+
if (a.body) {
|
|
60
|
+
// Truncate long bodies to a preview rather than dropping them entirely — the
|
|
61
|
+
// previous `length < 2000` guard omitted the body for exactly the longest
|
|
62
|
+
// answers (same class as the question-body fix above; answers get a more
|
|
63
|
+
// generous bound since the body is the whole point of an answer).
|
|
64
|
+
const body = a.body.length > 2000 ? `${a.body.substring(0, 2000)}...` : a.body;
|
|
65
|
+
output += `\n### Answer Body\n\n${body}\n`;
|
|
61
66
|
}
|
|
62
67
|
|
|
63
68
|
output += '\n---\n\n';
|
|
@@ -11,6 +11,16 @@ import { readJsonCapped, BodyTooLargeError } from '../utils/http-body.ts';
|
|
|
11
11
|
|
|
12
12
|
const API_BASE = 'https://api.stackexchange.com/2.3';
|
|
13
13
|
|
|
14
|
+
// How long quota-exhaustion evidence keeps the request gate armed. quotaRemaining
|
|
15
|
+
// is only refreshed from a successful response, and the tool-level gate
|
|
16
|
+
// (stackexchange/index.ts) blocks every request while isQuotaExhausted() — so a
|
|
17
|
+
// plain `quotaRemaining <= 0` check could never observe SE's real daily reset and
|
|
18
|
+
// wedged the process-wide client until restart or an API-key change. After the
|
|
19
|
+
// cooldown one probe request is allowed: it either refreshes the quota (clearing
|
|
20
|
+
// the stamp) or shows 0 / throttles again, re-arming the gate — at most one
|
|
21
|
+
// wasted request per cooldown.
|
|
22
|
+
const QUOTA_PROBE_COOLDOWN_MS = 10 * 60 * 1000;
|
|
23
|
+
|
|
14
24
|
export interface RequestOptions {
|
|
15
25
|
method: string;
|
|
16
26
|
endpoint: string;
|
|
@@ -22,6 +32,9 @@ export class StackExchangeClient {
|
|
|
22
32
|
private readonly _timeout: number;
|
|
23
33
|
private quotaRemaining = 300;
|
|
24
34
|
private quotaMax = 300;
|
|
35
|
+
// Timestamp of the last quota-exhaustion evidence (a response reporting
|
|
36
|
+
// quota_remaining <= 0, or a throttle_violation API error). null = no evidence.
|
|
37
|
+
private quotaExhaustedAt: number | null = null;
|
|
25
38
|
private requestCount = 0;
|
|
26
39
|
private lastBackoff: number | null = null;
|
|
27
40
|
private circuitBreaker: CircuitBreaker;
|
|
@@ -122,6 +135,12 @@ export class StackExchangeClient {
|
|
|
122
135
|
// Handle API errors
|
|
123
136
|
if (data.error_id) {
|
|
124
137
|
const errorName = data.error_name ?? 'unknown';
|
|
138
|
+
// A throttle_violation is quota-exhaustion evidence even though no
|
|
139
|
+
// quota_remaining arrives with it (this throw happens before the quota
|
|
140
|
+
// update below) — stamp it so the gate arms/re-arms.
|
|
141
|
+
if (errorName === 'throttle_violation') {
|
|
142
|
+
this.quotaExhaustedAt = Date.now();
|
|
143
|
+
}
|
|
125
144
|
metrics.increment('stackexchange_errors_total', 1, {
|
|
126
145
|
endpoint: options.endpoint,
|
|
127
146
|
error_id: data.error_id.toString(),
|
|
@@ -136,6 +155,15 @@ export class StackExchangeClient {
|
|
|
136
155
|
this.quotaRemaining = data.quota_remaining;
|
|
137
156
|
this.quotaMax = data.quota_max;
|
|
138
157
|
this.requestCount++;
|
|
158
|
+
// Stamp (or clear) exhaustion evidence: isQuotaExhausted() reads the
|
|
159
|
+
// stamp, not quotaRemaining — which can never refresh while the gate
|
|
160
|
+
// blocks requests — so a probe after the cooldown self-corrects once
|
|
161
|
+
// SE's daily reset has happened.
|
|
162
|
+
if (data.quota_remaining <= 0) {
|
|
163
|
+
this.quotaExhaustedAt = Date.now();
|
|
164
|
+
} else {
|
|
165
|
+
this.quotaExhaustedAt = null;
|
|
166
|
+
}
|
|
139
167
|
|
|
140
168
|
// Track quota metrics
|
|
141
169
|
metrics.setGauge('stackexchange_quota_remaining', this.quotaRemaining);
|
|
@@ -197,7 +225,13 @@ export class StackExchangeClient {
|
|
|
197
225
|
}
|
|
198
226
|
|
|
199
227
|
isQuotaExhausted(): boolean {
|
|
200
|
-
|
|
228
|
+
// True only within the probe cooldown of the last exhaustion evidence. After
|
|
229
|
+
// it elapses, one probe request is allowed through the tool-level gate; the
|
|
230
|
+
// probe's outcome refreshes or clears the stamp (see request()).
|
|
231
|
+
return (
|
|
232
|
+
this.quotaExhaustedAt !== null &&
|
|
233
|
+
Date.now() - this.quotaExhaustedAt < QUOTA_PROBE_COOLDOWN_MS
|
|
234
|
+
);
|
|
201
235
|
}
|
|
202
236
|
|
|
203
237
|
isQuotaLow(): boolean {
|
|
@@ -466,8 +466,14 @@ export function createResearchTool(iface?: ConfigInterface): ToolDefinition {
|
|
|
466
466
|
|
|
467
467
|
const errMsg = String(error).toLowerCase();
|
|
468
468
|
|
|
469
|
-
// Handle rate limits gracefully
|
|
470
|
-
|
|
469
|
+
// Handle rate limits gracefully. "429" is matched on a word boundary
|
|
470
|
+
// (mirroring messageIsTransient in web-research/retry-utils.ts): a substring
|
|
471
|
+
// test also matched digits embedded in a larger number — e.g. a context-
|
|
472
|
+
// overflow error quoting "you requested 142935 tokens" — misreporting the
|
|
473
|
+
// run as rate-limited. 'quota' stays unanchored, same trade-off as
|
|
474
|
+
// retry-utils: provider quota errors ("insufficient_quota") carry no
|
|
475
|
+
// rate/limit wording to anchor on.
|
|
476
|
+
if (/\b429\b/.test(errMsg) || errMsg.includes('rate limit') || errMsg.includes('too many requests') || errMsg.includes('quota')) {
|
|
471
477
|
logger.warn('[research] Run halted gracefully due to rate limit:', error);
|
|
472
478
|
if (ctx.hasUI) {
|
|
473
479
|
ctx.ui.notify('Research halted: API rate limit reached', 'warning');
|