@lincoln504/pi-research 1.2.0 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.mjs +1334 -750
- package/dist/thread-worker.mjs +194 -136
- package/docs/ARCHITECTURE.md +1 -1
- package/docs/CHANGELOG.md +129 -16
- package/package.json +1 -1
- package/scripts/cleanup.cjs +64 -2
- package/skills/pi-research/SKILL.md +1 -1
- package/src/cleanup/research-cleanup.ts +12 -4
- package/src/cli.ts +228 -70
- package/src/config.ts +78 -2
- package/src/core/interfaces/health-check-interfaces.ts +3 -39
- package/src/core/interfaces/planning-interfaces.ts +8 -0
- package/src/core/interfaces/service-names.ts +0 -1
- package/src/core/interfaces/state-manager-interfaces.ts +10 -2
- package/src/core/llm/agentic-repair.ts +14 -0
- package/src/core/planning-service.ts +43 -1
- package/src/core/service-initialization.ts +6 -14
- package/src/core/service-interfaces.ts +1 -1
- package/src/healthcheck/index.ts +7 -2
- package/src/infrastructure/browser/browser-client.ts +64 -9
- package/src/infrastructure/browser/browser-server.ts +24 -7
- package/src/infrastructure/browser/browser-task-scheduler.ts +58 -12
- package/src/infrastructure/browser/cleanup-utils.ts +50 -2
- package/src/infrastructure/browser/client-agent.ts +6 -6
- package/src/infrastructure/browser/config.ts +14 -3
- package/src/infrastructure/browser/scheduler-factory.ts +18 -3
- package/src/infrastructure/browser/thread-worker-messaging.ts +54 -2
- package/src/infrastructure/browser/thread-worker-types.ts +3 -1
- package/src/infrastructure/browser/thread-worker.mjs +194 -136
- package/src/infrastructure/browser/worker-pool-manager.ts +7 -2
- package/src/infrastructure/embedding/embedding-factory.ts +35 -6
- package/src/infrastructure/embedding/embedding-server.ts +40 -7
- package/src/infrastructure/file-lock-service.ts +72 -2
- package/src/infrastructure/knowledge-store-service.ts +48 -3
- package/src/infrastructure/process-lifecycle-service.ts +28 -5
- package/src/infrastructure/research-run-semaphore.ts +37 -3
- package/src/infrastructure/state/state-browser-api.ts +5 -2
- package/src/infrastructure/state/state-browser-manager.ts +15 -1
- package/src/infrastructure/state/state-manager-service.ts +4 -4
- package/src/infrastructure/state/state-manager.ts +15 -3
- package/src/knowledge/chunker.ts +88 -1
- package/src/knowledge/embedder.ts +136 -43
- package/src/knowledge/store.ts +66 -12
- package/src/logger.ts +5 -1
- package/src/observers/research-observer-impl.ts +31 -0
- package/src/orchestration/abort-utils.ts +60 -0
- package/src/orchestration/deep-research-orchestrator.ts +12 -2
- package/src/orchestration/headless-observer.ts +8 -0
- package/src/orchestration/research-orchestration-service.ts +63 -11
- package/src/orchestration/research-session-service.ts +37 -11
- package/src/orchestration/researcher-executor.ts +100 -20
- package/src/orchestration/researcher.ts +39 -20
- package/src/orchestration/session-state.ts +11 -0
- package/src/sdk.ts +125 -19
- package/src/security/index.ts +51 -30
- package/src/security/nvd-client.ts +7 -2
- package/src/skill-install/skill-installer.ts +18 -2
- package/src/stackexchange/index.ts +20 -5
- package/src/stackexchange/output/table.ts +7 -2
- package/src/stackexchange/rest-client.ts +35 -1
- package/src/tools/research-tool-definition.ts +42 -8
- package/src/tools/scrape.ts +48 -25
- package/src/tools/search.ts +10 -1
- package/src/tools/security.ts +68 -7
- package/src/tui/research-health.ts +18 -2
- package/src/utils/circuit-breaker.ts +9 -2
- package/src/utils/disk-space-checker.ts +11 -6
- package/src/utils/json-utils.ts +116 -76
- package/src/utils/llm-usage.ts +17 -1
- package/src/utils/log-rotation.ts +42 -10
- package/src/utils/log-utils.ts +40 -0
- package/src/utils/metrics-summary.ts +27 -9
- package/src/utils/stdio-capture.ts +63 -22
- package/src/utils/text-utils.ts +8 -1
- package/src/utils/tool-usage-tracker.ts +9 -2
- package/src/web-research/browser-search.ts +26 -7
- package/src/web-research/scraper-utils.ts +13 -0
- package/src/web-research/search.ts +6 -2
- package/src/web-research/types.ts +4 -2
- package/src/web-research/web-scraper.ts +65 -4
- package/src/core/health-check-service.ts +0 -265
|
@@ -7,7 +7,6 @@
|
|
|
7
7
|
import type { ExtensionContext, AgentToolResult } from '@earendil-works/pi-coding-agent';
|
|
8
8
|
import { type Model } from '@earendil-works/pi-ai';
|
|
9
9
|
import { logger } from '../logger.ts';
|
|
10
|
-
import { safeUnref } from '../utils/safe-unref.ts';
|
|
11
10
|
import { metrics } from '../utils/metrics.ts';
|
|
12
11
|
import { getSteeringMessages, consumeQueuedMessages, getActiveSteeringMessages, getFailedResearchers, getResearcherFailureReasons } from './session-state.ts';
|
|
13
12
|
import { MAX_EXTRA_ROUNDS_WITH_STEERING, resolveExcludedTools } from '../constants.ts';
|
|
@@ -245,6 +244,11 @@ export class DeepResearchOrchestrator {
|
|
|
245
244
|
reports: synthesisService.getAllReports(researchId),
|
|
246
245
|
previousPlan: planningService.getCurrentPlan(researchId),
|
|
247
246
|
totalResearchersPlanned: planningService.getTotalResearchersPlanned(researchId),
|
|
247
|
+
// Live, steering-extended budget — see maxRounds comment above. Without
|
|
248
|
+
// this, updatePlanForRound recomputes the BASE complexity-table value
|
|
249
|
+
// internally and understates the denominator once steering has extended
|
|
250
|
+
// the round budget, prematurely skewing its round-phase guidance to LATE.
|
|
251
|
+
maxRounds,
|
|
248
252
|
signal,
|
|
249
253
|
observer,
|
|
250
254
|
excludeTools: this.options.excludeTools,
|
|
@@ -294,11 +298,15 @@ export class DeepResearchOrchestrator {
|
|
|
294
298
|
clearTimeout(timeout);
|
|
295
299
|
reject(new Error('Research cancelled'));
|
|
296
300
|
};
|
|
301
|
+
// Deliberately REFERENCED: this sleep IS the run's sole pending
|
|
302
|
+
// operation at this moment (nothing else is in flight during a
|
|
303
|
+
// 'wait'), so an unref'd timer would let the process drain its
|
|
304
|
+
// event loop and exit 0 silently mid-run. Abortability, not
|
|
305
|
+
// unref, is what keeps cancellation responsive here.
|
|
297
306
|
const timeout = setTimeout(() => {
|
|
298
307
|
signal?.removeEventListener('abort', onAbort);
|
|
299
308
|
resolve();
|
|
300
309
|
}, 5000);
|
|
301
|
-
safeUnref(timeout);
|
|
302
310
|
signal?.addEventListener('abort', onAbort, { once: true });
|
|
303
311
|
});
|
|
304
312
|
|
|
@@ -474,6 +482,8 @@ export class DeepResearchOrchestrator {
|
|
|
474
482
|
previousPlan: planningService.getCurrentPlan(researchId),
|
|
475
483
|
totalResearchersPlanned: planningService.getTotalResearchersPlanned(researchId),
|
|
476
484
|
mustSynthesize: true,
|
|
485
|
+
// Live, steering-extended budget — see the in-loop call above.
|
|
486
|
+
maxRounds,
|
|
477
487
|
signal,
|
|
478
488
|
observer,
|
|
479
489
|
excludeTools: this.options.excludeTools,
|
|
@@ -139,6 +139,10 @@ export class HeadlessObserver implements ResearchObserver {
|
|
|
139
139
|
this.emit('researcher_progress', { id, status, tokens, cost });
|
|
140
140
|
}
|
|
141
141
|
|
|
142
|
+
onResearcherTokensHint(id: string, inputTokens: number): void {
|
|
143
|
+
this.emit('researcher_tokens_hint', { id, inputTokens });
|
|
144
|
+
}
|
|
145
|
+
|
|
142
146
|
onResearcherComplete(id: string, report: string): void {
|
|
143
147
|
this.emit('researcher_complete', { id, report });
|
|
144
148
|
}
|
|
@@ -167,6 +171,10 @@ export class HeadlessObserver implements ResearchObserver {
|
|
|
167
171
|
this.emit('evaluation_decision', { action, plan, round });
|
|
168
172
|
}
|
|
169
173
|
|
|
174
|
+
onSynthesisStart(): void {
|
|
175
|
+
this.emit('synthesis_start');
|
|
176
|
+
}
|
|
177
|
+
|
|
170
178
|
onComplete(result: string): void {
|
|
171
179
|
this.emit('complete', { result });
|
|
172
180
|
}
|
|
@@ -39,6 +39,7 @@ import { getCachedScrapedContent, normalizeUrl, cleanupSharedLinks } from '../ut
|
|
|
39
39
|
import { runResearcher } from './researcher-executor.ts';
|
|
40
40
|
import { ResearchRunSemaphore, ResearchRunCapacityError } from '../infrastructure/research-run-semaphore.ts';
|
|
41
41
|
import { recordResearcherFailure, shouldStopResearch, createResearchStopError } from './session-state.ts';
|
|
42
|
+
import { isAbortSentinel, boundSessionAbort } from './abort-utils.ts';
|
|
42
43
|
import type { ResearchSessionService } from './research-session-service.ts';
|
|
43
44
|
import { QuickResearchOrchestrator } from './quick-research-orchestrator.ts';
|
|
44
45
|
import { DeepResearchOrchestrator } from './deep-research-orchestrator.ts';
|
|
@@ -344,8 +345,34 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
|
|
|
344
345
|
// without this an early check, up to maxConcurrent-1 extra researchers could
|
|
345
346
|
// start past the stop threshold and waste a browser slot + an LLM call.
|
|
346
347
|
if (shouldStopResearch(sessionId, researchId, maxFailedResearchers)) {
|
|
347
|
-
|
|
348
|
-
|
|
348
|
+
// A SIGTERM mid-run disposes the container, and getService then throws
|
|
349
|
+
// "Cannot get service … during container disposal" — which would REPLACE
|
|
350
|
+
// the already-decided stop error with a raw service error. Teardown is
|
|
351
|
+
// happening either way, so a failure here must not gate the stop throw.
|
|
352
|
+
try {
|
|
353
|
+
const sessionService = await getService<ResearchSessionService>(ServiceNames.RESEARCH_SESSION_SERVICE, ctx, container);
|
|
354
|
+
// Bounded: abortAllSessions awaits each session's abort, and session.abort()
|
|
355
|
+
// can wedge on the very in-flight call it is cancelling — an unbounded await
|
|
356
|
+
// here would hang the fast-stop throw for as long as the wedged call takes.
|
|
357
|
+
// Proceed after the bound; the abandoned aborts drain in the background.
|
|
358
|
+
await boundSessionAbort(
|
|
359
|
+
sessionService.abortAllSessions(researchId),
|
|
360
|
+
() => logger.debug('[ResearchOrchestrationService] Fast-stop abortAllSessions did not settle within bound; proceeding with stop'),
|
|
361
|
+
);
|
|
362
|
+
} catch (abortErr) {
|
|
363
|
+
logger.debug(`[ResearchOrchestrationService] Fast-stop session abort unavailable (container disposing?): ${abortErr instanceof Error ? abortErr.message : String(abortErr)}`);
|
|
364
|
+
}
|
|
365
|
+
// abortAllSessions only reaches REGISTERED sessions; a researcher still
|
|
366
|
+
// building a session, in a retry backoff, or in its initial-links search
|
|
367
|
+
// loop self-stops via shouldStopResearch (researcher-executor.ts). Bound-
|
|
368
|
+
// await the active set so those self-stops settle BEFORE the stop error
|
|
369
|
+
// returns the run and releases its run-cap slot — bounded for the same
|
|
370
|
+
// reason as boundSessionAbort: one wedged researcher must not hang the
|
|
371
|
+
// already-decided stop.
|
|
372
|
+
await boundSessionAbort(
|
|
373
|
+
Promise.allSettled(active).then(() => undefined),
|
|
374
|
+
() => logger.debug('[ResearchOrchestrationService] Fast-stop: active researchers did not settle within bound; proceeding with stop'),
|
|
375
|
+
);
|
|
349
376
|
throw createResearchStopError(sessionId, researchId);
|
|
350
377
|
}
|
|
351
378
|
|
|
@@ -365,7 +392,7 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
|
|
|
365
392
|
}) : null;
|
|
366
393
|
try {
|
|
367
394
|
await Promise.race([...active, ...(abortPromise ? [abortPromise] : [])])
|
|
368
|
-
.catch(err => { if (err.message
|
|
395
|
+
.catch(err => { if (!isAbortSentinel(err.message)) throw err; });
|
|
369
396
|
} finally {
|
|
370
397
|
if (signal && onAbort) signal.removeEventListener('abort', onAbort);
|
|
371
398
|
}
|
|
@@ -394,11 +421,18 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
|
|
|
394
421
|
});
|
|
395
422
|
} catch (err) {
|
|
396
423
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
397
|
-
// A user cancellation (quit mid-run) surfaces here as an
|
|
398
|
-
//
|
|
399
|
-
//
|
|
400
|
-
//
|
|
401
|
-
|
|
424
|
+
// A user cancellation (quit mid-run) surfaces here as an abort-sentinel
|
|
425
|
+
// throw — either the race's bare 'Aborted' or the '<id>: Aborted' that
|
|
426
|
+
// ensureAssistantResponse raises for an externally-aborted session (the
|
|
427
|
+
// fast-stop path). Container disposal (SIGTERM mid-run) is the same
|
|
428
|
+
// clean-teardown class: the executor stops retrying on it, and counting
|
|
429
|
+
// its "during container disposal" throw as a researcher failure both
|
|
430
|
+
// paints a red slice with a bogus root cause and can trip
|
|
431
|
+
// shouldStopResearch during shutdown. None of these are researcher
|
|
432
|
+
// failures: don't log at ERROR, don't count toward the fast-stop
|
|
433
|
+
// threshold, don't paint the TUI slice red. Mirrors the guard in
|
|
434
|
+
// researcher-executor.ts.
|
|
435
|
+
if (signal?.aborted || isAbortSentinel(errMsg) || container?.isDisposing || errMsg.includes('during container disposal')) {
|
|
402
436
|
logger.debug(`[ResearchOrchestrationService] Researcher ${id} cancelled (aborted).`);
|
|
403
437
|
} else {
|
|
404
438
|
logger.error(`[ResearchOrchestrationService] Researcher ${id} failed: ${errMsg}`);
|
|
@@ -436,9 +470,27 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
|
|
|
436
470
|
}
|
|
437
471
|
|
|
438
472
|
if (shouldStopResearch(sessionId, researchId, maxFailedResearchers)) {
|
|
439
|
-
|
|
440
|
-
//
|
|
441
|
-
|
|
473
|
+
// getService can throw during container disposal — same guard as the
|
|
474
|
+
// pre-launch fast-stop above: the stop error must win, not a raw
|
|
475
|
+
// "Cannot get service … during container disposal".
|
|
476
|
+
try {
|
|
477
|
+
const sessionService = await getService<ResearchSessionService>(ServiceNames.RESEARCH_SESSION_SERVICE, ctx, container);
|
|
478
|
+
// Abort sessions specifically for this researchId, not the whole piSessionId.
|
|
479
|
+
// Bounded for the same reason as the pre-launch fast-stop above: a wedged
|
|
480
|
+
// session.abort() must not gate the stop error that is already decided.
|
|
481
|
+
await boundSessionAbort(
|
|
482
|
+
sessionService.abortAllSessions(researchId),
|
|
483
|
+
() => logger.debug('[ResearchOrchestrationService] Fast-stop abortAllSessions did not settle within bound; proceeding with stop'),
|
|
484
|
+
);
|
|
485
|
+
} catch (abortErr) {
|
|
486
|
+
logger.debug(`[ResearchOrchestrationService] Fast-stop session abort unavailable (container disposing?): ${abortErr instanceof Error ? abortErr.message : String(abortErr)}`);
|
|
487
|
+
}
|
|
488
|
+
// Bound-await the active set so unregistered researchers self-stop before
|
|
489
|
+
// the stop error releases the run-cap slot (see the pre-launch site).
|
|
490
|
+
await boundSessionAbort(
|
|
491
|
+
Promise.allSettled(active).then(() => undefined),
|
|
492
|
+
() => logger.debug('[ResearchOrchestrationService] Fast-stop: active researchers did not settle within bound; proceeding with stop'),
|
|
493
|
+
);
|
|
442
494
|
|
|
443
495
|
throw createResearchStopError(sessionId, researchId);
|
|
444
496
|
}
|
|
@@ -12,6 +12,7 @@ import type { AgentSession } from '@earendil-works/pi-coding-agent';
|
|
|
12
12
|
import { logger } from '../logger.ts';
|
|
13
13
|
import { ServiceLifecycle, type IService } from '../core/service-registry.ts';
|
|
14
14
|
import { ServiceNames } from '../core/interfaces/service-names.ts';
|
|
15
|
+
import { boundSessionAbort } from './abort-utils.ts';
|
|
15
16
|
|
|
16
17
|
/**
|
|
17
18
|
* Session entry with abort capability
|
|
@@ -99,26 +100,51 @@ export class ResearchSessionService implements IService {
|
|
|
99
100
|
}
|
|
100
101
|
|
|
101
102
|
/**
|
|
102
|
-
* Abort all active sessions for a specific sessionId, or all sessions if sessionId is omitted
|
|
103
|
+
* Abort all active sessions for a specific sessionId, or all sessions if sessionId is omitted.
|
|
104
|
+
*
|
|
105
|
+
* Each abort is bounded (boundSessionAbort): entry.abort() awaits the very
|
|
106
|
+
* in-flight call it is cancelling, which may never settle — and this method
|
|
107
|
+
* gates both the fast-stop throw (runResearchers) and run()'s finally via
|
|
108
|
+
* cleanup(), so one wedged session would otherwise hang the whole run's
|
|
109
|
+
* return. When the bound fires, teardown proceeds and the abandoned abort
|
|
110
|
+
* keeps draining in the background.
|
|
103
111
|
*/
|
|
104
112
|
async abortAllSessions(sessionId?: string): Promise<void> {
|
|
105
113
|
if (sessionId) {
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
114
|
+
// Read path: peek, never create — getSessionMap would resurrect an empty
|
|
115
|
+
// bucket for an already-completed researchId and leak it forever.
|
|
116
|
+
const sessionMap = this.peekSessionMap(sessionId);
|
|
117
|
+
if (!sessionMap) return;
|
|
118
|
+
// Snapshot BEFORE awaiting: only the entries aborted below may be removed.
|
|
119
|
+
// A session registered (or re-registered under the same id) DURING the
|
|
120
|
+
// bounded awaits — e.g. a researcher retry building a fresh session — was
|
|
121
|
+
// never aborted here; a blanket clear() would silently orphan it live.
|
|
122
|
+
const snapshot = Array.from(sessionMap.entries());
|
|
123
|
+
const aborts = snapshot.map(([, entry], index) =>
|
|
124
|
+
boundSessionAbort(
|
|
125
|
+
entry.abort().catch((err) => {
|
|
126
|
+
logger.warn(`[ResearchSessionService] Failed to abort session ${index}:`, err);
|
|
127
|
+
}),
|
|
128
|
+
() => logger.debug(`[ResearchSessionService] Session ${index} abort did not settle within bound; continuing teardown`),
|
|
129
|
+
)
|
|
111
130
|
);
|
|
112
131
|
await Promise.all(aborts);
|
|
113
|
-
|
|
114
|
-
|
|
132
|
+
for (const [id, entry] of snapshot) {
|
|
133
|
+
// Identity check: delete only the exact entry we aborted, not a
|
|
134
|
+
// same-id replacement registered while we awaited.
|
|
135
|
+
if (sessionMap.get(id) === entry) sessionMap.delete(id);
|
|
136
|
+
}
|
|
137
|
+
if (sessionMap.size === 0) this.sessions.delete(sessionId);
|
|
115
138
|
} else {
|
|
116
139
|
const aborts: Promise<void>[] = [];
|
|
117
140
|
for (const sessionMap of this.sessions.values()) {
|
|
118
141
|
aborts.push(...Array.from(sessionMap.values()).map((entry, index) =>
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
142
|
+
boundSessionAbort(
|
|
143
|
+
entry.abort().catch((err) => {
|
|
144
|
+
logger.warn(`[ResearchSessionService] Failed to abort session ${index}:`, err);
|
|
145
|
+
}),
|
|
146
|
+
() => logger.debug(`[ResearchSessionService] Session ${index} abort did not settle within bound; continuing teardown`),
|
|
147
|
+
)
|
|
122
148
|
));
|
|
123
149
|
}
|
|
124
150
|
await Promise.all(aborts);
|
|
@@ -19,7 +19,8 @@ import { getService, tryGetServiceContainerFromCtx } from '../core/service-regis
|
|
|
19
19
|
import type { ResearchSessionService } from './research-session-service.ts';
|
|
20
20
|
import { loadPrompt } from '../core/llm/prompts.ts';
|
|
21
21
|
import { injectCurrentDate } from '../core/llm/inject-date.ts';
|
|
22
|
-
import { recordResearcherFailure, getSteeringMessages } from './session-state.ts';
|
|
22
|
+
import { recordResearcherFailure, getSteeringMessages, shouldStopResearch } from './session-state.ts';
|
|
23
|
+
import { isAbortSentinel, boundSessionAbort } from './abort-utils.ts';
|
|
23
24
|
import type { RunResearcherOptions } from './orchestration-types.ts';
|
|
24
25
|
import { search } from '../web-research/search.ts';
|
|
25
26
|
|
|
@@ -46,6 +47,15 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
|
|
|
46
47
|
|
|
47
48
|
const id = String(researcherConfig.id);
|
|
48
49
|
const container = tryGetServiceContainerFromCtx(ctx);
|
|
50
|
+
|
|
51
|
+
// Fast-stop self-check: when the run's failure threshold trips, runResearchers'
|
|
52
|
+
// fast-stop path aborts only REGISTERED sessions — a researcher that is still
|
|
53
|
+
// acquiring initial links, sleeping between attempts, or rebuilding a session
|
|
54
|
+
// has none, and would otherwise sail on into a full LLM session + scrape burst
|
|
55
|
+
// AFTER the run already returned and released its run-cap slot. Every launch /
|
|
56
|
+
// retry boundary below consults this and exits quietly instead (the stop error
|
|
57
|
+
// is the run's outcome, not this researcher's failure).
|
|
58
|
+
const stopRequested = (): boolean => shouldStopResearch(sessionId, researchId, config.MAX_FAILED_RESEARCHERS);
|
|
49
59
|
observer?.onResearcherStart?.(id, researcherConfig.name, researcherConfig.goal, round);
|
|
50
60
|
metrics.increment('researchers_launched_total', 1, { mode: 'deep', complexity: String(complexity), round: String(round) });
|
|
51
61
|
|
|
@@ -81,12 +91,12 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
|
|
|
81
91
|
const maxLinkAttempts = config.RESEARCHER_MAX_RETRIES + 1;
|
|
82
92
|
for (let linkAttempt = 1; linkAttempt <= maxLinkAttempts && effectiveInitialLinks.length === 0; linkAttempt++) {
|
|
83
93
|
if (linkAttempt > 1) {
|
|
84
|
-
if (signal?.aborted || container?.isDisposing) break;
|
|
94
|
+
if (signal?.aborted || container?.isDisposing || stopRequested()) break;
|
|
85
95
|
const delay = Math.min(1000 * Math.pow(2, linkAttempt - 2), config.RESEARCHER_MAX_RETRY_DELAY_MS);
|
|
86
96
|
logger.warn(`[ResearcherExecutor] Researcher ${id} has no initial search results; retry ${linkAttempt - 1}/${config.RESEARCHER_MAX_RETRIES} after ${delay}ms`);
|
|
87
97
|
await new Promise(r => setTimeout(r, delay));
|
|
88
98
|
}
|
|
89
|
-
if (signal?.aborted || container?.isDisposing) break;
|
|
99
|
+
if (signal?.aborted || container?.isDisposing || stopRequested()) break;
|
|
90
100
|
try {
|
|
91
101
|
const queryResults = await search(retryQueries, config, signal, undefined, container);
|
|
92
102
|
const urls = new Set<string>();
|
|
@@ -99,6 +109,17 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
|
|
|
99
109
|
}
|
|
100
110
|
}
|
|
101
111
|
|
|
112
|
+
// A cancel/teardown/fast-stop lands here with zero links because the loop
|
|
113
|
+
// above breaks on abort/disposal/stop — that is a clean stop, not a "search
|
|
114
|
+
// produced nothing" researcher failure. Return quietly (mirrors the
|
|
115
|
+
// cancellation treatment in runResearchers) instead of recording a failure,
|
|
116
|
+
// bumping the skip metric, and painting a red failed slice with a bogus root
|
|
117
|
+
// cause on a plain cancel.
|
|
118
|
+
if (signal?.aborted || container?.isDisposing || stopRequested()) {
|
|
119
|
+
logger.debug(`[ResearcherExecutor] Researcher ${id} cancelled while acquiring initial links; skipping quietly.`);
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
|
|
102
123
|
if (effectiveInitialLinks.length === 0) {
|
|
103
124
|
logger.warn(`[ResearcherExecutor] Researcher ${id} has no initial search results or historical links after ${maxLinkAttempts} attempt(s); skipping.`);
|
|
104
125
|
// Use the resolved sessionId (= piSessionId), NOT the raw ctx.sessionId: when
|
|
@@ -124,11 +145,44 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
|
|
|
124
145
|
const deliveredSteeringIds = new Set<string>();
|
|
125
146
|
|
|
126
147
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
148
|
+
// Launch boundary: never build a session for a run that is already cancelled
|
|
149
|
+
// or fast-stopped. The abort sentinel keeps the wrapper's clean-cancel
|
|
150
|
+
// classification (runResearchers treats it exactly like the race's throw);
|
|
151
|
+
// the fast-stop exit is quiet — see stopRequested above.
|
|
152
|
+
if (signal?.aborted) throw new Error('Aborted');
|
|
153
|
+
if (stopRequested()) {
|
|
154
|
+
logger.debug(`[ResearcherExecutor] Researcher ${id} exiting before attempt ${attempt}: fast-stop threshold crossed.`);
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
127
157
|
if (attempt > 1) {
|
|
128
158
|
const delay = Math.min(1000 * Math.pow(2, attempt - 2), config.RESEARCHER_MAX_RETRY_DELAY_MS);
|
|
129
159
|
logger.warn(`[ResearcherExecutor] Researcher ${id} retry ${attempt - 1}/${config.RESEARCHER_MAX_RETRIES} after ${delay}ms`);
|
|
130
160
|
observer?.onResearcherProgress?.(id, 'retry');
|
|
131
|
-
|
|
161
|
+
// Abortable: a cancel during this backoff must wake the sleep immediately —
|
|
162
|
+
// an inert setTimeout held the loop for up to RESEARCHER_MAX_RETRY_DELAY_MS
|
|
163
|
+
// after the cancel and then re-entered a full attempt (fresh session +
|
|
164
|
+
// billed prompt). Mirrors the stagger sleep in runResearchers.
|
|
165
|
+
await new Promise<void>((resolve) => {
|
|
166
|
+
const timer = setTimeout(() => {
|
|
167
|
+
if (signal) signal.removeEventListener('abort', onAbort);
|
|
168
|
+
resolve();
|
|
169
|
+
}, delay);
|
|
170
|
+
const onAbort = () => { clearTimeout(timer); resolve(); };
|
|
171
|
+
if (signal) {
|
|
172
|
+
if (signal.aborted) onAbort();
|
|
173
|
+
else signal.addEventListener('abort', onAbort, { once: true });
|
|
174
|
+
}
|
|
175
|
+
});
|
|
176
|
+
// Re-launch boundary after the sleep: a cancel, fast-stop, or container
|
|
177
|
+
// teardown landing DURING the backoff must not re-enter a full attempt —
|
|
178
|
+
// this is exactly the window the fast-stop path cannot reach, because
|
|
179
|
+
// abortAllSessions only aborts registered sessions and a between-attempts
|
|
180
|
+
// researcher has none.
|
|
181
|
+
if (signal?.aborted) throw new Error('Aborted');
|
|
182
|
+
if (stopRequested() || container?.isDisposing) {
|
|
183
|
+
logger.debug(`[ResearcherExecutor] Researcher ${id} exiting after backoff: fast-stop or container teardown.`);
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
132
186
|
}
|
|
133
187
|
|
|
134
188
|
// Generate prompt for this attempt, incorporating ALL steering messages delivered so far
|
|
@@ -229,6 +283,32 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
|
|
|
229
283
|
throw registrationError;
|
|
230
284
|
}
|
|
231
285
|
|
|
286
|
+
// Post-registration self-check: this session is now visible to
|
|
287
|
+
// abortAllSessions(), but that scan is point-in-time — a stop/cancel that
|
|
288
|
+
// landed anywhere between createResearcherSession() resolving and
|
|
289
|
+
// registerSession() actually running (the `await getService(...)` above can
|
|
290
|
+
// yield the event loop) would have missed this session entirely, and every
|
|
291
|
+
// other guard in this function only fires BEFORE session creation or
|
|
292
|
+
// BETWEEN attempts. Nothing between here and session.prompt() below awaits
|
|
293
|
+
// anything, so this single check closes the whole window: either it fires
|
|
294
|
+
// here, or the run is not stopping and prompt() is about to make this
|
|
295
|
+
// session's registration meaningful. Mirrors the top-of-loop launch
|
|
296
|
+
// boundary above exactly (signal throws, fast-stop returns quietly) —
|
|
297
|
+
// deliberately NOT container?.isDisposing: like that boundary, this is
|
|
298
|
+
// still attempt 1's launch, and a disposing container must let this
|
|
299
|
+
// attempt's own session.prompt() fail naturally into the catch block's
|
|
300
|
+
// disposal classification rather than being pre-empted here.
|
|
301
|
+
if (signal?.aborted || stopRequested()) {
|
|
302
|
+
logger.debug(`[ResearcherExecutor] Researcher ${id} exiting right after session creation: fast-stop/cancel landed before prompt() was reached.`);
|
|
303
|
+
await boundSessionAbort(
|
|
304
|
+
session.abort().catch((err) => logger.warn(`[ResearcherExecutor] Failed to abort researcher session ${id} on post-creation fast-stop:`, err)),
|
|
305
|
+
() => logger.warn(`[ResearcherExecutor] Researcher ${id} session abort did not settle within 10s; continuing cleanup`),
|
|
306
|
+
);
|
|
307
|
+
sessionService.unregisterSession(researchId, id);
|
|
308
|
+
if (signal?.aborted) throw new Error('Aborted');
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
311
|
+
|
|
232
312
|
let lastSteeringCheck = Date.now();
|
|
233
313
|
|
|
234
314
|
// User-supplied observer callbacks (SDK consumers) are invoked from inside
|
|
@@ -399,13 +479,19 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
|
|
|
399
479
|
logger.debug(`[ResearcherExecutor] Researcher ${id} salvage attempt failed:`, salvageErr);
|
|
400
480
|
}
|
|
401
481
|
|
|
402
|
-
// Skip retries when the run is being torn down: an aborted signal,
|
|
403
|
-
//
|
|
404
|
-
//
|
|
405
|
-
//
|
|
406
|
-
//
|
|
407
|
-
//
|
|
408
|
-
|
|
482
|
+
// Skip retries when the run is being torn down: an aborted signal, the abort
|
|
483
|
+
// sentinel, or the service container entering disposal (SIGTERM/quit mid-run,
|
|
484
|
+
// which throws "…during container disposal" from getService). The sentinel has
|
|
485
|
+
// TWO forms — the race's bare 'Aborted' above AND the '<id>: Aborted' that
|
|
486
|
+
// ensureAssistantResponse throws when a fast-stop abortAllSessions() aborted
|
|
487
|
+
// this session externally (prompt() then resolves normally) — so match it via
|
|
488
|
+
// isAbortSentinel, not exact equality: the prefixed form used to slip past
|
|
489
|
+
// this guard and RETRY a researcher the fast-stop path had just aborted.
|
|
490
|
+
// Retrying during disposal is futile — services are being destroyed — and it
|
|
491
|
+
// relaunches search bursts into a WorkerPool that dispose() is simultaneously
|
|
492
|
+
// destroying, producing a storm of "Cannot execute a task on destroying pool"
|
|
493
|
+
// errors.
|
|
494
|
+
if (signal?.aborted || isAbortSentinel(errMsg) || errMsg.includes('during container disposal') || container?.isDisposing) {
|
|
409
495
|
logger.debug(`[ResearcherExecutor] Researcher ${id} aborted or container disposing, skipping retries.`);
|
|
410
496
|
break; // Break out of the attempt loop
|
|
411
497
|
}
|
|
@@ -423,18 +509,12 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
|
|
|
423
509
|
// never settle — never let per-attempt cleanup hang the retry loop on it. The
|
|
424
510
|
// retry builds a fresh session; the abandoned abort keeps draining in the
|
|
425
511
|
// background and its session dies with the run's teardown.
|
|
426
|
-
await
|
|
512
|
+
await boundSessionAbort(
|
|
427
513
|
session.abort().catch((err) => {
|
|
428
514
|
logger.warn(`[ResearcherExecutor] Failed to abort researcher session ${id}:`, err);
|
|
429
515
|
}),
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
logger.warn(`[ResearcherExecutor] Researcher ${id} session abort did not settle within 10s; continuing cleanup`);
|
|
433
|
-
resolve();
|
|
434
|
-
}, 10_000);
|
|
435
|
-
t.unref?.();
|
|
436
|
-
}),
|
|
437
|
-
]);
|
|
516
|
+
() => logger.warn(`[ResearcherExecutor] Researcher ${id} session abort did not settle within 10s; continuing cleanup`),
|
|
517
|
+
);
|
|
438
518
|
sessionService?.unregisterSession(researchId, id);
|
|
439
519
|
|
|
440
520
|
// Restore the default thinking label now that the researcher is done.
|
|
@@ -23,6 +23,7 @@ import { logger } from '../logger.ts';
|
|
|
23
23
|
|
|
24
24
|
import { resolveResearchModel } from '../core/llm/research-model-resolver.ts';
|
|
25
25
|
import { getRuntimeForRegistry } from '../core/llm/model-registry-factory.ts';
|
|
26
|
+
import { boundSessionAbort } from './abort-utils.ts';
|
|
26
27
|
|
|
27
28
|
export interface CreateResearcherSessionOptions {
|
|
28
29
|
cwd: string;
|
|
@@ -158,27 +159,45 @@ export async function createResearcherSession(options: CreateResearcherSessionOp
|
|
|
158
159
|
thinkingLevel: config?.LLM_THINKING_LEVEL ?? 'off',
|
|
159
160
|
});
|
|
160
161
|
|
|
161
|
-
//
|
|
162
|
-
//
|
|
163
|
-
//
|
|
164
|
-
//
|
|
165
|
-
//
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
162
|
+
// The session above is LIVE from here on, but the caller only receives it at
|
|
163
|
+
// the return below — a throw anywhere in this tail (a host UI callback, the
|
|
164
|
+
// validity check) would rethrow a session nobody owns, leaking it unaborted.
|
|
165
|
+
// Abort it best-effort (bounded: session.abort() awaits the in-flight call it
|
|
166
|
+
// cancels and may never settle) before rethrowing.
|
|
167
|
+
try {
|
|
168
|
+
// Customize the agent thinking label for researchers (e.g. "Researcher 1.1")
|
|
169
|
+
// so non-TUI surfaces can tell concurrent researchers apart. Skip this in the
|
|
170
|
+
// research TUI (mode === 'tui'): the panel already shows every researcher, and
|
|
171
|
+
// the host renders this label as dim/italic thinking text ABOVE the panel —
|
|
172
|
+
// stray output the panel was meant to replace. Use the researcherId passed by
|
|
173
|
+
// the caller (not regex on the prompt, since the ID is never templated in).
|
|
174
|
+
if (extensionCtx.mode !== 'tui' && extensionCtx.hasUI && typeof extensionCtx.ui.setHiddenThinkingLabel === 'function') {
|
|
175
|
+
extensionCtx.ui.setHiddenThinkingLabel(
|
|
176
|
+
researcherId ? `Researcher ${researcherId}` : undefined
|
|
177
|
+
);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
logger.log(`[Researcher] Created session with model=${modelToUse?.id || 'unknown'}`);
|
|
181
|
+
|
|
182
|
+
if (!result || !result.session) {
|
|
183
|
+
throw new Error('Session creation returned invalid result');
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
sessionRef.session = result.session;
|
|
187
|
+
|
|
188
|
+
return { session: result.session, resolvedModel: modelToUse };
|
|
189
|
+
} catch (tailError) {
|
|
190
|
+
const session = result?.session;
|
|
191
|
+
if (session && typeof session.abort === 'function') {
|
|
192
|
+
await boundSessionAbort(
|
|
193
|
+
(async () => session.abort())().catch((abortErr) => {
|
|
194
|
+
logger.warn('[Researcher] Failed to abort never-returned session:', abortErr);
|
|
195
|
+
}),
|
|
196
|
+
() => logger.warn('[Researcher] Never-returned session abort did not settle within bound; continuing'),
|
|
197
|
+
);
|
|
198
|
+
}
|
|
199
|
+
throw tailError;
|
|
171
200
|
}
|
|
172
|
-
|
|
173
|
-
logger.log(`[Researcher] Created session with model=${modelToUse?.id || 'unknown'}`);
|
|
174
|
-
|
|
175
|
-
if (!result || !result.session) {
|
|
176
|
-
throw new Error('Session creation returned invalid result');
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
sessionRef.session = result.session;
|
|
180
|
-
|
|
181
|
-
return { session: result.session, resolvedModel: modelToUse };
|
|
182
201
|
} catch (error) {
|
|
183
202
|
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
184
203
|
throw new Error(`Failed to create researcher session: ${errorMsg}`, {
|
|
@@ -520,6 +520,17 @@ export function endResearchSession(piSessionId: string | undefined, researchId:
|
|
|
520
520
|
// but preserve steering messages if any remain (they might arrive between
|
|
521
521
|
// the last research end and the next research start)
|
|
522
522
|
if (state.order.length === 0 && state.panels.size === 0) {
|
|
523
|
+
// Drop steering messages this run already consumed ('active') — and any
|
|
524
|
+
// 'popped' remnants, which are likewise already resolved — before deciding
|
|
525
|
+
// whether to keep the session alive. Without this, a message the run that
|
|
526
|
+
// just ended already consumed stayed in the array and leaked into a later,
|
|
527
|
+
// unrelated run's steering (both getActiveSteeringMessages/getSteeringMessages
|
|
528
|
+
// reads and the before_agent_start injection gate), since nothing distinguishes
|
|
529
|
+
// "steering for THIS run" from "steering already applied to a finished run".
|
|
530
|
+
// Only messages still 'queued' (arrived in the gap, never attached to any run)
|
|
531
|
+
// are genuinely eligible for the gap-preservation this block performs.
|
|
532
|
+
state.steeringMessages = state.steeringMessages.filter(m => m.status === 'queued');
|
|
533
|
+
|
|
523
534
|
// Only fully delete if there are no remaining steering messages
|
|
524
535
|
const hasRemainingSteering = state.steeringMessages.length > 0;
|
|
525
536
|
if (!hasRemainingSteering) {
|