@lincoln504/pi-research 1.0.6 → 1.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.mjs +250 -104
- package/dist/prompts/research-tool-usage.md +1 -0
- package/dist/thread-worker.mjs +9 -3
- package/docs/ARCHITECTURE.md +19 -4
- package/docs/CHANGELOG.md +129 -0
- package/docs/CONFIGURATION.md +2 -1
- package/docs/SDK.md +4 -1
- package/package.json +18 -16
- package/skills/pi-research/SKILL.md +1 -1
- package/src/cli.ts +62 -13
- package/src/config.ts +12 -4
- package/src/core/llm/model-registry-factory.ts +77 -25
- package/src/index.ts +11 -5
- package/src/infrastructure/browser/task-execution-service.ts +15 -3
- package/src/infrastructure/browser/thread-worker.mjs +9 -3
- package/src/orchestration/research-orchestration-service.ts +6 -6
- package/src/orchestration/research-synthesis-service.ts +31 -20
- package/src/orchestration/researcher-executor.ts +51 -13
- package/src/orchestration/researcher.ts +12 -1
- package/src/orchestration/session-state.ts +27 -4
- package/src/prompts/research-tool-usage.md +1 -0
- package/src/sdk.ts +58 -5
- package/src/utils/make-resource-loader.ts +13 -23
- package/src/web-research/browser-search.ts +17 -5
- package/src/youtube/index.ts +5 -4
|
@@ -8,7 +8,8 @@
|
|
|
8
8
|
* Previously this logic was duplicated (#29) — any bug fix now updates both paths.
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
import {
|
|
11
|
+
import { ModelRegistry, ModelRuntime, getAgentDir } from '@earendil-works/pi-coding-agent';
|
|
12
|
+
import { InMemoryCredentialStore } from '@earendil-works/pi-ai';
|
|
12
13
|
import type { Model } from '@earendil-works/pi-ai';
|
|
13
14
|
import * as path from 'node:path';
|
|
14
15
|
import * as fs from 'node:fs';
|
|
@@ -103,42 +104,93 @@ function buildHeaders(provider?: string): Record<string, string> {
|
|
|
103
104
|
/**
|
|
104
105
|
* Create a ModelRegistry with the given credentials.
|
|
105
106
|
*
|
|
107
|
+
* pi 0.80.8 replaced `AuthStorage`/`ModelRegistry.create()` with the async
|
|
108
|
+
* `ModelRuntime`: `ModelRegistry` is now a SYNCHRONOUS compatibility facade that
|
|
109
|
+
* wraps a `ModelRuntime` and still exposes `getAll`/`getAvailable`/`find`/
|
|
110
|
+
* `getApiKeyAndHeaders` — so every consumer of the registry (resolveModel,
|
|
111
|
+
* safeGet*, the research-model resolver) stays synchronous. Only this factory
|
|
112
|
+
* became async (constructing the ModelRuntime is inherently async) and its two
|
|
113
|
+
* callers await it. Mirrors how pi itself builds a registry
|
|
114
|
+
* (`sdk.js`: `await ModelRuntime.create({ authPath, modelsPath })`).
|
|
115
|
+
*
|
|
106
116
|
* @param apiKey - Optional explicit API key
|
|
107
117
|
* @param provider - Optional provider name to key the API key under
|
|
108
118
|
* @returns A configured ModelRegistry
|
|
109
119
|
*/
|
|
110
|
-
export function buildModelRegistry(apiKey?: string, provider?: string): ModelRegistry {
|
|
120
|
+
export async function buildModelRegistry(apiKey?: string, provider?: string): Promise<ModelRegistry> {
|
|
111
121
|
const agentDir = getAgentDir();
|
|
112
122
|
const modelsJsonPath = path.join(agentDir, 'models.json');
|
|
113
123
|
const authPath = path.join(agentDir, 'auth.json');
|
|
124
|
+
const hasModels = fs.existsSync(modelsJsonPath);
|
|
125
|
+
const hasAuth = fs.existsSync(authPath);
|
|
126
|
+
|
|
127
|
+
// Build the runtime from the user's pi config (models.json + auth.json) when
|
|
128
|
+
// EITHER file is present, otherwise a bare runtime that reads env-var keys and
|
|
129
|
+
// built-in providers. allowModelNetwork:false preserves the prior behavior of
|
|
130
|
+
// NOT doing a network model-catalog refresh at construction (the old
|
|
131
|
+
// ModelRegistry.create only read the local files). Env provider keys
|
|
132
|
+
// (e.g. OPENAI_API_KEY) are resolved via pi-ai's provider auth CONTEXT, not the
|
|
133
|
+
// credential store, so they are honored in every branch below; embedded
|
|
134
|
+
// models.json apiKeys likewise — auth.json presence must not gate models.json.
|
|
135
|
+
//
|
|
136
|
+
// When auth.json does NOT exist, an explicit in-memory credential store is
|
|
137
|
+
// passed instead of letting ModelRuntime default to the file at authPath:
|
|
138
|
+
// AuthStorage's file backend CREATES ~/.pi/agent/ and an empty auth.json as a
|
|
139
|
+
// construction side effect (mkdir+write). Without this, every CLI invocation —
|
|
140
|
+
// including read-only ones like `status` — would silently create a pi config
|
|
141
|
+
// dir on machines that never installed pi, and would hard-fail in read-only-
|
|
142
|
+
// $HOME environments (containers/CI) where the old in-memory path needed no
|
|
143
|
+
// filesystem write access at all.
|
|
144
|
+
const runtime = await ModelRuntime.create({
|
|
145
|
+
authPath: hasAuth ? authPath : undefined,
|
|
146
|
+
credentials: hasAuth ? undefined : new InMemoryCredentialStore(),
|
|
147
|
+
modelsPath: hasModels ? modelsJsonPath : null,
|
|
148
|
+
allowModelNetwork: false,
|
|
149
|
+
});
|
|
114
150
|
|
|
151
|
+
// Explicit key: seed it on the runtime after construction. NOTE: unlike the old
|
|
152
|
+
// AuthStorage.inMemory({ [provider]: ... }) seeding, this MERGES — the explicit
|
|
153
|
+
// key wins for its own provider (runtime overrides are checked first), but any
|
|
154
|
+
// providers already configured in auth.json remain visible in getAvailable().
|
|
155
|
+
// That broadening is acceptable because explicit-key flows always pair the key
|
|
156
|
+
// with an explicit model (enforced in cli.ts pre-flight), so resolution never
|
|
157
|
+
// falls back to "first available".
|
|
115
158
|
if (apiKey && provider) {
|
|
116
|
-
//
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
fs.existsSync(modelsJsonPath) ? modelsJsonPath : undefined,
|
|
123
|
-
);
|
|
159
|
+
// allowNetwork:false is REQUIRED here, not just consistent with create():
|
|
160
|
+
// setRuntimeApiKey triggers a model-catalog refresh for the newly configured
|
|
161
|
+
// provider, and its default options open live catalog connections — verified
|
|
162
|
+
// to hang indefinitely (no timeout at this layer) in network-restricted
|
|
163
|
+
// environments, which would wedge every explicit-API-key pre-flight.
|
|
164
|
+
await runtime.setRuntimeApiKey(provider, apiKey, { allowNetwork: false });
|
|
124
165
|
}
|
|
125
166
|
|
|
126
|
-
//
|
|
127
|
-
//
|
|
128
|
-
//
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
const authStorage = AuthStorage.create(authPath);
|
|
134
|
-
return ModelRegistry.create(
|
|
135
|
-
authStorage,
|
|
136
|
-
fs.existsSync(modelsJsonPath) ? modelsJsonPath : undefined,
|
|
137
|
-
);
|
|
138
|
-
}
|
|
167
|
+
// The sync facade all our resolvers consume. getAvailable()/getApiKeyAndHeaders()
|
|
168
|
+
// read a cached snapshot populated by create(), so the sync reads below work
|
|
169
|
+
// without further awaits.
|
|
170
|
+
const registry = new ModelRegistry(runtime);
|
|
171
|
+
registryRuntimes.set(registry, runtime);
|
|
172
|
+
return registry;
|
|
173
|
+
}
|
|
139
174
|
|
|
140
|
-
|
|
141
|
-
|
|
175
|
+
/**
|
|
176
|
+
* The ModelRuntime backing each registry this factory built. Researcher sessions
|
|
177
|
+
* must be created with `modelRuntime` (pi 0.80.8+ replaced the old
|
|
178
|
+
* `modelRegistry` session option): without it, createAgentSession() builds its
|
|
179
|
+
* OWN runtime from the default ~/.pi/agent paths, which never sees an explicit
|
|
180
|
+
* API key seeded via setRuntimeApiKey above — standalone PI_RESEARCH_API_KEY
|
|
181
|
+
* users with no (or a different-provider) auth.json would pass pre-flight and
|
|
182
|
+
* then fail auth on every researcher LLM call.
|
|
183
|
+
*/
|
|
184
|
+
const registryRuntimes = new WeakMap<ModelRegistry, ModelRuntime>();
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Return the ModelRuntime behind a registry built by {@link buildModelRegistry},
|
|
188
|
+
* or undefined for registries built elsewhere (e.g. the pi host's own
|
|
189
|
+
* ExtensionContext.modelRegistry — for those, createAgentSession()'s default
|
|
190
|
+
* runtime construction from the host's agent dir is exactly right).
|
|
191
|
+
*/
|
|
192
|
+
export function getRuntimeForRegistry(registry: unknown): ModelRuntime | undefined {
|
|
193
|
+
return registry instanceof Object ? registryRuntimes.get(registry as ModelRegistry) : undefined;
|
|
142
194
|
}
|
|
143
195
|
|
|
144
196
|
/**
|
package/src/index.ts
CHANGED
|
@@ -53,6 +53,8 @@ export {
|
|
|
53
53
|
getLastRunSummary,
|
|
54
54
|
getLastErrorReport,
|
|
55
55
|
getLastRunStats,
|
|
56
|
+
getLastResearcherOutcome,
|
|
57
|
+
type ResearcherOutcome,
|
|
56
58
|
getSessionMetrics,
|
|
57
59
|
getResearchHealth,
|
|
58
60
|
runQuickResearch,
|
|
@@ -88,10 +90,14 @@ function extractResultText(result: AgentToolResult<unknown>): string {
|
|
|
88
90
|
* Pi Research Extension
|
|
89
91
|
*/
|
|
90
92
|
export default async function (pi: ExtensionAPI) {
|
|
91
|
-
// Runtime version check — must match the @earendil-works/* dependency minimum (>=0.80.
|
|
92
|
-
// 0.80.
|
|
93
|
-
//
|
|
94
|
-
//
|
|
93
|
+
// Runtime version check — must match the @earendil-works/* dependency minimum (>=0.80.8).
|
|
94
|
+
// 0.80.8 is the floor because pi 0.80.8 introduced ModelRuntime and removed
|
|
95
|
+
// AuthStorage/ModelRegistry.create(); buildModelRegistry (model-registry-factory.ts)
|
|
96
|
+
// unconditionally calls ModelRuntime.create(), and createAgentSession() is invoked
|
|
97
|
+
// without the removed `modelRegistry` option (researcher.ts), relying on the
|
|
98
|
+
// 0.80.8+ host auto-building its ModelRuntime. This in-host check is the real
|
|
99
|
+
// enforcement point: when running as an extension the host's version is NOT
|
|
100
|
+
// constrained by npm's resolution of our own package.json range.
|
|
95
101
|
const versionParts = PI_VERSION.split('.').map(Number);
|
|
96
102
|
const major = versionParts[0] ?? 0;
|
|
97
103
|
const minor = versionParts[1] ?? 0;
|
|
@@ -102,7 +108,7 @@ export default async function (pi: ExtensionAPI) {
|
|
|
102
108
|
`Please ensure pi-coding-agent is installed correctly.`,
|
|
103
109
|
);
|
|
104
110
|
}
|
|
105
|
-
const minMajor = 0, minMinor = 80, minPatch =
|
|
111
|
+
const minMajor = 0, minMinor = 80, minPatch = 8;
|
|
106
112
|
const tooOld = major < minMajor
|
|
107
113
|
|| (major === minMajor && minor < minMinor)
|
|
108
114
|
|| (major === minMajor && minor === minMinor && patch < minPatch);
|
|
@@ -99,11 +99,17 @@ export async function runBrowserTask<T>(
|
|
|
99
99
|
await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
|
|
100
100
|
} else {
|
|
101
101
|
// True socket/connection error — restart the scheduler with thundering herd guard.
|
|
102
|
+
// Do NOT force-clear remote leader state here: a transient socket
|
|
103
|
+
// hiccup against a live-but-busy leader is not proof the leader is
|
|
104
|
+
// dead. forceSchedulerRestart(false) still clears state when its own
|
|
105
|
+
// PID+start-time liveness probe confirms the registered leader is
|
|
106
|
+
// actually gone; it only skips clearing when that probe finds the
|
|
107
|
+
// leader alive, which is exactly the case we must not disturb.
|
|
102
108
|
const now = Date.now();
|
|
103
109
|
if (now - lastRestartTime > RESTART_COOLDOWN_MS) {
|
|
104
110
|
lastRestartTime = now;
|
|
105
111
|
logger.warn(`[BrowserManager] Forcing scheduler restart and retrying...`);
|
|
106
|
-
await forceSchedulerRestart(
|
|
112
|
+
await forceSchedulerRestart(false, container);
|
|
107
113
|
await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
|
|
108
114
|
} else {
|
|
109
115
|
logger.warn(`[BrowserManager] Scheduler restart recently triggered, waiting for pool idle...`);
|
|
@@ -141,11 +147,14 @@ export async function runBrowserHealthCheck(config?: Config, retries = 1, signal
|
|
|
141
147
|
if (isPoolShutdownError(error)) {
|
|
142
148
|
await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
|
|
143
149
|
} else {
|
|
150
|
+
// See runBrowserTask: don't force-clear remote leader state on a bare
|
|
151
|
+
// transient socket error — let the liveness probe inside
|
|
152
|
+
// forceSchedulerRestart decide.
|
|
144
153
|
const now = Date.now();
|
|
145
154
|
if (now - lastRestartTime > RESTART_COOLDOWN_MS) {
|
|
146
155
|
lastRestartTime = now;
|
|
147
156
|
logger.warn(`[BrowserManager] Forcing scheduler restart and retrying...`);
|
|
148
|
-
await forceSchedulerRestart(
|
|
157
|
+
await forceSchedulerRestart(false, container);
|
|
149
158
|
await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
|
|
150
159
|
} else {
|
|
151
160
|
await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
|
|
@@ -189,11 +198,14 @@ export async function runWorkerSearch(query: string, config?: Config, signal?: A
|
|
|
189
198
|
if (isPoolShutdownError(error)) {
|
|
190
199
|
await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
|
|
191
200
|
} else {
|
|
201
|
+
// See runBrowserTask: don't force-clear remote leader state on a bare
|
|
202
|
+
// transient socket error — let the liveness probe inside
|
|
203
|
+
// forceSchedulerRestart decide.
|
|
192
204
|
const now = Date.now();
|
|
193
205
|
if (now - lastRestartTime > RESTART_COOLDOWN_MS) {
|
|
194
206
|
lastRestartTime = now;
|
|
195
207
|
logger.warn(`[BrowserManager] Forcing scheduler restart and retrying...`);
|
|
196
|
-
await forceSchedulerRestart(
|
|
208
|
+
await forceSchedulerRestart(false, container);
|
|
197
209
|
await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
|
|
198
210
|
} else {
|
|
199
211
|
await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
|
|
@@ -2219,6 +2219,8 @@ var ConfigSchema = Type.Object({
|
|
|
2219
2219
|
RESEARCHER_MAX_RETRIES: Type.Number({ minimum: 0, maximum: 5, default: 2 }),
|
|
2220
2220
|
/** Base delay between retries in milliseconds (default: 2000, range: 100-10000) */
|
|
2221
2221
|
RESEARCHER_MAX_RETRY_DELAY_MS: Type.Number({ minimum: 100, maximum: 1e4, default: 2e3 }),
|
|
2222
|
+
/** Unique researcher failures that abort the entire run (default: 2, range: 1-10) */
|
|
2223
|
+
MAX_FAILED_RESEARCHERS: Type.Number({ minimum: 1, maximum: 10, default: 2 }),
|
|
2222
2224
|
/** Target depth for recursive research (default: 1, range: 1-3) */
|
|
2223
2225
|
DEFAULT_RESEARCH_DEPTH: Type.Number({ minimum: 1, maximum: 3, default: 1 }),
|
|
2224
2226
|
/** Number of batches to allow for a single scrape tool call (default: 2, 0=unlimited) */
|
|
@@ -2276,9 +2278,13 @@ var ConfigSchema = Type.Object({
|
|
|
2276
2278
|
* never killed before it has used its full nav budget plus time waiting in the queue.
|
|
2277
2279
|
* (default: 10000) */
|
|
2278
2280
|
BROWSER_TASK_TIMEOUT_MS: Type.Number({ minimum: 2e3, maximum: 12e4, default: 1e4 }),
|
|
2279
|
-
/** Timeout for coordinator/evaluator/repair/knowledge LLM calls in ms (default: 300000 = 5 min, range: 60s-
|
|
2280
|
-
* Not exposed in TUI — controlled via PI_RESEARCH_LLM_TIMEOUT_MS env var.
|
|
2281
|
-
|
|
2281
|
+
/** Timeout for coordinator/evaluator/repair/knowledge LLM calls in ms (default: 300000 = 5 min, range: 60s-30min).
|
|
2282
|
+
* Not exposed in TUI — controlled via PI_RESEARCH_LLM_TIMEOUT_MS env var.
|
|
2283
|
+
* Ceiling raised to 30min (matching RESEARCHER_TIMEOUT_MS): a slow model on a
|
|
2284
|
+
* large-context synthesis/plan call can legitimately run >10min, and the old
|
|
2285
|
+
* 600000 ceiling silently clamped an explicit PI_RESEARCH_LLM_TIMEOUT_MS
|
|
2286
|
+
* (e.g. 900000), causing those calls to time out mid-generation. */
|
|
2287
|
+
LLM_TIMEOUT_MS: Type.Number({ minimum: 6e4, maximum: 18e5, default: 3e5 }),
|
|
2282
2288
|
/** Chain-of-thought "thinking" level for the engine's own LLM calls (coordinator,
|
|
2283
2289
|
* evaluator, synthesis, JSON-repair, knowledge extraction) AND the researcher
|
|
2284
2290
|
* sub-agents. Default 'off': these calls emit structured JSON / cited reports, not
|
|
@@ -37,7 +37,7 @@ import type { Config } from '../config.ts';
|
|
|
37
37
|
import { getConfig, DEFAULTS } from '../config.ts';
|
|
38
38
|
import { getCachedScrapedContent, normalizeUrl, cleanupSharedLinks } from '../utils/shared-links.ts';
|
|
39
39
|
import { runResearcher } from './researcher-executor.ts';
|
|
40
|
-
import { recordResearcherFailure, shouldStopResearch,
|
|
40
|
+
import { recordResearcherFailure, shouldStopResearch, createResearchStopError } from './session-state.ts';
|
|
41
41
|
import type { ResearchSessionService } from './research-session-service.ts';
|
|
42
42
|
import { QuickResearchOrchestrator } from './quick-research-orchestrator.ts';
|
|
43
43
|
import { DeepResearchOrchestrator } from './deep-research-orchestrator.ts';
|
|
@@ -291,6 +291,7 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
|
|
|
291
291
|
const active = new Set<Promise<void>>();
|
|
292
292
|
// Honour MAX_CONCURRENT_RESEARCHERS — prevents resource spikes when a plan has many researchers.
|
|
293
293
|
const maxConcurrent: number = (orchestratorOptions.config as Config)?.MAX_CONCURRENT_RESEARCHERS ?? DEFAULTS.MAX_CONCURRENT_RESEARCHERS;
|
|
294
|
+
const maxFailedResearchers: number = (orchestratorOptions.config as Config)?.MAX_FAILED_RESEARCHERS ?? DEFAULTS.MAX_FAILED_RESEARCHERS;
|
|
294
295
|
|
|
295
296
|
for (const configItem of researchers) {
|
|
296
297
|
if (signal?.aborted) break;
|
|
@@ -299,10 +300,10 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
|
|
|
299
300
|
// post-launch check below only fires after a researcher is in-flight, so
|
|
300
301
|
// without this an early check, up to maxConcurrent-1 extra researchers could
|
|
301
302
|
// start past the stop threshold and waste a browser slot + an LLM call.
|
|
302
|
-
if (shouldStopResearch(sessionId, researchId)) {
|
|
303
|
+
if (shouldStopResearch(sessionId, researchId, maxFailedResearchers)) {
|
|
303
304
|
const sessionService = await getService<ResearchSessionService>(ServiceNames.RESEARCH_SESSION_SERVICE, ctx, container);
|
|
304
305
|
await sessionService.abortAllSessions(researchId);
|
|
305
|
-
throw
|
|
306
|
+
throw createResearchStopError(sessionId, researchId);
|
|
306
307
|
}
|
|
307
308
|
|
|
308
309
|
// Enforce the concurrency cap before launching the next researcher.
|
|
@@ -391,13 +392,12 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
|
|
|
391
392
|
if (signal?.aborted) break;
|
|
392
393
|
}
|
|
393
394
|
|
|
394
|
-
if (shouldStopResearch(sessionId, researchId)) {
|
|
395
|
+
if (shouldStopResearch(sessionId, researchId, maxFailedResearchers)) {
|
|
395
396
|
const sessionService = await getService<ResearchSessionService>(ServiceNames.RESEARCH_SESSION_SERVICE, ctx, container);
|
|
396
397
|
// Abort sessions specifically for this researchId, not the whole piSessionId
|
|
397
398
|
await sessionService.abortAllSessions(researchId);
|
|
398
399
|
|
|
399
|
-
|
|
400
|
-
throw new Error(stopMessage);
|
|
400
|
+
throw createResearchStopError(sessionId, researchId);
|
|
401
401
|
}
|
|
402
402
|
}
|
|
403
403
|
|
|
@@ -178,28 +178,39 @@ export class ResearchSynthesisService implements IService {
|
|
|
178
178
|
let globalCitations: GlobalCitation[] =
|
|
179
179
|
reports.size > 0 ? normalizeCitations(reports).globalCitations : [];
|
|
180
180
|
|
|
181
|
-
// Robustness fallback:
|
|
182
|
-
//
|
|
183
|
-
//
|
|
184
|
-
//
|
|
185
|
-
//
|
|
186
|
-
//
|
|
187
|
-
//
|
|
188
|
-
|
|
181
|
+
// Robustness fallback: reconcile against ground-truth provenance — the URLs
|
|
182
|
+
// ACTUALLY, successfully scraped this session (registerScrapedLinks only records
|
|
183
|
+
// HTTP-success fetches). This runs even when SOME citations parsed: a
|
|
184
|
+
// multi-researcher run where only one researcher's report has an unparseable/
|
|
185
|
+
// missing "CITED LINKS" block still needs that researcher's real sources folded
|
|
186
|
+
// in — gating this on globalCitations being entirely EMPTY only ever caught the
|
|
187
|
+
// all-researchers-failed-to-cite case and silently dropped a partial researcher's
|
|
188
|
+
// sources whenever ANY other researcher cited normally. Only URLs not already
|
|
189
|
+
// represented (by normalized URL) are added, so parsed citations — richer, with
|
|
190
|
+
// descriptions and source tags — are never overwritten.
|
|
191
|
+
if (reports.size > 0) {
|
|
189
192
|
const scraped = getScrapedLinks(sessionId);
|
|
190
193
|
if (scraped.length > 0) {
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
194
|
+
const known = new Set(globalCitations.map((c) => normalizeUrl(c.url)));
|
|
195
|
+
const missing = scraped.filter((url) => !known.has(normalizeUrl(url)));
|
|
196
|
+
if (missing.length > 0) {
|
|
197
|
+
const startId = globalCitations.reduce((max, c) => Math.max(max, c.id), 0) + 1;
|
|
198
|
+
// Label per actual provenance: the pool also contains transcript watch
|
|
199
|
+
// URLs the youtube_transcript tool registered — calling those "Scrape"
|
|
200
|
+
// misattributes the report's real source material.
|
|
201
|
+
globalCitations = [
|
|
202
|
+
...globalCitations,
|
|
203
|
+
...missing.map((url, i) => ({
|
|
204
|
+
id: startId + i,
|
|
205
|
+
url,
|
|
206
|
+
description: '',
|
|
207
|
+
source: isTranscribedLink(sessionId, url) ? 'YouTube Transcript' : 'Scrape',
|
|
208
|
+
})),
|
|
209
|
+
];
|
|
210
|
+
logger.warn(
|
|
211
|
+
`[ResearchSynthesisService] Folded in ${missing.length} source(s) from scrape/transcript provenance not covered by parsed citations.`,
|
|
212
|
+
);
|
|
213
|
+
}
|
|
203
214
|
}
|
|
204
215
|
}
|
|
205
216
|
|
|
@@ -21,6 +21,7 @@ import { loadPrompt } from '../core/llm/prompts.ts';
|
|
|
21
21
|
import { injectCurrentDate } from '../core/llm/inject-date.ts';
|
|
22
22
|
import { recordResearcherFailure, getSteeringMessages } from './session-state.ts';
|
|
23
23
|
import type { RunResearcherOptions } from './orchestration-types.ts';
|
|
24
|
+
import { search } from '../web-research/search.ts';
|
|
24
25
|
|
|
25
26
|
/**
|
|
26
27
|
* Run a single researcher with retries
|
|
@@ -62,22 +63,59 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
|
|
|
62
63
|
}
|
|
63
64
|
|
|
64
65
|
const researcherPromptTemplate = loadPrompt('researcher');
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
//
|
|
69
|
-
//
|
|
70
|
-
//
|
|
71
|
-
//
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
66
|
+
|
|
67
|
+
let effectiveInitialLinks = initialLinks;
|
|
68
|
+
if (effectiveInitialLinks.length === 0 && historicalUrls.length === 0) {
|
|
69
|
+
// The plan-wide initial search burst (run once, before any researcher starts)
|
|
70
|
+
// can come back empty for THIS researcher's queries specifically because the
|
|
71
|
+
// shared browser pool was mid-recovery (e.g. leader-election churn, see
|
|
72
|
+
// task-execution-service.ts) at that exact moment — not because the topic has
|
|
73
|
+
// no results. Unlike every other researcher failure mode, this one used to
|
|
74
|
+
// `return` immediately, bypassing RESEARCHER_MAX_RETRIES entirely: a single
|
|
75
|
+
// transient pool hiccup counted as a full, unretried researcher failure and,
|
|
76
|
+
// combined with MAX_FAILED_RESEARCHERS, could abort a run with several other
|
|
77
|
+
// researchers still healthy and in flight. Give it the same bounded retry
|
|
78
|
+
// budget as the rest of this function, re-issuing a fresh search for this
|
|
79
|
+
// researcher's own queries before giving up.
|
|
80
|
+
const retryQueries = researcherConfig.queries?.length ? researcherConfig.queries : [researcherConfig.goal];
|
|
81
|
+
const maxLinkAttempts = config.RESEARCHER_MAX_RETRIES + 1;
|
|
82
|
+
for (let linkAttempt = 1; linkAttempt <= maxLinkAttempts && effectiveInitialLinks.length === 0; linkAttempt++) {
|
|
83
|
+
if (linkAttempt > 1) {
|
|
84
|
+
if (signal?.aborted || container?.isDisposing) break;
|
|
85
|
+
const delay = Math.min(1000 * Math.pow(2, linkAttempt - 2), config.RESEARCHER_MAX_RETRY_DELAY_MS);
|
|
86
|
+
logger.warn(`[ResearcherExecutor] Researcher ${id} has no initial search results; retry ${linkAttempt - 1}/${config.RESEARCHER_MAX_RETRIES} after ${delay}ms`);
|
|
87
|
+
await new Promise(r => setTimeout(r, delay));
|
|
88
|
+
}
|
|
89
|
+
if (signal?.aborted || container?.isDisposing) break;
|
|
90
|
+
try {
|
|
91
|
+
const queryResults = await search(retryQueries, config, signal, undefined, container);
|
|
92
|
+
const urls = new Set<string>();
|
|
93
|
+
for (const qr of queryResults) {
|
|
94
|
+
for (const r of qr.results || []) urls.add(r.url);
|
|
95
|
+
}
|
|
96
|
+
effectiveInitialLinks = [...urls];
|
|
97
|
+
} catch (err) {
|
|
98
|
+
logger.debug(`[ResearcherExecutor] Researcher ${id} link-retry search burst failed:`, err);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
if (effectiveInitialLinks.length === 0) {
|
|
103
|
+
logger.warn(`[ResearcherExecutor] Researcher ${id} has no initial search results or historical links after ${maxLinkAttempts} attempt(s); skipping.`);
|
|
104
|
+
// Use the resolved sessionId (= piSessionId), NOT the raw ctx.sessionId: when
|
|
105
|
+
// ctx.sessionId is unset but sessionManager supplies a real id (common in SDK
|
|
106
|
+
// use), recording under ctx.sessionId would file this no-links skip under a
|
|
107
|
+
// different id than shouldStopResearch() checks, so the fast-stop guard would
|
|
108
|
+
// under-count exactly the broad-search-failure case it exists to catch.
|
|
109
|
+
recordResearcherFailure(sessionId, researchId, id, 'no initial search results or historical links (search produced nothing usable)');
|
|
110
|
+
metrics.increment('researcher_skipped_total', 1, { mode: 'deep', complexity: String(complexity), reason: 'no_initial_links' });
|
|
111
|
+
observer?.onResearcherFailure?.(id, 'No initial search results or historical links available');
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
76
114
|
}
|
|
77
115
|
|
|
78
116
|
let evidenceSection = '';
|
|
79
|
-
if (
|
|
80
|
-
evidenceSection = `## Evidence Provided\nInitial search results provided the following URLs to investigate:\n${
|
|
117
|
+
if (effectiveInitialLinks.length > 0) {
|
|
118
|
+
evidenceSection = `## Evidence Provided\nInitial search results provided the following URLs to investigate:\n${effectiveInitialLinks.map(l => `- ${l}`).join('\n')}`;
|
|
81
119
|
}
|
|
82
120
|
|
|
83
121
|
const maxAttempts = config.RESEARCHER_MAX_RETRIES + 1;
|
|
@@ -22,6 +22,7 @@ import type { Config } from '../config.ts';
|
|
|
22
22
|
import { logger } from '../logger.ts';
|
|
23
23
|
|
|
24
24
|
import { resolveResearchModel } from '../core/llm/research-model-resolver.ts';
|
|
25
|
+
import { getRuntimeForRegistry } from '../core/llm/model-registry-factory.ts';
|
|
25
26
|
|
|
26
27
|
export interface CreateResearcherSessionOptions {
|
|
27
28
|
cwd: string;
|
|
@@ -137,7 +138,17 @@ export async function createResearcherSession(options: CreateResearcherSessionOp
|
|
|
137
138
|
sessionManager: SessionManager.inMemory(), // Each researcher gets its own isolated session
|
|
138
139
|
settingsManager: researcherSettings,
|
|
139
140
|
model: modelToUse,
|
|
140
|
-
modelRegistry
|
|
141
|
+
// pi 0.80.8 replaced the SDK's `modelRegistry`/`authStorage` session options
|
|
142
|
+
// with the async `modelRuntime`. Pass the runtime backing OUR registry when
|
|
143
|
+
// this run built one (standalone CLI/SDK): it carries any explicit API key
|
|
144
|
+
// seeded via setRuntimeApiKey, which the session's own default runtime
|
|
145
|
+
// (auto-built from ~/.pi/agent/{auth,models}.json) would never see — without
|
|
146
|
+
// this, standalone PI_RESEARCH_API_KEY users fail auth on every researcher
|
|
147
|
+
// LLM call despite passing pre-flight. For registries we did NOT build
|
|
148
|
+
// (the pi host's ExtensionContext.modelRegistry), this resolves to
|
|
149
|
+
// undefined and the host auto-builds from its own agent dir — correct
|
|
150
|
+
// in-host, where auth lives in the host's config.
|
|
151
|
+
modelRuntime: getRuntimeForRegistry(modelRegistry),
|
|
141
152
|
resourceLoader: makeResourceLoader(systemPrompt),
|
|
142
153
|
// Thinking/reasoning is disabled for researcher agents: they do retrieval and
|
|
143
154
|
// synthesis from scraped pages, not open-ended reasoning. Keeping it off reduces
|
|
@@ -605,16 +605,18 @@ export function getFailedResearchers(piSessionId: string | undefined, researchId
|
|
|
605
605
|
}
|
|
606
606
|
|
|
607
607
|
/**
|
|
608
|
-
*
|
|
608
|
+
* Default (and fallback, for callers not passing a resolved Config) max allowed
|
|
609
|
+
* unique failed researchers before the run aborts. Configurable per-run via
|
|
610
|
+
* Config.MAX_FAILED_RESEARCHERS / PI_RESEARCH_MAX_FAILED_RESEARCHERS.
|
|
609
611
|
*/
|
|
610
|
-
const
|
|
612
|
+
const DEFAULT_MAX_FAILED_RESEARCHERS = 2;
|
|
611
613
|
|
|
612
614
|
/**
|
|
613
615
|
* Check if research should stop due to too many unique failures
|
|
614
616
|
*/
|
|
615
|
-
export function shouldStopResearch(piSessionId: string | undefined, researchId: string): boolean {
|
|
617
|
+
export function shouldStopResearch(piSessionId: string | undefined, researchId: string, maxFailedResearchers: number = DEFAULT_MAX_FAILED_RESEARCHERS): boolean {
|
|
616
618
|
const sid = normalizeSessionId(piSessionId);
|
|
617
|
-
return getFailedResearchers(sid, researchId).length >=
|
|
619
|
+
return getFailedResearchers(sid, researchId).length >= maxFailedResearchers;
|
|
618
620
|
}
|
|
619
621
|
|
|
620
622
|
/**
|
|
@@ -659,6 +661,27 @@ export function getResearchStopMessage(piSessionId: string | undefined, research
|
|
|
659
661
|
].join('\n');
|
|
660
662
|
}
|
|
661
663
|
|
|
664
|
+
/**
|
|
665
|
+
* Stable error code for a fail-fast research-stop error. cli.ts's reportError()
|
|
666
|
+
* checks this BEFORE its 'api key'-substring config-error heuristic: the
|
|
667
|
+
* boilerplate advice line above ("...API key / context settings are valid.")
|
|
668
|
+
* is appended to every research-stop message regardless of cause, so a
|
|
669
|
+
* substring match alone misclassifies every worker-pool/infra-driven stop as
|
|
670
|
+
* a config error (exit 78) instead of a software error (exit 70).
|
|
671
|
+
*/
|
|
672
|
+
export const RESEARCH_STOPPED_ERROR_CODE = 'RESEARCH_STOPPED';
|
|
673
|
+
|
|
674
|
+
/**
|
|
675
|
+
* Build the Error to throw when shouldStopResearch() trips, tagged with
|
|
676
|
+
* RESEARCH_STOPPED_ERROR_CODE so callers can classify it without re-parsing
|
|
677
|
+
* the (human-oriented, boilerplate-laden) message text.
|
|
678
|
+
*/
|
|
679
|
+
export function createResearchStopError(piSessionId: string | undefined, researchId: string): Error & { code: string } {
|
|
680
|
+
const err = new Error(getResearchStopMessage(piSessionId, researchId)) as Error & { code: string };
|
|
681
|
+
err.code = RESEARCH_STOPPED_ERROR_CODE;
|
|
682
|
+
return err;
|
|
683
|
+
}
|
|
684
|
+
|
|
662
685
|
/**
|
|
663
686
|
* Get all active research panels in a Pi session, in display order (newest first)
|
|
664
687
|
*/
|
|
@@ -85,6 +85,7 @@ The coordinator will plan as many researchers as needed (up to the max). You do
|
|
|
85
85
|
|
|
86
86
|
**Multiple Topics:**
|
|
87
87
|
- If you need to research multiple unrelated topics of **truly distinct scope**, you may emit multiple `research` calls **simultaneously in a single turn**. Each call gets its own isolated research run with its own coordinator and researchers.
|
|
88
|
+
- **Maximum 4 simultaneous calls** — Do NOT make more than 4 `research` calls in a single turn, even if you have more unrelated topics. Exceeding 4 simultaneous research runs can overload the system and degrade performance. If you have more than 4 truly distinct topics, they can likely be consolidated into fewer calls (group related subtopics, use broader queries, or batch across multiple turns).
|
|
88
89
|
- Example: `research("bananas")` and `research("quantum computing")` in the same turn — these have zero overlap.
|
|
89
90
|
- **Group related topics**: If topics share ANY scope or could inform each other, combine them into a single `research` call. The internal coordinator will decompose them into parallel researcher agents. Do NOT split related sub-topics into separate calls.
|
|
90
91
|
- Example: `research("Python async performance and Rust async performance comparison")` — NOT separate calls for Python and Rust, since the comparison IS the research goal.
|