@lincoln504/pi-research 1.4.1 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cli.mjs +449 -187
- package/dist/prompts/research-knowledge-relevance-triage.md +2 -0
- package/dist/prompts/research-knowledge-search-extractor.md +2 -0
- package/dist/prompts/researcher.md +2 -14
- package/dist/thread-worker.mjs +91 -59
- package/docs/AGENT-SKILL.md +4 -3
- package/docs/CHANGELOG.md +120 -0
- package/docs/CONFIGURATION.md +2 -2
- package/docs/SDK.md +1 -1
- package/package.json +1 -1
- package/skills/pi-research/SKILL.md +1 -1
- package/src/cli.ts +44 -3
- package/src/config.ts +36 -13
- package/src/constants.ts +23 -4
- package/src/core/llm/research-model-resolver.ts +38 -0
- package/src/core/planning-service.ts +67 -9
- package/src/index.ts +3 -1
- package/src/infrastructure/browser/browser-task-scheduler.ts +16 -9
- package/src/infrastructure/browser/config.ts +40 -2
- package/src/infrastructure/browser/thread-worker-browser.ts +69 -48
- package/src/infrastructure/browser/thread-worker-messaging.ts +65 -4
- package/src/infrastructure/browser/thread-worker.mjs +91 -59
- package/src/infrastructure/browser/thread-worker.ts +13 -5
- package/src/infrastructure/embedding/embedding-server.ts +54 -0
- package/src/knowledge/embedder-init.ts +3 -2
- package/src/knowledge/embedder.ts +11 -3
- package/src/knowledge/index.ts +36 -10
- package/src/knowledge/store.ts +85 -5
- package/src/observers/research-observer-impl.ts +14 -6
- package/src/orchestration/deep-research-orchestrator.ts +3 -1
- package/src/orchestration/quick-research-orchestrator.ts +29 -10
- package/src/orchestration/research-synthesis-service.ts +32 -0
- package/src/orchestration/researcher-executor.ts +31 -16
- package/src/orchestration/researcher.ts +2 -2
- package/src/orchestration/session-state.ts +35 -4
- package/src/prompts/research-knowledge-relevance-triage.md +2 -0
- package/src/prompts/research-knowledge-search-extractor.md +2 -0
- package/src/prompts/researcher.md +2 -14
- package/src/sdk.ts +16 -0
- package/src/skill-install/skill-installer.ts +5 -1
- package/src/stackexchange/index.ts +3 -0
- package/src/stackexchange/rest-client.ts +7 -12
- package/src/tools/index.ts +2 -1
- package/src/tools/research-knowledge-search.ts +37 -4
- package/src/tools/research-tool-definition.ts +54 -12
- package/src/tools/scrape.ts +8 -1
- package/src/tools/security.ts +13 -10
- package/src/utils/coverage-digest.ts +1 -1
- package/src/utils/host-config.ts +12 -1
- package/src/utils/log-utils.ts +15 -2
- package/src/utils/metrics-summary.ts +21 -1
- package/src/utils/tool-usage-tracker.ts +6 -3
- package/src/web-research/browser-search.ts +11 -4
- package/src/web-research/web-scraper.ts +36 -16
- package/src/youtube/index.ts +11 -1
- package/src/youtube/potoken.ts +8 -1
- package/src/youtube/transcript-client.ts +56 -5
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
name: pi-research
|
|
3
3
|
description: "Multi-agent web research for any coding agent. Use for anything requiring current information from the internet — news, trends, product/service/company facts, statistics, people, places, events, APIs, library behavior, or 'what is X / how does X work' questions. Also use to search a local knowledge store of previously researched findings before doing live research. Powered by the pi-research engine (search, stealth scraping, security databases, Stack Exchange). Not for reading local files, running project commands, or analyzing this repository's own source."
|
|
4
4
|
license: MIT
|
|
5
|
-
metadata: { "author": "Lincoln Deen", "version": "1.
|
|
5
|
+
metadata: { "author": "Lincoln Deen", "version": "1.5.0", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
|
|
6
6
|
allowed-tools: Bash(node:*)
|
|
7
7
|
---
|
|
8
8
|
|
package/src/cli.ts
CHANGED
|
@@ -150,7 +150,7 @@ function pretty(obj: unknown): string {
|
|
|
150
150
|
* lifecycle event to stderr. Keeps a long research run visible without spamming
|
|
151
151
|
* stdout (the report goes to stdout, untouched).
|
|
152
152
|
*/
|
|
153
|
-
function makeProgressObserver(): HeadlessObserverOptions {
|
|
153
|
+
export function makeProgressObserver(): HeadlessObserverOptions {
|
|
154
154
|
return {
|
|
155
155
|
enableLogging: false,
|
|
156
156
|
onProgress: (event: string, data?: any) => {
|
|
@@ -196,9 +196,24 @@ function makeProgressObserver(): HeadlessObserverOptions {
|
|
|
196
196
|
}
|
|
197
197
|
break;
|
|
198
198
|
}
|
|
199
|
-
case 'complete':
|
|
199
|
+
case 'complete': {
|
|
200
200
|
toStderr(` • complete (${data?.result?.length ?? 0} chars)\n`);
|
|
201
|
+
// Degradation signal for callers that read stderr (the agent skill
|
|
202
|
+
// does): a run that completes but is weakly grounded or retrieved no
|
|
203
|
+
// sources at all looks identical to a healthy run on exit code and
|
|
204
|
+
// the "complete" line alone. Two real runs on 2026-08-20 shipped
|
|
205
|
+
// degraded reports (1 citation / 0 sources) with nothing on stderr to
|
|
206
|
+
// distinguish them. The markers are the synthesis service's own
|
|
207
|
+
// notices, appended by ensureCitedLinks.
|
|
208
|
+
const reportText = typeof data?.result === 'string' ? data.result : '';
|
|
209
|
+
if (reportText.includes('No web sources were successfully retrieved') ||
|
|
210
|
+
reportText.includes('None of the links above could be verified')) {
|
|
211
|
+
toStderr(' • WARNING: no web sources could be retrieved — the report is unverified\n');
|
|
212
|
+
} else if (reportText.includes('Grounding notice: only')) {
|
|
213
|
+
toStderr(' • WARNING: weakly grounded report (fewer than 3 verified sources) — treat unattributed claims as unverified\n');
|
|
214
|
+
}
|
|
201
215
|
break;
|
|
216
|
+
}
|
|
202
217
|
case 'error':
|
|
203
218
|
toStderr(` • error: ${data?.message}\n`);
|
|
204
219
|
break;
|
|
@@ -738,7 +753,9 @@ export async function cmdKnowledge(queries: string[], json?: boolean): Promise<n
|
|
|
738
753
|
config: getConfig(process.cwd(), 'cli'),
|
|
739
754
|
ignoreGlobalConfig: true,
|
|
740
755
|
});
|
|
741
|
-
|
|
756
|
+
// "N queries", not "N query/s" — same rationale as the search_start line above:
|
|
757
|
+
// the slash form reads as a rate and renders "1 query/" in the singular.
|
|
758
|
+
toStderr(`[pi-research] searching knowledge store (${queries.length} ${queries.length === 1 ? 'query' : 'queries'})…\n`);
|
|
742
759
|
result = await searchKnowledge(queries, abortController.signal);
|
|
743
760
|
} catch (err) {
|
|
744
761
|
return reportError(err, 'knowledge search', json);
|
|
@@ -1660,10 +1677,34 @@ async function main(argv: string[]): Promise<number> {
|
|
|
1660
1677
|
// left the SDK's handler to tear the container down underneath a live orchestrator
|
|
1661
1678
|
// — exactly the race the CLI installs its own handlers to avoid.
|
|
1662
1679
|
process.on('SIGHUP', () => onSignal('SIGHUP'));
|
|
1680
|
+
// SIGQUIT: documented (see exitCodeForSignal's comment and the exit-code
|
|
1681
|
+
// tables in docs/SDK.md and SKILL.md) as handled the same way as SIGHUP,
|
|
1682
|
+
// but no listener was ever registered for it — an unhandled SIGQUIT hits
|
|
1683
|
+
// Node's default disposition (immediate termination, no abort, no drain),
|
|
1684
|
+
// silently breaking that documented guarantee. Has no Windows equivalent;
|
|
1685
|
+
// registering the listener there is a documented no-op, so no guard needed.
|
|
1686
|
+
process.on('SIGQUIT', () => onSignal('SIGQUIT'));
|
|
1663
1687
|
if (process.platform === 'win32') {
|
|
1664
1688
|
process.on('SIGBREAK', () => onSignal('SIGBREAK'));
|
|
1665
1689
|
}
|
|
1666
1690
|
|
|
1691
|
+
// Test-only seam: idle here, right after every handler above is registered,
|
|
1692
|
+
// instead of parsing argv/running a command. Lets a subprocess test deliver
|
|
1693
|
+
// a real OS signal and observe THIS handler run (process.exit with the
|
|
1694
|
+
// derived code) rather than Node's default disposition (process killed by
|
|
1695
|
+
// the signal, no code) — the exact distinction that would have caught the
|
|
1696
|
+
// missing SIGQUIT registration this seam was added for. Never set in
|
|
1697
|
+
// production; see the SIGQUIT subprocess test in test/unit/cli.test.ts.
|
|
1698
|
+
if (process.env['PI_RESEARCH_TEST_HANG_FOR_SIGNAL'] === '1') {
|
|
1699
|
+
process.stdout.write('PI_RESEARCH_TEST_READY\n');
|
|
1700
|
+
// A bare unresolved Promise does not keep the event loop alive by
|
|
1701
|
+
// itself (only real libuv handles do — an installed signal listener
|
|
1702
|
+
// isn't one either), so without a ref'd timer the process would exit 0
|
|
1703
|
+
// on its own the instant this synchronous section finishes, before any
|
|
1704
|
+
// signal could ever arrive.
|
|
1705
|
+
await new Promise<never>(() => setInterval(() => {}, 1000));
|
|
1706
|
+
}
|
|
1707
|
+
|
|
1667
1708
|
let parsed: ParsedArgs;
|
|
1668
1709
|
try {
|
|
1669
1710
|
parsed = parseArgs(argv);
|
package/src/config.ts
CHANGED
|
@@ -82,10 +82,16 @@ export const ConfigSchema = Type.Object({
|
|
|
82
82
|
SEARCH_TIMEOUT_MS: Type.Number({ minimum: 5000, maximum: 120000, default: 45000 }),
|
|
83
83
|
/** TUI refresh debounce in milliseconds (default: 100ms) */
|
|
84
84
|
TUI_REFRESH_DEBOUNCE_MS: Type.Number({ minimum: 0, maximum: 1000, default: 100 }),
|
|
85
|
-
/**
|
|
86
|
-
*
|
|
87
|
-
* SEARCH_TIMEOUT_MS + this, a scrape after SCRAPE_TIMEOUT_MS + this
|
|
88
|
-
*
|
|
85
|
+
/** Overhead margin (ms) added on top of each browser operation's own nav timeout to
|
|
86
|
+
* form its scheduler task-timeout ceiling: a search task times out after
|
|
87
|
+
* SEARCH_TIMEOUT_MS + this, a scrape after SCRAPE_TIMEOUT_MS + this — covering
|
|
88
|
+
* per-task work beyond raw navigation (page setup, result parsing). Queue wait is
|
|
89
|
+
* NOT part of this margin: the deadline is armed on dispatch, not on enqueue, so
|
|
90
|
+
* time spent waiting for a free worker never counts against it (see
|
|
91
|
+
* browser-task-scheduler.ts). Cold browser-launch/context-creation time is also
|
|
92
|
+
* separate from this margin — COLD_START_ALLOWANCE_MS
|
|
93
|
+
* (infrastructure/browser/config.ts) covers that unconditionally on top of it,
|
|
94
|
+
* since it isn't something a user should need to tune.
|
|
89
95
|
* (default: 10000) */
|
|
90
96
|
BROWSER_TASK_TIMEOUT_MS: Type.Number({ minimum: 2000, maximum: 120000, default: 10000 }),
|
|
91
97
|
/** Timeout for coordinator/research-lead/repair/knowledge LLM calls in ms (default: 300000 = 5 min, range: 60s-30min).
|
|
@@ -745,16 +751,23 @@ function loadEnvFiles(cwd: string, iface?: ConfigInterface): Record<string, stri
|
|
|
745
751
|
try {
|
|
746
752
|
logger.warn(`[config] WARNING: .pi-research.env files are deprecated. Settings will be auto-migrated to the centralized registry. Remove ${legacyPath} after migration.`);
|
|
747
753
|
legacyEnv = parseDotEnv(fs.readFileSync(legacyPath, 'utf-8'));
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
//
|
|
751
|
-
//
|
|
752
|
-
//
|
|
753
|
-
//
|
|
754
|
-
//
|
|
754
|
+
// Only LOCAL_SCOPE_KEYS from this run's resolution too — same boundary
|
|
755
|
+
// step 4 below already enforces for the registry this file migrates
|
|
756
|
+
// into ("the registry is project-scoped storage... never user-scoped
|
|
757
|
+
// keys"). This file lives in the CWD, which for this tool is routinely
|
|
758
|
+
// an arbitrary, potentially untrusted repository a coding agent was
|
|
759
|
+
// just pointed at — applying every key unconditionally let a dropped-in
|
|
760
|
+
// .pi-research.env silently control user-scoped settings (RESEARCH_MODEL,
|
|
761
|
+
// EMBEDDING_DEVICE, DISABLED_TOOLS, export destinations, ...) for that
|
|
762
|
+
// run with no confirmation, for any setting the user hadn't already set
|
|
763
|
+
// via a real environment variable (which still wins over all of this).
|
|
764
|
+
// Non-local keys are ignored here exactly as they already are for the
|
|
765
|
+
// registry — this predecessor mechanism should not be MORE permissive
|
|
766
|
+
// than the successor it's migrating into.
|
|
755
767
|
const legacyLocal = Object.fromEntries(
|
|
756
768
|
Object.entries(legacyEnv).filter(([k]) => LOCAL_SCOPE_KEYS.has(k))
|
|
757
769
|
);
|
|
770
|
+
Object.assign(merged, legacyLocal);
|
|
758
771
|
const existingLocal = Object.fromEntries(
|
|
759
772
|
Object.entries(findRegistryEntry(registry, normalizedCwd) ?? {}).filter(([k]) => LOCAL_SCOPE_KEYS.has(k))
|
|
760
773
|
);
|
|
@@ -793,8 +806,14 @@ function loadEnvFiles(cwd: string, iface?: ConfigInterface): Record<string, stri
|
|
|
793
806
|
}
|
|
794
807
|
}
|
|
795
808
|
} else if (Object.keys(merged).length === 0 && !fs.existsSync(legacyPath) && !fs.existsSync(globalPath)) {
|
|
796
|
-
// 5. Warning for missing config
|
|
797
|
-
|
|
809
|
+
// 5. Warning for missing config — once per workspace per process. getConfig
|
|
810
|
+
// is called on many hot paths (every tool invocation, health checks), and
|
|
811
|
+
// repeating this advisory each time made it the single most common line in
|
|
812
|
+
// the shared log (362 entries in one observed day), burying real warnings.
|
|
813
|
+
if (!warnedMissingConfigWorkspaces.has(normalizedCwd)) {
|
|
814
|
+
warnedMissingConfigWorkspaces.add(normalizedCwd);
|
|
815
|
+
logger.warn(`[config] No configuration found for workspace: ${cwd}. Using code defaults. Run /research-config to configure.`);
|
|
816
|
+
}
|
|
798
817
|
}
|
|
799
818
|
|
|
800
819
|
warnLegacyEnvVarsOnce(merged);
|
|
@@ -819,6 +838,10 @@ const LEGACY_ENV_VARS: Record<string, string> = {
|
|
|
819
838
|
|
|
820
839
|
let warnedLegacyEnvVars = false;
|
|
821
840
|
|
|
841
|
+
// Workspaces already warned about having no configuration (see loadEnvFiles) —
|
|
842
|
+
// per-process dedup so the advisory appears once, not on every getConfig call.
|
|
843
|
+
const warnedMissingConfigWorkspaces = new Set<string>();
|
|
844
|
+
|
|
822
845
|
/** Warn once per process about set-but-dead 0.1.x env vars (upgrade aid; never throws). */
|
|
823
846
|
function warnLegacyEnvVarsOnce(merged: Record<string, string>): void {
|
|
824
847
|
if (warnedLegacyEnvVars) return;
|
package/src/constants.ts
CHANGED
|
@@ -46,11 +46,26 @@ export function getMaxGatheringCalls(config?: Config): number {
|
|
|
46
46
|
/**
|
|
47
47
|
* Get the maximum scrape batches from config.
|
|
48
48
|
* This function should be used instead of MAX_SCRAPE_CALLS to support dynamic configuration.
|
|
49
|
+
*
|
|
50
|
+
* @param cachingActive - Pass `true` when prompt caching is active for this
|
|
51
|
+
* run's resolved model (see isPromptCachingActiveForModel in
|
|
52
|
+
* core/llm/research-model-resolver.ts). With caching active, the EFFECTIVE
|
|
53
|
+
* maximum is the configured value plus one: a cached run's system-prompt/
|
|
54
|
+
* tool-definition prefix is cheap to re-send, so the same token budget
|
|
55
|
+
* covers one more scrape batch per researcher. Every consumer of the batch
|
|
56
|
+
* count (tracker enforcement, prompt text, progress units) must go through
|
|
57
|
+
* this function with the same cachingActive value so they all agree on the
|
|
58
|
+
* effective number. The "unlimited" sentinel (0 or >99) is unchanged —
|
|
59
|
+
* adding one to unlimited is meaningless.
|
|
49
60
|
*/
|
|
50
|
-
export function getMaxScrapeBatches(config?: Config): number {
|
|
61
|
+
export function getMaxScrapeBatches(config?: Config, cachingActive = false): number {
|
|
51
62
|
try {
|
|
52
63
|
const batches = (config || getConfig()).MAX_SCRAPE_BATCHES;
|
|
53
|
-
|
|
64
|
+
if (batches === 0 || batches > 99) return 999999;
|
|
65
|
+
// Cap at 99: a configured 99 (the documented max) plus one would land in the
|
|
66
|
+
// >99 "unlimited" band that every renderer checks — the prompt would say
|
|
67
|
+
// "unlimited batches" while enforcement actually blocked at 100.
|
|
68
|
+
return cachingActive ? Math.min(batches + 1, 99) : batches;
|
|
54
69
|
} catch {
|
|
55
70
|
return 2; // Fallback to default
|
|
56
71
|
}
|
|
@@ -63,9 +78,13 @@ export const MAX_SCRAPE_URLS = 6;
|
|
|
63
78
|
* Get the units per researcher for the progress bar.
|
|
64
79
|
* Units = 1 (for start/search) + number of scrape batches.
|
|
65
80
|
* When batches are unlimited (MAX_SCRAPE_BATCHES=0), caps at 4 for a sane progress estimate.
|
|
81
|
+
*
|
|
82
|
+
* Relative to the EFFECTIVE batch count: pass the run's cachingActive (from
|
|
83
|
+
* isPromptCachingActiveForModel over the resolved model) so progress units
|
|
84
|
+
* agree with the limit the tracker actually enforces.
|
|
66
85
|
*/
|
|
67
|
-
export function getUnitsPerResearcher(config?: Config): number {
|
|
68
|
-
const batches = getMaxScrapeBatches(config);
|
|
86
|
+
export function getUnitsPerResearcher(config?: Config, cachingActive = false): number {
|
|
87
|
+
const batches = getMaxScrapeBatches(config, cachingActive);
|
|
69
88
|
return 1 + Math.min(batches, 4);
|
|
70
89
|
}
|
|
71
90
|
|
|
@@ -89,3 +89,41 @@ export function resolveResearchModel(options: {
|
|
|
89
89
|
|
|
90
90
|
throw new Error(buildNoModelAvailableMessage());
|
|
91
91
|
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Best-effort, CONSERVATIVE signal for "will pi-ai apply prompt caching for
|
|
95
|
+
* this resolved model." pi-ai owns all cache-mechanism logic internally (this
|
|
96
|
+
* codebase never constructs cache_control blocks itself) and exposes no
|
|
97
|
+
* public API to ask in advance for most providers, so this can only answer
|
|
98
|
+
* for the two client-visible cases:
|
|
99
|
+
*
|
|
100
|
+
* - `api === 'anthropic-messages'`: caching is native and on by default (no
|
|
101
|
+
* capability gate at all — see pi-ai's anthropic-messages provider).
|
|
102
|
+
* - `api === 'openai-completions'` with `compat.cacheControlFormat ===
|
|
103
|
+
* 'anthropic'`: an explicit opt-in to Anthropic-style cache_control
|
|
104
|
+
* emulation (e.g. an OpenRouter/gateway route to an Anthropic model
|
|
105
|
+
* configured with this override).
|
|
106
|
+
*
|
|
107
|
+
* Every other case returns false even though caching may actually be
|
|
108
|
+
* active: pi's generated model catalog sets `cacheControlFormat` on zero
|
|
109
|
+
* models, so plain `openai-completions` traffic (this project's custom
|
|
110
|
+
* providers — glm-coding, cerebras, zai, OpenRouter, ...) has its cache
|
|
111
|
+
* compat auto-detected by pi-ai's own private, unexported heuristic INSIDE
|
|
112
|
+
* the live request — not knowable client-side, before or after model
|
|
113
|
+
* resolution. Providers with fully implicit/automatic caching
|
|
114
|
+
* (openai-responses family, google-generative-ai, google-vertex,
|
|
115
|
+
* mistral-conversations, bedrock-converse-stream) have no "will it be
|
|
116
|
+
* active" signal at all — it depends on provider-side runtime behavior
|
|
117
|
+
* (minimum prompt length, server cache state). Treat this as "caching is
|
|
118
|
+
* DEFINITELY active" — a false negative is expected and fine for providers
|
|
119
|
+
* outside these two cases, but a false positive would be a real bug.
|
|
120
|
+
*/
|
|
121
|
+
export function isPromptCachingActiveForModel(model: Model<any> | undefined): boolean {
|
|
122
|
+
if (!model) return false;
|
|
123
|
+
if (model.api === 'anthropic-messages') return true;
|
|
124
|
+
if (model.api === 'openai-completions') {
|
|
125
|
+
const compat = model.compat as { cacheControlFormat?: string } | undefined;
|
|
126
|
+
return compat?.cacheControlFormat === 'anthropic';
|
|
127
|
+
}
|
|
128
|
+
return false;
|
|
129
|
+
}
|
|
@@ -220,8 +220,19 @@ function isDegradableLlmError(error: unknown, signal?: AbortSignal): boolean {
|
|
|
220
220
|
export function salvageReportText(raw: string): string {
|
|
221
221
|
const trimmed = raw.trim();
|
|
222
222
|
if (trimmed.length <= 50) return '';
|
|
223
|
+
// Envelope sniff. Models routinely wrap the JSON envelope in a markdown code
|
|
224
|
+
// fence (```json … ```); a fenced envelope starts with backticks, not '{', so a
|
|
225
|
+
// bare startsWith('{') check shipped one verbatim as a user-facing report
|
|
226
|
+
// (observed in production 2026-08-20: a ```json-fenced {"action":"synthesize",…}
|
|
227
|
+
// blob, internal protocol fields included, published as the final report).
|
|
228
|
+
// Strip one leading fence line and a trailing fence before the shape check.
|
|
229
|
+
let sniff = trimmed;
|
|
230
|
+
const fence = sniff.match(/^```[a-zA-Z0-9_-]*[ \t]*\r?\n/);
|
|
231
|
+
if (fence) {
|
|
232
|
+
sniff = sniff.slice(fence[0].length).replace(/\r?\n```\s*$/, '').trim();
|
|
233
|
+
}
|
|
223
234
|
// Looks like a (broken) JSON envelope rather than a report — suppress it.
|
|
224
|
-
if (
|
|
235
|
+
if (sniff.startsWith('{') && /"(?:action|content|researchers)"\s*:/.test(sniff)) {
|
|
225
236
|
return '';
|
|
226
237
|
}
|
|
227
238
|
return raw;
|
|
@@ -717,14 +728,38 @@ export class PlanningService implements IPlanningService {
|
|
|
717
728
|
};
|
|
718
729
|
|
|
719
730
|
const responseText = isRouter
|
|
720
|
-
? await
|
|
721
|
-
...
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
731
|
+
? await (() => {
|
|
732
|
+
const routerMessage = buildRouterMessage(...this.buildRouterEvidence(options, round), runContext);
|
|
733
|
+
// The router carries no reduce mechanism the way synthesizeCorpus does — a
|
|
734
|
+
// routing decision made from PARTITIONED evidence isn't naturally mergeable
|
|
735
|
+
// the way synthesis prose is, and this call's output is a structured JSON
|
|
736
|
+
// delegate plan, not free text. Its input is bounded by team size in the
|
|
737
|
+
// common case (digests for earlier rounds are already capped — see
|
|
738
|
+
// MAX_DIGEST_CHARS), but nothing caps an individual researcher's fresh report
|
|
739
|
+
// body, so a round with a few unusually long reports can still overflow the
|
|
740
|
+
// window with no reduce path to fall back on. Warn rather than fail silently;
|
|
741
|
+
// the provider's own rejection is still the actual backstop, same as the
|
|
742
|
+
// synthesizer's over-budget-after-every-pass case above.
|
|
743
|
+
const window = model.contextWindow ?? DEFAULT_MODEL_CONTEXT_WINDOW;
|
|
744
|
+
const estimatedTokens = routerMessage.length / CHARS_PER_TOKEN + ROUTER_MAX_TOKENS + SYNTHESIS_OVERHEAD_TOKENS;
|
|
745
|
+
if (estimatedTokens > window) {
|
|
746
|
+
logger.warn(
|
|
747
|
+
`[PlanningService] Router request for ${model.id} is an estimated ` +
|
|
748
|
+
`${Math.round(estimatedTokens)} tokens against a ${window}-token context window ` +
|
|
749
|
+
`(evidence ${routerMessage.length} chars, output ceiling ${ROUTER_MAX_TOKENS} tokens). ` +
|
|
750
|
+
`Unlike the synthesizer, the router has no reduce path for an over-budget round — ` +
|
|
751
|
+
`sending it anyway, the provider may reject this request.`,
|
|
752
|
+
);
|
|
753
|
+
}
|
|
754
|
+
return this.callLead({
|
|
755
|
+
...callOptions,
|
|
756
|
+
systemPrompt: populate(''),
|
|
757
|
+
userMessage: routerMessage,
|
|
758
|
+
maxTokens: ROUTER_MAX_TOKENS,
|
|
759
|
+
component: 'router',
|
|
760
|
+
label: 'router-updatePlanForRound',
|
|
761
|
+
});
|
|
762
|
+
})()
|
|
728
763
|
: await this.synthesizeCorpus({ ...callOptions, reports, runContext, populate });
|
|
729
764
|
|
|
730
765
|
// Extract and validate JSON
|
|
@@ -1115,6 +1150,29 @@ export class PlanningService implements IPlanningService {
|
|
|
1115
1150
|
);
|
|
1116
1151
|
}
|
|
1117
1152
|
|
|
1153
|
+
// budgetChars alone can UNDER-report overflow risk once it hits MIN_SYNTHESIS_CORPUS_CHARS:
|
|
1154
|
+
// below that point it is a fixed floor, not a value derived from the window, so a reduced
|
|
1155
|
+
// corpus that fits comfortably under the floor can still combine with a large global source
|
|
1156
|
+
// list and/or run context (neither bounded, both already subtracted from budgetChars but not
|
|
1157
|
+
// from THIS check) to overflow the real window — the same failure class the budget exists to
|
|
1158
|
+
// prevent, just triggered by those two inputs growing instead of the corpus. Check the actual
|
|
1159
|
+
// total request against the actual window directly, independent of any floor.
|
|
1160
|
+
const window = args.model.contextWindow ?? DEFAULT_MODEL_CONTEXT_WINDOW;
|
|
1161
|
+
const estimatedTotalTokens =
|
|
1162
|
+
(finalSize + globalSourceList.length + runContext.length) / CHARS_PER_TOKEN +
|
|
1163
|
+
config.SYNTHESIS_MAX_TOKENS + SYNTHESIS_OVERHEAD_TOKENS;
|
|
1164
|
+
if (estimatedTotalTokens > window) {
|
|
1165
|
+
logger.warn(
|
|
1166
|
+
`[PlanningService] Synthesis request for ${args.model.id} is an estimated ` +
|
|
1167
|
+
`${Math.round(estimatedTotalTokens)} tokens against a ${window}-token context window ` +
|
|
1168
|
+
`(corpus ${finalSize} chars, source list ${globalSourceList.length} chars, run context ` +
|
|
1169
|
+
`${runContext.length} chars, output ceiling ${config.SYNTHESIS_MAX_TOKENS} tokens) — the ` +
|
|
1170
|
+
`source list and/or run context are large enough on their own to overflow the window even ` +
|
|
1171
|
+
`though the reduced corpus fits its budget. Sending it anyway — the provider may reject ` +
|
|
1172
|
+
`this request.`,
|
|
1173
|
+
);
|
|
1174
|
+
}
|
|
1175
|
+
|
|
1118
1176
|
return this.callLead({
|
|
1119
1177
|
...args,
|
|
1120
1178
|
systemPrompt: populate(reduced ? MERGE_SYNTHESIS_SECTION : ''),
|
package/src/index.ts
CHANGED
|
@@ -556,7 +556,9 @@ export default async function (pi: ExtensionAPI) {
|
|
|
556
556
|
|
|
557
557
|
// Researcher sub-agent sessions already carry the authoritative steering copy:
|
|
558
558
|
// the executor (deep) and quick orchestrator inject "ADDITIONAL USER GUIDANCE"/
|
|
559
|
-
// "ADDITIONAL CONSIDERATIONS" into the researcher
|
|
559
|
+
// "ADDITIONAL CONSIDERATIONS" into the researcher's initial user message themselves
|
|
560
|
+
// (kept OUT of the system prompt so its cacheable prefix stays byte-identical
|
|
561
|
+
// across researchers — see researcher-executor.ts).
|
|
560
562
|
// Appending the same messages again here delivered every steering message
|
|
561
563
|
// TWICE to every researcher — return the prompt untouched. (The coordinator and
|
|
562
564
|
// the research lead never pass through this hook at all: they run via
|
|
@@ -23,7 +23,7 @@ import type { IScheduler, TaskDispatchListener } from '../../core/interfaces/sch
|
|
|
23
23
|
import { cleanupOrphanedCamoufoxProcesses, getBrowserPidsForWorkers, killBrowserProcesses } from './browser-cleanup.ts';
|
|
24
24
|
import { raceWithDeadline } from '../../utils/safe-unref.ts';
|
|
25
25
|
import { PriorityTaskQueue } from './priority-task-queue.ts';
|
|
26
|
-
import { getHealthCheckBudgetMs } from './config.ts';
|
|
26
|
+
import { getHealthCheckBudgetMs, COLD_START_ALLOWANCE_MS } from './config.ts';
|
|
27
27
|
|
|
28
28
|
/**
|
|
29
29
|
* Grace added to the worker's own task budget before the dead-worker backstop
|
|
@@ -273,12 +273,17 @@ export class BrowserTaskScheduler implements IScheduler {
|
|
|
273
273
|
const pool = await (await this.getWorkerPoolManager()).ensurePool(config);
|
|
274
274
|
const startTime = Date.now();
|
|
275
275
|
|
|
276
|
-
// Task timeout = the search nav budget (SEARCH_TIMEOUT_MS) +
|
|
277
|
-
//
|
|
278
|
-
//
|
|
279
|
-
//
|
|
276
|
+
// Task timeout = the search nav budget (SEARCH_TIMEOUT_MS) + an overhead margin
|
|
277
|
+
// (BROWSER_TASK_TIMEOUT_MS) + a cold-start allowance (COLD_START_ALLOWANCE_MS).
|
|
278
|
+
// Deriving from SEARCH_TIMEOUT_MS keeps the task ceiling coherent with the
|
|
279
|
+
// worker's own nav timeout — using a flat base here would kill a search before
|
|
280
|
+
// it had used its full nav budget. The cold-start term matters because eager
|
|
281
|
+
// browser warmup is deliberately disabled: the first search dispatched to a
|
|
282
|
+
// freshly created or just-reset worker pays a real Camoufox launch + context
|
|
283
|
+
// creation inline, before any navigation even starts — without this term that
|
|
284
|
+
// legitimate cold start reads as a dead/wedged worker (see COLD_START_ALLOWANCE_MS).
|
|
280
285
|
const cfg = config || getConfig();
|
|
281
|
-
const timeoutMs = cfg.SEARCH_TIMEOUT_MS + cfg.BROWSER_TASK_TIMEOUT_MS;
|
|
286
|
+
const timeoutMs = cfg.SEARCH_TIMEOUT_MS + cfg.BROWSER_TASK_TIMEOUT_MS + COLD_START_ALLOWANCE_MS;
|
|
282
287
|
|
|
283
288
|
// The deadline measures EXECUTION, not queue wait: the timer is armed by
|
|
284
289
|
// `startDeadline()` from inside the enqueued callback, i.e. once the queue has
|
|
@@ -413,11 +418,13 @@ export class BrowserTaskScheduler implements IScheduler {
|
|
|
413
418
|
this.resetIdleTimer();
|
|
414
419
|
const pool = await (await this.getWorkerPoolManager()).ensurePool(config);
|
|
415
420
|
const startTime = Date.now();
|
|
416
|
-
// Task timeout = the scrape nav budget (SCRAPE_TIMEOUT_MS) +
|
|
417
|
-
//
|
|
421
|
+
// Task timeout = the scrape nav budget (SCRAPE_TIMEOUT_MS) + an overhead margin,
|
|
422
|
+
// mirroring the search path so both stay coherent. The mocked branch skips the
|
|
423
|
+
// cold-start allowance deliberately: PI_RESEARCH_MOCK_SCRAPE implies no real
|
|
424
|
+
// Camoufox launch is expected for scrape tasks, so there is nothing to budget for.
|
|
418
425
|
const cfg = config || getConfig();
|
|
419
426
|
const isMocking = process.env['PI_RESEARCH_MOCK_SCRAPE'] === 'true';
|
|
420
|
-
const timeoutMs = cfg.SCRAPE_TIMEOUT_MS + (isMocking ? 5000 : cfg.BROWSER_TASK_TIMEOUT_MS);
|
|
427
|
+
const timeoutMs = cfg.SCRAPE_TIMEOUT_MS + (isMocking ? 5000 : cfg.BROWSER_TASK_TIMEOUT_MS + COLD_START_ALLOWANCE_MS);
|
|
421
428
|
|
|
422
429
|
// Armed on dispatch, not on enqueue — see runSearch for why.
|
|
423
430
|
// See runSearch: dispatch time is tracked for measurement, not for the deadline.
|
|
@@ -346,13 +346,50 @@ function isFullMockMode(): boolean {
|
|
|
346
346
|
* and a budget below that sum makes the fallback ladder unreachable — so raising
|
|
347
347
|
* HEALTH_CHECK_TIMEOUT_MS used to make the health check strictly worse.
|
|
348
348
|
*
|
|
349
|
-
* Floored at the historical 105s
|
|
349
|
+
* Floored at the historical 105s, kept as a floor rather than a default: the
|
|
350
|
+
* derived value now exceeds it even at default timeouts once COLD_START_ALLOWANCE_MS
|
|
351
|
+
* (below) is folded into `longestSlotHold` — a cold Firefox launch was always able to
|
|
352
|
+
* hold a slot for close to that long, the 105s floor just didn't know it. The floor
|
|
353
|
+
* still matters for a hypothetical config where the derived value would otherwise land
|
|
354
|
+
* below it.
|
|
350
355
|
*/
|
|
351
356
|
export const HEALTHCHECK_MIN_BUDGET_MS = 45_000 + 60_000;
|
|
352
357
|
|
|
353
358
|
/** Navigation attempts executeHealthCheck makes: primary, retry, fallback endpoint. */
|
|
354
359
|
const HEALTHCHECK_NAV_ATTEMPTS = 3;
|
|
355
360
|
|
|
361
|
+
/**
|
|
362
|
+
* Hard ceiling on a single Camoufox browser launch (see launchOnce() in
|
|
363
|
+
* thread-worker-browser.ts, the single source of truth this mirrors — CI runners
|
|
364
|
+
* (2 vCPU) can take up to ~60s to start Firefox under load, floored generously
|
|
365
|
+
* above that).
|
|
366
|
+
*/
|
|
367
|
+
export const BROWSER_LAUNCH_TIMEOUT_MS = 90_000;
|
|
368
|
+
|
|
369
|
+
/**
|
|
370
|
+
* Hard ceiling on creating one BrowserContext against an already-launched browser
|
|
371
|
+
* (see acquireTaskContext() in thread-worker-browser.ts, the single source of
|
|
372
|
+
* truth this mirrors).
|
|
373
|
+
*/
|
|
374
|
+
export const CONTEXT_CREATION_TIMEOUT_MS = 30_000;
|
|
375
|
+
|
|
376
|
+
/**
|
|
377
|
+
* Worst-case time a worker can spend getting a browser+context ready before it
|
|
378
|
+
* starts a task's actual work. Eager warmup is deliberately disabled (thundering-
|
|
379
|
+
* herd avoidance — see thread-worker.ts), so the first task dispatched to a
|
|
380
|
+
* freshly created or just-reset worker pays this in full, inline with the task
|
|
381
|
+
* itself. Every deadline that bounds how long a task may legitimately hold a
|
|
382
|
+
* worker slot has to include this on top of its own nav budget, or a completely
|
|
383
|
+
* normal cold start on a fresh pool reads as a dead/wedged worker — the false
|
|
384
|
+
* positive `runSearch`/`runScrape`'s task-timeout ceiling and this budget were
|
|
385
|
+
* both built to eliminate, just triggered by a different clock.
|
|
386
|
+
*
|
|
387
|
+
* Deliberately NOT part of BROWSER_TASK_TIMEOUT_MS: that value is user-tunable
|
|
388
|
+
* overhead for per-task work, and cold-start cost isn't something a user should
|
|
389
|
+
* need to account for by hand when adjusting it.
|
|
390
|
+
*/
|
|
391
|
+
export const COLD_START_ALLOWANCE_MS = BROWSER_LAUNCH_TIMEOUT_MS + CONTEXT_CREATION_TIMEOUT_MS;
|
|
392
|
+
|
|
356
393
|
export function getHealthCheckBudgetMs(config?: Config): number {
|
|
357
394
|
const c = (config || getConfig()) as Partial<Config>;
|
|
358
395
|
// Each field is read defensively. A missing or non-finite value would otherwise
|
|
@@ -363,7 +400,8 @@ export function getHealthCheckBudgetMs(config?: Config): number {
|
|
|
363
400
|
typeof value === 'number' && Number.isFinite(value) ? value : fallback;
|
|
364
401
|
const longestSlotHold =
|
|
365
402
|
Math.max(ms(c.SEARCH_TIMEOUT_MS, 45_000), ms(c.SCRAPE_TIMEOUT_MS, 15_000))
|
|
366
|
-
+ ms(c.BROWSER_TASK_TIMEOUT_MS, 10_000)
|
|
403
|
+
+ ms(c.BROWSER_TASK_TIMEOUT_MS, 10_000)
|
|
404
|
+
+ COLD_START_ALLOWANCE_MS;
|
|
367
405
|
// Mirrors the worker's own floor in executeHealthCheck.
|
|
368
406
|
const probeWork = HEALTHCHECK_NAV_ATTEMPTS * Math.max(10_000, ms(c.HEALTH_CHECK_TIMEOUT_MS, 10_000));
|
|
369
407
|
return Math.max(HEALTHCHECK_MIN_BUDGET_MS, longestSlotHold + probeWork);
|