@lincoln504/pi-research 1.4.1 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/README.md +1 -1
  2. package/dist/cli.mjs +449 -187
  3. package/dist/prompts/research-knowledge-relevance-triage.md +2 -0
  4. package/dist/prompts/research-knowledge-search-extractor.md +2 -0
  5. package/dist/prompts/researcher.md +2 -14
  6. package/dist/thread-worker.mjs +91 -59
  7. package/docs/AGENT-SKILL.md +4 -3
  8. package/docs/CHANGELOG.md +120 -0
  9. package/docs/CONFIGURATION.md +2 -2
  10. package/docs/SDK.md +1 -1
  11. package/package.json +1 -1
  12. package/skills/pi-research/SKILL.md +1 -1
  13. package/src/cli.ts +44 -3
  14. package/src/config.ts +36 -13
  15. package/src/constants.ts +23 -4
  16. package/src/core/llm/research-model-resolver.ts +38 -0
  17. package/src/core/planning-service.ts +67 -9
  18. package/src/index.ts +3 -1
  19. package/src/infrastructure/browser/browser-task-scheduler.ts +16 -9
  20. package/src/infrastructure/browser/config.ts +40 -2
  21. package/src/infrastructure/browser/thread-worker-browser.ts +69 -48
  22. package/src/infrastructure/browser/thread-worker-messaging.ts +65 -4
  23. package/src/infrastructure/browser/thread-worker.mjs +91 -59
  24. package/src/infrastructure/browser/thread-worker.ts +13 -5
  25. package/src/infrastructure/embedding/embedding-server.ts +54 -0
  26. package/src/knowledge/embedder-init.ts +3 -2
  27. package/src/knowledge/embedder.ts +11 -3
  28. package/src/knowledge/index.ts +36 -10
  29. package/src/knowledge/store.ts +85 -5
  30. package/src/observers/research-observer-impl.ts +14 -6
  31. package/src/orchestration/deep-research-orchestrator.ts +3 -1
  32. package/src/orchestration/quick-research-orchestrator.ts +29 -10
  33. package/src/orchestration/research-synthesis-service.ts +32 -0
  34. package/src/orchestration/researcher-executor.ts +31 -16
  35. package/src/orchestration/researcher.ts +2 -2
  36. package/src/orchestration/session-state.ts +35 -4
  37. package/src/prompts/research-knowledge-relevance-triage.md +2 -0
  38. package/src/prompts/research-knowledge-search-extractor.md +2 -0
  39. package/src/prompts/researcher.md +2 -14
  40. package/src/sdk.ts +16 -0
  41. package/src/skill-install/skill-installer.ts +5 -1
  42. package/src/stackexchange/index.ts +3 -0
  43. package/src/stackexchange/rest-client.ts +7 -12
  44. package/src/tools/index.ts +2 -1
  45. package/src/tools/research-knowledge-search.ts +37 -4
  46. package/src/tools/research-tool-definition.ts +54 -12
  47. package/src/tools/scrape.ts +8 -1
  48. package/src/tools/security.ts +13 -10
  49. package/src/utils/coverage-digest.ts +1 -1
  50. package/src/utils/host-config.ts +12 -1
  51. package/src/utils/log-utils.ts +15 -2
  52. package/src/utils/metrics-summary.ts +21 -1
  53. package/src/utils/tool-usage-tracker.ts +6 -3
  54. package/src/web-research/browser-search.ts +11 -4
  55. package/src/web-research/web-scraper.ts +36 -16
  56. package/src/youtube/index.ts +11 -1
  57. package/src/youtube/potoken.ts +8 -1
  58. package/src/youtube/transcript-client.ts +56 -5
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@lincoln504/pi-research",
3
- "version": "1.4.1",
3
+ "version": "1.5.0",
4
4
  "description": "Web research for your agents with smart and safe tooling + knowledge store",
5
5
  "type": "module",
6
6
  "private": false,
@@ -2,7 +2,7 @@
2
2
  name: pi-research
3
3
  description: "Multi-agent web research for any coding agent. Use for anything requiring current information from the internet — news, trends, product/service/company facts, statistics, people, places, events, APIs, library behavior, or 'what is X / how does X work' questions. Also use to search a local knowledge store of previously researched findings before doing live research. Powered by the pi-research engine (search, stealth scraping, security databases, Stack Exchange). Not for reading local files, running project commands, or analyzing this repository's own source."
4
4
  license: MIT
5
- metadata: { "author": "Lincoln Deen", "version": "1.4.1", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
5
+ metadata: { "author": "Lincoln Deen", "version": "1.5.0", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
6
6
  allowed-tools: Bash(node:*)
7
7
  ---
8
8
 
package/src/cli.ts CHANGED
@@ -150,7 +150,7 @@ function pretty(obj: unknown): string {
150
150
  * lifecycle event to stderr. Keeps a long research run visible without spamming
151
151
  * stdout (the report goes to stdout, untouched).
152
152
  */
153
- function makeProgressObserver(): HeadlessObserverOptions {
153
+ export function makeProgressObserver(): HeadlessObserverOptions {
154
154
  return {
155
155
  enableLogging: false,
156
156
  onProgress: (event: string, data?: any) => {
@@ -196,9 +196,24 @@ function makeProgressObserver(): HeadlessObserverOptions {
196
196
  }
197
197
  break;
198
198
  }
199
- case 'complete':
199
+ case 'complete': {
200
200
  toStderr(` • complete (${data?.result?.length ?? 0} chars)\n`);
201
+ // Degradation signal for callers that read stderr (the agent skill
202
+ // does): a run that completes but is weakly grounded or retrieved no
203
+ // sources at all looks identical to a healthy run on exit code and
204
+ // the "complete" line alone. Two real runs on 2026-08-20 shipped
205
+ // degraded reports (1 citation / 0 sources) with nothing on stderr to
206
+ // distinguish them. The markers are the synthesis service's own
207
+ // notices, appended by ensureCitedLinks.
208
+ const reportText = typeof data?.result === 'string' ? data.result : '';
209
+ if (reportText.includes('No web sources were successfully retrieved') ||
210
+ reportText.includes('None of the links above could be verified')) {
211
+ toStderr(' • WARNING: no web sources could be retrieved — the report is unverified\n');
212
+ } else if (reportText.includes('Grounding notice: only')) {
213
+ toStderr(' • WARNING: weakly grounded report (fewer than 3 verified sources) — treat unattributed claims as unverified\n');
214
+ }
201
215
  break;
216
+ }
202
217
  case 'error':
203
218
  toStderr(` • error: ${data?.message}\n`);
204
219
  break;
@@ -738,7 +753,9 @@ export async function cmdKnowledge(queries: string[], json?: boolean): Promise<n
738
753
  config: getConfig(process.cwd(), 'cli'),
739
754
  ignoreGlobalConfig: true,
740
755
  });
741
- toStderr(`[pi-research] searching knowledge store (${queries.length} query/${queries.length === 1 ? '' : 's'})…\n`);
756
+ // "N queries", not "N query/s" — same rationale as the search_start line above:
757
+ // the slash form reads as a rate and renders "1 query/" in the singular.
758
+ toStderr(`[pi-research] searching knowledge store (${queries.length} ${queries.length === 1 ? 'query' : 'queries'})…\n`);
742
759
  result = await searchKnowledge(queries, abortController.signal);
743
760
  } catch (err) {
744
761
  return reportError(err, 'knowledge search', json);
@@ -1660,10 +1677,34 @@ async function main(argv: string[]): Promise<number> {
1660
1677
  // left the SDK's handler to tear the container down underneath a live orchestrator
1661
1678
  // — exactly the race the CLI installs its own handlers to avoid.
1662
1679
  process.on('SIGHUP', () => onSignal('SIGHUP'));
1680
+ // SIGQUIT: documented (see exitCodeForSignal's comment and the exit-code
1681
+ // tables in docs/SDK.md and SKILL.md) as handled the same way as SIGHUP,
1682
+ // but no listener was ever registered for it — an unhandled SIGQUIT hits
1683
+ // Node's default disposition (immediate termination, no abort, no drain),
1684
+ // silently breaking that documented guarantee. Has no Windows equivalent;
1685
+ // registering the listener there is a documented no-op, so no guard needed.
1686
+ process.on('SIGQUIT', () => onSignal('SIGQUIT'));
1663
1687
  if (process.platform === 'win32') {
1664
1688
  process.on('SIGBREAK', () => onSignal('SIGBREAK'));
1665
1689
  }
1666
1690
 
1691
+ // Test-only seam: idle here, right after every handler above is registered,
1692
+ // instead of parsing argv/running a command. Lets a subprocess test deliver
1693
+ // a real OS signal and observe THIS handler run (process.exit with the
1694
+ // derived code) rather than Node's default disposition (process killed by
1695
+ // the signal, no code) — the exact distinction that would have caught the
1696
+ // missing SIGQUIT registration this seam was added for. Never set in
1697
+ // production; see the SIGQUIT subprocess test in test/unit/cli.test.ts.
1698
+ if (process.env['PI_RESEARCH_TEST_HANG_FOR_SIGNAL'] === '1') {
1699
+ process.stdout.write('PI_RESEARCH_TEST_READY\n');
1700
+ // A bare unresolved Promise does not keep the event loop alive by
1701
+ // itself (only real libuv handles do — an installed signal listener
1702
+ // isn't one either), so without a ref'd timer the process would exit 0
1703
+ // on its own the instant this synchronous section finishes, before any
1704
+ // signal could ever arrive.
1705
+ await new Promise<never>(() => setInterval(() => {}, 1000));
1706
+ }
1707
+
1667
1708
  let parsed: ParsedArgs;
1668
1709
  try {
1669
1710
  parsed = parseArgs(argv);
package/src/config.ts CHANGED
@@ -82,10 +82,16 @@ export const ConfigSchema = Type.Object({
82
82
  SEARCH_TIMEOUT_MS: Type.Number({ minimum: 5000, maximum: 120000, default: 45000 }),
83
83
  /** TUI refresh debounce in milliseconds (default: 100ms) */
84
84
  TUI_REFRESH_DEBOUNCE_MS: Type.Number({ minimum: 0, maximum: 1000, default: 100 }),
85
- /** Queue-wait / overhead margin (ms) added on top of each browser operation's own nav
86
- * timeout to form its scheduler task-timeout ceiling: a search task times out after
87
- * SEARCH_TIMEOUT_MS + this, a scrape after SCRAPE_TIMEOUT_MS + this. Sized so a task is
88
- * never killed before it has used its full nav budget plus time waiting in the queue.
85
+ /** Overhead margin (ms) added on top of each browser operation's own nav timeout to
86
+ * form its scheduler task-timeout ceiling: a search task times out after
87
+ * SEARCH_TIMEOUT_MS + this, a scrape after SCRAPE_TIMEOUT_MS + this — covering
88
+ * per-task work beyond raw navigation (page setup, result parsing). Queue wait is
89
+ * NOT part of this margin: the deadline is armed on dispatch, not on enqueue, so
90
+ * time spent waiting for a free worker never counts against it (see
91
+ * browser-task-scheduler.ts). Cold browser-launch/context-creation time is also
92
+ * separate from this margin — COLD_START_ALLOWANCE_MS
93
+ * (infrastructure/browser/config.ts) covers that unconditionally on top of it,
94
+ * since it isn't something a user should need to tune.
89
95
  * (default: 10000) */
90
96
  BROWSER_TASK_TIMEOUT_MS: Type.Number({ minimum: 2000, maximum: 120000, default: 10000 }),
91
97
  /** Timeout for coordinator/research-lead/repair/knowledge LLM calls in ms (default: 300000 = 5 min, range: 60s-30min).
@@ -745,16 +751,23 @@ function loadEnvFiles(cwd: string, iface?: ConfigInterface): Record<string, stri
745
751
  try {
746
752
  logger.warn(`[config] WARNING: .pi-research.env files are deprecated. Settings will be auto-migrated to the centralized registry. Remove ${legacyPath} after migration.`);
747
753
  legacyEnv = parseDotEnv(fs.readFileSync(legacyPath, 'utf-8'));
748
- Object.assign(merged, legacyEnv);
749
-
750
- // Auto-migrate legacy settings to the central registry — but only LOCAL_SCOPE_KEYS. The
751
- // registry is project-scoped storage by design (saveConfig writes only these keys to it);
752
- // persisting user-scoped legacy keys would freeze them per-workspace and silently override
753
- // the user's global config.env forever. Non-local legacy keys still apply to THIS resolution
754
- // via the Object.assign(merged, legacyEnv) above — they just aren't persisted.
754
+ // Only LOCAL_SCOPE_KEYS from this run's resolution too — same boundary
755
+ // step 4 below already enforces for the registry this file migrates
756
+ // into ("the registry is project-scoped storage... never user-scoped
757
+ // keys"). This file lives in the CWD, which for this tool is routinely
758
+ // an arbitrary, potentially untrusted repository a coding agent was
759
+ // just pointed at — applying every key unconditionally let a dropped-in
760
+ // .pi-research.env silently control user-scoped settings (RESEARCH_MODEL,
761
+ // EMBEDDING_DEVICE, DISABLED_TOOLS, export destinations, ...) for that
762
+ // run with no confirmation, for any setting the user hadn't already set
763
+ // via a real environment variable (which still wins over all of this).
764
+ // Non-local keys are ignored here exactly as they already are for the
765
+ // registry — this predecessor mechanism should not be MORE permissive
766
+ // than the successor it's migrating into.
755
767
  const legacyLocal = Object.fromEntries(
756
768
  Object.entries(legacyEnv).filter(([k]) => LOCAL_SCOPE_KEYS.has(k))
757
769
  );
770
+ Object.assign(merged, legacyLocal);
758
771
  const existingLocal = Object.fromEntries(
759
772
  Object.entries(findRegistryEntry(registry, normalizedCwd) ?? {}).filter(([k]) => LOCAL_SCOPE_KEYS.has(k))
760
773
  );
@@ -793,8 +806,14 @@ function loadEnvFiles(cwd: string, iface?: ConfigInterface): Record<string, stri
793
806
  }
794
807
  }
795
808
  } else if (Object.keys(merged).length === 0 && !fs.existsSync(legacyPath) && !fs.existsSync(globalPath)) {
796
- // 5. Warning for missing config
797
- logger.warn(`[config] No configuration found for workspace: ${cwd}. Using code defaults. Run /research-config to configure.`);
809
+ // 5. Warning for missing config — once per workspace per process. getConfig
810
+ // is called on many hot paths (every tool invocation, health checks), and
811
+ // repeating this advisory each time made it the single most common line in
812
+ // the shared log (362 entries in one observed day), burying real warnings.
813
+ if (!warnedMissingConfigWorkspaces.has(normalizedCwd)) {
814
+ warnedMissingConfigWorkspaces.add(normalizedCwd);
815
+ logger.warn(`[config] No configuration found for workspace: ${cwd}. Using code defaults. Run /research-config to configure.`);
816
+ }
798
817
  }
799
818
 
800
819
  warnLegacyEnvVarsOnce(merged);
@@ -819,6 +838,10 @@ const LEGACY_ENV_VARS: Record<string, string> = {
819
838
 
820
839
  let warnedLegacyEnvVars = false;
821
840
 
841
+ // Workspaces already warned about having no configuration (see loadEnvFiles) —
842
+ // per-process dedup so the advisory appears once, not on every getConfig call.
843
+ const warnedMissingConfigWorkspaces = new Set<string>();
844
+
822
845
  /** Warn once per process about set-but-dead 0.1.x env vars (upgrade aid; never throws). */
823
846
  function warnLegacyEnvVarsOnce(merged: Record<string, string>): void {
824
847
  if (warnedLegacyEnvVars) return;
package/src/constants.ts CHANGED
@@ -46,11 +46,26 @@ export function getMaxGatheringCalls(config?: Config): number {
46
46
  /**
47
47
  * Get the maximum scrape batches from config.
48
48
  * This function should be used instead of MAX_SCRAPE_CALLS to support dynamic configuration.
49
+ *
50
+ * @param cachingActive - Pass `true` when prompt caching is active for this
51
+ * run's resolved model (see isPromptCachingActiveForModel in
52
+ * core/llm/research-model-resolver.ts). With caching active, the EFFECTIVE
53
+ * maximum is the configured value plus one: a cached run's system-prompt/
54
+ * tool-definition prefix is cheap to re-send, so the same token budget
55
+ * covers one more scrape batch per researcher. Every consumer of the batch
56
+ * count (tracker enforcement, prompt text, progress units) must go through
57
+ * this function with the same cachingActive value so they all agree on the
58
+ * effective number. The "unlimited" sentinel (0 or >99) is unchanged —
59
+ * adding one to unlimited is meaningless.
49
60
  */
50
- export function getMaxScrapeBatches(config?: Config): number {
61
+ export function getMaxScrapeBatches(config?: Config, cachingActive = false): number {
51
62
  try {
52
63
  const batches = (config || getConfig()).MAX_SCRAPE_BATCHES;
53
- return batches === 0 || batches > 99 ? 999999 : batches;
64
+ if (batches === 0 || batches > 99) return 999999;
65
+ // Cap at 99: a configured 99 (the documented max) plus one would land in the
66
+ // >99 "unlimited" band that every renderer checks — the prompt would say
67
+ // "unlimited batches" while enforcement actually blocked at 100.
68
+ return cachingActive ? Math.min(batches + 1, 99) : batches;
54
69
  } catch {
55
70
  return 2; // Fallback to default
56
71
  }
@@ -63,9 +78,13 @@ export const MAX_SCRAPE_URLS = 6;
63
78
  * Get the units per researcher for the progress bar.
64
79
  * Units = 1 (for start/search) + number of scrape batches.
65
80
  * When batches are unlimited (MAX_SCRAPE_BATCHES=0), caps at 4 for a sane progress estimate.
81
+ *
82
+ * Relative to the EFFECTIVE batch count: pass the run's cachingActive (from
83
+ * isPromptCachingActiveForModel over the resolved model) so progress units
84
+ * agree with the limit the tracker actually enforces.
66
85
  */
67
- export function getUnitsPerResearcher(config?: Config): number {
68
- const batches = getMaxScrapeBatches(config);
86
+ export function getUnitsPerResearcher(config?: Config, cachingActive = false): number {
87
+ const batches = getMaxScrapeBatches(config, cachingActive);
69
88
  return 1 + Math.min(batches, 4);
70
89
  }
71
90
 
@@ -89,3 +89,41 @@ export function resolveResearchModel(options: {
89
89
 
90
90
  throw new Error(buildNoModelAvailableMessage());
91
91
  }
92
+
93
+ /**
94
+ * Best-effort, CONSERVATIVE signal for "will pi-ai apply prompt caching for
95
+ * this resolved model." pi-ai owns all cache-mechanism logic internally (this
96
+ * codebase never constructs cache_control blocks itself) and exposes no
97
+ * public API to ask in advance for most providers, so this can only answer
98
+ * for the two client-visible cases:
99
+ *
100
+ * - `api === 'anthropic-messages'`: caching is native and on by default (no
101
+ * capability gate at all — see pi-ai's anthropic-messages provider).
102
+ * - `api === 'openai-completions'` with `compat.cacheControlFormat ===
103
+ * 'anthropic'`: an explicit opt-in to Anthropic-style cache_control
104
+ * emulation (e.g. an OpenRouter/gateway route to an Anthropic model
105
+ * configured with this override).
106
+ *
107
+ * Every other case returns false even though caching may actually be
108
+ * active: pi's generated model catalog sets `cacheControlFormat` on zero
109
+ * models, so plain `openai-completions` traffic (this project's custom
110
+ * providers — glm-coding, cerebras, zai, OpenRouter, ...) has its cache
111
+ * compat auto-detected by pi-ai's own private, unexported heuristic INSIDE
112
+ * the live request — not knowable client-side, before or after model
113
+ * resolution. Providers with fully implicit/automatic caching
114
+ * (openai-responses family, google-generative-ai, google-vertex,
115
+ * mistral-conversations, bedrock-converse-stream) have no "will it be
116
+ * active" signal at all — it depends on provider-side runtime behavior
117
+ * (minimum prompt length, server cache state). Treat this as "caching is
118
+ * DEFINITELY active" — a false negative is expected and fine for providers
119
+ * outside these two cases, but a false positive would be a real bug.
120
+ */
121
+ export function isPromptCachingActiveForModel(model: Model<any> | undefined): boolean {
122
+ if (!model) return false;
123
+ if (model.api === 'anthropic-messages') return true;
124
+ if (model.api === 'openai-completions') {
125
+ const compat = model.compat as { cacheControlFormat?: string } | undefined;
126
+ return compat?.cacheControlFormat === 'anthropic';
127
+ }
128
+ return false;
129
+ }
@@ -220,8 +220,19 @@ function isDegradableLlmError(error: unknown, signal?: AbortSignal): boolean {
220
220
  export function salvageReportText(raw: string): string {
221
221
  const trimmed = raw.trim();
222
222
  if (trimmed.length <= 50) return '';
223
+ // Envelope sniff. Models routinely wrap the JSON envelope in a markdown code
224
+ // fence (```json … ```); a fenced envelope starts with backticks, not '{', so a
225
+ // bare startsWith('{') check shipped one verbatim as a user-facing report
226
+ // (observed in production 2026-08-20: a ```json-fenced {"action":"synthesize",…}
227
+ // blob, internal protocol fields included, published as the final report).
228
+ // Strip one leading fence line and a trailing fence before the shape check.
229
+ let sniff = trimmed;
230
+ const fence = sniff.match(/^```[a-zA-Z0-9_-]*[ \t]*\r?\n/);
231
+ if (fence) {
232
+ sniff = sniff.slice(fence[0].length).replace(/\r?\n```\s*$/, '').trim();
233
+ }
223
234
  // Looks like a (broken) JSON envelope rather than a report — suppress it.
224
- if (trimmed.startsWith('{') && /"(?:action|content|researchers)"\s*:/.test(trimmed)) {
235
+ if (sniff.startsWith('{') && /"(?:action|content|researchers)"\s*:/.test(sniff)) {
225
236
  return '';
226
237
  }
227
238
  return raw;
@@ -717,14 +728,38 @@ export class PlanningService implements IPlanningService {
717
728
  };
718
729
 
719
730
  const responseText = isRouter
720
- ? await this.callLead({
721
- ...callOptions,
722
- systemPrompt: populate(''),
723
- userMessage: buildRouterMessage(...this.buildRouterEvidence(options, round), runContext),
724
- maxTokens: ROUTER_MAX_TOKENS,
725
- component: 'router',
726
- label: 'router-updatePlanForRound',
727
- })
731
+ ? await (() => {
732
+ const routerMessage = buildRouterMessage(...this.buildRouterEvidence(options, round), runContext);
733
+ // The router carries no reduce mechanism the way synthesizeCorpus does — a
734
+ // routing decision made from PARTITIONED evidence isn't naturally mergeable
735
+ // the way synthesis prose is, and this call's output is a structured JSON
736
+ // delegate plan, not free text. Its input is bounded by team size in the
737
+ // common case (digests for earlier rounds are already capped — see
738
+ // MAX_DIGEST_CHARS), but nothing caps an individual researcher's fresh report
739
+ // body, so a round with a few unusually long reports can still overflow the
740
+ // window with no reduce path to fall back on. Warn rather than fail silently;
741
+ // the provider's own rejection is still the actual backstop, same as the
742
+ // synthesizer's over-budget-after-every-pass case above.
743
+ const window = model.contextWindow ?? DEFAULT_MODEL_CONTEXT_WINDOW;
744
+ const estimatedTokens = routerMessage.length / CHARS_PER_TOKEN + ROUTER_MAX_TOKENS + SYNTHESIS_OVERHEAD_TOKENS;
745
+ if (estimatedTokens > window) {
746
+ logger.warn(
747
+ `[PlanningService] Router request for ${model.id} is an estimated ` +
748
+ `${Math.round(estimatedTokens)} tokens against a ${window}-token context window ` +
749
+ `(evidence ${routerMessage.length} chars, output ceiling ${ROUTER_MAX_TOKENS} tokens). ` +
750
+ `Unlike the synthesizer, the router has no reduce path for an over-budget round — ` +
751
+ `sending it anyway, the provider may reject this request.`,
752
+ );
753
+ }
754
+ return this.callLead({
755
+ ...callOptions,
756
+ systemPrompt: populate(''),
757
+ userMessage: routerMessage,
758
+ maxTokens: ROUTER_MAX_TOKENS,
759
+ component: 'router',
760
+ label: 'router-updatePlanForRound',
761
+ });
762
+ })()
728
763
  : await this.synthesizeCorpus({ ...callOptions, reports, runContext, populate });
729
764
 
730
765
  // Extract and validate JSON
@@ -1115,6 +1150,29 @@ export class PlanningService implements IPlanningService {
1115
1150
  );
1116
1151
  }
1117
1152
 
1153
+ // budgetChars alone can UNDER-report overflow risk once it hits MIN_SYNTHESIS_CORPUS_CHARS:
1154
+ // below that point it is a fixed floor, not a value derived from the window, so a reduced
1155
+ // corpus that fits comfortably under the floor can still combine with a large global source
1156
+ // list and/or run context (neither bounded, both already subtracted from budgetChars but not
1157
+ // from THIS check) to overflow the real window — the same failure class the budget exists to
1158
+ // prevent, just triggered by those two inputs growing instead of the corpus. Check the actual
1159
+ // total request against the actual window directly, independent of any floor.
1160
+ const window = args.model.contextWindow ?? DEFAULT_MODEL_CONTEXT_WINDOW;
1161
+ const estimatedTotalTokens =
1162
+ (finalSize + globalSourceList.length + runContext.length) / CHARS_PER_TOKEN +
1163
+ config.SYNTHESIS_MAX_TOKENS + SYNTHESIS_OVERHEAD_TOKENS;
1164
+ if (estimatedTotalTokens > window) {
1165
+ logger.warn(
1166
+ `[PlanningService] Synthesis request for ${args.model.id} is an estimated ` +
1167
+ `${Math.round(estimatedTotalTokens)} tokens against a ${window}-token context window ` +
1168
+ `(corpus ${finalSize} chars, source list ${globalSourceList.length} chars, run context ` +
1169
+ `${runContext.length} chars, output ceiling ${config.SYNTHESIS_MAX_TOKENS} tokens) — the ` +
1170
+ `source list and/or run context are large enough on their own to overflow the window even ` +
1171
+ `though the reduced corpus fits its budget. Sending it anyway — the provider may reject ` +
1172
+ `this request.`,
1173
+ );
1174
+ }
1175
+
1118
1176
  return this.callLead({
1119
1177
  ...args,
1120
1178
  systemPrompt: populate(reduced ? MERGE_SYNTHESIS_SECTION : ''),
package/src/index.ts CHANGED
@@ -556,7 +556,9 @@ export default async function (pi: ExtensionAPI) {
556
556
 
557
557
  // Researcher sub-agent sessions already carry the authoritative steering copy:
558
558
  // the executor (deep) and quick orchestrator inject "ADDITIONAL USER GUIDANCE"/
559
- // "ADDITIONAL CONSIDERATIONS" into the researcher system prompt themselves.
559
+ // "ADDITIONAL CONSIDERATIONS" into the researcher's initial user message themselves
560
+ // (kept OUT of the system prompt so its cacheable prefix stays byte-identical
561
+ // across researchers — see researcher-executor.ts).
560
562
  // Appending the same messages again here delivered every steering message
561
563
  // TWICE to every researcher — return the prompt untouched. (The coordinator and
562
564
  // the research lead never pass through this hook at all: they run via
@@ -23,7 +23,7 @@ import type { IScheduler, TaskDispatchListener } from '../../core/interfaces/sch
23
23
  import { cleanupOrphanedCamoufoxProcesses, getBrowserPidsForWorkers, killBrowserProcesses } from './browser-cleanup.ts';
24
24
  import { raceWithDeadline } from '../../utils/safe-unref.ts';
25
25
  import { PriorityTaskQueue } from './priority-task-queue.ts';
26
- import { getHealthCheckBudgetMs } from './config.ts';
26
+ import { getHealthCheckBudgetMs, COLD_START_ALLOWANCE_MS } from './config.ts';
27
27
 
28
28
  /**
29
29
  * Grace added to the worker's own task budget before the dead-worker backstop
@@ -273,12 +273,17 @@ export class BrowserTaskScheduler implements IScheduler {
273
273
  const pool = await (await this.getWorkerPoolManager()).ensurePool(config);
274
274
  const startTime = Date.now();
275
275
 
276
- // Task timeout = the search nav budget (SEARCH_TIMEOUT_MS) + a queue-wait/overhead
277
- // margin (BROWSER_TASK_TIMEOUT_MS). Deriving from SEARCH_TIMEOUT_MS keeps the task
278
- // ceiling coherent with the worker's own nav timeout — using a flat base here would
279
- // kill a search before it had used its full nav budget.
276
+ // Task timeout = the search nav budget (SEARCH_TIMEOUT_MS) + an overhead margin
277
+ // (BROWSER_TASK_TIMEOUT_MS) + a cold-start allowance (COLD_START_ALLOWANCE_MS).
278
+ // Deriving from SEARCH_TIMEOUT_MS keeps the task ceiling coherent with the
279
+ // worker's own nav timeout — using a flat base here would kill a search before
280
+ // it had used its full nav budget. The cold-start term matters because eager
281
+ // browser warmup is deliberately disabled: the first search dispatched to a
282
+ // freshly created or just-reset worker pays a real Camoufox launch + context
283
+ // creation inline, before any navigation even starts — without this term that
284
+ // legitimate cold start reads as a dead/wedged worker (see COLD_START_ALLOWANCE_MS).
280
285
  const cfg = config || getConfig();
281
- const timeoutMs = cfg.SEARCH_TIMEOUT_MS + cfg.BROWSER_TASK_TIMEOUT_MS;
286
+ const timeoutMs = cfg.SEARCH_TIMEOUT_MS + cfg.BROWSER_TASK_TIMEOUT_MS + COLD_START_ALLOWANCE_MS;
282
287
 
283
288
  // The deadline measures EXECUTION, not queue wait: the timer is armed by
284
289
  // `startDeadline()` from inside the enqueued callback, i.e. once the queue has
@@ -413,11 +418,13 @@ export class BrowserTaskScheduler implements IScheduler {
413
418
  this.resetIdleTimer();
414
419
  const pool = await (await this.getWorkerPoolManager()).ensurePool(config);
415
420
  const startTime = Date.now();
416
- // Task timeout = the scrape nav budget (SCRAPE_TIMEOUT_MS) + the queue-wait/overhead
417
- // margin (BROWSER_TASK_TIMEOUT_MS), mirroring the search path so both stay coherent.
421
+ // Task timeout = the scrape nav budget (SCRAPE_TIMEOUT_MS) + an overhead margin,
422
+ // mirroring the search path so both stay coherent. The mocked branch skips the
423
+ // cold-start allowance deliberately: PI_RESEARCH_MOCK_SCRAPE implies no real
424
+ // Camoufox launch is expected for scrape tasks, so there is nothing to budget for.
418
425
  const cfg = config || getConfig();
419
426
  const isMocking = process.env['PI_RESEARCH_MOCK_SCRAPE'] === 'true';
420
- const timeoutMs = cfg.SCRAPE_TIMEOUT_MS + (isMocking ? 5000 : cfg.BROWSER_TASK_TIMEOUT_MS);
427
+ const timeoutMs = cfg.SCRAPE_TIMEOUT_MS + (isMocking ? 5000 : cfg.BROWSER_TASK_TIMEOUT_MS + COLD_START_ALLOWANCE_MS);
421
428
 
422
429
  // Armed on dispatch, not on enqueue — see runSearch for why.
423
430
  // See runSearch: dispatch time is tracked for measurement, not for the deadline.
@@ -346,13 +346,50 @@ function isFullMockMode(): boolean {
346
346
  * and a budget below that sum makes the fallback ladder unreachable — so raising
347
347
  * HEALTH_CHECK_TIMEOUT_MS used to make the health check strictly worse.
348
348
  *
349
- * Floored at the historical 105s so default installs are unchanged.
349
+ * Floored at the historical 105s, kept as a floor rather than a default: the
350
+ * derived value now exceeds it even at default timeouts once COLD_START_ALLOWANCE_MS
351
+ * (below) is folded into `longestSlotHold` — a cold Firefox launch was always able to
352
+ * hold a slot for close to that long, the 105s floor just didn't know it. The floor
353
+ * still matters for a hypothetical config where the derived value would otherwise land
354
+ * below it.
350
355
  */
351
356
  export const HEALTHCHECK_MIN_BUDGET_MS = 45_000 + 60_000;
352
357
 
353
358
  /** Navigation attempts executeHealthCheck makes: primary, retry, fallback endpoint. */
354
359
  const HEALTHCHECK_NAV_ATTEMPTS = 3;
355
360
 
361
+ /**
362
+ * Hard ceiling on a single Camoufox browser launch (see launchOnce() in
363
+ * thread-worker-browser.ts, the single source of truth this mirrors — CI runners
364
+ * (2 vCPU) can take up to ~60s to start Firefox under load, floored generously
365
+ * above that).
366
+ */
367
+ export const BROWSER_LAUNCH_TIMEOUT_MS = 90_000;
368
+
369
+ /**
370
+ * Hard ceiling on creating one BrowserContext against an already-launched browser
371
+ * (see acquireTaskContext() in thread-worker-browser.ts, the single source of
372
+ * truth this mirrors).
373
+ */
374
+ export const CONTEXT_CREATION_TIMEOUT_MS = 30_000;
375
+
376
+ /**
377
+ * Worst-case time a worker can spend getting a browser+context ready before it
378
+ * starts a task's actual work. Eager warmup is deliberately disabled (thundering-
379
+ * herd avoidance — see thread-worker.ts), so the first task dispatched to a
380
+ * freshly created or just-reset worker pays this in full, inline with the task
381
+ * itself. Every deadline that bounds how long a task may legitimately hold a
382
+ * worker slot has to include this on top of its own nav budget, or a completely
383
+ * normal cold start on a fresh pool reads as a dead/wedged worker — the false
384
+ * positive `runSearch`/`runScrape`'s task-timeout ceiling and this budget were
385
+ * both built to eliminate, just triggered by a different clock.
386
+ *
387
+ * Deliberately NOT part of BROWSER_TASK_TIMEOUT_MS: that value is user-tunable
388
+ * overhead for per-task work, and cold-start cost isn't something a user should
389
+ * need to account for by hand when adjusting it.
390
+ */
391
+ export const COLD_START_ALLOWANCE_MS = BROWSER_LAUNCH_TIMEOUT_MS + CONTEXT_CREATION_TIMEOUT_MS;
392
+
356
393
  export function getHealthCheckBudgetMs(config?: Config): number {
357
394
  const c = (config || getConfig()) as Partial<Config>;
358
395
  // Each field is read defensively. A missing or non-finite value would otherwise
@@ -363,7 +400,8 @@ export function getHealthCheckBudgetMs(config?: Config): number {
363
400
  typeof value === 'number' && Number.isFinite(value) ? value : fallback;
364
401
  const longestSlotHold =
365
402
  Math.max(ms(c.SEARCH_TIMEOUT_MS, 45_000), ms(c.SCRAPE_TIMEOUT_MS, 15_000))
366
- + ms(c.BROWSER_TASK_TIMEOUT_MS, 10_000);
403
+ + ms(c.BROWSER_TASK_TIMEOUT_MS, 10_000)
404
+ + COLD_START_ALLOWANCE_MS;
367
405
  // Mirrors the worker's own floor in executeHealthCheck.
368
406
  const probeWork = HEALTHCHECK_NAV_ATTEMPTS * Math.max(10_000, ms(c.HEALTH_CHECK_TIMEOUT_MS, 10_000));
369
407
  return Math.max(HEALTHCHECK_MIN_BUDGET_MS, longestSlotHold + probeWork);