@lincoln504/pi-research 1.3.10 → 1.3.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/README.md +1 -1
  2. package/dist/cli.mjs +264 -61
  3. package/dist/prompts/system-coordinator.md +1 -1
  4. package/dist/prompts/system-lead-evaluator.md +1 -1
  5. package/dist/thread-worker.mjs +31 -21
  6. package/docs/ARCHITECTURE.md +60 -22
  7. package/docs/CHANGELOG.md +103 -1
  8. package/docs/CONFIGURATION.md +6 -5
  9. package/package.json +18 -10
  10. package/scripts/cleanup.cjs +26 -5
  11. package/scripts/setup.cjs +25 -1
  12. package/skills/pi-research/SKILL.md +1 -1
  13. package/src/cli.ts +94 -13
  14. package/src/config.ts +6 -15
  15. package/src/constants.ts +13 -0
  16. package/src/core/service-registry.ts +19 -3
  17. package/src/infrastructure/browser/config.ts +36 -5
  18. package/src/infrastructure/browser/task-execution-service.ts +12 -2
  19. package/src/infrastructure/browser/thread-worker-browser.ts +34 -18
  20. package/src/infrastructure/browser/thread-worker.mjs +31 -21
  21. package/src/infrastructure/gpu-resource-service.ts +15 -1
  22. package/src/knowledge/embedder-init.ts +34 -2
  23. package/src/knowledge/store.ts +75 -6
  24. package/src/knowledge/writer-queue.ts +28 -0
  25. package/src/orchestration/research-orchestration-service.ts +35 -1
  26. package/src/orchestration/research-synthesis-service.ts +4 -2
  27. package/src/orchestration/researcher.ts +2 -13
  28. package/src/prompts/system-coordinator.md +1 -1
  29. package/src/prompts/system-lead-evaluator.md +1 -1
  30. package/src/sdk.ts +10 -1
  31. package/src/security/nvd-client.ts +55 -4
  32. package/src/security/osv-client.ts +13 -1
  33. package/src/tools/research-knowledge-search.ts +18 -1
  34. package/src/tools/research-tool-definition.ts +8 -1
  35. package/src/utils/citation-utils.ts +4 -2
  36. package/src/utils/log-utils.ts +16 -1
  37. package/src/utils/text-utils.ts +17 -3
  38. package/src/web-research/scraper-utils.ts +55 -14
package/src/cli.ts CHANGED
@@ -44,6 +44,8 @@ import { exportResearchReport, appendExportMessage } from './utils/research-expo
44
44
  import { getConfig, getGlobalConfigDir, getGlobalEnvFilePath, getInterfaceEnvFilePath, getStateDir, saveConfig, resetConfig, describeKnowledgeStoreMode, isProjectScopedKey, parseDotEnv } from './config.ts';
45
45
  import { getAgentDir } from '@earendil-works/pi-coding-agent';
46
46
  import { buildModelRegistry, safeGetAvailable } from './core/llm/model-registry-factory.ts';
47
+ import { RESEARCH_TOOL_NAMES } from './constants.ts';
48
+ import { isBrowserAvailable } from './infrastructure/browser/config.ts';
47
49
 
48
50
  // ---------------------------------------------------------------------------
49
51
  // Constants
@@ -234,7 +236,8 @@ export class ConfigFileError extends Error {}
234
236
  * the real environment, so real shell exports always win.
235
237
  */
236
238
  export function bridgeConfigEnv(explicitConfigPath?: string): { path: string; loaded: boolean } {
237
- const basePath = explicitConfigPath ?? getGlobalEnvFilePath();
239
+ const globalBasePath = getGlobalEnvFilePath();
240
+ const basePath = explicitConfigPath ?? globalBasePath;
238
241
  const cliOverlayPath = getInterfaceEnvFilePath('cli');
239
242
  let loaded = false;
240
243
  // Merge base then overlay into one map FIRST (overlay wins), and only then
@@ -252,13 +255,18 @@ export function bridgeConfigEnv(explicitConfigPath?: string): { path: string; lo
252
255
  // the user passed on this exact invocation is the most specific intent, and the
253
256
  // ambient cli.env overlay silently overriding it re-created the very failure
254
257
  // this bridge exists to fix (a key set in `--config ci.env` simply not applying).
258
+ // The GLOBAL base always stays underneath: this bridge exists for env-only keys
259
+ // (PI_RESEARCH_API_KEY & co.) that getConfig never carries, so dropping the base
260
+ // here silently dropped credentials stored only in config.env — `--config extra.env`
261
+ // exit-78'd on a correctly configured machine while the help text promised
262
+ // "base config.env keys it does not set still apply".
255
263
  const layers = explicitConfigPath !== undefined
256
- ? [cliOverlayPath, basePath]
257
- : [basePath, cliOverlayPath];
264
+ ? [globalBasePath, cliOverlayPath, explicitConfigPath]
265
+ : [globalBasePath, cliOverlayPath];
258
266
  for (const filePath of layers) {
259
267
  // Only the EXPLICIT --config file fails loudly (see ConfigFileError above);
260
268
  // the ambient layers keep their missing/unreadable tolerance unchanged.
261
- const isExplicit = explicitConfigPath !== undefined && filePath === basePath;
269
+ const isExplicit = explicitConfigPath !== undefined && filePath === explicitConfigPath;
262
270
  try {
263
271
  if (!fs.existsSync(filePath)) {
264
272
  if (isExplicit) throw new ConfigFileError(`--config file not found: ${filePath}`);
@@ -495,11 +503,20 @@ async function detectCredentials(explicitModel?: string): Promise<CredentialDete
495
503
  modelFrom,
496
504
  piAuthPresent,
497
505
  piModelsPresent,
498
- problem:
499
- 'No model or API key is configured for pi-research. Provide credentials via environment ' +
500
- 'variables (PI_RESEARCH_API_KEY + PI_RESEARCH_PROVIDER + PI_RESEARCH_MODEL) or, if you use ' +
501
- "pi, via pi's configuration (a provider entry in ~/.pi/agent/auth.json, or a per-provider " +
502
- 'apiKey in ~/.pi/agent/models.json).',
506
+ // A model configured with no usable key is a different (and misleading to
507
+ // conflate) gap than nothing configured at all: SKILL.md tells agents to relay
508
+ // this text verbatim, so "No model … is configured" pointed users who HAD set
509
+ // PI_RESEARCH_MODEL at the wrong knob.
510
+ problem: model
511
+ ? `A model is configured (${model}) but no API key is available for ` +
512
+ `${provider ? `provider "${provider}"` : 'its provider'}. Provide one via the ` +
513
+ 'PI_RESEARCH_API_KEY environment variable (with PI_RESEARCH_PROVIDER), or, if you use pi, ' +
514
+ "via pi's configuration (a provider entry in ~/.pi/agent/auth.json, or a per-provider " +
515
+ 'apiKey in ~/.pi/agent/models.json).'
516
+ : 'No model or API key is configured for pi-research. Provide credentials via environment ' +
517
+ 'variables (PI_RESEARCH_API_KEY + PI_RESEARCH_PROVIDER + PI_RESEARCH_MODEL) or, if you use ' +
518
+ "pi, via pi's configuration (a provider entry in ~/.pi/agent/auth.json, or a per-provider " +
519
+ 'apiKey in ~/.pi/agent/models.json).',
503
520
  };
504
521
  }
505
522
 
@@ -604,6 +621,14 @@ export async function cmdResearch(args: ResearchArgs): Promise<number> {
604
621
  ...(args.initialLinks ? { initialLinks: args.initialLinks } : {}),
605
622
  }, abortController.signal);
606
623
 
624
+ // Snapshot the run diagnostics NOW, before any further await: on a cancelled
625
+ // run, onSignal's fire-and-forget safeShutdown runs concurrently with this
626
+ // tail and resets these SDK globals — reading them at JSON-emit time made
627
+ // `researchers`/`trackedErrors` silently vanish from a cancelled-but-
628
+ // successful payload depending on who won the race.
629
+ const outcome = getLastResearcherOutcome();
630
+ const errorReport = getLastErrorReport();
631
+
607
632
  // Optionally persist the report to a file (opt-in via
608
633
  // PI_RESEARCH_REPORT_EXPORT_ENABLED). The SDK is a pure library and does not
609
634
  // write files; the CLI front-end does, mirroring the pi tool path.
@@ -636,8 +661,7 @@ export async function cmdResearch(args: ResearchArgs): Promise<number> {
636
661
  // (ISSUES-ANALYSIS-2026-07-09.md, Symptom 4): a caller reading only stdout/JSON
637
662
  // had no way to distinguish "thin report, little material out there" from "thin
638
663
  // report because most researchers died mid-run" without opening the log file.
639
- const outcome = getLastResearcherOutcome();
640
- const errorReport = getLastErrorReport();
664
+ // (outcome/errorReport were snapshotted right after the run resolved — see above.)
641
665
  toStdout(pretty({
642
666
  ok: true,
643
667
  depth,
@@ -719,11 +743,20 @@ export async function cmdKnowledge(queries: string[], json?: boolean): Promise<n
719
743
  await safeShutdown();
720
744
  }
721
745
 
746
+ // Same latch as cmdResearch's post-run check: the abort signal's checks inside
747
+ // searchKnowledge are point-in-time, so a Ctrl-C landing after the last one lets
748
+ // the search resolve normally — without this, an interrupted run raced onSignal's
749
+ // flushAndExit and could report ok:true / exit 0. The rationale comment above
750
+ // named exactly this failure; only the throwing half was previously handled.
751
+ if (cancellationRequested) {
752
+ exit = cancellationSignal ? exitCodeForSignal(cancellationSignal) : EXIT.CANCELLED;
753
+ }
754
+
722
755
  if (json) {
723
756
  // `ok: true` makes `ok` a reliable success/failure discriminant across every
724
757
  // knowledge JSON variant (disabled-store above, reportError's ok:false below,
725
758
  // and this success shape) — additive only, every existing field is unchanged.
726
- toStdout(pretty({ ok: true, ...result }));
759
+ toStdout(pretty({ ok: true, ...(cancellationRequested ? { cancelled: true } : {}), ...result }));
727
760
  } else {
728
761
  toStdout((result.text.endsWith('\n') ? result.text : result.text + '\n'));
729
762
  }
@@ -738,6 +771,18 @@ async function cmdStatus(json?: boolean): Promise<number> {
738
771
  const det = await detectCredentials();
739
772
  const paths = resolvedConfigPaths();
740
773
  const cfg = getConfig(process.cwd(), 'cli');
774
+ // The stealth browser is installed by this package's `postinstall`
775
+ // (scripts/setup.cjs), which npm 11.19+ BLOCKS BY DEFAULT — it now warns
776
+ // "packages have install scripts not yet covered by allowScripts" and carries
777
+ // on, so the browser is simply absent on an otherwise successful install.
778
+ // Every other native artifact survives that (onnxruntime, sharp, esbuild and
779
+ // webgpu all ship prebuilt platform packages), which is exactly what makes
780
+ // this one worth reporting: nothing else fails, and without this the first
781
+ // symptom is a scrape falling back mid-run. A missing browser degrades
782
+ // capability rather than breaking the tool — the fetch path still serves most
783
+ // pages — so it is reported alongside `ready` rather than folded into it.
784
+ // isBrowserAvailable() is a resolve + existsSync; it never launches anything.
785
+ const browserInstalled = isBrowserAvailable();
741
786
  const summary = {
742
787
  package: '@lincoln504/pi-research',
743
788
  version: PKG_VERSION,
@@ -751,6 +796,14 @@ async function cmdStatus(json?: boolean): Promise<number> {
751
796
  modelFrom: det.modelFrom,
752
797
  problem: det.problem ?? null,
753
798
  },
799
+ browser: {
800
+ installed: browserInstalled,
801
+ // Same remedy the health check prints, so both surfaces name one command.
802
+ fix: browserInstalled ? null : 'npx camoufox-js fetch',
803
+ note: browserInstalled
804
+ ? null
805
+ : 'Stealth browser not installed — scraping falls back to plain fetch, which some sites block. npm 11.19+ blocks install scripts by default, which skips this package\'s postinstall.',
806
+ },
754
807
  knowledgeStoreMode: cfg.KNOWLEDGE_STORE_MODE,
755
808
  defaultDepth: cfg.DEFAULT_RESEARCH_DEPTH,
756
809
  paths,
@@ -764,6 +817,10 @@ async function cmdStatus(json?: boolean): Promise<number> {
764
817
  toStdout(`pi-research ${PKG_VERSION}\n`);
765
818
  toStdout(`ready: ${summary.ready ? 'yes' : 'no'}\n\n`);
766
819
  toStdout(configBlock(det, summary.ready ? undefined : '— fix the problem above, then re-run.') + '\n');
820
+ if (!summary.browser.installed) {
821
+ toStdout(`\nstealth browser: NOT INSTALLED — run: ${summary.browser.fix}\n`);
822
+ toStdout(` ${summary.browser.note}\n`);
823
+ }
767
824
  toStdout(`\nknowledge store mode: ${cfg.KNOWLEDGE_STORE_MODE} (default depth: ${cfg.DEFAULT_RESEARCH_DEPTH})\n`);
768
825
  return EXIT.OK;
769
826
  }
@@ -1032,7 +1089,17 @@ export async function reportError(err: unknown, what: string, json?: boolean): P
1032
1089
  // --json: emit a structured error on stdout so a machine consumer never has to
1033
1090
  // special-case plain-text stderr. Mirrors the { ok: true, ... } success shape.
1034
1091
  if (json) {
1035
- const payload: Record<string, unknown> = { ok: false, error: msg, exitCode };
1092
+ // A cancellation reports itself as a cancellation, not as whatever component
1093
+ // happened to notice the abort first. The internal message is genuinely
1094
+ // misleading here — a Ctrl-C unwinding through the planner surfaces as
1095
+ // "Coordinator failed: Request was aborted", so a consumer relaying `error`
1096
+ // verbatim (which SKILL.md instructs agents to do) tells the user a component
1097
+ // broke when they simply stopped the run. The plain-text branch below already
1098
+ // refuses to say "failed" for a cancel; JSON said it anyway. The original
1099
+ // wording is preserved under `cause` so nothing diagnostic is lost.
1100
+ const payload: Record<string, unknown> = isCancellation
1101
+ ? { ok: false, error: `pi-research ${what} cancelled.`, cause: msg, exitCode }
1102
+ : { ok: false, error: msg, exitCode };
1036
1103
  // Machine-readable "this will succeed later" flag, so a consumer can back off
1037
1104
  // and retry without string-matching the human message. Gated on cancellation:
1038
1105
  // a Ctrl-C landing in the same window a queue-wait expires must never emit
@@ -1158,6 +1225,7 @@ COMMANDS
1158
1225
  --dry-run | --json Plan only; JSON output.
1159
1226
 
1160
1227
  help, --help, -h Show this help.
1228
+ --version, -v Print the pi-research version.
1161
1229
 
1162
1230
  CONFIGURE
1163
1231
  Credentials are resolved in this order (first match wins):
@@ -1372,6 +1440,15 @@ export function parseArgs(argv: string[]): ParsedArgs {
1372
1440
  } else if (a === '--exclude-tools') {
1373
1441
  const v = takeOptionValue(rest, ++i, '--exclude-tools');
1374
1442
  excludeTools = v.split(',').map((s) => s.trim()).filter(Boolean);
1443
+ // Unknown names fail fast: the merge downstream is a blind Set union, so a
1444
+ // typo ("securty_search") silently excluded nothing while the user believed
1445
+ // the tool was off for the whole (billed) run.
1446
+ const unknown = excludeTools.filter((t) => !RESEARCH_TOOL_NAMES.includes(t));
1447
+ if (unknown.length > 0) {
1448
+ throw new UsageError(
1449
+ `--exclude-tools: unknown tool name${unknown.length === 1 ? '' : 's'} ` +
1450
+ `${unknown.map((t) => `"${t}"`).join(', ')}. Valid names: ${RESEARCH_TOOL_NAMES.join(', ')}.`);
1451
+ }
1375
1452
  if (excludeTools.length === 0) excludeTools = undefined;
1376
1453
  } else if (a === '--initial-links') {
1377
1454
  // Consume until the next flag or end. Validate each entry: these are templated
@@ -1392,6 +1469,10 @@ export function parseArgs(argv: string[]): ParsedArgs {
1392
1469
  }
1393
1470
  j++;
1394
1471
  }
1472
+ // Every other value-taking flag fails fast on a missing value; a bare
1473
+ // `--initial-links` (immediately followed by another flag or end of args)
1474
+ // silently proceeding with zero links was the one exception.
1475
+ if (j === i + 1) throw new UsageError('--initial-links requires at least one URL.');
1395
1476
  i = j - 1;
1396
1477
  } else if (a === '--config') {
1397
1478
  configPath = takeOptionValue(rest, ++i, '--config');
package/src/config.ts CHANGED
@@ -122,15 +122,6 @@ export const ConfigSchema = Type.Object({
122
122
  * the model's real maxTokens at call time.
123
123
  * Not exposed in TUI — controlled via PI_RESEARCH_SYNTHESIS_MAX_TOKENS env var. */
124
124
  SYNTHESIS_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 32768 }),
125
- /** Max OUTPUT tokens per researcher LLM call (deep and quick sub-agent sessions).
126
- * Without a cap the session inherits the model's full catalog ceiling as the
127
- * per-request max_tokens — ~230k for large-context models — which providers that
128
- * budget-check max_tokens up front (OpenRouter 402s against remaining credits)
129
- * reject outright on low balances, even though the actual response would cost
130
- * cents. Default 16384: researcher turns are tool calls plus a cited report, far
131
- * below this. Clamped to the model's real ceiling at session build time.
132
- * Not exposed in TUI — controlled via PI_RESEARCH_RESEARCHER_MAX_TOKENS env var. */
133
- RESEARCHER_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 16384 }),
134
125
  /** LLM Model override for researcher sub-agents and knowledge synthesis.
135
126
  * Format: provider/model-id (e.g. google/gemini-2.0-flash-001) or just model-id.
136
127
  * When set, this overrides ctx.model for researcher sub-agents (both deep and quick)
@@ -167,7 +158,8 @@ export const ConfigSchema = Type.Object({
167
158
  /**
168
159
  * Comma-separated list of research tools to DISABLE for every researcher in a run
169
160
  * (e.g. "scrape", or "youtube_transcript,stackexchange"). Valid tool names:
170
- * search, scrape, security_search, stackexchange, youtube_transcript, grep.
161
+ * search, scrape, security_search, stackexchange, youtube_transcript, grep, read
162
+ * (RESEARCH_TOOL_NAMES in constants.ts is the canonical list).
171
163
  * Disable-only: the list is merged into the per-run excludeTools stream, so disabled
172
164
  * tools are both removed from each researcher's toolset AND named in the
173
165
  * coordinator/evaluator "DISABLED TOOLS" prompt section. Env-only
@@ -242,7 +234,6 @@ const USER_MIGRATION_KEYS = [
242
234
  'PI_RESEARCH_LLM_THINKING_LEVEL',
243
235
  'PI_RESEARCH_PLANNING_MAX_TOKENS',
244
236
  'PI_RESEARCH_SYNTHESIS_MAX_TOKENS',
245
- 'PI_RESEARCH_RESEARCHER_MAX_TOKENS',
246
237
  'PI_RESEARCH_MIGRATION_STRATEGY',
247
238
  'PI_RESEARCH_CONSOLE_LOG',
248
239
  'PI_RESEARCH_MODEL',
@@ -875,7 +866,6 @@ export function saveConfig(config: Config, scope: 'local' | 'user' = 'local', cw
875
866
  PI_RESEARCH_LLM_THINKING_LEVEL: config.LLM_THINKING_LEVEL,
876
867
  PI_RESEARCH_PLANNING_MAX_TOKENS: String(config.PLANNING_MAX_TOKENS),
877
868
  PI_RESEARCH_SYNTHESIS_MAX_TOKENS: String(config.SYNTHESIS_MAX_TOKENS),
878
- PI_RESEARCH_RESEARCHER_MAX_TOKENS: String(config.RESEARCHER_MAX_TOKENS),
879
869
  PI_RESEARCH_MIGRATION_STRATEGY: config.MIGRATION_STRATEGY,
880
870
  PI_RESEARCH_CONSOLE_LOG: String(config.CONSOLE_LOG),
881
871
  ...(config.RESEARCH_MODEL ? { PI_RESEARCH_MODEL: config.RESEARCH_MODEL } : {}),
@@ -1106,7 +1096,6 @@ export function createConfig(env: Record<string, string | undefined>, processEnv
1106
1096
  LLM_THINKING_LEVEL: parseEnvEnum(e, 'PI_RESEARCH_LLM_THINKING_LEVEL', ['off', 'minimal', 'low', 'medium', 'high'] as const, DEFAULTS.LLM_THINKING_LEVEL),
1107
1097
  PLANNING_MAX_TOKENS: parseEnvNumber(e, 'PI_RESEARCH_PLANNING_MAX_TOKENS', DEFAULTS.PLANNING_MAX_TOKENS, 1024, 131072),
1108
1098
  SYNTHESIS_MAX_TOKENS: parseEnvNumber(e, 'PI_RESEARCH_SYNTHESIS_MAX_TOKENS', DEFAULTS.SYNTHESIS_MAX_TOKENS, 1024, 131072),
1109
- RESEARCHER_MAX_TOKENS: parseEnvNumber(e, 'PI_RESEARCH_RESEARCHER_MAX_TOKENS', DEFAULTS.RESEARCHER_MAX_TOKENS, 1024, 131072),
1110
1099
  MIGRATION_STRATEGY: parseEnvEnum(e, 'PI_RESEARCH_MIGRATION_STRATEGY', ['drop', 're-embed', 'backup'] as const, DEFAULTS.MIGRATION_STRATEGY),
1111
1100
  CONSOLE_LOG: parseEnvBool(e, 'PI_RESEARCH_CONSOLE_LOG', DEFAULTS.CONSOLE_LOG),
1112
1101
  RESEARCH_MODEL: parseEnvString(e, 'PI_RESEARCH_MODEL', DEFAULTS.RESEARCH_MODEL),
@@ -1226,7 +1215,9 @@ function parseEnvNumber(env: Record<string, string | undefined>, key: string, de
1226
1215
  }
1227
1216
 
1228
1217
  function parseEnvBool(env: Record<string, string | undefined>, key: string, def: boolean): boolean {
1229
- const v = env[key];
1218
+ // Trim like parseEnvNumber does: file values arrive trimmed via parseDotEnv, but a
1219
+ // real env var `PI_RESEARCH_X='true '` otherwise silently read false.
1220
+ const v = env[key]?.trim();
1230
1221
  if (v === undefined || v === '') return def;
1231
1222
  // Strict by design: only the literal 'true' (any case) is truthy; every other value is false.
1232
1223
  // This is intentional and asserted in config.test.ts ('1'/'yes' → false) — do not loosen it.
@@ -1249,7 +1240,7 @@ function parseEnvEnum<T extends string>(
1249
1240
  allowed: readonly T[],
1250
1241
  def: T,
1251
1242
  ): T {
1252
- const raw = env[key];
1243
+ const raw = env[key]?.trim();
1253
1244
  if (raw === undefined || raw === '') return def;
1254
1245
  if ((allowed as readonly string[]).includes(raw)) return raw as T;
1255
1246
  logger.warn(`[config] Invalid ${key}="${raw}" (expected one of: ${allowed.join(', ')}). Using default "${def}".`);
package/src/constants.ts CHANGED
@@ -182,6 +182,19 @@ export const MAX_QUERIES_PER_RESEARCHER_LEVEL_3 = 20;
182
182
  */
183
183
  export const DEFAULT_EXCLUDED_TOOLS: readonly string[] = ['grep'];
184
184
 
185
+ /**
186
+ * The complete set of excludable research tool names — the vocabulary
187
+ * `--exclude-tools` / `excludeTools` / PI_RESEARCH_DISABLED_TOOLS accept.
188
+ * Kept here (next to DEFAULT_EXCLUDED_TOOLS) so validation and the exclusion
189
+ * machinery cannot drift; the DISABLED_TOOLS doc in config.ts names the same set.
190
+ * `read` is a researcher tool too (local file reads) — excluding it is a
191
+ * legitimate hardening move, and omitting it from this list broke exactly that
192
+ * on the CLI when validation was introduced.
193
+ */
194
+ export const RESEARCH_TOOL_NAMES: readonly string[] = [
195
+ 'search', 'scrape', 'security_search', 'stackexchange', 'youtube_transcript', 'grep', 'read',
196
+ ];
197
+
185
198
  /**
186
199
  * Resolve the effective researcher exclusion list.
187
200
  *
@@ -178,10 +178,26 @@ export class ServiceContainer {
178
178
  options: ServiceContainerOptions = {}
179
179
  ): Promise<void> {
180
180
  const mergedOptions = { ...this.defaultOptions, ...options, allowOverwrite: true };
181
-
181
+
182
182
  // Dispose existing service if present
183
- if (this.services.has(name)) {
184
- const registration = this.services.get(name)!;
183
+ const registration = this.services.get(name);
184
+ if (registration) {
185
+ // Join an already-in-flight clear()/replace()/disposeAll() on this
186
+ // registration before touching its instance — the same guard those paths
187
+ // use on each other. Without it, a concurrent teardown and this call both
188
+ // dispose the SAME instance, and register() below then swaps the whole
189
+ // registration object out from under the in-flight one, orphaning any
190
+ // instance it constructs.
191
+ // while(), not if(): a newer disposal can begin while we are awaiting this
192
+ // one, exactly as get() documents for the same field.
193
+ while (registration.disposalPromise) {
194
+ await registration.disposalPromise.catch(() => { /* disposal errors are logged by clear()/replace() */ });
195
+ }
196
+ // Settle an in-flight initialization too, so we never discard a service
197
+ // that is still being built — mirrors replace().
198
+ if (registration.initializationPromise) {
199
+ await registration.initializationPromise.catch(() => { /* failed init cleaned up after itself */ });
200
+ }
185
201
  if (registration.instance && registration.instance.dispose) {
186
202
  try {
187
203
  await registration.instance.dispose();
@@ -31,15 +31,40 @@ function getWindowsCamoufoxDir(): string {
31
31
  return join(homedir(), 'AppData', 'Local', 'camoufox', 'camoufox', 'Cache');
32
32
  }
33
33
 
34
+ /**
35
+ * The custom camoufox location, if the user set one.
36
+ *
37
+ * CAMOUFOX_INSTALL_DIR takes precedence because it is camoufox-js's OWN
38
+ * variable and the only one that can ever move the install: camoufox-js ≥0.12.0
39
+ * resolves INSTALL_DIR from it. PLAYWRIGHT_BROWSERS_PATH is kept as a documented
40
+ * alias, but on its own it only ever moved where we LOOK — camoufox went on
41
+ * installing to the default cache, so setting it made the browser permanently
42
+ * "not found" right after a successful download. Both are exported onward to
43
+ * camoufox-js (getBrowserEnv, scripts/setup.cjs) so that install and lookup
44
+ * cannot disagree.
45
+ *
46
+ * CAVEAT, deliberate: the pinned camoufox-js (0.10.x — see
47
+ * thread-worker-browser.ts for why it cannot move yet) hardcodes
48
+ * `userCacheDir("camoufox")` and honours NEITHER variable, so on the shipped
49
+ * pin a custom location still only relocates lookup and the underlying bug
50
+ * remains. This plumbing is correct and inert rather than wrong: it starts
51
+ * working the moment the pin can advance, and costs nothing until then. Until
52
+ * that happens, treat a custom browser directory as unsupported.
53
+ */
54
+ function getCustomCamoufoxDir(): string | undefined {
55
+ return process.env['CAMOUFOX_INSTALL_DIR'] || process.env['PLAYWRIGHT_BROWSERS_PATH'] || undefined;
56
+ }
57
+
34
58
  /**
35
59
  * Get the camoufox binary cache directory.
36
- * Uses PLAYWRIGHT_BROWSERS_PATH if set, otherwise the standard user cache location
60
+ * Uses the custom location above if set, otherwise the standard user cache location
37
61
  * that camoufox uses by default (~/.cache/camoufox on Linux).
38
62
  * We do NOT override HOME — that trick was unreliable and caused install/runtime mismatches.
39
63
  */
40
64
  export function getBrowserCacheDir(): string {
41
- if (process.env['PLAYWRIGHT_BROWSERS_PATH']) {
42
- return process.env['PLAYWRIGHT_BROWSERS_PATH'];
65
+ const custom = getCustomCamoufoxDir();
66
+ if (custom) {
67
+ return custom;
43
68
  }
44
69
  // Mirror getCamoufoxBinaryPath() so both functions agree on the cache location
45
70
  const osPlatform = platform();
@@ -109,11 +134,17 @@ export function getBrowserProfileDir(config?: Config): string {
109
134
  */
110
135
  export function getBrowserEnv(config?: Config): NodeJS.ProcessEnv {
111
136
  const env: NodeJS.ProcessEnv = { ...process.env };
112
- const customPath = process.env['PLAYWRIGHT_BROWSERS_PATH'];
137
+ const customPath = getCustomCamoufoxDir();
113
138
  if (customPath) {
114
139
  env['PLAYWRIGHT_BROWSERS_PATH'] = customPath;
140
+ // The one that camoufox-js actually reads. Without it the worker's
141
+ // camoufox launches from the DEFAULT cache while everything on this side
142
+ // resolves the custom path — the install/runtime mismatch this file's
143
+ // header warns about, just arriving through a different door.
144
+ env['CAMOUFOX_INSTALL_DIR'] = customPath;
115
145
  } else {
116
146
  delete env['PLAYWRIGHT_BROWSERS_PATH'];
147
+ delete env['CAMOUFOX_INSTALL_DIR'];
117
148
  }
118
149
  // Tell thread-workers where to write their lifecycle/error log. Precedence:
119
150
  // 1. A user-set PI_RESEARCH_LOG_FILE (already copied from process.env above) wins —
@@ -164,7 +195,7 @@ export function ensureBrowserCacheDir(): string {
164
195
  * Matches camoufox-js's own resolution logic.
165
196
  */
166
197
  export function getCamoufoxBinaryPath(): string {
167
- const customPath = process.env['PLAYWRIGHT_BROWSERS_PATH'];
198
+ const customPath = getCustomCamoufoxDir();
168
199
  if (customPath) {
169
200
  return customPath;
170
201
  }
@@ -136,9 +136,19 @@ async function isRegisteredLeaderAlive(container: ServiceContainer): Promise<boo
136
136
  return result;
137
137
  }
138
138
 
139
- /** Test seam: drop the coalesced liveness probe. */
140
- export function _resetLeaderLivenessProbeForTests(): void {
139
+ /**
140
+ * Test seam: drop BOTH module-level globals that survive between tests — the
141
+ * coalesced liveness probe (1s window) and the restart cooldown (10s window).
142
+ *
143
+ * Both are far longer than a test file's runtime, so a test that reaches
144
+ * restartSchedulerWithHerdGuard leaves state that silently short-circuits the
145
+ * next one: isRegisteredLeaderAliveUncached / forceSchedulerRestart simply are
146
+ * not re-invoked. Today nothing observes that only because the one test that
147
+ * reaches this path happens to run last — ordering, not a guarantee.
148
+ */
149
+ export function _resetTaskExecutionGlobalsForTests(): void {
141
150
  leaderLivenessProbe = null;
151
+ lastRestartTime = 0;
142
152
  }
143
153
 
144
154
  /**
@@ -134,29 +134,45 @@ export async function initBrowser(): Promise<void> {
134
134
  // (consumers install without our lockfile, so a floating range resolves
135
135
  // to a newer, broken version):
136
136
  //
137
- // • camoufox-js ^0.10.2 → fetches camoufox Firefox 135 / beta.24, the
138
- // NEWEST camoufox with binaries for every OS we support (Linux x64/
139
- // arm64, macOS x64/arm64, Windows x64). FF146/FF150 dropped the
140
- // Windows build entirely, so the 0.11.x line breaks Windows installs.
137
+ // • camoufox-js ^0.10.2 → the two ORIGINAL reasons for this pin have
138
+ // both lapsed (camoufox restored Windows assets in v152.0.4-beta.26,
139
+ // 2026-07-16; and impit's `only-allow pnpm` guard existed only in
140
+ // 0.13.1/0.14.0), but a THIRD, worse one replaced them. 0.12.0
141
+ // requires `better-sqlite3 ^13.0.1`, and better-sqlite3 13 DROPPED
142
+ // PREBUILT BINARIES: 12.x installs via `prebuild-install || node-gyp
143
+ // rebuild`, 13.x has no install script at all, so npm falls back to
144
+ // auto-running `node-gyp rebuild` for its binding.gyp and every
145
+ // install needs a full C++ toolchain. Measured: `npm ci` fails
146
+ // outright on the Windows CI runner ("could not find a version of
147
+ // Visual Studio 2017 or newer"), which is most Windows consumers.
148
+ // Upgrading again means checking better-sqlite3 first, not camoufox.
141
149
  // • playwright-core 1.60.0 (exact) → playwright-core and the camoufox
142
150
  // binary are one matched Juggler-protocol pair. 1.61 added a
143
- // viewport.isMobile field the FF135 build REJECTS ("property
151
+ // viewport.isMobile field the build REJECTS ("property
144
152
  // viewport.isMobile not described in this scheme"), failing every
145
- // launch below. 1.60.0 is the newest playwright the FF135 Juggler
146
- // accepts.
153
+ // launch below. Corroborated upstream: camoufox-js 0.12.0 declares
154
+ // `peerDependencies: playwright-core "<1.61.0"`, the same bound this
155
+ // pin has held by hand.
147
156
  // • impit 0.13.0 (exact, a direct dep though only camoufox-js uses it)
148
- // → impit 0.13.1/0.14.0 shipped a `preinstall: npx only-allow pnpm`
149
- // guard (an upstream mistake, fixed in 0.14.1+) that hard-fails
150
- // `npm install -g`. Within camoufox-js's ^0.13.0 range 0.13.0 is the
151
- // only guard-free version; a direct-dep pin is the ONLY way to force
152
- // it for consumers (npm `overrides` don't propagate to installers of
153
- // a published package).
157
+ // → camoufox-js 0.10.2 requires `impit ^0.13.0`, and within that range
158
+ // 0.13.0 is the only version free of the `preinstall: npx only-allow
159
+ // pnpm` guard (an upstream mistake in 0.13.1/0.14.0, dropped again in
160
+ // 0.14.1) that hard-fails `npm install -g`. The exact pin is also the
161
+ // ONLY way to force a version for consumers, since npm `overrides`
162
+ // don't propagate to installers of a published package.
154
163
  //
155
- // Removal: upgrade all three together when camoufox ships stable
156
- // cross-platform (incl. Windows) FF150 binaries — bumping camoufox-js to
157
- // ≥0.11.1 also drops the impit pin (0.11.1 requires the fixed impit
158
- // ^0.14.1) and needs playwright re-pinned to the new build's Juggler.
159
- // Always re-verify a real headless run on Linux/macOS/Windows after.
164
+ // What is NOT pinned: the browser BINARY. `camoufox-js fetch` takes no
165
+ // version argument — it walks the GitHub releases newest-first and takes
166
+ // the first non-prerelease one with an asset for this OS/arch. A consumer
167
+ // therefore gets whatever camoufox published most recently, whichever
168
+ // camoufox-js they have, and a future release could break launches with no
169
+ // change on our side. Both the current newest (v152.0.4-beta.28, Firefox
170
+ // 152) and the older v135.0.1-beta.24 an existing cache may hold were
171
+ // verified launching and driving under playwright-core 1.60.0.
172
+ //
173
+ // Always re-verify a REAL headless run after touching any of this: the
174
+ // unit and integration suites mock the browser, so a Juggler mismatch
175
+ // reaches production with a fully green suite.
160
176
  let CamoufoxModule: any;
161
177
  try {
162
178
  CamoufoxModule = await import('camoufox-js');
@@ -99,7 +99,10 @@ var init_log_utils = __esm({
99
99
  `${B_LEFT}(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})${B_RIGHT}`,
100
100
  "g"
101
101
  );
102
- JWT_PATTERN = new RegExp(`${B_LEFT}eyJ[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+`, "g");
102
+ JWT_PATTERN = new RegExp(
103
+ `${B_LEFT}eyJ[A-Za-z0-9_-]{1,1024}\\.[A-Za-z0-9_-]{1,8192}\\.[A-Za-z0-9_-]{1,4096}`,
104
+ "g"
105
+ );
103
106
  BASIC_AUTH_PATTERN = new RegExp(`${B_LEFT}Basic\\s+[A-Za-z0-9+/]{16,}={0,2}`, "gi");
104
107
  BEARER_AUTH_PATTERN = new RegExp(`${B_LEFT}Bearer\\s+[A-Za-z0-9._~+/-]+=*`, "gi");
105
108
  PROVIDER_TOKEN_PATTERN = new RegExp(
@@ -1509,6 +1512,7 @@ var scraper_utils_exports = {};
1509
1512
  __export(scraper_utils_exports, {
1510
1513
  createJsMarkdownConverter: () => createJsMarkdownConverter,
1511
1514
  createNativeMarkdownConverter: () => createNativeMarkdownConverter,
1515
+ disposeSsrfSafeFetcher: () => disposeSsrfSafeFetcher,
1512
1516
  extractDomain: () => extractDomain,
1513
1517
  formatErrorWithCause: () => formatErrorWithCause,
1514
1518
  getRandomUserAgent: () => getRandomUserAgent,
@@ -1644,19 +1648,33 @@ function isSsrfBlockError(error) {
1644
1648
  }
1645
1649
  return false;
1646
1650
  }
1647
- async function getSsrfSafeFetcher() {
1648
- if (cachedSsrfFetcher !== void 0) return cachedSsrfFetcher;
1651
+ function getSsrfSafeFetcher() {
1652
+ cachedSsrfFetcher ??= (async () => {
1653
+ try {
1654
+ const { Agent, fetch: undiciFetch } = await import("undici");
1655
+ return {
1656
+ fetch: undiciFetch,
1657
+ dispatcher: new Agent({ connect: { lookup: ssrfSafeLookup } })
1658
+ };
1659
+ } catch (e) {
1660
+ logger.warn(`[Scrapers] undici unavailable; connect-time SSRF pinning disabled (request-time validation still active): ${e instanceof Error ? e.message : String(e)}`);
1661
+ return null;
1662
+ }
1663
+ })();
1664
+ return cachedSsrfFetcher;
1665
+ }
1666
+ async function disposeSsrfSafeFetcher() {
1667
+ const pending = cachedSsrfFetcher;
1668
+ cachedSsrfFetcher = void 0;
1669
+ if (pending === void 0) return;
1670
+ const cached = await pending.catch(() => null);
1671
+ const closable = cached?.dispatcher;
1672
+ if (!closable?.close) return;
1649
1673
  try {
1650
- const { Agent, fetch: undiciFetch } = await import("undici");
1651
- cachedSsrfFetcher = {
1652
- fetch: undiciFetch,
1653
- dispatcher: new Agent({ connect: { lookup: ssrfSafeLookup } })
1654
- };
1674
+ await closable.close();
1655
1675
  } catch (e) {
1656
- logger.warn(`[Scrapers] undici unavailable; connect-time SSRF pinning disabled (request-time validation still active): ${e instanceof Error ? e.message : String(e)}`);
1657
- cachedSsrfFetcher = null;
1676
+ logger.debug(`[Scrapers] Error closing SSRF dispatcher: ${e instanceof Error ? e.message : String(e)}`);
1658
1677
  }
1659
- return cachedSsrfFetcher;
1660
1678
  }
1661
1679
  function formatErrorWithCause(err) {
1662
1680
  try {
@@ -2544,15 +2562,6 @@ var ConfigSchema = Type.Object({
2544
2562
  * the model's real maxTokens at call time.
2545
2563
  * Not exposed in TUI — controlled via PI_RESEARCH_SYNTHESIS_MAX_TOKENS env var. */
2546
2564
  SYNTHESIS_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 32768 }),
2547
- /** Max OUTPUT tokens per researcher LLM call (deep and quick sub-agent sessions).
2548
- * Without a cap the session inherits the model's full catalog ceiling as the
2549
- * per-request max_tokens — ~230k for large-context models — which providers that
2550
- * budget-check max_tokens up front (OpenRouter 402s against remaining credits)
2551
- * reject outright on low balances, even though the actual response would cost
2552
- * cents. Default 16384: researcher turns are tool calls plus a cited report, far
2553
- * below this. Clamped to the model's real ceiling at session build time.
2554
- * Not exposed in TUI — controlled via PI_RESEARCH_RESEARCHER_MAX_TOKENS env var. */
2555
- RESEARCHER_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 16384 }),
2556
2565
  /** LLM Model override for researcher sub-agents and knowledge synthesis.
2557
2566
  * Format: provider/model-id (e.g. google/gemini-2.0-flash-001) or just model-id.
2558
2567
  * When set, this overrides ctx.model for researcher sub-agents (both deep and quick)
@@ -2589,7 +2598,8 @@ var ConfigSchema = Type.Object({
2589
2598
  /**
2590
2599
  * Comma-separated list of research tools to DISABLE for every researcher in a run
2591
2600
  * (e.g. "scrape", or "youtube_transcript,stackexchange"). Valid tool names:
2592
- * search, scrape, security_search, stackexchange, youtube_transcript, grep.
2601
+ * search, scrape, security_search, stackexchange, youtube_transcript, grep, read
2602
+ * (RESEARCH_TOOL_NAMES in constants.ts is the canonical list).
2593
2603
  * Disable-only: the list is merged into the per-run excludeTools stream, so disabled
2594
2604
  * tools are both removed from each researcher's toolset AND named in the
2595
2605
  * coordinator/evaluator "DISABLED TOOLS" prompt section. Env-only
@@ -27,7 +27,21 @@ export class GPUResourceService implements IGPUResourceService {
27
27
  lifecycle = ServiceLifecycle.UNINITIALIZED;
28
28
 
29
29
  private processLifecycle: IProcessLifecycle | null = null;
30
- private gpuLockStaleThresholdMs: number = 1000 * 60 * 5; // 5 minutes default
30
+ /**
31
+ * Backstop for a holder that is alive but wedged. Step 3 already reclaims from
32
+ * *dead* owners via PID+startTime liveness, so this only ever fires against a
33
+ * process we just proved is running — which makes it a lock STEAL, and the
34
+ * threshold must therefore exceed the longest legitimate hold.
35
+ *
36
+ * It previously did not: at 5 minutes it exactly equalled the default
37
+ * EMBEDDING_MODEL_INIT_TIMEOUT_MS (300000, schema max 600000), and the
38
+ * embedder holds this lock across the whole model download + pipeline load.
39
+ * A first-time HuggingFace fetch running near its own permitted budget would
40
+ * have the GPU lock taken out from under it, admitting exactly the concurrent
41
+ * WebGPU/Dawn init this lock exists to prevent. Keep this strictly above that
42
+ * schema maximum.
43
+ */
44
+ private gpuLockStaleThresholdMs: number = 1000 * 60 * 15; // 15 min > EMBEDDING_MODEL_INIT_TIMEOUT_MS max (10 min)
31
45
 
32
46
  constructor(options?: GPUResourceServiceOptions) {
33
47
  if (options) {