@lincoln504/pi-research 1.4.1 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/README.md +1 -1
  2. package/dist/cli.mjs +449 -187
  3. package/dist/prompts/research-knowledge-relevance-triage.md +2 -0
  4. package/dist/prompts/research-knowledge-search-extractor.md +2 -0
  5. package/dist/prompts/researcher.md +2 -14
  6. package/dist/thread-worker.mjs +91 -59
  7. package/docs/AGENT-SKILL.md +4 -3
  8. package/docs/CHANGELOG.md +120 -0
  9. package/docs/CONFIGURATION.md +2 -2
  10. package/docs/SDK.md +1 -1
  11. package/package.json +1 -1
  12. package/skills/pi-research/SKILL.md +1 -1
  13. package/src/cli.ts +44 -3
  14. package/src/config.ts +36 -13
  15. package/src/constants.ts +23 -4
  16. package/src/core/llm/research-model-resolver.ts +38 -0
  17. package/src/core/planning-service.ts +67 -9
  18. package/src/index.ts +3 -1
  19. package/src/infrastructure/browser/browser-task-scheduler.ts +16 -9
  20. package/src/infrastructure/browser/config.ts +40 -2
  21. package/src/infrastructure/browser/thread-worker-browser.ts +69 -48
  22. package/src/infrastructure/browser/thread-worker-messaging.ts +65 -4
  23. package/src/infrastructure/browser/thread-worker.mjs +91 -59
  24. package/src/infrastructure/browser/thread-worker.ts +13 -5
  25. package/src/infrastructure/embedding/embedding-server.ts +54 -0
  26. package/src/knowledge/embedder-init.ts +3 -2
  27. package/src/knowledge/embedder.ts +11 -3
  28. package/src/knowledge/index.ts +36 -10
  29. package/src/knowledge/store.ts +85 -5
  30. package/src/observers/research-observer-impl.ts +14 -6
  31. package/src/orchestration/deep-research-orchestrator.ts +3 -1
  32. package/src/orchestration/quick-research-orchestrator.ts +29 -10
  33. package/src/orchestration/research-synthesis-service.ts +32 -0
  34. package/src/orchestration/researcher-executor.ts +31 -16
  35. package/src/orchestration/researcher.ts +2 -2
  36. package/src/orchestration/session-state.ts +35 -4
  37. package/src/prompts/research-knowledge-relevance-triage.md +2 -0
  38. package/src/prompts/research-knowledge-search-extractor.md +2 -0
  39. package/src/prompts/researcher.md +2 -14
  40. package/src/sdk.ts +16 -0
  41. package/src/skill-install/skill-installer.ts +5 -1
  42. package/src/stackexchange/index.ts +3 -0
  43. package/src/stackexchange/rest-client.ts +7 -12
  44. package/src/tools/index.ts +2 -1
  45. package/src/tools/research-knowledge-search.ts +37 -4
  46. package/src/tools/research-tool-definition.ts +54 -12
  47. package/src/tools/scrape.ts +8 -1
  48. package/src/tools/security.ts +13 -10
  49. package/src/utils/coverage-digest.ts +1 -1
  50. package/src/utils/host-config.ts +12 -1
  51. package/src/utils/log-utils.ts +15 -2
  52. package/src/utils/metrics-summary.ts +21 -1
  53. package/src/utils/tool-usage-tracker.ts +6 -3
  54. package/src/web-research/browser-search.ts +11 -4
  55. package/src/web-research/web-scraper.ts +36 -16
  56. package/src/youtube/index.ts +11 -1
  57. package/src/youtube/potoken.ts +8 -1
  58. package/src/youtube/transcript-client.ts +56 -5
@@ -11,21 +11,23 @@
11
11
  import { platform } from 'node:os';
12
12
  import { setupMocking } from './thread-worker-messaging.ts';
13
13
  import { redactSecrets } from '../../utils/log-utils.ts';
14
- import { resolveHeadlessMode } from './config.ts';
14
+ import { resolveHeadlessMode, BROWSER_LAUNCH_TIMEOUT_MS, CONTEXT_CREATION_TIMEOUT_MS } from './config.ts';
15
15
 
16
16
  let browser: any = null;
17
- let context: any = null;
18
17
  let initPromise: Promise<void> | null = null;
19
18
  let workerId: string = '';
20
19
 
21
20
  // How many tasks this worker process currently has in flight. poolifier's
22
21
  // tasksQueueOptions.concurrency (WORKER_CONCURRENCY, default 2 — see
23
22
  // worker-pool-manager.ts) dispatches more than one task at a time to the SAME
24
- // worker process, and every dispatched task shares this module's browser/
25
- // context singletons (each gets its own page via createPageSafe(context), but
26
- // the context/browser themselves are one shared instance). resetBrowser()
27
- // uses this to avoid tearing that shared instance down out from under a
28
- // sibling task that is still actively using it — see resetBrowser() below.
23
+ // worker process, and every dispatched task shares this module's browser
24
+ // singleton (each gets its own FRESH BrowserContext via acquireTaskContext()
25
+ // below — a real Firefox process is too expensive to launch per task, but a
26
+ // context is cheap, and per-task contexts keep cookies/storage/route-mocking
27
+ // from one task bleeding into a concurrent or later, unrelated one).
28
+ // resetBrowser() uses this to avoid tearing the shared BROWSER down out from
29
+ // under a sibling task that is still actively using it — see resetBrowser()
30
+ // below.
29
31
  let activeTaskCount = 0;
30
32
  // A reset that resetBrowser() deferred because sibling tasks were still
31
33
  // active, to be applied once the last one finishes.
@@ -119,12 +121,12 @@ function logToDebugFile(level: string, ...args: any[]): void {
119
121
  * Initialize the browser instance
120
122
  */
121
123
  export async function initBrowser(): Promise<void> {
122
- if (isBrowserConnected() && context) return;
124
+ if (isBrowserConnected()) return;
123
125
  if (initPromise) return initPromise;
124
126
 
125
127
  initPromise = (async () => {
126
128
  try {
127
- if (!isBrowserConnected() || !context) {
129
+ if (!isBrowserConnected()) {
128
130
  logToDebugFile('INFO', `[Worker-${workerId}] Initializing browser instance...`);
129
131
 
130
132
  // ---------------------------------------------------------------------
@@ -196,8 +198,12 @@ export async function initBrowser(): Promise<void> {
196
198
  //
197
199
  // 90s launch timeout: CI runners (2 vCPU) need up to 60s to start Firefox
198
200
  // when it has both CPUs. The old 45s limit was too tight and caused silent
199
- // init failures under resource pressure.
200
- const launchTimeoutMs = 90000;
201
+ // init failures under resource pressure. Imported from config.ts (single
202
+ // source of truth) rather than a local literal — the scheduler's task
203
+ // deadlines (browser-task-scheduler.ts) and the healthcheck budget
204
+ // (getHealthCheckBudgetMs) both have to budget for this exact worst case,
205
+ // via COLD_START_ALLOWANCE_MS.
206
+ const launchTimeoutMs = BROWSER_LAUNCH_TIMEOUT_MS;
201
207
  const launchOnce = async (headless: boolean | 'virtual') => {
202
208
  const launchPromise = Camoufox({
203
209
  headless,
@@ -295,35 +301,6 @@ export async function initBrowser(): Promise<void> {
295
301
  }
296
302
 
297
303
  browser = launchedBrowser;
298
- // newContext() can hang if the browser process becomes unresponsive immediately
299
- // after launch (e.g. OOM, GPU crash). Guard with a hard timeout.
300
- const contextPromise = browser.newContext();
301
- contextPromise.catch((err: Error) => logToDebugFile('DEBUG', `[ThreadWorker] Background browser context rejection: ${err.message}`));
302
- let contextTimeoutId: NodeJS.Timeout | undefined;
303
- let contextRaceLost = false;
304
- // Same adoption as the launch above. Here the catch below CAN close the
305
- // browser (it is assigned by now), which disposes the stray context with it —
306
- // but only when initBrowser is entered with a dead browser, and it leaves the
307
- // context open for the window in between. Closing it directly is unconditional.
308
- contextPromise.then((c: any) => {
309
- if (!contextRaceLost) return;
310
- logToDebugFile('WARN', `[Worker-${workerId}] Browser context creation completed after its deadline; closing the orphaned context.`);
311
- try { c?.close?.(); } catch { /* already gone */ }
312
- }).catch(() => { /* rejection already logged above */ });
313
- context = await Promise.race([
314
- contextPromise,
315
- new Promise<never>((_, reject) => {
316
- contextTimeoutId = setTimeout(() => {
317
- contextRaceLost = true;
318
- reject(new Error('Browser context creation timed out after 30000ms'));
319
- }, 30000);
320
- })
321
- ]);
322
- if (contextTimeoutId) clearTimeout(contextTimeoutId);
323
-
324
- // Setup mocking for CI if enabled
325
- await setupMocking(context);
326
-
327
304
  logToDebugFile('INFO', `[Worker-${workerId}] Browser initialized.`);
328
305
  }
329
306
  } catch (e: unknown) {
@@ -334,7 +311,6 @@ export async function initBrowser(): Promise<void> {
334
311
  Promise.resolve(browser.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed browser close error during failed init: ${err.message}`));
335
312
  }
336
313
  browser = null;
337
- context = null;
338
314
  const msg = e instanceof Error ? e.message : String(e);
339
315
 
340
316
  if (msg.includes('Camoufox is not installed') || msg.includes('Version information not found')) {
@@ -361,16 +337,60 @@ export async function initBrowser(): Promise<void> {
361
337
  }
362
338
 
363
339
  /**
364
- * Get the current browser context
340
+ * Create a FRESH BrowserContext for one task, against the shared browser
341
+ * process. Call once per task, right after initBrowser() resolves; the
342
+ * caller owns the returned context and must close it itself (in a `finally`,
343
+ * covering the success, error, AND timeout-abort paths) once the task ends.
344
+ *
345
+ * Never reused across tasks — that was the bug this replaced. Two tasks
346
+ * dispatched concurrently to the same worker (WORKER_CONCURRENCY > 1, the
347
+ * default) used to share one BrowserContext, so cookies/storage a site set
348
+ * for one task's request were visible to a different, unrelated task's later
349
+ * request to the same site — a content-consistency gap with no counterpart
350
+ * anywhere else in this codebase, which scopes browser-adjacent state
351
+ * (circuit breakers, scrape caches) per session deliberately.
352
+ *
353
+ * The launch race/timeout/orphan-adoption pattern below is the exact one
354
+ * initBrowser() used when context creation lived there (moved here wholesale,
355
+ * not reinvented): newContext() can hang if the browser process is
356
+ * unresponsive (OOM, GPU crash), so it is guarded with a hard timeout, and a
357
+ * context that finishes creating AFTER that timeout is adopted and closed
358
+ * rather than left to leak.
365
359
  */
366
- export function getContext(): any {
367
- return context;
360
+ export async function acquireTaskContext(): Promise<any> {
361
+ const contextPromise = browser.newContext();
362
+ contextPromise.catch((err: Error) => logToDebugFile('DEBUG', `[ThreadWorker] Background browser context rejection: ${err.message}`));
363
+ let contextTimeoutId: NodeJS.Timeout | undefined;
364
+ let contextRaceLost = false;
365
+ contextPromise.then((c: any) => {
366
+ if (!contextRaceLost) return;
367
+ logToDebugFile('WARN', `[Worker-${workerId}] Browser context creation completed after its deadline; closing the orphaned context.`);
368
+ try { c?.close?.(); } catch { /* already gone */ }
369
+ }).catch(() => { /* rejection already logged above */ });
370
+ let newContext: any;
371
+ try {
372
+ newContext = await Promise.race([
373
+ contextPromise,
374
+ new Promise<never>((_, reject) => {
375
+ contextTimeoutId = setTimeout(() => {
376
+ contextRaceLost = true;
377
+ reject(new Error(`Browser context creation timed out after ${CONTEXT_CREATION_TIMEOUT_MS}ms`));
378
+ }, CONTEXT_CREATION_TIMEOUT_MS);
379
+ })
380
+ ]);
381
+ } finally {
382
+ if (contextTimeoutId) clearTimeout(contextTimeoutId);
383
+ }
384
+
385
+ // Route interception is per-context, so mocking must be (re-)applied to
386
+ // every fresh context, not just the first one ever created.
387
+ await setupMocking(newContext);
388
+
389
+ return newContext;
368
390
  }
369
391
 
370
392
  function doResetBrowser(): void {
371
- if (context) Promise.resolve(context.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed context close error during reset: ${err.message}`));
372
393
  if (browser) Promise.resolve(browser.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed browser close error during reset: ${err.message}`));
373
- context = null;
374
394
  browser = null;
375
395
  }
376
396
 
@@ -407,7 +427,9 @@ export async function cleanupBrowser(): Promise<void> {
407
427
  const timeoutMs = 2000;
408
428
 
409
429
  const cleanup = async () => {
410
- if (context) await Promise.resolve(context.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed context close error during cleanup: ${err.message}`));
430
+ // Closes every context still open under it (any task whose own close()
431
+ // hasn't run yet) as a side effect — no separate context bookkeeping
432
+ // needed here now that contexts are task-owned, not module-owned.
411
433
  if (browser) await Promise.resolve(browser.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed browser close error during cleanup: ${err.message}`));
412
434
  };
413
435
 
@@ -419,7 +441,6 @@ export async function cleanupBrowser(): Promise<void> {
419
441
  } catch {
420
442
  // Ignore timeout — we just want to ensure cleanup doesn't hang indefinitely
421
443
  } finally {
422
- context = null;
423
444
  browser = null;
424
445
  }
425
446
  }
@@ -433,6 +433,13 @@ export async function executeScrapeTask(
433
433
  let poisonedError: Error | null = null;
434
434
  const pendingAddrChecks: Array<Promise<void>> = [];
435
435
 
436
+ // Set by the response-side early-abort check (see page.on('response') below) when the
437
+ // main-frame document's OWN headers already declare a PDF larger than MAX_PDF_SIZE.
438
+ // Declared outside the try for the same reason as poisonedError: the catch below needs
439
+ // to surface this instead of the secondary 'Target closed' error the triggering
440
+ // page.close() produces.
441
+ let oversizedPdfError: Error | null = null;
442
+
436
443
  try {
437
444
  logToDebugFile('DEBUG', `[Worker-${workerId}] Starting scrape for: ${url}`);
438
445
 
@@ -511,6 +518,46 @@ export async function executeScrapeTask(
511
518
  try {
512
519
  if (resp.request().resourceType() === 'document' && resp.frame() === page.mainFrame()) {
513
520
  finalMainFrameAddr = resp.serverAddr().catch(() => null);
521
+
522
+ // Early-abort a declared-oversized PDF as soon as its headers arrive, rather
523
+ // than waiting for page.goto() to finish downloading the whole thing first.
524
+ // 'response' fires on headers-received, well before the body is fully
525
+ // transferred — for a direct navigation to a PDF URL, Playwright's goto()
526
+ // does not resolve until the download completes, so without this the full
527
+ // (possibly multi-hundred-MB) body is pulled into the browser process before
528
+ // the PDF branch below ever gets to check Content-Length.
529
+ //
530
+ // This closes the gap only for a server that HONESTLY declares an oversized
531
+ // Content-Length up front. A chunked-transfer response (no Content-Length) or
532
+ // one that understates its length and then sends far more cannot be caught
533
+ // this way — Playwright's public API exposes no incremental
534
+ // bytes-received/streaming hook to abort mid-body on either engine. That
535
+ // residual case is bounded instead by the task's own SCRAPE_TIMEOUT_MS +
536
+ // BROWSER_TASK_TIMEOUT_MS deadline (see the onAbort/page.close() wiring
537
+ // above), which forcibly closes the page — and with it any in-flight
538
+ // download — once the task's time budget is exhausted, converting an
539
+ // otherwise-unbounded read into one bounded by (bandwidth x that budget).
540
+ const check2: Promise<void> = Promise.resolve(resp.headerValue('content-type'))
541
+ .then(async (ct: string | null) => {
542
+ // Matches the PDF branch below AND the fetch layer's own detection
543
+ // (web-scraper.ts): a server can mislabel or omit content-type, so a
544
+ // `.pdf` URL extension is an equally-trusted signal, not just the header.
545
+ // resp.url() — THIS response's URL, not the original task URL: goto()
546
+ // follows redirects, and a download endpoint without `.pdf` that
547
+ // redirects to the real `.pdf` file must still arm the early abort
548
+ // (same redirect-aware fix the fetch layer got via currentUrl).
549
+ const looksLikePdf = ct?.includes('application/pdf') || String(resp.url()).toLowerCase().endsWith('.pdf');
550
+ if (!looksLikePdf || oversizedPdfError || poisonedError) return;
551
+ const declared = parseInt((await resp.headerValue('content-length').catch(() => null)) || '', 10);
552
+ if (Number.isFinite(declared) && declared > MAX_PDF_SIZE) {
553
+ const sizeMB = Math.round(declared / 1024 / 1024);
554
+ oversizedPdfError = new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
555
+ logToDebugFile('DEBUG', `[Worker-${workerId}] Aborting oversized PDF (${sizeMB}MB declared) before download completes: ${url}`);
556
+ page.close().catch(() => {});
557
+ }
558
+ })
559
+ .catch(() => {});
560
+ pendingAddrChecks.push(check2);
514
561
  }
515
562
  const check: Promise<void> = Promise.resolve(resp.serverAddr())
516
563
  .then((addr: { ipAddress?: string } | null) => {
@@ -562,7 +609,14 @@ export async function executeScrapeTask(
562
609
  // 403/503 — the HTML branch defers to the challenge wait below for that.
563
610
  const status = response?.status?.() ?? 0;
564
611
 
565
- if (contentType.includes('application/pdf')) {
612
+ // `.pdf` extension is an equally-trusted signal alongside the header (matches
613
+ // the fetch layer's own detection in web-scraper.ts): a server that mislabels
614
+ // or omits content-type would otherwise fall into the HTML branch below, whose
615
+ // page.content() would serialize Firefox's PDF-viewer chrome instead of
616
+ // extracting the document — a silent content loss for exactly the URLs this
617
+ // feature advertises as supported, and specifically for the malformed/oddly-
618
+ // served PDFs most likely to reach this browser fallback in the first place.
619
+ if (contentType.includes('application/pdf') || url.toLowerCase().endsWith('.pdf')) {
566
620
  if (!response) throw new Error(`[Worker] No response received for PDF URL: ${url}`);
567
621
  // An errored "PDF" is an error page: fail BEFORE buffering its body.
568
622
  if (status >= 400) throw new Error(`HTTP ${status}`);
@@ -576,6 +630,10 @@ export async function executeScrapeTask(
576
630
  const sizeMB = Math.round(declaredLength / 1024 / 1024);
577
631
  throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
578
632
  }
633
+ // The response-side early-abort check above (page.on('response')) may already have
634
+ // decided this and be in the middle of closing the page — skip the (redundant,
635
+ // about-to-fail) download attempt rather than race it.
636
+ if (oversizedPdfError) throw oversizedPdfError;
579
637
  const buffer = await response.body();
580
638
  // ...then the actual bytes: the header is advisory and absent on chunked
581
639
  // responses, so the byte count is the check that actually bounds memory.
@@ -692,9 +750,12 @@ export async function executeScrapeTask(
692
750
  return { contentType, html, jitter };
693
751
  } catch (error) {
694
752
  await page.close().catch((err: any) => logToDebugFile('DEBUG', `[ThreadWorker] Swallowed page close/wait error: ${err.message || String(err)}`));
695
- // A poison-triggered page.close() makes the in-flight goto/content() reject
696
- // with 'Target closed'; report the SSRF poisoning (a benign-classified
697
- // 'Fetch blocked:' failure), not the secondary teardown error.
753
+ // An early-abort-triggered page.close() (oversized PDF, or SSRF poisoning) makes
754
+ // the in-flight goto/content()/body() reject with 'Target closed'; report the
755
+ // actual reason (a benign-classified failure) instead of that secondary teardown
756
+ // error. oversizedPdfError takes priority since it can only be set on the PDF path,
757
+ // where poisonedError (an HTML-only DOM-exfil concern) is not meaningful.
758
+ if (oversizedPdfError) throw oversizedPdfError;
698
759
  if (poisonedError) throw poisonedError;
699
760
  throw error;
700
761
  } finally {
@@ -71,6 +71,8 @@ function redactSecrets(message) {
71
71
  }
72
72
  if (out.length > MAX_LOG_MESSAGE_LENGTH) {
73
73
  out = `${out.slice(0, keep)}\u2026[truncated ${Math.max(rawLength, out.length) - keep} chars]`;
74
+ } else if (rawLength > scanEnd) {
75
+ out = `${out}\u2026[truncated ${rawLength - scanEnd} chars]`;
74
76
  }
75
77
  return out;
76
78
  }
@@ -247,7 +249,7 @@ var init_error_tracker = __esm({
247
249
  * @param error - The error to track (Error object, string message, or unknown)
248
250
  * @param context - Optional context information about where the error occurred
249
251
  */
250
- trackError(error, context2 = {}) {
252
+ trackError(error, context = {}) {
251
253
  let message;
252
254
  if (error instanceof Error) {
253
255
  message = error.message;
@@ -276,15 +278,15 @@ var init_error_tracker = __esm({
276
278
  }
277
279
  pattern.count++;
278
280
  pattern.lastSeen = now;
279
- if (context2.domain) {
280
- const existing = pattern.domainCounts.get(context2.domain);
281
+ if (context.domain) {
282
+ const existing = pattern.domainCounts.get(context.domain);
281
283
  if (existing !== void 0) {
282
- pattern.domainCounts.set(context2.domain, existing + 1);
284
+ pattern.domainCounts.set(context.domain, existing + 1);
283
285
  } else if (pattern.domainCounts.size < this.MAX_DOMAINS_PER_PATTERN) {
284
- pattern.domainCounts.set(context2.domain, 1);
286
+ pattern.domainCounts.set(context.domain, 1);
285
287
  }
286
288
  }
287
- pattern.contexts.push(context2);
289
+ pattern.contexts.push(context);
288
290
  if (pattern.contexts.length > this.MAX_CONTEXTS_PER_PATTERN) {
289
291
  pattern.contexts.shift();
290
292
  }
@@ -1109,8 +1111,8 @@ var init_logger = __esm({
1109
1111
  error(...args) {
1110
1112
  const errArg = args.find((arg) => arg instanceof Error);
1111
1113
  if (errArg) {
1112
- const context2 = getLogContext();
1113
- errorTracker.trackError(errArg, context2);
1114
+ const context = getLogContext();
1115
+ errorTracker.trackError(errArg, context);
1114
1116
  }
1115
1117
  this.emit("ERROR" /* ERROR */, ...args);
1116
1118
  }
@@ -2025,7 +2027,7 @@ var workerId2 = "";
2025
2027
  var pageCreationLock = Promise.resolve();
2026
2028
  var PAGE_CREATION_HARD_RELEASE_MS = 5 * 60 * 1e3;
2027
2029
  var wedgedContext = null;
2028
- async function createPageSafe(context2) {
2030
+ async function createPageSafe(context) {
2029
2031
  let release;
2030
2032
  const nextLock = new Promise((resolve) => {
2031
2033
  release = resolve;
@@ -2033,11 +2035,11 @@ async function createPageSafe(context2) {
2033
2035
  const currentLock = pageCreationLock;
2034
2036
  pageCreationLock = currentLock.then(() => nextLock);
2035
2037
  await currentLock;
2036
- if (context2 === wedgedContext) {
2038
+ if (context === wedgedContext) {
2037
2039
  release();
2038
2040
  throw new Error("Browser context is wedged from a prior page-creation call that never settled; a new browser context is required.");
2039
2041
  }
2040
- const pagePromise = context2.newPage();
2042
+ const pagePromise = context.newPage();
2041
2043
  let hardReleased = false;
2042
2044
  const releaseOnce = () => {
2043
2045
  if (hardReleased) return;
@@ -2047,17 +2049,17 @@ async function createPageSafe(context2) {
2047
2049
  };
2048
2050
  const hardReleaseTimer = setTimeout(() => {
2049
2051
  logToDebugFile2("WARN", "[ThreadWorker] page-creation mutex force-released \u2014 newPage() did not settle within the hard timeout");
2050
- wedgedContext = context2;
2052
+ wedgedContext = context;
2051
2053
  releaseOnce();
2052
2054
  }, PAGE_CREATION_HARD_RELEASE_MS);
2053
2055
  safeUnref(hardReleaseTimer);
2054
2056
  pagePromise.then(
2055
2057
  () => {
2056
- if (wedgedContext === context2) wedgedContext = null;
2058
+ if (wedgedContext === context) wedgedContext = null;
2057
2059
  releaseOnce();
2058
2060
  },
2059
2061
  () => {
2060
- if (wedgedContext === context2) wedgedContext = null;
2062
+ if (wedgedContext === context) wedgedContext = null;
2061
2063
  releaseOnce();
2062
2064
  }
2063
2065
  );
@@ -2119,12 +2121,12 @@ function logToDebugFile2(level, ...args) {
2119
2121
  } catch {
2120
2122
  }
2121
2123
  }
2122
- async function setupMocking(context2) {
2124
+ async function setupMocking(context) {
2123
2125
  const mockSearch = process.env["PI_RESEARCH_MOCK_SEARCH"] === "true";
2124
2126
  const mockScrape = process.env["PI_RESEARCH_MOCK_SCRAPE"] === "true";
2125
2127
  if (!mockSearch && !mockScrape) return;
2126
2128
  logToDebugFile2("INFO", `[Worker-${workerId2}] Setting up browser mocking (Search: ${mockSearch}, Scrape: ${mockScrape})`);
2127
- await context2.route("**", (route, request) => {
2129
+ await context.route("**", (route, request) => {
2128
2130
  const urlStr = request.url();
2129
2131
  let url;
2130
2132
  try {
@@ -2297,6 +2299,7 @@ async function executeScrapeTask(_context, url, signal) {
2297
2299
  }
2298
2300
  let poisonedError = null;
2299
2301
  const pendingAddrChecks = [];
2302
+ let oversizedPdfError = null;
2300
2303
  try {
2301
2304
  logToDebugFile2("DEBUG", `[Worker-${workerId2}] Starting scrape for: ${url}`);
2302
2305
  const { validateUrlForSSRF: validateForWorker, validateUrlForSSRFSync: validateForWorkerSync } = await Promise.resolve().then(() => (init_scraper_utils(), scraper_utils_exports));
@@ -2328,6 +2331,20 @@ async function executeScrapeTask(_context, url, signal) {
2328
2331
  try {
2329
2332
  if (resp.request().resourceType() === "document" && resp.frame() === page.mainFrame()) {
2330
2333
  finalMainFrameAddr = resp.serverAddr().catch(() => null);
2334
+ const check2 = Promise.resolve(resp.headerValue("content-type")).then(async (ct) => {
2335
+ const looksLikePdf = ct?.includes("application/pdf") || String(resp.url()).toLowerCase().endsWith(".pdf");
2336
+ if (!looksLikePdf || oversizedPdfError || poisonedError) return;
2337
+ const declared = parseInt(await resp.headerValue("content-length").catch(() => null) || "", 10);
2338
+ if (Number.isFinite(declared) && declared > MAX_PDF_SIZE) {
2339
+ const sizeMB = Math.round(declared / 1024 / 1024);
2340
+ oversizedPdfError = new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
2341
+ logToDebugFile2("DEBUG", `[Worker-${workerId2}] Aborting oversized PDF (${sizeMB}MB declared) before download completes: ${url}`);
2342
+ page.close().catch(() => {
2343
+ });
2344
+ }
2345
+ }).catch(() => {
2346
+ });
2347
+ pendingAddrChecks.push(check2);
2331
2348
  }
2332
2349
  const check = Promise.resolve(resp.serverAddr()).then((addr) => {
2333
2350
  const ip = addr?.ipAddress;
@@ -2357,7 +2374,7 @@ async function executeScrapeTask(_context, url, signal) {
2357
2374
  }
2358
2375
  const contentType = await response?.headerValue("content-type") || "";
2359
2376
  const status = response?.status?.() ?? 0;
2360
- if (contentType.includes("application/pdf")) {
2377
+ if (contentType.includes("application/pdf") || url.toLowerCase().endsWith(".pdf")) {
2361
2378
  if (!response) throw new Error(`[Worker] No response received for PDF URL: ${url}`);
2362
2379
  if (status >= 400) throw new Error(`HTTP ${status}`);
2363
2380
  const declaredLength = parseInt(await response.headerValue("content-length").catch(() => null) || "", 10);
@@ -2365,6 +2382,7 @@ async function executeScrapeTask(_context, url, signal) {
2365
2382
  const sizeMB = Math.round(declaredLength / 1024 / 1024);
2366
2383
  throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
2367
2384
  }
2385
+ if (oversizedPdfError) throw oversizedPdfError;
2368
2386
  const buffer = await response.body();
2369
2387
  if (buffer.byteLength > MAX_PDF_SIZE) {
2370
2388
  const sizeMB = Math.round(buffer.byteLength / 1024 / 1024);
@@ -2422,6 +2440,7 @@ async function executeScrapeTask(_context, url, signal) {
2422
2440
  return { contentType, html, jitter };
2423
2441
  } catch (error) {
2424
2442
  await page.close().catch((err) => logToDebugFile2("DEBUG", `[ThreadWorker] Swallowed page close/wait error: ${err.message || String(err)}`));
2443
+ if (oversizedPdfError) throw oversizedPdfError;
2425
2444
  if (poisonedError) throw poisonedError;
2426
2445
  throw error;
2427
2446
  } finally {
@@ -2566,10 +2585,16 @@ var ConfigSchema = Type.Object({
2566
2585
  SEARCH_TIMEOUT_MS: Type.Number({ minimum: 5e3, maximum: 12e4, default: 45e3 }),
2567
2586
  /** TUI refresh debounce in milliseconds (default: 100ms) */
2568
2587
  TUI_REFRESH_DEBOUNCE_MS: Type.Number({ minimum: 0, maximum: 1e3, default: 100 }),
2569
- /** Queue-wait / overhead margin (ms) added on top of each browser operation's own nav
2570
- * timeout to form its scheduler task-timeout ceiling: a search task times out after
2571
- * SEARCH_TIMEOUT_MS + this, a scrape after SCRAPE_TIMEOUT_MS + this. Sized so a task is
2572
- * never killed before it has used its full nav budget plus time waiting in the queue.
2588
+ /** Overhead margin (ms) added on top of each browser operation's own nav timeout to
2589
+ * form its scheduler task-timeout ceiling: a search task times out after
2590
+ * SEARCH_TIMEOUT_MS + this, a scrape after SCRAPE_TIMEOUT_MS + this — covering
2591
+ * per-task work beyond raw navigation (page setup, result parsing). Queue wait is
2592
+ * NOT part of this margin: the deadline is armed on dispatch, not on enqueue, so
2593
+ * time spent waiting for a free worker never counts against it (see
2594
+ * browser-task-scheduler.ts). Cold browser-launch/context-creation time is also
2595
+ * separate from this margin — COLD_START_ALLOWANCE_MS
2596
+ * (infrastructure/browser/config.ts) covers that unconditionally on top of it,
2597
+ * since it isn't something a user should need to tune.
2573
2598
  * (default: 10000) */
2574
2599
  BROWSER_TASK_TIMEOUT_MS: Type.Number({ minimum: 2e3, maximum: 12e4, default: 1e4 }),
2575
2600
  /** Timeout for coordinator/research-lead/repair/knowledge LLM calls in ms (default: 300000 = 5 min, range: 60s-30min).
@@ -2669,10 +2694,12 @@ function resolveHeadlessMode() {
2669
2694
  return true;
2670
2695
  }
2671
2696
  var HEALTHCHECK_MIN_BUDGET_MS = 45e3 + 6e4;
2697
+ var BROWSER_LAUNCH_TIMEOUT_MS = 9e4;
2698
+ var CONTEXT_CREATION_TIMEOUT_MS = 3e4;
2699
+ var COLD_START_ALLOWANCE_MS = BROWSER_LAUNCH_TIMEOUT_MS + CONTEXT_CREATION_TIMEOUT_MS;
2672
2700
 
2673
2701
  // src/infrastructure/browser/thread-worker-browser.ts
2674
2702
  var browser = null;
2675
- var context = null;
2676
2703
  var initPromise = null;
2677
2704
  var workerId3 = "";
2678
2705
  var activeTaskCount = 0;
@@ -2726,11 +2753,11 @@ function logToDebugFile3(level, ...args) {
2726
2753
  }
2727
2754
  }
2728
2755
  async function initBrowser() {
2729
- if (isBrowserConnected() && context) return;
2756
+ if (isBrowserConnected()) return;
2730
2757
  if (initPromise) return initPromise;
2731
2758
  initPromise = (async () => {
2732
2759
  try {
2733
- if (!isBrowserConnected() || !context) {
2760
+ if (!isBrowserConnected()) {
2734
2761
  logToDebugFile3("INFO", `[Worker-${workerId3}] Initializing browser instance...`);
2735
2762
  let CamoufoxModule;
2736
2763
  try {
@@ -2739,7 +2766,7 @@ async function initBrowser() {
2739
2766
  throw new Error(`[Worker] camoufox-js not found in node_modules. Please run 'npm install'. Original error: ${e instanceof Error ? e.message : String(e)}`, { cause: e });
2740
2767
  }
2741
2768
  const { Camoufox } = CamoufoxModule;
2742
- const launchTimeoutMs = 9e4;
2769
+ const launchTimeoutMs = BROWSER_LAUNCH_TIMEOUT_MS;
2743
2770
  const launchOnce = async (headless) => {
2744
2771
  const launchPromise = Camoufox({
2745
2772
  headless,
@@ -2822,30 +2849,6 @@ async function initBrowser() {
2822
2849
  }
2823
2850
  }
2824
2851
  browser = launchedBrowser;
2825
- const contextPromise = browser.newContext();
2826
- contextPromise.catch((err) => logToDebugFile3("DEBUG", `[ThreadWorker] Background browser context rejection: ${err.message}`));
2827
- let contextTimeoutId;
2828
- let contextRaceLost = false;
2829
- contextPromise.then((c) => {
2830
- if (!contextRaceLost) return;
2831
- logToDebugFile3("WARN", `[Worker-${workerId3}] Browser context creation completed after its deadline; closing the orphaned context.`);
2832
- try {
2833
- c?.close?.();
2834
- } catch {
2835
- }
2836
- }).catch(() => {
2837
- });
2838
- context = await Promise.race([
2839
- contextPromise,
2840
- new Promise((_, reject) => {
2841
- contextTimeoutId = setTimeout(() => {
2842
- contextRaceLost = true;
2843
- reject(new Error("Browser context creation timed out after 30000ms"));
2844
- }, 3e4);
2845
- })
2846
- ]);
2847
- if (contextTimeoutId) clearTimeout(contextTimeoutId);
2848
- await setupMocking(context);
2849
2852
  logToDebugFile3("INFO", `[Worker-${workerId3}] Browser initialized.`);
2850
2853
  }
2851
2854
  } catch (e) {
@@ -2853,7 +2856,6 @@ async function initBrowser() {
2853
2856
  Promise.resolve(browser.close()).catch((err) => logToDebugFile3("DEBUG", `[Worker-${workerId3}] Swallowed browser close error during failed init: ${err.message}`));
2854
2857
  }
2855
2858
  browser = null;
2856
- context = null;
2857
2859
  const msg = e instanceof Error ? e.message : String(e);
2858
2860
  if (msg.includes("Camoufox is not installed") || msg.includes("Version information not found")) {
2859
2861
  throw new Error(`[Worker] Browser binaries not found. Please run 'npx camoufox-js fetch' to install them.`, { cause: e });
@@ -2872,13 +2874,39 @@ async function initBrowser() {
2872
2874
  })();
2873
2875
  return initPromise;
2874
2876
  }
2875
- function getContext() {
2876
- return context;
2877
+ async function acquireTaskContext() {
2878
+ const contextPromise = browser.newContext();
2879
+ contextPromise.catch((err) => logToDebugFile3("DEBUG", `[ThreadWorker] Background browser context rejection: ${err.message}`));
2880
+ let contextTimeoutId;
2881
+ let contextRaceLost = false;
2882
+ contextPromise.then((c) => {
2883
+ if (!contextRaceLost) return;
2884
+ logToDebugFile3("WARN", `[Worker-${workerId3}] Browser context creation completed after its deadline; closing the orphaned context.`);
2885
+ try {
2886
+ c?.close?.();
2887
+ } catch {
2888
+ }
2889
+ }).catch(() => {
2890
+ });
2891
+ let newContext;
2892
+ try {
2893
+ newContext = await Promise.race([
2894
+ contextPromise,
2895
+ new Promise((_, reject) => {
2896
+ contextTimeoutId = setTimeout(() => {
2897
+ contextRaceLost = true;
2898
+ reject(new Error(`Browser context creation timed out after ${CONTEXT_CREATION_TIMEOUT_MS}ms`));
2899
+ }, CONTEXT_CREATION_TIMEOUT_MS);
2900
+ })
2901
+ ]);
2902
+ } finally {
2903
+ if (contextTimeoutId) clearTimeout(contextTimeoutId);
2904
+ }
2905
+ await setupMocking(newContext);
2906
+ return newContext;
2877
2907
  }
2878
2908
  function doResetBrowser() {
2879
- if (context) Promise.resolve(context.close()).catch((err) => logToDebugFile3("DEBUG", `[Worker-${workerId3}] Swallowed context close error during reset: ${err.message}`));
2880
2909
  if (browser) Promise.resolve(browser.close()).catch((err) => logToDebugFile3("DEBUG", `[Worker-${workerId3}] Swallowed browser close error during reset: ${err.message}`));
2881
- context = null;
2882
2910
  browser = null;
2883
2911
  }
2884
2912
  function resetBrowser() {
@@ -2892,7 +2920,6 @@ function resetBrowser() {
2892
2920
  async function cleanupBrowser2() {
2893
2921
  const timeoutMs = 2e3;
2894
2922
  const cleanup = async () => {
2895
- if (context) await Promise.resolve(context.close()).catch((err) => logToDebugFile3("DEBUG", `[Worker-${workerId3}] Swallowed context close error during cleanup: ${err.message}`));
2896
2923
  if (browser) await Promise.resolve(browser.close()).catch((err) => logToDebugFile3("DEBUG", `[Worker-${workerId3}] Swallowed browser close error during cleanup: ${err.message}`));
2897
2924
  };
2898
2925
  try {
@@ -2902,7 +2929,6 @@ async function cleanupBrowser2() {
2902
2929
  ]);
2903
2930
  } catch {
2904
2931
  } finally {
2905
- context = null;
2906
2932
  browser = null;
2907
2933
  }
2908
2934
  }
@@ -2951,20 +2977,22 @@ async function runTask(data) {
2951
2977
  taskTimer = setTimeout(() => abortController.abort(new Error(`Task timed out after ${taskTimeoutMs}ms`)), taskTimeoutMs);
2952
2978
  }
2953
2979
  taskStarted();
2980
+ let taskContext;
2954
2981
  try {
2955
2982
  await initBrowser();
2956
2983
  const initMs = Date.now() - startTime;
2984
+ taskContext = await acquireTaskContext();
2957
2985
  let result;
2958
2986
  if (type === "search") {
2959
2987
  if (!query) throw new Error("Search task requires a query");
2960
- const searchResult = await executeSearchTask(getContext(), query, abortController.signal);
2988
+ const searchResult = await executeSearchTask(taskContext, query, abortController.signal);
2961
2989
  result = { results: searchResult.results, duration: Date.now() - startTime, jitter: searchResult.jitter };
2962
2990
  } else if (type === "scrape") {
2963
2991
  if (!url) throw new Error("Scrape task requires a URL");
2964
- const scrapeResult = await executeScrapeTask(getContext(), url, abortController.signal);
2992
+ const scrapeResult = await executeScrapeTask(taskContext, url, abortController.signal);
2965
2993
  result = { ...scrapeResult, duration: Date.now() - startTime };
2966
2994
  } else if (type === "healthcheck") {
2967
- const healthResult = await executeHealthCheck(getContext(), initMs, abortController.signal);
2995
+ const healthResult = await executeHealthCheck(taskContext, initMs, abortController.signal);
2968
2996
  result = { ...healthResult, duration: Date.now() - startTime };
2969
2997
  } else {
2970
2998
  result = { error: "Unknown task type", duration: Date.now() - startTime };
@@ -2987,6 +3015,10 @@ async function runTask(data) {
2987
3015
  };
2988
3016
  } finally {
2989
3017
  if (taskTimer !== void 0) clearTimeout(taskTimer);
3018
+ if (taskContext) {
3019
+ Promise.resolve(taskContext.close()).catch(() => {
3020
+ });
3021
+ }
2990
3022
  taskFinished();
2991
3023
  }
2992
3024
  }