@lincoln504/pi-research 1.4.1 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cli.mjs +449 -187
- package/dist/prompts/research-knowledge-relevance-triage.md +2 -0
- package/dist/prompts/research-knowledge-search-extractor.md +2 -0
- package/dist/prompts/researcher.md +2 -14
- package/dist/thread-worker.mjs +91 -59
- package/docs/AGENT-SKILL.md +4 -3
- package/docs/CHANGELOG.md +120 -0
- package/docs/CONFIGURATION.md +2 -2
- package/docs/SDK.md +1 -1
- package/package.json +1 -1
- package/skills/pi-research/SKILL.md +1 -1
- package/src/cli.ts +44 -3
- package/src/config.ts +36 -13
- package/src/constants.ts +23 -4
- package/src/core/llm/research-model-resolver.ts +38 -0
- package/src/core/planning-service.ts +67 -9
- package/src/index.ts +3 -1
- package/src/infrastructure/browser/browser-task-scheduler.ts +16 -9
- package/src/infrastructure/browser/config.ts +40 -2
- package/src/infrastructure/browser/thread-worker-browser.ts +69 -48
- package/src/infrastructure/browser/thread-worker-messaging.ts +65 -4
- package/src/infrastructure/browser/thread-worker.mjs +91 -59
- package/src/infrastructure/browser/thread-worker.ts +13 -5
- package/src/infrastructure/embedding/embedding-server.ts +54 -0
- package/src/knowledge/embedder-init.ts +3 -2
- package/src/knowledge/embedder.ts +11 -3
- package/src/knowledge/index.ts +36 -10
- package/src/knowledge/store.ts +85 -5
- package/src/observers/research-observer-impl.ts +14 -6
- package/src/orchestration/deep-research-orchestrator.ts +3 -1
- package/src/orchestration/quick-research-orchestrator.ts +29 -10
- package/src/orchestration/research-synthesis-service.ts +32 -0
- package/src/orchestration/researcher-executor.ts +31 -16
- package/src/orchestration/researcher.ts +2 -2
- package/src/orchestration/session-state.ts +35 -4
- package/src/prompts/research-knowledge-relevance-triage.md +2 -0
- package/src/prompts/research-knowledge-search-extractor.md +2 -0
- package/src/prompts/researcher.md +2 -14
- package/src/sdk.ts +16 -0
- package/src/skill-install/skill-installer.ts +5 -1
- package/src/stackexchange/index.ts +3 -0
- package/src/stackexchange/rest-client.ts +7 -12
- package/src/tools/index.ts +2 -1
- package/src/tools/research-knowledge-search.ts +37 -4
- package/src/tools/research-tool-definition.ts +54 -12
- package/src/tools/scrape.ts +8 -1
- package/src/tools/security.ts +13 -10
- package/src/utils/coverage-digest.ts +1 -1
- package/src/utils/host-config.ts +12 -1
- package/src/utils/log-utils.ts +15 -2
- package/src/utils/metrics-summary.ts +21 -1
- package/src/utils/tool-usage-tracker.ts +6 -3
- package/src/web-research/browser-search.ts +11 -4
- package/src/web-research/web-scraper.ts +36 -16
- package/src/youtube/index.ts +11 -1
- package/src/youtube/potoken.ts +8 -1
- package/src/youtube/transcript-client.ts +56 -5
|
@@ -11,21 +11,23 @@
|
|
|
11
11
|
import { platform } from 'node:os';
|
|
12
12
|
import { setupMocking } from './thread-worker-messaging.ts';
|
|
13
13
|
import { redactSecrets } from '../../utils/log-utils.ts';
|
|
14
|
-
import { resolveHeadlessMode } from './config.ts';
|
|
14
|
+
import { resolveHeadlessMode, BROWSER_LAUNCH_TIMEOUT_MS, CONTEXT_CREATION_TIMEOUT_MS } from './config.ts';
|
|
15
15
|
|
|
16
16
|
let browser: any = null;
|
|
17
|
-
let context: any = null;
|
|
18
17
|
let initPromise: Promise<void> | null = null;
|
|
19
18
|
let workerId: string = '';
|
|
20
19
|
|
|
21
20
|
// How many tasks this worker process currently has in flight. poolifier's
|
|
22
21
|
// tasksQueueOptions.concurrency (WORKER_CONCURRENCY, default 2 — see
|
|
23
22
|
// worker-pool-manager.ts) dispatches more than one task at a time to the SAME
|
|
24
|
-
// worker process, and every dispatched task shares this module's browser
|
|
25
|
-
//
|
|
26
|
-
//
|
|
27
|
-
//
|
|
28
|
-
//
|
|
23
|
+
// worker process, and every dispatched task shares this module's browser
|
|
24
|
+
// singleton (each gets its own FRESH BrowserContext via acquireTaskContext()
|
|
25
|
+
// below — a real Firefox process is too expensive to launch per task, but a
|
|
26
|
+
// context is cheap, and per-task contexts keep cookies/storage/route-mocking
|
|
27
|
+
// from one task bleeding into a concurrent or later, unrelated one).
|
|
28
|
+
// resetBrowser() uses this to avoid tearing the shared BROWSER down out from
|
|
29
|
+
// under a sibling task that is still actively using it — see resetBrowser()
|
|
30
|
+
// below.
|
|
29
31
|
let activeTaskCount = 0;
|
|
30
32
|
// A reset that resetBrowser() deferred because sibling tasks were still
|
|
31
33
|
// active, to be applied once the last one finishes.
|
|
@@ -119,12 +121,12 @@ function logToDebugFile(level: string, ...args: any[]): void {
|
|
|
119
121
|
* Initialize the browser instance
|
|
120
122
|
*/
|
|
121
123
|
export async function initBrowser(): Promise<void> {
|
|
122
|
-
if (isBrowserConnected()
|
|
124
|
+
if (isBrowserConnected()) return;
|
|
123
125
|
if (initPromise) return initPromise;
|
|
124
126
|
|
|
125
127
|
initPromise = (async () => {
|
|
126
128
|
try {
|
|
127
|
-
if (!isBrowserConnected()
|
|
129
|
+
if (!isBrowserConnected()) {
|
|
128
130
|
logToDebugFile('INFO', `[Worker-${workerId}] Initializing browser instance...`);
|
|
129
131
|
|
|
130
132
|
// ---------------------------------------------------------------------
|
|
@@ -196,8 +198,12 @@ export async function initBrowser(): Promise<void> {
|
|
|
196
198
|
//
|
|
197
199
|
// 90s launch timeout: CI runners (2 vCPU) need up to 60s to start Firefox
|
|
198
200
|
// when it has both CPUs. The old 45s limit was too tight and caused silent
|
|
199
|
-
// init failures under resource pressure.
|
|
200
|
-
|
|
201
|
+
// init failures under resource pressure. Imported from config.ts (single
|
|
202
|
+
// source of truth) rather than a local literal — the scheduler's task
|
|
203
|
+
// deadlines (browser-task-scheduler.ts) and the healthcheck budget
|
|
204
|
+
// (getHealthCheckBudgetMs) both have to budget for this exact worst case,
|
|
205
|
+
// via COLD_START_ALLOWANCE_MS.
|
|
206
|
+
const launchTimeoutMs = BROWSER_LAUNCH_TIMEOUT_MS;
|
|
201
207
|
const launchOnce = async (headless: boolean | 'virtual') => {
|
|
202
208
|
const launchPromise = Camoufox({
|
|
203
209
|
headless,
|
|
@@ -295,35 +301,6 @@ export async function initBrowser(): Promise<void> {
|
|
|
295
301
|
}
|
|
296
302
|
|
|
297
303
|
browser = launchedBrowser;
|
|
298
|
-
// newContext() can hang if the browser process becomes unresponsive immediately
|
|
299
|
-
// after launch (e.g. OOM, GPU crash). Guard with a hard timeout.
|
|
300
|
-
const contextPromise = browser.newContext();
|
|
301
|
-
contextPromise.catch((err: Error) => logToDebugFile('DEBUG', `[ThreadWorker] Background browser context rejection: ${err.message}`));
|
|
302
|
-
let contextTimeoutId: NodeJS.Timeout | undefined;
|
|
303
|
-
let contextRaceLost = false;
|
|
304
|
-
// Same adoption as the launch above. Here the catch below CAN close the
|
|
305
|
-
// browser (it is assigned by now), which disposes the stray context with it —
|
|
306
|
-
// but only when initBrowser is entered with a dead browser, and it leaves the
|
|
307
|
-
// context open for the window in between. Closing it directly is unconditional.
|
|
308
|
-
contextPromise.then((c: any) => {
|
|
309
|
-
if (!contextRaceLost) return;
|
|
310
|
-
logToDebugFile('WARN', `[Worker-${workerId}] Browser context creation completed after its deadline; closing the orphaned context.`);
|
|
311
|
-
try { c?.close?.(); } catch { /* already gone */ }
|
|
312
|
-
}).catch(() => { /* rejection already logged above */ });
|
|
313
|
-
context = await Promise.race([
|
|
314
|
-
contextPromise,
|
|
315
|
-
new Promise<never>((_, reject) => {
|
|
316
|
-
contextTimeoutId = setTimeout(() => {
|
|
317
|
-
contextRaceLost = true;
|
|
318
|
-
reject(new Error('Browser context creation timed out after 30000ms'));
|
|
319
|
-
}, 30000);
|
|
320
|
-
})
|
|
321
|
-
]);
|
|
322
|
-
if (contextTimeoutId) clearTimeout(contextTimeoutId);
|
|
323
|
-
|
|
324
|
-
// Setup mocking for CI if enabled
|
|
325
|
-
await setupMocking(context);
|
|
326
|
-
|
|
327
304
|
logToDebugFile('INFO', `[Worker-${workerId}] Browser initialized.`);
|
|
328
305
|
}
|
|
329
306
|
} catch (e: unknown) {
|
|
@@ -334,7 +311,6 @@ export async function initBrowser(): Promise<void> {
|
|
|
334
311
|
Promise.resolve(browser.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed browser close error during failed init: ${err.message}`));
|
|
335
312
|
}
|
|
336
313
|
browser = null;
|
|
337
|
-
context = null;
|
|
338
314
|
const msg = e instanceof Error ? e.message : String(e);
|
|
339
315
|
|
|
340
316
|
if (msg.includes('Camoufox is not installed') || msg.includes('Version information not found')) {
|
|
@@ -361,16 +337,60 @@ export async function initBrowser(): Promise<void> {
|
|
|
361
337
|
}
|
|
362
338
|
|
|
363
339
|
/**
|
|
364
|
-
*
|
|
340
|
+
* Create a FRESH BrowserContext for one task, against the shared browser
|
|
341
|
+
* process. Call once per task, right after initBrowser() resolves; the
|
|
342
|
+
* caller owns the returned context and must close it itself (in a `finally`,
|
|
343
|
+
* covering the success, error, AND timeout-abort paths) once the task ends.
|
|
344
|
+
*
|
|
345
|
+
* Never reused across tasks — that was the bug this replaced. Two tasks
|
|
346
|
+
* dispatched concurrently to the same worker (WORKER_CONCURRENCY > 1, the
|
|
347
|
+
* default) used to share one BrowserContext, so cookies/storage a site set
|
|
348
|
+
* for one task's request were visible to a different, unrelated task's later
|
|
349
|
+
* request to the same site — a content-consistency gap with no counterpart
|
|
350
|
+
* anywhere else in this codebase, which scopes browser-adjacent state
|
|
351
|
+
* (circuit breakers, scrape caches) per session deliberately.
|
|
352
|
+
*
|
|
353
|
+
* The launch race/timeout/orphan-adoption pattern below is the exact one
|
|
354
|
+
* initBrowser() used when context creation lived there (moved here wholesale,
|
|
355
|
+
* not reinvented): newContext() can hang if the browser process is
|
|
356
|
+
* unresponsive (OOM, GPU crash), so it is guarded with a hard timeout, and a
|
|
357
|
+
* context that finishes creating AFTER that timeout is adopted and closed
|
|
358
|
+
* rather than left to leak.
|
|
365
359
|
*/
|
|
366
|
-
export function
|
|
367
|
-
|
|
360
|
+
export async function acquireTaskContext(): Promise<any> {
|
|
361
|
+
const contextPromise = browser.newContext();
|
|
362
|
+
contextPromise.catch((err: Error) => logToDebugFile('DEBUG', `[ThreadWorker] Background browser context rejection: ${err.message}`));
|
|
363
|
+
let contextTimeoutId: NodeJS.Timeout | undefined;
|
|
364
|
+
let contextRaceLost = false;
|
|
365
|
+
contextPromise.then((c: any) => {
|
|
366
|
+
if (!contextRaceLost) return;
|
|
367
|
+
logToDebugFile('WARN', `[Worker-${workerId}] Browser context creation completed after its deadline; closing the orphaned context.`);
|
|
368
|
+
try { c?.close?.(); } catch { /* already gone */ }
|
|
369
|
+
}).catch(() => { /* rejection already logged above */ });
|
|
370
|
+
let newContext: any;
|
|
371
|
+
try {
|
|
372
|
+
newContext = await Promise.race([
|
|
373
|
+
contextPromise,
|
|
374
|
+
new Promise<never>((_, reject) => {
|
|
375
|
+
contextTimeoutId = setTimeout(() => {
|
|
376
|
+
contextRaceLost = true;
|
|
377
|
+
reject(new Error(`Browser context creation timed out after ${CONTEXT_CREATION_TIMEOUT_MS}ms`));
|
|
378
|
+
}, CONTEXT_CREATION_TIMEOUT_MS);
|
|
379
|
+
})
|
|
380
|
+
]);
|
|
381
|
+
} finally {
|
|
382
|
+
if (contextTimeoutId) clearTimeout(contextTimeoutId);
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
// Route interception is per-context, so mocking must be (re-)applied to
|
|
386
|
+
// every fresh context, not just the first one ever created.
|
|
387
|
+
await setupMocking(newContext);
|
|
388
|
+
|
|
389
|
+
return newContext;
|
|
368
390
|
}
|
|
369
391
|
|
|
370
392
|
function doResetBrowser(): void {
|
|
371
|
-
if (context) Promise.resolve(context.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed context close error during reset: ${err.message}`));
|
|
372
393
|
if (browser) Promise.resolve(browser.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed browser close error during reset: ${err.message}`));
|
|
373
|
-
context = null;
|
|
374
394
|
browser = null;
|
|
375
395
|
}
|
|
376
396
|
|
|
@@ -407,7 +427,9 @@ export async function cleanupBrowser(): Promise<void> {
|
|
|
407
427
|
const timeoutMs = 2000;
|
|
408
428
|
|
|
409
429
|
const cleanup = async () => {
|
|
410
|
-
|
|
430
|
+
// Closes every context still open under it (any task whose own close()
|
|
431
|
+
// hasn't run yet) as a side effect — no separate context bookkeeping
|
|
432
|
+
// needed here now that contexts are task-owned, not module-owned.
|
|
411
433
|
if (browser) await Promise.resolve(browser.close()).catch((err: Error) => logToDebugFile('DEBUG', `[Worker-${workerId}] Swallowed browser close error during cleanup: ${err.message}`));
|
|
412
434
|
};
|
|
413
435
|
|
|
@@ -419,7 +441,6 @@ export async function cleanupBrowser(): Promise<void> {
|
|
|
419
441
|
} catch {
|
|
420
442
|
// Ignore timeout — we just want to ensure cleanup doesn't hang indefinitely
|
|
421
443
|
} finally {
|
|
422
|
-
context = null;
|
|
423
444
|
browser = null;
|
|
424
445
|
}
|
|
425
446
|
}
|
|
@@ -433,6 +433,13 @@ export async function executeScrapeTask(
|
|
|
433
433
|
let poisonedError: Error | null = null;
|
|
434
434
|
const pendingAddrChecks: Array<Promise<void>> = [];
|
|
435
435
|
|
|
436
|
+
// Set by the response-side early-abort check (see page.on('response') below) when the
|
|
437
|
+
// main-frame document's OWN headers already declare a PDF larger than MAX_PDF_SIZE.
|
|
438
|
+
// Declared outside the try for the same reason as poisonedError: the catch below needs
|
|
439
|
+
// to surface this instead of the secondary 'Target closed' error the triggering
|
|
440
|
+
// page.close() produces.
|
|
441
|
+
let oversizedPdfError: Error | null = null;
|
|
442
|
+
|
|
436
443
|
try {
|
|
437
444
|
logToDebugFile('DEBUG', `[Worker-${workerId}] Starting scrape for: ${url}`);
|
|
438
445
|
|
|
@@ -511,6 +518,46 @@ export async function executeScrapeTask(
|
|
|
511
518
|
try {
|
|
512
519
|
if (resp.request().resourceType() === 'document' && resp.frame() === page.mainFrame()) {
|
|
513
520
|
finalMainFrameAddr = resp.serverAddr().catch(() => null);
|
|
521
|
+
|
|
522
|
+
// Early-abort a declared-oversized PDF as soon as its headers arrive, rather
|
|
523
|
+
// than waiting for page.goto() to finish downloading the whole thing first.
|
|
524
|
+
// 'response' fires on headers-received, well before the body is fully
|
|
525
|
+
// transferred — for a direct navigation to a PDF URL, Playwright's goto()
|
|
526
|
+
// does not resolve until the download completes, so without this the full
|
|
527
|
+
// (possibly multi-hundred-MB) body is pulled into the browser process before
|
|
528
|
+
// the PDF branch below ever gets to check Content-Length.
|
|
529
|
+
//
|
|
530
|
+
// This closes the gap only for a server that HONESTLY declares an oversized
|
|
531
|
+
// Content-Length up front. A chunked-transfer response (no Content-Length) or
|
|
532
|
+
// one that understates its length and then sends far more cannot be caught
|
|
533
|
+
// this way — Playwright's public API exposes no incremental
|
|
534
|
+
// bytes-received/streaming hook to abort mid-body on either engine. That
|
|
535
|
+
// residual case is bounded instead by the task's own SCRAPE_TIMEOUT_MS +
|
|
536
|
+
// BROWSER_TASK_TIMEOUT_MS deadline (see the onAbort/page.close() wiring
|
|
537
|
+
// above), which forcibly closes the page — and with it any in-flight
|
|
538
|
+
// download — once the task's time budget is exhausted, converting an
|
|
539
|
+
// otherwise-unbounded read into one bounded by (bandwidth x that budget).
|
|
540
|
+
const check2: Promise<void> = Promise.resolve(resp.headerValue('content-type'))
|
|
541
|
+
.then(async (ct: string | null) => {
|
|
542
|
+
// Matches the PDF branch below AND the fetch layer's own detection
|
|
543
|
+
// (web-scraper.ts): a server can mislabel or omit content-type, so a
|
|
544
|
+
// `.pdf` URL extension is an equally-trusted signal, not just the header.
|
|
545
|
+
// resp.url() — THIS response's URL, not the original task URL: goto()
|
|
546
|
+
// follows redirects, and a download endpoint without `.pdf` that
|
|
547
|
+
// redirects to the real `.pdf` file must still arm the early abort
|
|
548
|
+
// (same redirect-aware fix the fetch layer got via currentUrl).
|
|
549
|
+
const looksLikePdf = ct?.includes('application/pdf') || String(resp.url()).toLowerCase().endsWith('.pdf');
|
|
550
|
+
if (!looksLikePdf || oversizedPdfError || poisonedError) return;
|
|
551
|
+
const declared = parseInt((await resp.headerValue('content-length').catch(() => null)) || '', 10);
|
|
552
|
+
if (Number.isFinite(declared) && declared > MAX_PDF_SIZE) {
|
|
553
|
+
const sizeMB = Math.round(declared / 1024 / 1024);
|
|
554
|
+
oversizedPdfError = new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
|
|
555
|
+
logToDebugFile('DEBUG', `[Worker-${workerId}] Aborting oversized PDF (${sizeMB}MB declared) before download completes: ${url}`);
|
|
556
|
+
page.close().catch(() => {});
|
|
557
|
+
}
|
|
558
|
+
})
|
|
559
|
+
.catch(() => {});
|
|
560
|
+
pendingAddrChecks.push(check2);
|
|
514
561
|
}
|
|
515
562
|
const check: Promise<void> = Promise.resolve(resp.serverAddr())
|
|
516
563
|
.then((addr: { ipAddress?: string } | null) => {
|
|
@@ -562,7 +609,14 @@ export async function executeScrapeTask(
|
|
|
562
609
|
// 403/503 — the HTML branch defers to the challenge wait below for that.
|
|
563
610
|
const status = response?.status?.() ?? 0;
|
|
564
611
|
|
|
565
|
-
|
|
612
|
+
// `.pdf` extension is an equally-trusted signal alongside the header (matches
|
|
613
|
+
// the fetch layer's own detection in web-scraper.ts): a server that mislabels
|
|
614
|
+
// or omits content-type would otherwise fall into the HTML branch below, whose
|
|
615
|
+
// page.content() would serialize Firefox's PDF-viewer chrome instead of
|
|
616
|
+
// extracting the document — a silent content loss for exactly the URLs this
|
|
617
|
+
// feature advertises as supported, and specifically for the malformed/oddly-
|
|
618
|
+
// served PDFs most likely to reach this browser fallback in the first place.
|
|
619
|
+
if (contentType.includes('application/pdf') || url.toLowerCase().endsWith('.pdf')) {
|
|
566
620
|
if (!response) throw new Error(`[Worker] No response received for PDF URL: ${url}`);
|
|
567
621
|
// An errored "PDF" is an error page: fail BEFORE buffering its body.
|
|
568
622
|
if (status >= 400) throw new Error(`HTTP ${status}`);
|
|
@@ -576,6 +630,10 @@ export async function executeScrapeTask(
|
|
|
576
630
|
const sizeMB = Math.round(declaredLength / 1024 / 1024);
|
|
577
631
|
throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
|
|
578
632
|
}
|
|
633
|
+
// The response-side early-abort check above (page.on('response')) may already have
|
|
634
|
+
// decided this and be in the middle of closing the page — skip the (redundant,
|
|
635
|
+
// about-to-fail) download attempt rather than race it.
|
|
636
|
+
if (oversizedPdfError) throw oversizedPdfError;
|
|
579
637
|
const buffer = await response.body();
|
|
580
638
|
// ...then the actual bytes: the header is advisory and absent on chunked
|
|
581
639
|
// responses, so the byte count is the check that actually bounds memory.
|
|
@@ -692,9 +750,12 @@ export async function executeScrapeTask(
|
|
|
692
750
|
return { contentType, html, jitter };
|
|
693
751
|
} catch (error) {
|
|
694
752
|
await page.close().catch((err: any) => logToDebugFile('DEBUG', `[ThreadWorker] Swallowed page close/wait error: ${err.message || String(err)}`));
|
|
695
|
-
//
|
|
696
|
-
// with 'Target closed'; report the
|
|
697
|
-
//
|
|
753
|
+
// An early-abort-triggered page.close() (oversized PDF, or SSRF poisoning) makes
|
|
754
|
+
// the in-flight goto/content()/body() reject with 'Target closed'; report the
|
|
755
|
+
// actual reason (a benign-classified failure) instead of that secondary teardown
|
|
756
|
+
// error. oversizedPdfError takes priority since it can only be set on the PDF path,
|
|
757
|
+
// where poisonedError (an HTML-only DOM-exfil concern) is not meaningful.
|
|
758
|
+
if (oversizedPdfError) throw oversizedPdfError;
|
|
698
759
|
if (poisonedError) throw poisonedError;
|
|
699
760
|
throw error;
|
|
700
761
|
} finally {
|
|
@@ -71,6 +71,8 @@ function redactSecrets(message) {
|
|
|
71
71
|
}
|
|
72
72
|
if (out.length > MAX_LOG_MESSAGE_LENGTH) {
|
|
73
73
|
out = `${out.slice(0, keep)}\u2026[truncated ${Math.max(rawLength, out.length) - keep} chars]`;
|
|
74
|
+
} else if (rawLength > scanEnd) {
|
|
75
|
+
out = `${out}\u2026[truncated ${rawLength - scanEnd} chars]`;
|
|
74
76
|
}
|
|
75
77
|
return out;
|
|
76
78
|
}
|
|
@@ -247,7 +249,7 @@ var init_error_tracker = __esm({
|
|
|
247
249
|
* @param error - The error to track (Error object, string message, or unknown)
|
|
248
250
|
* @param context - Optional context information about where the error occurred
|
|
249
251
|
*/
|
|
250
|
-
trackError(error,
|
|
252
|
+
trackError(error, context = {}) {
|
|
251
253
|
let message;
|
|
252
254
|
if (error instanceof Error) {
|
|
253
255
|
message = error.message;
|
|
@@ -276,15 +278,15 @@ var init_error_tracker = __esm({
|
|
|
276
278
|
}
|
|
277
279
|
pattern.count++;
|
|
278
280
|
pattern.lastSeen = now;
|
|
279
|
-
if (
|
|
280
|
-
const existing = pattern.domainCounts.get(
|
|
281
|
+
if (context.domain) {
|
|
282
|
+
const existing = pattern.domainCounts.get(context.domain);
|
|
281
283
|
if (existing !== void 0) {
|
|
282
|
-
pattern.domainCounts.set(
|
|
284
|
+
pattern.domainCounts.set(context.domain, existing + 1);
|
|
283
285
|
} else if (pattern.domainCounts.size < this.MAX_DOMAINS_PER_PATTERN) {
|
|
284
|
-
pattern.domainCounts.set(
|
|
286
|
+
pattern.domainCounts.set(context.domain, 1);
|
|
285
287
|
}
|
|
286
288
|
}
|
|
287
|
-
pattern.contexts.push(
|
|
289
|
+
pattern.contexts.push(context);
|
|
288
290
|
if (pattern.contexts.length > this.MAX_CONTEXTS_PER_PATTERN) {
|
|
289
291
|
pattern.contexts.shift();
|
|
290
292
|
}
|
|
@@ -1109,8 +1111,8 @@ var init_logger = __esm({
|
|
|
1109
1111
|
error(...args) {
|
|
1110
1112
|
const errArg = args.find((arg) => arg instanceof Error);
|
|
1111
1113
|
if (errArg) {
|
|
1112
|
-
const
|
|
1113
|
-
errorTracker.trackError(errArg,
|
|
1114
|
+
const context = getLogContext();
|
|
1115
|
+
errorTracker.trackError(errArg, context);
|
|
1114
1116
|
}
|
|
1115
1117
|
this.emit("ERROR" /* ERROR */, ...args);
|
|
1116
1118
|
}
|
|
@@ -2025,7 +2027,7 @@ var workerId2 = "";
|
|
|
2025
2027
|
var pageCreationLock = Promise.resolve();
|
|
2026
2028
|
var PAGE_CREATION_HARD_RELEASE_MS = 5 * 60 * 1e3;
|
|
2027
2029
|
var wedgedContext = null;
|
|
2028
|
-
async function createPageSafe(
|
|
2030
|
+
async function createPageSafe(context) {
|
|
2029
2031
|
let release;
|
|
2030
2032
|
const nextLock = new Promise((resolve) => {
|
|
2031
2033
|
release = resolve;
|
|
@@ -2033,11 +2035,11 @@ async function createPageSafe(context2) {
|
|
|
2033
2035
|
const currentLock = pageCreationLock;
|
|
2034
2036
|
pageCreationLock = currentLock.then(() => nextLock);
|
|
2035
2037
|
await currentLock;
|
|
2036
|
-
if (
|
|
2038
|
+
if (context === wedgedContext) {
|
|
2037
2039
|
release();
|
|
2038
2040
|
throw new Error("Browser context is wedged from a prior page-creation call that never settled; a new browser context is required.");
|
|
2039
2041
|
}
|
|
2040
|
-
const pagePromise =
|
|
2042
|
+
const pagePromise = context.newPage();
|
|
2041
2043
|
let hardReleased = false;
|
|
2042
2044
|
const releaseOnce = () => {
|
|
2043
2045
|
if (hardReleased) return;
|
|
@@ -2047,17 +2049,17 @@ async function createPageSafe(context2) {
|
|
|
2047
2049
|
};
|
|
2048
2050
|
const hardReleaseTimer = setTimeout(() => {
|
|
2049
2051
|
logToDebugFile2("WARN", "[ThreadWorker] page-creation mutex force-released \u2014 newPage() did not settle within the hard timeout");
|
|
2050
|
-
wedgedContext =
|
|
2052
|
+
wedgedContext = context;
|
|
2051
2053
|
releaseOnce();
|
|
2052
2054
|
}, PAGE_CREATION_HARD_RELEASE_MS);
|
|
2053
2055
|
safeUnref(hardReleaseTimer);
|
|
2054
2056
|
pagePromise.then(
|
|
2055
2057
|
() => {
|
|
2056
|
-
if (wedgedContext ===
|
|
2058
|
+
if (wedgedContext === context) wedgedContext = null;
|
|
2057
2059
|
releaseOnce();
|
|
2058
2060
|
},
|
|
2059
2061
|
() => {
|
|
2060
|
-
if (wedgedContext ===
|
|
2062
|
+
if (wedgedContext === context) wedgedContext = null;
|
|
2061
2063
|
releaseOnce();
|
|
2062
2064
|
}
|
|
2063
2065
|
);
|
|
@@ -2119,12 +2121,12 @@ function logToDebugFile2(level, ...args) {
|
|
|
2119
2121
|
} catch {
|
|
2120
2122
|
}
|
|
2121
2123
|
}
|
|
2122
|
-
async function setupMocking(
|
|
2124
|
+
async function setupMocking(context) {
|
|
2123
2125
|
const mockSearch = process.env["PI_RESEARCH_MOCK_SEARCH"] === "true";
|
|
2124
2126
|
const mockScrape = process.env["PI_RESEARCH_MOCK_SCRAPE"] === "true";
|
|
2125
2127
|
if (!mockSearch && !mockScrape) return;
|
|
2126
2128
|
logToDebugFile2("INFO", `[Worker-${workerId2}] Setting up browser mocking (Search: ${mockSearch}, Scrape: ${mockScrape})`);
|
|
2127
|
-
await
|
|
2129
|
+
await context.route("**", (route, request) => {
|
|
2128
2130
|
const urlStr = request.url();
|
|
2129
2131
|
let url;
|
|
2130
2132
|
try {
|
|
@@ -2297,6 +2299,7 @@ async function executeScrapeTask(_context, url, signal) {
|
|
|
2297
2299
|
}
|
|
2298
2300
|
let poisonedError = null;
|
|
2299
2301
|
const pendingAddrChecks = [];
|
|
2302
|
+
let oversizedPdfError = null;
|
|
2300
2303
|
try {
|
|
2301
2304
|
logToDebugFile2("DEBUG", `[Worker-${workerId2}] Starting scrape for: ${url}`);
|
|
2302
2305
|
const { validateUrlForSSRF: validateForWorker, validateUrlForSSRFSync: validateForWorkerSync } = await Promise.resolve().then(() => (init_scraper_utils(), scraper_utils_exports));
|
|
@@ -2328,6 +2331,20 @@ async function executeScrapeTask(_context, url, signal) {
|
|
|
2328
2331
|
try {
|
|
2329
2332
|
if (resp.request().resourceType() === "document" && resp.frame() === page.mainFrame()) {
|
|
2330
2333
|
finalMainFrameAddr = resp.serverAddr().catch(() => null);
|
|
2334
|
+
const check2 = Promise.resolve(resp.headerValue("content-type")).then(async (ct) => {
|
|
2335
|
+
const looksLikePdf = ct?.includes("application/pdf") || String(resp.url()).toLowerCase().endsWith(".pdf");
|
|
2336
|
+
if (!looksLikePdf || oversizedPdfError || poisonedError) return;
|
|
2337
|
+
const declared = parseInt(await resp.headerValue("content-length").catch(() => null) || "", 10);
|
|
2338
|
+
if (Number.isFinite(declared) && declared > MAX_PDF_SIZE) {
|
|
2339
|
+
const sizeMB = Math.round(declared / 1024 / 1024);
|
|
2340
|
+
oversizedPdfError = new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
|
|
2341
|
+
logToDebugFile2("DEBUG", `[Worker-${workerId2}] Aborting oversized PDF (${sizeMB}MB declared) before download completes: ${url}`);
|
|
2342
|
+
page.close().catch(() => {
|
|
2343
|
+
});
|
|
2344
|
+
}
|
|
2345
|
+
}).catch(() => {
|
|
2346
|
+
});
|
|
2347
|
+
pendingAddrChecks.push(check2);
|
|
2331
2348
|
}
|
|
2332
2349
|
const check = Promise.resolve(resp.serverAddr()).then((addr) => {
|
|
2333
2350
|
const ip = addr?.ipAddress;
|
|
@@ -2357,7 +2374,7 @@ async function executeScrapeTask(_context, url, signal) {
|
|
|
2357
2374
|
}
|
|
2358
2375
|
const contentType = await response?.headerValue("content-type") || "";
|
|
2359
2376
|
const status = response?.status?.() ?? 0;
|
|
2360
|
-
if (contentType.includes("application/pdf")) {
|
|
2377
|
+
if (contentType.includes("application/pdf") || url.toLowerCase().endsWith(".pdf")) {
|
|
2361
2378
|
if (!response) throw new Error(`[Worker] No response received for PDF URL: ${url}`);
|
|
2362
2379
|
if (status >= 400) throw new Error(`HTTP ${status}`);
|
|
2363
2380
|
const declaredLength = parseInt(await response.headerValue("content-length").catch(() => null) || "", 10);
|
|
@@ -2365,6 +2382,7 @@ async function executeScrapeTask(_context, url, signal) {
|
|
|
2365
2382
|
const sizeMB = Math.round(declaredLength / 1024 / 1024);
|
|
2366
2383
|
throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
|
|
2367
2384
|
}
|
|
2385
|
+
if (oversizedPdfError) throw oversizedPdfError;
|
|
2368
2386
|
const buffer = await response.body();
|
|
2369
2387
|
if (buffer.byteLength > MAX_PDF_SIZE) {
|
|
2370
2388
|
const sizeMB = Math.round(buffer.byteLength / 1024 / 1024);
|
|
@@ -2422,6 +2440,7 @@ async function executeScrapeTask(_context, url, signal) {
|
|
|
2422
2440
|
return { contentType, html, jitter };
|
|
2423
2441
|
} catch (error) {
|
|
2424
2442
|
await page.close().catch((err) => logToDebugFile2("DEBUG", `[ThreadWorker] Swallowed page close/wait error: ${err.message || String(err)}`));
|
|
2443
|
+
if (oversizedPdfError) throw oversizedPdfError;
|
|
2425
2444
|
if (poisonedError) throw poisonedError;
|
|
2426
2445
|
throw error;
|
|
2427
2446
|
} finally {
|
|
@@ -2566,10 +2585,16 @@ var ConfigSchema = Type.Object({
|
|
|
2566
2585
|
SEARCH_TIMEOUT_MS: Type.Number({ minimum: 5e3, maximum: 12e4, default: 45e3 }),
|
|
2567
2586
|
/** TUI refresh debounce in milliseconds (default: 100ms) */
|
|
2568
2587
|
TUI_REFRESH_DEBOUNCE_MS: Type.Number({ minimum: 0, maximum: 1e3, default: 100 }),
|
|
2569
|
-
/**
|
|
2570
|
-
*
|
|
2571
|
-
* SEARCH_TIMEOUT_MS + this, a scrape after SCRAPE_TIMEOUT_MS + this
|
|
2572
|
-
*
|
|
2588
|
+
/** Overhead margin (ms) added on top of each browser operation's own nav timeout to
|
|
2589
|
+
* form its scheduler task-timeout ceiling: a search task times out after
|
|
2590
|
+
* SEARCH_TIMEOUT_MS + this, a scrape after SCRAPE_TIMEOUT_MS + this — covering
|
|
2591
|
+
* per-task work beyond raw navigation (page setup, result parsing). Queue wait is
|
|
2592
|
+
* NOT part of this margin: the deadline is armed on dispatch, not on enqueue, so
|
|
2593
|
+
* time spent waiting for a free worker never counts against it (see
|
|
2594
|
+
* browser-task-scheduler.ts). Cold browser-launch/context-creation time is also
|
|
2595
|
+
* separate from this margin — COLD_START_ALLOWANCE_MS
|
|
2596
|
+
* (infrastructure/browser/config.ts) covers that unconditionally on top of it,
|
|
2597
|
+
* since it isn't something a user should need to tune.
|
|
2573
2598
|
* (default: 10000) */
|
|
2574
2599
|
BROWSER_TASK_TIMEOUT_MS: Type.Number({ minimum: 2e3, maximum: 12e4, default: 1e4 }),
|
|
2575
2600
|
/** Timeout for coordinator/research-lead/repair/knowledge LLM calls in ms (default: 300000 = 5 min, range: 60s-30min).
|
|
@@ -2669,10 +2694,12 @@ function resolveHeadlessMode() {
|
|
|
2669
2694
|
return true;
|
|
2670
2695
|
}
|
|
2671
2696
|
var HEALTHCHECK_MIN_BUDGET_MS = 45e3 + 6e4;
|
|
2697
|
+
var BROWSER_LAUNCH_TIMEOUT_MS = 9e4;
|
|
2698
|
+
var CONTEXT_CREATION_TIMEOUT_MS = 3e4;
|
|
2699
|
+
var COLD_START_ALLOWANCE_MS = BROWSER_LAUNCH_TIMEOUT_MS + CONTEXT_CREATION_TIMEOUT_MS;
|
|
2672
2700
|
|
|
2673
2701
|
// src/infrastructure/browser/thread-worker-browser.ts
|
|
2674
2702
|
var browser = null;
|
|
2675
|
-
var context = null;
|
|
2676
2703
|
var initPromise = null;
|
|
2677
2704
|
var workerId3 = "";
|
|
2678
2705
|
var activeTaskCount = 0;
|
|
@@ -2726,11 +2753,11 @@ function logToDebugFile3(level, ...args) {
|
|
|
2726
2753
|
}
|
|
2727
2754
|
}
|
|
2728
2755
|
async function initBrowser() {
|
|
2729
|
-
if (isBrowserConnected()
|
|
2756
|
+
if (isBrowserConnected()) return;
|
|
2730
2757
|
if (initPromise) return initPromise;
|
|
2731
2758
|
initPromise = (async () => {
|
|
2732
2759
|
try {
|
|
2733
|
-
if (!isBrowserConnected()
|
|
2760
|
+
if (!isBrowserConnected()) {
|
|
2734
2761
|
logToDebugFile3("INFO", `[Worker-${workerId3}] Initializing browser instance...`);
|
|
2735
2762
|
let CamoufoxModule;
|
|
2736
2763
|
try {
|
|
@@ -2739,7 +2766,7 @@ async function initBrowser() {
|
|
|
2739
2766
|
throw new Error(`[Worker] camoufox-js not found in node_modules. Please run 'npm install'. Original error: ${e instanceof Error ? e.message : String(e)}`, { cause: e });
|
|
2740
2767
|
}
|
|
2741
2768
|
const { Camoufox } = CamoufoxModule;
|
|
2742
|
-
const launchTimeoutMs =
|
|
2769
|
+
const launchTimeoutMs = BROWSER_LAUNCH_TIMEOUT_MS;
|
|
2743
2770
|
const launchOnce = async (headless) => {
|
|
2744
2771
|
const launchPromise = Camoufox({
|
|
2745
2772
|
headless,
|
|
@@ -2822,30 +2849,6 @@ async function initBrowser() {
|
|
|
2822
2849
|
}
|
|
2823
2850
|
}
|
|
2824
2851
|
browser = launchedBrowser;
|
|
2825
|
-
const contextPromise = browser.newContext();
|
|
2826
|
-
contextPromise.catch((err) => logToDebugFile3("DEBUG", `[ThreadWorker] Background browser context rejection: ${err.message}`));
|
|
2827
|
-
let contextTimeoutId;
|
|
2828
|
-
let contextRaceLost = false;
|
|
2829
|
-
contextPromise.then((c) => {
|
|
2830
|
-
if (!contextRaceLost) return;
|
|
2831
|
-
logToDebugFile3("WARN", `[Worker-${workerId3}] Browser context creation completed after its deadline; closing the orphaned context.`);
|
|
2832
|
-
try {
|
|
2833
|
-
c?.close?.();
|
|
2834
|
-
} catch {
|
|
2835
|
-
}
|
|
2836
|
-
}).catch(() => {
|
|
2837
|
-
});
|
|
2838
|
-
context = await Promise.race([
|
|
2839
|
-
contextPromise,
|
|
2840
|
-
new Promise((_, reject) => {
|
|
2841
|
-
contextTimeoutId = setTimeout(() => {
|
|
2842
|
-
contextRaceLost = true;
|
|
2843
|
-
reject(new Error("Browser context creation timed out after 30000ms"));
|
|
2844
|
-
}, 3e4);
|
|
2845
|
-
})
|
|
2846
|
-
]);
|
|
2847
|
-
if (contextTimeoutId) clearTimeout(contextTimeoutId);
|
|
2848
|
-
await setupMocking(context);
|
|
2849
2852
|
logToDebugFile3("INFO", `[Worker-${workerId3}] Browser initialized.`);
|
|
2850
2853
|
}
|
|
2851
2854
|
} catch (e) {
|
|
@@ -2853,7 +2856,6 @@ async function initBrowser() {
|
|
|
2853
2856
|
Promise.resolve(browser.close()).catch((err) => logToDebugFile3("DEBUG", `[Worker-${workerId3}] Swallowed browser close error during failed init: ${err.message}`));
|
|
2854
2857
|
}
|
|
2855
2858
|
browser = null;
|
|
2856
|
-
context = null;
|
|
2857
2859
|
const msg = e instanceof Error ? e.message : String(e);
|
|
2858
2860
|
if (msg.includes("Camoufox is not installed") || msg.includes("Version information not found")) {
|
|
2859
2861
|
throw new Error(`[Worker] Browser binaries not found. Please run 'npx camoufox-js fetch' to install them.`, { cause: e });
|
|
@@ -2872,13 +2874,39 @@ async function initBrowser() {
|
|
|
2872
2874
|
})();
|
|
2873
2875
|
return initPromise;
|
|
2874
2876
|
}
|
|
2875
|
-
function
|
|
2876
|
-
|
|
2877
|
+
async function acquireTaskContext() {
|
|
2878
|
+
const contextPromise = browser.newContext();
|
|
2879
|
+
contextPromise.catch((err) => logToDebugFile3("DEBUG", `[ThreadWorker] Background browser context rejection: ${err.message}`));
|
|
2880
|
+
let contextTimeoutId;
|
|
2881
|
+
let contextRaceLost = false;
|
|
2882
|
+
contextPromise.then((c) => {
|
|
2883
|
+
if (!contextRaceLost) return;
|
|
2884
|
+
logToDebugFile3("WARN", `[Worker-${workerId3}] Browser context creation completed after its deadline; closing the orphaned context.`);
|
|
2885
|
+
try {
|
|
2886
|
+
c?.close?.();
|
|
2887
|
+
} catch {
|
|
2888
|
+
}
|
|
2889
|
+
}).catch(() => {
|
|
2890
|
+
});
|
|
2891
|
+
let newContext;
|
|
2892
|
+
try {
|
|
2893
|
+
newContext = await Promise.race([
|
|
2894
|
+
contextPromise,
|
|
2895
|
+
new Promise((_, reject) => {
|
|
2896
|
+
contextTimeoutId = setTimeout(() => {
|
|
2897
|
+
contextRaceLost = true;
|
|
2898
|
+
reject(new Error(`Browser context creation timed out after ${CONTEXT_CREATION_TIMEOUT_MS}ms`));
|
|
2899
|
+
}, CONTEXT_CREATION_TIMEOUT_MS);
|
|
2900
|
+
})
|
|
2901
|
+
]);
|
|
2902
|
+
} finally {
|
|
2903
|
+
if (contextTimeoutId) clearTimeout(contextTimeoutId);
|
|
2904
|
+
}
|
|
2905
|
+
await setupMocking(newContext);
|
|
2906
|
+
return newContext;
|
|
2877
2907
|
}
|
|
2878
2908
|
function doResetBrowser() {
|
|
2879
|
-
if (context) Promise.resolve(context.close()).catch((err) => logToDebugFile3("DEBUG", `[Worker-${workerId3}] Swallowed context close error during reset: ${err.message}`));
|
|
2880
2909
|
if (browser) Promise.resolve(browser.close()).catch((err) => logToDebugFile3("DEBUG", `[Worker-${workerId3}] Swallowed browser close error during reset: ${err.message}`));
|
|
2881
|
-
context = null;
|
|
2882
2910
|
browser = null;
|
|
2883
2911
|
}
|
|
2884
2912
|
function resetBrowser() {
|
|
@@ -2892,7 +2920,6 @@ function resetBrowser() {
|
|
|
2892
2920
|
async function cleanupBrowser2() {
|
|
2893
2921
|
const timeoutMs = 2e3;
|
|
2894
2922
|
const cleanup = async () => {
|
|
2895
|
-
if (context) await Promise.resolve(context.close()).catch((err) => logToDebugFile3("DEBUG", `[Worker-${workerId3}] Swallowed context close error during cleanup: ${err.message}`));
|
|
2896
2923
|
if (browser) await Promise.resolve(browser.close()).catch((err) => logToDebugFile3("DEBUG", `[Worker-${workerId3}] Swallowed browser close error during cleanup: ${err.message}`));
|
|
2897
2924
|
};
|
|
2898
2925
|
try {
|
|
@@ -2902,7 +2929,6 @@ async function cleanupBrowser2() {
|
|
|
2902
2929
|
]);
|
|
2903
2930
|
} catch {
|
|
2904
2931
|
} finally {
|
|
2905
|
-
context = null;
|
|
2906
2932
|
browser = null;
|
|
2907
2933
|
}
|
|
2908
2934
|
}
|
|
@@ -2951,20 +2977,22 @@ async function runTask(data) {
|
|
|
2951
2977
|
taskTimer = setTimeout(() => abortController.abort(new Error(`Task timed out after ${taskTimeoutMs}ms`)), taskTimeoutMs);
|
|
2952
2978
|
}
|
|
2953
2979
|
taskStarted();
|
|
2980
|
+
let taskContext;
|
|
2954
2981
|
try {
|
|
2955
2982
|
await initBrowser();
|
|
2956
2983
|
const initMs = Date.now() - startTime;
|
|
2984
|
+
taskContext = await acquireTaskContext();
|
|
2957
2985
|
let result;
|
|
2958
2986
|
if (type === "search") {
|
|
2959
2987
|
if (!query) throw new Error("Search task requires a query");
|
|
2960
|
-
const searchResult = await executeSearchTask(
|
|
2988
|
+
const searchResult = await executeSearchTask(taskContext, query, abortController.signal);
|
|
2961
2989
|
result = { results: searchResult.results, duration: Date.now() - startTime, jitter: searchResult.jitter };
|
|
2962
2990
|
} else if (type === "scrape") {
|
|
2963
2991
|
if (!url) throw new Error("Scrape task requires a URL");
|
|
2964
|
-
const scrapeResult = await executeScrapeTask(
|
|
2992
|
+
const scrapeResult = await executeScrapeTask(taskContext, url, abortController.signal);
|
|
2965
2993
|
result = { ...scrapeResult, duration: Date.now() - startTime };
|
|
2966
2994
|
} else if (type === "healthcheck") {
|
|
2967
|
-
const healthResult = await executeHealthCheck(
|
|
2995
|
+
const healthResult = await executeHealthCheck(taskContext, initMs, abortController.signal);
|
|
2968
2996
|
result = { ...healthResult, duration: Date.now() - startTime };
|
|
2969
2997
|
} else {
|
|
2970
2998
|
result = { error: "Unknown task type", duration: Date.now() - startTime };
|
|
@@ -2987,6 +3015,10 @@ async function runTask(data) {
|
|
|
2987
3015
|
};
|
|
2988
3016
|
} finally {
|
|
2989
3017
|
if (taskTimer !== void 0) clearTimeout(taskTimer);
|
|
3018
|
+
if (taskContext) {
|
|
3019
|
+
Promise.resolve(taskContext.close()).catch(() => {
|
|
3020
|
+
});
|
|
3021
|
+
}
|
|
2990
3022
|
taskFinished();
|
|
2991
3023
|
}
|
|
2992
3024
|
}
|