@lincoln504/pi-research 1.0.10 → 1.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.mjs +444 -58
- package/dist/thread-worker.mjs +39 -1
- package/docs/CHANGELOG.md +26 -0
- package/package.json +5 -2
- package/skills/pi-research/SKILL.md +1 -1
- package/src/core/interfaces/service-names.ts +2 -0
- package/src/infrastructure/browser/browser-client.ts +19 -1
- package/src/infrastructure/browser/browser-error-utils.ts +24 -1
- package/src/infrastructure/browser/browser-server.ts +70 -0
- package/src/infrastructure/browser/browser-task-scheduler.ts +19 -3
- package/src/infrastructure/browser/scheduler-factory.ts +1 -1
- package/src/infrastructure/browser/thread-worker.mjs +39 -1
- package/src/infrastructure/process-lifecycle-service.ts +21 -3
- package/src/infrastructure/research-run-semaphore.ts +290 -0
- package/src/infrastructure/service-initialization.ts +20 -0
- package/src/orchestration/deep-research-orchestrator.ts +11 -0
- package/src/orchestration/research-orchestration-service.ts +35 -0
- package/src/orchestration/research-synthesis-service.ts +171 -15
- package/src/web-research/retry-utils.ts +9 -3
- package/src/web-research/scraper-utils.ts +43 -0
- package/src/web-research/web-scraper.ts +67 -40
package/dist/thread-worker.mjs
CHANGED
|
@@ -1356,7 +1356,10 @@ function isTransientSocketError(error) {
|
|
|
1356
1356
|
err.message.includes("Worker pool is shutting down") || // Poolifier throws this when pool.execute() is called while the pool is being
|
|
1357
1357
|
// destroyed (e.g., during scheduler restart). Treat as transient — the pool will
|
|
1358
1358
|
// be ready for new tasks once destruction completes and a new pool is initialized.
|
|
1359
|
-
err.message.includes("Cannot execute a task on destroying pool") || err.message.includes("destroying pool")
|
|
1359
|
+
err.message.includes("Cannot execute a task on destroying pool") || err.message.includes("destroying pool") || // The leader answered 503 because it is shutting down after losing
|
|
1360
|
+
// leadership. Retrying re-elects and reaches the new leader, so this is
|
|
1361
|
+
// transient in exactly the same sense as a pool drain.
|
|
1362
|
+
err.message.includes("Browser pool leader is draining"));
|
|
1360
1363
|
}
|
|
1361
1364
|
function isCloudflareBlockError(error) {
|
|
1362
1365
|
if (!error || typeof error !== "object") return false;
|
|
@@ -1368,6 +1371,11 @@ function isTaskTimeoutError(error) {
|
|
|
1368
1371
|
const err = error;
|
|
1369
1372
|
return typeof err.message === "string" && (err.message.includes("timed out after") || err.message.includes("Search task timed out") || err.message.includes("Scrape task timed out") || err.message.includes("Health check timed out") || err.message.includes("[BrowserClient] Request to"));
|
|
1370
1373
|
}
|
|
1374
|
+
function isServerDrainingError(error) {
|
|
1375
|
+
if (!error || typeof error !== "object") return false;
|
|
1376
|
+
const err = error;
|
|
1377
|
+
return typeof err.message === "string" && err.message.includes("Browser pool leader is draining");
|
|
1378
|
+
}
|
|
1371
1379
|
function isPoolShutdownError(error) {
|
|
1372
1380
|
if (!error || typeof error !== "object") return false;
|
|
1373
1381
|
const err = error;
|
|
@@ -1389,6 +1397,7 @@ var init_browser_error_utils = __esm({
|
|
|
1389
1397
|
name: "BrowserPool",
|
|
1390
1398
|
isTransientError: (error) => {
|
|
1391
1399
|
if (isPoolShutdownError(error)) return false;
|
|
1400
|
+
if (isServerDrainingError(error)) return false;
|
|
1392
1401
|
if (isCloudflareBlockError(error)) return false;
|
|
1393
1402
|
if (isTaskTimeoutError(error)) return false;
|
|
1394
1403
|
return isTransientSocketError(error);
|
|
@@ -1404,6 +1413,7 @@ __export(scraper_utils_exports, {
|
|
|
1404
1413
|
createJsMarkdownConverter: () => createJsMarkdownConverter,
|
|
1405
1414
|
createNativeMarkdownConverter: () => createNativeMarkdownConverter,
|
|
1406
1415
|
extractDomain: () => extractDomain,
|
|
1416
|
+
formatErrorWithCause: () => formatErrorWithCause,
|
|
1407
1417
|
getRandomUserAgent: () => getRandomUserAgent,
|
|
1408
1418
|
getSsrfSafeDispatcher: () => getSsrfSafeDispatcher,
|
|
1409
1419
|
isBenignScrapeFailure: () => isBenignScrapeFailure,
|
|
@@ -1535,6 +1545,34 @@ async function getSsrfSafeDispatcher() {
|
|
|
1535
1545
|
}
|
|
1536
1546
|
return cachedSsrfDispatcher;
|
|
1537
1547
|
}
|
|
1548
|
+
function formatErrorWithCause(err) {
|
|
1549
|
+
try {
|
|
1550
|
+
const parts = [];
|
|
1551
|
+
let current = err;
|
|
1552
|
+
for (let depth = 0; current !== null && current !== void 0 && depth < 5; depth++) {
|
|
1553
|
+
let segment;
|
|
1554
|
+
if (current instanceof Error) {
|
|
1555
|
+
segment = current.message || current.name;
|
|
1556
|
+
const code = current.code;
|
|
1557
|
+
if (typeof code === "string" && code && !segment.includes(code)) {
|
|
1558
|
+
segment = `${segment} [${code}]`;
|
|
1559
|
+
}
|
|
1560
|
+
} else if (typeof current === "string") {
|
|
1561
|
+
segment = current;
|
|
1562
|
+
} else {
|
|
1563
|
+
segment = String(current);
|
|
1564
|
+
}
|
|
1565
|
+
if (segment && !parts.includes(segment)) parts.push(segment);
|
|
1566
|
+
const next = current.cause;
|
|
1567
|
+
if (next === current) break;
|
|
1568
|
+
current = next;
|
|
1569
|
+
}
|
|
1570
|
+
if (parts.length === 0) return String(err);
|
|
1571
|
+
return parts.length === 1 ? parts[0] : parts.join(" \u2190 ");
|
|
1572
|
+
} catch {
|
|
1573
|
+
return String(err);
|
|
1574
|
+
}
|
|
1575
|
+
}
|
|
1538
1576
|
function isPrivateIp(ip) {
|
|
1539
1577
|
const parts = ip.split(".").map(Number);
|
|
1540
1578
|
if (parts.length !== 4 || parts.some((p) => isNaN(p))) return false;
|
package/docs/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,32 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [1.0.12] - 2026-08-03
|
|
9
|
+
|
|
10
|
+
### Fixed
|
|
11
|
+
- **Run-cap / file-lock / leader-election could misreport a live process as dead (Windows/macOS) — the root cause of the flaky `run-semaphore-multiprocess` integration failures on Windows CI.** `ProcessLifecycleService.isProcessAlive(pid, expectedStartTime)` returned `false` ("dead") whenever `process.kill(pid, 0)` confirmed the process was alive but the follow-up **start-time lookup failed**. On Linux that null reliably means the process exited in the microsecond gap between the `signal(0)` and the `/proc/{pid}/stat` read, so `false` was correct — but on Windows/macOS the start time comes from a subprocess (`powershell Get-Process` / `ps`) that can fail or time out under load while the process is still very much alive. Reporting a confirmed-alive PID as dead is catastrophic for *every* caller: a live lock / leader / run-cap owner is reclaimed from under it, so two writers collide (lost update) or the run-cap admits extra concurrent runs. The v1.0.11 Windows CI failure manifested both ways depending on timing — `expected 3 to be 4` (two holders collided on one slot) and `expected +0 to be 3` (a 5th process stole a live holder's slot and was admitted past the cap). The fix re-confirms liveness instead of guessing: when the start-time lookup returns null it issues a second `signal(0)`; if the process is still alive it reports `true`, and only if that probe throws (the process genuinely died in the interim) does the outer handler return `false`. A real death is therefore still detected within one poll tick (~100 ms); the only behavioural change is that a confirmed-alive process is never again reclaimed on a transient start-time lookup failure. PID-reuse protection is unchanged when the start time is resolvable (the `±1s` slack comparison still applies). This is a strict safety improvement across all consumers — file locks, the run-cap semaphore, GPU locks, the embedding leader, browser scheduler and state sessions — none of which ever wanted a live owner treated as dead.
|
|
12
|
+
- **Stack Exchange integration tests no longer fail when the upstream API returns empty results** — the Stack Exchange `search` endpoints intermittently answer `200 OK` (quota consumed, no error) with a zero-item `items:[]` for queries that normally return thousands of results (verified 2026-08-03: `q=javascript` on Stack Overflow returned `[]` from a residential IP). This is an upstream API degradation / contract drift, not a pi-research code fault and not a datacenter-IP block, yet it hard-failed the serial integration suite on all three CI OSes and thereby blocked every release whose code was otherwise identical to a previously-green build. The integration-test environment-skip helper (`isNetworkUnavailable`) now also recognizes a successful-but-empty Stack Exchange response (anchored on the tool-specific `**API Quota:**` footer plus the zero-item body) and skips the affected tests visibly via `ctx.skip()` — the same treatment already given to transport errors and `429`/`5xx` throttling. The anchor is emitted only by the Stack Exchange tool, so it cannot mask a regression in any other tool, and a populated result still exercises the assertions in full.
|
|
13
|
+
|
|
14
|
+
## [1.0.11] - 2026-08-02
|
|
15
|
+
|
|
16
|
+
### Added
|
|
17
|
+
- **Cross-process research run-cap** — a machine-wide counting semaphore (`ResearchRunSemaphore`) now gates every `runResearch()` entry on one of N file-lock slots. This is the root-cause fix for the parallel-run instability (60+ error storms and, on constrained hosts, process aborts) that occurred when several research runs saturated the single shared leader-elected browser/embedding pool at once. Holders are reclaimed immediately when their PID exits (PID + startTime guards against PID reuse); *live* holders are never stolen, because a run legitimately holds a slot for minutes. The cap **fails open** on any internal/IO error (a semaphore bug can never break research) and **fails fast** with a distinct `ResearchRunCapacityError` only on genuine capacity exhaustion. (Phase 1 of the multi-run hardening plan.)
|
|
18
|
+
- **Browser leader `drain` / 503 handover** — when the elected leader loses leadership (election loss, idle timeout, or shutdown), it now enters a draining state that answers new requests with an explicit `503 { error: "draining" }` *before* closing the listener, instead of letting connected clients stall on a connect timeout or bare reset. The client classifies this as a transient "re-elect and retry" signal (logged at DEBUG, recorded under its own `draining` errorType for forensics) and it is deliberately excluded from the circuit breaker so a routine leadership handover never reads as an incident.
|
|
19
|
+
|
|
20
|
+
### Fixed
|
|
21
|
+
- **Citation rewriting regression — `[removed]` tokens, list/code corruption, `[0]` deletion** — the prior placeholder-citation defense rewrote synthesis bodies with a global space collapse that flattened nested lists and 4-space code blocks, deleted bracketed integers *inside* quoted code (`items[12]`), treated `[0]` as a citation, and emitted broken `[removed]` tokens for dropped citations. All replaced by a structure-preserving rewrite: fenced blocks and inline-code spans are skipped entirely, line-leading indentation is preserved, dangling markers are removed cleanly (never as a visible token), and `[N]` outside a plausible citation range (`1..200`) and `[0]` are left untouched.
|
|
22
|
+
- **Provenance gate mass-dropping legitimate citations** — the gate that previously DROPPED citations tagged Scrape/Transcript when their URL was absent from the successful-scrape pool conflated a *blocked* scrape (real URL, content unretrieved) with fabrication: under the common high-scrape-failure rate (bot blocks on GitHub / Microsoft / etc.) it mass-deleted real citations. The gate is removed; every parsed citation is now kept (deduped by URL). Fabrication defense rests on URL dedup plus a new prose-URL sweep instead of punishing failed scrapes.
|
|
23
|
+
- **Fabricated inline URLs in synthesis prose** — inline `https?://` tokens in the report body whose URL was not retrieved this session are now redacted to `[link removed: not found in retrieved sources]` (verified inline references are kept). A matching caveat block is appended when a model-written `CITED LINKS` section survives a run that retrieved nothing, so no report ships fully-sourced-looking with no warning on a zero-retrieval run.
|
|
24
|
+
- **Fetch-layer scrape had no retry** — the fetch layer was the only scrape layer with no transient retry, so a single socket/DNS blip (ECONNRESET, ENOTFOUND, `UND_ERR_*`, a bare `fetch failed`) cost a full 10–30× slower browser render. It now retries once (250 ms, gated on `isTransientError`) before falling back to the stealth browser; SSRF rejections, size caps and 4xx rethrow immediately. Error tracking is recorded exactly once per URL (moved out of the retried layer) rather than once per attempt.
|
|
25
|
+
- **Uninformative `fetch failed` scrape logs** — Node/undici wraps the real transport reason under `error.cause`; logging only `String(err)` recorded the uninformative top-level message and discarded it. A new `formatErrorWithCause` walks the cause chain (bounded, `.code`-folding, defensive) so every fetch-failure line is self-diagnosing; it is now used consistently across the scrape path.
|
|
26
|
+
- **Final synthesis body never logged** — every other layer (researcher prompts/responses, searches, scrapes) was logged at DEBUG, but the synthesis itself — the actual bytes returned to the caller — was the one conspicuous exception, making post-hoc diagnosis of citation/output reports impossible. The full final and fallback synthesis bodies are now logged at DEBUG.
|
|
27
|
+
- **`abortableDelay` could exit the process mid-operation** — the timer was unconditionally `unref`'d, so a foreground backoff that was the sole pending handle let Node drain the loop and silently truncate work. A `keepAlive` option (used by the new foreground fetch retry) holds the loop until the operation completes; background callers are unaffected.
|
|
28
|
+
|
|
29
|
+
## [1.0.10] - 2026-07-30
|
|
30
|
+
|
|
31
|
+
### Fixed
|
|
32
|
+
- **pi 0.83.0 compatibility — `ResourceLoader` contract** — pi 0.83.0 extended the `ResourceLoader` contract with `getSystemPromptSource()` and `getAppendSystemPromptSources()` (consumed by interactive mode's startup context listing for file-backed `SYSTEM.md`/`APPEND_SYSTEM.md`). The in-memory sub-session loader now implements both, returning "no backing file" — accurate for prompts passed as text. The extra methods are harmless on pre-0.83 hosts, so the supported range stays `>=0.80.8 <1`. Audited the 0.82.1→0.83.0 dist diffs: the only other contract change (`ExtensionContext.scopedModels`) is provided by pi's own session code, and the mock runtime path has been immune since v1.0.9 via `createExtensionRuntime()`.
|
|
33
|
+
|
|
8
34
|
## [1.0.9] - 2026-07-28
|
|
9
35
|
|
|
10
36
|
### Fixed
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lincoln504/pi-research",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.12",
|
|
4
4
|
"description": "A set of web research tools (search, scrape, and others) and research agent system that uses them safely and efficiently.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"private": false,
|
|
@@ -142,7 +142,10 @@
|
|
|
142
142
|
"ua-parser-js": "^1.0.41",
|
|
143
143
|
"protobufjs": "7.6.5",
|
|
144
144
|
"vite": "^8.0.16",
|
|
145
|
-
"adm-zip": "^0.6.0"
|
|
145
|
+
"adm-zip": "^0.6.0",
|
|
146
|
+
"jsdom": {
|
|
147
|
+
"undici": "^7.29.0"
|
|
148
|
+
}
|
|
146
149
|
},
|
|
147
150
|
"devDependencies": {
|
|
148
151
|
"@eslint/js": "^10.0.1",
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
name: pi-research
|
|
3
3
|
description: "Multi-agent web research for any coding agent. Use for anything requiring current information from the internet — news, trends, product/service/company facts, statistics, people, places, events, APIs, library behavior, or 'what is X / how does X work' questions. Also use to search a local knowledge store of previously researched findings before doing live research. Powered by the pi-research engine (search, stealth scraping, security databases, Stack Exchange). Not for reading local files, running project commands, or analyzing this repository's own source."
|
|
4
4
|
license: MIT
|
|
5
|
-
metadata: { "author": "Lincoln Deen", "version": "1.0.
|
|
5
|
+
metadata: { "author": "Lincoln Deen", "version": "1.0.12", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
|
|
6
6
|
allowed-tools: Bash(node:*)
|
|
7
7
|
---
|
|
8
8
|
|
|
@@ -31,6 +31,8 @@ export const ServiceNames = {
|
|
|
31
31
|
// Research session services
|
|
32
32
|
RESEARCH_SESSION_SERVICE: 'research-session-service',
|
|
33
33
|
RESEARCH_SYNTHESIS_SERVICE: 'research-synthesis-service',
|
|
34
|
+
// Cross-process cap on concurrent research runs (Phase 1 run-cap)
|
|
35
|
+
RESEARCH_RUN_SEMAPHORE: 'research-run-semaphore',
|
|
34
36
|
} as const;
|
|
35
37
|
|
|
36
38
|
/**
|
|
@@ -103,7 +103,25 @@ export class BrowserClient implements IScheduler {
|
|
|
103
103
|
const duration = Date.now() - start;
|
|
104
104
|
try {
|
|
105
105
|
const parsed = JSON.parse(body);
|
|
106
|
-
if (res.statusCode
|
|
106
|
+
if (res.statusCode === 503 && parsed?.error === 'draining') {
|
|
107
|
+
// Planned leadership handover, not a fault. Distinct message so
|
|
108
|
+
// isServerDrainingError can classify it: retry paths treat it as
|
|
109
|
+
// transient and the circuit breaker ignores it. Logged at DEBUG,
|
|
110
|
+
// not ERROR, so a routine re-election doesn't read as an incident.
|
|
111
|
+
// It is still recorded in the error tracker (this file records every
|
|
112
|
+
// non-200 uniformly) under its own `draining` errorType, which is
|
|
113
|
+
// what keeps handovers distinguishable from real 503s in forensics.
|
|
114
|
+
const error = new Error(
|
|
115
|
+
`Browser pool leader is draining (${parsed.reason || 'leadership-lost'}) — re-elect and retry`,
|
|
116
|
+
);
|
|
117
|
+
logger.debug(`[BrowserClient] Leader draining on ${path}; re-election required.`);
|
|
118
|
+
errorTracker.trackError(error, {
|
|
119
|
+
component: 'browser-manager',
|
|
120
|
+
operation,
|
|
121
|
+
errorType: 'draining',
|
|
122
|
+
});
|
|
123
|
+
reject(error);
|
|
124
|
+
} else if (res.statusCode !== 200) {
|
|
107
125
|
const error = new Error(parsed.error || `HTTP ${res.statusCode}`);
|
|
108
126
|
errorTracker.trackError(error, {
|
|
109
127
|
component: 'browser-manager',
|
|
@@ -31,7 +31,11 @@ export function isTransientSocketError(error: unknown): boolean {
|
|
|
31
31
|
// destroyed (e.g., during scheduler restart). Treat as transient — the pool will
|
|
32
32
|
// be ready for new tasks once destruction completes and a new pool is initialized.
|
|
33
33
|
err.message.includes('Cannot execute a task on destroying pool') ||
|
|
34
|
-
err.message.includes('destroying pool')
|
|
34
|
+
err.message.includes('destroying pool') ||
|
|
35
|
+
// The leader answered 503 because it is shutting down after losing
|
|
36
|
+
// leadership. Retrying re-elects and reaches the new leader, so this is
|
|
37
|
+
// transient in exactly the same sense as a pool drain.
|
|
38
|
+
err.message.includes('Browser pool leader is draining')
|
|
35
39
|
);
|
|
36
40
|
}
|
|
37
41
|
|
|
@@ -68,6 +72,22 @@ export function isTaskTimeoutError(error: unknown): boolean {
|
|
|
68
72
|
);
|
|
69
73
|
}
|
|
70
74
|
|
|
75
|
+
/**
|
|
76
|
+
* Check if an error is a "leader is draining" rejection (HTTP 503 from
|
|
77
|
+
* {@link BrowserServer.beginDrain}).
|
|
78
|
+
*
|
|
79
|
+
* The leader answers 503 while tearing down after losing leadership, so this is
|
|
80
|
+
* an expected transition signal rather than a fault: the correct response is to
|
|
81
|
+
* re-elect and retry against the new leader. It must not count toward the
|
|
82
|
+
* circuit breaker — a leadership handover would otherwise look like a burst of
|
|
83
|
+
* infrastructure failures and open the breaker for every subsequent request.
|
|
84
|
+
*/
|
|
85
|
+
export function isServerDrainingError(error: unknown): boolean {
|
|
86
|
+
if (!error || typeof error !== 'object') return false;
|
|
87
|
+
const err = error as NodeError;
|
|
88
|
+
return typeof err.message === 'string' && err.message.includes('Browser pool leader is draining');
|
|
89
|
+
}
|
|
90
|
+
|
|
71
91
|
/**
|
|
72
92
|
* Check if an error is specifically a pool-shutdown / pool-drain error.
|
|
73
93
|
* For these errors the scheduler itself is fine — only the WorkerPool is temporarily
|
|
@@ -107,6 +127,9 @@ const DEFAULT_BREAKER_CONFIG: any = {
|
|
|
107
127
|
isTransientError: (error: unknown) => {
|
|
108
128
|
// Pool-drain transients: pool recovers on its own — don't count.
|
|
109
129
|
if (isPoolShutdownError(error)) return false;
|
|
130
|
+
// Leadership handover (503 draining) is a planned transition, not a
|
|
131
|
+
// failure — counting it would open the breaker on every re-election.
|
|
132
|
+
if (isServerDrainingError(error)) return false;
|
|
110
133
|
// Cloudflare blocks are content-layer rejections, not infra failures — don't count.
|
|
111
134
|
if (isCloudflareBlockError(error)) return false;
|
|
112
135
|
// Task-level timeouts (slow/unresponsive sites) are expected during parallel
|
|
@@ -32,9 +32,57 @@ export function getBrowserServerAuthSecret(): string {
|
|
|
32
32
|
export class BrowserServer {
|
|
33
33
|
private server: http.Server | null = null;
|
|
34
34
|
private port: number = 0;
|
|
35
|
+
/**
|
|
36
|
+
* Live client sockets. Tracked so teardown can tell whether anyone is still
|
|
37
|
+
* attached (`hasActiveClients`) rather than guessing, and so drain has a
|
|
38
|
+
* meaningful count to report.
|
|
39
|
+
*/
|
|
40
|
+
private connections: Set<import('node:net').Socket> = new Set();
|
|
41
|
+
private draining = false;
|
|
42
|
+
/**
|
|
43
|
+
* Why this server stopped accepting work. Reported to the client so a log line
|
|
44
|
+
* says which teardown it hit — a lost election, an idle timeout, or process
|
|
45
|
+
* shutdown all reach the same code path but are very different incidents.
|
|
46
|
+
*/
|
|
47
|
+
private drainReason = 'shutting-down';
|
|
35
48
|
|
|
36
49
|
constructor(private options: BrowserServerOptions) {}
|
|
37
50
|
|
|
51
|
+
/**
|
|
52
|
+
* Enter draining mode: every *subsequent* request is answered `503` instead of
|
|
53
|
+
* being served or left to hit a closed socket. Requests already past this
|
|
54
|
+
* check run to completion, so drain never truncates in-flight work.
|
|
55
|
+
*
|
|
56
|
+
* This exists for leadership-loss teardown. Without it a client that is
|
|
57
|
+
* mid-conversation with a leader that just lost the election waits out a
|
|
58
|
+
* connect timeout or gets a bare ECONNREFUSED/RST once the listener closes,
|
|
59
|
+
* and only then re-elects. An explicit 503 turns that silent stall into an
|
|
60
|
+
* immediate, unambiguous "this leader is gone — re-elect now".
|
|
61
|
+
*
|
|
62
|
+
* Idempotent: calling it while already draining is a no-op.
|
|
63
|
+
*/
|
|
64
|
+
beginDrain(reason = 'shutting-down'): void {
|
|
65
|
+
if (this.draining) return;
|
|
66
|
+
this.draining = true;
|
|
67
|
+
this.drainReason = reason;
|
|
68
|
+
logger.log(`[BrowserServer] Draining (${reason}) — refusing new requests with 503 (${this.connections.size} client connection(s) attached).`);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** True once {@link beginDrain} has been called. */
|
|
72
|
+
isDraining(): boolean {
|
|
73
|
+
return this.draining;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Whether any client socket is currently attached. */
|
|
77
|
+
hasActiveClients(): boolean {
|
|
78
|
+
return this.connections.size > 0;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Number of attached client sockets (observability / teardown decisions). */
|
|
82
|
+
getConnectionCount(): number {
|
|
83
|
+
return this.connections.size;
|
|
84
|
+
}
|
|
85
|
+
|
|
38
86
|
async start(): Promise<number> {
|
|
39
87
|
return new Promise((resolve, reject) => {
|
|
40
88
|
this.server = http.createServer(async (req, res) => {
|
|
@@ -49,6 +97,20 @@ export class BrowserServer {
|
|
|
49
97
|
return;
|
|
50
98
|
}
|
|
51
99
|
|
|
100
|
+
// Checked after auth so an unauthenticated caller cannot probe
|
|
101
|
+
// leadership state, and before routing so no new work is started
|
|
102
|
+
// against a pool that is being torn down. `Connection: close`
|
|
103
|
+
// stops the client reusing this socket for a retry that would
|
|
104
|
+
// only 503 again.
|
|
105
|
+
if (this.draining) {
|
|
106
|
+
res.writeHead(503, {
|
|
107
|
+
'Content-Type': 'application/json',
|
|
108
|
+
'Connection': 'close',
|
|
109
|
+
});
|
|
110
|
+
res.end(JSON.stringify({ error: 'draining', reason: this.drainReason }));
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
|
|
52
114
|
if (req.method !== 'POST') {
|
|
53
115
|
res.writeHead(405);
|
|
54
116
|
res.end('Method Not Allowed');
|
|
@@ -138,6 +200,13 @@ export class BrowserServer {
|
|
|
138
200
|
});
|
|
139
201
|
});
|
|
140
202
|
|
|
203
|
+
this.server.on('connection', (socket) => {
|
|
204
|
+
this.connections.add(socket);
|
|
205
|
+
socket.on('close', () => {
|
|
206
|
+
this.connections.delete(socket);
|
|
207
|
+
});
|
|
208
|
+
});
|
|
209
|
+
|
|
141
210
|
this.server.listen(0, '127.0.0.1', () => {
|
|
142
211
|
const addr = this.server?.address();
|
|
143
212
|
if (addr && typeof addr === 'object') {
|
|
@@ -162,6 +231,7 @@ export class BrowserServer {
|
|
|
162
231
|
return new Promise((resolve) => {
|
|
163
232
|
this.server?.close(() => {
|
|
164
233
|
this.server = null;
|
|
234
|
+
this.connections.clear();
|
|
165
235
|
resolve();
|
|
166
236
|
});
|
|
167
237
|
});
|
|
@@ -108,7 +108,7 @@ export class BrowserTaskScheduler implements IScheduler {
|
|
|
108
108
|
if (this.consecutiveLeadershipMisses >= this.LEADERSHIP_MISS_THRESHOLD) {
|
|
109
109
|
metrics.increment('browser_leadership_lost_total', 1);
|
|
110
110
|
logger.error(`[Scheduler] Leadership threshold exceeded (${this.consecutiveLeadershipMisses} misses), shutting down pool...`);
|
|
111
|
-
await this.shutdown();
|
|
111
|
+
await this.shutdown('leadership-lost');
|
|
112
112
|
return;
|
|
113
113
|
}
|
|
114
114
|
} else {
|
|
@@ -145,7 +145,7 @@ export class BrowserTaskScheduler implements IScheduler {
|
|
|
145
145
|
// getService throwing during a concurrent container disposal) would otherwise surface
|
|
146
146
|
// as an unhandledRejection from this fire-and-forget timer. Swallow it — an idle
|
|
147
147
|
// teardown that fails is non-fatal; the next run gets a fresh scheduler regardless.
|
|
148
|
-
this.shutdown().catch(err => logger.warn('[Scheduler] Idle-timeout shutdown failed (non-fatal):', err));
|
|
148
|
+
this.shutdown('idle-timeout').catch(err => logger.warn('[Scheduler] Idle-timeout shutdown failed (non-fatal):', err));
|
|
149
149
|
}, this.IDLE_TIMEOUT_MS);
|
|
150
150
|
if (this.idleTimer.unref) this.idleTimer.unref();
|
|
151
151
|
}
|
|
@@ -374,10 +374,26 @@ export class BrowserTaskScheduler implements IScheduler {
|
|
|
374
374
|
return result;
|
|
375
375
|
}
|
|
376
376
|
|
|
377
|
-
|
|
377
|
+
/**
|
|
378
|
+
* @param reason why the pool is going away. Reported verbatim to clients in the
|
|
379
|
+
* drain 503, so a lost election, an idle timeout and a process shutdown are
|
|
380
|
+
* distinguishable in a client-side log rather than all reading as one.
|
|
381
|
+
*/
|
|
382
|
+
async shutdown(reason = 'shutting-down') {
|
|
378
383
|
if (this.isShuttingDown) return;
|
|
379
384
|
this.isShuttingDown = true;
|
|
380
385
|
|
|
386
|
+
// Start draining before any of the teardown below. Clearing state and
|
|
387
|
+
// stopping the pool involves awaits, and until the listener actually
|
|
388
|
+
// closes a connected client would otherwise keep issuing requests against
|
|
389
|
+
// a pool that is already going away — the leadership-loss case this
|
|
390
|
+
// guards. Draining first turns those into an immediate 503 so the client
|
|
391
|
+
// re-elects at once instead of waiting for a reset.
|
|
392
|
+
if (this.server?.hasActiveClients()) {
|
|
393
|
+
logger.log(`[Scheduler] Draining ${this.server.getConnectionCount()} client connection(s) before shutdown (${reason}).`);
|
|
394
|
+
}
|
|
395
|
+
this.server?.beginDrain(reason);
|
|
396
|
+
|
|
381
397
|
if (this.idleTimer) {
|
|
382
398
|
clearTimeout(this.idleTimer);
|
|
383
399
|
this.idleTimer = null;
|
|
@@ -139,7 +139,7 @@ export async function forceSchedulerRestart(forceClearRemoteState: boolean = fal
|
|
|
139
139
|
|
|
140
140
|
if (oldScheduler && 'schedulerId' in oldScheduler && oldScheduler.schedulerId) {
|
|
141
141
|
if (oldScheduler instanceof BrowserTaskScheduler) {
|
|
142
|
-
const shutdownPromise: Promise<void> = oldScheduler.shutdown().catch((err: unknown) => {
|
|
142
|
+
const shutdownPromise: Promise<void> = oldScheduler.shutdown('leadership-lost').catch((err: unknown) => {
|
|
143
143
|
logger.warn('[Scheduler] Error during old scheduler shutdown after restart:', err);
|
|
144
144
|
});
|
|
145
145
|
schedulerService.setPendingShutdownPromise(shutdownPromise);
|
|
@@ -1356,7 +1356,10 @@ function isTransientSocketError(error) {
|
|
|
1356
1356
|
err.message.includes("Worker pool is shutting down") || // Poolifier throws this when pool.execute() is called while the pool is being
|
|
1357
1357
|
// destroyed (e.g., during scheduler restart). Treat as transient — the pool will
|
|
1358
1358
|
// be ready for new tasks once destruction completes and a new pool is initialized.
|
|
1359
|
-
err.message.includes("Cannot execute a task on destroying pool") || err.message.includes("destroying pool")
|
|
1359
|
+
err.message.includes("Cannot execute a task on destroying pool") || err.message.includes("destroying pool") || // The leader answered 503 because it is shutting down after losing
|
|
1360
|
+
// leadership. Retrying re-elects and reaches the new leader, so this is
|
|
1361
|
+
// transient in exactly the same sense as a pool drain.
|
|
1362
|
+
err.message.includes("Browser pool leader is draining"));
|
|
1360
1363
|
}
|
|
1361
1364
|
function isCloudflareBlockError(error) {
|
|
1362
1365
|
if (!error || typeof error !== "object") return false;
|
|
@@ -1368,6 +1371,11 @@ function isTaskTimeoutError(error) {
|
|
|
1368
1371
|
const err = error;
|
|
1369
1372
|
return typeof err.message === "string" && (err.message.includes("timed out after") || err.message.includes("Search task timed out") || err.message.includes("Scrape task timed out") || err.message.includes("Health check timed out") || err.message.includes("[BrowserClient] Request to"));
|
|
1370
1373
|
}
|
|
1374
|
+
function isServerDrainingError(error) {
|
|
1375
|
+
if (!error || typeof error !== "object") return false;
|
|
1376
|
+
const err = error;
|
|
1377
|
+
return typeof err.message === "string" && err.message.includes("Browser pool leader is draining");
|
|
1378
|
+
}
|
|
1371
1379
|
function isPoolShutdownError(error) {
|
|
1372
1380
|
if (!error || typeof error !== "object") return false;
|
|
1373
1381
|
const err = error;
|
|
@@ -1389,6 +1397,7 @@ var init_browser_error_utils = __esm({
|
|
|
1389
1397
|
name: "BrowserPool",
|
|
1390
1398
|
isTransientError: (error) => {
|
|
1391
1399
|
if (isPoolShutdownError(error)) return false;
|
|
1400
|
+
if (isServerDrainingError(error)) return false;
|
|
1392
1401
|
if (isCloudflareBlockError(error)) return false;
|
|
1393
1402
|
if (isTaskTimeoutError(error)) return false;
|
|
1394
1403
|
return isTransientSocketError(error);
|
|
@@ -1404,6 +1413,7 @@ __export(scraper_utils_exports, {
|
|
|
1404
1413
|
createJsMarkdownConverter: () => createJsMarkdownConverter,
|
|
1405
1414
|
createNativeMarkdownConverter: () => createNativeMarkdownConverter,
|
|
1406
1415
|
extractDomain: () => extractDomain,
|
|
1416
|
+
formatErrorWithCause: () => formatErrorWithCause,
|
|
1407
1417
|
getRandomUserAgent: () => getRandomUserAgent,
|
|
1408
1418
|
getSsrfSafeDispatcher: () => getSsrfSafeDispatcher,
|
|
1409
1419
|
isBenignScrapeFailure: () => isBenignScrapeFailure,
|
|
@@ -1535,6 +1545,34 @@ async function getSsrfSafeDispatcher() {
|
|
|
1535
1545
|
}
|
|
1536
1546
|
return cachedSsrfDispatcher;
|
|
1537
1547
|
}
|
|
1548
|
+
function formatErrorWithCause(err) {
|
|
1549
|
+
try {
|
|
1550
|
+
const parts = [];
|
|
1551
|
+
let current = err;
|
|
1552
|
+
for (let depth = 0; current !== null && current !== void 0 && depth < 5; depth++) {
|
|
1553
|
+
let segment;
|
|
1554
|
+
if (current instanceof Error) {
|
|
1555
|
+
segment = current.message || current.name;
|
|
1556
|
+
const code = current.code;
|
|
1557
|
+
if (typeof code === "string" && code && !segment.includes(code)) {
|
|
1558
|
+
segment = `${segment} [${code}]`;
|
|
1559
|
+
}
|
|
1560
|
+
} else if (typeof current === "string") {
|
|
1561
|
+
segment = current;
|
|
1562
|
+
} else {
|
|
1563
|
+
segment = String(current);
|
|
1564
|
+
}
|
|
1565
|
+
if (segment && !parts.includes(segment)) parts.push(segment);
|
|
1566
|
+
const next = current.cause;
|
|
1567
|
+
if (next === current) break;
|
|
1568
|
+
current = next;
|
|
1569
|
+
}
|
|
1570
|
+
if (parts.length === 0) return String(err);
|
|
1571
|
+
return parts.length === 1 ? parts[0] : parts.join(" \u2190 ");
|
|
1572
|
+
} catch {
|
|
1573
|
+
return String(err);
|
|
1574
|
+
}
|
|
1575
|
+
}
|
|
1538
1576
|
function isPrivateIp(ip) {
|
|
1539
1577
|
const parts = ip.split(".").map(Number);
|
|
1540
1578
|
if (parts.length !== 4 || parts.some((p) => isNaN(p))) return false;
|
|
@@ -119,9 +119,27 @@ export class ProcessLifecycleService implements IProcessLifecycle {
|
|
|
119
119
|
if (expectedStartTime !== undefined && expectedStartTime !== null) {
|
|
120
120
|
const actualStartTime = await this.getProcessStartTime(pid);
|
|
121
121
|
if (actualStartTime === null) {
|
|
122
|
-
//
|
|
123
|
-
//
|
|
124
|
-
|
|
122
|
+
// signal(0) above confirmed a live process at this PID, but we couldn't
|
|
123
|
+
// resolve its start time to verify it's the SAME instance (the PID-reuse
|
|
124
|
+
// guard). The two failure modes collapse to the same null here but must
|
|
125
|
+
// NOT be treated identically:
|
|
126
|
+
// • Linux: usually the process exited in the microsecond gap between
|
|
127
|
+
// signal(0) and the /proc/{pid}/stat read → genuinely dead.
|
|
128
|
+
// • Windows/macOS: far more often the start-time lookup subprocess
|
|
129
|
+
// (powershell `Get-Process` / `ps`) simply failed or hit its timeout
|
|
130
|
+
// while the process is still very much alive → NOT dead.
|
|
131
|
+
// Reporting that confirmed-alive PID as DEAD is catastrophic for every
|
|
132
|
+
// caller — a live lock/leader/run-cap owner is reclaimed from under it
|
|
133
|
+
// → two writers collide (lost update) or the run-cap admits extra
|
|
134
|
+
// concurrent runs (the flaky duplicate-slot/cap-breach we saw on Windows
|
|
135
|
+
// CI). So re-confirm liveness instead of guessing: if signal(0) STILL
|
|
136
|
+
// succeeds the process is alive → return true; if it now throws it died
|
|
137
|
+
// between the two probes and the outer catch returns false. A genuine
|
|
138
|
+
// death is therefore still detected within one poll tick (~100ms); the
|
|
139
|
+
// only behavioural change is that a confirmed-alive PID is never again
|
|
140
|
+
// misreported as dead on a transient start-time lookup failure.
|
|
141
|
+
process.kill(pid, 0); // re-confirm; a throw (process died) → outer catch → false
|
|
142
|
+
return true;
|
|
125
143
|
}
|
|
126
144
|
// Allow ±1s slack. The macOS/BSD fallback derives start time as
|
|
127
145
|
// floor(Date.now()/1000) - `ps etimes` (elapsed whole seconds); both terms are
|