@lincoln504/pi-research 1.0.10 → 1.0.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1356,7 +1356,10 @@ function isTransientSocketError(error) {
1356
1356
  err.message.includes("Worker pool is shutting down") || // Poolifier throws this when pool.execute() is called while the pool is being
1357
1357
  // destroyed (e.g., during scheduler restart). Treat as transient — the pool will
1358
1358
  // be ready for new tasks once destruction completes and a new pool is initialized.
1359
- err.message.includes("Cannot execute a task on destroying pool") || err.message.includes("destroying pool"));
1359
+ err.message.includes("Cannot execute a task on destroying pool") || err.message.includes("destroying pool") || // The leader answered 503 because it is shutting down after losing
1360
+ // leadership. Retrying re-elects and reaches the new leader, so this is
1361
+ // transient in exactly the same sense as a pool drain.
1362
+ err.message.includes("Browser pool leader is draining"));
1360
1363
  }
1361
1364
  function isCloudflareBlockError(error) {
1362
1365
  if (!error || typeof error !== "object") return false;
@@ -1368,6 +1371,11 @@ function isTaskTimeoutError(error) {
1368
1371
  const err = error;
1369
1372
  return typeof err.message === "string" && (err.message.includes("timed out after") || err.message.includes("Search task timed out") || err.message.includes("Scrape task timed out") || err.message.includes("Health check timed out") || err.message.includes("[BrowserClient] Request to"));
1370
1373
  }
1374
+ function isServerDrainingError(error) {
1375
+ if (!error || typeof error !== "object") return false;
1376
+ const err = error;
1377
+ return typeof err.message === "string" && err.message.includes("Browser pool leader is draining");
1378
+ }
1371
1379
  function isPoolShutdownError(error) {
1372
1380
  if (!error || typeof error !== "object") return false;
1373
1381
  const err = error;
@@ -1389,6 +1397,7 @@ var init_browser_error_utils = __esm({
1389
1397
  name: "BrowserPool",
1390
1398
  isTransientError: (error) => {
1391
1399
  if (isPoolShutdownError(error)) return false;
1400
+ if (isServerDrainingError(error)) return false;
1392
1401
  if (isCloudflareBlockError(error)) return false;
1393
1402
  if (isTaskTimeoutError(error)) return false;
1394
1403
  return isTransientSocketError(error);
@@ -1404,6 +1413,7 @@ __export(scraper_utils_exports, {
1404
1413
  createJsMarkdownConverter: () => createJsMarkdownConverter,
1405
1414
  createNativeMarkdownConverter: () => createNativeMarkdownConverter,
1406
1415
  extractDomain: () => extractDomain,
1416
+ formatErrorWithCause: () => formatErrorWithCause,
1407
1417
  getRandomUserAgent: () => getRandomUserAgent,
1408
1418
  getSsrfSafeDispatcher: () => getSsrfSafeDispatcher,
1409
1419
  isBenignScrapeFailure: () => isBenignScrapeFailure,
@@ -1535,6 +1545,34 @@ async function getSsrfSafeDispatcher() {
1535
1545
  }
1536
1546
  return cachedSsrfDispatcher;
1537
1547
  }
1548
+ function formatErrorWithCause(err) {
1549
+ try {
1550
+ const parts = [];
1551
+ let current = err;
1552
+ for (let depth = 0; current !== null && current !== void 0 && depth < 5; depth++) {
1553
+ let segment;
1554
+ if (current instanceof Error) {
1555
+ segment = current.message || current.name;
1556
+ const code = current.code;
1557
+ if (typeof code === "string" && code && !segment.includes(code)) {
1558
+ segment = `${segment} [${code}]`;
1559
+ }
1560
+ } else if (typeof current === "string") {
1561
+ segment = current;
1562
+ } else {
1563
+ segment = String(current);
1564
+ }
1565
+ if (segment && !parts.includes(segment)) parts.push(segment);
1566
+ const next = current.cause;
1567
+ if (next === current) break;
1568
+ current = next;
1569
+ }
1570
+ if (parts.length === 0) return String(err);
1571
+ return parts.length === 1 ? parts[0] : parts.join(" \u2190 ");
1572
+ } catch {
1573
+ return String(err);
1574
+ }
1575
+ }
1538
1576
  function isPrivateIp(ip) {
1539
1577
  const parts = ip.split(".").map(Number);
1540
1578
  if (parts.length !== 4 || parts.some((p) => isNaN(p))) return false;
package/docs/CHANGELOG.md CHANGED
@@ -5,6 +5,32 @@ All notable changes to this project will be documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
+ ## [1.0.12] - 2026-08-03
9
+
10
+ ### Fixed
11
+ - **Run-cap / file-lock / leader-election could misreport a live process as dead (Windows/macOS) — the root cause of the flaky `run-semaphore-multiprocess` integration failures on Windows CI.** `ProcessLifecycleService.isProcessAlive(pid, expectedStartTime)` returned `false` ("dead") whenever `process.kill(pid, 0)` confirmed the process was alive but the follow-up **start-time lookup failed**. On Linux that null reliably means the process exited in the microsecond gap between the `signal(0)` and the `/proc/{pid}/stat` read, so `false` was correct — but on Windows/macOS the start time comes from a subprocess (`powershell Get-Process` / `ps`) that can fail or time out under load while the process is still very much alive. Reporting a confirmed-alive PID as dead is catastrophic for *every* caller: a live lock / leader / run-cap owner is reclaimed from under it, so two writers collide (lost update) or the run-cap admits extra concurrent runs. The v1.0.11 Windows CI failure manifested both ways depending on timing — `expected 3 to be 4` (two holders collided on one slot) and `expected +0 to be 3` (a 5th process stole a live holder's slot and was admitted past the cap). The fix re-confirms liveness instead of guessing: when the start-time lookup returns null it issues a second `signal(0)`; if the process is still alive it reports `true`, and only if that probe throws (the process genuinely died in the interim) does the outer handler return `false`. A real death is therefore still detected within one poll tick (~100 ms); the only behavioural change is that a confirmed-alive process is never again reclaimed on a transient start-time lookup failure. PID-reuse protection is unchanged when the start time is resolvable (the `±1s` slack comparison still applies). This is a strict safety improvement across all consumers — file locks, the run-cap semaphore, GPU locks, the embedding leader, browser scheduler and state sessions — none of which ever wanted a live owner treated as dead.
12
+ - **Stack Exchange integration tests no longer fail when the upstream API returns empty results** — the Stack Exchange `search` endpoints intermittently answer `200 OK` (quota consumed, no error) with a zero-item `items:[]` for queries that normally return thousands of results (verified 2026-08-03: `q=javascript` on Stack Overflow returned `[]` from a residential IP). This is an upstream API degradation / contract drift, not a pi-research code fault and not a datacenter-IP block, yet it hard-failed the serial integration suite on all three CI OSes and thereby blocked every release whose code was otherwise identical to a previously-green build. The integration-test environment-skip helper (`isNetworkUnavailable`) now also recognizes a successful-but-empty Stack Exchange response (anchored on the tool-specific `**API Quota:**` footer plus the zero-item body) and skips the affected tests visibly via `ctx.skip()` — the same treatment already given to transport errors and `429`/`5xx` throttling. The anchor is emitted only by the Stack Exchange tool, so it cannot mask a regression in any other tool, and a populated result still exercises the assertions in full.
13
+
14
+ ## [1.0.11] - 2026-08-02
15
+
16
+ ### Added
17
+ - **Cross-process research run-cap** — a machine-wide counting semaphore (`ResearchRunSemaphore`) now gates every `runResearch()` entry on one of N file-lock slots. This is the root-cause fix for the parallel-run instability (60+ error storms and, on constrained hosts, process aborts) that occurred when several research runs saturated the single shared leader-elected browser/embedding pool at once. Holders are reclaimed immediately when their PID exits (PID + startTime guards against PID reuse); *live* holders are never stolen, because a run legitimately holds a slot for minutes. The cap **fails open** on any internal/IO error (a semaphore bug can never break research) and **fails fast** with a distinct `ResearchRunCapacityError` only on genuine capacity exhaustion. (Phase 1 of the multi-run hardening plan.)
18
+ - **Browser leader `drain` / 503 handover** — when the elected leader loses leadership (election loss, idle timeout, or shutdown), it now enters a draining state that answers new requests with an explicit `503 { error: "draining" }` *before* closing the listener, instead of letting connected clients stall on a connect timeout or bare reset. The client classifies this as a transient "re-elect and retry" signal (logged at DEBUG, recorded under its own `draining` errorType for forensics) and it is deliberately excluded from the circuit breaker so a routine leadership handover never reads as an incident.
19
+
20
+ ### Fixed
21
+ - **Citation rewriting regression — `[removed]` tokens, list/code corruption, `[0]` deletion** — the prior placeholder-citation defense rewrote synthesis bodies with a global space collapse that flattened nested lists and 4-space code blocks, deleted bracketed integers *inside* quoted code (`items[12]`), treated `[0]` as a citation, and emitted broken `[removed]` tokens for dropped citations. All replaced by a structure-preserving rewrite: fenced blocks and inline-code spans are skipped entirely, line-leading indentation is preserved, dangling markers are removed cleanly (never as a visible token), and `[N]` outside a plausible citation range (`1..200`) and `[0]` are left untouched.
22
+ - **Provenance gate mass-dropping legitimate citations** — the gate that previously DROPPED citations tagged Scrape/Transcript when their URL was absent from the successful-scrape pool conflated a *blocked* scrape (real URL, content unretrieved) with fabrication: under the common high-scrape-failure rate (bot blocks on GitHub / Microsoft / etc.) it mass-deleted real citations. The gate is removed; every parsed citation is now kept (deduped by URL). Fabrication defense rests on URL dedup plus a new prose-URL sweep instead of punishing failed scrapes.
23
+ - **Fabricated inline URLs in synthesis prose** — inline `https?://` tokens in the report body whose URL was not retrieved this session are now redacted to `[link removed: not found in retrieved sources]` (verified inline references are kept). A matching caveat block is appended when a model-written `CITED LINKS` section survives a run that retrieved nothing, so no report ships fully-sourced-looking with no warning on a zero-retrieval run.
24
+ - **Fetch-layer scrape had no retry** — the fetch layer was the only scrape layer with no transient retry, so a single socket/DNS blip (ECONNRESET, ENOTFOUND, `UND_ERR_*`, a bare `fetch failed`) cost a full 10–30× slower browser render. It now retries once (250 ms, gated on `isTransientError`) before falling back to the stealth browser; SSRF rejections, size caps and 4xx rethrow immediately. Error tracking is recorded exactly once per URL (moved out of the retried layer) rather than once per attempt.
25
+ - **Uninformative `fetch failed` scrape logs** — Node/undici wraps the real transport reason under `error.cause`; logging only `String(err)` recorded the uninformative top-level message and discarded it. A new `formatErrorWithCause` walks the cause chain (bounded, `.code`-folding, defensive) so every fetch-failure line is self-diagnosing; it is now used consistently across the scrape path.
26
+ - **Final synthesis body never logged** — every other layer (researcher prompts/responses, searches, scrapes) was logged at DEBUG, but the synthesis itself — the actual bytes returned to the caller — was the one conspicuous exception, making post-hoc diagnosis of citation/output reports impossible. The full final and fallback synthesis bodies are now logged at DEBUG.
27
+ - **`abortableDelay` could exit the process mid-operation** — the timer was unconditionally `unref`'d, so a foreground backoff that was the sole pending handle let Node drain the loop and silently truncate work. A `keepAlive` option (used by the new foreground fetch retry) holds the loop until the operation completes; background callers are unaffected.
28
+
29
+ ## [1.0.10] - 2026-07-30
30
+
31
+ ### Fixed
32
+ - **pi 0.83.0 compatibility — `ResourceLoader` contract** — pi 0.83.0 extended the `ResourceLoader` contract with `getSystemPromptSource()` and `getAppendSystemPromptSources()` (consumed by interactive mode's startup context listing for file-backed `SYSTEM.md`/`APPEND_SYSTEM.md`). The in-memory sub-session loader now implements both, returning "no backing file" — accurate for prompts passed as text. The extra methods are harmless on pre-0.83 hosts, so the supported range stays `>=0.80.8 <1`. Audited the 0.82.1→0.83.0 dist diffs: the only other contract change (`ExtensionContext.scopedModels`) is provided by pi's own session code, and the mock runtime path has been immune since v1.0.9 via `createExtensionRuntime()`.
33
+
8
34
  ## [1.0.9] - 2026-07-28
9
35
 
10
36
  ### Fixed
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@lincoln504/pi-research",
3
- "version": "1.0.10",
3
+ "version": "1.0.12",
4
4
  "description": "A set of web research tools (search, scrape, and others) and research agent system that uses them safely and efficiently.",
5
5
  "type": "module",
6
6
  "private": false,
@@ -142,7 +142,10 @@
142
142
  "ua-parser-js": "^1.0.41",
143
143
  "protobufjs": "7.6.5",
144
144
  "vite": "^8.0.16",
145
- "adm-zip": "^0.6.0"
145
+ "adm-zip": "^0.6.0",
146
+ "jsdom": {
147
+ "undici": "^7.29.0"
148
+ }
146
149
  },
147
150
  "devDependencies": {
148
151
  "@eslint/js": "^10.0.1",
@@ -2,7 +2,7 @@
2
2
  name: pi-research
3
3
  description: "Multi-agent web research for any coding agent. Use for anything requiring current information from the internet — news, trends, product/service/company facts, statistics, people, places, events, APIs, library behavior, or 'what is X / how does X work' questions. Also use to search a local knowledge store of previously researched findings before doing live research. Powered by the pi-research engine (search, stealth scraping, security databases, Stack Exchange). Not for reading local files, running project commands, or analyzing this repository's own source."
4
4
  license: MIT
5
- metadata: { "author": "Lincoln Deen", "version": "1.0.10", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
5
+ metadata: { "author": "Lincoln Deen", "version": "1.0.12", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
6
6
  allowed-tools: Bash(node:*)
7
7
  ---
8
8
 
@@ -31,6 +31,8 @@ export const ServiceNames = {
31
31
  // Research session services
32
32
  RESEARCH_SESSION_SERVICE: 'research-session-service',
33
33
  RESEARCH_SYNTHESIS_SERVICE: 'research-synthesis-service',
34
+ // Cross-process cap on concurrent research runs (Phase 1 run-cap)
35
+ RESEARCH_RUN_SEMAPHORE: 'research-run-semaphore',
34
36
  } as const;
35
37
 
36
38
  /**
@@ -103,7 +103,25 @@ export class BrowserClient implements IScheduler {
103
103
  const duration = Date.now() - start;
104
104
  try {
105
105
  const parsed = JSON.parse(body);
106
- if (res.statusCode !== 200) {
106
+ if (res.statusCode === 503 && parsed?.error === 'draining') {
107
+ // Planned leadership handover, not a fault. Distinct message so
108
+ // isServerDrainingError can classify it: retry paths treat it as
109
+ // transient and the circuit breaker ignores it. Logged at DEBUG,
110
+ // not ERROR, so a routine re-election doesn't read as an incident.
111
+ // It is still recorded in the error tracker (this file records every
112
+ // non-200 uniformly) under its own `draining` errorType, which is
113
+ // what keeps handovers distinguishable from real 503s in forensics.
114
+ const error = new Error(
115
+ `Browser pool leader is draining (${parsed.reason || 'leadership-lost'}) — re-elect and retry`,
116
+ );
117
+ logger.debug(`[BrowserClient] Leader draining on ${path}; re-election required.`);
118
+ errorTracker.trackError(error, {
119
+ component: 'browser-manager',
120
+ operation,
121
+ errorType: 'draining',
122
+ });
123
+ reject(error);
124
+ } else if (res.statusCode !== 200) {
107
125
  const error = new Error(parsed.error || `HTTP ${res.statusCode}`);
108
126
  errorTracker.trackError(error, {
109
127
  component: 'browser-manager',
@@ -31,7 +31,11 @@ export function isTransientSocketError(error: unknown): boolean {
31
31
  // destroyed (e.g., during scheduler restart). Treat as transient — the pool will
32
32
  // be ready for new tasks once destruction completes and a new pool is initialized.
33
33
  err.message.includes('Cannot execute a task on destroying pool') ||
34
- err.message.includes('destroying pool')
34
+ err.message.includes('destroying pool') ||
35
+ // The leader answered 503 because it is shutting down after losing
36
+ // leadership. Retrying re-elects and reaches the new leader, so this is
37
+ // transient in exactly the same sense as a pool drain.
38
+ err.message.includes('Browser pool leader is draining')
35
39
  );
36
40
  }
37
41
 
@@ -68,6 +72,22 @@ export function isTaskTimeoutError(error: unknown): boolean {
68
72
  );
69
73
  }
70
74
 
75
+ /**
76
+ * Check if an error is a "leader is draining" rejection (HTTP 503 from
77
+ * {@link BrowserServer.beginDrain}).
78
+ *
79
+ * The leader answers 503 while tearing down after losing leadership, so this is
80
+ * an expected transition signal rather than a fault: the correct response is to
81
+ * re-elect and retry against the new leader. It must not count toward the
82
+ * circuit breaker — a leadership handover would otherwise look like a burst of
83
+ * infrastructure failures and open the breaker for every subsequent request.
84
+ */
85
+ export function isServerDrainingError(error: unknown): boolean {
86
+ if (!error || typeof error !== 'object') return false;
87
+ const err = error as NodeError;
88
+ return typeof err.message === 'string' && err.message.includes('Browser pool leader is draining');
89
+ }
90
+
71
91
  /**
72
92
  * Check if an error is specifically a pool-shutdown / pool-drain error.
73
93
  * For these errors the scheduler itself is fine — only the WorkerPool is temporarily
@@ -107,6 +127,9 @@ const DEFAULT_BREAKER_CONFIG: any = {
107
127
  isTransientError: (error: unknown) => {
108
128
  // Pool-drain transients: pool recovers on its own — don't count.
109
129
  if (isPoolShutdownError(error)) return false;
130
+ // Leadership handover (503 draining) is a planned transition, not a
131
+ // failure — counting it would open the breaker on every re-election.
132
+ if (isServerDrainingError(error)) return false;
110
133
  // Cloudflare blocks are content-layer rejections, not infra failures — don't count.
111
134
  if (isCloudflareBlockError(error)) return false;
112
135
  // Task-level timeouts (slow/unresponsive sites) are expected during parallel
@@ -32,9 +32,57 @@ export function getBrowserServerAuthSecret(): string {
32
32
  export class BrowserServer {
33
33
  private server: http.Server | null = null;
34
34
  private port: number = 0;
35
+ /**
36
+ * Live client sockets. Tracked so teardown can tell whether anyone is still
37
+ * attached (`hasActiveClients`) rather than guessing, and so drain has a
38
+ * meaningful count to report.
39
+ */
40
+ private connections: Set<import('node:net').Socket> = new Set();
41
+ private draining = false;
42
+ /**
43
+ * Why this server stopped accepting work. Reported to the client so a log line
44
+ * says which teardown it hit — a lost election, an idle timeout, or process
45
+ * shutdown all reach the same code path but are very different incidents.
46
+ */
47
+ private drainReason = 'shutting-down';
35
48
 
36
49
  constructor(private options: BrowserServerOptions) {}
37
50
 
51
+ /**
52
+ * Enter draining mode: every *subsequent* request is answered `503` instead of
53
+ * being served or left to hit a closed socket. Requests already past this
54
+ * check run to completion, so drain never truncates in-flight work.
55
+ *
56
+ * This exists for leadership-loss teardown. Without it a client that is
57
+ * mid-conversation with a leader that just lost the election waits out a
58
+ * connect timeout or gets a bare ECONNREFUSED/RST once the listener closes,
59
+ * and only then re-elects. An explicit 503 turns that silent stall into an
60
+ * immediate, unambiguous "this leader is gone — re-elect now".
61
+ *
62
+ * Idempotent: calling it while already draining is a no-op.
63
+ */
64
+ beginDrain(reason = 'shutting-down'): void {
65
+ if (this.draining) return;
66
+ this.draining = true;
67
+ this.drainReason = reason;
68
+ logger.log(`[BrowserServer] Draining (${reason}) — refusing new requests with 503 (${this.connections.size} client connection(s) attached).`);
69
+ }
70
+
71
+ /** True once {@link beginDrain} has been called. */
72
+ isDraining(): boolean {
73
+ return this.draining;
74
+ }
75
+
76
+ /** Whether any client socket is currently attached. */
77
+ hasActiveClients(): boolean {
78
+ return this.connections.size > 0;
79
+ }
80
+
81
+ /** Number of attached client sockets (observability / teardown decisions). */
82
+ getConnectionCount(): number {
83
+ return this.connections.size;
84
+ }
85
+
38
86
  async start(): Promise<number> {
39
87
  return new Promise((resolve, reject) => {
40
88
  this.server = http.createServer(async (req, res) => {
@@ -49,6 +97,20 @@ export class BrowserServer {
49
97
  return;
50
98
  }
51
99
 
100
+ // Checked after auth so an unauthenticated caller cannot probe
101
+ // leadership state, and before routing so no new work is started
102
+ // against a pool that is being torn down. `Connection: close`
103
+ // stops the client reusing this socket for a retry that would
104
+ // only 503 again.
105
+ if (this.draining) {
106
+ res.writeHead(503, {
107
+ 'Content-Type': 'application/json',
108
+ 'Connection': 'close',
109
+ });
110
+ res.end(JSON.stringify({ error: 'draining', reason: this.drainReason }));
111
+ return;
112
+ }
113
+
52
114
  if (req.method !== 'POST') {
53
115
  res.writeHead(405);
54
116
  res.end('Method Not Allowed');
@@ -138,6 +200,13 @@ export class BrowserServer {
138
200
  });
139
201
  });
140
202
 
203
+ this.server.on('connection', (socket) => {
204
+ this.connections.add(socket);
205
+ socket.on('close', () => {
206
+ this.connections.delete(socket);
207
+ });
208
+ });
209
+
141
210
  this.server.listen(0, '127.0.0.1', () => {
142
211
  const addr = this.server?.address();
143
212
  if (addr && typeof addr === 'object') {
@@ -162,6 +231,7 @@ export class BrowserServer {
162
231
  return new Promise((resolve) => {
163
232
  this.server?.close(() => {
164
233
  this.server = null;
234
+ this.connections.clear();
165
235
  resolve();
166
236
  });
167
237
  });
@@ -108,7 +108,7 @@ export class BrowserTaskScheduler implements IScheduler {
108
108
  if (this.consecutiveLeadershipMisses >= this.LEADERSHIP_MISS_THRESHOLD) {
109
109
  metrics.increment('browser_leadership_lost_total', 1);
110
110
  logger.error(`[Scheduler] Leadership threshold exceeded (${this.consecutiveLeadershipMisses} misses), shutting down pool...`);
111
- await this.shutdown();
111
+ await this.shutdown('leadership-lost');
112
112
  return;
113
113
  }
114
114
  } else {
@@ -145,7 +145,7 @@ export class BrowserTaskScheduler implements IScheduler {
145
145
  // getService throwing during a concurrent container disposal) would otherwise surface
146
146
  // as an unhandledRejection from this fire-and-forget timer. Swallow it — an idle
147
147
  // teardown that fails is non-fatal; the next run gets a fresh scheduler regardless.
148
- this.shutdown().catch(err => logger.warn('[Scheduler] Idle-timeout shutdown failed (non-fatal):', err));
148
+ this.shutdown('idle-timeout').catch(err => logger.warn('[Scheduler] Idle-timeout shutdown failed (non-fatal):', err));
149
149
  }, this.IDLE_TIMEOUT_MS);
150
150
  if (this.idleTimer.unref) this.idleTimer.unref();
151
151
  }
@@ -374,10 +374,26 @@ export class BrowserTaskScheduler implements IScheduler {
374
374
  return result;
375
375
  }
376
376
 
377
- async shutdown() {
377
+ /**
378
+ * @param reason why the pool is going away. Reported verbatim to clients in the
379
+ * drain 503, so a lost election, an idle timeout and a process shutdown are
380
+ * distinguishable in a client-side log rather than all reading as one.
381
+ */
382
+ async shutdown(reason = 'shutting-down') {
378
383
  if (this.isShuttingDown) return;
379
384
  this.isShuttingDown = true;
380
385
 
386
+ // Start draining before any of the teardown below. Clearing state and
387
+ // stopping the pool involves awaits, and until the listener actually
388
+ // closes a connected client would otherwise keep issuing requests against
389
+ // a pool that is already going away — the leadership-loss case this
390
+ // guards. Draining first turns those into an immediate 503 so the client
391
+ // re-elects at once instead of waiting for a reset.
392
+ if (this.server?.hasActiveClients()) {
393
+ logger.log(`[Scheduler] Draining ${this.server.getConnectionCount()} client connection(s) before shutdown (${reason}).`);
394
+ }
395
+ this.server?.beginDrain(reason);
396
+
381
397
  if (this.idleTimer) {
382
398
  clearTimeout(this.idleTimer);
383
399
  this.idleTimer = null;
@@ -139,7 +139,7 @@ export async function forceSchedulerRestart(forceClearRemoteState: boolean = fal
139
139
 
140
140
  if (oldScheduler && 'schedulerId' in oldScheduler && oldScheduler.schedulerId) {
141
141
  if (oldScheduler instanceof BrowserTaskScheduler) {
142
- const shutdownPromise: Promise<void> = oldScheduler.shutdown().catch((err: unknown) => {
142
+ const shutdownPromise: Promise<void> = oldScheduler.shutdown('leadership-lost').catch((err: unknown) => {
143
143
  logger.warn('[Scheduler] Error during old scheduler shutdown after restart:', err);
144
144
  });
145
145
  schedulerService.setPendingShutdownPromise(shutdownPromise);
@@ -1356,7 +1356,10 @@ function isTransientSocketError(error) {
1356
1356
  err.message.includes("Worker pool is shutting down") || // Poolifier throws this when pool.execute() is called while the pool is being
1357
1357
  // destroyed (e.g., during scheduler restart). Treat as transient — the pool will
1358
1358
  // be ready for new tasks once destruction completes and a new pool is initialized.
1359
- err.message.includes("Cannot execute a task on destroying pool") || err.message.includes("destroying pool"));
1359
+ err.message.includes("Cannot execute a task on destroying pool") || err.message.includes("destroying pool") || // The leader answered 503 because it is shutting down after losing
1360
+ // leadership. Retrying re-elects and reaches the new leader, so this is
1361
+ // transient in exactly the same sense as a pool drain.
1362
+ err.message.includes("Browser pool leader is draining"));
1360
1363
  }
1361
1364
  function isCloudflareBlockError(error) {
1362
1365
  if (!error || typeof error !== "object") return false;
@@ -1368,6 +1371,11 @@ function isTaskTimeoutError(error) {
1368
1371
  const err = error;
1369
1372
  return typeof err.message === "string" && (err.message.includes("timed out after") || err.message.includes("Search task timed out") || err.message.includes("Scrape task timed out") || err.message.includes("Health check timed out") || err.message.includes("[BrowserClient] Request to"));
1370
1373
  }
1374
+ function isServerDrainingError(error) {
1375
+ if (!error || typeof error !== "object") return false;
1376
+ const err = error;
1377
+ return typeof err.message === "string" && err.message.includes("Browser pool leader is draining");
1378
+ }
1371
1379
  function isPoolShutdownError(error) {
1372
1380
  if (!error || typeof error !== "object") return false;
1373
1381
  const err = error;
@@ -1389,6 +1397,7 @@ var init_browser_error_utils = __esm({
1389
1397
  name: "BrowserPool",
1390
1398
  isTransientError: (error) => {
1391
1399
  if (isPoolShutdownError(error)) return false;
1400
+ if (isServerDrainingError(error)) return false;
1392
1401
  if (isCloudflareBlockError(error)) return false;
1393
1402
  if (isTaskTimeoutError(error)) return false;
1394
1403
  return isTransientSocketError(error);
@@ -1404,6 +1413,7 @@ __export(scraper_utils_exports, {
1404
1413
  createJsMarkdownConverter: () => createJsMarkdownConverter,
1405
1414
  createNativeMarkdownConverter: () => createNativeMarkdownConverter,
1406
1415
  extractDomain: () => extractDomain,
1416
+ formatErrorWithCause: () => formatErrorWithCause,
1407
1417
  getRandomUserAgent: () => getRandomUserAgent,
1408
1418
  getSsrfSafeDispatcher: () => getSsrfSafeDispatcher,
1409
1419
  isBenignScrapeFailure: () => isBenignScrapeFailure,
@@ -1535,6 +1545,34 @@ async function getSsrfSafeDispatcher() {
1535
1545
  }
1536
1546
  return cachedSsrfDispatcher;
1537
1547
  }
1548
+ function formatErrorWithCause(err) {
1549
+ try {
1550
+ const parts = [];
1551
+ let current = err;
1552
+ for (let depth = 0; current !== null && current !== void 0 && depth < 5; depth++) {
1553
+ let segment;
1554
+ if (current instanceof Error) {
1555
+ segment = current.message || current.name;
1556
+ const code = current.code;
1557
+ if (typeof code === "string" && code && !segment.includes(code)) {
1558
+ segment = `${segment} [${code}]`;
1559
+ }
1560
+ } else if (typeof current === "string") {
1561
+ segment = current;
1562
+ } else {
1563
+ segment = String(current);
1564
+ }
1565
+ if (segment && !parts.includes(segment)) parts.push(segment);
1566
+ const next = current.cause;
1567
+ if (next === current) break;
1568
+ current = next;
1569
+ }
1570
+ if (parts.length === 0) return String(err);
1571
+ return parts.length === 1 ? parts[0] : parts.join(" \u2190 ");
1572
+ } catch {
1573
+ return String(err);
1574
+ }
1575
+ }
1538
1576
  function isPrivateIp(ip) {
1539
1577
  const parts = ip.split(".").map(Number);
1540
1578
  if (parts.length !== 4 || parts.some((p) => isNaN(p))) return false;
@@ -119,9 +119,27 @@ export class ProcessLifecycleService implements IProcessLifecycle {
119
119
  if (expectedStartTime !== undefined && expectedStartTime !== null) {
120
120
  const actualStartTime = await this.getProcessStartTime(pid);
121
121
  if (actualStartTime === null) {
122
- // If we can't get start time but expected it, we can't be sure it's the same process
123
- // On Linux this usually means the process just exited between the signal and the read.
124
- return false;
122
+ // signal(0) above confirmed a live process at this PID, but we couldn't
123
+ // resolve its start time to verify it's the SAME instance (the PID-reuse
124
+ // guard). The two failure modes collapse to the same null here but must
125
+ // NOT be treated identically:
126
+ // • Linux: usually the process exited in the microsecond gap between
127
+ // signal(0) and the /proc/{pid}/stat read → genuinely dead.
128
+ // • Windows/macOS: far more often the start-time lookup subprocess
129
+ // (powershell `Get-Process` / `ps`) simply failed or hit its timeout
130
+ // while the process is still very much alive → NOT dead.
131
+ // Reporting that confirmed-alive PID as DEAD is catastrophic for every
132
+ // caller — a live lock/leader/run-cap owner is reclaimed from under it
133
+ // → two writers collide (lost update) or the run-cap admits extra
134
+ // concurrent runs (the flaky duplicate-slot/cap-breach we saw on Windows
135
+ // CI). So re-confirm liveness instead of guessing: if signal(0) STILL
136
+ // succeeds the process is alive → return true; if it now throws it died
137
+ // between the two probes and the outer catch returns false. A genuine
138
+ // death is therefore still detected within one poll tick (~100ms); the
139
+ // only behavioural change is that a confirmed-alive PID is never again
140
+ // misreported as dead on a transient start-time lookup failure.
141
+ process.kill(pid, 0); // re-confirm; a throw (process died) → outer catch → false
142
+ return true;
125
143
  }
126
144
  // Allow ±1s slack. The macOS/BSD fallback derives start time as
127
145
  // floor(Date.now()/1000) - `ps etimes` (elapsed whole seconds); both terms are