@lincoln504/pi-research 1.7.10 → 1.7.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent-skill/pi-research/SKILL.md +1 -1
- package/dist/cli.mjs +66 -5
- package/dist/thread-worker.mjs +12 -1
- package/docs/CHANGELOG.md +12 -0
- package/package.json +1 -1
- package/src/infrastructure/browser/thread-worker-lifecycle.ts +76 -2
- package/src/infrastructure/browser/thread-worker.mjs +12 -1
- package/src/infrastructure/browser/thread-worker.ts +6 -0
- package/src/infrastructure/browser/worker-pool-manager.ts +53 -3
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
name: pi-research
|
|
3
3
|
description: "Multi-agent web research for any coding agent. Use for anything requiring current information from the internet — news, trends, product/service/company facts, statistics, people, places, events, APIs, library behavior, or 'what is X / how does X work' questions. Also use to search a local knowledge store of previously researched findings before doing live research. Powered by the pi-research engine (search, stealth scraping, security databases, Stack Exchange). Not for reading local files, running project commands, or analyzing this repository's own source."
|
|
4
4
|
license: MIT
|
|
5
|
-
metadata: { "author": "Lincoln Deen", "version": "1.7.
|
|
5
|
+
metadata: { "author": "Lincoln Deen", "version": "1.7.11", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
|
|
6
6
|
allowed-tools: Bash(node:*)
|
|
7
7
|
---
|
|
8
8
|
|
package/dist/cli.mjs
CHANGED
|
@@ -16617,6 +16617,7 @@ init_metrics();
|
|
|
16617
16617
|
init_config();
|
|
16618
16618
|
import { dirname as dirname10, join as join14 } from "node:path";
|
|
16619
16619
|
import { existsSync as existsSync9 } from "node:fs";
|
|
16620
|
+
import { execFileSync } from "node:child_process";
|
|
16620
16621
|
import { fileURLToPath as fileURLToPath3 } from "node:url";
|
|
16621
16622
|
import { FixedClusterPool, WorkerChoiceStrategies } from "poolifier";
|
|
16622
16623
|
|
|
@@ -16922,6 +16923,30 @@ function setupMasterIpcErrorHandler() {
|
|
|
16922
16923
|
const w = cluster.workers?.[id];
|
|
16923
16924
|
if (w) guardClusterWorker(w);
|
|
16924
16925
|
}
|
|
16926
|
+
guardWorkerEmitReemit();
|
|
16927
|
+
}
|
|
16928
|
+
var workerEmitGuardPatched = false;
|
|
16929
|
+
var workerEmitOriginal = null;
|
|
16930
|
+
function guardWorkerEmitReemit() {
|
|
16931
|
+
if (workerEmitGuardPatched || cluster.isWorker) return;
|
|
16932
|
+
const Worker2 = cluster.Worker;
|
|
16933
|
+
if (!Worker2 || !Worker2.prototype) return;
|
|
16934
|
+
workerEmitGuardPatched = true;
|
|
16935
|
+
const origEmit = Worker2.prototype.emit;
|
|
16936
|
+
if (typeof origEmit !== "function") return;
|
|
16937
|
+
workerEmitOriginal = origEmit;
|
|
16938
|
+
Worker2.prototype.emit = function(event, ...args) {
|
|
16939
|
+
if (event === "error" && typeof this.listenerCount === "function" && this.listenerCount("error") === 0) {
|
|
16940
|
+
this.on("error", (err) => {
|
|
16941
|
+
if (isBenignClusterIpcError(err)) {
|
|
16942
|
+
logToDebugFile("DEBUG", `[Master] Last-resort guard swallowed listener-less IPC error: ${err?.code || err?.message}`);
|
|
16943
|
+
return;
|
|
16944
|
+
}
|
|
16945
|
+
logToDebugFile("WARN", `[Master] Last-resort guard handled listener-less worker error: ${err?.stack || err?.message || err}`);
|
|
16946
|
+
});
|
|
16947
|
+
}
|
|
16948
|
+
return origEmit.apply(this, [event, ...args]);
|
|
16949
|
+
};
|
|
16925
16950
|
}
|
|
16926
16951
|
function guardClusterWorker(worker) {
|
|
16927
16952
|
if (!worker || worker.__piIpcGuarded) return;
|
|
@@ -17020,6 +17045,34 @@ var WorkerPoolManager = class {
|
|
|
17020
17045
|
// Handle for the schedulePoolReset() destroy timer, so shutdown() can cancel a pending
|
|
17021
17046
|
// reset instead of letting it fire after teardown and clobber a freshly-rebuilt pool.
|
|
17022
17047
|
_resetTimer = null;
|
|
17048
|
+
// One in-flight destroy promise per pool instance. Both the auto-recovery timer
|
|
17049
|
+
// and shutdown() can try to destroy the same pool (observed 2026-10-08 during a
|
|
17050
|
+
// leader-handover storm: "Cannot destroy an already destroying pool"). A second
|
|
17051
|
+
// destroy() racing the first wipes worker listeners (terminate →
|
|
17052
|
+
// removeAllListeners) while the first is still sending kill messages — the fatal
|
|
17053
|
+
// IPC re-emit window. Sharing one promise per pool makes that race impossible.
|
|
17054
|
+
_destroyInFlight = /* @__PURE__ */ new Map();
|
|
17055
|
+
/**
|
|
17056
|
+
* Serialize pool destruction: the first caller wins; later callers await the
|
|
17057
|
+
* same promise instead of racing a second destroy(). Rejections that merely
|
|
17058
|
+
* report a concurrent/finished destroy are treated as success — the pool is
|
|
17059
|
+
* gone either way, and re-throwing them would turn a benign race into a
|
|
17060
|
+
* logged (and potentially escalated) failure.
|
|
17061
|
+
*/
|
|
17062
|
+
destroyPoolOnce(deadPool) {
|
|
17063
|
+
let p = this._destroyInFlight.get(deadPool);
|
|
17064
|
+
if (!p) {
|
|
17065
|
+
p = deadPool.destroy().catch((e) => {
|
|
17066
|
+
if (e instanceof Error && /already (destroying|destroyed|starting)/i.test(e.message)) {
|
|
17067
|
+
return;
|
|
17068
|
+
}
|
|
17069
|
+
throw new Error(`Worker pool destroy failed: ${e instanceof Error ? e.message : String(e)}`, { cause: e });
|
|
17070
|
+
});
|
|
17071
|
+
void p.then(() => this._destroyInFlight.delete(deadPool));
|
|
17072
|
+
this._destroyInFlight.set(deadPool, p);
|
|
17073
|
+
}
|
|
17074
|
+
return p;
|
|
17075
|
+
}
|
|
17023
17076
|
/**
|
|
17024
17077
|
* Ensure the pool is initialized with the current config.
|
|
17025
17078
|
* Recreates the pool if the worker count has changed.
|
|
@@ -17054,7 +17107,7 @@ var WorkerPoolManager = class {
|
|
|
17054
17107
|
if (this.pool && (this.currentWorkerCount !== maxWorkers || this.currentWorkerConcurrency !== workerConcurrency)) {
|
|
17055
17108
|
logger.log(`[WorkerPoolManager] Pool shape changed (workers ${this.currentWorkerCount} -> ${maxWorkers}, concurrency ${this.currentWorkerConcurrency} -> ${workerConcurrency}), recreating pool...`);
|
|
17056
17109
|
this.poolEpoch++;
|
|
17057
|
-
await this.pool
|
|
17110
|
+
await this.destroyPoolOnce(this.pool);
|
|
17058
17111
|
this.pool = null;
|
|
17059
17112
|
}
|
|
17060
17113
|
this.currentWorkerCount = maxWorkers;
|
|
@@ -17074,6 +17127,14 @@ var WorkerPoolManager = class {
|
|
|
17074
17127
|
`Browser worker bundle is missing at ${workerPath}. It is produced by the build step; reinstall the extension (e.g. \`pi update --extensions\`) or run \`npm install\` in the extension directory so the prepare/build script regenerates it.`
|
|
17075
17128
|
);
|
|
17076
17129
|
}
|
|
17130
|
+
try {
|
|
17131
|
+
execFileSync(process.execPath, ["--check", workerPath], { stdio: "pipe" });
|
|
17132
|
+
} catch (e) {
|
|
17133
|
+
throw new Error(
|
|
17134
|
+
`Browser worker bundle at ${workerPath} does not parse (${e instanceof Error ? e.message : String(e)}). Reinstall the extension (e.g. \`pi update --extensions\`) so the build step regenerates it.`,
|
|
17135
|
+
{ cause: e }
|
|
17136
|
+
);
|
|
17137
|
+
}
|
|
17077
17138
|
const myEpoch = ++this.poolEpoch;
|
|
17078
17139
|
this.pool = new FixedClusterPool(maxWorkers, workerPath, {
|
|
17079
17140
|
env: browserEnv,
|
|
@@ -17235,7 +17296,7 @@ var WorkerPoolManager = class {
|
|
|
17235
17296
|
try {
|
|
17236
17297
|
if (this.isShuttingDown || this.generation !== myGen) return;
|
|
17237
17298
|
this.poolEpoch++;
|
|
17238
|
-
if (deadPool) await
|
|
17299
|
+
if (deadPool) await this.destroyPoolOnce(deadPool);
|
|
17239
17300
|
logger.info("[WorkerPoolManager] Auto-recovery: old pool destroyed.");
|
|
17240
17301
|
} catch (err) {
|
|
17241
17302
|
logger.warn("[WorkerPoolManager] Auto-recovery: error destroying old pool:", err);
|
|
@@ -17276,7 +17337,7 @@ var WorkerPoolManager = class {
|
|
|
17276
17337
|
if (this.pool) {
|
|
17277
17338
|
try {
|
|
17278
17339
|
this.poolEpoch++;
|
|
17279
|
-
const destroyPromise = this.pool
|
|
17340
|
+
const destroyPromise = this.destroyPoolOnce(this.pool);
|
|
17280
17341
|
destroyPromise.catch((err) => logger.debug(`[WorkerPoolManager] Background pool destroy rejection: ${err.message}`));
|
|
17281
17342
|
await raceWithDeadline(destroyPromise, 5e3);
|
|
17282
17343
|
} catch (e) {
|
|
@@ -18510,7 +18571,7 @@ init_logger();
|
|
|
18510
18571
|
|
|
18511
18572
|
// src/healthcheck/index.ts
|
|
18512
18573
|
init_config();
|
|
18513
|
-
import { execFileSync } from "node:child_process";
|
|
18574
|
+
import { execFileSync as execFileSync2 } from "node:child_process";
|
|
18514
18575
|
init_service_registry();
|
|
18515
18576
|
init_service_interfaces();
|
|
18516
18577
|
|
|
@@ -18602,7 +18663,7 @@ async function checkBrowserCapability() {
|
|
|
18602
18663
|
if (isBrowserAvailable() || mockMode) {
|
|
18603
18664
|
if (!mockMode && resolveHeadlessMode() === "virtual") {
|
|
18604
18665
|
try {
|
|
18605
|
-
|
|
18666
|
+
execFileSync2("which", ["Xvfb"], { stdio: "ignore" });
|
|
18606
18667
|
} catch {
|
|
18607
18668
|
return {
|
|
18608
18669
|
healthy: false,
|
package/dist/thread-worker.mjs
CHANGED
|
@@ -1958,12 +1958,22 @@ function setupIpcErrorHandler() {
|
|
|
1958
1958
|
return;
|
|
1959
1959
|
}
|
|
1960
1960
|
cluster.worker.on("error", (err) => {
|
|
1961
|
-
if (err
|
|
1961
|
+
if (isBenignClusterIpcError(err)) {
|
|
1962
1962
|
return;
|
|
1963
1963
|
}
|
|
1964
1964
|
throw err;
|
|
1965
1965
|
});
|
|
1966
1966
|
}
|
|
1967
|
+
var BENIGN_IPC_CODES = /* @__PURE__ */ new Set(["EPIPE", "ERR_IPC_CHANNEL_CLOSED", "ECONNRESET"]);
|
|
1968
|
+
function isBenignClusterIpcError(err) {
|
|
1969
|
+
if (!err) return false;
|
|
1970
|
+
if (typeof err.code === "string" && BENIGN_IPC_CODES.has(err.code)) return true;
|
|
1971
|
+
const msg = String(err.message || err);
|
|
1972
|
+
return /\b(EPIPE|ECONNRESET)\b/.test(msg) || /channel closed/i.test(msg);
|
|
1973
|
+
}
|
|
1974
|
+
function markWorkerBootstrap(id) {
|
|
1975
|
+
logToDebugFile("WARN", `[Worker-${id}] Bootstrap complete (pid ${process2.pid})`);
|
|
1976
|
+
}
|
|
1967
1977
|
function setupUncaughtExceptionHandler() {
|
|
1968
1978
|
process2.on("uncaughtException", (err) => {
|
|
1969
1979
|
if (err && err.message && err.message.includes("reading 'url'") && err.stack && err.stack.includes("coreBundle.js")) {
|
|
@@ -3232,6 +3242,7 @@ setBrowserCleanup(cleanupBrowser2);
|
|
|
3232
3242
|
if (process3.env["GITHUB_ACTIONS"] !== "true") {
|
|
3233
3243
|
setupOrphanProtection();
|
|
3234
3244
|
}
|
|
3245
|
+
markWorkerBootstrap(workerId4);
|
|
3235
3246
|
var FULL_MOCK_MODE = process3.env["PI_RESEARCH_MOCK_SEARCH"] === "true" && process3.env["PI_RESEARCH_MOCK_SCRAPE"] === "true";
|
|
3236
3247
|
async function runTask(data) {
|
|
3237
3248
|
if (!data) {
|
package/docs/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [1.7.11] - 2026-10-10
|
|
9
|
+
|
|
10
|
+
### Fixed
|
|
11
|
+
|
|
12
|
+
- **The browser worker pool crash class observed on 2026-10-08 is closed at four layers.** On a pi host, an `ERR_IPC_CHANNEL_CLOSED` raised by a write to a worker's closed IPC channel reached a `cluster.Worker` with no `error` listener and threw inside `EventEmitter` → `uncaughtException` → the pi process exited mid-research. The full chain was reconstructed from the host's crash log: a storm of worker exits with code 1 (16+ in under a second, **zero worker-side log lines**) fed the 3-consecutive-error threshold, the auto-recovery 1 s timer fired into a leader-handover storm that tore down and rebuilt the pool three times in ten seconds with an explicit `Cannot destroy an already destroying pool` double-destroy, and the fatal `destroy()` → `sendKillMessageToWorker` → `Worker.send` hit the closed channel at 22:00:30.12. Fixes, in `src/infrastructure/browser/thread-worker-lifecycle.ts` and `worker-pool-manager.ts`:
|
|
13
|
+
- **The worker-side IPC handler now swallows the same benign set as the master side** (`EPIPE`, `ECONNRESET`, `ERR_IPC_CHANNEL_CLOSED`, `channel closed`, via the shared `isBenignClusterIpcError`). The old narrow check (`ERR_IPC_CHANNEL_CLOSED` only) re-threw `EPIPE`/`ECONNRESET` during teardown churn as an uncaught worker-side exception — a code-1 exit that fed the consecutive-error counter driving the storm. Non-benign errors still re-throw (the worker's uncaught handler logs and exits, by design).
|
|
14
|
+
- **Pool destruction is serialized.** The auto-recovery timer, `shutdown()`, and the shape-change re-init all destroy through one in-flight promise per pool (`destroyPoolOnce`), so a second `destroy()` can no longer race the first — the double-destroy that wiped worker listeners (`terminate` → `removeAllListeners`) while the first destroy was still sending kill messages, which opened the fatal re-emit window. Rejections that merely report a concurrent/finished destroy are swallowed as success.
|
|
15
|
+
- **A `cluster.Worker` prototype last-resort guard.** `setupMasterIpcErrorHandler` also patches `Worker.prototype.emit` so no Worker can ever emit `error` without a listener, whatever wipe path, timing, or pool generation is involved. The per-worker re-attach patch already survives poolifier's `removeAllListeners`; this makes the crash class unreachable rather than merely guarded. The listener it attaches can only be the first on a listener-less Worker (the exact state where `EventEmitter` throws), so it never shadows poolifier's own error accounting.
|
|
16
|
+
- **A corrupted worker bundle now fails loudly at pool init.** A bundle that exists but is truncated or overwritten fails with a bare `SyntaxError` in every forked worker — code-1 exits with no worker-side log, the same silent-churn signature the missing-bundle guard could not catch. `ensurePool` now runs `node --check` on `thread-worker.mjs` (~40 ms) and throws an actionable reinstall error on a parse failure. (Observed 2026-10-09 on a consumer git checkout that had also lost `src/cleanup/research-cleanup.ts`, breaking the research tool with `Cannot find module` — a partial-install corruption class, now loud for the bundle and diagnosable for the rest.)
|
|
17
|
+
- **Each worker writes a startup marker** (`[Worker-<id>] Bootstrap complete (pid …)`, WARN, always recorded). A non-zero exit with no marker means the worker died during top-level module evaluation, before its handlers were installed — the exact signature of the 2026-10-08 silent churn storm, now visible in post-mortems.
|
|
18
|
+
- Tests: new `test/unit/infrastructure/browser/thread-worker-lifecycle.test.ts` (widened swallow, re-attach patch, prototype last-resort guard plus the no-patch control that throws, bootstrap marker) and destroy-serialization cases in `worker-pool-manager.test.ts` (auto-recovery timer vs concurrent `shutdown()` sharing one `destroy()` call; "already destroying" rejection swallowed; parse-guard failure). Full unit suite 3318, integration 103+57, lint, type-check (TypeScript 6 and 7), depcruise and `npm audit` (0) pass.
|
|
19
|
+
|
|
8
20
|
## [1.7.10] - 2026-10-08
|
|
9
21
|
|
|
10
22
|
### Changed
|
package/package.json
CHANGED
|
@@ -20,14 +20,23 @@ export function setWorkerId(id: string): void {
|
|
|
20
20
|
}
|
|
21
21
|
|
|
22
22
|
/**
|
|
23
|
-
* Handle
|
|
23
|
+
* Handle benign closed-channel IPC errors when poolifier tears down a worker's
|
|
24
|
+
* channel. Symmetric with the master-side classification (isBenignClusterIpcError):
|
|
25
|
+
* the write failed because the pipe is already gone, which is not a health signal.
|
|
26
|
+
*
|
|
27
|
+
* The old narrow check (ERR_IPC_CHANNEL_CLOSED only) re-threw EPIPE/ECONNRESET as an
|
|
28
|
+
* uncaught worker-side exception → exit code 1. A worker that dies that way during
|
|
29
|
+
* teardown churn feeds the master's consecutive-error counter and re-spawns — the
|
|
30
|
+
* self-sustaining worker storm behind the 2026-10-08 crash chain. Swallowing the
|
|
31
|
+
* same benign set here stops that feedback; genuinely non-benign errors still
|
|
32
|
+
* re-throw (the worker's uncaught handler logs and exits, as designed).
|
|
24
33
|
*/
|
|
25
34
|
export function setupIpcErrorHandler(): void {
|
|
26
35
|
if (!cluster.isWorker || !cluster.worker) {
|
|
27
36
|
return;
|
|
28
37
|
}
|
|
29
38
|
cluster.worker.on('error', (err: any) => {
|
|
30
|
-
if (err
|
|
39
|
+
if (isBenignClusterIpcError(err)) {
|
|
31
40
|
return;
|
|
32
41
|
}
|
|
33
42
|
throw err;
|
|
@@ -92,6 +101,10 @@ export function setupMasterIpcErrorHandler(): void {
|
|
|
92
101
|
const w = (cluster.workers as any)?.[id];
|
|
93
102
|
if (w) guardClusterWorker(w);
|
|
94
103
|
}
|
|
104
|
+
|
|
105
|
+
// Prototype-level backstop: no cluster.Worker can ever emit 'error' without
|
|
106
|
+
// a listener, whatever path wiped its per-instance listeners.
|
|
107
|
+
guardWorkerEmitReemit();
|
|
95
108
|
}
|
|
96
109
|
|
|
97
110
|
/** Reset for tests — re-arms the one-time install guard. */
|
|
@@ -99,6 +112,55 @@ export function __resetMasterIpcGuardForTests(): void {
|
|
|
99
112
|
masterIpcGuardInstalled = false;
|
|
100
113
|
}
|
|
101
114
|
|
|
115
|
+
let workerEmitGuardPatched = false;
|
|
116
|
+
let workerEmitOriginal: unknown = null;
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Last-resort defense against the crash class observed on 2026-10-08: an 'error'
|
|
120
|
+
* event emitted on a cluster.Worker with NO listeners throws inside EventEmitter
|
|
121
|
+
* (uncaughtException → the pi host exits). The per-worker guard above is re-armed
|
|
122
|
+
* after poolifier's removeAllListeners() by the re-attach patch in
|
|
123
|
+
* guardClusterWorker, but this patches the Worker PROTOTYPE so that no Worker can
|
|
124
|
+
* ever emit 'error' without a listener — whatever wipe path, re-emit timing, or
|
|
125
|
+
* pool generation is involved.
|
|
126
|
+
*
|
|
127
|
+
* The listener attached here can only be the first listener on a listener-less
|
|
128
|
+
* Worker (that is the exact state where EventEmitter would throw), so it can never
|
|
129
|
+
* shadow poolifier's own error accounting, which is always attached at fork time.
|
|
130
|
+
* Idempotent; master process only. Installed once by setupMasterIpcErrorHandler().
|
|
131
|
+
*/
|
|
132
|
+
export function guardWorkerEmitReemit(): void {
|
|
133
|
+
if (workerEmitGuardPatched || cluster.isWorker) return;
|
|
134
|
+
const Worker = (cluster as unknown as { Worker?: { prototype: { emit?: unknown } } }).Worker;
|
|
135
|
+
if (!Worker || !Worker.prototype) return;
|
|
136
|
+
workerEmitGuardPatched = true;
|
|
137
|
+
const origEmit = Worker.prototype.emit;
|
|
138
|
+
if (typeof origEmit !== 'function') return;
|
|
139
|
+
workerEmitOriginal = origEmit;
|
|
140
|
+
Worker.prototype.emit = function (this: any, event: string | symbol, ...args: unknown[]) {
|
|
141
|
+
if (event === 'error' && typeof this.listenerCount === 'function' && this.listenerCount('error') === 0) {
|
|
142
|
+
this.on('error', (err: any) => {
|
|
143
|
+
if (isBenignClusterIpcError(err)) {
|
|
144
|
+
logToDebugFile('DEBUG', `[Master] Last-resort guard swallowed listener-less IPC error: ${err?.code || err?.message}`);
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
logToDebugFile('WARN', `[Master] Last-resort guard handled listener-less worker error: ${err?.stack || err?.message || err}`);
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
return origEmit.apply(this, [event, ...args]);
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** Reset for tests — restores the original prototype emit and re-arms the flag. */
|
|
155
|
+
export function __resetWorkerEmitGuardForTests(): void {
|
|
156
|
+
if (workerEmitGuardPatched && workerEmitOriginal) {
|
|
157
|
+
const Worker = (cluster as unknown as { Worker?: { prototype: { emit?: unknown } } }).Worker;
|
|
158
|
+
if (Worker && Worker.prototype) Worker.prototype.emit = workerEmitOriginal;
|
|
159
|
+
workerEmitOriginal = null;
|
|
160
|
+
}
|
|
161
|
+
workerEmitGuardPatched = false;
|
|
162
|
+
}
|
|
163
|
+
|
|
102
164
|
function guardClusterWorker(worker: any): void {
|
|
103
165
|
if (!worker || worker.__piIpcGuarded) return;
|
|
104
166
|
worker.__piIpcGuarded = true;
|
|
@@ -137,6 +199,18 @@ function guardClusterWorker(worker: any): void {
|
|
|
137
199
|
}
|
|
138
200
|
}
|
|
139
201
|
|
|
202
|
+
/**
|
|
203
|
+
* Worker bootstrap marker. WARN (always recorded; INFO/DEBUG are gated behind
|
|
204
|
+
* PI_RESEARCH_DEBUG). If a worker later exits non-zero WITHOUT this line in the
|
|
205
|
+
* log, it died during top-level module evaluation — before its error handlers
|
|
206
|
+
* were installed — and left no diagnostic behind. The 2026-10-08 churn storm
|
|
207
|
+
* had exactly that signature (a burst of code-1 exits with zero worker-side log
|
|
208
|
+
* lines); this marker makes that class visible in post-mortems.
|
|
209
|
+
*/
|
|
210
|
+
export function markWorkerBootstrap(id: string): void {
|
|
211
|
+
logToDebugFile('WARN', `[Worker-${id}] Bootstrap complete (pid ${process.pid})`);
|
|
212
|
+
}
|
|
213
|
+
|
|
140
214
|
/**
|
|
141
215
|
* Handle Uncaught Exceptions in Worker
|
|
142
216
|
*/
|
|
@@ -1958,12 +1958,22 @@ function setupIpcErrorHandler() {
|
|
|
1958
1958
|
return;
|
|
1959
1959
|
}
|
|
1960
1960
|
cluster.worker.on("error", (err) => {
|
|
1961
|
-
if (err
|
|
1961
|
+
if (isBenignClusterIpcError(err)) {
|
|
1962
1962
|
return;
|
|
1963
1963
|
}
|
|
1964
1964
|
throw err;
|
|
1965
1965
|
});
|
|
1966
1966
|
}
|
|
1967
|
+
var BENIGN_IPC_CODES = /* @__PURE__ */ new Set(["EPIPE", "ERR_IPC_CHANNEL_CLOSED", "ECONNRESET"]);
|
|
1968
|
+
function isBenignClusterIpcError(err) {
|
|
1969
|
+
if (!err) return false;
|
|
1970
|
+
if (typeof err.code === "string" && BENIGN_IPC_CODES.has(err.code)) return true;
|
|
1971
|
+
const msg = String(err.message || err);
|
|
1972
|
+
return /\b(EPIPE|ECONNRESET)\b/.test(msg) || /channel closed/i.test(msg);
|
|
1973
|
+
}
|
|
1974
|
+
function markWorkerBootstrap(id) {
|
|
1975
|
+
logToDebugFile("WARN", `[Worker-${id}] Bootstrap complete (pid ${process2.pid})`);
|
|
1976
|
+
}
|
|
1967
1977
|
function setupUncaughtExceptionHandler() {
|
|
1968
1978
|
process2.on("uncaughtException", (err) => {
|
|
1969
1979
|
if (err && err.message && err.message.includes("reading 'url'") && err.stack && err.stack.includes("coreBundle.js")) {
|
|
@@ -3232,6 +3242,7 @@ setBrowserCleanup(cleanupBrowser2);
|
|
|
3232
3242
|
if (process3.env["GITHUB_ACTIONS"] !== "true") {
|
|
3233
3243
|
setupOrphanProtection();
|
|
3234
3244
|
}
|
|
3245
|
+
markWorkerBootstrap(workerId4);
|
|
3235
3246
|
var FULL_MOCK_MODE = process3.env["PI_RESEARCH_MOCK_SEARCH"] === "true" && process3.env["PI_RESEARCH_MOCK_SCRAPE"] === "true";
|
|
3236
3247
|
async function runTask(data) {
|
|
3237
3248
|
if (!data) {
|
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
setupOrphanProtection,
|
|
20
20
|
createKillHandler,
|
|
21
21
|
setBrowserCleanup,
|
|
22
|
+
markWorkerBootstrap,
|
|
22
23
|
} from './thread-worker-lifecycle.ts';
|
|
23
24
|
import {
|
|
24
25
|
setWorkerId,
|
|
@@ -60,6 +61,11 @@ if (process.env['GITHUB_ACTIONS'] !== 'true') {
|
|
|
60
61
|
setupOrphanProtection();
|
|
61
62
|
}
|
|
62
63
|
|
|
64
|
+
// Startup marker (always recorded). A non-zero exit with no marker in the log
|
|
65
|
+
// means the worker died during top-level module evaluation, before the handlers
|
|
66
|
+
// above were installed — the silent churn signature of the 2026-10-08 crash chain.
|
|
67
|
+
markWorkerBootstrap(workerId);
|
|
68
|
+
|
|
63
69
|
// When both search and scrape are mocked, skip Firefox entirely in task handlers
|
|
64
70
|
// and in the eager warm-up below. Firefox startup in cluster workers takes 60-90s
|
|
65
71
|
// on constrained CI runners; mocked tasks don't need a real browser at all.
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
import { dirname, join } from 'node:path';
|
|
9
9
|
import { existsSync } from 'node:fs';
|
|
10
|
+
import { execFileSync } from 'node:child_process';
|
|
10
11
|
import { fileURLToPath } from 'node:url';
|
|
11
12
|
import { FixedClusterPool, WorkerChoiceStrategies } from 'poolifier';
|
|
12
13
|
import { logger } from '../../logger.ts';
|
|
@@ -110,6 +111,35 @@ export class WorkerPoolManager implements IService {
|
|
|
110
111
|
// Handle for the schedulePoolReset() destroy timer, so shutdown() can cancel a pending
|
|
111
112
|
// reset instead of letting it fire after teardown and clobber a freshly-rebuilt pool.
|
|
112
113
|
private _resetTimer: ReturnType<typeof setTimeout> | null = null;
|
|
114
|
+
// One in-flight destroy promise per pool instance. Both the auto-recovery timer
|
|
115
|
+
// and shutdown() can try to destroy the same pool (observed 2026-10-08 during a
|
|
116
|
+
// leader-handover storm: "Cannot destroy an already destroying pool"). A second
|
|
117
|
+
// destroy() racing the first wipes worker listeners (terminate →
|
|
118
|
+
// removeAllListeners) while the first is still sending kill messages — the fatal
|
|
119
|
+
// IPC re-emit window. Sharing one promise per pool makes that race impossible.
|
|
120
|
+
private _destroyInFlight: Map<object, Promise<void>> = new Map();
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Serialize pool destruction: the first caller wins; later callers await the
|
|
124
|
+
* same promise instead of racing a second destroy(). Rejections that merely
|
|
125
|
+
* report a concurrent/finished destroy are treated as success — the pool is
|
|
126
|
+
* gone either way, and re-throwing them would turn a benign race into a
|
|
127
|
+
* logged (and potentially escalated) failure.
|
|
128
|
+
*/
|
|
129
|
+
private destroyPoolOnce(deadPool: object): Promise<void> {
|
|
130
|
+
let p = this._destroyInFlight.get(deadPool);
|
|
131
|
+
if (!p) {
|
|
132
|
+
p = (deadPool as { destroy: () => Promise<void> }).destroy().catch((e: unknown) => {
|
|
133
|
+
if (e instanceof Error && /already (destroying|destroyed|starting)/i.test(e.message)) {
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
throw new Error(`Worker pool destroy failed: ${e instanceof Error ? e.message : String(e)}`, { cause: e });
|
|
137
|
+
});
|
|
138
|
+
void p.then(() => this._destroyInFlight.delete(deadPool));
|
|
139
|
+
this._destroyInFlight.set(deadPool, p);
|
|
140
|
+
}
|
|
141
|
+
return p;
|
|
142
|
+
}
|
|
113
143
|
|
|
114
144
|
constructor(
|
|
115
145
|
private readonly onPoolError?: (error: Error, consecutiveErrors: number) => void
|
|
@@ -169,7 +199,7 @@ export class WorkerPoolManager implements IService {
|
|
|
169
199
|
if (this.pool && (this.currentWorkerCount !== maxWorkers || this.currentWorkerConcurrency !== workerConcurrency)) {
|
|
170
200
|
logger.log(`[WorkerPoolManager] Pool shape changed (workers ${this.currentWorkerCount} -> ${maxWorkers}, concurrency ${this.currentWorkerConcurrency} -> ${workerConcurrency}), recreating pool...`);
|
|
171
201
|
this.poolEpoch++; // intentional destroy: silence the dying pool's exit events
|
|
172
|
-
await this.pool
|
|
202
|
+
await this.destroyPoolOnce(this.pool);
|
|
173
203
|
this.pool = null;
|
|
174
204
|
}
|
|
175
205
|
|
|
@@ -237,6 +267,23 @@ export class WorkerPoolManager implements IService {
|
|
|
237
267
|
);
|
|
238
268
|
}
|
|
239
269
|
|
|
270
|
+
// A bundle that EXISTS but is corrupted (truncated or partially overwritten
|
|
271
|
+
// file — observed 2026-10-08: a git checkout missing files) fails with a bare
|
|
272
|
+
// SyntaxError in every forked worker — code-1 exits with no worker-side log,
|
|
273
|
+
// the same silent churn signature as a missing bundle. Parse (not execute)
|
|
274
|
+
// the source here so a corrupted bundle fails loudly at pool init instead of
|
|
275
|
+
// masking as worker churn. `node --check` parses .mjs as ESM (~40 ms).
|
|
276
|
+
try {
|
|
277
|
+
execFileSync(process.execPath, ['--check', workerPath], { stdio: 'pipe' });
|
|
278
|
+
} catch (e) {
|
|
279
|
+
throw new Error(
|
|
280
|
+
`Browser worker bundle at ${workerPath} does not parse ` +
|
|
281
|
+
`(${e instanceof Error ? e.message : String(e)}). ` +
|
|
282
|
+
`Reinstall the extension (e.g. \`pi update --extensions\`) so the build step regenerates it.`,
|
|
283
|
+
{ cause: e },
|
|
284
|
+
);
|
|
285
|
+
}
|
|
286
|
+
|
|
240
287
|
// Identity token for THIS pool instance. The handlers below compare it against
|
|
241
288
|
// the live `this.poolEpoch` so a superseded pool's trailing events are ignored.
|
|
242
289
|
const myEpoch = ++this.poolEpoch;
|
|
@@ -472,7 +519,7 @@ export class WorkerPoolManager implements IService {
|
|
|
472
519
|
// and nulling shared state would clobber a pool rebuilt after the shutdown.
|
|
473
520
|
if (this.isShuttingDown || this.generation !== myGen) return;
|
|
474
521
|
this.poolEpoch++; // intentional destroy: silence the dying pool's exit events
|
|
475
|
-
if (deadPool) await
|
|
522
|
+
if (deadPool) await this.destroyPoolOnce(deadPool);
|
|
476
523
|
logger.info('[WorkerPoolManager] Auto-recovery: old pool destroyed.');
|
|
477
524
|
} catch (err) {
|
|
478
525
|
logger.warn('[WorkerPoolManager] Auto-recovery: error destroying old pool:', err);
|
|
@@ -535,7 +582,10 @@ export class WorkerPoolManager implements IService {
|
|
|
535
582
|
// in their killHandler (context.close / browser.close via Playwright), so
|
|
536
583
|
// allow enough time for those to complete before the IPC channel closes.
|
|
537
584
|
this.poolEpoch++; // intentional destroy: silence the dying pool's exit events
|
|
538
|
-
|
|
585
|
+
// Shared promise: if the auto-recovery timer is already destroying this pool
|
|
586
|
+
// (it fired before shutdown cleared the handle), await that destroy instead of
|
|
587
|
+
// racing a second one — the double-destroy race of the 2026-10-08 crash chain.
|
|
588
|
+
const destroyPromise = this.destroyPoolOnce(this.pool);
|
|
539
589
|
destroyPromise.catch((err: Error) => logger.debug(`[WorkerPoolManager] Background pool destroy rejection: ${err.message}`));
|
|
540
590
|
await raceWithDeadline(destroyPromise, 5000);
|
|
541
591
|
} catch (e) {
|