@lincoln504/pi-research 1.3.3 → 1.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cli.mjs +177 -72
- package/dist/thread-worker.mjs +56 -37
- package/docs/CHANGELOG.md +39 -0
- package/package.json +1 -1
- package/skills/pi-research/SKILL.md +1 -1
- package/src/core/service-registry.ts +37 -3
- package/src/healthcheck/index.ts +9 -7
- package/src/infrastructure/browser/task-execution-service.ts +21 -5
- package/src/infrastructure/browser/thread-worker-browser.ts +20 -1
- package/src/infrastructure/browser/thread-worker-messaging.ts +30 -22
- package/src/infrastructure/browser/thread-worker.mjs +56 -37
- package/src/infrastructure/knowledge-store-service.ts +27 -2
- package/src/knowledge/chunker.ts +14 -1
- package/src/orchestration/quick-research-orchestrator.ts +13 -2
- package/src/orchestration/research-orchestration-service.ts +16 -3
- package/src/sdk.ts +11 -2
- package/src/security/github-advisories.ts +13 -4
- package/src/skill-install/skill-installer.ts +40 -5
- package/src/stackexchange/index.ts +15 -0
- package/src/stackexchange/rest-client.ts +13 -24
- package/src/utils/input-validation.ts +0 -10
- package/src/utils/json-utils.ts +0 -78
- package/src/utils/log-utils.ts +32 -33
- package/src/utils/metrics.ts +0 -8
- package/src/utils/shared-links.ts +0 -14
- package/src/utils/stdio-capture.ts +44 -7
- package/src/utils/url-utils.ts +7 -1
- package/src/web-research/scraper-utils.ts +8 -3
- package/src/web-research/web-scraper.ts +33 -1
package/dist/thread-worker.mjs
CHANGED
|
@@ -45,12 +45,7 @@ function formatArg(arg) {
|
|
|
45
45
|
return String(arg);
|
|
46
46
|
}
|
|
47
47
|
function redactSecrets(message) {
|
|
48
|
-
let
|
|
49
|
-
if (message.length > keep) {
|
|
50
|
-
const lastKeptUnit = message.charCodeAt(keep - 1);
|
|
51
|
-
if (lastKeptUnit >= 55296 && lastKeptUnit <= 56319) keep -= 1;
|
|
52
|
-
}
|
|
53
|
-
let out = message.length > MAX_LOG_MESSAGE_LENGTH ? `${message.slice(0, keep)}\u2026[truncated ${message.length - keep} chars]` : message;
|
|
48
|
+
let out = message;
|
|
54
49
|
out = out.replace(ANSI_PATTERN, "");
|
|
55
50
|
out = out.replace(URL_CREDENTIALS_PATTERN, "$1[REDACTED]@");
|
|
56
51
|
out = out.replace(JWT_PATTERN, "[REDACTED]");
|
|
@@ -61,6 +56,14 @@ function redactSecrets(message) {
|
|
|
61
56
|
out = out.replace(KNOWN_TOKEN_PATTERN, "[REDACTED]");
|
|
62
57
|
out = out.replace(PROVIDER_TOKEN_PATTERN, "[REDACTED]");
|
|
63
58
|
out = out.replace(LONG_HEX_SECRET_PATTERN, "[REDACTED]");
|
|
59
|
+
let keep = MAX_LOG_MESSAGE_LENGTH;
|
|
60
|
+
if (out.length > keep) {
|
|
61
|
+
const lastKeptUnit = out.charCodeAt(keep - 1);
|
|
62
|
+
if (lastKeptUnit >= 55296 && lastKeptUnit <= 56319) keep -= 1;
|
|
63
|
+
}
|
|
64
|
+
if (out.length > MAX_LOG_MESSAGE_LENGTH) {
|
|
65
|
+
out = `${out.slice(0, keep)}\u2026[truncated ${out.length - keep} chars]`;
|
|
66
|
+
}
|
|
64
67
|
return out;
|
|
65
68
|
}
|
|
66
69
|
function stripTerminalEscapes(message) {
|
|
@@ -75,7 +78,7 @@ var init_log_utils = __esm({
|
|
|
75
78
|
ANSI_PATTERN = /\x1b\[[0-9;]*[A-Za-z]/g;
|
|
76
79
|
URL_CREDENTIALS_PATTERN = /([a-z][a-z0-9+.-]*:\/\/)[^/\s:@]+:[^/\s:@]+@/gi;
|
|
77
80
|
COOKIE_HEADER_PATTERN = /\b(set[_-]?cookie|cookie)\b(["']?\s*[:=]\s*["']?)([^\r\n]+)/gi;
|
|
78
|
-
SENSITIVE_KV_PATTERN = /\b(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key)\b(["']?\s*[:=]\s*["']?)([^\s"',&)]+)/gi;
|
|
81
|
+
SENSITIVE_KV_PATTERN = /\b(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key|key)\b(["']?\s*[:=]\s*["']?)([^\s"',&)]+)/gi;
|
|
79
82
|
KNOWN_TOKEN_PATTERN = /\b(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})\b/g;
|
|
80
83
|
JWT_PATTERN = /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g;
|
|
81
84
|
BASIC_AUTH_PATTERN = /\bBasic\s+[A-Za-z0-9+/]{16,}={0,2}/gi;
|
|
@@ -588,7 +591,10 @@ async function captureStdio(logFile, hasSufficientDiskSpace, task, sessionId) {
|
|
|
588
591
|
unkeyedCaptureDepth++;
|
|
589
592
|
isAnyLoggerCapturingOutput = true;
|
|
590
593
|
}
|
|
591
|
-
const
|
|
594
|
+
const stderrDecoder = new TextDecoder();
|
|
595
|
+
const stdoutDecoder = new TextDecoder();
|
|
596
|
+
const fd1SyncDecoder = new TextDecoder();
|
|
597
|
+
const fd2SyncDecoder = new TextDecoder();
|
|
592
598
|
const originalStderrWrite = process.stderr.write;
|
|
593
599
|
const originalStdoutWrite = process.stdout.write;
|
|
594
600
|
const originalFsWriteSync = fs4.writeSync;
|
|
@@ -634,7 +640,11 @@ async function captureStdio(logFile, hasSufficientDiskSpace, task, sessionId) {
|
|
|
634
640
|
console.debug = createConsolePatch("debug", logFile, hasSufficientDiskSpace);
|
|
635
641
|
process.stderr.write = (chunk, encodingOrCb, callback) => {
|
|
636
642
|
const cb = typeof encodingOrCb === "function" ? encodingOrCb : callback;
|
|
637
|
-
const message = typeof chunk === "string" ? chunk :
|
|
643
|
+
const message = typeof chunk === "string" ? chunk : stderrDecoder.decode(chunk, { stream: true });
|
|
644
|
+
if (message.length === 0) {
|
|
645
|
+
if (typeof cb === "function") cb();
|
|
646
|
+
return true;
|
|
647
|
+
}
|
|
638
648
|
if (!hasSufficientDiskSpace()) {
|
|
639
649
|
if (typeof cb === "function") cb();
|
|
640
650
|
return true;
|
|
@@ -655,7 +665,7 @@ async function captureStdio(logFile, hasSufficientDiskSpace, task, sessionId) {
|
|
|
655
665
|
return true;
|
|
656
666
|
};
|
|
657
667
|
process.stdout.write = (chunk, encodingOrCb, callback) => {
|
|
658
|
-
const message = typeof chunk === "string" ? chunk :
|
|
668
|
+
const message = typeof chunk === "string" ? chunk : stdoutDecoder.decode(chunk, { stream: true });
|
|
659
669
|
const isNative = isNativeLog(message);
|
|
660
670
|
const complexTui = isComplexTui(message);
|
|
661
671
|
const boxDrawing = isBoxDrawing(message);
|
|
@@ -709,7 +719,11 @@ async function captureStdio(logFile, hasSufficientDiskSpace, task, sessionId) {
|
|
|
709
719
|
if (!descriptor || (descriptor.writable || descriptor.set)) {
|
|
710
720
|
fs4.writeSync = (fd, chunk, ...args) => {
|
|
711
721
|
if (fd === 1 || fd === 2) {
|
|
712
|
-
const
|
|
722
|
+
const fdDecoder = fd === 1 ? fd1SyncDecoder : fd2SyncDecoder;
|
|
723
|
+
const message = typeof chunk === "string" ? chunk : fdDecoder.decode(chunk, { stream: true });
|
|
724
|
+
if (fd === 2 && message.length === 0) {
|
|
725
|
+
return typeof chunk === "string" ? Buffer.from(chunk).length : chunk.length;
|
|
726
|
+
}
|
|
713
727
|
const isNative = isNativeLog(message);
|
|
714
728
|
const boxDrawing = isBoxDrawing(message);
|
|
715
729
|
const complexTui = isComplexTui(message);
|
|
@@ -1489,6 +1503,7 @@ function isBenignScrapeFailure(error) {
|
|
|
1489
1503
|
return msg.startsWith("Fetch blocked:") || // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
|
|
1490
1504
|
msg.startsWith("Fetch returned stub:") || // nav-only / near-empty page
|
|
1491
1505
|
msg.startsWith("Could not extract content from PDF") || // malformed/encrypted/scanned-only PDF
|
|
1506
|
+
msg.startsWith("PDF too large") || // exceeds the deliberate 100MB policy cap
|
|
1492
1507
|
/^HTTP \d{3}\b/.test(msg) || // remote 4xx/5xx status
|
|
1493
1508
|
isCloudflareBlockError(error) || isTaskTimeoutError(error) || isPoolShutdownError(error);
|
|
1494
1509
|
}
|
|
@@ -1914,32 +1929,29 @@ async function createPageSafe(context2) {
|
|
|
1914
1929
|
const currentLock = pageCreationLock;
|
|
1915
1930
|
pageCreationLock = currentLock.then(() => nextLock);
|
|
1916
1931
|
await currentLock;
|
|
1932
|
+
const pagePromise = context2.newPage();
|
|
1933
|
+
pagePromise.then(release, release);
|
|
1934
|
+
pagePromise.catch((err) => logToDebugFile2("DEBUG", `[ThreadWorker] Background page creation rejection: ${err.message}`));
|
|
1935
|
+
let timeoutId;
|
|
1917
1936
|
try {
|
|
1918
|
-
const
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
}
|
|
1937
|
-
}).catch(() => {
|
|
1938
|
-
});
|
|
1939
|
-
throw err;
|
|
1940
|
-
}
|
|
1941
|
-
} finally {
|
|
1942
|
-
release();
|
|
1937
|
+
const result = await Promise.race([
|
|
1938
|
+
pagePromise,
|
|
1939
|
+
new Promise((_, reject) => {
|
|
1940
|
+
timeoutId = setTimeout(() => reject(new Error("Browser page creation timed out after 60000ms")), 6e4);
|
|
1941
|
+
})
|
|
1942
|
+
]);
|
|
1943
|
+
if (timeoutId) clearTimeout(timeoutId);
|
|
1944
|
+
return result;
|
|
1945
|
+
} catch (err) {
|
|
1946
|
+
if (timeoutId) clearTimeout(timeoutId);
|
|
1947
|
+
pagePromise.then((p) => {
|
|
1948
|
+
try {
|
|
1949
|
+
p?.close?.();
|
|
1950
|
+
} catch {
|
|
1951
|
+
}
|
|
1952
|
+
}).catch(() => {
|
|
1953
|
+
});
|
|
1954
|
+
throw err;
|
|
1943
1955
|
}
|
|
1944
1956
|
}
|
|
1945
1957
|
function setWorkerId2(id) {
|
|
@@ -2339,6 +2351,7 @@ init_logger();
|
|
|
2339
2351
|
|
|
2340
2352
|
// src/utils/url-utils.ts
|
|
2341
2353
|
init_logger();
|
|
2354
|
+
init_log_utils();
|
|
2342
2355
|
|
|
2343
2356
|
// src/config.ts
|
|
2344
2357
|
var ConfigSchema = Type.Object({
|
|
@@ -2508,6 +2521,7 @@ var initPromise = null;
|
|
|
2508
2521
|
var workerId3 = "";
|
|
2509
2522
|
var activeTaskCount = 0;
|
|
2510
2523
|
var resetPending = false;
|
|
2524
|
+
var resetPendingForBrowser = null;
|
|
2511
2525
|
function setWorkerId3(id) {
|
|
2512
2526
|
workerId3 = id;
|
|
2513
2527
|
}
|
|
@@ -2518,7 +2532,11 @@ function taskFinished() {
|
|
|
2518
2532
|
activeTaskCount = Math.max(0, activeTaskCount - 1);
|
|
2519
2533
|
if (activeTaskCount === 0 && resetPending) {
|
|
2520
2534
|
resetPending = false;
|
|
2521
|
-
|
|
2535
|
+
const capturedBrowser = resetPendingForBrowser;
|
|
2536
|
+
resetPendingForBrowser = null;
|
|
2537
|
+
if (browser === capturedBrowser) {
|
|
2538
|
+
doResetBrowser();
|
|
2539
|
+
}
|
|
2522
2540
|
}
|
|
2523
2541
|
}
|
|
2524
2542
|
var isBrowserConnected = () => {
|
|
@@ -2655,6 +2673,7 @@ function doResetBrowser() {
|
|
|
2655
2673
|
function resetBrowser() {
|
|
2656
2674
|
if (activeTaskCount > 1) {
|
|
2657
2675
|
resetPending = true;
|
|
2676
|
+
resetPendingForBrowser = browser;
|
|
2658
2677
|
return;
|
|
2659
2678
|
}
|
|
2660
2679
|
doResetBrowser();
|
package/docs/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,45 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [1.3.4] - 2026-08-09
|
|
9
|
+
|
|
10
|
+
Two rounds on the clean 1.3.3 tree: a full-tree re-audit (10 independent
|
|
11
|
+
reviews, each a disjoint `src/` slice) followed by a targeted security/
|
|
12
|
+
cross-platform/dead-code sweep (parallel reviews of secret-redaction
|
|
13
|
+
call sites, path/URL handling, Windows/macOS compatibility, and unused
|
|
14
|
+
exports). Every fix below is paired with a regression test confirmed (via
|
|
15
|
+
a local revert-and-rerun) to fail against the pre-fix code.
|
|
16
|
+
|
|
17
|
+
### Fixed
|
|
18
|
+
|
|
19
|
+
- **`createPageSafe()`'s page-creation mutex released before `newPage()` itself had settled.** The lock released in an outer `finally` as soon as its internal 60s race settled — including when the timeout won, while the real `context.newPage()` call was still pending in the background. That let the next queued caller start its own `newPage()` concurrently with the still-unsettled prior one — exactly the concurrent-`newPage()` Firefox/Juggler deadlock scenario the lock exists to prevent, and it happens precisely when the browser is already stalling under stress. Release is now tied to the underlying `newPage()` promise itself settling, independent of which branch of the timeout race resolves.
|
|
20
|
+
- **A deferred `resetBrowser()` could tear down a later, unrelated, healthy browser.** The deferred-reset flag was a bare boolean with no identity tied to which browser instance it was requested against. A sibling task's own crash could take the *immediate* reset path (activeTaskCount ≤ 1) and relaunch a fresh browser without ever clearing the stale deferred flag; once the task counter later hit zero, the stale flag fired against whatever browser was current — tearing down a healthy instance a subsequent task had just started using. The flag now captures the specific browser instance at defer time and only applies if that instance is still current.
|
|
21
|
+
- **`ServiceContainer.disposeAll()` could double-invoke a service's `dispose()`.** Teardown snapshotted "instances to dispose" without waiting for a `clear()`/`replace()` disposal already in flight for the same service — if that disposal had called `instance.dispose()` but not yet nulled/replaced `registration.instance`, the snapshot included it, and the DAG-teardown loop called `dispose()` a second time on the same instance, concurrently with the original call. `get()` already guarded this window for reads; `disposeAll()` now waits it out the same way before reading instances.
|
|
22
|
+
- **The `WRITER_QUEUE` DI registration cached its first writer-queue object forever.** Once resolved, the registry never re-resolved it — but the writer queue is rebuilt whenever the knowledge store re-initializes (a Knowledge Mode or cwd change via `/research-config`), so every write after such a change silently landed on a disposed, closed writer queue. `storeLinkDescriptions` now resolves the writer queue directly from the already-fresh `KnowledgeStoreService` instead of through the stale cache.
|
|
23
|
+
- **`KnowledgeStoreService.dispose()` lacked the identity guard `initialize()` already has.** Two concurrent `dispose()` calls arriving while an `initialize()` was in flight could both pass the early checks and both build their own teardown closure from the same instance snapshot — double-invoking `dispose()` on the embedder/store/writer-queue, the same class of race fixed in the service registry above.
|
|
24
|
+
- **Chunker UTF-16 surrogate corruption at tight overlap ratios.** At a `targetSize`/`overlap` combination that left less than one code unit of slack (e.g. `targetSize:2, overlap:1`), the surrogate-pair read-side guard's two escape branches (pull back / push forward) could both be simultaneously blocked, leaving a chunk boundary mid-surrogate-pair and opening the next chunk with a lone low surrogate — silently corrupted on UTF-8 encode. The push-forward bound was off by one; landing exactly on `end` (zero overlap for that transition) is a safe position that was wrongly excluded.
|
|
25
|
+
- **`redactSecrets()` truncated before redacting.** A secret straddling the 10,000-char truncation boundary had its patterns matched against the already-cut string — a credential split mid-token was too short for any pattern to recognize, leaking a fragment in the clear right before the "…[truncated N chars]" marker. Redaction now always runs on the full message before truncation.
|
|
26
|
+
- **The same truncate-before-redact bug had already spread to five other call sites** that built their own short log-line summaries by truncating `error.message` before ever passing it through `redactSecrets` — `task-execution-service.ts` (5 retry-path WARN logs), `sdk.ts`'s end-of-run error summary, and `url-utils.ts`'s unparseable-URL debug log. A failing scrape/navigation routinely echoes the failing URL verbatim, which can carry userinfo credentials; each site now redacts the full message before truncating for display.
|
|
27
|
+
- **`redactSecrets()`'s key/value pattern had no bare `key` alternative.** Several provider APIs (e.g. StackExchange's `?key=<apikey>` query parameter) name their credential with no qualifying prefix like `api_` or `private_`, so it fell through every existing pattern uncaught. Added `key` to the sensitive-key-name alternation, after the more specific `*_key` variants so those still match first.
|
|
28
|
+
- **`stdio-capture.ts`'s `TextDecoder` instances were missing `{stream: true}`.** A single native write/writeSync call can split a multi-byte UTF-8 character's bytes across two separate calls; without streaming mode each half decoded independently to U+FFFD instead of reconstructing the original character. Also split the previously-shared decoder into one per logical byte stream (stdout, stderr, FD1, FD2) — a shared decoder's carried-over partial bytes are only valid within one continuous stream, and mixing streams could try to complete one stream's dangling sequence with another's bytes.
|
|
29
|
+
- **Path traversal in `github-advisories.ts`'s repo-scoped search.** An owner or repo name of exactly `.` or `..` passed the character-allowlist regex (dots are permitted, for names like `node.js`) and, once interpolated into the API path, was collapsed by the URL parser before the request was sent — `owner=".."` turned `/repos/../rate_limit/...` into a request for `/rate_limit/...`, escaping the intended `/repos/{owner}/{name}/` scope with this tool's own GitHub credentials. The allowlist now additionally rejects a segment that is entirely dots.
|
|
30
|
+
- **PDF-extraction failures conflated a broken native install with a routine malformed PDF.** `pdf-oxide-wasm` failing to load (missing prebuilt binary, corrupted install) meant PDF extraction was broken for every URL, not just one — but it shared a catch block with genuine per-PDF parse failures, so both logged at `debug` and were classified as the same benign, routine outcome. The two are now caught separately: a native-module load failure surfaces at `error` with its own metric label and is never classified as benign. Also added the missing `PDF too large` case to the benign-failure classifier (it exceeds a deliberate size cap, exactly as routine as a malformed PDF, but had never been added).
|
|
31
|
+
- **Skill-installer copy-version laundering.** An ordinary re-run of `installSkill()` against an already-installed owned *copy* stamped the manifest's recorded version to the current package version without ever refreshing the actual on-disk copy — permanently defeating `reconcileSkillInstalls()`'s later staleness check, which compares the recorded version against current. The unrefreshed branch now preserves whatever version was already recorded instead of claiming it's current.
|
|
32
|
+
- **`installSkill()`'s stale-symlink re-point could silently report success after deleting the skill.** The re-point unlinks the old symlink, then creates the new one; if the second step failed (e.g. a Windows junction cannot cross volumes, and `pi update` can relocate the package to a different drive) the error escaped to an outer catch that fell through to reporting "already-installed" — even though the skill directory no longer existed on disk at all. It now falls back to a copy, mirroring the fresh-install path's existing fallback for the identical failure.
|
|
33
|
+
- **StackExchange REST client ignored an already-aborted signal.** The client chained an external `AbortSignal` via a bare `addEventListener('abort', ...)`, which never fires for a signal that was already aborted before the call started — an already-cancelled request still went out to the network. Replaced the hand-rolled `AbortController` + listener wiring with `createTimeoutSignal` (already used by the `security/*.ts` clients), whose `AbortSignal.any` combination correctly produces an already-aborted signal up front.
|
|
34
|
+
- **StackExchange `format=compact` with `command=get` fell through to a raw JSON dump.** The compact formatter only branched on an array result (search/user/site); `get`'s single-question-with-answers object shape had no matching branch, unlike the table formatter's equivalent case. Wired up the existing (previously unused) `formatQuestionsCompact`/`formatAnswersCompact` compact formatters for this shape.
|
|
35
|
+
- **A misleading, permanently-zero "sessions" healthcheck stat.** The `StateManager` health diagnostic reported `stats.activeSessions` from `state.sessions` — a field nothing in production code ever writes to (the session-tracking write API has no live caller anywhere in the codebase), so the figure was always 0, presented as if it were live data. Removed from the diagnostic; the underlying write-side API is still dead code, flagged for a future cleanup round rather than a risky mid-release refactor of state-manager's DI wiring.
|
|
36
|
+
|
|
37
|
+
### Removed
|
|
38
|
+
|
|
39
|
+
- `neutralizeControlChars()` (superseded by `stripTerminalEscapes()`, which already performs the same control-character neutralization as part of a superset of its job; zero remaining callers).
|
|
40
|
+
- Six other dead exported utilities with zero production callers, independently verified: `didResearcherScrape`/`getResearcherScrapes` (`shared-links.ts`), `validateComplexity` (`input-validation.ts`), `normalizeStringArrayDetailed`/`NormalizeArrayResult` (`json-utils.ts`), `getCurrentRunRegistry` (`metrics.ts` — its own internal consumer already reads the underlying `AsyncLocalStorage` directly).
|
|
41
|
+
|
|
42
|
+
Patch, not minor: every fix is an internal correctness/security/robustness
|
|
43
|
+
change with no change to pi-research's own CLI, tool, or SDK contract —
|
|
44
|
+
matching this project's established precedent for audit-round releases
|
|
45
|
+
(1.3.1, 1.3.2, 1.3.3).
|
|
46
|
+
|
|
8
47
|
## [1.3.3] - 2026-08-07
|
|
9
48
|
|
|
10
49
|
pi 0.84.1 compatibility. Triggered by updating the local `pi` install and
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
name: pi-research
|
|
3
3
|
description: "Multi-agent web research for any coding agent. Use for anything requiring current information from the internet — news, trends, product/service/company facts, statistics, people, places, events, APIs, library behavior, or 'what is X / how does X work' questions. Also use to search a local knowledge store of previously researched findings before doing live research. Powered by the pi-research engine (search, stealth scraping, security databases, Stack Exchange). Not for reading local files, running project commands, or analyzing this repository's own source."
|
|
4
4
|
license: MIT
|
|
5
|
-
metadata: { "author": "Lincoln Deen", "version": "1.3.
|
|
5
|
+
metadata: { "author": "Lincoln Deen", "version": "1.3.4", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
|
|
6
6
|
allowed-tools: Bash(node:*)
|
|
7
7
|
---
|
|
8
8
|
|
|
@@ -18,9 +18,10 @@ import { AsyncLocalStorage } from 'node:async_hooks';
|
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
/**
|
|
21
|
-
* How long teardown waits for an initialization
|
|
22
|
-
*
|
|
23
|
-
*
|
|
21
|
+
* How long teardown waits for an initialization or a clear()/replace()
|
|
22
|
+
* disposal that is still in flight. Long enough to cover a normal service
|
|
23
|
+
* coming up or tearing down, short enough that a service blocked on
|
|
24
|
+
* something slow (a model download) cannot hold shutdown open indefinitely.
|
|
24
25
|
*/
|
|
25
26
|
const DISPOSE_INFLIGHT_INIT_TIMEOUT_MS = 5_000;
|
|
26
27
|
|
|
@@ -521,6 +522,39 @@ export class ServiceContainer {
|
|
|
521
522
|
if (settleTimer) clearTimeout(settleTimer);
|
|
522
523
|
}
|
|
523
524
|
|
|
525
|
+
// A concurrent clear()/replace() can be mid-disposal right now: its
|
|
526
|
+
// async IIFE has already called instance.dispose() (or is about to)
|
|
527
|
+
// but hasn't yet nulled/replaced registration.instance. Snapshotting
|
|
528
|
+
// activeServices without waiting for this would include that service,
|
|
529
|
+
// and the loop below would call instance.dispose() a SECOND time on
|
|
530
|
+
// the very same instance concurrently with clear()/replace()'s own
|
|
531
|
+
// call — get() already guards against reading mid-disposal state (see
|
|
532
|
+
// its own disposalPromise wait above); teardown needs the same guard
|
|
533
|
+
// before it reads/touches registration.instance below. Bounded for the
|
|
534
|
+
// same reason as the init wait: a service stuck disposing must not
|
|
535
|
+
// hold overall teardown open indefinitely — the snapshot below still
|
|
536
|
+
// catches whatever state is left after the timeout.
|
|
537
|
+
const inFlightDisposals = Array.from(this.services.values())
|
|
538
|
+
.map((r) => r.disposalPromise)
|
|
539
|
+
.filter((p): p is Promise<void> => p != null);
|
|
540
|
+
if (inFlightDisposals.length > 0) {
|
|
541
|
+
if (this.defaultOptions.enableLogging) {
|
|
542
|
+
logger.debug(`[ServiceContainer] Waiting for ${inFlightDisposals.length} in-flight disposal(s) before teardown`);
|
|
543
|
+
}
|
|
544
|
+
let disposalSettleTimer: ReturnType<typeof setTimeout> | undefined;
|
|
545
|
+
await Promise.race([
|
|
546
|
+
Promise.allSettled(inFlightDisposals),
|
|
547
|
+
new Promise<void>((resolve) => {
|
|
548
|
+
disposalSettleTimer = setTimeout(() => {
|
|
549
|
+
logger.warn('[ServiceContainer] Timed out waiting for in-flight service disposal; proceeding with teardown');
|
|
550
|
+
resolve();
|
|
551
|
+
}, DISPOSE_INFLIGHT_INIT_TIMEOUT_MS);
|
|
552
|
+
disposalSettleTimer.unref?.();
|
|
553
|
+
}),
|
|
554
|
+
]);
|
|
555
|
+
if (disposalSettleTimer) clearTimeout(disposalSettleTimer);
|
|
556
|
+
}
|
|
557
|
+
|
|
524
558
|
const activeServices = Array.from(this.services.keys()).filter(name =>
|
|
525
559
|
this.services.get(name)?.instance !== null
|
|
526
560
|
);
|
package/src/healthcheck/index.ts
CHANGED
|
@@ -233,17 +233,19 @@ export function registerHealthChecks(registry: IHealthRegistryService, container
|
|
|
233
233
|
registry.register('StateManager', async () => {
|
|
234
234
|
try {
|
|
235
235
|
const stateManager = await getService<IStateManager>(ServiceNames.STATE_MANAGER, { container }, container);
|
|
236
|
-
const stats = await stateManager.getMetrics();
|
|
237
236
|
const gpuOwner = await stateManager.getGpuOwner();
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
237
|
+
|
|
238
|
+
// No `sessions` field here: the session-tracking API (addSession/
|
|
239
|
+
// removeSession/updateHeartbeat) that would ever populate state.sessions
|
|
240
|
+
// has no production caller anywhere in the codebase — stats.activeSessions
|
|
241
|
+
// was always 0, permanently, presenting fake data as a live diagnostic.
|
|
242
|
+
return {
|
|
243
|
+
healthy: true,
|
|
244
|
+
diagnostic: {
|
|
242
245
|
status: 'operational',
|
|
243
|
-
sessions: stats.activeSessions,
|
|
244
246
|
gpuLocked: !!gpuOwner,
|
|
245
247
|
gpuOwner: gpuOwner?.sessionId || 'none'
|
|
246
|
-
}
|
|
248
|
+
}
|
|
247
249
|
};
|
|
248
250
|
} catch (e) {
|
|
249
251
|
return { healthy: false, error: `State manager healthcheck failed: ${e instanceof Error ? e.message : String(e)}` };
|
|
@@ -9,6 +9,7 @@ import type { Config } from '../../config.ts';
|
|
|
9
9
|
import type { SearchResult } from '../../web-research/types.ts';
|
|
10
10
|
import type { BrowserTask } from '../../types/index.ts';
|
|
11
11
|
import { logger } from '../../logger.ts';
|
|
12
|
+
import { redactSecrets } from '../../utils/log-utils.ts';
|
|
12
13
|
import { errorTracker } from '../../utils/error-tracker.ts';
|
|
13
14
|
import { getBrowserCircuitBreaker, browserCircuitBreaker, isTransientSocketError, isPoolShutdownError, isTaskTimeoutError, isCloudflareBlockError } from './browser-error-utils.ts';
|
|
14
15
|
import { getScheduler as _getScheduler, forceSchedulerRestart as _forceSchedulerRestart } from './scheduler-factory.ts';
|
|
@@ -42,6 +43,21 @@ const LEADER_HANDOVER_RETRIES = 3;
|
|
|
42
43
|
/** Backoff between handover attempts — long enough for an election to complete. */
|
|
43
44
|
const HANDOVER_BACKOFF_MS = 1200;
|
|
44
45
|
|
|
46
|
+
/**
|
|
47
|
+
* A short, log-line-sized summary of an error — redacted BEFORE truncating.
|
|
48
|
+
* A raw error.message can echo the failing URL verbatim (Playwright/undici
|
|
49
|
+
* navigation errors routinely do), and a URL can carry userinfo credentials
|
|
50
|
+
* (https://user:pass@host/...). Truncating first — the old `.substring(0,
|
|
51
|
+
* 100)` at each call site below — can cut a credential mid-token, leaving a
|
|
52
|
+
* short unredacted fragment in the log: redactSecrets' patterns need to see
|
|
53
|
+
* the whole token to recognize it. Same class of bug as the truncate-before-
|
|
54
|
+
* redact fix in log-utils.ts itself; redact the full message first here too.
|
|
55
|
+
*/
|
|
56
|
+
export function briefErrorMessage(error: unknown): string {
|
|
57
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
58
|
+
return redactSecrets(msg).substring(0, 100);
|
|
59
|
+
}
|
|
60
|
+
|
|
45
61
|
/**
|
|
46
62
|
* Recover from a leader handover before retrying.
|
|
47
63
|
*
|
|
@@ -225,7 +241,7 @@ export async function runBrowserTask<T>(
|
|
|
225
241
|
taskType: type,
|
|
226
242
|
errorType: 'leader_handover',
|
|
227
243
|
});
|
|
228
|
-
logger.warn(`[BrowserManager] Leader handover during ${type} task (handover attempts left: ${handoverRetries}): ${(error
|
|
244
|
+
logger.warn(`[BrowserManager] Leader handover during ${type} task (handover attempts left: ${handoverRetries}): ${briefErrorMessage(error)}...`);
|
|
229
245
|
await recoverFromLeaderHandover(container);
|
|
230
246
|
await new Promise((resolve) => setTimeout(resolve, HANDOVER_BACKOFF_MS + Math.floor(Math.random() * 400)));
|
|
231
247
|
return runBrowserTask<T>(taskOrUrl, type, config, signal, retries, container, handoverRetries - 1);
|
|
@@ -238,7 +254,7 @@ export async function runBrowserTask<T>(
|
|
|
238
254
|
taskType: type,
|
|
239
255
|
errorType: 'transient_socket_error',
|
|
240
256
|
});
|
|
241
|
-
logger.warn(`[BrowserManager] Transient socket error during ${type} task (retries left: ${retries}): ${(error
|
|
257
|
+
logger.warn(`[BrowserManager] Transient socket error during ${type} task (retries left: ${retries}): ${briefErrorMessage(error)}...`);
|
|
242
258
|
|
|
243
259
|
if (isPoolShutdownError(error)) {
|
|
244
260
|
// Handover budget already spent — fall back to a plain drain wait.
|
|
@@ -280,7 +296,7 @@ export async function runBrowserHealthCheck(config?: Config, retries = 1, signal
|
|
|
280
296
|
operation: 'healthcheck',
|
|
281
297
|
errorType: 'transient_socket_error',
|
|
282
298
|
});
|
|
283
|
-
logger.warn(`[BrowserManager] Transient socket error during healthcheck (retries left: ${retries}): ${(error
|
|
299
|
+
logger.warn(`[BrowserManager] Transient socket error during healthcheck (retries left: ${retries}): ${briefErrorMessage(error)}...`);
|
|
284
300
|
|
|
285
301
|
if (isPoolShutdownError(error)) {
|
|
286
302
|
// Same stale-handle trap as the task paths: waiting alone leaves the
|
|
@@ -329,7 +345,7 @@ export async function runWorkerSearch(query: string, config?: Config, signal?: A
|
|
|
329
345
|
query,
|
|
330
346
|
errorType: 'leader_handover',
|
|
331
347
|
});
|
|
332
|
-
logger.warn(`[BrowserManager] Leader handover during search (handover attempts left: ${handoverRetries}): ${(error
|
|
348
|
+
logger.warn(`[BrowserManager] Leader handover during search (handover attempts left: ${handoverRetries}): ${briefErrorMessage(error)}...`);
|
|
333
349
|
await recoverFromLeaderHandover(container);
|
|
334
350
|
await new Promise((resolve) => setTimeout(resolve, HANDOVER_BACKOFF_MS + Math.floor(Math.random() * 400)));
|
|
335
351
|
return runWorkerSearch(query, config, signal, retries, sessionId, container, handoverRetries - 1);
|
|
@@ -342,7 +358,7 @@ export async function runWorkerSearch(query: string, config?: Config, signal?: A
|
|
|
342
358
|
query,
|
|
343
359
|
errorType: 'transient_socket_error',
|
|
344
360
|
});
|
|
345
|
-
logger.warn(`[BrowserManager] Transient socket error during search (retries left: ${retries}): ${(error
|
|
361
|
+
logger.warn(`[BrowserManager] Transient socket error during search (retries left: ${retries}): ${briefErrorMessage(error)}...`);
|
|
346
362
|
|
|
347
363
|
if (isPoolShutdownError(error)) {
|
|
348
364
|
await waitForBrowserPoolIdle(15000).catch((err) => logger.debug('Wait for browser idle timed out or failed:', err));
|
|
@@ -30,6 +30,12 @@ let activeTaskCount = 0;
|
|
|
30
30
|
// A reset that resetBrowser() deferred because sibling tasks were still
|
|
31
31
|
// active, to be applied once the last one finishes.
|
|
32
32
|
let resetPending = false;
|
|
33
|
+
// The browser instance resetPending was requested against, captured at defer
|
|
34
|
+
// time. An intervening resetBrowser() call from a SIBLING task can take the
|
|
35
|
+
// immediate path (doResetBrowser() + a fresh initBrowser() relaunch) while
|
|
36
|
+
// this flag is still pending — see taskFinished() below for why the flag
|
|
37
|
+
// alone isn't enough to know whether it's still safe to apply.
|
|
38
|
+
let resetPendingForBrowser: any = null;
|
|
33
39
|
|
|
34
40
|
/**
|
|
35
41
|
* Set the worker ID for logging purposes
|
|
@@ -48,7 +54,19 @@ export function taskFinished(): void {
|
|
|
48
54
|
activeTaskCount = Math.max(0, activeTaskCount - 1);
|
|
49
55
|
if (activeTaskCount === 0 && resetPending) {
|
|
50
56
|
resetPending = false;
|
|
51
|
-
|
|
57
|
+
const capturedBrowser = resetPendingForBrowser;
|
|
58
|
+
resetPendingForBrowser = null;
|
|
59
|
+
// A sibling task's OWN crash can call resetBrowser() while this flag is
|
|
60
|
+
// still pending; if that call sees activeTaskCount <= 1 it takes the
|
|
61
|
+
// IMMEDIATE path (doResetBrowser() below), which tears the browser down
|
|
62
|
+
// and lets the next task's initBrowser() relaunch a brand-new, unrelated
|
|
63
|
+
// instance — all without ever touching this flag. Applying a stale
|
|
64
|
+
// deferred reset unconditionally would then tear down that later,
|
|
65
|
+
// healthy instance instead of being the no-op it should be. Only apply
|
|
66
|
+
// it if the browser is still the exact instance it was requested against.
|
|
67
|
+
if (browser === capturedBrowser) {
|
|
68
|
+
doResetBrowser();
|
|
69
|
+
}
|
|
52
70
|
}
|
|
53
71
|
}
|
|
54
72
|
|
|
@@ -301,6 +319,7 @@ function doResetBrowser(): void {
|
|
|
301
319
|
export function resetBrowser(): void {
|
|
302
320
|
if (activeTaskCount > 1) {
|
|
303
321
|
resetPending = true;
|
|
322
|
+
resetPendingForBrowser = browser;
|
|
304
323
|
return;
|
|
305
324
|
}
|
|
306
325
|
doResetBrowser();
|
|
@@ -16,35 +16,43 @@ import { MAX_HTML_SIZE, MAX_PDF_SIZE } from '../../web-research/scraper-types.ts
|
|
|
16
16
|
*/
|
|
17
17
|
let pageCreationLock = Promise.resolve();
|
|
18
18
|
|
|
19
|
-
|
|
19
|
+
/** Exported for unit testing of the page-creation mutex's timeout behavior. */
|
|
20
|
+
export async function createPageSafe(context: any): Promise<any> {
|
|
20
21
|
let release!: () => void;
|
|
21
22
|
const nextLock = new Promise<void>(resolve => { release = resolve; });
|
|
22
23
|
const currentLock = pageCreationLock;
|
|
23
24
|
pageCreationLock = currentLock.then(() => nextLock);
|
|
24
25
|
|
|
25
26
|
await currentLock;
|
|
27
|
+
const pagePromise = context.newPage();
|
|
28
|
+
// Release the mutex only once newPage() itself has actually settled — NOT when
|
|
29
|
+
// the timeout race below settles. Releasing in an outer `finally` (the old
|
|
30
|
+
// shape) fired as soon as the 60s timeout won the race, while newPage() was
|
|
31
|
+
// still pending in the background — so the NEXT queued caller started its OWN
|
|
32
|
+
// newPage() concurrently with this still-unsettled one, exactly the concurrent-
|
|
33
|
+
// newPage() scenario this lock exists to prevent (see the comment above), and
|
|
34
|
+
// it happens precisely when the browser is already under the stress that
|
|
35
|
+
// causes 60s+ page-creation stalls in the first place. Attached before the
|
|
36
|
+
// race starts so it's independent of which branch below returns/throws.
|
|
37
|
+
pagePromise.then(release, release);
|
|
38
|
+
pagePromise.catch((err: Error) => logToDebugFile('DEBUG', `[ThreadWorker] Background page creation rejection: ${err.message}`));
|
|
39
|
+
|
|
40
|
+
let timeoutId: NodeJS.Timeout | undefined;
|
|
26
41
|
try {
|
|
27
|
-
const
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
} catch (
|
|
40
|
-
|
|
41
|
-
// The timeout (or another error) won the race. If newPage() later resolves,
|
|
42
|
-
// close that orphaned page so it doesn't accumulate in the long-lived context.
|
|
43
|
-
pagePromise.then((p: any) => { try { p?.close?.(); } catch { /* already gone */ } }).catch(() => {});
|
|
44
|
-
throw err;
|
|
45
|
-
}
|
|
46
|
-
} finally {
|
|
47
|
-
release();
|
|
42
|
+
const result = await Promise.race([
|
|
43
|
+
pagePromise,
|
|
44
|
+
new Promise<never>((_, reject) => {
|
|
45
|
+
timeoutId = setTimeout(() => reject(new Error('Browser page creation timed out after 60000ms')), 60000);
|
|
46
|
+
})
|
|
47
|
+
]);
|
|
48
|
+
if (timeoutId) clearTimeout(timeoutId);
|
|
49
|
+
return result;
|
|
50
|
+
} catch (err) {
|
|
51
|
+
if (timeoutId) clearTimeout(timeoutId);
|
|
52
|
+
// The timeout (or another error) won the race. If newPage() later resolves,
|
|
53
|
+
// close that orphaned page so it doesn't accumulate in the long-lived context.
|
|
54
|
+
pagePromise.then((p: any) => { try { p?.close?.(); } catch { /* already gone */ } }).catch(() => {});
|
|
55
|
+
throw err;
|
|
48
56
|
}
|
|
49
57
|
}
|
|
50
58
|
|