@lincoln504/pi-research 1.4.1 → 1.4.3

package/dist/thread-worker.mjs

@@ -71,6 +71,8 @@ function redactSecrets(message) {
   }
   if (out.length > MAX_LOG_MESSAGE_LENGTH) {
     out = `${out.slice(0, keep)}\u2026[truncated ${Math.max(rawLength, out.length) - keep} chars]`;
+  } else if (rawLength > scanEnd) {
+    out = `${out}\u2026[truncated ${rawLength - scanEnd} chars]`;
   }
   return out;
 }
@@ -247,7 +249,7 @@ var init_error_tracker = __esm({
        * @param error - The error to track (Error object, string message, or unknown)
        * @param context - Optional context information about where the error occurred
        */
-      trackError(error, context2 = {}) {
+      trackError(error, context = {}) {
         let message;
         if (error instanceof Error) {
           message = error.message;
@@ -276,15 +278,15 @@ var init_error_tracker = __esm({
         }
         pattern.count++;
         pattern.lastSeen = now;
-        if (context2.domain) {
-          const existing = pattern.domainCounts.get(context2.domain);
+        if (context.domain) {
+          const existing = pattern.domainCounts.get(context.domain);
           if (existing !== void 0) {
-            pattern.domainCounts.set(context2.domain, existing + 1);
+            pattern.domainCounts.set(context.domain, existing + 1);
           } else if (pattern.domainCounts.size < this.MAX_DOMAINS_PER_PATTERN) {
-            pattern.domainCounts.set(context2.domain, 1);
+            pattern.domainCounts.set(context.domain, 1);
           }
         }
-        pattern.contexts.push(context2);
+        pattern.contexts.push(context);
         if (pattern.contexts.length > this.MAX_CONTEXTS_PER_PATTERN) {
           pattern.contexts.shift();
         }
@@ -1109,8 +1111,8 @@ var init_logger = __esm({
       error(...args) {
         const errArg = args.find((arg) => arg instanceof Error);
         if (errArg) {
-          const context2 = getLogContext();
-          errorTracker.trackError(errArg, context2);
+          const context = getLogContext();
+          errorTracker.trackError(errArg, context);
         }
         this.emit("ERROR" /* ERROR */, ...args);
       }
@@ -2025,7 +2027,7 @@ var workerId2 = "";
 var pageCreationLock = Promise.resolve();
 var PAGE_CREATION_HARD_RELEASE_MS = 5 * 60 * 1e3;
 var wedgedContext = null;
-async function createPageSafe(context2) {
+async function createPageSafe(context) {
   let release;
   const nextLock = new Promise((resolve) => {
     release = resolve;
@@ -2033,11 +2035,11 @@ async function createPageSafe(context2) {
   const currentLock = pageCreationLock;
   pageCreationLock = currentLock.then(() => nextLock);
   await currentLock;
-  if (context2 === wedgedContext) {
+  if (context === wedgedContext) {
     release();
     throw new Error("Browser context is wedged from a prior page-creation call that never settled; a new browser context is required.");
   }
-  const pagePromise = context2.newPage();
+  const pagePromise = context.newPage();
   let hardReleased = false;
   const releaseOnce = () => {
     if (hardReleased) return;
@@ -2047,17 +2049,17 @@ async function createPageSafe(context2) {
   };
   const hardReleaseTimer = setTimeout(() => {
     logToDebugFile2("WARN", "[ThreadWorker] page-creation mutex force-released \u2014 newPage() did not settle within the hard timeout");
-    wedgedContext = context2;
+    wedgedContext = context;
     releaseOnce();
   }, PAGE_CREATION_HARD_RELEASE_MS);
   safeUnref(hardReleaseTimer);
   pagePromise.then(
     () => {
-      if (wedgedContext === context2) wedgedContext = null;
+      if (wedgedContext === context) wedgedContext = null;
       releaseOnce();
     },
     () => {
-      if (wedgedContext === context2) wedgedContext = null;
+      if (wedgedContext === context) wedgedContext = null;
       releaseOnce();
     }
   );
@@ -2119,12 +2121,12 @@ function logToDebugFile2(level, ...args) {
   } catch {
   }
 }
-async function setupMocking(context2) {
+async function setupMocking(context) {
   const mockSearch = process.env["PI_RESEARCH_MOCK_SEARCH"] === "true";
   const mockScrape = process.env["PI_RESEARCH_MOCK_SCRAPE"] === "true";
   if (!mockSearch && !mockScrape) return;
   logToDebugFile2("INFO", `[Worker-${workerId2}] Setting up browser mocking (Search: ${mockSearch}, Scrape: ${mockScrape})`);
-  await context2.route("**", (route, request) => {
+  await context.route("**", (route, request) => {
     const urlStr = request.url();
     let url;
     try {
@@ -2297,6 +2299,7 @@ async function executeScrapeTask(_context, url, signal) {
   }
   let poisonedError = null;
   const pendingAddrChecks = [];
+  let oversizedPdfError = null;
   try {
     logToDebugFile2("DEBUG", `[Worker-${workerId2}] Starting scrape for: ${url}`);
     const { validateUrlForSSRF: validateForWorker, validateUrlForSSRFSync: validateForWorkerSync } = await Promise.resolve().then(() => (init_scraper_utils(), scraper_utils_exports));
@@ -2328,6 +2331,20 @@ async function executeScrapeTask(_context, url, signal) {
       try {
         if (resp.request().resourceType() === "document" && resp.frame() === page.mainFrame()) {
           finalMainFrameAddr = resp.serverAddr().catch(() => null);
+          const check2 = Promise.resolve(resp.headerValue("content-type")).then(async (ct) => {
+            const looksLikePdf = ct?.includes("application/pdf") || url.toLowerCase().endsWith(".pdf");
+            if (!looksLikePdf || oversizedPdfError || poisonedError) return;
+            const declared = parseInt(await resp.headerValue("content-length").catch(() => null) || "", 10);
+            if (Number.isFinite(declared) && declared > MAX_PDF_SIZE) {
+              const sizeMB = Math.round(declared / 1024 / 1024);
+              oversizedPdfError = new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
+              logToDebugFile2("DEBUG", `[Worker-${workerId2}] Aborting oversized PDF (${sizeMB}MB declared) before download completes: ${url}`);
+              page.close().catch(() => {
+              });
+            }
+          }).catch(() => {
+          });
+          pendingAddrChecks.push(check2);
         }
         const check = Promise.resolve(resp.serverAddr()).then((addr) => {
           const ip = addr?.ipAddress;
@@ -2357,7 +2374,7 @@ async function executeScrapeTask(_context, url, signal) {
     }
     const contentType = await response?.headerValue("content-type") || "";
     const status = response?.status?.() ?? 0;
-    if (contentType.includes("application/pdf")) {
+    if (contentType.includes("application/pdf") || url.toLowerCase().endsWith(".pdf")) {
       if (!response) throw new Error(`[Worker] No response received for PDF URL: ${url}`);
       if (status >= 400) throw new Error(`HTTP ${status}`);
       const declaredLength = parseInt(await response.headerValue("content-length").catch(() => null) || "", 10);
@@ -2365,6 +2382,7 @@ async function executeScrapeTask(_context, url, signal) {
         const sizeMB = Math.round(declaredLength / 1024 / 1024);
         throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
       }
+      if (oversizedPdfError) throw oversizedPdfError;
       const buffer = await response.body();
       if (buffer.byteLength > MAX_PDF_SIZE) {
         const sizeMB = Math.round(buffer.byteLength / 1024 / 1024);
@@ -2422,6 +2440,7 @@ async function executeScrapeTask(_context, url, signal) {
     return { contentType, html, jitter };
   } catch (error) {
     await page.close().catch((err) => logToDebugFile2("DEBUG", `[ThreadWorker] Swallowed page close/wait error: ${err.message || String(err)}`));
+    if (oversizedPdfError) throw oversizedPdfError;
     if (poisonedError) throw poisonedError;
     throw error;
   } finally {
@@ -2566,10 +2585,16 @@ var ConfigSchema = Type.Object({
   SEARCH_TIMEOUT_MS: Type.Number({ minimum: 5e3, maximum: 12e4, default: 45e3 }),
   /** TUI refresh debounce in milliseconds (default: 100ms) */
   TUI_REFRESH_DEBOUNCE_MS: Type.Number({ minimum: 0, maximum: 1e3, default: 100 }),
-  /** Queue-wait / overhead margin (ms) added on top of each browser operation's own nav
-   *  timeout to form its scheduler task-timeout ceiling: a search task times out after
-   *  SEARCH_TIMEOUT_MS + this, a scrape after SCRAPE_TIMEOUT_MS + this. Sized so a task is
-   *  never killed before it has used its full nav budget plus time waiting in the queue.
+  /** Overhead margin (ms) added on top of each browser operation's own nav timeout to
+   *  form its scheduler task-timeout ceiling: a search task times out after
+   *  SEARCH_TIMEOUT_MS + this, a scrape after SCRAPE_TIMEOUT_MS + this — covering
+   *  per-task work beyond raw navigation (page setup, result parsing). Queue wait is
+   *  NOT part of this margin: the deadline is armed on dispatch, not on enqueue, so
+   *  time spent waiting for a free worker never counts against it (see
+   *  browser-task-scheduler.ts). Cold browser-launch/context-creation time is also
+   *  separate from this margin — COLD_START_ALLOWANCE_MS
+   *  (infrastructure/browser/config.ts) covers that unconditionally on top of it,
+   *  since it isn't something a user should need to tune.
    *  (default: 10000) */
   BROWSER_TASK_TIMEOUT_MS: Type.Number({ minimum: 2e3, maximum: 12e4, default: 1e4 }),
   /** Timeout for coordinator/research-lead/repair/knowledge LLM calls in ms (default: 300000 = 5 min, range: 60s-30min).
@@ -2669,10 +2694,12 @@ function resolveHeadlessMode() {
   return true;
 }
 var HEALTHCHECK_MIN_BUDGET_MS = 45e3 + 6e4;
+var BROWSER_LAUNCH_TIMEOUT_MS = 9e4;
+var CONTEXT_CREATION_TIMEOUT_MS = 3e4;
+var COLD_START_ALLOWANCE_MS = BROWSER_LAUNCH_TIMEOUT_MS + CONTEXT_CREATION_TIMEOUT_MS;
 
 // src/infrastructure/browser/thread-worker-browser.ts
 var browser = null;
-var context = null;
 var initPromise = null;
 var workerId3 = "";
 var activeTaskCount = 0;
@@ -2726,11 +2753,11 @@ function logToDebugFile3(level, ...args) {
   }
 }
 async function initBrowser() {
-  if (isBrowserConnected() && context) return;
+  if (isBrowserConnected()) return;
   if (initPromise) return initPromise;
   initPromise = (async () => {
     try {
-      if (!isBrowserConnected() || !context) {
+      if (!isBrowserConnected()) {
         logToDebugFile3("INFO", `[Worker-${workerId3}] Initializing browser instance...`);
         let CamoufoxModule;
         try {
@@ -2739,7 +2766,7 @@ async function initBrowser() {
           throw new Error(`[Worker] camoufox-js not found in node_modules. Please run 'npm install'. Original error: ${e instanceof Error ? e.message : String(e)}`, { cause: e });
         }
         const { Camoufox } = CamoufoxModule;
-        const launchTimeoutMs = 9e4;
+        const launchTimeoutMs = BROWSER_LAUNCH_TIMEOUT_MS;
         const launchOnce = async (headless) => {
           const launchPromise = Camoufox({
             headless,
@@ -2822,30 +2849,6 @@ async function initBrowser() {
           }
         }
         browser = launchedBrowser;
-        const contextPromise = browser.newContext();
-        contextPromise.catch((err) => logToDebugFile3("DEBUG", `[ThreadWorker] Background browser context rejection: ${err.message}`));
-        let contextTimeoutId;
-        let contextRaceLost = false;
-        contextPromise.then((c) => {
-          if (!contextRaceLost) return;
-          logToDebugFile3("WARN", `[Worker-${workerId3}] Browser context creation completed after its deadline; closing the orphaned context.`);
-          try {
-            c?.close?.();
-          } catch {
-          }
-        }).catch(() => {
-        });
-        context = await Promise.race([
-          contextPromise,
-          new Promise((_, reject) => {
-            contextTimeoutId = setTimeout(() => {
-              contextRaceLost = true;
-              reject(new Error("Browser context creation timed out after 30000ms"));
-            }, 3e4);
-          })
-        ]);
-        if (contextTimeoutId) clearTimeout(contextTimeoutId);
-        await setupMocking(context);
         logToDebugFile3("INFO", `[Worker-${workerId3}] Browser initialized.`);
       }
     } catch (e) {
@@ -2853,7 +2856,6 @@ async function initBrowser() {
         Promise.resolve(browser.close()).catch((err) => logToDebugFile3("DEBUG", `[Worker-${workerId3}] Swallowed browser close error during failed init: ${err.message}`));
       }
       browser = null;
-      context = null;
       const msg = e instanceof Error ? e.message : String(e);
       if (msg.includes("Camoufox is not installed") || msg.includes("Version information not found")) {
         throw new Error(`[Worker] Browser binaries not found. Please run 'npx camoufox-js fetch' to install them.`, { cause: e });
@@ -2872,13 +2874,39 @@ async function initBrowser() {
   })();
   return initPromise;
 }
-function getContext() {
-  return context;
+async function acquireTaskContext() {
+  const contextPromise = browser.newContext();
+  contextPromise.catch((err) => logToDebugFile3("DEBUG", `[ThreadWorker] Background browser context rejection: ${err.message}`));
+  let contextTimeoutId;
+  let contextRaceLost = false;
+  contextPromise.then((c) => {
+    if (!contextRaceLost) return;
+    logToDebugFile3("WARN", `[Worker-${workerId3}] Browser context creation completed after its deadline; closing the orphaned context.`);
+    try {
+      c?.close?.();
+    } catch {
+    }
+  }).catch(() => {
+  });
+  let newContext;
+  try {
+    newContext = await Promise.race([
+      contextPromise,
+      new Promise((_, reject) => {
+        contextTimeoutId = setTimeout(() => {
+          contextRaceLost = true;
+          reject(new Error(`Browser context creation timed out after ${CONTEXT_CREATION_TIMEOUT_MS}ms`));
+        }, CONTEXT_CREATION_TIMEOUT_MS);
+      })
+    ]);
+  } finally {
+    if (contextTimeoutId) clearTimeout(contextTimeoutId);
+  }
+  await setupMocking(newContext);
+  return newContext;
 }
 function doResetBrowser() {
-  if (context) Promise.resolve(context.close()).catch((err) => logToDebugFile3("DEBUG", `[Worker-${workerId3}] Swallowed context close error during reset: ${err.message}`));
   if (browser) Promise.resolve(browser.close()).catch((err) => logToDebugFile3("DEBUG", `[Worker-${workerId3}] Swallowed browser close error during reset: ${err.message}`));
-  context = null;
   browser = null;
 }
 function resetBrowser() {
@@ -2892,7 +2920,6 @@ function resetBrowser() {
 async function cleanupBrowser2() {
   const timeoutMs = 2e3;
   const cleanup = async () => {
-    if (context) await Promise.resolve(context.close()).catch((err) => logToDebugFile3("DEBUG", `[Worker-${workerId3}] Swallowed context close error during cleanup: ${err.message}`));
     if (browser) await Promise.resolve(browser.close()).catch((err) => logToDebugFile3("DEBUG", `[Worker-${workerId3}] Swallowed browser close error during cleanup: ${err.message}`));
   };
   try {
@@ -2902,7 +2929,6 @@ async function cleanupBrowser2() {
     ]);
   } catch {
   } finally {
-    context = null;
     browser = null;
   }
 }
@@ -2951,20 +2977,22 @@ async function runTask(data) {
     taskTimer = setTimeout(() => abortController.abort(new Error(`Task timed out after ${taskTimeoutMs}ms`)), taskTimeoutMs);
   }
   taskStarted();
+  let taskContext;
   try {
     await initBrowser();
     const initMs = Date.now() - startTime;
+    taskContext = await acquireTaskContext();
     let result;
     if (type === "search") {
       if (!query) throw new Error("Search task requires a query");
-      const searchResult = await executeSearchTask(getContext(), query, abortController.signal);
+      const searchResult = await executeSearchTask(taskContext, query, abortController.signal);
       result = { results: searchResult.results, duration: Date.now() - startTime, jitter: searchResult.jitter };
     } else if (type === "scrape") {
       if (!url) throw new Error("Scrape task requires a URL");
-      const scrapeResult = await executeScrapeTask(getContext(), url, abortController.signal);
+      const scrapeResult = await executeScrapeTask(taskContext, url, abortController.signal);
       result = { ...scrapeResult, duration: Date.now() - startTime };
     } else if (type === "healthcheck") {
-      const healthResult = await executeHealthCheck(getContext(), initMs, abortController.signal);
+      const healthResult = await executeHealthCheck(taskContext, initMs, abortController.signal);
       result = { ...healthResult, duration: Date.now() - startTime };
     } else {
       result = { error: "Unknown task type", duration: Date.now() - startTime };
@@ -2987,6 +3015,10 @@ async function runTask(data) {
     };
   } finally {
     if (taskTimer !== void 0) clearTimeout(taskTimer);
+    if (taskContext) {
+      Promise.resolve(taskContext.close()).catch(() => {
+      });
+    }
     taskFinished();
   }
 }

package/docs/CHANGELOG.md

@@ -5,6 +5,82 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+## [1.4.3] - 2026-08-19
+
+v1.4.2 was tagged and pushed but never published: the release workflow's
+`publish` job restructuring below (removing `actions/checkout` entirely, to
+close the OIDC-exposure gap) also removed `.nvmrc`, which `actions/setup-node`
+needs on disk — the job failed at the very first step, before ever reaching
+`npm publish`. No package was published under 1.4.2; this is that fix,
+re-tagged as 1.4.3.
+
+### Fixed
+
+- **The `publish` job failed immediately: `actions/setup-node`'s `node-version-file: '.nvmrc'` requires that file on disk, and the job no longer checked out the repository at all.** Restored via a minimal sparse checkout of just `.nvmrc` — checkout itself executes no dependency code, so the actual guarantee the OIDC-exposure fix established (no `npm ci`/`install`, hence no untrusted lifecycle scripts, in the credential-holding job) is unaffected.
+
+## [1.4.2] - 2026-08-19 (tagged, never published — see 1.4.3)
+
+Five further fresh-eyes audit rounds on top of 1.4.1, covering subsystems the
+1.4.1 audit hadn't reached: scripts/config/CLI, knowledge-store/embedding, and
+browser/network first; then orchestration router/synthesizer, locks/scheduler,
+tools, repo hygiene; then knowledge/embedding, CLI/SDK, security/YouTube
+internals, web-scraper/PDF, and CI/release scripts again from a second pass;
+then a re-audit of the first three rounds' own diffs plus CI/release workflow
+robustness and state/lock lifecycle; then a fifth round covering release/CI
+credential handling end to end, web-scraper redirect classification, tool
+budget/validation/steering-concurrency gaps, embedding-server shutdown vs. its
+own GPU serialization queue, and knowledge-store migration manifest
+persistence.
+
+### Fixed
+
+- **Every browser task dispatched to the same worker process shared one Playwright `BrowserContext`**, letting cookies/storage bleed across concurrent, unrelated research sessions (`WORKER_CONCURRENCY` defaults to 2). Never a deliberate isolation trade-off — just an unexamined resource-sharing default, inconsistent with the rest of the codebase's per-session scoping (circuit breakers, scrape caches). Each task now gets its own fresh context via `acquireTaskContext()`, closed in its own `finally`; the browser process itself stays shared.
+- **Knowledge-store corruption auto-recovery wiped the entire shared `knowledge_db` directory** — every project's data, every global entry, and every `_backup_*` safety net — the instant any single workspace hit a corrupted LanceDB manifest. Scoped the recovery to delete only the active table's own directory.
+- **A WASM PDF document leaked native memory whenever both `toMarkdownAll()` and the per-page fallback failed** — the routine case for a malformed/encrypted PDF, in a browser-server process that stays warm across many sessions. `doc.free()` now runs unconditionally in a `finally`.
+- **`PI_RESEARCH_CONFIG_DIR_NAME` (feeds `config.env`/state/`knowledge_db` paths, all under `HOME`) had no path-traversal guard**, unlike `cleanup.cjs`'s own copy of the same resolution — a value like `../../tmp/evil` could silently relocate where API keys get written. Applied the guard both places now share.
+- CPU-fallback re-warmup after a WebGPU failure hardcoded `'mean'` pooling instead of the model's actual `poolingMode` (wrong for bge-m3/Qwen3-Embedding models) — inconsequential for the dummy warmup inference, but inconsistent with every other `warmupPipeline` call site in the embedder. Fixed to use the real pooling mode.
+- Three CodeQL alerts: a predictable `Date.now()`/`Math.random()` temp-dir name in a test (`js/insecure-temporary-file`) replaced with `mkdtemp`; a dead assignment to `lastAnnouncedRound` with no later read; a dead `?? ''` on an already-non-nullable string parameter.
+- **Browser task deadlines never budgeted for a worker's first real Camoufox launch.** Search/scrape/healthcheck timeouts assumed a warm worker, but eager warmup is deliberately disabled, so a normal cold start (up to 120s combined launch + context creation) on a fresh or just-reset worker could be misreported as "worker likely died mid-task", cascading toward unnecessary pool teardown. Added `COLD_START_ALLOWANCE_MS` as a single source of truth, folded into the scheduler's task ceilings, the healthcheck budget, and `browser-search.ts`'s mirrored per-query timeout.
+- **The synthesizer's and router's overflow warnings only checked the reduced corpus against its own budget.** Below `MIN_SYNTHESIS_CORPUS_CHARS` that budget is a fixed floor, not a value derived from the model's context window, so a large global source list or run context could still overflow the real window with no warning. Both now also check the total against the real window directly.
+- **`stackexchange_quota_low_total`/`_exhausted_total` were incremented as a side effect of `getQuotaInfo()`**, a method called up to three times per tool invocation for display — recording one real event as 2-3x the count. Moved the increments to the single gate check that actually observes each event.
+- **The release workflow granted `id-token: write` (the npm OIDC trusted-publishing credential) to the whole workflow**, not just the job that publishes. Every job runs `npm ci` with full lifecycle scripts before its own lint/audit/test steps execute, so a malicious postinstall/prepare script in any dependency could have requested and exchanged that credential from a job that never legitimately needed it. Scoped to only the `publish` job.
+- **A direct browser navigation to an oversized PDF downloaded the whole body before the size cap ever ran.** Playwright's `page.goto()` does not resolve until the full response finishes downloading, so a multi-hundred-MB PDF was pulled entirely into the worker process — risking an OOM kill that strands sibling tasks — before the existing post-goto size check could reject it. `executeScrapeTask` now inspects the main-frame response's Content-Length as soon as headers arrive and closes the page immediately when it already declares an oversized PDF, cutting the download off in flight rather than letting it finish first. A chunked or dishonestly-labelled response still falls through to the existing post-download byte check. The browser scrape layer also now falls back to a `.pdf` URL extension when content-type is missing or mislabelled (e.g. `application/octet-stream`), matching the fetch layer — previously such a response was serialized as HTML, silently losing the document.
+- **The YouTube timedtext fetch validated only the first-hop URL against its trusted-host allowlist**, then followed redirects with default (`follow`) semantics and no revalidation — a 3xx `Location` header could have carried the PoToken-bearing request to an untrusted host. Now follows redirects manually, re-validating each hop against the same allowlist, mirroring `web-scraper.ts`'s existing SSRF redirect pattern.
+- **The YouTube BotGuard attestation-server response was read with a bare `.json()` call** — no `.ok` check and no size cap on an externally-controlled body. Now checks the response status and reads through the codebase's existing `readJsonCapped`.
+- **The embedder's input-truncation guard could split a UTF-16 surrogate pair at the cut boundary**, emitting an invalid lone high surrogate into the tokenizer for text containing an astral character (emoji, rare CJK) near the cap. Backs off one unit, matching the equivalent guard already in `truncateWithMarker`.
+- **`initResearchSDK()` called while a shutdown was still in flight silently no-op'd, believing the SDK was ready.** `isInitialized` only flips to `false` near the end of `_doShutdown()` (after container disposal), so a caller landing in that window hit the "already initialized" branch and returned immediately while the container was still mid-teardown. `initResearchSDK()` now waits out any in-flight shutdown first, mirroring `shutdownResearchSDK()`'s existing symmetric wait on an in-flight init.
+- **A CI job could hang silently for its full timeout window with no diagnosable cause.** `integration-test (ubuntu-latest)` burned its whole 60-minute budget when `npx playwright install-deps firefox` (apt-based) stalled, with no step-level bound to fail it fast. Added `timeout-minutes` to every bare apt-get/brew/choco/npx-network step across `ci.yml` and `release.yml`. A follow-up pass found the same exposure one level up: 6 jobs (including release.yml's OIDC-privileged `publish` job) had no job-level timeout at all, defaulting to GitHub's 6-hour ceiling — bounded all of them.
+- **`validate` (the 3-OS pack/install/execute job) had no `needs:` gate on `lint`/`type-check`**, unlike its sibling jobs `integration-test` and `dep-graph`, which explicitly gate for the documented reason of not burning multi-OS runner minutes on an already-red build. Added the same gate.
+- **The YouTube timedtext/BotGuard fetches (above) used the plain global `fetch`, with no connect-time SSRF pin** — the host allowlist constrains which hostname a redirect can name, but not what that hostname resolves to at the moment the socket opens, and this codebase already treats a resolver answering differently between validation and connect as a real threat elsewhere (the DNS-rebinding backstop in `thread-worker-messaging.ts`, which applies even to already-trusted hosts). Wired in the same `getSsrfSafeFetcher()` dispatcher-paired fetch already used by `web-scraper.ts`.
+- **The release workflow's `publish` job held the npm OIDC trusted-publishing credential (`id-token: write`) for its entire duration, including its own `npm ci`** — which runs full dependency lifecycle scripts *before* any of that same job's lint/audit-gate/type-check/test steps execute. A compromised transitive dependency's postinstall script could have requested and exchanged that credential before a single check had a chance to fail the build — the exact risk `test-install`'s own comment already reasoned about for *other* jobs, just not the one that actually holds the credential. Restructured: `test-install` now also runs the full check suite (lint/audit-gate/type-check/deps:check/test:unit) and uploads the built, verified tarball as an artifact; `publish` (the only job with `id-token: write`) no longer runs `npm ci`, `npm install`, or any other step that executes a dependency's lifecycle scripts — it only downloads that pre-vetted tarball and runs `npm publish` on the file directly, which triggers no lifecycle scripts at all.
+- **`ci.yml`'s `dep-graph` job (`contents: write`) checked out with default `persist-credentials: true`**, writing the write-scoped `GITHUB_TOKEN` into `.git/config` on disk *before* its own `npm ci` (on any cache miss, on every branch push) ran untrusted lifecycle scripts — a compromised dependency's postinstall could `git push` an arbitrary commit straight to the branch with no review, no OIDC exchange needed since the credential was already materialized on disk. Set `persist-credentials: false` on checkout; the token is now only ever pulled into memory in the final commit-and-push step, via an explicit authenticated remote URL.
+- **`ci.yml`'s `validate` job passed a live `GITHUB_TOKEN` to `npm ci` alongside `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1`** — but `setup.cjs` checks that skip flag first and returns before ever reaching the GitHub-releases fetch the token exists to authenticate, so every other dependency's postinstall/prepare script in the tree got a live token for no functional reason. Removed.
+- **The fetch-layer scraper classified a response as PDF-vs-HTML using the pre-redirect URL, not the followed one.** `scrapeWithFetch`'s manual redirect loop tracks the real destination in `currentUrl`, but both the pre-download size-cap check and the actual extraction dispatch checked `url.toLowerCase().endsWith('.pdf')` against the stale original parameter. A legitimate large PDF served from a URL without a `.pdf` suffix (common for document-management/CDN download endpoints) that redirects to its real file could be rejected under the 25MB HTML cap instead of the 100MB PDF cap; the reverse case let a non-PDF landing page bypass the smaller HTML cap. Both checks now use `currentUrl`.
+- **`security_search` spent one of the researcher's shared `MAX_GATHERING_CALLS` slots on a request that did zero work** — the same "malformed call forfeits a real call" bug class 1.4.1 fixed for `youtube_transcript`, left in a later gate within this tool. `options.tracker.recordCall()` ran before the semantic checks that reject unknown database names or an unrecognized severity, both of which return without ever querying a database. Moved the budget-charging call after every semantic-validation reject.
+- **A finishing headless (non-TUI) `research` call could wipe a concurrent sibling headless run's queued/active steering messages.** `teardownUi()`'s own steering-clear guard (`getPiActivePanels(...).length <= 1`) is blind to headless runs — `getPiActivePanels` is populated only by the TUI's `registerSessionPanel`, so for headless sessions it always read 0, making the guard a no-op — and the headless `cleanup()` closure then cleared steering a second time, completely unconditionally. Added `getActiveResearchRunCount()` (backed by the `aborts` map, populated by both the TUI and headless branches) as a mode-agnostic concurrency signal, and removed the unconditional clear.
+- **The top-level `research` tool was the one tool in this codebase that never validated its own parameters against its declared schema** — every sibling tool (`search`, `scrape`, `security_search`, `stackexchange`, `youtube_transcript`, `research_knowledge_search`) explicitly re-checks with `Value.Check` before trusting `params`, since the host does not enforce this. `research-tool-definition.ts` cast `params as ResearchParams` with no runtime check, relying entirely on incidental downstream validation. Added the same gate, using a structural-only variant of the schema for the runtime check so the existing, better-messaged `validateInitialLinks()` bounds errors ("at most 20 URLs...", "at most 2048 characters...") still surface instead of being masked by a generic rejection.
+- **`uninstallSkill([])` (an explicit empty tool-id list) was indistinguishable from `uninstallSkill(undefined)` and silently uninstalled everything**, because the gate was `toolIds && toolIds.length && !toolIds.includes(tool)` — `[].length` is falsy, so the whole condition short-circuited to "keep nothing filtered out." No production call site currently passes `[]`, but it was a live trap for any future caller and inconsistent with this file's other list params, where empty means none. Now `toolIds !== undefined` gates the filter, so `undefined` still means "every entry" and `[]` correctly means "none."
+- **`redactSecrets()` could silently drop the tail of an oversized log message with no truncation marker.** The `REDACT_SCAN_LENGTH` (40,000 char) cut discards everything past that point *before* redaction runs, but the final "…[truncated N chars]" marker was only appended when the *post-redaction* length still exceeded `MAX_LOG_MESSAGE_LENGTH` (10,000). A message whose scanned prefix was dominated by content the redaction patterns collapse heavily (many long hex/JWT runs, each replaced by the much shorter `[REDACTED]`) could end up under that threshold even though real content beyond the scan cut was discarded — reading as complete while silently hiding the tail of a large scraped page or provider error body. Now also appends the marker when the scan cut alone truncated the message, even if redaction's own output stays under the length cap.
+- **`EmbeddingServer.shutdown()` could dispose the embedder pipeline while a legitimately slow (not hung) inference was still running on it.** The `SerialQueue` exists specifically to prevent concurrent native GPU calls (a prior segfault class) and deliberately holds its slot past its own 120s soft timeout for exactly this reason — but `shutdown()` runs on ordinary leadership loss too, not just the poisoned-queue path, and its only guard against an in-flight call was `Embedder.dispose()`'s own 5-second drain, an order of magnitude under what the queue itself considers still-healthy work. A leader serving a large `embedMany` batch that legitimately took 20-30 seconds, racing a leadership-check tick, could have its pipeline disposed out from under the still-running call — the exact concurrent-native-access hazard the queue exists to prevent, just triggered by disposal instead of a second call. Added `SerialQueue.waitForIdle()`, awaited before any `disposeEmbedder` shutdown disposes the pipeline.
+- **A re-embed migration could silently orphan its own freshly-migrated data.** After successfully renaming the temp migration directory to the canonical `knowledge.lance` name, `migrationReEmbed` called `saveManifest()` to persist that new name but discarded its (documented, non-throwing) `false`-on-failure return value — unlike the pre-drop save nineteen lines earlier, which does check it. If that save failed (disk pressure, a killed process), the manifest was left pointing at the now-renamed-away temp name; the next process start would find no such directory, silently create a brand-new EMPTY table under it, and strand every re-embedded document in the unreferenced canonical directory with no recovery path. Now checked: on failure, the rename is reverted (restoring the manifest/disk agreement the earlier successful save already established) rather than thrown as a migration failure — throwing here would have tripped the caller's re-embed-failure fallback to the `backup` strategy, which would have needlessly relocated this perfectly good, fully re-embedded data to a `_backup_*` directory over what is fundamentally a single failed file write.
+
+### Deferred
+
+- A CLI SIGINT-during-init race — already closed in practice by the `initResearchSDK()` fix above, since `shutdownResearchSDK()` already awaits an in-flight init before tearing down.
+- `scripts/cleanup.cjs`'s hardcoded `.pi` config-dir fallback (vs. the runtime's `getConfigDirName()`, which additionally consults the host `pi` package) — latent, since it only diverges if pi itself ever renames its config directory, and requiring the host package from an uninstall-time script that must never fail was judged not worth the risk for a currently-inert gap.
+- A knowledge-store migration-vs-concurrent-write race (`addDocuments()` not covered by the same cross-process lock `open()` uses) — architecturally complex, needs a dedicated pass.
+- A low-confidence, unreachable-at-current-call-sites race in `clearEmbeddingInstance()`, flagged by CodeQL but not backed by a concrete interleaving — reviewed and deliberately left alone.
+
+### Verified
+
+- Every fix carries a regression test proven to fail against the pre-fix code and pass after (stash-and-rerun discipline).
+- Full suites green across all five rounds: unit tests (2726-2747 depending on round), 57 + 103 integration (real browsers, including a live YouTube network fetch), lint, both type-checks, `deps:check`, the production audit gate, the build, and a live end-to-end research run against the built/published CLI.
+- The fourth round's own changes re-verified after the fact: full unit suite (2740), lint, type-check, and `deps:check` all green; both workflow YAML files parse; the transcript-client test suite (which stubs global `fetch`) needed an accompanying `getSsrfSafeFetcher` mock to keep exercising the stub once the fetch path switched to the dispatcher-paired fetch — added, matching the same pattern already used by `web-research`'s own tests.
+- The fifth round's own changes re-verified: full unit suite (2747, up from 2740 — six new regression tests plus a schema-validation test file), lint, both type-checks, `deps:check`, the production build, and the tarball manifest check all green; both workflow YAML files parse. Also fixed in the same round, discovered by the new schema-validation gate itself: the runtime check initially used the full public parameter schema (bounds included), which caught `initialLinks` count/length violations before the existing, better-messaged `validateInitialLinks()` check could — regressing two of its error-message assertions. Split into a structural-only runtime-check schema so those specific messages still win.
+- A test-quality-only fix alongside the production fixes above: `browser-server.test.ts`'s headers-sent-guard test asserted "no unhandledRejection surfaced" after a fixed 25ms wait, which fails OPEN (silently loses coverage, not a flaky red build) on a slow/contended CI runner. Now flushes several real event-loop turns before a longer (100ms) backstop.
+
 ## [1.4.1] - 2026-08-17
 
 v1.4.0 was tagged and pushed but never published: CI failed on every commit from the

package/docs/CONFIGURATION.md

@@ -102,7 +102,7 @@ Timeouts
 | `PI_RESEARCH_LLM_TIMEOUT_MS` | `300000` | 60000–1800000 | Coordinator / research-lead / repair / knowledge LLM call timeout. |
 | `PI_RESEARCH_SCRAPE_TIMEOUT_MS` | `15000` | 5000–120000 | Per-page scrape (page-load) timeout. |
 | `PI_RESEARCH_SEARCH_TIMEOUT_MS` | `45000` | 5000–120000 | Browser search page timeout. |
-| `PI_RESEARCH_BROWSER_TASK_TIMEOUT_MS` | `10000` | 2000–120000 | Overhead margin added to each browser op's own timeout (a search task ceiling is `SEARCH_TIMEOUT_MS` + this; a scrape is `SCRAPE_TIMEOUT_MS` + this). These ceilings bound **execution**: a task's clock starts when a worker picks it up, so time spent queued behind other work is not charged against it and raising this to cover a busy pool is neither needed nor effective. |
+| `PI_RESEARCH_BROWSER_TASK_TIMEOUT_MS` | `10000` | 2000–120000 | Overhead margin added to each browser op's own timeout (a search task ceiling is `SEARCH_TIMEOUT_MS` + this + a fixed ~120s cold-start allowance; a scrape is `SCRAPE_TIMEOUT_MS` + this + the same allowance). The cold-start allowance covers a worker's first real browser launch + context creation, which is not user-tunable. These ceilings bound **execution**: a task's clock starts when a worker picks it up, so time spent queued behind other work is not charged against it and raising this to cover a busy pool is neither needed nor effective. |
 | `PI_RESEARCH_HEALTH_CHECK_TIMEOUT_MS` | `10000` | 2000–120000 | Pre-flight health-check timeout. |
 
 LLM output & reasoning

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lincoln504/pi-research",
-  "version": "1.4.1",
+  "version": "1.4.3",
   "description": "Web research for your agents with smart and safe tooling + knowledge store",
   "type": "module",
   "private": false,

package/skills/pi-research/SKILL.md

@@ -2,7 +2,7 @@
 name: pi-research
 description: "Multi-agent web research for any coding agent. Use for anything requiring current information from the internet — news, trends, product/service/company facts, statistics, people, places, events, APIs, library behavior, or 'what is X / how does X work' questions. Also use to search a local knowledge store of previously researched findings before doing live research. Powered by the pi-research engine (search, stealth scraping, security databases, Stack Exchange). Not for reading local files, running project commands, or analyzing this repository's own source."
 license: MIT
-metadata: { "author": "Lincoln Deen", "version": "1.4.1", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
+metadata: { "author": "Lincoln Deen", "version": "1.4.3", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
 allowed-tools: Bash(node:*)
 ---
 

package/src/config.ts

@@ -82,10 +82,16 @@ export const ConfigSchema = Type.Object({
   SEARCH_TIMEOUT_MS: Type.Number({ minimum: 5000, maximum: 120000, default: 45000 }),
   /** TUI refresh debounce in milliseconds (default: 100ms) */
   TUI_REFRESH_DEBOUNCE_MS: Type.Number({ minimum: 0, maximum: 1000, default: 100 }),
-  /** Queue-wait / overhead margin (ms) added on top of each browser operation's own nav
-   *  timeout to form its scheduler task-timeout ceiling: a search task times out after
-   *  SEARCH_TIMEOUT_MS + this, a scrape after SCRAPE_TIMEOUT_MS + this. Sized so a task is
-   *  never killed before it has used its full nav budget plus time waiting in the queue.
+  /** Overhead margin (ms) added on top of each browser operation's own nav timeout to
+   *  form its scheduler task-timeout ceiling: a search task times out after
+   *  SEARCH_TIMEOUT_MS + this, a scrape after SCRAPE_TIMEOUT_MS + this — covering
+   *  per-task work beyond raw navigation (page setup, result parsing). Queue wait is
+   *  NOT part of this margin: the deadline is armed on dispatch, not on enqueue, so
+   *  time spent waiting for a free worker never counts against it (see
+   *  browser-task-scheduler.ts). Cold browser-launch/context-creation time is also
+   *  separate from this margin — COLD_START_ALLOWANCE_MS
+   *  (infrastructure/browser/config.ts) covers that unconditionally on top of it,
+   *  since it isn't something a user should need to tune.
    *  (default: 10000) */
   BROWSER_TASK_TIMEOUT_MS: Type.Number({ minimum: 2000, maximum: 120000, default: 10000 }),
   /** Timeout for coordinator/research-lead/repair/knowledge LLM calls in ms (default: 300000 = 5 min, range: 60s-30min).

package/src/core/planning-service.ts

@@ -717,14 +717,38 @@ export class PlanningService implements IPlanningService {
       };
 
       const responseText = isRouter
-        ? await this.callLead({
-            ...callOptions,
-            systemPrompt: populate(''),
-            userMessage: buildRouterMessage(...this.buildRouterEvidence(options, round), runContext),
-            maxTokens: ROUTER_MAX_TOKENS,
-            component: 'router',
-            label: 'router-updatePlanForRound',
-          })
+        ? await (() => {
+            const routerMessage = buildRouterMessage(...this.buildRouterEvidence(options, round), runContext);
+            // The router carries no reduce mechanism the way synthesizeCorpus does — a
+            // routing decision made from PARTITIONED evidence isn't naturally mergeable
+            // the way synthesis prose is, and this call's output is a structured JSON
+            // delegate plan, not free text. Its input is bounded by team size in the
+            // common case (digests for earlier rounds are already capped — see
+            // MAX_DIGEST_CHARS), but nothing caps an individual researcher's fresh report
+            // body, so a round with a few unusually long reports can still overflow the
+            // window with no reduce path to fall back on. Warn rather than fail silently;
+            // the provider's own rejection is still the actual backstop, same as the
+            // synthesizer's over-budget-after-every-pass case above.
+            const window = model.contextWindow ?? DEFAULT_MODEL_CONTEXT_WINDOW;
+            const estimatedTokens = routerMessage.length / CHARS_PER_TOKEN + ROUTER_MAX_TOKENS + SYNTHESIS_OVERHEAD_TOKENS;
+            if (estimatedTokens > window) {
+              logger.warn(
+                `[PlanningService] Router request for ${model.id} is an estimated ` +
+                `${Math.round(estimatedTokens)} tokens against a ${window}-token context window ` +
+                `(evidence ${routerMessage.length} chars, output ceiling ${ROUTER_MAX_TOKENS} tokens). ` +
+                `Unlike the synthesizer, the router has no reduce path for an over-budget round — ` +
+                `sending it anyway, the provider may reject this request.`,
+              );
+            }
+            return this.callLead({
+              ...callOptions,
+              systemPrompt: populate(''),
+              userMessage: routerMessage,
+              maxTokens: ROUTER_MAX_TOKENS,
+              component: 'router',
+              label: 'router-updatePlanForRound',
+            });
+          })()
         : await this.synthesizeCorpus({ ...callOptions, reports, runContext, populate });
 
       // Extract and validate JSON
@@ -1115,6 +1139,29 @@ export class PlanningService implements IPlanningService {
       );
     }
 
+    // budgetChars alone can UNDER-report overflow risk once it hits MIN_SYNTHESIS_CORPUS_CHARS:
+    // below that point it is a fixed floor, not a value derived from the window, so a reduced
+    // corpus that fits comfortably under the floor can still combine with a large global source
+    // list and/or run context (neither bounded, both already subtracted from budgetChars but not
+    // from THIS check) to overflow the real window — the same failure class the budget exists to
+    // prevent, just triggered by those two inputs growing instead of the corpus. Check the actual
+    // total request against the actual window directly, independent of any floor.
+    const window = args.model.contextWindow ?? DEFAULT_MODEL_CONTEXT_WINDOW;
+    const estimatedTotalTokens =
+      (finalSize + globalSourceList.length + runContext.length) / CHARS_PER_TOKEN +
+      config.SYNTHESIS_MAX_TOKENS + SYNTHESIS_OVERHEAD_TOKENS;
+    if (estimatedTotalTokens > window) {
+      logger.warn(
+        `[PlanningService] Synthesis request for ${args.model.id} is an estimated ` +
+        `${Math.round(estimatedTotalTokens)} tokens against a ${window}-token context window ` +
+        `(corpus ${finalSize} chars, source list ${globalSourceList.length} chars, run context ` +
+        `${runContext.length} chars, output ceiling ${config.SYNTHESIS_MAX_TOKENS} tokens) — the ` +
+        `source list and/or run context are large enough on their own to overflow the window even ` +
+        `though the reduced corpus fits its budget. Sending it anyway — the provider may reject ` +
+        `this request.`,
+      );
+    }
+
     return this.callLead({
       ...args,
       systemPrompt: populate(reduced ? MERGE_SYNTHESIS_SECTION : ''),