@lincoln504/pi-research 1.4.3 → 1.5.1

package/docs/SDK.md

@@ -1,6 +1,6 @@
 ## SDK
 
-The programmatic SDK for scripts, CI, and custom tooling. For configuration (the
+A high-level research SDK for scripts, CI, and custom tooling. For configuration (the
 layering model, the TUI settings, and every environment variable) see
 [CONFIGURATION.md](CONFIGURATION.md).
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lincoln504/pi-research",
-  "version": "1.4.3",
+  "version": "1.5.1",
   "description": "Web research for your agents with smart and safe tooling + knowledge store",
   "type": "module",
   "private": false,
@@ -158,7 +158,8 @@
     "adm-zip": "^0.6.0",
     "jsdom": {
       "undici": "^7.29.0"
-    }
+    },
+    "sharp": "^0.35.0"
   },
   "devDependencies": {
     "@eslint/js": "^10.0.1",

package/skills/pi-research/SKILL.md

@@ -2,7 +2,7 @@
 name: pi-research
 description: "Multi-agent web research for any coding agent. Use for anything requiring current information from the internet — news, trends, product/service/company facts, statistics, people, places, events, APIs, library behavior, or 'what is X / how does X work' questions. Also use to search a local knowledge store of previously researched findings before doing live research. Powered by the pi-research engine (search, stealth scraping, security databases, Stack Exchange). Not for reading local files, running project commands, or analyzing this repository's own source."
 license: MIT
-metadata: { "author": "Lincoln Deen", "version": "1.4.3", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
+metadata: { "author": "Lincoln Deen", "version": "1.5.1", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
 allowed-tools: Bash(node:*)
 ---
 

package/src/cli.ts

@@ -150,7 +150,7 @@ function pretty(obj: unknown): string {
  * lifecycle event to stderr. Keeps a long research run visible without spamming
  * stdout (the report goes to stdout, untouched).
  */
-function makeProgressObserver(): HeadlessObserverOptions {
+export function makeProgressObserver(): HeadlessObserverOptions {
   return {
     enableLogging: false,
     onProgress: (event: string, data?: any) => {
@@ -196,9 +196,24 @@ function makeProgressObserver(): HeadlessObserverOptions {
           }
           break;
         }
-        case 'complete':
+        case 'complete': {
           toStderr(`  • complete (${data?.result?.length ?? 0} chars)\n`);
+          // Degradation signal for callers that read stderr (the agent skill
+          // does): a run that completes but is weakly grounded or retrieved no
+          // sources at all looks identical to a healthy run on exit code and
+          // the "complete" line alone. Two real runs on 2026-08-20 shipped
+          // degraded reports (1 citation / 0 sources) with nothing on stderr to
+          // distinguish them. The markers are the synthesis service's own
+          // notices, appended by ensureCitedLinks.
+          const reportText = typeof data?.result === 'string' ? data.result : '';
+          if (reportText.includes('No web sources were successfully retrieved') ||
+              reportText.includes('None of the links above could be verified')) {
+            toStderr('  • WARNING: no web sources could be retrieved — the report is unverified\n');
+          } else if (reportText.includes('Grounding notice: only')) {
+            toStderr('  • WARNING: weakly grounded report (fewer than 3 cited sources) — treat unattributed claims as unverified\n');
+          }
           break;
+        }
         case 'error':
           toStderr(`  • error: ${data?.message}\n`);
           break;
@@ -738,7 +753,9 @@ export async function cmdKnowledge(queries: string[], json?: boolean): Promise<n
       config: getConfig(process.cwd(), 'cli'),
       ignoreGlobalConfig: true,
     });
-    toStderr(`[pi-research] searching knowledge store (${queries.length} query/${queries.length === 1 ? '' : 's'})…\n`);
+    // "N queries", not "N query/s" — same rationale as the search_start line above:
+    // the slash form reads as a rate and renders "1 query/" in the singular.
+    toStderr(`[pi-research] searching knowledge store (${queries.length} ${queries.length === 1 ? 'query' : 'queries'})…\n`);
     result = await searchKnowledge(queries, abortController.signal);
   } catch (err) {
     return reportError(err, 'knowledge search', json);
@@ -1660,10 +1677,34 @@ async function main(argv: string[]): Promise<number> {
   // left the SDK's handler to tear the container down underneath a live orchestrator
   // — exactly the race the CLI installs its own handlers to avoid.
   process.on('SIGHUP', () => onSignal('SIGHUP'));
+  // SIGQUIT: documented (see exitCodeForSignal's comment and the exit-code
+  // tables in docs/SDK.md and SKILL.md) as handled the same way as SIGHUP,
+  // but no listener was ever registered for it — an unhandled SIGQUIT hits
+  // Node's default disposition (immediate termination, no abort, no drain),
+  // silently breaking that documented guarantee. Has no Windows equivalent;
+  // registering the listener there is a documented no-op, so no guard needed.
+  process.on('SIGQUIT', () => onSignal('SIGQUIT'));
   if (process.platform === 'win32') {
     process.on('SIGBREAK', () => onSignal('SIGBREAK'));
   }
 
+  // Test-only seam: idle here, right after every handler above is registered,
+  // instead of parsing argv/running a command. Lets a subprocess test deliver
+  // a real OS signal and observe THIS handler run (process.exit with the
+  // derived code) rather than Node's default disposition (process killed by
+  // the signal, no code) — the exact distinction that would have caught the
+  // missing SIGQUIT registration this seam was added for. Never set in
+  // production; see the SIGQUIT subprocess test in test/unit/cli.test.ts.
+  if (process.env['PI_RESEARCH_TEST_HANG_FOR_SIGNAL'] === '1') {
+    process.stdout.write('PI_RESEARCH_TEST_READY\n');
+    // A bare unresolved Promise does not keep the event loop alive by
+    // itself (only real libuv handles do — an installed signal listener
+    // isn't one either), so without a ref'd timer the process would exit 0
+    // on its own the instant this synchronous section finishes, before any
+    // signal could ever arrive.
+    await new Promise<never>(() => setInterval(() => {}, 1000));
+  }
+
   let parsed: ParsedArgs;
   try {
     parsed = parseArgs(argv);

package/src/config.ts

@@ -245,6 +245,11 @@ const USER_MIGRATION_KEYS = [
   'PI_RESEARCH_PLANNING_MAX_TOKENS',
   'PI_RESEARCH_SYNTHESIS_MAX_TOKENS',
   'PI_RESEARCH_MIGRATION_STRATEGY',
+  'PI_RESEARCH_MAX_FAILED_RESEARCHERS',
+  'PI_RESEARCH_YOUTUBE_TRANSCRIPT_MAX_VIDEOS',
+  'PI_RESEARCH_YOUTUBE_TRANSCRIPT_TIMEOUT_MS',
+  'PI_RESEARCH_YOUTUBE_TRANSCRIPT_LANG',
+  'PI_RESEARCH_YOUTUBE_QUERY_EVERY_N',
   'PI_RESEARCH_CONSOLE_LOG',
   'PI_RESEARCH_MODEL',
   'PI_RESEARCH_KNOWLEDGE_DIR',
@@ -751,16 +756,23 @@ function loadEnvFiles(cwd: string, iface?: ConfigInterface): Record<string, stri
     try {
       logger.warn(`[config] WARNING: .pi-research.env files are deprecated. Settings will be auto-migrated to the centralized registry. Remove ${legacyPath} after migration.`);
       legacyEnv = parseDotEnv(fs.readFileSync(legacyPath, 'utf-8'));
-      Object.assign(merged, legacyEnv);
-      
-      // Auto-migrate legacy settings to the central registry — but only LOCAL_SCOPE_KEYS. The
-      // registry is project-scoped storage by design (saveConfig writes only these keys to it);
-      // persisting user-scoped legacy keys would freeze them per-workspace and silently override
-      // the user's global config.env forever. Non-local legacy keys still apply to THIS resolution
-      // via the Object.assign(merged, legacyEnv) above — they just aren't persisted.
+      // Only LOCAL_SCOPE_KEYS from this run's resolution too — same boundary
+      // step 4 below already enforces for the registry this file migrates
+      // into ("the registry is project-scoped storage... never user-scoped
+      // keys"). This file lives in the CWD, which for this tool is routinely
+      // an arbitrary, potentially untrusted repository a coding agent was
+      // just pointed at — applying every key unconditionally let a dropped-in
+      // .pi-research.env silently control user-scoped settings (RESEARCH_MODEL,
+      // EMBEDDING_DEVICE, DISABLED_TOOLS, export destinations, ...) for that
+      // run with no confirmation, for any setting the user hadn't already set
+      // via a real environment variable (which still wins over all of this).
+      // Non-local keys are ignored here exactly as they already are for the
+      // registry — this predecessor mechanism should not be MORE permissive
+      // than the successor it's migrating into.
       const legacyLocal = Object.fromEntries(
         Object.entries(legacyEnv).filter(([k]) => LOCAL_SCOPE_KEYS.has(k))
       );
+      Object.assign(merged, legacyLocal);
       const existingLocal = Object.fromEntries(
         Object.entries(findRegistryEntry(registry, normalizedCwd) ?? {}).filter(([k]) => LOCAL_SCOPE_KEYS.has(k))
       );
@@ -799,8 +811,14 @@ function loadEnvFiles(cwd: string, iface?: ConfigInterface): Record<string, stri
       }
     }
   } else if (Object.keys(merged).length === 0 && !fs.existsSync(legacyPath) && !fs.existsSync(globalPath)) {
-    // 5. Warning for missing config
-    logger.warn(`[config] No configuration found for workspace: ${cwd}. Using code defaults. Run /research-config to configure.`);
+    // 5. Warning for missing config — once per workspace per process. getConfig
+    // is called on many hot paths (every tool invocation, health checks), and
+    // repeating this advisory each time made it the single most common line in
+    // the shared log (362 entries in one observed day), burying real warnings.
+    if (!warnedMissingConfigWorkspaces.has(normalizedCwd)) {
+      warnedMissingConfigWorkspaces.add(normalizedCwd);
+      logger.warn(`[config] No configuration found for workspace: ${cwd}. Using code defaults. Run /research-config to configure.`);
+    }
   }
 
   warnLegacyEnvVarsOnce(merged);
@@ -825,6 +843,10 @@ const LEGACY_ENV_VARS: Record<string, string> = {
 
 let warnedLegacyEnvVars = false;
 
+// Workspaces already warned about having no configuration (see loadEnvFiles) —
+// per-process dedup so the advisory appears once, not on every getConfig call.
+const warnedMissingConfigWorkspaces = new Set<string>();
+
 /** Warn once per process about set-but-dead 0.1.x env vars (upgrade aid; never throws). */
 function warnLegacyEnvVarsOnce(merged: Record<string, string>): void {
   if (warnedLegacyEnvVars) return;

package/src/constants.ts

@@ -46,11 +46,26 @@ export function getMaxGatheringCalls(config?: Config): number {
 /**
  * Get the maximum scrape batches from config.
  * This function should be used instead of MAX_SCRAPE_CALLS to support dynamic configuration.
+ *
+ * @param cachingActive - Pass `true` when prompt caching is active for this
+ * run's resolved model (see isPromptCachingActiveForModel in
+ * core/llm/research-model-resolver.ts). With caching active, the EFFECTIVE
+ * maximum is the configured value plus one: a cached run's system-prompt/
+ * tool-definition prefix is cheap to re-send, so the same token budget
+ * covers one more scrape batch per researcher. Every consumer of the batch
+ * count (tracker enforcement, prompt text, progress units) must go through
+ * this function with the same cachingActive value so they all agree on the
+ * effective number. The "unlimited" sentinel (0 or >99) is unchanged —
+ * adding one to unlimited is meaningless.
  */
-export function getMaxScrapeBatches(config?: Config): number {
+export function getMaxScrapeBatches(config?: Config, cachingActive = false): number {
   try {
     const batches = (config || getConfig()).MAX_SCRAPE_BATCHES;
-    return batches === 0 || batches > 99 ? 999999 : batches;
+    if (batches === 0 || batches > 99) return 999999;
+    // Cap at 99: a configured 99 (the documented max) plus one would land in the
+    // >99 "unlimited" band that every renderer checks — the prompt would say
+    // "unlimited batches" while enforcement actually blocked at 100.
+    return cachingActive ? Math.min(batches + 1, 99) : batches;
   } catch {
     return 2; // Fallback to default
   }
@@ -63,9 +78,13 @@ export const MAX_SCRAPE_URLS = 6;
  * Get the units per researcher for the progress bar.
  * Units = 1 (for start/search) + number of scrape batches.
  * When batches are unlimited (MAX_SCRAPE_BATCHES=0), caps at 4 for a sane progress estimate.
+ *
+ * Relative to the EFFECTIVE batch count: pass the run's cachingActive (from
+ * isPromptCachingActiveForModel over the resolved model) so progress units
+ * agree with the limit the tracker actually enforces.
  */
-export function getUnitsPerResearcher(config?: Config): number {
-  const batches = getMaxScrapeBatches(config);
+export function getUnitsPerResearcher(config?: Config, cachingActive = false): number {
+  const batches = getMaxScrapeBatches(config, cachingActive);
   return 1 + Math.min(batches, 4);
 }
 

package/src/core/interfaces/state-manager-interfaces.ts

@@ -8,6 +8,14 @@ import type { IService } from '../service-registry.ts';
  * State manager service interface
  */
 export interface IStateManager extends IService {
+  /**
+   * True once a newer-build state file forced this process into read-only mode
+   * (writes suppressed) — state-file-based coordination (GPU lock, embedding
+   * leader election) is then void and consumers must degrade to a
+   * coordination-free mode. Optional so structural test doubles need not
+   * implement it; callers use `isReadOnly?.()` (absent ⇒ not read-only).
+   */
+  isReadOnly?(): boolean;
   readState(): Promise<any>;
   writeState(state: any): Promise<void>;
   updateState(updater: (state: any) => any | Promise<any>): Promise<void>;

package/src/core/llm/research-model-resolver.ts

@@ -89,3 +89,41 @@ export function resolveResearchModel(options: {
 
   throw new Error(buildNoModelAvailableMessage());
 }
+
+/**
+ * Best-effort, CONSERVATIVE signal for "will pi-ai apply prompt caching for
+ * this resolved model." pi-ai owns all cache-mechanism logic internally (this
+ * codebase never constructs cache_control blocks itself) and exposes no
+ * public API to ask in advance for most providers, so this can only answer
+ * for the two client-visible cases:
+ *
+ * - `api === 'anthropic-messages'`: caching is native and on by default (no
+ *   capability gate at all — see pi-ai's anthropic-messages provider).
+ * - `api === 'openai-completions'` with `compat.cacheControlFormat ===
+ *   'anthropic'`: an explicit opt-in to Anthropic-style cache_control
+ *   emulation (e.g. an OpenRouter/gateway route to an Anthropic model
+ *   configured with this override).
+ *
+ * Every other case returns false even though caching may actually be
+ * active: pi's generated model catalog sets `cacheControlFormat` on zero
+ * models, so plain `openai-completions` traffic (this project's custom
+ * providers — glm-coding, cerebras, zai, OpenRouter, ...) has its cache
+ * compat auto-detected by pi-ai's own private, unexported heuristic INSIDE
+ * the live request — not knowable client-side, before or after model
+ * resolution. Providers with fully implicit/automatic caching
+ * (openai-responses family, google-generative-ai, google-vertex,
+ * mistral-conversations, bedrock-converse-stream) have no "will it be
+ * active" signal at all — it depends on provider-side runtime behavior
+ * (minimum prompt length, server cache state). Treat this as "caching is
+ * DEFINITELY active" — a false negative is expected and fine for providers
+ * outside these two cases, but a false positive would be a real bug.
+ */
+export function isPromptCachingActiveForModel(model: Model<any> | undefined): boolean {
+  if (!model) return false;
+  if (model.api === 'anthropic-messages') return true;
+  if (model.api === 'openai-completions') {
+    const compat = model.compat as { cacheControlFormat?: string } | undefined;
+    return compat?.cacheControlFormat === 'anthropic';
+  }
+  return false;
+}

package/src/core/planning-service.ts

@@ -220,8 +220,19 @@ function isDegradableLlmError(error: unknown, signal?: AbortSignal): boolean {
 export function salvageReportText(raw: string): string {
   const trimmed = raw.trim();
   if (trimmed.length <= 50) return '';
+  // Envelope sniff. Models routinely wrap the JSON envelope in a markdown code
+  // fence (```json … ```); a fenced envelope starts with backticks, not '{', so a
+  // bare startsWith('{') check shipped one verbatim as a user-facing report
+  // (observed in production 2026-08-20: a ```json-fenced {"action":"synthesize",…}
+  // blob, internal protocol fields included, published as the final report).
+  // Strip one leading fence line and a trailing fence before the shape check.
+  let sniff = trimmed;
+  const fence = sniff.match(/^```[a-zA-Z0-9_-]*[ \t]*\r?\n/);
+  if (fence) {
+    sniff = sniff.slice(fence[0].length).replace(/\r?\n```\s*$/, '').trim();
+  }
   // Looks like a (broken) JSON envelope rather than a report — suppress it.
-  if (trimmed.startsWith('{') && /"(?:action|content|researchers)"\s*:/.test(trimmed)) {
+  if (sniff.startsWith('{') && /"(?:action|content|researchers)"\s*:/.test(sniff)) {
     return '';
   }
   return raw;
@@ -637,7 +648,10 @@ export class PlanningService implements IPlanningService {
 
     // Both lead prompts are deliberately ROUND-INVARIANT: they interpolate only values
     // that are fixed for the whole run (complexity, team size, query budget, the
-    // disabled-tool list). Everything that changes between rounds — root query, round
+    // disabled-tool list). One deliberate exception: the total-rounds figure inside
+    // complexity_guidance moves if steering extends maxRounds mid-run — the evaluator
+    // must see the true budget, so that steer costs one cache-prefix invalidation.
+    // Everything that changes between rounds — root query, round
     // number, agenda, executed queries, steering, round-phase guidance — is appended to
     // the END of the user message instead (see runContext below).
     //
@@ -871,7 +885,15 @@ export class PlanningService implements IPlanningService {
           return capped;
       }
 
-      this.currentPlans.set(sessionId, finalPlan);
+      // A 'wait' plan is NOT stored: it carries no agenda (schema-legal with
+      // researchers: []), and storing it clobbered the last real delegate plan
+      // that the empty-after-cap fallback above and the retry's previousPlan /
+      // initialAgendaSection all read — so a rare 'wait' followed by one
+      // unparseable/transient retry degraded to an early empty synthesize
+      // instead of continuing the prior agenda.
+      if (finalPlan.action !== 'wait') {
+        this.currentPlans.set(sessionId, finalPlan);
+      }
       return finalPlan;
     } catch (err) {
       // A genuine cancellation must propagate so the orchestrator can abort cleanly.
@@ -1200,7 +1222,20 @@ function unwrapPartialSynthesis(text: string, service: { parseJsonPlan(t: string
       return content;
     }
   } catch {
-    // Not an envelope — it is prose that happens to mention "content". Use it as written.
+    // parseJsonPlan rejects a `{"content": "..."}` envelope with no "action"
+    // (its non-synthesize guard demands a researchers array) — but that is
+    // still an envelope, not prose, and returning it raw put the braces in the
+    // final report: the exact failure this unwrap exists to prevent. Try the
+    // action-less shape directly before giving up.
+    try {
+      const bare = JSON.parse(trimmed) as { content?: unknown };
+      if (bare && typeof bare === 'object' && typeof bare.content === 'string' && bare.content.trim()) {
+        logger.debug('[PlanningService] Partial synthesis returned an action-less JSON envelope; unwrapped its content');
+        return bare.content.trim();
+      }
+    } catch {
+      // Genuinely not JSON — prose that happens to mention "content". Use as written.
+    }
   }
   return trimmed;
 }

package/src/index.ts

@@ -556,7 +556,9 @@ export default async function (pi: ExtensionAPI) {
 
     // Researcher sub-agent sessions already carry the authoritative steering copy:
     // the executor (deep) and quick orchestrator inject "ADDITIONAL USER GUIDANCE"/
-    // "ADDITIONAL CONSIDERATIONS" into the researcher system prompt themselves.
+    // "ADDITIONAL CONSIDERATIONS" into the researcher's initial user message themselves
+    // (kept OUT of the system prompt so its cacheable prefix stays byte-identical
+    // across researchers — see researcher-executor.ts).
     // Appending the same messages again here delivered every steering message
     // TWICE to every researcher — return the prompt untouched. (The coordinator and
     // the research lead never pass through this hook at all: they run via

package/src/infrastructure/browser/thread-worker-browser.ts

@@ -243,6 +243,16 @@ export async function initBrowser(): Promise<void> {
             firefox_user_prefs: {
               'media.volume_scale': '0.0',
               'media.autoplay.default': 5,
+              // SSRF: WebSocket connections bypass BOTH layers of the browser
+              // scrape defense — page.route('**') intercepts HTTP(S) only, and
+              // the serverAddr() rebinding backstop hangs off page.on('response'),
+              // which a WS handshake never fires. A hostile scraped page could
+              // therefore `new WebSocket('ws://10.x.x.x/...')` to probe or reach
+              // internal hosts with no validation, and write any reply into the
+              // DOM that page.content() returns. The scraper extracts static
+              // content (domcontentloaded), so nothing it needs arrives over WS —
+              // disable the channel outright at the engine level.
+              'network.websocket.max-connections': 0,
             },
           });
           // Guard with a hard timeout. We catch the launchPromise rejection so it

package/src/infrastructure/browser/thread-worker-messaging.ts

@@ -241,8 +241,14 @@ async function extractSearchResults(page: any): Promise<any[]> {
       let url = link.href;
       try {
         const u = new URL(url);
+        // searchParams.get() already percent-decodes the redirect target once.
+        // A second decodeURIComponent here double-decoded targets whose
+        // canonical form itself contains escapes (%26/%3F in query values,
+        // %2F path slashes, %27 in titles) — structurally corrupting them so
+        // the later scrape fetched the wrong resource — and threw URIError on
+        // a lone `%`, silently dropping the result via the hostname filter.
         const uddg = u.searchParams.get('uddg');
-        if (uddg) url = decodeURIComponent(uddg || '');
+        if (uddg) url = uddg;
       } catch {
         // ignore
       }
@@ -542,7 +548,11 @@ export async function executeScrapeTask(
               // Matches the PDF branch below AND the fetch layer's own detection
               // (web-scraper.ts): a server can mislabel or omit content-type, so a
               // `.pdf` URL extension is an equally-trusted signal, not just the header.
-              const looksLikePdf = ct?.includes('application/pdf') || url.toLowerCase().endsWith('.pdf');
+              // resp.url() — THIS response's URL, not the original task URL: goto()
+              // follows redirects, and a download endpoint without `.pdf` that
+              // redirects to the real `.pdf` file must still arm the early abort
+              // (same redirect-aware fix the fetch layer got via currentUrl).
+              const looksLikePdf = ct?.includes('application/pdf') || String(resp.url()).toLowerCase().endsWith('.pdf');
               if (!looksLikePdf || oversizedPdfError || poisonedError) return;
               const declared = parseInt((await resp.headerValue('content-length').catch(() => null)) || '', 10);
               if (Number.isFinite(declared) && declared > MAX_PDF_SIZE) {
@@ -612,7 +622,14 @@ export async function executeScrapeTask(
     // extracting the document — a silent content loss for exactly the URLs this
     // feature advertises as supported, and specifically for the malformed/oddly-
     // served PDFs most likely to reach this browser fallback in the first place.
-    if (contentType.includes('application/pdf') || url.toLowerCase().endsWith('.pdf')) {
+    // response.url(), NOT the original task `url`: goto() follows redirects, so
+    // classification must use the URL that actually answered — a download
+    // endpoint without `.pdf` redirecting to the real `.pdf` file must take the
+    // PDF branch, and a `.pdf` task URL redirecting to an HTML page must NOT
+    // (same two-sided redirect-aware fix the fetch layer got via currentUrl,
+    // and the early-abort check above already applies via resp.url()).
+    const finalUrl = String(response?.url?.() ?? url);
+    if (contentType.includes('application/pdf') || finalUrl.toLowerCase().endsWith('.pdf')) {
       if (!response) throw new Error(`[Worker] No response received for PDF URL: ${url}`);
       // An errored "PDF" is an error page: fail BEFORE buffering its body.
       if (status >= 400) throw new Error(`HTTP ${status}`);

package/src/infrastructure/browser/thread-worker.mjs

@@ -1571,6 +1571,8 @@ function isBenignScrapeFailure(error) {
   msg.startsWith("Fetch returned stub:") || // nav-only / near-empty page
   msg.startsWith("Could not extract content from PDF") || // malformed/encrypted/scanned-only PDF
   msg.startsWith("PDF too large") || // exceeds the deliberate 100MB policy cap
+  msg.startsWith("HTML response too large") || // fetch-layer 25MB policy cap — same class as the PDF cap
+  msg.startsWith("Browser HTML too large") || // browser-layer 25MB policy cap
   /^HTTP \d{3}\b/.test(msg) || // remote 4xx/5xx status
   isCloudflareBlockError(error) || isTaskTimeoutError(error) || isPoolShutdownError(error);
 }
@@ -2190,7 +2192,7 @@ async function extractSearchResults(page) {
       try {
         const u = new URL(url);
         const uddg = u.searchParams.get("uddg");
-        if (uddg) url = decodeURIComponent(uddg || "");
+        if (uddg) url = uddg;
       } catch {
       }
       let urlHostname = "";
@@ -2332,7 +2334,7 @@ async function executeScrapeTask(_context, url, signal) {
         if (resp.request().resourceType() === "document" && resp.frame() === page.mainFrame()) {
           finalMainFrameAddr = resp.serverAddr().catch(() => null);
           const check2 = Promise.resolve(resp.headerValue("content-type")).then(async (ct) => {
-            const looksLikePdf = ct?.includes("application/pdf") || url.toLowerCase().endsWith(".pdf");
+            const looksLikePdf = ct?.includes("application/pdf") || String(resp.url()).toLowerCase().endsWith(".pdf");
             if (!looksLikePdf || oversizedPdfError || poisonedError) return;
             const declared = parseInt(await resp.headerValue("content-length").catch(() => null) || "", 10);
             if (Number.isFinite(declared) && declared > MAX_PDF_SIZE) {
@@ -2374,7 +2376,8 @@ async function executeScrapeTask(_context, url, signal) {
     }
     const contentType = await response?.headerValue("content-type") || "";
     const status = response?.status?.() ?? 0;
-    if (contentType.includes("application/pdf") || url.toLowerCase().endsWith(".pdf")) {
+    const finalUrl = String(response?.url?.() ?? url);
+    if (contentType.includes("application/pdf") || finalUrl.toLowerCase().endsWith(".pdf")) {
       if (!response) throw new Error(`[Worker] No response received for PDF URL: ${url}`);
       if (status >= 400) throw new Error(`HTTP ${status}`);
       const declaredLength = parseInt(await response.headerValue("content-length").catch(() => null) || "", 10);
@@ -2805,7 +2808,17 @@ async function initBrowser() {
             // silent either, so the mute is applied unconditionally.
             firefox_user_prefs: {
               "media.volume_scale": "0.0",
-              "media.autoplay.default": 5
+              "media.autoplay.default": 5,
+              // SSRF: WebSocket connections bypass BOTH layers of the browser
+              // scrape defense — page.route('**') intercepts HTTP(S) only, and
+              // the serverAddr() rebinding backstop hangs off page.on('response'),
+              // which a WS handshake never fires. A hostile scraped page could
+              // therefore `new WebSocket('ws://10.x.x.x/...')` to probe or reach
+              // internal hosts with no validation, and write any reply into the
+              // DOM that page.content() returns. The scraper extracts static
+              // content (domcontentloaded), so nothing it needs arrives over WS —
+              // disable the channel outright at the engine level.
+              "network.websocket.max-connections": 0
             }
           });
           launchPromise.catch((err) => logToDebugFile3("DEBUG", `[ThreadWorker] Background browser launch rejection: ${err.message}`));

package/src/infrastructure/embedding/embedding-factory.ts

@@ -162,6 +162,31 @@ export async function getEmbedder(config?: Config, _attempt = 0): Promise<IEmbed
     // read → the entry omits it and liveness falls back to a bare signal-0 check.
     const myStartTime = await processLifecycle.getCurrentProcessStartTime();
 
+    // Read-only state (a NEWER build's state file is on disk — mixed-version
+    // window): every state write is suppressed, so the port=-1 candidacy claim
+    // below would never be visible to other processes, every process would
+    // "win" its own election, and the GPU lock would likewise always read as
+    // unowned — the exact concurrent-GPU-init segfault class election + lock
+    // exist to prevent. Degrade to a coordination-free mode instead: no
+    // election, local in-process embedder forced onto CPU (concurrent CPU
+    // inference across processes is safe; the exclusion protects GPU/WebGPU
+    // contexts, and a CPU device never touches the GPU lock). Slower embedding
+    // for the mixed-version window only; normal service resumes once every
+    // session runs the same build.
+    const localCpuForReadOnlyState = (): IEmbedder => {
+      logger.warn(
+        '[EmbeddingFactory] State file is read-only (written by a newer build): cross-process leader election and the GPU lock cannot coordinate in this mode. Skipping election and embedding locally on CPU until all sessions run the same build.',
+      );
+      const embedder = buildInProcessEmbedder({ ...cfg, EMBEDDING_DEVICE: 'cpu' }, stateManager);
+      _embeddingInstance = embedder;
+      _cachedModel = cfg.EMBEDDING_MODEL;
+      _cachedDevice = cfg.EMBEDDING_DEVICE;
+      return embedder;
+    };
+    if (stateManager.isReadOnly?.()) {
+      return localCpuForReadOnlyState();
+    }
+
     // ---- Phase 1: Atomically claim candidacy under the state lock ----
     // Writing port=-1 acts as a mutex: other processes that see it will wait
     // for the real port rather than starting their own model initialization.
@@ -200,6 +225,15 @@ export async function getEmbedder(config?: Config, _attempt = 0): Promise<IEmbed
       return state;
     });
 
+    // Re-check AFTER the claim: the read-only flag is set lazily by the first
+    // read that encounters the newer file — which can be this very updateState.
+    // In that case iAmCandidate is true but the claim write was silently
+    // suppressed: no other process can see it, so proceeding as leader is the
+    // concurrent-election hazard the pre-check above exists to avoid.
+    if (iAmCandidate && stateManager.isReadOnly?.()) {
+      return localCpuForReadOnlyState();
+    }
+
     // A live leader serving a DIFFERENT model must not be adopted: two sessions
     // with different EMBEDDING_MODELs would cross-embed (same-dimension models
     // corrupt the vector space silently). Fall back to a local in-process

package/src/infrastructure/embedding/embedding-server.ts

@@ -178,19 +178,39 @@ class SerialQueue {
   }
 
   /**
-   * Resolve once no native call is in flight. Used by EmbeddingServer.shutdown()
-   * to wait out a still-running inference before disposing the pipeline —
-   * without this, disposal could run concurrently with a legitimately slow (not
-   * hung) embed/embedMany call on the same re-entrant-locked GPU context, the
-   * exact native-segfault class this queue exists to prevent. No timeout: a
-   * promise that truly never settles is already an accepted, pre-existing
-   * leak-to-process-exit scenario elsewhere in this class (see the poison path's
+   * Resolve once no native call is in flight AND no queued task is still due
+   * to run. Used by EmbeddingServer.shutdown() to wait out a still-running
+   * inference before disposing the pipeline — without this, disposal could
+   * run concurrently with a legitimately slow (not hung) embed/embedMany call
+   * on the same re-entrant-locked GPU context, the exact native-segfault
+   * class this queue exists to prevent. No timeout: a promise that truly
+   * never settles is already an accepted, pre-existing leak-to-process-exit
+   * scenario elsewhere in this class (see the poison path's
    * disposeEmbedder=false handling) — better to leak than risk concurrent
    * native access.
+   *
+   * Checking only `currentWork` is NOT enough with a second task already
+   * queued: a task's `run()` clears `currentWork` to null in its `finally`
+   * (once its native call has settled), and `pump()` only claims it again for
+   * the next task on a LATER microtask tick, once `run()` for the current
+   * task has fully unwound and `await entry.run()` resumes. If a caller's own
+   * check happens to land in that gap, `currentWork` reads as null even
+   * though a second native call is about to start (or a queue in the process
+   * of unwinding to its next task hasn't reached it yet) — exactly the window
+   * disposal must not run in. `running` stays true for the queue's entire
+   * drain, across every task in `tasks`, not just the current one, so it has
+   * no equivalent gap and closes this one.
    */
   async waitForIdle(): Promise<void> {
-    while (this.currentWork) {
-      await this.currentWork.catch(() => {});
+    while (this.currentWork || this.running) {
+      if (this.currentWork) {
+        await this.currentWork.catch(() => {});
+      } else {
+        // Nothing executing right now, but pump() is still mid-drain (about
+        // to claim the next queued task, or unwinding) — yield a microtask
+        // so it can advance instead of reporting idle into this exact gap.
+        await Promise.resolve();
+      }
     }
   }
 
@@ -330,6 +350,11 @@ export class EmbeddingServer implements IEmbedder {
     // stuck native thread (itself hanging) or touch a device-lost context. Clearing
     // the state registration + closing the HTTP server is enough for re-election;
     // the wedged native resources are abandoned to eventual process exit.
+    // Latch the wedge on the Embedder itself too: it stays registered for
+    // process-exit cleanup (registerGlobalEmbedder), and without the latch that
+    // exit task's dispose() would call pipeline.dispose() on the wedged session
+    // anyway — re-creating at teardown the exact hang this path avoids.
+    this.embedder.markPipelineWedged();
     void this.shutdown(false);
     this.externalOnPoison?.(err);
   }

package/src/infrastructure/state/state-manager.ts

@@ -198,6 +198,21 @@ export class StateManager {
     };
   }
 
+  /**
+   * True once this process has seen a newer-build state file and entered
+   * read-only mode (writes suppressed, reads served from an in-memory default).
+   * In this mode NO cross-process coordination that lives in the state file
+   * works: a GPU-lock acquire always sees "no owner" and the embedding
+   * leader-election CAS always sees an empty slot, so every read-only process
+   * would self-elect — the concurrent-GPU-init class those mechanisms exist to
+   * prevent. Consumers that rely on that exclusion (the embedding factory) must
+   * check this and degrade to a coordination-free mode (local CPU embedding).
+   * The flag is set lazily by the first read that encounters the newer file.
+   */
+  isReadOnly(): boolean {
+    return this.stateTooNew;
+  }
+
   // ==================== Core State Operations ====================
 
   /**