@lincoln504/pi-research 1.6.10 → 1.6.11

package/README.md

@@ -18,11 +18,14 @@
   <a href="docs/CONFIGURATION.md">Configuration</a>
   ·
   <a href="docs/ARCHITECTURE.md">Architecture</a>
+  ·
+  <a href="docs/DOCUMENTATION-ES.md">ES</a>
+  ·
+  <a href="docs/DOCUMENTATION-ZH.md">中文研究</a>
 </p>
 
 ---
 
-<a href="https://github.com/Lincoln504/pi-research/actions/workflows/ci.yml"><img alt="Build status" src="https://img.shields.io/github/actions/workflow/status/Lincoln504/pi-research/ci.yml?style=flat-square&branch=main" /></a><br />
 <a href="https://www.npmjs.com/package/@lincoln504/pi-research"><img alt="npm version" src="https://img.shields.io/npm/v/@lincoln504/pi-research.svg?style=flat-square" /></a><br />
 <a href="LICENSE"><img alt="License: MIT" src="https://img.shields.io/badge/license-MIT-97ca00?style=flat-square" /></a><br />
 <a href="https://github.com/Lincoln504/pi-research"><img alt="GitHub repository" src="https://img.shields.io/badge/GitHub-Lincoln504%2Fpi--research-181717?style=flat-square&logo=github&logoColor=white" /></a>

package/agent-skill/pi-research/SKILL.md

@@ -2,7 +2,7 @@
 name: pi-research
 description: "Multi-agent web research for any coding agent. Use for anything requiring current information from the internet — news, trends, product/service/company facts, statistics, people, places, events, APIs, library behavior, or 'what is X / how does X work' questions. Also use to search a local knowledge store of previously researched findings before doing live research. Powered by the pi-research engine (search, stealth scraping, security databases, Stack Exchange). Not for reading local files, running project commands, or analyzing this repository's own source."
 license: MIT
-metadata: { "author": "Lincoln Deen", "version": "1.6.10", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
+metadata: { "author": "Lincoln Deen", "version": "1.6.11", "package": "@lincoln504/pi-research", "engine": "pi-research-sdk", "openclaw": { "requires": { "bins": ["node"] } } }
 allowed-tools: Bash(node:*)
 ---
 

package/dist/cli.mjs

@@ -2797,6 +2797,7 @@ function saveConfig(config, scope = "local", cwd = process.cwd(), changedKeys) {
     PI_RESEARCH_AVG_TOKENS_PER_SCRAPE: String(config.AVG_TOKENS_PER_SCRAPE),
     PI_RESEARCH_MAX_CONCURRENT_SCRAPES: String(config.MAX_CONCURRENT_SCRAPES),
     PI_RESEARCH_MAX_SCRAPE_URLS: String(config.MAX_SCRAPE_URLS),
+    PI_RESEARCH_SCRAPE_SECOND_PAGE: String(config.SCRAPE_SECOND_PAGE),
     PI_RESEARCH_YOUTUBE_TRANSCRIPT_MAX_VIDEOS: String(config.YOUTUBE_TRANSCRIPT_MAX_VIDEOS),
     PI_RESEARCH_YOUTUBE_TRANSCRIPT_TIMEOUT_MS: String(config.YOUTUBE_TRANSCRIPT_TIMEOUT_MS),
     PI_RESEARCH_YOUTUBE_TRANSCRIPT_LANG: config.YOUTUBE_TRANSCRIPT_LANG,
@@ -2959,6 +2960,7 @@ function createConfig(env, processEnv) {
     AVG_TOKENS_PER_SCRAPE: parseEnvNumber(e, "PI_RESEARCH_AVG_TOKENS_PER_SCRAPE", DEFAULTS.AVG_TOKENS_PER_SCRAPE, 500, 1e4),
     MAX_CONCURRENT_SCRAPES: parseEnvNumber(e, "PI_RESEARCH_MAX_CONCURRENT_SCRAPES", DEFAULTS.MAX_CONCURRENT_SCRAPES, 1, 20, true),
     MAX_SCRAPE_URLS: parseEnvNumber(e, "PI_RESEARCH_MAX_SCRAPE_URLS", DEFAULTS.MAX_SCRAPE_URLS, 1, 20, true),
+    SCRAPE_SECOND_PAGE: parseEnvBool(e, "PI_RESEARCH_SCRAPE_SECOND_PAGE", DEFAULTS.SCRAPE_SECOND_PAGE),
     YOUTUBE_TRANSCRIPT_MAX_VIDEOS: parseEnvNumber(e, "PI_RESEARCH_YOUTUBE_TRANSCRIPT_MAX_VIDEOS", DEFAULTS.YOUTUBE_TRANSCRIPT_MAX_VIDEOS, 1, 5, true),
     YOUTUBE_TRANSCRIPT_TIMEOUT_MS: parseEnvNumber(e, "PI_RESEARCH_YOUTUBE_TRANSCRIPT_TIMEOUT_MS", DEFAULTS.YOUTUBE_TRANSCRIPT_TIMEOUT_MS, 5e3, 12e4),
     YOUTUBE_TRANSCRIPT_LANG: parseEnvString(e, "PI_RESEARCH_YOUTUBE_TRANSCRIPT_LANG", DEFAULTS.YOUTUBE_TRANSCRIPT_LANG),
@@ -3105,6 +3107,8 @@ var init_config = __esm({
       MAX_CONCURRENT_SCRAPES: Type.Number({ minimum: 1, maximum: 20, default: 3 }),
       /** Maximum URLs fetched per scrape batch; extras are listed under "Not Fetched — Over Batch Cap" (default: 8, range: 1-20) */
       MAX_SCRAPE_URLS: Type.Number({ minimum: 1, maximum: 20, default: 8 }),
+      /** Also fetch DuckDuckGo results page 2 for every search query (default: false). Configured via env/config file; not surfaced in the config TUI. */
+      SCRAPE_SECOND_PAGE: Type.Boolean({ default: false }),
       /** Max YouTube videos transcribed per youtube_transcript call (default: 3, range 1-5) */
       YOUTUBE_TRANSCRIPT_MAX_VIDEOS: Type.Number({ minimum: 1, maximum: 5, default: 3 }),
       /** Per-video timeout for the youtube_transcript tool in ms (default: 20000, range 5-120s) */
@@ -3171,11 +3175,12 @@ var init_config = __esm({
        *  the model's real maxTokens at call time.
        *  Not exposed in TUI — controlled via PI_RESEARCH_SYNTHESIS_MAX_TOKENS env var. */
       SYNTHESIS_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 32768 }),
-      /** LLM Model override for researcher sub-agents and knowledge synthesis.
+      /** LLM Model override for the evaluator/synthesis rounds and researcher sub-agents.
        *  Format: provider/model-id (e.g. google/gemini-2.0-flash-001) or just model-id.
-       *  When set, this overrides ctx.model for researcher sub-agents (both deep and quick)
-       *  and the knowledge synthesis background LLM. The coordinator and the research lead
-       *  always use the caller's model (ctx.model).
+       *  When set, this governs the evaluator/synthesis rounds and the researcher
+       *  sub-agents (both deep and quick) and the knowledge synthesis background LLM.
+       *  The initial coordinator (round 1 planning) uses the parent chat's model (ctx.model)
+       *  for prompt-cache continuity with the conversation that triggered research.
        */
       RESEARCH_MODEL: Type.Optional(Type.String()),
       /** Explicit directory for the knowledge store database (overrides default) */
@@ -3242,6 +3247,7 @@ var init_config = __esm({
       "PI_RESEARCH_AVG_TOKENS_PER_SCRAPE",
       "PI_RESEARCH_MAX_CONCURRENT_SCRAPES",
       "PI_RESEARCH_MAX_SCRAPE_URLS",
+      "PI_RESEARCH_SCRAPE_SECOND_PAGE",
       "PI_RESEARCH_HEALTH_CHECK_TIMEOUT_MS",
       "PI_RESEARCH_SEARCH_TIMEOUT_MS",
       "PI_RESEARCH_TUI_REFRESH_DEBOUNCE_MS",
@@ -4069,7 +4075,7 @@ function withTimeout2(promise, timeoutMs, label, signal) {
 }
 function messageIsTransient(message) {
   const m = message.toLowerCase();
-  if (m.includes("econnrefused") || m.includes("enotfound") || m.includes("timeout") || m.includes("etimedout") || m.includes("econnreset") || m.includes("fetch failed") || m.includes("terminated") || m.includes("other side closed") || m.includes("socket hang up") || m.includes("und_err")) {
+  if (m.includes("econnrefused") || m.includes("enotfound") || m.includes("timeout") || m.includes("etimedout") || m.includes("econnreset") || m.includes("fetch failed") || m.includes("terminated") || m.includes("other side closed") || m.includes("socket hang up") || m.includes("und_err") || m.includes("connection error")) {
     return true;
   }
   if (/\b429\b/.test(m) || /\brate\b/.test(m) || m.includes("quota")) {
@@ -11510,13 +11516,29 @@ You are in the late phase of research. Set a higher threshold for delegation:
   }
 }
 function parseJsonPlan(text) {
-  const result = extractJson(text, "object");
+  let normalized = text.trim();
+  if (normalized.startsWith('"')) {
+    try {
+      const unwrapped = JSON.parse(normalized);
+      if (typeof unwrapped === "string") normalized = unwrapped.trim();
+    } catch {
+    }
+  }
+  const result = extractJson(normalized, "object");
   if (!result.success || !result.value) {
     const preview = text.length > 100 ? text.slice(0, 100) + "..." : text;
     throw new Error(`Failed to extract valid JSON plan: ${result.error}. Raw response preview: "${preview}"`);
   }
+  const candidate = result.value;
+  if (candidate && typeof candidate === "object" && !Array.isArray(candidate)) {
+    for (const key of ["researchers", "allQueries", "content", "title", "action", "fallback"]) {
+      if (candidate[key] === null) {
+        delete candidate[key];
+      }
+    }
+  }
   try {
-    const coerced = Value3.Convert(ResearchPlanSchema, result.value);
+    const coerced = Value3.Convert(ResearchPlanSchema, candidate);
     if (!Value3.Check(ResearchPlanSchema, coerced)) {
       const errors = [...Value3.Errors(ResearchPlanSchema, coerced)];
       const errorMsg = errors.map((e) => `${e.path}: ${e.message}`).join(", ");
@@ -12676,6 +12698,7 @@ function getBrowserEnv(config) {
   const c = config || getConfig();
   env["PI_RESEARCH_SCRAPE_TIMEOUT_MS"] = String(c.SCRAPE_TIMEOUT_MS);
   env["PI_RESEARCH_SEARCH_TIMEOUT_MS"] = String(c.SEARCH_TIMEOUT_MS);
+  env["PI_RESEARCH_SCRAPE_SECOND_PAGE"] = String(c.SCRAPE_SECOND_PAGE);
   env["PI_RESEARCH_HEALTH_CHECK_TIMEOUT_MS"] = String(c.HEALTH_CHECK_TIMEOUT_MS);
   return env;
 }
@@ -16523,7 +16546,7 @@ var WorkerPoolManager = class {
             }
           },
           exitHandler: (code) => {
-            if (code !== 0 && this.poolEpoch === myEpoch) {
+            if (code !== 0 && !this.isShuttingDown && this.poolEpoch === myEpoch) {
               logger.error(`[WorkerPoolManager] Worker exited with code ${code}`);
               this.consecutiveErrors++;
               if (this.consecutiveErrors >= 3) {
@@ -24621,7 +24644,8 @@ var DeepResearchOrchestrator = class {
     const { model, query, complexity, researchId, ctx } = this.options;
     const observer = this.observer;
     const container = tryGetServiceContainerFromCtx(ctx);
-    const coordinatorModel = ctx.model ?? model;
+    const initialCoordinatorModel = ctx.model ?? model;
+    const evaluatorModel = model;
     const orchestrationService = await this.getOrchestrationService();
     const planningService = await this.getPlanningService();
     await orchestrationService.cleanupResearchServices(void 0, researchId, ctx, this.config);
@@ -24685,7 +24709,7 @@ var DeepResearchOrchestrator = class {
             sessionId: researchId,
             query,
             complexity,
-            model: coordinatorModel,
+            model: initialCoordinatorModel,
             modelRegistry: ctx.modelRegistry,
             cwd: ctx.cwd,
             config: this.config,
@@ -24705,7 +24729,7 @@ var DeepResearchOrchestrator = class {
             query,
             complexity,
             round: this.currentRound,
-            model: coordinatorModel,
+            model: evaluatorModel,
             modelRegistry: ctx.modelRegistry,
             cwd: ctx.cwd,
             config: this.config,
@@ -24870,7 +24894,7 @@ var DeepResearchOrchestrator = class {
         query,
         complexity,
         round: this.currentRound,
-        model: coordinatorModel,
+        model: evaluatorModel,
         modelRegistry: ctx.modelRegistry,
         cwd: ctx.cwd,
         config: this.config,
@@ -26073,6 +26097,7 @@ async function initializeOrchestrationServices(ctx, container = getServiceContai
 
 // src/sdk.ts
 init_service_registry();
+init_safe_unref();
 init_service_interfaces();
 init_logger();
 init_log_utils();
@@ -26307,6 +26332,17 @@ async function _doInit(options = {}) {
     }
     isInitialized = true;
     logger.log("[SDK] Research SDK initialized successfully");
+    shutdownManager.register(async () => {
+      if (!globalContainer) return;
+      const schedulerService = tryGetService(ServiceNames.SCHEDULER, globalContainer);
+      const scheduler = schedulerService?.getScheduler?.();
+      if (!scheduler) return;
+      try {
+        await raceWithDeadline(scheduler.shutdown(), 12e3);
+      } catch (err) {
+        logger.error("[SDK] Error stopping scheduler during shutdown cleanup:", err);
+      }
+    });
     _registerSignalHandlers();
   } catch (err) {
     logger.error("[SDK] Initialization failed:", err);

package/dist/thread-worker.mjs

@@ -2014,6 +2014,8 @@ function createKillHandler() {
 
 // src/infrastructure/browser/thread-worker-browser.ts
 import { platform as platform2 } from "node:os";
+import { readFileSync, existsSync as existsSync3 } from "node:fs";
+import { homedir as homedir2 } from "node:os";
 
 // src/infrastructure/browser/thread-worker-messaging.ts
 init_log_utils();
@@ -2143,15 +2145,23 @@ async function setupMocking(context) {
     const isDuckDuckGo = url.hostname === "duckduckgo.com" || url.hostname === "lite.duckduckgo.com" || url.hostname.endsWith(".duckduckgo.com");
     if (mockSearch && isDuckDuckGo) {
       logToDebugFile2("DEBUG", `[Worker-${workerId2}] Mocking search response for: ${urlStr}`);
-      route.fulfill({
-        status: 200,
-        contentType: "text/html",
-        body: `
-          <html><head><title>DuckDuckGo Lite</title></head><body>
-            <form action="/lite/" method="post">
-              <input name="q" type="text">
-              <input type="submit" value="Search">
-            </form>
+      const isSecondPageRequest = request.method() === "POST" && typeof request.postData() === "string" && /(?:^|&)s=2(?:&|$)/.test(request.postData());
+      const mockResultsHtml = isSecondPageRequest ? `
+            <table class="result-link">
+              <tr>
+                <td><a class="result-link" href="https://example.com/mock-result-3?uddg=https%3A%2F%2Fexample.com%2Ftarget-3">Mock Result 3</a></td>
+              </tr>
+              <tr>
+                <td class="result-snippet">Page-two mocked search result snippet.</td>
+              </tr>
+              <tr>
+                <td><a class="result-link" href="https://example.com/mock-result-4?uddg=https%3A%2F%2Fexample.com%2Ftarget-4">Mock Result 4</a></td>
+              </tr>
+              <tr>
+                <td class="result-snippet">Another page-two mocked snippet to verify second-page extraction.</td>
+              </tr>
+            </table>
+          ` : `
             <table class="result-link">
               <tr>
                 <td><a class="result-link" href="https://example.com/mock-result-1?uddg=https%3A%2F%2Fexample.com%2Ftarget-1">Mock Result 1</a></td>
@@ -2166,6 +2176,23 @@ async function setupMocking(context) {
                 <td class="result-snippet">Another mocked snippet to verify multiple result extraction.</td>
               </tr>
             </table>
+            <form action="/lite/" method="post">
+              <input type="hidden" name="q" value="">
+              <input type="hidden" name="s" value="2">
+              <input type="hidden" name="dc" value="3">
+              <input type="submit" value="Next">
+            </form>
+          `;
+      route.fulfill({
+        status: 200,
+        contentType: "text/html",
+        body: `
+          <html><head><title>DuckDuckGo Lite</title></head><body>
+            <form action="/lite/" method="post">
+              <input name="q" type="text">
+              <input type="submit" value="Search">
+            </form>
+            ${mockResultsHtml}
           </body></html>
         `
       });
@@ -2186,10 +2213,11 @@ async function setupMocking(context) {
 async function extractSearchResults(page) {
   return await page.evaluate(() => {
     const found = [];
-    const links = Array.from(document.querySelectorAll("a.result-link"));
+    const links = Array.from(document.querySelectorAll("a.result-link, a.result__a"));
     links.forEach((link) => {
-      const row = link.closest("tr");
-      const snippet = row?.nextElementSibling?.querySelector("td.result-snippet")?.textContent?.trim() || "";
+      const row = link.closest("tr") || link.closest(".result");
+      const snippetEl = row?.querySelector("td.result-snippet") || row?.querySelector("a.result__snippet") || row?.querySelector(".result__snippet") || row?.nextElementSibling?.querySelector("td.result-snippet");
+      const snippet = snippetEl?.textContent?.trim() || "";
       const title = link.textContent?.trim() || "";
       let url = link.href;
       try {
@@ -2230,14 +2258,41 @@ async function executeSearchTask(_context, query, signal) {
     if (process.env["PI_RESEARCH_MOCK_SEARCH"] === "true") {
       await page.goto(`https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(query)}`, { waitUntil: "domcontentloaded" });
     } else {
-      await page.goto("https://lite.duckduckgo.com/lite/", { waitUntil: "domcontentloaded" });
-      await page.fill('input[name="q"]', query);
-      await Promise.all([
-        page.waitForNavigation({ waitUntil: "domcontentloaded" }),
-        page.keyboard.press("Enter")
-      ]);
+      const qurl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
+      const nav = await page.goto(qurl, { waitUntil: "domcontentloaded" });
+      if ((!nav || nav.status && nav.status() > 399) && !await page.$("a.result__a, a.result-link")) {
+        await page.goto(`https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(query)}`, { waitUntil: "domcontentloaded" });
+      }
+      try {
+        await page.waitForSelector("a.result__a, a.result-link", { timeout: 8e3 });
+      } catch {
+      }
     }
     const results = await extractSearchResults(page);
+    const scrapeSecondPage = process.env["PI_RESEARCH_SCRAPE_SECOND_PAGE"] === "true";
+    if (scrapeSecondPage && results.length > 0) {
+      try {
+        const nextButton = page.locator('form input[type="submit"][value*="next" i]').first();
+        if (await nextButton.count() > 0) {
+          logToDebugFile2("DEBUG", `[Worker-${workerId2}] SCRAPE_SECOND_PAGE: fetching results page 2 for: ${query}`);
+          await Promise.all([
+            page.waitForNavigation({ waitUntil: "domcontentloaded" }),
+            nextButton.click()
+          ]);
+          const seen = new Set(results.map((r) => r.url));
+          for (const r of await extractSearchResults(page)) {
+            if (!seen.has(r.url)) {
+              seen.add(r.url);
+              results.push(r);
+            }
+          }
+        } else {
+          logToDebugFile2("DEBUG", `[Worker-${workerId2}] SCRAPE_SECOND_PAGE: no next-page control (single page of results) for: ${query}`);
+        }
+      } catch (err) {
+        logToDebugFile2("WARN", `[Worker-${workerId2}] SCRAPE_SECOND_PAGE: page-2 fetch failed (keeping page-1 results) for: ${query} \u2014 ${err?.message || String(err)}`);
+      }
+    }
     await page.close();
     const jitterBase = process.env["PI_RESEARCH_MOCK_SEARCH"] === "true" ? 10 : 500;
     const jitterRange = process.env["PI_RESEARCH_MOCK_SEARCH"] === "true" ? 10 : 1e3;
@@ -2669,6 +2724,8 @@ var ConfigSchema = Type.Object({
   MAX_CONCURRENT_SCRAPES: Type.Number({ minimum: 1, maximum: 20, default: 3 }),
   /** Maximum URLs fetched per scrape batch; extras are listed under "Not Fetched — Over Batch Cap" (default: 8, range: 1-20) */
   MAX_SCRAPE_URLS: Type.Number({ minimum: 1, maximum: 20, default: 8 }),
+  /** Also fetch DuckDuckGo results page 2 for every search query (default: false). Configured via env/config file; not surfaced in the config TUI. */
+  SCRAPE_SECOND_PAGE: Type.Boolean({ default: false }),
   /** Max YouTube videos transcribed per youtube_transcript call (default: 3, range 1-5) */
   YOUTUBE_TRANSCRIPT_MAX_VIDEOS: Type.Number({ minimum: 1, maximum: 5, default: 3 }),
   /** Per-video timeout for the youtube_transcript tool in ms (default: 20000, range 5-120s) */
@@ -2735,11 +2792,12 @@ var ConfigSchema = Type.Object({
    *  the model's real maxTokens at call time.
    *  Not exposed in TUI — controlled via PI_RESEARCH_SYNTHESIS_MAX_TOKENS env var. */
   SYNTHESIS_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 32768 }),
-  /** LLM Model override for researcher sub-agents and knowledge synthesis.
+  /** LLM Model override for the evaluator/synthesis rounds and researcher sub-agents.
    *  Format: provider/model-id (e.g. google/gemini-2.0-flash-001) or just model-id.
-   *  When set, this overrides ctx.model for researcher sub-agents (both deep and quick)
-   *  and the knowledge synthesis background LLM. The coordinator and the research lead
-   *  always use the caller's model (ctx.model).
+   *  When set, this governs the evaluator/synthesis rounds and the researcher
+   *  sub-agents (both deep and quick) and the knowledge synthesis background LLM.
+   *  The initial coordinator (round 1 planning) uses the parent chat's model (ctx.model)
+   *  for prompt-cache continuity with the conversation that triggered research.
    */
   RESEARCH_MODEL: Type.Optional(Type.String()),
   /** Explicit directory for the knowledge store database (overrides default) */
@@ -2799,6 +2857,25 @@ var CONTEXT_CREATION_TIMEOUT_MS = 3e4;
 var COLD_START_ALLOWANCE_MS = BROWSER_LAUNCH_TIMEOUT_MS + CONTEXT_CREATION_TIMEOUT_MS;
 
 // src/infrastructure/browser/thread-worker-browser.ts
+function loadSocksProxyPrefs() {
+  try {
+    const p = `${homedir2()}/.pi/research/proxy.json`;
+    if (!existsSync3(p)) return {};
+    const cfg = JSON.parse(readFileSync(p, "utf8"));
+    const host = cfg.host ?? "127.0.0.1";
+    const port = cfg.port ?? 1080;
+    return {
+      "network.proxy.type": 1,
+      "network.proxy.socks": host,
+      "network.proxy.socks_port": port,
+      "network.proxy.socks_remote_dns": true,
+      "network.proxy.socks_version": 5,
+      "network.proxy.no_proxies_on": ""
+    };
+  } catch {
+    return {};
+  }
+}
 var browser = null;
 var initPromise = null;
 var workerId3 = "";
@@ -2904,6 +2981,7 @@ async function initBrowser() {
             // but headless Firefox on macOS/Windows is not guaranteed to be
             // silent either, so the mute is applied unconditionally.
             firefox_user_prefs: {
+              ...loadSocksProxyPrefs(),
               "media.volume_scale": "0.0",
               "media.autoplay.default": 5,
               // SSRF: WebSocket connections bypass BOTH layers of the browser

package/docs/CHANGELOG.md

@@ -5,6 +5,39 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.6.11] - 2026-09-04
+
+### Changed
+
+- **New `RESEARCH_MODEL` config setting (env `PI_RESEARCH_MODEL`) separates the research engine's model from the host pi session's model.** When unset, everything runs on the host model exactly as before — the setting is strictly optional. When set, the split is: round-1 planning (`generatePlan`) keeps running on the host session's model (`ctx.model`, falling back to `RESEARCH_MODEL`) for prompt-cache continuity with the ongoing conversation, while round-2+ plan updates and the final synthesis — the calls that do the long-form research writing — run on `RESEARCH_MODEL`. Wired through the config schema (TypeBox string, `src/config.ts`), strict env parsing, `saveConfig` persistence, and documented in `docs/CONFIGURATION.md`; `src/sdk.ts` and the `research_knowledge_search` tool text describe the split.
+
+- **Dependency upgrades across the board; `npm audit` reports 0 vulnerabilities.** pi extension packages (`@earendil-works/*`) moved to 0.85.0 with `PI_MIN_VERSION` / `PI_TESTED_MAX_VERSION` following, and `@earendil-works/pi-server` added; the dev stack moved to vitest 5, jsdom 30, eslint 10.10, and dependency-cruiser 18. Three upgrades were deliberately held back by verified constraints: typescript stays `< 6.1.0` (@typescript-eslint 8.69's peer ceiling), playwright-core stays `1.60.0` (camoufox-js requires `< 1.61.0`), and apache-arrow stays `21.1.0` (21.2.0 breaks LanceDB's IPC reads — docs/ARCHITECTURE.md documents why).
+- **New boolean config setting `SCRAPE_SECOND_PAGE` (env `PI_RESEARCH_SCRAPE_SECOND_PAGE`, default `false`): when enabled, every search query also fetches DuckDuckGo Lite's results page 2, roughly doubling the candidate links each query contributes.** The thread-worker's search task (`src/infrastructure/browser/thread-worker-messaging.ts`) now clicks DDG Lite's own next-page form (the hidden `s`/`dc` offsets + "Next" submit at the bottom of the results table) rather than hand-building a page-2 URL, so the offsets the site itself issued stay authoritative whatever per-page count DuckDuckGo currently serves; page-2 results are URL-deduped against page 1 in-worker, and cross-query dedup continues to happen in `performSearch`'s `seenUrls` set. Failure policy: page-1 results are already complete and usable, so any page-2 problem — no next control (single page of results), click/navigation timeout, abort-triggered page close — degrades to page-1-only with a WARN line, never a failed query. The setting follows the full knob integration path: Config schema (TypeBox boolean, `src/config.ts`), strict env parsing (`parseEnvBool`, literal `true` only), the env allowlist, `saveConfig` persistence, and the worker env bridge in `getBrowserEnv` (workers read it from their `process.env` copy). The mock DDG handler now also serves a DISTINCT page-2 result set when a `s=2` POST arrives, making the whole flow end-to-end testable offline; six new worker tests pin the merge, dedup, flag-off, no-next-control, click-failure, and empty-page-1 paths (`test/unit/infrastructure/thread-worker-search-second-page.test.ts`), plus config parse/persist round-trip tests. Not surfaced in the `/research-config` TUI menu, matching every other env-only scrape knob.
+
+### Fixed
+
+- **Ctrl+C could quit the entire pi process when the user pressed it twice to abort a research run — and one physical press could clear the editor AND abort the run 150ms later.** Root causes found by reading pi's actual input pipeline (dist/modes/interactive/interactive-mode.js + @earendil-works/pi-tui) and confirmed against the live logs: (1) Extension `onTerminalInput` listeners run BEFORE the focused editor and may consume keys, and pi's `handleCtrlC()` is TIME-based, not content-based — the first press clears the editor and arms a window, and a SECOND press within 500ms calls `shutdown()`, exiting the whole pi process; there is no "abort research" step in it. The previous fix made the extension abort the active run when the editor was empty but still let the key fall through, so the natural double-press gesture (clear, then abort-now) landed inside pi's 500ms window and killed pi itself. (2) Pi runs the Kitty keyboard protocol with flag 2 (press/repeat/release reporting), so every physical key sends TWO sequences ~100-200ms apart — and Tui.handleTerminalInput filters releases only for the focused component, AFTER extension listeners run, while `matchesKey` cannot tell them apart (`matchesKittySequence` parses codepoint+modifier but never the event type, so a ctrl+c RELEASE `\x1b[99;5:3u` matches `matchesKey(data, 'ctrl+c')` exactly like a press). Observed live 2026-09-03 (log lines 150ms apart, same pid): one physical Ctrl+C with text in the editor logged "with editor text — clearing only", then the RELEASE event matched ctrl+c against the editor pi had just cleared and aborted the research run; the same double-fire masqueraded as "Esc" aborts in older logs. The cancel policy in `src/tui/tui-controller.ts` now (a) drops key-RELEASE events outright — pi's own components never see them, so this matches pi's semantics exactly; key repeats stay, as genuine held-key intent — and (b) fully owns Ctrl+C while a run is active: editor has text → fall through (pi clears, run continues); editor empty AND run active → abort the run AND consume the key so pi's exit path can never fire; editor empty AND no run → untouched native behavior (clear/arm, double-press exit keeps working). Esc keeps its panic-cancel role and still falls through to pi's app.interrupt. Verified that consuming Ctrl+C cannot break copying: no `tui.input.copy` handler exists in pi's runtime (message copy is Ctrl+X, terminal selection copy is the emulator's own). Thirteen policy tests pin the contract, including real Kitty press/release/repeat sequences (`test/unit/tui/tui-controller.test.ts`).
+
+- **Doubly-encoded plan payloads no longer fail plan parsing, and explicit nulls for optional plan fields are treated as absent.** Two live-failure classes from 2026-09-02 (`src/core/planning-utils.ts`): some models (observed: deepseek-v4-flash via OpenRouter) call `submit_plan` with the plan JSON-encoded INSIDE a string argument, so the toolCall re-serialization hands `parseJsonPlan` a doubly-encoded payload whose raw `{`-scan cannot parse (stray `\"` after `{`), failing with the misleading "No valid JSON found" and dropping the plan into the costly agentic-repair path — `parseJsonPlan` now unwraps a top-level JSON string literal once before extraction (safe by construction: any object resolvable in the outer string is resolvable in the inner one). Separately, models emit optional schema fields (`researchers`/`allQueries`/`content`/`title`/`action`/`fallback`) as explicit `null`: under TypeBox 1.3.7 a null optional field reports a ROOT error ("undefined: must be object") instead of pointing at the field — reproducing "Plan validation failed: undefined: must be object" in the logs — and pushed a perfectly valid `research-complete` synthesize decision into agentic repair. The six optional keys (the complete optional-field list of `ResearchPlanSchema`) are now dropped when null before validation; required-field nulls inside researcher objects still fail validation as they should. Both behaviors are pinned by new tests.
+
+- **Worker-pool auto-recovery no longer counts shutdown-window worker deaths as health faults (`src/infrastructure/browser/worker-pool-manager.ts`).** The SIGTERM teardown tears the process down while workers die with code null mid-DAG-disposal; the exitHandler counted those as unexpected deaths, logging ERROR and scheduling pointless auto-recovery on a dying process — all 73 null-code exits across 15 pids in the 2026-09-01/02 logs coincided with SDK SIGTERM shutdown, tripping "3 consecutive exits → pool unhealthy" once per dying session, with zero exits during live research. The exitHandler now skips health-fault accounting while `isShuttingDown` is set; the existing `poolEpoch` check still guards all intentional-destroy paths, and post-shutdown pool rebuilds start clean.
+
+- **The browser scheduler is now stopped as the FIRST cleanup task of an SDK shutdown (`src/sdk.ts`), and this runs on every teardown path — signal handlers and cooperative `shutdownResearchSDK()` alike.** Previously the sdk_shutdown cleanup list was empty for scheduler teardown: the scheduler's cluster workers died mid-DAG-disposal with code null while the worker pool was still live, which the pool's exitHandler counted as unexpected deaths ("3 consecutive exits → pool unhealthy") on a process that was exiting anyway. Stopping the scheduler first puts those deaths inside the pool's intentional-destroy window. The task is bounded by `raceWithDeadline(..., 12000)` well inside the 15s signal force-exit deadline, both teardown paths are idempotent (the later DAG dispose no-ops), and per-init registration cannot stack because `runCleanup` detaches the task set after running.
+
+- **The OpenAI SDK's `APIConnectionError` message ("Connection error.") is now classified as transient (`src/web-research/retry-utils.ts`).** pi-ai surfaces this message shape for openai-completions providers like OpenRouter when the fetch itself fails before any response (refused / reset / DNS / TLS) — the SDK throws this class only for fetch-level transport failures, after its own 2 internal retries are exhausted, so the string is a reliable transient signal. Seen live 2026-09-02: an OpenRouter planning call died in 4s with zero retries because the pattern was missing, and the same miss made `isDegradableLlmError` throw instead of degrading to the fallback plan — both layers gate on this matcher.
+
+- **The DDG search path no longer POSTs the form endpoint; it drives a direct GET of the `html` endpoint with a wait for results, and degrades to the Lite layout when the HTML endpoint serves nothing (`src/infrastructure/browser/thread-worker-messaging.ts`).** The form POST was the first thing DDG's bot defenses throttled (live 2026-09-02: every query timed out in `page.goto`), while a plain GET of `https://html.duckduckgo.com/html/?q=…` still renders the full result list. The task now navigates to the GET URL, waits for a result anchor (`a.result__a` / `a.result-link`) to appear (bounded, abort-aware), and falls back to the Lite host layout when the HTML endpoint returns an empty page. Snippet extraction was rewritten to match both layouts at once: in the HTML layout the snippet (`a.result__snippet` / `.result__snippet`) sits inside the same `div.result` container as the title link, while in the Lite layout each result is a TWO-row pair — the link `<tr>` and a following snippet `<tr>` — so `extractSearchResults` first tries the inside-row selectors, then `row.nextElementSibling`'s `td.result-snippet` (the original rewrite searched only inside the row, silently dropping every Lite snippet). A `s=2` POST is still used for page 2 under `SCRAPE_SECOND_PAGE`, reusing the offsets the site itself issued.
+
+- **Browser workers now route through the local SOCKS proxy when `~/.pi/research/proxy.json` is present (`src/infrastructure/browser/thread-worker-browser.ts`).** Some egress paths can reach only one of DuckDuckGo's front-door IPs while the system resolver hands out the unreachable one, blackholing every DDG hostname for every HTTP client. When the proxy descriptor file exists (`{"host":"127.0.0.1","port":1080}`), the worker launches the browser with `--proxy-server=socks5://host:port` and remote DNS on, so hostname resolution happens at the proxy — which pins `*.duckduckgo.com` to verified-live front-door IPs and uses normal DNS for everything else. The file-based contract (not env) was chosen deliberately: a long-lived leader process picks up proxy changes without a restart, and a missing/invalid file is swallowed to `{}` so the browser launches direct.
+
+- **The bundled SOCKS proxy (`scripts/ddg-socks-proxy.mjs`) now catches the accept-then-stall front-door pattern and rotates its starting pin.** The dead front-door ACCEPTS the TCP handshake and then never answers the TLS ClientHello, so the original data-deadline — cleared on `connect` — was disarmed exactly when it was needed: clients hung until their own 45s timeout with the fallback never engaging. The data-arrival deadline now SPANS connect plus the post-connect ClientHello wait and clears only on the first upstream byte; the success reply goes out at connect (the SOCKS client cannot send its first payload until it gets the reply, and no transparent retry is possible after that), so a post-reply stall fast-fails both ends (~8s instead of 45s) and rotates the starting pin for the NEXT connection — the live front-door flips with the egress path, so the pin list self-heals toward whichever IP last worked. Two more hardening fixes in the same pass: a client that pipelined its payload with the request (instead of waiting for the success reply) no longer loses those buffered bytes, and a latent double-callback on post-handoff socket errors was fixed.
+
+- **The README's CI build-status badge was removed** (cosmetic; the npm version, license, and repository badges remain, markup verified intact).
+
+### Verified
+
+- 3,046 unit tests over 241 files, ESLint clean on the touched source, all four type-checks (TS 6 src + tests, TS 7 native src + tests), build, and `npm ci --dry-run` manifest verification; `npm audit` reports 0 vulnerabilities.
+
 ## [1.6.10] - 2026-09-02
 
 ### Changed

package/docs/CONFIGURATION.md

@@ -12,7 +12,11 @@ environment-variable reference, and finally how the configuration layers resolve
 
 Run `/research-config` in the pi extension to open an interactive menu. Selecting a
 setting and pressing `Enter` / `Space` cycles its value; the change is saved
-immediately. A setting is written to one of two scopes:
+immediately. (On non-TUI hosts — RPC, web hub, print, JSON, SDK — the menu cannot
+render: `/research-config` explains why and answers with the headless diagnostics
+`/research-config health` and `/research-config knowledge-status`. Settings are
+still read from the environment and config files there, and `PI_RESEARCH_*`
+variables still take effect.) A setting is written to one of two scopes:
 
 - `[project]` — saved per working directory in the central project registry, so
   a given repo can carry its own value without changing your global default.
@@ -72,6 +76,7 @@ Research
 | `PI_RESEARCH_MAX_GATHERING_CALLS` | `12` | 1–100 | Shared web-gathering calls per researcher (`search` + `security_search` + `stackexchange` + `youtube_transcript`). |
 | `PI_RESEARCH_MAX_CONCURRENT_SCRAPES` | `3` | 1–20 | Concurrent URLs fetched per scrape batch. |
 | `PI_RESEARCH_MAX_SCRAPE_URLS` | `8` | 1–20 | Maximum URLs fetched per scrape batch. URLs past the cap are listed under "Not Fetched — Over Batch Cap" and must be requested in a later batch. (Promoted from a hardcoded constant so it is env/config-file tunable like every other scrape knob.) |
+| `PI_RESEARCH_SCRAPE_SECOND_PAGE` | `false` | true/false | Also fetch DuckDuckGo results page 2 for every search query, roughly doubling the candidate links each query contributes. Each query's page-2 fetch degrades gracefully — if page 2 fails or does not exist, the page-1 results are kept. |
 | `PI_RESEARCH_MAX_RETRIES` | `2` | 0–5 | Retries per researcher request. |
 | `PI_RESEARCH_RETRY_DELAY_MS` | `2000` | 100–10000 | Base delay between retries. |
 | `PI_RESEARCH_MAX_FAILED_RESEARCHERS` | `2` | 1–10 | Unique researcher failures that abort the whole run. Raise to let slower, still-in-flight researchers finish before giving up. |
@@ -79,7 +84,7 @@ Research
 | `PI_RESEARCH_WORKER_CONCURRENCY` | `2` | 1–10 | Tasks per worker process. |
 | `PI_RESEARCH_MAX_CONCURRENT_RUNS` | `3` | ≥1 | Machine-wide cap on research runs executing at the same time, across **every** process (CLI, agent skill, pi extension, SDK). Runs beyond the cap queue rather than fail. All concurrent runs share one leader-elected browser/embedding pool, so oversubscribing it degrades every run at once. |
 | `PI_RESEARCH_RUN_ACQUIRE_TIMEOUT_MS` | `600000` | ≥0 | How long a run queues for a free slot before failing with "maximum concurrent research runs reached" (CLI exit `75`). `0` = fail immediately instead of queueing. |
-| `PI_RESEARCH_MODEL` | _(pi: session model; CLI/skill: required)_ | — | The model research runs on. **Required for the standalone CLI / agent skill** — they use only this configured model (never the model selected inside the pi extension) and refuse to start without one (the CLI's per-run `--model` flag also satisfies this). On the SDK it selects the session model when no `model` option is given. In the pi extension it overrides researcher sub-agents and knowledge synthesis, while the coordinator and research lead keep using the session model. Accepts `provider/id` or a bare model id. |
+| `PI_RESEARCH_MODEL` | _(pi: session model; CLI/skill: required)_ | — | The model research runs on. **Required for the standalone CLI / agent skill** — they use only this configured model (never the model selected inside the pi extension) and refuse to start without one (the CLI's per-run `--model` flag also satisfies this). On the SDK it selects the session model when no `model` option is given. In the pi extension it overrides the evaluator/synthesis rounds and researcher sub-agents and knowledge synthesis. The initial coordinator (round 1 planning) uses the pi session model for prompt-cache continuity with the conversation that triggered research. Accepts `provider/id` or a bare model id. |
 | `PI_RESEARCH_DISABLED_TOOLS` | _(none)_ | — | Comma-separated research tools to disable for a run (`search`, `scrape`, `security_search`, `stackexchange`, `youtube_transcript`, `grep`, `read`). Removed from every researcher's toolset and named in the coordinator and research-lead prompts. Strictly additive — it can only remove capabilities, never grant one, and it stacks on top of the default exclusions rather than replacing them. An unrecognized name excludes nothing and is warned about rather than failing the run. |
 | `PI_RESEARCH_REPORT_EXPORT_ENABLED` (TUI) | `false` | — | Front-ends write a Markdown report to disk and surface its path. |
 | `PI_RESEARCH_REPORT_EXPORT_DIR` | _(smart cwd)_ | — | Pin exported reports to a fixed directory, bypassing the cwd-relative resolution. Useful for the agent skill, which runs from the host agent's arbitrary directory. |

package/docs/PI-EXTENSION.md

@@ -19,7 +19,7 @@ Three slash commands are also registered:
 | Command | Description |
 |---------|-------------|
 | `/research <query>` | Invokes the `research` tool directly at the configured default depth (`PI_RESEARCH_DEFAULT_RESEARCH_DEPTH`, 1 by default) — a plain live run with no LLM turn. It does not parse an inline depth and does **not** consult the knowledge store; use `/knowledge-store <query>` for a store-only lookup. |
-| `/research-config` | Opens the interactive TUI settings dashboard. |
+| `/research-config` | Opens the interactive TUI settings dashboard. On non-TUI hosts (RPC, web hub, print, JSON, SDK) the menu cannot render, so the command reports why and answers with the headless diagnostics that do work there: `/research-config health` (system health) and `/research-config knowledge-status` (knowledge-store status). |
 | `/knowledge-store <query>` | Searches the local knowledge store for a query and returns a synthesised answer from previously researched findings. Unavailable when Knowledge Mode is `none`. The store auto-manages its own compaction, so there is no maintenance subcommand. |
 
 ![Running a live investigation with the /research slash command](https://raw.githubusercontent.com/Lincoln504/pi-research/main/docs/media/01-slash-research.gif)
@@ -57,6 +57,7 @@ During a run pi-research renders a live progress panel:
 | Key | Action |
 |-----|--------|
 | `Escape` | Cancel active research |
+| `Ctrl+C` | With text in the editor: clears the editor only. With an empty editor: cancels the run (same as `Escape`). |
 | Arrow keys | Navigate the `/research-config` menu |
 | `Enter` / `Space` | Cycle a setting's value |
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lincoln504/pi-research",
-  "version": "1.6.10",
+  "version": "1.6.11",
   "description": "Web research for your agents with smart and safe tooling + knowledge store",
   "type": "module",
   "private": false,
@@ -114,28 +114,29 @@
     "image": "https://raw.githubusercontent.com/Lincoln504/pi-research/main/docs/media/hero.png"
   },
   "dependencies": {
-    "@earendil-works/pi-ai": ">=0.84.0 <1",
-    "@earendil-works/pi-coding-agent": ">=0.84.0 <1",
-    "@earendil-works/pi-tui": ">=0.84.0 <1",
+    "@earendil-works/pi-ai": ">=0.85.0 <1",
+    "@earendil-works/pi-coding-agent": ">=0.85.0 <1",
+    "@earendil-works/pi-tui": ">=0.85.0 <1",
+    "@earendil-works/pi-server": ">=0.85.0 <1",
     "@kreuzberg/html-to-markdown-node": "^3.7.2",
-    "@lancedb/lancedb": "0.37.1",
+    "@lancedb/lancedb": "0.38.0",
     "apache-arrow": "21.1.0",
     "bgutils-js": "^4.0.3",
     "camoufox-js": "^0.12.0",
-    "esbuild": "^0.28.1",
+    "esbuild": "^0.28.2",
     "impit": "0.14.4",
-    "jsdom": "^29.1.1",
+    "jsdom": "^30.0.1",
     "node-html-markdown": "^2.0.0",
-    "pdf-oxide-wasm": "^0.3.75",
+    "pdf-oxide-wasm": "^0.3.77",
     "playwright-core": "1.60.0",
     "poolifier": "^5.3.2",
-    "typebox": "1.3.7",
-    "undici": "^8.9.0",
+    "typebox": "1.3.26",
+    "undici": "^8.10.2",
     "youtubei.js": "^18.0.0"
   },
   "optionalDependencies": {
     "@huggingface/transformers": "4.2.0",
-    "webgpu": "^0.4.0"
+    "webgpu": "^0.6.0"
   },
   "overrides": {
     "lodash.isequal": "npm:fast-deep-equal@^3.1.3",
@@ -148,21 +149,22 @@
     "vite": "^8.0.16",
     "adm-zip": "^0.6.0",
     "jsdom": {
-      "undici": "^7.29.0"
+      "undici": "^8.10.2"
     },
     "sharp": "^0.35.0"
   },
   "devDependencies": {
     "@eslint/js": "^10.0.1",
     "@types/jsdom": "^30.0.0",
-    "@typescript-eslint/eslint-plugin": "^8.65.0",
-    "@typescript-eslint/parser": "^8.65.0",
-    "@vitest/coverage-v8": "^4.1.10",
-    "dependency-cruiser": "^17.4.3",
+    "@typescript-eslint/eslint-plugin": "^8.69.0",
+    "@typescript-eslint/parser": "^8.69.0",
+    "@vitest/coverage-v8": "^5.0.0",
+    "dependency-cruiser": "^18.2.0",
     "depngn": "^0.6.0",
-    "eslint": "^10.8.0",
-    "npm-check-updates": "^22.2.9",
+    "eslint": "^10.10.0",
+    "npm-check-updates": "^23.1.0",
     "typescript": ">=6.0.3 <6.1.0",
-    "vitest": "^4.1.10"
+    "vite": "^8.0.16",
+    "vitest": "^5.0.0"
   }
 }

package/src/config.ts

@@ -75,6 +75,8 @@ export const ConfigSchema = Type.Object({
   MAX_CONCURRENT_SCRAPES: Type.Number({ minimum: 1, maximum: 20, default: 3 }),
   /** Maximum URLs fetched per scrape batch; extras are listed under "Not Fetched — Over Batch Cap" (default: 8, range: 1-20) */
   MAX_SCRAPE_URLS: Type.Number({ minimum: 1, maximum: 20, default: 8 }),
+  /** Also fetch DuckDuckGo results page 2 for every search query (default: false). Configured via env/config file; not surfaced in the config TUI. */
+  SCRAPE_SECOND_PAGE: Type.Boolean({ default: false }),
   /** Max YouTube videos transcribed per youtube_transcript call (default: 3, range 1-5) */
   YOUTUBE_TRANSCRIPT_MAX_VIDEOS: Type.Number({ minimum: 1, maximum: 5, default: 3 }),
   /** Per-video timeout for the youtube_transcript tool in ms (default: 20000, range 5-120s) */
@@ -141,11 +143,12 @@ export const ConfigSchema = Type.Object({
    *  the model's real maxTokens at call time.
    *  Not exposed in TUI — controlled via PI_RESEARCH_SYNTHESIS_MAX_TOKENS env var. */
   SYNTHESIS_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 32768 }),
-  /** LLM Model override for researcher sub-agents and knowledge synthesis.
+  /** LLM Model override for the evaluator/synthesis rounds and researcher sub-agents.
    *  Format: provider/model-id (e.g. google/gemini-2.0-flash-001) or just model-id.
-   *  When set, this overrides ctx.model for researcher sub-agents (both deep and quick)
-   *  and the knowledge synthesis background LLM. The coordinator and the research lead
-   *  always use the caller's model (ctx.model).
+   *  When set, this governs the evaluator/synthesis rounds and the researcher
+   *  sub-agents (both deep and quick) and the knowledge synthesis background LLM.
+   *  The initial coordinator (round 1 planning) uses the parent chat's model (ctx.model)
+   *  for prompt-cache continuity with the conversation that triggered research.
    */
   RESEARCH_MODEL: Type.Optional(Type.String()),
   /** Explicit directory for the knowledge store database (overrides default) */
@@ -246,6 +249,7 @@ const USER_MIGRATION_KEYS = [
   'PI_RESEARCH_AVG_TOKENS_PER_SCRAPE',
   'PI_RESEARCH_MAX_CONCURRENT_SCRAPES',
   'PI_RESEARCH_MAX_SCRAPE_URLS',
+  'PI_RESEARCH_SCRAPE_SECOND_PAGE',
   'PI_RESEARCH_HEALTH_CHECK_TIMEOUT_MS',
   'PI_RESEARCH_SEARCH_TIMEOUT_MS',
   'PI_RESEARCH_TUI_REFRESH_DEBOUNCE_MS',
@@ -930,6 +934,7 @@ export function saveConfig(config: Config, scope: 'local' | 'user' = 'local', cw
     PI_RESEARCH_AVG_TOKENS_PER_SCRAPE: String(config.AVG_TOKENS_PER_SCRAPE),
     PI_RESEARCH_MAX_CONCURRENT_SCRAPES: String(config.MAX_CONCURRENT_SCRAPES),
     PI_RESEARCH_MAX_SCRAPE_URLS: String(config.MAX_SCRAPE_URLS),
+    PI_RESEARCH_SCRAPE_SECOND_PAGE: String(config.SCRAPE_SECOND_PAGE),
     PI_RESEARCH_YOUTUBE_TRANSCRIPT_MAX_VIDEOS: String(config.YOUTUBE_TRANSCRIPT_MAX_VIDEOS),
     PI_RESEARCH_YOUTUBE_TRANSCRIPT_TIMEOUT_MS: String(config.YOUTUBE_TRANSCRIPT_TIMEOUT_MS),
     PI_RESEARCH_YOUTUBE_TRANSCRIPT_LANG: config.YOUTUBE_TRANSCRIPT_LANG,
@@ -1158,6 +1163,7 @@ export function createConfig(env: Record<string, string | undefined>, processEnv
     AVG_TOKENS_PER_SCRAPE: parseEnvNumber(e, 'PI_RESEARCH_AVG_TOKENS_PER_SCRAPE', DEFAULTS.AVG_TOKENS_PER_SCRAPE, 500, 10000),
     MAX_CONCURRENT_SCRAPES: parseEnvNumber(e, 'PI_RESEARCH_MAX_CONCURRENT_SCRAPES', DEFAULTS.MAX_CONCURRENT_SCRAPES, 1, 20, true),
     MAX_SCRAPE_URLS: parseEnvNumber(e, 'PI_RESEARCH_MAX_SCRAPE_URLS', DEFAULTS.MAX_SCRAPE_URLS, 1, 20, true),
+    SCRAPE_SECOND_PAGE: parseEnvBool(e, 'PI_RESEARCH_SCRAPE_SECOND_PAGE', DEFAULTS.SCRAPE_SECOND_PAGE),
     YOUTUBE_TRANSCRIPT_MAX_VIDEOS: parseEnvNumber(e, 'PI_RESEARCH_YOUTUBE_TRANSCRIPT_MAX_VIDEOS', DEFAULTS.YOUTUBE_TRANSCRIPT_MAX_VIDEOS, 1, 5, true),
     YOUTUBE_TRANSCRIPT_TIMEOUT_MS: parseEnvNumber(e, 'PI_RESEARCH_YOUTUBE_TRANSCRIPT_TIMEOUT_MS', DEFAULTS.YOUTUBE_TRANSCRIPT_TIMEOUT_MS, 5000, 120000),
     YOUTUBE_TRANSCRIPT_LANG: parseEnvString(e, 'PI_RESEARCH_YOUTUBE_TRANSCRIPT_LANG', DEFAULTS.YOUTUBE_TRANSCRIPT_LANG)!,