@lincoln504/pi-research 1.6.10 → 1.6.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -75,7 +75,7 @@ export function compareVersions(a: SemverParts, b: SemverParts): number {
75
75
  * option originally existed to prevent. The floor moved to close that gap,
76
76
  * not just because the type changed.
77
77
  */
78
- export const PI_MIN_VERSION: SemverParts = { major: 0, minor: 84, patch: 0 };
78
+ export const PI_MIN_VERSION: SemverParts = { major: 0, minor: 85, patch: 0 };
79
79
 
80
80
  /**
81
81
  * Newest host line this release was actually exercised against (CI + a real run).
@@ -83,7 +83,7 @@ export const PI_MIN_VERSION: SemverParts = { major: 0, minor: 84, patch: 0 };
83
83
  * Compared on MAJOR.MINOR only: a patch bump within a tested line is not a new
84
84
  * surface, and warning on it would be noise.
85
85
  */
86
- export const PI_TESTED_MAX_VERSION: SemverParts = { major: 0, minor: 84, patch: 4 };
86
+ export const PI_TESTED_MAX_VERSION: SemverParts = { major: 0, minor: 85, patch: 0 };
87
87
 
88
88
  export type PiCompatibilityLevel = 'ok' | 'unparseable' | 'too-old' | 'untested';
89
89
 
@@ -160,16 +160,48 @@ You are in the late phase of research. Set a higher threshold for delegation:
160
160
  * Parse JSON from LLM response into ResearchPlan
161
161
  */
162
162
  export function parseJsonPlan(text: string): ResearchPlan {
163
- const result = extractJson<unknown>(text, 'object');
163
+ // Unwrap a doubly-encoded payload up front. Some models (observed: deepseek-v4-flash
164
+ // via OpenRouter, 2026-09-02) call the submit_plan tool with the plan JSON-encoded
165
+ // INSIDE a string argument, so the toolCall re-serialization path hands us a
166
+ // doubly-encoded payload — extractJson('object') then fails outright with the
167
+ // misleading "No valid JSON found" instead of exposing the plan it contains.
168
+ let normalized = text.trim();
169
+ if (normalized.startsWith('"')) {
170
+ try {
171
+ const unwrapped = JSON.parse(normalized);
172
+ if (typeof unwrapped === 'string') normalized = unwrapped.trim();
173
+ } catch { /* not a JSON string literal — extract below reports the real state */ }
174
+ }
175
+
176
+ const result = extractJson<unknown>(normalized, 'object');
164
177
  if (!result.success || !result.value) {
165
178
  const preview = text.length > 100 ? text.slice(0, 100) + '...' : text;
166
179
  throw new Error(`Failed to extract valid JSON plan: ${result.error}. Raw response preview: "${preview}"`);
167
180
  }
168
181
 
182
+ const candidate: unknown = result.value;
183
+
184
+ // Normalize explicit nulls for OPTIONAL fields. The schema treats researchers /
185
+ // allQueries / content / title / action / fallback as optional, but models
186
+ // frequently emit them as null (observed live 2026-09-02: a "research-complete"
187
+ // router answer with researchers:null, allQueries:null). With TypeBox 1.3.7 a null
188
+ // optional field makes Value.Errors report a ROOT error (path undefined, message
189
+ // "must be object") instead of pointing at the field — reproducing exactly the
190
+ // "Plan validation failed: undefined: must be object" seen in the logs — and the
191
+ // plan fell into the costly agentic-repair path even though the model's decision
192
+ // (synthesize) was perfectly valid. Dropping the key is exactly what the model meant.
193
+ if (candidate && typeof candidate === 'object' && !Array.isArray(candidate)) {
194
+ for (const key of ['researchers', 'allQueries', 'content', 'title', 'action', 'fallback'] as const) {
195
+ if ((candidate as Record<string, unknown>)[key] === null) {
196
+ delete (candidate as Record<string, unknown>)[key];
197
+ }
198
+ }
199
+ }
200
+
169
201
  // Robust validation using TypeBox
170
202
  try {
171
203
  // 1. Convert/Coerce values (e.g. string numbers to numbers, single values to arrays if possible)
172
- const coerced = Value.Convert(ResearchPlanSchema, result.value);
204
+ const coerced = Value.Convert(ResearchPlanSchema, candidate);
173
205
 
174
206
  // 2. Validate against schema
175
207
  if (!Value.Check(ResearchPlanSchema, coerced)) {
@@ -171,6 +171,9 @@ export function getBrowserEnv(config?: Config): NodeJS.ProcessEnv {
171
171
  const c = config || getConfig();
172
172
  env['PI_RESEARCH_SCRAPE_TIMEOUT_MS'] = String(c.SCRAPE_TIMEOUT_MS);
173
173
  env['PI_RESEARCH_SEARCH_TIMEOUT_MS'] = String(c.SEARCH_TIMEOUT_MS);
174
+ // Search results page 2: the thread-worker reads this directly from its
175
+ // process.env copy inside executeSearchTask.
176
+ env['PI_RESEARCH_SCRAPE_SECOND_PAGE'] = String(c.SCRAPE_SECOND_PAGE);
174
177
  env['PI_RESEARCH_HEALTH_CHECK_TIMEOUT_MS'] = String(c.HEALTH_CHECK_TIMEOUT_MS);
175
178
  return env;
176
179
  }
@@ -9,10 +9,38 @@
9
9
  */
10
10
 
11
11
  import { platform } from 'node:os';
12
+ import { readFileSync, existsSync } from 'node:fs';
13
+ import { homedir } from 'node:os';
12
14
  import { setupMocking } from './thread-worker-messaging.ts';
13
15
  import { redactSecrets } from '../../utils/log-utils.ts';
14
16
  import { resolveHeadlessMode, BROWSER_LAUNCH_TIMEOUT_MS, CONTEXT_CREATION_TIMEOUT_MS } from './config.ts';
15
17
 
18
+ // Egress DNS workaround (see scripts/ddg-socks-proxy.mjs): when
19
+ // ~/.pi/research/proxy.json exists with {"host":"127.0.0.1","port":1080}, launch
20
+ // the browser through that SOCKS proxy. Needed because the system resolver
21
+ // returns an unreachable DuckDuckGo front-door IP from this machine's egress
22
+ // path; the local proxy pins DDG hostnames to the verified-live IP. Uses a
23
+ // file rather than env so a long-lived leader picks it up without a restart.
24
+ function loadSocksProxyPrefs(): Record<string, string | number | boolean> {
25
+ try {
26
+ const p = `${homedir()}/.pi/research/proxy.json`;
27
+ if (!existsSync(p)) return {};
28
+ const cfg = JSON.parse(readFileSync(p, 'utf8')) as { host?: string; port?: number };
29
+ const host = cfg.host ?? '127.0.0.1';
30
+ const port = cfg.port ?? 1080;
31
+ return {
32
+ 'network.proxy.type': 1,
33
+ 'network.proxy.socks': host,
34
+ 'network.proxy.socks_port': port,
35
+ 'network.proxy.socks_remote_dns': true,
36
+ 'network.proxy.socks_version': 5,
37
+ 'network.proxy.no_proxies_on': '',
38
+ };
39
+ } catch {
40
+ return {};
41
+ }
42
+ }
43
+
16
44
  let browser: any = null;
17
45
  let initPromise: Promise<void> | null = null;
18
46
  let workerId: string = '';
@@ -240,6 +268,7 @@ export async function initBrowser(): Promise<void> {
240
268
  // but headless Firefox on macOS/Windows is not guaranteed to be
241
269
  // silent either, so the mute is applied unconditionally.
242
270
  firefox_user_prefs: {
271
+ ...loadSocksProxyPrefs(),
243
272
  'media.volume_scale': '0.0',
244
273
  'media.autoplay.default': 5,
245
274
  // SSRF: WebSocket connections bypass BOTH layers of the browser
@@ -182,15 +182,33 @@ export async function setupMocking(context: any): Promise<void> {
182
182
 
183
183
  if (mockSearch && isDuckDuckGo) {
184
184
  logToDebugFile('DEBUG', `[Worker-${workerId}] Mocking search response for: ${urlStr}`);
185
- route.fulfill({
186
- status: 200,
187
- contentType: 'text/html',
188
- body: `
189
- <html><head><title>DuckDuckGo Lite</title></head><body>
190
- <form action="/lite/" method="post">
191
- <input name="q" type="text">
192
- <input type="submit" value="Search">
193
- </form>
185
+ // When SCRAPE_SECOND_PAGE is on, the mock also exposes DDG's next-page
186
+ // form (hidden `s`/`dc` offsets + a "Next" submit) and serves a DISTINCT
187
+ // result set when a page-2 POST arrives — identified by the `s=2` form
188
+ // field in the request body. This mirrors the real flow (page 1 = a GET,
189
+ // page 2 = POST with `s`), so the second-page scrape is
190
+ // end-to-end testable without a live network.
191
+ const isSecondPageRequest = request.method() === 'POST' &&
192
+ typeof request.postData() === 'string' &&
193
+ /(?:^|&)s=2(?:&|$)/.test(request.postData()!);
194
+ const mockResultsHtml = isSecondPageRequest
195
+ ? `
196
+ <table class="result-link">
197
+ <tr>
198
+ <td><a class="result-link" href="https://example.com/mock-result-3?uddg=https%3A%2F%2Fexample.com%2Ftarget-3">Mock Result 3</a></td>
199
+ </tr>
200
+ <tr>
201
+ <td class="result-snippet">Page-two mocked search result snippet.</td>
202
+ </tr>
203
+ <tr>
204
+ <td><a class="result-link" href="https://example.com/mock-result-4?uddg=https%3A%2F%2Fexample.com%2Ftarget-4">Mock Result 4</a></td>
205
+ </tr>
206
+ <tr>
207
+ <td class="result-snippet">Another page-two mocked snippet to verify second-page extraction.</td>
208
+ </tr>
209
+ </table>
210
+ `
211
+ : `
194
212
  <table class="result-link">
195
213
  <tr>
196
214
  <td><a class="result-link" href="https://example.com/mock-result-1?uddg=https%3A%2F%2Fexample.com%2Ftarget-1">Mock Result 1</a></td>
@@ -205,6 +223,23 @@ export async function setupMocking(context: any): Promise<void> {
205
223
  <td class="result-snippet">Another mocked snippet to verify multiple result extraction.</td>
206
224
  </tr>
207
225
  </table>
226
+ <form action="/lite/" method="post">
227
+ <input type="hidden" name="q" value="">
228
+ <input type="hidden" name="s" value="2">
229
+ <input type="hidden" name="dc" value="3">
230
+ <input type="submit" value="Next">
231
+ </form>
232
+ `;
233
+ route.fulfill({
234
+ status: 200,
235
+ contentType: 'text/html',
236
+ body: `
237
+ <html><head><title>DuckDuckGo Lite</title></head><body>
238
+ <form action="/lite/" method="post">
239
+ <input name="q" type="text">
240
+ <input type="submit" value="Search">
241
+ </form>
242
+ ${mockResultsHtml}
208
243
  </body></html>
209
244
  `
210
245
  });
@@ -233,10 +268,20 @@ async function extractSearchResults(page: any): Promise<any[]> {
233
268
  const found: any[] = [];
234
269
  // @ts-ignore - document is available in browser context
235
270
  // eslint-disable-next-line no-undef
236
- const links = Array.from(document.querySelectorAll('a.result-link'));
271
+ const links = Array.from(document.querySelectorAll('a.result-link, a.result__a'));
237
272
  links.forEach((link: any) => {
238
- const row = link.closest('tr');
239
- const snippet = row?.nextElementSibling?.querySelector('td.result-snippet')?.textContent?.trim() || '';
273
+ // lite layout: <a class="result-link"> inside a <tr>; the snippet lives in
274
+ // the NEXT sibling <tr>'s td.result-snippet (each result is a two-row pair:
275
+ // link row + snippet row). Fall back to inside-row for robustness.
276
+ // html layout: <a class="result__a"> inside div.result; snippet in
277
+ // a.result__snippet / .result__snippet inside the same container.
278
+ const row = link.closest('tr') || link.closest('.result');
279
+ const snippetEl =
280
+ row?.querySelector('td.result-snippet') ||
281
+ row?.querySelector('a.result__snippet') ||
282
+ row?.querySelector('.result__snippet') ||
283
+ row?.nextElementSibling?.querySelector('td.result-snippet');
284
+ const snippet = snippetEl?.textContent?.trim() || '';
240
285
  const title = link.textContent?.trim() || '';
241
286
  let url = link.href;
242
287
  try {
@@ -309,17 +354,69 @@ export async function executeSearchTask(
309
354
  if (process.env['PI_RESEARCH_MOCK_SEARCH'] === 'true') {
310
355
  await page.goto(`https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(query)}`, { waitUntil: 'domcontentloaded' });
311
356
  } else {
312
- // Relaxed timeout to accommodate network latency and concurrent worker load
313
- await page.goto('https://lite.duckduckgo.com/lite/', { waitUntil: 'domcontentloaded' });
314
- await page.fill('input[name="q"]', query);
315
- await Promise.all([
316
- page.waitForNavigation({ waitUntil: 'domcontentloaded' }),
317
- page.keyboard.press('Enter')
318
- ]);
357
+ // Direct GET with the query inline. The old path (goto the empty lite landing
358
+ // → fill input[name=q] → Enter → waitForNavigation) broke when DDG changed the
359
+ // landing interaction: the submit↔navigation race extract()ed the empty form
360
+ // page, so EVERY search returned zero results for days while the same browsers
361
+ // fetched real results via a plain ?q= GET (observed 2026-09-02..03). We drive
362
+ // the html endpoint — its result__a markup is static in the initial HTML and is
363
+ // what the extractor matches — and wait for the result anchors to actually be
364
+ // in the DOM (domcontentloaded can fire before DDG paints them).
365
+ const qurl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
366
+ const nav = await page.goto(qurl, { waitUntil: 'domcontentloaded' });
367
+ if ((!nav || (nav.status && nav.status() > 399)) && !(await page.$('a.result__a, a.result-link'))) {
368
+ // HTTP error or no anchors: fall back to the lite endpoint (result-link
369
+ // class, also directly GET-able).
370
+ await page.goto(`https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(query)}`, { waitUntil: 'domcontentloaded' });
371
+ }
372
+ try {
373
+ await page.waitForSelector('a.result__a, a.result-link', { timeout: 8000 });
374
+ } catch {
375
+ // Results may legitimately be absent (anomaly/no-results page); extract
376
+ // whatever is there (possibly nothing) rather than throwing.
377
+ }
319
378
  }
320
379
 
321
380
  const results = await extractSearchResults(page);
322
381
 
382
+ // Optional second results page (SCRAPE_SECOND_PAGE config setting, threaded
383
+ // to workers as PI_RESEARCH_SCRAPE_SECOND_PAGE via getBrowserEnv). DuckDuckGo
384
+ // paginates with a small POST form at the bottom of the results table
385
+ // (hidden `s`/`dc` offsets plus a submit labelled "Next"); clicking THAT form
386
+ // (rather than hand-building a page-2 URL) reuses the offsets the site itself
387
+ // issued, so it stays correct whatever per-page count DDG currently serves.
388
+ // If the primary endpoint doesn't serve the control (or the click fails),
389
+ // page-2 degrades to page-1-only — see the failure policy below.
390
+ // Failure policy: page-1 results are already in hand and fully usable, so any
391
+ // page-2 problem (no next control, click/navigation timeout, abort-triggered
392
+ // page close) degrades to returning page 1 alone with a WARN — it must never
393
+ // fail a query that succeeded on page 1. Cross-page and cross-query dedup
394
+ // happen upstream in performSearch's seenUrls set.
395
+ const scrapeSecondPage = process.env['PI_RESEARCH_SCRAPE_SECOND_PAGE'] === 'true';
396
+ if (scrapeSecondPage && results.length > 0) {
397
+ try {
398
+ const nextButton = page.locator('form input[type="submit"][value*="next" i]').first();
399
+ if (await nextButton.count() > 0) {
400
+ logToDebugFile('DEBUG', `[Worker-${workerId}] SCRAPE_SECOND_PAGE: fetching results page 2 for: ${query}`);
401
+ await Promise.all([
402
+ page.waitForNavigation({ waitUntil: 'domcontentloaded' }),
403
+ nextButton.click(),
404
+ ]);
405
+ const seen = new Set(results.map((r: any) => r.url));
406
+ for (const r of await extractSearchResults(page)) {
407
+ if (!seen.has(r.url)) {
408
+ seen.add(r.url);
409
+ results.push(r);
410
+ }
411
+ }
412
+ } else {
413
+ logToDebugFile('DEBUG', `[Worker-${workerId}] SCRAPE_SECOND_PAGE: no next-page control (single page of results) for: ${query}`);
414
+ }
415
+ } catch (err: any) {
416
+ logToDebugFile('WARN', `[Worker-${workerId}] SCRAPE_SECOND_PAGE: page-2 fetch failed (keeping page-1 results) for: ${query} — ${err?.message || String(err)}`);
417
+ }
418
+ }
419
+
323
420
  await page.close();
324
421
 
325
422
  // Reduce jitter in CI/mock mode to speed up tests
@@ -2014,6 +2014,8 @@ function createKillHandler() {
2014
2014
 
2015
2015
  // src/infrastructure/browser/thread-worker-browser.ts
2016
2016
  import { platform as platform2 } from "node:os";
2017
+ import { readFileSync, existsSync as existsSync3 } from "node:fs";
2018
+ import { homedir as homedir2 } from "node:os";
2017
2019
 
2018
2020
  // src/infrastructure/browser/thread-worker-messaging.ts
2019
2021
  init_log_utils();
@@ -2143,15 +2145,23 @@ async function setupMocking(context) {
2143
2145
  const isDuckDuckGo = url.hostname === "duckduckgo.com" || url.hostname === "lite.duckduckgo.com" || url.hostname.endsWith(".duckduckgo.com");
2144
2146
  if (mockSearch && isDuckDuckGo) {
2145
2147
  logToDebugFile2("DEBUG", `[Worker-${workerId2}] Mocking search response for: ${urlStr}`);
2146
- route.fulfill({
2147
- status: 200,
2148
- contentType: "text/html",
2149
- body: `
2150
- <html><head><title>DuckDuckGo Lite</title></head><body>
2151
- <form action="/lite/" method="post">
2152
- <input name="q" type="text">
2153
- <input type="submit" value="Search">
2154
- </form>
2148
+ const isSecondPageRequest = request.method() === "POST" && typeof request.postData() === "string" && /(?:^|&)s=2(?:&|$)/.test(request.postData());
2149
+ const mockResultsHtml = isSecondPageRequest ? `
2150
+ <table class="result-link">
2151
+ <tr>
2152
+ <td><a class="result-link" href="https://example.com/mock-result-3?uddg=https%3A%2F%2Fexample.com%2Ftarget-3">Mock Result 3</a></td>
2153
+ </tr>
2154
+ <tr>
2155
+ <td class="result-snippet">Page-two mocked search result snippet.</td>
2156
+ </tr>
2157
+ <tr>
2158
+ <td><a class="result-link" href="https://example.com/mock-result-4?uddg=https%3A%2F%2Fexample.com%2Ftarget-4">Mock Result 4</a></td>
2159
+ </tr>
2160
+ <tr>
2161
+ <td class="result-snippet">Another page-two mocked snippet to verify second-page extraction.</td>
2162
+ </tr>
2163
+ </table>
2164
+ ` : `
2155
2165
  <table class="result-link">
2156
2166
  <tr>
2157
2167
  <td><a class="result-link" href="https://example.com/mock-result-1?uddg=https%3A%2F%2Fexample.com%2Ftarget-1">Mock Result 1</a></td>
@@ -2166,6 +2176,23 @@ async function setupMocking(context) {
2166
2176
  <td class="result-snippet">Another mocked snippet to verify multiple result extraction.</td>
2167
2177
  </tr>
2168
2178
  </table>
2179
+ <form action="/lite/" method="post">
2180
+ <input type="hidden" name="q" value="">
2181
+ <input type="hidden" name="s" value="2">
2182
+ <input type="hidden" name="dc" value="3">
2183
+ <input type="submit" value="Next">
2184
+ </form>
2185
+ `;
2186
+ route.fulfill({
2187
+ status: 200,
2188
+ contentType: "text/html",
2189
+ body: `
2190
+ <html><head><title>DuckDuckGo Lite</title></head><body>
2191
+ <form action="/lite/" method="post">
2192
+ <input name="q" type="text">
2193
+ <input type="submit" value="Search">
2194
+ </form>
2195
+ ${mockResultsHtml}
2169
2196
  </body></html>
2170
2197
  `
2171
2198
  });
@@ -2186,10 +2213,11 @@ async function setupMocking(context) {
2186
2213
  async function extractSearchResults(page) {
2187
2214
  return await page.evaluate(() => {
2188
2215
  const found = [];
2189
- const links = Array.from(document.querySelectorAll("a.result-link"));
2216
+ const links = Array.from(document.querySelectorAll("a.result-link, a.result__a"));
2190
2217
  links.forEach((link) => {
2191
- const row = link.closest("tr");
2192
- const snippet = row?.nextElementSibling?.querySelector("td.result-snippet")?.textContent?.trim() || "";
2218
+ const row = link.closest("tr") || link.closest(".result");
2219
+ const snippetEl = row?.querySelector("td.result-snippet") || row?.querySelector("a.result__snippet") || row?.querySelector(".result__snippet") || row?.nextElementSibling?.querySelector("td.result-snippet");
2220
+ const snippet = snippetEl?.textContent?.trim() || "";
2193
2221
  const title = link.textContent?.trim() || "";
2194
2222
  let url = link.href;
2195
2223
  try {
@@ -2230,14 +2258,41 @@ async function executeSearchTask(_context, query, signal) {
2230
2258
  if (process.env["PI_RESEARCH_MOCK_SEARCH"] === "true") {
2231
2259
  await page.goto(`https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(query)}`, { waitUntil: "domcontentloaded" });
2232
2260
  } else {
2233
- await page.goto("https://lite.duckduckgo.com/lite/", { waitUntil: "domcontentloaded" });
2234
- await page.fill('input[name="q"]', query);
2235
- await Promise.all([
2236
- page.waitForNavigation({ waitUntil: "domcontentloaded" }),
2237
- page.keyboard.press("Enter")
2238
- ]);
2261
+ const qurl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
2262
+ const nav = await page.goto(qurl, { waitUntil: "domcontentloaded" });
2263
+ if ((!nav || nav.status && nav.status() > 399) && !await page.$("a.result__a, a.result-link")) {
2264
+ await page.goto(`https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(query)}`, { waitUntil: "domcontentloaded" });
2265
+ }
2266
+ try {
2267
+ await page.waitForSelector("a.result__a, a.result-link", { timeout: 8e3 });
2268
+ } catch {
2269
+ }
2239
2270
  }
2240
2271
  const results = await extractSearchResults(page);
2272
+ const scrapeSecondPage = process.env["PI_RESEARCH_SCRAPE_SECOND_PAGE"] === "true";
2273
+ if (scrapeSecondPage && results.length > 0) {
2274
+ try {
2275
+ const nextButton = page.locator('form input[type="submit"][value*="next" i]').first();
2276
+ if (await nextButton.count() > 0) {
2277
+ logToDebugFile2("DEBUG", `[Worker-${workerId2}] SCRAPE_SECOND_PAGE: fetching results page 2 for: ${query}`);
2278
+ await Promise.all([
2279
+ page.waitForNavigation({ waitUntil: "domcontentloaded" }),
2280
+ nextButton.click()
2281
+ ]);
2282
+ const seen = new Set(results.map((r) => r.url));
2283
+ for (const r of await extractSearchResults(page)) {
2284
+ if (!seen.has(r.url)) {
2285
+ seen.add(r.url);
2286
+ results.push(r);
2287
+ }
2288
+ }
2289
+ } else {
2290
+ logToDebugFile2("DEBUG", `[Worker-${workerId2}] SCRAPE_SECOND_PAGE: no next-page control (single page of results) for: ${query}`);
2291
+ }
2292
+ } catch (err) {
2293
+ logToDebugFile2("WARN", `[Worker-${workerId2}] SCRAPE_SECOND_PAGE: page-2 fetch failed (keeping page-1 results) for: ${query} \u2014 ${err?.message || String(err)}`);
2294
+ }
2295
+ }
2241
2296
  await page.close();
2242
2297
  const jitterBase = process.env["PI_RESEARCH_MOCK_SEARCH"] === "true" ? 10 : 500;
2243
2298
  const jitterRange = process.env["PI_RESEARCH_MOCK_SEARCH"] === "true" ? 10 : 1e3;
@@ -2669,6 +2724,8 @@ var ConfigSchema = Type.Object({
2669
2724
  MAX_CONCURRENT_SCRAPES: Type.Number({ minimum: 1, maximum: 20, default: 3 }),
2670
2725
  /** Maximum URLs fetched per scrape batch; extras are listed under "Not Fetched — Over Batch Cap" (default: 8, range: 1-20) */
2671
2726
  MAX_SCRAPE_URLS: Type.Number({ minimum: 1, maximum: 20, default: 8 }),
2727
+ /** Also fetch DuckDuckGo results page 2 for every search query (default: false). Configured via env/config file; not surfaced in the config TUI. */
2728
+ SCRAPE_SECOND_PAGE: Type.Boolean({ default: false }),
2672
2729
  /** Max YouTube videos transcribed per youtube_transcript call (default: 3, range 1-5) */
2673
2730
  YOUTUBE_TRANSCRIPT_MAX_VIDEOS: Type.Number({ minimum: 1, maximum: 5, default: 3 }),
2674
2731
  /** Per-video timeout for the youtube_transcript tool in ms (default: 20000, range 5-120s) */
@@ -2735,11 +2792,12 @@ var ConfigSchema = Type.Object({
2735
2792
  * the model's real maxTokens at call time.
2736
2793
  * Not exposed in TUI — controlled via PI_RESEARCH_SYNTHESIS_MAX_TOKENS env var. */
2737
2794
  SYNTHESIS_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 32768 }),
2738
- /** LLM Model override for researcher sub-agents and knowledge synthesis.
2795
+ /** LLM Model override for the evaluator/synthesis rounds and researcher sub-agents.
2739
2796
  * Format: provider/model-id (e.g. google/gemini-2.0-flash-001) or just model-id.
2740
- * When set, this overrides ctx.model for researcher sub-agents (both deep and quick)
2741
- * and the knowledge synthesis background LLM. The coordinator and the research lead
2742
- * always use the caller's model (ctx.model).
2797
+ * When set, this governs the evaluator/synthesis rounds and the researcher
2798
+ * sub-agents (both deep and quick) and the knowledge synthesis background LLM.
2799
+ * The initial coordinator (round 1 planning) uses the parent chat's model (ctx.model)
2800
+ * for prompt-cache continuity with the conversation that triggered research.
2743
2801
  */
2744
2802
  RESEARCH_MODEL: Type.Optional(Type.String()),
2745
2803
  /** Explicit directory for the knowledge store database (overrides default) */
@@ -2799,6 +2857,25 @@ var CONTEXT_CREATION_TIMEOUT_MS = 3e4;
2799
2857
  var COLD_START_ALLOWANCE_MS = BROWSER_LAUNCH_TIMEOUT_MS + CONTEXT_CREATION_TIMEOUT_MS;
2800
2858
 
2801
2859
  // src/infrastructure/browser/thread-worker-browser.ts
2860
+ function loadSocksProxyPrefs() {
2861
+ try {
2862
+ const p = `${homedir2()}/.pi/research/proxy.json`;
2863
+ if (!existsSync3(p)) return {};
2864
+ const cfg = JSON.parse(readFileSync(p, "utf8"));
2865
+ const host = cfg.host ?? "127.0.0.1";
2866
+ const port = cfg.port ?? 1080;
2867
+ return {
2868
+ "network.proxy.type": 1,
2869
+ "network.proxy.socks": host,
2870
+ "network.proxy.socks_port": port,
2871
+ "network.proxy.socks_remote_dns": true,
2872
+ "network.proxy.socks_version": 5,
2873
+ "network.proxy.no_proxies_on": ""
2874
+ };
2875
+ } catch {
2876
+ return {};
2877
+ }
2878
+ }
2802
2879
  var browser = null;
2803
2880
  var initPromise = null;
2804
2881
  var workerId3 = "";
@@ -2904,6 +2981,7 @@ async function initBrowser() {
2904
2981
  // but headless Firefox on macOS/Windows is not guaranteed to be
2905
2982
  // silent either, so the mute is applied unconditionally.
2906
2983
  firefox_user_prefs: {
2984
+ ...loadSocksProxyPrefs(),
2907
2985
  "media.volume_scale": "0.0",
2908
2986
  "media.autoplay.default": 5,
2909
2987
  // SSRF: WebSocket connections bypass BOTH layers of the browser
@@ -284,7 +284,16 @@ export class WorkerPoolManager implements IService {
284
284
  // Every intentional destroy bumps poolEpoch first, so a null-code exit
285
285
  // that still matches the live epoch is an unexpected death and must count
286
286
  // toward auto-recovery like any other abnormal exit.
287
- if (code !== 0 && this.poolEpoch === myEpoch) {
287
+ //
288
+ // EXCEPT while this instance is mid-shutdown: the SIGTERM path tears the
289
+ // process down and workers die with code null while teardown (DAG disposal,
290
+ // server stop, browser kill) is still unwinding. Counting those deaths as
291
+ // health faults logged ERROR + scheduled a pointless auto-recovery on a
292
+ // dying process — every one of the 73 null-code exits across 15 pids in
293
+ // the 2026-09-01/02 logs coincided with SDK SIGTERM shutdown, tripping
294
+ // "3 consecutive exits → pool unhealthy" once per dying session. Not a
295
+ // queue-capacity or OOM problem: zero exits occurred during live research.
296
+ if (code !== 0 && !this.isShuttingDown && this.poolEpoch === myEpoch) {
288
297
  logger.error(`[WorkerPoolManager] Worker exited with code ${code}`);
289
298
  this.consecutiveErrors++;
290
299
  if (this.consecutiveErrors >= 3) {
@@ -104,13 +104,15 @@ export class DeepResearchOrchestrator {
104
104
  const observer = this.observer;
105
105
  const container = tryGetServiceContainerFromCtx(ctx);
106
106
 
107
- // RESEARCH_MODEL contract (config.ts): the coordinator and evaluator always
108
- // use the caller's model (ctx.model); RESEARCH_MODEL overrides only the
109
- // researcher sub-agents. `model` is the resolved research model — it equals
110
- // ctx.model on the SDK/CLI path, but on the pi-tool path it is the
111
- // RESEARCH_MODEL override, so planning must key off ctx.model explicitly.
112
- // Fall back to `model` when the host exposes no ctx.model.
113
- const coordinatorModel = (ctx.model as Model<any> | undefined) ?? model;
107
+ // The initial coordinator/evaluator (round 1 planning) uses the
108
+ // parent chat's model (ctx.model) so the prompt-cache prefix is
109
+ // shared with the conversation that preceded the research tool call —
110
+ // same provider, same model, same cache lineage.
111
+ // Later evaluator rounds (round 2+ routing) and final synthesis
112
+ // use the research model (RESEARCH_MODEL override or host model),
113
+ // which is also what the researcher sub-agents use.
114
+ const initialCoordinatorModel = (ctx.model as Model<any> | undefined) ?? model;
115
+ const evaluatorModel = model;
114
116
 
115
117
  const orchestrationService = await this.getOrchestrationService();
116
118
  const planningService = await this.getPlanningService();
@@ -249,7 +251,7 @@ export class DeepResearchOrchestrator {
249
251
  sessionId: researchId,
250
252
  query,
251
253
  complexity,
252
- model: coordinatorModel,
254
+ model: initialCoordinatorModel,
253
255
  modelRegistry: ctx.modelRegistry,
254
256
  cwd: ctx.cwd,
255
257
  config: this.config,
@@ -274,7 +276,7 @@ export class DeepResearchOrchestrator {
274
276
  query: query,
275
277
  complexity,
276
278
  round: this.currentRound,
277
- model: coordinatorModel,
279
+ model: evaluatorModel,
278
280
  modelRegistry: ctx.modelRegistry,
279
281
  cwd: ctx.cwd,
280
282
  config: this.config,
@@ -568,7 +570,7 @@ export class DeepResearchOrchestrator {
568
570
  query: query,
569
571
  complexity,
570
572
  round: this.currentRound,
571
- model: coordinatorModel,
573
+ model: evaluatorModel,
572
574
  modelRegistry: ctx.modelRegistry,
573
575
  cwd: ctx.cwd,
574
576
  config: this.config,