@lincoln504/pi-research 1.6.10 → 1.6.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/agent-skill/pi-research/SKILL.md +1 -1
- package/dist/cli.mjs +48 -12
- package/dist/thread-worker.mjs +100 -22
- package/docs/CHANGELOG.md +33 -0
- package/docs/CONFIGURATION.md +7 -2
- package/docs/PI-EXTENSION.md +2 -1
- package/package.json +21 -19
- package/src/config.ts +10 -4
- package/src/core/pi-version.ts +2 -2
- package/src/core/planning-utils.ts +34 -2
- package/src/infrastructure/browser/config.ts +3 -0
- package/src/infrastructure/browser/thread-worker-browser.ts +29 -0
- package/src/infrastructure/browser/thread-worker-messaging.ts +116 -19
- package/src/infrastructure/browser/thread-worker.mjs +100 -22
- package/src/infrastructure/browser/worker-pool-manager.ts +10 -1
- package/src/orchestration/deep-research-orchestrator.ts +12 -10
- package/src/sdk.ts +38 -11
- package/src/tools/research-knowledge-search.ts +1 -1
- package/src/tui/tui-controller.ts +57 -15
- package/src/web-research/retry-utils.ts +11 -1
package/src/core/pi-version.ts
CHANGED
|
@@ -75,7 +75,7 @@ export function compareVersions(a: SemverParts, b: SemverParts): number {
|
|
|
75
75
|
* option originally existed to prevent. The floor moved to close that gap,
|
|
76
76
|
* not just because the type changed.
|
|
77
77
|
*/
|
|
78
|
-
export const PI_MIN_VERSION: SemverParts = { major: 0, minor:
|
|
78
|
+
export const PI_MIN_VERSION: SemverParts = { major: 0, minor: 85, patch: 0 };
|
|
79
79
|
|
|
80
80
|
/**
|
|
81
81
|
* Newest host line this release was actually exercised against (CI + a real run).
|
|
@@ -83,7 +83,7 @@ export const PI_MIN_VERSION: SemverParts = { major: 0, minor: 84, patch: 0 };
|
|
|
83
83
|
* Compared on MAJOR.MINOR only: a patch bump within a tested line is not a new
|
|
84
84
|
* surface, and warning on it would be noise.
|
|
85
85
|
*/
|
|
86
|
-
export const PI_TESTED_MAX_VERSION: SemverParts = { major: 0, minor:
|
|
86
|
+
export const PI_TESTED_MAX_VERSION: SemverParts = { major: 0, minor: 85, patch: 0 };
|
|
87
87
|
|
|
88
88
|
export type PiCompatibilityLevel = 'ok' | 'unparseable' | 'too-old' | 'untested';
|
|
89
89
|
|
|
@@ -160,16 +160,48 @@ You are in the late phase of research. Set a higher threshold for delegation:
|
|
|
160
160
|
* Parse JSON from LLM response into ResearchPlan
|
|
161
161
|
*/
|
|
162
162
|
export function parseJsonPlan(text: string): ResearchPlan {
|
|
163
|
-
|
|
163
|
+
// Unwrap a doubly-encoded payload up front. Some models (observed: deepseek-v4-flash
|
|
164
|
+
// via OpenRouter, 2026-09-02) call the submit_plan tool with the plan JSON-encoded
|
|
165
|
+
// INSIDE a string argument, so the toolCall re-serialization path hands us a
|
|
166
|
+
// doubly-encoded payload — extractJson('object') then fails outright with the
|
|
167
|
+
// misleading "No valid JSON found" instead of exposing the plan it contains.
|
|
168
|
+
let normalized = text.trim();
|
|
169
|
+
if (normalized.startsWith('"')) {
|
|
170
|
+
try {
|
|
171
|
+
const unwrapped = JSON.parse(normalized);
|
|
172
|
+
if (typeof unwrapped === 'string') normalized = unwrapped.trim();
|
|
173
|
+
} catch { /* not a JSON string literal — extract below reports the real state */ }
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
const result = extractJson<unknown>(normalized, 'object');
|
|
164
177
|
if (!result.success || !result.value) {
|
|
165
178
|
const preview = text.length > 100 ? text.slice(0, 100) + '...' : text;
|
|
166
179
|
throw new Error(`Failed to extract valid JSON plan: ${result.error}. Raw response preview: "${preview}"`);
|
|
167
180
|
}
|
|
168
181
|
|
|
182
|
+
const candidate: unknown = result.value;
|
|
183
|
+
|
|
184
|
+
// Normalize explicit nulls for OPTIONAL fields. The schema treats researchers /
|
|
185
|
+
// allQueries / content / title / action / fallback as optional, but models
|
|
186
|
+
// frequently emit them as null (observed live 2026-09-02: a "research-complete"
|
|
187
|
+
// router answer with researchers:null, allQueries:null). With TypeBox 1.3.7 a null
|
|
188
|
+
// optional field makes Value.Errors report a ROOT error (path undefined, message
|
|
189
|
+
// "must be object") instead of pointing at the field — reproducing exactly the
|
|
190
|
+
// "Plan validation failed: undefined: must be object" seen in the logs — and the
|
|
191
|
+
// plan fell into the costly agentic-repair path even though the model's decision
|
|
192
|
+
// (synthesize) was perfectly valid. Dropping the key is exactly what the model meant.
|
|
193
|
+
if (candidate && typeof candidate === 'object' && !Array.isArray(candidate)) {
|
|
194
|
+
for (const key of ['researchers', 'allQueries', 'content', 'title', 'action', 'fallback'] as const) {
|
|
195
|
+
if ((candidate as Record<string, unknown>)[key] === null) {
|
|
196
|
+
delete (candidate as Record<string, unknown>)[key];
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
169
201
|
// Robust validation using TypeBox
|
|
170
202
|
try {
|
|
171
203
|
// 1. Convert/Coerce values (e.g. string numbers to numbers, single values to arrays if possible)
|
|
172
|
-
const coerced = Value.Convert(ResearchPlanSchema,
|
|
204
|
+
const coerced = Value.Convert(ResearchPlanSchema, candidate);
|
|
173
205
|
|
|
174
206
|
// 2. Validate against schema
|
|
175
207
|
if (!Value.Check(ResearchPlanSchema, coerced)) {
|
|
@@ -171,6 +171,9 @@ export function getBrowserEnv(config?: Config): NodeJS.ProcessEnv {
|
|
|
171
171
|
const c = config || getConfig();
|
|
172
172
|
env['PI_RESEARCH_SCRAPE_TIMEOUT_MS'] = String(c.SCRAPE_TIMEOUT_MS);
|
|
173
173
|
env['PI_RESEARCH_SEARCH_TIMEOUT_MS'] = String(c.SEARCH_TIMEOUT_MS);
|
|
174
|
+
// Search results page 2: the thread-worker reads this directly from its
|
|
175
|
+
// process.env copy inside executeSearchTask.
|
|
176
|
+
env['PI_RESEARCH_SCRAPE_SECOND_PAGE'] = String(c.SCRAPE_SECOND_PAGE);
|
|
174
177
|
env['PI_RESEARCH_HEALTH_CHECK_TIMEOUT_MS'] = String(c.HEALTH_CHECK_TIMEOUT_MS);
|
|
175
178
|
return env;
|
|
176
179
|
}
|
|
@@ -9,10 +9,38 @@
|
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
11
|
import { platform } from 'node:os';
|
|
12
|
+
import { readFileSync, existsSync } from 'node:fs';
|
|
13
|
+
import { homedir } from 'node:os';
|
|
12
14
|
import { setupMocking } from './thread-worker-messaging.ts';
|
|
13
15
|
import { redactSecrets } from '../../utils/log-utils.ts';
|
|
14
16
|
import { resolveHeadlessMode, BROWSER_LAUNCH_TIMEOUT_MS, CONTEXT_CREATION_TIMEOUT_MS } from './config.ts';
|
|
15
17
|
|
|
18
|
+
// Egress DNS workaround (see scripts/ddg-socks-proxy.mjs): when
|
|
19
|
+
// ~/.pi/research/proxy.json exists with {"host":"127.0.0.1","port":1080}, launch
|
|
20
|
+
// the browser through that SOCKS proxy. Needed because the system resolver
|
|
21
|
+
// returns an unreachable DuckDuckGo front-door IP from this machine's egress
|
|
22
|
+
// path; the local proxy pins DDG hostnames to the verified-live IP. Uses a
|
|
23
|
+
// file rather than env so a long-lived leader picks it up without a restart.
|
|
24
|
+
function loadSocksProxyPrefs(): Record<string, string | number | boolean> {
|
|
25
|
+
try {
|
|
26
|
+
const p = `${homedir()}/.pi/research/proxy.json`;
|
|
27
|
+
if (!existsSync(p)) return {};
|
|
28
|
+
const cfg = JSON.parse(readFileSync(p, 'utf8')) as { host?: string; port?: number };
|
|
29
|
+
const host = cfg.host ?? '127.0.0.1';
|
|
30
|
+
const port = cfg.port ?? 1080;
|
|
31
|
+
return {
|
|
32
|
+
'network.proxy.type': 1,
|
|
33
|
+
'network.proxy.socks': host,
|
|
34
|
+
'network.proxy.socks_port': port,
|
|
35
|
+
'network.proxy.socks_remote_dns': true,
|
|
36
|
+
'network.proxy.socks_version': 5,
|
|
37
|
+
'network.proxy.no_proxies_on': '',
|
|
38
|
+
};
|
|
39
|
+
} catch {
|
|
40
|
+
return {};
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
16
44
|
let browser: any = null;
|
|
17
45
|
let initPromise: Promise<void> | null = null;
|
|
18
46
|
let workerId: string = '';
|
|
@@ -240,6 +268,7 @@ export async function initBrowser(): Promise<void> {
|
|
|
240
268
|
// but headless Firefox on macOS/Windows is not guaranteed to be
|
|
241
269
|
// silent either, so the mute is applied unconditionally.
|
|
242
270
|
firefox_user_prefs: {
|
|
271
|
+
...loadSocksProxyPrefs(),
|
|
243
272
|
'media.volume_scale': '0.0',
|
|
244
273
|
'media.autoplay.default': 5,
|
|
245
274
|
// SSRF: WebSocket connections bypass BOTH layers of the browser
|
|
@@ -182,15 +182,33 @@ export async function setupMocking(context: any): Promise<void> {
|
|
|
182
182
|
|
|
183
183
|
if (mockSearch && isDuckDuckGo) {
|
|
184
184
|
logToDebugFile('DEBUG', `[Worker-${workerId}] Mocking search response for: ${urlStr}`);
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
185
|
+
// When SCRAPE_SECOND_PAGE is on, the mock also exposes DDG's next-page
|
|
186
|
+
// form (hidden `s`/`dc` offsets + a "Next" submit) and serves a DISTINCT
|
|
187
|
+
// result set when a page-2 POST arrives — identified by the `s=2` form
|
|
188
|
+
// field in the request body. This mirrors the real flow (page 1 = a GET,
|
|
189
|
+
// page 2 = POST with `s`), so the second-page scrape is
|
|
190
|
+
// end-to-end testable without a live network.
|
|
191
|
+
const isSecondPageRequest = request.method() === 'POST' &&
|
|
192
|
+
typeof request.postData() === 'string' &&
|
|
193
|
+
/(?:^|&)s=2(?:&|$)/.test(request.postData()!);
|
|
194
|
+
const mockResultsHtml = isSecondPageRequest
|
|
195
|
+
? `
|
|
196
|
+
<table class="result-link">
|
|
197
|
+
<tr>
|
|
198
|
+
<td><a class="result-link" href="https://example.com/mock-result-3?uddg=https%3A%2F%2Fexample.com%2Ftarget-3">Mock Result 3</a></td>
|
|
199
|
+
</tr>
|
|
200
|
+
<tr>
|
|
201
|
+
<td class="result-snippet">Page-two mocked search result snippet.</td>
|
|
202
|
+
</tr>
|
|
203
|
+
<tr>
|
|
204
|
+
<td><a class="result-link" href="https://example.com/mock-result-4?uddg=https%3A%2F%2Fexample.com%2Ftarget-4">Mock Result 4</a></td>
|
|
205
|
+
</tr>
|
|
206
|
+
<tr>
|
|
207
|
+
<td class="result-snippet">Another page-two mocked snippet to verify second-page extraction.</td>
|
|
208
|
+
</tr>
|
|
209
|
+
</table>
|
|
210
|
+
`
|
|
211
|
+
: `
|
|
194
212
|
<table class="result-link">
|
|
195
213
|
<tr>
|
|
196
214
|
<td><a class="result-link" href="https://example.com/mock-result-1?uddg=https%3A%2F%2Fexample.com%2Ftarget-1">Mock Result 1</a></td>
|
|
@@ -205,6 +223,23 @@ export async function setupMocking(context: any): Promise<void> {
|
|
|
205
223
|
<td class="result-snippet">Another mocked snippet to verify multiple result extraction.</td>
|
|
206
224
|
</tr>
|
|
207
225
|
</table>
|
|
226
|
+
<form action="/lite/" method="post">
|
|
227
|
+
<input type="hidden" name="q" value="">
|
|
228
|
+
<input type="hidden" name="s" value="2">
|
|
229
|
+
<input type="hidden" name="dc" value="3">
|
|
230
|
+
<input type="submit" value="Next">
|
|
231
|
+
</form>
|
|
232
|
+
`;
|
|
233
|
+
route.fulfill({
|
|
234
|
+
status: 200,
|
|
235
|
+
contentType: 'text/html',
|
|
236
|
+
body: `
|
|
237
|
+
<html><head><title>DuckDuckGo Lite</title></head><body>
|
|
238
|
+
<form action="/lite/" method="post">
|
|
239
|
+
<input name="q" type="text">
|
|
240
|
+
<input type="submit" value="Search">
|
|
241
|
+
</form>
|
|
242
|
+
${mockResultsHtml}
|
|
208
243
|
</body></html>
|
|
209
244
|
`
|
|
210
245
|
});
|
|
@@ -233,10 +268,20 @@ async function extractSearchResults(page: any): Promise<any[]> {
|
|
|
233
268
|
const found: any[] = [];
|
|
234
269
|
// @ts-ignore - document is available in browser context
|
|
235
270
|
// eslint-disable-next-line no-undef
|
|
236
|
-
const links = Array.from(document.querySelectorAll('a.result-link'));
|
|
271
|
+
const links = Array.from(document.querySelectorAll('a.result-link, a.result__a'));
|
|
237
272
|
links.forEach((link: any) => {
|
|
238
|
-
|
|
239
|
-
|
|
273
|
+
// lite layout: <a class="result-link"> inside a <tr>; the snippet lives in
|
|
274
|
+
// the NEXT sibling <tr>'s td.result-snippet (each result is a two-row pair:
|
|
275
|
+
// link row + snippet row). Fall back to inside-row for robustness.
|
|
276
|
+
// html layout: <a class="result__a"> inside div.result; snippet in
|
|
277
|
+
// a.result__snippet / .result__snippet inside the same container.
|
|
278
|
+
const row = link.closest('tr') || link.closest('.result');
|
|
279
|
+
const snippetEl =
|
|
280
|
+
row?.querySelector('td.result-snippet') ||
|
|
281
|
+
row?.querySelector('a.result__snippet') ||
|
|
282
|
+
row?.querySelector('.result__snippet') ||
|
|
283
|
+
row?.nextElementSibling?.querySelector('td.result-snippet');
|
|
284
|
+
const snippet = snippetEl?.textContent?.trim() || '';
|
|
240
285
|
const title = link.textContent?.trim() || '';
|
|
241
286
|
let url = link.href;
|
|
242
287
|
try {
|
|
@@ -309,17 +354,69 @@ export async function executeSearchTask(
|
|
|
309
354
|
if (process.env['PI_RESEARCH_MOCK_SEARCH'] === 'true') {
|
|
310
355
|
await page.goto(`https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(query)}`, { waitUntil: 'domcontentloaded' });
|
|
311
356
|
} else {
|
|
312
|
-
//
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
357
|
+
// Direct GET with the query inline. The old path (goto the empty lite landing
|
|
358
|
+
// → fill input[name=q] → Enter → waitForNavigation) broke when DDG changed the
|
|
359
|
+
// landing interaction: the submit↔navigation race extract()ed the empty form
|
|
360
|
+
// page, so EVERY search returned zero results for days while the same browsers
|
|
361
|
+
// fetched real results via a plain ?q= GET (observed 2026-09-02..03). We drive
|
|
362
|
+
// the html endpoint — its result__a markup is static in the initial HTML and is
|
|
363
|
+
// what the extractor matches — and wait for the result anchors to actually be
|
|
364
|
+
// in the DOM (domcontentloaded can fire before DDG paints them).
|
|
365
|
+
const qurl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
|
|
366
|
+
const nav = await page.goto(qurl, { waitUntil: 'domcontentloaded' });
|
|
367
|
+
if ((!nav || (nav.status && nav.status() > 399)) && !(await page.$('a.result__a, a.result-link'))) {
|
|
368
|
+
// HTTP error or no anchors: fall back to the lite endpoint (result-link
|
|
369
|
+
// class, also directly GET-able).
|
|
370
|
+
await page.goto(`https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(query)}`, { waitUntil: 'domcontentloaded' });
|
|
371
|
+
}
|
|
372
|
+
try {
|
|
373
|
+
await page.waitForSelector('a.result__a, a.result-link', { timeout: 8000 });
|
|
374
|
+
} catch {
|
|
375
|
+
// Results may legitimately be absent (anomaly/no-results page); extract
|
|
376
|
+
// whatever is there (possibly nothing) rather than throwing.
|
|
377
|
+
}
|
|
319
378
|
}
|
|
320
379
|
|
|
321
380
|
const results = await extractSearchResults(page);
|
|
322
381
|
|
|
382
|
+
// Optional second results page (SCRAPE_SECOND_PAGE config setting, threaded
|
|
383
|
+
// to workers as PI_RESEARCH_SCRAPE_SECOND_PAGE via getBrowserEnv). DuckDuckGo
|
|
384
|
+
// paginates with a small POST form at the bottom of the results table
|
|
385
|
+
// (hidden `s`/`dc` offsets plus a submit labelled "Next"); clicking THAT form
|
|
386
|
+
// (rather than hand-building a page-2 URL) reuses the offsets the site itself
|
|
387
|
+
// issued, so it stays correct whatever per-page count DDG currently serves.
|
|
388
|
+
// If the primary endpoint doesn't serve the control (or the click fails),
|
|
389
|
+
// page-2 degrades to page-1-only — see the failure policy below.
|
|
390
|
+
// Failure policy: page-1 results are already in hand and fully usable, so any
|
|
391
|
+
// page-2 problem (no next control, click/navigation timeout, abort-triggered
|
|
392
|
+
// page close) degrades to returning page 1 alone with a WARN — it must never
|
|
393
|
+
// fail a query that succeeded on page 1. Cross-page and cross-query dedup
|
|
394
|
+
// happen upstream in performSearch's seenUrls set.
|
|
395
|
+
const scrapeSecondPage = process.env['PI_RESEARCH_SCRAPE_SECOND_PAGE'] === 'true';
|
|
396
|
+
if (scrapeSecondPage && results.length > 0) {
|
|
397
|
+
try {
|
|
398
|
+
const nextButton = page.locator('form input[type="submit"][value*="next" i]').first();
|
|
399
|
+
if (await nextButton.count() > 0) {
|
|
400
|
+
logToDebugFile('DEBUG', `[Worker-${workerId}] SCRAPE_SECOND_PAGE: fetching results page 2 for: ${query}`);
|
|
401
|
+
await Promise.all([
|
|
402
|
+
page.waitForNavigation({ waitUntil: 'domcontentloaded' }),
|
|
403
|
+
nextButton.click(),
|
|
404
|
+
]);
|
|
405
|
+
const seen = new Set(results.map((r: any) => r.url));
|
|
406
|
+
for (const r of await extractSearchResults(page)) {
|
|
407
|
+
if (!seen.has(r.url)) {
|
|
408
|
+
seen.add(r.url);
|
|
409
|
+
results.push(r);
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
} else {
|
|
413
|
+
logToDebugFile('DEBUG', `[Worker-${workerId}] SCRAPE_SECOND_PAGE: no next-page control (single page of results) for: ${query}`);
|
|
414
|
+
}
|
|
415
|
+
} catch (err: any) {
|
|
416
|
+
logToDebugFile('WARN', `[Worker-${workerId}] SCRAPE_SECOND_PAGE: page-2 fetch failed (keeping page-1 results) for: ${query} — ${err?.message || String(err)}`);
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
|
|
323
420
|
await page.close();
|
|
324
421
|
|
|
325
422
|
// Reduce jitter in CI/mock mode to speed up tests
|
|
@@ -2014,6 +2014,8 @@ function createKillHandler() {
|
|
|
2014
2014
|
|
|
2015
2015
|
// src/infrastructure/browser/thread-worker-browser.ts
|
|
2016
2016
|
import { platform as platform2 } from "node:os";
|
|
2017
|
+
import { readFileSync, existsSync as existsSync3 } from "node:fs";
|
|
2018
|
+
import { homedir as homedir2 } from "node:os";
|
|
2017
2019
|
|
|
2018
2020
|
// src/infrastructure/browser/thread-worker-messaging.ts
|
|
2019
2021
|
init_log_utils();
|
|
@@ -2143,15 +2145,23 @@ async function setupMocking(context) {
|
|
|
2143
2145
|
const isDuckDuckGo = url.hostname === "duckduckgo.com" || url.hostname === "lite.duckduckgo.com" || url.hostname.endsWith(".duckduckgo.com");
|
|
2144
2146
|
if (mockSearch && isDuckDuckGo) {
|
|
2145
2147
|
logToDebugFile2("DEBUG", `[Worker-${workerId2}] Mocking search response for: ${urlStr}`);
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
<
|
|
2153
|
-
|
|
2154
|
-
|
|
2148
|
+
const isSecondPageRequest = request.method() === "POST" && typeof request.postData() === "string" && /(?:^|&)s=2(?:&|$)/.test(request.postData());
|
|
2149
|
+
const mockResultsHtml = isSecondPageRequest ? `
|
|
2150
|
+
<table class="result-link">
|
|
2151
|
+
<tr>
|
|
2152
|
+
<td><a class="result-link" href="https://example.com/mock-result-3?uddg=https%3A%2F%2Fexample.com%2Ftarget-3">Mock Result 3</a></td>
|
|
2153
|
+
</tr>
|
|
2154
|
+
<tr>
|
|
2155
|
+
<td class="result-snippet">Page-two mocked search result snippet.</td>
|
|
2156
|
+
</tr>
|
|
2157
|
+
<tr>
|
|
2158
|
+
<td><a class="result-link" href="https://example.com/mock-result-4?uddg=https%3A%2F%2Fexample.com%2Ftarget-4">Mock Result 4</a></td>
|
|
2159
|
+
</tr>
|
|
2160
|
+
<tr>
|
|
2161
|
+
<td class="result-snippet">Another page-two mocked snippet to verify second-page extraction.</td>
|
|
2162
|
+
</tr>
|
|
2163
|
+
</table>
|
|
2164
|
+
` : `
|
|
2155
2165
|
<table class="result-link">
|
|
2156
2166
|
<tr>
|
|
2157
2167
|
<td><a class="result-link" href="https://example.com/mock-result-1?uddg=https%3A%2F%2Fexample.com%2Ftarget-1">Mock Result 1</a></td>
|
|
@@ -2166,6 +2176,23 @@ async function setupMocking(context) {
|
|
|
2166
2176
|
<td class="result-snippet">Another mocked snippet to verify multiple result extraction.</td>
|
|
2167
2177
|
</tr>
|
|
2168
2178
|
</table>
|
|
2179
|
+
<form action="/lite/" method="post">
|
|
2180
|
+
<input type="hidden" name="q" value="">
|
|
2181
|
+
<input type="hidden" name="s" value="2">
|
|
2182
|
+
<input type="hidden" name="dc" value="3">
|
|
2183
|
+
<input type="submit" value="Next">
|
|
2184
|
+
</form>
|
|
2185
|
+
`;
|
|
2186
|
+
route.fulfill({
|
|
2187
|
+
status: 200,
|
|
2188
|
+
contentType: "text/html",
|
|
2189
|
+
body: `
|
|
2190
|
+
<html><head><title>DuckDuckGo Lite</title></head><body>
|
|
2191
|
+
<form action="/lite/" method="post">
|
|
2192
|
+
<input name="q" type="text">
|
|
2193
|
+
<input type="submit" value="Search">
|
|
2194
|
+
</form>
|
|
2195
|
+
${mockResultsHtml}
|
|
2169
2196
|
</body></html>
|
|
2170
2197
|
`
|
|
2171
2198
|
});
|
|
@@ -2186,10 +2213,11 @@ async function setupMocking(context) {
|
|
|
2186
2213
|
async function extractSearchResults(page) {
|
|
2187
2214
|
return await page.evaluate(() => {
|
|
2188
2215
|
const found = [];
|
|
2189
|
-
const links = Array.from(document.querySelectorAll("a.result-link"));
|
|
2216
|
+
const links = Array.from(document.querySelectorAll("a.result-link, a.result__a"));
|
|
2190
2217
|
links.forEach((link) => {
|
|
2191
|
-
const row = link.closest("tr");
|
|
2192
|
-
const
|
|
2218
|
+
const row = link.closest("tr") || link.closest(".result");
|
|
2219
|
+
const snippetEl = row?.querySelector("td.result-snippet") || row?.querySelector("a.result__snippet") || row?.querySelector(".result__snippet") || row?.nextElementSibling?.querySelector("td.result-snippet");
|
|
2220
|
+
const snippet = snippetEl?.textContent?.trim() || "";
|
|
2193
2221
|
const title = link.textContent?.trim() || "";
|
|
2194
2222
|
let url = link.href;
|
|
2195
2223
|
try {
|
|
@@ -2230,14 +2258,41 @@ async function executeSearchTask(_context, query, signal) {
|
|
|
2230
2258
|
if (process.env["PI_RESEARCH_MOCK_SEARCH"] === "true") {
|
|
2231
2259
|
await page.goto(`https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(query)}`, { waitUntil: "domcontentloaded" });
|
|
2232
2260
|
} else {
|
|
2233
|
-
|
|
2234
|
-
await page.
|
|
2235
|
-
await
|
|
2236
|
-
page.
|
|
2237
|
-
|
|
2238
|
-
|
|
2261
|
+
const qurl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
|
|
2262
|
+
const nav = await page.goto(qurl, { waitUntil: "domcontentloaded" });
|
|
2263
|
+
if ((!nav || nav.status && nav.status() > 399) && !await page.$("a.result__a, a.result-link")) {
|
|
2264
|
+
await page.goto(`https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(query)}`, { waitUntil: "domcontentloaded" });
|
|
2265
|
+
}
|
|
2266
|
+
try {
|
|
2267
|
+
await page.waitForSelector("a.result__a, a.result-link", { timeout: 8e3 });
|
|
2268
|
+
} catch {
|
|
2269
|
+
}
|
|
2239
2270
|
}
|
|
2240
2271
|
const results = await extractSearchResults(page);
|
|
2272
|
+
const scrapeSecondPage = process.env["PI_RESEARCH_SCRAPE_SECOND_PAGE"] === "true";
|
|
2273
|
+
if (scrapeSecondPage && results.length > 0) {
|
|
2274
|
+
try {
|
|
2275
|
+
const nextButton = page.locator('form input[type="submit"][value*="next" i]').first();
|
|
2276
|
+
if (await nextButton.count() > 0) {
|
|
2277
|
+
logToDebugFile2("DEBUG", `[Worker-${workerId2}] SCRAPE_SECOND_PAGE: fetching results page 2 for: ${query}`);
|
|
2278
|
+
await Promise.all([
|
|
2279
|
+
page.waitForNavigation({ waitUntil: "domcontentloaded" }),
|
|
2280
|
+
nextButton.click()
|
|
2281
|
+
]);
|
|
2282
|
+
const seen = new Set(results.map((r) => r.url));
|
|
2283
|
+
for (const r of await extractSearchResults(page)) {
|
|
2284
|
+
if (!seen.has(r.url)) {
|
|
2285
|
+
seen.add(r.url);
|
|
2286
|
+
results.push(r);
|
|
2287
|
+
}
|
|
2288
|
+
}
|
|
2289
|
+
} else {
|
|
2290
|
+
logToDebugFile2("DEBUG", `[Worker-${workerId2}] SCRAPE_SECOND_PAGE: no next-page control (single page of results) for: ${query}`);
|
|
2291
|
+
}
|
|
2292
|
+
} catch (err) {
|
|
2293
|
+
logToDebugFile2("WARN", `[Worker-${workerId2}] SCRAPE_SECOND_PAGE: page-2 fetch failed (keeping page-1 results) for: ${query} \u2014 ${err?.message || String(err)}`);
|
|
2294
|
+
}
|
|
2295
|
+
}
|
|
2241
2296
|
await page.close();
|
|
2242
2297
|
const jitterBase = process.env["PI_RESEARCH_MOCK_SEARCH"] === "true" ? 10 : 500;
|
|
2243
2298
|
const jitterRange = process.env["PI_RESEARCH_MOCK_SEARCH"] === "true" ? 10 : 1e3;
|
|
@@ -2669,6 +2724,8 @@ var ConfigSchema = Type.Object({
|
|
|
2669
2724
|
MAX_CONCURRENT_SCRAPES: Type.Number({ minimum: 1, maximum: 20, default: 3 }),
|
|
2670
2725
|
/** Maximum URLs fetched per scrape batch; extras are listed under "Not Fetched — Over Batch Cap" (default: 8, range: 1-20) */
|
|
2671
2726
|
MAX_SCRAPE_URLS: Type.Number({ minimum: 1, maximum: 20, default: 8 }),
|
|
2727
|
+
/** Also fetch DuckDuckGo results page 2 for every search query (default: false). Configured via env/config file; not surfaced in the config TUI. */
|
|
2728
|
+
SCRAPE_SECOND_PAGE: Type.Boolean({ default: false }),
|
|
2672
2729
|
/** Max YouTube videos transcribed per youtube_transcript call (default: 3, range 1-5) */
|
|
2673
2730
|
YOUTUBE_TRANSCRIPT_MAX_VIDEOS: Type.Number({ minimum: 1, maximum: 5, default: 3 }),
|
|
2674
2731
|
/** Per-video timeout for the youtube_transcript tool in ms (default: 20000, range 5-120s) */
|
|
@@ -2735,11 +2792,12 @@ var ConfigSchema = Type.Object({
|
|
|
2735
2792
|
* the model's real maxTokens at call time.
|
|
2736
2793
|
* Not exposed in TUI — controlled via PI_RESEARCH_SYNTHESIS_MAX_TOKENS env var. */
|
|
2737
2794
|
SYNTHESIS_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 32768 }),
|
|
2738
|
-
/** LLM Model override for researcher sub-agents
|
|
2795
|
+
/** LLM Model override for the evaluator/synthesis rounds and researcher sub-agents.
|
|
2739
2796
|
* Format: provider/model-id (e.g. google/gemini-2.0-flash-001) or just model-id.
|
|
2740
|
-
* When set, this
|
|
2741
|
-
* and the knowledge synthesis background LLM.
|
|
2742
|
-
*
|
|
2797
|
+
* When set, this governs the evaluator/synthesis rounds and the researcher
|
|
2798
|
+
* sub-agents (both deep and quick) and the knowledge synthesis background LLM.
|
|
2799
|
+
* The initial coordinator (round 1 planning) uses the parent chat's model (ctx.model)
|
|
2800
|
+
* for prompt-cache continuity with the conversation that triggered research.
|
|
2743
2801
|
*/
|
|
2744
2802
|
RESEARCH_MODEL: Type.Optional(Type.String()),
|
|
2745
2803
|
/** Explicit directory for the knowledge store database (overrides default) */
|
|
@@ -2799,6 +2857,25 @@ var CONTEXT_CREATION_TIMEOUT_MS = 3e4;
|
|
|
2799
2857
|
var COLD_START_ALLOWANCE_MS = BROWSER_LAUNCH_TIMEOUT_MS + CONTEXT_CREATION_TIMEOUT_MS;
|
|
2800
2858
|
|
|
2801
2859
|
// src/infrastructure/browser/thread-worker-browser.ts
|
|
2860
|
+
function loadSocksProxyPrefs() {
|
|
2861
|
+
try {
|
|
2862
|
+
const p = `${homedir2()}/.pi/research/proxy.json`;
|
|
2863
|
+
if (!existsSync3(p)) return {};
|
|
2864
|
+
const cfg = JSON.parse(readFileSync(p, "utf8"));
|
|
2865
|
+
const host = cfg.host ?? "127.0.0.1";
|
|
2866
|
+
const port = cfg.port ?? 1080;
|
|
2867
|
+
return {
|
|
2868
|
+
"network.proxy.type": 1,
|
|
2869
|
+
"network.proxy.socks": host,
|
|
2870
|
+
"network.proxy.socks_port": port,
|
|
2871
|
+
"network.proxy.socks_remote_dns": true,
|
|
2872
|
+
"network.proxy.socks_version": 5,
|
|
2873
|
+
"network.proxy.no_proxies_on": ""
|
|
2874
|
+
};
|
|
2875
|
+
} catch {
|
|
2876
|
+
return {};
|
|
2877
|
+
}
|
|
2878
|
+
}
|
|
2802
2879
|
var browser = null;
|
|
2803
2880
|
var initPromise = null;
|
|
2804
2881
|
var workerId3 = "";
|
|
@@ -2904,6 +2981,7 @@ async function initBrowser() {
|
|
|
2904
2981
|
// but headless Firefox on macOS/Windows is not guaranteed to be
|
|
2905
2982
|
// silent either, so the mute is applied unconditionally.
|
|
2906
2983
|
firefox_user_prefs: {
|
|
2984
|
+
...loadSocksProxyPrefs(),
|
|
2907
2985
|
"media.volume_scale": "0.0",
|
|
2908
2986
|
"media.autoplay.default": 5,
|
|
2909
2987
|
// SSRF: WebSocket connections bypass BOTH layers of the browser
|
|
@@ -284,7 +284,16 @@ export class WorkerPoolManager implements IService {
|
|
|
284
284
|
// Every intentional destroy bumps poolEpoch first, so a null-code exit
|
|
285
285
|
// that still matches the live epoch is an unexpected death and must count
|
|
286
286
|
// toward auto-recovery like any other abnormal exit.
|
|
287
|
-
|
|
287
|
+
//
|
|
288
|
+
// EXCEPT while this instance is mid-shutdown: the SIGTERM path tears the
|
|
289
|
+
// process down and workers die with code null while teardown (DAG disposal,
|
|
290
|
+
// server stop, browser kill) is still unwinding. Counting those deaths as
|
|
291
|
+
// health faults logged ERROR + scheduled a pointless auto-recovery on a
|
|
292
|
+
// dying process — every one of the 73 null-code exits across 15 pids in
|
|
293
|
+
// the 2026-09-01/02 logs coincided with SDK SIGTERM shutdown, tripping
|
|
294
|
+
// "3 consecutive exits → pool unhealthy" once per dying session. Not a
|
|
295
|
+
// queue-capacity or OOM problem: zero exits occurred during live research.
|
|
296
|
+
if (code !== 0 && !this.isShuttingDown && this.poolEpoch === myEpoch) {
|
|
288
297
|
logger.error(`[WorkerPoolManager] Worker exited with code ${code}`);
|
|
289
298
|
this.consecutiveErrors++;
|
|
290
299
|
if (this.consecutiveErrors >= 3) {
|
|
@@ -104,13 +104,15 @@ export class DeepResearchOrchestrator {
|
|
|
104
104
|
const observer = this.observer;
|
|
105
105
|
const container = tryGetServiceContainerFromCtx(ctx);
|
|
106
106
|
|
|
107
|
-
//
|
|
108
|
-
//
|
|
109
|
-
//
|
|
110
|
-
//
|
|
111
|
-
//
|
|
112
|
-
//
|
|
113
|
-
|
|
107
|
+
// The initial coordinator/evaluator (round 1 planning) uses the
|
|
108
|
+
// parent chat's model (ctx.model) so the prompt-cache prefix is
|
|
109
|
+
// shared with the conversation that preceded the research tool call —
|
|
110
|
+
// same provider, same model, same cache lineage.
|
|
111
|
+
// Later evaluator rounds (round 2+ routing) and final synthesis
|
|
112
|
+
// use the research model (RESEARCH_MODEL override or host model),
|
|
113
|
+
// which is also what the researcher sub-agents use.
|
|
114
|
+
const initialCoordinatorModel = (ctx.model as Model<any> | undefined) ?? model;
|
|
115
|
+
const evaluatorModel = model;
|
|
114
116
|
|
|
115
117
|
const orchestrationService = await this.getOrchestrationService();
|
|
116
118
|
const planningService = await this.getPlanningService();
|
|
@@ -249,7 +251,7 @@ export class DeepResearchOrchestrator {
|
|
|
249
251
|
sessionId: researchId,
|
|
250
252
|
query,
|
|
251
253
|
complexity,
|
|
252
|
-
model:
|
|
254
|
+
model: initialCoordinatorModel,
|
|
253
255
|
modelRegistry: ctx.modelRegistry,
|
|
254
256
|
cwd: ctx.cwd,
|
|
255
257
|
config: this.config,
|
|
@@ -274,7 +276,7 @@ export class DeepResearchOrchestrator {
|
|
|
274
276
|
query: query,
|
|
275
277
|
complexity,
|
|
276
278
|
round: this.currentRound,
|
|
277
|
-
model:
|
|
279
|
+
model: evaluatorModel,
|
|
278
280
|
modelRegistry: ctx.modelRegistry,
|
|
279
281
|
cwd: ctx.cwd,
|
|
280
282
|
config: this.config,
|
|
@@ -568,7 +570,7 @@ export class DeepResearchOrchestrator {
|
|
|
568
570
|
query: query,
|
|
569
571
|
complexity,
|
|
570
572
|
round: this.currentRound,
|
|
571
|
-
model:
|
|
573
|
+
model: evaluatorModel,
|
|
572
574
|
modelRegistry: ctx.modelRegistry,
|
|
573
575
|
cwd: ctx.cwd,
|
|
574
576
|
config: this.config,
|